refactor(file): 重构文件删除逻辑确保S3与数据库同步清理

- 删除文件时先删S3对象再删FileMeta记录,防止出现孤儿文件或残留对象
- 在DeleteFileCommandHandler中注入FileDbContext以支持数据库操作
- 更新文件删除逻辑:精确匹配bucket+key删除对应的FileMeta记录
- 添加DatabaseInitializer类统一处理启动期数据库初始化流程
- 将原有的数据库迁移、种子数据等逻辑抽取到独立的初始化服务中
- 新增HostObjectAccessors类简化ABP与ASP.NET Core管线的双向访问配置
- 添加平台模块废弃权限清理功能,移除OrderManagement分组的残留权限
- 实现完整的单元测试覆盖文件删除的各种场景和边界情况
This commit is contained in:
xiangning 2026-07-03 09:42:05 +08:00
parent d0650e200a
commit bdde5b9b8a
6 changed files with 380 additions and 82 deletions

View File

@ -0,0 +1,111 @@
using BuildFlow.Infrastructure.Persistence;
using BuildFlow.Platform.Infrastructure.Persistence;
using File.Infrastructure.Persistence;
using IM.Infrastructure.Mongo;
using IM.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Diagnostics;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Workflow.Infrastructure.Persistence;
using RagSeedData = BuildFlow.Infrastructure.Persistence.SeedData;
using PlatformSeedData = BuildFlow.Platform.Infrastructure.Persistence.SeedData;
using WorkflowSeedData = Workflow.Infrastructure.Persistence.SeedData;
namespace BuildFlow.Host.Configuration;
/// <summary>
/// 启动期数据库初始化:按固定顺序完成「迁移 → IM MongoDB 建索引 → 种子数据」。
/// 容错策略:单个 DbContext / 单个步骤失败只记录警告并跳过,不影响其他库与主流程启动,
/// 避免任一外部库抖动拖垮整个 Host。日志走 <see cref="ILogger"/>(可被 OTel 采集),
/// 输出格式与 <see cref="Services.BackgroundServices"/> 一致。
/// </summary>
public static class DatabaseInitializer
{
public static async Task RunAsync(IServiceProvider rootProvider)
{
using var scope = rootProvider.CreateScope();
var sp = scope.ServiceProvider;
var logger = sp.GetRequiredService<ILoggerFactory>().CreateLogger(nameof(DatabaseInitializer));
// 1. 各模块 DbContext 自动迁移(顺序:rag → platform → file → im → workflow)
// workflow 传 suppressPendingChanges:true:该模块注册 DbContext 时未抑制
// PendingModelChangesWarning(有未提交的模型变更但不影响已有 migration),单独处理避免告警。
await MigrateAsync<RagDbContext>(sp, logger);
await MigrateAsync<PlatformDbContext>(sp, logger);
await MigrateAsync<FileDbContext>(sp, logger);
await MigrateAsync<ImDbContext>(sp, logger);
await MigrateAsync<WorkflowDbContext>(sp, logger, suppressPendingChanges: true);
// 2. IM MongoDB 建索引(可选,Mongo 不可用时跳过)
await SafeRunAsync(
async () =>
{
var mongoContext = sp.GetService<MongoDbContext>();
if (mongoContext is not null)
await mongoContext.CreateIndexesAsync();
},
logger,
"IM MongoDB 建索引");
// 3. 种子数据:平台(权限/角色/用户/菜单)+ workflow(组件注册)
await SafeRunAsync(
async () =>
{
var platformDb = sp.GetRequiredService<PlatformDbContext>();
await PlatformSeedData.SeedAsync(platformDb);
var wfDb = sp.GetRequiredService<WorkflowDbContext>();
await WorkflowSeedData.SeedAsync(wfDb);
},
logger,
"Seed");
}
/// <summary>
/// 迁移单个 DbContext,失败仅记录警告。<paramref name="suppressPendingChanges"/> 为 true 时,
/// 用独立的 options(抑制 PendingModelChangesWarning)新建临时 DbContext 完成迁移,避免污染主注册。
/// </summary>
private static async Task MigrateAsync<T>(
IServiceProvider sp,
ILogger logger,
bool suppressPendingChanges = false)
where T : DbContext
{
try
{
if (suppressPendingChanges)
{
var db = sp.GetRequiredService<T>();
var options = new DbContextOptionsBuilder<T>()
.UseNpgsql(db.Database.GetConnectionString())
.ConfigureWarnings(w => w.Ignore(RelationalEventId.PendingModelChangesWarning))
.Options;
await using var tmp = (T)Activator.CreateInstance(typeof(T), options)!;
await tmp.Database.MigrateAsync();
}
else
{
await sp.GetRequiredService<T>().Database.MigrateAsync();
}
logger.LogInformation("[ok] {Name} migrated", typeof(T).Name);
}
catch (Exception ex)
{
logger.LogWarning(ex, "[warn] {Name} migrate 跳过: {Message}", typeof(T).Name, ex.Message);
}
}
/// <summary>包裹一个启动步骤,失败仅记录警告(含步骤名)不中断主流程。</summary>
private static async Task SafeRunAsync(Func<Task> action, ILogger logger, string stepName)
{
try
{
await action();
}
catch (Exception ex)
{
logger.LogWarning(ex, "[warn] {Step} 跳过: {Message}", stepName, ex.Message);
}
}
}

View File

@ -0,0 +1,34 @@
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Routing;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Volo.Abp.DependencyInjection;
namespace BuildFlow.Host.Configuration;
/// <summary>
/// 注册 ABP 与 ASP.NET Core 管线之间的双向访问桥。
/// ABP 模块的 <c>OnApplicationInitialization</c> 通过 <c>context.GetApplicationBuilder()</c> /
/// <c>context.GetEndpointRouteBuilder()</c> 取回这些 ObjectAccessor,从而在 ABP 模块里装配中间件、
/// 映射 Hub 等。每个 accessor 同时注册「具体类型」与 <c>IObjectAccessor&lt;T&gt;</c> 接口两份,
/// 兼容 ABP 内部按接口解析与宿主按具体类型解析两种获取方式。
/// </summary>
public static class HostObjectAccessors
{
public static IServiceCollection AddHostObjectAccessors(this IServiceCollection services)
{
services.AddAccessor(new ObjectAccessor<IApplicationBuilder>());
services.AddAccessor(new ObjectAccessor<WebApplication>());
services.AddAccessor(new ObjectAccessor<IHost>());
services.AddAccessor(new ObjectAccessor<IEndpointRouteBuilder>());
return services;
}
/// <summary>同时注册 accessor 的具体类型与 <see cref="IObjectAccessor{T}"/> 接口,单一注册点避免重复。</summary>
private static void AddAccessor<T>(this IServiceCollection services, ObjectAccessor<T> accessor)
where T : class
{
services.AddSingleton(accessor);
services.AddSingleton<IObjectAccessor<T>>(accessor);
}
}

View File

@ -1,10 +1,6 @@
using BuildFlow.Host;
using File.Infrastructure.Persistence;
using IM.Infrastructure.Mongo;
using IM.Infrastructure.Persistence;
using BuildFlow.Host.Configuration;
using OpenTelemetry.Logs;
using Volo.Abp.DependencyInjection;
using Workflow.Infrastructure.Persistence;
var builder = WebApplication.CreateBuilder(args);
@ -15,85 +11,16 @@ builder.Logging.AddOpenTelemetry(logging =>
logging.AddOtlpExporter(options => options.Endpoint = new Uri(otlpEndpoint));
});
// ABP 模块化引导:注册 ObjectAccessor,打通 ABP 模块与 ASP.NET Core 管线的双向访问。
var appBuilderAccessor = new ObjectAccessor<Microsoft.AspNetCore.Builder.IApplicationBuilder>();
var webAppAccessor = new ObjectAccessor<WebApplication>();
var hostAccessor = new ObjectAccessor<Microsoft.Extensions.Hosting.IHost>();
var endpointAccessor = new ObjectAccessor<Microsoft.AspNetCore.Routing.IEndpointRouteBuilder>();
builder.Services.AddSingleton(appBuilderAccessor);
builder.Services.AddSingleton<IObjectAccessor<Microsoft.AspNetCore.Builder.IApplicationBuilder>>(appBuilderAccessor);
builder.Services.AddSingleton(webAppAccessor);
builder.Services.AddSingleton<IObjectAccessor<Microsoft.AspNetCore.Builder.WebApplication>>(webAppAccessor);
builder.Services.AddSingleton(hostAccessor);
builder.Services.AddSingleton<IObjectAccessor<Microsoft.Extensions.Hosting.IHost>>(hostAccessor);
builder.Services.AddSingleton(endpointAccessor);
builder.Services.AddSingleton<IObjectAccessor<Microsoft.AspNetCore.Routing.IEndpointRouteBuilder>>(endpointAccessor);
// BuildFlow Host 引导:按 DependsOn 顺序加载各模块的 Application/Infrastructure 模块
// ABP 模块化引导:
// 1) 注册 ObjectAccessor,打通 ABP 模块与 ASP.NET Core 管线(模块内 MapHub/UseMiddleware 依赖)
// 2) 按 [DependsOn] 加载 platform/rag/file/im/workflow 各模块的 Application/Infrastructure 层
builder.Services.AddHostObjectAccessors();
await builder.AddApplicationAsync<BuildFlowHostModule>();
var app = builder.Build();
await app.InitializeApplicationAsync();
// 启动时自动迁移所有模块的数据库(容错:单库失败不影响其他库)
using (var scope = app.Services.CreateScope())
{
var sp = scope.ServiceProvider;
// 各模块 DbContext 自动 Migrate(有 migration 的)或 EnsureCreated(无 migration 的)
// rag/workflow 均有未提交的模型变更(PendingModelChanges),已在模块注册时抑制该警告
await MigrateDbAsync<BuildFlow.Infrastructure.Persistence.RagDbContext>(sp);
await MigrateDbAsync<BuildFlow.Platform.Infrastructure.Persistence.PlatformDbContext>(sp);
await MigrateDbAsync<FileDbContext>(sp);
await MigrateDbAsync<ImDbContext>(sp);
await MigrateDbAsync<WorkflowDbContext>(sp, suppressPendingChanges: true);
// IM MongoDB 建索引
var mongoContext = sp.GetService<MongoDbContext>();
if (mongoContext is not null)
{
try { await mongoContext.CreateIndexesAsync(); }
catch (Exception ex) { Console.WriteLine($"[warn] IM MongoDB 建索引跳过: {ex.GetType().Name}"); }
}
// seed 数据
try
{
// 平台种子(权限/角色/用户/菜单)——用 PlatformDbContext
var platformDb = sp.GetRequiredService<BuildFlow.Platform.Infrastructure.Persistence.PlatformDbContext>();
await BuildFlow.Platform.Infrastructure.Persistence.SeedData.SeedAsync(platformDb);
// workflow 种子(组件注册)
var wfDb = sp.GetRequiredService<WorkflowDbContext>();
await SeedData.SeedAsync(wfDb);
}
catch (Exception ex) { Console.WriteLine($"[warn] Seed 跳过: {ex.Message}"); }
}
// 启动期数据库初始化:迁移各库 + IM MongoDB 建索引 + 种子数据(单库失败不影响其他库)
await DatabaseInitializer.RunAsync(app.Services);
await app.RunAsync();
// ===== 辅助方法 =====
static async Task MigrateDbAsync<T>(IServiceProvider sp, bool suppressPendingChanges = false) where T : DbContext
{
try
{
var db = sp.GetRequiredService<T>();
if (suppressPendingChanges)
{
// 抑制 PendingModelChangesWarning(workflow 有未提交的模型变更但不影响已有 migration)
var optionsBuilder = new DbContextOptionsBuilder<T>();
optionsBuilder.UseNpgsql(db.Database.GetConnectionString());
optionsBuilder.ConfigureWarnings(w => w.Ignore(Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning));
var tmpDb = (T)Activator.CreateInstance(typeof(T), optionsBuilder.Options)!;
await tmpDb.Database.MigrateAsync();
await tmpDb.DisposeAsync();
}
else
{
await db.Database.MigrateAsync();
}
Console.WriteLine($"[ok] {typeof(T).Name} migrated");
}
catch (Exception ex) { Console.WriteLine($"[warn] {typeof(T).Name} migrate 跳过: {ex.Message}"); }
}

View File

@ -1,17 +1,36 @@
using File.Application.Events;
using File.Application.Features.Files.Commands;
using File.Domain.Entities;
using File.Infrastructure.Persistence;
using File.Infrastructure.Storage;
using MediatR;
using Microsoft.EntityFrameworkCore;
namespace File.Application.Features.Files.Handlers;
/// <summary>删除文件命令处理器。删 S3 对象后发布 FileDeletedEvent。</summary>
public class DeleteFileCommandHandler(IS3StorageService s3, IMediator mediator)
/// <summary>
/// 删除文件命令处理器。先删 S3 对象,再删 FileMeta 记录,最后发布 FileDeletedEvent。
/// 两者必须同步删除:若只删 S3 不删库,文件夹视图(读 DB)仍会显示已失效的文件;
/// 若只删库不删 S3,对象存储会残留孤儿对象。S3 删除失败时阻断(避免库删了对象还在)。
/// </summary>
public class DeleteFileCommandHandler(IS3StorageService s3, FileDbContext db, IMediator mediator)
: IRequestHandler<DeleteFileCommand, Unit>
{
public async Task<Unit> Handle(DeleteFileCommand req, CancellationToken ct)
{
// 1. 删 S3 对象(先删存储,成功后再清库记录,保证不会出现"库没了但对象还在"的孤儿)
await s3.DeleteFileAsync(req.BucketName, req.ObjectKey, ct);
// 2. 删 FileMeta 记录(按 bucket + key 精确匹配,可能有 0~N 条)
var metas = await db.Files
.Where(f => f.S3Bucket == req.BucketName && f.S3Key == req.ObjectKey)
.ToListAsync(ct);
if (metas.Count > 0)
{
db.Files.RemoveRange(metas);
await db.SaveChangesAsync(ct);
}
await mediator.Publish(new FileDeletedEvent(req.BucketName, req.ObjectKey), ct);
return Unit.Value;
}

View File

@ -13,6 +13,11 @@ public static class SeedData
// 权限
await SeedPermissionsAsync(db);
// 清理已废弃的 OrderManagement 分组残留权限及其角色关联。
// 该分组在业务下线后已从种子移除,但历史数据库中仍有残留记录,
// 会导致权限管理树出现空分组。此清理幂等,删完后不再命中。
await PurgeDeprecatedOrderPermissionsAsync(db);
// 角色
if (!await db.Roles.AnyAsync())
{
@ -109,6 +114,46 @@ public static class SeedData
await db.SaveChangesAsync();
}
/// <summary>
/// 清理已废弃的 OrderManagement 分组权限及其角色关联。
/// 业务下线后该分组已从种子移除,但历史库残留会导致权限树出现空分组。
/// 权限实体实现 ISoftDelete,Remove 由 AuditInterceptor 转为软删除,
/// 默认查询带全局过滤器,管理端不再可见;RolePermission 无软删除,按物理删除处理。
/// 幂等:残留清理完毕后不再命中任何记录。
/// </summary>
private static async Task PurgeDeprecatedOrderPermissionsAsync(PlatformDbContext db)
{
// 先定位废弃分组的权限 Id(IgnoreQueryFilters 以兼顾此前可能已软删的记录,统一清理其关联)。
var deprecatedPermissionIds = await db.Permissions
.IgnoreQueryFilters()
.Where(p => p.Group == "OrderManagement")
.Select(p => p.Id)
.ToListAsync();
if (deprecatedPermissionIds.Count == 0) return;
// 物理删除角色-权限关联(该表仅 IAuditable,无软删除)。
var rolePermissions = await db.RolePermissions
.Where(rp => deprecatedPermissionIds.Contains(rp.PermissionId))
.ToListAsync();
if (rolePermissions.Count > 0)
{
db.RolePermissions.RemoveRange(rolePermissions);
}
// 软删除权限本身(ISoftDelete -> AuditInterceptor 置 IsDeleted=true,管理端默认查询不再返回)。
var permissions = await db.Permissions
.IgnoreQueryFilters()
.Where(p => deprecatedPermissionIds.Contains(p.Id) && !p.IsDeleted)
.ToListAsync();
if (permissions.Count > 0)
{
db.Permissions.RemoveRange(permissions);
}
// 仅有关联清理而无权限本身待删时,也需提交关联删除。
await db.SaveChangesAsync();
}
private static async Task EnsureDefaultRolePermissionsAsync(PlatformDbContext db)
{
var superAdminRoleId = Guid.Parse("20000000-0000-0000-0000-000000000001");

View File

@ -0,0 +1,162 @@
using BuildFlow.Shared.Domain.Common;
using BuildFlow.Shared.Infrastructure;
using File.Application.Features.Files.Commands;
using File.Application.Features.Files.Handlers;
using File.Domain.Entities;
using File.Infrastructure.Persistence;
using File.Infrastructure.Storage;
using FluentAssertions;
using Microsoft.EntityFrameworkCore;
using Moq;
using Xunit;
namespace BuildFlow.File.Tests;
/// <summary>
/// 删除文件命令处理器测试。核心回归:删除必须同时清理 S3 对象和 FileMeta 记录,
/// 否则文件夹视图(读 DB)仍显示已失效文件,或 S3 残留孤儿对象。
/// </summary>
public class DeleteFileCommandHandlerTests
{
private const string Bucket = "build-flow";
private const string Key = "multipart/abc123";
/// <summary>构造测试依赖:InMemory DB(挂审计拦截器自动填充非空字段)+ Mock S3 + Mock Mediator。</summary>
private static (FileDbContext db, Mock<IS3StorageService> s3, Mock<MediatR.IMediator> mediator) CreateDeps(string dbName)
{
var user = new Mock<ICurrentUserContext>();
user.SetupGet(u => u.UserId).Returns("user-001");
user.SetupGet(u => u.IPAddress).Returns("127.0.0.1");
var s3 = new Mock<IS3StorageService>();
var mediator = new Mock<MediatR.IMediator>();
var options = new DbContextOptionsBuilder<FileDbContext>()
.UseInMemoryDatabase(dbName)
.AddInterceptors(new AuditInterceptor(user.Object))
.Options;
return (new FileDbContext(options), s3, mediator);
}
/// <summary>构造一条 FileMeta 记录。</summary>
private static FileMeta NewMeta(string bucket, string key, string owner = "user-001")
=> new()
{
Id = Guid.NewGuid(),
Name = key,
S3Key = key,
S3Bucket = bucket,
Size = 1024,
ContentType = "text/plain",
OwnerId = owner,
};
[Fact]
public async Task Delete_存在FileMeta_同时删S3和数据库记录()
{
// Arrange:DB 有一条匹配的 FileMeta
var (db, s3, mediator) = CreateDeps(nameof(Delete_存在FileMeta_同时删S3和数据库记录));
db.Files.Add(NewMeta(Bucket, Key));
await db.SaveChangesAsync();
var handler = new DeleteFileCommandHandler(s3.Object, db, mediator.Object);
// Act
await handler.Handle(new DeleteFileCommand(Bucket, Key), CancellationToken.None);
// Assert:S3 对象已删
s3.Verify(x => x.DeleteFileAsync(Bucket, Key, It.IsAny<CancellationToken>()), Times.Once);
// Assert:FileMeta 记录已删(核心回归点——之前漏删导致文件夹视图仍显示)
var remaining = await db.Files.ToListAsync();
remaining.Should().BeEmpty("删除后 FileMeta 记录必须同步清理");
}
[Fact]
public async Task Delete_无FileMeta记录_仍删S3对象_不报错()
{
// Arrange:DB 无匹配记录(如根目录直传未落库、或已被其他途径清理)
var (db, s3, mediator) = CreateDeps(nameof(Delete_无FileMeta记录_仍删S3对象_不报错));
var handler = new DeleteFileCommandHandler(s3.Object, db, mediator.Object);
// Act
await handler.Handle(new DeleteFileCommand(Bucket, Key), CancellationToken.None);
// Assert:S3 仍尝试删除(幂等,删不存在的对象也返回成功)
s3.Verify(x => x.DeleteFileAsync(Bucket, Key, It.IsAny<CancellationToken>()), Times.Once);
}
[Fact]
public async Task Delete_多条同bucket同key的FileMeta_全部清理()
{
// Arrange:极端场景——同 bucket+key 存在多条 FileMeta(历史数据或并发落库)
var (db, s3, mediator) = CreateDeps(nameof(Delete_多条同bucket同key的FileMeta_全部清理));
db.Files.Add(NewMeta(Bucket, Key, "user-001"));
db.Files.Add(NewMeta(Bucket, Key, "user-002"));
await db.SaveChangesAsync();
var handler = new DeleteFileCommandHandler(s3.Object, db, mediator.Object);
// Act
await handler.Handle(new DeleteFileCommand(Bucket, Key), CancellationToken.None);
// Assert:两条记录都被清理
var remaining = await db.Files.Where(f => f.S3Bucket == Bucket && f.S3Key == Key).ToListAsync();
remaining.Should().BeEmpty("同 bucket+key 的所有 FileMeta 都应被清理");
}
[Fact]
public async Task Delete_不影响其他文件的FileMeta()
{
// Arrange:删 A 文件,B 文件应不受影响
var (db, s3, mediator) = CreateDeps(nameof(Delete_不影响其他文件的FileMeta));
db.Files.Add(NewMeta(Bucket, "multipart/aaa"));
db.Files.Add(NewMeta(Bucket, "multipart/bbb"));
await db.SaveChangesAsync();
var handler = new DeleteFileCommandHandler(s3.Object, db, mediator.Object);
// Act:只删 aaa
await handler.Handle(new DeleteFileCommand(Bucket, "multipart/aaa"), CancellationToken.None);
// Assert:aaa 没了,bbb 还在
var remaining = await db.Files.ToListAsync();
remaining.Should().HaveCount(1);
remaining[0].S3Key.Should().Be("multipart/bbb");
}
[Fact]
public async Task Delete_S3删除失败_抛异常不删数据库()
{
// Arrange:S3 删除抛异常(网络故障/权限不足等)
var (db, s3, mediator) = CreateDeps(nameof(Delete_S3删除失败_抛异常不删数据库));
db.Files.Add(NewMeta(Bucket, Key));
await db.SaveChangesAsync();
s3.Setup(x => x.DeleteFileAsync(Bucket, Key, It.IsAny<CancellationToken>()))
.ThrowsAsync(new Exception("S3 连接失败"));
var handler = new DeleteFileCommandHandler(s3.Object, db, mediator.Object);
// Act
var act = () => handler.Handle(new DeleteFileCommand(Bucket, Key), CancellationToken.None);
// Assert:异常向上抛
await act.Should().ThrowAsync<Exception>().WithMessage("S3 连接失败");
// Assert:FileMeta 仍在(S3 删失败时不删库,避免库删了但对象还在的孤儿)
var remaining = await db.Files.ToListAsync();
remaining.Should().HaveCount(1, "S3 删除失败时不应清理数据库记录");
}
[Fact]
public async Task Delete_发布FileDeletedEvent()
{
// Arrange
var (db, s3, mediator) = CreateDeps(nameof(Delete_发布FileDeletedEvent));
var handler = new DeleteFileCommandHandler(s3.Object, db, mediator.Object);
// Act
await handler.Handle(new DeleteFileCommand(Bucket, Key), CancellationToken.None);
// Assert:事件已发布(供其他模块响应,如清理缓存)
mediator.Verify(
x => x.Publish(It.IsAny<global::File.Application.Events.FileDeletedEvent>(), It.IsAny<CancellationToken>()),
Times.Once);
}
}