feat: 文件大小回填与文件夹递归总大小计算

- IS3StorageService 新增 GetObjectSizeAsync(HeadObject 取 ContentLength)
- 新增 FileSizeBackfillHelper:读路径惰性回填历史 Size=0 的文件
- 新增 FolderSizeCalculator:后序 DFS 计算文件夹递归总大小
- GetRootFiles/GetFolder 加载时回填;GetFolderTree 计算并返回 FolderDto.Size
- FolderDto 增加 Size 字段,FolderMapper.ToDto 适配

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
ljw559 2026-07-06 00:45:46 +08:00
parent d45d655a5a
commit 4ec64c773a
9 changed files with 183 additions and 13 deletions

View File

@ -0,0 +1,52 @@
using File.Domain.Entities;
using File.Infrastructure.Persistence;
using File.Infrastructure.Storage;
namespace File.Application.Common;
/// <summary>
/// 文件大小回填助手:对 Size==0 的 FileMeta,从 S3 HeadObject 取真实 ContentLength 写回。
/// 用于读路径(列表加载)惰性修复早期未记录大小的旧数据——首次加载即回填并持久化,
/// 后续加载直接命中 DB 不再访问 S3。对象缺失或空文件保留 Size=0;单个失败不中断整体。
/// </summary>
public static class FileSizeBackfillHelper
{
public static async Task BackfillAsync(
FileDbContext db,
IS3StorageService s3,
IReadOnlyList<FileMeta> files,
CancellationToken ct)
{
var stale = files.Where(f => f.Size == 0).ToList();
if (stale.Count == 0)
{
return;
}
var updated = 0;
foreach (var file in stale)
{
try
{
var size = await s3.GetObjectSizeAsync(file.S3Bucket, file.S3Key, ct);
if (size is null || size <= 0)
{
// 对象缺失(已被外部清理)或空文件:保留 Size=0
continue;
}
file.Size = size.Value;
updated++;
}
catch
{
// 单个对象查询失败不中断整体回填,保留 Size=0 下次可重试
}
}
if (updated > 0)
{
await db.SaveChangesAsync(ct);
}
}
}

View File

@ -0,0 +1,57 @@
using File.Domain.Entities;
namespace File.Application.Common;
/// <summary>
/// 文件夹递归大小计算器:给定全部文件夹 + 每个文件夹的直接文件总大小,
/// 用后序 DFS 算出每个文件夹的递归总大小(自身文件 + 所有子孙文件夹文件)。
/// 用于 getFolderTree 在返回前一次性算出所有文件夹的 Size。
/// </summary>
public static class FolderSizeCalculator
{
public static Dictionary<Guid, long> Compute(
IReadOnlyList<Folder> folders,
IReadOnlyDictionary<Guid, long> filesSizeByFolder)
{
// parentId → 子文件夹 id 列表(根文件夹 parentId=null 归到 Guid.Empty 桶,不会被当文件夹遍历)
var childrenByParent = folders
.GroupBy(f => f.ParentId ?? Guid.Empty)
.ToDictionary(g => g.Key, g => g.Select(f => f.Id).ToList());
var result = new Dictionary<Guid, long>(folders.Count);
var inStack = new HashSet<Guid>(); // 防环(理论上 ParentId 不该成环,保险起见)
long Dfs(Guid id)
{
if (result.TryGetValue(id, out var cached))
{
return cached;
}
if (!inStack.Add(id))
{
return 0; // 检测到环,断开避免死循环
}
var sum = filesSizeByFolder.GetValueOrDefault(id, 0L);
if (childrenByParent.TryGetValue(id, out var children))
{
foreach (var childId in children)
{
sum += Dfs(childId);
}
}
inStack.Remove(id);
result[id] = sum;
return sum;
}
foreach (var folder in folders)
{
Dfs(folder.Id);
}
return result;
}
}

View File

@ -6,5 +6,5 @@ namespace File.Application.Features.Folders.Commands;
/// <summary>Folder ↔ FolderDto 映射工具。</summary> /// <summary>Folder ↔ FolderDto 映射工具。</summary>
internal static class FolderMapper internal static class FolderMapper
{ {
public static FolderDto ToDto(Folder f) => new(f.Id, f.ParentId, f.Name, f.OwnerId, f.CreatedAt, f.UpdatedAt); public static FolderDto ToDto(Folder f, long size = 0) => new(f.Id, f.ParentId, f.Name, f.OwnerId, f.CreatedAt, f.UpdatedAt, size);
} }

View File

@ -3,5 +3,5 @@
namespace File.Application.Features.Folders.DTOs; namespace File.Application.Features.Folders.DTOs;
/// <summary>文件夹 DTO。</summary> /// <summary>文件夹 DTO。Size = 该文件夹下所有文件递归汇总的字节大小(由 getFolderTree 计算)。</summary>
public record FolderDto(Guid Id, Guid? ParentId, string Name, string OwnerId, DateTime CreatedAt, DateTime UpdatedAt); public record FolderDto(Guid Id, Guid? ParentId, string Name, string OwnerId, DateTime CreatedAt, DateTime UpdatedAt, long Size = 0);

View File

@ -1,15 +1,23 @@
using BuildFlow.Shared.Domain.Common; using BuildFlow.Shared.Domain.Common;
using File.Application.Common;
using File.Application.Features.Folders.Commands; using File.Application.Features.Folders.Commands;
using File.Application.Features.Folders.DTOs; using File.Application.Features.Folders.DTOs;
using File.Application.Features.Folders.Queries; using File.Application.Features.Folders.Queries;
using File.Infrastructure.Persistence; using File.Infrastructure.Persistence;
using File.Infrastructure.Storage;
using MediatR; using MediatR;
namespace File.Application.Features.Folders.Handlers; namespace File.Application.Features.Folders.Handlers;
/// <summary>查询单文件夹处理器。校验归属后聚合子文件夹与文件清单。</summary> /// <summary>
public class GetFolderQueryHandler(FileDbContext db, ICurrentUserContext currentUser) /// 查询单文件夹处理器。校验归属后聚合子文件夹与文件清单。
: IRequestHandler<GetFolderQuery, FolderWithChildrenDto?> /// 加载时对 Size==0 的旧文件记录惰性回填(从 S3 HeadObject 取真实大小写回),
/// 这样移动文件进文件夹后再加载也会自动补全大小。
/// </summary>
public class GetFolderQueryHandler(
FileDbContext db,
ICurrentUserContext currentUser,
IS3StorageService s3) : IRequestHandler<GetFolderQuery, FolderWithChildrenDto?>
{ {
public async Task<FolderWithChildrenDto?> Handle(GetFolderQuery req, CancellationToken ct) public async Task<FolderWithChildrenDto?> Handle(GetFolderQuery req, CancellationToken ct)
{ {
@ -23,9 +31,11 @@ public class GetFolderQueryHandler(FileDbContext db, ICurrentUserContext current
.OrderBy(f => f.Name) .OrderBy(f => f.Name)
.ToListAsync(ct); .ToListAsync(ct);
await FileSizeBackfillHelper.BackfillAsync(db, s3, files, ct);
return new FolderWithChildrenDto( return new FolderWithChildrenDto(
FolderMapper.ToDto(folder), FolderMapper.ToDto(folder),
subFolders.Select(FolderMapper.ToDto).ToList(), subFolders.Select(f => FolderMapper.ToDto(f)).ToList(),
files.Select(f => new FileMetaDto(f.Id, f.FolderId, f.Name, f.S3Key, f.S3Bucket, files.Select(f => new FileMetaDto(f.Id, f.FolderId, f.Name, f.S3Key, f.S3Bucket,
f.Size, f.ContentType, f.OwnerId, f.CreatedAt, f.UpdatedAt)).ToList()); f.Size, f.ContentType, f.OwnerId, f.CreatedAt, f.UpdatedAt)).ToList());
} }

View File

@ -1,20 +1,43 @@
using BuildFlow.Shared.Domain.Common; using BuildFlow.Shared.Domain.Common;
using File.Application.Common;
using File.Application.Features.Folders.Commands; using File.Application.Features.Folders.Commands;
using File.Application.Features.Folders.DTOs; using File.Application.Features.Folders.DTOs;
using File.Application.Features.Folders.Queries; using File.Application.Features.Folders.Queries;
using File.Infrastructure.Persistence; using File.Infrastructure.Persistence;
using File.Infrastructure.Storage;
using MediatR; using MediatR;
namespace File.Application.Features.Folders.Handlers; namespace File.Application.Features.Folders.Handlers;
/// <summary>文件夹树查询处理器。返回当前用户范围内的全部文件夹。</summary> /// <summary>
public class GetFolderTreeQueryHandler(FileDbContext db, ICurrentUserContext currentUser) /// 文件夹树查询处理器。返回当前用户范围内的全部文件夹。
: IRequestHandler<GetFolderTreeQuery, List<FolderDto>> /// 同时计算每个文件夹的递归总大小(自身文件 + 所有子孙文件夹文件),写入 FolderDto.Size。
/// 计算前先回填历史 Size=0 的文件,保证汇总准确。
/// </summary>
public class GetFolderTreeQueryHandler(
FileDbContext db,
ICurrentUserContext currentUser,
IS3StorageService s3) : IRequestHandler<GetFolderTreeQuery, List<FolderDto>>
{ {
public async Task<List<FolderDto>> Handle(GetFolderTreeQuery req, CancellationToken ct) public async Task<List<FolderDto>> Handle(GetFolderTreeQuery req, CancellationToken ct)
{ {
var owner = currentUser.GetRequiredUserId(); var owner = currentUser.GetRequiredUserId();
var folders = await db.Folders.Where(f => f.OwnerId == owner).ToListAsync(ct); var folders = await db.Folders.Where(f => f.OwnerId == owner).ToListAsync(ct);
return folders.Select(FolderMapper.ToDto).ToList();
// 加载该用户全部文件:先回填历史 Size=0 记录(保证文件夹大小准确),再按文件夹汇总
var files = await db.Files.Where(f => f.OwnerId == owner).ToListAsync(ct);
await FileSizeBackfillHelper.BackfillAsync(db, s3, files, ct);
var filesSizeByFolder = files
.Where(f => f.FolderId.HasValue)
.GroupBy(f => f.FolderId!.Value)
.ToDictionary(g => g.Key, g => g.Sum(f => f.Size));
var sizes = FolderSizeCalculator.Compute(folders, filesSizeByFolder);
return folders
.Select(f => FolderMapper.ToDto(f, sizes.GetValueOrDefault(f.Id, 0L)))
.ToList();
} }
} }

View File

@ -1,7 +1,9 @@
using BuildFlow.Shared.Domain.Common; using BuildFlow.Shared.Domain.Common;
using File.Application.Common;
using File.Application.Features.Folders.DTOs; using File.Application.Features.Folders.DTOs;
using File.Application.Features.Folders.Queries; using File.Application.Features.Folders.Queries;
using File.Infrastructure.Persistence; using File.Infrastructure.Persistence;
using File.Infrastructure.Storage;
using MediatR; using MediatR;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
@ -10,9 +12,12 @@ namespace File.Application.Features.Folders.Handlers;
/// <summary> /// <summary>
/// 查询根级文件处理器。返回当前用户名下 FolderId=null 的所有 FileMeta(按名称排序)。 /// 查询根级文件处理器。返回当前用户名下 FolderId=null 的所有 FileMeta(按名称排序)。
/// 与 GetFolderQueryHandler 配合,使根目录与子文件夹共享同一数据源(DB),而非 S3 列表。 /// 与 GetFolderQueryHandler 配合,使根目录与子文件夹共享同一数据源(DB),而非 S3 列表。
/// 加载时对 Size==0 的旧记录惰性回填(从 S3 HeadObject 取真实大小写回)。
/// </summary> /// </summary>
public class GetRootFilesQueryHandler(FileDbContext db, ICurrentUserContext currentUser) public class GetRootFilesQueryHandler(
: IRequestHandler<GetRootFilesQuery, List<FileMetaDto>> FileDbContext db,
ICurrentUserContext currentUser,
IS3StorageService s3) : IRequestHandler<GetRootFilesQuery, List<FileMetaDto>>
{ {
public async Task<List<FileMetaDto>> Handle(GetRootFilesQuery req, CancellationToken ct) public async Task<List<FileMetaDto>> Handle(GetRootFilesQuery req, CancellationToken ct)
{ {
@ -22,6 +27,8 @@ public class GetRootFilesQueryHandler(FileDbContext db, ICurrentUserContext curr
.OrderBy(f => f.Name) .OrderBy(f => f.Name)
.ToListAsync(ct); .ToListAsync(ct);
await FileSizeBackfillHelper.BackfillAsync(db, s3, files, ct);
return files.Select(f => new FileMetaDto(f.Id, f.FolderId, f.Name, f.S3Key, f.S3Bucket, return files.Select(f => new FileMetaDto(f.Id, f.FolderId, f.Name, f.S3Key, f.S3Bucket,
f.Size, f.ContentType, f.OwnerId, f.CreatedAt, f.UpdatedAt)).ToList(); f.Size, f.ContentType, f.OwnerId, f.CreatedAt, f.UpdatedAt)).ToList();
} }

View File

@ -26,6 +26,12 @@ public interface IS3StorageService
/// </summary> /// </summary>
Task<bool> ObjectExistsAsync(string bucket, string key, CancellationToken ct); Task<bool> ObjectExistsAsync(string bucket, string key, CancellationToken ct);
/// <summary>
/// 取对象字节大小(HeadObject → ContentLength)。对象不存在返回 null。
/// 用于回填历史 FileMeta.Size=0 的记录(早期未记录大小的旧数据)。
/// </summary>
Task<long?> GetObjectSizeAsync(string bucket, string key, CancellationToken ct);
// 分片上传 // 分片上传
Task<string> CreateMultipartUploadAsync(string bucket, string key, CancellationToken ct); Task<string> CreateMultipartUploadAsync(string bucket, string key, CancellationToken ct);
Task<string> UploadPartAsync(string bucket, string key, string uploadId, int partNumber, Stream data, CancellationToken ct); Task<string> UploadPartAsync(string bucket, string key, string uploadId, int partNumber, Stream data, CancellationToken ct);

View File

@ -166,6 +166,21 @@ public class S3StorageService : IS3StorageService
catch (Exception ex) { throw Wrap(ex); } catch (Exception ex) { throw Wrap(ex); }
} }
public async Task<long?> GetObjectSizeAsync(string bucket, string key, CancellationToken ct)
{
try
{
// HeadObject 取 ContentLength;不存在抛 NotFound → 返回 null
var resp = await _client.GetObjectMetadataAsync(bucket, key, ct);
return resp?.ContentLength;
}
catch (Amazon.S3.AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
{
return null;
}
catch (Exception ex) { throw Wrap(ex); }
}
public async Task<string> CreateMultipartUploadAsync(string bucket, string key, CancellationToken ct) public async Task<string> CreateMultipartUploadAsync(string bucket, string key, CancellationToken ct)
{ {
try try