feat: 文件大小回填与文件夹递归总大小计算
- IS3StorageService 新增 GetObjectSizeAsync(HeadObject 取 ContentLength) - 新增 FileSizeBackfillHelper:读路径惰性回填历史 Size=0 的文件 - 新增 FolderSizeCalculator:后序 DFS 计算文件夹递归总大小 - GetRootFiles/GetFolder 加载时回填;GetFolderTree 计算并返回 FolderDto.Size - FolderDto 增加 Size 字段,FolderMapper.ToDto 适配 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
d45d655a5a
commit
4ec64c773a
@ -0,0 +1,52 @@
|
||||
using File.Domain.Entities;
|
||||
using File.Infrastructure.Persistence;
|
||||
using File.Infrastructure.Storage;
|
||||
|
||||
namespace File.Application.Common;
|
||||
|
||||
/// <summary>
|
||||
/// 文件大小回填助手:对 Size==0 的 FileMeta,从 S3 HeadObject 取真实 ContentLength 写回。
|
||||
/// 用于读路径(列表加载)惰性修复早期未记录大小的旧数据——首次加载即回填并持久化,
|
||||
/// 后续加载直接命中 DB 不再访问 S3。对象缺失或空文件保留 Size=0;单个失败不中断整体。
|
||||
/// </summary>
|
||||
public static class FileSizeBackfillHelper
|
||||
{
|
||||
public static async Task BackfillAsync(
|
||||
FileDbContext db,
|
||||
IS3StorageService s3,
|
||||
IReadOnlyList<FileMeta> files,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var stale = files.Where(f => f.Size == 0).ToList();
|
||||
if (stale.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var updated = 0;
|
||||
foreach (var file in stale)
|
||||
{
|
||||
try
|
||||
{
|
||||
var size = await s3.GetObjectSizeAsync(file.S3Bucket, file.S3Key, ct);
|
||||
if (size is null || size <= 0)
|
||||
{
|
||||
// 对象缺失(已被外部清理)或空文件:保留 Size=0
|
||||
continue;
|
||||
}
|
||||
|
||||
file.Size = size.Value;
|
||||
updated++;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// 单个对象查询失败不中断整体回填,保留 Size=0 下次可重试
|
||||
}
|
||||
}
|
||||
|
||||
if (updated > 0)
|
||||
{
|
||||
await db.SaveChangesAsync(ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
57
src/modules/file/Application/Common/FolderSizeCalculator.cs
Normal file
57
src/modules/file/Application/Common/FolderSizeCalculator.cs
Normal file
@ -0,0 +1,57 @@
|
||||
using File.Domain.Entities;
|
||||
|
||||
namespace File.Application.Common;
|
||||
|
||||
/// <summary>
|
||||
/// 文件夹递归大小计算器:给定全部文件夹 + 每个文件夹的直接文件总大小,
|
||||
/// 用后序 DFS 算出每个文件夹的递归总大小(自身文件 + 所有子孙文件夹文件)。
|
||||
/// 用于 getFolderTree 在返回前一次性算出所有文件夹的 Size。
|
||||
/// </summary>
|
||||
public static class FolderSizeCalculator
|
||||
{
|
||||
public static Dictionary<Guid, long> Compute(
|
||||
IReadOnlyList<Folder> folders,
|
||||
IReadOnlyDictionary<Guid, long> filesSizeByFolder)
|
||||
{
|
||||
// parentId → 子文件夹 id 列表(根文件夹 parentId=null 归到 Guid.Empty 桶,不会被当文件夹遍历)
|
||||
var childrenByParent = folders
|
||||
.GroupBy(f => f.ParentId ?? Guid.Empty)
|
||||
.ToDictionary(g => g.Key, g => g.Select(f => f.Id).ToList());
|
||||
|
||||
var result = new Dictionary<Guid, long>(folders.Count);
|
||||
var inStack = new HashSet<Guid>(); // 防环(理论上 ParentId 不该成环,保险起见)
|
||||
|
||||
long Dfs(Guid id)
|
||||
{
|
||||
if (result.TryGetValue(id, out var cached))
|
||||
{
|
||||
return cached;
|
||||
}
|
||||
|
||||
if (!inStack.Add(id))
|
||||
{
|
||||
return 0; // 检测到环,断开避免死循环
|
||||
}
|
||||
|
||||
var sum = filesSizeByFolder.GetValueOrDefault(id, 0L);
|
||||
if (childrenByParent.TryGetValue(id, out var children))
|
||||
{
|
||||
foreach (var childId in children)
|
||||
{
|
||||
sum += Dfs(childId);
|
||||
}
|
||||
}
|
||||
|
||||
inStack.Remove(id);
|
||||
result[id] = sum;
|
||||
return sum;
|
||||
}
|
||||
|
||||
foreach (var folder in folders)
|
||||
{
|
||||
Dfs(folder.Id);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@ -6,5 +6,5 @@ namespace File.Application.Features.Folders.Commands;
|
||||
/// <summary>Folder ↔ FolderDto 映射工具。</summary>
|
||||
internal static class FolderMapper
|
||||
{
|
||||
public static FolderDto ToDto(Folder f) => new(f.Id, f.ParentId, f.Name, f.OwnerId, f.CreatedAt, f.UpdatedAt);
|
||||
public static FolderDto ToDto(Folder f, long size = 0) => new(f.Id, f.ParentId, f.Name, f.OwnerId, f.CreatedAt, f.UpdatedAt, size);
|
||||
}
|
||||
|
||||
@ -3,5 +3,5 @@
|
||||
namespace File.Application.Features.Folders.DTOs;
|
||||
|
||||
|
||||
/// <summary>文件夹 DTO。</summary>
|
||||
public record FolderDto(Guid Id, Guid? ParentId, string Name, string OwnerId, DateTime CreatedAt, DateTime UpdatedAt);
|
||||
/// <summary>文件夹 DTO。Size = 该文件夹下所有文件递归汇总的字节大小(由 getFolderTree 计算)。</summary>
|
||||
public record FolderDto(Guid Id, Guid? ParentId, string Name, string OwnerId, DateTime CreatedAt, DateTime UpdatedAt, long Size = 0);
|
||||
|
||||
@ -1,15 +1,23 @@
|
||||
using BuildFlow.Shared.Domain.Common;
|
||||
using File.Application.Common;
|
||||
using File.Application.Features.Folders.Commands;
|
||||
using File.Application.Features.Folders.DTOs;
|
||||
using File.Application.Features.Folders.Queries;
|
||||
using File.Infrastructure.Persistence;
|
||||
using File.Infrastructure.Storage;
|
||||
using MediatR;
|
||||
|
||||
namespace File.Application.Features.Folders.Handlers;
|
||||
|
||||
/// <summary>查询单文件夹处理器。校验归属后聚合子文件夹与文件清单。</summary>
|
||||
public class GetFolderQueryHandler(FileDbContext db, ICurrentUserContext currentUser)
|
||||
: IRequestHandler<GetFolderQuery, FolderWithChildrenDto?>
|
||||
/// <summary>
|
||||
/// 查询单文件夹处理器。校验归属后聚合子文件夹与文件清单。
|
||||
/// 加载时对 Size==0 的旧文件记录惰性回填(从 S3 HeadObject 取真实大小写回),
|
||||
/// 这样移动文件进文件夹后再加载也会自动补全大小。
|
||||
/// </summary>
|
||||
public class GetFolderQueryHandler(
|
||||
FileDbContext db,
|
||||
ICurrentUserContext currentUser,
|
||||
IS3StorageService s3) : IRequestHandler<GetFolderQuery, FolderWithChildrenDto?>
|
||||
{
|
||||
public async Task<FolderWithChildrenDto?> Handle(GetFolderQuery req, CancellationToken ct)
|
||||
{
|
||||
@ -23,9 +31,11 @@ public class GetFolderQueryHandler(FileDbContext db, ICurrentUserContext current
|
||||
.OrderBy(f => f.Name)
|
||||
.ToListAsync(ct);
|
||||
|
||||
await FileSizeBackfillHelper.BackfillAsync(db, s3, files, ct);
|
||||
|
||||
return new FolderWithChildrenDto(
|
||||
FolderMapper.ToDto(folder),
|
||||
subFolders.Select(FolderMapper.ToDto).ToList(),
|
||||
subFolders.Select(f => FolderMapper.ToDto(f)).ToList(),
|
||||
files.Select(f => new FileMetaDto(f.Id, f.FolderId, f.Name, f.S3Key, f.S3Bucket,
|
||||
f.Size, f.ContentType, f.OwnerId, f.CreatedAt, f.UpdatedAt)).ToList());
|
||||
}
|
||||
|
||||
@ -1,20 +1,43 @@
|
||||
using BuildFlow.Shared.Domain.Common;
|
||||
using File.Application.Common;
|
||||
using File.Application.Features.Folders.Commands;
|
||||
using File.Application.Features.Folders.DTOs;
|
||||
using File.Application.Features.Folders.Queries;
|
||||
using File.Infrastructure.Persistence;
|
||||
using File.Infrastructure.Storage;
|
||||
using MediatR;
|
||||
|
||||
namespace File.Application.Features.Folders.Handlers;
|
||||
|
||||
/// <summary>文件夹树查询处理器。返回当前用户范围内的全部文件夹。</summary>
|
||||
public class GetFolderTreeQueryHandler(FileDbContext db, ICurrentUserContext currentUser)
|
||||
: IRequestHandler<GetFolderTreeQuery, List<FolderDto>>
|
||||
/// <summary>
|
||||
/// 文件夹树查询处理器。返回当前用户范围内的全部文件夹。
|
||||
/// 同时计算每个文件夹的递归总大小(自身文件 + 所有子孙文件夹文件),写入 FolderDto.Size。
|
||||
/// 计算前先回填历史 Size=0 的文件,保证汇总准确。
|
||||
/// </summary>
|
||||
public class GetFolderTreeQueryHandler(
|
||||
FileDbContext db,
|
||||
ICurrentUserContext currentUser,
|
||||
IS3StorageService s3) : IRequestHandler<GetFolderTreeQuery, List<FolderDto>>
|
||||
{
|
||||
public async Task<List<FolderDto>> Handle(GetFolderTreeQuery req, CancellationToken ct)
|
||||
{
|
||||
var owner = currentUser.GetRequiredUserId();
|
||||
|
||||
var folders = await db.Folders.Where(f => f.OwnerId == owner).ToListAsync(ct);
|
||||
return folders.Select(FolderMapper.ToDto).ToList();
|
||||
|
||||
// 加载该用户全部文件:先回填历史 Size=0 记录(保证文件夹大小准确),再按文件夹汇总
|
||||
var files = await db.Files.Where(f => f.OwnerId == owner).ToListAsync(ct);
|
||||
await FileSizeBackfillHelper.BackfillAsync(db, s3, files, ct);
|
||||
|
||||
var filesSizeByFolder = files
|
||||
.Where(f => f.FolderId.HasValue)
|
||||
.GroupBy(f => f.FolderId!.Value)
|
||||
.ToDictionary(g => g.Key, g => g.Sum(f => f.Size));
|
||||
|
||||
var sizes = FolderSizeCalculator.Compute(folders, filesSizeByFolder);
|
||||
|
||||
return folders
|
||||
.Select(f => FolderMapper.ToDto(f, sizes.GetValueOrDefault(f.Id, 0L)))
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
using BuildFlow.Shared.Domain.Common;
|
||||
using File.Application.Common;
|
||||
using File.Application.Features.Folders.DTOs;
|
||||
using File.Application.Features.Folders.Queries;
|
||||
using File.Infrastructure.Persistence;
|
||||
using File.Infrastructure.Storage;
|
||||
using MediatR;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
@ -10,9 +12,12 @@ namespace File.Application.Features.Folders.Handlers;
|
||||
/// <summary>
|
||||
/// 查询根级文件处理器。返回当前用户名下 FolderId=null 的所有 FileMeta(按名称排序)。
|
||||
/// 与 GetFolderQueryHandler 配合,使根目录与子文件夹共享同一数据源(DB),而非 S3 列表。
|
||||
/// 加载时对 Size==0 的旧记录惰性回填(从 S3 HeadObject 取真实大小写回)。
|
||||
/// </summary>
|
||||
public class GetRootFilesQueryHandler(FileDbContext db, ICurrentUserContext currentUser)
|
||||
: IRequestHandler<GetRootFilesQuery, List<FileMetaDto>>
|
||||
public class GetRootFilesQueryHandler(
|
||||
FileDbContext db,
|
||||
ICurrentUserContext currentUser,
|
||||
IS3StorageService s3) : IRequestHandler<GetRootFilesQuery, List<FileMetaDto>>
|
||||
{
|
||||
public async Task<List<FileMetaDto>> Handle(GetRootFilesQuery req, CancellationToken ct)
|
||||
{
|
||||
@ -22,6 +27,8 @@ public class GetRootFilesQueryHandler(FileDbContext db, ICurrentUserContext curr
|
||||
.OrderBy(f => f.Name)
|
||||
.ToListAsync(ct);
|
||||
|
||||
await FileSizeBackfillHelper.BackfillAsync(db, s3, files, ct);
|
||||
|
||||
return files.Select(f => new FileMetaDto(f.Id, f.FolderId, f.Name, f.S3Key, f.S3Bucket,
|
||||
f.Size, f.ContentType, f.OwnerId, f.CreatedAt, f.UpdatedAt)).ToList();
|
||||
}
|
||||
|
||||
@ -26,6 +26,12 @@ public interface IS3StorageService
|
||||
/// </summary>
|
||||
Task<bool> ObjectExistsAsync(string bucket, string key, CancellationToken ct);
|
||||
|
||||
/// <summary>
|
||||
/// 取对象字节大小(HeadObject → ContentLength)。对象不存在返回 null。
|
||||
/// 用于回填历史 FileMeta.Size=0 的记录(早期未记录大小的旧数据)。
|
||||
/// </summary>
|
||||
Task<long?> GetObjectSizeAsync(string bucket, string key, CancellationToken ct);
|
||||
|
||||
// 分片上传
|
||||
Task<string> CreateMultipartUploadAsync(string bucket, string key, CancellationToken ct);
|
||||
Task<string> UploadPartAsync(string bucket, string key, string uploadId, int partNumber, Stream data, CancellationToken ct);
|
||||
|
||||
@ -166,6 +166,21 @@ public class S3StorageService : IS3StorageService
|
||||
catch (Exception ex) { throw Wrap(ex); }
|
||||
}
|
||||
|
||||
public async Task<long?> GetObjectSizeAsync(string bucket, string key, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
// HeadObject 取 ContentLength;不存在抛 NotFound → 返回 null
|
||||
var resp = await _client.GetObjectMetadataAsync(bucket, key, ct);
|
||||
return resp?.ContentLength;
|
||||
}
|
||||
catch (Amazon.S3.AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
catch (Exception ex) { throw Wrap(ex); }
|
||||
}
|
||||
|
||||
public async Task<string> CreateMultipartUploadAsync(string bucket, string key, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user