perf: stream paged excel exports

This commit is contained in:
xiangning 2026-07-24 10:57:02 +08:00
parent de4241e5b5
commit 4683d09488
15 changed files with 460 additions and 53 deletions

View File

@ -28,7 +28,21 @@ public class DownloadExportEndpoint(IMediator mediator) : Endpoint<DownloadExpor
var disposition = new ContentDisposition { FileName = result.FileName, Inline = false };
HttpContext.Response.Headers.ContentDisposition = disposition.ToString();
await HttpContext.Response.Body.WriteAsync(result.Content.AsMemory(0, result.Content.Length), ct);
await HttpContext.Response.Body.FlushAsync(ct);
try
{
await using var stream = new FileStream(
result.FilePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
81_920,
FileOptions.Asynchronous | FileOptions.SequentialScan);
await stream.CopyToAsync(HttpContext.Response.Body, 81_920, ct);
await HttpContext.Response.Body.FlushAsync(ct);
}
finally
{
System.IO.File.Delete(result.FilePath);
}
}
}

View File

@ -35,24 +35,58 @@ public class FormSchemaExcelService : IFormSchemaExcelService
}
/// <inheritdoc />
public byte[] BuildExport(string schemaJson, string formName, IEnumerable<string> dataJsonRows)
public async Task WriteExportAsync(
string schemaJson,
string formName,
IAsyncEnumerable<string> dataJsonRows,
Stream output,
CancellationToken ct)
{
var fields = ExtractFields(schemaJson);
using var wb = new XLWorkbook();
var ws = wb.AddWorksheet(SanitizeSheetName(formName));
WriteHeader(ws, fields);
var rowIndex = 2;
foreach (var dataJson in dataJsonRows)
using var document = SpreadsheetDocument.Create(
output,
SpreadsheetDocumentType.Workbook,
true);
var workbookPart = document.AddWorkbookPart();
workbookPart.Workbook = new Workbook();
var worksheetPart = workbookPart.AddNewPart<WorksheetPart>();
var sheets = workbookPart.Workbook.AppendChild(new Sheets());
sheets.Append(new Sheet
{
using var doc = System.Text.Json.JsonDocument.Parse(dataJson);
WriteDataRow(ws, fields, rowIndex, doc.RootElement);
rowIndex++;
Id = workbookPart.GetIdOfPart(worksheetPart),
SheetId = 1,
Name = SanitizeSheetName(formName),
});
using (var writer = OpenXmlWriter.Create(worksheetPart))
{
writer.WriteStartElement(new Worksheet());
writer.WriteStartElement(new SheetData());
WriteExportRow(
writer,
1,
fields.Select(field => string.IsNullOrWhiteSpace(field.Title) ? field.Path : field.Title));
uint rowIndex = 2;
await foreach (var dataJson in dataJsonRows.WithCancellation(ct))
{
ct.ThrowIfCancellationRequested();
using var json = System.Text.Json.JsonDocument.Parse(dataJson);
WriteExportRow(
writer,
rowIndex,
fields.Select(field =>
TryReadFieldValue(json.RootElement, field.Path, out var value)
? value
: string.Empty));
rowIndex++;
}
writer.WriteEndElement();
writer.WriteEndElement();
}
using var ms = new MemoryStream();
wb.SaveAs(ms);
return ms.ToArray();
workbookPart.Workbook.Save();
}
/// <inheritdoc />
@ -164,17 +198,24 @@ public class FormSchemaExcelService : IFormSchemaExcelService
ws.Row(rowNumber).Style.Font.FontColor = XLColor.Gray;
}
private static void WriteDataRow(IXLWorksheet ws, IReadOnlyList<FieldSummary> fields, int rowNumber, System.Text.Json.JsonElement root)
private static void WriteExportRow(
OpenXmlWriter writer,
uint rowIndex,
IEnumerable<string> values)
{
for (var i = 0; i < fields.Count; i++)
writer.WriteStartElement(new Row { RowIndex = rowIndex });
foreach (var value in values)
{
var field = fields[i];
var cell = ws.Cell(rowNumber, i + 1);
if (TryReadFieldValue(root, field.Path, out var raw))
var text = new Text(value)
{
cell.Value = raw;
}
Space = SpaceProcessingModeValues.Preserve,
};
writer.WriteElement(new Cell(new InlineString(text))
{
DataType = CellValues.InlineString,
});
}
writer.WriteEndElement();
}
private static Dictionary<string, int> BuildHeaderMap(

View File

@ -9,8 +9,13 @@ public interface IFormSchemaExcelService
/// <summary>根据 SchemaJson 生成导入模版 xlsx 字节流(仅表头 + 必填批注)。</summary>
byte[] BuildTemplate(string schemaJson, string formName);
/// <summary>根据 SchemaJson 字段顺序导出已有 FormData 列表为 xlsx 字节流。</summary>
byte[] BuildExport(string schemaJson, string formName, IEnumerable<string> dataJsonRows);
/// <summary>按 SchemaJson 字段顺序将异步数据行顺序写入 xlsx 输出流。</summary>
Task WriteExportAsync(
string schemaJson,
string formName,
IAsyncEnumerable<string> dataJsonRows,
Stream output,
CancellationToken ct);
/// <summary>从 xlsx 流逐行解析数据,并在枚举阶段强制执行行数与单元格长度上限。</summary>
IEnumerable<ExcelParsedRow> Parse(

View File

@ -1,6 +1,7 @@
using MediatR;
using Workflow.Application.BuildFlow.Common.Services;
using Workflow.Application.BuildFlow.Exports.Queries;
using Workflow.Application.BuildFlow.Imports.DTOs;
using Workflow.Infrastructure.Persistence;
namespace Workflow.Application.BuildFlow.Exports.Handlers;
@ -8,25 +9,27 @@ namespace Workflow.Application.BuildFlow.Exports.Handlers;
/// <summary>下载导入模版处理器。按 FormDefinition.SchemaJson 字段生成 xlsx。</summary>
public class DownloadImportTemplateQueryHandler(
WorkflowDbContext db,
IFormSchemaExcelService excelService) : IRequestHandler<DownloadImportTemplateQuery, ExportFormDataResult>
IFormSchemaExcelService excelService) : IRequestHandler<DownloadImportTemplateQuery, ImportTemplateResult>
{
public async Task<ExportFormDataResult> Handle(DownloadImportTemplateQuery req, CancellationToken ct)
public async Task<ImportTemplateResult> Handle(DownloadImportTemplateQuery req, CancellationToken ct)
{
var form = await db.FormDefinitions
.AsNoTracking()
.Where(f => f.Id == req.FormDefinitionId)
.Select(f => new { f.Name, f.SchemaJson })
.FirstOrDefaultAsync(ct)
?? throw new BusinessException($"表单定义 {req.FormDefinitionId} 不存在");
?? throw new NotFoundException(
ErrorCode.NotFound,
$"表单定义 {req.FormDefinitionId} 不存在");
var content = excelService.BuildTemplate(form.SchemaJson ?? "{}", form.Name);
var fileName = $"{SanitizeFileName(form.Name)}-导入模版.xlsx";
return new ExportFormDataResult(content, fileName, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
return new ImportTemplateResult(content, fileName, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
}
private static string SanitizeFileName(string name)
{
var invalid = Path.GetInvalidFileNameChars();
return new string(name.Where(c => !invalid.Contains(c)).ToArray()).Trim();
return string.Concat(name.Where(c => !invalid.Contains(c))).Trim();
}
}

View File

@ -1,15 +1,22 @@
using System.Runtime.CompilerServices;
using MediatR;
using Microsoft.Extensions.Options;
using Workflow.Application.BuildFlow.Common.Services;
using Workflow.Application.BuildFlow.Exports.Queries;
using Workflow.Application.Reliability;
using Workflow.Infrastructure.Persistence;
namespace Workflow.Application.BuildFlow.Exports.Handlers;
/// <summary>导出表单数据处理器。拉取所有 FormData.DataJson,按 Schema 字段顺序输出 xlsx。</summary>
/// <summary>按稳定 keyset 分页读取 FormData,并顺序写入临时 xlsx 文件。</summary>
public class ExportFormDataQueryHandler(
WorkflowDbContext db,
IFormSchemaExcelService excelService) : IRequestHandler<ExportFormDataQuery, ExportFormDataResult>
IFormSchemaExcelService excelService,
IOptions<WorkflowJobOptions> options) : IRequestHandler<ExportFormDataQuery, ExportFormDataResult>
{
private static readonly string ExportDirectory =
Path.Combine(Path.GetTempPath(), "buildflow-exports");
public async Task<ExportFormDataResult> Handle(ExportFormDataQuery req, CancellationToken ct)
{
var form = await db.FormDefinitions
@ -17,23 +24,95 @@ public class ExportFormDataQueryHandler(
.Where(f => f.Id == req.FormDefinitionId)
.Select(f => new { f.Name, f.SchemaJson })
.FirstOrDefaultAsync(ct)
?? throw new BusinessException($"表单定义 {req.FormDefinitionId} 不存在");
?? throw new NotFoundException(
ErrorCode.NotFound,
$"表单定义 {req.FormDefinitionId} 不存在");
var dataRows = await db.FormData
.AsNoTracking()
.Where(d => d.FormDefinitionId == req.FormDefinitionId)
.OrderBy(d => d.CreatedAt)
.Select(d => d.DataJson)
.ToListAsync(ct);
var content = excelService.BuildExport(form.SchemaJson ?? "{}", form.Name, dataRows);
Directory.CreateDirectory(ExportDirectory);
var filePath = Path.Combine(
ExportDirectory,
$"buildflow-export-{Guid.NewGuid():N}.xlsx");
var fileName = $"{SanitizeFileName(form.Name)}-{DateTime.UtcNow:yyyyMMddHHmmss}.xlsx";
return new ExportFormDataResult(content, fileName, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
try
{
await using (var output = new FileStream(
filePath,
FileMode.CreateNew,
FileAccess.ReadWrite,
FileShare.None,
81_920,
FileOptions.Asynchronous | FileOptions.SequentialScan))
{
await excelService.WriteExportAsync(
form.SchemaJson ?? "{}",
form.Name,
ReadRowsAsync(
req.FormDefinitionId,
options.Value.Import.ExportPageSize,
ct),
output,
ct);
await output.FlushAsync(ct);
}
return new ExportFormDataResult(
filePath,
fileName,
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
}
catch
{
File.Delete(filePath);
throw;
}
}
private async IAsyncEnumerable<string> ReadRowsAsync(
Guid formDefinitionId,
int pageSize,
[EnumeratorCancellation] CancellationToken ct)
{
DateTime? lastCreatedAt = null;
var lastId = Guid.Empty;
while (true)
{
var query = db.FormData
.AsNoTracking()
.Where(data => data.FormDefinitionId == formDefinitionId);
if (lastCreatedAt.HasValue)
{
var cursorCreatedAt = lastCreatedAt.Value;
var cursorId = lastId;
query = query.Where(data =>
data.CreatedAt > cursorCreatedAt
|| (data.CreatedAt == cursorCreatedAt && data.Id.CompareTo(cursorId) > 0));
}
var pageCount = 0;
await foreach (var row in query
.OrderBy(data => data.CreatedAt)
.ThenBy(data => data.Id)
.Select(data => new { data.CreatedAt, data.Id, data.DataJson })
.Take(pageSize)
.AsAsyncEnumerable()
.WithCancellation(ct))
{
pageCount++;
lastCreatedAt = row.CreatedAt;
lastId = row.Id;
yield return row.DataJson;
}
if (pageCount < pageSize)
{
yield break;
}
}
}
private static string SanitizeFileName(string name)
{
var invalid = Path.GetInvalidFileNameChars();
return new string(name.Where(c => !invalid.Contains(c)).ToArray()).Trim();
return string.Concat(name.Where(c => !invalid.Contains(c))).Trim();
}
}

View File

@ -1,6 +1,7 @@
using MediatR;
using Workflow.Application.BuildFlow.Imports.DTOs;
namespace Workflow.Application.BuildFlow.Exports.Queries;
/// <summary>下载导入模版查询。按 FormDefinition.SchemaJson 字段生成 xlsx。</summary>
public sealed record DownloadImportTemplateQuery(Guid FormDefinitionId) : IRequest<ExportFormDataResult>;
public sealed record DownloadImportTemplateQuery(Guid FormDefinitionId) : IRequest<ImportTemplateResult>;

View File

@ -2,5 +2,5 @@ using MediatR;
namespace Workflow.Application.BuildFlow.Exports.Queries;
/// <summary>导出表单数据为 xlsx 字节流查询。返回 xlsx 文件字节。</summary>
/// <summary>导出表单数据为临时 xlsx 文件的查询。</summary>
public sealed record ExportFormDataQuery(Guid FormDefinitionId) : IRequest<ExportFormDataResult>;

View File

@ -1,4 +1,7 @@
namespace Workflow.Application.BuildFlow.Exports.Queries;
/// <summary>导出结果:xlsx 字节流 + 建议文件名。</summary>
public sealed record ExportFormDataResult(byte[] Content, string FileName, string ContentType);
/// <summary>导出结果:临时 xlsx 文件路径、建议文件名和响应内容类型。</summary>
public sealed record ExportFormDataResult(
string FilePath,
string FileName,
string ContentType);

View File

@ -0,0 +1,7 @@
namespace Workflow.Application.BuildFlow.Imports.DTOs;
/// <summary>导入模板下载结果:小型 xlsx 内容、建议文件名和响应内容类型。</summary>
public sealed record ImportTemplateResult(
byte[] Content,
string FileName,
string ContentType);

View File

@ -82,7 +82,7 @@ public class FormSchemaExcelServiceTests
}
[Fact]
public void BuildExport_OutputsDataRowsByFieldOrder()
public async Task WriteExportAsync_OutputsDataRowsByFieldOrder()
{
var dataRows = new[]
{
@ -90,12 +90,29 @@ public class FormSchemaExcelServiceTests
"""{"name":"王五","amount":300,"approved":false}""",
};
var bytes = _service.BuildExport(SampleSchema, "测试表单", dataRows);
using var ms = new MemoryStream();
await _service.WriteExportAsync(
SampleSchema,
"测试表单",
ToAsyncEnumerable(dataRows),
ms,
CancellationToken.None);
ms.Position = 0;
using var ms = new MemoryStream(bytes);
using var wb = new ClosedXML.Excel.XLWorkbook(ms);
var ws = wb.Worksheets.First();
ws.Cell(2, 1).GetString().Should().Be("李四");
ws.Cell(3, 1).GetString().Should().Be("王五");
}
private static async IAsyncEnumerable<string> ToAsyncEnumerable(
IEnumerable<string> rows)
{
foreach (var row in rows)
{
yield return row;
}
await Task.CompletedTask;
}
}

View File

@ -0,0 +1,38 @@
using Workflow.Application.BuildFlow.Common.Services;
namespace BuildFlow.Workflow.Tests.Performance;
/// <summary>在导出写入阶段触发取消或指定异常,并记录实际输出路径。</summary>
public sealed class CancellingExportExcelService(
CancellationTokenSource? cancellation = null,
Exception? exception = null) : IFormSchemaExcelService
{
/// <summary>handler 传入的临时文件路径。</summary>
public string? OutputPath { get; private set; }
public byte[] BuildTemplate(string schemaJson, string formName) => [];
public IEnumerable<ExcelParsedRow> Parse(
string schemaJson,
Stream xlsxStream,
int maxRows,
int maxCellTextLength) => [];
/// <summary>记录路径后触发测试指定的失败。</summary>
public Task WriteExportAsync(
string schemaJson,
string formName,
IAsyncEnumerable<string> dataJsonRows,
Stream output,
CancellationToken ct)
{
OutputPath = (output as FileStream)?.Name;
if (cancellation is not null)
{
cancellation.Cancel();
ct.ThrowIfCancellationRequested();
}
throw exception ?? new InvalidOperationException("测试导出写入失败");
}
}

View File

@ -0,0 +1,39 @@
using System.Data.Common;
using Microsoft.EntityFrameworkCore.Diagnostics;
namespace BuildFlow.Workflow.Tests.Performance;
/// <summary>仅统计读取 b_form_data 的分页 SELECT 命令。</summary>
public sealed class CountingExportPageInterceptor : DbCommandInterceptor
{
/// <summary>已执行的表单数据分页查询数。</summary>
public int PageSelectCount { get; private set; }
public override InterceptionResult<DbDataReader> ReaderExecuting(
DbCommand command,
CommandEventData eventData,
InterceptionResult<DbDataReader> result)
{
Count(command);
return result;
}
public override ValueTask<InterceptionResult<DbDataReader>> ReaderExecutingAsync(
DbCommand command,
CommandEventData eventData,
InterceptionResult<DbDataReader> result,
CancellationToken cancellationToken = default)
{
Count(command);
return ValueTask.FromResult(result);
}
private void Count(DbCommand command)
{
if (command.CommandText.TrimStart().StartsWith("SELECT", StringComparison.OrdinalIgnoreCase)
&& command.CommandText.Contains("b_form_data", StringComparison.OrdinalIgnoreCase))
{
PageSelectCount++;
}
}
}

View File

@ -8,10 +8,12 @@ public sealed class EnumerableFormSchemaExcelService(IEnumerable<ExcelParsedRow>
{
public byte[] BuildTemplate(string schemaJson, string formName) => [];
public byte[] BuildExport(
public Task WriteExportAsync(
string schemaJson,
string formName,
IEnumerable<string> dataJsonRows) => [];
IAsyncEnumerable<string> dataJsonRows,
Stream output,
CancellationToken ct) => Task.CompletedTask;
public IEnumerable<ExcelParsedRow> Parse(
string schemaJson,

View File

@ -0,0 +1,156 @@
using ClosedXML.Excel;
using FluentAssertions;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
using Workflow.Application.BuildFlow.Common.Services;
using Workflow.Application.BuildFlow.Exports.Handlers;
using Workflow.Application.BuildFlow.Exports.Queries;
using Workflow.Application.Reliability;
using Workflow.Domain.Entities;
using Workflow.Domain.Enums;
using Xunit;
namespace BuildFlow.Workflow.Tests.Performance;
/// <summary>Excel 导出的 keyset 分页、顺序写入和临时文件清理边界测试。</summary>
[Collection(Reliability.PostgresWorkflowCollection.Name)]
public class OpenXmlExportBoundaryTests(Reliability.PostgresWorkflowFixture fixture)
{
[Fact]
public async Task TwelveHundredOneRowsUseThreePagesAndKeepStableOrder()
{
await ResetAsync();
var formId = await SeedFormAsync();
await SeedRowsAsync(formId, 1_201);
var interceptor = new CountingExportPageInterceptor();
await using var db = fixture.CreateDbContext(interceptor);
var handler = CreateHandler(db, new FormSchemaExcelService(), exportPageSize: 500);
string? filePath = null;
try
{
var result = await handler.Handle(
new ExportFormDataQuery(formId),
CancellationToken.None);
filePath = result.FilePath;
interceptor.PageSelectCount.Should().Be(3);
File.Exists(filePath).Should().BeTrue();
typeof(ExportFormDataResult).GetProperties()
.Should().NotContain(property => property.PropertyType == typeof(byte[]));
using var workbook = new XLWorkbook(filePath);
var worksheet = workbook.Worksheets.First();
worksheet.LastRowUsed()!.RowNumber().Should().Be(1_202);
for (var index = 0; index < 1_201; index++)
{
worksheet.Cell(index + 2, 1).GetString()
.Should().Be($"row-{index:D4}");
}
}
finally
{
if (filePath is not null)
{
File.Delete(filePath);
}
}
}
[Fact]
public async Task CancellationDeletesTemporaryExportFile()
{
await ResetAsync();
var formId = await SeedFormAsync();
using var cts = new CancellationTokenSource();
var excel = new CancellingExportExcelService(cts);
await using var db = fixture.CreateDbContext();
var handler = CreateHandler(db, excel, exportPageSize: 500);
var act = () => handler.Handle(new ExportFormDataQuery(formId), cts.Token);
await act.Should().ThrowAsync<OperationCanceledException>();
excel.OutputPath.Should().NotBeNull();
File.Exists(excel.OutputPath).Should().BeFalse();
}
[Fact]
public async Task WriterFailureDeletesTemporaryExportFile()
{
await ResetAsync();
var formId = await SeedFormAsync();
var excel = new CancellingExportExcelService(
exception: new InvalidOperationException("writer failed"));
await using var db = fixture.CreateDbContext();
var handler = CreateHandler(db, excel, exportPageSize: 500);
var act = () => handler.Handle(new ExportFormDataQuery(formId), CancellationToken.None);
await act.Should().ThrowAsync<InvalidOperationException>();
excel.OutputPath.Should().NotBeNull();
File.Exists(excel.OutputPath).Should().BeFalse();
}
private static ExportFormDataQueryHandler CreateHandler(
WorkflowDbContext db,
IFormSchemaExcelService excelService,
int exportPageSize)
=> new(
db,
excelService,
Options.Create(new WorkflowJobOptions
{
Import = new WorkflowJobOptions.ImportJobSection
{
ExportPageSize = exportPageSize,
},
}));
private async Task ResetAsync()
{
await using var db = fixture.CreateDbContext();
await db.FormData.ExecuteDeleteAsync();
await db.FormDefinitions.ExecuteDeleteAsync();
}
private async Task<Guid> SeedFormAsync()
{
await using var db = fixture.CreateDbContext();
var formId = Guid.NewGuid();
db.FormDefinitions.Add(new FormDefinition
{
Id = formId,
Name = "分页导出测试",
Code = $"export-{formId:N}",
Version = 1,
SchemaJson = Schema,
Status = FormStatus.Published,
});
await db.SaveChangesAsync();
return formId;
}
private async Task SeedRowsAsync(Guid formId, int count)
{
await using var db = fixture.CreateDbContext();
var start = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
db.FormData.AddRange(Enumerable.Range(0, count).Select(index => new FormData
{
Id = Guid.NewGuid(),
FormDefinitionId = formId,
InstanceId = Guid.NewGuid(),
DataJson = $$"""{"name":"row-{{index:D4}}"}""",
CreatedAt = start.AddMilliseconds(index),
UpdatedAt = start.AddMilliseconds(index),
}));
await db.SaveChangesAsync();
}
private const string Schema = """
{
"type": "object",
"properties": {
"name": { "type": "string", "title": "名称", "x-component": "Input" }
}
}
""";
}

View File

@ -213,10 +213,12 @@ public class ImportJobRecoveryTests(PostgresWorkflowFixture fixture)
{
public byte[] BuildTemplate(string schemaJson, string formName) => [];
public byte[] BuildExport(
public Task WriteExportAsync(
string schemaJson,
string formName,
IEnumerable<string> dataJsonRows) => [];
IAsyncEnumerable<string> dataJsonRows,
Stream output,
CancellationToken ct) => Task.CompletedTask;
public IEnumerable<ExcelParsedRow> Parse(
string schemaJson,