perf: persist imports in bounded batches
This commit is contained in:
parent
4bedd17741
commit
65566dc112
@ -117,8 +117,9 @@ public class BuildFlowHostModule : AbpModule
|
|||||||
&& options.Import.StartupDelaySeconds > 0
|
&& options.Import.StartupDelaySeconds > 0
|
||||||
&& options.Import.BatchSize > 0
|
&& options.Import.BatchSize > 0
|
||||||
&& options.Import.LeaseSeconds > 0
|
&& options.Import.LeaseSeconds > 0
|
||||||
&& options.Import.MaxAttempts > 0,
|
&& options.Import.MaxAttempts > 0
|
||||||
"Workflow:Jobs 的间隔、批量、租约、重试延迟和最大尝试次数必须为正数。")
|
&& options.Import.RowsPerBatch is > 0 and <= 1000,
|
||||||
|
"Workflow:Jobs 的间隔、批量、租约、重试延迟和最大尝试次数必须为正数,导入事务行数不得超过 1000。")
|
||||||
.ValidateOnStart();
|
.ValidateOnStart();
|
||||||
services.AddNotifications(config);
|
services.AddNotifications(config);
|
||||||
services.AddHostedService<TimeoutSchedulerService>();
|
services.AddHostedService<TimeoutSchedulerService>();
|
||||||
|
|||||||
@ -68,7 +68,8 @@ public sealed class ImportJobProcessorService(
|
|||||||
_leaseOwner,
|
_leaseOwner,
|
||||||
importOptions.BatchSize,
|
importOptions.BatchSize,
|
||||||
TimeSpan.FromSeconds(importOptions.LeaseSeconds),
|
TimeSpan.FromSeconds(importOptions.LeaseSeconds),
|
||||||
importOptions.MaxAttempts),
|
importOptions.MaxAttempts,
|
||||||
|
importOptions.RowsPerBatch),
|
||||||
ct);
|
ct);
|
||||||
|
|
||||||
if (processed > 0)
|
if (processed > 0)
|
||||||
|
|||||||
@ -107,7 +107,8 @@
|
|||||||
"StartupDelaySeconds": 5,
|
"StartupDelaySeconds": 5,
|
||||||
"BatchSize": 5,
|
"BatchSize": 5,
|
||||||
"LeaseSeconds": 300,
|
"LeaseSeconds": 300,
|
||||||
"MaxAttempts": 5
|
"MaxAttempts": 5,
|
||||||
|
"RowsPerBatch": 200
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|||||||
@ -9,4 +9,5 @@ public sealed record ProcessAvailableImportJobsCommand(
|
|||||||
string LeaseOwner,
|
string LeaseOwner,
|
||||||
int BatchSize,
|
int BatchSize,
|
||||||
TimeSpan LeaseDuration,
|
TimeSpan LeaseDuration,
|
||||||
int MaxAttempts) : IRequest<int>;
|
int MaxAttempts,
|
||||||
|
int RowsPerBatch) : IRequest<int>;
|
||||||
|
|||||||
@ -32,6 +32,7 @@ public sealed class ProcessAvailableImportJobsCommandHandler(
|
|||||||
jobId,
|
jobId,
|
||||||
request.LeaseOwner,
|
request.LeaseOwner,
|
||||||
request.LeaseDuration,
|
request.LeaseDuration,
|
||||||
|
request.RowsPerBatch,
|
||||||
ct);
|
ct);
|
||||||
}
|
}
|
||||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||||
|
|||||||
@ -8,5 +8,6 @@ public interface IImportJobProcessor
|
|||||||
Guid jobId,
|
Guid jobId,
|
||||||
string leaseOwner,
|
string leaseOwner,
|
||||||
TimeSpan leaseDuration,
|
TimeSpan leaseDuration,
|
||||||
|
int rowsPerBatch,
|
||||||
CancellationToken ct);
|
CancellationToken ct);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,10 +1,7 @@
|
|||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using Workflow.Application.BuildFlow.Common.Services;
|
using Workflow.Application.BuildFlow.Common.Services;
|
||||||
using Workflow.Application.BuildFlow.Imports.Services;
|
using Workflow.Application.BuildFlow.Imports.Services;
|
||||||
using Workflow.Application.Form.Aggregates;
|
using Workflow.Application.Form.Schema;
|
||||||
using Workflow.Application.Form.Derivation;
|
|
||||||
using Workflow.Application.Form.FormData.Commands;
|
|
||||||
using Workflow.Application.Form.FormData.Handlers;
|
|
||||||
using Workflow.Domain.Entities;
|
using Workflow.Domain.Entities;
|
||||||
using Workflow.Domain.Enums;
|
using Workflow.Domain.Enums;
|
||||||
using Workflow.Infrastructure.Persistence;
|
using Workflow.Infrastructure.Persistence;
|
||||||
@ -12,14 +9,15 @@ using Workflow.Infrastructure.Persistence;
|
|||||||
namespace Workflow.Application.BuildFlow.Imports.Handlers;
|
namespace Workflow.Application.BuildFlow.Imports.Handlers;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// 导入任务核心处理器。
|
/// 导入任务核心处理器。惰性枚举 Excel 行并按有界批次提交表单数据、进度和租约心跳。
|
||||||
/// 流程:Pending → Running → 解析 xlsx → 逐行提交 FormData → 写回 Success/Failed/PartialSuccess + ErrorJson。
|
|
||||||
/// 异常会被捕获并写回 ErrorMessage + Status=Failed,避免单个任务失败拖垮后台服务。
|
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public class ImportJobProcessor(
|
public class ImportJobProcessor(
|
||||||
WorkflowDbContext db,
|
WorkflowDbContext db,
|
||||||
IFormSchemaExcelService excelService) : IImportJobProcessor
|
IFormSchemaExcelService excelService,
|
||||||
|
ImportBatchWriter batchWriter) : IImportJobProcessor
|
||||||
{
|
{
|
||||||
|
private const int MaxRowsPerBatch = 1000;
|
||||||
|
|
||||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||||
{
|
{
|
||||||
WriteIndented = false,
|
WriteIndented = false,
|
||||||
@ -29,10 +27,18 @@ public class ImportJobProcessor(
|
|||||||
Guid jobId,
|
Guid jobId,
|
||||||
string leaseOwner,
|
string leaseOwner,
|
||||||
TimeSpan leaseDuration,
|
TimeSpan leaseDuration,
|
||||||
|
int rowsPerBatch,
|
||||||
CancellationToken ct)
|
CancellationToken ct)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
|
if (rowsPerBatch is <= 0 or > MaxRowsPerBatch)
|
||||||
|
{
|
||||||
|
throw new BusinessException(
|
||||||
|
ErrorCode.InvalidParam,
|
||||||
|
$"导入批次行数必须在 1 到 {MaxRowsPerBatch} 之间");
|
||||||
|
}
|
||||||
|
|
||||||
var job = await db.BuildFlowImportJobs
|
var job = await db.BuildFlowImportJobs
|
||||||
.AsNoTracking()
|
.AsNoTracking()
|
||||||
.FirstOrDefaultAsync(x =>
|
.FirstOrDefaultAsync(x =>
|
||||||
@ -40,99 +46,88 @@ public class ImportJobProcessor(
|
|||||||
&& x.Status == BuildFlowImportJobStatus.Running
|
&& x.Status == BuildFlowImportJobStatus.Running
|
||||||
&& x.LeaseOwner == leaseOwner,
|
&& x.LeaseOwner == leaseOwner,
|
||||||
ct);
|
ct);
|
||||||
if (job is null) return;
|
if (job is null)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
var form = await db.FormDefinitions
|
var form = await db.FormDefinitions
|
||||||
.AsNoTracking()
|
.AsNoTracking()
|
||||||
.Where(f => f.Id == job.FormDefinitionId)
|
.Where(x => x.Id == job.FormDefinitionId)
|
||||||
.Select(f => new { f.SchemaJson })
|
.Select(x => new { x.SchemaJson })
|
||||||
.FirstOrDefaultAsync(ct);
|
.FirstOrDefaultAsync(ct);
|
||||||
if (form is null)
|
if (form is null)
|
||||||
{
|
{
|
||||||
throw new NotFoundException(ErrorCode.NotFound, $"表单定义 {job.FormDefinitionId} 不存在");
|
throw new NotFoundException(
|
||||||
|
ErrorCode.NotFound,
|
||||||
|
$"表单定义 {job.FormDefinitionId} 不存在");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var schemaJson = form.SchemaJson
|
||||||
|
?? """{"type":"object","properties":{}}""";
|
||||||
|
|
||||||
if (!File.Exists(job.FileUrl))
|
if (!File.Exists(job.FileUrl))
|
||||||
{
|
{
|
||||||
throw new BusinessException(ErrorCode.Business, $"上传文件不存在:{job.FileUrl}");
|
throw new BusinessException(
|
||||||
|
ErrorCode.Business,
|
||||||
|
$"上传文件不存在:{job.FileUrl}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var preparedSchema = PreparedFormSchema.Create(schemaJson);
|
||||||
var fileBytes = await File.ReadAllBytesAsync(job.FileUrl, ct);
|
var fileBytes = await File.ReadAllBytesAsync(job.FileUrl, ct);
|
||||||
var rows = excelService.Parse(form.SchemaJson ?? "{}", fileBytes);
|
var committedCursor = job.SuccessRows + job.FailedRows;
|
||||||
var nextRowIndex = await InitializeProgressAsync(
|
var batch = new List<ImportBatchItem>(rowsPerBatch);
|
||||||
jobId,
|
|
||||||
leaseOwner,
|
|
||||||
leaseDuration,
|
|
||||||
ct);
|
|
||||||
if (nextRowIndex is null) return;
|
|
||||||
|
|
||||||
var submitHandler = new SubmitFormDataCommandHandler(
|
|
||||||
db,
|
|
||||||
new FormValueDerivationService([], [], db),
|
|
||||||
new FormAggregateRefreshService(db));
|
|
||||||
|
|
||||||
var rowIndex = 0;
|
var rowIndex = 0;
|
||||||
foreach (var row in rows)
|
|
||||||
|
foreach (var row in excelService.Parse(schemaJson, fileBytes))
|
||||||
{
|
{
|
||||||
ct.ThrowIfCancellationRequested();
|
ct.ThrowIfCancellationRequested();
|
||||||
if (rowIndex < nextRowIndex.Value)
|
if (rowIndex >= committedCursor)
|
||||||
{
|
{
|
||||||
rowIndex++;
|
batch.Add(new ImportBatchItem(rowIndex, row));
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (row.Errors.Count > 0)
|
|
||||||
{
|
|
||||||
var recorded = await CommitFailedRowAsync(
|
|
||||||
jobId,
|
|
||||||
leaseOwner,
|
|
||||||
leaseDuration,
|
|
||||||
row.RowNumber,
|
|
||||||
row.Errors.Select(e => e.Message).ToList(),
|
|
||||||
rowIndex + 1,
|
|
||||||
ct);
|
|
||||||
if (!recorded) return;
|
|
||||||
rowIndex++;
|
rowIndex++;
|
||||||
continue;
|
if (batch.Count == rowsPerBatch)
|
||||||
}
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
{
|
||||||
var committed = await CommitSuccessfulRowAsync(
|
var committed = await CommitBatchAsync(
|
||||||
jobId,
|
jobId,
|
||||||
leaseOwner,
|
leaseOwner,
|
||||||
leaseDuration,
|
leaseDuration,
|
||||||
job.FormDefinitionId,
|
job.FormDefinitionId,
|
||||||
rowIndex,
|
preparedSchema,
|
||||||
row.JsonObject,
|
batch,
|
||||||
submitHandler,
|
|
||||||
ct);
|
ct);
|
||||||
if (!committed) return;
|
if (!committed)
|
||||||
}
|
|
||||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
|
||||||
{
|
{
|
||||||
throw;
|
return;
|
||||||
}
|
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
var recorded = await CommitFailedRowAsync(
|
|
||||||
jobId,
|
|
||||||
leaseOwner,
|
|
||||||
leaseDuration,
|
|
||||||
row.RowNumber,
|
|
||||||
[ex.Message],
|
|
||||||
rowIndex + 1,
|
|
||||||
ct);
|
|
||||||
if (!recorded) return;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
rowIndex++;
|
batch.Clear();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (nextRowIndex.Value > rowIndex)
|
if (committedCursor > rowIndex)
|
||||||
{
|
{
|
||||||
throw new BusinessException(
|
throw new BusinessException(
|
||||||
ErrorCode.Business,
|
ErrorCode.Business,
|
||||||
$"导入任务进度游标 {nextRowIndex.Value} 超过总行数 {rowIndex}");
|
$"导入任务进度游标 {committedCursor} 超过总行数 {rowIndex}");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (batch.Count > 0)
|
||||||
|
{
|
||||||
|
var committed = await CommitBatchAsync(
|
||||||
|
jobId,
|
||||||
|
leaseOwner,
|
||||||
|
leaseDuration,
|
||||||
|
job.FormDefinitionId,
|
||||||
|
preparedSchema,
|
||||||
|
batch,
|
||||||
|
ct);
|
||||||
|
if (!committed)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
await CompleteJobAsync(jobId, leaseOwner, rowIndex, ct);
|
await CompleteJobAsync(jobId, leaseOwner, rowIndex, ct);
|
||||||
@ -148,35 +143,13 @@ public class ImportJobProcessor(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task<int?> InitializeProgressAsync(
|
private async Task<bool> CommitBatchAsync(
|
||||||
Guid jobId,
|
|
||||||
string leaseOwner,
|
|
||||||
TimeSpan leaseDuration,
|
|
||||||
CancellationToken ct)
|
|
||||||
{
|
|
||||||
db.ChangeTracker.Clear();
|
|
||||||
await using var transaction = await db.Database.BeginTransactionAsync(ct);
|
|
||||||
var job = await LockOwnedJobAsync(jobId, leaseOwner, ct);
|
|
||||||
if (job is null)
|
|
||||||
{
|
|
||||||
await transaction.RollbackAsync(ct);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
RenewLease(job, leaseDuration);
|
|
||||||
await db.SaveChangesAsync(ct);
|
|
||||||
await transaction.CommitAsync(ct);
|
|
||||||
return job.SuccessRows + job.FailedRows;
|
|
||||||
}
|
|
||||||
|
|
||||||
private async Task<bool> CommitSuccessfulRowAsync(
|
|
||||||
Guid jobId,
|
Guid jobId,
|
||||||
string leaseOwner,
|
string leaseOwner,
|
||||||
TimeSpan leaseDuration,
|
TimeSpan leaseDuration,
|
||||||
Guid formDefinitionId,
|
Guid formDefinitionId,
|
||||||
int rowIndex,
|
PreparedFormSchema schema,
|
||||||
string dataJson,
|
IReadOnlyList<ImportBatchItem> batch,
|
||||||
SubmitFormDataCommandHandler submitHandler,
|
|
||||||
CancellationToken ct)
|
CancellationToken ct)
|
||||||
{
|
{
|
||||||
db.ChangeTracker.Clear();
|
db.ChangeTracker.Clear();
|
||||||
@ -188,41 +161,22 @@ public class ImportJobProcessor(
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
var instanceId = DeterministicInstanceId(jobId, rowIndex);
|
var result = await batchWriter.WriteAsync(
|
||||||
await submitHandler.Handle(
|
jobId,
|
||||||
new SubmitFormDataCommand(formDefinitionId, instanceId, dataJson),
|
formDefinitionId,
|
||||||
|
schema,
|
||||||
|
batch,
|
||||||
ct);
|
ct);
|
||||||
job.SuccessRows++;
|
job.SuccessRows += result.SuccessCount;
|
||||||
job.TotalRows = Math.Max(job.TotalRows, rowIndex + 1);
|
job.FailedRows += result.Errors.Count;
|
||||||
RenewLease(job, leaseDuration);
|
job.TotalRows = Math.Max(job.TotalRows, batch[^1].RowIndex + 1);
|
||||||
await db.SaveChangesAsync(ct);
|
if (result.Errors.Count > 0)
|
||||||
await transaction.CommitAsync(ct);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
private async Task<bool> CommitFailedRowAsync(
|
|
||||||
Guid jobId,
|
|
||||||
string leaseOwner,
|
|
||||||
TimeSpan leaseDuration,
|
|
||||||
int rowNumber,
|
|
||||||
IReadOnlyList<string> messages,
|
|
||||||
int processedRows,
|
|
||||||
CancellationToken ct)
|
|
||||||
{
|
{
|
||||||
db.ChangeTracker.Clear();
|
|
||||||
await using var transaction = await db.Database.BeginTransactionAsync(ct);
|
|
||||||
var job = await LockOwnedJobAsync(jobId, leaseOwner, ct);
|
|
||||||
if (job is null)
|
|
||||||
{
|
|
||||||
await transaction.RollbackAsync(ct);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
var errors = DeserializeErrors(job.ErrorJson);
|
var errors = DeserializeErrors(job.ErrorJson);
|
||||||
errors.Add(new ImportRowError(rowNumber, messages));
|
errors.AddRange(result.Errors);
|
||||||
job.FailedRows++;
|
|
||||||
job.TotalRows = Math.Max(job.TotalRows, processedRows);
|
|
||||||
job.ErrorJson = JsonSerializer.Serialize(errors, JsonOptions);
|
job.ErrorJson = JsonSerializer.Serialize(errors, JsonOptions);
|
||||||
|
}
|
||||||
|
|
||||||
RenewLease(job, leaseDuration);
|
RenewLease(job, leaseDuration);
|
||||||
await db.SaveChangesAsync(ct);
|
await db.SaveChangesAsync(ct);
|
||||||
await transaction.CommitAsync(ct);
|
await transaction.CommitAsync(ct);
|
||||||
@ -329,13 +283,4 @@ public class ImportJobProcessor(
|
|||||||
|
|
||||||
private static string Truncate(string value, int maxLength)
|
private static string Truncate(string value, int maxLength)
|
||||||
=> value.Length <= maxLength ? value : value[..maxLength];
|
=> value.Length <= maxLength ? value : value[..maxLength];
|
||||||
|
|
||||||
private static Guid DeterministicInstanceId(Guid jobId, int rowIndex)
|
|
||||||
{
|
|
||||||
// 组合 jobId 与 rowIndex 生成确定性 Guid,便于行级聚合。
|
|
||||||
var bytes = jobId.ToByteArray();
|
|
||||||
bytes[0] = (byte)(rowIndex & 0xFF);
|
|
||||||
bytes[1] = (byte)((rowIndex >> 8) & 0xFF);
|
|
||||||
return new Guid(bytes);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -53,5 +53,8 @@ public class WorkflowJobOptions
|
|||||||
|
|
||||||
/// <summary>最大领取尝试次数。</summary>
|
/// <summary>最大领取尝试次数。</summary>
|
||||||
public int MaxAttempts { get; set; } = 5;
|
public int MaxAttempts { get; set; } = 5;
|
||||||
|
|
||||||
|
/// <summary>单个导入事务最多处理的 Excel 行数。</summary>
|
||||||
|
public int RowsPerBatch { get; set; } = 200;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,22 @@
|
|||||||
|
using Workflow.Application.BuildFlow.Common.Services;
|
||||||
|
|
||||||
|
namespace BuildFlow.Workflow.Tests.Performance;
|
||||||
|
|
||||||
|
/// <summary>枚举到指定索引时触发取消的 Excel 行序列。</summary>
|
||||||
|
public sealed class CancellingExcelRows(
|
||||||
|
IReadOnlyList<ExcelParsedRow> rows,
|
||||||
|
int cancelAtIndex,
|
||||||
|
Action cancel) : IEnumerable<ExcelParsedRow>
|
||||||
|
{
|
||||||
|
public IEnumerator<ExcelParsedRow> GetEnumerator()
|
||||||
|
{
|
||||||
|
for (var index = 0; index < rows.Count; index++)
|
||||||
|
{
|
||||||
|
if (index == cancelAtIndex) cancel();
|
||||||
|
yield return rows[index];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
|
||||||
|
=> GetEnumerator();
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
using System.Data.Common;
|
||||||
|
using Microsoft.EntityFrameworkCore.Diagnostics;
|
||||||
|
|
||||||
|
namespace BuildFlow.Workflow.Tests.Performance;
|
||||||
|
|
||||||
|
/// <summary>统计 EF 显式事务提交次数的测试拦截器。</summary>
|
||||||
|
public sealed class CountingTransactionInterceptor : DbTransactionInterceptor
|
||||||
|
{
|
||||||
|
/// <summary>已提交事务数量。</summary>
|
||||||
|
public int CommitCount { get; private set; }
|
||||||
|
|
||||||
|
public override void TransactionCommitted(
|
||||||
|
DbTransaction transaction,
|
||||||
|
TransactionEndEventData eventData)
|
||||||
|
=> CommitCount++;
|
||||||
|
|
||||||
|
public override Task TransactionCommittedAsync(
|
||||||
|
DbTransaction transaction,
|
||||||
|
TransactionEndEventData eventData,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
CommitCount++;
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,17 @@
|
|||||||
|
using Workflow.Application.BuildFlow.Common.Services;
|
||||||
|
|
||||||
|
namespace BuildFlow.Workflow.Tests.Performance;
|
||||||
|
|
||||||
|
/// <summary>从给定枚举返回导入行的 Excel 服务测试替身。</summary>
|
||||||
|
public sealed class EnumerableFormSchemaExcelService(IEnumerable<ExcelParsedRow> rows)
|
||||||
|
: IFormSchemaExcelService
|
||||||
|
{
|
||||||
|
public byte[] BuildTemplate(string schemaJson, string formName) => [];
|
||||||
|
|
||||||
|
public byte[] BuildExport(
|
||||||
|
string schemaJson,
|
||||||
|
string formName,
|
||||||
|
IEnumerable<string> dataJsonRows) => [];
|
||||||
|
|
||||||
|
public IEnumerable<ExcelParsedRow> Parse(string schemaJson, byte[] xlsxBytes) => rows;
|
||||||
|
}
|
||||||
@ -0,0 +1,203 @@
|
|||||||
|
using BuildFlow.Workflow.Tests.Reliability;
|
||||||
|
using FluentAssertions;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Workflow.Application.BuildFlow.Common.Services;
|
||||||
|
using Workflow.Application.BuildFlow.Imports.Handlers;
|
||||||
|
using Workflow.Application.BuildFlow.Imports.Services;
|
||||||
|
using Workflow.Application.Form.Aggregates;
|
||||||
|
using Workflow.Application.Form.Derivation;
|
||||||
|
using Workflow.Domain.Entities;
|
||||||
|
using Workflow.Domain.Enums;
|
||||||
|
using Xunit;
|
||||||
|
|
||||||
|
namespace BuildFlow.Workflow.Tests.Performance;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 导入任务有界批次事务和取消恢复测试。
|
||||||
|
/// </summary>
|
||||||
|
[Collection(PostgresWorkflowCollection.Name)]
|
||||||
|
public class ImportJobBatchingTests(PostgresWorkflowFixture fixture)
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public async Task FourHundredOneRowsUseThreeBatchCommitsAndOneCompletionCommit()
|
||||||
|
{
|
||||||
|
await ResetAsync();
|
||||||
|
var filePath = await CreateTempFileAsync();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var seeded = await SeedImportAsync(filePath);
|
||||||
|
var rows = Enumerable.Range(0, 401)
|
||||||
|
.Select(index => ValidRow(index + 2, $"row-{index}"))
|
||||||
|
.ToArray();
|
||||||
|
var transactions = new CountingTransactionInterceptor();
|
||||||
|
await using var db = fixture.CreateDbContext(transactions);
|
||||||
|
var processor = CreateProcessor(db, rows);
|
||||||
|
|
||||||
|
await processor.ProcessAsync(
|
||||||
|
seeded.JobId,
|
||||||
|
seeded.LeaseOwner,
|
||||||
|
TimeSpan.FromMinutes(5),
|
||||||
|
200,
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
transactions.CommitCount.Should().Be(4);
|
||||||
|
await using var verify = fixture.CreateDbContext();
|
||||||
|
var job = await verify.BuildFlowImportJobs.SingleAsync(x => x.Id == seeded.JobId);
|
||||||
|
job.Status.Should().Be(BuildFlowImportJobStatus.Success);
|
||||||
|
job.TotalRows.Should().Be(401);
|
||||||
|
job.SuccessRows.Should().Be(401);
|
||||||
|
job.FailedRows.Should().Be(0);
|
||||||
|
job.LeaseOwner.Should().BeNull();
|
||||||
|
(await verify.FormData.CountAsync(x =>
|
||||||
|
x.FormDefinitionId == seeded.FormDefinitionId)).Should().Be(401);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
File.Delete(filePath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task CancellationBeforeRowTwoHundredOneKeepsFirstBatchCommitted()
|
||||||
|
{
|
||||||
|
await ResetAsync();
|
||||||
|
var filePath = await CreateTempFileAsync();
|
||||||
|
using var cts = new CancellationTokenSource();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var seeded = await SeedImportAsync(filePath);
|
||||||
|
var rows = new CancellingExcelRows(
|
||||||
|
Enumerable.Range(0, 401)
|
||||||
|
.Select(index => ValidRow(index + 2, $"row-{index}"))
|
||||||
|
.ToArray(),
|
||||||
|
cancelAtIndex: 200,
|
||||||
|
cts.Cancel);
|
||||||
|
await using var db = fixture.CreateDbContext();
|
||||||
|
var processor = CreateProcessor(db, rows);
|
||||||
|
|
||||||
|
var act = () => processor.ProcessAsync(
|
||||||
|
seeded.JobId,
|
||||||
|
seeded.LeaseOwner,
|
||||||
|
TimeSpan.FromMinutes(5),
|
||||||
|
200,
|
||||||
|
cts.Token);
|
||||||
|
|
||||||
|
await act.Should().ThrowAsync<OperationCanceledException>();
|
||||||
|
await using var verify = fixture.CreateDbContext();
|
||||||
|
var job = await verify.BuildFlowImportJobs.SingleAsync(x => x.Id == seeded.JobId);
|
||||||
|
job.Status.Should().Be(BuildFlowImportJobStatus.Pending);
|
||||||
|
job.TotalRows.Should().Be(200);
|
||||||
|
job.SuccessRows.Should().Be(200);
|
||||||
|
job.FailedRows.Should().Be(0);
|
||||||
|
job.LeaseOwner.Should().BeNull();
|
||||||
|
(await verify.FormData.CountAsync(x =>
|
||||||
|
x.FormDefinitionId == seeded.FormDefinitionId)).Should().Be(200);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
File.Delete(filePath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ExistingFormWithNullSchemaUsesEmptyObjectSchema()
|
||||||
|
{
|
||||||
|
await ResetAsync();
|
||||||
|
var filePath = await CreateTempFileAsync();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var seeded = await SeedImportAsync(filePath, schemaJson: null);
|
||||||
|
await using var db = fixture.CreateDbContext();
|
||||||
|
var processor = CreateProcessor(db, [ValidRow(2, "row")]);
|
||||||
|
|
||||||
|
await processor.ProcessAsync(
|
||||||
|
seeded.JobId,
|
||||||
|
seeded.LeaseOwner,
|
||||||
|
TimeSpan.FromMinutes(5),
|
||||||
|
1,
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
await using var verify = fixture.CreateDbContext();
|
||||||
|
var job = await verify.BuildFlowImportJobs.SingleAsync(x => x.Id == seeded.JobId);
|
||||||
|
job.Status.Should().Be(BuildFlowImportJobStatus.Success);
|
||||||
|
job.SuccessRows.Should().Be(1);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
File.Delete(filePath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ImportJobProcessor CreateProcessor(
|
||||||
|
WorkflowDbContext db,
|
||||||
|
IEnumerable<ExcelParsedRow> rows)
|
||||||
|
{
|
||||||
|
var derivation = new FormValueDerivationService([], [], db);
|
||||||
|
var aggregates = new FormAggregateRefreshService(db);
|
||||||
|
return new ImportJobProcessor(
|
||||||
|
db,
|
||||||
|
new EnumerableFormSchemaExcelService(rows),
|
||||||
|
new ImportBatchWriter(db, derivation, aggregates));
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task ResetAsync()
|
||||||
|
{
|
||||||
|
await using var db = fixture.CreateDbContext();
|
||||||
|
await db.FormData.ExecuteDeleteAsync();
|
||||||
|
await db.BuildFlowImportJobs.ExecuteDeleteAsync();
|
||||||
|
await db.FormDefinitions.ExecuteDeleteAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<(Guid JobId, Guid FormDefinitionId, string LeaseOwner)> SeedImportAsync(
|
||||||
|
string filePath,
|
||||||
|
string? schemaJson = Schema)
|
||||||
|
{
|
||||||
|
await using var db = fixture.CreateDbContext();
|
||||||
|
var formDefinitionId = Guid.NewGuid();
|
||||||
|
var jobId = Guid.NewGuid();
|
||||||
|
const string leaseOwner = "batch-import-worker";
|
||||||
|
db.FormDefinitions.Add(new FormDefinition
|
||||||
|
{
|
||||||
|
Id = formDefinitionId,
|
||||||
|
Name = "批量导入测试表单",
|
||||||
|
Code = $"batch-import-{formDefinitionId:N}",
|
||||||
|
Version = 1,
|
||||||
|
SchemaJson = schemaJson,
|
||||||
|
Status = FormStatus.Published,
|
||||||
|
});
|
||||||
|
db.BuildFlowImportJobs.Add(new BuildFlowImportJob
|
||||||
|
{
|
||||||
|
Id = jobId,
|
||||||
|
FormDefinitionId = formDefinitionId,
|
||||||
|
Status = BuildFlowImportJobStatus.Running,
|
||||||
|
FileUrl = filePath,
|
||||||
|
OriginalFileName = "batch.xlsx",
|
||||||
|
LeaseOwner = leaseOwner,
|
||||||
|
LeaseExpiresAt = DateTime.UtcNow.AddMinutes(5),
|
||||||
|
AttemptCount = 1,
|
||||||
|
LastHeartbeatAt = DateTime.UtcNow,
|
||||||
|
StartedAt = DateTime.UtcNow,
|
||||||
|
});
|
||||||
|
await db.SaveChangesAsync();
|
||||||
|
return (jobId, formDefinitionId, leaseOwner);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ExcelParsedRow ValidRow(int rowNumber, string name)
|
||||||
|
=> new(rowNumber, $$"""{"name":"{{name}}"}""", []);
|
||||||
|
|
||||||
|
private static async Task<string> CreateTempFileAsync()
|
||||||
|
{
|
||||||
|
var path = Path.Combine(Path.GetTempPath(), $"buildflow-import-{Guid.NewGuid():N}.xlsx");
|
||||||
|
await File.WriteAllBytesAsync(path, [1, 2, 3]);
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
private const string Schema = """
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"name": { "type": "string", "required": true, "x-component": "Input" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
""";
|
||||||
|
}
|
||||||
@ -97,7 +97,8 @@ public class ImportJobClaimTests(PostgresWorkflowFixture fixture)
|
|||||||
owner,
|
owner,
|
||||||
BatchSize: 10,
|
BatchSize: 10,
|
||||||
LeaseDuration: TimeSpan.FromMinutes(5),
|
LeaseDuration: TimeSpan.FromMinutes(5),
|
||||||
MaxAttempts: 5);
|
MaxAttempts: 5,
|
||||||
|
RowsPerBatch: 200);
|
||||||
|
|
||||||
private static ProcessAvailableImportJobsCommandHandler CreateHandler(
|
private static ProcessAvailableImportJobsCommandHandler CreateHandler(
|
||||||
WorkflowDbContext db,
|
WorkflowDbContext db,
|
||||||
@ -144,9 +145,10 @@ public class ImportJobClaimTests(PostgresWorkflowFixture fixture)
|
|||||||
Guid jobId,
|
Guid jobId,
|
||||||
string leaseOwner,
|
string leaseOwner,
|
||||||
TimeSpan leaseDuration,
|
TimeSpan leaseDuration,
|
||||||
|
int rowsPerBatch,
|
||||||
CancellationToken ct)
|
CancellationToken ct)
|
||||||
{
|
{
|
||||||
Calls.Enqueue(new ImportJobCall(jobId, leaseOwner, leaseDuration));
|
Calls.Enqueue(new ImportJobCall(jobId, leaseOwner, leaseDuration, rowsPerBatch));
|
||||||
return Task.CompletedTask;
|
return Task.CompletedTask;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -154,5 +156,6 @@ public class ImportJobClaimTests(PostgresWorkflowFixture fixture)
|
|||||||
private sealed record ImportJobCall(
|
private sealed record ImportJobCall(
|
||||||
Guid JobId,
|
Guid JobId,
|
||||||
string LeaseOwner,
|
string LeaseOwner,
|
||||||
TimeSpan LeaseDuration);
|
TimeSpan LeaseDuration,
|
||||||
|
int RowsPerBatch);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -2,6 +2,9 @@ using FluentAssertions;
|
|||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Workflow.Application.BuildFlow.Common.Services;
|
using Workflow.Application.BuildFlow.Common.Services;
|
||||||
using Workflow.Application.BuildFlow.Imports.Handlers;
|
using Workflow.Application.BuildFlow.Imports.Handlers;
|
||||||
|
using Workflow.Application.BuildFlow.Imports.Services;
|
||||||
|
using Workflow.Application.Form.Aggregates;
|
||||||
|
using Workflow.Application.Form.Derivation;
|
||||||
using Workflow.Domain.Entities;
|
using Workflow.Domain.Entities;
|
||||||
using Workflow.Domain.Enums;
|
using Workflow.Domain.Enums;
|
||||||
using Xunit;
|
using Xunit;
|
||||||
@ -23,18 +26,18 @@ public class ImportJobRecoveryTests(PostgresWorkflowFixture fixture)
|
|||||||
{
|
{
|
||||||
var seeded = await SeedImportAsync(filePath, successRows: 1);
|
var seeded = await SeedImportAsync(filePath, successRows: 1);
|
||||||
await using var db = fixture.CreateDbContext();
|
await using var db = fixture.CreateDbContext();
|
||||||
var processor = new ImportJobProcessor(
|
var processor = CreateProcessor(
|
||||||
db,
|
db,
|
||||||
new StubExcelService(
|
|
||||||
[
|
[
|
||||||
ValidRow(2, """{"name":"first"}"""),
|
ValidRow(2, """{"name":"first"}"""),
|
||||||
ValidRow(3, """{"name":"second"}"""),
|
ValidRow(3, """{"name":"second"}"""),
|
||||||
]));
|
]);
|
||||||
|
|
||||||
await processor.ProcessAsync(
|
await processor.ProcessAsync(
|
||||||
seeded.JobId,
|
seeded.JobId,
|
||||||
seeded.LeaseOwner,
|
seeded.LeaseOwner,
|
||||||
TimeSpan.FromMinutes(5),
|
TimeSpan.FromMinutes(5),
|
||||||
|
1,
|
||||||
CancellationToken.None);
|
CancellationToken.None);
|
||||||
|
|
||||||
await using var verify = fixture.CreateDbContext();
|
await using var verify = fixture.CreateDbContext();
|
||||||
@ -61,14 +64,15 @@ public class ImportJobRecoveryTests(PostgresWorkflowFixture fixture)
|
|||||||
{
|
{
|
||||||
var seeded = await SeedImportAsync(filePath);
|
var seeded = await SeedImportAsync(filePath);
|
||||||
await using var db = fixture.CreateDbContext();
|
await using var db = fixture.CreateDbContext();
|
||||||
var processor = new ImportJobProcessor(
|
var processor = CreateProcessor(
|
||||||
db,
|
db,
|
||||||
new StubExcelService([ValidRow(2, """{"name":"only"}""")]));
|
[ValidRow(2, """{"name":"only"}""")]);
|
||||||
|
|
||||||
await processor.ProcessAsync(
|
await processor.ProcessAsync(
|
||||||
seeded.JobId,
|
seeded.JobId,
|
||||||
seeded.LeaseOwner,
|
seeded.LeaseOwner,
|
||||||
TimeSpan.FromMinutes(5),
|
TimeSpan.FromMinutes(5),
|
||||||
|
1,
|
||||||
CancellationToken.None);
|
CancellationToken.None);
|
||||||
|
|
||||||
await using var verify = fixture.CreateDbContext();
|
await using var verify = fixture.CreateDbContext();
|
||||||
@ -102,12 +106,13 @@ public class ImportJobRecoveryTests(PostgresWorkflowFixture fixture)
|
|||||||
cancelAtIndex: 1,
|
cancelAtIndex: 1,
|
||||||
cts.Cancel);
|
cts.Cancel);
|
||||||
await using var db = fixture.CreateDbContext();
|
await using var db = fixture.CreateDbContext();
|
||||||
var processor = new ImportJobProcessor(db, new StubExcelService(rows));
|
var processor = CreateProcessor(db, rows);
|
||||||
|
|
||||||
var act = () => processor.ProcessAsync(
|
var act = () => processor.ProcessAsync(
|
||||||
seeded.JobId,
|
seeded.JobId,
|
||||||
seeded.LeaseOwner,
|
seeded.LeaseOwner,
|
||||||
TimeSpan.FromMinutes(5),
|
TimeSpan.FromMinutes(5),
|
||||||
|
1,
|
||||||
cts.Token);
|
cts.Token);
|
||||||
|
|
||||||
await act.Should().ThrowAsync<OperationCanceledException>();
|
await act.Should().ThrowAsync<OperationCanceledException>();
|
||||||
@ -184,6 +189,18 @@ public class ImportJobRecoveryTests(PostgresWorkflowFixture fixture)
|
|||||||
private static ExcelParsedRow ValidRow(int rowNumber, string json)
|
private static ExcelParsedRow ValidRow(int rowNumber, string json)
|
||||||
=> new(rowNumber, json, []);
|
=> new(rowNumber, json, []);
|
||||||
|
|
||||||
|
private static ImportJobProcessor CreateProcessor(
|
||||||
|
WorkflowDbContext db,
|
||||||
|
IReadOnlyList<ExcelParsedRow> rows)
|
||||||
|
{
|
||||||
|
var derivation = new FormValueDerivationService([], [], db);
|
||||||
|
var aggregates = new FormAggregateRefreshService(db);
|
||||||
|
return new ImportJobProcessor(
|
||||||
|
db,
|
||||||
|
new StubExcelService(rows),
|
||||||
|
new ImportBatchWriter(db, derivation, aggregates));
|
||||||
|
}
|
||||||
|
|
||||||
private static async Task<string> CreateTempFileAsync()
|
private static async Task<string> CreateTempFileAsync()
|
||||||
{
|
{
|
||||||
var path = Path.Combine(Path.GetTempPath(), $"buildflow-import-{Guid.NewGuid():N}.xlsx");
|
var path = Path.Combine(Path.GetTempPath(), $"buildflow-import-{Guid.NewGuid():N}.xlsx");
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user