From 1e51e6a9650e2cec4bee5eedcfe1385badce0520 Mon Sep 17 00:00:00 2001 From: xiangning Date: Thu, 23 Jul 2026 17:59:37 +0800 Subject: [PATCH] feat: bound Obsidian filesystem access --- src/BuildFlow.Host/appsettings.json | 8 + .../Obsidian/ObsidianOptions.cs | 22 ++ .../Obsidian/ObsidianVaultFile.cs | 11 + .../Obsidian/ObsidianVaultScanner.cs | 340 ++++++++++++++++++ .../Infrastructure/RAGInfrastructureModule.cs | 8 + .../ObsidianVaultScannerCollection.cs | 12 + .../ObsidianVaultScannerTests.cs | 174 +++++++++ 7 files changed, 575 insertions(+) create mode 100644 src/modules/rag/Infrastructure/Obsidian/ObsidianOptions.cs create mode 100644 src/modules/rag/Infrastructure/Obsidian/ObsidianVaultFile.cs create mode 100644 src/modules/rag/Infrastructure/Obsidian/ObsidianVaultScanner.cs create mode 100644 tests/BuildFlow.Rag.Tests/ObsidianVaultScannerCollection.cs create mode 100644 tests/BuildFlow.Rag.Tests/ObsidianVaultScannerTests.cs diff --git a/src/BuildFlow.Host/appsettings.json b/src/BuildFlow.Host/appsettings.json index d6de368..3f0f4aa 100644 --- a/src/BuildFlow.Host/appsettings.json +++ b/src/BuildFlow.Host/appsettings.json @@ -84,6 +84,14 @@ "MaxTokens": 2000, "Temperature": 0.7 }, + "Rag": { + "Obsidian": { + "AllowedRoots": [], + "MaxFiles": 10000, + "MaxFileBytes": 10485760, + "MaxTotalBytes": 536870912 + } + }, "Monitoring": { "Scheduler": { "PollIntervalSeconds": 60, diff --git a/src/modules/rag/Infrastructure/Obsidian/ObsidianOptions.cs b/src/modules/rag/Infrastructure/Obsidian/ObsidianOptions.cs new file mode 100644 index 0000000..5e8bceb --- /dev/null +++ b/src/modules/rag/Infrastructure/Obsidian/ObsidianOptions.cs @@ -0,0 +1,22 @@ +namespace BuildFlow.Infrastructure.Obsidian; + +/// +/// Obsidian 服务端目录访问与容量限制配置。 +/// +public sealed class ObsidianOptions +{ + /// 配置节名称。 + public const string SectionName = "Rag:Obsidian"; + + /// 允许访问的服务端根目录；空集合表示禁用目录同步。 + public string[] AllowedRoots { get; set; } = []; + + /// 单次同步允许的最大 Markdown 文件数量。 + public int MaxFiles { get; set; } = 10_000; + + /// 单个 Markdown 文件允许的最大字节数。 + public long MaxFileBytes { get; set; } = 10 * 1024 * 1024; + + /// 单次同步全部 Markdown 文件允许的最大累计字节数。 + public long MaxTotalBytes { get; set; } = 512 * 1024 * 1024; +} diff --git a/src/modules/rag/Infrastructure/Obsidian/ObsidianVaultFile.cs b/src/modules/rag/Infrastructure/Obsidian/ObsidianVaultFile.cs new file mode 100644 index 0000000..f1a30a5 --- /dev/null +++ b/src/modules/rag/Infrastructure/Obsidian/ObsidianVaultFile.cs @@ -0,0 +1,11 @@ +namespace BuildFlow.Infrastructure.Obsidian; + +/// +/// 通过服务端路径和容量安全检查后的 Obsidian Markdown 文件元数据。 +/// +public sealed record ObsidianVaultFile( + string FullPath, + string RelativePath, + string FileName, + long FileSize, + string ContentHash); diff --git a/src/modules/rag/Infrastructure/Obsidian/ObsidianVaultScanner.cs b/src/modules/rag/Infrastructure/Obsidian/ObsidianVaultScanner.cs new file mode 100644 index 0000000..407524d --- /dev/null +++ b/src/modules/rag/Infrastructure/Obsidian/ObsidianVaultScanner.cs @@ -0,0 +1,340 @@ +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using BuildFlow.Shared.Domain.Exceptions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace BuildFlow.Infrastructure.Obsidian; + +/// +/// 在允许根目录内流式枚举 Obsidian Markdown 文件，并以流式 SHA256 生成增量同步元数据。 +/// +public sealed class ObsidianVaultScanner( + IOptions options, + ILogger logger) +{ + private static readonly HashSet IgnoredDirectories = + new(StringComparer.OrdinalIgnoreCase) { ".obsidian", ".trash", ".git", "node_modules" }; + private readonly ObsidianOptions _options = options.Value; + + /// + /// 流式扫描指定 Vault。遍历过程中持续执行数量和大小门禁，不在内存中缓存文件内容。 + /// + public async IAsyncEnumerable ScanAsync( + string requestedDirectory, + [EnumeratorCancellation] CancellationToken ct = default) + { + var (vaultRoot, allowedRoot) = ResolveVaultRoot(requestedDirectory); + var pending = new Stack<(DirectoryInfo Directory, string RelativeDirectory)>(); + var visited = new HashSet(PathComparer); + pending.Push((new DirectoryInfo(vaultRoot), string.Empty)); + var fileCount = 0; + long totalBytes = 0; + + while (pending.TryPop(out var current)) + { + ThrowIfCancellationRequested(ct, current.Directory.FullName); + var currentPath = ResolveEntry(current.Directory, allowedRoot); + if (!visited.Add(currentPath)) continue; + + using var entries = new DirectoryInfo(currentPath) + .EnumerateFileSystemInfos() + .GetEnumerator(); + + while (MoveNext(entries, currentPath, out var entry)) + { + ThrowIfCancellationRequested(ct, currentPath); + if (entry is null) continue; + + FileAttributes attributes; + try + { + attributes = entry.Attributes; + } + catch (FileNotFoundException ex) + { + logger.LogWarning(ex, "Obsidian 路径在扫描期间消失：{Path}", entry.FullName); + continue; + } + catch (DirectoryNotFoundException ex) + { + logger.LogWarning(ex, "Obsidian 路径在扫描期间消失：{Path}", entry.FullName); + continue; + } + + if ((attributes & FileAttributes.Directory) != 0) + { + if (IgnoredDirectories.Contains(entry.Name)) continue; + var childPath = ResolveEntry(entry, allowedRoot); + var childRelative = string.IsNullOrEmpty(current.RelativeDirectory) + ? entry.Name + : Path.Combine(current.RelativeDirectory, entry.Name); + pending.Push((new DirectoryInfo(childPath), childRelative)); + continue; + } + + if (!string.Equals(entry.Extension, ".md", StringComparison.OrdinalIgnoreCase)) + continue; + + var physicalFile = ResolveEntry(entry, allowedRoot); + var fileLength = ReadFileLength(physicalFile); + if (fileLength is null) continue; + + fileCount++; + if (fileCount > _options.MaxFiles) + { + throw new BusinessException( + ErrorCode.TooLarge, + "Obsidian 文件数量超过配置上限"); + } + + if (fileLength.Value > _options.MaxFileBytes) + { + throw new BusinessException( + ErrorCode.TooLarge, + $"Obsidian 文件超过单文件大小上限：{entry.Name}"); + } + + if (fileLength.Value > _options.MaxTotalBytes - totalBytes) + { + throw new BusinessException( + ErrorCode.TooLarge, + "Obsidian 文件累计大小超过配置上限"); + } + totalBytes += fileLength.Value; + + byte[] digest; + try + { + await using var stream = new FileStream( + physicalFile, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + 64 * 1024, + FileOptions.Asynchronous | FileOptions.SequentialScan); + digest = await SHA256.HashDataAsync(stream, ct); + } + catch (FileNotFoundException ex) + { + logger.LogWarning(ex, "Obsidian 文件在读取前消失：{Path}", physicalFile); + continue; + } + catch (DirectoryNotFoundException ex) + { + logger.LogWarning(ex, "Obsidian 文件目录在读取前消失：{Path}", physicalFile); + continue; + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + logger.LogInformation("Obsidian 文件扫描已取消：{Path}", physicalFile); + throw; + } + catch (UnauthorizedAccessException ex) + { + logger.LogWarning(ex, "Obsidian 文件无读取权限：{Path}", physicalFile); + throw new BusinessException(ErrorCode.Business, "Obsidian 文件无读取权限"); + } + catch (IOException ex) + { + logger.LogWarning(ex, "Obsidian 文件读取失败：{Path}", physicalFile); + throw new BusinessException(ErrorCode.Business, "Obsidian 文件读取失败"); + } + + var relativePath = string.IsNullOrEmpty(current.RelativeDirectory) + ? entry.Name + : Path.Combine(current.RelativeDirectory, entry.Name); + yield return new ObsidianVaultFile( + physicalFile, + relativePath.Replace('\\', '/'), + entry.Name, + fileLength.Value, + Convert.ToHexString(digest).ToLowerInvariant()); + } + } + } + + private (string VaultRoot, string AllowedRoot) ResolveVaultRoot(string requestedDirectory) + { + if (_options.AllowedRoots.Length == 0) + throw new BusinessException(ErrorCode.InvalidParam, "未配置允许的 Obsidian 根目录"); + if (string.IsNullOrWhiteSpace(requestedDirectory)) + throw new BusinessException(ErrorCode.InvalidParam, "Obsidian 路径不能为空"); + + var segments = requestedDirectory.Split( + [Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar], + StringSplitOptions.RemoveEmptyEntries); + if (segments.Contains("..", StringComparer.Ordinal)) + throw new BusinessException(ErrorCode.InvalidParam, "Obsidian 路径不能包含父目录跳转"); + + string requested; + try + { + requested = TrimPath(Path.GetFullPath(requestedDirectory)); + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException) + { + throw new BusinessException(ErrorCode.InvalidParam, "Obsidian 路径格式无效"); + } + + foreach (var configuredRoot in _options.AllowedRoots.Where(x => !string.IsNullOrWhiteSpace(x))) + { + var lexicalRoot = TrimPath(Path.GetFullPath(configuredRoot)); + if (!Directory.Exists(lexicalRoot) || !IsWithin(requested, lexicalRoot)) continue; + + var physicalRoot = ResolveLink(new DirectoryInfo(lexicalRoot)); + var relative = Path.GetRelativePath(lexicalRoot, requested); + var physicalVault = ResolveRelativeDirectory(physicalRoot, relative); + EnsureWithin(physicalVault, physicalRoot); + return (physicalVault, physicalRoot); + } + + throw new BusinessException(ErrorCode.InvalidParam, "Obsidian 路径不在允许的根目录内"); + } + + private string ResolveRelativeDirectory(string physicalRoot, string relative) + { + var current = physicalRoot; + if (relative == ".") return current; + + foreach (var segment in relative.Split( + [Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar], + StringSplitOptions.RemoveEmptyEntries)) + { + var next = new DirectoryInfo(Path.Combine(current, segment)); + if (!next.Exists) + throw new BusinessException(ErrorCode.InvalidParam, "Vault 目录不存在"); + current = ResolveLink(next); + EnsureWithin(current, physicalRoot); + } + + return current; + } + + private string ResolveEntry(FileSystemInfo entry, string allowedRoot) + { + string resolved; + try + { + resolved = ResolveLink(entry); + } + catch (FileNotFoundException ex) + { + logger.LogWarning(ex, "Obsidian 路径在解析时消失：{Path}", entry.FullName); + throw new BusinessException(ErrorCode.Business, "Obsidian 路径在扫描期间消失"); + } + catch (UnauthorizedAccessException ex) + { + logger.LogWarning(ex, "Obsidian 路径无访问权限：{Path}", entry.FullName); + throw new BusinessException(ErrorCode.Business, "Obsidian 路径无访问权限"); + } + catch (IOException ex) + { + logger.LogWarning(ex, "Obsidian 路径解析失败：{Path}", entry.FullName); + throw new BusinessException(ErrorCode.Business, "Obsidian 路径解析失败"); + } + + EnsureWithin(resolved, allowedRoot); + return resolved; + } + + private static string ResolveLink(FileSystemInfo entry) + { + if ((entry.Attributes & FileAttributes.ReparsePoint) == 0) + return TrimPath(Path.GetFullPath(entry.FullName)); + + var target = entry.ResolveLinkTarget(returnFinalTarget: true) + ?? throw new BusinessException(ErrorCode.InvalidParam, "Obsidian 符号链接目标不存在"); + return TrimPath(Path.GetFullPath(target.FullName)); + } + + private long? ReadFileLength(string physicalFile) + { + try + { + var fileInfo = new FileInfo(physicalFile); + fileInfo.Refresh(); + if (fileInfo.Exists) return fileInfo.Length; + logger.LogWarning("Obsidian 文件在扫描期间消失：{Path}", physicalFile); + return null; + } + catch (FileNotFoundException ex) + { + logger.LogWarning(ex, "Obsidian 文件在扫描期间消失：{Path}", physicalFile); + return null; + } + catch (UnauthorizedAccessException ex) + { + logger.LogWarning(ex, "Obsidian 文件无读取权限：{Path}", physicalFile); + throw new BusinessException(ErrorCode.Business, "Obsidian 文件无读取权限"); + } + catch (IOException ex) + { + logger.LogWarning(ex, "Obsidian 文件元数据读取失败：{Path}", physicalFile); + throw new BusinessException(ErrorCode.Business, "Obsidian 文件元数据读取失败"); + } + } + + private bool MoveNext( + IEnumerator entries, + string directory, + out FileSystemInfo? entry) + { + try + { + if (!entries.MoveNext()) + { + entry = null; + return false; + } + + entry = entries.Current; + return true; + } + catch (UnauthorizedAccessException ex) + { + logger.LogWarning(ex, "Obsidian 目录无枚举权限：{Path}", directory); + throw new BusinessException(ErrorCode.Business, "Obsidian 目录无枚举权限"); + } + catch (IOException ex) + { + logger.LogWarning(ex, "Obsidian 目录枚举失败：{Path}", directory); + throw new BusinessException(ErrorCode.Business, "Obsidian 目录枚举失败"); + } + } + + private void ThrowIfCancellationRequested(CancellationToken ct, string path) + { + if (!ct.IsCancellationRequested) return; + logger.LogInformation("Obsidian 目录扫描已取消：{Path}", path); + ct.ThrowIfCancellationRequested(); + } + + private static void EnsureWithin(string path, string root) + { + if (!IsWithin(path, root)) + { + throw new BusinessException( + ErrorCode.InvalidParam, + "Obsidian 符号链接逃逸允许根目录"); + } + } + + private static bool IsWithin(string path, string root) + { + var relative = Path.GetRelativePath(root, path); + return relative == "." || + (!Path.IsPathRooted(relative) && + relative != ".." && + !relative.StartsWith($"..{Path.DirectorySeparatorChar}", PathComparison) && + !relative.StartsWith($"..{Path.AltDirectorySeparatorChar}", PathComparison)); + } + + private static string TrimPath(string path) => Path.TrimEndingDirectorySeparator(path); + + private static StringComparison PathComparison => + OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + + private static StringComparer PathComparer => + OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; +} diff --git a/src/modules/rag/Infrastructure/RAGInfrastructureModule.cs b/src/modules/rag/Infrastructure/RAGInfrastructureModule.cs index d4d9cff..91c0cd2 100644 --- a/src/modules/rag/Infrastructure/RAGInfrastructureModule.cs +++ b/src/modules/rag/Infrastructure/RAGInfrastructureModule.cs @@ -8,6 +8,7 @@ using BuildFlow.Infrastructure.Cache; using BuildFlow.Infrastructure.Extractors; using BuildFlow.Infrastructure.Files; using BuildFlow.Infrastructure.Messaging; +using BuildFlow.Infrastructure.Obsidian; using BuildFlow.Infrastructure.Persistence; using BuildFlow.Shared.Infrastructure; using Volo.Abp.Modularity; @@ -50,6 +51,13 @@ public class RAGInfrastructureModule : AbpModule services.AddScoped(); services.AddSingleton(); + services.AddOptions() + .Bind(config.GetSection(ObsidianOptions.SectionName)) + .Validate(x => x.MaxFiles > 0, "Rag:Obsidian:MaxFiles 必须大于 0") + .Validate(x => x.MaxFileBytes > 0, "Rag:Obsidian:MaxFileBytes 必须大于 0") + .Validate(x => x.MaxTotalBytes > 0, "Rag:Obsidian:MaxTotalBytes 必须大于 0"); + services.AddScoped(); + // 文档提取器：各格式提取器 + 工厂按扩展名分发（参考 Dify ExtractProcessor） services.AddSingleton(); services.AddSingleton(); diff --git a/tests/BuildFlow.Rag.Tests/ObsidianVaultScannerCollection.cs b/tests/BuildFlow.Rag.Tests/ObsidianVaultScannerCollection.cs new file mode 100644 index 0000000..81cc7e1 --- /dev/null +++ b/tests/BuildFlow.Rag.Tests/ObsidianVaultScannerCollection.cs @@ -0,0 +1,12 @@ +using Xunit; + +namespace BuildFlow.Rag.Tests; + +/// +/// 隔离 Obsidian 扫描器的托管内存测量，避免并行测试制造分配噪声。 +/// +[CollectionDefinition("ObsidianVaultScanner", DisableParallelization = true)] +public sealed class ObsidianVaultScannerCollection +{ + public const string Name = "ObsidianVaultScanner"; +} diff --git a/tests/BuildFlow.Rag.Tests/ObsidianVaultScannerTests.cs b/tests/BuildFlow.Rag.Tests/ObsidianVaultScannerTests.cs new file mode 100644 index 0000000..39c54f4 --- /dev/null +++ b/tests/BuildFlow.Rag.Tests/ObsidianVaultScannerTests.cs @@ -0,0 +1,174 @@ +using System.Security.Cryptography; +using BuildFlow.Infrastructure.Obsidian; +using BuildFlow.Shared.Domain.Errors; +using BuildFlow.Shared.Domain.Exceptions; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Xunit; + +namespace BuildFlow.Rag.Tests; + +/// +/// Obsidian 服务端目录访问和容量边界测试。 +/// +[Collection(ObsidianVaultScannerCollection.Name)] +public sealed class ObsidianVaultScannerTests : IDisposable +{ + private readonly string _testRoot = Path.Combine( + Path.GetTempPath(), + $"buildflow-obsidian-{Guid.NewGuid():N}"); + private readonly string _allowedRoot; + private readonly string _outsideRoot; + + public ObsidianVaultScannerTests() + { + _allowedRoot = Directory.CreateDirectory(Path.Combine(_testRoot, "allowed")).FullName; + _outsideRoot = Directory.CreateDirectory(Path.Combine(_testRoot, "outside")).FullName; + } + + [Fact] + public async Task OutsideAllowedRootShouldBeRejected() + { + var scanner = CreateScanner(_allowedRoot); + + var act = () => CollectAsync(scanner.ScanAsync(_outsideRoot)); + + await act.Should().ThrowAsync() + .Where(x => x.Code == ErrorCode.InvalidParam); + } + + [Fact] + public async Task ParentDirectoryTraversalShouldBeRejected() + { + var traversal = Path.Combine(_allowedRoot, "child", "..", "..", "outside"); + var scanner = CreateScanner(_allowedRoot); + + var act = () => CollectAsync(scanner.ScanAsync(traversal)); + + await act.Should().ThrowAsync() + .Where(x => x.Code == ErrorCode.InvalidParam); + } + + [Fact] + public async Task SymlinkEscapingAllowedRootShouldBeRejected() + { + var link = Path.Combine(_allowedRoot, "escape"); + Directory.CreateSymbolicLink(link, _outsideRoot); + await File.WriteAllTextAsync(Path.Combine(_outsideRoot, "secret.md"), "secret"); + var scanner = CreateScanner(_allowedRoot); + + var act = () => CollectAsync(scanner.ScanAsync(_allowedRoot)); + + await act.Should().ThrowAsync() + .Where(x => x.Code == ErrorCode.InvalidParam); + } + + [Fact] + public async Task FileCountLimitShouldUseTooLargeCode() + { + await File.WriteAllTextAsync(Path.Combine(_allowedRoot, "1.md"), "1"); + await File.WriteAllTextAsync(Path.Combine(_allowedRoot, "2.md"), "2"); + var scanner = CreateScanner(_allowedRoot, maxFiles: 1); + + var act = () => CollectAsync(scanner.ScanAsync(_allowedRoot)); + + await act.Should().ThrowAsync() + .Where(x => x.Code == ErrorCode.TooLarge); + } + + [Fact] + public async Task SingleAndTotalSizeLimitsShouldUseTooLargeCode() + { + await File.WriteAllBytesAsync(Path.Combine(_allowedRoot, "large.md"), new byte[32]); + var singleScanner = CreateScanner(_allowedRoot, maxFileBytes: 16); + var singleAct = () => CollectAsync(singleScanner.ScanAsync(_allowedRoot)); + await singleAct.Should().ThrowAsync() + .Where(x => x.Code == ErrorCode.TooLarge); + + await File.WriteAllBytesAsync(Path.Combine(_allowedRoot, "small.md"), new byte[8]); + var totalScanner = CreateScanner( + _allowedRoot, + maxFileBytes: 64, + maxTotalBytes: 32); + var totalAct = () => CollectAsync(totalScanner.ScanAsync(_allowedRoot)); + await totalAct.Should().ThrowAsync() + .Where(x => x.Code == ErrorCode.TooLarge); + } + + [Fact] + public async Task ValidVaultShouldReturnRelativePathSizeAndStreamingHash() + { + var nested = Directory.CreateDirectory(Path.Combine(_allowedRoot, "Daily")); + var path = Path.Combine(nested.FullName, "note.md"); + await File.WriteAllTextAsync(path, "hello"); + + var files = await CollectAsync(CreateScanner(_allowedRoot).ScanAsync(_allowedRoot)); + + files.Should().ContainSingle(); + files[0].RelativePath.Should().Be("Daily/note.md"); + files[0].FileSize.Should().Be(5); + files[0].ContentHash.Should().Be( + Convert.ToHexString(SHA256.HashData("hello"u8)).ToLowerInvariant()); + } + + [Fact] + public async Task LargeVaultShouldNotAllocateTheCombinedFileSize() + { + for (var i = 0; i < 8; i++) + { + await using var stream = File.Create(Path.Combine(_allowedRoot, $"{i}.md")); + stream.SetLength(8L * 1024 * 1024); + } + + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + var baseline = GC.GetTotalMemory(forceFullCollection: true); + var peak = baseline; + var count = 0; + + await foreach (var _ in CreateScanner( + _allowedRoot, + maxFileBytes: 9L * 1024 * 1024, + maxTotalBytes: 70L * 1024 * 1024).ScanAsync(_allowedRoot)) + { + count++; + peak = Math.Max(peak, GC.GetTotalMemory(forceFullCollection: false)); + } + + count.Should().Be(8); + (peak - baseline).Should().BeLessThan(32L * 1024 * 1024); + } + + private static ObsidianVaultScanner CreateScanner( + string allowedRoot, + int maxFiles = 100, + long maxFileBytes = 16L * 1024 * 1024, + long maxTotalBytes = 128L * 1024 * 1024) + { + var options = Options.Create(new ObsidianOptions + { + AllowedRoots = [allowedRoot], + MaxFiles = maxFiles, + MaxFileBytes = maxFileBytes, + MaxTotalBytes = maxTotalBytes, + }); + return new ObsidianVaultScanner( + options, + NullLogger.Instance); + } + + private static async Task> CollectAsync( + IAsyncEnumerable source) + { + var result = new List(); + await foreach (var item in source) result.Add(item); + return result; + } + + public void Dispose() + { + if (Directory.Exists(_testRoot)) Directory.Delete(_testRoot, recursive: true); + } +}