feat: bound Obsidian filesystem access

This commit is contained in:
xiangning 2026-07-23 17:59:37 +08:00
parent 3f6082a9b1
commit 1e51e6a965
7 changed files with 575 additions and 0 deletions

View File

@ -84,6 +84,14 @@
"MaxTokens": 2000,
"Temperature": 0.7
},
"Rag": {
"Obsidian": {
"AllowedRoots": [],
"MaxFiles": 10000,
"MaxFileBytes": 10485760,
"MaxTotalBytes": 536870912
}
},
"Monitoring": {
"Scheduler": {
"PollIntervalSeconds": 60,

View File

@ -0,0 +1,22 @@
namespace BuildFlow.Infrastructure.Obsidian;
/// <summary>
/// Obsidian 服务端目录访问与容量限制配置。
/// </summary>
public sealed class ObsidianOptions
{
/// <summary>配置节名称。</summary>
public const string SectionName = "Rag:Obsidian";
/// <summary>允许访问的服务端根目录;空集合表示禁用目录同步。</summary>
public string[] AllowedRoots { get; set; } = [];
/// <summary>单次同步允许的最大 Markdown 文件数量。</summary>
public int MaxFiles { get; set; } = 10_000;
/// <summary>单个 Markdown 文件允许的最大字节数。</summary>
public long MaxFileBytes { get; set; } = 10 * 1024 * 1024;
/// <summary>单次同步全部 Markdown 文件允许的最大累计字节数。</summary>
public long MaxTotalBytes { get; set; } = 512 * 1024 * 1024;
}

View File

@ -0,0 +1,11 @@
namespace BuildFlow.Infrastructure.Obsidian;
/// <summary>
/// 通过服务端路径和容量安全检查后的 Obsidian Markdown 文件元数据。
/// </summary>
public sealed record ObsidianVaultFile(
string FullPath,
string RelativePath,
string FileName,
long FileSize,
string ContentHash);

View File

@ -0,0 +1,340 @@
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using BuildFlow.Shared.Domain.Exceptions;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace BuildFlow.Infrastructure.Obsidian;
/// <summary>
/// 在允许根目录内流式枚举 Obsidian Markdown 文件,并以流式 SHA256 生成增量同步元数据。
/// </summary>
public sealed class ObsidianVaultScanner(
IOptions<ObsidianOptions> options,
ILogger<ObsidianVaultScanner> logger)
{
private static readonly HashSet<string> IgnoredDirectories =
new(StringComparer.OrdinalIgnoreCase) { ".obsidian", ".trash", ".git", "node_modules" };
private readonly ObsidianOptions _options = options.Value;
/// <summary>
/// 流式扫描指定 Vault。遍历过程中持续执行数量和大小门禁,不在内存中缓存文件内容。
/// </summary>
public async IAsyncEnumerable<ObsidianVaultFile> ScanAsync(
string requestedDirectory,
[EnumeratorCancellation] CancellationToken ct = default)
{
var (vaultRoot, allowedRoot) = ResolveVaultRoot(requestedDirectory);
var pending = new Stack<(DirectoryInfo Directory, string RelativeDirectory)>();
var visited = new HashSet<string>(PathComparer);
pending.Push((new DirectoryInfo(vaultRoot), string.Empty));
var fileCount = 0;
long totalBytes = 0;
while (pending.TryPop(out var current))
{
ThrowIfCancellationRequested(ct, current.Directory.FullName);
var currentPath = ResolveEntry(current.Directory, allowedRoot);
if (!visited.Add(currentPath)) continue;
using var entries = new DirectoryInfo(currentPath)
.EnumerateFileSystemInfos()
.GetEnumerator();
while (MoveNext(entries, currentPath, out var entry))
{
ThrowIfCancellationRequested(ct, currentPath);
if (entry is null) continue;
FileAttributes attributes;
try
{
attributes = entry.Attributes;
}
catch (FileNotFoundException ex)
{
logger.LogWarning(ex, "Obsidian 路径在扫描期间消失:{Path}", entry.FullName);
continue;
}
catch (DirectoryNotFoundException ex)
{
logger.LogWarning(ex, "Obsidian 路径在扫描期间消失:{Path}", entry.FullName);
continue;
}
if ((attributes & FileAttributes.Directory) != 0)
{
if (IgnoredDirectories.Contains(entry.Name)) continue;
var childPath = ResolveEntry(entry, allowedRoot);
var childRelative = string.IsNullOrEmpty(current.RelativeDirectory)
? entry.Name
: Path.Combine(current.RelativeDirectory, entry.Name);
pending.Push((new DirectoryInfo(childPath), childRelative));
continue;
}
if (!string.Equals(entry.Extension, ".md", StringComparison.OrdinalIgnoreCase))
continue;
var physicalFile = ResolveEntry(entry, allowedRoot);
var fileLength = ReadFileLength(physicalFile);
if (fileLength is null) continue;
fileCount++;
if (fileCount > _options.MaxFiles)
{
throw new BusinessException(
ErrorCode.TooLarge,
"Obsidian 文件数量超过配置上限");
}
if (fileLength.Value > _options.MaxFileBytes)
{
throw new BusinessException(
ErrorCode.TooLarge,
$"Obsidian 文件超过单文件大小上限:{entry.Name}");
}
if (fileLength.Value > _options.MaxTotalBytes - totalBytes)
{
throw new BusinessException(
ErrorCode.TooLarge,
"Obsidian 文件累计大小超过配置上限");
}
totalBytes += fileLength.Value;
byte[] digest;
try
{
await using var stream = new FileStream(
physicalFile,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
64 * 1024,
FileOptions.Asynchronous | FileOptions.SequentialScan);
digest = await SHA256.HashDataAsync(stream, ct);
}
catch (FileNotFoundException ex)
{
logger.LogWarning(ex, "Obsidian 文件在读取前消失:{Path}", physicalFile);
continue;
}
catch (DirectoryNotFoundException ex)
{
logger.LogWarning(ex, "Obsidian 文件目录在读取前消失:{Path}", physicalFile);
continue;
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
logger.LogInformation("Obsidian 文件扫描已取消:{Path}", physicalFile);
throw;
}
catch (UnauthorizedAccessException ex)
{
logger.LogWarning(ex, "Obsidian 文件无读取权限:{Path}", physicalFile);
throw new BusinessException(ErrorCode.Business, "Obsidian 文件无读取权限");
}
catch (IOException ex)
{
logger.LogWarning(ex, "Obsidian 文件读取失败:{Path}", physicalFile);
throw new BusinessException(ErrorCode.Business, "Obsidian 文件读取失败");
}
var relativePath = string.IsNullOrEmpty(current.RelativeDirectory)
? entry.Name
: Path.Combine(current.RelativeDirectory, entry.Name);
yield return new ObsidianVaultFile(
physicalFile,
relativePath.Replace('\\', '/'),
entry.Name,
fileLength.Value,
Convert.ToHexString(digest).ToLowerInvariant());
}
}
}
private (string VaultRoot, string AllowedRoot) ResolveVaultRoot(string requestedDirectory)
{
if (_options.AllowedRoots.Length == 0)
throw new BusinessException(ErrorCode.InvalidParam, "未配置允许的 Obsidian 根目录");
if (string.IsNullOrWhiteSpace(requestedDirectory))
throw new BusinessException(ErrorCode.InvalidParam, "Obsidian 路径不能为空");
var segments = requestedDirectory.Split(
[Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar],
StringSplitOptions.RemoveEmptyEntries);
if (segments.Contains("..", StringComparer.Ordinal))
throw new BusinessException(ErrorCode.InvalidParam, "Obsidian 路径不能包含父目录跳转");
string requested;
try
{
requested = TrimPath(Path.GetFullPath(requestedDirectory));
}
catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException)
{
throw new BusinessException(ErrorCode.InvalidParam, "Obsidian 路径格式无效");
}
foreach (var configuredRoot in _options.AllowedRoots.Where(x => !string.IsNullOrWhiteSpace(x)))
{
var lexicalRoot = TrimPath(Path.GetFullPath(configuredRoot));
if (!Directory.Exists(lexicalRoot) || !IsWithin(requested, lexicalRoot)) continue;
var physicalRoot = ResolveLink(new DirectoryInfo(lexicalRoot));
var relative = Path.GetRelativePath(lexicalRoot, requested);
var physicalVault = ResolveRelativeDirectory(physicalRoot, relative);
EnsureWithin(physicalVault, physicalRoot);
return (physicalVault, physicalRoot);
}
throw new BusinessException(ErrorCode.InvalidParam, "Obsidian 路径不在允许的根目录内");
}
private string ResolveRelativeDirectory(string physicalRoot, string relative)
{
var current = physicalRoot;
if (relative == ".") return current;
foreach (var segment in relative.Split(
[Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar],
StringSplitOptions.RemoveEmptyEntries))
{
var next = new DirectoryInfo(Path.Combine(current, segment));
if (!next.Exists)
throw new BusinessException(ErrorCode.InvalidParam, "Vault 目录不存在");
current = ResolveLink(next);
EnsureWithin(current, physicalRoot);
}
return current;
}
private string ResolveEntry(FileSystemInfo entry, string allowedRoot)
{
string resolved;
try
{
resolved = ResolveLink(entry);
}
catch (FileNotFoundException ex)
{
logger.LogWarning(ex, "Obsidian 路径在解析时消失:{Path}", entry.FullName);
throw new BusinessException(ErrorCode.Business, "Obsidian 路径在扫描期间消失");
}
catch (UnauthorizedAccessException ex)
{
logger.LogWarning(ex, "Obsidian 路径无访问权限:{Path}", entry.FullName);
throw new BusinessException(ErrorCode.Business, "Obsidian 路径无访问权限");
}
catch (IOException ex)
{
logger.LogWarning(ex, "Obsidian 路径解析失败:{Path}", entry.FullName);
throw new BusinessException(ErrorCode.Business, "Obsidian 路径解析失败");
}
EnsureWithin(resolved, allowedRoot);
return resolved;
}
private static string ResolveLink(FileSystemInfo entry)
{
if ((entry.Attributes & FileAttributes.ReparsePoint) == 0)
return TrimPath(Path.GetFullPath(entry.FullName));
var target = entry.ResolveLinkTarget(returnFinalTarget: true)
?? throw new BusinessException(ErrorCode.InvalidParam, "Obsidian 符号链接目标不存在");
return TrimPath(Path.GetFullPath(target.FullName));
}
private long? ReadFileLength(string physicalFile)
{
try
{
var fileInfo = new FileInfo(physicalFile);
fileInfo.Refresh();
if (fileInfo.Exists) return fileInfo.Length;
logger.LogWarning("Obsidian 文件在扫描期间消失:{Path}", physicalFile);
return null;
}
catch (FileNotFoundException ex)
{
logger.LogWarning(ex, "Obsidian 文件在扫描期间消失:{Path}", physicalFile);
return null;
}
catch (UnauthorizedAccessException ex)
{
logger.LogWarning(ex, "Obsidian 文件无读取权限:{Path}", physicalFile);
throw new BusinessException(ErrorCode.Business, "Obsidian 文件无读取权限");
}
catch (IOException ex)
{
logger.LogWarning(ex, "Obsidian 文件元数据读取失败:{Path}", physicalFile);
throw new BusinessException(ErrorCode.Business, "Obsidian 文件元数据读取失败");
}
}
private bool MoveNext(
IEnumerator<FileSystemInfo> entries,
string directory,
out FileSystemInfo? entry)
{
try
{
if (!entries.MoveNext())
{
entry = null;
return false;
}
entry = entries.Current;
return true;
}
catch (UnauthorizedAccessException ex)
{
logger.LogWarning(ex, "Obsidian 目录无枚举权限:{Path}", directory);
throw new BusinessException(ErrorCode.Business, "Obsidian 目录无枚举权限");
}
catch (IOException ex)
{
logger.LogWarning(ex, "Obsidian 目录枚举失败:{Path}", directory);
throw new BusinessException(ErrorCode.Business, "Obsidian 目录枚举失败");
}
}
private void ThrowIfCancellationRequested(CancellationToken ct, string path)
{
if (!ct.IsCancellationRequested) return;
logger.LogInformation("Obsidian 目录扫描已取消:{Path}", path);
ct.ThrowIfCancellationRequested();
}
private static void EnsureWithin(string path, string root)
{
if (!IsWithin(path, root))
{
throw new BusinessException(
ErrorCode.InvalidParam,
"Obsidian 符号链接逃逸允许根目录");
}
}
private static bool IsWithin(string path, string root)
{
var relative = Path.GetRelativePath(root, path);
return relative == "." ||
(!Path.IsPathRooted(relative) &&
relative != ".." &&
!relative.StartsWith($"..{Path.DirectorySeparatorChar}", PathComparison) &&
!relative.StartsWith($"..{Path.AltDirectorySeparatorChar}", PathComparison));
}
private static string TrimPath(string path) => Path.TrimEndingDirectorySeparator(path);
private static StringComparison PathComparison =>
OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
private static StringComparer PathComparer =>
OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal;
}

View File

@ -8,6 +8,7 @@ using BuildFlow.Infrastructure.Cache;
using BuildFlow.Infrastructure.Extractors;
using BuildFlow.Infrastructure.Files;
using BuildFlow.Infrastructure.Messaging;
using BuildFlow.Infrastructure.Obsidian;
using BuildFlow.Infrastructure.Persistence;
using BuildFlow.Shared.Infrastructure;
using Volo.Abp.Modularity;
@ -50,6 +51,13 @@ public class RAGInfrastructureModule : AbpModule
services.AddScoped<ITextChunker, TextChunker>();
services.AddSingleton<IChatMessageCache, ChatMessageCache>();
services.AddOptions<ObsidianOptions>()
.Bind(config.GetSection(ObsidianOptions.SectionName))
.Validate(x => x.MaxFiles > 0, "Rag:Obsidian:MaxFiles 必须大于 0")
.Validate(x => x.MaxFileBytes > 0, "Rag:Obsidian:MaxFileBytes 必须大于 0")
.Validate(x => x.MaxTotalBytes > 0, "Rag:Obsidian:MaxTotalBytes 必须大于 0");
services.AddScoped<ObsidianVaultScanner>();
// 文档提取器:各格式提取器 + 工厂按扩展名分发(参考 Dify ExtractProcessor)
services.AddSingleton<MarkdownExtractor>();
services.AddSingleton<PdfExtractor>();

View File

@ -0,0 +1,12 @@
using Xunit;
namespace BuildFlow.Rag.Tests;
/// <summary>
/// 隔离 Obsidian 扫描器的托管内存测量,避免并行测试制造分配噪声。
/// </summary>
[CollectionDefinition("ObsidianVaultScanner", DisableParallelization = true)]
public sealed class ObsidianVaultScannerCollection
{
public const string Name = "ObsidianVaultScanner";
}

View File

@ -0,0 +1,174 @@
using System.Security.Cryptography;
using BuildFlow.Infrastructure.Obsidian;
using BuildFlow.Shared.Domain.Errors;
using BuildFlow.Shared.Domain.Exceptions;
using FluentAssertions;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Xunit;
namespace BuildFlow.Rag.Tests;
/// <summary>
/// Obsidian 服务端目录访问和容量边界测试。
/// </summary>
[Collection(ObsidianVaultScannerCollection.Name)]
public sealed class ObsidianVaultScannerTests : IDisposable
{
private readonly string _testRoot = Path.Combine(
Path.GetTempPath(),
$"buildflow-obsidian-{Guid.NewGuid():N}");
private readonly string _allowedRoot;
private readonly string _outsideRoot;
public ObsidianVaultScannerTests()
{
_allowedRoot = Directory.CreateDirectory(Path.Combine(_testRoot, "allowed")).FullName;
_outsideRoot = Directory.CreateDirectory(Path.Combine(_testRoot, "outside")).FullName;
}
[Fact]
public async Task OutsideAllowedRootShouldBeRejected()
{
var scanner = CreateScanner(_allowedRoot);
var act = () => CollectAsync(scanner.ScanAsync(_outsideRoot));
await act.Should().ThrowAsync<BusinessException>()
.Where(x => x.Code == ErrorCode.InvalidParam);
}
[Fact]
public async Task ParentDirectoryTraversalShouldBeRejected()
{
var traversal = Path.Combine(_allowedRoot, "child", "..", "..", "outside");
var scanner = CreateScanner(_allowedRoot);
var act = () => CollectAsync(scanner.ScanAsync(traversal));
await act.Should().ThrowAsync<BusinessException>()
.Where(x => x.Code == ErrorCode.InvalidParam);
}
[Fact]
public async Task SymlinkEscapingAllowedRootShouldBeRejected()
{
var link = Path.Combine(_allowedRoot, "escape");
Directory.CreateSymbolicLink(link, _outsideRoot);
await File.WriteAllTextAsync(Path.Combine(_outsideRoot, "secret.md"), "secret");
var scanner = CreateScanner(_allowedRoot);
var act = () => CollectAsync(scanner.ScanAsync(_allowedRoot));
await act.Should().ThrowAsync<BusinessException>()
.Where(x => x.Code == ErrorCode.InvalidParam);
}
[Fact]
public async Task FileCountLimitShouldUseTooLargeCode()
{
await File.WriteAllTextAsync(Path.Combine(_allowedRoot, "1.md"), "1");
await File.WriteAllTextAsync(Path.Combine(_allowedRoot, "2.md"), "2");
var scanner = CreateScanner(_allowedRoot, maxFiles: 1);
var act = () => CollectAsync(scanner.ScanAsync(_allowedRoot));
await act.Should().ThrowAsync<BusinessException>()
.Where(x => x.Code == ErrorCode.TooLarge);
}
[Fact]
public async Task SingleAndTotalSizeLimitsShouldUseTooLargeCode()
{
await File.WriteAllBytesAsync(Path.Combine(_allowedRoot, "large.md"), new byte[32]);
var singleScanner = CreateScanner(_allowedRoot, maxFileBytes: 16);
var singleAct = () => CollectAsync(singleScanner.ScanAsync(_allowedRoot));
await singleAct.Should().ThrowAsync<BusinessException>()
.Where(x => x.Code == ErrorCode.TooLarge);
await File.WriteAllBytesAsync(Path.Combine(_allowedRoot, "small.md"), new byte[8]);
var totalScanner = CreateScanner(
_allowedRoot,
maxFileBytes: 64,
maxTotalBytes: 32);
var totalAct = () => CollectAsync(totalScanner.ScanAsync(_allowedRoot));
await totalAct.Should().ThrowAsync<BusinessException>()
.Where(x => x.Code == ErrorCode.TooLarge);
}
[Fact]
public async Task ValidVaultShouldReturnRelativePathSizeAndStreamingHash()
{
var nested = Directory.CreateDirectory(Path.Combine(_allowedRoot, "Daily"));
var path = Path.Combine(nested.FullName, "note.md");
await File.WriteAllTextAsync(path, "hello");
var files = await CollectAsync(CreateScanner(_allowedRoot).ScanAsync(_allowedRoot));
files.Should().ContainSingle();
files[0].RelativePath.Should().Be("Daily/note.md");
files[0].FileSize.Should().Be(5);
files[0].ContentHash.Should().Be(
Convert.ToHexString(SHA256.HashData("hello"u8)).ToLowerInvariant());
}
[Fact]
public async Task LargeVaultShouldNotAllocateTheCombinedFileSize()
{
for (var i = 0; i < 8; i++)
{
await using var stream = File.Create(Path.Combine(_allowedRoot, $"{i}.md"));
stream.SetLength(8L * 1024 * 1024);
}
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
var baseline = GC.GetTotalMemory(forceFullCollection: true);
var peak = baseline;
var count = 0;
await foreach (var _ in CreateScanner(
_allowedRoot,
maxFileBytes: 9L * 1024 * 1024,
maxTotalBytes: 70L * 1024 * 1024).ScanAsync(_allowedRoot))
{
count++;
peak = Math.Max(peak, GC.GetTotalMemory(forceFullCollection: false));
}
count.Should().Be(8);
(peak - baseline).Should().BeLessThan(32L * 1024 * 1024);
}
private static ObsidianVaultScanner CreateScanner(
string allowedRoot,
int maxFiles = 100,
long maxFileBytes = 16L * 1024 * 1024,
long maxTotalBytes = 128L * 1024 * 1024)
{
var options = Options.Create(new ObsidianOptions
{
AllowedRoots = [allowedRoot],
MaxFiles = maxFiles,
MaxFileBytes = maxFileBytes,
MaxTotalBytes = maxTotalBytes,
});
return new ObsidianVaultScanner(
options,
NullLogger<ObsidianVaultScanner>.Instance);
}
private static async Task<List<ObsidianVaultFile>> CollectAsync(
IAsyncEnumerable<ObsidianVaultFile> source)
{
var result = new List<ObsidianVaultFile>();
await foreach (var item in source) result.Add(item);
return result;
}
public void Dispose()
{
if (Directory.Exists(_testRoot)) Directory.Delete(_testRoot, recursive: true);
}
}