feat: bound Obsidian filesystem access
This commit is contained in:
parent
3f6082a9b1
commit
1e51e6a965
@ -84,6 +84,14 @@
|
|||||||
"MaxTokens": 2000,
|
"MaxTokens": 2000,
|
||||||
"Temperature": 0.7
|
"Temperature": 0.7
|
||||||
},
|
},
|
||||||
|
"Rag": {
|
||||||
|
"Obsidian": {
|
||||||
|
"AllowedRoots": [],
|
||||||
|
"MaxFiles": 10000,
|
||||||
|
"MaxFileBytes": 10485760,
|
||||||
|
"MaxTotalBytes": 536870912
|
||||||
|
}
|
||||||
|
},
|
||||||
"Monitoring": {
|
"Monitoring": {
|
||||||
"Scheduler": {
|
"Scheduler": {
|
||||||
"PollIntervalSeconds": 60,
|
"PollIntervalSeconds": 60,
|
||||||
|
|||||||
22
src/modules/rag/Infrastructure/Obsidian/ObsidianOptions.cs
Normal file
22
src/modules/rag/Infrastructure/Obsidian/ObsidianOptions.cs
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
namespace BuildFlow.Infrastructure.Obsidian;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Obsidian 服务端目录访问与容量限制配置。
|
||||||
|
/// </summary>
|
||||||
|
public sealed class ObsidianOptions
|
||||||
|
{
|
||||||
|
/// <summary>配置节名称。</summary>
|
||||||
|
public const string SectionName = "Rag:Obsidian";
|
||||||
|
|
||||||
|
/// <summary>允许访问的服务端根目录;空集合表示禁用目录同步。</summary>
|
||||||
|
public string[] AllowedRoots { get; set; } = [];
|
||||||
|
|
||||||
|
/// <summary>单次同步允许的最大 Markdown 文件数量。</summary>
|
||||||
|
public int MaxFiles { get; set; } = 10_000;
|
||||||
|
|
||||||
|
/// <summary>单个 Markdown 文件允许的最大字节数。</summary>
|
||||||
|
public long MaxFileBytes { get; set; } = 10 * 1024 * 1024;
|
||||||
|
|
||||||
|
/// <summary>单次同步全部 Markdown 文件允许的最大累计字节数。</summary>
|
||||||
|
public long MaxTotalBytes { get; set; } = 512 * 1024 * 1024;
|
||||||
|
}
|
||||||
11
src/modules/rag/Infrastructure/Obsidian/ObsidianVaultFile.cs
Normal file
11
src/modules/rag/Infrastructure/Obsidian/ObsidianVaultFile.cs
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
namespace BuildFlow.Infrastructure.Obsidian;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 通过服务端路径和容量安全检查后的 Obsidian Markdown 文件元数据。
|
||||||
|
/// </summary>
|
||||||
|
public sealed record ObsidianVaultFile(
|
||||||
|
string FullPath,
|
||||||
|
string RelativePath,
|
||||||
|
string FileName,
|
||||||
|
long FileSize,
|
||||||
|
string ContentHash);
|
||||||
340
src/modules/rag/Infrastructure/Obsidian/ObsidianVaultScanner.cs
Normal file
340
src/modules/rag/Infrastructure/Obsidian/ObsidianVaultScanner.cs
Normal file
@ -0,0 +1,340 @@
|
|||||||
|
using System.Runtime.CompilerServices;
|
||||||
|
using System.Security.Cryptography;
|
||||||
|
using BuildFlow.Shared.Domain.Exceptions;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
|
namespace BuildFlow.Infrastructure.Obsidian;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 在允许根目录内流式枚举 Obsidian Markdown 文件,并以流式 SHA256 生成增量同步元数据。
|
||||||
|
/// </summary>
|
||||||
|
public sealed class ObsidianVaultScanner(
|
||||||
|
IOptions<ObsidianOptions> options,
|
||||||
|
ILogger<ObsidianVaultScanner> logger)
|
||||||
|
{
|
||||||
|
private static readonly HashSet<string> IgnoredDirectories =
|
||||||
|
new(StringComparer.OrdinalIgnoreCase) { ".obsidian", ".trash", ".git", "node_modules" };
|
||||||
|
private readonly ObsidianOptions _options = options.Value;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 流式扫描指定 Vault。遍历过程中持续执行数量和大小门禁,不在内存中缓存文件内容。
|
||||||
|
/// </summary>
|
||||||
|
public async IAsyncEnumerable<ObsidianVaultFile> ScanAsync(
|
||||||
|
string requestedDirectory,
|
||||||
|
[EnumeratorCancellation] CancellationToken ct = default)
|
||||||
|
{
|
||||||
|
var (vaultRoot, allowedRoot) = ResolveVaultRoot(requestedDirectory);
|
||||||
|
var pending = new Stack<(DirectoryInfo Directory, string RelativeDirectory)>();
|
||||||
|
var visited = new HashSet<string>(PathComparer);
|
||||||
|
pending.Push((new DirectoryInfo(vaultRoot), string.Empty));
|
||||||
|
var fileCount = 0;
|
||||||
|
long totalBytes = 0;
|
||||||
|
|
||||||
|
while (pending.TryPop(out var current))
|
||||||
|
{
|
||||||
|
ThrowIfCancellationRequested(ct, current.Directory.FullName);
|
||||||
|
var currentPath = ResolveEntry(current.Directory, allowedRoot);
|
||||||
|
if (!visited.Add(currentPath)) continue;
|
||||||
|
|
||||||
|
using var entries = new DirectoryInfo(currentPath)
|
||||||
|
.EnumerateFileSystemInfos()
|
||||||
|
.GetEnumerator();
|
||||||
|
|
||||||
|
while (MoveNext(entries, currentPath, out var entry))
|
||||||
|
{
|
||||||
|
ThrowIfCancellationRequested(ct, currentPath);
|
||||||
|
if (entry is null) continue;
|
||||||
|
|
||||||
|
FileAttributes attributes;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
attributes = entry.Attributes;
|
||||||
|
}
|
||||||
|
catch (FileNotFoundException ex)
|
||||||
|
{
|
||||||
|
logger.LogWarning(ex, "Obsidian 路径在扫描期间消失:{Path}", entry.FullName);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
catch (DirectoryNotFoundException ex)
|
||||||
|
{
|
||||||
|
logger.LogWarning(ex, "Obsidian 路径在扫描期间消失:{Path}", entry.FullName);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((attributes & FileAttributes.Directory) != 0)
|
||||||
|
{
|
||||||
|
if (IgnoredDirectories.Contains(entry.Name)) continue;
|
||||||
|
var childPath = ResolveEntry(entry, allowedRoot);
|
||||||
|
var childRelative = string.IsNullOrEmpty(current.RelativeDirectory)
|
||||||
|
? entry.Name
|
||||||
|
: Path.Combine(current.RelativeDirectory, entry.Name);
|
||||||
|
pending.Push((new DirectoryInfo(childPath), childRelative));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!string.Equals(entry.Extension, ".md", StringComparison.OrdinalIgnoreCase))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
var physicalFile = ResolveEntry(entry, allowedRoot);
|
||||||
|
var fileLength = ReadFileLength(physicalFile);
|
||||||
|
if (fileLength is null) continue;
|
||||||
|
|
||||||
|
fileCount++;
|
||||||
|
if (fileCount > _options.MaxFiles)
|
||||||
|
{
|
||||||
|
throw new BusinessException(
|
||||||
|
ErrorCode.TooLarge,
|
||||||
|
"Obsidian 文件数量超过配置上限");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (fileLength.Value > _options.MaxFileBytes)
|
||||||
|
{
|
||||||
|
throw new BusinessException(
|
||||||
|
ErrorCode.TooLarge,
|
||||||
|
$"Obsidian 文件超过单文件大小上限:{entry.Name}");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (fileLength.Value > _options.MaxTotalBytes - totalBytes)
|
||||||
|
{
|
||||||
|
throw new BusinessException(
|
||||||
|
ErrorCode.TooLarge,
|
||||||
|
"Obsidian 文件累计大小超过配置上限");
|
||||||
|
}
|
||||||
|
totalBytes += fileLength.Value;
|
||||||
|
|
||||||
|
byte[] digest;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await using var stream = new FileStream(
|
||||||
|
physicalFile,
|
||||||
|
FileMode.Open,
|
||||||
|
FileAccess.Read,
|
||||||
|
FileShare.Read,
|
||||||
|
64 * 1024,
|
||||||
|
FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||||
|
digest = await SHA256.HashDataAsync(stream, ct);
|
||||||
|
}
|
||||||
|
catch (FileNotFoundException ex)
|
||||||
|
{
|
||||||
|
logger.LogWarning(ex, "Obsidian 文件在读取前消失:{Path}", physicalFile);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
catch (DirectoryNotFoundException ex)
|
||||||
|
{
|
||||||
|
logger.LogWarning(ex, "Obsidian 文件目录在读取前消失:{Path}", physicalFile);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
logger.LogInformation("Obsidian 文件扫描已取消:{Path}", physicalFile);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
catch (UnauthorizedAccessException ex)
|
||||||
|
{
|
||||||
|
logger.LogWarning(ex, "Obsidian 文件无读取权限:{Path}", physicalFile);
|
||||||
|
throw new BusinessException(ErrorCode.Business, "Obsidian 文件无读取权限");
|
||||||
|
}
|
||||||
|
catch (IOException ex)
|
||||||
|
{
|
||||||
|
logger.LogWarning(ex, "Obsidian 文件读取失败:{Path}", physicalFile);
|
||||||
|
throw new BusinessException(ErrorCode.Business, "Obsidian 文件读取失败");
|
||||||
|
}
|
||||||
|
|
||||||
|
var relativePath = string.IsNullOrEmpty(current.RelativeDirectory)
|
||||||
|
? entry.Name
|
||||||
|
: Path.Combine(current.RelativeDirectory, entry.Name);
|
||||||
|
yield return new ObsidianVaultFile(
|
||||||
|
physicalFile,
|
||||||
|
relativePath.Replace('\\', '/'),
|
||||||
|
entry.Name,
|
||||||
|
fileLength.Value,
|
||||||
|
Convert.ToHexString(digest).ToLowerInvariant());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private (string VaultRoot, string AllowedRoot) ResolveVaultRoot(string requestedDirectory)
|
||||||
|
{
|
||||||
|
if (_options.AllowedRoots.Length == 0)
|
||||||
|
throw new BusinessException(ErrorCode.InvalidParam, "未配置允许的 Obsidian 根目录");
|
||||||
|
if (string.IsNullOrWhiteSpace(requestedDirectory))
|
||||||
|
throw new BusinessException(ErrorCode.InvalidParam, "Obsidian 路径不能为空");
|
||||||
|
|
||||||
|
var segments = requestedDirectory.Split(
|
||||||
|
[Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar],
|
||||||
|
StringSplitOptions.RemoveEmptyEntries);
|
||||||
|
if (segments.Contains("..", StringComparer.Ordinal))
|
||||||
|
throw new BusinessException(ErrorCode.InvalidParam, "Obsidian 路径不能包含父目录跳转");
|
||||||
|
|
||||||
|
string requested;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
requested = TrimPath(Path.GetFullPath(requestedDirectory));
|
||||||
|
}
|
||||||
|
catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException)
|
||||||
|
{
|
||||||
|
throw new BusinessException(ErrorCode.InvalidParam, "Obsidian 路径格式无效");
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var configuredRoot in _options.AllowedRoots.Where(x => !string.IsNullOrWhiteSpace(x)))
|
||||||
|
{
|
||||||
|
var lexicalRoot = TrimPath(Path.GetFullPath(configuredRoot));
|
||||||
|
if (!Directory.Exists(lexicalRoot) || !IsWithin(requested, lexicalRoot)) continue;
|
||||||
|
|
||||||
|
var physicalRoot = ResolveLink(new DirectoryInfo(lexicalRoot));
|
||||||
|
var relative = Path.GetRelativePath(lexicalRoot, requested);
|
||||||
|
var physicalVault = ResolveRelativeDirectory(physicalRoot, relative);
|
||||||
|
EnsureWithin(physicalVault, physicalRoot);
|
||||||
|
return (physicalVault, physicalRoot);
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new BusinessException(ErrorCode.InvalidParam, "Obsidian 路径不在允许的根目录内");
|
||||||
|
}
|
||||||
|
|
||||||
|
private string ResolveRelativeDirectory(string physicalRoot, string relative)
|
||||||
|
{
|
||||||
|
var current = physicalRoot;
|
||||||
|
if (relative == ".") return current;
|
||||||
|
|
||||||
|
foreach (var segment in relative.Split(
|
||||||
|
[Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar],
|
||||||
|
StringSplitOptions.RemoveEmptyEntries))
|
||||||
|
{
|
||||||
|
var next = new DirectoryInfo(Path.Combine(current, segment));
|
||||||
|
if (!next.Exists)
|
||||||
|
throw new BusinessException(ErrorCode.InvalidParam, "Vault 目录不存在");
|
||||||
|
current = ResolveLink(next);
|
||||||
|
EnsureWithin(current, physicalRoot);
|
||||||
|
}
|
||||||
|
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
|
||||||
|
private string ResolveEntry(FileSystemInfo entry, string allowedRoot)
|
||||||
|
{
|
||||||
|
string resolved;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
resolved = ResolveLink(entry);
|
||||||
|
}
|
||||||
|
catch (FileNotFoundException ex)
|
||||||
|
{
|
||||||
|
logger.LogWarning(ex, "Obsidian 路径在解析时消失:{Path}", entry.FullName);
|
||||||
|
throw new BusinessException(ErrorCode.Business, "Obsidian 路径在扫描期间消失");
|
||||||
|
}
|
||||||
|
catch (UnauthorizedAccessException ex)
|
||||||
|
{
|
||||||
|
logger.LogWarning(ex, "Obsidian 路径无访问权限:{Path}", entry.FullName);
|
||||||
|
throw new BusinessException(ErrorCode.Business, "Obsidian 路径无访问权限");
|
||||||
|
}
|
||||||
|
catch (IOException ex)
|
||||||
|
{
|
||||||
|
logger.LogWarning(ex, "Obsidian 路径解析失败:{Path}", entry.FullName);
|
||||||
|
throw new BusinessException(ErrorCode.Business, "Obsidian 路径解析失败");
|
||||||
|
}
|
||||||
|
|
||||||
|
EnsureWithin(resolved, allowedRoot);
|
||||||
|
return resolved;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string ResolveLink(FileSystemInfo entry)
|
||||||
|
{
|
||||||
|
if ((entry.Attributes & FileAttributes.ReparsePoint) == 0)
|
||||||
|
return TrimPath(Path.GetFullPath(entry.FullName));
|
||||||
|
|
||||||
|
var target = entry.ResolveLinkTarget(returnFinalTarget: true)
|
||||||
|
?? throw new BusinessException(ErrorCode.InvalidParam, "Obsidian 符号链接目标不存在");
|
||||||
|
return TrimPath(Path.GetFullPath(target.FullName));
|
||||||
|
}
|
||||||
|
|
||||||
|
private long? ReadFileLength(string physicalFile)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var fileInfo = new FileInfo(physicalFile);
|
||||||
|
fileInfo.Refresh();
|
||||||
|
if (fileInfo.Exists) return fileInfo.Length;
|
||||||
|
logger.LogWarning("Obsidian 文件在扫描期间消失:{Path}", physicalFile);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
catch (FileNotFoundException ex)
|
||||||
|
{
|
||||||
|
logger.LogWarning(ex, "Obsidian 文件在扫描期间消失:{Path}", physicalFile);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
catch (UnauthorizedAccessException ex)
|
||||||
|
{
|
||||||
|
logger.LogWarning(ex, "Obsidian 文件无读取权限:{Path}", physicalFile);
|
||||||
|
throw new BusinessException(ErrorCode.Business, "Obsidian 文件无读取权限");
|
||||||
|
}
|
||||||
|
catch (IOException ex)
|
||||||
|
{
|
||||||
|
logger.LogWarning(ex, "Obsidian 文件元数据读取失败:{Path}", physicalFile);
|
||||||
|
throw new BusinessException(ErrorCode.Business, "Obsidian 文件元数据读取失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private bool MoveNext(
|
||||||
|
IEnumerator<FileSystemInfo> entries,
|
||||||
|
string directory,
|
||||||
|
out FileSystemInfo? entry)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (!entries.MoveNext())
|
||||||
|
{
|
||||||
|
entry = null;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
entry = entries.Current;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
catch (UnauthorizedAccessException ex)
|
||||||
|
{
|
||||||
|
logger.LogWarning(ex, "Obsidian 目录无枚举权限:{Path}", directory);
|
||||||
|
throw new BusinessException(ErrorCode.Business, "Obsidian 目录无枚举权限");
|
||||||
|
}
|
||||||
|
catch (IOException ex)
|
||||||
|
{
|
||||||
|
logger.LogWarning(ex, "Obsidian 目录枚举失败:{Path}", directory);
|
||||||
|
throw new BusinessException(ErrorCode.Business, "Obsidian 目录枚举失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ThrowIfCancellationRequested(CancellationToken ct, string path)
|
||||||
|
{
|
||||||
|
if (!ct.IsCancellationRequested) return;
|
||||||
|
logger.LogInformation("Obsidian 目录扫描已取消:{Path}", path);
|
||||||
|
ct.ThrowIfCancellationRequested();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void EnsureWithin(string path, string root)
|
||||||
|
{
|
||||||
|
if (!IsWithin(path, root))
|
||||||
|
{
|
||||||
|
throw new BusinessException(
|
||||||
|
ErrorCode.InvalidParam,
|
||||||
|
"Obsidian 符号链接逃逸允许根目录");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsWithin(string path, string root)
|
||||||
|
{
|
||||||
|
var relative = Path.GetRelativePath(root, path);
|
||||||
|
return relative == "." ||
|
||||||
|
(!Path.IsPathRooted(relative) &&
|
||||||
|
relative != ".." &&
|
||||||
|
!relative.StartsWith($"..{Path.DirectorySeparatorChar}", PathComparison) &&
|
||||||
|
!relative.StartsWith($"..{Path.AltDirectorySeparatorChar}", PathComparison));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string TrimPath(string path) => Path.TrimEndingDirectorySeparator(path);
|
||||||
|
|
||||||
|
private static StringComparison PathComparison =>
|
||||||
|
OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
|
||||||
|
|
||||||
|
private static StringComparer PathComparer =>
|
||||||
|
OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal;
|
||||||
|
}
|
||||||
@ -8,6 +8,7 @@ using BuildFlow.Infrastructure.Cache;
|
|||||||
using BuildFlow.Infrastructure.Extractors;
|
using BuildFlow.Infrastructure.Extractors;
|
||||||
using BuildFlow.Infrastructure.Files;
|
using BuildFlow.Infrastructure.Files;
|
||||||
using BuildFlow.Infrastructure.Messaging;
|
using BuildFlow.Infrastructure.Messaging;
|
||||||
|
using BuildFlow.Infrastructure.Obsidian;
|
||||||
using BuildFlow.Infrastructure.Persistence;
|
using BuildFlow.Infrastructure.Persistence;
|
||||||
using BuildFlow.Shared.Infrastructure;
|
using BuildFlow.Shared.Infrastructure;
|
||||||
using Volo.Abp.Modularity;
|
using Volo.Abp.Modularity;
|
||||||
@ -50,6 +51,13 @@ public class RAGInfrastructureModule : AbpModule
|
|||||||
services.AddScoped<ITextChunker, TextChunker>();
|
services.AddScoped<ITextChunker, TextChunker>();
|
||||||
services.AddSingleton<IChatMessageCache, ChatMessageCache>();
|
services.AddSingleton<IChatMessageCache, ChatMessageCache>();
|
||||||
|
|
||||||
|
services.AddOptions<ObsidianOptions>()
|
||||||
|
.Bind(config.GetSection(ObsidianOptions.SectionName))
|
||||||
|
.Validate(x => x.MaxFiles > 0, "Rag:Obsidian:MaxFiles 必须大于 0")
|
||||||
|
.Validate(x => x.MaxFileBytes > 0, "Rag:Obsidian:MaxFileBytes 必须大于 0")
|
||||||
|
.Validate(x => x.MaxTotalBytes > 0, "Rag:Obsidian:MaxTotalBytes 必须大于 0");
|
||||||
|
services.AddScoped<ObsidianVaultScanner>();
|
||||||
|
|
||||||
// 文档提取器:各格式提取器 + 工厂按扩展名分发(参考 Dify ExtractProcessor)
|
// 文档提取器:各格式提取器 + 工厂按扩展名分发(参考 Dify ExtractProcessor)
|
||||||
services.AddSingleton<MarkdownExtractor>();
|
services.AddSingleton<MarkdownExtractor>();
|
||||||
services.AddSingleton<PdfExtractor>();
|
services.AddSingleton<PdfExtractor>();
|
||||||
|
|||||||
12
tests/BuildFlow.Rag.Tests/ObsidianVaultScannerCollection.cs
Normal file
12
tests/BuildFlow.Rag.Tests/ObsidianVaultScannerCollection.cs
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
using Xunit;
|
||||||
|
|
||||||
|
namespace BuildFlow.Rag.Tests;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 隔离 Obsidian 扫描器的托管内存测量,避免并行测试制造分配噪声。
|
||||||
|
/// </summary>
|
||||||
|
[CollectionDefinition("ObsidianVaultScanner", DisableParallelization = true)]
|
||||||
|
public sealed class ObsidianVaultScannerCollection
|
||||||
|
{
|
||||||
|
public const string Name = "ObsidianVaultScanner";
|
||||||
|
}
|
||||||
174
tests/BuildFlow.Rag.Tests/ObsidianVaultScannerTests.cs
Normal file
174
tests/BuildFlow.Rag.Tests/ObsidianVaultScannerTests.cs
Normal file
@ -0,0 +1,174 @@
|
|||||||
|
using System.Security.Cryptography;
|
||||||
|
using BuildFlow.Infrastructure.Obsidian;
|
||||||
|
using BuildFlow.Shared.Domain.Errors;
|
||||||
|
using BuildFlow.Shared.Domain.Exceptions;
|
||||||
|
using FluentAssertions;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
using Xunit;
|
||||||
|
|
||||||
|
namespace BuildFlow.Rag.Tests;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Obsidian 服务端目录访问和容量边界测试。
|
||||||
|
/// </summary>
|
||||||
|
[Collection(ObsidianVaultScannerCollection.Name)]
|
||||||
|
public sealed class ObsidianVaultScannerTests : IDisposable
|
||||||
|
{
|
||||||
|
private readonly string _testRoot = Path.Combine(
|
||||||
|
Path.GetTempPath(),
|
||||||
|
$"buildflow-obsidian-{Guid.NewGuid():N}");
|
||||||
|
private readonly string _allowedRoot;
|
||||||
|
private readonly string _outsideRoot;
|
||||||
|
|
||||||
|
public ObsidianVaultScannerTests()
|
||||||
|
{
|
||||||
|
_allowedRoot = Directory.CreateDirectory(Path.Combine(_testRoot, "allowed")).FullName;
|
||||||
|
_outsideRoot = Directory.CreateDirectory(Path.Combine(_testRoot, "outside")).FullName;
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task OutsideAllowedRootShouldBeRejected()
|
||||||
|
{
|
||||||
|
var scanner = CreateScanner(_allowedRoot);
|
||||||
|
|
||||||
|
var act = () => CollectAsync(scanner.ScanAsync(_outsideRoot));
|
||||||
|
|
||||||
|
await act.Should().ThrowAsync<BusinessException>()
|
||||||
|
.Where(x => x.Code == ErrorCode.InvalidParam);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ParentDirectoryTraversalShouldBeRejected()
|
||||||
|
{
|
||||||
|
var traversal = Path.Combine(_allowedRoot, "child", "..", "..", "outside");
|
||||||
|
var scanner = CreateScanner(_allowedRoot);
|
||||||
|
|
||||||
|
var act = () => CollectAsync(scanner.ScanAsync(traversal));
|
||||||
|
|
||||||
|
await act.Should().ThrowAsync<BusinessException>()
|
||||||
|
.Where(x => x.Code == ErrorCode.InvalidParam);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task SymlinkEscapingAllowedRootShouldBeRejected()
|
||||||
|
{
|
||||||
|
var link = Path.Combine(_allowedRoot, "escape");
|
||||||
|
Directory.CreateSymbolicLink(link, _outsideRoot);
|
||||||
|
await File.WriteAllTextAsync(Path.Combine(_outsideRoot, "secret.md"), "secret");
|
||||||
|
var scanner = CreateScanner(_allowedRoot);
|
||||||
|
|
||||||
|
var act = () => CollectAsync(scanner.ScanAsync(_allowedRoot));
|
||||||
|
|
||||||
|
await act.Should().ThrowAsync<BusinessException>()
|
||||||
|
.Where(x => x.Code == ErrorCode.InvalidParam);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task FileCountLimitShouldUseTooLargeCode()
|
||||||
|
{
|
||||||
|
await File.WriteAllTextAsync(Path.Combine(_allowedRoot, "1.md"), "1");
|
||||||
|
await File.WriteAllTextAsync(Path.Combine(_allowedRoot, "2.md"), "2");
|
||||||
|
var scanner = CreateScanner(_allowedRoot, maxFiles: 1);
|
||||||
|
|
||||||
|
var act = () => CollectAsync(scanner.ScanAsync(_allowedRoot));
|
||||||
|
|
||||||
|
await act.Should().ThrowAsync<BusinessException>()
|
||||||
|
.Where(x => x.Code == ErrorCode.TooLarge);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task SingleAndTotalSizeLimitsShouldUseTooLargeCode()
|
||||||
|
{
|
||||||
|
await File.WriteAllBytesAsync(Path.Combine(_allowedRoot, "large.md"), new byte[32]);
|
||||||
|
var singleScanner = CreateScanner(_allowedRoot, maxFileBytes: 16);
|
||||||
|
var singleAct = () => CollectAsync(singleScanner.ScanAsync(_allowedRoot));
|
||||||
|
await singleAct.Should().ThrowAsync<BusinessException>()
|
||||||
|
.Where(x => x.Code == ErrorCode.TooLarge);
|
||||||
|
|
||||||
|
await File.WriteAllBytesAsync(Path.Combine(_allowedRoot, "small.md"), new byte[8]);
|
||||||
|
var totalScanner = CreateScanner(
|
||||||
|
_allowedRoot,
|
||||||
|
maxFileBytes: 64,
|
||||||
|
maxTotalBytes: 32);
|
||||||
|
var totalAct = () => CollectAsync(totalScanner.ScanAsync(_allowedRoot));
|
||||||
|
await totalAct.Should().ThrowAsync<BusinessException>()
|
||||||
|
.Where(x => x.Code == ErrorCode.TooLarge);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ValidVaultShouldReturnRelativePathSizeAndStreamingHash()
|
||||||
|
{
|
||||||
|
var nested = Directory.CreateDirectory(Path.Combine(_allowedRoot, "Daily"));
|
||||||
|
var path = Path.Combine(nested.FullName, "note.md");
|
||||||
|
await File.WriteAllTextAsync(path, "hello");
|
||||||
|
|
||||||
|
var files = await CollectAsync(CreateScanner(_allowedRoot).ScanAsync(_allowedRoot));
|
||||||
|
|
||||||
|
files.Should().ContainSingle();
|
||||||
|
files[0].RelativePath.Should().Be("Daily/note.md");
|
||||||
|
files[0].FileSize.Should().Be(5);
|
||||||
|
files[0].ContentHash.Should().Be(
|
||||||
|
Convert.ToHexString(SHA256.HashData("hello"u8)).ToLowerInvariant());
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task LargeVaultShouldNotAllocateTheCombinedFileSize()
|
||||||
|
{
|
||||||
|
for (var i = 0; i < 8; i++)
|
||||||
|
{
|
||||||
|
await using var stream = File.Create(Path.Combine(_allowedRoot, $"{i}.md"));
|
||||||
|
stream.SetLength(8L * 1024 * 1024);
|
||||||
|
}
|
||||||
|
|
||||||
|
GC.Collect();
|
||||||
|
GC.WaitForPendingFinalizers();
|
||||||
|
GC.Collect();
|
||||||
|
var baseline = GC.GetTotalMemory(forceFullCollection: true);
|
||||||
|
var peak = baseline;
|
||||||
|
var count = 0;
|
||||||
|
|
||||||
|
await foreach (var _ in CreateScanner(
|
||||||
|
_allowedRoot,
|
||||||
|
maxFileBytes: 9L * 1024 * 1024,
|
||||||
|
maxTotalBytes: 70L * 1024 * 1024).ScanAsync(_allowedRoot))
|
||||||
|
{
|
||||||
|
count++;
|
||||||
|
peak = Math.Max(peak, GC.GetTotalMemory(forceFullCollection: false));
|
||||||
|
}
|
||||||
|
|
||||||
|
count.Should().Be(8);
|
||||||
|
(peak - baseline).Should().BeLessThan(32L * 1024 * 1024);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ObsidianVaultScanner CreateScanner(
|
||||||
|
string allowedRoot,
|
||||||
|
int maxFiles = 100,
|
||||||
|
long maxFileBytes = 16L * 1024 * 1024,
|
||||||
|
long maxTotalBytes = 128L * 1024 * 1024)
|
||||||
|
{
|
||||||
|
var options = Options.Create(new ObsidianOptions
|
||||||
|
{
|
||||||
|
AllowedRoots = [allowedRoot],
|
||||||
|
MaxFiles = maxFiles,
|
||||||
|
MaxFileBytes = maxFileBytes,
|
||||||
|
MaxTotalBytes = maxTotalBytes,
|
||||||
|
});
|
||||||
|
return new ObsidianVaultScanner(
|
||||||
|
options,
|
||||||
|
NullLogger<ObsidianVaultScanner>.Instance);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task<List<ObsidianVaultFile>> CollectAsync(
|
||||||
|
IAsyncEnumerable<ObsidianVaultFile> source)
|
||||||
|
{
|
||||||
|
var result = new List<ObsidianVaultFile>();
|
||||||
|
await foreach (var item in source) result.Add(item);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
if (Directory.Exists(_testRoot)) Directory.Delete(_testRoot, recursive: true);
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user