75 lines
2.7 KiB
C#
75 lines
2.7 KiB
C#
using BuildFlow.Infrastructure.AI;
|
||
using FluentAssertions;
|
||
using Xunit;
|
||
|
||
namespace BuildFlow.Rag.Tests;
|
||
|
||
/// <summary>
|
||
/// [R5] 关键词提取器单元测试：Ngram 滑窗（确定性兜底）与 jieba 分词（生产默认）。
|
||
/// 纯本地计算，不依赖 PG/Redis 等基础设施。
|
||
/// </summary>
|
||
public class KeywordExtractorTests
|
||
{
|
||
[Fact]
|
||
public void Ngram_ChineseShortSegment_KeptWhole()
|
||
{
|
||
var result = new NgramKeywordExtractor().Extract("知识库是什么");
|
||
// 连续中文段 "知识库是什么"（6 字 > 4）应产生 2-4 字滑窗子串
|
||
result.Should().Contain("知识库");
|
||
result.Should().Contain("是什么");
|
||
result.Should().NotContain("知"); // 单字 n-gram 不产生
|
||
}
|
||
|
||
[Fact]
|
||
public void Ngram_EnglishWords_LowercasedAndFiltered()
|
||
{
|
||
var result = new NgramKeywordExtractor().Extract("What is the Vector Database");
|
||
result.Should().Contain("vector").And.Contain("database");
|
||
result.Should().NotContain("what").And.NotContain("is").And.NotContain("the"); // 停用词
|
||
}
|
||
|
||
[Fact]
|
||
public void Ngram_Deduplicates()
|
||
{
|
||
var result = new NgramKeywordExtractor().Extract("检索 检索");
|
||
result.Count.Should().Be(result.Distinct().Count());
|
||
}
|
||
|
||
[Fact]
|
||
public void Jieba_RealWords_NoNoiseSubstrings()
|
||
{
|
||
var result = new JiebaKeywordExtractor().Extract("如何搭建企业知识库检索系统");
|
||
// 分词应产出真实词汇，而非全量 n-gram 子串
|
||
result.Should().Contain("知识库");
|
||
result.Should().Contain("检索");
|
||
// 全量 2-4 字滑窗会产生 "何搭" 之类噪声子串，jieba 不应出现
|
||
result.Should().NotContain("何搭");
|
||
// 停用词 "如何" 应被过滤
|
||
result.Should().NotContain("如何");
|
||
}
|
||
|
||
[Fact]
|
||
public void Jieba_EnglishLowercased()
|
||
{
|
||
var result = new JiebaKeywordExtractor().Extract("pgvector Hybrid Search");
|
||
result.Should().Contain("pgvector").And.Contain("hybrid").And.Contain("search");
|
||
}
|
||
|
||
[Fact]
|
||
public void Jieba_EmptyInput_ReturnsEmpty()
|
||
{
|
||
new JiebaKeywordExtractor().Extract("").Should().BeEmpty();
|
||
new NgramKeywordExtractor().Extract("").Should().BeEmpty();
|
||
}
|
||
|
||
[Fact]
|
||
public void Jieba_ComparedToNgram_FewerKeywords()
|
||
{
|
||
const string question = "如何搭建企业知识库检索系统";
|
||
var jiebaCount = new JiebaKeywordExtractor().Extract(question).Count;
|
||
var ngramCount = new NgramKeywordExtractor().Extract(question).Count;
|
||
// R5 的核心收益：真词数量显著少于 n-gram 子串数量（召回 SQL 参数收敛）
|
||
jiebaCount.Should().BeLessThan(ngramCount);
|
||
}
|
||
}
|