1 line
104 KiB
JSON
1 line
104 KiB
JSON
{"repo": "arabidopsisdev/recitehelper", "files": {"ReciteHelper.Infrastructure/Services/KnowledgeBaseService.cs": {"21288821229eb8a1da7f5a38d248fdfd10104f80": {"t": 1785746215.5563538, "n": 19499, "h": "b55de41d91f179bc", "c": "using OpenAI;\nusing OpenAI.Chat;\nusing ReciteHelper.Core.Entities;\nusing ReciteHelper.Core.Interfaces.Configuration;\nusing ReciteHelper.Core.Interfaces.Services;\nusing System.ClientModel;\nusing System.Collections.Concurrent;\nusing System.Text;\nusing System.Text.Json;\nusing System.Text.RegularExpressions;\nusing OpenAI.Embeddings;\nusing ReciteHelper.Core.ValueObjects;\nusing ReciteHelper.Core.DTOs;\n\nnamespace ReciteHelper.Infrastructure.Services\n{\n public class KnowledgeBaseService(\n IConfigService cfgService,\n IAiChatService aiChatService,\n HostedModelService hostedModelService) : IKnowledgeBaseService\n {\n private readonly int _chunkSize = 1500;\n private readonly int _overlap = 50;\n\n private ChatClient? _chatClient;\n private EmbeddingClient? _embedClient;\n\n private const int MaxConcurrentRequests = 4;\n private const int BatchSize = 10;\n\n private IConfigService configService = cfgService;\n\n public async Task<FileVectorStore> Build(string projectPath, string text)\n {\n if (string.IsNullOrWhiteSpace(text))\n throw new InvalidOperationException(\"知识库构建失败：题库文本为空。\");\n\n var slices = Slice(text);\n if (slices.Count == 0)\n throw new InvalidOperationException(\"知识库构建失败：未切分出有效文本。\");\n\n var cfg = await configService.LoadAsync();\n var src = new CancellationTokenSource();\n\n if (!string.IsNullOrWhiteSpace(cfg.DeepSeekKey))\n {\n _chatClient = new OpenAIClient(new ApiKeyCredential(cfg.DeepSeekKey!), new OpenAIClientOptions\n {\n Endpoint = new Uri(\"https://api.deepseek.com\")\n }).GetChatClient(\"deepseek-v4-flash\");\n }\n\n if (!string.IsNullOrWhiteSpace(cfg.QwenKey))\n _embedClient = CreateQwenEmbeddingClient(cfg.QwenKey!);\n\n var cluster = await ClusterAsync(slices, src.Token);\n var embed = await EmbedAsync(cluster, src.Token);\n var elements = BuildVectorEntries(cluster, embed, src.Token);\n var fvs = BuildVectorStore(projectPath, elements);\n\n return fvs;\n }\n\n public async Task<IReadOnlyList<KnowledgeBaseMatch>> SearchAsync(\n FileVectorStore store,\n string query,\n int topK,\n CancellationToken cancellationToken = default)\n {\n if (store.Entries.Count == 0 || string.IsNullOrWhiteSpace(query) || topK <= 0)\n return [];\n\n var cfg = await configService.LoadAsync();\n var queryVector = !string.IsNullOrWhiteSpace(cfg.QwenKey)\n ? await GenerateQwenEmbeddingAsync(cfg.QwenKey, query.Trim(), cancellationToken)\n : (await hostedModelService.EmbedTextsAsync([query.Trim()], cancellationToken))[0];\n\n return store.Search(queryVector, topK)\n .Select(result => new KnowledgeBaseMatch(\n CreateMatchTitle(result.Entry),\n result.Entry.Text,\n result.Score))\n .ToList();\n }\n\n public async Task<IReadOnlyList<float[]>> EmbedTextsAsync(\n IReadOnlyList<string> texts,\n CancellationToken cancellationToken = default)\n {\n var normalizedTexts = texts\n .Select(text => text?.Trim() ?? string.Empty)\n .ToList();\n if (normalizedTexts.Count == 0)\n return [];\n if (normalizedTexts.Any(string.IsNullOrWhiteSpace))\n throw new ArgumentException(\"待向量化文本不能为空。\", nameof(texts));\n\n var cfg = await configService.LoadAsync();\n if (string.IsNullOrWhiteSpace(cfg.QwenKey))\n return await hostedModelService.EmbedTextsAsync(normalizedTexts, cancellationToken);\n\n var embedClient = CreateQwenEmbeddingClient(cfg.QwenKey);\n var results = new float[normalizedTexts.Count][];\n var batches = normalizedTexts\n .Select((text, index) => new { Text = text, Index = index })\n .Chunk(BatchSize)\n .ToList();\n\n using var semaphore = new SemaphoreSlim(MaxConcurrentRequests);\n var tasks = batches.Select(async batch =>\n {\n await semaphore.WaitAsync(cancellationToken);\n try\n {\n var response = await embedClient.GenerateEmbeddingsAsync(\n batch.Select(item => item.Text).ToList(),\n options: null,\n cancellationToken);\n\n for (var index = 0; index < batch.Length; index++)\n results[batch[index].Index] = response.Value[index].ToFloats().ToArray();\n }\n finally\n {\n semaphore.Release();\n }\n });\n\n await Task.WhenAll(tasks);\n return results;\n }\n\n private static EmbeddingClient CreateQwenEmbeddingClient(string qwenKey)\n {\n return new OpenAIClient(\n new ApiKeyCredential(qwenKey),\n new OpenAIClientOptions\n {\n Endpoint = new Uri(\"https://dashscope.aliyuncs.com/compatible-mode/v1\")\n }).GetEmbeddingClient(\"text-embedding-v4\");\n }\n\n private static async Task<float[]> GenerateQwenEmbeddingAsync(\n string qwenKey,\n string text,\n CancellationToken cancellationToken)\n {\n var embedClient = CreateQwenEmbeddingClient(qwenKey);\n var response = await embedClient.GenerateEmbeddingsAsync(\n [text],\n options: null,\n cancellationToken);\n return response.Value[0].ToFloats().ToArray();\n }\n\n private static string CreateMatchTitle(VectorEntry entry)\n {\n if (!string.IsNullOrWhiteSpace(entry.Semantics.Summary))\n return entry.Semantics.Summary;\n\n var tags = string.Join(\"、\", entry.Semantics.Tags.Where(x => !string.IsNullOrWhiteSpace(x)));\n if (!string.IsNullOrWhiteSpace(tags))\n return tags;\n\n const int maxLength = 24;\n return entry.Text.Length <= maxLength\n ? entry.Text\n : $\"{entry.Text[..maxLength]}...\";\n }\n\n public List<string> Slice(string text)\n {\n var slices = SplitText(text);\n\n return slices.Select(x => x.Replace(\"\\n\", \"\").Trim()).ToList();\n }\n\n\n public List<string> SplitText(string text)\n {\n if (string.IsNullOrWhiteSpace(text))\n return new List<string>();\n\n text = CleanText(text);\n var paragraphs = SplitByParagraphs(text);\n\n var chunks = new List<string>();\n foreach (var para in paragraphs)\n {\n chunks.AddRange(SplitParagraph(para));\n }\n\n return chunks;\n }\n\n private string CleanText(string text)\n {\n text = Regex.Replace(text, @\"\\n\\s*\\n\", \"\\n\\n\");\n text = text.Trim();\n return text;\n }\n\n private List<string> SplitByParagraphs(string text)\n {\n var paragraphs = new List<string>();\n var currentPara = new StringBuilder();\n\n using var reader = new StringReader(text);\n string? line;\n\n while ((line = reader.ReadLine()) != null)\n {\n if (string.IsNullOrWhiteSpace(line))\n {\n if (currentPara.Length > 0)\n {\n paragraphs.Add(currentPara.ToString().Trim());\n currentPara.Clear();\n }\n }\n else\n {\n currentPara.AppendLine(line);\n }\n }\n\n if (currentPara.Length > 0)\n {\n paragraphs.Add(currentPara.ToString().Trim());\n }\n\n return paragraphs;\n }\n\n private List<string> SplitParagraph(string paragraph)\n {\n if (paragraph.Length <= _chunkSize)\n {\n return new List<string> { paragraph };\n }\n\n var chunks = new List<string>();\n var sentences = SplitIntoSentences(paragraph);\n\n var currentChunk = new StringBuilder();\n\n foreach (var sentence in sentences)\n {\n if (currentChunk.Length + sentence.Length + 1 > _chunkSize && currentChunk.Length > 0)\n {\n chunks.Add(currentChunk.ToString().Trim());\n\n string overlap = GetOverlap(currentChunk.ToString());\n currentChunk.Clear();\n currentChunk.Append(overlap);\n }\n\n currentChunk.Append(sentence);\n }\n\n if (currentChunk.Length > 0)\n {\n chunks.Add(currentChunk.ToString().Trim());\n }\n\n return chunks;\n }\n\n private List<string> SplitIntoSentences(string text)\n {\n var separators = new[] { \"。\", \"！\", \"？\", \"；\", \".\", \"!\", \"?\", \";\" };\n\n var sentences = new List<string>();\n var currentSentence = new StringBuilder();\n\n for (int i = 0; i < text.Length; i++)\n {\n currentSentence.Append(text[i]);\n\n foreach (var sep in separators)\n {\n if (text[i].ToString() == sep)\n {\n sentences.Add(currentSentence.ToString());\n currentSentence.Clear();\n break;\n }\n }\n }\n\n if (currentSentence.Length > 0)\n {\n sentences.Add(currentSentence.ToString());\n }\n\n return sentences;\n }\n\n private string GetOverlap(string chunk)\n {\n if (chunk.Length <= _overlap)\n return chunk;\n\n int start = chunk.Length - _overlap;\n\n int actualStart = chunk.Length;\n for (int i = start; i < chunk.Length; i++)\n {\n if (i > 0 && (chunk[i - 1] == '。' || chunk[i - 1] == '！' || chunk[i - 1] == '？'))\n {\n actualStart = i;\n break;\n }\n }\n\n if (actualStart >= chunk.Length)\n actualStart = start;\n\n return chunk.Substring(actualStart);\n }\n\n public async Task<Dictionary<Semantics, string>> ClusterAsync(List<string> chunks, CancellationToken cts = default)\n {\n var results = new ConcurrentDictionary<Semantics, string>();\n\n await Parallel.ForEachAsync(chunks,\n new ParallelOptions { MaxDegreeOfParallelism = MaxConcurrentRequests, CancellationToken = cts },\n async (chunk, _) =>\n {\n var segmentedSemantics = await ProcessChunkAsync(chunk);\n foreach (var item in segmentedSemantics)\n {\n results.TryAdd(item.Item1, item.Item2);\n }\n });\n\n return results.OrderBy(x => x.Key.Id).ToDictionary(x => x.Key, x => x.Value);\n }\n\n /// <summary>\n /// Process single chunk: segment by atomic knowledge points and generate semantics\n /// </summary>\n private async Task<List<(Semantics, string)>> ProcessChunkAsync(string chunk)\n {\n var prompt = BuildPrompt(chunk);\n\n const string instructions = \"你是一个专业的文本分析专家。请完成以下任务：\\n\" +\n \"1. 将输入的文本块按照知识点的原子性进行分割\\n\" +\n \"2. 确保每个分割后的段落只表达一个独立、完整的知识点\\n\" +\n \"3. 为每个知识点段落生成2-3个关键词标签\\n\" +\n \"4. 生成不超过20字的单句摘要\\n\" +\n \"5. 对于被截断的知识点，如果有明确的语义信息进行补全，否则直接丢弃\" +\n \"6. 返回JSON数组，每个元素包含text、tags、summary字段，无其他内容，不要添加markdown代码块结构\";\n\n if (_chatClient is null)\n {\n var hostedResponse = await aiChatService.RunAsync(string.Empty, prompt, instructions);\n return ParseResponse(ExtractJsonContent(hostedResponse));\n }\n\n var messages = new List<ChatMessage>\n {\n new SystemChatMessage(instructions),\n new UserChatMessage(prompt)\n };\n\n var response = await _chatClient.CompleteChatAsync(messages, new ChatCompletionOptions\n {\n Temperature = 0.3f,\n MaxOutputTokenCount = 66666\n });\n\n var json = ExtractJsonContent(response.Value.Content[0].Text);\n return ParseResponse(json);\n }\n\n private static string BuildPrompt(string chunk)\n {\n var sb = new StringBuilder();\n sb.AppendLine(\"请处理以下文本块：\");\n sb.AppendLine();\n sb.AppendLine(chunk);\n sb.AppendLine();\n sb.AppendLine(\"返回JSON数组格式，示例：\");\n sb.AppendLine(\"[\");\n sb.AppendLine(\" {\\\"text\\\": \\\"分割后的知识点1\\\", \\\"tags\\\": [\\\"标签1\\\", \\\"标签2\\\"], \\\"summary\\\": \\\"摘要\\\"},\");\n sb.AppendLine(\" {\\\"text\\\": \\\"分割后的知识点2\\\", \\\"tags\\\": [\\\"标签1\\\", \\\"标签2\\\"], \\\"summary\\\": \\\"摘要\\\"}\");\n sb.AppendLine(\"]\");\n\n return sb.ToString();\n }\n\n /// <summary>\n /// Parse response and return semantics-text pairs with auto-incremented IDs\n /// </summary>\n private static List<(Semantics, string)> ParseResponse(string json)\n {\n var result = new List<(Semantics, string)>();\n var idCounter = 0;\n\n using var doc = JsonDocument.Parse(json);\n var array = doc.RootElement.EnumerateArray().ToList();\n\n foreach (var item in array)\n {\n var text = item.TryGetProperty(\"text\", out var textProp)\n ? textProp.GetString() ?? \"\"\n : \"\";\n\n if (string.IsNullOrWhiteSpace(text))\n continue;\n\n var semantics = new Semantics\n {\n Id = idCounter++,\n Tags = item.TryGetProperty(\"tags\", out var tags)\n ? tags.EnumerateArray()\n .Select(t => t.GetString() ?? \"\")\n .Where(t => !string.IsNullOrWhiteSpace(t))\n .ToList()\n : [],\n Summary = item.TryGetProperty(\"summary\", out var summary)\n ? (summary.GetString() ?? \"\").Trim()\n : \"\"\n };\n\n result.Add((semantics, text.Trim()));\n }\n\n\n return result;\n }\n\n private static string ExtractJsonContent(string content)\n {\n if (string.IsNullOrWhiteSpace(content))\n return \"[]\";\n\n var trimmed = content.Trim();\n var fenceStart = trimmed.IndexOf(\"```\", StringComparison.Ordinal);\n if (fenceStart >= 0)\n {\n var jsonStart = trimmed.IndexOf('\\n', fenceStart);\n if (jsonStart >= 0)\n {\n var fenceEnd = trimmed.LastIndexOf(\"```\", StringComparison.Ordinal);\n trimmed = fenceEnd > jsonStart\n ? trimmed[(jsonStart + 1)..fenceEnd].Trim()\n : trimmed[(jsonStart + 1)..].Trim();\n }\n }\n\n var arrayStart = trimmed.IndexOf('[');\n var arrayEnd = trimmed.LastIndexOf(']');\n return arrayStart >= 0 && arrayEnd > arrayStart\n ? trimmed[arrayStart..(arrayEnd + 1)]\n : trimmed;\n }\n\n public async Task<Dictionary<Semantics, float[]>> EmbedAsync(\n Dictionary<Semantics, string> semanticChunks,\n CancellationToken cts)\n {\n if (semanticChunks.Count == 0) return [];\n\n var semanticsList = semanticChunks.Keys.ToList();\n var textsList = semanticChunks.Values.ToList();\n\n // Results array maintains index correspondence with input order\n var results = new float[textsList.Count][];\n\n var batches = textsList\n .Select((text, index) => new { text, index })\n .Chunk(BatchSize)\n .ToList();\n\n if (_embedClient is null)\n {\n var vectors = await hostedModelService.EmbedTextsAsync(textsList, cts);\n var hostedResult = new Dictionary<Semantics, float[]>();\n for (var index = 0; index < semanticsList.Count; index++)\n hostedResult[semanticsList[index]] = vectors[index];\n\n return hostedResult;\n }\n\n using var semaphore = new SemaphoreSlim(MaxConcurrentRequests);\n\n var tasks = batches.Select(async batch =>\n {\n await semaphore.WaitAsync(cts);\n try\n {\n var batchTexts = batch.Select(b => b.text).ToList();\n var response = await _embedClient.GenerateEmbeddingsAsync(batchTexts, options: null, cts);\n\n for (int i = 0; i < batch.Length; i++)\n results[batch[i].index] = response.Value[i].ToFloats().ToArray();\n }\n finally\n {\n semaphore.Release();\n }\n });\n\n await Task.WhenAll(tasks);\n\n // Reconstruct the result dictionary maintaining Semantics-Embedding pairing\n var resultDictionary = new Dictionary<Semantics, float[]>();\n for (int i = 0; i < semanticsList.Count; i++)\n {\n resultDictionary[semanticsList[i]] = results[i];\n }\n\n return resultDictionary;\n }\n\n public List<VectorEntry> BuildVectorEntries(\n Dictionary<Semantics, string> semanticChunks,\n Dictionary<Semantics, float[]> vectors,\n CancellationToken cts)\n {\n var entries = semanticChunks\n .Select((kvp, idx) => new VectorEntry\n {\n Id = idx,\n Semantics = new Semantics\n {\n Id = idx,\n Tags = kvp.Key.Tags,\n Summary = kvp.Key.Summary\n },\n Text = kvp.Value,\n Vector = vectors[kvp.Key],\n SourceFile = \"question-bank\",\n CreatedAt = DateTime.UtcNow\n })\n .ToList();\n\n Console.WriteLine($\"知识库已构建：{entries.Count} 个向量。\");\n return entries;\n }\n\n public FileVectorStore BuildVectorStore(string projectPath, List<VectorEntry> entries)\n {\n var fvs = new FileVectorStore(projectPath);\n fvs.AddRange(entries);\n return fvs;\n }\n\n }\n}\n"}, "1d90a68d9b606d170ee38ab0151fb3be6af5785a": {"t": 1785760672.0380714, "n": 18458, "h": "c0ea3c0c9a228782", "c": "using OpenAI;\nusing OpenAI.Chat;\nusing ReciteHelper.Core.Entities;\nusing ReciteHelper.Core.Interfaces.Configuration;\nusing ReciteHelper.Core.Interfaces.Services;\nusing System.ClientModel;\nusing System.Collections.Concurrent;\nusing System.Text;\nusing System.Text.Json;\nusing System.Text.RegularExpressions;\nusing OpenAI.Embeddings;\nusing ReciteHelper.Core.ValueObjects;\nusing ReciteHelper.Core.DTOs;\n\nnamespace ReciteHelper.Infrastructure.Services\n{\n public class KnowledgeBaseService(IConfigService cfgService) : IKnowledgeBaseService\n {\n private readonly int _chunkSize = 1500;\n private readonly int _overlap = 50;\n\n private ChatClient _chatClient;\n private EmbeddingClient _embedClient;\n\n private const int MaxConcurrentRequests = 4;\n private const int BatchSize = 10;\n\n private IConfigService configService = cfgService;\n\n public async Task<FileVectorStore> Build(string projectPath, string text)\n {\n if (string.IsNullOrWhiteSpace(text))\n throw new InvalidOperationException(\"知识库构建失败：题库文本为空。\");\n\n var slices = Slice(text);\n if (slices.Count == 0)\n throw new InvalidOperationException(\"知识库构建失败：未切分出有效文本。\");\n\n var cfg = await configService.LoadAsync();\n if (string.IsNullOrWhiteSpace(cfg.DeepSeekKey))\n throw new InvalidOperationException(\"知识库构建失败：未配置 DeepSeek Key。请在 Config.xml 中配置 DeepSeekKey。\");\n\n if (string.IsNullOrWhiteSpace(cfg.QwenKey))\n throw new InvalidOperationException(\"知识库构建失败：未配置 Qwen Key。\");\n\n var src = new CancellationTokenSource();\n\n _chatClient = new OpenAIClient(new ApiKeyCredential(cfg.DeepSeekKey!), new OpenAIClientOptions\n {\n Endpoint = new Uri(\"https://api.deepseek.com\")\n }).GetChatClient(\"deepseek-v4-flash\");\n _embedClient = CreateQwenEmbeddingClient(cfg.QwenKey!);\n\n var cluster = await ClusterAsync(slices, src.Token);\n var embed = await EmbedAsync(cluster, src.Token);\n var elements = BuildVectorEntries(cluster, embed, src.Token);\n var fvs = BuildVectorStore(projectPath, elements);\n\n return fvs;\n }\n\n public async Task<IReadOnlyList<KnowledgeBaseMatch>> SearchAsync(\n FileVectorStore store,\n string query,\n int topK,\n CancellationToken cancellationToken = default)\n {\n if (store.Entries.Count == 0 || string.IsNullOrWhiteSpace(query) || topK <= 0)\n return [];\n\n var cfg = await configService.LoadAsync();\n if (string.IsNullOrWhiteSpace(cfg.QwenKey))\n throw new InvalidOperationException(\"知识库查询失败：未配置 Qwen Key。\");\n\n var embedClient = CreateQwenEmbeddingClient(cfg.QwenKey);\n\n var response = await embedClient.GenerateEmbeddingsAsync(\n [query.Trim()],\n options: null,\n cancellationToken);\n var queryVector = response.Value[0].ToFloats().ToArray();\n\n return store.Search(queryVector, topK)\n .Select(result => new KnowledgeBaseMatch(\n CreateMatchTitle(result.Entry),\n result.Entry.Text,\n result.Score))\n .ToList();\n }\n\n public async Task<IReadOnlyList<float[]>> EmbedTextsAsync(\n IReadOnlyList<string> texts,\n CancellationToken cancellationToken = default)\n {\n var normalizedTexts = texts\n .Select(text => text?.Trim() ?? string.Empty)\n .ToList();\n if (normalizedTexts.Count == 0)\n return [];\n if (normalizedTexts.Any(string.IsNullOrWhiteSpace))\n throw new ArgumentException(\"待向量化文本不能为空。\", nameof(texts));\n\n var cfg = await configService.LoadAsync();\n if (string.IsNullOrWhiteSpace(cfg.QwenKey))\n throw new InvalidOperationException(\"语义相似度计算失败：未配置 Qwen Key。\");\n\n var embedClient = CreateQwenEmbeddingClient(cfg.QwenKey);\n var results = new float[normalizedTexts.Count][];\n var batches = normalizedTexts\n .Select((text, index) => new { Text = text, Index = index })\n .Chunk(BatchSize)\n .ToList();\n\n using var semaphore = new SemaphoreSlim(MaxConcurrentRequests);\n var tasks = batches.Select(async batch =>\n {\n await semaphore.WaitAsync(cancellationToken);\n try\n {\n var response = await embedClient.GenerateEmbeddingsAsync(\n batch.Select(item => item.Text).ToList(),\n options: null,\n cancellationToken);\n\n for (var index = 0; index < batch.Length; index++)\n results[batch[index].Index] = response.Value[index].ToFloats().ToArray();\n }\n finally\n {\n semaphore.Release();\n }\n });\n\n await Task.WhenAll(tasks);\n return results;\n }\n\n private static EmbeddingClient CreateQwenEmbeddingClient(string qwenKey)\n {\n return new OpenAIClient(\n new ApiKeyCredential(qwenKey),\n new OpenAIClientOptions\n {\n Endpoint = new Uri(\"https://dashscope.aliyuncs.com/compatible-mode/v1\")\n }).GetEmbeddingClient(\"text-embedding-v4\");\n }\n\n private static string CreateMatchTitle(VectorEntry entry)\n {\n if (!string.IsNullOrWhiteSpace(entry.Semantics.Summary))\n return entry.Semantics.Summary;\n\n var tags = string.Join(\"、\", entry.Semantics.Tags.Where(x => !string.IsNullOrWhiteSpace(x)));\n if (!string.IsNullOrWhiteSpace(tags))\n return tags;\n\n const int maxLength = 24;\n return entry.Text.Length <= maxLength\n ? entry.Text\n : $\"{entry.Text[..maxLength]}...\";\n }\n\n public List<string> Slice(string text)\n {\n var slices = SplitText(text);\n\n return slices.Select(x => x.Replace(\"\\n\", \"\").Trim()).ToList();\n }\n\n\n public List<string> SplitText(string text)\n {\n if (string.IsNullOrWhiteSpace(text))\n return new List<string>();\n\n text = CleanText(text);\n var paragraphs = SplitByParagraphs(text);\n\n var chunks = new List<string>();\n foreach (var para in paragraphs)\n {\n chunks.AddRange(SplitParagraph(para));\n }\n\n return chunks;\n }\n\n private string CleanText(string text)\n {\n text = Regex.Replace(text, @\"\\n\\s*\\n\", \"\\n\\n\");\n text = text.Trim();\n return text;\n }\n\n private List<string> SplitByParagraphs(string text)\n {\n var paragraphs = new List<string>();\n var currentPara = new StringBuilder();\n\n using var reader = new StringReader(text);\n string? line;\n\n while ((line = reader.ReadLine()) != null)\n {\n if (string.IsNullOrWhiteSpace(line))\n {\n if (currentPara.Length > 0)\n {\n paragraphs.Add(currentPara.ToString().Trim());\n currentPara.Clear();\n }\n }\n else\n {\n currentPara.AppendLine(line);\n }\n }\n\n if (currentPara.Length > 0)\n {\n paragraphs.Add(currentPara.ToString().Trim());\n }\n\n return paragraphs;\n }\n\n private List<string> SplitParagraph(string paragraph)\n {\n if (paragraph.Length <= _chunkSize)\n {\n return new List<string> { paragraph };\n }\n\n var chunks = new List<string>();\n var sentences = SplitIntoSentences(paragraph);\n\n var currentChunk = new StringBuilder();\n\n foreach (var sentence in sentences)\n {\n if (currentChunk.Length + sentence.Length + 1 > _chunkSize && currentChunk.Length > 0)\n {\n chunks.Add(currentChunk.ToString().Trim());\n\n string overlap = GetOverlap(currentChunk.ToString());\n currentChunk.Clear();\n currentChunk.Append(overlap);\n }\n\n currentChunk.Append(sentence);\n }\n\n if (currentChunk.Length > 0)\n {\n chunks.Add(currentChunk.ToString().Trim());\n }\n\n return chunks;\n }\n\n private List<string> SplitIntoSentences(string text)\n {\n var separators = new[] { \"。\", \"！\", \"？\", \"；\", \".\", \"!\", \"?\", \";\" };\n\n var sentences = new List<string>();\n var currentSentence = new StringBuilder();\n\n for (int i = 0; i < text.Length; i++)\n {\n currentSentence.Append(text[i]);\n\n foreach (var sep in separators)\n {\n if (text[i].ToString() == sep)\n {\n sentences.Add(currentSentence.ToString());\n currentSentence.Clear();\n break;\n }\n }\n }\n\n if (currentSentence.Length > 0)\n {\n sentences.Add(currentSentence.ToString());\n }\n\n return sentences;\n }\n\n private string GetOverlap(string chunk)\n {\n if (chunk.Length <= _overlap)\n return chunk;\n\n int start = chunk.Length - _overlap;\n\n int actualStart = chunk.Length;\n for (int i = start; i < chunk.Length; i++)\n {\n if (i > 0 && (chunk[i - 1] == '。' || chunk[i - 1] == '！' || chunk[i - 1] == '？'))\n {\n actualStart = i;\n break;\n }\n }\n\n if (actualStart >= chunk.Length)\n actualStart = start;\n\n return chunk.Substring(actualStart);\n }\n\n public async Task<Dictionary<Semantics, string>> ClusterAsync(List<string> chunks, CancellationToken cts = default)\n {\n var results = new ConcurrentDictionary<Semantics, string>();\n\n await Parallel.ForEachAsync(chunks,\n new ParallelOptions { MaxDegreeOfParallelism = MaxConcurrentRequests, CancellationToken = cts },\n async (chunk, _) =>\n {\n var segmentedSemantics = await ProcessChunkAsync(chunk);\n foreach (var item in segmentedSemantics)\n {\n results.TryAdd(item.Item1, item.Item2);\n }\n });\n\n return results.OrderBy(x => x.Key.Id).ToDictionary(x => x.Key, x => x.Value);\n }\n\n /// <summary>\n /// Process single chunk: segment by atomic knowledge points and generate semantics\n /// </summary>\n private async Task<List<(Semantics, string)>> ProcessChunkAsync(string chunk)\n {\n var prompt = BuildPrompt(chunk);\n\n var messages = new List<ChatMessage>\n {\n new SystemChatMessage(\"你是一个专业的文本分析专家。请完成以下任务：\\n\" +\n \"1. 将输入的文本块按照知识点的原子性进行分割\\n\" +\n \"2. 确保每个分割后的段落只表达一个独立、完整的知识点\\n\" +\n \"3. 为每个知识点段落生成2-3个关键词标签\\n\" +\n \"4. 生成不超过20字的单句摘要\\n\" +\n \"5. 对于被截断的知识点，如果有明确的语义信息进行补全，否则直接丢弃\" +\n \"6. 返回JSON数组，每个元素包含text、tags、summary字段，无其他内容，不要添加markdown代码块结构\"),\n new UserChatMessage(prompt)\n };\n\n var response = await _chatClient.CompleteChatAsync(messages, new ChatCompletionOptions\n {\n Temperature = 0.3f,\n MaxOutputTokenCount = 66666\n });\n\n var json = ExtractJsonContent(response.Value.Content[0].Text);\n return ParseResponse(json);\n }\n\n private static string BuildPrompt(string chunk)\n {\n var sb = new StringBuilder();\n sb.AppendLine(\"请处理以下文本块：\");\n sb.AppendLine();\n sb.AppendLine(chunk);\n sb.AppendLine();\n sb.AppendLine(\"返回JSON数组格式，示例：\");\n sb.AppendLine(\"[\");\n sb.AppendLine(\" {\\\"text\\\": \\\"分割后的知识点1\\\", \\\"tags\\\": [\\\"标签1\\\", \\\"标签2\\\"], \\\"summary\\\": \\\"摘要\\\"},\");\n sb.AppendLine(\" {\\\"text\\\": \\\"分割后的知识点2\\\", \\\"tags\\\": [\\\"标签1\\\", \\\"标签2\\\"], \\\"summary\\\": \\\"摘要\\\"}\");\n sb.AppendLine(\"]\");\n\n return sb.ToString();\n }\n\n /// <summary>\n /// Parse response and return semantics-text pairs with auto-incremented IDs\n /// </summary>\n private static List<(Semantics, string)> ParseResponse(string json)\n {\n var result = new List<(Semantics, string)>();\n var idCounter = 0;\n\n using var doc = JsonDocument.Parse(json);\n var array = doc.RootElement.EnumerateArray().ToList();\n\n foreach (var item in array)\n {\n var text = item.TryGetProperty(\"text\", out var textProp)\n ? textProp.GetString() ?? \"\"\n : \"\";\n\n if (string.IsNullOrWhiteSpace(text))\n continue;\n\n var semantics = new Semantics\n {\n Id = idCounter++,\n Tags = item.TryGetProperty(\"tags\", out var tags)\n ? tags.EnumerateArray()\n .Select(t => t.GetString() ?? \"\")\n .Where(t => !string.IsNullOrWhiteSpace(t))\n .ToList()\n : [],\n Summary = item.TryGetProperty(\"summary\", out var summary)\n ? (summary.GetString() ?? \"\").Trim()\n : \"\"\n };\n\n result.Add((semantics, text.Trim()));\n }\n\n\n return result;\n }\n\n private static string ExtractJsonContent(string content)\n {\n if (string.IsNullOrWhiteSpace(content))\n return \"[]\";\n\n var trimmed = content.Trim();\n var fenceStart = trimmed.IndexOf(\"```\", StringComparison.Ordinal);\n if (fenceStart >= 0)\n {\n var jsonStart = trimmed.IndexOf('\\n', fenceStart);\n if (jsonStart >= 0)\n {\n var fenceEnd = trimmed.LastIndexOf(\"```\", StringComparison.Ordinal);\n trimmed = fenceEnd > jsonStart\n ? trimmed[(jsonStart + 1)..fenceEnd].Trim()\n : trimmed[(jsonStart + 1)..].Trim();\n }\n }\n\n var arrayStart = trimmed.IndexOf('[');\n var arrayEnd = trimmed.LastIndexOf(']');\n return arrayStart >= 0 && arrayEnd > arrayStart\n ? trimmed[arrayStart..(arrayEnd + 1)]\n : trimmed;\n }\n\n public async Task<Dictionary<Semantics, float[]>> EmbedAsync(\n Dictionary<Semantics, string> semanticChunks,\n CancellationToken cts)\n {\n if (semanticChunks.Count == 0) return [];\n\n var semanticsList = semanticChunks.Keys.ToList();\n var textsList = semanticChunks.Values.ToList();\n\n // Results array maintains index correspondence with input order\n var results = new float[textsList.Count][];\n\n var batches = textsList\n .Select((text, index) => new { text, index })\n .Chunk(BatchSize)\n .ToList();\n\n using var semaphore = new SemaphoreSlim(MaxConcurrentRequests);\n\n var tasks = batches.Select(async batch =>\n {\n await semaphore.WaitAsync(cts);\n try\n {\n var batchTexts = batch.Select(b => b.text).ToList();\n var response = await _embedClient.GenerateEmbeddingsAsync(batchTexts, options: null, cts);\n\n for (int i = 0; i < batch.Length; i++)\n results[batch[i].index] = response.Value[i].ToFloats().ToArray();\n }\n finally\n {\n semaphore.Release();\n }\n });\n\n await Task.WhenAll(tasks);\n\n // Reconstruct the result dictionary maintaining Semantics-Embedding pairing\n var resultDictionary = new Dictionary<Semantics, float[]>();\n for (int i = 0; i < semanticsList.Count; i++)\n {\n resultDictionary[semanticsList[i]] = results[i];\n }\n\n return resultDictionary;\n }\n\n public List<VectorEntry> BuildVectorEntries(\n Dictionary<Semantics, string> semanticChunks,\n Dictionary<Semantics, float[]> vectors,\n CancellationToken cts)\n {\n var entries = semanticChunks\n .Select((kvp, idx) => new VectorEntry\n {\n Id = idx,\n Semantics = new Semantics\n {\n Id = idx,\n Tags = kvp.Key.Tags,\n Summary = kvp.Key.Summary\n },\n Text = kvp.Value,\n Vector = vectors[kvp.Key],\n SourceFile = \"question-bank\",\n CreatedAt = DateTime.UtcNow\n })\n .ToList();\n\n Console.WriteLine($\"知识库已构建：{entries.Count} 个向量。\");\n return entries;\n }\n\n public FileVectorStore BuildVectorStore(string projectPath, List<VectorEntry> entries)\n {\n var fvs = new FileVectorStore(projectPath);\n fvs.AddRange(entries);\n return fvs;\n }\n\n }\n}\n"}, "8cc55d133d0be99b98e6b8c1a05ec6dada13c4dd": {"t": 1785760673.7215776, "n": 16618, "h": "9c81b2d22fe8af60", "c": "using OpenAI;\nusing OpenAI.Chat;\nusing ReciteHelper.Core.Entities;\nusing ReciteHelper.Core.Interfaces.Configuration;\nusing ReciteHelper.Core.Interfaces.Services;\nusing System.ClientModel;\nusing System.Collections.Concurrent;\nusing System.Text;\nusing System.Text.Json;\nusing System.Text.RegularExpressions;\nusing OpenAI.Embeddings;\nusing ReciteHelper.Core.ValueObjects;\nusing ReciteHelper.Core.DTOs;\n\nnamespace ReciteHelper.Infrastructure.Services\n{\n public class KnowledgeBaseService(IConfigService cfgService) : IKnowledgeBaseService\n {\n private readonly int _chunkSize = 1500;\n private readonly int _overlap = 50;\n\n private ChatClient _chatClient;\n private EmbeddingClient _embedClient;\n\n private const int MaxConcurrentRequests = 4;\n private const int BatchSize = 10;\n\n private IConfigService configService = cfgService;\n\n public async Task<FileVectorStore> Build(string projectPath, string text)\n {\n if (string.IsNullOrWhiteSpace(text))\n throw new InvalidOperationException(\"知识库构建失败：题库文本为空。\");\n\n var slices = Slice(text);\n if (slices.Count == 0)\n throw new InvalidOperationException(\"知识库构建失败：未切分出有效文本。\");\n\n var cfg = await configService.LoadAsync();\n if (string.IsNullOrWhiteSpace(cfg.DeepSeekKey))\n throw new InvalidOperationException(\"知识库构建失败：未配置 DeepSeek Key。请在 Config.xml 中配置 DeepSeekKey。\");\n\n if (string.IsNullOrWhiteSpace(cfg.QwenKey))\n throw new InvalidOperationException(\"知识库构建失败：未配置 Qwen Key。\");\n\n var src = new CancellationTokenSource();\n\n _chatClient = new OpenAIClient(new ApiKeyCredential(cfg.DeepSeekKey!), new OpenAIClientOptions\n {\n Endpoint = new Uri(\"https://api.deepseek.com\")\n }).GetChatClient(\"deepseek-v4-flash\");\n _embedClient = new OpenAIClient(new ApiKeyCredential(cfg.QwenKey!), new OpenAIClientOptions\n {\n Endpoint = new Uri(\"https://dashscope.aliyuncs.com/compatible-mode/v1\")\n }).GetEmbeddingClient(\"text-embedding-v4\");\n\n var cluster = await ClusterAsync(slices, src.Token);\n var embed = await EmbedAsync(cluster, src.Token);\n var elements = BuildVectorEntries(cluster, embed, src.Token);\n var fvs = BuildVectorStore(projectPath, elements);\n\n return fvs;\n }\n\n public async Task<IReadOnlyList<KnowledgeBaseMatch>> SearchAsync(\n FileVectorStore store,\n string query,\n int topK,\n CancellationToken cancellationToken = default)\n {\n if (store.Entries.Count == 0 || string.IsNullOrWhiteSpace(query) || topK <= 0)\n return [];\n\n var cfg = await configService.LoadAsync();\n if (string.IsNullOrWhiteSpace(cfg.QwenKey))\n throw new InvalidOperationException(\"知识库查询失败：未配置 Qwen Key。\");\n\n var embedClient = new OpenAIClient(\n new ApiKeyCredential(cfg.QwenKey),\n new OpenAIClientOptions\n {\n Endpoint = new Uri(\"https://dashscope.aliyuncs.com/compatible-mode/v1\")\n }).GetEmbeddingClient(\"text-embedding-v4\");\n\n var response = await embedClient.GenerateEmbeddingsAsync(\n [query.Trim()],\n options: null,\n cancellationToken);\n var queryVector = response.Value[0].ToFloats().ToArray();\n\n return store.Search(queryVector, topK)\n .Select(result => new KnowledgeBaseMatch(\n CreateMatchTitle(result.Entry),\n result.Entry.Text,\n result.Score))\n .ToList();\n }\n\n private static string CreateMatchTitle(VectorEntry entry)\n {\n if (!string.IsNullOrWhiteSpace(entry.Semantics.Summary))\n return entry.Semantics.Summary;\n\n var tags = string.Join(\"、\", entry.Semantics.Tags.Where(x => !string.IsNullOrWhiteSpace(x)));\n if (!string.IsNullOrWhiteSpace(tags))\n return tags;\n\n const int maxLength = 24;\n return entry.Text.Length <= maxLength\n ? entry.Text\n : $\"{entry.Text[..maxLength]}...\";\n }\n\n public List<string> Slice(string text)\n {\n var slices = SplitText(text);\n\n return slices.Select(x => x.Replace(\"\\n\", \"\").Trim()).ToList();\n }\n\n\n public List<string> SplitText(string text)\n {\n if (string.IsNullOrWhiteSpace(text))\n return new List<string>();\n\n text = CleanText(text);\n var paragraphs = SplitByParagraphs(text);\n\n var chunks = new List<string>();\n foreach (var para in paragraphs)\n {\n chunks.AddRange(SplitParagraph(para));\n }\n\n return chunks;\n }\n\n private string CleanText(string text)\n {\n text = Regex.Replace(text, @\"\\n\\s*\\n\", \"\\n\\n\");\n text = text.Trim();\n return text;\n }\n\n private List<string> SplitByParagraphs(string text)\n {\n var paragraphs = new List<string>();\n var currentPara = new StringBuilder();\n\n using var reader = new StringReader(text);\n string? line;\n\n while ((line = reader.ReadLine()) != null)\n {\n if (string.IsNullOrWhiteSpace(line))\n {\n if (currentPara.Length > 0)\n {\n paragraphs.Add(currentPara.ToString().Trim());\n currentPara.Clear();\n }\n }\n else\n {\n currentPara.AppendLine(line);\n }\n }\n\n if (currentPara.Length > 0)\n {\n paragraphs.Add(currentPara.ToString().Trim());\n }\n\n return paragraphs;\n }\n\n private List<string> SplitParagraph(string paragraph)\n {\n if (paragraph.Length <= _chunkSize)\n {\n return new List<string> { paragraph };\n }\n\n var chunks = new List<string>();\n var sentences = SplitIntoSentences(paragraph);\n\n var currentChunk = new StringBuilder();\n\n foreach (var sentence in sentences)\n {\n if (currentChunk.Length + sentence.Length + 1 > _chunkSize && currentChunk.Length > 0)\n {\n chunks.Add(currentChunk.ToString().Trim());\n\n string overlap = GetOverlap(currentChunk.ToString());\n currentChunk.Clear();\n currentChunk.Append(overlap);\n }\n\n currentChunk.Append(sentence);\n }\n\n if (currentChunk.Length > 0)\n {\n chunks.Add(currentChunk.ToString().Trim());\n }\n\n return chunks;\n }\n\n private List<string> SplitIntoSentences(string text)\n {\n var separators = new[] { \"。\", \"！\", \"？\", \"；\", \".\", \"!\", \"?\", \";\" };\n\n var sentences = new List<string>();\n var currentSentence = new StringBuilder();\n\n for (int i = 0; i < text.Length; i++)\n {\n currentSentence.Append(text[i]);\n\n foreach (var sep in separators)\n {\n if (text[i].ToString() == sep)\n {\n sentences.Add(currentSentence.ToString());\n currentSentence.Clear();\n break;\n }\n }\n }\n\n if (currentSentence.Length > 0)\n {\n sentences.Add(currentSentence.ToString());\n }\n\n return sentences;\n }\n\n private string GetOverlap(string chunk)\n {\n if (chunk.Length <= _overlap)\n return chunk;\n\n int start = chunk.Length - _overlap;\n\n int actualStart = chunk.Length;\n for (int i = start; i < chunk.Length; i++)\n {\n if (i > 0 && (chunk[i - 1] == '。' || chunk[i - 1] == '！' || chunk[i - 1] == '？'))\n {\n actualStart = i;\n break;\n }\n }\n\n if (actualStart >= chunk.Length)\n actualStart = start;\n\n return chunk.Substring(actualStart);\n }\n\n public async Task<Dictionary<Semantics, string>> ClusterAsync(List<string> chunks, CancellationToken cts = default)\n {\n var results = new ConcurrentDictionary<Semantics, string>();\n\n await Parallel.ForEachAsync(chunks,\n new ParallelOptions { MaxDegreeOfParallelism = MaxConcurrentRequests, CancellationToken = cts },\n async (chunk, _) =>\n {\n var segmentedSemantics = await ProcessChunkAsync(chunk);\n foreach (var item in segmentedSemantics)\n {\n results.TryAdd(item.Item1, item.Item2);\n }\n });\n\n return results.OrderBy(x => x.Key.Id).ToDictionary(x => x.Key, x => x.Value);\n }\n\n /// <summary>\n /// Process single chunk: segment by atomic knowledge points and generate semantics\n /// </summary>\n private async Task<List<(Semantics, string)>> ProcessChunkAsync(string chunk)\n {\n var prompt = BuildPrompt(chunk);\n\n var messages = new List<ChatMessage>\n {\n new SystemChatMessage(\"你是一个专业的文本分析专家。请完成以下任务：\\n\" +\n \"1. 将输入的文本块按照知识点的原子性进行分割\\n\" +\n \"2. 确保每个分割后的段落只表达一个独立、完整的知识点\\n\" +\n \"3. 为每个知识点段落生成2-3个关键词标签\\n\" +\n \"4. 生成不超过20字的单句摘要\\n\" +\n \"5. 对于被截断的知识点，如果有明确的语义信息进行补全，否则直接丢弃\" +\n \"6. 返回JSON数组，每个元素包含text、tags、summary字段，无其他内容，不要添加markdown代码块结构\"),\n new UserChatMessage(prompt)\n };\n\n var response = await _chatClient.CompleteChatAsync(messages, new ChatCompletionOptions\n {\n Temperature = 0.3f,\n MaxOutputTokenCount = 66666\n });\n\n var json = ExtractJsonContent(response.Value.Content[0].Text);\n return ParseResponse(json);\n }\n\n private static string BuildPrompt(string chunk)\n {\n var sb = new StringBuilder();\n sb.AppendLine(\"请处理以下文本块：\");\n sb.AppendLine();\n sb.AppendLine(chunk);\n sb.AppendLine();\n sb.AppendLine(\"返回JSON数组格式，示例：\");\n sb.AppendLine(\"[\");\n sb.AppendLine(\" {\\\"text\\\": \\\"分割后的知识点1\\\", \\\"tags\\\": [\\\"标签1\\\", \\\"标签2\\\"], \\\"summary\\\": \\\"摘要\\\"},\");\n sb.AppendLine(\" {\\\"text\\\": \\\"分割后的知识点2\\\", \\\"tags\\\": [\\\"标签1\\\", \\\"标签2\\\"], \\\"summary\\\": \\\"摘要\\\"}\");\n sb.AppendLine(\"]\");\n\n return sb.ToString();\n }\n\n /// <summary>\n /// Parse response and return semantics-text pairs with auto-incremented IDs\n /// </summary>\n private static List<(Semantics, string)> ParseResponse(string json)\n {\n var result = new List<(Semantics, string)>();\n var idCounter = 0;\n\n using var doc = JsonDocument.Parse(json);\n var array = doc.RootElement.EnumerateArray().ToList();\n\n foreach (var item in array)\n {\n var text = item.TryGetProperty(\"text\", out var textProp)\n ? textProp.GetString() ?? \"\"\n : \"\";\n\n if (string.IsNullOrWhiteSpace(text))\n continue;\n\n var semantics = new Semantics\n {\n Id = idCounter++,\n Tags = item.TryGetProperty(\"tags\", out var tags)\n ? tags.EnumerateArray()\n .Select(t => t.GetString() ?? \"\")\n .Where(t => !string.IsNullOrWhiteSpace(t))\n .ToList()\n : [],\n Summary = item.TryGetProperty(\"summary\", out var summary)\n ? (summary.GetString() ?? \"\").Trim()\n : \"\"\n };\n\n result.Add((semantics, text.Trim()));\n }\n\n\n return result;\n }\n\n private static string ExtractJsonContent(string content)\n {\n if (string.IsNullOrWhiteSpace(content))\n return \"[]\";\n\n var trimmed = content.Trim();\n var fenceStart = trimmed.IndexOf(\"```\", StringComparison.Ordinal);\n if (fenceStart >= 0)\n {\n var jsonStart = trimmed.IndexOf('\\n', fenceStart);\n if (jsonStart >= 0)\n {\n var fenceEnd = trimmed.LastIndexOf(\"```\", StringComparison.Ordinal);\n trimmed = fenceEnd > jsonStart\n ? trimmed[(jsonStart + 1)..fenceEnd].Trim()\n : trimmed[(jsonStart + 1)..].Trim();\n }\n }\n\n var arrayStart = trimmed.IndexOf('[');\n var arrayEnd = trimmed.LastIndexOf(']');\n return arrayStart >= 0 && arrayEnd > arrayStart\n ? trimmed[arrayStart..(arrayEnd + 1)]\n : trimmed;\n }\n\n public async Task<Dictionary<Semantics, float[]>> EmbedAsync(\n Dictionary<Semantics, string> semanticChunks,\n CancellationToken cts)\n {\n if (semanticChunks.Count == 0) return [];\n\n var semanticsList = semanticChunks.Keys.ToList();\n var textsList = semanticChunks.Values.ToList();\n\n // Results array maintains index correspondence with input order\n var results = new float[textsList.Count][];\n\n var batches = textsList\n .Select((text, index) => new { text, index })\n .Chunk(BatchSize)\n .ToList();\n\n using var semaphore = new SemaphoreSlim(MaxConcurrentRequests);\n\n var tasks = batches.Select(async batch =>\n {\n await semaphore.WaitAsync(cts);\n try\n {\n var batchTexts = batch.Select(b => b.text).ToList();\n var response = await _embedClient.GenerateEmbeddingsAsync(batchTexts, options: null, cts);\n\n for (int i = 0; i < batch.Length; i++)\n results[batch[i].index] = response.Value[i].ToFloats().ToArray();\n }\n finally\n {\n semaphore.Release();\n }\n });\n\n await Task.WhenAll(tasks);\n\n // Reconstruct the result dictionary maintaining Semantics-Embedding pairing\n var resultDictionary = new Dictionary<Semantics, float[]>();\n for (int i = 0; i < semanticsList.Count; i++)\n {\n resultDictionary[semanticsList[i]] = results[i];\n }\n\n return resultDictionary;\n }\n\n public List<VectorEntry> BuildVectorEntries(\n Dictionary<Semantics, string> semanticChunks,\n Dictionary<Semantics, float[]> vectors,\n CancellationToken cts)\n {\n var entries = semanticChunks\n .Select((kvp, idx) => new VectorEntry\n {\n Id = idx,\n Semantics = new Semantics\n {\n Id = idx,\n Tags = kvp.Key.Tags,\n Summary = kvp.Key.Summary\n },\n Text = kvp.Value,\n Vector = vectors[kvp.Key],\n SourceFile = \"question-bank\",\n CreatedAt = DateTime.UtcNow\n })\n .ToList();\n\n Console.WriteLine($\"知识库已构建：{entries.Count} 个向量。\");\n return entries;\n }\n\n public FileVectorStore BuildVectorStore(string projectPath, List<VectorEntry> entries)\n {\n var fvs = new FileVectorStore(projectPath);\n fvs.AddRange(entries);\n return fvs;\n }\n\n }\n}\n"}, "b08be962bb60251384f0260d4f85ebd0c02821b5": {"t": 1785760686.8974004, "n": 16588, "h": "f9778429b3d7a221", "c": "using OpenAI;\nusing OpenAI.Chat;\nusing ReciteHelper.Core.Entities;\nusing ReciteHelper.Core.Interfaces.Configuration;\nusing ReciteHelper.Core.Interfaces.Services;\nusing System.ClientModel;\nusing System.Collections.Concurrent;\nusing System.Text;\nusing System.Text.Json;\nusing System.Text.RegularExpressions;\nusing OpenAI.Embeddings;\nusing ReciteHelper.Core.ValueObjects;\nusing ReciteHelper.Core.DTOs;\n\nnamespace ReciteHelper.Infrastructure.Services\n{\n public class KnowledgeBaseService(IConfigService cfgService) : IKnowledgeBaseService\n {\n private readonly int _chunkSize = 1500;\n private readonly int _overlap = 50;\n\n private ChatClient _chatClient;\n private EmbeddingClient _embedClient;\n\n private const int MaxConcurrentRequests = 4;\n private const int BatchSize = 10;\n\n private IConfigService configService = cfgService;\n\n public async Task<FileVectorStore> Build(string projectPath, string text)\n {\n if (string.IsNullOrWhiteSpace(text))\n throw new InvalidOperationException(\"知识库构建失败：题库文本为空。\");\n\n var slices = Slice(text);\n if (slices.Count == 0)\n throw new InvalidOperationException(\"知识库构建失败：未切分出有效文本。\");\n\n var cfg = await configService.LoadAsync();\n if (string.IsNullOrWhiteSpace(cfg.DeepSeekKey))\n throw new InvalidOperationException(\"知识库构建失败：未配置 DeepSeek Key。\");\n\n if (string.IsNullOrWhiteSpace(cfg.QwenKey))\n throw new InvalidOperationException(\"知识库构建失败：未配置 Qwen Key。\");\n\n var src = new CancellationTokenSource();\n\n _chatClient = new OpenAIClient(new ApiKeyCredential(cfg.DeepSeekKey!), new OpenAIClientOptions\n {\n Endpoint = new Uri(\"https://api.deepseek.com\")\n }).GetChatClient(\"deepseek-v4-flash\");\n _embedClient = new OpenAIClient(new ApiKeyCredential(cfg.QwenKey!), new OpenAIClientOptions\n {\n Endpoint = new Uri(\"https://dashscope.aliyuncs.com/compatible-mode/v1\")\n }).GetEmbeddingClient(\"text-embedding-v4\");\n\n var cluster = await ClusterAsync(slices, src.Token);\n var embed = await EmbedAsync(cluster, src.Token);\n var elements = BuildVectorEntries(cluster, embed, src.Token);\n var fvs = BuildVectorStore(projectPath, elements);\n\n return fvs;\n }\n\n public async Task<IReadOnlyList<KnowledgeBaseMatch>> SearchAsync(\n FileVectorStore store,\n string query,\n int topK,\n CancellationToken cancellationToken = default)\n {\n if (store.Entries.Count == 0 || string.IsNullOrWhiteSpace(query) || topK <= 0)\n return [];\n\n var cfg = await configService.LoadAsync();\n if (string.IsNullOrWhiteSpace(cfg.QwenKey))\n throw new InvalidOperationException(\"知识库查询失败：未配置 Qwen Key。\");\n\n var embedClient = new OpenAIClient(\n new ApiKeyCredential(cfg.QwenKey),\n new OpenAIClientOptions\n {\n Endpoint = new Uri(\"https://dashscope.aliyuncs.com/compatible-mode/v1\")\n }).GetEmbeddingClient(\"text-embedding-v4\");\n\n var response = await embedClient.GenerateEmbeddingsAsync(\n [query.Trim()],\n options: null,\n cancellationToken);\n var queryVector = response.Value[0].ToFloats().ToArray();\n\n return store.Search(queryVector, topK)\n .Select(result => new KnowledgeBaseMatch(\n CreateMatchTitle(result.Entry),\n result.Entry.Text,\n result.Score))\n .ToList();\n }\n\n private static string CreateMatchTitle(VectorEntry entry)\n {\n if (!string.IsNullOrWhiteSpace(entry.Semantics.Summary))\n return entry.Semantics.Summary;\n\n var tags = string.Join(\"、\", entry.Semantics.Tags.Where(x => !string.IsNullOrWhiteSpace(x)));\n if (!string.IsNullOrWhiteSpace(tags))\n return tags;\n\n const int maxLength = 24;\n return entry.Text.Length <= maxLength\n ? entry.Text\n : $\"{entry.Text[..maxLength]}...\";\n }\n\n public List<string> Slice(string text)\n {\n var slices = SplitText(text);\n\n return slices.Select(x => x.Replace(\"\\n\", \"\").Trim()).ToList();\n }\n\n\n public List<string> SplitText(string text)\n {\n if (string.IsNullOrWhiteSpace(text))\n return new List<string>();\n\n text = CleanText(text);\n var paragraphs = SplitByParagraphs(text);\n\n var chunks = new List<string>();\n foreach (var para in paragraphs)\n {\n chunks.AddRange(SplitParagraph(para));\n }\n\n return chunks;\n }\n\n private string CleanText(string text)\n {\n text = Regex.Replace(text, @\"\\n\\s*\\n\", \"\\n\\n\");\n text = text.Trim();\n return text;\n }\n\n private List<string> SplitByParagraphs(string text)\n {\n var paragraphs = new List<string>();\n var currentPara = new StringBuilder();\n\n using var reader = new StringReader(text);\n string? line;\n\n while ((line = reader.ReadLine()) != null)\n {\n if (string.IsNullOrWhiteSpace(line))\n {\n if (currentPara.Length > 0)\n {\n paragraphs.Add(currentPara.ToString().Trim());\n currentPara.Clear();\n }\n }\n else\n {\n currentPara.AppendLine(line);\n }\n }\n\n if (currentPara.Length > 0)\n {\n paragraphs.Add(currentPara.ToString().Trim());\n }\n\n return paragraphs;\n }\n\n private List<string> SplitParagraph(string paragraph)\n {\n if (paragraph.Length <= _chunkSize)\n {\n return new List<string> { paragraph };\n }\n\n var chunks = new List<string>();\n var sentences = SplitIntoSentences(paragraph);\n\n var currentChunk = new StringBuilder();\n\n foreach (var sentence in sentences)\n {\n if (currentChunk.Length + sentence.Length + 1 > _chunkSize && currentChunk.Length > 0)\n {\n chunks.Add(currentChunk.ToString().Trim());\n\n string overlap = GetOverlap(currentChunk.ToString());\n currentChunk.Clear();\n currentChunk.Append(overlap);\n }\n\n currentChunk.Append(sentence);\n }\n\n if (currentChunk.Length > 0)\n {\n chunks.Add(currentChunk.ToString().Trim());\n }\n\n return chunks;\n }\n\n private List<string> SplitIntoSentences(string text)\n {\n var separators = new[] { \"。\", \"！\", \"？\", \"；\", \".\", \"!\", \"?\", \";\" };\n\n var sentences = new List<string>();\n var currentSentence = new StringBuilder();\n\n for (int i = 0; i < text.Length; i++)\n {\n currentSentence.Append(text[i]);\n\n foreach (var sep in separators)\n {\n if (text[i].ToString() == sep)\n {\n sentences.Add(currentSentence.ToString());\n currentSentence.Clear();\n break;\n }\n }\n }\n\n if (currentSentence.Length > 0)\n {\n sentences.Add(currentSentence.ToString());\n }\n\n return sentences;\n }\n\n private string GetOverlap(string chunk)\n {\n if (chunk.Length <= _overlap)\n return chunk;\n\n int start = chunk.Length - _overlap;\n\n int actualStart = chunk.Length;\n for (int i = start; i < chunk.Length; i++)\n {\n if (i > 0 && (chunk[i - 1] == '。' || chunk[i - 1] == '！' || chunk[i - 1] == '？'))\n {\n actualStart = i;\n break;\n }\n }\n\n if (actualStart >= chunk.Length)\n actualStart = start;\n\n return chunk.Substring(actualStart);\n }\n\n public async Task<Dictionary<Semantics, string>> ClusterAsync(List<string> chunks, CancellationToken cts = default)\n {\n var results = new ConcurrentDictionary<Semantics, string>();\n\n await Parallel.ForEachAsync(chunks,\n new ParallelOptions { MaxDegreeOfParallelism = MaxConcurrentRequests, CancellationToken = cts },\n async (chunk, _) =>\n {\n var segmentedSemantics = await ProcessChunkAsync(chunk);\n foreach (var item in segmentedSemantics)\n {\n results.TryAdd(item.Item1, item.Item2);\n }\n });\n\n return results.OrderBy(x => x.Key.Id).ToDictionary(x => x.Key, x => x.Value);\n }\n\n /// <summary>\n /// Process single chunk: segment by atomic knowledge points and generate semantics\n /// </summary>\n private async Task<List<(Semantics, string)>> ProcessChunkAsync(string chunk)\n {\n var prompt = BuildPrompt(chunk);\n\n var messages = new List<ChatMessage>\n {\n new SystemChatMessage(\"你是一个专业的文本分析专家。请完成以下任务：\\n\" +\n \"1. 将输入的文本块按照知识点的原子性进行分割\\n\" +\n \"2. 确保每个分割后的段落只表达一个独立、完整的知识点\\n\" +\n \"3. 为每个知识点段落生成2-3个关键词标签\\n\" +\n \"4. 生成不超过20字的单句摘要\\n\" +\n \"5. 对于被截断的知识点，如果有明确的语义信息进行补全，否则直接丢弃\" +\n \"6. 返回JSON数组，每个元素包含text、tags、summary字段，无其他内容，不要添加markdown代码块结构\"),\n new UserChatMessage(prompt)\n };\n\n var response = await _chatClient.CompleteChatAsync(messages, new ChatCompletionOptions\n {\n Temperature = 0.3f,\n MaxOutputTokenCount = 66666\n });\n\n var json = ExtractJsonContent(response.Value.Content[0].Text);\n return ParseResponse(json);\n }\n\n private static string BuildPrompt(string chunk)\n {\n var sb = new StringBuilder();\n sb.AppendLine(\"请处理以下文本块：\");\n sb.AppendLine();\n sb.AppendLine(chunk);\n sb.AppendLine();\n sb.AppendLine(\"返回JSON数组格式，示例：\");\n sb.AppendLine(\"[\");\n sb.AppendLine(\" {\\\"text\\\": \\\"分割后的知识点1\\\", \\\"tags\\\": [\\\"标签1\\\", \\\"标签2\\\"], \\\"summary\\\": \\\"摘要\\\"},\");\n sb.AppendLine(\" {\\\"text\\\": \\\"分割后的知识点2\\\", \\\"tags\\\": [\\\"标签1\\\", \\\"标签2\\\"], \\\"summary\\\": \\\"摘要\\\"}\");\n sb.AppendLine(\"]\");\n\n return sb.ToString();\n }\n\n /// <summary>\n /// Parse response and return semantics-text pairs with auto-incremented IDs\n /// </summary>\n private static List<(Semantics, string)> ParseResponse(string json)\n {\n var result = new List<(Semantics, string)>();\n var idCounter = 0;\n\n using var doc = JsonDocument.Parse(json);\n var array = doc.RootElement.EnumerateArray().ToList();\n\n foreach (var item in array)\n {\n var text = item.TryGetProperty(\"text\", out var textProp)\n ? textProp.GetString() ?? \"\"\n : \"\";\n\n if (string.IsNullOrWhiteSpace(text))\n continue;\n\n var semantics = new Semantics\n {\n Id = idCounter++,\n Tags = item.TryGetProperty(\"tags\", out var tags)\n ? tags.EnumerateArray()\n .Select(t => t.GetString() ?? \"\")\n .Where(t => !string.IsNullOrWhiteSpace(t))\n .ToList()\n : [],\n Summary = item.TryGetProperty(\"summary\", out var summary)\n ? (summary.GetString() ?? \"\").Trim()\n : \"\"\n };\n\n result.Add((semantics, text.Trim()));\n }\n\n\n return result;\n }\n\n private static string ExtractJsonContent(string content)\n {\n if (string.IsNullOrWhiteSpace(content))\n return \"[]\";\n\n var trimmed = content.Trim();\n var fenceStart = trimmed.IndexOf(\"```\", StringComparison.Ordinal);\n if (fenceStart >= 0)\n {\n var jsonStart = trimmed.IndexOf('\\n', fenceStart);\n if (jsonStart >= 0)\n {\n var fenceEnd = trimmed.LastIndexOf(\"```\", StringComparison.Ordinal);\n trimmed = fenceEnd > jsonStart\n ? trimmed[(jsonStart + 1)..fenceEnd].Trim()\n : trimmed[(jsonStart + 1)..].Trim();\n }\n }\n\n var arrayStart = trimmed.IndexOf('[');\n var arrayEnd = trimmed.LastIndexOf(']');\n return arrayStart >= 0 && arrayEnd > arrayStart\n ? trimmed[arrayStart..(arrayEnd + 1)]\n : trimmed;\n }\n\n public async Task<Dictionary<Semantics, float[]>> EmbedAsync(\n Dictionary<Semantics, string> semanticChunks,\n CancellationToken cts)\n {\n if (semanticChunks.Count == 0) return [];\n\n var semanticsList = semanticChunks.Keys.ToList();\n var textsList = semanticChunks.Values.ToList();\n\n // Results array maintains index correspondence with input order\n var results = new float[textsList.Count][];\n\n var batches = textsList\n .Select((text, index) => new { text, index })\n .Chunk(BatchSize)\n .ToList();\n\n using var semaphore = new SemaphoreSlim(MaxConcurrentRequests);\n\n var tasks = batches.Select(async batch =>\n {\n await semaphore.WaitAsync(cts);\n try\n {\n var batchTexts = batch.Select(b => b.text).ToList();\n var response = await _embedClient.GenerateEmbeddingsAsync(batchTexts, options: null, cts);\n\n for (int i = 0; i < batch.Length; i++)\n results[batch[i].index] = response.Value[i].ToFloats().ToArray();\n }\n finally\n {\n semaphore.Release();\n }\n });\n\n await Task.WhenAll(tasks);\n\n // Reconstruct the result dictionary maintaining Semantics-Embedding pairing\n var resultDictionary = new Dictionary<Semantics, float[]>();\n for (int i = 0; i < semanticsList.Count; i++)\n {\n resultDictionary[semanticsList[i]] = results[i];\n }\n\n return resultDictionary;\n }\n\n public List<VectorEntry> BuildVectorEntries(\n Dictionary<Semantics, string> semanticChunks,\n Dictionary<Semantics, float[]> vectors,\n CancellationToken cts)\n {\n var entries = semanticChunks\n .Select((kvp, idx) => new VectorEntry\n {\n Id = idx,\n Semantics = new Semantics\n {\n Id = idx,\n Tags = kvp.Key.Tags,\n Summary = kvp.Key.Summary\n },\n Text = kvp.Value,\n Vector = vectors[kvp.Key],\n SourceFile = \"question-bank\",\n CreatedAt = DateTime.UtcNow\n })\n .ToList();\n\n Console.WriteLine($\"知识库已构建：{entries.Count} 个向量。\");\n return entries;\n }\n\n public FileVectorStore BuildVectorStore(string projectPath, List<VectorEntry> entries)\n {\n var fvs = new FileVectorStore(projectPath);\n fvs.AddRange(entries);\n return fvs;\n }\n\n }\n}\n"}, "59405879fe1154c28324fbb7be767592fa3db0e2": {"t": 1785760688.296928, "n": 14642, "h": "6be44b5aa969c08f", "c": "using OpenAI;\nusing OpenAI.Chat;\nusing ReciteHelper.Core.Entities;\nusing ReciteHelper.Core.Interfaces.Configuration;\nusing ReciteHelper.Core.Interfaces.Services;\nusing System.ClientModel;\nusing System.Collections.Concurrent;\nusing System.Text;\nusing System.Text.Json;\nusing System.Text.RegularExpressions;\nusing OpenAI.Embeddings;\nusing ReciteHelper.Core.ValueObjects;\n\nnamespace ReciteHelper.Infrastructure.Services\n{\n public class KnowledgeBaseService(IConfigService cfgService) : IKnowledgeBaseService\n {\n private readonly int _chunkSize = 1500;\n private readonly int _overlap = 50;\n\n private ChatClient _chatClient;\n private EmbeddingClient _embedClient;\n\n private const int MaxConcurrentRequests = 4;\n private const int BatchSize = 10;\n\n private IConfigService configService = cfgService;\n\n public async Task<FileVectorStore> Build(string projectPath, string text)\n {\n if (string.IsNullOrWhiteSpace(text))\n throw new InvalidOperationException(\"知识库构建失败：题库文本为空。\");\n\n var slices = Slice(text);\n if (slices.Count == 0)\n throw new InvalidOperationException(\"知识库构建失败：未切分出有效文本。\");\n\n var cfg = await configService.LoadAsync();\n if (string.IsNullOrWhiteSpace(cfg.DeepSeekKey))\n throw new InvalidOperationException(\"知识库构建失败：未配置 DeepSeek Key。\");\n\n if (string.IsNullOrWhiteSpace(cfg.QwenKey))\n throw new InvalidOperationException(\"知识库构建失败：未配置 Qwen Key。\");\n\n var src = new CancellationTokenSource();\n\n _chatClient = new OpenAIClient(new ApiKeyCredential(cfg.DeepSeekKey!), new OpenAIClientOptions\n {\n Endpoint = new Uri(\"https://api.deepseek.com\")\n }).GetChatClient(\"deepseek-v4-flash\");\n _embedClient = new OpenAIClient(new ApiKeyCredential(cfg.QwenKey!), new OpenAIClientOptions\n {\n Endpoint = new Uri(\"https://dashscope.aliyuncs.com/compatible-mode/v1\")\n }).GetEmbeddingClient(\"text-embedding-v4\");\n\n var cluster = await ClusterAsync(slices, src.Token);\n var embed = await EmbedAsync(cluster, src.Token);\n var elements = BuildVectorEntries(cluster, embed, src.Token);\n var fvs = BuildVectorStore(projectPath, elements);\n\n return fvs;\n }\n\n public List<string> Slice(string text)\n {\n var slices = SplitText(text);\n\n return slices.Select(x => x.Replace(\"\\n\", \"\").Trim()).ToList();\n }\n\n\n public List<string> SplitText(string text)\n {\n if (string.IsNullOrWhiteSpace(text))\n return new List<string>();\n\n text = CleanText(text);\n var paragraphs = SplitByParagraphs(text);\n\n var chunks = new List<string>();\n foreach (var para in paragraphs)\n {\n chunks.AddRange(SplitParagraph(para));\n }\n\n return chunks;\n }\n\n private string CleanText(string text)\n {\n text = Regex.Replace(text, @\"\\n\\s*\\n\", \"\\n\\n\");\n text = text.Trim();\n return text;\n }\n\n private List<string> SplitByParagraphs(string text)\n {\n var paragraphs = new List<string>();\n var currentPara = new StringBuilder();\n\n using var reader = new StringReader(text);\n string? line;\n\n while ((line = reader.ReadLine()) != null)\n {\n if (string.IsNullOrWhiteSpace(line))\n {\n if (currentPara.Length > 0)\n {\n paragraphs.Add(currentPara.ToString().Trim());\n currentPara.Clear();\n }\n }\n else\n {\n currentPara.AppendLine(line);\n }\n }\n\n if (currentPara.Length > 0)\n {\n paragraphs.Add(currentPara.ToString().Trim());\n }\n\n return paragraphs;\n }\n\n private List<string> SplitParagraph(string paragraph)\n {\n if (paragraph.Length <= _chunkSize)\n {\n return new List<string> { paragraph };\n }\n\n var chunks = new List<string>();\n var sentences = SplitIntoSentences(paragraph);\n\n var currentChunk = new StringBuilder();\n\n foreach (var sentence in sentences)\n {\n if (currentChunk.Length + sentence.Length + 1 > _chunkSize && currentChunk.Length > 0)\n {\n chunks.Add(currentChunk.ToString().Trim());\n\n string overlap = GetOverlap(currentChunk.ToString());\n currentChunk.Clear();\n currentChunk.Append(overlap);\n }\n\n currentChunk.Append(sentence);\n }\n\n if (currentChunk.Length > 0)\n {\n chunks.Add(currentChunk.ToString().Trim());\n }\n\n return chunks;\n }\n\n private List<string> SplitIntoSentences(string text)\n {\n var separators = new[] { \"。\", \"！\", \"？\", \"；\", \".\", \"!\", \"?\", \";\" };\n\n var sentences = new List<string>();\n var currentSentence = new StringBuilder();\n\n for (int i = 0; i < text.Length; i++)\n {\n currentSentence.Append(text[i]);\n\n foreach (var sep in separators)\n {\n if (text[i].ToString() == sep)\n {\n sentences.Add(currentSentence.ToString());\n currentSentence.Clear();\n break;\n }\n }\n }\n\n if (currentSentence.Length > 0)\n {\n sentences.Add(currentSentence.ToString());\n }\n\n return sentences;\n }\n\n private string GetOverlap(string chunk)\n {\n if (chunk.Length <= _overlap)\n return chunk;\n\n int start = chunk.Length - _overlap;\n\n int actualStart = chunk.Length;\n for (int i = start; i < chunk.Length; i++)\n {\n if (i > 0 && (chunk[i - 1] == '。' || chunk[i - 1] == '！' || chunk[i - 1] == '？'))\n {\n actualStart = i;\n break;\n }\n }\n\n if (actualStart >= chunk.Length)\n actualStart = start;\n\n return chunk.Substring(actualStart);\n }\n\n public async Task<Dictionary<Semantics, string>> ClusterAsync(List<string> chunks, CancellationToken cts = default)\n {\n var results = new ConcurrentDictionary<Semantics, string>();\n\n await Parallel.ForEachAsync(chunks,\n new ParallelOptions { MaxDegreeOfParallelism = MaxConcurrentRequests, CancellationToken = cts },\n async (chunk, _) =>\n {\n var segmentedSemantics = await ProcessChunkAsync(chunk);\n foreach (var item in segmentedSemantics)\n {\n results.TryAdd(item.Item1, item.Item2);\n }\n });\n\n return results.OrderBy(x => x.Key.Id).ToDictionary(x => x.Key, x => x.Value);\n }\n\n /// <summary>\n /// Process single chunk: segment by atomic knowledge points and generate semantics\n /// </summary>\n private async Task<List<(Semantics, string)>> ProcessChunkAsync(string chunk)\n {\n var prompt = BuildPrompt(chunk);\n\n var messages = new List<ChatMessage>\n {\n new SystemChatMessage(\"你是一个专业的文本分析专家。请完成以下任务：\\n\" +\n \"1. 将输入的文本块按照知识点的原子性进行分割\\n\" +\n \"2. 确保每个分割后的段落只表达一个独立、完整的知识点\\n\" +\n \"3. 为每个知识点段落生成2-3个关键词标签\\n\" +\n \"4. 生成不超过20字的单句摘要\\n\" +\n \"5. 对于被截断的知识点，如果有明确的语义信息进行补全，否则直接丢弃\" +\n \"6. 返回JSON数组，每个元素包含text、tags、summary字段，无其他内容，不要添加markdown代码块结构\"),\n new UserChatMessage(prompt)\n };\n\n var response = await _chatClient.CompleteChatAsync(messages, new ChatCompletionOptions\n {\n Temperature = 0.3f,\n MaxOutputTokenCount = 66666\n });\n\n var json = ExtractJsonContent(response.Value.Content[0].Text);\n return ParseResponse(json);\n }\n\n private static string BuildPrompt(string chunk)\n {\n var sb = new StringBuilder();\n sb.AppendLine(\"请处理以下文本块：\");\n sb.AppendLine();\n sb.AppendLine(chunk);\n sb.AppendLine();\n sb.AppendLine(\"返回JSON数组格式，示例：\");\n sb.AppendLine(\"[\");\n sb.AppendLine(\" {\\\"text\\\": \\\"分割后的知识点1\\\", \\\"tags\\\": [\\\"标签1\\\", \\\"标签2\\\"], \\\"summary\\\": \\\"摘要\\\"},\");\n sb.AppendLine(\" {\\\"text\\\": \\\"分割后的知识点2\\\", \\\"tags\\\": [\\\"标签1\\\", \\\"标签2\\\"], \\\"summary\\\": \\\"摘要\\\"}\");\n sb.AppendLine(\"]\");\n\n return sb.ToString();\n }\n\n /// <summary>\n /// Parse response and return semantics-text pairs with auto-incremented IDs\n /// </summary>\n private static List<(Semantics, string)> ParseResponse(string json)\n {\n var result = new List<(Semantics, string)>();\n var idCounter = 0;\n\n using var doc = JsonDocument.Parse(json);\n var array = doc.RootElement.EnumerateArray().ToList();\n\n foreach (var item in array)\n {\n var text = item.TryGetProperty(\"text\", out var textProp)\n ? textProp.GetString() ?? \"\"\n : \"\";\n\n if (string.IsNullOrWhiteSpace(text))\n continue;\n\n var semantics = new Semantics\n {\n Id = idCounter++,\n Tags = item.TryGetProperty(\"tags\", out var tags)\n ? tags.EnumerateArray()\n .Select(t => t.GetString() ?? \"\")\n .Where(t => !string.IsNullOrWhiteSpace(t))\n .ToList()\n : [],\n Summary = item.TryGetProperty(\"summary\", out var summary)\n ? (summary.GetString() ?? \"\").Trim()\n : \"\"\n };\n\n result.Add((semantics, text.Trim()));\n }\n\n\n return result;\n }\n\n private static string ExtractJsonContent(string content)\n {\n if (string.IsNullOrWhiteSpace(content))\n return \"[]\";\n\n var trimmed = content.Trim();\n var fenceStart = trimmed.IndexOf(\"```\", StringComparison.Ordinal);\n if (fenceStart >= 0)\n {\n var jsonStart = trimmed.IndexOf('\\n', fenceStart);\n if (jsonStart >= 0)\n {\n var fenceEnd = trimmed.LastIndexOf(\"```\", StringComparison.Ordinal);\n trimmed = fenceEnd > jsonStart\n ? trimmed[(jsonStart + 1)..fenceEnd].Trim()\n : trimmed[(jsonStart + 1)..].Trim();\n }\n }\n\n var arrayStart = trimmed.IndexOf('[');\n var arrayEnd = trimmed.LastIndexOf(']');\n return arrayStart >= 0 && arrayEnd > arrayStart\n ? trimmed[arrayStart..(arrayEnd + 1)]\n : trimmed;\n }\n\n public async Task<Dictionary<Semantics, float[]>> EmbedAsync(\n Dictionary<Semantics, string> semanticChunks,\n CancellationToken cts)\n {\n if (semanticChunks.Count == 0) return [];\n\n var semanticsList = semanticChunks.Keys.ToList();\n var textsList = semanticChunks.Values.ToList();\n\n // Results array maintains index correspondence with input order\n var results = new float[textsList.Count][];\n\n var batches = textsList\n .Select((text, index) => new { text, index })\n .Chunk(BatchSize)\n .ToList();\n\n using var semaphore = new SemaphoreSlim(MaxConcurrentRequests);\n\n var tasks = batches.Select(async batch =>\n {\n await semaphore.WaitAsync(cts);\n try\n {\n var batchTexts = batch.Select(b => b.text).ToList();\n var response = await _embedClient.GenerateEmbeddingsAsync(batchTexts, options: null, cts);\n\n for (int i = 0; i < batch.Length; i++)\n results[batch[i].index] = response.Value[i].ToFloats().ToArray();\n }\n finally\n {\n semaphore.Release();\n }\n });\n\n await Task.WhenAll(tasks);\n\n // Reconstruct the result dictionary maintaining Semantics-Embedding pairing\n var resultDictionary = new Dictionary<Semantics, float[]>();\n for (int i = 0; i < semanticsList.Count; i++)\n {\n resultDictionary[semanticsList[i]] = results[i];\n }\n\n return resultDictionary;\n }\n\n public List<VectorEntry> BuildVectorEntries(\n Dictionary<Semantics, string> semanticChunks,\n Dictionary<Semantics, float[]> vectors,\n CancellationToken cts)\n {\n var entries = semanticChunks\n .Select((kvp, idx) => new VectorEntry\n {\n Id = idx,\n Semantics = new Semantics\n {\n Id = idx,\n Tags = kvp.Key.Tags,\n Summary = kvp.Key.Summary\n },\n Text = kvp.Value,\n Vector = vectors[kvp.Key],\n SourceFile = \"question-bank\",\n CreatedAt = DateTime.UtcNow\n })\n .ToList();\n\n Console.WriteLine($\"知识库已构建：{entries.Count} 个向量。\");\n return entries;\n }\n\n public FileVectorStore BuildVectorStore(string projectPath, List<VectorEntry> entries)\n {\n var fvs = new FileVectorStore(projectPath);\n fvs.AddRange(entries);\n return fvs;\n }\n\n }\n}\n"}, "59e9fda6e2ab2d6af88aa7b19eb6980dae251c8b": {"t": 1785760689.7222912, "n": 12967, "h": "f7582fd4ee2e9491", "c": "using OpenAI;\nusing OpenAI.Chat;\nusing ReciteHelper.Core.Entities;\nusing ReciteHelper.Core.Interfaces.Configuration;\nusing ReciteHelper.Core.Interfaces.Services;\nusing System.ClientModel;\nusing System.Collections.Concurrent;\nusing System.Text;\nusing System.Text.Json;\nusing System.Text.RegularExpressions;\nusing OpenAI.Embeddings;\nusing ReciteHelper.Core.ValueObjects;\n\nnamespace ReciteHelper.Infrastructure.Services\n{\n internal class KnowledgeBaseService(IConfigService cfgService) : IKnowledgeBaseService\n {\n private readonly int _chunkSize = 1500;\n private readonly int _overlap = 50;\n\n private ChatClient _chatClient;\n private EmbeddingClient _embedClient;\n\n private const int MaxConcurrentRequests = 16;\n private const int BatchSize = 10;\n\n private IConfigService configService = cfgService;\n\n public async Task<FileVectorStore> Build(string projectPath, string text)\n {\n // Wow, such a magic pill!\n\n var slices = Slice(text);\n\n var cfg = await configService.LoadAsync();\n var src = new CancellationTokenSource();\n\n _chatClient = new OpenAIClient(new ApiKeyCredential(cfg.DeepSeekKey!), new OpenAIClientOptions\n {\n Endpoint = new Uri(\"https://api.deepseek.com\")\n }).GetChatClient(\"deepseek-v4-flash\");\n _embedClient = new OpenAIClient(new ApiKeyCredential(cfg.QwenKey!), new OpenAIClientOptions\n {\n Endpoint = new Uri(\"https://dashscope.aliyuncs.com/compatible-mode/v1\")\n }).GetEmbeddingClient(\"text-embedding-v4\");\n\n var cluster = await ClusterAsync(slices, src.Token);\n var embed = await EmbedAsync(cluster, src.Token);\n var elements = BuildVectorEntries(cluster, embed, src.Token);\n var fvs = BuildVectorStore(projectPath, elements);\n\n return fvs;\n }\n\n public List<string> Slice(string text)\n {\n var slices = SplitText(text);\n\n return slices.Select(x => x.Replace(\"\\n\", \"\").Trim()).ToList();\n }\n\n\n public List<string> SplitText(string text)\n {\n if (string.IsNullOrWhiteSpace(text))\n return new List<string>();\n\n text = CleanText(text);\n var paragraphs = SplitByParagraphs(text);\n\n var chunks = new List<string>();\n foreach (var para in paragraphs)\n {\n chunks.AddRange(SplitParagraph(para));\n }\n\n return chunks;\n }\n\n private string CleanText(string text)\n {\n text = Regex.Replace(text, @\"\\n\\s*\\n\", \"\\n\\n\");\n text = text.Trim();\n return text;\n }\n\n private List<string> SplitByParagraphs(string text)\n {\n var paragraphs = new List<string>();\n var currentPara = new StringBuilder();\n\n using var reader = new StringReader(text);\n string? line;\n\n while ((line = reader.ReadLine()) != null)\n {\n if (string.IsNullOrWhiteSpace(line))\n {\n if (currentPara.Length > 0)\n {\n paragraphs.Add(currentPara.ToString().Trim());\n currentPara.Clear();\n }\n }\n else\n {\n currentPara.AppendLine(line);\n }\n }\n\n if (currentPara.Length > 0)\n {\n paragraphs.Add(currentPara.ToString().Trim());\n }\n\n return paragraphs;\n }\n\n private List<string> SplitParagraph(string paragraph)\n {\n if (paragraph.Length <= _chunkSize)\n {\n return new List<string> { paragraph };\n }\n\n var chunks = new List<string>();\n var sentences = SplitIntoSentences(paragraph);\n\n var currentChunk = new StringBuilder();\n\n foreach (var sentence in sentences)\n {\n if (currentChunk.Length + sentence.Length + 1 > _chunkSize && currentChunk.Length > 0)\n {\n chunks.Add(currentChunk.ToString().Trim());\n\n string overlap = GetOverlap(currentChunk.ToString());\n currentChunk.Clear();\n currentChunk.Append(overlap);\n }\n\n currentChunk.Append(sentence);\n }\n\n if (currentChunk.Length > 0)\n {\n chunks.Add(currentChunk.ToString().Trim());\n }\n\n return chunks;\n }\n\n private List<string> SplitIntoSentences(string text)\n {\n var separators = new[] { \"。\", \"！\", \"？\", \"；\", \".\", \"!\", \"?\", \";\" };\n\n var sentences = new List<string>();\n var currentSentence = new StringBuilder();\n\n for (int i = 0; i < text.Length; i++)\n {\n currentSentence.Append(text[i]);\n\n foreach (var sep in separators)\n {\n if (text[i].ToString() == sep)\n {\n sentences.Add(currentSentence.ToString());\n currentSentence.Clear();\n break;\n }\n }\n }\n\n if (currentSentence.Length > 0)\n {\n sentences.Add(currentSentence.ToString());\n }\n\n return sentences;\n }\n\n private string GetOverlap(string chunk)\n {\n if (chunk.Length <= _overlap)\n return chunk;\n\n int start = chunk.Length - _overlap;\n\n int actualStart = chunk.Length;\n for (int i = start; i < chunk.Length; i++)\n {\n if (i > 0 && (chunk[i - 1] == '。' || chunk[i - 1] == '！' || chunk[i - 1] == '？'))\n {\n actualStart = i;\n break;\n }\n }\n\n if (actualStart >= chunk.Length)\n actualStart = start;\n\n return chunk.Substring(actualStart);\n }\n\n public async Task<Dictionary<Semantics, string>> ClusterAsync(List<string> chunks, CancellationToken cts = default)\n {\n var results = new ConcurrentDictionary<Semantics, string>();\n\n await Parallel.ForEachAsync(chunks,\n new ParallelOptions { MaxDegreeOfParallelism = MaxConcurrentRequests, CancellationToken = cts },\n async (chunk, _) =>\n {\n var segmentedSemantics = await ProcessChunkAsync(chunk);\n foreach (var item in segmentedSemantics)\n {\n results.TryAdd(item.Item1, item.Item2);\n }\n });\n\n return results.OrderBy(x => x.Key.Id).ToDictionary(x => x.Key, x => x.Value);\n }\n\n /// <summary>\n /// Process single chunk: segment by atomic knowledge points and generate semantics\n /// </summary>\n private async Task<List<(Semantics, string)>> ProcessChunkAsync(string chunk)\n {\n var prompt = BuildPrompt(chunk);\n\n var messages = new List<ChatMessage>\n {\n new SystemChatMessage(\"你是一个专业的文本分析专家。请完成以下任务：\\n\" +\n \"1. 将输入的文本块按照知识点的原子性进行分割\\n\" +\n \"2. 确保每个分割后的段落只表达一个独立、完整的知识点\\n\" +\n \"3. 为每个知识点段落生成2-3个关键词标签\\n\" +\n \"4. 生成不超过20字的单句摘要\\n\" +\n \"5. 对于被截断的知识点，如果有明确的语义信息进行补全，否则直接丢弃\" +\n \"6. 返回JSON数组，每个元素包含text、tags、summary字段，无其他内容，不要添加markdown代码块结构\"),\n new UserChatMessage(prompt)\n };\n\n var response = await _chatClient.CompleteChatAsync(messages, new ChatCompletionOptions\n {\n Temperature = 0.3f,\n MaxOutputTokenCount = 66666\n });\n\n var json = response.Value.Content[0].Text;\n return ParseResponse(json);\n }\n\n private static string BuildPrompt(string chunk)\n {\n var sb = new StringBuilder();\n sb.AppendLine(\"请处理以下文本块：\");\n sb.AppendLine();\n sb.AppendLine(chunk);\n sb.AppendLine();\n sb.AppendLine(\"返回JSON数组格式，示例：\");\n sb.AppendLine(\"[\");\n sb.AppendLine(\" {\\\"text\\\": \\\"分割后的知识点1\\\", \\\"tags\\\": [\\\"标签1\\\", \\\"标签2\\\"], \\\"summary\\\": \\\"摘要\\\"},\");\n sb.AppendLine(\" {\\\"text\\\": \\\"分割后的知识点2\\\", \\\"tags\\\": [\\\"标签1\\\", \\\"标签2\\\"], \\\"summary\\\": \\\"摘要\\\"}\");\n sb.AppendLine(\"]\");\n\n return sb.ToString();\n }\n\n /// <summary>\n /// Parse response and return semantics-text pairs with auto-incremented IDs\n /// </summary>\n private static List<(Semantics, string)> ParseResponse(string json)\n {\n var result = new List<(Semantics, string)>();\n var idCounter = 0;\n\n using var doc = JsonDocument.Parse(json);\n var array = doc.RootElement.EnumerateArray().ToList();\n\n foreach (var item in array)\n {\n var text = item.TryGetProperty(\"text\", out var textProp)\n ? textProp.GetString() ?? \"\"\n : \"\";\n\n if (string.IsNullOrWhiteSpace(text))\n continue;\n\n var semantics = new Semantics\n {\n Id = idCounter++,\n Tags = item.TryGetProperty(\"tags\", out var tags)\n ? tags.EnumerateArray()\n .Select(t => t.GetString() ?? \"\")\n .Where(t => !string.IsNullOrWhiteSpace(t))\n .ToList()\n : [],\n Summary = item.TryGetProperty(\"summary\", out var summary)\n ? (summary.GetString() ?? \"\").Trim()\n : \"\"\n };\n\n result.Add((semantics, text.Trim()));\n }\n\n\n return result;\n }\n\n public async Task<Dictionary<Semantics, float[]>> EmbedAsync(\n Dictionary<Semantics, string> semanticChunks,\n CancellationToken cts)\n {\n if (semanticChunks.Count == 0) return [];\n\n var semanticsList = semanticChunks.Keys.ToList();\n var textsList = semanticChunks.Values.ToList();\n\n // Results array maintains index correspondence with input order\n var results = new float[textsList.Count][];\n\n var batches = textsList\n .Select((text, index) => new { text, index })\n .Chunk(BatchSize)\n .ToList();\n\n using var semaphore = new SemaphoreSlim(MaxConcurrentRequests);\n\n var tasks = batches.Select(async batch =>\n {\n await semaphore.WaitAsync(cts);\n try\n {\n var batchTexts = batch.Select(b => b.text).ToList();\n var response = await _embedClient.GenerateEmbeddingsAsync(batchTexts, options: null, cts);\n\n for (int i = 0; i < batch.Length; i++)\n results[batch[i].index] = response.Value[i].ToFloats().ToArray();\n }\n finally\n {\n semaphore.Release();\n }\n });\n\n await Task.WhenAll(tasks);\n\n // Reconstruct the result dictionary maintaining Semantics-Embedding pairing\n var resultDictionary = new Dictionary<Semantics, float[]>();\n for (int i = 0; i < semanticsList.Count; i++)\n {\n resultDictionary[semanticsList[i]] = results[i];\n }\n\n return resultDictionary;\n }\n\n public List<VectorEntry> BuildVectorEntries(\n Dictionary<Semantics, string> semanticChunks,\n Dictionary<Semantics, float[]> vectors,\n CancellationToken cts)\n {\n var entries = semanticChunks\n .Select((kvp, idx) => new VectorEntry\n {\n Id = idx,\n Semantics = kvp.Key,\n Text = kvp.Value,\n Vector = vectors[kvp.Key],\n SourceFile = \"srcFile\",\n CreatedAt = DateTime.UtcNow\n })\n .ToList();\n\n Console.WriteLine($\"知识库已构建：{entries.Count} 个向量。\");\n return entries;\n }\n\n public FileVectorStore BuildVectorStore(string projectPath, List<VectorEntry> entries)\n {\n var fvs = new FileVectorStore(projectPath);\n fvs.AddRange(entries);\n return fvs;\n }\n\n }\n}\n"}}}} |