1 line
223 KiB
JSON
1 line
223 KiB
JSON
{"repo": "6roy/easyedu", "files": {"data/ds_data/data_processing/data_creation.ipynb": {"66a3ea61aafb398dd265cd5b7db6d15d06867452": {"t": 1785745507.809913, "n": 31621, "h": "e2ed3042fc064071", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 章节\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def extract_review_tips(text):\\n\",\n \" # 查找【复习提示】的起始位置\\n\",\n \" start_marker = \\\"【复习提示】\\\"\\n\",\n \" start_idx = text.find(start_marker)\\n\",\n \" \\n\",\n \" if start_idx == -1:\\n\",\n \" return None\\n\",\n \" \\n\",\n \" # 从标记后开始的文本\\n\",\n \" start_idx += len(start_marker)\\n\",\n \" \\n\",\n \" # 使用正则表达式找到下一个章节标记（如 1.1, 2.1, 3.1 等）\\n\",\n \" import re\\n\",\n \" text_after_tips = text[start_idx:]\\n\",\n \" chapter_match = re.search(r'\\\\n\\\\d+\\\\.\\\\d+', text_after_tips)\\n\",\n \" \\n\",\n \" if chapter_match:\\n\",\n \" end_idx = chapter_match.start()\\n\",\n \" review_tip = text_after_tips[:end_idx].strip()\\n\",\n \" else:\\n\",\n \" review_tip = text_after_tips.strip()\\n\",\n \" \\n\",\n \" return review_tip\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"# 使用示例\\n\",\n \"if __name__ == \\\"__main__\\\":\\n\",\n \" directory = \\\"/root/autodl-tmp/EasyEdu/data/DS2026_extracted/text\\\"\\n\",\n \" chapter_page = ['13', '24', '74', '121', '136', '210', '281', '349']\\n\",\n \" titles = ['绪论', '线性表', '栈和队列和数组', '串', '树和二叉树', '图', '查找', '排序']\\n\",\n \" chapters = []\\n\",\n \" for i, page in enumerate(chapter_page, 1):\\n\",\n \" file_path = os.path.join(directory, f'page_{page}.txt')\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" text = f.read()\\n\",\n \" review_tip = extract_review_tips(text)\\n\",\n \" if review_tip:\\n\",\n \" id = \\\"c0\\\" + str(i)\\n\",\n \" chapter = {\\n\",\n \" \\\"id\\\": id, # 章节唯一标识符\\n\",\n \" \\\"title\\\": titles[i-1], # 章节标题\\n\",\n \" \\\"parent_id\\\": \\\"\\\", # 父章节ID（可选）\\n\",\n \" \\\"order\\\": \\\"integer\\\", # 排序顺序\\n\",\n \" \\\"description\\\": review_tip, # 章节描述\\n\",\n \" \\\"knowledge_points\\\": [ # 包含的知识点ID列表\\n\",\n \" \\\"knowledge_point_id\\\"\\n\",\n \" ],\\n\",\n \" \\\"sub_chapters\\\": [ # 子章节ID列表\\n\",\n \" \\\"chapter_id\\\"\\n\",\n \" ]\\n\",\n \"}\\n\",\n \" chapters.append(chapter)\\n\",\n \" with open('/root/autodl-tmp/EasyEdu/data/questions/chapters.json', 'w', encoding='utf-8') as f:\\n\",\n \" json.dump(chapters, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 知识点\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def extract_knowledge_point(text, chapter_number):\\n\",\n \" import re\\n\",\n \" \\n\",\n \" # 使用正则表达式匹配章节标题和内容\\n\",\n \" section_pattern = rf'{chapter_number}\\\\.\\\\d+\\\\.\\\\d+'\\n\",\n \" \\n\",\n \" sections = []\\n\",\n \" matches = list(re.finditer(section_pattern, text))\\n\",\n \" \\n\",\n \" for i in range(len(matches)):\\n\",\n \" current_match = matches[i]\\n\",\n \" current_section = current_match.group()\\n\",\n \" start_pos = current_match.start()\\n\",\n \" \\n\",\n \" if i < len(matches) - 1:\\n\",\n \" end_pos = matches[i + 1].start()\\n\",\n \" content = text[start_pos:end_pos].strip()\\n\",\n \" else:\\n\",\n \" content = text[start_pos:].strip()\\n\",\n \" \\n\",\n \" if \\\"本节试题精选\\\" not in content and \\\"答案与解析\\\" not in content:\\n\",\n \" lines = content.split('\\\\n')\\n\",\n \" full_title = lines[0].strip()\\n\",\n \" title = full_title.split(current_section)[-1].strip()\\n\",\n \" description = '\\\\n'.join(lines[1:]).strip()\\n\",\n \" \\n\",\n \" sections.append({\\n\",\n \" 'section': current_section,\\n\",\n \" 'title': title,\\n\",\n \" 'description': description\\n\",\n \" })\\n\",\n \" \\n\",\n \" return sections\\n\",\n \"\\n\",\n \"def process_file(file_path, chapter_number):\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" text = f.read()\\n\",\n \" \\n\",\n \" sections = extract_knowledge_point(text, chapter_number)\\n\",\n \" \\n\",\n \" # for section in sections:\\n\",\n \" # print(f\\\"标题：{section['title']}\\\")\\n\",\n \" # print(section['description'])\\n\",\n \" # print(\\\"-\\\" * 50)\\n\",\n \" return sections\\n\",\n \"\\n\",\n \"\\n\",\n \"# 使用示例\\n\",\n \"directory = \\\"/root/autodl-tmp/EasyEdu/data/DS2026_extracted/text\\\"\\n\",\n \"chapter_number = \\\"4\\\" # 要提取的章节号\\n\",\n \"knowledge_points = []\\n\",\n \"for i in range(121, 136):\\n\",\n \" file_path = os.path.join(directory, f'page_{i}.txt')\\n\",\n \" sections = process_file(file_path, chapter_number)\\n\",\n \" for section in sections:\\n\",\n \" id = section['section'].split('.')\\n\",\n \" id = ''.join(id)\\n\",\n \" knowledgepoint = {\\n\",\n \" \\\"id\\\": \\\"kc0\\\"+id, # 知识点唯一标识符\\n\",\n \" \\\"title\\\": section['title'], # 知识点标题\\n\",\n \" \\\"chapter_id\\\": \\\" c0\\\"+chapter_number, # 所属章节ID\\n\",\n \" \\\"description\\\": section['description'], # 知识点详细描述\\n\",\n \" \\\"related_points\\\": [ # 关联知识点\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\", # 关联知识点ID\\n\",\n \" \\\"relation_type\\\": \\\"string\\\" # 关系类型: prerequisite, related, extension\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"questions\\\": [ # 相关问题ID列表\\n\",\n \" \\\"question_id\\\"\\n\",\n \" ]\\n\",\n \"}\\n\",\n \" knowledge_points.append(knowledgepoint)\\n\",\n \"with open(os.path.join(\\\"/root/autodl-tmp/EasyEdu/data/ds_data/knowledgepoints\\\",f'chapter_{chapter_number}.json'), 'w', encoding='utf-8') as f:\\n\",\n \" json.dump(knowledge_points, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 8,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def combine_chapter_content(directory, start_page, end_page):\\n\",\n \" \\\"\\\"\\\"合并章节内容\\\"\\\"\\\"\\n\",\n \" combined_text = \\\"\\\"\\n\",\n \" for i in range(start_page, end_page + 1):\\n\",\n \" file_path = os.path.join(directory, f'page_{i}.txt')\\n\",\n \" if os.path.exists(file_path):\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" combined_text += f.read() + \\\"\\\\n\\\"\\n\",\n \" return combined_text\\n\",\n \"\\n\",\n \"def extract_knowledge_point(text, chapter_number):\\n\",\n \" import re\\n\",\n \" \\n\",\n \" # 使用正则表达式匹配章节标题和内容\\n\",\n \" section_pattern = rf'{chapter_number}\\\\.\\\\d+\\\\.\\\\d+'\\n\",\n \" \\n\",\n \" sections = []\\n\",\n \" matches = list(re.finditer(section_pattern, text))\\n\",\n \" \\n\",\n \" for i in range(len(matches)):\\n\",\n \" current_match = matches[i]\\n\",\n \" current_section = current_match.group()\\n\",\n \" start_pos = current_match.start()\\n\",\n \" \\n\",\n \" if i < len(matches) - 1:\\n\",\n \" end_pos = matches[i + 1].start()\\n\",\n \" content = text[start_pos:end_pos].strip()\\n\",\n \" else:\\n\",\n \" content = text[start_pos:].strip()\\n\",\n \" \\n\",\n \" if \\\"本节试题精选\\\" not in content and \\\"答案与解析\\\" not in content:\\n\",\n \" lines = content.split('\\\\n')\\n\",\n \" full_title = lines[0].strip()\\n\",\n \" title = full_title.split(current_section)[-1].strip()\\n\",\n \" description = '\\\\n'.join(lines[1:]).strip()\\n\",\n \" \\n\",\n \" sections.append({\\n\",\n \" 'section': current_section,\\n\",\n \" 'title': title,\\n\",\n \" 'description': description\\n\",\n \" })\\n\",\n \" \\n\",\n \" return sections\\n\",\n \"\\n\",\n \"def process_chapter(directory, chapter_number, start_page, end_page):\\n\",\n \" # 先合并所有页面内容\\n\",\n \" combined_text = combine_chapter_content(directory, start_page, end_page)\\n\",\n \" \\n\",\n \" # 提取知识点\\n\",\n \" sections = extract_knowledge_point(combined_text, chapter_number)\\n\",\n \" \\n\",\n \" knowledge_points = []\\n\",\n \" for section in sections:\\n\",\n \" id = section['section'].split('.')\\n\",\n \" id = ''.join(id)\\n\",\n \" knowledgepoint = {\\n\",\n \" \\\"id\\\": \\\"kc0\\\"+id,\\n\",\n \" \\\"title\\\": section['title'],\\n\",\n \" \\\"chapter_id\\\": \\\"c0\\\"+chapter_number,\\n\",\n \" \\\"description\\\": section['description'],\\n\",\n \" \\\"related_points\\\": [\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\",\\n\",\n \" \\\"relation_type\\\": \\\"string\\\"\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"questions\\\": [\\\"question_id\\\"]\\n\",\n \" }\\n\",\n \" knowledge_points.append(knowledgepoint)\\n\",\n \" \\n\",\n \" # 保存结果\\n\",\n \" output_dir = \\\"/root/autodl-tmp/EasyEdu/data/ds_data/knowledgepoints\\\"\\n\",\n \" os.makedirs(output_dir, exist_ok=True)\\n\",\n \" output_path = os.path.join(output_dir, f'chapter_{chapter_number}.json')\\n\",\n \" with open(output_path, 'w', encoding='utf-8') as f:\\n\",\n \" json.dump(knowledge_points, f, ensure_ascii=False, indent=4)\\n\",\n \" \\n\",\n \" return knowledge_points\\n\",\n \"\\n\",\n \"# 使用示例\\n\",\n \"directory = \\\"/root/autodl-tmp/EasyEdu/data/DS2026_extracted/text\\\"\\n\",\n \"chapter_number = \\\"8\\\"\\n\",\n \"start_page = 349 # 章节开始页\\n\",\n \"end_page = 406 # 章节结束页\\n\",\n \"\\n\",\n \"knowledge_points = process_chapter(directory, chapter_number, start_page, end_page)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"question = {\\n\",\n \" \\\"id\\\": \\\"q023001\\\", # 问题唯一标识符\\n\",\n \" \\\"title\\\": \\\"线性表存储\\\", # 问题标题\\n\",\n \" \\\"content\\\": content, # 问题内容\\n\",\n \" \\\"difficulty\\\": \\\"integer\\\", # 难度等级 (1-5)\\n\",\n \" \\\"type\\\": \\\"concept\\\", # 问题类型 (\\\"concept\\\", \\\"calculation\\\", \\\"application\\\")\\n\",\n \" \\\"knowledge_points\\\": [ # 相关知识点ID列表\\n\",\n \" \\\"kc0221\\\",\\n\",\n \" \\\"kc0231\\\"\\n\",\n \" ],\\n\",\n \" \\\"related_questions\\\": [ # 相关问题（用于扩展）\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\", # 相关问题ID\\n\",\n \" \\\"relation_type\\\": \\\"string\\\" # 关系类型: \\\"extension\\\", \\\"application\\\", \\\"contrast\\\"\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"reference_answer\\\": { # 参考答案\\n\",\n \" \\\"content\\\": \\\"B\\\", # 答案内容\\n\",\n \" \\\"key_points\\\": [ # 关键点列表\\n\",\n \" \\\"string\\\"\\n\",\n \" ],\\n\",\n \" \\\"explanation\\\": \\\"\\\"\\\"两种存储结构适用于不同的场合，不能简单地说谁好谁坏，选项1错误。链式存储用指针表示逻辑结构，而指针的设置是任意的，因此比顺序存储结构能更方便地表示各种逻辑结构，选项\\n\",\n \"ⅡI正确。在顺序存储中，插入和删除结点需要移动大量元素，效率较低，选项IⅢI的描述刚好相反。\\n\",\n \"顺序存储结构既能随机存取又能顺序存取，而链式结构只能顺序存取，选项IV正确。\\\"\\\"\\\" # 详细解释\\n\",\n \" }\\n\",\n \"}\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 题目\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import re\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def combine_text_files(file_paths):\\n\",\n \" \\\"\\\"\\\"合并多个文本文件的内容\\\"\\\"\\\"\\n\",\n \" combined_text = \\\"\\\"\\n\",\n \" for file_path in file_paths:\\n\",\n \" if os.path.exists(file_path):\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" combined_text += f.read() + \\\"\\\\n\\\"\\n\",\n \" return combined_text\\n\",\n \"\\n\",\n \"def extract_choice_questions(text):\\n\",\n \" \\\"\\\"\\\"提取选择题部分\\\"\\\"\\\"\\n\",\n \" # 找到\\\"一、单项选择题\\\"的位置\\n\",\n \" start_index = text.find(\\\"一、单项选择题\\\")\\n\",\n \" if start_index == -1:\\n\",\n \" return \\\"\\\"\\n\",\n \" \\n\",\n \" # 从这个位置开始查找\\\"二、\\\"\\n\",\n \" text_after_start = text[start_index:]\\n\",\n \" end_index = text_after_start.find(\\\"二、\\\")\\n\",\n \" \\n\",\n \" if end_index != -1: # 找到了\\\"二、\\\"\\n\",\n \" return text_after_start[:end_index]\\n\",\n \" else:\\n\",\n \" return text_after_start\\n\",\n \"\\n\",\n \"def parse_questions(questions_text):\\n\",\n \" \\\"\\\"\\\"解析题目内容\\\"\\\"\\\"\\n\",\n \" questions = {}\\n\",\n \" current_question = []\\n\",\n \" current_number = None\\n\",\n \" \\n\",\n \" # 跳过\\\"一、单项选择题\\\"之前的内容\\n\",\n \" content = questions_text.split(\\\"一、单项选择题\\\", 1)[1]\\n\",\n \" \\n\",\n \" for line in content.split('\\\\n'):\\n\",\n \" # 匹配题号（形如 01. 02. 等）\\n\",\n \" match = re.match(r'^(\\\\d{2})\\\\.\\\\s*(.+)', line)\\n\",\n \" if match:\\n\",\n \" # 保存前一个题目\\n\",\n \" if current_number:\\n\",\n \" questions[current_number] = '\\\\n'.join(filter(None, current_question))\\n\",\n \" # 开始新题目\\n\",\n \" current_number = match.group(1)\\n\",\n \" current_question = [match.group(2)]\\n\",\n \" elif current_number and line.strip():\\n\",\n \" current_question.append(line.strip())\\n\",\n \" \\n\",\n \" # 保存最后一个题目\\n\",\n \" if current_number:\\n\",\n \" questions[current_number] = '\\\\n'.join(filter(None, current_question))\\n\",\n \" \\n\",\n \" return questions\\n\",\n \"\\n\",\n \"def parse_answers(answers_text):\\n\",\n \" \\\"\\\"\\\"解析答案内容\\\"\\\"\\\"\\n\",\n \" answers = {}\\n\",\n \" current_number = None\\n\",\n \" current_answer = None\\n\",\n \" current_explanation = []\\n\",\n \" \\n\",\n \" # 跳过\\\"一、单项选择题\\\"之前的内容\\n\",\n \" content = answers_text.split(\\\"一、单项选择题\\\", 1)[1]\\n\",\n \" \\n\",\n \" for line in content.split('\\\\n'):\\n\",\n \" # 匹配答案（形如 01.B 或 01. B）\\n\",\n \" match = re.match(r'^(\\\\d{2})\\\\.\\\\s*([A-D])\\\\s*$', line)\\n\",\n \" if match:\\n\",\n \" # 保存前一题的内容\\n\",\n \" if current_number:\\n\",\n \" answers[current_number] = {\\n\",\n \" \\\"答案\\\": current_answer,\\n\",\n \" \\\"解析\\\": '\\\\n'.join(filter(None, current_explanation))\\n\",\n \" }\\n\",\n \" # 开始新题目\\n\",\n \" current_number = match.group(1)\\n\",\n \" current_answer = match.group(2)\\n\",\n \" current_explanation = []\\n\",\n \" elif current_number and line.strip():\\n\",\n \" current_explanation.append(line.strip())\\n\",\n \" \\n\",\n \" # 保存最后一题的内容\\n\",\n \" if current_number:\\n\",\n \" answers[current_number] = {\\n\",\n \" \\\"答案\\\": current_answer,\\n\",\n \" \\\"解析\\\": '\\\\n'.join(filter(None, current_explanation))\\n\",\n \" }\\n\",\n \" \\n\",\n \" return answers\\n\",\n \"\\n\",\n \"def process_questions_and_answers(question_files, answer_files):\\n\",\n \" \\\"\\\"\\\"处理题目和答案文件\\\"\\\"\\\"\\n\",\n \" # 合并题目文件\\n\",\n \" questions_text = combine_text_files(question_files)\\n\",\n \" questions_section = extract_choice_questions(questions_text)\\n\",\n \" questions = parse_questions(questions_section)\\n\",\n \" \\n\",\n \" # 合并答案文件\\n\",\n \" answers_text = combine_text_files(answer_files)\\n\",\n \" answers_section = extract_choice_questions(answers_text)\\n\",\n \" answers = parse_answers(answers_section)\\n\",\n \" \\n\",\n \" # 合并题目和答案\\n\",\n \" result = []\\n\",\n \" for number in sorted(questions.keys()):\\n\",\n \" if number.isdigit() and 1 <= int(number) <= 36:\\n\",\n \" answer_info = answers.get(number, {\\\"答案\\\": \\\"未找到\\\", \\\"解析\\\": \\\"未找到解析\\\"})\\n\",\n \" item = {\\n\",\n \" \\\"题号\\\": number,\\n\",\n \" \\\"题目\\\": questions[number].strip(),\\n\",\n \" \\\"答案\\\": answer_info[\\\"答案\\\"],\\n\",\n \" \\\"解析\\\": answer_info[\\\"解析\\\"]\\n\",\n \" }\\n\",\n \" result.append(item)\\n\",\n \" \\n\",\n \" return result\\n\",\n \"\\n\",\n \"def final_process(id, kpid, qleft, qright, alleft, aright, output_path): \\n\",\n \" question_files = []\\n\",\n \" answer_files = []\\n\",\n \"\\n\",\n \" file = \\\"/root/autodl-tmp/EasyEdu/data/DS2026_extracted/text\\\"\\n\",\n \" id = id\\n\",\n \" kpid = kpid\\n\",\n \" for i in range(qleft,qright):\\n\",\n \" question_files.append(os.path.join(file,f\\\"page_{i}.txt\\\"))\\n\",\n \" for i in range(alleft,aright):\\n\",\n \" answer_files.append(os.path.join(file,f\\\"page_{i}.txt\\\"))\\n\",\n \"\\n\",\n \" results = process_questions_and_answers(question_files, answer_files)\\n\",\n \"\\n\",\n \" questions = []\\n\",\n \" for q in results:\\n\",\n \" question = {\\n\",\n \" \\\"id\\\": id+q[\\\"题号\\\"], # 问题唯一标识符\\n\",\n \" \\\"title\\\": \\\"title\\\", # 问题标题\\n\",\n \" \\\"content\\\": q[\\\"题目\\\"], # 问题内容\\n\",\n \" \\\"difficulty\\\": \\\"integer\\\", # 难度等级 (1-5)\\n\",\n \" \\\"type\\\": \\\"concept\\\", # 问题类型 (\\\"concept\\\", \\\"calculation\\\", \\\"application\\\")\\n\",\n \" \\\"knowledge_points\\\": kpid, # 相关知识点ID列表\\n\",\n \" \\\"related_questions\\\": [ # 相关问题（用于扩展）\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\", # 相关问题ID\\n\",\n \" \\\"relation_type\\\": \\\"string\\\" # 关系类型: \\\"extension\\\", \\\"application\\\", \\\"contrast\\\"\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"reference_answer\\\": { # 参考答案\\n\",\n \" \\\"content\\\": q[\\\"答案\\\"], # 答案内容\\n\",\n \" \\\"key_points\\\": [ # 关键点列表\\n\",\n \" \\\"string\\\"\\n\",\n \" ],\\n\",\n \" \\\"explanation\\\": q[\\\"解析\\\"] # 详细解释\\n\",\n \" }\\n\",\n \" }\\n\",\n \" questions.append(question)\\n\",\n \"\\n\",\n \" # 如果文件不存在，则创建文件\\n\",\n \" if not os.path.exists(output_path):\\n\",\n \" with open(output_path,\\\"w\\\",encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump([],f,ensure_ascii=False,indent=4)\\n\",\n \"\\n\",\n \" with open(output_path,\\\"r\\\",encoding=\\\"utf-8\\\") as f:\\n\",\n \" data = json.load(f)\\n\",\n \"\\n\",\n \" data += questions\\n\",\n \"\\n\",\n \" with open(output_path,\\\"w\\\",encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(data,f,ensure_ascii=False,indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 34,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"final_process(\\\"q0870\\\",[\\\"kc871\\\", \\\"kc872\\\", \\\"kc873\\\", \\\"kc874\\\", \\\"kc875\\\"],404, 406, 406, 410, \\\"/root/autodl-tmp/EasyEdu/data/ds_data/questions/chapter_8.json\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### 题目起名\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"load_dotenv()\\n\",\n \"from langchain_deepseek import ChatDeepSeek\\n\",\n \"from langchain_core.prompts import ChatPromptTemplate\\n\",\n \"from typing import TypedDict\\n\",\n \"\\n\",\n \"class QuestionName(TypedDict):\\n\",\n \" question_name: str\\n\",\n \"\\n\",\n \"\\n\",\n \"model = ChatDeepSeek(model=\\\"deepseek-chat\\\", api_key=os.getenv(\\\"DEEPSEEK_API_KEY\\\"))\\n\",\n \"\\n\",\n \"prompt = ChatPromptTemplate([\\n\",\n \" (\\\"system\\\", \\\"请根据以下题目内容，起一个可以概括题目内容，并且简洁的题目名称, 尽可能在15个字以内。\\\"),\\n\",\n \" (\\\"human\\\", \\\"题目内容：{content}\\\"),\\n\",\n \"])\\n\",\n \"\\n\",\n \"chain = prompt | model.with_structured_output(QuestionName)\\n\",\n \"\\n\",\n \"for i in range(8, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyEdu/data/ds_data/questions/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" data = json.load(f)\\n\",\n \" for question in data:\\n\",\n \" question_name = chain.invoke({\\\"content\\\": question[\\\"content\\\"]})\\n\",\n \" question[\\\"title\\\"] = question_name[\\\"question_name\\\"]\\n\",\n \" print(\\\"ok\\\")\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyEdu/data/ds_data/questions/chapter_{i}.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(data, f, ensure_ascii=False, indent=4)\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 题目对应章节信息\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"directory = \\\"/root/autodl-tmp/EasyEdu/data/ds_data/questions\\\"\\n\",\n \"\\n\",\n \"for i in range(2, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyEdu/data/ds_data/questions/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" questions = json.load(f)\\n\",\n \" for question in questions:\\n\",\n \" question[\\\"chapter\\\"] = f\\\"c0{i}\\\"\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyEdu/data/ds_data/questions/chapter_{i}.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(questions, f, ensure_ascii=False, indent=4)\\n\",\n \" print(f\\\"第{i}章节处理完成\\\")\\n\",\n \" print(\\\"=\\\"*20)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 章节信息补充\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 4,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyEdu/data/ds_data/chapters.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" chapters = json.load(f)\\n\",\n \"for chapter in chapters:\\n\",\n \" chapter[\\\"knowledge_points\\\"] = []\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyEdu/data/ds_data/chapters.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(chapters, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with open(\\\"/root/autodl-tmp/EasyEdu/data/ds_data/chapters.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" chapters = json.load(f)\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyEdu/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" all_knowledgepoints = json.load(f)\\n\",\n \"for chapter in chapters:\\n\",\n \" for kp in all_knowledgepoints:\\n\",\n \" if kp[\\\"chapter_id\\\"] == chapter[\\\"id\\\"]:\\n\",\n \" chapter[\\\"knowledge_points\\\"].append(kp[\\\"id\\\"])\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyEdu/data/ds_data/chapters.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(chapters, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 补充知识点的问题列表\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 12,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"for i in range(2, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyEdu/data/ds_data/questions/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" questions = json.load(f)\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyEdu/data/ds_data/knowledgepoints/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" knowledgepoints = json.load(f)\\n\",\n \" for question in questions:\\n\",\n \" for kp in question[\\\"knowledge_points\\\"]:\\n\",\n \" for kp_item in knowledgepoints:\\n\",\n \" if kp_item[\\\"id\\\"] == kp:\\n\",\n \" kp_item[\\\"questions\\\"].append(question[\\\"id\\\"])\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyEdu/data/ds_data/knowledgepoints/chapter_{i}.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(knowledgepoints, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 对知识点进行关联\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"all_knowledgepoints = []\\n\",\n \"for i in range(1, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyEdu/data/ds_data/knowledgepoints/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" knowledgepoints = json.load(f)\\n\",\n \" for kp in knowledgepoints:\\n\",\n \" kp[\\\"related_points\\\"] = []\\n\",\n \" all_knowledgepoints.append(kp)\\n\",\n \"with open(f\\\"/root/autodl-tmp/EasyEdu/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(all_knowledgepoints, f, ensure_ascii=False, indent=4)\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"len(all_knowledgepoints)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from langchain_deepseek import ChatDeepSeek\\n\",\n \"from langchain_core.prompts import ChatPromptTemplate\\n\",\n \"from typing import TypedDict, Literal\\n\",\n \"import os\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"load_dotenv()\\n\",\n \"import json\\n\",\n \"\\n\",\n \"# 关系类型固定为\\\"prerequisite\\\", \\\"related\\\", \\\"extension\\\", 如果没有关系则返回None\\n\",\n \"class KnowledgePointRelation(TypedDict):\\n\",\n \" related_knowledge_points: Literal[\\\"prerequisite\\\", \\\"related\\\", \\\"extension\\\", \\\"no_relation\\\"]\\n\",\n \"\\n\",\n \"prompt = ChatPromptTemplate([\\n\",\n \" (\\\"system\\\", \\\"当前的知识点为\\\\ntitle: {title}\\\\ndescription: {description}\\\\n判断用户输入的知识点与当前知识点之间的关系，关系类型固定为\\\\\\\"prerequisite\\\\\\\", \\\\\\\"extension\\\\\\\", 如果没有很明确的前后置关系则返回\\\\\\\"no_relation\\\\\\\"。\\\"),\\n\",\n \" (\\\"human\\\", \\\"用户输入的知识点为\\\\ntitle: {user_title}\\\\ndescription: {user_description}\\\"),\\n\",\n \"])\\n\",\n \"\\n\",\n \"model = ChatDeepSeek(model=\\\"deepseek-chat\\\", api_key=os.getenv(\\\"DEEPSEEK_API_KEY\\\"))\\n\",\n \"chain = prompt | model.with_structured_output(KnowledgePointRelation)\\n\",\n \"\\n\",\n \"# with open(f\\\"/root/autodl-tmp/EasyEdu/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \"# all_knowledgepoints = json.load(f)\\n\",\n \"for i in range(96, 101):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyEdu/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" all_knowledgepoints = json.load(f)\\n\",\n \" for j in range(i+1, 102):\\n\",\n \" relation = chain.invoke({\\\"title\\\": all_knowledgepoints[i][\\\"title\\\"], \\\"description\\\": all_knowledgepoints[i][\\\"description\\\"], \\\"user_title\\\": all_knowledgepoints[j][\\\"title\\\"], \\\"user_description\\\": all_knowledgepoints[j][\\\"description\\\"]})\\n\",\n \" if relation[\\\"related_knowledge_points\\\"] != \\\"no_relation\\\":\\n\",\n \" all_knowledgepoints[i][\\\"related_points\\\"].append({\\n\",\n \" \\\"id\\\": all_knowledgepoints[j][\\\"id\\\"],\\n\",\n \" \\\"relation_type\\\": relation[\\\"related_knowledge_points\\\"]\\n\",\n \" })\\n\",\n \" if relation[\\\"related_knowledge_points\\\"] == \\\"prerequisite\\\":\\n\",\n \" all_knowledgepoints[j][\\\"related_points\\\"].append({\\n\",\n \" \\\"id\\\": all_knowledgepoints[i][\\\"id\\\"],\\n\",\n \" \\\"relation_type\\\": \\\"extension\\\"\\n\",\n \" })\\n\",\n \" elif relation[\\\"related_knowledge_points\\\"] == \\\"extension\\\":\\n\",\n \" all_knowledgepoints[j][\\\"related_points\\\"].append({\\n\",\n \" \\\"id\\\": all_knowledgepoints[i][\\\"id\\\"],\\n\",\n \" \\\"relation_type\\\": \\\"prerequisite\\\"\\n\",\n \" })\\n\",\n \" \\n\",\n \" # print(f\\\"第{i+1}个知识点与第{j+1}个知识点处理完成\\\")\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyEdu/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(all_knowledgepoints, f, ensure_ascii=False, indent=4)\\n\",\n \"\\n\",\n \" print(f\\\"第{i+1}个知识点处理完成\\\")\\n\",\n \" print(\\\"=\\\"*20)\\n\",\n \"\\n\",\n \" \\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 知识点速览\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 10,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyEdu/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" all_knowledgepoints = json.load(f)\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 11,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"for knowledgepoint in all_knowledgepoints:\\n\",\n \" if \\\"summry\\\" in knowledgepoint:\\n\",\n \" # 将连续的两个换行替换为一个\\n\",\n \" summry = knowledgepoint[\\\"summry\\\"].replace(\\\"\\\\n\\\\n\\\", \\\"\\\\n\\\")\\n\",\n \" lines = summry.split(\\\"\\\\n\\\")\\n\",\n \" cleaned_lines = [line.strip() for line in lines]\\n\",\n \" knowledgepoint[\\\"summry\\\"] = \\\"\\\\n\\\".join(cleaned_lines)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 12,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with open(\\\"/root/autodl-tmp/EasyEdu/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(all_knowledgepoints, f, ensure_ascii=False, indent=4)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 索引\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 12,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from index_builder import KnowledgeIndexSystem\\n\",\n \"\\n\",\n \"system = KnowledgeIndexSystem.load_indices('/root/autodl-tmp/EasyEdu/data/ds_data/ds_indices.pkl')\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 11,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"'**数据** \\\\n- **定义**：信息的载体，可被计算机程序处理的符号集合（数、字符等），是程序加工的原料。 \\\\n\\\\n**数据元素** \\\\n- **定义**：数据的基本单位，由若干**数据项**（不可分割的最小单位）组成。 \\\\n- **示例**：学生记录（数据元素）包含学号、姓名、性别等数据项。 \\\\n\\\\n**数据对象** \\\\n- **定义**：相同性质数据元素的集合，是数据的子集。 \\\\n- **示例**：整数数据对象为集合 `N={0, ±1, ±2, ...}`。 \\\\n\\\\n**数据类型** \\\\n- **定义**：值的集合及其上定义的操作总称。 \\\\n- **分类**： \\\\n 1. **原子类型**：值不可再分（如`int`）。 \\\\n 2. **结构类型**：值可分解为多个成分（如`struct`）。 \\\\n 3. **抽象数据类型（ADT）**：数学模型及操作集合（如栈、队列的接口定义）。 \\\\n\\\\n**数据结构** \\\\n- **定义**：数据元素的集合及其间的特定关系，包含三方面： \\\\n 1. **逻辑结构**：数据间的抽象关系（线性、树形、图等）。 \\\\n 2. **存储结构**：逻辑结构在计算机中的实现（顺序存储、链式存储等）。 \\\\n 3. **数据运算**：对数据的基本操作（插入、删除、查找等）。 \\\\n- **关键点**：算法设计依赖逻辑结构，实现依赖存储结构。 \\\\n\\\\n**记忆提示**： \\\\n- 数据元素≈实体，数据项≈属性；ADT≈接口，数据结构≈接口+实现。'\"\n ]\n },\n \"execution_count\": 11,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"system.get_knowledge_point(\\\"kc0111\\\")['summry']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 9,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"chapters_knowledge_points = {}\\n\",\n \"for chapter_id, chapter_info in system.chapter_index.items():\\n\",\n \" chapters_knowledge_points[chapter_id] = chapter_info['knowledge_points']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 10,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"['kc0111', 'kc0112', 'kc0121', 'kc0122']\"\n ]\n },\n \"execution_count\": 10,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"chapters_knowledge_points[\\\"01\\\"]\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"EasyEdu\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.11\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n"}, "c31307c76080e23c2dd5e161ddc113604e6cf412": {"t": 1785753742.1049004, "n": 31591, "h": "56f80ecd269b4896", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 章节\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def extract_review_tips(text):\\n\",\n \" # 查找【复习提示】的起始位置\\n\",\n \" start_marker = \\\"【复习提示】\\\"\\n\",\n \" start_idx = text.find(start_marker)\\n\",\n \" \\n\",\n \" if start_idx == -1:\\n\",\n \" return None\\n\",\n \" \\n\",\n \" # 从标记后开始的文本\\n\",\n \" start_idx += len(start_marker)\\n\",\n \" \\n\",\n \" # 使用正则表达式找到下一个章节标记（如 1.1, 2.1, 3.1 等）\\n\",\n \" import re\\n\",\n \" text_after_tips = text[start_idx:]\\n\",\n \" chapter_match = re.search(r'\\\\n\\\\d+\\\\.\\\\d+', text_after_tips)\\n\",\n \" \\n\",\n \" if chapter_match:\\n\",\n \" end_idx = chapter_match.start()\\n\",\n \" review_tip = text_after_tips[:end_idx].strip()\\n\",\n \" else:\\n\",\n \" review_tip = text_after_tips.strip()\\n\",\n \" \\n\",\n \" return review_tip\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"# 使用示例\\n\",\n \"if __name__ == \\\"__main__\\\":\\n\",\n \" directory = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \" chapter_page = ['13', '24', '74', '121', '136', '210', '281', '349']\\n\",\n \" titles = ['绪论', '线性表', '栈和队列和数组', '串', '树和二叉树', '图', '查找', '排序']\\n\",\n \" chapters = []\\n\",\n \" for i, page in enumerate(chapter_page, 1):\\n\",\n \" file_path = os.path.join(directory, f'page_{page}.txt')\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" text = f.read()\\n\",\n \" review_tip = extract_review_tips(text)\\n\",\n \" if review_tip:\\n\",\n \" id = \\\"c0\\\" + str(i)\\n\",\n \" chapter = {\\n\",\n \" \\\"id\\\": id, # 章节唯一标识符\\n\",\n \" \\\"title\\\": titles[i-1], # 章节标题\\n\",\n \" \\\"parent_id\\\": \\\"\\\", # 父章节ID（可选）\\n\",\n \" \\\"order\\\": \\\"integer\\\", # 排序顺序\\n\",\n \" \\\"description\\\": review_tip, # 章节描述\\n\",\n \" \\\"knowledge_points\\\": [ # 包含的知识点ID列表\\n\",\n \" \\\"knowledge_point_id\\\"\\n\",\n \" ],\\n\",\n \" \\\"sub_chapters\\\": [ # 子章节ID列表\\n\",\n \" \\\"chapter_id\\\"\\n\",\n \" ]\\n\",\n \"}\\n\",\n \" chapters.append(chapter)\\n\",\n \" with open('/root/autodl-tmp/EasyDS/data/questions/chapters.json', 'w', encoding='utf-8') as f:\\n\",\n \" json.dump(chapters, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 知识点\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def extract_knowledge_point(text, chapter_number):\\n\",\n \" import re\\n\",\n \" \\n\",\n \" # 使用正则表达式匹配章节标题和内容\\n\",\n \" section_pattern = rf'{chapter_number}\\\\.\\\\d+\\\\.\\\\d+'\\n\",\n \" \\n\",\n \" sections = []\\n\",\n \" matches = list(re.finditer(section_pattern, text))\\n\",\n \" \\n\",\n \" for i in range(len(matches)):\\n\",\n \" current_match = matches[i]\\n\",\n \" current_section = current_match.group()\\n\",\n \" start_pos = current_match.start()\\n\",\n \" \\n\",\n \" if i < len(matches) - 1:\\n\",\n \" end_pos = matches[i + 1].start()\\n\",\n \" content = text[start_pos:end_pos].strip()\\n\",\n \" else:\\n\",\n \" content = text[start_pos:].strip()\\n\",\n \" \\n\",\n \" if \\\"本节试题精选\\\" not in content and \\\"答案与解析\\\" not in content:\\n\",\n \" lines = content.split('\\\\n')\\n\",\n \" full_title = lines[0].strip()\\n\",\n \" title = full_title.split(current_section)[-1].strip()\\n\",\n \" description = '\\\\n'.join(lines[1:]).strip()\\n\",\n \" \\n\",\n \" sections.append({\\n\",\n \" 'section': current_section,\\n\",\n \" 'title': title,\\n\",\n \" 'description': description\\n\",\n \" })\\n\",\n \" \\n\",\n \" return sections\\n\",\n \"\\n\",\n \"def process_file(file_path, chapter_number):\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" text = f.read()\\n\",\n \" \\n\",\n \" sections = extract_knowledge_point(text, chapter_number)\\n\",\n \" \\n\",\n \" # for section in sections:\\n\",\n \" # print(f\\\"标题：{section['title']}\\\")\\n\",\n \" # print(section['description'])\\n\",\n \" # print(\\\"-\\\" * 50)\\n\",\n \" return sections\\n\",\n \"\\n\",\n \"\\n\",\n \"# 使用示例\\n\",\n \"directory = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \"chapter_number = \\\"4\\\" # 要提取的章节号\\n\",\n \"knowledge_points = []\\n\",\n \"for i in range(121, 136):\\n\",\n \" file_path = os.path.join(directory, f'page_{i}.txt')\\n\",\n \" sections = process_file(file_path, chapter_number)\\n\",\n \" for section in sections:\\n\",\n \" id = section['section'].split('.')\\n\",\n \" id = ''.join(id)\\n\",\n \" knowledgepoint = {\\n\",\n \" \\\"id\\\": \\\"kc0\\\"+id, # 知识点唯一标识符\\n\",\n \" \\\"title\\\": section['title'], # 知识点标题\\n\",\n \" \\\"chapter_id\\\": \\\" c0\\\"+chapter_number, # 所属章节ID\\n\",\n \" \\\"description\\\": section['description'], # 知识点详细描述\\n\",\n \" \\\"related_points\\\": [ # 关联知识点\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\", # 关联知识点ID\\n\",\n \" \\\"relation_type\\\": \\\"string\\\" # 关系类型: prerequisite, related, extension\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"questions\\\": [ # 相关问题ID列表\\n\",\n \" \\\"question_id\\\"\\n\",\n \" ]\\n\",\n \"}\\n\",\n \" knowledge_points.append(knowledgepoint)\\n\",\n \"with open(os.path.join(\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints\\\",f'chapter_{chapter_number}.json'), 'w', encoding='utf-8') as f:\\n\",\n \" json.dump(knowledge_points, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 8,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def combine_chapter_content(directory, start_page, end_page):\\n\",\n \" \\\"\\\"\\\"合并章节内容\\\"\\\"\\\"\\n\",\n \" combined_text = \\\"\\\"\\n\",\n \" for i in range(start_page, end_page + 1):\\n\",\n \" file_path = os.path.join(directory, f'page_{i}.txt')\\n\",\n \" if os.path.exists(file_path):\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" combined_text += f.read() + \\\"\\\\n\\\"\\n\",\n \" return combined_text\\n\",\n \"\\n\",\n \"def extract_knowledge_point(text, chapter_number):\\n\",\n \" import re\\n\",\n \" \\n\",\n \" # 使用正则表达式匹配章节标题和内容\\n\",\n \" section_pattern = rf'{chapter_number}\\\\.\\\\d+\\\\.\\\\d+'\\n\",\n \" \\n\",\n \" sections = []\\n\",\n \" matches = list(re.finditer(section_pattern, text))\\n\",\n \" \\n\",\n \" for i in range(len(matches)):\\n\",\n \" current_match = matches[i]\\n\",\n \" current_section = current_match.group()\\n\",\n \" start_pos = current_match.start()\\n\",\n \" \\n\",\n \" if i < len(matches) - 1:\\n\",\n \" end_pos = matches[i + 1].start()\\n\",\n \" content = text[start_pos:end_pos].strip()\\n\",\n \" else:\\n\",\n \" content = text[start_pos:].strip()\\n\",\n \" \\n\",\n \" if \\\"本节试题精选\\\" not in content and \\\"答案与解析\\\" not in content:\\n\",\n \" lines = content.split('\\\\n')\\n\",\n \" full_title = lines[0].strip()\\n\",\n \" title = full_title.split(current_section)[-1].strip()\\n\",\n \" description = '\\\\n'.join(lines[1:]).strip()\\n\",\n \" \\n\",\n \" sections.append({\\n\",\n \" 'section': current_section,\\n\",\n \" 'title': title,\\n\",\n \" 'description': description\\n\",\n \" })\\n\",\n \" \\n\",\n \" return sections\\n\",\n \"\\n\",\n \"def process_chapter(directory, chapter_number, start_page, end_page):\\n\",\n \" # 先合并所有页面内容\\n\",\n \" combined_text = combine_chapter_content(directory, start_page, end_page)\\n\",\n \" \\n\",\n \" # 提取知识点\\n\",\n \" sections = extract_knowledge_point(combined_text, chapter_number)\\n\",\n \" \\n\",\n \" knowledge_points = []\\n\",\n \" for section in sections:\\n\",\n \" id = section['section'].split('.')\\n\",\n \" id = ''.join(id)\\n\",\n \" knowledgepoint = {\\n\",\n \" \\\"id\\\": \\\"kc0\\\"+id,\\n\",\n \" \\\"title\\\": section['title'],\\n\",\n \" \\\"chapter_id\\\": \\\"c0\\\"+chapter_number,\\n\",\n \" \\\"description\\\": section['description'],\\n\",\n \" \\\"related_points\\\": [\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\",\\n\",\n \" \\\"relation_type\\\": \\\"string\\\"\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"questions\\\": [\\\"question_id\\\"]\\n\",\n \" }\\n\",\n \" knowledge_points.append(knowledgepoint)\\n\",\n \" \\n\",\n \" # 保存结果\\n\",\n \" output_dir = \\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints\\\"\\n\",\n \" os.makedirs(output_dir, exist_ok=True)\\n\",\n \" output_path = os.path.join(output_dir, f'chapter_{chapter_number}.json')\\n\",\n \" with open(output_path, 'w', encoding='utf-8') as f:\\n\",\n \" json.dump(knowledge_points, f, ensure_ascii=False, indent=4)\\n\",\n \" \\n\",\n \" return knowledge_points\\n\",\n \"\\n\",\n \"# 使用示例\\n\",\n \"directory = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \"chapter_number = \\\"8\\\"\\n\",\n \"start_page = 349 # 章节开始页\\n\",\n \"end_page = 406 # 章节结束页\\n\",\n \"\\n\",\n \"knowledge_points = process_chapter(directory, chapter_number, start_page, end_page)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"question = {\\n\",\n \" \\\"id\\\": \\\"q023001\\\", # 问题唯一标识符\\n\",\n \" \\\"title\\\": \\\"线性表存储\\\", # 问题标题\\n\",\n \" \\\"content\\\": content, # 问题内容\\n\",\n \" \\\"difficulty\\\": \\\"integer\\\", # 难度等级 (1-5)\\n\",\n \" \\\"type\\\": \\\"concept\\\", # 问题类型 (\\\"concept\\\", \\\"calculation\\\", \\\"application\\\")\\n\",\n \" \\\"knowledge_points\\\": [ # 相关知识点ID列表\\n\",\n \" \\\"kc0221\\\",\\n\",\n \" \\\"kc0231\\\"\\n\",\n \" ],\\n\",\n \" \\\"related_questions\\\": [ # 相关问题（用于扩展）\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\", # 相关问题ID\\n\",\n \" \\\"relation_type\\\": \\\"string\\\" # 关系类型: \\\"extension\\\", \\\"application\\\", \\\"contrast\\\"\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"reference_answer\\\": { # 参考答案\\n\",\n \" \\\"content\\\": \\\"B\\\", # 答案内容\\n\",\n \" \\\"key_points\\\": [ # 关键点列表\\n\",\n \" \\\"string\\\"\\n\",\n \" ],\\n\",\n \" \\\"explanation\\\": \\\"\\\"\\\"两种存储结构适用于不同的场合，不能简单地说谁好谁坏，选项1错误。链式存储用指针表示逻辑结构，而指针的设置是任意的，因此比顺序存储结构能更方便地表示各种逻辑结构，选项\\n\",\n \"ⅡI正确。在顺序存储中，插入和删除结点需要移动大量元素，效率较低，选项IⅢI的描述刚好相反。\\n\",\n \"顺序存储结构既能随机存取又能顺序存取，而链式结构只能顺序存取，选项IV正确。\\\"\\\"\\\" # 详细解释\\n\",\n \" }\\n\",\n \"}\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 题目\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import re\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def combine_text_files(file_paths):\\n\",\n \" \\\"\\\"\\\"合并多个文本文件的内容\\\"\\\"\\\"\\n\",\n \" combined_text = \\\"\\\"\\n\",\n \" for file_path in file_paths:\\n\",\n \" if os.path.exists(file_path):\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" combined_text += f.read() + \\\"\\\\n\\\"\\n\",\n \" return combined_text\\n\",\n \"\\n\",\n \"def extract_choice_questions(text):\\n\",\n \" \\\"\\\"\\\"提取选择题部分\\\"\\\"\\\"\\n\",\n \" # 找到\\\"一、单项选择题\\\"的位置\\n\",\n \" start_index = text.find(\\\"一、单项选择题\\\")\\n\",\n \" if start_index == -1:\\n\",\n \" return \\\"\\\"\\n\",\n \" \\n\",\n \" # 从这个位置开始查找\\\"二、\\\"\\n\",\n \" text_after_start = text[start_index:]\\n\",\n \" end_index = text_after_start.find(\\\"二、\\\")\\n\",\n \" \\n\",\n \" if end_index != -1: # 找到了\\\"二、\\\"\\n\",\n \" return text_after_start[:end_index]\\n\",\n \" else:\\n\",\n \" return text_after_start\\n\",\n \"\\n\",\n \"def parse_questions(questions_text):\\n\",\n \" \\\"\\\"\\\"解析题目内容\\\"\\\"\\\"\\n\",\n \" questions = {}\\n\",\n \" current_question = []\\n\",\n \" current_number = None\\n\",\n \" \\n\",\n \" # 跳过\\\"一、单项选择题\\\"之前的内容\\n\",\n \" content = questions_text.split(\\\"一、单项选择题\\\", 1)[1]\\n\",\n \" \\n\",\n \" for line in content.split('\\\\n'):\\n\",\n \" # 匹配题号（形如 01. 02. 等）\\n\",\n \" match = re.match(r'^(\\\\d{2})\\\\.\\\\s*(.+)', line)\\n\",\n \" if match:\\n\",\n \" # 保存前一个题目\\n\",\n \" if current_number:\\n\",\n \" questions[current_number] = '\\\\n'.join(filter(None, current_question))\\n\",\n \" # 开始新题目\\n\",\n \" current_number = match.group(1)\\n\",\n \" current_question = [match.group(2)]\\n\",\n \" elif current_number and line.strip():\\n\",\n \" current_question.append(line.strip())\\n\",\n \" \\n\",\n \" # 保存最后一个题目\\n\",\n \" if current_number:\\n\",\n \" questions[current_number] = '\\\\n'.join(filter(None, current_question))\\n\",\n \" \\n\",\n \" return questions\\n\",\n \"\\n\",\n \"def parse_answers(answers_text):\\n\",\n \" \\\"\\\"\\\"解析答案内容\\\"\\\"\\\"\\n\",\n \" answers = {}\\n\",\n \" current_number = None\\n\",\n \" current_answer = None\\n\",\n \" current_explanation = []\\n\",\n \" \\n\",\n \" # 跳过\\\"一、单项选择题\\\"之前的内容\\n\",\n \" content = answers_text.split(\\\"一、单项选择题\\\", 1)[1]\\n\",\n \" \\n\",\n \" for line in content.split('\\\\n'):\\n\",\n \" # 匹配答案（形如 01.B 或 01. B）\\n\",\n \" match = re.match(r'^(\\\\d{2})\\\\.\\\\s*([A-D])\\\\s*$', line)\\n\",\n \" if match:\\n\",\n \" # 保存前一题的内容\\n\",\n \" if current_number:\\n\",\n \" answers[current_number] = {\\n\",\n \" \\\"答案\\\": current_answer,\\n\",\n \" \\\"解析\\\": '\\\\n'.join(filter(None, current_explanation))\\n\",\n \" }\\n\",\n \" # 开始新题目\\n\",\n \" current_number = match.group(1)\\n\",\n \" current_answer = match.group(2)\\n\",\n \" current_explanation = []\\n\",\n \" elif current_number and line.strip():\\n\",\n \" current_explanation.append(line.strip())\\n\",\n \" \\n\",\n \" # 保存最后一题的内容\\n\",\n \" if current_number:\\n\",\n \" answers[current_number] = {\\n\",\n \" \\\"答案\\\": current_answer,\\n\",\n \" \\\"解析\\\": '\\\\n'.join(filter(None, current_explanation))\\n\",\n \" }\\n\",\n \" \\n\",\n \" return answers\\n\",\n \"\\n\",\n \"def process_questions_and_answers(question_files, answer_files):\\n\",\n \" \\\"\\\"\\\"处理题目和答案文件\\\"\\\"\\\"\\n\",\n \" # 合并题目文件\\n\",\n \" questions_text = combine_text_files(question_files)\\n\",\n \" questions_section = extract_choice_questions(questions_text)\\n\",\n \" questions = parse_questions(questions_section)\\n\",\n \" \\n\",\n \" # 合并答案文件\\n\",\n \" answers_text = combine_text_files(answer_files)\\n\",\n \" answers_section = extract_choice_questions(answers_text)\\n\",\n \" answers = parse_answers(answers_section)\\n\",\n \" \\n\",\n \" # 合并题目和答案\\n\",\n \" result = []\\n\",\n \" for number in sorted(questions.keys()):\\n\",\n \" if number.isdigit() and 1 <= int(number) <= 36:\\n\",\n \" answer_info = answers.get(number, {\\\"答案\\\": \\\"未找到\\\", \\\"解析\\\": \\\"未找到解析\\\"})\\n\",\n \" item = {\\n\",\n \" \\\"题号\\\": number,\\n\",\n \" \\\"题目\\\": questions[number].strip(),\\n\",\n \" \\\"答案\\\": answer_info[\\\"答案\\\"],\\n\",\n \" \\\"解析\\\": answer_info[\\\"解析\\\"]\\n\",\n \" }\\n\",\n \" result.append(item)\\n\",\n \" \\n\",\n \" return result\\n\",\n \"\\n\",\n \"def final_process(id, kpid, qleft, qright, alleft, aright, output_path): \\n\",\n \" question_files = []\\n\",\n \" answer_files = []\\n\",\n \"\\n\",\n \" file = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \" id = id\\n\",\n \" kpid = kpid\\n\",\n \" for i in range(qleft,qright):\\n\",\n \" question_files.append(os.path.join(file,f\\\"page_{i}.txt\\\"))\\n\",\n \" for i in range(alleft,aright):\\n\",\n \" answer_files.append(os.path.join(file,f\\\"page_{i}.txt\\\"))\\n\",\n \"\\n\",\n \" results = process_questions_and_answers(question_files, answer_files)\\n\",\n \"\\n\",\n \" questions = []\\n\",\n \" for q in results:\\n\",\n \" question = {\\n\",\n \" \\\"id\\\": id+q[\\\"题号\\\"], # 问题唯一标识符\\n\",\n \" \\\"title\\\": \\\"title\\\", # 问题标题\\n\",\n \" \\\"content\\\": q[\\\"题目\\\"], # 问题内容\\n\",\n \" \\\"difficulty\\\": \\\"integer\\\", # 难度等级 (1-5)\\n\",\n \" \\\"type\\\": \\\"concept\\\", # 问题类型 (\\\"concept\\\", \\\"calculation\\\", \\\"application\\\")\\n\",\n \" \\\"knowledge_points\\\": kpid, # 相关知识点ID列表\\n\",\n \" \\\"related_questions\\\": [ # 相关问题（用于扩展）\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\", # 相关问题ID\\n\",\n \" \\\"relation_type\\\": \\\"string\\\" # 关系类型: \\\"extension\\\", \\\"application\\\", \\\"contrast\\\"\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"reference_answer\\\": { # 参考答案\\n\",\n \" \\\"content\\\": q[\\\"答案\\\"], # 答案内容\\n\",\n \" \\\"key_points\\\": [ # 关键点列表\\n\",\n \" \\\"string\\\"\\n\",\n \" ],\\n\",\n \" \\\"explanation\\\": q[\\\"解析\\\"] # 详细解释\\n\",\n \" }\\n\",\n \" }\\n\",\n \" questions.append(question)\\n\",\n \"\\n\",\n \" # 如果文件不存在，则创建文件\\n\",\n \" if not os.path.exists(output_path):\\n\",\n \" with open(output_path,\\\"w\\\",encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump([],f,ensure_ascii=False,indent=4)\\n\",\n \"\\n\",\n \" with open(output_path,\\\"r\\\",encoding=\\\"utf-8\\\") as f:\\n\",\n \" data = json.load(f)\\n\",\n \"\\n\",\n \" data += questions\\n\",\n \"\\n\",\n \" with open(output_path,\\\"w\\\",encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(data,f,ensure_ascii=False,indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 34,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"final_process(\\\"q0870\\\",[\\\"kc871\\\", \\\"kc872\\\", \\\"kc873\\\", \\\"kc874\\\", \\\"kc875\\\"],404, 406, 406, 410, \\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_8.json\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### 题目起名\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"load_dotenv()\\n\",\n \"from langchain_deepseek import ChatDeepSeek\\n\",\n \"from langchain_core.prompts import ChatPromptTemplate\\n\",\n \"from typing import TypedDict\\n\",\n \"\\n\",\n \"class QuestionName(TypedDict):\\n\",\n \" question_name: str\\n\",\n \"\\n\",\n \"\\n\",\n \"model = ChatDeepSeek(model=\\\"deepseek-chat\\\", api_key=os.getenv(\\\"DEEPSEEK_API_KEY\\\"))\\n\",\n \"\\n\",\n \"prompt = ChatPromptTemplate([\\n\",\n \" (\\\"system\\\", \\\"请根据以下题目内容，起一个可以概括题目内容，并且简洁的题目名称, 尽可能在15个字以内。\\\"),\\n\",\n \" (\\\"human\\\", \\\"题目内容：{content}\\\"),\\n\",\n \"])\\n\",\n \"\\n\",\n \"chain = prompt | model.with_structured_output(QuestionName)\\n\",\n \"\\n\",\n \"for i in range(8, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" data = json.load(f)\\n\",\n \" for question in data:\\n\",\n \" question_name = chain.invoke({\\\"content\\\": question[\\\"content\\\"]})\\n\",\n \" question[\\\"title\\\"] = question_name[\\\"question_name\\\"]\\n\",\n \" print(\\\"ok\\\")\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(data, f, ensure_ascii=False, indent=4)\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 题目对应章节信息\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"directory = \\\"/root/autodl-tmp/EasyDS/data/ds_data/questions\\\"\\n\",\n \"\\n\",\n \"for i in range(2, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" questions = json.load(f)\\n\",\n \" for question in questions:\\n\",\n \" question[\\\"chapter\\\"] = f\\\"c0{i}\\\"\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(questions, f, ensure_ascii=False, indent=4)\\n\",\n \" print(f\\\"第{i}章节处理完成\\\")\\n\",\n \" print(\\\"=\\\"*20)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 章节信息补充\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 4,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" chapters = json.load(f)\\n\",\n \"for chapter in chapters:\\n\",\n \" chapter[\\\"knowledge_points\\\"] = []\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(chapters, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" chapters = json.load(f)\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" all_knowledgepoints = json.load(f)\\n\",\n \"for chapter in chapters:\\n\",\n \" for kp in all_knowledgepoints:\\n\",\n \" if kp[\\\"chapter_id\\\"] == chapter[\\\"id\\\"]:\\n\",\n \" chapter[\\\"knowledge_points\\\"].append(kp[\\\"id\\\"])\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(chapters, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 补充知识点的问题列表\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 12,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"for i in range(2, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" questions = json.load(f)\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" knowledgepoints = json.load(f)\\n\",\n \" for question in questions:\\n\",\n \" for kp in question[\\\"knowledge_points\\\"]:\\n\",\n \" for kp_item in knowledgepoints:\\n\",\n \" if kp_item[\\\"id\\\"] == kp:\\n\",\n \" kp_item[\\\"questions\\\"].append(question[\\\"id\\\"])\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/chapter_{i}.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(knowledgepoints, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 对知识点进行关联\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"all_knowledgepoints = []\\n\",\n \"for i in range(1, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" knowledgepoints = json.load(f)\\n\",\n \" for kp in knowledgepoints:\\n\",\n \" kp[\\\"related_points\\\"] = []\\n\",\n \" all_knowledgepoints.append(kp)\\n\",\n \"with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(all_knowledgepoints, f, ensure_ascii=False, indent=4)\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"len(all_knowledgepoints)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from langchain_deepseek import ChatDeepSeek\\n\",\n \"from langchain_core.prompts import ChatPromptTemplate\\n\",\n \"from typing import TypedDict, Literal\\n\",\n \"import os\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"load_dotenv()\\n\",\n \"import json\\n\",\n \"\\n\",\n \"# 关系类型固定为\\\"prerequisite\\\", \\\"related\\\", \\\"extension\\\", 如果没有关系则返回None\\n\",\n \"class KnowledgePointRelation(TypedDict):\\n\",\n \" related_knowledge_points: Literal[\\\"prerequisite\\\", \\\"related\\\", \\\"extension\\\", \\\"no_relation\\\"]\\n\",\n \"\\n\",\n \"prompt = ChatPromptTemplate([\\n\",\n \" (\\\"system\\\", \\\"当前的知识点为\\\\ntitle: {title}\\\\ndescription: {description}\\\\n判断用户输入的知识点与当前知识点之间的关系，关系类型固定为\\\\\\\"prerequisite\\\\\\\", \\\\\\\"extension\\\\\\\", 如果没有很明确的前后置关系则返回\\\\\\\"no_relation\\\\\\\"。\\\"),\\n\",\n \" (\\\"human\\\", \\\"用户输入的知识点为\\\\ntitle: {user_title}\\\\ndescription: {user_description}\\\"),\\n\",\n \"])\\n\",\n \"\\n\",\n \"model = ChatDeepSeek(model=\\\"deepseek-chat\\\", api_key=os.getenv(\\\"DEEPSEEK_API_KEY\\\"))\\n\",\n \"chain = prompt | model.with_structured_output(KnowledgePointRelation)\\n\",\n \"\\n\",\n \"# with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \"# all_knowledgepoints = json.load(f)\\n\",\n \"for i in range(96, 101):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" all_knowledgepoints = json.load(f)\\n\",\n \" for j in range(i+1, 102):\\n\",\n \" relation = chain.invoke({\\\"title\\\": all_knowledgepoints[i][\\\"title\\\"], \\\"description\\\": all_knowledgepoints[i][\\\"description\\\"], \\\"user_title\\\": all_knowledgepoints[j][\\\"title\\\"], \\\"user_description\\\": all_knowledgepoints[j][\\\"description\\\"]})\\n\",\n \" if relation[\\\"related_knowledge_points\\\"] != \\\"no_relation\\\":\\n\",\n \" all_knowledgepoints[i][\\\"related_points\\\"].append({\\n\",\n \" \\\"id\\\": all_knowledgepoints[j][\\\"id\\\"],\\n\",\n \" \\\"relation_type\\\": relation[\\\"related_knowledge_points\\\"]\\n\",\n \" })\\n\",\n \" if relation[\\\"related_knowledge_points\\\"] == \\\"prerequisite\\\":\\n\",\n \" all_knowledgepoints[j][\\\"related_points\\\"].append({\\n\",\n \" \\\"id\\\": all_knowledgepoints[i][\\\"id\\\"],\\n\",\n \" \\\"relation_type\\\": \\\"extension\\\"\\n\",\n \" })\\n\",\n \" elif relation[\\\"related_knowledge_points\\\"] == \\\"extension\\\":\\n\",\n \" all_knowledgepoints[j][\\\"related_points\\\"].append({\\n\",\n \" \\\"id\\\": all_knowledgepoints[i][\\\"id\\\"],\\n\",\n \" \\\"relation_type\\\": \\\"prerequisite\\\"\\n\",\n \" })\\n\",\n \" \\n\",\n \" # print(f\\\"第{i+1}个知识点与第{j+1}个知识点处理完成\\\")\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(all_knowledgepoints, f, ensure_ascii=False, indent=4)\\n\",\n \"\\n\",\n \" print(f\\\"第{i+1}个知识点处理完成\\\")\\n\",\n \" print(\\\"=\\\"*20)\\n\",\n \"\\n\",\n \" \\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 知识点速览\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 10,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" all_knowledgepoints = json.load(f)\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 11,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"for knowledgepoint in all_knowledgepoints:\\n\",\n \" if \\\"summry\\\" in knowledgepoint:\\n\",\n \" # 将连续的两个换行替换为一个\\n\",\n \" summry = knowledgepoint[\\\"summry\\\"].replace(\\\"\\\\n\\\\n\\\", \\\"\\\\n\\\")\\n\",\n \" lines = summry.split(\\\"\\\\n\\\")\\n\",\n \" cleaned_lines = [line.strip() for line in lines]\\n\",\n \" knowledgepoint[\\\"summry\\\"] = \\\"\\\\n\\\".join(cleaned_lines)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 12,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(all_knowledgepoints, f, ensure_ascii=False, indent=4)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 索引\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 12,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from index_builder import KnowledgeIndexSystem\\n\",\n \"\\n\",\n \"system = KnowledgeIndexSystem.load_indices('/root/autodl-tmp/EasyDS/data/ds_data/ds_indices.pkl')\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 11,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"'**数据** \\\\n- **定义**：信息的载体，可被计算机程序处理的符号集合（数、字符等），是程序加工的原料。 \\\\n\\\\n**数据元素** \\\\n- **定义**：数据的基本单位，由若干**数据项**（不可分割的最小单位）组成。 \\\\n- **示例**：学生记录（数据元素）包含学号、姓名、性别等数据项。 \\\\n\\\\n**数据对象** \\\\n- **定义**：相同性质数据元素的集合，是数据的子集。 \\\\n- **示例**：整数数据对象为集合 `N={0, ±1, ±2, ...}`。 \\\\n\\\\n**数据类型** \\\\n- **定义**：值的集合及其上定义的操作总称。 \\\\n- **分类**： \\\\n 1. **原子类型**：值不可再分（如`int`）。 \\\\n 2. **结构类型**：值可分解为多个成分（如`struct`）。 \\\\n 3. **抽象数据类型（ADT）**：数学模型及操作集合（如栈、队列的接口定义）。 \\\\n\\\\n**数据结构** \\\\n- **定义**：数据元素的集合及其间的特定关系，包含三方面： \\\\n 1. **逻辑结构**：数据间的抽象关系（线性、树形、图等）。 \\\\n 2. **存储结构**：逻辑结构在计算机中的实现（顺序存储、链式存储等）。 \\\\n 3. **数据运算**：对数据的基本操作（插入、删除、查找等）。 \\\\n- **关键点**：算法设计依赖逻辑结构，实现依赖存储结构。 \\\\n\\\\n**记忆提示**： \\\\n- 数据元素≈实体，数据项≈属性；ADT≈接口，数据结构≈接口+实现。'\"\n ]\n },\n \"execution_count\": 11,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"system.get_knowledge_point(\\\"kc0111\\\")['summry']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 9,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"chapters_knowledge_points = {}\\n\",\n \"for chapter_id, chapter_info in system.chapter_index.items():\\n\",\n \" chapters_knowledge_points[chapter_id] = chapter_info['knowledge_points']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 10,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"['kc0111', 'kc0112', 'kc0121', 'kc0122']\"\n ]\n },\n \"execution_count\": 10,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"chapters_knowledge_points[\\\"01\\\"]\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"easyDS\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.11\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n"}, "2402601edffe8e2236211b3741919e83d43bb340": {"t": 1785753746.712644, "n": 30755, "h": "74b3af19930f85b3", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 章节\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def extract_review_tips(text):\\n\",\n \" # 查找【复习提示】的起始位置\\n\",\n \" start_marker = \\\"【复习提示】\\\"\\n\",\n \" start_idx = text.find(start_marker)\\n\",\n \" \\n\",\n \" if start_idx == -1:\\n\",\n \" return None\\n\",\n \" \\n\",\n \" # 从标记后开始的文本\\n\",\n \" start_idx += len(start_marker)\\n\",\n \" \\n\",\n \" # 使用正则表达式找到下一个章节标记（如 1.1, 2.1, 3.1 等）\\n\",\n \" import re\\n\",\n \" text_after_tips = text[start_idx:]\\n\",\n \" chapter_match = re.search(r'\\\\n\\\\d+\\\\.\\\\d+', text_after_tips)\\n\",\n \" \\n\",\n \" if chapter_match:\\n\",\n \" end_idx = chapter_match.start()\\n\",\n \" review_tip = text_after_tips[:end_idx].strip()\\n\",\n \" else:\\n\",\n \" review_tip = text_after_tips.strip()\\n\",\n \" \\n\",\n \" return review_tip\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"# 使用示例\\n\",\n \"if __name__ == \\\"__main__\\\":\\n\",\n \" directory = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \" chapter_page = ['13', '24', '74', '121', '136', '210', '281', '349']\\n\",\n \" titles = ['绪论', '线性表', '栈和队列和数组', '串', '树和二叉树', '图', '查找', '排序']\\n\",\n \" chapters = []\\n\",\n \" for i, page in enumerate(chapter_page, 1):\\n\",\n \" file_path = os.path.join(directory, f'page_{page}.txt')\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" text = f.read()\\n\",\n \" review_tip = extract_review_tips(text)\\n\",\n \" if review_tip:\\n\",\n \" id = \\\"c0\\\" + str(i)\\n\",\n \" chapter = {\\n\",\n \" \\\"id\\\": id, # 章节唯一标识符\\n\",\n \" \\\"title\\\": titles[i-1], # 章节标题\\n\",\n \" \\\"parent_id\\\": \\\"\\\", # 父章节ID（可选）\\n\",\n \" \\\"order\\\": \\\"integer\\\", # 排序顺序\\n\",\n \" \\\"description\\\": review_tip, # 章节描述\\n\",\n \" \\\"knowledge_points\\\": [ # 包含的知识点ID列表\\n\",\n \" \\\"knowledge_point_id\\\"\\n\",\n \" ],\\n\",\n \" \\\"sub_chapters\\\": [ # 子章节ID列表\\n\",\n \" \\\"chapter_id\\\"\\n\",\n \" ]\\n\",\n \"}\\n\",\n \" chapters.append(chapter)\\n\",\n \" with open('/root/autodl-tmp/EasyDS/data/questions/chapters.json', 'w', encoding='utf-8') as f:\\n\",\n \" json.dump(chapters, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 知识点\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def extract_knowledge_point(text, chapter_number):\\n\",\n \" import re\\n\",\n \" \\n\",\n \" # 使用正则表达式匹配章节标题和内容\\n\",\n \" section_pattern = rf'{chapter_number}\\\\.\\\\d+\\\\.\\\\d+'\\n\",\n \" \\n\",\n \" sections = []\\n\",\n \" matches = list(re.finditer(section_pattern, text))\\n\",\n \" \\n\",\n \" for i in range(len(matches)):\\n\",\n \" current_match = matches[i]\\n\",\n \" current_section = current_match.group()\\n\",\n \" start_pos = current_match.start()\\n\",\n \" \\n\",\n \" if i < len(matches) - 1:\\n\",\n \" end_pos = matches[i + 1].start()\\n\",\n \" content = text[start_pos:end_pos].strip()\\n\",\n \" else:\\n\",\n \" content = text[start_pos:].strip()\\n\",\n \" \\n\",\n \" if \\\"本节试题精选\\\" not in content and \\\"答案与解析\\\" not in content:\\n\",\n \" lines = content.split('\\\\n')\\n\",\n \" full_title = lines[0].strip()\\n\",\n \" title = full_title.split(current_section)[-1].strip()\\n\",\n \" description = '\\\\n'.join(lines[1:]).strip()\\n\",\n \" \\n\",\n \" sections.append({\\n\",\n \" 'section': current_section,\\n\",\n \" 'title': title,\\n\",\n \" 'description': description\\n\",\n \" })\\n\",\n \" \\n\",\n \" return sections\\n\",\n \"\\n\",\n \"def process_file(file_path, chapter_number):\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" text = f.read()\\n\",\n \" \\n\",\n \" sections = extract_knowledge_point(text, chapter_number)\\n\",\n \" \\n\",\n \" # for section in sections:\\n\",\n \" # print(f\\\"标题：{section['title']}\\\")\\n\",\n \" # print(section['description'])\\n\",\n \" # print(\\\"-\\\" * 50)\\n\",\n \" return sections\\n\",\n \"\\n\",\n \"\\n\",\n \"# 使用示例\\n\",\n \"directory = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \"chapter_number = \\\"4\\\" # 要提取的章节号\\n\",\n \"knowledge_points = []\\n\",\n \"for i in range(121, 136):\\n\",\n \" file_path = os.path.join(directory, f'page_{i}.txt')\\n\",\n \" sections = process_file(file_path, chapter_number)\\n\",\n \" for section in sections:\\n\",\n \" id = section['section'].split('.')\\n\",\n \" id = ''.join(id)\\n\",\n \" knowledgepoint = {\\n\",\n \" \\\"id\\\": \\\"kc0\\\"+id, # 知识点唯一标识符\\n\",\n \" \\\"title\\\": section['title'], # 知识点标题\\n\",\n \" \\\"chapter_id\\\": \\\" c0\\\"+chapter_number, # 所属章节ID\\n\",\n \" \\\"description\\\": section['description'], # 知识点详细描述\\n\",\n \" \\\"related_points\\\": [ # 关联知识点\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\", # 关联知识点ID\\n\",\n \" \\\"relation_type\\\": \\\"string\\\" # 关系类型: prerequisite, related, extension\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"questions\\\": [ # 相关问题ID列表\\n\",\n \" \\\"question_id\\\"\\n\",\n \" ]\\n\",\n \"}\\n\",\n \" knowledge_points.append(knowledgepoint)\\n\",\n \"with open(os.path.join(\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints\\\",f'chapter_{chapter_number}.json'), 'w', encoding='utf-8') as f:\\n\",\n \" json.dump(knowledge_points, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 8,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def combine_chapter_content(directory, start_page, end_page):\\n\",\n \" \\\"\\\"\\\"合并章节内容\\\"\\\"\\\"\\n\",\n \" combined_text = \\\"\\\"\\n\",\n \" for i in range(start_page, end_page + 1):\\n\",\n \" file_path = os.path.join(directory, f'page_{i}.txt')\\n\",\n \" if os.path.exists(file_path):\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" combined_text += f.read() + \\\"\\\\n\\\"\\n\",\n \" return combined_text\\n\",\n \"\\n\",\n \"def extract_knowledge_point(text, chapter_number):\\n\",\n \" import re\\n\",\n \" \\n\",\n \" # 使用正则表达式匹配章节标题和内容\\n\",\n \" section_pattern = rf'{chapter_number}\\\\.\\\\d+\\\\.\\\\d+'\\n\",\n \" \\n\",\n \" sections = []\\n\",\n \" matches = list(re.finditer(section_pattern, text))\\n\",\n \" \\n\",\n \" for i in range(len(matches)):\\n\",\n \" current_match = matches[i]\\n\",\n \" current_section = current_match.group()\\n\",\n \" start_pos = current_match.start()\\n\",\n \" \\n\",\n \" if i < len(matches) - 1:\\n\",\n \" end_pos = matches[i + 1].start()\\n\",\n \" content = text[start_pos:end_pos].strip()\\n\",\n \" else:\\n\",\n \" content = text[start_pos:].strip()\\n\",\n \" \\n\",\n \" if \\\"本节试题精选\\\" not in content and \\\"答案与解析\\\" not in content:\\n\",\n \" lines = content.split('\\\\n')\\n\",\n \" full_title = lines[0].strip()\\n\",\n \" title = full_title.split(current_section)[-1].strip()\\n\",\n \" description = '\\\\n'.join(lines[1:]).strip()\\n\",\n \" \\n\",\n \" sections.append({\\n\",\n \" 'section': current_section,\\n\",\n \" 'title': title,\\n\",\n \" 'description': description\\n\",\n \" })\\n\",\n \" \\n\",\n \" return sections\\n\",\n \"\\n\",\n \"def process_chapter(directory, chapter_number, start_page, end_page):\\n\",\n \" # 先合并所有页面内容\\n\",\n \" combined_text = combine_chapter_content(directory, start_page, end_page)\\n\",\n \" \\n\",\n \" # 提取知识点\\n\",\n \" sections = extract_knowledge_point(combined_text, chapter_number)\\n\",\n \" \\n\",\n \" knowledge_points = []\\n\",\n \" for section in sections:\\n\",\n \" id = section['section'].split('.')\\n\",\n \" id = ''.join(id)\\n\",\n \" knowledgepoint = {\\n\",\n \" \\\"id\\\": \\\"kc0\\\"+id,\\n\",\n \" \\\"title\\\": section['title'],\\n\",\n \" \\\"chapter_id\\\": \\\"c0\\\"+chapter_number,\\n\",\n \" \\\"description\\\": section['description'],\\n\",\n \" \\\"related_points\\\": [\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\",\\n\",\n \" \\\"relation_type\\\": \\\"string\\\"\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"questions\\\": [\\\"question_id\\\"]\\n\",\n \" }\\n\",\n \" knowledge_points.append(knowledgepoint)\\n\",\n \" \\n\",\n \" # 保存结果\\n\",\n \" output_dir = \\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints\\\"\\n\",\n \" os.makedirs(output_dir, exist_ok=True)\\n\",\n \" output_path = os.path.join(output_dir, f'chapter_{chapter_number}.json')\\n\",\n \" with open(output_path, 'w', encoding='utf-8') as f:\\n\",\n \" json.dump(knowledge_points, f, ensure_ascii=False, indent=4)\\n\",\n \" \\n\",\n \" return knowledge_points\\n\",\n \"\\n\",\n \"# 使用示例\\n\",\n \"directory = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \"chapter_number = \\\"8\\\"\\n\",\n \"start_page = 349 # 章节开始页\\n\",\n \"end_page = 406 # 章节结束页\\n\",\n \"\\n\",\n \"knowledge_points = process_chapter(directory, chapter_number, start_page, end_page)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"question = {\\n\",\n \" \\\"id\\\": \\\"q023001\\\", # 问题唯一标识符\\n\",\n \" \\\"title\\\": \\\"线性表存储\\\", # 问题标题\\n\",\n \" \\\"content\\\": content, # 问题内容\\n\",\n \" \\\"difficulty\\\": \\\"integer\\\", # 难度等级 (1-5)\\n\",\n \" \\\"type\\\": \\\"concept\\\", # 问题类型 (\\\"concept\\\", \\\"calculation\\\", \\\"application\\\")\\n\",\n \" \\\"knowledge_points\\\": [ # 相关知识点ID列表\\n\",\n \" \\\"kc0221\\\",\\n\",\n \" \\\"kc0231\\\"\\n\",\n \" ],\\n\",\n \" \\\"related_questions\\\": [ # 相关问题（用于扩展）\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\", # 相关问题ID\\n\",\n \" \\\"relation_type\\\": \\\"string\\\" # 关系类型: \\\"extension\\\", \\\"application\\\", \\\"contrast\\\"\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"reference_answer\\\": { # 参考答案\\n\",\n \" \\\"content\\\": \\\"B\\\", # 答案内容\\n\",\n \" \\\"key_points\\\": [ # 关键点列表\\n\",\n \" \\\"string\\\"\\n\",\n \" ],\\n\",\n \" \\\"explanation\\\": \\\"\\\"\\\"两种存储结构适用于不同的场合，不能简单地说谁好谁坏，选项1错误。链式存储用指针表示逻辑结构，而指针的设置是任意的，因此比顺序存储结构能更方便地表示各种逻辑结构，选项\\n\",\n \"ⅡI正确。在顺序存储中，插入和删除结点需要移动大量元素，效率较低，选项IⅢI的描述刚好相反。\\n\",\n \"顺序存储结构既能随机存取又能顺序存取，而链式结构只能顺序存取，选项IV正确。\\\"\\\"\\\" # 详细解释\\n\",\n \" }\\n\",\n \"}\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 题目\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import re\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def combine_text_files(file_paths):\\n\",\n \" \\\"\\\"\\\"合并多个文本文件的内容\\\"\\\"\\\"\\n\",\n \" combined_text = \\\"\\\"\\n\",\n \" for file_path in file_paths:\\n\",\n \" if os.path.exists(file_path):\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" combined_text += f.read() + \\\"\\\\n\\\"\\n\",\n \" return combined_text\\n\",\n \"\\n\",\n \"def extract_choice_questions(text):\\n\",\n \" \\\"\\\"\\\"提取选择题部分\\\"\\\"\\\"\\n\",\n \" # 找到\\\"一、单项选择题\\\"的位置\\n\",\n \" start_index = text.find(\\\"一、单项选择题\\\")\\n\",\n \" if start_index == -1:\\n\",\n \" return \\\"\\\"\\n\",\n \" \\n\",\n \" # 从这个位置开始查找\\\"二、\\\"\\n\",\n \" text_after_start = text[start_index:]\\n\",\n \" end_index = text_after_start.find(\\\"二、\\\")\\n\",\n \" \\n\",\n \" if end_index != -1: # 找到了\\\"二、\\\"\\n\",\n \" return text_after_start[:end_index]\\n\",\n \" else:\\n\",\n \" return text_after_start\\n\",\n \"\\n\",\n \"def parse_questions(questions_text):\\n\",\n \" \\\"\\\"\\\"解析题目内容\\\"\\\"\\\"\\n\",\n \" questions = {}\\n\",\n \" current_question = []\\n\",\n \" current_number = None\\n\",\n \" \\n\",\n \" # 跳过\\\"一、单项选择题\\\"之前的内容\\n\",\n \" content = questions_text.split(\\\"一、单项选择题\\\", 1)[1]\\n\",\n \" \\n\",\n \" for line in content.split('\\\\n'):\\n\",\n \" # 匹配题号（形如 01. 02. 等）\\n\",\n \" match = re.match(r'^(\\\\d{2})\\\\.\\\\s*(.+)', line)\\n\",\n \" if match:\\n\",\n \" # 保存前一个题目\\n\",\n \" if current_number:\\n\",\n \" questions[current_number] = '\\\\n'.join(filter(None, current_question))\\n\",\n \" # 开始新题目\\n\",\n \" current_number = match.group(1)\\n\",\n \" current_question = [match.group(2)]\\n\",\n \" elif current_number and line.strip():\\n\",\n \" current_question.append(line.strip())\\n\",\n \" \\n\",\n \" # 保存最后一个题目\\n\",\n \" if current_number:\\n\",\n \" questions[current_number] = '\\\\n'.join(filter(None, current_question))\\n\",\n \" \\n\",\n \" return questions\\n\",\n \"\\n\",\n \"def parse_answers(answers_text):\\n\",\n \" \\\"\\\"\\\"解析答案内容\\\"\\\"\\\"\\n\",\n \" answers = {}\\n\",\n \" current_number = None\\n\",\n \" current_answer = None\\n\",\n \" current_explanation = []\\n\",\n \" \\n\",\n \" # 跳过\\\"一、单项选择题\\\"之前的内容\\n\",\n \" content = answers_text.split(\\\"一、单项选择题\\\", 1)[1]\\n\",\n \" \\n\",\n \" for line in content.split('\\\\n'):\\n\",\n \" # 匹配答案（形如 01.B 或 01. B）\\n\",\n \" match = re.match(r'^(\\\\d{2})\\\\.\\\\s*([A-D])\\\\s*$', line)\\n\",\n \" if match:\\n\",\n \" # 保存前一题的内容\\n\",\n \" if current_number:\\n\",\n \" answers[current_number] = {\\n\",\n \" \\\"答案\\\": current_answer,\\n\",\n \" \\\"解析\\\": '\\\\n'.join(filter(None, current_explanation))\\n\",\n \" }\\n\",\n \" # 开始新题目\\n\",\n \" current_number = match.group(1)\\n\",\n \" current_answer = match.group(2)\\n\",\n \" current_explanation = []\\n\",\n \" elif current_number and line.strip():\\n\",\n \" current_explanation.append(line.strip())\\n\",\n \" \\n\",\n \" # 保存最后一题的内容\\n\",\n \" if current_number:\\n\",\n \" answers[current_number] = {\\n\",\n \" \\\"答案\\\": current_answer,\\n\",\n \" \\\"解析\\\": '\\\\n'.join(filter(None, current_explanation))\\n\",\n \" }\\n\",\n \" \\n\",\n \" return answers\\n\",\n \"\\n\",\n \"def process_questions_and_answers(question_files, answer_files):\\n\",\n \" \\\"\\\"\\\"处理题目和答案文件\\\"\\\"\\\"\\n\",\n \" # 合并题目文件\\n\",\n \" questions_text = combine_text_files(question_files)\\n\",\n \" questions_section = extract_choice_questions(questions_text)\\n\",\n \" questions = parse_questions(questions_section)\\n\",\n \" \\n\",\n \" # 合并答案文件\\n\",\n \" answers_text = combine_text_files(answer_files)\\n\",\n \" answers_section = extract_choice_questions(answers_text)\\n\",\n \" answers = parse_answers(answers_section)\\n\",\n \" \\n\",\n \" # 合并题目和答案\\n\",\n \" result = []\\n\",\n \" for number in sorted(questions.keys()):\\n\",\n \" if number.isdigit() and 1 <= int(number) <= 36:\\n\",\n \" answer_info = answers.get(number, {\\\"答案\\\": \\\"未找到\\\", \\\"解析\\\": \\\"未找到解析\\\"})\\n\",\n \" item = {\\n\",\n \" \\\"题号\\\": number,\\n\",\n \" \\\"题目\\\": questions[number].strip(),\\n\",\n \" \\\"答案\\\": answer_info[\\\"答案\\\"],\\n\",\n \" \\\"解析\\\": answer_info[\\\"解析\\\"]\\n\",\n \" }\\n\",\n \" result.append(item)\\n\",\n \" \\n\",\n \" return result\\n\",\n \"\\n\",\n \"def final_process(id, kpid, qleft, qright, alleft, aright, output_path): \\n\",\n \" question_files = []\\n\",\n \" answer_files = []\\n\",\n \"\\n\",\n \" file = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \" id = id\\n\",\n \" kpid = kpid\\n\",\n \" for i in range(qleft,qright):\\n\",\n \" question_files.append(os.path.join(file,f\\\"page_{i}.txt\\\"))\\n\",\n \" for i in range(alleft,aright):\\n\",\n \" answer_files.append(os.path.join(file,f\\\"page_{i}.txt\\\"))\\n\",\n \"\\n\",\n \" results = process_questions_and_answers(question_files, answer_files)\\n\",\n \"\\n\",\n \" questions = []\\n\",\n \" for q in results:\\n\",\n \" question = {\\n\",\n \" \\\"id\\\": id+q[\\\"题号\\\"], # 问题唯一标识符\\n\",\n \" \\\"title\\\": \\\"title\\\", # 问题标题\\n\",\n \" \\\"content\\\": q[\\\"题目\\\"], # 问题内容\\n\",\n \" \\\"difficulty\\\": \\\"integer\\\", # 难度等级 (1-5)\\n\",\n \" \\\"type\\\": \\\"concept\\\", # 问题类型 (\\\"concept\\\", \\\"calculation\\\", \\\"application\\\")\\n\",\n \" \\\"knowledge_points\\\": kpid, # 相关知识点ID列表\\n\",\n \" \\\"related_questions\\\": [ # 相关问题（用于扩展）\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\", # 相关问题ID\\n\",\n \" \\\"relation_type\\\": \\\"string\\\" # 关系类型: \\\"extension\\\", \\\"application\\\", \\\"contrast\\\"\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"reference_answer\\\": { # 参考答案\\n\",\n \" \\\"content\\\": q[\\\"答案\\\"], # 答案内容\\n\",\n \" \\\"key_points\\\": [ # 关键点列表\\n\",\n \" \\\"string\\\"\\n\",\n \" ],\\n\",\n \" \\\"explanation\\\": q[\\\"解析\\\"] # 详细解释\\n\",\n \" }\\n\",\n \" }\\n\",\n \" questions.append(question)\\n\",\n \"\\n\",\n \" # 如果文件不存在，则创建文件\\n\",\n \" if not os.path.exists(output_path):\\n\",\n \" with open(output_path,\\\"w\\\",encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump([],f,ensure_ascii=False,indent=4)\\n\",\n \"\\n\",\n \" with open(output_path,\\\"r\\\",encoding=\\\"utf-8\\\") as f:\\n\",\n \" data = json.load(f)\\n\",\n \"\\n\",\n \" data += questions\\n\",\n \"\\n\",\n \" with open(output_path,\\\"w\\\",encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(data,f,ensure_ascii=False,indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 34,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"final_process(\\\"q0870\\\",[\\\"kc871\\\", \\\"kc872\\\", \\\"kc873\\\", \\\"kc874\\\", \\\"kc875\\\"],404, 406, 406, 410, \\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_8.json\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### 题目起名\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"load_dotenv()\\n\",\n \"from langchain_deepseek import ChatDeepSeek\\n\",\n \"from langchain_core.prompts import ChatPromptTemplate\\n\",\n \"from typing import TypedDict\\n\",\n \"\\n\",\n \"class QuestionName(TypedDict):\\n\",\n \" question_name: str\\n\",\n \"\\n\",\n \"\\n\",\n \"model = ChatDeepSeek(model=\\\"deepseek-chat\\\", api_key=os.getenv(\\\"DEEPSEEK_API_KEY\\\"))\\n\",\n \"\\n\",\n \"prompt = ChatPromptTemplate([\\n\",\n \" (\\\"system\\\", \\\"请根据以下题目内容，起一个可以概括题目内容，并且简洁的题目名称, 尽可能在15个字以内。\\\"),\\n\",\n \" (\\\"human\\\", \\\"题目内容：{content}\\\"),\\n\",\n \"])\\n\",\n \"\\n\",\n \"chain = prompt | model.with_structured_output(QuestionName)\\n\",\n \"\\n\",\n \"for i in range(8, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" data = json.load(f)\\n\",\n \" for question in data:\\n\",\n \" question_name = chain.invoke({\\\"content\\\": question[\\\"content\\\"]})\\n\",\n \" question[\\\"title\\\"] = question_name[\\\"question_name\\\"]\\n\",\n \" print(\\\"ok\\\")\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(data, f, ensure_ascii=False, indent=4)\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 题目对应章节信息\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"directory = \\\"/root/autodl-tmp/EasyDS/data/ds_data/questions\\\"\\n\",\n \"\\n\",\n \"for i in range(2, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" questions = json.load(f)\\n\",\n \" for question in questions:\\n\",\n \" question[\\\"chapter\\\"] = f\\\"c0{i}\\\"\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(questions, f, ensure_ascii=False, indent=4)\\n\",\n \" print(f\\\"第{i}章节处理完成\\\")\\n\",\n \" print(\\\"=\\\"*20)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 章节信息补充\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 4,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" chapters = json.load(f)\\n\",\n \"for chapter in chapters:\\n\",\n \" chapter[\\\"knowledge_points\\\"] = []\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(chapters, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" chapters = json.load(f)\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" all_knowledgepoints = json.load(f)\\n\",\n \"for chapter in chapters:\\n\",\n \" for kp in all_knowledgepoints:\\n\",\n \" if kp[\\\"chapter_id\\\"] == chapter[\\\"id\\\"]:\\n\",\n \" chapter[\\\"knowledge_points\\\"].append(kp[\\\"id\\\"])\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(chapters, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 补充知识点的问题列表\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 12,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"for i in range(2, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" questions = json.load(f)\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" knowledgepoints = json.load(f)\\n\",\n \" for question in questions:\\n\",\n \" for kp in question[\\\"knowledge_points\\\"]:\\n\",\n \" for kp_item in knowledgepoints:\\n\",\n \" if kp_item[\\\"id\\\"] == kp:\\n\",\n \" kp_item[\\\"questions\\\"].append(question[\\\"id\\\"])\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/chapter_{i}.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(knowledgepoints, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 对知识点进行关联\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"all_knowledgepoints = []\\n\",\n \"for i in range(1, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" knowledgepoints = json.load(f)\\n\",\n \" for kp in knowledgepoints:\\n\",\n \" kp[\\\"related_points\\\"] = []\\n\",\n \" all_knowledgepoints.append(kp)\\n\",\n \"with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(all_knowledgepoints, f, ensure_ascii=False, indent=4)\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"len(all_knowledgepoints)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from langchain_deepseek import ChatDeepSeek\\n\",\n \"from langchain_core.prompts import ChatPromptTemplate\\n\",\n \"from typing import TypedDict, Literal\\n\",\n \"import os\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"load_dotenv()\\n\",\n \"import json\\n\",\n \"\\n\",\n \"# 关系类型固定为\\\"prerequisite\\\", \\\"related\\\", \\\"extension\\\", 如果没有关系则返回None\\n\",\n \"class KnowledgePointRelation(TypedDict):\\n\",\n \" related_knowledge_points: Literal[\\\"prerequisite\\\", \\\"related\\\", \\\"extension\\\", \\\"no_relation\\\"]\\n\",\n \"\\n\",\n \"prompt = ChatPromptTemplate([\\n\",\n \" (\\\"system\\\", \\\"当前的知识点为\\\\ntitle: {title}\\\\ndescription: {description}\\\\n判断用户输入的知识点与当前知识点之间的关系，关系类型固定为\\\\\\\"prerequisite\\\\\\\", \\\\\\\"extension\\\\\\\", 如果没有很明确的前后置关系则返回\\\\\\\"no_relation\\\\\\\"。\\\"),\\n\",\n \" (\\\"human\\\", \\\"用户输入的知识点为\\\\ntitle: {user_title}\\\\ndescription: {user_description}\\\"),\\n\",\n \"])\\n\",\n \"\\n\",\n \"model = ChatDeepSeek(model=\\\"deepseek-chat\\\", api_key=os.getenv(\\\"DEEPSEEK_API_KEY\\\"))\\n\",\n \"chain = prompt | model.with_structured_output(KnowledgePointRelation)\\n\",\n \"\\n\",\n \"# with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \"# all_knowledgepoints = json.load(f)\\n\",\n \"for i in range(96, 101):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" all_knowledgepoints = json.load(f)\\n\",\n \" for j in range(i+1, 102):\\n\",\n \" relation = chain.invoke({\\\"title\\\": all_knowledgepoints[i][\\\"title\\\"], \\\"description\\\": all_knowledgepoints[i][\\\"description\\\"], \\\"user_title\\\": all_knowledgepoints[j][\\\"title\\\"], \\\"user_description\\\": all_knowledgepoints[j][\\\"description\\\"]})\\n\",\n \" if relation[\\\"related_knowledge_points\\\"] != \\\"no_relation\\\":\\n\",\n \" all_knowledgepoints[i][\\\"related_points\\\"].append({\\n\",\n \" \\\"id\\\": all_knowledgepoints[j][\\\"id\\\"],\\n\",\n \" \\\"relation_type\\\": relation[\\\"related_knowledge_points\\\"]\\n\",\n \" })\\n\",\n \" if relation[\\\"related_knowledge_points\\\"] == \\\"prerequisite\\\":\\n\",\n \" all_knowledgepoints[j][\\\"related_points\\\"].append({\\n\",\n \" \\\"id\\\": all_knowledgepoints[i][\\\"id\\\"],\\n\",\n \" \\\"relation_type\\\": \\\"extension\\\"\\n\",\n \" })\\n\",\n \" elif relation[\\\"related_knowledge_points\\\"] == \\\"extension\\\":\\n\",\n \" all_knowledgepoints[j][\\\"related_points\\\"].append({\\n\",\n \" \\\"id\\\": all_knowledgepoints[i][\\\"id\\\"],\\n\",\n \" \\\"relation_type\\\": \\\"prerequisite\\\"\\n\",\n \" })\\n\",\n \" \\n\",\n \" # print(f\\\"第{i+1}个知识点与第{j+1}个知识点处理完成\\\")\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(all_knowledgepoints, f, ensure_ascii=False, indent=4)\\n\",\n \"\\n\",\n \" print(f\\\"第{i+1}个知识点处理完成\\\")\\n\",\n \" print(\\\"=\\\"*20)\\n\",\n \"\\n\",\n \" \\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 知识点速览\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" all_knowledgepoints = json.load(f)\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 索引\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 12,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from index_builder import KnowledgeIndexSystem\\n\",\n \"\\n\",\n \"system = KnowledgeIndexSystem.load_indices('/root/autodl-tmp/EasyDS/data/ds_data/ds_indices.pkl')\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 11,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"'**数据** \\\\n- **定义**：信息的载体，可被计算机程序处理的符号集合（数、字符等），是程序加工的原料。 \\\\n\\\\n**数据元素** \\\\n- **定义**：数据的基本单位，由若干**数据项**（不可分割的最小单位）组成。 \\\\n- **示例**：学生记录（数据元素）包含学号、姓名、性别等数据项。 \\\\n\\\\n**数据对象** \\\\n- **定义**：相同性质数据元素的集合，是数据的子集。 \\\\n- **示例**：整数数据对象为集合 `N={0, ±1, ±2, ...}`。 \\\\n\\\\n**数据类型** \\\\n- **定义**：值的集合及其上定义的操作总称。 \\\\n- **分类**： \\\\n 1. **原子类型**：值不可再分（如`int`）。 \\\\n 2. **结构类型**：值可分解为多个成分（如`struct`）。 \\\\n 3. **抽象数据类型（ADT）**：数学模型及操作集合（如栈、队列的接口定义）。 \\\\n\\\\n**数据结构** \\\\n- **定义**：数据元素的集合及其间的特定关系，包含三方面： \\\\n 1. **逻辑结构**：数据间的抽象关系（线性、树形、图等）。 \\\\n 2. **存储结构**：逻辑结构在计算机中的实现（顺序存储、链式存储等）。 \\\\n 3. **数据运算**：对数据的基本操作（插入、删除、查找等）。 \\\\n- **关键点**：算法设计依赖逻辑结构，实现依赖存储结构。 \\\\n\\\\n**记忆提示**： \\\\n- 数据元素≈实体，数据项≈属性；ADT≈接口，数据结构≈接口+实现。'\"\n ]\n },\n \"execution_count\": 11,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"system.get_knowledge_point(\\\"kc0111\\\")['summry']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 9,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"chapters_knowledge_points = {}\\n\",\n \"for chapter_id, chapter_info in system.chapter_index.items():\\n\",\n \" chapters_knowledge_points[chapter_id] = chapter_info['knowledge_points']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 10,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"['kc0111', 'kc0112', 'kc0121', 'kc0122']\"\n ]\n },\n \"execution_count\": 10,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"chapters_knowledge_points[\\\"01\\\"]\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"easyDS\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.11\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n"}, "3816c7f09f34d2857e4a427613d974f39bf60cfb": {"t": 1785753748.1094363, "n": 30754, "h": "79569dcbc576a758", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 章节\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def extract_review_tips(text):\\n\",\n \" # 查找【复习提示】的起始位置\\n\",\n \" start_marker = \\\"【复习提示】\\\"\\n\",\n \" start_idx = text.find(start_marker)\\n\",\n \" \\n\",\n \" if start_idx == -1:\\n\",\n \" return None\\n\",\n \" \\n\",\n \" # 从标记后开始的文本\\n\",\n \" start_idx += len(start_marker)\\n\",\n \" \\n\",\n \" # 使用正则表达式找到下一个章节标记（如 1.1, 2.1, 3.1 等）\\n\",\n \" import re\\n\",\n \" text_after_tips = text[start_idx:]\\n\",\n \" chapter_match = re.search(r'\\\\n\\\\d+\\\\.\\\\d+', text_after_tips)\\n\",\n \" \\n\",\n \" if chapter_match:\\n\",\n \" end_idx = chapter_match.start()\\n\",\n \" review_tip = text_after_tips[:end_idx].strip()\\n\",\n \" else:\\n\",\n \" review_tip = text_after_tips.strip()\\n\",\n \" \\n\",\n \" return review_tip\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"# 使用示例\\n\",\n \"if __name__ == \\\"__main__\\\":\\n\",\n \" directory = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \" chapter_page = ['13', '24', '74', '121', '136', '210', '281', '349']\\n\",\n \" titles = ['绪论', '线性表', '栈和队列和数组', '串', '树和二叉树', '图', '查找', '排序']\\n\",\n \" chapters = []\\n\",\n \" for i, page in enumerate(chapter_page, 1):\\n\",\n \" file_path = os.path.join(directory, f'page_{page}.txt')\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" text = f.read()\\n\",\n \" review_tip = extract_review_tips(text)\\n\",\n \" if review_tip:\\n\",\n \" id = \\\"c0\\\" + str(i)\\n\",\n \" chapter = {\\n\",\n \" \\\"id\\\": id, # 章节唯一标识符\\n\",\n \" \\\"title\\\": titles[i-1], # 章节标题\\n\",\n \" \\\"parent_id\\\": \\\"\\\", # 父章节ID（可选）\\n\",\n \" \\\"order\\\": \\\"integer\\\", # 排序顺序\\n\",\n \" \\\"description\\\": review_tip, # 章节描述\\n\",\n \" \\\"knowledge_points\\\": [ # 包含的知识点ID列表\\n\",\n \" \\\"knowledge_point_id\\\"\\n\",\n \" ],\\n\",\n \" \\\"sub_chapters\\\": [ # 子章节ID列表\\n\",\n \" \\\"chapter_id\\\"\\n\",\n \" ]\\n\",\n \"}\\n\",\n \" chapters.append(chapter)\\n\",\n \" with open('/root/autodl-tmp/EasyDS/data/questions/chapters.json', 'w', encoding='utf-8') as f:\\n\",\n \" json.dump(chapters, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 知识点\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def extract_knowledge_point(text, chapter_number):\\n\",\n \" import re\\n\",\n \" \\n\",\n \" # 使用正则表达式匹配章节标题和内容\\n\",\n \" section_pattern = rf'{chapter_number}\\\\.\\\\d+\\\\.\\\\d+'\\n\",\n \" \\n\",\n \" sections = []\\n\",\n \" matches = list(re.finditer(section_pattern, text))\\n\",\n \" \\n\",\n \" for i in range(len(matches)):\\n\",\n \" current_match = matches[i]\\n\",\n \" current_section = current_match.group()\\n\",\n \" start_pos = current_match.start()\\n\",\n \" \\n\",\n \" if i < len(matches) - 1:\\n\",\n \" end_pos = matches[i + 1].start()\\n\",\n \" content = text[start_pos:end_pos].strip()\\n\",\n \" else:\\n\",\n \" content = text[start_pos:].strip()\\n\",\n \" \\n\",\n \" if \\\"本节试题精选\\\" not in content and \\\"答案与解析\\\" not in content:\\n\",\n \" lines = content.split('\\\\n')\\n\",\n \" full_title = lines[0].strip()\\n\",\n \" title = full_title.split(current_section)[-1].strip()\\n\",\n \" description = '\\\\n'.join(lines[1:]).strip()\\n\",\n \" \\n\",\n \" sections.append({\\n\",\n \" 'section': current_section,\\n\",\n \" 'title': title,\\n\",\n \" 'description': description\\n\",\n \" })\\n\",\n \" \\n\",\n \" return sections\\n\",\n \"\\n\",\n \"def process_file(file_path, chapter_number):\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" text = f.read()\\n\",\n \" \\n\",\n \" sections = extract_knowledge_point(text, chapter_number)\\n\",\n \" \\n\",\n \" # for section in sections:\\n\",\n \" # print(f\\\"标题：{section['title']}\\\")\\n\",\n \" # print(section['description'])\\n\",\n \" # print(\\\"-\\\" * 50)\\n\",\n \" return sections\\n\",\n \"\\n\",\n \"\\n\",\n \"# 使用示例\\n\",\n \"directory = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \"chapter_number = \\\"4\\\" # 要提取的章节号\\n\",\n \"knowledge_points = []\\n\",\n \"for i in range(121, 136):\\n\",\n \" file_path = os.path.join(directory, f'page_{i}.txt')\\n\",\n \" sections = process_file(file_path, chapter_number)\\n\",\n \" for section in sections:\\n\",\n \" id = section['section'].split('.')\\n\",\n \" id = ''.join(id)\\n\",\n \" knowledgepoint = {\\n\",\n \" \\\"id\\\": \\\"kc0\\\"+id, # 知识点唯一标识符\\n\",\n \" \\\"title\\\": section['title'], # 知识点标题\\n\",\n \" \\\"chapter_id\\\": \\\" c0\\\"+chapter_number, # 所属章节ID\\n\",\n \" \\\"description\\\": section['description'], # 知识点详细描述\\n\",\n \" \\\"related_points\\\": [ # 关联知识点\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\", # 关联知识点ID\\n\",\n \" \\\"relation_type\\\": \\\"string\\\" # 关系类型: prerequisite, related, extension\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"questions\\\": [ # 相关问题ID列表\\n\",\n \" \\\"question_id\\\"\\n\",\n \" ]\\n\",\n \"}\\n\",\n \" knowledge_points.append(knowledgepoint)\\n\",\n \"with open(os.path.join(\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints\\\",f'chapter_{chapter_number}.json'), 'w', encoding='utf-8') as f:\\n\",\n \" json.dump(knowledge_points, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 8,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def combine_chapter_content(directory, start_page, end_page):\\n\",\n \" \\\"\\\"\\\"合并章节内容\\\"\\\"\\\"\\n\",\n \" combined_text = \\\"\\\"\\n\",\n \" for i in range(start_page, end_page + 1):\\n\",\n \" file_path = os.path.join(directory, f'page_{i}.txt')\\n\",\n \" if os.path.exists(file_path):\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" combined_text += f.read() + \\\"\\\\n\\\"\\n\",\n \" return combined_text\\n\",\n \"\\n\",\n \"def extract_knowledge_point(text, chapter_number):\\n\",\n \" import re\\n\",\n \" \\n\",\n \" # 使用正则表达式匹配章节标题和内容\\n\",\n \" section_pattern = rf'{chapter_number}\\\\.\\\\d+\\\\.\\\\d+'\\n\",\n \" \\n\",\n \" sections = []\\n\",\n \" matches = list(re.finditer(section_pattern, text))\\n\",\n \" \\n\",\n \" for i in range(len(matches)):\\n\",\n \" current_match = matches[i]\\n\",\n \" current_section = current_match.group()\\n\",\n \" start_pos = current_match.start()\\n\",\n \" \\n\",\n \" if i < len(matches) - 1:\\n\",\n \" end_pos = matches[i + 1].start()\\n\",\n \" content = text[start_pos:end_pos].strip()\\n\",\n \" else:\\n\",\n \" content = text[start_pos:].strip()\\n\",\n \" \\n\",\n \" if \\\"本节试题精选\\\" not in content and \\\"答案与解析\\\" not in content:\\n\",\n \" lines = content.split('\\\\n')\\n\",\n \" full_title = lines[0].strip()\\n\",\n \" title = full_title.split(current_section)[-1].strip()\\n\",\n \" description = '\\\\n'.join(lines[1:]).strip()\\n\",\n \" \\n\",\n \" sections.append({\\n\",\n \" 'section': current_section,\\n\",\n \" 'title': title,\\n\",\n \" 'description': description\\n\",\n \" })\\n\",\n \" \\n\",\n \" return sections\\n\",\n \"\\n\",\n \"def process_chapter(directory, chapter_number, start_page, end_page):\\n\",\n \" # 先合并所有页面内容\\n\",\n \" combined_text = combine_chapter_content(directory, start_page, end_page)\\n\",\n \" \\n\",\n \" # 提取知识点\\n\",\n \" sections = extract_knowledge_point(combined_text, chapter_number)\\n\",\n \" \\n\",\n \" knowledge_points = []\\n\",\n \" for section in sections:\\n\",\n \" id = section['section'].split('.')\\n\",\n \" id = ''.join(id)\\n\",\n \" knowledgepoint = {\\n\",\n \" \\\"id\\\": \\\"kc0\\\"+id,\\n\",\n \" \\\"title\\\": section['title'],\\n\",\n \" \\\"chapter_id\\\": \\\"c0\\\"+chapter_number,\\n\",\n \" \\\"description\\\": section['description'],\\n\",\n \" \\\"related_points\\\": [\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\",\\n\",\n \" \\\"relation_type\\\": \\\"string\\\"\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"questions\\\": [\\\"question_id\\\"]\\n\",\n \" }\\n\",\n \" knowledge_points.append(knowledgepoint)\\n\",\n \" \\n\",\n \" # 保存结果\\n\",\n \" output_dir = \\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints\\\"\\n\",\n \" os.makedirs(output_dir, exist_ok=True)\\n\",\n \" output_path = os.path.join(output_dir, f'chapter_{chapter_number}.json')\\n\",\n \" with open(output_path, 'w', encoding='utf-8') as f:\\n\",\n \" json.dump(knowledge_points, f, ensure_ascii=False, indent=4)\\n\",\n \" \\n\",\n \" return knowledge_points\\n\",\n \"\\n\",\n \"# 使用示例\\n\",\n \"directory = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \"chapter_number = \\\"8\\\"\\n\",\n \"start_page = 349 # 章节开始页\\n\",\n \"end_page = 406 # 章节结束页\\n\",\n \"\\n\",\n \"knowledge_points = process_chapter(directory, chapter_number, start_page, end_page)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"question = {\\n\",\n \" \\\"id\\\": \\\"q023001\\\", # 问题唯一标识符\\n\",\n \" \\\"title\\\": \\\"线性表存储\\\", # 问题标题\\n\",\n \" \\\"content\\\": content, # 问题内容\\n\",\n \" \\\"difficulty\\\": \\\"integer\\\", # 难度等级 (1-5)\\n\",\n \" \\\"type\\\": \\\"concept\\\", # 问题类型 (\\\"concept\\\", \\\"calculation\\\", \\\"application\\\")\\n\",\n \" \\\"knowledge_points\\\": [ # 相关知识点ID列表\\n\",\n \" \\\"kc0221\\\",\\n\",\n \" \\\"kc0231\\\"\\n\",\n \" ],\\n\",\n \" \\\"related_questions\\\": [ # 相关问题（用于扩展）\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\", # 相关问题ID\\n\",\n \" \\\"relation_type\\\": \\\"string\\\" # 关系类型: \\\"extension\\\", \\\"application\\\", \\\"contrast\\\"\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"reference_answer\\\": { # 参考答案\\n\",\n \" \\\"content\\\": \\\"B\\\", # 答案内容\\n\",\n \" \\\"key_points\\\": [ # 关键点列表\\n\",\n \" \\\"string\\\"\\n\",\n \" ],\\n\",\n \" \\\"explanation\\\": \\\"\\\"\\\"两种存储结构适用于不同的场合，不能简单地说谁好谁坏，选项1错误。链式存储用指针表示逻辑结构，而指针的设置是任意的，因此比顺序存储结构能更方便地表示各种逻辑结构，选项\\n\",\n \"ⅡI正确。在顺序存储中，插入和删除结点需要移动大量元素，效率较低，选项IⅢI的描述刚好相反。\\n\",\n \"顺序存储结构既能随机存取又能顺序存取，而链式结构只能顺序存取，选项IV正确。\\\"\\\"\\\" # 详细解释\\n\",\n \" }\\n\",\n \"}\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 题目\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import re\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def combine_text_files(file_paths):\\n\",\n \" \\\"\\\"\\\"合并多个文本文件的内容\\\"\\\"\\\"\\n\",\n \" combined_text = \\\"\\\"\\n\",\n \" for file_path in file_paths:\\n\",\n \" if os.path.exists(file_path):\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" combined_text += f.read() + \\\"\\\\n\\\"\\n\",\n \" return combined_text\\n\",\n \"\\n\",\n \"def extract_choice_questions(text):\\n\",\n \" \\\"\\\"\\\"提取选择题部分\\\"\\\"\\\"\\n\",\n \" # 找到\\\"一、单项选择题\\\"的位置\\n\",\n \" start_index = text.find(\\\"一、单项选择题\\\")\\n\",\n \" if start_index == -1:\\n\",\n \" return \\\"\\\"\\n\",\n \" \\n\",\n \" # 从这个位置开始查找\\\"二、\\\"\\n\",\n \" text_after_start = text[start_index:]\\n\",\n \" end_index = text_after_start.find(\\\"二、\\\")\\n\",\n \" \\n\",\n \" if end_index != -1: # 找到了\\\"二、\\\"\\n\",\n \" return text_after_start[:end_index]\\n\",\n \" else:\\n\",\n \" return text_after_start\\n\",\n \"\\n\",\n \"def parse_questions(questions_text):\\n\",\n \" \\\"\\\"\\\"解析题目内容\\\"\\\"\\\"\\n\",\n \" questions = {}\\n\",\n \" current_question = []\\n\",\n \" current_number = None\\n\",\n \" \\n\",\n \" # 跳过\\\"一、单项选择题\\\"之前的内容\\n\",\n \" content = questions_text.split(\\\"一、单项选择题\\\", 1)[1]\\n\",\n \" \\n\",\n \" for line in content.split('\\\\n'):\\n\",\n \" # 匹配题号（形如 01. 02. 等）\\n\",\n \" match = re.match(r'^(\\\\d{2})\\\\.\\\\s*(.+)', line)\\n\",\n \" if match:\\n\",\n \" # 保存前一个题目\\n\",\n \" if current_number:\\n\",\n \" questions[current_number] = '\\\\n'.join(filter(None, current_question))\\n\",\n \" # 开始新题目\\n\",\n \" current_number = match.group(1)\\n\",\n \" current_question = [match.group(2)]\\n\",\n \" elif current_number and line.strip():\\n\",\n \" current_question.append(line.strip())\\n\",\n \" \\n\",\n \" # 保存最后一个题目\\n\",\n \" if current_number:\\n\",\n \" questions[current_number] = '\\\\n'.join(filter(None, current_question))\\n\",\n \" \\n\",\n \" return questions\\n\",\n \"\\n\",\n \"def parse_answers(answers_text):\\n\",\n \" \\\"\\\"\\\"解析答案内容\\\"\\\"\\\"\\n\",\n \" answers = {}\\n\",\n \" current_number = None\\n\",\n \" current_answer = None\\n\",\n \" current_explanation = []\\n\",\n \" \\n\",\n \" # 跳过\\\"一、单项选择题\\\"之前的内容\\n\",\n \" content = answers_text.split(\\\"一、单项选择题\\\", 1)[1]\\n\",\n \" \\n\",\n \" for line in content.split('\\\\n'):\\n\",\n \" # 匹配答案（形如 01.B 或 01. B）\\n\",\n \" match = re.match(r'^(\\\\d{2})\\\\.\\\\s*([A-D])\\\\s*$', line)\\n\",\n \" if match:\\n\",\n \" # 保存前一题的内容\\n\",\n \" if current_number:\\n\",\n \" answers[current_number] = {\\n\",\n \" \\\"答案\\\": current_answer,\\n\",\n \" \\\"解析\\\": '\\\\n'.join(filter(None, current_explanation))\\n\",\n \" }\\n\",\n \" # 开始新题目\\n\",\n \" current_number = match.group(1)\\n\",\n \" current_answer = match.group(2)\\n\",\n \" current_explanation = []\\n\",\n \" elif current_number and line.strip():\\n\",\n \" current_explanation.append(line.strip())\\n\",\n \" \\n\",\n \" # 保存最后一题的内容\\n\",\n \" if current_number:\\n\",\n \" answers[current_number] = {\\n\",\n \" \\\"答案\\\": current_answer,\\n\",\n \" \\\"解析\\\": '\\\\n'.join(filter(None, current_explanation))\\n\",\n \" }\\n\",\n \" \\n\",\n \" return answers\\n\",\n \"\\n\",\n \"def process_questions_and_answers(question_files, answer_files):\\n\",\n \" \\\"\\\"\\\"处理题目和答案文件\\\"\\\"\\\"\\n\",\n \" # 合并题目文件\\n\",\n \" questions_text = combine_text_files(question_files)\\n\",\n \" questions_section = extract_choice_questions(questions_text)\\n\",\n \" questions = parse_questions(questions_section)\\n\",\n \" \\n\",\n \" # 合并答案文件\\n\",\n \" answers_text = combine_text_files(answer_files)\\n\",\n \" answers_section = extract_choice_questions(answers_text)\\n\",\n \" answers = parse_answers(answers_section)\\n\",\n \" \\n\",\n \" # 合并题目和答案\\n\",\n \" result = []\\n\",\n \" for number in sorted(questions.keys()):\\n\",\n \" if number.isdigit() and 1 <= int(number) <= 36:\\n\",\n \" answer_info = answers.get(number, {\\\"答案\\\": \\\"未找到\\\", \\\"解析\\\": \\\"未找到解析\\\"})\\n\",\n \" item = {\\n\",\n \" \\\"题号\\\": number,\\n\",\n \" \\\"题目\\\": questions[number].strip(),\\n\",\n \" \\\"答案\\\": answer_info[\\\"答案\\\"],\\n\",\n \" \\\"解析\\\": answer_info[\\\"解析\\\"]\\n\",\n \" }\\n\",\n \" result.append(item)\\n\",\n \" \\n\",\n \" return result\\n\",\n \"\\n\",\n \"def final_process(id, kpid, qleft, qright, alleft, aright, output_path): \\n\",\n \" question_files = []\\n\",\n \" answer_files = []\\n\",\n \"\\n\",\n \" file = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \" id = id\\n\",\n \" kpid = kpid\\n\",\n \" for i in range(qleft,qright):\\n\",\n \" question_files.append(os.path.join(file,f\\\"page_{i}.txt\\\"))\\n\",\n \" for i in range(alleft,aright):\\n\",\n \" answer_files.append(os.path.join(file,f\\\"page_{i}.txt\\\"))\\n\",\n \"\\n\",\n \" results = process_questions_and_answers(question_files, answer_files)\\n\",\n \"\\n\",\n \" questions = []\\n\",\n \" for q in results:\\n\",\n \" question = {\\n\",\n \" \\\"id\\\": id+q[\\\"题号\\\"], # 问题唯一标识符\\n\",\n \" \\\"title\\\": \\\"title\\\", # 问题标题\\n\",\n \" \\\"content\\\": q[\\\"题目\\\"], # 问题内容\\n\",\n \" \\\"difficulty\\\": \\\"integer\\\", # 难度等级 (1-5)\\n\",\n \" \\\"type\\\": \\\"concept\\\", # 问题类型 (\\\"concept\\\", \\\"calculation\\\", \\\"application\\\")\\n\",\n \" \\\"knowledge_points\\\": kpid, # 相关知识点ID列表\\n\",\n \" \\\"related_questions\\\": [ # 相关问题（用于扩展）\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\", # 相关问题ID\\n\",\n \" \\\"relation_type\\\": \\\"string\\\" # 关系类型: \\\"extension\\\", \\\"application\\\", \\\"contrast\\\"\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"reference_answer\\\": { # 参考答案\\n\",\n \" \\\"content\\\": q[\\\"答案\\\"], # 答案内容\\n\",\n \" \\\"key_points\\\": [ # 关键点列表\\n\",\n \" \\\"string\\\"\\n\",\n \" ],\\n\",\n \" \\\"explanation\\\": q[\\\"解析\\\"] # 详细解释\\n\",\n \" }\\n\",\n \" }\\n\",\n \" questions.append(question)\\n\",\n \"\\n\",\n \" # 如果文件不存在，则创建文件\\n\",\n \" if not os.path.exists(output_path):\\n\",\n \" with open(output_path,\\\"w\\\",encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump([],f,ensure_ascii=False,indent=4)\\n\",\n \"\\n\",\n \" with open(output_path,\\\"r\\\",encoding=\\\"utf-8\\\") as f:\\n\",\n \" data = json.load(f)\\n\",\n \"\\n\",\n \" data += questions\\n\",\n \"\\n\",\n \" with open(output_path,\\\"w\\\",encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(data,f,ensure_ascii=False,indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 34,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"final_process(\\\"q0870\\\",[\\\"kc871\\\", \\\"kc872\\\", \\\"kc873\\\", \\\"kc874\\\", \\\"kc875\\\"],404, 406, 406, 410, \\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_8.json\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### 题目起名\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"load_dotenv()\\n\",\n \"from langchain_deepseek import ChatDeepSeek\\n\",\n \"from langchain_core.prompts import ChatPromptTemplate\\n\",\n \"from typing import TypedDict\\n\",\n \"\\n\",\n \"class QuestionName(TypedDict):\\n\",\n \" question_name: str\\n\",\n \"\\n\",\n \"\\n\",\n \"model = ChatDeepSeek(model=\\\"deepseek-chat\\\", api_key=os.getenv(\\\"DEEPSEEK_API_KEY\\\"))\\n\",\n \"\\n\",\n \"prompt = ChatPromptTemplate([\\n\",\n \" (\\\"system\\\", \\\"请根据以下题目内容，起一个可以概括题目内容，并且简洁的题目名称, 尽可能在15个字以内。\\\"),\\n\",\n \" (\\\"human\\\", \\\"题目内容：{content}\\\"),\\n\",\n \"])\\n\",\n \"\\n\",\n \"chain = prompt | model.with_structured_output(QuestionName)\\n\",\n \"\\n\",\n \"for i in range(8, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" data = json.load(f)\\n\",\n \" for question in data:\\n\",\n \" question_name = chain.invoke({\\\"content\\\": question[\\\"content\\\"]})\\n\",\n \" question[\\\"title\\\"] = question_name[\\\"question_name\\\"]\\n\",\n \" print(\\\"ok\\\")\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(data, f, ensure_ascii=False, indent=4)\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 题目对应章节信息\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"directory = \\\"/root/autodl-tmp/EasyDS/data/ds_data/questions\\\"\\n\",\n \"\\n\",\n \"for i in range(2, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" questions = json.load(f)\\n\",\n \" for question in questions:\\n\",\n \" question[\\\"chapter\\\"] = f\\\"c0{i}\\\"\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(questions, f, ensure_ascii=False, indent=4)\\n\",\n \" print(f\\\"第{i}章节处理完成\\\")\\n\",\n \" print(\\\"=\\\"*20)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 章节信息补充\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 4,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" chapters = json.load(f)\\n\",\n \"for chapter in chapters:\\n\",\n \" chapter[\\\"knowledge_points\\\"] = []\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(chapters, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" chapters = json.load(f)\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" all_knowledgepoints = json.load(f)\\n\",\n \"for chapter in chapters:\\n\",\n \" for kp in all_knowledgepoints:\\n\",\n \" if kp[\\\"chapter_id\\\"] == chapter[\\\"id\\\"]:\\n\",\n \" chapter[\\\"knowledge_points\\\"].append(kp[\\\"id\\\"])\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(chapters, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 补充知识点的问题列表\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 12,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"for i in range(2, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" questions = json.load(f)\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" knowledgepoints = json.load(f)\\n\",\n \" for question in questions:\\n\",\n \" for kp in question[\\\"knowledge_points\\\"]:\\n\",\n \" for kp_item in knowledgepoints:\\n\",\n \" if kp_item[\\\"id\\\"] == kp:\\n\",\n \" kp_item[\\\"questions\\\"].append(question[\\\"id\\\"])\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/chapter_{i}.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(knowledgepoints, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 对知识点进行关联\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"all_knowledgepoints = []\\n\",\n \"for i in range(1, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" knowledgepoints = json.load(f)\\n\",\n \" for kp in knowledgepoints:\\n\",\n \" kp[\\\"related_points\\\"] = []\\n\",\n \" all_knowledgepoints.append(kp)\\n\",\n \"with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(all_knowledgepoints, f, ensure_ascii=False, indent=4)\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"len(all_knowledgepoints)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from langchain_deepseek import ChatDeepSeek\\n\",\n \"from langchain_core.prompts import ChatPromptTemplate\\n\",\n \"from typing import TypedDict, Literal\\n\",\n \"import os\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"load_dotenv()\\n\",\n \"import json\\n\",\n \"\\n\",\n \"# 关系类型固定为\\\"prerequisite\\\", \\\"related\\\", \\\"extension\\\", 如果没有关系则返回None\\n\",\n \"class KnowledgePointRelation(TypedDict):\\n\",\n \" related_knowledge_points: Literal[\\\"prerequisite\\\", \\\"related\\\", \\\"extension\\\", \\\"no_relation\\\"]\\n\",\n \"\\n\",\n \"prompt = ChatPromptTemplate([\\n\",\n \" (\\\"system\\\", \\\"当前的知识点为\\\\ntitle: {title}\\\\ndescription: {description}\\\\n判断用户输入的知识点与当前知识点之间的关系，关系类型固定为\\\\\\\"prerequisite\\\\\\\", \\\\\\\"extension\\\\\\\", 如果没有很明确的前后置关系则返回\\\\\\\"no_relation\\\\\\\"。\\\"),\\n\",\n \" (\\\"human\\\", \\\"用户输入的知识点为\\\\ntitle: {user_title}\\\\ndescription: {user_description}\\\"),\\n\",\n \"])\\n\",\n \"\\n\",\n \"model = ChatDeepSeek(model=\\\"deepseek-chat\\\", api_key=os.getenv(\\\"DEEPSEEK_API_KEY\\\"))\\n\",\n \"chain = prompt | model.with_structured_output(KnowledgePointRelation)\\n\",\n \"\\n\",\n \"# with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \"# all_knowledgepoints = json.load(f)\\n\",\n \"for i in range(96, 101):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" all_knowledgepoints = json.load(f)\\n\",\n \" for j in range(i+1, 102):\\n\",\n \" relation = chain.invoke({\\\"title\\\": all_knowledgepoints[i][\\\"title\\\"], \\\"description\\\": all_knowledgepoints[i][\\\"description\\\"], \\\"user_title\\\": all_knowledgepoints[j][\\\"title\\\"], \\\"user_description\\\": all_knowledgepoints[j][\\\"description\\\"]})\\n\",\n \" if relation[\\\"related_knowledge_points\\\"] != \\\"no_relation\\\":\\n\",\n \" all_knowledgepoints[i][\\\"related_points\\\"].append({\\n\",\n \" \\\"id\\\": all_knowledgepoints[j][\\\"id\\\"],\\n\",\n \" \\\"relation_type\\\": relation[\\\"related_knowledge_points\\\"]\\n\",\n \" })\\n\",\n \" if relation[\\\"related_knowledge_points\\\"] == \\\"prerequisite\\\":\\n\",\n \" all_knowledgepoints[j][\\\"related_points\\\"].append({\\n\",\n \" \\\"id\\\": all_knowledgepoints[i][\\\"id\\\"],\\n\",\n \" \\\"relation_type\\\": \\\"extension\\\"\\n\",\n \" })\\n\",\n \" elif relation[\\\"related_knowledge_points\\\"] == \\\"extension\\\":\\n\",\n \" all_knowledgepoints[j][\\\"related_points\\\"].append({\\n\",\n \" \\\"id\\\": all_knowledgepoints[i][\\\"id\\\"],\\n\",\n \" \\\"relation_type\\\": \\\"prerequisite\\\"\\n\",\n \" })\\n\",\n \" \\n\",\n \" # print(f\\\"第{i+1}个知识点与第{j+1}个知识点处理完成\\\")\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(all_knowledgepoints, f, ensure_ascii=False, indent=4)\\n\",\n \"\\n\",\n \" print(f\\\"第{i+1}个知识点处理完成\\\")\\n\",\n \" print(\\\"=\\\"*20)\\n\",\n \"\\n\",\n \" \\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 知识点速览\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" all_knowledgepoints = json.load(f)\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 索引\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from index_builder import KnowledgeIndexSystem\\n\",\n \"\\n\",\n \"system = KnowledgeIndexSystem.load_indices('/root/autodl-tmp/EasyDS/data/ds_data/ds_indices.pkl')\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 11,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"'**数据** \\\\n- **定义**：信息的载体，可被计算机程序处理的符号集合（数、字符等），是程序加工的原料。 \\\\n\\\\n**数据元素** \\\\n- **定义**：数据的基本单位，由若干**数据项**（不可分割的最小单位）组成。 \\\\n- **示例**：学生记录（数据元素）包含学号、姓名、性别等数据项。 \\\\n\\\\n**数据对象** \\\\n- **定义**：相同性质数据元素的集合，是数据的子集。 \\\\n- **示例**：整数数据对象为集合 `N={0, ±1, ±2, ...}`。 \\\\n\\\\n**数据类型** \\\\n- **定义**：值的集合及其上定义的操作总称。 \\\\n- **分类**： \\\\n 1. **原子类型**：值不可再分（如`int`）。 \\\\n 2. **结构类型**：值可分解为多个成分（如`struct`）。 \\\\n 3. **抽象数据类型（ADT）**：数学模型及操作集合（如栈、队列的接口定义）。 \\\\n\\\\n**数据结构** \\\\n- **定义**：数据元素的集合及其间的特定关系，包含三方面： \\\\n 1. **逻辑结构**：数据间的抽象关系（线性、树形、图等）。 \\\\n 2. **存储结构**：逻辑结构在计算机中的实现（顺序存储、链式存储等）。 \\\\n 3. **数据运算**：对数据的基本操作（插入、删除、查找等）。 \\\\n- **关键点**：算法设计依赖逻辑结构，实现依赖存储结构。 \\\\n\\\\n**记忆提示**： \\\\n- 数据元素≈实体，数据项≈属性；ADT≈接口，数据结构≈接口+实现。'\"\n ]\n },\n \"execution_count\": 11,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"system.get_knowledge_point(\\\"kc0111\\\")['summry']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 9,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"chapters_knowledge_points = {}\\n\",\n \"for chapter_id, chapter_info in system.chapter_index.items():\\n\",\n \" chapters_knowledge_points[chapter_id] = chapter_info['knowledge_points']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 10,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"['kc0111', 'kc0112', 'kc0121', 'kc0122']\"\n ]\n },\n \"execution_count\": 10,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"chapters_knowledge_points[\\\"01\\\"]\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"easyDS\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.11\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n"}, "8aa1b65e689429fbf4b18fa778ac80315c69f437": {"t": 1785753751.0150843, "n": 30573, "h": "f1e2c6d535318089", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 章节\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def extract_review_tips(text):\\n\",\n \" # 查找【复习提示】的起始位置\\n\",\n \" start_marker = \\\"【复习提示】\\\"\\n\",\n \" start_idx = text.find(start_marker)\\n\",\n \" \\n\",\n \" if start_idx == -1:\\n\",\n \" return None\\n\",\n \" \\n\",\n \" # 从标记后开始的文本\\n\",\n \" start_idx += len(start_marker)\\n\",\n \" \\n\",\n \" # 使用正则表达式找到下一个章节标记（如 1.1, 2.1, 3.1 等）\\n\",\n \" import re\\n\",\n \" text_after_tips = text[start_idx:]\\n\",\n \" chapter_match = re.search(r'\\\\n\\\\d+\\\\.\\\\d+', text_after_tips)\\n\",\n \" \\n\",\n \" if chapter_match:\\n\",\n \" end_idx = chapter_match.start()\\n\",\n \" review_tip = text_after_tips[:end_idx].strip()\\n\",\n \" else:\\n\",\n \" review_tip = text_after_tips.strip()\\n\",\n \" \\n\",\n \" return review_tip\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"# 使用示例\\n\",\n \"if __name__ == \\\"__main__\\\":\\n\",\n \" directory = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \" chapter_page = ['13', '24', '74', '121', '136', '210', '281', '349']\\n\",\n \" titles = ['绪论', '线性表', '栈和队列和数组', '串', '树和二叉树', '图', '查找', '排序']\\n\",\n \" chapters = []\\n\",\n \" for i, page in enumerate(chapter_page, 1):\\n\",\n \" file_path = os.path.join(directory, f'page_{page}.txt')\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" text = f.read()\\n\",\n \" review_tip = extract_review_tips(text)\\n\",\n \" if review_tip:\\n\",\n \" id = \\\"c0\\\" + str(i)\\n\",\n \" chapter = {\\n\",\n \" \\\"id\\\": id, # 章节唯一标识符\\n\",\n \" \\\"title\\\": titles[i-1], # 章节标题\\n\",\n \" \\\"parent_id\\\": \\\"\\\", # 父章节ID（可选）\\n\",\n \" \\\"order\\\": \\\"integer\\\", # 排序顺序\\n\",\n \" \\\"description\\\": review_tip, # 章节描述\\n\",\n \" \\\"knowledge_points\\\": [ # 包含的知识点ID列表\\n\",\n \" \\\"knowledge_point_id\\\"\\n\",\n \" ],\\n\",\n \" \\\"sub_chapters\\\": [ # 子章节ID列表\\n\",\n \" \\\"chapter_id\\\"\\n\",\n \" ]\\n\",\n \"}\\n\",\n \" chapters.append(chapter)\\n\",\n \" with open('/root/autodl-tmp/EasyDS/data/questions/chapters.json', 'w', encoding='utf-8') as f:\\n\",\n \" json.dump(chapters, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 知识点\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def extract_knowledge_point(text, chapter_number):\\n\",\n \" import re\\n\",\n \" \\n\",\n \" # 使用正则表达式匹配章节标题和内容\\n\",\n \" section_pattern = rf'{chapter_number}\\\\.\\\\d+\\\\.\\\\d+'\\n\",\n \" \\n\",\n \" sections = []\\n\",\n \" matches = list(re.finditer(section_pattern, text))\\n\",\n \" \\n\",\n \" for i in range(len(matches)):\\n\",\n \" current_match = matches[i]\\n\",\n \" current_section = current_match.group()\\n\",\n \" start_pos = current_match.start()\\n\",\n \" \\n\",\n \" if i < len(matches) - 1:\\n\",\n \" end_pos = matches[i + 1].start()\\n\",\n \" content = text[start_pos:end_pos].strip()\\n\",\n \" else:\\n\",\n \" content = text[start_pos:].strip()\\n\",\n \" \\n\",\n \" if \\\"本节试题精选\\\" not in content and \\\"答案与解析\\\" not in content:\\n\",\n \" lines = content.split('\\\\n')\\n\",\n \" full_title = lines[0].strip()\\n\",\n \" title = full_title.split(current_section)[-1].strip()\\n\",\n \" description = '\\\\n'.join(lines[1:]).strip()\\n\",\n \" \\n\",\n \" sections.append({\\n\",\n \" 'section': current_section,\\n\",\n \" 'title': title,\\n\",\n \" 'description': description\\n\",\n \" })\\n\",\n \" \\n\",\n \" return sections\\n\",\n \"\\n\",\n \"def process_file(file_path, chapter_number):\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" text = f.read()\\n\",\n \" \\n\",\n \" sections = extract_knowledge_point(text, chapter_number)\\n\",\n \" \\n\",\n \" # for section in sections:\\n\",\n \" # print(f\\\"标题：{section['title']}\\\")\\n\",\n \" # print(section['description'])\\n\",\n \" # print(\\\"-\\\" * 50)\\n\",\n \" return sections\\n\",\n \"\\n\",\n \"\\n\",\n \"# 使用示例\\n\",\n \"directory = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \"chapter_number = \\\"4\\\" # 要提取的章节号\\n\",\n \"knowledge_points = []\\n\",\n \"for i in range(121, 136):\\n\",\n \" file_path = os.path.join(directory, f'page_{i}.txt')\\n\",\n \" sections = process_file(file_path, chapter_number)\\n\",\n \" for section in sections:\\n\",\n \" id = section['section'].split('.')\\n\",\n \" id = ''.join(id)\\n\",\n \" knowledgepoint = {\\n\",\n \" \\\"id\\\": \\\"kc0\\\"+id, # 知识点唯一标识符\\n\",\n \" \\\"title\\\": section['title'], # 知识点标题\\n\",\n \" \\\"chapter_id\\\": \\\" c0\\\"+chapter_number, # 所属章节ID\\n\",\n \" \\\"description\\\": section['description'], # 知识点详细描述\\n\",\n \" \\\"related_points\\\": [ # 关联知识点\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\", # 关联知识点ID\\n\",\n \" \\\"relation_type\\\": \\\"string\\\" # 关系类型: prerequisite, related, extension\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"questions\\\": [ # 相关问题ID列表\\n\",\n \" \\\"question_id\\\"\\n\",\n \" ]\\n\",\n \"}\\n\",\n \" knowledge_points.append(knowledgepoint)\\n\",\n \"with open(os.path.join(\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints\\\",f'chapter_{chapter_number}.json'), 'w', encoding='utf-8') as f:\\n\",\n \" json.dump(knowledge_points, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 8,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def combine_chapter_content(directory, start_page, end_page):\\n\",\n \" \\\"\\\"\\\"合并章节内容\\\"\\\"\\\"\\n\",\n \" combined_text = \\\"\\\"\\n\",\n \" for i in range(start_page, end_page + 1):\\n\",\n \" file_path = os.path.join(directory, f'page_{i}.txt')\\n\",\n \" if os.path.exists(file_path):\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" combined_text += f.read() + \\\"\\\\n\\\"\\n\",\n \" return combined_text\\n\",\n \"\\n\",\n \"def extract_knowledge_point(text, chapter_number):\\n\",\n \" import re\\n\",\n \" \\n\",\n \" # 使用正则表达式匹配章节标题和内容\\n\",\n \" section_pattern = rf'{chapter_number}\\\\.\\\\d+\\\\.\\\\d+'\\n\",\n \" \\n\",\n \" sections = []\\n\",\n \" matches = list(re.finditer(section_pattern, text))\\n\",\n \" \\n\",\n \" for i in range(len(matches)):\\n\",\n \" current_match = matches[i]\\n\",\n \" current_section = current_match.group()\\n\",\n \" start_pos = current_match.start()\\n\",\n \" \\n\",\n \" if i < len(matches) - 1:\\n\",\n \" end_pos = matches[i + 1].start()\\n\",\n \" content = text[start_pos:end_pos].strip()\\n\",\n \" else:\\n\",\n \" content = text[start_pos:].strip()\\n\",\n \" \\n\",\n \" if \\\"本节试题精选\\\" not in content and \\\"答案与解析\\\" not in content:\\n\",\n \" lines = content.split('\\\\n')\\n\",\n \" full_title = lines[0].strip()\\n\",\n \" title = full_title.split(current_section)[-1].strip()\\n\",\n \" description = '\\\\n'.join(lines[1:]).strip()\\n\",\n \" \\n\",\n \" sections.append({\\n\",\n \" 'section': current_section,\\n\",\n \" 'title': title,\\n\",\n \" 'description': description\\n\",\n \" })\\n\",\n \" \\n\",\n \" return sections\\n\",\n \"\\n\",\n \"def process_chapter(directory, chapter_number, start_page, end_page):\\n\",\n \" # 先合并所有页面内容\\n\",\n \" combined_text = combine_chapter_content(directory, start_page, end_page)\\n\",\n \" \\n\",\n \" # 提取知识点\\n\",\n \" sections = extract_knowledge_point(combined_text, chapter_number)\\n\",\n \" \\n\",\n \" knowledge_points = []\\n\",\n \" for section in sections:\\n\",\n \" id = section['section'].split('.')\\n\",\n \" id = ''.join(id)\\n\",\n \" knowledgepoint = {\\n\",\n \" \\\"id\\\": \\\"kc0\\\"+id,\\n\",\n \" \\\"title\\\": section['title'],\\n\",\n \" \\\"chapter_id\\\": \\\"c0\\\"+chapter_number,\\n\",\n \" \\\"description\\\": section['description'],\\n\",\n \" \\\"related_points\\\": [\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\",\\n\",\n \" \\\"relation_type\\\": \\\"string\\\"\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"questions\\\": [\\\"question_id\\\"]\\n\",\n \" }\\n\",\n \" knowledge_points.append(knowledgepoint)\\n\",\n \" \\n\",\n \" # 保存结果\\n\",\n \" output_dir = \\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints\\\"\\n\",\n \" os.makedirs(output_dir, exist_ok=True)\\n\",\n \" output_path = os.path.join(output_dir, f'chapter_{chapter_number}.json')\\n\",\n \" with open(output_path, 'w', encoding='utf-8') as f:\\n\",\n \" json.dump(knowledge_points, f, ensure_ascii=False, indent=4)\\n\",\n \" \\n\",\n \" return knowledge_points\\n\",\n \"\\n\",\n \"# 使用示例\\n\",\n \"directory = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \"chapter_number = \\\"8\\\"\\n\",\n \"start_page = 349 # 章节开始页\\n\",\n \"end_page = 406 # 章节结束页\\n\",\n \"\\n\",\n \"knowledge_points = process_chapter(directory, chapter_number, start_page, end_page)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"question = {\\n\",\n \" \\\"id\\\": \\\"q023001\\\", # 问题唯一标识符\\n\",\n \" \\\"title\\\": \\\"线性表存储\\\", # 问题标题\\n\",\n \" \\\"content\\\": content, # 问题内容\\n\",\n \" \\\"difficulty\\\": \\\"integer\\\", # 难度等级 (1-5)\\n\",\n \" \\\"type\\\": \\\"concept\\\", # 问题类型 (\\\"concept\\\", \\\"calculation\\\", \\\"application\\\")\\n\",\n \" \\\"knowledge_points\\\": [ # 相关知识点ID列表\\n\",\n \" \\\"kc0221\\\",\\n\",\n \" \\\"kc0231\\\"\\n\",\n \" ],\\n\",\n \" \\\"related_questions\\\": [ # 相关问题（用于扩展）\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\", # 相关问题ID\\n\",\n \" \\\"relation_type\\\": \\\"string\\\" # 关系类型: \\\"extension\\\", \\\"application\\\", \\\"contrast\\\"\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"reference_answer\\\": { # 参考答案\\n\",\n \" \\\"content\\\": \\\"B\\\", # 答案内容\\n\",\n \" \\\"key_points\\\": [ # 关键点列表\\n\",\n \" \\\"string\\\"\\n\",\n \" ],\\n\",\n \" \\\"explanation\\\": \\\"\\\"\\\"两种存储结构适用于不同的场合，不能简单地说谁好谁坏，选项1错误。链式存储用指针表示逻辑结构，而指针的设置是任意的，因此比顺序存储结构能更方便地表示各种逻辑结构，选项\\n\",\n \"ⅡI正确。在顺序存储中，插入和删除结点需要移动大量元素，效率较低，选项IⅢI的描述刚好相反。\\n\",\n \"顺序存储结构既能随机存取又能顺序存取，而链式结构只能顺序存取，选项IV正确。\\\"\\\"\\\" # 详细解释\\n\",\n \" }\\n\",\n \"}\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 题目\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import re\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def combine_text_files(file_paths):\\n\",\n \" \\\"\\\"\\\"合并多个文本文件的内容\\\"\\\"\\\"\\n\",\n \" combined_text = \\\"\\\"\\n\",\n \" for file_path in file_paths:\\n\",\n \" if os.path.exists(file_path):\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" combined_text += f.read() + \\\"\\\\n\\\"\\n\",\n \" return combined_text\\n\",\n \"\\n\",\n \"def extract_choice_questions(text):\\n\",\n \" \\\"\\\"\\\"提取选择题部分\\\"\\\"\\\"\\n\",\n \" # 找到\\\"一、单项选择题\\\"的位置\\n\",\n \" start_index = text.find(\\\"一、单项选择题\\\")\\n\",\n \" if start_index == -1:\\n\",\n \" return \\\"\\\"\\n\",\n \" \\n\",\n \" # 从这个位置开始查找\\\"二、\\\"\\n\",\n \" text_after_start = text[start_index:]\\n\",\n \" end_index = text_after_start.find(\\\"二、\\\")\\n\",\n \" \\n\",\n \" if end_index != -1: # 找到了\\\"二、\\\"\\n\",\n \" return text_after_start[:end_index]\\n\",\n \" else:\\n\",\n \" return text_after_start\\n\",\n \"\\n\",\n \"def parse_questions(questions_text):\\n\",\n \" \\\"\\\"\\\"解析题目内容\\\"\\\"\\\"\\n\",\n \" questions = {}\\n\",\n \" current_question = []\\n\",\n \" current_number = None\\n\",\n \" \\n\",\n \" # 跳过\\\"一、单项选择题\\\"之前的内容\\n\",\n \" content = questions_text.split(\\\"一、单项选择题\\\", 1)[1]\\n\",\n \" \\n\",\n \" for line in content.split('\\\\n'):\\n\",\n \" # 匹配题号（形如 01. 02. 等）\\n\",\n \" match = re.match(r'^(\\\\d{2})\\\\.\\\\s*(.+)', line)\\n\",\n \" if match:\\n\",\n \" # 保存前一个题目\\n\",\n \" if current_number:\\n\",\n \" questions[current_number] = '\\\\n'.join(filter(None, current_question))\\n\",\n \" # 开始新题目\\n\",\n \" current_number = match.group(1)\\n\",\n \" current_question = [match.group(2)]\\n\",\n \" elif current_number and line.strip():\\n\",\n \" current_question.append(line.strip())\\n\",\n \" \\n\",\n \" # 保存最后一个题目\\n\",\n \" if current_number:\\n\",\n \" questions[current_number] = '\\\\n'.join(filter(None, current_question))\\n\",\n \" \\n\",\n \" return questions\\n\",\n \"\\n\",\n \"def parse_answers(answers_text):\\n\",\n \" \\\"\\\"\\\"解析答案内容\\\"\\\"\\\"\\n\",\n \" answers = {}\\n\",\n \" current_number = None\\n\",\n \" current_answer = None\\n\",\n \" current_explanation = []\\n\",\n \" \\n\",\n \" # 跳过\\\"一、单项选择题\\\"之前的内容\\n\",\n \" content = answers_text.split(\\\"一、单项选择题\\\", 1)[1]\\n\",\n \" \\n\",\n \" for line in content.split('\\\\n'):\\n\",\n \" # 匹配答案（形如 01.B 或 01. B）\\n\",\n \" match = re.match(r'^(\\\\d{2})\\\\.\\\\s*([A-D])\\\\s*$', line)\\n\",\n \" if match:\\n\",\n \" # 保存前一题的内容\\n\",\n \" if current_number:\\n\",\n \" answers[current_number] = {\\n\",\n \" \\\"答案\\\": current_answer,\\n\",\n \" \\\"解析\\\": '\\\\n'.join(filter(None, current_explanation))\\n\",\n \" }\\n\",\n \" # 开始新题目\\n\",\n \" current_number = match.group(1)\\n\",\n \" current_answer = match.group(2)\\n\",\n \" current_explanation = []\\n\",\n \" elif current_number and line.strip():\\n\",\n \" current_explanation.append(line.strip())\\n\",\n \" \\n\",\n \" # 保存最后一题的内容\\n\",\n \" if current_number:\\n\",\n \" answers[current_number] = {\\n\",\n \" \\\"答案\\\": current_answer,\\n\",\n \" \\\"解析\\\": '\\\\n'.join(filter(None, current_explanation))\\n\",\n \" }\\n\",\n \" \\n\",\n \" return answers\\n\",\n \"\\n\",\n \"def process_questions_and_answers(question_files, answer_files):\\n\",\n \" \\\"\\\"\\\"处理题目和答案文件\\\"\\\"\\\"\\n\",\n \" # 合并题目文件\\n\",\n \" questions_text = combine_text_files(question_files)\\n\",\n \" questions_section = extract_choice_questions(questions_text)\\n\",\n \" questions = parse_questions(questions_section)\\n\",\n \" \\n\",\n \" # 合并答案文件\\n\",\n \" answers_text = combine_text_files(answer_files)\\n\",\n \" answers_section = extract_choice_questions(answers_text)\\n\",\n \" answers = parse_answers(answers_section)\\n\",\n \" \\n\",\n \" # 合并题目和答案\\n\",\n \" result = []\\n\",\n \" for number in sorted(questions.keys()):\\n\",\n \" if number.isdigit() and 1 <= int(number) <= 36:\\n\",\n \" answer_info = answers.get(number, {\\\"答案\\\": \\\"未找到\\\", \\\"解析\\\": \\\"未找到解析\\\"})\\n\",\n \" item = {\\n\",\n \" \\\"题号\\\": number,\\n\",\n \" \\\"题目\\\": questions[number].strip(),\\n\",\n \" \\\"答案\\\": answer_info[\\\"答案\\\"],\\n\",\n \" \\\"解析\\\": answer_info[\\\"解析\\\"]\\n\",\n \" }\\n\",\n \" result.append(item)\\n\",\n \" \\n\",\n \" return result\\n\",\n \"\\n\",\n \"def final_process(id, kpid, qleft, qright, alleft, aright, output_path): \\n\",\n \" question_files = []\\n\",\n \" answer_files = []\\n\",\n \"\\n\",\n \" file = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \" id = id\\n\",\n \" kpid = kpid\\n\",\n \" for i in range(qleft,qright):\\n\",\n \" question_files.append(os.path.join(file,f\\\"page_{i}.txt\\\"))\\n\",\n \" for i in range(alleft,aright):\\n\",\n \" answer_files.append(os.path.join(file,f\\\"page_{i}.txt\\\"))\\n\",\n \"\\n\",\n \" results = process_questions_and_answers(question_files, answer_files)\\n\",\n \"\\n\",\n \" questions = []\\n\",\n \" for q in results:\\n\",\n \" question = {\\n\",\n \" \\\"id\\\": id+q[\\\"题号\\\"], # 问题唯一标识符\\n\",\n \" \\\"title\\\": \\\"title\\\", # 问题标题\\n\",\n \" \\\"content\\\": q[\\\"题目\\\"], # 问题内容\\n\",\n \" \\\"difficulty\\\": \\\"integer\\\", # 难度等级 (1-5)\\n\",\n \" \\\"type\\\": \\\"concept\\\", # 问题类型 (\\\"concept\\\", \\\"calculation\\\", \\\"application\\\")\\n\",\n \" \\\"knowledge_points\\\": kpid, # 相关知识点ID列表\\n\",\n \" \\\"related_questions\\\": [ # 相关问题（用于扩展）\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\", # 相关问题ID\\n\",\n \" \\\"relation_type\\\": \\\"string\\\" # 关系类型: \\\"extension\\\", \\\"application\\\", \\\"contrast\\\"\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"reference_answer\\\": { # 参考答案\\n\",\n \" \\\"content\\\": q[\\\"答案\\\"], # 答案内容\\n\",\n \" \\\"key_points\\\": [ # 关键点列表\\n\",\n \" \\\"string\\\"\\n\",\n \" ],\\n\",\n \" \\\"explanation\\\": q[\\\"解析\\\"] # 详细解释\\n\",\n \" }\\n\",\n \" }\\n\",\n \" questions.append(question)\\n\",\n \"\\n\",\n \" # 如果文件不存在，则创建文件\\n\",\n \" if not os.path.exists(output_path):\\n\",\n \" with open(output_path,\\\"w\\\",encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump([],f,ensure_ascii=False,indent=4)\\n\",\n \"\\n\",\n \" with open(output_path,\\\"r\\\",encoding=\\\"utf-8\\\") as f:\\n\",\n \" data = json.load(f)\\n\",\n \"\\n\",\n \" data += questions\\n\",\n \"\\n\",\n \" with open(output_path,\\\"w\\\",encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(data,f,ensure_ascii=False,indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 34,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"final_process(\\\"q0870\\\",[\\\"kc871\\\", \\\"kc872\\\", \\\"kc873\\\", \\\"kc874\\\", \\\"kc875\\\"],404, 406, 406, 410, \\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_8.json\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### 题目起名\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"load_dotenv()\\n\",\n \"from langchain_deepseek import ChatDeepSeek\\n\",\n \"from langchain_core.prompts import ChatPromptTemplate\\n\",\n \"from typing import TypedDict\\n\",\n \"\\n\",\n \"class QuestionName(TypedDict):\\n\",\n \" question_name: str\\n\",\n \"\\n\",\n \"\\n\",\n \"model = ChatDeepSeek(model=\\\"deepseek-chat\\\", api_key=os.getenv(\\\"DEEPSEEK_API_KEY\\\"))\\n\",\n \"\\n\",\n \"prompt = ChatPromptTemplate([\\n\",\n \" (\\\"system\\\", \\\"请根据以下题目内容，起一个可以概括题目内容，并且简洁的题目名称, 尽可能在15个字以内。\\\"),\\n\",\n \" (\\\"human\\\", \\\"题目内容：{content}\\\"),\\n\",\n \"])\\n\",\n \"\\n\",\n \"chain = prompt | model.with_structured_output(QuestionName)\\n\",\n \"\\n\",\n \"for i in range(8, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" data = json.load(f)\\n\",\n \" for question in data:\\n\",\n \" question_name = chain.invoke({\\\"content\\\": question[\\\"content\\\"]})\\n\",\n \" question[\\\"title\\\"] = question_name[\\\"question_name\\\"]\\n\",\n \" print(\\\"ok\\\")\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(data, f, ensure_ascii=False, indent=4)\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 题目对应章节信息\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"directory = \\\"/root/autodl-tmp/EasyDS/data/ds_data/questions\\\"\\n\",\n \"\\n\",\n \"for i in range(2, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" questions = json.load(f)\\n\",\n \" for question in questions:\\n\",\n \" question[\\\"chapter\\\"] = f\\\"c0{i}\\\"\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(questions, f, ensure_ascii=False, indent=4)\\n\",\n \" print(f\\\"第{i}章节处理完成\\\")\\n\",\n \" print(\\\"=\\\"*20)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 章节信息补充\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 4,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" chapters = json.load(f)\\n\",\n \"for chapter in chapters:\\n\",\n \" chapter[\\\"knowledge_points\\\"] = []\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(chapters, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" chapters = json.load(f)\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" all_knowledgepoints = json.load(f)\\n\",\n \"for chapter in chapters:\\n\",\n \" for kp in all_knowledgepoints:\\n\",\n \" if kp[\\\"chapter_id\\\"] == chapter[\\\"id\\\"]:\\n\",\n \" chapter[\\\"knowledge_points\\\"].append(kp[\\\"id\\\"])\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(chapters, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 补充知识点的问题列表\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 12,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"for i in range(2, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" questions = json.load(f)\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" knowledgepoints = json.load(f)\\n\",\n \" for question in questions:\\n\",\n \" for kp in question[\\\"knowledge_points\\\"]:\\n\",\n \" for kp_item in knowledgepoints:\\n\",\n \" if kp_item[\\\"id\\\"] == kp:\\n\",\n \" kp_item[\\\"questions\\\"].append(question[\\\"id\\\"])\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/chapter_{i}.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(knowledgepoints, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 对知识点进行关联\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"all_knowledgepoints = []\\n\",\n \"for i in range(1, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" knowledgepoints = json.load(f)\\n\",\n \" for kp in knowledgepoints:\\n\",\n \" kp[\\\"related_points\\\"] = []\\n\",\n \" all_knowledgepoints.append(kp)\\n\",\n \"with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(all_knowledgepoints, f, ensure_ascii=False, indent=4)\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"len(all_knowledgepoints)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from langchain_deepseek import ChatDeepSeek\\n\",\n \"from langchain_core.prompts import ChatPromptTemplate\\n\",\n \"from typing import TypedDict, Literal\\n\",\n \"import os\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"load_dotenv()\\n\",\n \"import json\\n\",\n \"\\n\",\n \"# 关系类型固定为\\\"prerequisite\\\", \\\"related\\\", \\\"extension\\\", 如果没有关系则返回None\\n\",\n \"class KnowledgePointRelation(TypedDict):\\n\",\n \" related_knowledge_points: Literal[\\\"prerequisite\\\", \\\"related\\\", \\\"extension\\\", \\\"no_relation\\\"]\\n\",\n \"\\n\",\n \"prompt = ChatPromptTemplate([\\n\",\n \" (\\\"system\\\", \\\"当前的知识点为\\\\ntitle: {title}\\\\ndescription: {description}\\\\n判断用户输入的知识点与当前知识点之间的关系，关系类型固定为\\\\\\\"prerequisite\\\\\\\", \\\\\\\"extension\\\\\\\", 如果没有很明确的前后置关系则返回\\\\\\\"no_relation\\\\\\\"。\\\"),\\n\",\n \" (\\\"human\\\", \\\"用户输入的知识点为\\\\ntitle: {user_title}\\\\ndescription: {user_description}\\\"),\\n\",\n \"])\\n\",\n \"\\n\",\n \"model = ChatDeepSeek(model=\\\"deepseek-chat\\\", api_key=os.getenv(\\\"DEEPSEEK_API_KEY\\\"))\\n\",\n \"chain = prompt | model.with_structured_output(KnowledgePointRelation)\\n\",\n \"\\n\",\n \"# with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \"# all_knowledgepoints = json.load(f)\\n\",\n \"for i in range(96, 101):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" all_knowledgepoints = json.load(f)\\n\",\n \" for j in range(i+1, 102):\\n\",\n \" relation = chain.invoke({\\\"title\\\": all_knowledgepoints[i][\\\"title\\\"], \\\"description\\\": all_knowledgepoints[i][\\\"description\\\"], \\\"user_title\\\": all_knowledgepoints[j][\\\"title\\\"], \\\"user_description\\\": all_knowledgepoints[j][\\\"description\\\"]})\\n\",\n \" if relation[\\\"related_knowledge_points\\\"] != \\\"no_relation\\\":\\n\",\n \" all_knowledgepoints[i][\\\"related_points\\\"].append({\\n\",\n \" \\\"id\\\": all_knowledgepoints[j][\\\"id\\\"],\\n\",\n \" \\\"relation_type\\\": relation[\\\"related_knowledge_points\\\"]\\n\",\n \" })\\n\",\n \" if relation[\\\"related_knowledge_points\\\"] == \\\"prerequisite\\\":\\n\",\n \" all_knowledgepoints[j][\\\"related_points\\\"].append({\\n\",\n \" \\\"id\\\": all_knowledgepoints[i][\\\"id\\\"],\\n\",\n \" \\\"relation_type\\\": \\\"extension\\\"\\n\",\n \" })\\n\",\n \" elif relation[\\\"related_knowledge_points\\\"] == \\\"extension\\\":\\n\",\n \" all_knowledgepoints[j][\\\"related_points\\\"].append({\\n\",\n \" \\\"id\\\": all_knowledgepoints[i][\\\"id\\\"],\\n\",\n \" \\\"relation_type\\\": \\\"prerequisite\\\"\\n\",\n \" })\\n\",\n \" \\n\",\n \" # print(f\\\"第{i+1}个知识点与第{j+1}个知识点处理完成\\\")\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(all_knowledgepoints, f, ensure_ascii=False, indent=4)\\n\",\n \"\\n\",\n \" print(f\\\"第{i+1}个知识点处理完成\\\")\\n\",\n \" print(\\\"=\\\"*20)\\n\",\n \"\\n\",\n \" \\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 知识点速览\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" all_knowledgepoints = json.load(f)\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 索引\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from index_builder import KnowledgeIndexSystem\\n\",\n \"\\n\",\n \"system = KnowledgeIndexSystem.load_indices('/root/autodl-tmp/EasyDS/data/ds_data/ds_indices.pkl')\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"{'id': '01',\\n\",\n \" 'title': '绪论',\\n\",\n \" 'parent_id': '',\\n\",\n \" 'order': 'integer',\\n\",\n \" 'description': '本章内容是数据结构概述。读者可通过对本章的学习，初步了解数据结\\\\n构的基本内容和基本方法。分析算法的时间复杂度和空间复杂度是本章重点，需要熟练掌握，算\\\\n法设计题通常都会要求分析时间复杂度、空间复杂度，同时会出现考查时间复杂度的选择题。',\\n\",\n \" 'knowledge_points': ['kc0111', 'kc0112', 'kc0121', 'kc0122'],\\n\",\n \" 'sub_chapters': ['chapter_id']}\"\n ]\n },\n \"execution_count\": 5,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"system.chapter_index[\\\"01\\\"]\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 12,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"{'id': 'q011002',\\n\",\n \" 'title': '非线性数据结构',\\n\",\n \" 'content': '下列四种数据结构中，（）是非线性数据结构。\\\\nA.树\\\\nB.字符串\\\\nC.队列\\\\nD.栈',\\n\",\n \" 'difficulty': 'integer',\\n\",\n \" 'type': 'concept',\\n\",\n \" 'knowledge_points': ['kc0111', 'kc0112'],\\n\",\n \" 'related_questions': [{'id': 'string', 'relation_type': 'string'}],\\n\",\n \" 'reference_answer': {'content': 'A',\\n\",\n \" 'key_points': ['string'],\\n\",\n \" 'explanation': '树和图是典型的非线性数据结构，其他选项都属于线性数据结构。'},\\n\",\n \" 'chapter': '01'}\"\n ]\n },\n \"execution_count\": 12,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"system.get_question(\\\"q011002\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"easyDS\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.11\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n"}, "49e0963ea54cebe5b7acb50e169a2f11042d47a8": {"t": 1785753752.9635696, "n": 30138, "h": "585adb9d10b51bdc", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 章节\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def extract_review_tips(text):\\n\",\n \" # 查找【复习提示】的起始位置\\n\",\n \" start_marker = \\\"【复习提示】\\\"\\n\",\n \" start_idx = text.find(start_marker)\\n\",\n \" \\n\",\n \" if start_idx == -1:\\n\",\n \" return None\\n\",\n \" \\n\",\n \" # 从标记后开始的文本\\n\",\n \" start_idx += len(start_marker)\\n\",\n \" \\n\",\n \" # 使用正则表达式找到下一个章节标记（如 1.1, 2.1, 3.1 等）\\n\",\n \" import re\\n\",\n \" text_after_tips = text[start_idx:]\\n\",\n \" chapter_match = re.search(r'\\\\n\\\\d+\\\\.\\\\d+', text_after_tips)\\n\",\n \" \\n\",\n \" if chapter_match:\\n\",\n \" end_idx = chapter_match.start()\\n\",\n \" review_tip = text_after_tips[:end_idx].strip()\\n\",\n \" else:\\n\",\n \" review_tip = text_after_tips.strip()\\n\",\n \" \\n\",\n \" return review_tip\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"# 使用示例\\n\",\n \"if __name__ == \\\"__main__\\\":\\n\",\n \" directory = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \" chapter_page = ['13', '24', '74', '121', '136', '210', '281', '349']\\n\",\n \" titles = ['绪论', '线性表', '栈和队列和数组', '串', '树和二叉树', '图', '查找', '排序']\\n\",\n \" chapters = []\\n\",\n \" for i, page in enumerate(chapter_page, 1):\\n\",\n \" file_path = os.path.join(directory, f'page_{page}.txt')\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" text = f.read()\\n\",\n \" review_tip = extract_review_tips(text)\\n\",\n \" if review_tip:\\n\",\n \" id = \\\"c0\\\" + str(i)\\n\",\n \" chapter = {\\n\",\n \" \\\"id\\\": id, # 章节唯一标识符\\n\",\n \" \\\"title\\\": titles[i-1], # 章节标题\\n\",\n \" \\\"parent_id\\\": \\\"\\\", # 父章节ID（可选）\\n\",\n \" \\\"order\\\": \\\"integer\\\", # 排序顺序\\n\",\n \" \\\"description\\\": review_tip, # 章节描述\\n\",\n \" \\\"knowledge_points\\\": [ # 包含的知识点ID列表\\n\",\n \" \\\"knowledge_point_id\\\"\\n\",\n \" ],\\n\",\n \" \\\"sub_chapters\\\": [ # 子章节ID列表\\n\",\n \" \\\"chapter_id\\\"\\n\",\n \" ]\\n\",\n \"}\\n\",\n \" chapters.append(chapter)\\n\",\n \" with open('/root/autodl-tmp/EasyDS/data/questions/chapters.json', 'w', encoding='utf-8') as f:\\n\",\n \" json.dump(chapters, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 知识点\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def extract_knowledge_point(text, chapter_number):\\n\",\n \" import re\\n\",\n \" \\n\",\n \" # 使用正则表达式匹配章节标题和内容\\n\",\n \" section_pattern = rf'{chapter_number}\\\\.\\\\d+\\\\.\\\\d+'\\n\",\n \" \\n\",\n \" sections = []\\n\",\n \" matches = list(re.finditer(section_pattern, text))\\n\",\n \" \\n\",\n \" for i in range(len(matches)):\\n\",\n \" current_match = matches[i]\\n\",\n \" current_section = current_match.group()\\n\",\n \" start_pos = current_match.start()\\n\",\n \" \\n\",\n \" if i < len(matches) - 1:\\n\",\n \" end_pos = matches[i + 1].start()\\n\",\n \" content = text[start_pos:end_pos].strip()\\n\",\n \" else:\\n\",\n \" content = text[start_pos:].strip()\\n\",\n \" \\n\",\n \" if \\\"本节试题精选\\\" not in content and \\\"答案与解析\\\" not in content:\\n\",\n \" lines = content.split('\\\\n')\\n\",\n \" full_title = lines[0].strip()\\n\",\n \" title = full_title.split(current_section)[-1].strip()\\n\",\n \" description = '\\\\n'.join(lines[1:]).strip()\\n\",\n \" \\n\",\n \" sections.append({\\n\",\n \" 'section': current_section,\\n\",\n \" 'title': title,\\n\",\n \" 'description': description\\n\",\n \" })\\n\",\n \" \\n\",\n \" return sections\\n\",\n \"\\n\",\n \"def process_file(file_path, chapter_number):\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" text = f.read()\\n\",\n \" \\n\",\n \" sections = extract_knowledge_point(text, chapter_number)\\n\",\n \" \\n\",\n \" # for section in sections:\\n\",\n \" # print(f\\\"标题：{section['title']}\\\")\\n\",\n \" # print(section['description'])\\n\",\n \" # print(\\\"-\\\" * 50)\\n\",\n \" return sections\\n\",\n \"\\n\",\n \"\\n\",\n \"# 使用示例\\n\",\n \"directory = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \"chapter_number = \\\"4\\\" # 要提取的章节号\\n\",\n \"knowledge_points = []\\n\",\n \"for i in range(121, 136):\\n\",\n \" file_path = os.path.join(directory, f'page_{i}.txt')\\n\",\n \" sections = process_file(file_path, chapter_number)\\n\",\n \" for section in sections:\\n\",\n \" id = section['section'].split('.')\\n\",\n \" id = ''.join(id)\\n\",\n \" knowledgepoint = {\\n\",\n \" \\\"id\\\": \\\"kc0\\\"+id, # 知识点唯一标识符\\n\",\n \" \\\"title\\\": section['title'], # 知识点标题\\n\",\n \" \\\"chapter_id\\\": \\\" c0\\\"+chapter_number, # 所属章节ID\\n\",\n \" \\\"description\\\": section['description'], # 知识点详细描述\\n\",\n \" \\\"related_points\\\": [ # 关联知识点\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\", # 关联知识点ID\\n\",\n \" \\\"relation_type\\\": \\\"string\\\" # 关系类型: prerequisite, related, extension\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"questions\\\": [ # 相关问题ID列表\\n\",\n \" \\\"question_id\\\"\\n\",\n \" ]\\n\",\n \"}\\n\",\n \" knowledge_points.append(knowledgepoint)\\n\",\n \"with open(os.path.join(\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints\\\",f'chapter_{chapter_number}.json'), 'w', encoding='utf-8') as f:\\n\",\n \" json.dump(knowledge_points, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 8,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def combine_chapter_content(directory, start_page, end_page):\\n\",\n \" \\\"\\\"\\\"合并章节内容\\\"\\\"\\\"\\n\",\n \" combined_text = \\\"\\\"\\n\",\n \" for i in range(start_page, end_page + 1):\\n\",\n \" file_path = os.path.join(directory, f'page_{i}.txt')\\n\",\n \" if os.path.exists(file_path):\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" combined_text += f.read() + \\\"\\\\n\\\"\\n\",\n \" return combined_text\\n\",\n \"\\n\",\n \"def extract_knowledge_point(text, chapter_number):\\n\",\n \" import re\\n\",\n \" \\n\",\n \" # 使用正则表达式匹配章节标题和内容\\n\",\n \" section_pattern = rf'{chapter_number}\\\\.\\\\d+\\\\.\\\\d+'\\n\",\n \" \\n\",\n \" sections = []\\n\",\n \" matches = list(re.finditer(section_pattern, text))\\n\",\n \" \\n\",\n \" for i in range(len(matches)):\\n\",\n \" current_match = matches[i]\\n\",\n \" current_section = current_match.group()\\n\",\n \" start_pos = current_match.start()\\n\",\n \" \\n\",\n \" if i < len(matches) - 1:\\n\",\n \" end_pos = matches[i + 1].start()\\n\",\n \" content = text[start_pos:end_pos].strip()\\n\",\n \" else:\\n\",\n \" content = text[start_pos:].strip()\\n\",\n \" \\n\",\n \" if \\\"本节试题精选\\\" not in content and \\\"答案与解析\\\" not in content:\\n\",\n \" lines = content.split('\\\\n')\\n\",\n \" full_title = lines[0].strip()\\n\",\n \" title = full_title.split(current_section)[-1].strip()\\n\",\n \" description = '\\\\n'.join(lines[1:]).strip()\\n\",\n \" \\n\",\n \" sections.append({\\n\",\n \" 'section': current_section,\\n\",\n \" 'title': title,\\n\",\n \" 'description': description\\n\",\n \" })\\n\",\n \" \\n\",\n \" return sections\\n\",\n \"\\n\",\n \"def process_chapter(directory, chapter_number, start_page, end_page):\\n\",\n \" # 先合并所有页面内容\\n\",\n \" combined_text = combine_chapter_content(directory, start_page, end_page)\\n\",\n \" \\n\",\n \" # 提取知识点\\n\",\n \" sections = extract_knowledge_point(combined_text, chapter_number)\\n\",\n \" \\n\",\n \" knowledge_points = []\\n\",\n \" for section in sections:\\n\",\n \" id = section['section'].split('.')\\n\",\n \" id = ''.join(id)\\n\",\n \" knowledgepoint = {\\n\",\n \" \\\"id\\\": \\\"kc0\\\"+id,\\n\",\n \" \\\"title\\\": section['title'],\\n\",\n \" \\\"chapter_id\\\": \\\"c0\\\"+chapter_number,\\n\",\n \" \\\"description\\\": section['description'],\\n\",\n \" \\\"related_points\\\": [\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\",\\n\",\n \" \\\"relation_type\\\": \\\"string\\\"\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"questions\\\": [\\\"question_id\\\"]\\n\",\n \" }\\n\",\n \" knowledge_points.append(knowledgepoint)\\n\",\n \" \\n\",\n \" # 保存结果\\n\",\n \" output_dir = \\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints\\\"\\n\",\n \" os.makedirs(output_dir, exist_ok=True)\\n\",\n \" output_path = os.path.join(output_dir, f'chapter_{chapter_number}.json')\\n\",\n \" with open(output_path, 'w', encoding='utf-8') as f:\\n\",\n \" json.dump(knowledge_points, f, ensure_ascii=False, indent=4)\\n\",\n \" \\n\",\n \" return knowledge_points\\n\",\n \"\\n\",\n \"# 使用示例\\n\",\n \"directory = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \"chapter_number = \\\"8\\\"\\n\",\n \"start_page = 349 # 章节开始页\\n\",\n \"end_page = 406 # 章节结束页\\n\",\n \"\\n\",\n \"knowledge_points = process_chapter(directory, chapter_number, start_page, end_page)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"question = {\\n\",\n \" \\\"id\\\": \\\"q023001\\\", # 问题唯一标识符\\n\",\n \" \\\"title\\\": \\\"线性表存储\\\", # 问题标题\\n\",\n \" \\\"content\\\": content, # 问题内容\\n\",\n \" \\\"difficulty\\\": \\\"integer\\\", # 难度等级 (1-5)\\n\",\n \" \\\"type\\\": \\\"concept\\\", # 问题类型 (\\\"concept\\\", \\\"calculation\\\", \\\"application\\\")\\n\",\n \" \\\"knowledge_points\\\": [ # 相关知识点ID列表\\n\",\n \" \\\"kc0221\\\",\\n\",\n \" \\\"kc0231\\\"\\n\",\n \" ],\\n\",\n \" \\\"related_questions\\\": [ # 相关问题（用于扩展）\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\", # 相关问题ID\\n\",\n \" \\\"relation_type\\\": \\\"string\\\" # 关系类型: \\\"extension\\\", \\\"application\\\", \\\"contrast\\\"\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"reference_answer\\\": { # 参考答案\\n\",\n \" \\\"content\\\": \\\"B\\\", # 答案内容\\n\",\n \" \\\"key_points\\\": [ # 关键点列表\\n\",\n \" \\\"string\\\"\\n\",\n \" ],\\n\",\n \" \\\"explanation\\\": \\\"\\\"\\\"两种存储结构适用于不同的场合，不能简单地说谁好谁坏，选项1错误。链式存储用指针表示逻辑结构，而指针的设置是任意的，因此比顺序存储结构能更方便地表示各种逻辑结构，选项\\n\",\n \"ⅡI正确。在顺序存储中，插入和删除结点需要移动大量元素，效率较低，选项IⅢI的描述刚好相反。\\n\",\n \"顺序存储结构既能随机存取又能顺序存取，而链式结构只能顺序存取，选项IV正确。\\\"\\\"\\\" # 详细解释\\n\",\n \" }\\n\",\n \"}\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 题目\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import re\\n\",\n \"import json\\n\",\n \"\\n\",\n \"def combine_text_files(file_paths):\\n\",\n \" \\\"\\\"\\\"合并多个文本文件的内容\\\"\\\"\\\"\\n\",\n \" combined_text = \\\"\\\"\\n\",\n \" for file_path in file_paths:\\n\",\n \" if os.path.exists(file_path):\\n\",\n \" with open(file_path, 'r', encoding='utf-8') as f:\\n\",\n \" combined_text += f.read() + \\\"\\\\n\\\"\\n\",\n \" return combined_text\\n\",\n \"\\n\",\n \"def extract_choice_questions(text):\\n\",\n \" \\\"\\\"\\\"提取选择题部分\\\"\\\"\\\"\\n\",\n \" # 找到\\\"一、单项选择题\\\"的位置\\n\",\n \" start_index = text.find(\\\"一、单项选择题\\\")\\n\",\n \" if start_index == -1:\\n\",\n \" return \\\"\\\"\\n\",\n \" \\n\",\n \" # 从这个位置开始查找\\\"二、\\\"\\n\",\n \" text_after_start = text[start_index:]\\n\",\n \" end_index = text_after_start.find(\\\"二、\\\")\\n\",\n \" \\n\",\n \" if end_index != -1: # 找到了\\\"二、\\\"\\n\",\n \" return text_after_start[:end_index]\\n\",\n \" else:\\n\",\n \" return text_after_start\\n\",\n \"\\n\",\n \"def parse_questions(questions_text):\\n\",\n \" \\\"\\\"\\\"解析题目内容\\\"\\\"\\\"\\n\",\n \" questions = {}\\n\",\n \" current_question = []\\n\",\n \" current_number = None\\n\",\n \" \\n\",\n \" # 跳过\\\"一、单项选择题\\\"之前的内容\\n\",\n \" content = questions_text.split(\\\"一、单项选择题\\\", 1)[1]\\n\",\n \" \\n\",\n \" for line in content.split('\\\\n'):\\n\",\n \" # 匹配题号（形如 01. 02. 等）\\n\",\n \" match = re.match(r'^(\\\\d{2})\\\\.\\\\s*(.+)', line)\\n\",\n \" if match:\\n\",\n \" # 保存前一个题目\\n\",\n \" if current_number:\\n\",\n \" questions[current_number] = '\\\\n'.join(filter(None, current_question))\\n\",\n \" # 开始新题目\\n\",\n \" current_number = match.group(1)\\n\",\n \" current_question = [match.group(2)]\\n\",\n \" elif current_number and line.strip():\\n\",\n \" current_question.append(line.strip())\\n\",\n \" \\n\",\n \" # 保存最后一个题目\\n\",\n \" if current_number:\\n\",\n \" questions[current_number] = '\\\\n'.join(filter(None, current_question))\\n\",\n \" \\n\",\n \" return questions\\n\",\n \"\\n\",\n \"def parse_answers(answers_text):\\n\",\n \" \\\"\\\"\\\"解析答案内容\\\"\\\"\\\"\\n\",\n \" answers = {}\\n\",\n \" current_number = None\\n\",\n \" current_answer = None\\n\",\n \" current_explanation = []\\n\",\n \" \\n\",\n \" # 跳过\\\"一、单项选择题\\\"之前的内容\\n\",\n \" content = answers_text.split(\\\"一、单项选择题\\\", 1)[1]\\n\",\n \" \\n\",\n \" for line in content.split('\\\\n'):\\n\",\n \" # 匹配答案（形如 01.B 或 01. B）\\n\",\n \" match = re.match(r'^(\\\\d{2})\\\\.\\\\s*([A-D])\\\\s*$', line)\\n\",\n \" if match:\\n\",\n \" # 保存前一题的内容\\n\",\n \" if current_number:\\n\",\n \" answers[current_number] = {\\n\",\n \" \\\"答案\\\": current_answer,\\n\",\n \" \\\"解析\\\": '\\\\n'.join(filter(None, current_explanation))\\n\",\n \" }\\n\",\n \" # 开始新题目\\n\",\n \" current_number = match.group(1)\\n\",\n \" current_answer = match.group(2)\\n\",\n \" current_explanation = []\\n\",\n \" elif current_number and line.strip():\\n\",\n \" current_explanation.append(line.strip())\\n\",\n \" \\n\",\n \" # 保存最后一题的内容\\n\",\n \" if current_number:\\n\",\n \" answers[current_number] = {\\n\",\n \" \\\"答案\\\": current_answer,\\n\",\n \" \\\"解析\\\": '\\\\n'.join(filter(None, current_explanation))\\n\",\n \" }\\n\",\n \" \\n\",\n \" return answers\\n\",\n \"\\n\",\n \"def process_questions_and_answers(question_files, answer_files):\\n\",\n \" \\\"\\\"\\\"处理题目和答案文件\\\"\\\"\\\"\\n\",\n \" # 合并题目文件\\n\",\n \" questions_text = combine_text_files(question_files)\\n\",\n \" questions_section = extract_choice_questions(questions_text)\\n\",\n \" questions = parse_questions(questions_section)\\n\",\n \" \\n\",\n \" # 合并答案文件\\n\",\n \" answers_text = combine_text_files(answer_files)\\n\",\n \" answers_section = extract_choice_questions(answers_text)\\n\",\n \" answers = parse_answers(answers_section)\\n\",\n \" \\n\",\n \" # 合并题目和答案\\n\",\n \" result = []\\n\",\n \" for number in sorted(questions.keys()):\\n\",\n \" if number.isdigit() and 1 <= int(number) <= 36:\\n\",\n \" answer_info = answers.get(number, {\\\"答案\\\": \\\"未找到\\\", \\\"解析\\\": \\\"未找到解析\\\"})\\n\",\n \" item = {\\n\",\n \" \\\"题号\\\": number,\\n\",\n \" \\\"题目\\\": questions[number].strip(),\\n\",\n \" \\\"答案\\\": answer_info[\\\"答案\\\"],\\n\",\n \" \\\"解析\\\": answer_info[\\\"解析\\\"]\\n\",\n \" }\\n\",\n \" result.append(item)\\n\",\n \" \\n\",\n \" return result\\n\",\n \"\\n\",\n \"def final_process(id, kpid, qleft, qright, alleft, aright, output_path): \\n\",\n \" question_files = []\\n\",\n \" answer_files = []\\n\",\n \"\\n\",\n \" file = \\\"/root/autodl-tmp/EasyDS/data/DS2026_extracted/text\\\"\\n\",\n \" id = id\\n\",\n \" kpid = kpid\\n\",\n \" for i in range(qleft,qright):\\n\",\n \" question_files.append(os.path.join(file,f\\\"page_{i}.txt\\\"))\\n\",\n \" for i in range(alleft,aright):\\n\",\n \" answer_files.append(os.path.join(file,f\\\"page_{i}.txt\\\"))\\n\",\n \"\\n\",\n \" results = process_questions_and_answers(question_files, answer_files)\\n\",\n \"\\n\",\n \" questions = []\\n\",\n \" for q in results:\\n\",\n \" question = {\\n\",\n \" \\\"id\\\": id+q[\\\"题号\\\"], # 问题唯一标识符\\n\",\n \" \\\"title\\\": \\\"title\\\", # 问题标题\\n\",\n \" \\\"content\\\": q[\\\"题目\\\"], # 问题内容\\n\",\n \" \\\"difficulty\\\": \\\"integer\\\", # 难度等级 (1-5)\\n\",\n \" \\\"type\\\": \\\"concept\\\", # 问题类型 (\\\"concept\\\", \\\"calculation\\\", \\\"application\\\")\\n\",\n \" \\\"knowledge_points\\\": kpid, # 相关知识点ID列表\\n\",\n \" \\\"related_questions\\\": [ # 相关问题（用于扩展）\\n\",\n \" {\\n\",\n \" \\\"id\\\": \\\"string\\\", # 相关问题ID\\n\",\n \" \\\"relation_type\\\": \\\"string\\\" # 关系类型: \\\"extension\\\", \\\"application\\\", \\\"contrast\\\"\\n\",\n \" }\\n\",\n \" ],\\n\",\n \" \\\"reference_answer\\\": { # 参考答案\\n\",\n \" \\\"content\\\": q[\\\"答案\\\"], # 答案内容\\n\",\n \" \\\"key_points\\\": [ # 关键点列表\\n\",\n \" \\\"string\\\"\\n\",\n \" ],\\n\",\n \" \\\"explanation\\\": q[\\\"解析\\\"] # 详细解释\\n\",\n \" }\\n\",\n \" }\\n\",\n \" questions.append(question)\\n\",\n \"\\n\",\n \" # 如果文件不存在，则创建文件\\n\",\n \" if not os.path.exists(output_path):\\n\",\n \" with open(output_path,\\\"w\\\",encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump([],f,ensure_ascii=False,indent=4)\\n\",\n \"\\n\",\n \" with open(output_path,\\\"r\\\",encoding=\\\"utf-8\\\") as f:\\n\",\n \" data = json.load(f)\\n\",\n \"\\n\",\n \" data += questions\\n\",\n \"\\n\",\n \" with open(output_path,\\\"w\\\",encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(data,f,ensure_ascii=False,indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 34,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"final_process(\\\"q0870\\\",[\\\"kc871\\\", \\\"kc872\\\", \\\"kc873\\\", \\\"kc874\\\", \\\"kc875\\\"],404, 406, 406, 410, \\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_8.json\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### 题目起名\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"load_dotenv()\\n\",\n \"from langchain_deepseek import ChatDeepSeek\\n\",\n \"from langchain_core.prompts import ChatPromptTemplate\\n\",\n \"from typing import TypedDict\\n\",\n \"\\n\",\n \"class QuestionName(TypedDict):\\n\",\n \" question_name: str\\n\",\n \"\\n\",\n \"\\n\",\n \"model = ChatDeepSeek(model=\\\"deepseek-chat\\\", api_key=os.getenv(\\\"DEEPSEEK_API_KEY\\\"))\\n\",\n \"\\n\",\n \"prompt = ChatPromptTemplate([\\n\",\n \" (\\\"system\\\", \\\"请根据以下题目内容，起一个可以概括题目内容，并且简洁的题目名称, 尽可能在15个字以内。\\\"),\\n\",\n \" (\\\"human\\\", \\\"题目内容：{content}\\\"),\\n\",\n \"])\\n\",\n \"\\n\",\n \"chain = prompt | model.with_structured_output(QuestionName)\\n\",\n \"\\n\",\n \"for i in range(8, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" data = json.load(f)\\n\",\n \" for question in data:\\n\",\n \" question_name = chain.invoke({\\\"content\\\": question[\\\"content\\\"]})\\n\",\n \" question[\\\"title\\\"] = question_name[\\\"question_name\\\"]\\n\",\n \" print(\\\"ok\\\")\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(data, f, ensure_ascii=False, indent=4)\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 题目对应章节信息\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"directory = \\\"/root/autodl-tmp/EasyDS/data/ds_data/questions\\\"\\n\",\n \"\\n\",\n \"for i in range(2, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" questions = json.load(f)\\n\",\n \" for question in questions:\\n\",\n \" question[\\\"chapter\\\"] = f\\\"c0{i}\\\"\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(questions, f, ensure_ascii=False, indent=4)\\n\",\n \" print(f\\\"第{i}章节处理完成\\\")\\n\",\n \" print(\\\"=\\\"*20)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 章节信息补充\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 4,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" chapters = json.load(f)\\n\",\n \"for chapter in chapters:\\n\",\n \" chapter[\\\"knowledge_points\\\"] = []\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(chapters, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" chapters = json.load(f)\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" all_knowledgepoints = json.load(f)\\n\",\n \"for chapter in chapters:\\n\",\n \" for kp in all_knowledgepoints:\\n\",\n \" if kp[\\\"chapter_id\\\"] == chapter[\\\"id\\\"]:\\n\",\n \" chapter[\\\"knowledge_points\\\"].append(kp[\\\"id\\\"])\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/chapters.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(chapters, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 补充知识点的问题列表\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 12,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"for i in range(2, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" questions = json.load(f)\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" knowledgepoints = json.load(f)\\n\",\n \" for question in questions:\\n\",\n \" for kp in question[\\\"knowledge_points\\\"]:\\n\",\n \" for kp_item in knowledgepoints:\\n\",\n \" if kp_item[\\\"id\\\"] == kp:\\n\",\n \" kp_item[\\\"questions\\\"].append(question[\\\"id\\\"])\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/chapter_{i}.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(knowledgepoints, f, ensure_ascii=False, indent=4)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 对知识点进行关联\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"all_knowledgepoints = []\\n\",\n \"for i in range(1, 9):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/chapter_{i}.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" knowledgepoints = json.load(f)\\n\",\n \" for kp in knowledgepoints:\\n\",\n \" kp[\\\"related_points\\\"] = []\\n\",\n \" all_knowledgepoints.append(kp)\\n\",\n \"with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(all_knowledgepoints, f, ensure_ascii=False, indent=4)\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"len(all_knowledgepoints)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from langchain_deepseek import ChatDeepSeek\\n\",\n \"from langchain_core.prompts import ChatPromptTemplate\\n\",\n \"from typing import TypedDict, Literal\\n\",\n \"import os\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"load_dotenv()\\n\",\n \"import json\\n\",\n \"\\n\",\n \"# 关系类型固定为\\\"prerequisite\\\", \\\"related\\\", \\\"extension\\\", 如果没有关系则返回None\\n\",\n \"class KnowledgePointRelation(TypedDict):\\n\",\n \" related_knowledge_points: Literal[\\\"prerequisite\\\", \\\"related\\\", \\\"extension\\\", \\\"no_relation\\\"]\\n\",\n \"\\n\",\n \"prompt = ChatPromptTemplate([\\n\",\n \" (\\\"system\\\", \\\"当前的知识点为\\\\ntitle: {title}\\\\ndescription: {description}\\\\n判断用户输入的知识点与当前知识点之间的关系，关系类型固定为\\\\\\\"prerequisite\\\\\\\", \\\\\\\"extension\\\\\\\", 如果没有很明确的前后置关系则返回\\\\\\\"no_relation\\\\\\\"。\\\"),\\n\",\n \" (\\\"human\\\", \\\"用户输入的知识点为\\\\ntitle: {user_title}\\\\ndescription: {user_description}\\\"),\\n\",\n \"])\\n\",\n \"\\n\",\n \"model = ChatDeepSeek(model=\\\"deepseek-chat\\\", api_key=os.getenv(\\\"DEEPSEEK_API_KEY\\\"))\\n\",\n \"chain = prompt | model.with_structured_output(KnowledgePointRelation)\\n\",\n \"\\n\",\n \"# with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \"# all_knowledgepoints = json.load(f)\\n\",\n \"for i in range(96, 101):\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" all_knowledgepoints = json.load(f)\\n\",\n \" for j in range(i+1, 102):\\n\",\n \" relation = chain.invoke({\\\"title\\\": all_knowledgepoints[i][\\\"title\\\"], \\\"description\\\": all_knowledgepoints[i][\\\"description\\\"], \\\"user_title\\\": all_knowledgepoints[j][\\\"title\\\"], \\\"user_description\\\": all_knowledgepoints[j][\\\"description\\\"]})\\n\",\n \" if relation[\\\"related_knowledge_points\\\"] != \\\"no_relation\\\":\\n\",\n \" all_knowledgepoints[i][\\\"related_points\\\"].append({\\n\",\n \" \\\"id\\\": all_knowledgepoints[j][\\\"id\\\"],\\n\",\n \" \\\"relation_type\\\": relation[\\\"related_knowledge_points\\\"]\\n\",\n \" })\\n\",\n \" if relation[\\\"related_knowledge_points\\\"] == \\\"prerequisite\\\":\\n\",\n \" all_knowledgepoints[j][\\\"related_points\\\"].append({\\n\",\n \" \\\"id\\\": all_knowledgepoints[i][\\\"id\\\"],\\n\",\n \" \\\"relation_type\\\": \\\"extension\\\"\\n\",\n \" })\\n\",\n \" elif relation[\\\"related_knowledge_points\\\"] == \\\"extension\\\":\\n\",\n \" all_knowledgepoints[j][\\\"related_points\\\"].append({\\n\",\n \" \\\"id\\\": all_knowledgepoints[i][\\\"id\\\"],\\n\",\n \" \\\"relation_type\\\": \\\"prerequisite\\\"\\n\",\n \" })\\n\",\n \" \\n\",\n \" # print(f\\\"第{i+1}个知识点与第{j+1}个知识点处理完成\\\")\\n\",\n \" with open(f\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(all_knowledgepoints, f, ensure_ascii=False, indent=4)\\n\",\n \"\\n\",\n \" print(f\\\"第{i+1}个知识点处理完成\\\")\\n\",\n \" print(\\\"=\\\"*20)\\n\",\n \"\\n\",\n \" \\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 知识点速览\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"\\n\",\n \"with open(\\\"/root/autodl-tmp/EasyDS/data/ds_data/knowledgepoints/all_knowledgepoints.json\\\", \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" all_knowledgepoints = json.load(f)\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 索引\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from index_builder import KnowledgeIndexSystem\\n\",\n \"\\n\",\n \"system = KnowledgeIndexSystem.load_indices('/root/autodl-tmp/EasyDS/data/ds_data/ds_indices.pkl')\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"system.get_question(\\\"q051001\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"system.get_knowledge_point(\\\"kc0111\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 12,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"directory = \\\"/root/autodl-tmp/EasyDS/data/ds_data/questions\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 17,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"for i in range(1, 9):\\n\",\n \" chapter = f\\\"/root/autodl-tmp/EasyDS/data/ds_data/questions/chapter_{i}.json\\\"\\n\",\n \" with open(chapter, \\\"r\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" questions = json.load(f)\\n\",\n \" for q in questions:\\n\",\n \" q[\\\"chapter\\\"] = q[\\\"chapter\\\"][1:]\\n\",\n \" with open(chapter, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n\",\n \" json.dump(questions, f, ensure_ascii=False, indent=4)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"easyDS\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.11\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n"}}}} |