hack/tools/scripts/llm-key-hunter/results/_cache/content/cgy-2313_rag-knowledge-base.json

{"repo": "cgy-2313/rag-knowledge-base", "files": {".env": {"118c4c54918a4fd515e7816ea4cf8fd20dea1f8e": {"t": 1785674119.531228, "n": 70, "h": "2f6ecc3af40f8a63", "c": "DEEPSEEK_API_KEY=你的DeepSeek API Key\nSILICONFLOW_API_KEY=你的硅基流动API Key\n"}}, "rag_chatbot_02.py": {"118c4c54918a4fd515e7816ea4cf8fd20dea1f8e": {"t": 1785745422.4480104, "n": 5133, "h": "400d36b44776cd64", "c": "import os\nimport shutil\nfrom dotenv import load_dotenv\nfrom langchain_openai import ChatOpenAI, OpenAIEmbeddings\nfrom langchain_community.vectorstores import Chroma\nfrom langchain_community.document_loaders import TextLoader, PyPDFLoader\nfrom langchain_text_splitters import RecursiveCharacterTextSplitter\nfrom langchain_core.prompts import ChatPromptTemplate\nfrom langchain_core.output_parsers import StrOutputParser\n\nload_dotenv()\n\nllm = ChatOpenAI(\n    model=\"deepseek-chat\",\n    api_key=os.getenv(\"DEEPSEEK_API_KEY\"),\n    base_url=\"https://api.deepseek.com\",\n    temperature=0\n)\n\nembeddings = OpenAIEmbeddings(\n    model=\"BAAI/bge-m3\",\n    api_key=os.getenv(\"SILICONFLOW_API_KEY\"),\n    base_url=\"https://api.siliconflow.cn/v1\"\n)\n\ndef load_documents_from_folder(folder_path):\n    all_docs = []\n    supported_extensions = [\".txt\", \".pdf\"]\n\n    for filename in os.listdir(folder_path):\n        ext = os.path.splitext(filename)[1].lower()\n        if ext not in supported_extensions:\n            continue\n\n        file_path = os.path.join(folder_path, filename)\n\n        try:\n            if ext == \".txt\":\n                loader = TextLoader(file_path, encoding=\"utf-8\")\n            elif ext == \".pdf\":\n                loader = PyPDFLoader(file_path)\n\n            docs = loader.load()\n            all_docs.extend(docs)\n            print(f\"已加载：{filename}，{len(docs)}页，\"\n                  f\"{sum(len(d.page_content) for d in docs)}字符\")\n\n        except Exception as e:\n            print(f\"加载失败：{filename}，原因：{e}\")\n\n    print(f\"\\n共加载{len(all_docs)}个文档页\")\n    return all_docs\n\n\ndef build_knowledge_base(folder_path, db_path=\"./rag_db\"):\n    if os.path.exists(db_path):\n        print(\"检测到已有知识库，直接加载...\")\n        vectorstore = Chroma(\n            persist_directory=db_path,\n            embedding_function=embeddings\n        )\n        print(f\"加载完成，共{vectorstore._collection.count()}个片段\")\n        return vectorstore\n\n    print(\"构建新知识库...\")\n    docs = load_documents_from_folder(folder_path)\n\n    if not docs:\n        raise ValueError(f\"文件夹{folder_path}里没有可加载的文件\")\n\n    splitter = RecursiveCharacterTextSplitter(\n        chunk_size=100,\n        chunk_overlap=20\n    )\n    chunks = splitter.split_documents(docs)\n    print(f\"切片完成：{len(chunks)}个片段\")\n\n    print(\"\\n片段来源分布：\")\n    sources = {}\n    for chunk in chunks:\n        source = os.path.basename(chunk.metadata.get(\"source\", \"未知\"))\n        sources[source] = sources.get(source, 0) + 1\n    for source, count in sources.items():\n        print(f\"  {source}：{count}个片段\")\n\n    vectorstore = Chroma.from_documents(\n        documents=chunks,\n        embedding=embeddings,\n        persist_directory=db_path\n    )\n    print(f\"\\n知识库构建完成，共{vectorstore._collection.count()}个向量\")\n    return vectorstore\n\n\n# ============================================================\n# 改动一：rag_answer加入chat_history参数\n# ============================================================\n\ndef rag_answer(question, vectorstore, chat_history=[]):\n    total = vectorstore._collection.count()\n    relevant_chunks = vectorstore.similarity_search(question, k=total)\n\n    context = \"\\n\\n\".join([\n        f\"片段{i+1}（来源：{os.path.basename(chunk.metadata.get('source', '未知'))}）：\"\n        f\"{chunk.page_content}\"\n        for i, chunk in enumerate(relevant_chunks)\n    ])\n\n    # 把对话历史拼成字符串放进system prompt\n    history_text = \"\"\n    if chat_history:\n        history_text = \"\\n\\n历史对话：\\n\"\n        for role, content in chat_history:\n            history_text += f\"{role}：{content}\\n\"\n\n    prompt = ChatPromptTemplate.from_messages([\n        (\"system\", \"\"\"你是一个公司内部知识库助手。\n请根据以下参考内容回答员工的问题。\n如果问题涉及代词（比如\"它\"、\"那\"、\"这个\"），请结合历史对话理解指代的内容。\n\n规则：\n1. 只根据参考内容回答，不要编造信息\n2. 如果参考内容里没有相关信息，直接说\"知识库中没有相关信息\"\n3. 回答要简洁清晰\n4. 回答末尾标注答案来自哪个文件\n\n参考内容：\n{context}\n{history_text}\"\"\"),\n        (\"user\", \"{question}\")\n    ])\n\n    chain = prompt | llm | StrOutputParser()\n    answer = chain.invoke({\n        \"context\": context,\n        \"history_text\": history_text,\n        \"question\": question\n    })\n    return answer, relevant_chunks\n\n\n# ============================================================\n# 改动二：run_rag_chatbot维护对话历史\n# ============================================================\n\ndef run_rag_chatbot():\n    print(\"=\" * 50)\n    print(\"构建知识库中...\")\n\n    if os.path.exists(\"./rag_db\"):\n        shutil.rmtree(\"./rag_db\")\n\n    vectorstore = build_knowledge_base(\"./docs\")\n\n    print(\"\\n\" + \"=\" * 50)\n    print(\"知识库问答系统启动\")\n    print(\"输入'退出'结束\")\n    print(\"=\" * 50)\n\n    # 新增：初始化对话历史\n    chat_history = []\n\n    while True:\n        question = input(\"\\n你的问题：\").strip()\n        if question == \"退出\":\n            print(\"再见！\")\n            break\n        if not question:\n            continue\n\n        print(\"\\n检索中...\")\n        answer, chunks = rag_answer(question, vectorstore, chat_history)\n        print(f\"\\n回答：\\n{answer}\")\n        print(f\"\\n参考片段来源：\")\n        for chunk in chunks:\n            source = os.path.basename(chunk.metadata.get(\"source\", \"未知\"))\n            print(f\"  {source}：{chunk.page_content[:30]}...\")\n\n        # 新增：把这轮对话存入历史\n        chat_history.append((\"用户\", question))\n        chat_history.append((\"助手\", answer))\n\n\nif __name__ == \"__main__\":\n    run_rag_chatbot()\n"}, "7bfb9c0f515612be8f33f237835988dc5817b8cb": {"t": 1785753017.0661898, "n": 5133, "h": "400d36b44776cd64", "c": "import os\nimport shutil\nfrom dotenv import load_dotenv\nfrom langchain_openai import ChatOpenAI, OpenAIEmbeddings\nfrom langchain_community.vectorstores import Chroma\nfrom langchain_community.document_loaders import TextLoader, PyPDFLoader\nfrom langchain_text_splitters import RecursiveCharacterTextSplitter\nfrom langchain_core.prompts import ChatPromptTemplate\nfrom langchain_core.output_parsers import StrOutputParser\n\nload_dotenv()\n\nllm = ChatOpenAI(\n    model=\"deepseek-chat\",\n    api_key=os.getenv(\"DEEPSEEK_API_KEY\"),\n    base_url=\"https://api.deepseek.com\",\n    temperature=0\n)\n\nembeddings = OpenAIEmbeddings(\n    model=\"BAAI/bge-m3\",\n    api_key=os.getenv(\"SILICONFLOW_API_KEY\"),\n    base_url=\"https://api.siliconflow.cn/v1\"\n)\n\ndef load_documents_from_folder(folder_path):\n    all_docs = []\n    supported_extensions = [\".txt\", \".pdf\"]\n\n    for filename in os.listdir(folder_path):\n        ext = os.path.splitext(filename)[1].lower()\n        if ext not in supported_extensions:\n            continue\n\n        file_path = os.path.join(folder_path, filename)\n\n        try:\n            if ext == \".txt\":\n                loader = TextLoader(file_path, encoding=\"utf-8\")\n            elif ext == \".pdf\":\n                loader = PyPDFLoader(file_path)\n\n            docs = loader.load()\n            all_docs.extend(docs)\n            print(f\"已加载：{filename}，{len(docs)}页，\"\n                  f\"{sum(len(d.page_content) for d in docs)}字符\")\n\n        except Exception as e:\n            print(f\"加载失败：{filename}，原因：{e}\")\n\n    print(f\"\\n共加载{len(all_docs)}个文档页\")\n    return all_docs\n\n\ndef build_knowledge_base(folder_path, db_path=\"./rag_db\"):\n    if os.path.exists(db_path):\n        print(\"检测到已有知识库，直接加载...\")\n        vectorstore = Chroma(\n            persist_directory=db_path,\n            embedding_function=embeddings\n        )\n        print(f\"加载完成，共{vectorstore._collection.count()}个片段\")\n        return vectorstore\n\n    print(\"构建新知识库...\")\n    docs = load_documents_from_folder(folder_path)\n\n    if not docs:\n        raise ValueError(f\"文件夹{folder_path}里没有可加载的文件\")\n\n    splitter = RecursiveCharacterTextSplitter(\n        chunk_size=100,\n        chunk_overlap=20\n    )\n    chunks = splitter.split_documents(docs)\n    print(f\"切片完成：{len(chunks)}个片段\")\n\n    print(\"\\n片段来源分布：\")\n    sources = {}\n    for chunk in chunks:\n        source = os.path.basename(chunk.metadata.get(\"source\", \"未知\"))\n        sources[source] = sources.get(source, 0) + 1\n    for source, count in sources.items():\n        print(f\"  {source}：{count}个片段\")\n\n    vectorstore = Chroma.from_documents(\n        documents=chunks,\n        embedding=embeddings,\n        persist_directory=db_path\n    )\n    print(f\"\\n知识库构建完成，共{vectorstore._collection.count()}个向量\")\n    return vectorstore\n\n\n# ============================================================\n# 改动一：rag_answer加入chat_history参数\n# ============================================================\n\ndef rag_answer(question, vectorstore, chat_history=[]):\n    total = vectorstore._collection.count()\n    relevant_chunks = vectorstore.similarity_search(question, k=total)\n\n    context = \"\\n\\n\".join([\n        f\"片段{i+1}（来源：{os.path.basename(chunk.metadata.get('source', '未知'))}）：\"\n        f\"{chunk.page_content}\"\n        for i, chunk in enumerate(relevant_chunks)\n    ])\n\n    # 把对话历史拼成字符串放进system prompt\n    history_text = \"\"\n    if chat_history:\n        history_text = \"\\n\\n历史对话：\\n\"\n        for role, content in chat_history:\n            history_text += f\"{role}：{content}\\n\"\n\n    prompt = ChatPromptTemplate.from_messages([\n        (\"system\", \"\"\"你是一个公司内部知识库助手。\n请根据以下参考内容回答员工的问题。\n如果问题涉及代词（比如\"它\"、\"那\"、\"这个\"），请结合历史对话理解指代的内容。\n\n规则：\n1. 只根据参考内容回答，不要编造信息\n2. 如果参考内容里没有相关信息，直接说\"知识库中没有相关信息\"\n3. 回答要简洁清晰\n4. 回答末尾标注答案来自哪个文件\n\n参考内容：\n{context}\n{history_text}\"\"\"),\n        (\"user\", \"{question}\")\n    ])\n\n    chain = prompt | llm | StrOutputParser()\n    answer = chain.invoke({\n        \"context\": context,\n        \"history_text\": history_text,\n        \"question\": question\n    })\n    return answer, relevant_chunks\n\n\n# ============================================================\n# 改动二：run_rag_chatbot维护对话历史\n# ============================================================\n\ndef run_rag_chatbot():\n    print(\"=\" * 50)\n    print(\"构建知识库中...\")\n\n    if os.path.exists(\"./rag_db\"):\n        shutil.rmtree(\"./rag_db\")\n\n    vectorstore = build_knowledge_base(\"./docs\")\n\n    print(\"\\n\" + \"=\" * 50)\n    print(\"知识库问答系统启动\")\n    print(\"输入'退出'结束\")\n    print(\"=\" * 50)\n\n    # 新增：初始化对话历史\n    chat_history = []\n\n    while True:\n        question = input(\"\\n你的问题：\").strip()\n        if question == \"退出\":\n            print(\"再见！\")\n            break\n        if not question:\n            continue\n\n        print(\"\\n检索中...\")\n        answer, chunks = rag_answer(question, vectorstore, chat_history)\n        print(f\"\\n回答：\\n{answer}\")\n        print(f\"\\n参考片段来源：\")\n        for chunk in chunks:\n            source = os.path.basename(chunk.metadata.get(\"source\", \"未知\"))\n            print(f\"  {source}：{chunk.page_content[:30]}...\")\n\n        # 新增：把这轮对话存入历史\n        chat_history.append((\"用户\", question))\n        chat_history.append((\"助手\", answer))\n\n\nif __name__ == \"__main__\":\n    run_rag_chatbot()\n"}}}}