OpenCode
68a9071cf1
Local branch had diverged from origin/master (sibling commits on the same base). Rewrote local history linearly on top of origin/master, folding in all local content: exa/hot-platform discovery hunters, leaks ledger, nightly sweep orchestrator, updated .gitignore and skill docs, plus the latest hunt output and vault state. Remote-only files (vault keys, channel scripts) were restored rather than dropped, so the resulting tree is a full union of both sides.
1 line
45 KiB
JSON
1 line
45 KiB
JSON
{"repo": "chensheg/multidrug-atlas", "files": {"multidrug_atlas_pipeline/literature_geo/llm_extractor.py": {"b44fe61ed98182d9c2fa9a3e34115a4f005e571b": {"t": 1785745431.710736, "n": 5978, "h": "0efeb404ad02139f", "c": "# llm_extractor.py\n\"\"\"\nDeepSeek LLM 中药信息提取器\n============================\n修复：JSON 解析鲁棒性（Markdown 剥离、空响应重试、reason 强制非空）\n\"\"\"\n\nimport os\nimport re\nimport json\nimport asyncio\nfrom typing import Optional\n\nfrom openai import AsyncOpenAI\nfrom config_logger import logger, retry\n\n# 引入全局密钥\ntry:\n import secret_config\n DEEPSEEK_API_KEY = secret_config.DEEPSEEK_API_KEY\nexcept ImportError:\n DEEPSEEK_API_KEY = \"placeholder_key\"\n\n# 动态获取 base_url 和 model\ntry:\n import secret_config as _sc\n _base_url = getattr(_sc, \"DEEPSEEK_BASE_URL\", \"https://api.deepseek.com\")\n _model = getattr(_sc, \"DEEPSEEK_MODEL\", \"deepseek-v4-pro\")\nexcept ImportError:\n _base_url = \"https://api.deepseek.com\"\n _model = \"deepseek-v4-pro\"\n\nclient = AsyncOpenAI(\n api_key=DEEPSEEK_API_KEY,\n base_url=_base_url,\n)\n\n\ndef _strip_markdown_json(text: str) -> str:\n \"\"\"从 LLM 响应中剥离 Markdown 代码块，提取纯 JSON。\"\"\"\n if not text:\n return text\n # 移除 ```json ... ``` 包裹\n match = re.search(r'```(?:json)?\\s*\\n?(.*?)\\n?\\s*```', text, re.DOTALL)\n if match:\n return match.group(1).strip()\n # 移除 ``` 包裹（无语言标识）\n match2 = re.search(r'```\\s*\\n?(.*?)\\n?\\s*```', text, re.DOTALL)\n if match2:\n return match2.group(1).strip()\n # 尝试直接提取 { ... } 块\n brace_match = re.search(r'\\{[^{}]*(?:\\{[^{}]*\\}[^{}]*)*\\}', text, re.DOTALL)\n if brace_match:\n return brace_match.group(0).strip()\n return text.strip()\n\n\ndef _enforce_reasoning(data: dict) -> dict:\n \"\"\"确保 reasoning 字段非空。\"\"\"\n if not data.get(\"reasoning\") or data[\"reasoning\"] in (\"null\", \"None\", \"\", None):\n is_tcm = data.get(\"is_tcm_perturbation\", False)\n if is_tcm:\n compound = data.get(\"compound_name\", \"未知化合物\")\n disease = data.get(\"target_disease\", \"未知疾病\")\n data[\"reasoning\"] = f\"涉及中药相关研究：{compound} → {disease}，涉及组学扰动。\"\n else:\n data[\"reasoning\"] = \"经LLM评估，该文献不涉及中药/天然产物相关组学扰动实验。\"\n return data\n\n\n@retry(max_retries=3, initial_delay=2, backoff_factor=2)\nasync def extract_tcm_info(title: str, abstract: str) -> dict:\n \"\"\"\n Asynchronously extracts TCM-related perturbation data using English prompts.\n Enhanced: Markdown stripping, empty response handling, reason enforcement.\n \"\"\"\n\n system_prompt = (\n \"You are an expert bioinformatics and TCM pharmacology assistant. \"\n \"Your task is to extract structured data from scientific abstracts regarding \"\n \"high-throughput omics perturbation studies, including transcriptomics, \"\n \"single-cell/spatial omics, proteomics, metabolomics, epigenomics, and genomics. \"\n \"You must return a valid JSON object. \"\n \"Do not include Markdown blocks, code fences, or any extra text. \"\n \"The 'reasoning' field is MANDATORY and must NEVER be null or empty.\"\n )\n\n user_prompt = f\"\"\"Analyze the following study:\nTitle: {title}\nAbstract: {abstract}\n\nExtract the information into the following JSON format:\n{{\n \"is_tcm_perturbation\": boolean, // Set true if the study involves TCM formulas, single herbs, extracts, or active monomers/natural products derived from TCM.\n \"perturbation_type\": string, // Choose from [\"Formula\", \"Herb\", \"Monomer\", \"Extract\", \"Unknown\"]\n \"omics_type\": string, // Choose from [\"bulk_transcriptomics\", \"single_cell_transcriptomics\", \"spatial_transcriptomics\", \"proteomics\", \"metabolomics\", \"epigenomics\", \"genomics\", \"multi_omics\", \"other\", \"unknown\"]\n \"compound_name\": string, // Name of the formula, herb, or specific compound (e.g., \"Alisol B 23-acetate\")\n \"source_herb\": string, // If it's a monomer/extract, name the source herb (e.g., \"Alismatis rhizoma\"). Else, null.\n \"herbal_ingredients\": list, // If a formula, list ingredients. If a monomer, return [].\n \"target_disease\": string, // The disease model (e.g., \"Acute Kidney Injury\")\n \"target_cells_or_tissues\": list, // List of affected cells or tissues.\n \"reasoning\": string // Briefly explain the rationale for is_tcm_perturbation (max 30 words). MANDATORY - never null.\n}}\n\"\"\"\n\n try:\n response = await client.chat.completions.create(\n model=_model,\n messages=[\n {\"role\": \"system\", \"content\": system_prompt},\n {\"role\": \"user\", \"content\": user_prompt},\n ],\n temperature=0.1,\n stream=False\n )\n\n raw_content = response.choices[0].message.content\n if not raw_content or raw_content.strip() == \"\":\n logger.warning(\"DeepSeek 返回空响应，触发重试\")\n raise ValueError(\"DeepSeek 返回空响应\")\n\n # 剥离 Markdown 代码块\n cleaned = _strip_markdown_json(raw_content)\n\n try:\n data = json.loads(cleaned)\n except json.JSONDecodeError as e:\n logger.warning(\n f\"DeepSeek JSON 解析失败 ({e})，尝试修复响应文本: {cleaned[:200]}...\"\n )\n # 最后一次尝试：正则提取\n brace = re.search(r'\\{.*\\}', cleaned, re.DOTALL)\n if brace:\n data = json.loads(brace.group(0))\n else:\n raise ValueError(f\"无法从 DeepSeek 响应中提取 JSON: {cleaned[:100]}\")\n\n # 确保 reasoning 非空\n data = _enforce_reasoning(data)\n\n return data\n\n except Exception as e:\n logger.error(f\"DeepSeek Extraction Error: {e}\")\n raise\n\n\nif __name__ == \"__main__\":\n async def test():\n test_title = \"A naturally occurring FXR agonist, alisol B 23-acetate, protects against renal ischemia-reperfusion injury\"\n test_abstract = \"We found that alisol B 23-acetate (ABA), a major active triterpenoid extracted from Alismatis rhizoma, a well-known traditional Chinese medicine, can activate renal FXR...\"\n\n print(\"Requesting DeepSeek...\")\n result = await extract_tcm_info(test_title, test_abstract)\n print(json.dumps(result, indent=4))\n\n import sys\n if sys.platform == 'win32':\n asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())\n asyncio.run(test())\n"}, "e75a7ae70118cdf1d57d07c227247ad1cd8ce894": {"t": 1786068243.968497, "n": 5978, "h": "0efeb404ad02139f", "c": "# llm_extractor.py\n\"\"\"\nDeepSeek LLM 中药信息提取器\n============================\n修复：JSON 解析鲁棒性（Markdown 剥离、空响应重试、reason 强制非空）\n\"\"\"\n\nimport os\nimport re\nimport json\nimport asyncio\nfrom typing import Optional\n\nfrom openai import AsyncOpenAI\nfrom config_logger import logger, retry\n\n# 引入全局密钥\ntry:\n import secret_config\n DEEPSEEK_API_KEY = secret_config.DEEPSEEK_API_KEY\nexcept ImportError:\n DEEPSEEK_API_KEY = \"placeholder_key\"\n\n# 动态获取 base_url 和 model\ntry:\n import secret_config as _sc\n _base_url = getattr(_sc, \"DEEPSEEK_BASE_URL\", \"https://api.deepseek.com\")\n _model = getattr(_sc, \"DEEPSEEK_MODEL\", \"deepseek-v4-pro\")\nexcept ImportError:\n _base_url = \"https://api.deepseek.com\"\n _model = \"deepseek-v4-pro\"\n\nclient = AsyncOpenAI(\n api_key=DEEPSEEK_API_KEY,\n base_url=_base_url,\n)\n\n\ndef _strip_markdown_json(text: str) -> str:\n \"\"\"从 LLM 响应中剥离 Markdown 代码块，提取纯 JSON。\"\"\"\n if not text:\n return text\n # 移除 ```json ... ``` 包裹\n match = re.search(r'```(?:json)?\\s*\\n?(.*?)\\n?\\s*```', text, re.DOTALL)\n if match:\n return match.group(1).strip()\n # 移除 ``` 包裹（无语言标识）\n match2 = re.search(r'```\\s*\\n?(.*?)\\n?\\s*```', text, re.DOTALL)\n if match2:\n return match2.group(1).strip()\n # 尝试直接提取 { ... } 块\n brace_match = re.search(r'\\{[^{}]*(?:\\{[^{}]*\\}[^{}]*)*\\}', text, re.DOTALL)\n if brace_match:\n return brace_match.group(0).strip()\n return text.strip()\n\n\ndef _enforce_reasoning(data: dict) -> dict:\n \"\"\"确保 reasoning 字段非空。\"\"\"\n if not data.get(\"reasoning\") or data[\"reasoning\"] in (\"null\", \"None\", \"\", None):\n is_tcm = data.get(\"is_tcm_perturbation\", False)\n if is_tcm:\n compound = data.get(\"compound_name\", \"未知化合物\")\n disease = data.get(\"target_disease\", \"未知疾病\")\n data[\"reasoning\"] = f\"涉及中药相关研究：{compound} → {disease}，涉及组学扰动。\"\n else:\n data[\"reasoning\"] = \"经LLM评估，该文献不涉及中药/天然产物相关组学扰动实验。\"\n return data\n\n\n@retry(max_retries=3, initial_delay=2, backoff_factor=2)\nasync def extract_tcm_info(title: str, abstract: str) -> dict:\n \"\"\"\n Asynchronously extracts TCM-related perturbation data using English prompts.\n Enhanced: Markdown stripping, empty response handling, reason enforcement.\n \"\"\"\n\n system_prompt = (\n \"You are an expert bioinformatics and TCM pharmacology assistant. \"\n \"Your task is to extract structured data from scientific abstracts regarding \"\n \"high-throughput omics perturbation studies, including transcriptomics, \"\n \"single-cell/spatial omics, proteomics, metabolomics, epigenomics, and genomics. \"\n \"You must return a valid JSON object. \"\n \"Do not include Markdown blocks, code fences, or any extra text. \"\n \"The 'reasoning' field is MANDATORY and must NEVER be null or empty.\"\n )\n\n user_prompt = f\"\"\"Analyze the following study:\nTitle: {title}\nAbstract: {abstract}\n\nExtract the information into the following JSON format:\n{{\n \"is_tcm_perturbation\": boolean, // Set true if the study involves TCM formulas, single herbs, extracts, or active monomers/natural products derived from TCM.\n \"perturbation_type\": string, // Choose from [\"Formula\", \"Herb\", \"Monomer\", \"Extract\", \"Unknown\"]\n \"omics_type\": string, // Choose from [\"bulk_transcriptomics\", \"single_cell_transcriptomics\", \"spatial_transcriptomics\", \"proteomics\", \"metabolomics\", \"epigenomics\", \"genomics\", \"multi_omics\", \"other\", \"unknown\"]\n \"compound_name\": string, // Name of the formula, herb, or specific compound (e.g., \"Alisol B 23-acetate\")\n \"source_herb\": string, // If it's a monomer/extract, name the source herb (e.g., \"Alismatis rhizoma\"). Else, null.\n \"herbal_ingredients\": list, // If a formula, list ingredients. If a monomer, return [].\n \"target_disease\": string, // The disease model (e.g., \"Acute Kidney Injury\")\n \"target_cells_or_tissues\": list, // List of affected cells or tissues.\n \"reasoning\": string // Briefly explain the rationale for is_tcm_perturbation (max 30 words). MANDATORY - never null.\n}}\n\"\"\"\n\n try:\n response = await client.chat.completions.create(\n model=_model,\n messages=[\n {\"role\": \"system\", \"content\": system_prompt},\n {\"role\": \"user\", \"content\": user_prompt},\n ],\n temperature=0.1,\n stream=False\n )\n\n raw_content = response.choices[0].message.content\n if not raw_content or raw_content.strip() == \"\":\n logger.warning(\"DeepSeek 返回空响应，触发重试\")\n raise ValueError(\"DeepSeek 返回空响应\")\n\n # 剥离 Markdown 代码块\n cleaned = _strip_markdown_json(raw_content)\n\n try:\n data = json.loads(cleaned)\n except json.JSONDecodeError as e:\n logger.warning(\n f\"DeepSeek JSON 解析失败 ({e})，尝试修复响应文本: {cleaned[:200]}...\"\n )\n # 最后一次尝试：正则提取\n brace = re.search(r'\\{.*\\}', cleaned, re.DOTALL)\n if brace:\n data = json.loads(brace.group(0))\n else:\n raise ValueError(f\"无法从 DeepSeek 响应中提取 JSON: {cleaned[:100]}\")\n\n # 确保 reasoning 非空\n data = _enforce_reasoning(data)\n\n return data\n\n except Exception as e:\n logger.error(f\"DeepSeek Extraction Error: {e}\")\n raise\n\n\nif __name__ == \"__main__\":\n async def test():\n test_title = \"A naturally occurring FXR agonist, alisol B 23-acetate, protects against renal ischemia-reperfusion injury\"\n test_abstract = \"We found that alisol B 23-acetate (ABA), a major active triterpenoid extracted from Alismatis rhizoma, a well-known traditional Chinese medicine, can activate renal FXR...\"\n\n print(\"Requesting DeepSeek...\")\n result = await extract_tcm_info(test_title, test_abstract)\n print(json.dumps(result, indent=4))\n\n import sys\n if sys.platform == 'win32':\n asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())\n asyncio.run(test())\n"}}, "multidrug_atlas_pipeline/literature_geo/figshare_fetcher.py": {"b44fe61ed98182d9c2fa9a3e34115a4f005e571b": {"t": 1785746642.8303888, "n": 13518, "h": "8c03897259d58a51", "c": "# figshare_fetcher.py\nimport os\nimport json\nimport time\nimport asyncio\nimport requests\nimport pandas as pd\nfrom openai import AsyncOpenAI\nfrom pathlib import Path\nfrom config_logger import logger\nimport aiohttp\n# 引入全局密钥\ntry:\n import secret_config\n DEEPSEEK_API_KEY = secret_config.DEEPSEEK_API_KEY\n FIGSHARE_TOKEN = secret_config.FIGSHARE_TOKEN\nexcept ImportError:\n DEEPSEEK_API_KEY = \"placeholder_key\"\n FIGSHARE_TOKEN = \"placeholder_token\"\n\n# 初始化 DeepSeek 异步客户端\nclient = AsyncOpenAI(\n api_key=DEEPSEEK_API_KEY,\n base_url=\"https://api.deepseek.com\"\n)\n\nFIGSHARE_BASE_URL = 'https://api.figshare.com/v2'\n\n# ==========================================\n# 核心功能 1：DeepSeek 数据质量智能研判\n# ==========================================\nasync def evaluate_figshare_data(title: str, description: str) -> dict:\n \"\"\"\n 使用 DeepSeek-v4-pro 判断 Figshare 的记录是否为有效的中药组学数据集\n \"\"\"\n system_prompt = \"\"\"你是一个专业的生物信息学与中药药理学数据审核员。\n你的任务是评估一段来自开放数据平台（如Figshare）的数据集描述，判断其是否包含真实的、可用于下游分析的中药/天然产物相关的组学数据（如RNA-seq、单细胞、空间组学、蛋白组、代谢组、表观组等）。\n很多时候用户上传的只是论文的补充表格（如纯文本的基因列表）而非原始数据。请仔细甄别。\n请只返回严格的 JSON 格式数据。\"\"\"\n\n user_prompt = f\"\"\"数据集标题: {title}\n数据集描述/摘要: {description}\n\n请分析并提取以下 JSON 格式信息：\n{{\n \"is_valid_omics_data\": boolean, // 是否真的是高通量组学数据（排除仅包含纯文本基因列表或无关图片的记录）\n \"is_tcm_related\": boolean, // 是否涉及中药、单体、植物提取物等\n \"omics_type\": string, // 组学类型（如 \"RNA-seq\", \"scRNA-seq\", \"Spatial transcriptomics\", \"Proteomics\", \"Metabolomics\", \"Epigenomics\", \"Unknown\"）\n \"organism\": string, // 实验物种（如 \"Human\", \"Mouse\", \"Cell line\"）\n \"reasoning\": string // 简要说明你的判断理由（限50字以内）\n}}\"\"\"\n\n try:\n response = await client.chat.completions.create(\n model=\"deepseek-v4-pro\", # 使用最新且速度最快的模型\n messages=[\n {\"role\": \"system\", \"content\": system_prompt},\n {\"role\": \"user\", \"content\": user_prompt},\n ],\n response_format={\"type\": \"json_object\"},\n temperature=0.1,\n stream=False\n )\n return json.loads(response.choices[0].message.content)\n except Exception as e:\n logger.error(f\"DeepSeek 解析失败: {e}\")\n return {\"is_valid_omics_data\": False, \"is_tcm_related\": False, \"reasoning\": \"LLM Error\"}\n\n# ==========================================\n# 核心功能 4：从 Figshare 提取 PMID (服务于 pubmed_searcher)\n# ==========================================\ndef search_figshare_for_pmids(query: str, limit: int = 500) -> list:\n \"\"\"策略5：在 Figshare 中检索数据集，带自动翻页功能\"\"\"\n # 建议针对 Figshare 简化检索词，放弃复杂的括号布尔逻辑\n simplified_query = \"Traditional Chinese Medicine omics\"\n logger.info(f\"\\n[策略 5] 开始 Figshare 数据挖掘，寻找潜藏的 PMID...\")\n logger.info(f\"Figshare 检索引擎较弱，使用简化关键词: '{simplified_query}'\")\n \n search_url = f\"{FIGSHARE_BASE_URL}/articles/search\"\n headers = {\"Accept\": \"application/json\", \"Content-Type\": \"application/json\"}\n \n all_pmids = set()\n page = 1\n page_size = 100\n total_fetched = 0\n\n # 匹配 \"PMID: 1234567\" 或 \"PubMed 1234567\" 等格式\n pmid_pattern = re.compile(r'(?:PMID|PubMed)[\\s:]*([1-9]\\d{6,7})', re.IGNORECASE)\n\n try:\n while total_fetched < limit:\n payload = {\n \"search_for\": simplified_query, \n \"page_size\": page_size,\n \"page\": page\n }\n \n r = requests.post(search_url, json=payload, headers=headers, timeout=30)\n r.raise_for_status()\n items = r.json()\n \n if not items:\n break # 已经没有更多结果了\n \n logger.info(f\"Figshare 第 {page} 页获取到 {len(items)} 条记录，正在深度解析提取 PMID...\")\n\n for item in items:\n item_id = item.get('id')\n try:\n # 获取详情页\n detail_res = requests.get(f\"{FIGSHARE_BASE_URL}/articles/{item_id}\", headers=headers, timeout=10)\n if detail_res.status_code == 200:\n data = detail_res.json()\n text_to_search = str(data.get('title', '')) + \" \" + str(data.get('description', ''))\n \n for custom_field in data.get('custom_fields', []):\n text_to_search += \" \" + str(custom_field.get('value', ''))\n\n # 提取全文中的 PMID\n matches = pmid_pattern.findall(text_to_search)\n for match in matches:\n all_pmids.add(match)\n \n # 提取参考文献中的 PMID\n for ref in data.get('references', []):\n if 'pubmed' in ref.lower() or 'ncbi.nlm.nih.gov' in ref.lower():\n ref_matches = re.findall(r'([1-9]\\d{6,7})', ref)\n if ref_matches:\n all_pmids.update(ref_matches)\n \n time.sleep(0.3) # 礼貌延迟\n except Exception as e:\n pass \n \n total_fetched += len(items)\n \n # 如果本页返回的数据少于 page_size，说明已经是最后一页了\n if len(items) < page_size:\n break\n \n page += 1\n\n except Exception as e:\n logger.error(f\"Figshare 检索异常中断: {e}\")\n \n unique_pmids = list(all_pmids)\n logger.info(f\"策略 5 完成！翻页扫描了 {total_fetched} 条记录，提取到 {len(unique_pmids)} 个关联 PMID\")\n return unique_pmids\n\nimport asyncio\nimport requests\nfrom config_logger import logger\n\nasync def fetch_figshare_by_pmid(pmid: str, session: aiohttp.ClientSession = None) -> list:\n \"\"\"根据 PMID 查找 Figshare 数据集 (已升级为原生 aiohttp 异步并发版)\"\"\"\n url = \"https://api.figshare.com/v2/articles/search\"\n payload = {\"search_for\": pmid}\n headers = {\"Accept\": \"application/json\", \"Content-Type\": \"application/json\"}\n \n # 定义代理地址 (与你 main.py 中保持一致)\n PROXY_URL = \"http://127.0.0.1:7897\"\n \n logger.debug(f\"PMID {pmid}: 开始获取 Figshare 关联数据...\")\n\n try:\n if session:\n # 加入 proxy=PROXY_URL\n async with session.post(url, json=payload, headers=headers, proxy=PROXY_URL, timeout=30) as resp:\n resp.raise_for_status()\n items = await resp.json()\n else:\n # 兼容性兜底，同样加上 trust_env 和 proxy\n async with aiohttp.ClientSession(trust_env=True) as tmp_session:\n async with tmp_session.post(url, json=payload, headers=headers, proxy=PROXY_URL, timeout=30) as resp:\n resp.raise_for_status()\n items = await resp.json()\n\n datasets = []\n for item in items:\n datasets.append({\n \"figshare_id\": item.get(\"id\"),\n \"title\": item.get(\"title\"),\n \"doi\": item.get(\"doi\"),\n \"url\": item.get(\"url\")\n })\n return datasets\n except Exception as e:\n logger.error(f\"Figshare 抓取失败 PMID {pmid}: {e}\")\n return []\n\n# ==========================================\n# 核心功能 2：Figshare 数据检索与文件抓取 (已修正)\n# ==========================================\nimport json\nimport time\nimport re\nimport requests # 换回标准的 requests\n\n# 确保定义了 BASE_URL\nFIGSHARE_BASE_URL = \"https://api.figshare.com/v2\"\n\nasync def fetch_and_filter_figshare(query: str, limit: int = 100) -> list:\n \"\"\"\n 检索 Figshare：遵循官方规范版 API 调用\n \"\"\"\n simplified_query = \"Traditional Chinese Medicine omics\" \n logger.info(f\"\\n开始检索 Figshare 数据... 实际发送的检索词: '{simplified_query}'\")\n \n # 1. 严格按照官方文档，搜索使用 POST 方法，端点为 /articles/search\n search_url = f\"{FIGSHARE_BASE_URL}/articles/search\"\n \n # 2. 搜索参数放在 JSON payload 中，关键字对应的字段为 \"search_for\"\n search_payload = {\n \"search_for\": simplified_query,\n \"page_size\": limit\n }\n \n # 3. 补全标准 API 请求头，明确声明交互格式\n headers = {\n \"Accept\": \"application/json\",\n \"Content-Type\": \"application/json\",\n \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\"\n }\n \n try:\n # 使用 requests.post 发生检索请求\n r = requests.post(\n search_url, \n json=search_payload,\n headers=headers,\n timeout=30\n )\n \n if r.status_code != 200:\n logger.error(f\"\\n❌ Figshare 服务器拒绝访问。状态码: {r.status_code}\")\n logger.error(f\">> 错误详情: {r.text}\")\n return []\n\n items = r.json()\n logger.info(f\"Figshare 初筛命中 {len(items)} 条记录，正在获取详情并进行 AI 研判...\")\n \n if not items:\n logger.error(\"警告：Figshare 返回了 0 条结果。\")\n return []\n \n except Exception as e:\n logger.error(f\"Figshare 检索请求彻底失败: {e}\")\n return []\n\n valid_datasets = []\n \n for item in items:\n item_id = item.get('id')\n detail_url = f\"{FIGSHARE_BASE_URL}/articles/{item_id}\"\n \n try:\n # 详情请求使用 GET，同样带上规范的 Headers\n detail_res = requests.get(\n detail_url, \n headers=headers,\n timeout=20\n )\n \n if detail_res.status_code != 200:\n logger.info(f\"跳过 ID {item_id}: 获取详情失败，状态码 {detail_res.status_code}\")\n continue\n \n detail_data = detail_res.json()\n title = detail_data.get('title', '')\n description = detail_data.get('description', '')\n \n clean_desc = re.sub('<[^<]+>', '', description)\n \n # 调用 DeepSeek 进行智能研判\n llm_judgment = await evaluate_figshare_data(title, clean_desc)\n \n if llm_judgment.get(\"is_valid_omics_data\") and llm_judgment.get(\"is_tcm_related\"):\n logger.info(f\"[命中] ID: {item_id} | {title[:40]}... | 理由: {llm_judgment.get('reasoning')}\")\n \n files_info = []\n for f in detail_data.get('files', []):\n files_info.append({\n \"file_name\": f.get('name'),\n \"download_url\": f.get('download_url'),\n \"size_bytes\": f.get('size')\n })\n \n dataset_record = {\n \"figshare_id\": item_id,\n \"title\": title,\n \"doi\": detail_data.get('doi', ''),\n \"omics_type\": llm_judgment.get(\"omics_type\"),\n \"organism\": llm_judgment.get(\"organism\"),\n \"files\": json.dumps(files_info),\n \"figshare_url\": detail_data.get('figshare_url')\n }\n valid_datasets.append(dataset_record)\n else:\n logger.error(f\"[排除] ID: {item_id} | 理由: {llm_judgment.get('reasoning')}\")\n \n except Exception as e:\n logger.error(f\"处理条目 {item_id} 时发生异常: {e}\")\n \n # 保持温柔的访问频率，避免触发并发限制\n time.sleep(1)\n\n return valid_datasets\n# ==========================================\n# 核心功能 3：自动下载符合条件的数据文件\n# ==========================================\ndef download_figshare_files(datasets: list, download_dir: str = \"figshare_downloads\"):\n \"\"\"\n 根据过滤后的数据集信息，自动下载实际的文件\n \"\"\"\n if not datasets:\n logger.warning(\"没有需要下载的数据。\")\n return\n \n os.makedirs(download_dir, exist_ok=True)\n \n for ds in datasets:\n item_id = ds['figshare_id']\n files_info = json.loads(ds['files'])\n \n # 为每个数据集创建一个独立的文件夹\n item_dir = os.path.join(download_dir, str(item_id))\n os.makedirs(item_dir, exist_ok=True)\n \n logger.info(f\"\\n准备下载数据集 {item_id} ({len(files_info)} 个文件)...\")\n \n for f in files_info:\n file_name = f['file_name']\n download_url = f['download_url']\n save_path = os.path.join(item_dir, file_name)\n \n if not download_url:\n continue\n \n logger.info(f\" -> 开始下载 {file_name} ...\")\n try:\n # 采用流式下载处理大文件\n with requests.get(download_url, stream=True, timeout=30) as r:\n r.raise_for_status()\n with open(save_path, 'wb') as f_out:\n for chunk in r.iter_content(chunk_size=8192):\n f_out.write(chunk)\n logger.info(f\" -> {file_name} 下载完成!\")\n except Exception as e:\n logger.error(f\" -> {file_name} 下载失败: {e}\")\n\n# ==========================================\n# 主流程\n# ==========================================\nasync def main():\n # 使用宽泛的搜索词，脏活累活交给后面的 DeepSeek 处理\n query = '(\"Traditional Chinese Medicine\" OR \"herb\" OR \"phytomedicine\") AND (\"RNA-seq\" OR \"transcriptome\" OR \"single-cell\" OR \"spatial transcriptomics\" OR \"proteomics\" OR \"metabolomics\" OR \"epigenomics\" OR \"methylation\")'\n logger.info(f\"开始独立执行 Figshare 孤儿数据集抓取，检索词: {query}\")\n\n # 1. 抓取并 AI 过滤\n valid_data = await fetch_and_filter_figshare(query, limit=999) # 测试时先搜50条\n \n # 2. 保存元数据到 CSV\n if valid_data:\n df = pd.DataFrame(valid_data)\n csv_name = \"figshare_tcm_omics_datasets.csv\"\n df.to_csv(csv_name, index=False)\n logger.info(f\"成功清洗出 {len(valid_data)} 个高质量数据集，元数据已保存至 {csv_name}\")\n \n # 3. 如果你想立刻下载文件，可以取消注释下面这行：\n download_figshare_files(valid_data)\n else:\n logger.error(\"\\n未找到符合 AI 标准的高质量数据集。\")\n\nif __name__ == \"__main__\":\n if os.name == 'nt':\n asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())\n asyncio.run(main())\n"}, "e75a7ae70118cdf1d57d07c227247ad1cd8ce894": {"t": 1786068247.7973433, "n": 13518, "h": "8c03897259d58a51", "c": "# figshare_fetcher.py\nimport os\nimport json\nimport time\nimport asyncio\nimport requests\nimport pandas as pd\nfrom openai import AsyncOpenAI\nfrom pathlib import Path\nfrom config_logger import logger\nimport aiohttp\n# 引入全局密钥\ntry:\n import secret_config\n DEEPSEEK_API_KEY = secret_config.DEEPSEEK_API_KEY\n FIGSHARE_TOKEN = secret_config.FIGSHARE_TOKEN\nexcept ImportError:\n DEEPSEEK_API_KEY = \"placeholder_key\"\n FIGSHARE_TOKEN = \"placeholder_token\"\n\n# 初始化 DeepSeek 异步客户端\nclient = AsyncOpenAI(\n api_key=DEEPSEEK_API_KEY,\n base_url=\"https://api.deepseek.com\"\n)\n\nFIGSHARE_BASE_URL = 'https://api.figshare.com/v2'\n\n# ==========================================\n# 核心功能 1：DeepSeek 数据质量智能研判\n# ==========================================\nasync def evaluate_figshare_data(title: str, description: str) -> dict:\n \"\"\"\n 使用 DeepSeek-v4-pro 判断 Figshare 的记录是否为有效的中药组学数据集\n \"\"\"\n system_prompt = \"\"\"你是一个专业的生物信息学与中药药理学数据审核员。\n你的任务是评估一段来自开放数据平台（如Figshare）的数据集描述，判断其是否包含真实的、可用于下游分析的中药/天然产物相关的组学数据（如RNA-seq、单细胞、空间组学、蛋白组、代谢组、表观组等）。\n很多时候用户上传的只是论文的补充表格（如纯文本的基因列表）而非原始数据。请仔细甄别。\n请只返回严格的 JSON 格式数据。\"\"\"\n\n user_prompt = f\"\"\"数据集标题: {title}\n数据集描述/摘要: {description}\n\n请分析并提取以下 JSON 格式信息：\n{{\n \"is_valid_omics_data\": boolean, // 是否真的是高通量组学数据（排除仅包含纯文本基因列表或无关图片的记录）\n \"is_tcm_related\": boolean, // 是否涉及中药、单体、植物提取物等\n \"omics_type\": string, // 组学类型（如 \"RNA-seq\", \"scRNA-seq\", \"Spatial transcriptomics\", \"Proteomics\", \"Metabolomics\", \"Epigenomics\", \"Unknown\"）\n \"organism\": string, // 实验物种（如 \"Human\", \"Mouse\", \"Cell line\"）\n \"reasoning\": string // 简要说明你的判断理由（限50字以内）\n}}\"\"\"\n\n try:\n response = await client.chat.completions.create(\n model=\"deepseek-v4-pro\", # 使用最新且速度最快的模型\n messages=[\n {\"role\": \"system\", \"content\": system_prompt},\n {\"role\": \"user\", \"content\": user_prompt},\n ],\n response_format={\"type\": \"json_object\"},\n temperature=0.1,\n stream=False\n )\n return json.loads(response.choices[0].message.content)\n except Exception as e:\n logger.error(f\"DeepSeek 解析失败: {e}\")\n return {\"is_valid_omics_data\": False, \"is_tcm_related\": False, \"reasoning\": \"LLM Error\"}\n\n# ==========================================\n# 核心功能 4：从 Figshare 提取 PMID (服务于 pubmed_searcher)\n# ==========================================\ndef search_figshare_for_pmids(query: str, limit: int = 500) -> list:\n \"\"\"策略5：在 Figshare 中检索数据集，带自动翻页功能\"\"\"\n # 建议针对 Figshare 简化检索词，放弃复杂的括号布尔逻辑\n simplified_query = \"Traditional Chinese Medicine omics\"\n logger.info(f\"\\n[策略 5] 开始 Figshare 数据挖掘，寻找潜藏的 PMID...\")\n logger.info(f\"Figshare 检索引擎较弱，使用简化关键词: '{simplified_query}'\")\n \n search_url = f\"{FIGSHARE_BASE_URL}/articles/search\"\n headers = {\"Accept\": \"application/json\", \"Content-Type\": \"application/json\"}\n \n all_pmids = set()\n page = 1\n page_size = 100\n total_fetched = 0\n\n # 匹配 \"PMID: 1234567\" 或 \"PubMed 1234567\" 等格式\n pmid_pattern = re.compile(r'(?:PMID|PubMed)[\\s:]*([1-9]\\d{6,7})', re.IGNORECASE)\n\n try:\n while total_fetched < limit:\n payload = {\n \"search_for\": simplified_query, \n \"page_size\": page_size,\n \"page\": page\n }\n \n r = requests.post(search_url, json=payload, headers=headers, timeout=30)\n r.raise_for_status()\n items = r.json()\n \n if not items:\n break # 已经没有更多结果了\n \n logger.info(f\"Figshare 第 {page} 页获取到 {len(items)} 条记录，正在深度解析提取 PMID...\")\n\n for item in items:\n item_id = item.get('id')\n try:\n # 获取详情页\n detail_res = requests.get(f\"{FIGSHARE_BASE_URL}/articles/{item_id}\", headers=headers, timeout=10)\n if detail_res.status_code == 200:\n data = detail_res.json()\n text_to_search = str(data.get('title', '')) + \" \" + str(data.get('description', ''))\n \n for custom_field in data.get('custom_fields', []):\n text_to_search += \" \" + str(custom_field.get('value', ''))\n\n # 提取全文中的 PMID\n matches = pmid_pattern.findall(text_to_search)\n for match in matches:\n all_pmids.add(match)\n \n # 提取参考文献中的 PMID\n for ref in data.get('references', []):\n if 'pubmed' in ref.lower() or 'ncbi.nlm.nih.gov' in ref.lower():\n ref_matches = re.findall(r'([1-9]\\d{6,7})', ref)\n if ref_matches:\n all_pmids.update(ref_matches)\n \n time.sleep(0.3) # 礼貌延迟\n except Exception as e:\n pass \n \n total_fetched += len(items)\n \n # 如果本页返回的数据少于 page_size，说明已经是最后一页了\n if len(items) < page_size:\n break\n \n page += 1\n\n except Exception as e:\n logger.error(f\"Figshare 检索异常中断: {e}\")\n \n unique_pmids = list(all_pmids)\n logger.info(f\"策略 5 完成！翻页扫描了 {total_fetched} 条记录，提取到 {len(unique_pmids)} 个关联 PMID\")\n return unique_pmids\n\nimport asyncio\nimport requests\nfrom config_logger import logger\n\nasync def fetch_figshare_by_pmid(pmid: str, session: aiohttp.ClientSession = None) -> list:\n \"\"\"根据 PMID 查找 Figshare 数据集 (已升级为原生 aiohttp 异步并发版)\"\"\"\n url = \"https://api.figshare.com/v2/articles/search\"\n payload = {\"search_for\": pmid}\n headers = {\"Accept\": \"application/json\", \"Content-Type\": \"application/json\"}\n \n # 定义代理地址 (与你 main.py 中保持一致)\n PROXY_URL = \"http://127.0.0.1:7897\"\n \n logger.debug(f\"PMID {pmid}: 开始获取 Figshare 关联数据...\")\n\n try:\n if session:\n # 加入 proxy=PROXY_URL\n async with session.post(url, json=payload, headers=headers, proxy=PROXY_URL, timeout=30) as resp:\n resp.raise_for_status()\n items = await resp.json()\n else:\n # 兼容性兜底，同样加上 trust_env 和 proxy\n async with aiohttp.ClientSession(trust_env=True) as tmp_session:\n async with tmp_session.post(url, json=payload, headers=headers, proxy=PROXY_URL, timeout=30) as resp:\n resp.raise_for_status()\n items = await resp.json()\n\n datasets = []\n for item in items:\n datasets.append({\n \"figshare_id\": item.get(\"id\"),\n \"title\": item.get(\"title\"),\n \"doi\": item.get(\"doi\"),\n \"url\": item.get(\"url\")\n })\n return datasets\n except Exception as e:\n logger.error(f\"Figshare 抓取失败 PMID {pmid}: {e}\")\n return []\n\n# ==========================================\n# 核心功能 2：Figshare 数据检索与文件抓取 (已修正)\n# ==========================================\nimport json\nimport time\nimport re\nimport requests # 换回标准的 requests\n\n# 确保定义了 BASE_URL\nFIGSHARE_BASE_URL = \"https://api.figshare.com/v2\"\n\nasync def fetch_and_filter_figshare(query: str, limit: int = 100) -> list:\n \"\"\"\n 检索 Figshare：遵循官方规范版 API 调用\n \"\"\"\n simplified_query = \"Traditional Chinese Medicine omics\" \n logger.info(f\"\\n开始检索 Figshare 数据... 实际发送的检索词: '{simplified_query}'\")\n \n # 1. 严格按照官方文档，搜索使用 POST 方法，端点为 /articles/search\n search_url = f\"{FIGSHARE_BASE_URL}/articles/search\"\n \n # 2. 搜索参数放在 JSON payload 中，关键字对应的字段为 \"search_for\"\n search_payload = {\n \"search_for\": simplified_query,\n \"page_size\": limit\n }\n \n # 3. 补全标准 API 请求头，明确声明交互格式\n headers = {\n \"Accept\": \"application/json\",\n \"Content-Type\": \"application/json\",\n \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\"\n }\n \n try:\n # 使用 requests.post 发生检索请求\n r = requests.post(\n search_url, \n json=search_payload,\n headers=headers,\n timeout=30\n )\n \n if r.status_code != 200:\n logger.error(f\"\\n❌ Figshare 服务器拒绝访问。状态码: {r.status_code}\")\n logger.error(f\">> 错误详情: {r.text}\")\n return []\n\n items = r.json()\n logger.info(f\"Figshare 初筛命中 {len(items)} 条记录，正在获取详情并进行 AI 研判...\")\n \n if not items:\n logger.error(\"警告：Figshare 返回了 0 条结果。\")\n return []\n \n except Exception as e:\n logger.error(f\"Figshare 检索请求彻底失败: {e}\")\n return []\n\n valid_datasets = []\n \n for item in items:\n item_id = item.get('id')\n detail_url = f\"{FIGSHARE_BASE_URL}/articles/{item_id}\"\n \n try:\n # 详情请求使用 GET，同样带上规范的 Headers\n detail_res = requests.get(\n detail_url, \n headers=headers,\n timeout=20\n )\n \n if detail_res.status_code != 200:\n logger.info(f\"跳过 ID {item_id}: 获取详情失败，状态码 {detail_res.status_code}\")\n continue\n \n detail_data = detail_res.json()\n title = detail_data.get('title', '')\n description = detail_data.get('description', '')\n \n clean_desc = re.sub('<[^<]+>', '', description)\n \n # 调用 DeepSeek 进行智能研判\n llm_judgment = await evaluate_figshare_data(title, clean_desc)\n \n if llm_judgment.get(\"is_valid_omics_data\") and llm_judgment.get(\"is_tcm_related\"):\n logger.info(f\"[命中] ID: {item_id} | {title[:40]}... | 理由: {llm_judgment.get('reasoning')}\")\n \n files_info = []\n for f in detail_data.get('files', []):\n files_info.append({\n \"file_name\": f.get('name'),\n \"download_url\": f.get('download_url'),\n \"size_bytes\": f.get('size')\n })\n \n dataset_record = {\n \"figshare_id\": item_id,\n \"title\": title,\n \"doi\": detail_data.get('doi', ''),\n \"omics_type\": llm_judgment.get(\"omics_type\"),\n \"organism\": llm_judgment.get(\"organism\"),\n \"files\": json.dumps(files_info),\n \"figshare_url\": detail_data.get('figshare_url')\n }\n valid_datasets.append(dataset_record)\n else:\n logger.error(f\"[排除] ID: {item_id} | 理由: {llm_judgment.get('reasoning')}\")\n \n except Exception as e:\n logger.error(f\"处理条目 {item_id} 时发生异常: {e}\")\n \n # 保持温柔的访问频率，避免触发并发限制\n time.sleep(1)\n\n return valid_datasets\n# ==========================================\n# 核心功能 3：自动下载符合条件的数据文件\n# ==========================================\ndef download_figshare_files(datasets: list, download_dir: str = \"figshare_downloads\"):\n \"\"\"\n 根据过滤后的数据集信息，自动下载实际的文件\n \"\"\"\n if not datasets:\n logger.warning(\"没有需要下载的数据。\")\n return\n \n os.makedirs(download_dir, exist_ok=True)\n \n for ds in datasets:\n item_id = ds['figshare_id']\n files_info = json.loads(ds['files'])\n \n # 为每个数据集创建一个独立的文件夹\n item_dir = os.path.join(download_dir, str(item_id))\n os.makedirs(item_dir, exist_ok=True)\n \n logger.info(f\"\\n准备下载数据集 {item_id} ({len(files_info)} 个文件)...\")\n \n for f in files_info:\n file_name = f['file_name']\n download_url = f['download_url']\n save_path = os.path.join(item_dir, file_name)\n \n if not download_url:\n continue\n \n logger.info(f\" -> 开始下载 {file_name} ...\")\n try:\n # 采用流式下载处理大文件\n with requests.get(download_url, stream=True, timeout=30) as r:\n r.raise_for_status()\n with open(save_path, 'wb') as f_out:\n for chunk in r.iter_content(chunk_size=8192):\n f_out.write(chunk)\n logger.info(f\" -> {file_name} 下载完成!\")\n except Exception as e:\n logger.error(f\" -> {file_name} 下载失败: {e}\")\n\n# ==========================================\n# 主流程\n# ==========================================\nasync def main():\n # 使用宽泛的搜索词，脏活累活交给后面的 DeepSeek 处理\n query = '(\"Traditional Chinese Medicine\" OR \"herb\" OR \"phytomedicine\") AND (\"RNA-seq\" OR \"transcriptome\" OR \"single-cell\" OR \"spatial transcriptomics\" OR \"proteomics\" OR \"metabolomics\" OR \"epigenomics\" OR \"methylation\")'\n logger.info(f\"开始独立执行 Figshare 孤儿数据集抓取，检索词: {query}\")\n\n # 1. 抓取并 AI 过滤\n valid_data = await fetch_and_filter_figshare(query, limit=999) # 测试时先搜50条\n \n # 2. 保存元数据到 CSV\n if valid_data:\n df = pd.DataFrame(valid_data)\n csv_name = \"figshare_tcm_omics_datasets.csv\"\n df.to_csv(csv_name, index=False)\n logger.info(f\"成功清洗出 {len(valid_data)} 个高质量数据集，元数据已保存至 {csv_name}\")\n \n # 3. 如果你想立刻下载文件，可以取消注释下面这行：\n download_figshare_files(valid_data)\n else:\n logger.error(\"\\n未找到符合 AI 标准的高质量数据集。\")\n\nif __name__ == \"__main__\":\n if os.name == 'nt':\n asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())\n asyncio.run(main())\n"}}}} |