{"repo": "chensheg/multidrug-atlas", "files": {"multidrug_atlas_pipeline/literature_geo/llm_extractor.py": {"b44fe61ed98182d9c2fa9a3e34115a4f005e571b": {"t": 1785745431.710736, "n": 5978, "h": "0efeb404ad02139f", "c": "# llm_extractor.py\n\"\"\"\nDeepSeek LLM 中药信息提取器\n============================\n修复：JSON 解析鲁棒性（Markdown 剥离、空响应重试、reason 强制非空）\n\"\"\"\n\nimport os\nimport re\nimport json\nimport asyncio\nfrom typing import Optional\n\nfrom openai import AsyncOpenAI\nfrom config_logger import logger, retry\n\n# 引入全局密钥\ntry:\n    import secret_config\n    DEEPSEEK_API_KEY = secret_config.DEEPSEEK_API_KEY\nexcept ImportError:\n    DEEPSEEK_API_KEY = \"placeholder_key\"\n\n# 动态获取 base_url 和 model\ntry:\n    import secret_config as _sc\n    _base_url = getattr(_sc, \"DEEPSEEK_BASE_URL\", \"https://api.deepseek.com\")\n    _model = getattr(_sc, \"DEEPSEEK_MODEL\", \"deepseek-v4-pro\")\nexcept ImportError:\n    _base_url = \"https://api.deepseek.com\"\n    _model = \"deepseek-v4-pro\"\n\nclient = AsyncOpenAI(\n    api_key=DEEPSEEK_API_KEY,\n    base_url=_base_url,\n)\n\n\ndef _strip_markdown_json(text: str) -> str:\n    \"\"\"从 LLM 响应中剥离 Markdown 代码块，提取纯 JSON。\"\"\"\n    if not text:\n        return text\n    # 移除 ```json ... ``` 包裹\n    match = re.search(r'```(?:json)?\\s*\\n?(.*?)\\n?\\s*```', text, re.DOTALL)\n    if match:\n        return match.group(1).strip()\n    # 移除 ``` 包裹（无语言标识）\n    match2 = re.search(r'```\\s*\\n?(.*?)\\n?\\s*```', text, re.DOTALL)\n    if match2:\n        return match2.group(1).strip()\n    # 尝试直接提取 { ... } 块\n    brace_match = re.search(r'\\{[^{}]*(?:\\{[^{}]*\\}[^{}]*)*\\}', text, re.DOTALL)\n    if brace_match:\n        return brace_match.group(0).strip()\n    return text.strip()\n\n\ndef _enforce_reasoning(data: dict) -> dict:\n    \"\"\"确保 reasoning 字段非空。\"\"\"\n    if not data.get(\"reasoning\") or data[\"reasoning\"] in (\"null\", \"None\", \"\", None):\n        is_tcm = data.get(\"is_tcm_perturbation\", False)\n        if is_tcm:\n            compound = data.get(\"compound_name\", \"未知化合物\")\n            disease = data.get(\"target_disease\", \"未知疾病\")\n            data[\"reasoning\"] = f\"涉及中药相关研究：{compound} → {disease}，涉及组学扰动。\"\n        else:\n            data[\"reasoning\"] = \"经LLM评估，该文献不涉及中药/天然产物相关组学扰动实验。\"\n    return data\n\n\n@retry(max_retries=3, initial_delay=2, backoff_factor=2)\nasync def extract_tcm_info(title: str, abstract: str) -> dict:\n    \"\"\"\n    Asynchronously extracts TCM-related perturbation data using English prompts.\n    Enhanced: Markdown stripping, empty response handling, reason enforcement.\n    \"\"\"\n\n    system_prompt = (\n        \"You are an expert bioinformatics and TCM pharmacology assistant. \"\n        \"Your task is to extract structured data from scientific abstracts regarding \"\n        \"high-throughput omics perturbation studies, including transcriptomics, \"\n        \"single-cell/spatial omics, proteomics, metabolomics, epigenomics, and genomics. \"\n        \"You must return a valid JSON object. \"\n        \"Do not include Markdown blocks, code fences, or any extra text. \"\n        \"The 'reasoning' field is MANDATORY and must NEVER be null or empty.\"\n    )\n\n    user_prompt = f\"\"\"Analyze the following study:\nTitle: {title}\nAbstract: {abstract}\n\nExtract the information into the following JSON format:\n{{\n    \"is_tcm_perturbation\": boolean, // Set true if the study involves TCM formulas, single herbs, extracts, or active monomers/natural products derived from TCM.\n    \"perturbation_type\": string, // Choose from [\"Formula\", \"Herb\", \"Monomer\", \"Extract\", \"Unknown\"]\n    \"omics_type\": string, // Choose from [\"bulk_transcriptomics\", \"single_cell_transcriptomics\", \"spatial_transcriptomics\", \"proteomics\", \"metabolomics\", \"epigenomics\", \"genomics\", \"multi_omics\", \"other\", \"unknown\"]\n    \"compound_name\": string, // Name of the formula, herb, or specific compound (e.g., \"Alisol B 23-acetate\")\n    \"source_herb\": string, // If it's a monomer/extract, name the source herb (e.g., \"Alismatis rhizoma\"). Else, null.\n    \"herbal_ingredients\": list, // If a formula, list ingredients. If a monomer, return [].\n    \"target_disease\": string, // The disease model (e.g., \"Acute Kidney Injury\")\n    \"target_cells_or_tissues\": list, // List of affected cells or tissues.\n    \"reasoning\": string // Briefly explain the rationale for is_tcm_perturbation (max 30 words). MANDATORY - never null.\n}}\n\"\"\"\n\n    try:\n        response = await client.chat.completions.create(\n            model=_model,\n            messages=[\n                {\"role\": \"system\", \"content\": system_prompt},\n                {\"role\": \"user\", \"content\": user_prompt},\n            ],\n            temperature=0.1,\n            stream=False\n        )\n\n        raw_content = response.choices[0].message.content\n        if not raw_content or raw_content.strip() == \"\":\n            logger.warning(\"DeepSeek 返回空响应，触发重试\")\n            raise ValueError(\"DeepSeek 返回空响应\")\n\n        # 剥离 Markdown 代码块\n        cleaned = _strip_markdown_json(raw_content)\n\n        try:\n            data = json.loads(cleaned)\n        except json.JSONDecodeError as e:\n            logger.warning(\n                f\"DeepSeek JSON 解析失败 ({e})，尝试修复响应文本: {cleaned[:200]}...\"\n            )\n            # 最后一次尝试：正则提取\n            brace = re.search(r'\\{.*\\}', cleaned, re.DOTALL)\n            if brace:\n                data = json.loads(brace.group(0))\n            else:\n                raise ValueError(f\"无法从 DeepSeek 响应中提取 JSON: {cleaned[:100]}\")\n\n        # 确保 reasoning 非空\n        data = _enforce_reasoning(data)\n\n        return data\n\n    except Exception as e:\n        logger.error(f\"DeepSeek Extraction Error: {e}\")\n        raise\n\n\nif __name__ == \"__main__\":\n    async def test():\n        test_title = \"A naturally occurring FXR agonist, alisol B 23-acetate, protects against renal ischemia-reperfusion injury\"\n        test_abstract = \"We found that alisol B 23-acetate (ABA), a major active triterpenoid extracted from Alismatis rhizoma, a well-known traditional Chinese medicine, can activate renal FXR...\"\n\n        print(\"Requesting DeepSeek...\")\n        result = await extract_tcm_info(test_title, test_abstract)\n        print(json.dumps(result, indent=4))\n\n    import sys\n    if sys.platform == 'win32':\n        asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())\n    asyncio.run(test())\n"}, "e75a7ae70118cdf1d57d07c227247ad1cd8ce894": {"t": 1786068243.968497, "n": 5978, "h": "0efeb404ad02139f", "c": "# llm_extractor.py\n\"\"\"\nDeepSeek LLM 中药信息提取器\n============================\n修复：JSON 解析鲁棒性（Markdown 剥离、空响应重试、reason 强制非空）\n\"\"\"\n\nimport os\nimport re\nimport json\nimport asyncio\nfrom typing import Optional\n\nfrom openai import AsyncOpenAI\nfrom config_logger import logger, retry\n\n# 引入全局密钥\ntry:\n    import secret_config\n    DEEPSEEK_API_KEY = secret_config.DEEPSEEK_API_KEY\nexcept ImportError:\n    DEEPSEEK_API_KEY = \"placeholder_key\"\n\n# 动态获取 base_url 和 model\ntry:\n    import secret_config as _sc\n    _base_url = getattr(_sc, \"DEEPSEEK_BASE_URL\", \"https://api.deepseek.com\")\n    _model = getattr(_sc, \"DEEPSEEK_MODEL\", \"deepseek-v4-pro\")\nexcept ImportError:\n    _base_url = \"https://api.deepseek.com\"\n    _model = \"deepseek-v4-pro\"\n\nclient = AsyncOpenAI(\n    api_key=DEEPSEEK_API_KEY,\n    base_url=_base_url,\n)\n\n\ndef _strip_markdown_json(text: str) -> str:\n    \"\"\"从 LLM 响应中剥离 Markdown 代码块，提取纯 JSON。\"\"\"\n    if not text:\n        return text\n    # 移除 ```json ... ``` 包裹\n    match = re.search(r'```(?:json)?\\s*\\n?(.*?)\\n?\\s*```', text, re.DOTALL)\n    if match:\n        return match.group(1).strip()\n    # 移除 ``` 包裹（无语言标识）\n    match2 = re.search(r'```\\s*\\n?(.*?)\\n?\\s*```', text, re.DOTALL)\n    if match2:\n        return match2.group(1).strip()\n    # 尝试直接提取 { ... } 块\n    brace_match = re.search(r'\\{[^{}]*(?:\\{[^{}]*\\}[^{}]*)*\\}', text, re.DOTALL)\n    if brace_match:\n        return brace_match.group(0).strip()\n    return text.strip()\n\n\ndef _enforce_reasoning(data: dict) -> dict:\n    \"\"\"确保 reasoning 字段非空。\"\"\"\n    if not data.get(\"reasoning\") or data[\"reasoning\"] in (\"null\", \"None\", \"\", None):\n        is_tcm = data.get(\"is_tcm_perturbation\", False)\n        if is_tcm:\n            compound = data.get(\"compound_name\", \"未知化合物\")\n            disease = data.get(\"target_disease\", \"未知疾病\")\n            data[\"reasoning\"] = f\"涉及中药相关研究：{compound} → {disease}，涉及组学扰动。\"\n        else:\n            data[\"reasoning\"] = \"经LLM评估，该文献不涉及中药/天然产物相关组学扰动实验。\"\n    return data\n\n\n@retry(max_retries=3, initial_delay=2, backoff_factor=2)\nasync def extract_tcm_info(title: str, abstract: str) -> dict:\n    \"\"\"\n    Asynchronously extracts TCM-related perturbation data using English prompts.\n    Enhanced: Markdown stripping, empty response handling, reason enforcement.\n    \"\"\"\n\n    system_prompt = (\n        \"You are an expert bioinformatics and TCM pharmacology assistant. \"\n        \"Your task is to extract structured data from scientific abstracts regarding \"\n        \"high-throughput omics perturbation studies, including transcriptomics, \"\n        \"single-cell/spatial omics, proteomics, metabolomics, epigenomics, and genomics. \"\n        \"You must return a valid JSON object. \"\n        \"Do not include Markdown blocks, code fences, or any extra text. \"\n        \"The 'reasoning' field is MANDATORY and must NEVER be null or empty.\"\n    )\n\n    user_prompt = f\"\"\"Analyze the following study:\nTitle: {title}\nAbstract: {abstract}\n\nExtract the information into the following JSON format:\n{{\n    \"is_tcm_perturbation\": boolean, // Set true if the study involves TCM formulas, single herbs, extracts, or active monomers/natural products derived from TCM.\n    \"perturbation_type\": string, // Choose from [\"Formula\", \"Herb\", \"Monomer\", \"Extract\", \"Unknown\"]\n    \"omics_type\": string, // Choose from [\"bulk_transcriptomics\", \"single_cell_transcriptomics\", \"spatial_transcriptomics\", \"proteomics\", \"metabolomics\", \"epigenomics\", \"genomics\", \"multi_omics\", \"other\", \"unknown\"]\n    \"compound_name\": string, // Name of the formula, herb, or specific compound (e.g., \"Alisol B 23-acetate\")\n    \"source_herb\": string, // If it's a monomer/extract, name the source herb (e.g., \"Alismatis rhizoma\"). Else, null.\n    \"herbal_ingredients\": list, // If a formula, list ingredients. If a monomer, return [].\n    \"target_disease\": string, // The disease model (e.g., \"Acute Kidney Injury\")\n    \"target_cells_or_tissues\": list, // List of affected cells or tissues.\n    \"reasoning\": string // Briefly explain the rationale for is_tcm_perturbation (max 30 words). MANDATORY - never null.\n}}\n\"\"\"\n\n    try:\n        response = await client.chat.completions.create(\n            model=_model,\n            messages=[\n                {\"role\": \"system\", \"content\": system_prompt},\n                {\"role\": \"user\", \"content\": user_prompt},\n            ],\n            temperature=0.1,\n            stream=False\n        )\n\n        raw_content = response.choices[0].message.content\n        if not raw_content or raw_content.strip() == \"\":\n            logger.warning(\"DeepSeek 返回空响应，触发重试\")\n            raise ValueError(\"DeepSeek 返回空响应\")\n\n        # 剥离 Markdown 代码块\n        cleaned = _strip_markdown_json(raw_content)\n\n        try:\n            data = json.loads(cleaned)\n        except json.JSONDecodeError as e:\n            logger.warning(\n                f\"DeepSeek JSON 解析失败 ({e})，尝试修复响应文本: {cleaned[:200]}...\"\n            )\n            # 最后一次尝试：正则提取\n            brace = re.search(r'\\{.*\\}', cleaned, re.DOTALL)\n            if brace:\n                data = json.loads(brace.group(0))\n            else:\n                raise ValueError(f\"无法从 DeepSeek 响应中提取 JSON: {cleaned[:100]}\")\n\n        # 确保 reasoning 非空\n        data = _enforce_reasoning(data)\n\n        return data\n\n    except Exception as e:\n        logger.error(f\"DeepSeek Extraction Error: {e}\")\n        raise\n\n\nif __name__ == \"__main__\":\n    async def test():\n        test_title = \"A naturally occurring FXR agonist, alisol B 23-acetate, protects against renal ischemia-reperfusion injury\"\n        test_abstract = \"We found that alisol B 23-acetate (ABA), a major active triterpenoid extracted from Alismatis rhizoma, a well-known traditional Chinese medicine, can activate renal FXR...\"\n\n        print(\"Requesting DeepSeek...\")\n        result = await extract_tcm_info(test_title, test_abstract)\n        print(json.dumps(result, indent=4))\n\n    import sys\n    if sys.platform == 'win32':\n        asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())\n    asyncio.run(test())\n"}}, "multidrug_atlas_pipeline/literature_geo/figshare_fetcher.py": {"b44fe61ed98182d9c2fa9a3e34115a4f005e571b": {"t": 1785746642.8303888, "n": 13518, "h": "8c03897259d58a51", "c": "# figshare_fetcher.py\nimport os\nimport json\nimport time\nimport asyncio\nimport requests\nimport pandas as pd\nfrom openai import AsyncOpenAI\nfrom pathlib import Path\nfrom config_logger import logger\nimport aiohttp\n# 引入全局密钥\ntry:\n    import secret_config\n    DEEPSEEK_API_KEY = secret_config.DEEPSEEK_API_KEY\n    FIGSHARE_TOKEN = secret_config.FIGSHARE_TOKEN\nexcept ImportError:\n    DEEPSEEK_API_KEY = \"placeholder_key\"\n    FIGSHARE_TOKEN = \"placeholder_token\"\n\n# 初始化 DeepSeek 异步客户端\nclient = AsyncOpenAI(\n    api_key=DEEPSEEK_API_KEY,\n    base_url=\"https://api.deepseek.com\"\n)\n\nFIGSHARE_BASE_URL = 'https://api.figshare.com/v2'\n\n# ==========================================\n# 核心功能 1：DeepSeek 数据质量智能研判\n# ==========================================\nasync def evaluate_figshare_data(title: str, description: str) -> dict:\n    \"\"\"\n    使用 DeepSeek-v4-pro 判断 Figshare 的记录是否为有效的中药组学数据集\n    \"\"\"\n    system_prompt = \"\"\"你是一个专业的生物信息学与中药药理学数据审核员。\n你的任务是评估一段来自开放数据平台（如Figshare）的数据集描述，判断其是否包含真实的、可用于下游分析的中药/天然产物相关的组学数据（如RNA-seq、单细胞、空间组学、蛋白组、代谢组、表观组等）。\n很多时候用户上传的只是论文的补充表格（如纯文本的基因列表）而非原始数据。请仔细甄别。\n请只返回严格的 JSON 格式数据。\"\"\"\n\n    user_prompt = f\"\"\"数据集标题: {title}\n数据集描述/摘要: {description}\n\n请分析并提取以下 JSON 格式信息：\n{{\n    \"is_valid_omics_data\": boolean, // 是否真的是高通量组学数据（排除仅包含纯文本基因列表或无关图片的记录）\n    \"is_tcm_related\": boolean, // 是否涉及中药、单体、植物提取物等\n    \"omics_type\": string, // 组学类型（如 \"RNA-seq\", \"scRNA-seq\", \"Spatial transcriptomics\", \"Proteomics\", \"Metabolomics\", \"Epigenomics\", \"Unknown\"）\n    \"organism\": string, // 实验物种（如 \"Human\", \"Mouse\", \"Cell line\"）\n    \"reasoning\": string // 简要说明你的判断理由（限50字以内）\n}}\"\"\"\n\n    try:\n        response = await client.chat.completions.create(\n            model=\"deepseek-v4-pro\", # 使用最新且速度最快的模型\n            messages=[\n                {\"role\": \"system\", \"content\": system_prompt},\n                {\"role\": \"user\", \"content\": user_prompt},\n            ],\n            response_format={\"type\": \"json_object\"},\n            temperature=0.1,\n            stream=False\n        )\n        return json.loads(response.choices[0].message.content)\n    except Exception as e:\n        logger.error(f\"DeepSeek 解析失败: {e}\")\n        return {\"is_valid_omics_data\": False, \"is_tcm_related\": False, \"reasoning\": \"LLM Error\"}\n\n# ==========================================\n# 核心功能 4：从 Figshare 提取 PMID (服务于 pubmed_searcher)\n# ==========================================\ndef search_figshare_for_pmids(query: str, limit: int = 500) -> list:\n    \"\"\"策略5：在 Figshare 中检索数据集，带自动翻页功能\"\"\"\n    # 建议针对 Figshare 简化检索词，放弃复杂的括号布尔逻辑\n    simplified_query = \"Traditional Chinese Medicine omics\"\n    logger.info(f\"\\n[策略 5] 开始 Figshare 数据挖掘，寻找潜藏的 PMID...\")\n    logger.info(f\"Figshare 检索引擎较弱，使用简化关键词: '{simplified_query}'\")\n    \n    search_url = f\"{FIGSHARE_BASE_URL}/articles/search\"\n    headers = {\"Accept\": \"application/json\", \"Content-Type\": \"application/json\"}\n    \n    all_pmids = set()\n    page = 1\n    page_size = 100\n    total_fetched = 0\n\n    # 匹配 \"PMID: 1234567\" 或 \"PubMed 1234567\" 等格式\n    pmid_pattern = re.compile(r'(?:PMID|PubMed)[\\s:]*([1-9]\\d{6,7})', re.IGNORECASE)\n\n    try:\n        while total_fetched < limit:\n            payload = {\n                \"search_for\": simplified_query, \n                \"page_size\": page_size,\n                \"page\": page\n            }\n            \n            r = requests.post(search_url, json=payload, headers=headers, timeout=30)\n            r.raise_for_status()\n            items = r.json()\n            \n            if not items:\n                break # 已经没有更多结果了\n                \n            logger.info(f\"Figshare 第 {page} 页获取到 {len(items)} 条记录，正在深度解析提取 PMID...\")\n\n            for item in items:\n                item_id = item.get('id')\n                try:\n                    # 获取详情页\n                    detail_res = requests.get(f\"{FIGSHARE_BASE_URL}/articles/{item_id}\", headers=headers, timeout=10)\n                    if detail_res.status_code == 200:\n                        data = detail_res.json()\n                        text_to_search = str(data.get('title', '')) + \" \" + str(data.get('description', ''))\n                        \n                        for custom_field in data.get('custom_fields', []):\n                            text_to_search += \" \" + str(custom_field.get('value', ''))\n\n                        # 提取全文中的 PMID\n                        matches = pmid_pattern.findall(text_to_search)\n                        for match in matches:\n                            all_pmids.add(match)\n                        \n                        # 提取参考文献中的 PMID\n                        for ref in data.get('references', []):\n                            if 'pubmed' in ref.lower() or 'ncbi.nlm.nih.gov' in ref.lower():\n                                ref_matches = re.findall(r'([1-9]\\d{6,7})', ref)\n                                if ref_matches:\n                                    all_pmids.update(ref_matches)\n                                    \n                    time.sleep(0.3) # 礼貌延迟\n                except Exception as e:\n                    pass \n                    \n            total_fetched += len(items)\n            \n            # 如果本页返回的数据少于 page_size，说明已经是最后一页了\n            if len(items) < page_size:\n                break\n                \n            page += 1\n\n    except Exception as e:\n        logger.error(f\"Figshare 检索异常中断: {e}\")\n        \n    unique_pmids = list(all_pmids)\n    logger.info(f\"策略 5 完成！翻页扫描了 {total_fetched} 条记录，提取到 {len(unique_pmids)} 个关联 PMID\")\n    return unique_pmids\n\nimport asyncio\nimport requests\nfrom config_logger import logger\n\nasync def fetch_figshare_by_pmid(pmid: str, session: aiohttp.ClientSession = None) -> list:\n    \"\"\"根据 PMID 查找 Figshare 数据集 (已升级为原生 aiohttp 异步并发版)\"\"\"\n    url = \"https://api.figshare.com/v2/articles/search\"\n    payload = {\"search_for\": pmid}\n    headers = {\"Accept\": \"application/json\", \"Content-Type\": \"application/json\"}\n    \n    # 定义代理地址 (与你 main.py 中保持一致)\n    PROXY_URL = \"http://127.0.0.1:7897\"\n    \n    logger.debug(f\"PMID {pmid}: 开始获取 Figshare 关联数据...\")\n\n    try:\n        if session:\n            # 加入 proxy=PROXY_URL\n            async with session.post(url, json=payload, headers=headers, proxy=PROXY_URL, timeout=30) as resp:\n                resp.raise_for_status()\n                items = await resp.json()\n        else:\n            # 兼容性兜底，同样加上 trust_env 和 proxy\n            async with aiohttp.ClientSession(trust_env=True) as tmp_session:\n                async with tmp_session.post(url, json=payload, headers=headers, proxy=PROXY_URL, timeout=30) as resp:\n                    resp.raise_for_status()\n                    items = await resp.json()\n\n        datasets = []\n        for item in items:\n            datasets.append({\n                \"figshare_id\": item.get(\"id\"),\n                \"title\": item.get(\"title\"),\n                \"doi\": item.get(\"doi\"),\n                \"url\": item.get(\"url\")\n            })\n        return datasets\n    except Exception as e:\n        logger.error(f\"Figshare 抓取失败 PMID {pmid}: {e}\")\n        return []\n\n# ==========================================\n# 核心功能 2：Figshare 数据检索与文件抓取 (已修正)\n# ==========================================\nimport json\nimport time\nimport re\nimport requests # 换回标准的 requests\n\n# 确保定义了 BASE_URL\nFIGSHARE_BASE_URL = \"https://api.figshare.com/v2\"\n\nasync def fetch_and_filter_figshare(query: str, limit: int = 100) -> list:\n    \"\"\"\n    检索 Figshare：遵循官方规范版 API 调用\n    \"\"\"\n    simplified_query = \"Traditional Chinese Medicine omics\" \n    logger.info(f\"\\n开始检索 Figshare 数据... 实际发送的检索词: '{simplified_query}'\")\n    \n    # 1. 严格按照官方文档，搜索使用 POST 方法，端点为 /articles/search\n    search_url = f\"{FIGSHARE_BASE_URL}/articles/search\"\n    \n    # 2. 搜索参数放在 JSON payload 中，关键字对应的字段为 \"search_for\"\n    search_payload = {\n        \"search_for\": simplified_query,\n        \"page_size\": limit\n    }\n    \n    # 3. 补全标准 API 请求头，明确声明交互格式\n    headers = {\n        \"Accept\": \"application/json\",\n        \"Content-Type\": \"application/json\",\n        \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\"\n    }\n    \n    try:\n        # 使用 requests.post 发生检索请求\n        r = requests.post(\n            search_url, \n            json=search_payload,\n            headers=headers,\n            timeout=30\n        )\n        \n        if r.status_code != 200:\n            logger.error(f\"\\n❌ Figshare 服务器拒绝访问。状态码: {r.status_code}\")\n            logger.error(f\">> 错误详情: {r.text}\")\n            return []\n\n        items = r.json()\n        logger.info(f\"Figshare 初筛命中 {len(items)} 条记录，正在获取详情并进行 AI 研判...\")\n        \n        if not items:\n            logger.error(\"警告：Figshare 返回了 0 条结果。\")\n            return []\n            \n    except Exception as e:\n        logger.error(f\"Figshare 检索请求彻底失败: {e}\")\n        return []\n\n    valid_datasets = []\n    \n    for item in items:\n        item_id = item.get('id')\n        detail_url = f\"{FIGSHARE_BASE_URL}/articles/{item_id}\"\n        \n        try:\n            # 详情请求使用 GET，同样带上规范的 Headers\n            detail_res = requests.get(\n                detail_url, \n                headers=headers,\n                timeout=20\n            )\n            \n            if detail_res.status_code != 200:\n                logger.info(f\"跳过 ID {item_id}: 获取详情失败，状态码 {detail_res.status_code}\")\n                continue\n                \n            detail_data = detail_res.json()\n            title = detail_data.get('title', '')\n            description = detail_data.get('description', '')\n            \n            clean_desc = re.sub('<[^<]+>', '', description)\n            \n            # 调用 DeepSeek 进行智能研判\n            llm_judgment = await evaluate_figshare_data(title, clean_desc)\n            \n            if llm_judgment.get(\"is_valid_omics_data\") and llm_judgment.get(\"is_tcm_related\"):\n                logger.info(f\"[命中] ID: {item_id} | {title[:40]}... | 理由: {llm_judgment.get('reasoning')}\")\n                \n                files_info = []\n                for f in detail_data.get('files', []):\n                    files_info.append({\n                        \"file_name\": f.get('name'),\n                        \"download_url\": f.get('download_url'),\n                        \"size_bytes\": f.get('size')\n                    })\n                \n                dataset_record = {\n                    \"figshare_id\": item_id,\n                    \"title\": title,\n                    \"doi\": detail_data.get('doi', ''),\n                    \"omics_type\": llm_judgment.get(\"omics_type\"),\n                    \"organism\": llm_judgment.get(\"organism\"),\n                    \"files\": json.dumps(files_info),\n                    \"figshare_url\": detail_data.get('figshare_url')\n                }\n                valid_datasets.append(dataset_record)\n            else:\n                logger.error(f\"[排除] ID: {item_id} | 理由: {llm_judgment.get('reasoning')}\")\n                \n        except Exception as e:\n            logger.error(f\"处理条目 {item_id} 时发生异常: {e}\")\n            \n        # 保持温柔的访问频率，避免触发并发限制\n        time.sleep(1)\n\n    return valid_datasets\n# ==========================================\n# 核心功能 3：自动下载符合条件的数据文件\n# ==========================================\ndef download_figshare_files(datasets: list, download_dir: str = \"figshare_downloads\"):\n    \"\"\"\n    根据过滤后的数据集信息，自动下载实际的文件\n    \"\"\"\n    if not datasets:\n        logger.warning(\"没有需要下载的数据。\")\n        return\n        \n    os.makedirs(download_dir, exist_ok=True)\n    \n    for ds in datasets:\n        item_id = ds['figshare_id']\n        files_info = json.loads(ds['files'])\n        \n        # 为每个数据集创建一个独立的文件夹\n        item_dir = os.path.join(download_dir, str(item_id))\n        os.makedirs(item_dir, exist_ok=True)\n        \n        logger.info(f\"\\n准备下载数据集 {item_id} ({len(files_info)} 个文件)...\")\n        \n        for f in files_info:\n            file_name = f['file_name']\n            download_url = f['download_url']\n            save_path = os.path.join(item_dir, file_name)\n            \n            if not download_url:\n                continue\n                \n            logger.info(f\" -> 开始下载 {file_name} ...\")\n            try:\n                # 采用流式下载处理大文件\n                with requests.get(download_url, stream=True, timeout=30) as r:\n                    r.raise_for_status()\n                    with open(save_path, 'wb') as f_out:\n                        for chunk in r.iter_content(chunk_size=8192):\n                            f_out.write(chunk)\n                logger.info(f\" -> {file_name} 下载完成!\")\n            except Exception as e:\n                logger.error(f\" -> {file_name} 下载失败: {e}\")\n\n# ==========================================\n# 主流程\n# ==========================================\nasync def main():\n    # 使用宽泛的搜索词，脏活累活交给后面的 DeepSeek 处理\n    query = '(\"Traditional Chinese Medicine\" OR \"herb\" OR \"phytomedicine\") AND (\"RNA-seq\" OR \"transcriptome\" OR \"single-cell\" OR \"spatial transcriptomics\" OR \"proteomics\" OR \"metabolomics\" OR \"epigenomics\" OR \"methylation\")'\n    logger.info(f\"开始独立执行 Figshare 孤儿数据集抓取，检索词: {query}\")\n\n    # 1. 抓取并 AI 过滤\n    valid_data = await fetch_and_filter_figshare(query, limit=999) # 测试时先搜50条\n    \n    # 2. 保存元数据到 CSV\n    if valid_data:\n        df = pd.DataFrame(valid_data)\n        csv_name = \"figshare_tcm_omics_datasets.csv\"\n        df.to_csv(csv_name, index=False)\n        logger.info(f\"成功清洗出 {len(valid_data)} 个高质量数据集，元数据已保存至 {csv_name}\")\n        \n        # 3. 如果你想立刻下载文件，可以取消注释下面这行：\n        download_figshare_files(valid_data)\n    else:\n        logger.error(\"\\n未找到符合 AI 标准的高质量数据集。\")\n\nif __name__ == \"__main__\":\n    if os.name == 'nt':\n        asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())\n    asyncio.run(main())\n"}, "e75a7ae70118cdf1d57d07c227247ad1cd8ce894": {"t": 1786068247.7973433, "n": 13518, "h": "8c03897259d58a51", "c": "# figshare_fetcher.py\nimport os\nimport json\nimport time\nimport asyncio\nimport requests\nimport pandas as pd\nfrom openai import AsyncOpenAI\nfrom pathlib import Path\nfrom config_logger import logger\nimport aiohttp\n# 引入全局密钥\ntry:\n    import secret_config\n    DEEPSEEK_API_KEY = secret_config.DEEPSEEK_API_KEY\n    FIGSHARE_TOKEN = secret_config.FIGSHARE_TOKEN\nexcept ImportError:\n    DEEPSEEK_API_KEY = \"placeholder_key\"\n    FIGSHARE_TOKEN = \"placeholder_token\"\n\n# 初始化 DeepSeek 异步客户端\nclient = AsyncOpenAI(\n    api_key=DEEPSEEK_API_KEY,\n    base_url=\"https://api.deepseek.com\"\n)\n\nFIGSHARE_BASE_URL = 'https://api.figshare.com/v2'\n\n# ==========================================\n# 核心功能 1：DeepSeek 数据质量智能研判\n# ==========================================\nasync def evaluate_figshare_data(title: str, description: str) -> dict:\n    \"\"\"\n    使用 DeepSeek-v4-pro 判断 Figshare 的记录是否为有效的中药组学数据集\n    \"\"\"\n    system_prompt = \"\"\"你是一个专业的生物信息学与中药药理学数据审核员。\n你的任务是评估一段来自开放数据平台（如Figshare）的数据集描述，判断其是否包含真实的、可用于下游分析的中药/天然产物相关的组学数据（如RNA-seq、单细胞、空间组学、蛋白组、代谢组、表观组等）。\n很多时候用户上传的只是论文的补充表格（如纯文本的基因列表）而非原始数据。请仔细甄别。\n请只返回严格的 JSON 格式数据。\"\"\"\n\n    user_prompt = f\"\"\"数据集标题: {title}\n数据集描述/摘要: {description}\n\n请分析并提取以下 JSON 格式信息：\n{{\n    \"is_valid_omics_data\": boolean, // 是否真的是高通量组学数据（排除仅包含纯文本基因列表或无关图片的记录）\n    \"is_tcm_related\": boolean, // 是否涉及中药、单体、植物提取物等\n    \"omics_type\": string, // 组学类型（如 \"RNA-seq\", \"scRNA-seq\", \"Spatial transcriptomics\", \"Proteomics\", \"Metabolomics\", \"Epigenomics\", \"Unknown\"）\n    \"organism\": string, // 实验物种（如 \"Human\", \"Mouse\", \"Cell line\"）\n    \"reasoning\": string // 简要说明你的判断理由（限50字以内）\n}}\"\"\"\n\n    try:\n        response = await client.chat.completions.create(\n            model=\"deepseek-v4-pro\", # 使用最新且速度最快的模型\n            messages=[\n                {\"role\": \"system\", \"content\": system_prompt},\n                {\"role\": \"user\", \"content\": user_prompt},\n            ],\n            response_format={\"type\": \"json_object\"},\n            temperature=0.1,\n            stream=False\n        )\n        return json.loads(response.choices[0].message.content)\n    except Exception as e:\n        logger.error(f\"DeepSeek 解析失败: {e}\")\n        return {\"is_valid_omics_data\": False, \"is_tcm_related\": False, \"reasoning\": \"LLM Error\"}\n\n# ==========================================\n# 核心功能 4：从 Figshare 提取 PMID (服务于 pubmed_searcher)\n# ==========================================\ndef search_figshare_for_pmids(query: str, limit: int = 500) -> list:\n    \"\"\"策略5：在 Figshare 中检索数据集，带自动翻页功能\"\"\"\n    # 建议针对 Figshare 简化检索词，放弃复杂的括号布尔逻辑\n    simplified_query = \"Traditional Chinese Medicine omics\"\n    logger.info(f\"\\n[策略 5] 开始 Figshare 数据挖掘，寻找潜藏的 PMID...\")\n    logger.info(f\"Figshare 检索引擎较弱，使用简化关键词: '{simplified_query}'\")\n    \n    search_url = f\"{FIGSHARE_BASE_URL}/articles/search\"\n    headers = {\"Accept\": \"application/json\", \"Content-Type\": \"application/json\"}\n    \n    all_pmids = set()\n    page = 1\n    page_size = 100\n    total_fetched = 0\n\n    # 匹配 \"PMID: 1234567\" 或 \"PubMed 1234567\" 等格式\n    pmid_pattern = re.compile(r'(?:PMID|PubMed)[\\s:]*([1-9]\\d{6,7})', re.IGNORECASE)\n\n    try:\n        while total_fetched < limit:\n            payload = {\n                \"search_for\": simplified_query, \n                \"page_size\": page_size,\n                \"page\": page\n            }\n            \n            r = requests.post(search_url, json=payload, headers=headers, timeout=30)\n            r.raise_for_status()\n            items = r.json()\n            \n            if not items:\n                break # 已经没有更多结果了\n                \n            logger.info(f\"Figshare 第 {page} 页获取到 {len(items)} 条记录，正在深度解析提取 PMID...\")\n\n            for item in items:\n                item_id = item.get('id')\n                try:\n                    # 获取详情页\n                    detail_res = requests.get(f\"{FIGSHARE_BASE_URL}/articles/{item_id}\", headers=headers, timeout=10)\n                    if detail_res.status_code == 200:\n                        data = detail_res.json()\n                        text_to_search = str(data.get('title', '')) + \" \" + str(data.get('description', ''))\n                        \n                        for custom_field in data.get('custom_fields', []):\n                            text_to_search += \" \" + str(custom_field.get('value', ''))\n\n                        # 提取全文中的 PMID\n                        matches = pmid_pattern.findall(text_to_search)\n                        for match in matches:\n                            all_pmids.add(match)\n                        \n                        # 提取参考文献中的 PMID\n                        for ref in data.get('references', []):\n                            if 'pubmed' in ref.lower() or 'ncbi.nlm.nih.gov' in ref.lower():\n                                ref_matches = re.findall(r'([1-9]\\d{6,7})', ref)\n                                if ref_matches:\n                                    all_pmids.update(ref_matches)\n                                    \n                    time.sleep(0.3) # 礼貌延迟\n                except Exception as e:\n                    pass \n                    \n            total_fetched += len(items)\n            \n            # 如果本页返回的数据少于 page_size，说明已经是最后一页了\n            if len(items) < page_size:\n                break\n                \n            page += 1\n\n    except Exception as e:\n        logger.error(f\"Figshare 检索异常中断: {e}\")\n        \n    unique_pmids = list(all_pmids)\n    logger.info(f\"策略 5 完成！翻页扫描了 {total_fetched} 条记录，提取到 {len(unique_pmids)} 个关联 PMID\")\n    return unique_pmids\n\nimport asyncio\nimport requests\nfrom config_logger import logger\n\nasync def fetch_figshare_by_pmid(pmid: str, session: aiohttp.ClientSession = None) -> list:\n    \"\"\"根据 PMID 查找 Figshare 数据集 (已升级为原生 aiohttp 异步并发版)\"\"\"\n    url = \"https://api.figshare.com/v2/articles/search\"\n    payload = {\"search_for\": pmid}\n    headers = {\"Accept\": \"application/json\", \"Content-Type\": \"application/json\"}\n    \n    # 定义代理地址 (与你 main.py 中保持一致)\n    PROXY_URL = \"http://127.0.0.1:7897\"\n    \n    logger.debug(f\"PMID {pmid}: 开始获取 Figshare 关联数据...\")\n\n    try:\n        if session:\n            # 加入 proxy=PROXY_URL\n            async with session.post(url, json=payload, headers=headers, proxy=PROXY_URL, timeout=30) as resp:\n                resp.raise_for_status()\n                items = await resp.json()\n        else:\n            # 兼容性兜底，同样加上 trust_env 和 proxy\n            async with aiohttp.ClientSession(trust_env=True) as tmp_session:\n                async with tmp_session.post(url, json=payload, headers=headers, proxy=PROXY_URL, timeout=30) as resp:\n                    resp.raise_for_status()\n                    items = await resp.json()\n\n        datasets = []\n        for item in items:\n            datasets.append({\n                \"figshare_id\": item.get(\"id\"),\n                \"title\": item.get(\"title\"),\n                \"doi\": item.get(\"doi\"),\n                \"url\": item.get(\"url\")\n            })\n        return datasets\n    except Exception as e:\n        logger.error(f\"Figshare 抓取失败 PMID {pmid}: {e}\")\n        return []\n\n# ==========================================\n# 核心功能 2：Figshare 数据检索与文件抓取 (已修正)\n# ==========================================\nimport json\nimport time\nimport re\nimport requests # 换回标准的 requests\n\n# 确保定义了 BASE_URL\nFIGSHARE_BASE_URL = \"https://api.figshare.com/v2\"\n\nasync def fetch_and_filter_figshare(query: str, limit: int = 100) -> list:\n    \"\"\"\n    检索 Figshare：遵循官方规范版 API 调用\n    \"\"\"\n    simplified_query = \"Traditional Chinese Medicine omics\" \n    logger.info(f\"\\n开始检索 Figshare 数据... 实际发送的检索词: '{simplified_query}'\")\n    \n    # 1. 严格按照官方文档，搜索使用 POST 方法，端点为 /articles/search\n    search_url = f\"{FIGSHARE_BASE_URL}/articles/search\"\n    \n    # 2. 搜索参数放在 JSON payload 中，关键字对应的字段为 \"search_for\"\n    search_payload = {\n        \"search_for\": simplified_query,\n        \"page_size\": limit\n    }\n    \n    # 3. 补全标准 API 请求头，明确声明交互格式\n    headers = {\n        \"Accept\": \"application/json\",\n        \"Content-Type\": \"application/json\",\n        \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\"\n    }\n    \n    try:\n        # 使用 requests.post 发生检索请求\n        r = requests.post(\n            search_url, \n            json=search_payload,\n            headers=headers,\n            timeout=30\n        )\n        \n        if r.status_code != 200:\n            logger.error(f\"\\n❌ Figshare 服务器拒绝访问。状态码: {r.status_code}\")\n            logger.error(f\">> 错误详情: {r.text}\")\n            return []\n\n        items = r.json()\n        logger.info(f\"Figshare 初筛命中 {len(items)} 条记录，正在获取详情并进行 AI 研判...\")\n        \n        if not items:\n            logger.error(\"警告：Figshare 返回了 0 条结果。\")\n            return []\n            \n    except Exception as e:\n        logger.error(f\"Figshare 检索请求彻底失败: {e}\")\n        return []\n\n    valid_datasets = []\n    \n    for item in items:\n        item_id = item.get('id')\n        detail_url = f\"{FIGSHARE_BASE_URL}/articles/{item_id}\"\n        \n        try:\n            # 详情请求使用 GET，同样带上规范的 Headers\n            detail_res = requests.get(\n                detail_url, \n                headers=headers,\n                timeout=20\n            )\n            \n            if detail_res.status_code != 200:\n                logger.info(f\"跳过 ID {item_id}: 获取详情失败，状态码 {detail_res.status_code}\")\n                continue\n                \n            detail_data = detail_res.json()\n            title = detail_data.get('title', '')\n            description = detail_data.get('description', '')\n            \n            clean_desc = re.sub('<[^<]+>', '', description)\n            \n            # 调用 DeepSeek 进行智能研判\n            llm_judgment = await evaluate_figshare_data(title, clean_desc)\n            \n            if llm_judgment.get(\"is_valid_omics_data\") and llm_judgment.get(\"is_tcm_related\"):\n                logger.info(f\"[命中] ID: {item_id} | {title[:40]}... | 理由: {llm_judgment.get('reasoning')}\")\n                \n                files_info = []\n                for f in detail_data.get('files', []):\n                    files_info.append({\n                        \"file_name\": f.get('name'),\n                        \"download_url\": f.get('download_url'),\n                        \"size_bytes\": f.get('size')\n                    })\n                \n                dataset_record = {\n                    \"figshare_id\": item_id,\n                    \"title\": title,\n                    \"doi\": detail_data.get('doi', ''),\n                    \"omics_type\": llm_judgment.get(\"omics_type\"),\n                    \"organism\": llm_judgment.get(\"organism\"),\n                    \"files\": json.dumps(files_info),\n                    \"figshare_url\": detail_data.get('figshare_url')\n                }\n                valid_datasets.append(dataset_record)\n            else:\n                logger.error(f\"[排除] ID: {item_id} | 理由: {llm_judgment.get('reasoning')}\")\n                \n        except Exception as e:\n            logger.error(f\"处理条目 {item_id} 时发生异常: {e}\")\n            \n        # 保持温柔的访问频率，避免触发并发限制\n        time.sleep(1)\n\n    return valid_datasets\n# ==========================================\n# 核心功能 3：自动下载符合条件的数据文件\n# ==========================================\ndef download_figshare_files(datasets: list, download_dir: str = \"figshare_downloads\"):\n    \"\"\"\n    根据过滤后的数据集信息，自动下载实际的文件\n    \"\"\"\n    if not datasets:\n        logger.warning(\"没有需要下载的数据。\")\n        return\n        \n    os.makedirs(download_dir, exist_ok=True)\n    \n    for ds in datasets:\n        item_id = ds['figshare_id']\n        files_info = json.loads(ds['files'])\n        \n        # 为每个数据集创建一个独立的文件夹\n        item_dir = os.path.join(download_dir, str(item_id))\n        os.makedirs(item_dir, exist_ok=True)\n        \n        logger.info(f\"\\n准备下载数据集 {item_id} ({len(files_info)} 个文件)...\")\n        \n        for f in files_info:\n            file_name = f['file_name']\n            download_url = f['download_url']\n            save_path = os.path.join(item_dir, file_name)\n            \n            if not download_url:\n                continue\n                \n            logger.info(f\" -> 开始下载 {file_name} ...\")\n            try:\n                # 采用流式下载处理大文件\n                with requests.get(download_url, stream=True, timeout=30) as r:\n                    r.raise_for_status()\n                    with open(save_path, 'wb') as f_out:\n                        for chunk in r.iter_content(chunk_size=8192):\n                            f_out.write(chunk)\n                logger.info(f\" -> {file_name} 下载完成!\")\n            except Exception as e:\n                logger.error(f\" -> {file_name} 下载失败: {e}\")\n\n# ==========================================\n# 主流程\n# ==========================================\nasync def main():\n    # 使用宽泛的搜索词，脏活累活交给后面的 DeepSeek 处理\n    query = '(\"Traditional Chinese Medicine\" OR \"herb\" OR \"phytomedicine\") AND (\"RNA-seq\" OR \"transcriptome\" OR \"single-cell\" OR \"spatial transcriptomics\" OR \"proteomics\" OR \"metabolomics\" OR \"epigenomics\" OR \"methylation\")'\n    logger.info(f\"开始独立执行 Figshare 孤儿数据集抓取，检索词: {query}\")\n\n    # 1. 抓取并 AI 过滤\n    valid_data = await fetch_and_filter_figshare(query, limit=999) # 测试时先搜50条\n    \n    # 2. 保存元数据到 CSV\n    if valid_data:\n        df = pd.DataFrame(valid_data)\n        csv_name = \"figshare_tcm_omics_datasets.csv\"\n        df.to_csv(csv_name, index=False)\n        logger.info(f\"成功清洗出 {len(valid_data)} 个高质量数据集，元数据已保存至 {csv_name}\")\n        \n        # 3. 如果你想立刻下载文件，可以取消注释下面这行：\n        download_figshare_files(valid_data)\n    else:\n        logger.error(\"\\n未找到符合 AI 标准的高质量数据集。\")\n\nif __name__ == \"__main__\":\n    if os.name == 'nt':\n        asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())\n    asyncio.run(main())\n"}}}}