{"repo": "bcefghj/interview-experience-spider", "files": {"config.example.json": {"79a681316ee347dd00245cf0c99550d0a96ca4f3": {"t": 1785603940.1530845, "n": 1040, "h": "5de10c3d4fee6689", "c": "{\n  \"_comment\": \"复制此文件为 config.json，填入你的配置。config.json 已加入 .gitignore，不会被上传。\",\n\n  \"nowcoder\": {\n    \"queries\": [\n      \"字节飞书后端实习\",\n      \"字节飞书 golang 面经\",\n      \"字节飞书服务端实习面经\",\n      \"字节飞书暑期实习面经\"\n    ],\n    \"max_pages\": 3,\n    \"delay_seconds\": 0.8\n  },\n\n  \"xiaohongshu\": {\n    \"enabled\": true,\n    \"_cookie_help\": \"见 docs/get-xhs-cookies.md，Cookie 约1天过期\",\n    \"cookies\": {\n      \"a1\": \"填入你的 a1 cookie 值\",\n      \"web_session\": \"填入你的 web_session cookie 值\",\n      \"webId\": \"填入你的 webId cookie 值\",\n      \"gid\": \"填入你的 gid cookie 值\"\n    },\n    \"queries\": [\n      \"字节飞书面试\",\n      \"字节飞书后端面经\",\n      \"字节飞书实习 golang\"\n    ],\n    \"max_pages\": 3,\n    \"delay_seconds\": 0.8\n  },\n\n  \"filter\": {\n    \"mode\": \"all\",\n    \"_mode_options\": \"all = 全部保留(去掉运营销售等) | backend = 只保留后端/Go相关\",\n    \"backend_keywords\": [\"后端\", \"golang\", \"go语言\", \"服务端\", \"backend\", \"全栈\"],\n    \"exclude_keywords\": [\"运营\", \"销售\", \"ux设计\", \"产品经理\", \"数据分析\", \"市场营销\"]\n  },\n\n  \"output\": {\n    \"dir\": \"./output\",\n    \"title\": \"面试备考手册\",\n    \"formats\": [\"md\", \"html\"],\n    \"max_content_length\": 2000\n  }\n}\n"}}, "generate_report.py": {"c984b42597f21f331271aaa47f8a19b17fcf9e5c": {"t": 1785603941.445472, "n": 13706, "h": "55b5d6e43a7b4f5e", "c": "#!/usr/bin/env python3\n\"\"\"\nMD/HTML 报告生成器\n用法：\n  python3 generate_report.py                                    # 使用 config.json\n  python3 generate_report.py --nc output/nc.json --xhs output/xhs.json\n  python3 generate_report.py --title \"字节飞书备考\" --filter backend\n\"\"\"\nimport json, re, os, sys, argparse\n\ntry:\n    import markdown as md_lib\nexcept ImportError:\n    print(\"❌ 缺少依赖: pip3 install markdown\")\n    sys.exit(1)\n\nBACKEND_KWS = [\"后端\", \"golang\", \"go语言\", \"服务端\", \"backend\", \"全栈\", \"开发工程师\"]\nEXCLUDE_KWS = [\"运营\", \"销售\", \"ux设计\", \"产品经理\", \"数据分析\", \"市场营销\", \"人力资源\"]\n\n\ndef filter_posts(posts: list, mode: str = \"all\") -> list:\n    if mode == \"backend\":\n        return [d for d in posts if any(\n            kw in (d.get(\"title\", \"\") + d.get(\"content\", \"\")).lower()\n            for kw in BACKEND_KWS\n        )]\n    else:  # all\n        return [d for d in posts if not any(\n            kw in (d.get(\"title\", \"\") + d.get(\"content\", \"\")).lower()\n            for kw in EXCLUDE_KWS\n        )]\n\n\ndef build_md(nc_posts: list, xhs_posts: list, title: str,\n             max_len: int = 2000) -> str:\n    lines = []\n    L = lines.append\n\n    nc_cnt  = len(nc_posts)\n    xhs_cnt = len(xhs_posts)\n\n    L(f\"# {title}\")\n    L(\"\")\n    L(f\"> 牛客网 **{nc_cnt}** 篇 + 小红书 **{xhs_cnt}** 篇，共 **{nc_cnt + xhs_cnt}** 篇真实面经\")\n    L(\"\")\n    L(\"---\")\n    L(\"\")\n\n    # 高频考点\n    L(\"## 第一部分：高频考点速查\")\n    L(\"\")\n\n    topics = [\n        (\"Golang / Go\", [\n            \"GMP 调度模型（G/M/P 的含义 + work stealing）\",\n            \"Goroutine vs 线程的区别\",\n            \"Channel 底层实现（hchan 结构 + 阻塞原理）\",\n            \"map 并发不安全 → sync.Map / Mutex 方案\",\n            \"GC 三色标记法 + 写屏障\",\n            \"slice 扩容规则（<256 翻倍，>256 增长 1.25）\",\n            \"defer LIFO + 闭包陷阱\",\n            \"interface 底层 iface/eface\",\n            \"内存逃逸分析\",\n        ]),\n        (\"MySQL\", [\n            \"B+ 树索引结构 + 为什么不用 B 树\",\n            \"聚簇索引 vs 非聚簇索引（回表/覆盖索引）\",\n            \"联合索引最左前缀 + 索引失效场景\",\n            \"MVCC 原理（undo log + read view）\",\n            \"四种隔离级别 + 幻读解决方案\",\n            \"redo log / undo log / bin log 区别\",\n            \"行锁 / 表锁 / 间隙锁 / Next-Key Lock\",\n            \"慢 SQL 排查：explain + 索引优化\",\n        ]),\n        (\"Redis\", [\n            \"为什么快：单线程 + IO多路复用 + 内存操作\",\n            \"五种数据结构底层实现\",\n            \"缓存穿透/击穿/雪崩 三问 ★★★\",\n            \"RDB vs AOF 持久化\",\n            \"分布式锁：SET NX EX + Redlock\",\n            \"内存淘汰：LRU vs LFU\",\n            \"热 Key / 大 Key 处理\",\n            \"集群：主从/哨兵/Cluster\",\n        ]),\n        (\"计算机网络\", [\n            \"TCP 三次握手/四次挥手 ★★★\",\n            \"TIME_WAIT（2MSL 原因）\",\n            \"TCP 可靠传输：序号/确认/重传/流控/拥塞控制\",\n            \"HTTP 1.0/1.1/2.0/3.0 区别\",\n            \"HTTPS TLS 握手过程\",\n            \"URL → 页面全流程 ★★★\",\n            \"DNS 解析（UDP 53 端口）\",\n        ]),\n        (\"操作系统\", [\n            \"进程/线程/协程区别\",\n            \"IO 多路复用：select/poll/epoll 区别\",\n            \"虚拟内存 + 缺页中断\",\n            \"死锁四条件 + 预防\",\n        ]),\n        (\"系统设计\", [\n            \"秒杀系统设计（防超卖）★★★\",\n            \"分布式 ID（雪花算法）\",\n            \"限流：令牌桶/漏桶/滑动窗口\",\n            \"短链服务设计\",\n            \"敏感词过滤（Trie）\",\n        ]),\n        (\"算法（手撕）\", [\n            \"★★★ LC3 无重复最长子串（滑动窗口）\",\n            \"★★★ LC128 最长连续序列（HashSet O(n)）\",\n            \"★★★ LC25 K个一组翻转链表\",\n            \"★★★ LC146 LRU 缓存\",\n            \"★★ LC102 二叉树层序遍历\",\n            \"★★ LC76 最小覆盖子串\",\n        ]),\n    ]\n\n    for i, (topic, qs) in enumerate(topics):\n        L(f\"### {i+1}. {topic}\")\n        L(\"\")\n        for q in qs:\n            L(f\"- {q}\")\n        L(\"\")\n\n    L(\"---\")\n    L(\"\")\n\n    # 牛客面经\n    L(f\"## 第二部分：牛客网面经（共 {nc_cnt} 篇，节选30篇原文）\")\n    L(\"\")\n    for i, d in enumerate(nc_posts[:30]):\n        title   = d.get(\"title\", \"无标题\")\n        url     = d.get(\"url\", \"\")\n        content = d.get(\"content\", \"\").strip()\n        if len(content) > max_len:\n            content = content[:max_len] + \"\\n\\n...（内容截断，完整见原链接）\"\n        L(f\"### {i+1}. {title}\")\n        L(\"\")\n        if url:\n            L(f\"> 来源：[{url}]({url})\")\n            L(\"\")\n        L(content)\n        L(\"\")\n        L(\"---\")\n        L(\"\")\n\n    # 小红书面经\n    L(f\"## 第三部分：小红书面经（共 {xhs_cnt} 篇，节选30篇原文）\")\n    L(\"\")\n    for i, d in enumerate(xhs_posts[:30]):\n        title   = d.get(\"title\", \"无标题\")\n        url     = d.get(\"url\", \"\")\n        content = d.get(\"content\", \"\").strip()\n        if len(content) > max_len:\n            content = content[:max_len] + \"\\n\\n...（内容截断）\"\n        L(f\"### {i+1}. {title}\")\n        L(\"\")\n        if url:\n            L(f\"> 来源：[小红书]({url})\")\n            L(\"\")\n        L(content)\n        L(\"\")\n        L(\"---\")\n        L(\"\")\n\n    # 附录\n    L(\"## 附录：全部面经索引\")\n    L(\"\")\n    L(f\"### 牛客网（{nc_cnt} 篇）\")\n    L(\"\")\n    for i, d in enumerate(nc_posts):\n        L(f\"{i+1}. [{d.get('title','无标题')}]({d.get('url','')})\")\n    L(\"\")\n    L(f\"### 小红书（{xhs_cnt} 篇）\")\n    L(\"\")\n    for i, d in enumerate(xhs_posts):\n        L(f\"{i+1}. [{d.get('title','无标题')}]({d.get('url','')})\")\n\n    return \"\\n\".join(lines)\n\n\n_CSS = \"\"\"\n:root{--bg:#0f1117;--sb:#161b22;--card:#1c2128;--bdr:#30363d;--tx:#e6edf3;--mt:#8b949e;--ac:#58a6ff;--ac2:#3fb950;--ac3:#d29922;}\n*{box-sizing:border-box;margin:0;padding:0;}\nbody{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI','PingFang SC','Microsoft YaHei',sans-serif;background:var(--bg);color:var(--tx);display:flex;min-height:100vh;font-size:15px;line-height:1.7;}\n#sb{width:270px;min-width:270px;background:var(--sb);border-right:1px solid var(--bdr);position:fixed;top:0;left:0;bottom:0;overflow-y:auto;padding:16px 0;z-index:100;}\n#sbh{padding:0 14px 14px;border-bottom:1px solid var(--bdr);margin-bottom:6px;}\n#sbh h2{font-size:12px;color:var(--mt);text-transform:uppercase;letter-spacing:.8px;}\n#sb ul{list-style:none;}\n#sb a{display:block;padding:5px 14px;color:var(--mt);text-decoration:none;font-size:13px;border-left:3px solid transparent;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;transition:all .15s;}\n#sb a:hover,#sb a.on{color:var(--ac);background:rgba(88,166,255,.07);border-left-color:var(--ac);}\n.h2a{font-weight:600;color:var(--tx)!important;}\n.h3a{padding-left:26px!important;font-size:12px!important;}\n#mn{margin-left:270px;flex:1;max-width:940px;padding:36px 48px;}\n.hero{background:linear-gradient(135deg,#1a2a4a,#0d1b2e);border:1px solid #1f6feb;border-radius:12px;padding:22px 26px;margin-bottom:28px;}\n.hero-t{font-size:20px;color:var(--ac);font-weight:700;margin-bottom:6px;}\n.badge{display:inline-block;padding:3px 9px;border-radius:20px;font-size:12px;font-weight:600;margin:3px 3px 0 0;}\n.bb{background:rgba(88,166,255,.15);color:#58a6ff;border:1px solid rgba(88,166,255,.3);}\n.bg{background:rgba(63,185,80,.15);color:#3fb950;border:1px solid rgba(63,185,80,.3);}\n.by{background:rgba(210,153,34,.15);color:#d29922;border:1px solid rgba(210,153,34,.3);}\nh1{display:none;}\nh2{font-size:19px;color:var(--ac);margin:36px 0 14px;padding-bottom:7px;border-bottom:1px solid var(--bdr);scroll-margin-top:20px;}\nh3{font-size:15px;color:var(--ac2);margin:22px 0 10px;scroll-margin-top:20px;}\np{margin:0 0 10px;}\nul,ol{padding-left:20px;margin:7px 0 10px;}\nli{margin:3px 0;}\na{color:var(--ac);text-decoration:none;}\na:hover{text-decoration:underline;}\nhr{border:none;border-top:1px solid var(--bdr);margin:28px 0;}\nblockquote{border-left:3px solid var(--ac3);padding:7px 14px;background:rgba(210,153,34,.07);border-radius:0 6px 6px 0;color:var(--mt);margin:10px 0;font-size:14px;}\ntable{width:100%;border-collapse:collapse;margin:14px 0;font-size:14px;}\nth{background:var(--card);color:var(--ac);padding:8px 11px;border:1px solid var(--bdr);text-align:left;}\ntd{padding:8px 11px;border:1px solid var(--bdr);vertical-align:top;}\ntr:nth-child(even) td{background:rgba(255,255,255,.015);}\ncode{font-family:'JetBrains Mono','Fira Code',Consolas,monospace;font-size:13px;background:var(--card);border:1px solid var(--bdr);padding:2px 5px;border-radius:4px;color:#e3b341;}\npre{background:var(--card);border:1px solid var(--bdr);border-radius:8px;padding:14px 18px;overflow-x:auto;margin:10px 0 14px;}\npre code{background:none;border:none;padding:0;color:#e6edf3;}\n#srch{width:100%;padding:7px 11px;background:var(--bg);border:1px solid var(--bdr);border-radius:6px;color:var(--tx);font-size:13px;margin-bottom:10px;outline:none;}\n#srch:focus{border-color:var(--ac);}\n#top{position:fixed;bottom:22px;right:22px;background:var(--ac);color:#000;width:38px;height:38px;border-radius:50%;display:flex;align-items:center;justify-content:center;font-size:17px;font-weight:bold;box-shadow:0 3px 10px rgba(0,0,0,.4);text-decoration:none;opacity:.8;}\n#top:hover{opacity:1;transform:scale(1.1);}\n#prog{position:fixed;top:0;left:270px;right:0;height:3px;background:linear-gradient(90deg,var(--ac),var(--ac2));transform-origin:left;transform:scaleX(0);transition:transform .1s;z-index:200;}\n@media(max-width:768px){#sb{display:none;}#mn{margin-left:0;padding:18px;}#prog{left:0;}}\n\"\"\"\n\n_JS = \"\"\"\nwindow.addEventListener('scroll',()=>{\n  const d=document.documentElement;\n  document.getElementById('prog').style.transform=`scaleX(${d.scrollTop/(d.scrollHeight-d.clientHeight)||0})`;\n});\nconst hs=document.querySelectorAll('h2,h3');\nconst ls=document.querySelectorAll('#sb a');\nhs.forEach(h=>new IntersectionObserver(es=>{es.forEach(e=>{\n  if(e.isIntersecting){ls.forEach(l=>l.classList.remove('on'));\n    const a=document.querySelector(`#sb a[href=\"#${e.target.id}\"]`);\n    if(a){a.classList.add('on');a.scrollIntoView({block:'nearest'});}\n  }\n});},{rootMargin:'-20px 0px -80% 0px'}).observe(h));\nfunction fs(q){q=q.toLowerCase();document.querySelectorAll('#sb li').forEach(li=>{\n  li.style.display=(!q||li.textContent.toLowerCase().includes(q))?'':'none';\n});}\n\"\"\"\n\n\ndef build_html(md_text: str, title: str, nc_cnt: int, xhs_cnt: int) -> str:\n    html_body = md_lib.markdown(\n        md_text,\n        extensions=[\"tables\", \"fenced_code\", \"toc\", \"nl2br\"],\n        extension_configs={\"toc\": {\"toc_depth\": \"2-3\"}}\n    )\n    toc_items = []\n    for m in re.finditer(r'<h([23])[^>]*id=\"([^\"]*)\"[^>]*>(.*?)</h\\1>', html_body, re.DOTALL):\n        lv, anc, txt = m.group(1), m.group(2), m.group(3)\n        clean = re.sub(r\"<[^>]+>\", \"\", txt).strip()\n        cls   = \"h2a\" if lv == \"2\" else \"h3a\"\n        toc_items.append(f'<li><a href=\"#{anc}\" class=\"{cls}\">{clean}</a></li>')\n    toc = \"<ul>\" + \"\".join(toc_items) + \"</ul>\"\n\n    return f\"\"\"<!DOCTYPE html>\n<html lang=\"zh-CN\">\n<head>\n<meta charset=\"UTF-8\">\n<meta name=\"viewport\" content=\"width=device-width,initial-scale=1.0\">\n<title>{title}</title>\n<style>{_CSS}</style>\n</head>\n<body>\n<div id=\"prog\"></div>\n<nav id=\"sb\">\n  <div id=\"sbh\">\n    <h2>📚 目录导航</h2>\n    <input id=\"srch\" type=\"text\" placeholder=\"搜索章节...\" oninput=\"fs(this.value)\">\n  </div>\n  {toc}\n</nav>\n<main id=\"mn\">\n  <div class=\"hero\">\n    <div class=\"hero-t\">{title}</div>\n    <p style=\"margin-top:7px\">\n      <span class=\"badge bb\">牛客网：{nc_cnt} 篇</span>\n      <span class=\"badge bg\">小红书：{xhs_cnt} 篇</span>\n      <span class=\"badge by\">合计：{nc_cnt + xhs_cnt} 篇</span>\n    </p>\n  </div>\n  {html_body}\n</main>\n<a href=\"#\" id=\"top\" title=\"返回顶部\">↑</a>\n<script>{_JS}</script>\n</body>\n</html>\"\"\"\n\n\ndef run(nc_file: str, xhs_file: str, output_dir: str, title: str,\n        filter_mode: str = \"all\", max_len: int = 2000,\n        formats: list = None, verbose: bool = True) -> dict:\n    \"\"\"生成报告\"\"\"\n    formats = formats or [\"md\", \"html\"]\n    os.makedirs(output_dir, exist_ok=True)\n\n    nc_posts  = json.load(open(nc_file,  encoding=\"utf-8\")) if os.path.exists(nc_file)  else []\n    xhs_posts = json.load(open(xhs_file, encoding=\"utf-8\")) if os.path.exists(xhs_file) else []\n\n    nc_posts  = filter_posts(nc_posts,  filter_mode)\n    xhs_posts = filter_posts(xhs_posts, filter_mode)\n\n    if verbose:\n        print(f\"📊 过滤模式: {filter_mode}\")\n        print(f\"   牛客: {len(nc_posts)} 篇 | 小红书: {len(xhs_posts)} 篇\")\n\n    md_text = build_md(nc_posts, xhs_posts, title, max_len)\n    results = {}\n\n    safe_title = re.sub(r'[\\\\/:*?\"<>|]', '_', title)\n\n    if \"md\" in formats:\n        md_path = os.path.join(output_dir, f\"{safe_title}.md\")\n        with open(md_path, \"w\", encoding=\"utf-8\") as f:\n            f.write(md_text)\n        results[\"md\"] = md_path\n        if verbose:\n            print(f\"✅ MD  → {md_path} ({len(md_text)//1024}KB)\")\n\n    if \"html\" in formats:\n        html_text = build_html(md_text, title, len(nc_posts), len(xhs_posts))\n        html_path = os.path.join(output_dir, f\"{safe_title}.html\")\n        with open(html_path, \"w\", encoding=\"utf-8\") as f:\n            f.write(html_text)\n        results[\"html\"] = html_path\n        if verbose:\n            print(f\"✅ HTML → {html_path} ({len(html_text)//1024}KB)\")\n\n    return results\n\n\ndef main():\n    parser = argparse.ArgumentParser(description=\"MD/HTML 报告生成器\")\n    parser.add_argument(\"--config\",  default=\"config.json\")\n    parser.add_argument(\"--nc\",      default=\"output/nowcoder.json\",     help=\"牛客数据 JSON\")\n    parser.add_argument(\"--xhs\",     default=\"output/xiaohongshu.json\",  help=\"小红书数据 JSON\")\n    parser.add_argument(\"--title\",   help=\"报告标题\")\n    parser.add_argument(\"--filter\",  choices=[\"all\", \"backend\"], default=\"all\")\n    parser.add_argument(\"--output\",  help=\"输出目录\")\n    args = parser.parse_args()\n\n    cfg = {}\n    if os.path.exists(args.config):\n        with open(args.config, encoding=\"utf-8\") as f:\n            cfg = json.load(f)\n\n    title      = args.title or cfg.get(\"output\", {}).get(\"title\", \"面经备考手册\")\n    output_dir = args.output or cfg.get(\"output\", {}).get(\"dir\", \"output\")\n    formats    = cfg.get(\"output\", {}).get(\"formats\", [\"md\", \"html\"])\n    filter_m   = cfg.get(\"filter\", {}).get(\"mode\", args.filter)\n    max_len    = cfg.get(\"output\", {}).get(\"max_content_length\", 2000)\n\n    run(nc_file=args.nc, xhs_file=args.xhs, output_dir=output_dir,\n        title=title, filter_mode=filter_m, max_len=max_len, formats=formats)\n\n\nif __name__ == \"__main__\":\n    main()\n"}}, "mcp_server.py": {"5e81eb8db7ffc59bc431f3912a31dcbad679197d": {"t": 1785603942.6252022, "n": 7477, "h": "a89de158df916eec", "c": "#!/usr/bin/env python3\n\"\"\"\n面经猎手 MCP Server\n让 Cursor / Claude 可以直接调用本工具\n\n配置方法（~/.cursor/mcp.json）：\n{\n  \"mcpServers\": {\n    \"interview-spider\": {\n      \"command\": \"python3\",\n      \"args\": [\"/path/to/interview-experience-spider/mcp_server.py\"]\n    }\n  }\n}\n\n然后在 Cursor 中说：\"帮我搜索字节飞书后端面经\"\n\"\"\"\nimport os, sys, json\n\nos.chdir(os.path.dirname(os.path.abspath(__file__)))\n\ntry:\n    from mcp.server.fastmcp import FastMCP\nexcept ImportError:\n    print(\"❌ 缺少 mcp 库，请运行: pip3 install mcp\", file=sys.stderr)\n    sys.exit(1)\n\nmcp = FastMCP(\"面经猎手 Interview Spider\")\n\nCONFIG_FILE = os.path.join(os.path.dirname(__file__), \"config.json\")\nOUTPUT_DIR  = \"output\"\n\n\ndef _load_config() -> dict:\n    if os.path.exists(CONFIG_FILE):\n        with open(CONFIG_FILE, encoding=\"utf-8\") as f:\n            return json.load(f)\n    return {}\n\n\n# ===== Tool 1: 搜索牛客网 =====\n@mcp.tool()\ndef search_nowcoder(\n    query: str,\n    pages: int = 3,\n    output_file: str = \"output/nowcoder.json\"\n) -> str:\n    \"\"\"\n    搜索牛客网面经，并保存到 JSON 文件。\n    \n    参数:\n      query       - 搜索关键词，如\"字节飞书后端实习\"\n      pages       - 搜索页数（默认3页，每页约20条）\n      output_file - 结果保存路径\n    \n    返回:\n      抓取结果摘要\n    \"\"\"\n    try:\n        from scrape_nowcoder import run\n        posts = run(\n            queries=[query],\n            max_pages=pages,\n            delay=0.8,\n            output_file=output_file,\n            verbose=False\n        )\n        titles = [f\"• {p['title'][:40]}\" for p in posts[:5]]\n        return (\n            f\"✅ 牛客网搜索完成\\n\"\n            f\"关键词：{query}\\n\"\n            f\"共找到 {len(posts)} 篇面经\\n\"\n            f\"保存到：{os.path.abspath(output_file)}\\n\\n\"\n            f\"前5篇标题：\\n\" + \"\\n\".join(titles)\n        )\n    except Exception as e:\n        return f\"❌ 搜索失败: {e}\"\n\n\n# ===== Tool 2: 搜索小红书 =====\n@mcp.tool()\ndef search_xiaohongshu(\n    query: str,\n    pages: int = 3,\n    output_file: str = \"output/xiaohongshu.json\"\n) -> str:\n    \"\"\"\n    搜索小红书面经（需要在 config.json 中配置 Cookie）。\n    \n    参数:\n      query       - 搜索关键词，如\"字节飞书面试\"\n      pages       - 搜索页数\n      output_file - 结果保存路径\n    \n    Cookie 配置见 docs/get-xhs-cookies.md\n    \"\"\"\n    cfg     = _load_config()\n    cookies = cfg.get(\"xiaohongshu\", {}).get(\"cookies\", {})\n    if not cookies.get(\"a1\") or \"填入\" in cookies.get(\"a1\", \"\"):\n        return (\n            \"❌ 小红书 Cookie 未配置\\n\"\n            \"请编辑 config.json，填入 a1/web_session/webId/gid\\n\"\n            \"获取方法见 docs/get-xhs-cookies.md\"\n        )\n    try:\n        from scrape_xhs import run\n        posts = run(\n            queries=[query],\n            cookies=cookies,\n            max_pages=pages,\n            delay=0.8,\n            output_file=output_file,\n            verbose=False\n        )\n        titles = [f\"• {p['title'][:40]}\" for p in posts[:5]]\n        return (\n            f\"✅ 小红书搜索完成\\n\"\n            f\"关键词：{query}\\n\"\n            f\"共找到 {len(posts)} 篇\\n\"\n            f\"保存到：{os.path.abspath(output_file)}\\n\\n\"\n            f\"前5篇标题：\\n\" + \"\\n\".join(titles)\n        )\n    except Exception as e:\n        return f\"❌ 搜索失败: {e}\"\n\n\n# ===== Tool 3: 生成报告 =====\n@mcp.tool()\ndef generate_report(\n    title: str = \"面经备考手册\",\n    nc_file: str = \"output/nowcoder.json\",\n    xhs_file: str = \"output/xiaohongshu.json\",\n    filter_mode: str = \"all\"\n) -> str:\n    \"\"\"\n    根据已抓取的面经数据，生成 MD + HTML 报告。\n    \n    参数:\n      title       - 报告标题\n      nc_file     - 牛客数据 JSON 路径\n      xhs_file    - 小红书数据 JSON 路径\n      filter_mode - \"all\"（全部）或 \"backend\"（只保留后端/Go相关）\n    \n    返回:\n      生成的文件路径\n    \"\"\"\n    try:\n        from generate_report import run\n        results = run(\n            nc_file=nc_file,\n            xhs_file=xhs_file,\n            output_dir=OUTPUT_DIR,\n            title=title,\n            filter_mode=filter_mode,\n            formats=[\"md\", \"html\"],\n            verbose=False\n        )\n        lines = [\"✅ 报告生成完成\"]\n        for fmt, path in results.items():\n            lines.append(f\"  {fmt.upper()} → {os.path.abspath(path)}\")\n        return \"\\n\".join(lines)\n    except Exception as e:\n        return f\"❌ 生成失败: {e}\"\n\n\n# ===== Tool 4: 一键全流程 =====\n@mcp.tool()\ndef collect_all(\n    company: str = \"字节飞书\",\n    position: str = \"后端实习\",\n    filter_mode: str = \"all\"\n) -> str:\n    \"\"\"\n    一键全流程：自动构建搜索关键词 → 爬取牛客+小红书 → 生成 HTML 报告。\n    \n    参数:\n      company     - 公司名，如\"字节飞书\"、\"腾讯微信\"\n      position    - 岗位名，如\"后端实习\"、\"golang开发\"\n      filter_mode - \"all\" 或 \"backend\"\n    \n    用法示例：\n      collect_all(\"字节飞书\", \"服务端实习\")\n      collect_all(\"腾讯\", \"C++客户端\")\n    \"\"\"\n    cfg     = _load_config()\n    cookies = cfg.get(\"xiaohongshu\", {}).get(\"cookies\", {})\n    has_xhs = bool(cookies.get(\"a1\") and \"填入\" not in cookies.get(\"a1\", \"\"))\n\n    queries = [\n        f\"{company}{position}\",\n        f\"{company} {position} 面经\",\n        f\"{company} {position} golang\",\n        f\"{company} {position} 一面 二面\",\n    ]\n\n    results_summary = [f\"🚀 开始收集：{company} {position}\\n\"]\n\n    # 爬取牛客\n    try:\n        from scrape_nowcoder import run as nc_run\n        nc_posts = nc_run(\n            queries=queries,\n            max_pages=cfg.get(\"nowcoder\", {}).get(\"max_pages\", 3),\n            delay=0.8,\n            output_file=f\"output/nowcoder.json\",\n            verbose=False\n        )\n        results_summary.append(f\"✅ 牛客网：{len(nc_posts)} 篇\")\n    except Exception as e:\n        results_summary.append(f\"❌ 牛客网失败: {e}\")\n        nc_posts = []\n\n    # 爬取小红书\n    if has_xhs:\n        try:\n            from scrape_xhs import run as xhs_run\n            xhs_posts = xhs_run(\n                queries=queries,\n                cookies=cookies,\n                max_pages=cfg.get(\"xiaohongshu\", {}).get(\"max_pages\", 3),\n                delay=0.8,\n                output_file=f\"output/xiaohongshu.json\",\n                verbose=False\n            )\n            results_summary.append(f\"✅ 小红书：{len(xhs_posts)} 篇\")\n        except Exception as e:\n            results_summary.append(f\"❌ 小红书失败: {e}\")\n    else:\n        results_summary.append(\"⚠️  小红书未配置 Cookie，已跳过\")\n\n    # 生成报告\n    try:\n        from generate_report import run as report_run\n        title   = f\"{company} {position} 面经备考手册\"\n        results = report_run(\n            nc_file=\"output/nowcoder.json\",\n            xhs_file=\"output/xiaohongshu.json\",\n            output_dir=OUTPUT_DIR,\n            title=title,\n            filter_mode=filter_mode,\n            formats=[\"md\", \"html\"],\n            verbose=False\n        )\n        for fmt, path in results.items():\n            results_summary.append(f\"📄 {fmt.upper()} → {os.path.abspath(path)}\")\n    except Exception as e:\n        results_summary.append(f\"❌ 报告生成失败: {e}\")\n\n    return \"\\n\".join(results_summary)\n\n\n# ===== Tool 5: 查看已有数据 =====\n@mcp.tool()\ndef list_collected(\n    source: str = \"all\"\n) -> str:\n    \"\"\"\n    查看已收集的面经数据摘要。\n    \n    参数:\n      source - \"nowcoder\"、\"xiaohongshu\" 或 \"all\"\n    \"\"\"\n    lines = []\n    files = {\n        \"nowcoder\":    \"output/nowcoder.json\",\n        \"xiaohongshu\": \"output/xiaohongshu.json\",\n    }\n    for src, path in files.items():\n        if source not in (\"all\", src):\n            continue\n        if os.path.exists(path):\n            data = json.load(open(path, encoding=\"utf-8\"))\n            lines.append(f\"📁 {src}（{len(data)} 篇）→ {os.path.abspath(path)}\")\n            for i, d in enumerate(data[:5]):\n                lines.append(f\"   {i+1}. {d.get('title','无标题')[:45]}\")\n            if len(data) > 5:\n                lines.append(f\"   ... 共 {len(data)} 篇\")\n        else:\n            lines.append(f\"📁 {src} → 暂无数据（{path} 不存在）\")\n        lines.append(\"\")\n    return \"\\n\".join(lines) if lines else \"暂无已收集的数据\"\n\n\nif __name__ == \"__main__\":\n    mcp.run()\n"}}, "run_all.py": {"a0aa20012e154875ee71fb1e9a0ccf919381fc49": {"t": 1785603944.1451764, "n": 5147, "h": "90819316a0498905", "c": "#!/usr/bin/env python3\n\"\"\"\n🚀 一键运行：搜索 + 抓取 + 生成报告\n用法：\n  python3 run_all.py                  # 使用 config.json\n  python3 run_all.py --filter backend # 只保留后端/Go 相关\n  python3 run_all.py --skip-nc        # 跳过牛客，只爬小红书\n  python3 run_all.py --skip-xhs       # 跳过小红书，只爬牛客\n\"\"\"\nimport os, sys, json, argparse, subprocess\n\ndef check_env():\n    \"\"\"检查运行环境\"\"\"\n    print(\"🔍 检查运行环境...\")\n\n    # Python 依赖\n    missing = []\n    for pkg in [\"requests\", \"execjs\", \"markdown\"]:\n        try:\n            __import__(pkg.replace(\"-\", \"_\"))\n        except ImportError:\n            missing.append(pkg)\n    if missing:\n        print(f\"❌ 缺少 Python 依赖: {', '.join(missing)}\")\n        print(f\"   请运行: pip3 install {' '.join(missing)}\")\n        sys.exit(1)\n\n    # Node.js 依赖（小红书签名需要）\n    node_modules = os.path.join(os.path.dirname(__file__), \"Spider_XHS_signing\", \"node_modules\")\n    if not os.path.exists(node_modules):\n        print(\"⚠️  小红书签名模块未初始化\")\n        print(\"   正在自动初始化...\")\n        signing_dir = os.path.join(os.path.dirname(__file__), \"Spider_XHS_signing\")\n        result = subprocess.run([\"npm\", \"install\"], cwd=signing_dir, capture_output=True, text=True)\n        if result.returncode != 0:\n            print(f\"   ❌ npm install 失败: {result.stderr}\")\n            print(f\"   请手动运行: cd Spider_XHS_signing && npm install\")\n        else:\n            print(\"   ✅ Node.js 依赖安装完成\")\n\n    # config.json\n    if not os.path.exists(\"config.json\"):\n        print(\"⚠️  未找到 config.json\")\n        if os.path.exists(\"config.example.json\"):\n            print(\"   提示：请先执行 cp config.example.json config.json 并填写配置\")\n        sys.exit(1)\n\n    print(\"✅ 环境检查通过\\n\")\n\n\ndef main():\n    parser = argparse.ArgumentParser(description=\"面经猎手 - 一键运行\")\n    parser.add_argument(\"--config\",   default=\"config.json\")\n    parser.add_argument(\"--filter\",   choices=[\"all\", \"backend\"], help=\"过滤模式（覆盖 config.json）\")\n    parser.add_argument(\"--skip-nc\",  action=\"store_true\", help=\"跳过牛客网爬取\")\n    parser.add_argument(\"--skip-xhs\", action=\"store_true\", help=\"跳过小红书爬取\")\n    parser.add_argument(\"--report-only\", action=\"store_true\", help=\"只生成报告（用已有数据）\")\n    args = parser.parse_args()\n\n    os.chdir(os.path.dirname(os.path.abspath(__file__)))\n    check_env()\n\n    with open(args.config, encoding=\"utf-8\") as f:\n        cfg = json.load(f)\n\n    output_dir = cfg.get(\"output\", {}).get(\"dir\", \"output\")\n    os.makedirs(output_dir, exist_ok=True)\n\n    nc_file  = os.path.join(output_dir, \"nowcoder.json\")\n    xhs_file = os.path.join(output_dir, \"xiaohongshu.json\")\n\n    # ===== 步骤1：爬取牛客 =====\n    if not args.report_only and not args.skip_nc:\n        print(\"=\" * 50)\n        print(\"📥 步骤1：爬取牛客网面经\")\n        print(\"=\" * 50)\n        from scrape_nowcoder import run as nc_run\n        queries   = cfg.get(\"nowcoder\", {}).get(\"queries\", [\"字节飞书面经\"])\n        max_pages = cfg.get(\"nowcoder\", {}).get(\"max_pages\", 3)\n        delay     = cfg.get(\"nowcoder\", {}).get(\"delay_seconds\", 0.8)\n        nc_run(queries=queries, max_pages=max_pages, delay=delay, output_file=nc_file)\n        print()\n    else:\n        print(\"⏭️  跳过牛客网爬取\\n\")\n\n    # ===== 步骤2：爬取小红书 =====\n    if not args.report_only and not args.skip_xhs:\n        xhs_cfg = cfg.get(\"xiaohongshu\", {})\n        if xhs_cfg.get(\"enabled\", True):\n            cookies = xhs_cfg.get(\"cookies\", {})\n            if not cookies.get(\"a1\") or \"填入\" in cookies.get(\"a1\", \"\"):\n                print(\"⚠️  小红书 Cookie 未配置，跳过小红书爬取\")\n                print(\"   配置方法见 docs/get-xhs-cookies.md\\n\")\n            else:\n                print(\"=\" * 50)\n                print(\"📥 步骤2：爬取小红书面经\")\n                print(\"=\" * 50)\n                from scrape_xhs import run as xhs_run\n                queries   = xhs_cfg.get(\"queries\", [\"字节飞书面试\"])\n                max_pages = xhs_cfg.get(\"max_pages\", 3)\n                delay     = xhs_cfg.get(\"delay_seconds\", 0.8)\n                xhs_run(queries=queries, cookies=cookies, max_pages=max_pages,\n                        delay=delay, output_file=xhs_file)\n                print()\n        else:\n            print(\"⏭️  config.json 中 xiaohongshu.enabled=false，跳过\\n\")\n    else:\n        print(\"⏭️  跳过小红书爬取\\n\")\n\n    # ===== 步骤3：生成报告 =====\n    print(\"=\" * 50)\n    print(\"📝 步骤3：生成 MD/HTML 报告\")\n    print(\"=\" * 50)\n    from generate_report import run as report_run\n\n    title      = cfg.get(\"output\", {}).get(\"title\", \"面经备考手册\")\n    formats    = cfg.get(\"output\", {}).get(\"formats\", [\"md\", \"html\"])\n    max_len    = cfg.get(\"output\", {}).get(\"max_content_length\", 2000)\n    filter_m   = args.filter or cfg.get(\"filter\", {}).get(\"mode\", \"all\")\n\n    results = report_run(\n        nc_file=nc_file, xhs_file=xhs_file,\n        output_dir=output_dir, title=title,\n        filter_mode=filter_m, max_len=max_len, formats=formats\n    )\n\n    print()\n    print(\"=\" * 50)\n    print(\"🎉 全部完成！\")\n    print(\"=\" * 50)\n    for fmt, path in results.items():\n        print(f\"   {fmt.upper()} → {os.path.abspath(path)}\")\n\n    # 自动用浏览器打开 HTML\n    if \"html\" in results:\n        import subprocess\n        try:\n            subprocess.Popen([\"open\", results[\"html\"]])\n            print(\"\\n   已自动在浏览器打开 HTML 文件 🚀\")\n        except Exception:\n            pass\n\n\nif __name__ == \"__main__\":\n    main()\n"}}, "scrape_nowcoder.py": {"8fe322649d07bcfcabe18babb412c7ee8f2b882b": {"t": 1785603945.4394035, "n": 8488, "h": "ad7d3dcc4393ba4a", "c": "#!/usr/bin/env python3\n\"\"\"\n牛客网面经爬虫\n用法：\n  python3 scrape_nowcoder.py                          # 使用 config.json\n  python3 scrape_nowcoder.py --query \"字节飞书\" --pages 5\n  python3 scrape_nowcoder.py --output output/nc.json\n\"\"\"\nimport re, time, json, argparse, sys, os\n\ntry:\n    import requests\nexcept ImportError:\n    print(\"❌ 缺少依赖，请先运行: pip3 install requests\")\n    sys.exit(1)\n\nSEARCH_API  = \"https://gw-c.nowcoder.com/api/sparta/pc/search\"\nDISCUSS_API = \"https://gw-c.nowcoder.com/api/sparta/detail/content-data/detail\"\nFEED_URL    = \"https://www.nowcoder.com/feed/main/detail\"\n\nHEADERS = {\n    \"Content-Type\": \"application/json; charset=UTF-8\",\n    \"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36\",\n}\n\n\ndef html_to_text(html: str) -> str:\n    if not html:\n        return \"\"\n    html = re.sub(r\"<script[^>]*>.*?</script>\", \"\", html, flags=re.DOTALL | re.IGNORECASE)\n    html = re.sub(r\"<style[^>]*>.*?</style>\", \"\", html, flags=re.DOTALL | re.IGNORECASE)\n    html = re.sub(r\"</?(p|div|br|h[1-6]|li|tr)[^>]*>\", \"\\n\", html, flags=re.IGNORECASE)\n    html = re.sub(r\"<[^>]+>\", \"\", html)\n    for old, new in [(\"&nbsp;\", \" \"), (\"&amp;\", \"&\"), (\"&lt;\", \"<\"),\n                     (\"&gt;\", \">\"), (\"&quot;\", '\"'), (\"&#39;\", \"'\"), (\"\\xa0\", \" \")]:\n        html = html.replace(old, new)\n    html = re.sub(r\"\\n\\s*\\n\", \"\\n\\n\", html)\n    return html.strip()\n\n\ndef search(query: str, max_pages: int = 3, delay: float = 0.8) -> list:\n    \"\"\"搜索牛客网面经，返回帖子元信息列表\"\"\"\n    results = []\n    seen = set()\n    for page in range(1, max_pages + 1):\n        try:\n            payload = {\n                \"type\": \"all\",\n                \"query\": query,\n                \"page\": page,\n                \"tag\": [{\"name\": \"面经\", \"id\": 818, \"count\": None}],\n                \"order\": \"create\",\n                \"gioParams\": {\"searchFrom_var\": \"顶部导航栏\", \"searchEnter_var\": \"主站\"},\n            }\n            resp = requests.post(SEARCH_API, json=payload, headers=HEADERS, timeout=15)\n            data = resp.json()\n            if not data.get(\"success\"):\n                break\n            records = data.get(\"data\", {}).get(\"records\", [])\n            if not records:\n                break\n            for r in records:\n                rc_type = r.get(\"rc_type\", 0)\n                rd = r.get(\"data\", {})\n                if rc_type == 201:\n                    md = rd.get(\"momentData\", {})\n                    if md:\n                        uid   = md.get(\"uuid\", \"\")\n                        title = md.get(\"title\", \"\")\n                        if uid and uid not in seen:\n                            seen.add(uid)\n                            results.append({\"rc_type\": 201, \"uuid\": uid, \"title\": title, \"query\": query})\n                elif rc_type == 207:\n                    cd = rd.get(\"contentData\", {})\n                    if cd:\n                        cid   = str(cd.get(\"id\", \"\"))\n                        title = cd.get(\"title\", \"\")\n                        if cid and cid not in seen:\n                            seen.add(cid)\n                            results.append({\"rc_type\": 207, \"content_id\": cid, \"title\": title, \"query\": query})\n            total_page = data.get(\"data\", {}).get(\"totalPage\", 1)\n            if page >= total_page:\n                break\n            time.sleep(delay)\n        except Exception as e:\n            print(f\"  ⚠️  搜索第{page}页出错: {e}\")\n            break\n    return results\n\n\ndef fetch_feed(uuid: str) -> dict | None:\n    \"\"\"抓取 Feed 类型帖子正文\"\"\"\n    try:\n        resp = requests.get(\n            f\"{FEED_URL}/{uuid}\",\n            headers={\"User-Agent\": HEADERS[\"User-Agent\"]},\n            timeout=15\n        )\n        html = resp.text\n        if \"内容不存在\" in html:\n            return None\n        title_m = re.search(r'\"title\":\"([^\"]+)\"', html)\n        title   = title_m.group(1) if title_m else \"\"\n        content = \"\"\n        cm = re.search(\n            r'<div[^>]*class=\"[^\"]*feed-content-text[^\"]*\"[^>]*>(.*?)</div>',\n            html, re.DOTALL | re.IGNORECASE\n        )\n        if cm:\n            content = html_to_text(cm.group(1))\n        if not content:\n            for m in re.findall(r'\"content\":\"([^\"]{100,})\"', html):\n                content = m.replace(\"\\\\n\", \"\\n\").replace(\"\\\\u002F\", \"/\").replace(\"\\\\t\", \"\\t\")\n                break\n        return {\"title\": title, \"content\": content, \"url\": f\"{FEED_URL}/{uuid}\"}\n    except Exception as e:\n        print(f\"  ⚠️  Feed 抓取失败 {uuid}: {e}\")\n        return None\n\n\ndef fetch_discuss(content_id: str) -> dict | None:\n    \"\"\"抓取 Discuss 类型帖子正文\"\"\"\n    try:\n        resp = requests.get(\n            f\"{DISCUSS_API}/{content_id}\",\n            headers={\"User-Agent\": HEADERS[\"User-Agent\"]},\n            timeout=15\n        )\n        data = resp.json()\n        if not data.get(\"success\"):\n            return None\n        cd    = data.get(\"data\", {})\n        rich  = cd.get(\"richText\", \"\") or cd.get(\"content\", \"\")\n        return {\n            \"title\":   cd.get(\"title\", \"\"),\n            \"content\": html_to_text(rich),\n            \"url\":     f\"https://www.nowcoder.com/discuss/{content_id}\",\n        }\n    except Exception as e:\n        print(f\"  ⚠️  Discuss 抓取失败 {content_id}: {e}\")\n        return None\n\n\ndef run(queries: list, max_pages: int = 3, delay: float = 0.8,\n        output_file: str = \"output/nowcoder.json\", verbose: bool = True) -> list:\n    \"\"\"主函数：搜索 + 抓取正文，返回结果列表\"\"\"\n    os.makedirs(os.path.dirname(output_file) or \".\", exist_ok=True)\n\n    # 第一步：搜索，去重\n    all_posts = {}\n    for q in queries:\n        if verbose:\n            print(f\"🔍 搜索: {q}\")\n        results = search(q, max_pages=max_pages, delay=delay)\n        new = 0\n        for r in results:\n            key = r.get(\"uuid\") or r.get(\"content_id\")\n            if key and key not in all_posts:\n                all_posts[key] = r\n                new += 1\n        if verbose:\n            print(f\"   新增 {new} 条（累计 {len(all_posts)} 条）\")\n        time.sleep(1)\n\n    if verbose:\n        print(f\"\\n📄 共 {len(all_posts)} 条，开始抓取正文...\\n\")\n\n    # 第二步：抓取正文\n    collected = []\n    for i, (key, info) in enumerate(all_posts.items()):\n        title_short = (info.get(\"title\") or key)[:50]\n        if verbose:\n            print(f\"[{i+1}/{len(all_posts)}] {title_short}\")\n\n        if info[\"rc_type\"] == 201:\n            detail = fetch_feed(info[\"uuid\"])\n        else:\n            detail = fetch_discuss(info[\"content_id\"])\n\n        if detail and len(detail.get(\"content\", \"\")) > 80:\n            collected.append({\n                \"title\":   detail[\"title\"] or info[\"title\"],\n                \"content\": detail[\"content\"],\n                \"url\":     detail[\"url\"],\n                \"query\":   info[\"query\"],\n                \"source\":  \"nowcoder\",\n            })\n            if verbose:\n                print(f\"   ✅ {len(detail['content'])} 字\")\n        else:\n            if verbose:\n                print(f\"   ❌ 无正文\")\n        time.sleep(delay)\n\n    with open(output_file, \"w\", encoding=\"utf-8\") as f:\n        json.dump(collected, f, ensure_ascii=False, indent=2)\n\n    if verbose:\n        print(f\"\\n✅ 牛客面经抓取完成：{len(collected)}/{len(all_posts)} 篇有效\")\n        print(f\"📁 保存到: {output_file}\")\n\n    return collected\n\n\ndef main():\n    parser = argparse.ArgumentParser(description=\"牛客网面经爬虫\")\n    parser.add_argument(\"--config\", default=\"config.json\", help=\"配置文件路径\")\n    parser.add_argument(\"--query\",  help=\"单个搜索关键词（覆盖 config.json）\")\n    parser.add_argument(\"--pages\",  type=int, help=\"每个关键词搜索页数\")\n    parser.add_argument(\"--output\", default=\"output/nowcoder.json\", help=\"输出 JSON 文件路径\")\n    args = parser.parse_args()\n\n    queries   = [args.query] if args.query else None\n    max_pages = args.pages\n\n    if not queries:\n        if os.path.exists(args.config):\n            with open(args.config, encoding=\"utf-8\") as f:\n                cfg = json.load(f)\n            queries   = cfg.get(\"nowcoder\", {}).get(\"queries\", [\"字节飞书面经\"])\n            max_pages = max_pages or cfg.get(\"nowcoder\", {}).get(\"max_pages\", 3)\n            delay     = cfg.get(\"nowcoder\", {}).get(\"delay_seconds\", 0.8)\n            output    = cfg.get(\"output\", {}).get(\"dir\", \"output\") + \"/nowcoder.json\"\n        else:\n            print(f\"⚠️  未找到 {args.config}，使用默认关键词\")\n            queries   = [\"字节飞书后端实习\"]\n            max_pages = 3\n            delay     = 0.8\n            output    = args.output\n    else:\n        max_pages = max_pages or 3\n        delay     = 0.8\n        output    = args.output\n\n    run(queries=queries, max_pages=max_pages, delay=delay, output_file=output)\n\n\nif __name__ == \"__main__\":\n    main()\n"}}, "scrape_xhs.py": {"2ec5d1a1f0aea92e838ff142fd001e0c2a6be515": {"t": 1785603947.9690757, "n": 8907, "h": "68289d32781c29b3", "c": "#!/usr/bin/env python3\n\"\"\"\n小红书面经爬虫（需要 Cookie + Spider_XHS 签名）\n用法：\n  python3 scrape_xhs.py                          # 使用 config.json\n  python3 scrape_xhs.py --query \"字节飞书\" --config config.json\n  python3 scrape_xhs.py --output output/xhs.json\n如何获取 Cookie？见 docs/get-xhs-cookies.md\n\"\"\"\nimport re, time, json, argparse, sys, os\n\ntry:\n    import requests\nexcept ImportError:\n    print(\"❌ 缺少依赖，请先运行: pip3 install requests\")\n    sys.exit(1)\n\n# 设置 Spider_XHS_signing 路径\nSCRIPT_DIR   = os.path.dirname(os.path.abspath(__file__))\nSIGNING_DIR  = os.path.join(SCRIPT_DIR, \"Spider_XHS_signing\")\nNODE_MODULES = os.path.join(SIGNING_DIR, \"node_modules\")\n\nif not os.path.exists(NODE_MODULES):\n    print(\"❌ 未找到 Spider_XHS_signing/node_modules\")\n    print(\"   请先运行: cd Spider_XHS_signing && npm install\")\n    sys.exit(1)\n\nsys.path.insert(0, SIGNING_DIR)\nos.environ[\"NODE_PATH\"] = NODE_MODULES\n\ntry:\n    from xhs_utils.xhs_util import generate_xs_xs_common, generate_search_id\nexcept ImportError as e:\n    print(f\"❌ 导入签名工具失败: {e}\")\n    print(\"   请确保已安装 PyExecJS: pip3 install PyExecJS\")\n    sys.exit(1)\n\n\ndef build_headers(a1: str, api_path: str, data_str: str = \"\", method: str = \"POST\",\n                  extra_cookies: dict = None) -> dict:\n    xs, xt, xs_common = generate_xs_xs_common(a1, api_path, data_str, method)\n    cookies = {\n        \"a1\":          a1,\n        \"web_session\": extra_cookies.get(\"web_session\", \"\"),\n        \"webId\":       extra_cookies.get(\"webId\", \"\"),\n        \"gid\":         extra_cookies.get(\"gid\", \"\"),\n    }\n    cookie_str = \"; \".join(f\"{k}={v}\" for k, v in cookies.items() if v)\n    return {\n        \"User-Agent\":      \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36\",\n        \"Referer\":         \"https://www.xiaohongshu.com/\",\n        \"Origin\":          \"https://www.xiaohongshu.com\",\n        \"Accept\":          \"application/json, text/plain, */*\",\n        \"Accept-Language\": \"zh-CN,zh;q=0.9\",\n        \"Content-Type\":    \"application/json;charset=UTF-8\",\n        \"Cookie\":          cookie_str,\n        \"x-s\":             xs,\n        \"x-t\":             str(xt),\n        \"x-s-common\":      xs_common,\n    }\n\n\ndef search_notes(keyword: str, page: int = 1, page_size: int = 20,\n                 cookies: dict = None) -> dict | None:\n    \"\"\"搜索小红书笔记\"\"\"\n    api = \"/api/sns/web/v1/search/notes\"\n    payload = {\n        \"keyword\":   keyword,\n        \"page\":      page,\n        \"page_size\": page_size,\n        \"search_id\": generate_search_id(),\n        \"sort\":      \"general\",\n        \"note_type\": 0,\n    }\n    data_str = json.dumps(payload, separators=(\",\", \":\"))\n    headers = build_headers(cookies.get(\"a1\", \"\"), api, data_str, \"POST\", cookies)\n    try:\n        resp = requests.post(\n            f\"https://edith.xiaohongshu.com{api}\",\n            data=data_str,\n            headers=headers,\n            timeout=15\n        )\n        return resp.json()\n    except Exception as e:\n        print(f\"  ⚠️  搜索异常: {e}\")\n        return None\n\n\ndef fetch_note(note_id: str, xsec_token: str = \"\", cookies: dict = None) -> dict | None:\n    \"\"\"获取笔记详情\"\"\"\n    api = \"/api/sns/web/v1/feed\"\n    payload = {\n        \"source_note_id\": note_id,\n        \"image_formats\":  [\"jpg\", \"webp\", \"avif\"],\n        \"extra\":          {\"need_body_topic\": \"1\"},\n        \"xsec_source\":    \"pc_search\",\n        \"xsec_token\":     xsec_token,\n    }\n    data_str = json.dumps(payload, separators=(\",\", \":\"))\n    headers = build_headers(cookies.get(\"a1\", \"\"), api, data_str, \"POST\", cookies)\n    try:\n        resp = requests.post(\n            f\"https://edith.xiaohongshu.com{api}\",\n            data=data_str,\n            headers=headers,\n            timeout=15\n        )\n        result = resp.json()\n        if result.get(\"success\"):\n            items = result.get(\"data\", {}).get(\"items\", [])\n            if items:\n                note = items[0].get(\"note_card\", {})\n                return {\n                    \"title\":   note.get(\"title\", \"\"),\n                    \"content\": note.get(\"desc\", \"\"),\n                }\n        return None\n    except Exception as e:\n        print(f\"  ⚠️  笔记抓取异常 {note_id}: {e}\")\n        return None\n\n\ndef is_relevant(title: str, content: str, keywords: list) -> bool:\n    \"\"\"判断内容是否相关\"\"\"\n    text = (title + content).lower()\n    return any(kw.lower() in text for kw in keywords)\n\n\ndef run(queries: list, cookies: dict, max_pages: int = 3, delay: float = 0.8,\n        filter_keywords: list = None, output_file: str = \"output/xiaohongshu.json\",\n        verbose: bool = True) -> list:\n    \"\"\"主函数：搜索 + 抓取正文，返回结果列表\"\"\"\n    os.makedirs(os.path.dirname(output_file) or \".\", exist_ok=True)\n\n    if not cookies.get(\"a1\"):\n        print(\"❌ 未配置小红书 Cookie，请在 config.json 中填写\")\n        print(\"   见 docs/get-xhs-cookies.md\")\n        return []\n\n    # 第一步：搜索\n    all_notes = {}\n    for kw in queries:\n        if verbose:\n            print(f\"🔍 搜索: {kw}\")\n        for page in range(1, max_pages + 1):\n            result = search_notes(kw, page=page, page_size=20, cookies=cookies)\n            if not result:\n                break\n            if not result.get(\"success\"):\n                code = result.get(\"code\", \"?\")\n                msg  = result.get(\"msg\", \"\")\n                if verbose:\n                    print(f\"   ❌ code={code} msg={msg}\")\n                if code == 461:\n                    print(\"   ⚠️  Cookie 已过期，请重新获取！见 docs/get-xhs-cookies.md\")\n                break\n            items = result.get(\"data\", {}).get(\"items\", [])\n            new = 0\n            for item in items:\n                nid        = item.get(\"id\", \"\")\n                note_card  = item.get(\"note_card\", {})\n                title      = note_card.get(\"display_title\") or note_card.get(\"title\", \"\")\n                xsec_token = item.get(\"xsec_token\", \"\")\n                if nid and nid not in all_notes:\n                    all_notes[nid] = {\"note_id\": nid, \"title\": title,\n                                      \"xsec_token\": xsec_token, \"keyword\": kw}\n                    new += 1\n            if verbose:\n                print(f\"   页{page}: {len(items)} 条，新增 {new}（累计 {len(all_notes)}）\")\n            if len(items) < 20:\n                break\n            time.sleep(delay)\n        time.sleep(1)\n\n    if verbose:\n        print(f\"\\n📄 共 {len(all_notes)} 条，开始抓取正文...\\n\")\n\n    # 第二步：抓取正文\n    collected = []\n    for i, (nid, info) in enumerate(all_notes.items()):\n        short = (info.get(\"title\") or nid)[:50]\n        if verbose:\n            print(f\"[{i+1}/{len(all_notes)}] {short}\")\n\n        detail = fetch_note(nid, info.get(\"xsec_token\", \"\"), cookies)\n        if detail and len(detail.get(\"content\", \"\")) > 30:\n            title   = detail[\"title\"] or info.get(\"title\", \"\")\n            content = detail[\"content\"]\n            # 如果设置了过滤关键词，只保留相关内容\n            if filter_keywords and not is_relevant(title, content, filter_keywords):\n                if verbose:\n                    print(f\"   - 不相关，跳过\")\n                time.sleep(0.3)\n                continue\n            collected.append({\n                \"title\":   title,\n                \"content\": content,\n                \"url\":     f\"https://www.xiaohongshu.com/explore/{nid}\",\n                \"source\":  \"xiaohongshu\",\n                \"keyword\": info.get(\"keyword\", \"\"),\n            })\n            if verbose:\n                print(f\"   ✅ {len(content)} 字\")\n        else:\n            if verbose:\n                print(f\"   ❌ 无正文\")\n        time.sleep(delay)\n\n    with open(output_file, \"w\", encoding=\"utf-8\") as f:\n        json.dump(collected, f, ensure_ascii=False, indent=2)\n\n    if verbose:\n        print(f\"\\n✅ 小红书面经抓取完成：{len(collected)}/{len(all_notes)} 篇有效\")\n        print(f\"📁 保存到: {output_file}\")\n\n    return collected\n\n\ndef main():\n    parser = argparse.ArgumentParser(description=\"小红书面经爬虫\")\n    parser.add_argument(\"--config\",  default=\"config.json\", help=\"配置文件路径\")\n    parser.add_argument(\"--query\",   help=\"单个搜索关键词\")\n    parser.add_argument(\"--pages\",   type=int, help=\"每个关键词搜索页数\")\n    parser.add_argument(\"--output\",  default=\"output/xiaohongshu.json\", help=\"输出 JSON 路径\")\n    args = parser.parse_args()\n\n    if not os.path.exists(args.config):\n        print(f\"❌ 未找到配置文件: {args.config}\")\n        print(f\"   请先执行: cp config.example.json config.json 并填入 Cookie\")\n        sys.exit(1)\n\n    with open(args.config, encoding=\"utf-8\") as f:\n        cfg = json.load(f)\n\n    cookies    = cfg.get(\"xiaohongshu\", {}).get(\"cookies\", {})\n    queries    = [args.query] if args.query else cfg.get(\"xiaohongshu\", {}).get(\"queries\", [])\n    max_pages  = args.pages or cfg.get(\"xiaohongshu\", {}).get(\"max_pages\", 3)\n    delay      = cfg.get(\"xiaohongshu\", {}).get(\"delay_seconds\", 0.8)\n    output     = args.output\n    filter_kws = None  # 不过滤，全部保留\n\n    run(queries=queries, cookies=cookies, max_pages=max_pages, delay=delay,\n        filter_keywords=filter_kws, output_file=output)\n\n\nif __name__ == \"__main__\":\n    main()\n"}}}}