1 line
52 KiB
JSON
1 line
52 KiB
JSON
{"repo": "bcefghj/interview-experience-spider", "files": {"config.example.json": {"79a681316ee347dd00245cf0c99550d0a96ca4f3": {"t": 1785603940.1530845, "n": 1040, "h": "5de10c3d4fee6689", "c": "{\n \"_comment\": \"复制此文件为 config.json，填入你的配置。config.json 已加入 .gitignore，不会被上传。\",\n\n \"nowcoder\": {\n \"queries\": [\n \"字节飞书后端实习\",\n \"字节飞书 golang 面经\",\n \"字节飞书服务端实习面经\",\n \"字节飞书暑期实习面经\"\n ],\n \"max_pages\": 3,\n \"delay_seconds\": 0.8\n },\n\n \"xiaohongshu\": {\n \"enabled\": true,\n \"_cookie_help\": \"见 docs/get-xhs-cookies.md，Cookie 约1天过期\",\n \"cookies\": {\n \"a1\": \"填入你的 a1 cookie 值\",\n \"web_session\": \"填入你的 web_session cookie 值\",\n \"webId\": \"填入你的 webId cookie 值\",\n \"gid\": \"填入你的 gid cookie 值\"\n },\n \"queries\": [\n \"字节飞书面试\",\n \"字节飞书后端面经\",\n \"字节飞书实习 golang\"\n ],\n \"max_pages\": 3,\n \"delay_seconds\": 0.8\n },\n\n \"filter\": {\n \"mode\": \"all\",\n \"_mode_options\": \"all = 全部保留(去掉运营销售等) | backend = 只保留后端/Go相关\",\n \"backend_keywords\": [\"后端\", \"golang\", \"go语言\", \"服务端\", \"backend\", \"全栈\"],\n \"exclude_keywords\": [\"运营\", \"销售\", \"ux设计\", \"产品经理\", \"数据分析\", \"市场营销\"]\n },\n\n \"output\": {\n \"dir\": \"./output\",\n \"title\": \"面试备考手册\",\n \"formats\": [\"md\", \"html\"],\n \"max_content_length\": 2000\n }\n}\n"}}, "generate_report.py": {"c984b42597f21f331271aaa47f8a19b17fcf9e5c": {"t": 1785603941.445472, "n": 13706, "h": "55b5d6e43a7b4f5e", "c": "#!/usr/bin/env python3\n\"\"\"\nMD/HTML 报告生成器\n用法：\n python3 generate_report.py # 使用 config.json\n python3 generate_report.py --nc output/nc.json --xhs output/xhs.json\n python3 generate_report.py --title \"字节飞书备考\" --filter backend\n\"\"\"\nimport json, re, os, sys, argparse\n\ntry:\n import markdown as md_lib\nexcept ImportError:\n print(\"❌ 缺少依赖: pip3 install markdown\")\n sys.exit(1)\n\nBACKEND_KWS = [\"后端\", \"golang\", \"go语言\", \"服务端\", \"backend\", \"全栈\", \"开发工程师\"]\nEXCLUDE_KWS = [\"运营\", \"销售\", \"ux设计\", \"产品经理\", \"数据分析\", \"市场营销\", \"人力资源\"]\n\n\ndef filter_posts(posts: list, mode: str = \"all\") -> list:\n if mode == \"backend\":\n return [d for d in posts if any(\n kw in (d.get(\"title\", \"\") + d.get(\"content\", \"\")).lower()\n for kw in BACKEND_KWS\n )]\n else: # all\n return [d for d in posts if not any(\n kw in (d.get(\"title\", \"\") + d.get(\"content\", \"\")).lower()\n for kw in EXCLUDE_KWS\n )]\n\n\ndef build_md(nc_posts: list, xhs_posts: list, title: str,\n max_len: int = 2000) -> str:\n lines = []\n L = lines.append\n\n nc_cnt = len(nc_posts)\n xhs_cnt = len(xhs_posts)\n\n L(f\"# {title}\")\n L(\"\")\n L(f\"> 牛客网 **{nc_cnt}** 篇 + 小红书 **{xhs_cnt}** 篇，共 **{nc_cnt + xhs_cnt}** 篇真实面经\")\n L(\"\")\n L(\"---\")\n L(\"\")\n\n # 高频考点\n L(\"## 第一部分：高频考点速查\")\n L(\"\")\n\n topics = [\n (\"Golang / Go\", [\n \"GMP 调度模型（G/M/P 的含义 + work stealing）\",\n \"Goroutine vs 线程的区别\",\n \"Channel 底层实现（hchan 结构 + 阻塞原理）\",\n \"map 并发不安全 → sync.Map / Mutex 方案\",\n \"GC 三色标记法 + 写屏障\",\n \"slice 扩容规则（<256 翻倍，>256 增长 1.25）\",\n \"defer LIFO + 闭包陷阱\",\n \"interface 底层 iface/eface\",\n \"内存逃逸分析\",\n ]),\n (\"MySQL\", [\n \"B+ 树索引结构 + 为什么不用 B 树\",\n \"聚簇索引 vs 非聚簇索引（回表/覆盖索引）\",\n \"联合索引最左前缀 + 索引失效场景\",\n \"MVCC 原理（undo log + read view）\",\n \"四种隔离级别 + 幻读解决方案\",\n \"redo log / undo log / bin log 区别\",\n \"行锁 / 表锁 / 间隙锁 / Next-Key Lock\",\n \"慢 SQL 排查：explain + 索引优化\",\n ]),\n (\"Redis\", [\n \"为什么快：单线程 + IO多路复用 + 内存操作\",\n \"五种数据结构底层实现\",\n \"缓存穿透/击穿/雪崩 三问 ★★★\",\n \"RDB vs AOF 持久化\",\n \"分布式锁：SET NX EX + Redlock\",\n \"内存淘汰：LRU vs LFU\",\n \"热 Key / 大 Key 处理\",\n \"集群：主从/哨兵/Cluster\",\n ]),\n (\"计算机网络\", [\n \"TCP 三次握手/四次挥手 ★★★\",\n \"TIME_WAIT（2MSL 原因）\",\n \"TCP 可靠传输：序号/确认/重传/流控/拥塞控制\",\n \"HTTP 1.0/1.1/2.0/3.0 区别\",\n \"HTTPS TLS 握手过程\",\n \"URL → 页面全流程 ★★★\",\n \"DNS 解析（UDP 53 端口）\",\n ]),\n (\"操作系统\", [\n \"进程/线程/协程区别\",\n \"IO 多路复用：select/poll/epoll 区别\",\n \"虚拟内存 + 缺页中断\",\n \"死锁四条件 + 预防\",\n ]),\n (\"系统设计\", [\n \"秒杀系统设计（防超卖）★★★\",\n \"分布式 ID（雪花算法）\",\n \"限流：令牌桶/漏桶/滑动窗口\",\n \"短链服务设计\",\n \"敏感词过滤（Trie）\",\n ]),\n (\"算法（手撕）\", [\n \"★★★ LC3 无重复最长子串（滑动窗口）\",\n \"★★★ LC128 最长连续序列（HashSet O(n)）\",\n \"★★★ LC25 K个一组翻转链表\",\n \"★★★ LC146 LRU 缓存\",\n \"★★ LC102 二叉树层序遍历\",\n \"★★ LC76 最小覆盖子串\",\n ]),\n ]\n\n for i, (topic, qs) in enumerate(topics):\n L(f\"### {i+1}. {topic}\")\n L(\"\")\n for q in qs:\n L(f\"- {q}\")\n L(\"\")\n\n L(\"---\")\n L(\"\")\n\n # 牛客面经\n L(f\"## 第二部分：牛客网面经（共 {nc_cnt} 篇，节选30篇原文）\")\n L(\"\")\n for i, d in enumerate(nc_posts[:30]):\n title = d.get(\"title\", \"无标题\")\n url = d.get(\"url\", \"\")\n content = d.get(\"content\", \"\").strip()\n if len(content) > max_len:\n content = content[:max_len] + \"\\n\\n...（内容截断，完整见原链接）\"\n L(f\"### {i+1}. {title}\")\n L(\"\")\n if url:\n L(f\"> 来源：[{url}]({url})\")\n L(\"\")\n L(content)\n L(\"\")\n L(\"---\")\n L(\"\")\n\n # 小红书面经\n L(f\"## 第三部分：小红书面经（共 {xhs_cnt} 篇，节选30篇原文）\")\n L(\"\")\n for i, d in enumerate(xhs_posts[:30]):\n title = d.get(\"title\", \"无标题\")\n url = d.get(\"url\", \"\")\n content = d.get(\"content\", \"\").strip()\n if len(content) > max_len:\n content = content[:max_len] + \"\\n\\n...（内容截断）\"\n L(f\"### {i+1}. {title}\")\n L(\"\")\n if url:\n L(f\"> 来源：[小红书]({url})\")\n L(\"\")\n L(content)\n L(\"\")\n L(\"---\")\n L(\"\")\n\n # 附录\n L(\"## 附录：全部面经索引\")\n L(\"\")\n L(f\"### 牛客网（{nc_cnt} 篇）\")\n L(\"\")\n for i, d in enumerate(nc_posts):\n L(f\"{i+1}. [{d.get('title','无标题')}]({d.get('url','')})\")\n L(\"\")\n L(f\"### 小红书（{xhs_cnt} 篇）\")\n L(\"\")\n for i, d in enumerate(xhs_posts):\n L(f\"{i+1}. [{d.get('title','无标题')}]({d.get('url','')})\")\n\n return \"\\n\".join(lines)\n\n\n_CSS = \"\"\"\n:root{--bg:#0f1117;--sb:#161b22;--card:#1c2128;--bdr:#30363d;--tx:#e6edf3;--mt:#8b949e;--ac:#58a6ff;--ac2:#3fb950;--ac3:#d29922;}\n*{box-sizing:border-box;margin:0;padding:0;}\nbody{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI','PingFang SC','Microsoft YaHei',sans-serif;background:var(--bg);color:var(--tx);display:flex;min-height:100vh;font-size:15px;line-height:1.7;}\n#sb{width:270px;min-width:270px;background:var(--sb);border-right:1px solid var(--bdr);position:fixed;top:0;left:0;bottom:0;overflow-y:auto;padding:16px 0;z-index:100;}\n#sbh{padding:0 14px 14px;border-bottom:1px solid var(--bdr);margin-bottom:6px;}\n#sbh h2{font-size:12px;color:var(--mt);text-transform:uppercase;letter-spacing:.8px;}\n#sb ul{list-style:none;}\n#sb a{display:block;padding:5px 14px;color:var(--mt);text-decoration:none;font-size:13px;border-left:3px solid transparent;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;transition:all .15s;}\n#sb a:hover,#sb a.on{color:var(--ac);background:rgba(88,166,255,.07);border-left-color:var(--ac);}\n.h2a{font-weight:600;color:var(--tx)!important;}\n.h3a{padding-left:26px!important;font-size:12px!important;}\n#mn{margin-left:270px;flex:1;max-width:940px;padding:36px 48px;}\n.hero{background:linear-gradient(135deg,#1a2a4a,#0d1b2e);border:1px solid #1f6feb;border-radius:12px;padding:22px 26px;margin-bottom:28px;}\n.hero-t{font-size:20px;color:var(--ac);font-weight:700;margin-bottom:6px;}\n.badge{display:inline-block;padding:3px 9px;border-radius:20px;font-size:12px;font-weight:600;margin:3px 3px 0 0;}\n.bb{background:rgba(88,166,255,.15);color:#58a6ff;border:1px solid rgba(88,166,255,.3);}\n.bg{background:rgba(63,185,80,.15);color:#3fb950;border:1px solid rgba(63,185,80,.3);}\n.by{background:rgba(210,153,34,.15);color:#d29922;border:1px solid rgba(210,153,34,.3);}\nh1{display:none;}\nh2{font-size:19px;color:var(--ac);margin:36px 0 14px;padding-bottom:7px;border-bottom:1px solid var(--bdr);scroll-margin-top:20px;}\nh3{font-size:15px;color:var(--ac2);margin:22px 0 10px;scroll-margin-top:20px;}\np{margin:0 0 10px;}\nul,ol{padding-left:20px;margin:7px 0 10px;}\nli{margin:3px 0;}\na{color:var(--ac);text-decoration:none;}\na:hover{text-decoration:underline;}\nhr{border:none;border-top:1px solid var(--bdr);margin:28px 0;}\nblockquote{border-left:3px solid var(--ac3);padding:7px 14px;background:rgba(210,153,34,.07);border-radius:0 6px 6px 0;color:var(--mt);margin:10px 0;font-size:14px;}\ntable{width:100%;border-collapse:collapse;margin:14px 0;font-size:14px;}\nth{background:var(--card);color:var(--ac);padding:8px 11px;border:1px solid var(--bdr);text-align:left;}\ntd{padding:8px 11px;border:1px solid var(--bdr);vertical-align:top;}\ntr:nth-child(even) td{background:rgba(255,255,255,.015);}\ncode{font-family:'JetBrains Mono','Fira Code',Consolas,monospace;font-size:13px;background:var(--card);border:1px solid var(--bdr);padding:2px 5px;border-radius:4px;color:#e3b341;}\npre{background:var(--card);border:1px solid var(--bdr);border-radius:8px;padding:14px 18px;overflow-x:auto;margin:10px 0 14px;}\npre code{background:none;border:none;padding:0;color:#e6edf3;}\n#srch{width:100%;padding:7px 11px;background:var(--bg);border:1px solid var(--bdr);border-radius:6px;color:var(--tx);font-size:13px;margin-bottom:10px;outline:none;}\n#srch:focus{border-color:var(--ac);}\n#top{position:fixed;bottom:22px;right:22px;background:var(--ac);color:#000;width:38px;height:38px;border-radius:50%;display:flex;align-items:center;justify-content:center;font-size:17px;font-weight:bold;box-shadow:0 3px 10px rgba(0,0,0,.4);text-decoration:none;opacity:.8;}\n#top:hover{opacity:1;transform:scale(1.1);}\n#prog{position:fixed;top:0;left:270px;right:0;height:3px;background:linear-gradient(90deg,var(--ac),var(--ac2));transform-origin:left;transform:scaleX(0);transition:transform .1s;z-index:200;}\n@media(max-width:768px){#sb{display:none;}#mn{margin-left:0;padding:18px;}#prog{left:0;}}\n\"\"\"\n\n_JS = \"\"\"\nwindow.addEventListener('scroll',()=>{\n const d=document.documentElement;\n document.getElementById('prog').style.transform=`scaleX(${d.scrollTop/(d.scrollHeight-d.clientHeight)||0})`;\n});\nconst hs=document.querySelectorAll('h2,h3');\nconst ls=document.querySelectorAll('#sb a');\nhs.forEach(h=>new IntersectionObserver(es=>{es.forEach(e=>{\n if(e.isIntersecting){ls.forEach(l=>l.classList.remove('on'));\n const a=document.querySelector(`#sb a[href=\"#${e.target.id}\"]`);\n if(a){a.classList.add('on');a.scrollIntoView({block:'nearest'});}\n }\n});},{rootMargin:'-20px 0px -80% 0px'}).observe(h));\nfunction fs(q){q=q.toLowerCase();document.querySelectorAll('#sb li').forEach(li=>{\n li.style.display=(!q||li.textContent.toLowerCase().includes(q))?'':'none';\n});}\n\"\"\"\n\n\ndef build_html(md_text: str, title: str, nc_cnt: int, xhs_cnt: int) -> str:\n html_body = md_lib.markdown(\n md_text,\n extensions=[\"tables\", \"fenced_code\", \"toc\", \"nl2br\"],\n extension_configs={\"toc\": {\"toc_depth\": \"2-3\"}}\n )\n toc_items = []\n for m in re.finditer(r'<h([23])[^>]*id=\"([^\"]*)\"[^>]*>(.*?)</h\\1>', html_body, re.DOTALL):\n lv, anc, txt = m.group(1), m.group(2), m.group(3)\n clean = re.sub(r\"<[^>]+>\", \"\", txt).strip()\n cls = \"h2a\" if lv == \"2\" else \"h3a\"\n toc_items.append(f'<li><a href=\"#{anc}\" class=\"{cls}\">{clean}</a></li>')\n toc = \"<ul>\" + \"\".join(toc_items) + \"</ul>\"\n\n return f\"\"\"<!DOCTYPE html>\n<html lang=\"zh-CN\">\n<head>\n<meta charset=\"UTF-8\">\n<meta name=\"viewport\" content=\"width=device-width,initial-scale=1.0\">\n<title>{title}</title>\n<style>{_CSS}</style>\n</head>\n<body>\n<div id=\"prog\"></div>\n<nav id=\"sb\">\n <div id=\"sbh\">\n <h2>📚 目录导航</h2>\n <input id=\"srch\" type=\"text\" placeholder=\"搜索章节...\" oninput=\"fs(this.value)\">\n </div>\n {toc}\n</nav>\n<main id=\"mn\">\n <div class=\"hero\">\n <div class=\"hero-t\">{title}</div>\n <p style=\"margin-top:7px\">\n <span class=\"badge bb\">牛客网：{nc_cnt} 篇</span>\n <span class=\"badge bg\">小红书：{xhs_cnt} 篇</span>\n <span class=\"badge by\">合计：{nc_cnt + xhs_cnt} 篇</span>\n </p>\n </div>\n {html_body}\n</main>\n<a href=\"#\" id=\"top\" title=\"返回顶部\">↑</a>\n<script>{_JS}</script>\n</body>\n</html>\"\"\"\n\n\ndef run(nc_file: str, xhs_file: str, output_dir: str, title: str,\n filter_mode: str = \"all\", max_len: int = 2000,\n formats: list = None, verbose: bool = True) -> dict:\n \"\"\"生成报告\"\"\"\n formats = formats or [\"md\", \"html\"]\n os.makedirs(output_dir, exist_ok=True)\n\n nc_posts = json.load(open(nc_file, encoding=\"utf-8\")) if os.path.exists(nc_file) else []\n xhs_posts = json.load(open(xhs_file, encoding=\"utf-8\")) if os.path.exists(xhs_file) else []\n\n nc_posts = filter_posts(nc_posts, filter_mode)\n xhs_posts = filter_posts(xhs_posts, filter_mode)\n\n if verbose:\n print(f\"📊 过滤模式: {filter_mode}\")\n print(f\" 牛客: {len(nc_posts)} 篇 | 小红书: {len(xhs_posts)} 篇\")\n\n md_text = build_md(nc_posts, xhs_posts, title, max_len)\n results = {}\n\n safe_title = re.sub(r'[\\\\/:*?\"<>|]', '_', title)\n\n if \"md\" in formats:\n md_path = os.path.join(output_dir, f\"{safe_title}.md\")\n with open(md_path, \"w\", encoding=\"utf-8\") as f:\n f.write(md_text)\n results[\"md\"] = md_path\n if verbose:\n print(f\"✅ MD → {md_path} ({len(md_text)//1024}KB)\")\n\n if \"html\" in formats:\n html_text = build_html(md_text, title, len(nc_posts), len(xhs_posts))\n html_path = os.path.join(output_dir, f\"{safe_title}.html\")\n with open(html_path, \"w\", encoding=\"utf-8\") as f:\n f.write(html_text)\n results[\"html\"] = html_path\n if verbose:\n print(f\"✅ HTML → {html_path} ({len(html_text)//1024}KB)\")\n\n return results\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"MD/HTML 报告生成器\")\n parser.add_argument(\"--config\", default=\"config.json\")\n parser.add_argument(\"--nc\", default=\"output/nowcoder.json\", help=\"牛客数据 JSON\")\n parser.add_argument(\"--xhs\", default=\"output/xiaohongshu.json\", help=\"小红书数据 JSON\")\n parser.add_argument(\"--title\", help=\"报告标题\")\n parser.add_argument(\"--filter\", choices=[\"all\", \"backend\"], default=\"all\")\n parser.add_argument(\"--output\", help=\"输出目录\")\n args = parser.parse_args()\n\n cfg = {}\n if os.path.exists(args.config):\n with open(args.config, encoding=\"utf-8\") as f:\n cfg = json.load(f)\n\n title = args.title or cfg.get(\"output\", {}).get(\"title\", \"面经备考手册\")\n output_dir = args.output or cfg.get(\"output\", {}).get(\"dir\", \"output\")\n formats = cfg.get(\"output\", {}).get(\"formats\", [\"md\", \"html\"])\n filter_m = cfg.get(\"filter\", {}).get(\"mode\", args.filter)\n max_len = cfg.get(\"output\", {}).get(\"max_content_length\", 2000)\n\n run(nc_file=args.nc, xhs_file=args.xhs, output_dir=output_dir,\n title=title, filter_mode=filter_m, max_len=max_len, formats=formats)\n\n\nif __name__ == \"__main__\":\n main()\n"}}, "mcp_server.py": {"5e81eb8db7ffc59bc431f3912a31dcbad679197d": {"t": 1785603942.6252022, "n": 7477, "h": "a89de158df916eec", "c": "#!/usr/bin/env python3\n\"\"\"\n面经猎手 MCP Server\n让 Cursor / Claude 可以直接调用本工具\n\n配置方法（~/.cursor/mcp.json）：\n{\n \"mcpServers\": {\n \"interview-spider\": {\n \"command\": \"python3\",\n \"args\": [\"/path/to/interview-experience-spider/mcp_server.py\"]\n }\n }\n}\n\n然后在 Cursor 中说：\"帮我搜索字节飞书后端面经\"\n\"\"\"\nimport os, sys, json\n\nos.chdir(os.path.dirname(os.path.abspath(__file__)))\n\ntry:\n from mcp.server.fastmcp import FastMCP\nexcept ImportError:\n print(\"❌ 缺少 mcp 库，请运行: pip3 install mcp\", file=sys.stderr)\n sys.exit(1)\n\nmcp = FastMCP(\"面经猎手 Interview Spider\")\n\nCONFIG_FILE = os.path.join(os.path.dirname(__file__), \"config.json\")\nOUTPUT_DIR = \"output\"\n\n\ndef _load_config() -> dict:\n if os.path.exists(CONFIG_FILE):\n with open(CONFIG_FILE, encoding=\"utf-8\") as f:\n return json.load(f)\n return {}\n\n\n# ===== Tool 1: 搜索牛客网 =====\n@mcp.tool()\ndef search_nowcoder(\n query: str,\n pages: int = 3,\n output_file: str = \"output/nowcoder.json\"\n) -> str:\n \"\"\"\n 搜索牛客网面经，并保存到 JSON 文件。\n \n 参数:\n query - 搜索关键词，如\"字节飞书后端实习\"\n pages - 搜索页数（默认3页，每页约20条）\n output_file - 结果保存路径\n \n 返回:\n 抓取结果摘要\n \"\"\"\n try:\n from scrape_nowcoder import run\n posts = run(\n queries=[query],\n max_pages=pages,\n delay=0.8,\n output_file=output_file,\n verbose=False\n )\n titles = [f\"• {p['title'][:40]}\" for p in posts[:5]]\n return (\n f\"✅ 牛客网搜索完成\\n\"\n f\"关键词：{query}\\n\"\n f\"共找到 {len(posts)} 篇面经\\n\"\n f\"保存到：{os.path.abspath(output_file)}\\n\\n\"\n f\"前5篇标题：\\n\" + \"\\n\".join(titles)\n )\n except Exception as e:\n return f\"❌ 搜索失败: {e}\"\n\n\n# ===== Tool 2: 搜索小红书 =====\n@mcp.tool()\ndef search_xiaohongshu(\n query: str,\n pages: int = 3,\n output_file: str = \"output/xiaohongshu.json\"\n) -> str:\n \"\"\"\n 搜索小红书面经（需要在 config.json 中配置 Cookie）。\n \n 参数:\n query - 搜索关键词，如\"字节飞书面试\"\n pages - 搜索页数\n output_file - 结果保存路径\n \n Cookie 配置见 docs/get-xhs-cookies.md\n \"\"\"\n cfg = _load_config()\n cookies = cfg.get(\"xiaohongshu\", {}).get(\"cookies\", {})\n if not cookies.get(\"a1\") or \"填入\" in cookies.get(\"a1\", \"\"):\n return (\n \"❌ 小红书 Cookie 未配置\\n\"\n \"请编辑 config.json，填入 a1/web_session/webId/gid\\n\"\n \"获取方法见 docs/get-xhs-cookies.md\"\n )\n try:\n from scrape_xhs import run\n posts = run(\n queries=[query],\n cookies=cookies,\n max_pages=pages,\n delay=0.8,\n output_file=output_file,\n verbose=False\n )\n titles = [f\"• {p['title'][:40]}\" for p in posts[:5]]\n return (\n f\"✅ 小红书搜索完成\\n\"\n f\"关键词：{query}\\n\"\n f\"共找到 {len(posts)} 篇\\n\"\n f\"保存到：{os.path.abspath(output_file)}\\n\\n\"\n f\"前5篇标题：\\n\" + \"\\n\".join(titles)\n )\n except Exception as e:\n return f\"❌ 搜索失败: {e}\"\n\n\n# ===== Tool 3: 生成报告 =====\n@mcp.tool()\ndef generate_report(\n title: str = \"面经备考手册\",\n nc_file: str = \"output/nowcoder.json\",\n xhs_file: str = \"output/xiaohongshu.json\",\n filter_mode: str = \"all\"\n) -> str:\n \"\"\"\n 根据已抓取的面经数据，生成 MD + HTML 报告。\n \n 参数:\n title - 报告标题\n nc_file - 牛客数据 JSON 路径\n xhs_file - 小红书数据 JSON 路径\n filter_mode - \"all\"（全部）或 \"backend\"（只保留后端/Go相关）\n \n 返回:\n 生成的文件路径\n \"\"\"\n try:\n from generate_report import run\n results = run(\n nc_file=nc_file,\n xhs_file=xhs_file,\n output_dir=OUTPUT_DIR,\n title=title,\n filter_mode=filter_mode,\n formats=[\"md\", \"html\"],\n verbose=False\n )\n lines = [\"✅ 报告生成完成\"]\n for fmt, path in results.items():\n lines.append(f\" {fmt.upper()} → {os.path.abspath(path)}\")\n return \"\\n\".join(lines)\n except Exception as e:\n return f\"❌ 生成失败: {e}\"\n\n\n# ===== Tool 4: 一键全流程 =====\n@mcp.tool()\ndef collect_all(\n company: str = \"字节飞书\",\n position: str = \"后端实习\",\n filter_mode: str = \"all\"\n) -> str:\n \"\"\"\n 一键全流程：自动构建搜索关键词 → 爬取牛客+小红书 → 生成 HTML 报告。\n \n 参数:\n company - 公司名，如\"字节飞书\"、\"腾讯微信\"\n position - 岗位名，如\"后端实习\"、\"golang开发\"\n filter_mode - \"all\" 或 \"backend\"\n \n 用法示例：\n collect_all(\"字节飞书\", \"服务端实习\")\n collect_all(\"腾讯\", \"C++客户端\")\n \"\"\"\n cfg = _load_config()\n cookies = cfg.get(\"xiaohongshu\", {}).get(\"cookies\", {})\n has_xhs = bool(cookies.get(\"a1\") and \"填入\" not in cookies.get(\"a1\", \"\"))\n\n queries = [\n f\"{company}{position}\",\n f\"{company} {position} 面经\",\n f\"{company} {position} golang\",\n f\"{company} {position} 一面 二面\",\n ]\n\n results_summary = [f\"🚀 开始收集：{company} {position}\\n\"]\n\n # 爬取牛客\n try:\n from scrape_nowcoder import run as nc_run\n nc_posts = nc_run(\n queries=queries,\n max_pages=cfg.get(\"nowcoder\", {}).get(\"max_pages\", 3),\n delay=0.8,\n output_file=f\"output/nowcoder.json\",\n verbose=False\n )\n results_summary.append(f\"✅ 牛客网：{len(nc_posts)} 篇\")\n except Exception as e:\n results_summary.append(f\"❌ 牛客网失败: {e}\")\n nc_posts = []\n\n # 爬取小红书\n if has_xhs:\n try:\n from scrape_xhs import run as xhs_run\n xhs_posts = xhs_run(\n queries=queries,\n cookies=cookies,\n max_pages=cfg.get(\"xiaohongshu\", {}).get(\"max_pages\", 3),\n delay=0.8,\n output_file=f\"output/xiaohongshu.json\",\n verbose=False\n )\n results_summary.append(f\"✅ 小红书：{len(xhs_posts)} 篇\")\n except Exception as e:\n results_summary.append(f\"❌ 小红书失败: {e}\")\n else:\n results_summary.append(\"⚠️ 小红书未配置 Cookie，已跳过\")\n\n # 生成报告\n try:\n from generate_report import run as report_run\n title = f\"{company} {position} 面经备考手册\"\n results = report_run(\n nc_file=\"output/nowcoder.json\",\n xhs_file=\"output/xiaohongshu.json\",\n output_dir=OUTPUT_DIR,\n title=title,\n filter_mode=filter_mode,\n formats=[\"md\", \"html\"],\n verbose=False\n )\n for fmt, path in results.items():\n results_summary.append(f\"📄 {fmt.upper()} → {os.path.abspath(path)}\")\n except Exception as e:\n results_summary.append(f\"❌ 报告生成失败: {e}\")\n\n return \"\\n\".join(results_summary)\n\n\n# ===== Tool 5: 查看已有数据 =====\n@mcp.tool()\ndef list_collected(\n source: str = \"all\"\n) -> str:\n \"\"\"\n 查看已收集的面经数据摘要。\n \n 参数:\n source - \"nowcoder\"、\"xiaohongshu\" 或 \"all\"\n \"\"\"\n lines = []\n files = {\n \"nowcoder\": \"output/nowcoder.json\",\n \"xiaohongshu\": \"output/xiaohongshu.json\",\n }\n for src, path in files.items():\n if source not in (\"all\", src):\n continue\n if os.path.exists(path):\n data = json.load(open(path, encoding=\"utf-8\"))\n lines.append(f\"📁 {src}（{len(data)} 篇）→ {os.path.abspath(path)}\")\n for i, d in enumerate(data[:5]):\n lines.append(f\" {i+1}. {d.get('title','无标题')[:45]}\")\n if len(data) > 5:\n lines.append(f\" ... 共 {len(data)} 篇\")\n else:\n lines.append(f\"📁 {src} → 暂无数据（{path} 不存在）\")\n lines.append(\"\")\n return \"\\n\".join(lines) if lines else \"暂无已收集的数据\"\n\n\nif __name__ == \"__main__\":\n mcp.run()\n"}}, "run_all.py": {"a0aa20012e154875ee71fb1e9a0ccf919381fc49": {"t": 1785603944.1451764, "n": 5147, "h": "90819316a0498905", "c": "#!/usr/bin/env python3\n\"\"\"\n🚀 一键运行：搜索 + 抓取 + 生成报告\n用法：\n python3 run_all.py # 使用 config.json\n python3 run_all.py --filter backend # 只保留后端/Go 相关\n python3 run_all.py --skip-nc # 跳过牛客，只爬小红书\n python3 run_all.py --skip-xhs # 跳过小红书，只爬牛客\n\"\"\"\nimport os, sys, json, argparse, subprocess\n\ndef check_env():\n \"\"\"检查运行环境\"\"\"\n print(\"🔍 检查运行环境...\")\n\n # Python 依赖\n missing = []\n for pkg in [\"requests\", \"execjs\", \"markdown\"]:\n try:\n __import__(pkg.replace(\"-\", \"_\"))\n except ImportError:\n missing.append(pkg)\n if missing:\n print(f\"❌ 缺少 Python 依赖: {', '.join(missing)}\")\n print(f\" 请运行: pip3 install {' '.join(missing)}\")\n sys.exit(1)\n\n # Node.js 依赖（小红书签名需要）\n node_modules = os.path.join(os.path.dirname(__file__), \"Spider_XHS_signing\", \"node_modules\")\n if not os.path.exists(node_modules):\n print(\"⚠️ 小红书签名模块未初始化\")\n print(\" 正在自动初始化...\")\n signing_dir = os.path.join(os.path.dirname(__file__), \"Spider_XHS_signing\")\n result = subprocess.run([\"npm\", \"install\"], cwd=signing_dir, capture_output=True, text=True)\n if result.returncode != 0:\n print(f\" ❌ npm install 失败: {result.stderr}\")\n print(f\" 请手动运行: cd Spider_XHS_signing && npm install\")\n else:\n print(\" ✅ Node.js 依赖安装完成\")\n\n # config.json\n if not os.path.exists(\"config.json\"):\n print(\"⚠️ 未找到 config.json\")\n if os.path.exists(\"config.example.json\"):\n print(\" 提示：请先执行 cp config.example.json config.json 并填写配置\")\n sys.exit(1)\n\n print(\"✅ 环境检查通过\\n\")\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"面经猎手 - 一键运行\")\n parser.add_argument(\"--config\", default=\"config.json\")\n parser.add_argument(\"--filter\", choices=[\"all\", \"backend\"], help=\"过滤模式（覆盖 config.json）\")\n parser.add_argument(\"--skip-nc\", action=\"store_true\", help=\"跳过牛客网爬取\")\n parser.add_argument(\"--skip-xhs\", action=\"store_true\", help=\"跳过小红书爬取\")\n parser.add_argument(\"--report-only\", action=\"store_true\", help=\"只生成报告（用已有数据）\")\n args = parser.parse_args()\n\n os.chdir(os.path.dirname(os.path.abspath(__file__)))\n check_env()\n\n with open(args.config, encoding=\"utf-8\") as f:\n cfg = json.load(f)\n\n output_dir = cfg.get(\"output\", {}).get(\"dir\", \"output\")\n os.makedirs(output_dir, exist_ok=True)\n\n nc_file = os.path.join(output_dir, \"nowcoder.json\")\n xhs_file = os.path.join(output_dir, \"xiaohongshu.json\")\n\n # ===== 步骤1：爬取牛客 =====\n if not args.report_only and not args.skip_nc:\n print(\"=\" * 50)\n print(\"📥 步骤1：爬取牛客网面经\")\n print(\"=\" * 50)\n from scrape_nowcoder import run as nc_run\n queries = cfg.get(\"nowcoder\", {}).get(\"queries\", [\"字节飞书面经\"])\n max_pages = cfg.get(\"nowcoder\", {}).get(\"max_pages\", 3)\n delay = cfg.get(\"nowcoder\", {}).get(\"delay_seconds\", 0.8)\n nc_run(queries=queries, max_pages=max_pages, delay=delay, output_file=nc_file)\n print()\n else:\n print(\"⏭️ 跳过牛客网爬取\\n\")\n\n # ===== 步骤2：爬取小红书 =====\n if not args.report_only and not args.skip_xhs:\n xhs_cfg = cfg.get(\"xiaohongshu\", {})\n if xhs_cfg.get(\"enabled\", True):\n cookies = xhs_cfg.get(\"cookies\", {})\n if not cookies.get(\"a1\") or \"填入\" in cookies.get(\"a1\", \"\"):\n print(\"⚠️ 小红书 Cookie 未配置，跳过小红书爬取\")\n print(\" 配置方法见 docs/get-xhs-cookies.md\\n\")\n else:\n print(\"=\" * 50)\n print(\"📥 步骤2：爬取小红书面经\")\n print(\"=\" * 50)\n from scrape_xhs import run as xhs_run\n queries = xhs_cfg.get(\"queries\", [\"字节飞书面试\"])\n max_pages = xhs_cfg.get(\"max_pages\", 3)\n delay = xhs_cfg.get(\"delay_seconds\", 0.8)\n xhs_run(queries=queries, cookies=cookies, max_pages=max_pages,\n delay=delay, output_file=xhs_file)\n print()\n else:\n print(\"⏭️ config.json 中 xiaohongshu.enabled=false，跳过\\n\")\n else:\n print(\"⏭️ 跳过小红书爬取\\n\")\n\n # ===== 步骤3：生成报告 =====\n print(\"=\" * 50)\n print(\"📝 步骤3：生成 MD/HTML 报告\")\n print(\"=\" * 50)\n from generate_report import run as report_run\n\n title = cfg.get(\"output\", {}).get(\"title\", \"面经备考手册\")\n formats = cfg.get(\"output\", {}).get(\"formats\", [\"md\", \"html\"])\n max_len = cfg.get(\"output\", {}).get(\"max_content_length\", 2000)\n filter_m = args.filter or cfg.get(\"filter\", {}).get(\"mode\", \"all\")\n\n results = report_run(\n nc_file=nc_file, xhs_file=xhs_file,\n output_dir=output_dir, title=title,\n filter_mode=filter_m, max_len=max_len, formats=formats\n )\n\n print()\n print(\"=\" * 50)\n print(\"🎉 全部完成！\")\n print(\"=\" * 50)\n for fmt, path in results.items():\n print(f\" {fmt.upper()} → {os.path.abspath(path)}\")\n\n # 自动用浏览器打开 HTML\n if \"html\" in results:\n import subprocess\n try:\n subprocess.Popen([\"open\", results[\"html\"]])\n print(\"\\n 已自动在浏览器打开 HTML 文件 🚀\")\n except Exception:\n pass\n\n\nif __name__ == \"__main__\":\n main()\n"}}, "scrape_nowcoder.py": {"8fe322649d07bcfcabe18babb412c7ee8f2b882b": {"t": 1785603945.4394035, "n": 8488, "h": "ad7d3dcc4393ba4a", "c": "#!/usr/bin/env python3\n\"\"\"\n牛客网面经爬虫\n用法：\n python3 scrape_nowcoder.py # 使用 config.json\n python3 scrape_nowcoder.py --query \"字节飞书\" --pages 5\n python3 scrape_nowcoder.py --output output/nc.json\n\"\"\"\nimport re, time, json, argparse, sys, os\n\ntry:\n import requests\nexcept ImportError:\n print(\"❌ 缺少依赖，请先运行: pip3 install requests\")\n sys.exit(1)\n\nSEARCH_API = \"https://gw-c.nowcoder.com/api/sparta/pc/search\"\nDISCUSS_API = \"https://gw-c.nowcoder.com/api/sparta/detail/content-data/detail\"\nFEED_URL = \"https://www.nowcoder.com/feed/main/detail\"\n\nHEADERS = {\n \"Content-Type\": \"application/json; charset=UTF-8\",\n \"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36\",\n}\n\n\ndef html_to_text(html: str) -> str:\n if not html:\n return \"\"\n html = re.sub(r\"<script[^>]*>.*?</script>\", \"\", html, flags=re.DOTALL | re.IGNORECASE)\n html = re.sub(r\"<style[^>]*>.*?</style>\", \"\", html, flags=re.DOTALL | re.IGNORECASE)\n html = re.sub(r\"</?(p|div|br|h[1-6]|li|tr)[^>]*>\", \"\\n\", html, flags=re.IGNORECASE)\n html = re.sub(r\"<[^>]+>\", \"\", html)\n for old, new in [(\" \", \" \"), (\"&\", \"&\"), (\"<\", \"<\"),\n (\">\", \">\"), (\""\", '\"'), (\"'\", \"'\"), (\"\\xa0\", \" \")]:\n html = html.replace(old, new)\n html = re.sub(r\"\\n\\s*\\n\", \"\\n\\n\", html)\n return html.strip()\n\n\ndef search(query: str, max_pages: int = 3, delay: float = 0.8) -> list:\n \"\"\"搜索牛客网面经，返回帖子元信息列表\"\"\"\n results = []\n seen = set()\n for page in range(1, max_pages + 1):\n try:\n payload = {\n \"type\": \"all\",\n \"query\": query,\n \"page\": page,\n \"tag\": [{\"name\": \"面经\", \"id\": 818, \"count\": None}],\n \"order\": \"create\",\n \"gioParams\": {\"searchFrom_var\": \"顶部导航栏\", \"searchEnter_var\": \"主站\"},\n }\n resp = requests.post(SEARCH_API, json=payload, headers=HEADERS, timeout=15)\n data = resp.json()\n if not data.get(\"success\"):\n break\n records = data.get(\"data\", {}).get(\"records\", [])\n if not records:\n break\n for r in records:\n rc_type = r.get(\"rc_type\", 0)\n rd = r.get(\"data\", {})\n if rc_type == 201:\n md = rd.get(\"momentData\", {})\n if md:\n uid = md.get(\"uuid\", \"\")\n title = md.get(\"title\", \"\")\n if uid and uid not in seen:\n seen.add(uid)\n results.append({\"rc_type\": 201, \"uuid\": uid, \"title\": title, \"query\": query})\n elif rc_type == 207:\n cd = rd.get(\"contentData\", {})\n if cd:\n cid = str(cd.get(\"id\", \"\"))\n title = cd.get(\"title\", \"\")\n if cid and cid not in seen:\n seen.add(cid)\n results.append({\"rc_type\": 207, \"content_id\": cid, \"title\": title, \"query\": query})\n total_page = data.get(\"data\", {}).get(\"totalPage\", 1)\n if page >= total_page:\n break\n time.sleep(delay)\n except Exception as e:\n print(f\" ⚠️ 搜索第{page}页出错: {e}\")\n break\n return results\n\n\ndef fetch_feed(uuid: str) -> dict | None:\n \"\"\"抓取 Feed 类型帖子正文\"\"\"\n try:\n resp = requests.get(\n f\"{FEED_URL}/{uuid}\",\n headers={\"User-Agent\": HEADERS[\"User-Agent\"]},\n timeout=15\n )\n html = resp.text\n if \"内容不存在\" in html:\n return None\n title_m = re.search(r'\"title\":\"([^\"]+)\"', html)\n title = title_m.group(1) if title_m else \"\"\n content = \"\"\n cm = re.search(\n r'<div[^>]*class=\"[^\"]*feed-content-text[^\"]*\"[^>]*>(.*?)</div>',\n html, re.DOTALL | re.IGNORECASE\n )\n if cm:\n content = html_to_text(cm.group(1))\n if not content:\n for m in re.findall(r'\"content\":\"([^\"]{100,})\"', html):\n content = m.replace(\"\\\\n\", \"\\n\").replace(\"\\\\u002F\", \"/\").replace(\"\\\\t\", \"\\t\")\n break\n return {\"title\": title, \"content\": content, \"url\": f\"{FEED_URL}/{uuid}\"}\n except Exception as e:\n print(f\" ⚠️ Feed 抓取失败 {uuid}: {e}\")\n return None\n\n\ndef fetch_discuss(content_id: str) -> dict | None:\n \"\"\"抓取 Discuss 类型帖子正文\"\"\"\n try:\n resp = requests.get(\n f\"{DISCUSS_API}/{content_id}\",\n headers={\"User-Agent\": HEADERS[\"User-Agent\"]},\n timeout=15\n )\n data = resp.json()\n if not data.get(\"success\"):\n return None\n cd = data.get(\"data\", {})\n rich = cd.get(\"richText\", \"\") or cd.get(\"content\", \"\")\n return {\n \"title\": cd.get(\"title\", \"\"),\n \"content\": html_to_text(rich),\n \"url\": f\"https://www.nowcoder.com/discuss/{content_id}\",\n }\n except Exception as e:\n print(f\" ⚠️ Discuss 抓取失败 {content_id}: {e}\")\n return None\n\n\ndef run(queries: list, max_pages: int = 3, delay: float = 0.8,\n output_file: str = \"output/nowcoder.json\", verbose: bool = True) -> list:\n \"\"\"主函数：搜索 + 抓取正文，返回结果列表\"\"\"\n os.makedirs(os.path.dirname(output_file) or \".\", exist_ok=True)\n\n # 第一步：搜索，去重\n all_posts = {}\n for q in queries:\n if verbose:\n print(f\"🔍 搜索: {q}\")\n results = search(q, max_pages=max_pages, delay=delay)\n new = 0\n for r in results:\n key = r.get(\"uuid\") or r.get(\"content_id\")\n if key and key not in all_posts:\n all_posts[key] = r\n new += 1\n if verbose:\n print(f\" 新增 {new} 条（累计 {len(all_posts)} 条）\")\n time.sleep(1)\n\n if verbose:\n print(f\"\\n📄 共 {len(all_posts)} 条，开始抓取正文...\\n\")\n\n # 第二步：抓取正文\n collected = []\n for i, (key, info) in enumerate(all_posts.items()):\n title_short = (info.get(\"title\") or key)[:50]\n if verbose:\n print(f\"[{i+1}/{len(all_posts)}] {title_short}\")\n\n if info[\"rc_type\"] == 201:\n detail = fetch_feed(info[\"uuid\"])\n else:\n detail = fetch_discuss(info[\"content_id\"])\n\n if detail and len(detail.get(\"content\", \"\")) > 80:\n collected.append({\n \"title\": detail[\"title\"] or info[\"title\"],\n \"content\": detail[\"content\"],\n \"url\": detail[\"url\"],\n \"query\": info[\"query\"],\n \"source\": \"nowcoder\",\n })\n if verbose:\n print(f\" ✅ {len(detail['content'])} 字\")\n else:\n if verbose:\n print(f\" ❌ 无正文\")\n time.sleep(delay)\n\n with open(output_file, \"w\", encoding=\"utf-8\") as f:\n json.dump(collected, f, ensure_ascii=False, indent=2)\n\n if verbose:\n print(f\"\\n✅ 牛客面经抓取完成：{len(collected)}/{len(all_posts)} 篇有效\")\n print(f\"📁 保存到: {output_file}\")\n\n return collected\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"牛客网面经爬虫\")\n parser.add_argument(\"--config\", default=\"config.json\", help=\"配置文件路径\")\n parser.add_argument(\"--query\", help=\"单个搜索关键词（覆盖 config.json）\")\n parser.add_argument(\"--pages\", type=int, help=\"每个关键词搜索页数\")\n parser.add_argument(\"--output\", default=\"output/nowcoder.json\", help=\"输出 JSON 文件路径\")\n args = parser.parse_args()\n\n queries = [args.query] if args.query else None\n max_pages = args.pages\n\n if not queries:\n if os.path.exists(args.config):\n with open(args.config, encoding=\"utf-8\") as f:\n cfg = json.load(f)\n queries = cfg.get(\"nowcoder\", {}).get(\"queries\", [\"字节飞书面经\"])\n max_pages = max_pages or cfg.get(\"nowcoder\", {}).get(\"max_pages\", 3)\n delay = cfg.get(\"nowcoder\", {}).get(\"delay_seconds\", 0.8)\n output = cfg.get(\"output\", {}).get(\"dir\", \"output\") + \"/nowcoder.json\"\n else:\n print(f\"⚠️ 未找到 {args.config}，使用默认关键词\")\n queries = [\"字节飞书后端实习\"]\n max_pages = 3\n delay = 0.8\n output = args.output\n else:\n max_pages = max_pages or 3\n delay = 0.8\n output = args.output\n\n run(queries=queries, max_pages=max_pages, delay=delay, output_file=output)\n\n\nif __name__ == \"__main__\":\n main()\n"}}, "scrape_xhs.py": {"2ec5d1a1f0aea92e838ff142fd001e0c2a6be515": {"t": 1785603947.9690757, "n": 8907, "h": "68289d32781c29b3", "c": "#!/usr/bin/env python3\n\"\"\"\n小红书面经爬虫（需要 Cookie + Spider_XHS 签名）\n用法：\n python3 scrape_xhs.py # 使用 config.json\n python3 scrape_xhs.py --query \"字节飞书\" --config config.json\n python3 scrape_xhs.py --output output/xhs.json\n如何获取 Cookie？见 docs/get-xhs-cookies.md\n\"\"\"\nimport re, time, json, argparse, sys, os\n\ntry:\n import requests\nexcept ImportError:\n print(\"❌ 缺少依赖，请先运行: pip3 install requests\")\n sys.exit(1)\n\n# 设置 Spider_XHS_signing 路径\nSCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))\nSIGNING_DIR = os.path.join(SCRIPT_DIR, \"Spider_XHS_signing\")\nNODE_MODULES = os.path.join(SIGNING_DIR, \"node_modules\")\n\nif not os.path.exists(NODE_MODULES):\n print(\"❌ 未找到 Spider_XHS_signing/node_modules\")\n print(\" 请先运行: cd Spider_XHS_signing && npm install\")\n sys.exit(1)\n\nsys.path.insert(0, SIGNING_DIR)\nos.environ[\"NODE_PATH\"] = NODE_MODULES\n\ntry:\n from xhs_utils.xhs_util import generate_xs_xs_common, generate_search_id\nexcept ImportError as e:\n print(f\"❌ 导入签名工具失败: {e}\")\n print(\" 请确保已安装 PyExecJS: pip3 install PyExecJS\")\n sys.exit(1)\n\n\ndef build_headers(a1: str, api_path: str, data_str: str = \"\", method: str = \"POST\",\n extra_cookies: dict = None) -> dict:\n xs, xt, xs_common = generate_xs_xs_common(a1, api_path, data_str, method)\n cookies = {\n \"a1\": a1,\n \"web_session\": extra_cookies.get(\"web_session\", \"\"),\n \"webId\": extra_cookies.get(\"webId\", \"\"),\n \"gid\": extra_cookies.get(\"gid\", \"\"),\n }\n cookie_str = \"; \".join(f\"{k}={v}\" for k, v in cookies.items() if v)\n return {\n \"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36\",\n \"Referer\": \"https://www.xiaohongshu.com/\",\n \"Origin\": \"https://www.xiaohongshu.com\",\n \"Accept\": \"application/json, text/plain, */*\",\n \"Accept-Language\": \"zh-CN,zh;q=0.9\",\n \"Content-Type\": \"application/json;charset=UTF-8\",\n \"Cookie\": cookie_str,\n \"x-s\": xs,\n \"x-t\": str(xt),\n \"x-s-common\": xs_common,\n }\n\n\ndef search_notes(keyword: str, page: int = 1, page_size: int = 20,\n cookies: dict = None) -> dict | None:\n \"\"\"搜索小红书笔记\"\"\"\n api = \"/api/sns/web/v1/search/notes\"\n payload = {\n \"keyword\": keyword,\n \"page\": page,\n \"page_size\": page_size,\n \"search_id\": generate_search_id(),\n \"sort\": \"general\",\n \"note_type\": 0,\n }\n data_str = json.dumps(payload, separators=(\",\", \":\"))\n headers = build_headers(cookies.get(\"a1\", \"\"), api, data_str, \"POST\", cookies)\n try:\n resp = requests.post(\n f\"https://edith.xiaohongshu.com{api}\",\n data=data_str,\n headers=headers,\n timeout=15\n )\n return resp.json()\n except Exception as e:\n print(f\" ⚠️ 搜索异常: {e}\")\n return None\n\n\ndef fetch_note(note_id: str, xsec_token: str = \"\", cookies: dict = None) -> dict | None:\n \"\"\"获取笔记详情\"\"\"\n api = \"/api/sns/web/v1/feed\"\n payload = {\n \"source_note_id\": note_id,\n \"image_formats\": [\"jpg\", \"webp\", \"avif\"],\n \"extra\": {\"need_body_topic\": \"1\"},\n \"xsec_source\": \"pc_search\",\n \"xsec_token\": xsec_token,\n }\n data_str = json.dumps(payload, separators=(\",\", \":\"))\n headers = build_headers(cookies.get(\"a1\", \"\"), api, data_str, \"POST\", cookies)\n try:\n resp = requests.post(\n f\"https://edith.xiaohongshu.com{api}\",\n data=data_str,\n headers=headers,\n timeout=15\n )\n result = resp.json()\n if result.get(\"success\"):\n items = result.get(\"data\", {}).get(\"items\", [])\n if items:\n note = items[0].get(\"note_card\", {})\n return {\n \"title\": note.get(\"title\", \"\"),\n \"content\": note.get(\"desc\", \"\"),\n }\n return None\n except Exception as e:\n print(f\" ⚠️ 笔记抓取异常 {note_id}: {e}\")\n return None\n\n\ndef is_relevant(title: str, content: str, keywords: list) -> bool:\n \"\"\"判断内容是否相关\"\"\"\n text = (title + content).lower()\n return any(kw.lower() in text for kw in keywords)\n\n\ndef run(queries: list, cookies: dict, max_pages: int = 3, delay: float = 0.8,\n filter_keywords: list = None, output_file: str = \"output/xiaohongshu.json\",\n verbose: bool = True) -> list:\n \"\"\"主函数：搜索 + 抓取正文，返回结果列表\"\"\"\n os.makedirs(os.path.dirname(output_file) or \".\", exist_ok=True)\n\n if not cookies.get(\"a1\"):\n print(\"❌ 未配置小红书 Cookie，请在 config.json 中填写\")\n print(\" 见 docs/get-xhs-cookies.md\")\n return []\n\n # 第一步：搜索\n all_notes = {}\n for kw in queries:\n if verbose:\n print(f\"🔍 搜索: {kw}\")\n for page in range(1, max_pages + 1):\n result = search_notes(kw, page=page, page_size=20, cookies=cookies)\n if not result:\n break\n if not result.get(\"success\"):\n code = result.get(\"code\", \"?\")\n msg = result.get(\"msg\", \"\")\n if verbose:\n print(f\" ❌ code={code} msg={msg}\")\n if code == 461:\n print(\" ⚠️ Cookie 已过期，请重新获取！见 docs/get-xhs-cookies.md\")\n break\n items = result.get(\"data\", {}).get(\"items\", [])\n new = 0\n for item in items:\n nid = item.get(\"id\", \"\")\n note_card = item.get(\"note_card\", {})\n title = note_card.get(\"display_title\") or note_card.get(\"title\", \"\")\n xsec_token = item.get(\"xsec_token\", \"\")\n if nid and nid not in all_notes:\n all_notes[nid] = {\"note_id\": nid, \"title\": title,\n \"xsec_token\": xsec_token, \"keyword\": kw}\n new += 1\n if verbose:\n print(f\" 页{page}: {len(items)} 条，新增 {new}（累计 {len(all_notes)}）\")\n if len(items) < 20:\n break\n time.sleep(delay)\n time.sleep(1)\n\n if verbose:\n print(f\"\\n📄 共 {len(all_notes)} 条，开始抓取正文...\\n\")\n\n # 第二步：抓取正文\n collected = []\n for i, (nid, info) in enumerate(all_notes.items()):\n short = (info.get(\"title\") or nid)[:50]\n if verbose:\n print(f\"[{i+1}/{len(all_notes)}] {short}\")\n\n detail = fetch_note(nid, info.get(\"xsec_token\", \"\"), cookies)\n if detail and len(detail.get(\"content\", \"\")) > 30:\n title = detail[\"title\"] or info.get(\"title\", \"\")\n content = detail[\"content\"]\n # 如果设置了过滤关键词，只保留相关内容\n if filter_keywords and not is_relevant(title, content, filter_keywords):\n if verbose:\n print(f\" - 不相关，跳过\")\n time.sleep(0.3)\n continue\n collected.append({\n \"title\": title,\n \"content\": content,\n \"url\": f\"https://www.xiaohongshu.com/explore/{nid}\",\n \"source\": \"xiaohongshu\",\n \"keyword\": info.get(\"keyword\", \"\"),\n })\n if verbose:\n print(f\" ✅ {len(content)} 字\")\n else:\n if verbose:\n print(f\" ❌ 无正文\")\n time.sleep(delay)\n\n with open(output_file, \"w\", encoding=\"utf-8\") as f:\n json.dump(collected, f, ensure_ascii=False, indent=2)\n\n if verbose:\n print(f\"\\n✅ 小红书面经抓取完成：{len(collected)}/{len(all_notes)} 篇有效\")\n print(f\"📁 保存到: {output_file}\")\n\n return collected\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"小红书面经爬虫\")\n parser.add_argument(\"--config\", default=\"config.json\", help=\"配置文件路径\")\n parser.add_argument(\"--query\", help=\"单个搜索关键词\")\n parser.add_argument(\"--pages\", type=int, help=\"每个关键词搜索页数\")\n parser.add_argument(\"--output\", default=\"output/xiaohongshu.json\", help=\"输出 JSON 路径\")\n args = parser.parse_args()\n\n if not os.path.exists(args.config):\n print(f\"❌ 未找到配置文件: {args.config}\")\n print(f\" 请先执行: cp config.example.json config.json 并填入 Cookie\")\n sys.exit(1)\n\n with open(args.config, encoding=\"utf-8\") as f:\n cfg = json.load(f)\n\n cookies = cfg.get(\"xiaohongshu\", {}).get(\"cookies\", {})\n queries = [args.query] if args.query else cfg.get(\"xiaohongshu\", {}).get(\"queries\", [])\n max_pages = args.pages or cfg.get(\"xiaohongshu\", {}).get(\"max_pages\", 3)\n delay = cfg.get(\"xiaohongshu\", {}).get(\"delay_seconds\", 0.8)\n output = args.output\n filter_kws = None # 不过滤，全部保留\n\n run(queries=queries, cookies=cookies, max_pages=max_pages, delay=delay,\n filter_keywords=filter_kws, output_file=output)\n\n\nif __name__ == \"__main__\":\n main()\n"}}}} |