1 line
28 KiB
JSON
1 line
28 KiB
JSON
{"repo": "3261559688-oss/crime-map-recife-", "files": {"scripts/llm_verify.py": {"6c4ef1b69a83b6ff142fa8ab1c411c8c73fbcfbf": {"t": 1785596012.326627, "n": 8549, "h": "2b2d1f676a9d36a9", "c": "#!/usr/bin/env python3\n\"\"\"\nLLM 精准校验脚本\n------------------------------------------------------------\n对 public/rss_incidents.json 每条做 LLM 二次校验：\n 1) 是否真的为犯罪事件（过滤政治/娱乐/广告）\n 2) 真实发生州/市/街区\n 3) 重新分类犯罪类型\n 4) 给出置信度 0-100\n\n支持 3 个 LLM Provider:\n - deepseek (api.deepseek.com) # ⭐ 推荐：便宜 + 葡语强\n - openai (api.openai.com gpt-4o-mini) # 备选\n - kat (公司内部，免费) # 内部备选\n\n用法:\n # Dry-run（不真的调 LLM，输出预期 prompt 给 5 条）\n python3 scripts/llm_verify.py --dry-run\n\n # 真正跑（需要 API KEY）\n export DEEPSEEK_API_KEY=\"sk-...\"\n python3 scripts/llm_verify.py --provider deepseek --limit 50\n\n # 全量（约 2547 条，DeepSeek 大约 ¥1.3）\n python3 scripts/llm_verify.py --provider deepseek --limit 0\n\n输出:\n public/rss_incidents.json 会被原地更新，新增字段:\n llm_verified : bool 是否真的是犯罪事件\n llm_score : int 置信度 0-100\n llm_state : str LLM 判定的州（2 字母代码）\n llm_city : str LLM 判定的城市\n llm_neighbor : str 街区（如有）\n llm_type : str 犯罪类型（重分类）\n llm_reason : str 简短理由\n llm_at : str 校验时间戳\n\"\"\"\nimport os\nimport sys\nimport json\nimport time\nimport argparse\nimport urllib.request\nimport urllib.error\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom pathlib import Path\n\nROOT = Path(__file__).resolve().parents[1]\nDATA = ROOT / 'public' / 'rss_incidents.json'\n\n# 巴西标准化州列表（供 LLM 选择，避免乱写）\nBR_STATES = ['AC','AL','AP','AM','BA','CE','DF','ES','GO','MA','MT','MS','MG','PA','PB','PR','PE','PI','RJ','RN','RS','RO','RR','SC','SP','SE','TO']\nCRIME_TYPES = ['homicidio','roubo','furto','estupro','trafico','sequestro','violencia','policia','faccao','fraude','veiculo','menor','outros','NAO_CRIME']\n\nPROMPT_SYSTEM = \"\"\"Você é um analista que classifica notícias brasileiras sobre crimes.\n\nPara cada manchete fornecida, responda APENAS em JSON com os campos:\n{\n \"is_crime\": true | false,\n \"score\": 0-100,\n \"state\": \"<sigla 2 letras dos 27 estados brasileiros, ex: SP, RJ, PE>\",\n \"city\": \"<nome da cidade>\",\n \"neighborhood\": \"<bairro se mencionado, senão ''>\",\n \"type\": \"homicidio|roubo|furto|estupro|trafico|sequestro|violencia|policia|faccao|fraude|veiculo|menor|outros|NAO_CRIME\",\n \"reason\": \"<frase curta em português>\"\n}\n\nREGRAS:\n- is_crime=false para notícias políticas, esportivas, econômicas, celebridades.\n- score reflete sua confiança (90+ = alta certeza).\n- state DEVE ser uma das 27 siglas. Se for nacional/desconhecido, use \"BR\".\n- Se a manchete cita múltiplos estados, escolha onde o EVENTO ocorreu.\n- \"trafico\" = tráfico de drogas.\n- \"faccao\" = atividade do CV, PCC, ADA ou outra facção criminosa.\n- \"violencia\" = briga, espancamento sem morte.\n- Responda APENAS o JSON, sem markdown.\"\"\"\n\ndef build_user_prompt(item):\n return f\"Manchete: {item['title']}\\nFonte: {item.get('source','')}\\nLink: {item.get('link','')[:80]}\"\n\n# ---------------- LLM Providers ----------------\ndef call_deepseek(items, api_key, model='deepseek-chat'):\n \"\"\"批量调用 DeepSeek。每次 1 条（DeepSeek 不支持批 prompt，但便宜）。\"\"\"\n results = []\n for it in items:\n body = {\n 'model': model,\n 'messages':[\n {'role':'system','content':PROMPT_SYSTEM},\n {'role':'user','content':build_user_prompt(it)},\n ],\n 'temperature':0,\n 'response_format':{'type':'json_object'},\n }\n req = urllib.request.Request(\n 'https://api.deepseek.com/v1/chat/completions',\n data=json.dumps(body).encode(),\n headers={'Authorization':f'Bearer {api_key}','Content-Type':'application/json'},\n )\n try:\n with urllib.request.urlopen(req, timeout=30) as r:\n resp = json.loads(r.read())\n content = resp['choices'][0]['message']['content']\n results.append(json.loads(content))\n except Exception as e:\n print(f' ⚠️ {it[\"id\"]}: {e}', file=sys.stderr)\n results.append(None)\n return results\n\ndef call_openai(items, api_key, model='gpt-4o-mini'):\n results = []\n for it in items:\n body = {\n 'model': model,\n 'messages':[\n {'role':'system','content':PROMPT_SYSTEM},\n {'role':'user','content':build_user_prompt(it)},\n ],\n 'temperature':0,\n 'response_format':{'type':'json_object'},\n }\n req = urllib.request.Request(\n 'https://api.openai.com/v1/chat/completions',\n data=json.dumps(body).encode(),\n headers={'Authorization':f'Bearer {api_key}','Content-Type':'application/json'},\n )\n try:\n with urllib.request.urlopen(req, timeout=30) as r:\n resp = json.loads(r.read())\n content = resp['choices'][0]['message']['content']\n results.append(json.loads(content))\n except Exception as e:\n print(f' ⚠️ {it[\"id\"]}: {e}', file=sys.stderr)\n results.append(None)\n return results\n\ndef call_kat(items, api_key=None):\n \"\"\"公司内部 KAT-Coder（占位，根据实际网关地址填）\"\"\"\n print('⚠️ KAT provider 还需要你提供内部网关地址')\n return [None]*len(items)\n\nPROVIDERS = {'deepseek':call_deepseek,'openai':call_openai,'kat':call_kat}\n\n# ---------------- Main ----------------\ndef main():\n parser = argparse.ArgumentParser()\n parser.add_argument('--provider', choices=PROVIDERS.keys(), default='deepseek')\n parser.add_argument('--limit', type=int, default=20, help='处理多少条；0=全量')\n parser.add_argument('--dry-run', action='store_true', help='不调 LLM，只打印 prompt')\n parser.add_argument('--workers', type=int, default=4)\n parser.add_argument('--only-default', action='store_true', help='只校验 city_method=default 的（最可能错）')\n parser.add_argument('--skip-verified', action='store_true', help='跳过已校验过的')\n args = parser.parse_args()\n\n with open(DATA) as f:\n items = json.load(f)\n print(f'📦 加载 {len(items)} 条')\n\n todo = items\n if args.only_default:\n todo = [x for x in todo if x.get('city_method')=='default']\n print(f' 过滤 city_method=default: {len(todo)} 条')\n if args.skip_verified:\n todo = [x for x in todo if 'llm_verified' not in x]\n print(f' 跳过已校验: {len(todo)} 条剩余')\n if args.limit > 0:\n todo = todo[:args.limit]\n print(f'🎯 本次处理 {len(todo)} 条')\n\n if args.dry_run:\n print('\\n========== Dry Run (前 3 条 prompt) ==========')\n for it in todo[:3]:\n print(f'\\n--- {it[\"id\"]} ---')\n print('USER:', build_user_prompt(it))\n print('\\n（dry-run 完成，未真正调用 LLM）')\n return\n\n api_key = os.environ.get({\n 'deepseek':'DEEPSEEK_API_KEY',\n 'openai':'OPENAI_API_KEY',\n 'kat':'KAT_API_KEY',\n }[args.provider])\n if not api_key and args.provider!='kat':\n print(f'❌ 请设置环境变量 {args.provider.upper()}_API_KEY')\n return\n\n fn = PROVIDERS[args.provider]\n # 简单分批并发\n batch_size = max(1, len(todo)//args.workers)\n batches = [todo[i:i+batch_size] for i in range(0,len(todo),batch_size)]\n print(f'⚡ 启动 {len(batches)} 个并发 worker (batch={batch_size})')\n\n t0=time.time()\n all_results = [None]*len(todo)\n with ThreadPoolExecutor(max_workers=args.workers) as ex:\n futures = {ex.submit(fn, b, api_key): (i*batch_size, b) for i,b in enumerate(batches)}\n for fut in as_completed(futures):\n offset, b = futures[fut]\n r = fut.result()\n for j, rr in enumerate(r):\n all_results[offset+j] = rr\n done=sum(1 for x in all_results if x is not None)\n print(f' ✓ {done}/{len(todo)}')\n\n # 回写\n by_id = {it['id']:it for it in items}\n ok=0; fail=0; not_crime=0\n now_iso=time.strftime('%Y-%m-%dT%H:%M:%S')\n for it, r in zip(todo, all_results):\n if not r: fail+=1; continue\n if not r.get('is_crime'): not_crime+=1\n target = by_id[it['id']]\n target['llm_verified'] = r.get('is_crime',False)\n target['llm_score'] = r.get('score',0)\n target['llm_state'] = r.get('state','')\n target['llm_city'] = r.get('city','')\n target['llm_neighbor'] = r.get('neighborhood','')\n target['llm_type'] = r.get('type','')\n target['llm_reason'] = r.get('reason','')[:200]\n target['llm_at'] = now_iso\n target['llm_model'] = args.provider\n ok+=1\n\n with open(DATA,'w') as f:\n json.dump(items, f, ensure_ascii=False, indent=0)\n\n dt=time.time()-t0\n print(f'\\n📊 完成 {ok} 条成功 / {fail} 条失败 / {not_crime} 条标记为非犯罪 (耗时 {dt:.1f}s)')\n print(f'✅ 已回写 {DATA}')\n\nif __name__=='__main__':\n main()\n"}, "04c174e57573370bddae069177126acdfce610e2": {"t": 1785747373.034803, "n": 8549, "h": "2b2d1f676a9d36a9", "c": "#!/usr/bin/env python3\n\"\"\"\nLLM 精准校验脚本\n------------------------------------------------------------\n对 public/rss_incidents.json 每条做 LLM 二次校验：\n 1) 是否真的为犯罪事件（过滤政治/娱乐/广告）\n 2) 真实发生州/市/街区\n 3) 重新分类犯罪类型\n 4) 给出置信度 0-100\n\n支持 3 个 LLM Provider:\n - deepseek (api.deepseek.com) # ⭐ 推荐：便宜 + 葡语强\n - openai (api.openai.com gpt-4o-mini) # 备选\n - kat (公司内部，免费) # 内部备选\n\n用法:\n # Dry-run（不真的调 LLM，输出预期 prompt 给 5 条）\n python3 scripts/llm_verify.py --dry-run\n\n # 真正跑（需要 API KEY）\n export DEEPSEEK_API_KEY=\"sk-...\"\n python3 scripts/llm_verify.py --provider deepseek --limit 50\n\n # 全量（约 2547 条，DeepSeek 大约 ¥1.3）\n python3 scripts/llm_verify.py --provider deepseek --limit 0\n\n输出:\n public/rss_incidents.json 会被原地更新，新增字段:\n llm_verified : bool 是否真的是犯罪事件\n llm_score : int 置信度 0-100\n llm_state : str LLM 判定的州（2 字母代码）\n llm_city : str LLM 判定的城市\n llm_neighbor : str 街区（如有）\n llm_type : str 犯罪类型（重分类）\n llm_reason : str 简短理由\n llm_at : str 校验时间戳\n\"\"\"\nimport os\nimport sys\nimport json\nimport time\nimport argparse\nimport urllib.request\nimport urllib.error\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom pathlib import Path\n\nROOT = Path(__file__).resolve().parents[1]\nDATA = ROOT / 'public' / 'rss_incidents.json'\n\n# 巴西标准化州列表（供 LLM 选择，避免乱写）\nBR_STATES = ['AC','AL','AP','AM','BA','CE','DF','ES','GO','MA','MT','MS','MG','PA','PB','PR','PE','PI','RJ','RN','RS','RO','RR','SC','SP','SE','TO']\nCRIME_TYPES = ['homicidio','roubo','furto','estupro','trafico','sequestro','violencia','policia','faccao','fraude','veiculo','menor','outros','NAO_CRIME']\n\nPROMPT_SYSTEM = \"\"\"Você é um analista que classifica notícias brasileiras sobre crimes.\n\nPara cada manchete fornecida, responda APENAS em JSON com os campos:\n{\n \"is_crime\": true | false,\n \"score\": 0-100,\n \"state\": \"<sigla 2 letras dos 27 estados brasileiros, ex: SP, RJ, PE>\",\n \"city\": \"<nome da cidade>\",\n \"neighborhood\": \"<bairro se mencionado, senão ''>\",\n \"type\": \"homicidio|roubo|furto|estupro|trafico|sequestro|violencia|policia|faccao|fraude|veiculo|menor|outros|NAO_CRIME\",\n \"reason\": \"<frase curta em português>\"\n}\n\nREGRAS:\n- is_crime=false para notícias políticas, esportivas, econômicas, celebridades.\n- score reflete sua confiança (90+ = alta certeza).\n- state DEVE ser uma das 27 siglas. Se for nacional/desconhecido, use \"BR\".\n- Se a manchete cita múltiplos estados, escolha onde o EVENTO ocorreu.\n- \"trafico\" = tráfico de drogas.\n- \"faccao\" = atividade do CV, PCC, ADA ou outra facção criminosa.\n- \"violencia\" = briga, espancamento sem morte.\n- Responda APENAS o JSON, sem markdown.\"\"\"\n\ndef build_user_prompt(item):\n return f\"Manchete: {item['title']}\\nFonte: {item.get('source','')}\\nLink: {item.get('link','')[:80]}\"\n\n# ---------------- LLM Providers ----------------\ndef call_deepseek(items, api_key, model='deepseek-chat'):\n \"\"\"批量调用 DeepSeek。每次 1 条（DeepSeek 不支持批 prompt，但便宜）。\"\"\"\n results = []\n for it in items:\n body = {\n 'model': model,\n 'messages':[\n {'role':'system','content':PROMPT_SYSTEM},\n {'role':'user','content':build_user_prompt(it)},\n ],\n 'temperature':0,\n 'response_format':{'type':'json_object'},\n }\n req = urllib.request.Request(\n 'https://api.deepseek.com/v1/chat/completions',\n data=json.dumps(body).encode(),\n headers={'Authorization':f'Bearer {api_key}','Content-Type':'application/json'},\n )\n try:\n with urllib.request.urlopen(req, timeout=30) as r:\n resp = json.loads(r.read())\n content = resp['choices'][0]['message']['content']\n results.append(json.loads(content))\n except Exception as e:\n print(f' ⚠️ {it[\"id\"]}: {e}', file=sys.stderr)\n results.append(None)\n return results\n\ndef call_openai(items, api_key, model='gpt-4o-mini'):\n results = []\n for it in items:\n body = {\n 'model': model,\n 'messages':[\n {'role':'system','content':PROMPT_SYSTEM},\n {'role':'user','content':build_user_prompt(it)},\n ],\n 'temperature':0,\n 'response_format':{'type':'json_object'},\n }\n req = urllib.request.Request(\n 'https://api.openai.com/v1/chat/completions',\n data=json.dumps(body).encode(),\n headers={'Authorization':f'Bearer {api_key}','Content-Type':'application/json'},\n )\n try:\n with urllib.request.urlopen(req, timeout=30) as r:\n resp = json.loads(r.read())\n content = resp['choices'][0]['message']['content']\n results.append(json.loads(content))\n except Exception as e:\n print(f' ⚠️ {it[\"id\"]}: {e}', file=sys.stderr)\n results.append(None)\n return results\n\ndef call_kat(items, api_key=None):\n \"\"\"公司内部 KAT-Coder（占位，根据实际网关地址填）\"\"\"\n print('⚠️ KAT provider 还需要你提供内部网关地址')\n return [None]*len(items)\n\nPROVIDERS = {'deepseek':call_deepseek,'openai':call_openai,'kat':call_kat}\n\n# ---------------- Main ----------------\ndef main():\n parser = argparse.ArgumentParser()\n parser.add_argument('--provider', choices=PROVIDERS.keys(), default='deepseek')\n parser.add_argument('--limit', type=int, default=20, help='处理多少条；0=全量')\n parser.add_argument('--dry-run', action='store_true', help='不调 LLM，只打印 prompt')\n parser.add_argument('--workers', type=int, default=4)\n parser.add_argument('--only-default', action='store_true', help='只校验 city_method=default 的（最可能错）')\n parser.add_argument('--skip-verified', action='store_true', help='跳过已校验过的')\n args = parser.parse_args()\n\n with open(DATA) as f:\n items = json.load(f)\n print(f'📦 加载 {len(items)} 条')\n\n todo = items\n if args.only_default:\n todo = [x for x in todo if x.get('city_method')=='default']\n print(f' 过滤 city_method=default: {len(todo)} 条')\n if args.skip_verified:\n todo = [x for x in todo if 'llm_verified' not in x]\n print(f' 跳过已校验: {len(todo)} 条剩余')\n if args.limit > 0:\n todo = todo[:args.limit]\n print(f'🎯 本次处理 {len(todo)} 条')\n\n if args.dry_run:\n print('\\n========== Dry Run (前 3 条 prompt) ==========')\n for it in todo[:3]:\n print(f'\\n--- {it[\"id\"]} ---')\n print('USER:', build_user_prompt(it))\n print('\\n（dry-run 完成，未真正调用 LLM）')\n return\n\n api_key = os.environ.get({\n 'deepseek':'DEEPSEEK_API_KEY',\n 'openai':'OPENAI_API_KEY',\n 'kat':'KAT_API_KEY',\n }[args.provider])\n if not api_key and args.provider!='kat':\n print(f'❌ 请设置环境变量 {args.provider.upper()}_API_KEY')\n return\n\n fn = PROVIDERS[args.provider]\n # 简单分批并发\n batch_size = max(1, len(todo)//args.workers)\n batches = [todo[i:i+batch_size] for i in range(0,len(todo),batch_size)]\n print(f'⚡ 启动 {len(batches)} 个并发 worker (batch={batch_size})')\n\n t0=time.time()\n all_results = [None]*len(todo)\n with ThreadPoolExecutor(max_workers=args.workers) as ex:\n futures = {ex.submit(fn, b, api_key): (i*batch_size, b) for i,b in enumerate(batches)}\n for fut in as_completed(futures):\n offset, b = futures[fut]\n r = fut.result()\n for j, rr in enumerate(r):\n all_results[offset+j] = rr\n done=sum(1 for x in all_results if x is not None)\n print(f' ✓ {done}/{len(todo)}')\n\n # 回写\n by_id = {it['id']:it for it in items}\n ok=0; fail=0; not_crime=0\n now_iso=time.strftime('%Y-%m-%dT%H:%M:%S')\n for it, r in zip(todo, all_results):\n if not r: fail+=1; continue\n if not r.get('is_crime'): not_crime+=1\n target = by_id[it['id']]\n target['llm_verified'] = r.get('is_crime',False)\n target['llm_score'] = r.get('score',0)\n target['llm_state'] = r.get('state','')\n target['llm_city'] = r.get('city','')\n target['llm_neighbor'] = r.get('neighborhood','')\n target['llm_type'] = r.get('type','')\n target['llm_reason'] = r.get('reason','')[:200]\n target['llm_at'] = now_iso\n target['llm_model'] = args.provider\n ok+=1\n\n with open(DATA,'w') as f:\n json.dump(items, f, ensure_ascii=False, indent=0)\n\n dt=time.time()-t0\n print(f'\\n📊 完成 {ok} 条成功 / {fail} 条失败 / {not_crime} 条标记为非犯罪 (耗时 {dt:.1f}s)')\n print(f'✅ 已回写 {DATA}')\n\nif __name__=='__main__':\n main()\n"}, "23a5fd5f2e7c85edc9ab88996e93288f3bc996ca": {"t": 1785772895.5503025, "n": 8549, "h": "2b2d1f676a9d36a9", "c": "#!/usr/bin/env python3\n\"\"\"\nLLM 精准校验脚本\n------------------------------------------------------------\n对 public/rss_incidents.json 每条做 LLM 二次校验：\n 1) 是否真的为犯罪事件（过滤政治/娱乐/广告）\n 2) 真实发生州/市/街区\n 3) 重新分类犯罪类型\n 4) 给出置信度 0-100\n\n支持 3 个 LLM Provider:\n - deepseek (api.deepseek.com) # ⭐ 推荐：便宜 + 葡语强\n - openai (api.openai.com gpt-4o-mini) # 备选\n - kat (公司内部，免费) # 内部备选\n\n用法:\n # Dry-run（不真的调 LLM，输出预期 prompt 给 5 条）\n python3 scripts/llm_verify.py --dry-run\n\n # 真正跑（需要 API KEY）\n export DEEPSEEK_API_KEY=\"sk-...\"\n python3 scripts/llm_verify.py --provider deepseek --limit 50\n\n # 全量（约 2547 条，DeepSeek 大约 ¥1.3）\n python3 scripts/llm_verify.py --provider deepseek --limit 0\n\n输出:\n public/rss_incidents.json 会被原地更新，新增字段:\n llm_verified : bool 是否真的是犯罪事件\n llm_score : int 置信度 0-100\n llm_state : str LLM 判定的州（2 字母代码）\n llm_city : str LLM 判定的城市\n llm_neighbor : str 街区（如有）\n llm_type : str 犯罪类型（重分类）\n llm_reason : str 简短理由\n llm_at : str 校验时间戳\n\"\"\"\nimport os\nimport sys\nimport json\nimport time\nimport argparse\nimport urllib.request\nimport urllib.error\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom pathlib import Path\n\nROOT = Path(__file__).resolve().parents[1]\nDATA = ROOT / 'public' / 'rss_incidents.json'\n\n# 巴西标准化州列表（供 LLM 选择，避免乱写）\nBR_STATES = ['AC','AL','AP','AM','BA','CE','DF','ES','GO','MA','MT','MS','MG','PA','PB','PR','PE','PI','RJ','RN','RS','RO','RR','SC','SP','SE','TO']\nCRIME_TYPES = ['homicidio','roubo','furto','estupro','trafico','sequestro','violencia','policia','faccao','fraude','veiculo','menor','outros','NAO_CRIME']\n\nPROMPT_SYSTEM = \"\"\"Você é um analista que classifica notícias brasileiras sobre crimes.\n\nPara cada manchete fornecida, responda APENAS em JSON com os campos:\n{\n \"is_crime\": true | false,\n \"score\": 0-100,\n \"state\": \"<sigla 2 letras dos 27 estados brasileiros, ex: SP, RJ, PE>\",\n \"city\": \"<nome da cidade>\",\n \"neighborhood\": \"<bairro se mencionado, senão ''>\",\n \"type\": \"homicidio|roubo|furto|estupro|trafico|sequestro|violencia|policia|faccao|fraude|veiculo|menor|outros|NAO_CRIME\",\n \"reason\": \"<frase curta em português>\"\n}\n\nREGRAS:\n- is_crime=false para notícias políticas, esportivas, econômicas, celebridades.\n- score reflete sua confiança (90+ = alta certeza).\n- state DEVE ser uma das 27 siglas. Se for nacional/desconhecido, use \"BR\".\n- Se a manchete cita múltiplos estados, escolha onde o EVENTO ocorreu.\n- \"trafico\" = tráfico de drogas.\n- \"faccao\" = atividade do CV, PCC, ADA ou outra facção criminosa.\n- \"violencia\" = briga, espancamento sem morte.\n- Responda APENAS o JSON, sem markdown.\"\"\"\n\ndef build_user_prompt(item):\n return f\"Manchete: {item['title']}\\nFonte: {item.get('source','')}\\nLink: {item.get('link','')[:80]}\"\n\n# ---------------- LLM Providers ----------------\ndef call_deepseek(items, api_key, model='deepseek-chat'):\n \"\"\"批量调用 DeepSeek。每次 1 条（DeepSeek 不支持批 prompt，但便宜）。\"\"\"\n results = []\n for it in items:\n body = {\n 'model': model,\n 'messages':[\n {'role':'system','content':PROMPT_SYSTEM},\n {'role':'user','content':build_user_prompt(it)},\n ],\n 'temperature':0,\n 'response_format':{'type':'json_object'},\n }\n req = urllib.request.Request(\n 'https://api.deepseek.com/v1/chat/completions',\n data=json.dumps(body).encode(),\n headers={'Authorization':f'Bearer {api_key}','Content-Type':'application/json'},\n )\n try:\n with urllib.request.urlopen(req, timeout=30) as r:\n resp = json.loads(r.read())\n content = resp['choices'][0]['message']['content']\n results.append(json.loads(content))\n except Exception as e:\n print(f' ⚠️ {it[\"id\"]}: {e}', file=sys.stderr)\n results.append(None)\n return results\n\ndef call_openai(items, api_key, model='gpt-4o-mini'):\n results = []\n for it in items:\n body = {\n 'model': model,\n 'messages':[\n {'role':'system','content':PROMPT_SYSTEM},\n {'role':'user','content':build_user_prompt(it)},\n ],\n 'temperature':0,\n 'response_format':{'type':'json_object'},\n }\n req = urllib.request.Request(\n 'https://api.openai.com/v1/chat/completions',\n data=json.dumps(body).encode(),\n headers={'Authorization':f'Bearer {api_key}','Content-Type':'application/json'},\n )\n try:\n with urllib.request.urlopen(req, timeout=30) as r:\n resp = json.loads(r.read())\n content = resp['choices'][0]['message']['content']\n results.append(json.loads(content))\n except Exception as e:\n print(f' ⚠️ {it[\"id\"]}: {e}', file=sys.stderr)\n results.append(None)\n return results\n\ndef call_kat(items, api_key=None):\n \"\"\"公司内部 KAT-Coder（占位，根据实际网关地址填）\"\"\"\n print('⚠️ KAT provider 还需要你提供内部网关地址')\n return [None]*len(items)\n\nPROVIDERS = {'deepseek':call_deepseek,'openai':call_openai,'kat':call_kat}\n\n# ---------------- Main ----------------\ndef main():\n parser = argparse.ArgumentParser()\n parser.add_argument('--provider', choices=PROVIDERS.keys(), default='deepseek')\n parser.add_argument('--limit', type=int, default=20, help='处理多少条；0=全量')\n parser.add_argument('--dry-run', action='store_true', help='不调 LLM，只打印 prompt')\n parser.add_argument('--workers', type=int, default=4)\n parser.add_argument('--only-default', action='store_true', help='只校验 city_method=default 的（最可能错）')\n parser.add_argument('--skip-verified', action='store_true', help='跳过已校验过的')\n args = parser.parse_args()\n\n with open(DATA) as f:\n items = json.load(f)\n print(f'📦 加载 {len(items)} 条')\n\n todo = items\n if args.only_default:\n todo = [x for x in todo if x.get('city_method')=='default']\n print(f' 过滤 city_method=default: {len(todo)} 条')\n if args.skip_verified:\n todo = [x for x in todo if 'llm_verified' not in x]\n print(f' 跳过已校验: {len(todo)} 条剩余')\n if args.limit > 0:\n todo = todo[:args.limit]\n print(f'🎯 本次处理 {len(todo)} 条')\n\n if args.dry_run:\n print('\\n========== Dry Run (前 3 条 prompt) ==========')\n for it in todo[:3]:\n print(f'\\n--- {it[\"id\"]} ---')\n print('USER:', build_user_prompt(it))\n print('\\n（dry-run 完成，未真正调用 LLM）')\n return\n\n api_key = os.environ.get({\n 'deepseek':'DEEPSEEK_API_KEY',\n 'openai':'OPENAI_API_KEY',\n 'kat':'KAT_API_KEY',\n }[args.provider])\n if not api_key and args.provider!='kat':\n print(f'❌ 请设置环境变量 {args.provider.upper()}_API_KEY')\n return\n\n fn = PROVIDERS[args.provider]\n # 简单分批并发\n batch_size = max(1, len(todo)//args.workers)\n batches = [todo[i:i+batch_size] for i in range(0,len(todo),batch_size)]\n print(f'⚡ 启动 {len(batches)} 个并发 worker (batch={batch_size})')\n\n t0=time.time()\n all_results = [None]*len(todo)\n with ThreadPoolExecutor(max_workers=args.workers) as ex:\n futures = {ex.submit(fn, b, api_key): (i*batch_size, b) for i,b in enumerate(batches)}\n for fut in as_completed(futures):\n offset, b = futures[fut]\n r = fut.result()\n for j, rr in enumerate(r):\n all_results[offset+j] = rr\n done=sum(1 for x in all_results if x is not None)\n print(f' ✓ {done}/{len(todo)}')\n\n # 回写\n by_id = {it['id']:it for it in items}\n ok=0; fail=0; not_crime=0\n now_iso=time.strftime('%Y-%m-%dT%H:%M:%S')\n for it, r in zip(todo, all_results):\n if not r: fail+=1; continue\n if not r.get('is_crime'): not_crime+=1\n target = by_id[it['id']]\n target['llm_verified'] = r.get('is_crime',False)\n target['llm_score'] = r.get('score',0)\n target['llm_state'] = r.get('state','')\n target['llm_city'] = r.get('city','')\n target['llm_neighbor'] = r.get('neighborhood','')\n target['llm_type'] = r.get('type','')\n target['llm_reason'] = r.get('reason','')[:200]\n target['llm_at'] = now_iso\n target['llm_model'] = args.provider\n ok+=1\n\n with open(DATA,'w') as f:\n json.dump(items, f, ensure_ascii=False, indent=0)\n\n dt=time.time()-t0\n print(f'\\n📊 完成 {ok} 条成功 / {fail} 条失败 / {not_crime} 条标记为非犯罪 (耗时 {dt:.1f}s)')\n print(f'✅ 已回写 {DATA}')\n\nif __name__=='__main__':\n main()\n"}}}} |