1 line
23 KiB
JSON
1 line
23 KiB
JSON
{"repo": "berbadov/carissuetopicmodelling", "files": {"vectorApproach/scripts/generate_issue_knowledge_official.py": {"9d65bd53b788794857f46cd6b740793bf2b16e27": {"t": 1785746647.4278095, "n": 10919, "h": "ea4bed313443a181", "c": "#!/usr/bin/env python3\n\"\"\"\ngenerate_issue_knowledge_official.py\n──────────────────────────────────────\nLLM interpretation of BERTopic topics from official sources\n(NHTSA TSBs, recalls, CarComplaints, CarProblemZoo).\n\nUsage:\n DEEPSEEK_API_KEY=<key> python scripts/generate_issue_knowledge_official.py\n\"\"\"\n\nimport json\nimport logging\nimport os\nimport sys\nimport time\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom pathlib import Path\n\nimport pandas as pd\nimport yaml\nfrom openai import OpenAI\n\nlogging.basicConfig(level=logging.INFO, format=\"%(asctime)s %(levelname)s %(message)s\", datefmt=\"%H:%M:%S\")\nlog = logging.getLogger(__name__)\n\nROOT = Path(__file__).resolve().parent.parent\nDATA_DIR = ROOT / \"data\" / \"processed\"\nSCAFFOLD_PATH = ROOT / \"data\" / \"scaffolds\" / \"vw_golf.yaml\"\nMODEL = \"deepseek-chat\"\n\n\ndef load_scaffold(path):\n with open(path, encoding=\"utf-8\") as f:\n return yaml.safe_load(f)\n\nSCAFFOLD = load_scaffold(SCAFFOLD_PATH)\n\napi_key = os.environ.get(\"DEEPSEEK_API_KEY\")\nif not api_key:\n sys.exit(\"DEEPSEEK_API_KEY not set\")\nclient = OpenAI(api_key=api_key, base_url=\"https://api.deepseek.com/v1\")\n\n\ndef load_data() -> dict:\n log.info(\"Loading official BERTopic outputs ...\")\n enriched = pd.read_csv(DATA_DIR / \"official_thread_enriched.csv\")\n effects = pd.read_csv(DATA_DIR / \"official_covariate_effects.csv\")\n llm_input = pd.read_csv(DATA_DIR / \"official_llm_input.csv\")\n top_terms = pd.read_csv(DATA_DIR / \"official_top_terms.csv\")\n prepared = pd.read_csv(DATA_DIR / \"official_prepared.csv\", dtype=str, low_memory=False)\n log.info(\"Loaded. %d topics, %d enriched docs.\", len(top_terms), len(enriched))\n return dict(enriched=enriched, effects=effects, llm_input=llm_input,\n top_terms=top_terms, prepared=prepared)\n\n\ndef build_bundle(topic_id: int, data: dict) -> dict:\n enriched = data[\"enriched\"]\n llm_input = data[\"llm_input\"]\n top_terms = data[\"top_terms\"]\n prepared = data[\"prepared\"]\n\n tt_row = top_terms[top_terms[\"topic\"] == topic_id].iloc[0]\n li_row = llm_input[llm_input[\"topic\"] == topic_id].iloc[0]\n\n topic_docs = enriched[enriched[\"dominant_topic\"] == topic_id]\n n = len(topic_docs)\n\n # Engine spec breakdown\n spec_counts = topic_docs[\"engine_spec\"].value_counts().reset_index()\n spec_counts.columns = [\"engine_spec\", \"n\"]\n spec_counts[\"pct\"] = (spec_counts[\"n\"] / n * 100).round(1)\n\n # Year breakdown\n year_counts = topic_docs[\"prod_year\"].dropna().value_counts().head(8).reset_index()\n year_counts.columns = [\"year\", \"n\"]\n\n # Source breakdown\n src_counts = topic_docs[\"source\"].value_counts().reset_index()\n src_counts.columns = [\"source\", \"n\"]\n\n # Top snippets\n topic_enr = topic_docs.copy()\n topic_enr[\"topic_gamma\"] = pd.to_numeric(topic_enr.get(\"topic_gamma\", 0), errors=\"coerce\").fillna(0)\n left = topic_enr.drop(columns=[\"technical_score\"], errors=\"ignore\")\n snippet_df = left.merge(\n prepared[[\"doc_name\", \"txt\", \"technical_score\"]].drop_duplicates(\"doc_name\"),\n on=\"doc_name\", how=\"inner\",\n )\n if len(snippet_df) > 0:\n snippet_df[\"technical_score\"] = pd.to_numeric(snippet_df[\"technical_score\"], errors=\"coerce\").fillna(0)\n snippet_df[\"rank\"] = snippet_df[\"topic_gamma\"] * (1 + snippet_df[\"technical_score\"].clip(upper=8) / 8)\n snippets = [str(s)[:400] for s in snippet_df.nlargest(5, \"rank\")[\"txt\"].tolist()]\n else:\n snippets = []\n\n mid = li_row.get(\"mileage_median_miles\")\n p20 = li_row.get(\"mileage_p20_miles\")\n p80 = li_row.get(\"mileage_p80_miles\")\n\n return {\n \"topic_id\": topic_id,\n \"terms\": str(tt_row[\"terms_prob\"]),\n \"prevalence_pct\": round(float(li_row[\"prevalence_pct\"]), 1),\n \"chronic_signal\": round(float(li_row.get(\"chronic_signal\", 0)), 3),\n \"thread_count\": int(li_row[\"thread_count\"]),\n \"source_breakdown\": str(li_row.get(\"source_breakdown\", \"\")),\n \"mileage_median_miles\": int(mid) if pd.notna(mid) else None,\n \"mileage_p20_miles\": int(p20) if pd.notna(p20) else None,\n \"mileage_p80_miles\": int(p80) if pd.notna(p80) else None,\n \"engine_breakdown\": spec_counts.to_dict(\"records\"),\n \"year_breakdown\": year_counts.to_dict(\"records\"),\n \"source_detail\": src_counts.to_dict(\"records\"),\n \"snippets\": snippets,\n }\n\n\ndef build_scaffold_context(scaffold: dict) -> str:\n lines = []\n meta = scaffold.get(\"meta\", {})\n lines.append(f\"Vehicle: {meta.get('make','?')} {meta.get('model','?')}\")\n for ef in scaffold.get(\"engine_families\", []):\n disps = \", \".join(ef.get(\"displacements\", []))\n yr = ef.get(\"year_range\", [])\n lines.append(f\" {ef['code']} | {ef['fuel_type']} | {disps} | {yr[0]}–{yr[1]}\")\n for issue in ef.get(\"known_issues\", []):\n if issue.get(\"issue\") != \"none_major\":\n lines.append(f\" • {issue['issue']}: {issue.get('notes','').strip()}\")\n for tx in scaffold.get(\"transmissions\", []):\n compat = \", \".join(tx.get(\"compatible_engines\", []))\n yr = tx.get(\"year_range\", [])\n lines.append(f\" {tx['code']} | {tx['type']} | {compat} | {yr[0]}–{yr[1]}\")\n for issue in tx.get(\"known_issues\", []):\n lines.append(f\" • {issue['issue']}: {issue.get('notes','').strip()}\")\n return \"\\n\".join(lines)\n\nSCAFFOLD_CONTEXT = build_scaffold_context(SCAFFOLD)\n\nSYSTEM_PROMPT = (\n \"You are an automotive reliability analyst. \"\n \"You will receive a BERTopic topic bundle aggregated from multiple official sources: \"\n \"NHTSA Technical Service Bulletins (manufacturer repair instructions), \"\n \"NHTSA safety recalls (defect/consequence/remedy), \"\n \"CarComplaints.com owner narratives, and CarProblemZoo owner reports — all for VW Golf. \"\n \"Sources span US and European market vehicles. \"\n \"Engine specs: 2.0_TSI (GTI/R), 2.0_TDI (diesel), 1.8_TSI, 1.4_TSI, 1.6_TDI, 1.5_TSI etc. \"\n \"Return ONLY valid JSON — no markdown, no code fences.\"\n)\n\n\ndef format_prompt(bundle: dict) -> str:\n mid = bundle[\"mileage_median_miles\"]\n p20 = bundle[\"mileage_p20_miles\"]\n p80 = bundle[\"mileage_p80_miles\"]\n mileage_line = f\"Median: {mid} miles | P20–P80: {p20}–{p80}\" if mid else \"Insufficient mileage data\"\n\n return f\"\"\"Analyze this multi-source BERTopic topic and return structured JSON.\n\n=== VEHICLE KNOWLEDGE ===\n{SCAFFOLD_CONTEXT}\n\n=== TOPIC {bundle[\"topic_id\"]} ===\n\nTerms: {bundle[\"terms\"]}\nPrevalence: {bundle[\"prevalence_pct\"]}% ({bundle[\"thread_count\"]} documents)\nSource breakdown: {bundle[\"source_breakdown\"]}\nMileage: {mileage_line}\n\nEngine spec breakdown:\n{json.dumps(bundle[\"engine_breakdown\"], indent=2)}\n\nProduction year breakdown:\n{json.dumps(bundle[\"year_breakdown\"], indent=2)}\n\nRepresentative snippets:\n{chr(10).join(f\"[{i+1}] {s}\" for i, s in enumerate(bundle[\"snippets\"]))}\n\n=== OUTPUT FORMAT (JSON only) ===\n\n{{\n \"topic_id\": {bundle[\"topic_id\"]},\n \"label\": \"descriptive label\",\n \"label_short\": \"2-3 word short label\",\n \"system_component\": \"engine | gearbox | cooling | electrical | suspension | exhaust | brakes | battery | lighting | other\",\n \"issue_type\": \"chronic_failure | intermittent_fault | wear_item | sensor_fault | fluid_leak | noise | safety_recall | other\",\n \"severity\": \"low | medium | high\",\n \"confidence\": \"low | medium | high\",\n \"onset_mileage_typical_miles\": null,\n \"onset_mileage_range\": \"e.g. 40k-80k miles or null\",\n \"affected_engines\": [\"e.g. 1.4_TSI\", \"2.0_TSI\"],\n \"affected_years\": \"e.g. 2010-2014 or null\",\n \"known_part_codes\": [{{\"code\": \"part number\", \"name\": \"part name\", \"notes\": \"optional\"}}],\n \"engine_notes\": \"1 sentence on engine-specific patterns or null\",\n \"prevalence_pct\": {bundle[\"prevalence_pct\"]},\n \"chronic_signal\": {bundle[\"chronic_signal\"]},\n \"summary\": \"2 sentences describing the issue pattern based on the combined source data\",\n \"warning_signs\": [\"sign 1\", \"sign 2\"],\n \"inspection_advice\": \"1-2 sentences on what to check when buying a used VW Golf\",\n \"data_quality\": \"low | medium | high\",\n \"thread_count\": {bundle[\"thread_count\"]},\n \"notes\": \"source caveats (e.g. US-only data, recall-dominated) or null\"\n}}\"\"\"\n\n\ndef call_deepseek(prompt: str, topic_id: int) -> dict | None:\n for attempt in range(1, 4):\n try:\n log.info(\"T%d attempt %d\", topic_id, attempt)\n resp = client.chat.completions.create(\n model=MODEL,\n messages=[{\"role\": \"system\", \"content\": SYSTEM_PROMPT},\n {\"role\": \"user\", \"content\": prompt}],\n temperature=0.2, max_tokens=2000,\n )\n raw = resp.choices[0].message.content.strip()\n if raw.startswith(\"```\"):\n raw = raw.split(\"```\", 2)[1]\n if raw.startswith(\"json\"):\n raw = raw[4:]\n raw = raw.strip()\n return json.loads(raw)\n except json.JSONDecodeError as e:\n log.warning(\"T%d JSON error: %s\", topic_id, e)\n except Exception as e:\n log.warning(\"T%d API error: %s\", topic_id, e)\n if attempt < 3:\n time.sleep(5 * attempt)\n return None\n\n\ndef process_topic(topic_id: int, data: dict) -> dict:\n bundle = build_bundle(topic_id, data)\n result = call_deepseek(format_prompt(bundle), topic_id)\n return result or {\"topic_id\": topic_id, \"error\": \"failed_after_retries\"}\n\n\ndef main():\n data = load_data()\n topic_ids = sorted(data[\"top_terms\"][\"topic\"].tolist())\n log.info(\"Topics: %s\", topic_ids)\n\n json_path = DATA_DIR / \"issue_knowledge_official.json\"\n results_map: dict[int, dict] = {}\n if json_path.exists():\n for r in json.loads(json_path.read_text(encoding=\"utf-8\")):\n if \"error\" not in r:\n results_map[r[\"topic_id\"]] = r\n\n todo = [t for t in topic_ids if t not in results_map]\n log.info(\"To process: %s\", todo)\n\n with ThreadPoolExecutor(max_workers=3) as pool:\n futures = {pool.submit(process_topic, t, data): t for t in todo}\n for future in as_completed(futures):\n t = futures[future]\n try:\n results_map[t] = future.result()\n log.info(\"T%d done\", t)\n except Exception as e:\n results_map[t] = {\"topic_id\": t, \"error\": str(e)}\n\n results = [results_map[t] for t in topic_ids]\n json_path.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding=\"utf-8\")\n log.info(\"Saved: %s\", json_path)\n\n flat = [{k: (json.dumps(v) if isinstance(v, (list, dict)) else v) for k, v in r.items()} for r in results]\n pd.DataFrame(flat).to_csv(DATA_DIR / \"issue_knowledge_official.csv\", index=False)\n log.info(\"Done. %d topics.\", len(topic_ids))\n\n\nif __name__ == \"__main__\":\n main()\n"}, "951d38a1c31f2cf267a81b4eff7dc315bb2384f5": {"t": 1785766551.6025126, "n": 10919, "h": "ea4bed313443a181", "c": "#!/usr/bin/env python3\n\"\"\"\ngenerate_issue_knowledge_official.py\n──────────────────────────────────────\nLLM interpretation of BERTopic topics from official sources\n(NHTSA TSBs, recalls, CarComplaints, CarProblemZoo).\n\nUsage:\n DEEPSEEK_API_KEY=<key> python scripts/generate_issue_knowledge_official.py\n\"\"\"\n\nimport json\nimport logging\nimport os\nimport sys\nimport time\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom pathlib import Path\n\nimport pandas as pd\nimport yaml\nfrom openai import OpenAI\n\nlogging.basicConfig(level=logging.INFO, format=\"%(asctime)s %(levelname)s %(message)s\", datefmt=\"%H:%M:%S\")\nlog = logging.getLogger(__name__)\n\nROOT = Path(__file__).resolve().parent.parent\nDATA_DIR = ROOT / \"data\" / \"processed\"\nSCAFFOLD_PATH = ROOT / \"data\" / \"scaffolds\" / \"vw_golf.yaml\"\nMODEL = \"deepseek-chat\"\n\n\ndef load_scaffold(path):\n with open(path, encoding=\"utf-8\") as f:\n return yaml.safe_load(f)\n\nSCAFFOLD = load_scaffold(SCAFFOLD_PATH)\n\napi_key = os.environ.get(\"DEEPSEEK_API_KEY\")\nif not api_key:\n sys.exit(\"DEEPSEEK_API_KEY not set\")\nclient = OpenAI(api_key=api_key, base_url=\"https://api.deepseek.com/v1\")\n\n\ndef load_data() -> dict:\n log.info(\"Loading official BERTopic outputs ...\")\n enriched = pd.read_csv(DATA_DIR / \"official_thread_enriched.csv\")\n effects = pd.read_csv(DATA_DIR / \"official_covariate_effects.csv\")\n llm_input = pd.read_csv(DATA_DIR / \"official_llm_input.csv\")\n top_terms = pd.read_csv(DATA_DIR / \"official_top_terms.csv\")\n prepared = pd.read_csv(DATA_DIR / \"official_prepared.csv\", dtype=str, low_memory=False)\n log.info(\"Loaded. %d topics, %d enriched docs.\", len(top_terms), len(enriched))\n return dict(enriched=enriched, effects=effects, llm_input=llm_input,\n top_terms=top_terms, prepared=prepared)\n\n\ndef build_bundle(topic_id: int, data: dict) -> dict:\n enriched = data[\"enriched\"]\n llm_input = data[\"llm_input\"]\n top_terms = data[\"top_terms\"]\n prepared = data[\"prepared\"]\n\n tt_row = top_terms[top_terms[\"topic\"] == topic_id].iloc[0]\n li_row = llm_input[llm_input[\"topic\"] == topic_id].iloc[0]\n\n topic_docs = enriched[enriched[\"dominant_topic\"] == topic_id]\n n = len(topic_docs)\n\n # Engine spec breakdown\n spec_counts = topic_docs[\"engine_spec\"].value_counts().reset_index()\n spec_counts.columns = [\"engine_spec\", \"n\"]\n spec_counts[\"pct\"] = (spec_counts[\"n\"] / n * 100).round(1)\n\n # Year breakdown\n year_counts = topic_docs[\"prod_year\"].dropna().value_counts().head(8).reset_index()\n year_counts.columns = [\"year\", \"n\"]\n\n # Source breakdown\n src_counts = topic_docs[\"source\"].value_counts().reset_index()\n src_counts.columns = [\"source\", \"n\"]\n\n # Top snippets\n topic_enr = topic_docs.copy()\n topic_enr[\"topic_gamma\"] = pd.to_numeric(topic_enr.get(\"topic_gamma\", 0), errors=\"coerce\").fillna(0)\n left = topic_enr.drop(columns=[\"technical_score\"], errors=\"ignore\")\n snippet_df = left.merge(\n prepared[[\"doc_name\", \"txt\", \"technical_score\"]].drop_duplicates(\"doc_name\"),\n on=\"doc_name\", how=\"inner\",\n )\n if len(snippet_df) > 0:\n snippet_df[\"technical_score\"] = pd.to_numeric(snippet_df[\"technical_score\"], errors=\"coerce\").fillna(0)\n snippet_df[\"rank\"] = snippet_df[\"topic_gamma\"] * (1 + snippet_df[\"technical_score\"].clip(upper=8) / 8)\n snippets = [str(s)[:400] for s in snippet_df.nlargest(5, \"rank\")[\"txt\"].tolist()]\n else:\n snippets = []\n\n mid = li_row.get(\"mileage_median_miles\")\n p20 = li_row.get(\"mileage_p20_miles\")\n p80 = li_row.get(\"mileage_p80_miles\")\n\n return {\n \"topic_id\": topic_id,\n \"terms\": str(tt_row[\"terms_prob\"]),\n \"prevalence_pct\": round(float(li_row[\"prevalence_pct\"]), 1),\n \"chronic_signal\": round(float(li_row.get(\"chronic_signal\", 0)), 3),\n \"thread_count\": int(li_row[\"thread_count\"]),\n \"source_breakdown\": str(li_row.get(\"source_breakdown\", \"\")),\n \"mileage_median_miles\": int(mid) if pd.notna(mid) else None,\n \"mileage_p20_miles\": int(p20) if pd.notna(p20) else None,\n \"mileage_p80_miles\": int(p80) if pd.notna(p80) else None,\n \"engine_breakdown\": spec_counts.to_dict(\"records\"),\n \"year_breakdown\": year_counts.to_dict(\"records\"),\n \"source_detail\": src_counts.to_dict(\"records\"),\n \"snippets\": snippets,\n }\n\n\ndef build_scaffold_context(scaffold: dict) -> str:\n lines = []\n meta = scaffold.get(\"meta\", {})\n lines.append(f\"Vehicle: {meta.get('make','?')} {meta.get('model','?')}\")\n for ef in scaffold.get(\"engine_families\", []):\n disps = \", \".join(ef.get(\"displacements\", []))\n yr = ef.get(\"year_range\", [])\n lines.append(f\" {ef['code']} | {ef['fuel_type']} | {disps} | {yr[0]}–{yr[1]}\")\n for issue in ef.get(\"known_issues\", []):\n if issue.get(\"issue\") != \"none_major\":\n lines.append(f\" • {issue['issue']}: {issue.get('notes','').strip()}\")\n for tx in scaffold.get(\"transmissions\", []):\n compat = \", \".join(tx.get(\"compatible_engines\", []))\n yr = tx.get(\"year_range\", [])\n lines.append(f\" {tx['code']} | {tx['type']} | {compat} | {yr[0]}–{yr[1]}\")\n for issue in tx.get(\"known_issues\", []):\n lines.append(f\" • {issue['issue']}: {issue.get('notes','').strip()}\")\n return \"\\n\".join(lines)\n\nSCAFFOLD_CONTEXT = build_scaffold_context(SCAFFOLD)\n\nSYSTEM_PROMPT = (\n \"You are an automotive reliability analyst. \"\n \"You will receive a BERTopic topic bundle aggregated from multiple official sources: \"\n \"NHTSA Technical Service Bulletins (manufacturer repair instructions), \"\n \"NHTSA safety recalls (defect/consequence/remedy), \"\n \"CarComplaints.com owner narratives, and CarProblemZoo owner reports — all for VW Golf. \"\n \"Sources span US and European market vehicles. \"\n \"Engine specs: 2.0_TSI (GTI/R), 2.0_TDI (diesel), 1.8_TSI, 1.4_TSI, 1.6_TDI, 1.5_TSI etc. \"\n \"Return ONLY valid JSON — no markdown, no code fences.\"\n)\n\n\ndef format_prompt(bundle: dict) -> str:\n mid = bundle[\"mileage_median_miles\"]\n p20 = bundle[\"mileage_p20_miles\"]\n p80 = bundle[\"mileage_p80_miles\"]\n mileage_line = f\"Median: {mid} miles | P20–P80: {p20}–{p80}\" if mid else \"Insufficient mileage data\"\n\n return f\"\"\"Analyze this multi-source BERTopic topic and return structured JSON.\n\n=== VEHICLE KNOWLEDGE ===\n{SCAFFOLD_CONTEXT}\n\n=== TOPIC {bundle[\"topic_id\"]} ===\n\nTerms: {bundle[\"terms\"]}\nPrevalence: {bundle[\"prevalence_pct\"]}% ({bundle[\"thread_count\"]} documents)\nSource breakdown: {bundle[\"source_breakdown\"]}\nMileage: {mileage_line}\n\nEngine spec breakdown:\n{json.dumps(bundle[\"engine_breakdown\"], indent=2)}\n\nProduction year breakdown:\n{json.dumps(bundle[\"year_breakdown\"], indent=2)}\n\nRepresentative snippets:\n{chr(10).join(f\"[{i+1}] {s}\" for i, s in enumerate(bundle[\"snippets\"]))}\n\n=== OUTPUT FORMAT (JSON only) ===\n\n{{\n \"topic_id\": {bundle[\"topic_id\"]},\n \"label\": \"descriptive label\",\n \"label_short\": \"2-3 word short label\",\n \"system_component\": \"engine | gearbox | cooling | electrical | suspension | exhaust | brakes | battery | lighting | other\",\n \"issue_type\": \"chronic_failure | intermittent_fault | wear_item | sensor_fault | fluid_leak | noise | safety_recall | other\",\n \"severity\": \"low | medium | high\",\n \"confidence\": \"low | medium | high\",\n \"onset_mileage_typical_miles\": null,\n \"onset_mileage_range\": \"e.g. 40k-80k miles or null\",\n \"affected_engines\": [\"e.g. 1.4_TSI\", \"2.0_TSI\"],\n \"affected_years\": \"e.g. 2010-2014 or null\",\n \"known_part_codes\": [{{\"code\": \"part number\", \"name\": \"part name\", \"notes\": \"optional\"}}],\n \"engine_notes\": \"1 sentence on engine-specific patterns or null\",\n \"prevalence_pct\": {bundle[\"prevalence_pct\"]},\n \"chronic_signal\": {bundle[\"chronic_signal\"]},\n \"summary\": \"2 sentences describing the issue pattern based on the combined source data\",\n \"warning_signs\": [\"sign 1\", \"sign 2\"],\n \"inspection_advice\": \"1-2 sentences on what to check when buying a used VW Golf\",\n \"data_quality\": \"low | medium | high\",\n \"thread_count\": {bundle[\"thread_count\"]},\n \"notes\": \"source caveats (e.g. US-only data, recall-dominated) or null\"\n}}\"\"\"\n\n\ndef call_deepseek(prompt: str, topic_id: int) -> dict | None:\n for attempt in range(1, 4):\n try:\n log.info(\"T%d attempt %d\", topic_id, attempt)\n resp = client.chat.completions.create(\n model=MODEL,\n messages=[{\"role\": \"system\", \"content\": SYSTEM_PROMPT},\n {\"role\": \"user\", \"content\": prompt}],\n temperature=0.2, max_tokens=2000,\n )\n raw = resp.choices[0].message.content.strip()\n if raw.startswith(\"```\"):\n raw = raw.split(\"```\", 2)[1]\n if raw.startswith(\"json\"):\n raw = raw[4:]\n raw = raw.strip()\n return json.loads(raw)\n except json.JSONDecodeError as e:\n log.warning(\"T%d JSON error: %s\", topic_id, e)\n except Exception as e:\n log.warning(\"T%d API error: %s\", topic_id, e)\n if attempt < 3:\n time.sleep(5 * attempt)\n return None\n\n\ndef process_topic(topic_id: int, data: dict) -> dict:\n bundle = build_bundle(topic_id, data)\n result = call_deepseek(format_prompt(bundle), topic_id)\n return result or {\"topic_id\": topic_id, \"error\": \"failed_after_retries\"}\n\n\ndef main():\n data = load_data()\n topic_ids = sorted(data[\"top_terms\"][\"topic\"].tolist())\n log.info(\"Topics: %s\", topic_ids)\n\n json_path = DATA_DIR / \"issue_knowledge_official.json\"\n results_map: dict[int, dict] = {}\n if json_path.exists():\n for r in json.loads(json_path.read_text(encoding=\"utf-8\")):\n if \"error\" not in r:\n results_map[r[\"topic_id\"]] = r\n\n todo = [t for t in topic_ids if t not in results_map]\n log.info(\"To process: %s\", todo)\n\n with ThreadPoolExecutor(max_workers=3) as pool:\n futures = {pool.submit(process_topic, t, data): t for t in todo}\n for future in as_completed(futures):\n t = futures[future]\n try:\n results_map[t] = future.result()\n log.info(\"T%d done\", t)\n except Exception as e:\n results_map[t] = {\"topic_id\": t, \"error\": str(e)}\n\n results = [results_map[t] for t in topic_ids]\n json_path.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding=\"utf-8\")\n log.info(\"Saved: %s\", json_path)\n\n flat = [{k: (json.dumps(v) if isinstance(v, (list, dict)) else v) for k, v in r.items()} for r in results]\n pd.DataFrame(flat).to_csv(DATA_DIR / \"issue_knowledge_official.csv\", index=False)\n log.info(\"Done. %d topics.\", len(topic_ids))\n\n\nif __name__ == \"__main__\":\n main()\n"}}}} |