1 line
11 KiB
JSON
1 line
11 KiB
JSON
{"repo": "chenchenzhang569-bit/igcse-revision", "files": {"scripts/unify_clean_answers.py": {"c313a5694867abfbd25c4524f38bb13aefc0f3ef": {"t": 1785747514.1219583, "n": 5248, "h": "3f7a4ef2b6359e88", "c": "#!/usr/bin/env python3\n\"\"\"Unify clean_answer_text: extract from SME answer_text only, same approach as fix_explanations.py.\nOne source, one logic — consistent with clean_explanation.\n\"\"\"\n\nimport sys, json, time, requests, argparse\n\nSRV_KEY = \"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImFvbmRsZHF3d3Z0dHdwZXJ2cmZxIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImlhdCI6MTc3ODI2NDM4MSwiZXhwIjoyMDkzODQwMzgxfQ.OYuqkYVvPuU02cKDntfTWiqZwkzY0dceO0DMTOA4U88\"\nDS_KEY = \"sk-8a31bc8a84ab416d97dab6ee8c19ffae\"\nSUPABASE_URL = \"https://aondldqwwvttwpervrfq.supabase.co\"\nHEADERS = {\"apikey\": SRV_KEY, \"Authorization\": f\"Bearer {SRV_KEY}\"}\nHEADERS_P = {**HEADERS, \"Content-Type\": \"application/json\", \"Prefer\": \"return=minimal\"}\n\nSYSTEM_PROMPT = \"\"\"You extract clean answer values from an IGCSE math mark scheme (answer_text). The answer_text is the SME original — extract values from it, do NOT invent or generate answers.\n\nRules:\n1. Extract ALL sub-part answer values from answer_text. Preserve labels: (i), (ii), (a), (b), 1., 2.\n2. Strip ALL mark scheme artifacts: [1], [2], [n], \"mark for\", \"Award\", \"Incorrect\", table HTML, \"bold\", etc.\n3. Strip ALL LaTeX: $, \\\\frac, \\\\times, \\\\div, \\\\text{}, \\\\begin, \\\\end, \\\\bold, etc.\n4. Replace English operation words: \"Subtract\"→\"-\", \"Add\"→\"+\", \"Multiply by\"→\"×\", \"Divide by\"→\"÷\", \"times\"→\"×\"\n5. Use \"||\" between sub-parts. Use \",\" between multiple acceptable answers for same part.\n6. Keep numbers and math symbols only: × ÷ √ ² ³ → ± ≤ ≥ °\n7. For word-only answers (like \"Subtract 7\"), convert to symbol form (\"-7\")\n8. Output as JSON: {\"clean_answer\": \"...\"}\n\nCRITICAL: Every value MUST come from answer_text. Do NOT change numbers. Do NOT add answers not in answer_text.\nDo NOT repeat the same value across different sub-parts — each (i), (ii) etc. gets its own values.\"\"\"\n\n\ndef call_deepseek(answer_text, question_text):\n user_msg = f\"\"\"Question (for context only):\n{question_text[:400]}\n\nSME answer_text (EXTRACT values from here):\n{answer_text[:1200]}\n\nExtract the clean answer values from answer_text above. Every value must come from answer_text.\"\"\"\n\n payload = {\n \"model\": \"deepseek-chat\",\n \"messages\": [\n {\"role\": \"system\", \"content\": SYSTEM_PROMPT},\n {\"role\": \"user\", \"content\": user_msg},\n ],\n \"temperature\": 0.1,\n \"max_tokens\": 600,\n \"response_format\": {\"type\": \"json_object\"},\n }\n for attempt in range(3):\n try:\n r = requests.post(\n \"https://api.deepseek.com/v1/chat/completions\",\n headers={\"Authorization\": f\"Bearer {DS_KEY}\", \"Content-Type\": \"application/json\"},\n json=payload, timeout=60,\n )\n r.raise_for_status()\n content = r.json()[\"choices\"][0][\"message\"][\"content\"]\n result = json.loads(content)\n return result.get(\"clean_answer\", \"\")\n except Exception as e:\n if attempt < 2:\n time.sleep(2)\n return None\n\n\ndef main():\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--offset\", type=int, default=0)\n parser.add_argument(\"--limit\", type=int, default=500)\n args = parser.parse_args()\n\n # Fetch structured questions with answer_text\n r = requests.get(\n f\"{SUPABASE_URL}/rest/v1/questions\",\n headers=HEADERS,\n params={\n \"select\": \"id,question_text,answer_text\",\n \"question_type\": \"eq.structured\",\n \"answer_text\": \"not.is.null\",\n \"order\": \"id\",\n \"offset\": args.offset,\n \"limit\": args.limit,\n },\n timeout=30,\n )\n questions = r.json()\n if not isinstance(questions, list):\n print(f\"ERROR: {type(questions)} - {questions[:200]}\")\n return\n\n total = len(questions)\n print(f\"Processing {total} questions (offset={args.offset}, limit={args.limit})\")\n\n done = errors = skipped = 0\n start = time.time()\n\n for q in questions:\n qid = q[\"id\"]\n answer_text = q.get(\"answer_text\") or \"\"\n question_text = q.get(\"question_text\") or \"\"\n\n if not answer_text.strip():\n skipped += 1\n continue\n\n new_ca = call_deepseek(answer_text, question_text)\n if not new_ca:\n errors += 1\n print(f\" [{done}/{total}] FAILED {qid[:12]}...\")\n time.sleep(1)\n continue\n\n r_patch = requests.patch(\n f\"{SUPABASE_URL}/rest/v1/questions?id=eq.{qid}\",\n headers=HEADERS_P,\n json={\"clean_answer_text\": new_ca},\n timeout=30,\n )\n if r_patch.status_code == 204:\n done += 1\n else:\n errors += 1\n\n elapsed = time.time() - start\n rate = done / elapsed if elapsed > 0 else 0\n eta = (total - done - errors - skipped) / rate if rate > 0 else 0\n print(f\" [{done}/{total}] {qid[:12]}... | {r_patch.status_code} | {rate:.1f}/s | ETA {eta/60:.0f}m | err:{errors} | skip:{skipped}\")\n\n time.sleep(0.3)\n\n elapsed = time.time() - start\n print(f\"\\nDone! offset={args.offset} limit={args.limit} | Done: {done}, Errors: {errors}, Skipped: {skipped}, Time: {elapsed/60:.1f}m\")\n\n\nif __name__ == \"__main__\":\n main()\n"}, "16503d6953224ec7837269b5732b29945e82836b": {"t": 1785773691.8255548, "n": 5248, "h": "3f7a4ef2b6359e88", "c": "#!/usr/bin/env python3\n\"\"\"Unify clean_answer_text: extract from SME answer_text only, same approach as fix_explanations.py.\nOne source, one logic — consistent with clean_explanation.\n\"\"\"\n\nimport sys, json, time, requests, argparse\n\nSRV_KEY = \"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImFvbmRsZHF3d3Z0dHdwZXJ2cmZxIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImlhdCI6MTc3ODI2NDM4MSwiZXhwIjoyMDkzODQwMzgxfQ.OYuqkYVvPuU02cKDntfTWiqZwkzY0dceO0DMTOA4U88\"\nDS_KEY = \"sk-8a31bc8a84ab416d97dab6ee8c19ffae\"\nSUPABASE_URL = \"https://aondldqwwvttwpervrfq.supabase.co\"\nHEADERS = {\"apikey\": SRV_KEY, \"Authorization\": f\"Bearer {SRV_KEY}\"}\nHEADERS_P = {**HEADERS, \"Content-Type\": \"application/json\", \"Prefer\": \"return=minimal\"}\n\nSYSTEM_PROMPT = \"\"\"You extract clean answer values from an IGCSE math mark scheme (answer_text). The answer_text is the SME original — extract values from it, do NOT invent or generate answers.\n\nRules:\n1. Extract ALL sub-part answer values from answer_text. Preserve labels: (i), (ii), (a), (b), 1., 2.\n2. Strip ALL mark scheme artifacts: [1], [2], [n], \"mark for\", \"Award\", \"Incorrect\", table HTML, \"bold\", etc.\n3. Strip ALL LaTeX: $, \\\\frac, \\\\times, \\\\div, \\\\text{}, \\\\begin, \\\\end, \\\\bold, etc.\n4. Replace English operation words: \"Subtract\"→\"-\", \"Add\"→\"+\", \"Multiply by\"→\"×\", \"Divide by\"→\"÷\", \"times\"→\"×\"\n5. Use \"||\" between sub-parts. Use \",\" between multiple acceptable answers for same part.\n6. Keep numbers and math symbols only: × ÷ √ ² ³ → ± ≤ ≥ °\n7. For word-only answers (like \"Subtract 7\"), convert to symbol form (\"-7\")\n8. Output as JSON: {\"clean_answer\": \"...\"}\n\nCRITICAL: Every value MUST come from answer_text. Do NOT change numbers. Do NOT add answers not in answer_text.\nDo NOT repeat the same value across different sub-parts — each (i), (ii) etc. gets its own values.\"\"\"\n\n\ndef call_deepseek(answer_text, question_text):\n user_msg = f\"\"\"Question (for context only):\n{question_text[:400]}\n\nSME answer_text (EXTRACT values from here):\n{answer_text[:1200]}\n\nExtract the clean answer values from answer_text above. Every value must come from answer_text.\"\"\"\n\n payload = {\n \"model\": \"deepseek-chat\",\n \"messages\": [\n {\"role\": \"system\", \"content\": SYSTEM_PROMPT},\n {\"role\": \"user\", \"content\": user_msg},\n ],\n \"temperature\": 0.1,\n \"max_tokens\": 600,\n \"response_format\": {\"type\": \"json_object\"},\n }\n for attempt in range(3):\n try:\n r = requests.post(\n \"https://api.deepseek.com/v1/chat/completions\",\n headers={\"Authorization\": f\"Bearer {DS_KEY}\", \"Content-Type\": \"application/json\"},\n json=payload, timeout=60,\n )\n r.raise_for_status()\n content = r.json()[\"choices\"][0][\"message\"][\"content\"]\n result = json.loads(content)\n return result.get(\"clean_answer\", \"\")\n except Exception as e:\n if attempt < 2:\n time.sleep(2)\n return None\n\n\ndef main():\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--offset\", type=int, default=0)\n parser.add_argument(\"--limit\", type=int, default=500)\n args = parser.parse_args()\n\n # Fetch structured questions with answer_text\n r = requests.get(\n f\"{SUPABASE_URL}/rest/v1/questions\",\n headers=HEADERS,\n params={\n \"select\": \"id,question_text,answer_text\",\n \"question_type\": \"eq.structured\",\n \"answer_text\": \"not.is.null\",\n \"order\": \"id\",\n \"offset\": args.offset,\n \"limit\": args.limit,\n },\n timeout=30,\n )\n questions = r.json()\n if not isinstance(questions, list):\n print(f\"ERROR: {type(questions)} - {questions[:200]}\")\n return\n\n total = len(questions)\n print(f\"Processing {total} questions (offset={args.offset}, limit={args.limit})\")\n\n done = errors = skipped = 0\n start = time.time()\n\n for q in questions:\n qid = q[\"id\"]\n answer_text = q.get(\"answer_text\") or \"\"\n question_text = q.get(\"question_text\") or \"\"\n\n if not answer_text.strip():\n skipped += 1\n continue\n\n new_ca = call_deepseek(answer_text, question_text)\n if not new_ca:\n errors += 1\n print(f\" [{done}/{total}] FAILED {qid[:12]}...\")\n time.sleep(1)\n continue\n\n r_patch = requests.patch(\n f\"{SUPABASE_URL}/rest/v1/questions?id=eq.{qid}\",\n headers=HEADERS_P,\n json={\"clean_answer_text\": new_ca},\n timeout=30,\n )\n if r_patch.status_code == 204:\n done += 1\n else:\n errors += 1\n\n elapsed = time.time() - start\n rate = done / elapsed if elapsed > 0 else 0\n eta = (total - done - errors - skipped) / rate if rate > 0 else 0\n print(f\" [{done}/{total}] {qid[:12]}... | {r_patch.status_code} | {rate:.1f}/s | ETA {eta/60:.0f}m | err:{errors} | skip:{skipped}\")\n\n time.sleep(0.3)\n\n elapsed = time.time() - start\n print(f\"\\nDone! offset={args.offset} limit={args.limit} | Done: {done}, Errors: {errors}, Skipped: {skipped}, Time: {elapsed/60:.1f}m\")\n\n\nif __name__ == \"__main__\":\n main()\n"}}}} |