1 line
2.5 KiB
JSON
1 line
2.5 KiB
JSON
{"repo": "chenghengli/eval-awareness", "files": {"main.py": {"641a132e61edd67c245a1ea5daf70cfce83bd274": {"t": 1785674355.592089, "n": 2220, "h": "257c8ae283134734", "c": "\"\"\"\nEval Detection Experiment — unified entry point.\n\nPipeline steps (run in order for a fresh experiment):\n python main.py build-pairs # Step 1: TF-IDF pairing of eval vs deploy\n python main.py generate-dataset # Step 2: Build forced-choice JSONL files\n\nRun forced-choice experiment on a model:\n python main.py run \\\\\n --model allenai/olmo-3.1-32b-instruct \\\\\n --provider openrouter \\\\\n --api-key sk-or-v1-... \\\\\n --epochs 4 --threads 30 --phase both\n\nRun pool-based experiment on a model:\n python main.py run-pool \\\\\n --model allenai/olmo-3.1-32b-instruct \\\\\n --provider openrouter \\\\\n --api-key sk-or-v1-... \\\\\n --epochs 4 --threads 30\n\nPool setup stages (run once to generate pool data):\n python main.py pool --stage assign\n python main.py pool --stage pilot\n python main.py pool --stage analyze-pilot --results-dir results/pool/MODEL/DATE\n python main.py pool --stage focused\n python main.py pool --stage rank --results-dir results/pool/MODEL/DATE\n\nAnalysis:\n python main.py analyze results/MODEL/DATE/with_sys.jsonl\n python main.py analyze-all\n\nSupported providers: openrouter, openai, anthropic\n\"\"\"\n\nimport subprocess\nimport sys\nfrom pathlib import Path\n\nROOT = Path(__file__).resolve().parent\nSRC = ROOT / \"src\"\n\nCOMMANDS = {\n \"build-pairs\": SRC / \"data\" / \"build_pairs.py\",\n \"generate-dataset\": SRC / \"data\" / \"generate_dataset.py\",\n \"run\": SRC / \"runners\" / \"run_experiment.py\",\n \"run-pool\": SRC / \"runners\" / \"run_pool_experiment.py\",\n \"pool\": SRC / \"pool\" / \"pool_experiment.py\",\n \"analyze\": SRC / \"analysis\" / \"analyze_results.py\",\n \"analyze-all\": SRC / \"analysis\" / \"analyze_all.py\",\n}\n\n\ndef main():\n if len(sys.argv) < 2 or sys.argv[1] in (\"-h\", \"--help\"):\n print(__doc__)\n sys.exit(0)\n\n cmd = sys.argv[1]\n rest = sys.argv[2:]\n\n if cmd not in COMMANDS:\n print(f\"error: unknown command '{cmd}'\")\n print(f\"available commands: {', '.join(COMMANDS)}\")\n sys.exit(1)\n\n result = subprocess.run([sys.executable, str(COMMANDS[cmd])] + rest)\n sys.exit(result.returncode)\n\n\nif __name__ == \"__main__\":\n main()\n"}}}} |