1 line
44 KiB
JSON
1 line
44 KiB
JSON
{"repo": "chencx2023/edureflect-ai", "files": {"results/EduReflect_Alignment_Colab_results_120.ipynb": {"8d51bb583b74642dba71fd7f83ca0a76e8e05a3a": {"t": 1785745480.5427246, "n": 40502, "h": "b5323b8ac4dcb284", "c": "{\n \"nbformat\": 4,\n \"nbformat_minor\": 0,\n \"metadata\": {\n \"colab\": {\n \"provenance\": [],\n \"machine_shape\": \"hm\",\n \"gpuType\": \"A100\"\n },\n \"kernelspec\": {\n \"name\": \"python3\",\n \"display_name\": \"Python 3\"\n },\n \"language_info\": {\n \"name\": \"python\"\n },\n \"accelerator\": \"GPU\"\n },\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"0g8RTasHyi_F\"\n },\n \"source\": [\n \"# EduReflect-Alignment\\n\",\n \"本 notebook **只做评测，不训练模型，也不重新生成数据**。主实验使用已冻结的 120 条平衡 test；建议使用 T4 或更好的 GPU。\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"metadata\": {\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"id\": \"b4q1IHY6yi_H\",\n \"outputId\": \"85835d47-201a-4ad6-f97d-0d50be6ace91\"\n },\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"name\": \"stdout\",\n \"text\": [\n \"Mounted at /content/drive\\n\",\n \"/content/drive/MyDrive/EduReflect-Alignment\\n\",\n \"\\u001b[2K \\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\\u001b[0m \\u001b[32m44.0/44.0 kB\\u001b[0m \\u001b[31m4.4 MB/s\\u001b[0m eta \\u001b[36m0:00:00\\u001b[0m\\n\",\n \"\\u001b[2K \\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\\u001b[0m \\u001b[32m12.0/12.0 MB\\u001b[0m \\u001b[31m148.3 MB/s\\u001b[0m eta \\u001b[36m0:00:00\\u001b[0m\\n\",\n \"\\u001b[2K \\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\\u001b[0m \\u001b[32m557.0/557.0 kB\\u001b[0m \\u001b[31m49.3 MB/s\\u001b[0m eta \\u001b[36m0:00:00\\u001b[0m\\n\",\n \"\\u001b[2K \\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\\u001b[0m \\u001b[32m531.0/531.0 kB\\u001b[0m \\u001b[31m46.6 MB/s\\u001b[0m eta \\u001b[36m0:00:00\\u001b[0m\\n\",\n \"\\u001b[2K \\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\\u001b[0m \\u001b[32m566.4/566.4 kB\\u001b[0m \\u001b[31m51.7 MB/s\\u001b[0m eta \\u001b[36m0:00:00\\u001b[0m\\n\",\n \"\\u001b[?25h\"\n ]\n }\n ],\n \"source\": [\n \"from google.colab import drive\\n\",\n \"drive.mount('/content/drive')\\n\",\n \"%cd /content/drive/MyDrive/EduReflect-Alignment\\n\",\n \"!pip install -q -r requirements-colab.txt\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [],\n \"metadata\": {\n \"id\": \"3jyfhMxAyjp4\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"id\": \"ePBQkPOAyi_I\",\n \"outputId\": \"653c3074-8ce0-42f6-8116-072263d8531e\"\n },\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"name\": \"stdout\",\n \"text\": [\n \"{\\n\",\n \" \\\"validation_samples\\\": 12,\\n\",\n \" \\\"test_samples\\\": 120,\\n\",\n \" \\\"total_samples\\\": 132,\\n\",\n \" \\\"validation_categories\\\": {\\n\",\n \" \\\"concept\\\": 3,\\n\",\n \" \\\"calculation\\\": 3,\\n\",\n \" \\\"pedagogy\\\": 3,\\n\",\n \" \\\"abstention\\\": 3\\n\",\n \" },\\n\",\n \" \\\"test_categories\\\": {\\n\",\n \" \\\"concept\\\": 30,\\n\",\n \" \\\"calculation\\\": 30,\\n\",\n \" \\\"pedagogy\\\": 30,\\n\",\n \" \\\"abstention\\\": 30\\n\",\n \" },\\n\",\n \" \\\"test_difficulties\\\": {\\n\",\n \" \\\"medium\\\": 40,\\n\",\n \" \\\"easy\\\": 40,\\n\",\n \" \\\"hard\\\": 40\\n\",\n \" },\\n\",\n \" \\\"test_subjects\\\": {\\n\",\n \" \\\"medicine_biology\\\": 20,\\n\",\n \" \\\"computer_science_ai\\\": 20,\\n\",\n \" \\\"mathematics_statistics\\\": 20,\\n\",\n \" \\\"economics_business\\\": 20,\\n\",\n \" \\\"humanities_social_sciences\\\": 20,\\n\",\n \" \\\"natural_sciences\\\": 20\\n\",\n \" },\\n\",\n \" \\\"errors\\\": []\\n\",\n \"}\\n\",\n \"{\\n\",\n \" \\\"sft\\\": {\\n\",\n \" \\\"samples\\\": 1000,\\n\",\n \" \\\"missing_required_fields\\\": 0,\\n\",\n \" \\\"output_avg_chars\\\": 335.008,\\n\",\n \" \\\"output_emoji_rate\\\": 0.148,\\n\",\n \" \\\"duplicate_outputs\\\": 0\\n\",\n \" },\\n\",\n \" \\\"preferences\\\": {\\n\",\n \" \\\"samples\\\": 471,\\n\",\n \" \\\"chosen_avg_chars\\\": 200.92569002123142,\\n\",\n \" \\\"rejected_avg_chars\\\": 191.44373673036094,\\n\",\n \" \\\"chosen_rejected_length_ratio\\\": 1.0495286680714206,\\n\",\n \" \\\"chosen_emoji_rate\\\": 0.0,\\n\",\n \" \\\"rejected_emoji_rate\\\": 0.0,\\n\",\n \" \\\"clean_pair_rate\\\": 1.0,\\n\",\n \" \\\"duplicate_prompts\\\": 0,\\n\",\n \" \\\"issues\\\": {}\\n\",\n \" }\\n\",\n \"}\\n\"\n ]\n }\n ],\n \"source\": [\n \"!python scripts/validate_benchmark.py\\n\",\n \"!python scripts/audit_data.py --strict\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"H02UanXFyi_I\"\n },\n \"source\": [\n \"## Base / SFT / DPO 主实验（120 条 test）\\n\",\n \"四类题型各30条、三档难度各40条、六个学科组各20条。运行中断后重新执行同一单元会自动续跑。\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"id\": \"5GAsdWd5yi_J\",\n \"outputId\": \"91a33b7b-29ba-40a9-8608-3ff22368c989\"\n },\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"name\": \"stdout\",\n \"text\": [\n \"base: all requested predictions already exist\\n\",\n \"sft: all requested predictions already exist\\n\",\n \"dpo: all requested predictions already exist\\n\",\n \"Wrote 36 predictions to results/checkpoints_validation/predictions.jsonl\\n\"\n ]\n }\n ],\n \"source\": [\n \"!python scripts/evaluate_alignment.py \\\\\\n\",\n \" --data data/alignment_validation.jsonl \\\\\\n\",\n \" --checkpoints base,sft,dpo \\\\\\n\",\n \" --reflection-rounds 0 \\\\\\n\",\n \" --output results/checkpoints_validation/predictions.jsonl\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"id\": \"mcHmpoN5yi_J\",\n \"outputId\": \"ff892de8-b0dd-4edc-e939-cae7b3727eae\"\n },\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"name\": \"stdout\",\n \"text\": [\n \"base: all requested predictions already exist\\n\",\n \"sft: all requested predictions already exist\\n\",\n \"dpo: all requested predictions already exist\\n\",\n \"Wrote 360 predictions to results/checkpoints_test_120/predictions.jsonl\\n\"\n ]\n }\n ],\n \"source\": [\n \"!python scripts/evaluate_alignment.py \\\\\\n\",\n \" --data data/alignment_test_120.jsonl \\\\\\n\",\n \" --checkpoints base,sft,dpo \\\\\\n\",\n \" --reflection-rounds 0 \\\\\\n\",\n \" --output results/checkpoints_test_120/predictions.jsonl\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"wwH0eSUyyi_J\"\n },\n \"source\": [\n \"## Judge\\n\",\n \"优先使用 DeepSeek 等不同模型家族；只有 DashScope Key 时，将后续命令的 provider 改为 dashscope 并加 `--model qwen-plus`。120 条 test 中有60道开放题，三个 checkpoint 共需评审180个回答，脚本支持断点续跑。\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"metadata\": {\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"id\": \"-4pleQfMyi_J\",\n \"outputId\": \"beaedc35-2191-4d94-d396-0678cfcc1063\"\n },\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"DeepSeek API key: ··········\\n\"\n ]\n }\n ],\n \"source\": [\n \"import getpass, os\\n\",\n \"os.environ['DEEPSEEK_API_KEY'] = getpass.getpass('DeepSeek API key: ')\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"id\": \"D6iOYbCkyi_K\",\n \"outputId\": \"0a90febb-b9fe-4924-ac38-18a627397d04\"\n },\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"name\": \"stdout\",\n \"text\": [\n \"Wrote 18 judge rows to results/checkpoints_validation/judge_scores.jsonl\\n\",\n \"[1/180] judged test_concept_01 / base\\n\",\n \"[2/180] judged test_concept_02 / base\\n\",\n \"[3/180] judged test_concept_03 / base\\n\",\n \"[4/180] judged test_concept_04 / base\\n\",\n \"[5/180] judged test_concept_05 / base\\n\",\n \"[6/180] judged test_concept_06 / base\\n\",\n \"[7/180] judged test_concept_07 / base\\n\",\n \"[8/180] judged test_concept_08 / base\\n\",\n \"[9/180] judged test_concept_09 / base\\n\",\n \"[10/180] judged test_concept_10 / base\\n\",\n \"[11/180] judged test_concept_11 / base\\n\",\n \"[12/180] judged test_concept_12 / base\\n\",\n \"[13/180] judged test_pedagogy_01 / base\\n\",\n \"[14/180] judged test_pedagogy_02 / base\\n\",\n \"[15/180] judged test_pedagogy_03 / base\\n\",\n \"[16/180] judged test_pedagogy_04 / base\\n\",\n \"[17/180] judged test_pedagogy_05 / base\\n\",\n \"[18/180] judged test_pedagogy_06 / base\\n\",\n \"[19/180] judged test_pedagogy_07 / base\\n\",\n \"[20/180] judged test_pedagogy_08 / base\\n\",\n \"[21/180] judged test_pedagogy_09 / base\\n\",\n \"[22/180] judged test_pedagogy_10 / base\\n\",\n \"[23/180] judged test_pedagogy_11 / base\\n\",\n \"[24/180] judged test_pedagogy_12 / base\\n\",\n \"[25/180] judged test_concept_13 / base\\n\",\n \"[26/180] judged test_concept_14 / base\\n\",\n \"[27/180] judged test_concept_15 / base\\n\",\n \"[28/180] judged test_concept_16 / base\\n\",\n \"[29/180] judged test_concept_17 / base\\n\",\n \"[30/180] judged test_concept_18 / base\\n\",\n \"[31/180] judged test_concept_19 / base\\n\",\n \"[32/180] judged test_concept_20 / base\\n\",\n \"[33/180] judged test_concept_21 / base\\n\",\n \"[34/180] judged test_concept_22 / base\\n\",\n \"[35/180] judged test_concept_23 / base\\n\",\n \"[36/180] judged test_concept_24 / base\\n\",\n \"[37/180] judged test_concept_25 / base\\n\",\n \"[38/180] judged test_concept_26 / base\\n\",\n \"[39/180] judged test_concept_27 / base\\n\",\n \"[40/180] judged test_concept_28 / base\\n\",\n \"[41/180] judged test_concept_29 / base\\n\",\n \"[42/180] judged test_concept_30 / base\\n\",\n \"[43/180] judged test_pedagogy_13 / base\\n\",\n \"[44/180] judged test_pedagogy_14 / base\\n\",\n \"[45/180] judged test_pedagogy_15 / base\\n\",\n \"[46/180] judged test_pedagogy_16 / base\\n\",\n \"[47/180] judged test_pedagogy_17 / base\\n\",\n \"[48/180] judged test_pedagogy_18 / base\\n\",\n \"[49/180] judged test_pedagogy_19 / base\\n\",\n \"[50/180] judged test_pedagogy_20 / base\\n\",\n \"[51/180] judged test_pedagogy_21 / base\\n\",\n \"[52/180] judged test_pedagogy_22 / base\\n\",\n \"[53/180] judged test_pedagogy_23 / base\\n\",\n \"[54/180] judged test_pedagogy_24 / base\\n\",\n \"[55/180] judged test_pedagogy_25 / base\\n\",\n \"[56/180] judged test_pedagogy_26 / base\\n\",\n \"[57/180] judged test_pedagogy_27 / base\\n\",\n \"[58/180] judged test_pedagogy_28 / base\\n\",\n \"[59/180] judged test_pedagogy_29 / base\\n\",\n \"[60/180] judged test_pedagogy_30 / base\\n\",\n \"[61/180] judged test_concept_01 / sft\\n\",\n \"[62/180] judged test_concept_02 / sft\\n\",\n \"[63/180] judged test_concept_03 / sft\\n\",\n \"[64/180] judged test_concept_04 / sft\\n\",\n \"[65/180] judged test_concept_05 / sft\\n\",\n \"[66/180] judged test_concept_06 / sft\\n\",\n \"[67/180] judged test_concept_07 / sft\\n\",\n \"[68/180] judged test_concept_08 / sft\\n\",\n \"[69/180] judged test_concept_09 / sft\\n\",\n \"[70/180] judged test_concept_10 / sft\\n\",\n \"[71/180] judged test_concept_11 / sft\\n\",\n \"[72/180] judged test_concept_12 / sft\\n\",\n \"[73/180] judged test_pedagogy_01 / sft\\n\",\n \"[74/180] judged test_pedagogy_02 / sft\\n\",\n \"[75/180] judged test_pedagogy_03 / sft\\n\",\n \"[76/180] judged test_pedagogy_04 / sft\\n\",\n \"[77/180] judged test_pedagogy_05 / sft\\n\",\n \"[78/180] judged test_pedagogy_06 / sft\\n\",\n \"[79/180] judged test_pedagogy_07 / sft\\n\",\n \"[80/180] judged test_pedagogy_08 / sft\\n\",\n \"[81/180] judged test_pedagogy_09 / sft\\n\",\n \"[82/180] judged test_pedagogy_10 / sft\\n\",\n \"[83/180] judged test_pedagogy_11 / sft\\n\",\n \"[84/180] judged test_pedagogy_12 / sft\\n\",\n \"[85/180] judged test_concept_13 / sft\\n\",\n \"[86/180] judged test_concept_14 / sft\\n\",\n \"[87/180] judged test_concept_15 / sft\\n\",\n \"[88/180] judged test_concept_16 / sft\\n\",\n \"[89/180] judged test_concept_17 / sft\\n\",\n \"[90/180] judged test_concept_18 / sft\\n\",\n \"[91/180] judged test_concept_19 / sft\\n\",\n \"[92/180] judged test_concept_20 / sft\\n\",\n \"[93/180] judged test_concept_21 / sft\\n\",\n \"[94/180] judged test_concept_22 / sft\\n\",\n \"[95/180] judged test_concept_23 / sft\\n\",\n \"[96/180] judged test_concept_24 / sft\\n\",\n \"[97/180] judged test_concept_25 / sft\\n\",\n \"[98/180] judged test_concept_26 / sft\\n\",\n \"[99/180] judged test_concept_27 / sft\\n\",\n \"[100/180] judged test_concept_28 / sft\\n\",\n \"[101/180] judged test_concept_29 / sft\\n\",\n \"[102/180] judged test_concept_30 / sft\\n\",\n \"[103/180] judged test_pedagogy_13 / sft\\n\",\n \"[104/180] judged test_pedagogy_14 / sft\\n\",\n \"[105/180] judged test_pedagogy_15 / sft\\n\",\n \"[106/180] judged test_pedagogy_16 / sft\\n\",\n \"[107/180] judged test_pedagogy_17 / sft\\n\",\n \"[108/180] judged test_pedagogy_18 / sft\\n\",\n \"[109/180] judged test_pedagogy_19 / sft\\n\",\n \"[110/180] judged test_pedagogy_20 / sft\\n\",\n \"[111/180] judged test_pedagogy_21 / sft\\n\",\n \"[112/180] judged test_pedagogy_22 / sft\\n\",\n \"[113/180] judged test_pedagogy_23 / sft\\n\",\n \"[114/180] judged test_pedagogy_24 / sft\\n\",\n \"[115/180] judged test_pedagogy_25 / sft\\n\",\n \"[116/180] judged test_pedagogy_26 / sft\\n\",\n \"[117/180] judged test_pedagogy_27 / sft\\n\",\n \"[118/180] judged test_pedagogy_28 / sft\\n\",\n \"[119/180] judged test_pedagogy_29 / sft\\n\",\n \"[120/180] judged test_pedagogy_30 / sft\\n\",\n \"[121/180] judged test_concept_01 / dpo\\n\",\n \"[122/180] judged test_concept_02 / dpo\\n\",\n \"[123/180] judged test_concept_03 / dpo\\n\",\n \"[124/180] judged test_concept_04 / dpo\\n\",\n \"[125/180] judged test_concept_05 / dpo\\n\",\n \"[126/180] judged test_concept_06 / dpo\\n\",\n \"[127/180] judged test_concept_07 / dpo\\n\",\n \"[128/180] judged test_concept_08 / dpo\\n\",\n \"[129/180] judged test_concept_09 / dpo\\n\",\n \"[130/180] judged test_concept_10 / dpo\\n\",\n \"[131/180] judged test_concept_11 / dpo\\n\",\n \"[132/180] judged test_concept_12 / dpo\\n\",\n \"[133/180] judged test_pedagogy_01 / dpo\\n\",\n \"[134/180] judged test_pedagogy_02 / dpo\\n\",\n \"[135/180] judged test_pedagogy_03 / dpo\\n\",\n \"[136/180] judged test_pedagogy_04 / dpo\\n\",\n \"[137/180] judged test_pedagogy_05 / dpo\\n\",\n \"[138/180] judged test_pedagogy_06 / dpo\\n\",\n \"[139/180] judged test_pedagogy_07 / dpo\\n\",\n \"[140/180] judged test_pedagogy_08 / dpo\\n\",\n \"[141/180] judged test_pedagogy_09 / dpo\\n\",\n \"[142/180] judged test_pedagogy_10 / dpo\\n\",\n \"[143/180] judged test_pedagogy_11 / dpo\\n\",\n \"[144/180] judged test_pedagogy_12 / dpo\\n\",\n \"[145/180] judged test_concept_13 / dpo\\n\",\n \"[146/180] judged test_concept_14 / dpo\\n\",\n \"[147/180] judged test_concept_15 / dpo\\n\",\n \"[148/180] judged test_concept_16 / dpo\\n\",\n \"[149/180] judged test_concept_17 / dpo\\n\",\n \"[150/180] judged test_concept_18 / dpo\\n\",\n \"[151/180] judged test_concept_19 / dpo\\n\",\n \"[152/180] judged test_concept_20 / dpo\\n\",\n \"[153/180] judged test_concept_21 / dpo\\n\",\n \"[154/180] judged test_concept_22 / dpo\\n\",\n \"[155/180] judged test_concept_23 / dpo\\n\",\n \"[156/180] judged test_concept_24 / dpo\\n\",\n \"[157/180] judged test_concept_25 / dpo\\n\",\n \"[158/180] judged test_concept_26 / dpo\\n\",\n \"[159/180] judged test_concept_27 / dpo\\n\",\n \"[160/180] judged test_concept_28 / dpo\\n\",\n \"[161/180] judged test_concept_29 / dpo\\n\",\n \"[162/180] judged test_concept_30 / dpo\\n\",\n \"[163/180] judged test_pedagogy_13 / dpo\\n\",\n \"[164/180] judged test_pedagogy_14 / dpo\\n\",\n \"[165/180] judged test_pedagogy_15 / dpo\\n\",\n \"[166/180] judged test_pedagogy_16 / dpo\\n\",\n \"[167/180] judged test_pedagogy_17 / dpo\\n\",\n \"[168/180] judged test_pedagogy_18 / dpo\\n\",\n \"[169/180] judged test_pedagogy_19 / dpo\\n\",\n \"[170/180] judged test_pedagogy_20 / dpo\\n\",\n \"[171/180] judged test_pedagogy_21 / dpo\\n\",\n \"[172/180] judged test_pedagogy_22 / dpo\\n\",\n \"[173/180] judged test_pedagogy_23 / dpo\\n\",\n \"[174/180] judged test_pedagogy_24 / dpo\\n\",\n \"[175/180] judged test_pedagogy_25 / dpo\\n\",\n \"[176/180] judged test_pedagogy_26 / dpo\\n\",\n \"[177/180] judged test_pedagogy_27 / dpo\\n\",\n \"[178/180] judged test_pedagogy_28 / dpo\\n\",\n \"[179/180] judged test_pedagogy_29 / dpo\\n\",\n \"[180/180] judged test_pedagogy_30 / dpo\\n\",\n \"Wrote 180 judge rows to results/checkpoints_test_120/judge_scores.jsonl\\n\"\n ]\n }\n ],\n \"source\": [\n \"!python scripts/judge_open_answers.py \\\\\\n\",\n \" --predictions results/checkpoints_validation/predictions.jsonl \\\\\\n\",\n \" --provider deepseek \\\\\\n\",\n \" --output results/checkpoints_validation/judge_scores.jsonl\\n\",\n \"\\n\",\n \"!python scripts/judge_open_answers.py \\\\\\n\",\n \" --predictions results/checkpoints_test_120/predictions.jsonl \\\\\\n\",\n \" --provider deepseek \\\\\\n\",\n \" --output results/checkpoints_test_120/judge_scores.jsonl\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"id\": \"28iIbAZDyi_K\",\n \"outputId\": \"70b71bfe-e511-4f24-8a3e-5d12cc30a5b4\"\n },\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"name\": \"stdout\",\n \"text\": [\n \"[\\n\",\n \" {\\n\",\n \" \\\"method\\\": \\\"base\\\",\\n\",\n \" \\\"samples\\\": 120,\\n\",\n \" \\\"labelled_samples\\\": 120,\\n\",\n \" \\\"accuracy\\\": 0.43333333333333335,\\n\",\n \" \\\"accuracy_ci95_low\\\": 0.3416666666666667,\\n\",\n \" \\\"accuracy_ci95_high\\\": 0.525,\\n\",\n \" \\\"ECE\\\": 0.35631831519238877,\\n\",\n \" \\\"Brier\\\": 0.3896907390642678,\\n\",\n \" \\\"mean_confidence\\\": 0.2867758023693804,\\n\",\n \" \\\"judge_mean\\\": 3.6233333333333335,\\n\",\n \" \\\"judge_std\\\": 2.4237781591910905,\\n\",\n \" \\\"avg_answer_chars\\\": 229.89166666666668,\\n\",\n \" \\\"emoji_rate\\\": 0.0,\\n\",\n \" \\\"avg_latency_ms\\\": 4188.796842000019\\n\",\n \" },\\n\",\n \" {\\n\",\n \" \\\"method\\\": \\\"dpo\\\",\\n\",\n \" \\\"samples\\\": 120,\\n\",\n \" \\\"labelled_samples\\\": 120,\\n\",\n \" \\\"accuracy\\\": 0.39166666666666666,\\n\",\n \" \\\"accuracy_ci95_low\\\": 0.30833333333333335,\\n\",\n \" \\\"accuracy_ci95_high\\\": 0.475,\\n\",\n \" \\\"ECE\\\": 0.22732827016939325,\\n\",\n \" \\\"Brier\\\": 0.3225722129323905,\\n\",\n \" \\\"mean_confidence\\\": 0.32398260451790756,\\n\",\n \" \\\"judge_mean\\\": 4.405,\\n\",\n \" \\\"judge_std\\\": 2.138859588421608,\\n\",\n \" \\\"avg_answer_chars\\\": 228.83333333333334,\\n\",\n \" \\\"emoji_rate\\\": 0.008333333333333333,\\n\",\n \" \\\"avg_latency_ms\\\": 8440.784955249974\\n\",\n \" },\\n\",\n \" {\\n\",\n \" \\\"method\\\": \\\"sft\\\",\\n\",\n \" \\\"samples\\\": 120,\\n\",\n \" \\\"labelled_samples\\\": 120,\\n\",\n \" \\\"accuracy\\\": 0.4,\\n\",\n \" \\\"accuracy_ci95_low\\\": 0.30833333333333335,\\n\",\n \" \\\"accuracy_ci95_high\\\": 0.49166666666666664,\\n\",\n \" \\\"ECE\\\": 0.2149007183484543,\\n\",\n \" \\\"Brier\\\": 0.29956070398156004,\\n\",\n \" \\\"mean_confidence\\\": 0.35979992134509503,\\n\",\n \" \\\"judge_mean\\\": 4.125,\\n\",\n \" \\\"judge_std\\\": 2.1082264214423927,\\n\",\n \" \\\"avg_answer_chars\\\": 160.7,\\n\",\n \" \\\"emoji_rate\\\": 0.008333333333333333,\\n\",\n \" \\\"avg_latency_ms\\\": 5900.1180156832825\\n\",\n \" }\\n\",\n \"]\\n\"\n ]\n }\n ],\n \"source\": [\n \"!python scripts/analyze_alignment.py \\\\\\n\",\n \" --predictions results/checkpoints_test_120/predictions.jsonl \\\\\\n\",\n \" --judge-scores results/checkpoints_test_120/judge_scores.jsonl \\\\\\n\",\n \" --output-dir results/checkpoints_test_120/analysis\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"6JkpRE-6yi_K\"\n },\n \"source\": [\n \"## 可选：DPO 的 0 / 1 / 2 轮反思消融\\n\",\n \"第一项优化只要求完成上面的主实验；本节计算量较大，可在主实验结束后另行运行。\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"id\": \"IjY8k-IGyi_K\",\n \"outputId\": \"09c6b0bc-59bb-4571-f2e5-d17bd07805fc\"\n },\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"name\": \"stdout\",\n \"text\": [\n \"Loading dpo: adapters/dpo_1p5b\\n\",\n \"tokenizer_config.json: 7.30kB [00:00, 31.8MB/s]\\n\",\n \"vocab.json: 2.78MB [00:00, 86.0MB/s]\\n\",\n \"merges.txt: 1.67MB [00:00, 136MB/s]\\n\",\n \"tokenizer.json: 7.03MB [00:00, 183MB/s]\\n\",\n \"config.json: 100% 660/660 [00:00<00:00, 5.80MB/s]\\n\",\n \"model.safetensors: 100% 3.09G/3.09G [00:06<00:00, 467MB/s]\\n\",\n \"generation_config.json: 100% 242/242 [00:00<00:00, 2.42MB/s]\\n\",\n \"[56/120] dpo: test_concept_20\\n\",\n \"[57/120] dpo: test_concept_21\\n\",\n \"[58/120] dpo: test_concept_22\\n\",\n \"[59/120] dpo: test_concept_23\\n\",\n \"[60/120] dpo: test_concept_24\\n\",\n \"[61/120] dpo: test_concept_25\\n\",\n \"[62/120] dpo: test_concept_26\\n\",\n \"[63/120] dpo: test_concept_27\\n\",\n \"[64/120] dpo: test_concept_28\\n\",\n \"[65/120] dpo: test_concept_29\\n\",\n \"[66/120] dpo: test_concept_30\\n\",\n \"[67/120] dpo: test_calc_13\\n\",\n \"[68/120] dpo: test_calc_14\\n\",\n \"[69/120] dpo: test_calc_15\\n\",\n \"[70/120] dpo: test_calc_16\\n\",\n \"[71/120] dpo: test_calc_17\\n\",\n \"[72/120] dpo: test_calc_18\\n\",\n \"[73/120] dpo: test_calc_19\\n\",\n \"[74/120] dpo: test_calc_20\\n\",\n \"[75/120] dpo: test_calc_21\\n\",\n \"[76/120] dpo: test_calc_22\\n\",\n \"[77/120] dpo: test_calc_23\\n\",\n \"[78/120] dpo: test_calc_24\\n\",\n \"[79/120] dpo: test_calc_25\\n\",\n \"[80/120] dpo: test_calc_26\\n\",\n \"[81/120] dpo: test_calc_27\\n\",\n \"[82/120] dpo: test_calc_28\\n\",\n \"[83/120] dpo: test_calc_29\\n\",\n \"[84/120] dpo: test_calc_30\\n\",\n \"[85/120] dpo: test_pedagogy_13\\n\",\n \"[86/120] dpo: test_pedagogy_14\\n\",\n \"[87/120] dpo: test_pedagogy_15\\n\",\n \"[88/120] dpo: test_pedagogy_16\\n\",\n \"[89/120] dpo: test_pedagogy_17\\n\",\n \"[90/120] dpo: test_pedagogy_18\\n\",\n \"[91/120] dpo: test_pedagogy_19\\n\",\n \"[92/120] dpo: test_pedagogy_20\\n\",\n \"[93/120] dpo: test_pedagogy_21\\n\",\n \"[94/120] dpo: test_pedagogy_22\\n\",\n \"[95/120] dpo: test_pedagogy_23\\n\",\n \"[96/120] dpo: test_pedagogy_24\\n\",\n \"[97/120] dpo: test_pedagogy_25\\n\",\n \"[98/120] dpo: test_pedagogy_26\\n\",\n \"[99/120] dpo: test_pedagogy_27\\n\",\n \"[100/120] dpo: test_pedagogy_28\\n\",\n \"[101/120] dpo: test_pedagogy_29\\n\",\n \"[102/120] dpo: test_pedagogy_30\\n\",\n \"[103/120] dpo: test_abstain_13\\n\",\n \"[104/120] dpo: test_abstain_14\\n\",\n \"[105/120] dpo: test_abstain_15\\n\",\n \"[106/120] dpo: test_abstain_16\\n\",\n \"[107/120] dpo: test_abstain_17\\n\",\n \"[108/120] dpo: test_abstain_18\\n\",\n \"[109/120] dpo: test_abstain_19\\n\",\n \"[110/120] dpo: test_abstain_20\\n\",\n \"[111/120] dpo: test_abstain_21\\n\",\n \"[112/120] dpo: test_abstain_22\\n\",\n \"[113/120] dpo: test_abstain_23\\n\",\n \"[114/120] dpo: test_abstain_24\\n\",\n \"[115/120] dpo: test_abstain_25\\n\",\n \"[116/120] dpo: test_abstain_26\\n\",\n \"[117/120] dpo: test_abstain_27\\n\",\n \"[118/120] dpo: test_abstain_28\\n\",\n \"[119/120] dpo: test_abstain_29\\n\",\n \"[120/120] dpo: test_abstain_30\\n\",\n \"Wrote 360 predictions to results/reflection_test_120/predictions.jsonl\\n\"\n ]\n }\n ],\n \"source\": [\n \"!python scripts/evaluate_alignment.py \\\\\\n\",\n \" --data data/alignment_test_120.jsonl \\\\\\n\",\n \" --checkpoints dpo \\\\\\n\",\n \" --reflection-rounds 0,1,2 \\\\\\n\",\n \" --output results/reflection_test_120/predictions.jsonl\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 4,\n \"metadata\": {\n \"id\": \"rLPWZqBtyi_K\",\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"outputId\": \"1817da74-667b-4abf-eec3-96337b587ea6\"\n },\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"name\": \"stdout\",\n \"text\": [\n \"[1/180] judged test_concept_01 / dpo\\n\",\n \"[2/180] judged test_concept_01 / dpo\\n\",\n \"[3/180] judged test_concept_01 / dpo\\n\",\n \"[4/180] judged test_concept_02 / dpo\\n\",\n \"[5/180] judged test_concept_02 / dpo\\n\",\n \"[6/180] judged test_concept_02 / dpo\\n\",\n \"[7/180] judged test_concept_03 / dpo\\n\",\n \"[8/180] judged test_concept_03 / dpo\\n\",\n \"[9/180] judged test_concept_03 / dpo\\n\",\n \"[10/180] judged test_concept_04 / dpo\\n\",\n \"[11/180] judged test_concept_04 / dpo\\n\",\n \"[12/180] judged test_concept_04 / dpo\\n\",\n \"[13/180] judged test_concept_05 / dpo\\n\",\n \"[14/180] judged test_concept_05 / dpo\\n\",\n \"[15/180] judged test_concept_05 / dpo\\n\",\n \"[16/180] judged test_concept_06 / dpo\\n\",\n \"[17/180] judged test_concept_06 / dpo\\n\",\n \"[18/180] judged test_concept_06 / dpo\\n\",\n \"[19/180] judged test_concept_07 / dpo\\n\",\n \"[20/180] judged test_concept_07 / dpo\\n\",\n \"[21/180] judged test_concept_07 / dpo\\n\",\n \"[22/180] judged test_concept_08 / dpo\\n\",\n \"[23/180] judged test_concept_08 / dpo\\n\",\n \"[24/180] judged test_concept_08 / dpo\\n\",\n \"[25/180] judged test_concept_09 / dpo\\n\",\n \"[26/180] judged test_concept_09 / dpo\\n\",\n \"[27/180] judged test_concept_09 / dpo\\n\",\n \"[28/180] judged test_concept_10 / dpo\\n\",\n \"[29/180] judged test_concept_10 / dpo\\n\",\n \"[30/180] judged test_concept_10 / dpo\\n\",\n \"[31/180] judged test_concept_11 / dpo\\n\",\n \"[32/180] judged test_concept_11 / dpo\\n\",\n \"[33/180] judged test_concept_11 / dpo\\n\",\n \"[34/180] judged test_concept_12 / dpo\\n\",\n \"[35/180] judged test_concept_12 / dpo\\n\",\n \"[36/180] judged test_concept_12 / dpo\\n\",\n \"[37/180] judged test_pedagogy_01 / dpo\\n\",\n \"[38/180] judged test_pedagogy_01 / dpo\\n\",\n \"[39/180] judged test_pedagogy_01 / dpo\\n\",\n \"[40/180] judged test_pedagogy_02 / dpo\\n\",\n \"[41/180] judged test_pedagogy_02 / dpo\\n\",\n \"[42/180] judged test_pedagogy_02 / dpo\\n\",\n \"[43/180] judged test_pedagogy_03 / dpo\\n\",\n \"[44/180] judged test_pedagogy_03 / dpo\\n\",\n \"[45/180] judged test_pedagogy_03 / dpo\\n\",\n \"[46/180] judged test_pedagogy_04 / dpo\\n\",\n \"[47/180] judged test_pedagogy_04 / dpo\\n\",\n \"[48/180] judged test_pedagogy_04 / dpo\\n\",\n \"[49/180] judged test_pedagogy_05 / dpo\\n\",\n \"[50/180] judged test_pedagogy_05 / dpo\\n\",\n \"[51/180] judged test_pedagogy_05 / dpo\\n\",\n \"[52/180] judged test_pedagogy_06 / dpo\\n\",\n \"[53/180] judged test_pedagogy_06 / dpo\\n\",\n \"[54/180] judged test_pedagogy_06 / dpo\\n\",\n \"[55/180] judged test_pedagogy_07 / dpo\\n\",\n \"[56/180] judged test_pedagogy_07 / dpo\\n\",\n \"[57/180] judged test_pedagogy_07 / dpo\\n\",\n \"[58/180] judged test_pedagogy_08 / dpo\\n\",\n \"[59/180] judged test_pedagogy_08 / dpo\\n\",\n \"[60/180] judged test_pedagogy_08 / dpo\\n\",\n \"[61/180] judged test_pedagogy_09 / dpo\\n\",\n \"[62/180] judged test_pedagogy_09 / dpo\\n\",\n \"[63/180] judged test_pedagogy_09 / dpo\\n\",\n \"[64/180] judged test_pedagogy_10 / dpo\\n\",\n \"[65/180] judged test_pedagogy_10 / dpo\\n\",\n \"[66/180] judged test_pedagogy_10 / dpo\\n\",\n \"[67/180] judged test_pedagogy_11 / dpo\\n\",\n \"[68/180] judged test_pedagogy_11 / dpo\\n\",\n \"[69/180] judged test_pedagogy_11 / dpo\\n\",\n \"[70/180] judged test_pedagogy_12 / dpo\\n\",\n \"[71/180] judged test_pedagogy_12 / dpo\\n\",\n \"[72/180] judged test_pedagogy_12 / dpo\\n\",\n \"[73/180] judged test_concept_13 / dpo\\n\",\n \"[74/180] judged test_concept_13 / dpo\\n\",\n \"[75/180] judged test_concept_13 / dpo\\n\",\n \"[76/180] judged test_concept_14 / dpo\\n\",\n \"[77/180] judged test_concept_14 / dpo\\n\",\n \"[78/180] judged test_concept_14 / dpo\\n\",\n \"[79/180] judged test_concept_15 / dpo\\n\",\n \"[80/180] judged test_concept_15 / dpo\\n\",\n \"[81/180] judged test_concept_15 / dpo\\n\",\n \"[82/180] judged test_concept_16 / dpo\\n\",\n \"[83/180] judged test_concept_16 / dpo\\n\",\n \"[84/180] judged test_concept_16 / dpo\\n\",\n \"[85/180] judged test_concept_17 / dpo\\n\",\n \"[86/180] judged test_concept_17 / dpo\\n\",\n \"[87/180] judged test_concept_17 / dpo\\n\",\n \"[88/180] judged test_concept_18 / dpo\\n\",\n \"[89/180] judged test_concept_18 / dpo\\n\",\n \"[90/180] judged test_concept_18 / dpo\\n\",\n \"[91/180] judged test_concept_19 / dpo\\n\",\n \"[92/180] judged test_concept_19 / dpo\\n\",\n \"[93/180] judged test_concept_19 / dpo\\n\",\n \"[94/180] judged test_concept_20 / dpo\\n\",\n \"[95/180] judged test_concept_20 / dpo\\n\",\n \"[96/180] judged test_concept_20 / dpo\\n\",\n \"[97/180] judged test_concept_21 / dpo\\n\",\n \"[98/180] judged test_concept_21 / dpo\\n\",\n \"[99/180] judged test_concept_21 / dpo\\n\",\n \"[100/180] judged test_concept_22 / dpo\\n\",\n \"[101/180] judged test_concept_22 / dpo\\n\",\n \"[102/180] judged test_concept_22 / dpo\\n\",\n \"[103/180] judged test_concept_23 / dpo\\n\",\n \"[104/180] judged test_concept_23 / dpo\\n\",\n \"[105/180] judged test_concept_23 / dpo\\n\",\n \"[106/180] judged test_concept_24 / dpo\\n\",\n \"[107/180] judged test_concept_24 / dpo\\n\",\n \"[108/180] judged test_concept_24 / dpo\\n\",\n \"[109/180] judged test_concept_25 / dpo\\n\",\n \"[110/180] judged test_concept_25 / dpo\\n\",\n \"[111/180] judged test_concept_25 / dpo\\n\",\n \"[112/180] judged test_concept_26 / dpo\\n\",\n \"[113/180] judged test_concept_26 / dpo\\n\",\n \"[114/180] judged test_concept_26 / dpo\\n\",\n \"[115/180] judged test_concept_27 / dpo\\n\",\n \"[116/180] judged test_concept_27 / dpo\\n\",\n \"[117/180] judged test_concept_27 / dpo\\n\",\n \"[118/180] judged test_concept_28 / dpo\\n\",\n \"[119/180] judged test_concept_28 / dpo\\n\",\n \"[120/180] judged test_concept_28 / dpo\\n\",\n \"[121/180] judged test_concept_29 / dpo\\n\",\n \"[122/180] judged test_concept_29 / dpo\\n\",\n \"[123/180] judged test_concept_29 / dpo\\n\",\n \"[124/180] judged test_concept_30 / dpo\\n\",\n \"[125/180] judged test_concept_30 / dpo\\n\",\n \"[126/180] judged test_concept_30 / dpo\\n\",\n \"[127/180] judged test_pedagogy_13 / dpo\\n\",\n \"[128/180] judged test_pedagogy_13 / dpo\\n\",\n \"[129/180] judged test_pedagogy_13 / dpo\\n\",\n \"[130/180] judged test_pedagogy_14 / dpo\\n\",\n \"[131/180] judged test_pedagogy_14 / dpo\\n\",\n \"[132/180] judged test_pedagogy_14 / dpo\\n\",\n \"[133/180] judged test_pedagogy_15 / dpo\\n\",\n \"[134/180] judged test_pedagogy_15 / dpo\\n\",\n \"[135/180] judged test_pedagogy_15 / dpo\\n\",\n \"[136/180] judged test_pedagogy_16 / dpo\\n\",\n \"[137/180] judged test_pedagogy_16 / dpo\\n\",\n \"[138/180] judged test_pedagogy_16 / dpo\\n\",\n \"[139/180] judged test_pedagogy_17 / dpo\\n\",\n \"[140/180] judged test_pedagogy_17 / dpo\\n\",\n \"[141/180] judged test_pedagogy_17 / dpo\\n\",\n \"[142/180] judged test_pedagogy_18 / dpo\\n\",\n \"[143/180] judged test_pedagogy_18 / dpo\\n\",\n \"[144/180] judged test_pedagogy_18 / dpo\\n\",\n \"[145/180] judged test_pedagogy_19 / dpo\\n\",\n \"[146/180] judged test_pedagogy_19 / dpo\\n\",\n \"[147/180] judged test_pedagogy_19 / dpo\\n\",\n \"[148/180] judged test_pedagogy_20 / dpo\\n\",\n \"[149/180] judged test_pedagogy_20 / dpo\\n\",\n \"[150/180] judged test_pedagogy_20 / dpo\\n\",\n \"[151/180] judged test_pedagogy_21 / dpo\\n\",\n \"[152/180] judged test_pedagogy_21 / dpo\\n\",\n \"[153/180] judged test_pedagogy_21 / dpo\\n\",\n \"[154/180] judged test_pedagogy_22 / dpo\\n\",\n \"[155/180] judged test_pedagogy_22 / dpo\\n\",\n \"[156/180] judged test_pedagogy_22 / dpo\\n\",\n \"[157/180] judged test_pedagogy_23 / dpo\\n\",\n \"[158/180] judged test_pedagogy_23 / dpo\\n\",\n \"[159/180] judged test_pedagogy_23 / dpo\\n\",\n \"[160/180] judged test_pedagogy_24 / dpo\\n\",\n \"[161/180] judged test_pedagogy_24 / dpo\\n\",\n \"[162/180] judged test_pedagogy_24 / dpo\\n\",\n \"[163/180] judged test_pedagogy_25 / dpo\\n\",\n \"[164/180] judged test_pedagogy_25 / dpo\\n\",\n \"[165/180] judged test_pedagogy_25 / dpo\\n\",\n \"[166/180] judged test_pedagogy_26 / dpo\\n\",\n \"[167/180] judged test_pedagogy_26 / dpo\\n\",\n \"[168/180] judged test_pedagogy_26 / dpo\\n\",\n \"[169/180] judged test_pedagogy_27 / dpo\\n\",\n \"[170/180] judged test_pedagogy_27 / dpo\\n\",\n \"[171/180] judged test_pedagogy_27 / dpo\\n\",\n \"[172/180] judged test_pedagogy_28 / dpo\\n\",\n \"[173/180] judged test_pedagogy_28 / dpo\\n\",\n \"[174/180] judged test_pedagogy_28 / dpo\\n\",\n \"[175/180] judged test_pedagogy_29 / dpo\\n\",\n \"[176/180] judged test_pedagogy_29 / dpo\\n\",\n \"[177/180] judged test_pedagogy_29 / dpo\\n\",\n \"[178/180] judged test_pedagogy_30 / dpo\\n\",\n \"[179/180] judged test_pedagogy_30 / dpo\\n\",\n \"[180/180] judged test_pedagogy_30 / dpo\\n\",\n \"Wrote 180 judge rows to results/reflection_test_120/judge_scores.jsonl\\n\",\n \"[\\n\",\n \" {\\n\",\n \" \\\"method\\\": \\\"dpo\\\",\\n\",\n \" \\\"samples\\\": 120,\\n\",\n \" \\\"labelled_samples\\\": 120,\\n\",\n \" \\\"accuracy\\\": 0.4,\\n\",\n \" \\\"accuracy_ci95_low\\\": 0.31666666666666665,\\n\",\n \" \\\"accuracy_ci95_high\\\": 0.48333333333333334,\\n\",\n \" \\\"ECE\\\": 0.23566160350272658,\\n\",\n \" \\\"Brier\\\": 0.32439021952157143,\\n\",\n \" \\\"mean_confidence\\\": 0.32398260451790756,\\n\",\n \" \\\"judge_mean\\\": 4.406666666666666,\\n\",\n \" \\\"judge_std\\\": 2.1453826387539046,\\n\",\n \" \\\"avg_answer_chars\\\": 228.83333333333334,\\n\",\n \" \\\"emoji_rate\\\": 0.008333333333333333,\\n\",\n \" \\\"avg_latency_ms\\\": 8202.53317289168\\n\",\n \" },\\n\",\n \" {\\n\",\n \" \\\"method\\\": \\\"dpo+reflect1\\\",\\n\",\n \" \\\"samples\\\": 120,\\n\",\n \" \\\"labelled_samples\\\": 120,\\n\",\n \" \\\"accuracy\\\": 0.2833333333333333,\\n\",\n \" \\\"accuracy_ci95_low\\\": 0.20833333333333334,\\n\",\n \" \\\"accuracy_ci95_high\\\": 0.36666666666666664,\\n\",\n \" \\\"ECE\\\": 0.28994651200267035,\\n\",\n \" \\\"Brier\\\": 0.2874677666138387,\\n\",\n \" \\\"mean_confidence\\\": 0.3407057497038126,\\n\",\n \" \\\"judge_mean\\\": 2.856666666666667,\\n\",\n \" \\\"judge_std\\\": 1.9834370670303751,\\n\",\n \" \\\"avg_answer_chars\\\": 258.85833333333335,\\n\",\n \" \\\"emoji_rate\\\": 0.008333333333333333,\\n\",\n \" \\\"avg_latency_ms\\\": 17899.895929958362\\n\",\n \" },\\n\",\n \" {\\n\",\n \" \\\"method\\\": \\\"dpo+reflect2\\\",\\n\",\n \" \\\"samples\\\": 120,\\n\",\n \" \\\"labelled_samples\\\": 120,\\n\",\n \" \\\"accuracy\\\": 0.20833333333333334,\\n\",\n \" \\\"accuracy_ci95_low\\\": 0.13333333333333333,\\n\",\n \" \\\"accuracy_ci95_high\\\": 0.2833333333333333,\\n\",\n \" \\\"ECE\\\": 0.31259612623662525,\\n\",\n \" \\\"Brier\\\": 0.275575007122457,\\n\",\n \" \\\"mean_confidence\\\": 0.3551854133877593,\\n\",\n \" \\\"judge_mean\\\": 2.76,\\n\",\n \" \\\"judge_std\\\": 1.959349598070142,\\n\",\n \" \\\"avg_answer_chars\\\": 259.26666666666665,\\n\",\n \" \\\"emoji_rate\\\": 0.0,\\n\",\n \" \\\"avg_latency_ms\\\": 27613.496549425097\\n\",\n \" }\\n\",\n \"]\\n\"\n ]\n }\n ],\n \"source\": [\n \"!python scripts/judge_open_answers.py \\\\\\n\",\n \" --predictions results/reflection_test_120/predictions.jsonl \\\\\\n\",\n \" --provider deepseek \\\\\\n\",\n \" --output results/reflection_test_120/judge_scores.jsonl\\n\",\n \"\\n\",\n \"!python scripts/analyze_alignment.py \\\\\\n\",\n \" --predictions results/reflection_test_120/predictions.jsonl \\\\\\n\",\n \" --judge-scores results/reflection_test_120/judge_scores.jsonl \\\\\\n\",\n \" --output-dir results/reflection_test_120/analysis\"\n ]\n }\n ]\n}"}}}} |