1 line
18 KiB
JSON
1 line
18 KiB
JSON
{"repo": "chenling03/lfqa-e", "files": {"codes/api_models_eval.py": {"f22577c35c858f2a8a0c134e4d5f55956f1e58ba": {"t": 1785746504.5006642, "n": 5071, "h": "18fee65b7041d6f4", "c": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\nCall GPT-4o on every answer‑pair record and save full model responses to Excel, including\nan extra column with the last line of each model response.\n\nUsage\n-----\npython evaluate_pairs_to_xlsx_v2.py <language>\n\"\"\"\n\nimport json\nimport sys\nfrom pathlib import Path\nfrom typing import Tuple\n\nimport openai\nfrom openpyxl import Workbook\n\nopenai.api_key = \"your_api_key\"\nopenai.base_url = \"your_base_url\" \nMODEL_NAME = \"deepseek-reasoner\"\n\n\ndef build_prompt(\n question: str,\n reference: str,\n ans1: str,\n ans2: str,\n language: str\n) -> str:\n \"\"\"Construct the evaluation prompt fed to the GPT model based on the language.\"\"\"\n if language == 'zh':\n return f\"\"\"现有如下问题：\n\n问题：{question}\n该问题的**标准答案**如下：{reference}\n\n我们现在得到了两份学生对该问题的回答。\n回答一的内容如下：{ans1}\n回答二的内容如下：{ans2}\n\n现在，你应当以该问题的内容为上下文背景，必须以标准答案为判决依据，判决答案一和答案二中哪份更贴近标准答案的内容。\n现在，请你先展开简单的分析，随后给出结论：回答一或回答二更优秀\n有时，回答一与回答二和标准答案相比也有可能不相上下。\n\n你的回答必须在最后一行以如下形式给出：\n因此，【回答一】更优秀 或 因此，【回答二】更优秀\n若你认为这两个回答不相上下，你应当在最后一行以如下形式给出：\n因此，【两个回答相同】\n\"\"\"\n else:\n # Default to English prompt\n return f\"\"\"We have the following question:\n\nQuestion: {question}\nThe reference (standard) answer to this question is as follows: {reference}\n\nWe now have two student responses to this question.\nResponse 1 is as follows: {ans1}\nResponse 2 is as follows: {ans2}\n\nNow, you should evaluate the two responses based on the content of the question, using the standard answer as the sole basis for judgment.\nDetermine which of the two—Response 1 or Response 2—is closer to the reference answer in terms of content.\n\nPlease begin with a brief analysis, and then provide your final judgment in one of the following forms:\n\nIf one response is better:\n\"Therefore, [Response 1] is better.\" or \"Therefore, [Response 2] is better.\"\n\nIf the two responses are roughly equal in quality:\n\"Therefore, [Both responses are equal].\"\"\" # noqa: E501\n\n\ndef call_gpt(prompt: str) -> str:\n \"\"\"Send the prompt to the GPT model and return its reply.\"\"\"\n response = openai.chat.completions.create(\n model=MODEL_NAME,\n messages=[{\"role\": \"user\", \"content\": prompt}],\n temperature=0,\n )\n return response.choices[0].message.content.strip()\n\n\nPAIR_KEYS = (\n \"student_answer_a_vs_student_answer_b\",\n \"student_answer_a_vs_model_answer_a\",\n \"student_answer_a_vs_model_answer_b\",\n \"student_answer_b_vs_model_answer_a\",\n \"student_answer_b_vs_model_answer_b\",\n \"model_answer_a_vs_model_answer_b\",\n)\n\n\ndef extract_pair_and_fields(item: dict) -> Tuple[str, str, str]:\n \"\"\"Return the pair key and the corresponding field names for the left and right answers.\"\"\"\n for key in PAIR_KEYS:\n if key in item:\n left_base, right_base = key.split(\"_vs_\")\n return key, f\"{left_base}\", f\"{right_base}\"\n raise ValueError(\"cannot find pair key!\")\n\n\ndef main(language: str) -> None:\n # Determine the input JSON and output Excel paths based on the language\n if language == 'zh':\n input_json = Path(\"../data/LFQA-E-zh.json\").resolve()\n output_xlsx = Path(f\"../results/{MODEL_NAME}_zh.xlsx\").resolve()\n else:\n input_json = Path(\"../data/LFQA-E-en.json\").resolve()\n output_xlsx = Path(f\"../results/{MODEL_NAME}_en.xlsx\").resolve()\n\n items = json.loads(input_json.read_text(encoding=\"utf-8\"))\n\n wb = Workbook()\n ws = wb.active\n ws.title = \"full_model_responses\"\n # ➕ Add an extra column header for the last line of the model response\n ws.append([\n \"pair_type\",\n \"model_response\",\n \"human_label_raw\",\n \"model_response_verdict\",\n ])\n\n for idx, item in enumerate(items, 1):\n pair_key, left_field, right_field = extract_pair_and_fields(item)\n\n question = item[\"question\"]\n reference = item.get(\"Concise_Reference\") or item.get(\"Concise_Reference\", \"\")\n ans1 = item.get(left_field, \"\")\n ans2 = item.get(right_field, \"\")\n human_raw = item[pair_key]\n\n prompt = build_prompt(question, reference, ans1, ans2, language)\n\n try:\n gpt_reply = call_gpt(prompt)\n except Exception as e:\n print(f\"[{idx}] invoke failed: {e}\")\n gpt_reply = f\"ERROR: {e}\"\n\n # Extract the last non‑empty line from the model response\n last_line = gpt_reply.splitlines()[-1].strip() if gpt_reply else \"\"\n\n # Write a row including the extra column\n ws.append([pair_key, gpt_reply, human_raw, last_line])\n print(f\"[{idx:>4}/{len(items)}] {pair_key} → finished\")\n\n wb.save(output_xlsx)\n print(f\"\\nAll finished, results saved to {output_xlsx}\")\n\n\nif __name__ == \"__main__\":\n if len(sys.argv) != 2:\n print(\"usage: python evaluate_pairs_to_xlsx_v2.py <language>\")\n print(\"language should be 'zh' for Chinese or 'en' for English.\")\n sys.exit(1)\n\n language = sys.argv[1].lower()\n\n if language not in ['zh', 'en']:\n print(\"Error: language should be 'zh' for Chinese or 'en' for English.\")\n sys.exit(1)\n\n main(language)\n"}, "4773319bfd05b235fbe10f92f94534ff3f827aea": {"t": 1785765049.521846, "n": 5071, "h": "18fee65b7041d6f4", "c": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\nCall GPT-4o on every answer‑pair record and save full model responses to Excel, including\nan extra column with the last line of each model response.\n\nUsage\n-----\npython evaluate_pairs_to_xlsx_v2.py <language>\n\"\"\"\n\nimport json\nimport sys\nfrom pathlib import Path\nfrom typing import Tuple\n\nimport openai\nfrom openpyxl import Workbook\n\nopenai.api_key = \"your_api_key\"\nopenai.base_url = \"your_base_url\" \nMODEL_NAME = \"deepseek-reasoner\"\n\n\ndef build_prompt(\n question: str,\n reference: str,\n ans1: str,\n ans2: str,\n language: str\n) -> str:\n \"\"\"Construct the evaluation prompt fed to the GPT model based on the language.\"\"\"\n if language == 'zh':\n return f\"\"\"现有如下问题：\n\n问题：{question}\n该问题的**标准答案**如下：{reference}\n\n我们现在得到了两份学生对该问题的回答。\n回答一的内容如下：{ans1}\n回答二的内容如下：{ans2}\n\n现在，你应当以该问题的内容为上下文背景，必须以标准答案为判决依据，判决答案一和答案二中哪份更贴近标准答案的内容。\n现在，请你先展开简单的分析，随后给出结论：回答一或回答二更优秀\n有时，回答一与回答二和标准答案相比也有可能不相上下。\n\n你的回答必须在最后一行以如下形式给出：\n因此，【回答一】更优秀 或 因此，【回答二】更优秀\n若你认为这两个回答不相上下，你应当在最后一行以如下形式给出：\n因此，【两个回答相同】\n\"\"\"\n else:\n # Default to English prompt\n return f\"\"\"We have the following question:\n\nQuestion: {question}\nThe reference (standard) answer to this question is as follows: {reference}\n\nWe now have two student responses to this question.\nResponse 1 is as follows: {ans1}\nResponse 2 is as follows: {ans2}\n\nNow, you should evaluate the two responses based on the content of the question, using the standard answer as the sole basis for judgment.\nDetermine which of the two—Response 1 or Response 2—is closer to the reference answer in terms of content.\n\nPlease begin with a brief analysis, and then provide your final judgment in one of the following forms:\n\nIf one response is better:\n\"Therefore, [Response 1] is better.\" or \"Therefore, [Response 2] is better.\"\n\nIf the two responses are roughly equal in quality:\n\"Therefore, [Both responses are equal].\"\"\" # noqa: E501\n\n\ndef call_gpt(prompt: str) -> str:\n \"\"\"Send the prompt to the GPT model and return its reply.\"\"\"\n response = openai.chat.completions.create(\n model=MODEL_NAME,\n messages=[{\"role\": \"user\", \"content\": prompt}],\n temperature=0,\n )\n return response.choices[0].message.content.strip()\n\n\nPAIR_KEYS = (\n \"student_answer_a_vs_student_answer_b\",\n \"student_answer_a_vs_model_answer_a\",\n \"student_answer_a_vs_model_answer_b\",\n \"student_answer_b_vs_model_answer_a\",\n \"student_answer_b_vs_model_answer_b\",\n \"model_answer_a_vs_model_answer_b\",\n)\n\n\ndef extract_pair_and_fields(item: dict) -> Tuple[str, str, str]:\n \"\"\"Return the pair key and the corresponding field names for the left and right answers.\"\"\"\n for key in PAIR_KEYS:\n if key in item:\n left_base, right_base = key.split(\"_vs_\")\n return key, f\"{left_base}\", f\"{right_base}\"\n raise ValueError(\"cannot find pair key!\")\n\n\ndef main(language: str) -> None:\n # Determine the input JSON and output Excel paths based on the language\n if language == 'zh':\n input_json = Path(\"../data/LFQA-E-zh.json\").resolve()\n output_xlsx = Path(f\"../results/{MODEL_NAME}_zh.xlsx\").resolve()\n else:\n input_json = Path(\"../data/LFQA-E-en.json\").resolve()\n output_xlsx = Path(f\"../results/{MODEL_NAME}_en.xlsx\").resolve()\n\n items = json.loads(input_json.read_text(encoding=\"utf-8\"))\n\n wb = Workbook()\n ws = wb.active\n ws.title = \"full_model_responses\"\n # ➕ Add an extra column header for the last line of the model response\n ws.append([\n \"pair_type\",\n \"model_response\",\n \"human_label_raw\",\n \"model_response_verdict\",\n ])\n\n for idx, item in enumerate(items, 1):\n pair_key, left_field, right_field = extract_pair_and_fields(item)\n\n question = item[\"question\"]\n reference = item.get(\"Concise_Reference\") or item.get(\"Concise_Reference\", \"\")\n ans1 = item.get(left_field, \"\")\n ans2 = item.get(right_field, \"\")\n human_raw = item[pair_key]\n\n prompt = build_prompt(question, reference, ans1, ans2, language)\n\n try:\n gpt_reply = call_gpt(prompt)\n except Exception as e:\n print(f\"[{idx}] invoke failed: {e}\")\n gpt_reply = f\"ERROR: {e}\"\n\n # Extract the last non‑empty line from the model response\n last_line = gpt_reply.splitlines()[-1].strip() if gpt_reply else \"\"\n\n # Write a row including the extra column\n ws.append([pair_key, gpt_reply, human_raw, last_line])\n print(f\"[{idx:>4}/{len(items)}] {pair_key} → finished\")\n\n wb.save(output_xlsx)\n print(f\"\\nAll finished, results saved to {output_xlsx}\")\n\n\nif __name__ == \"__main__\":\n if len(sys.argv) != 2:\n print(\"usage: python evaluate_pairs_to_xlsx_v2.py <language>\")\n print(\"language should be 'zh' for Chinese or 'en' for English.\")\n sys.exit(1)\n\n language = sys.argv[1].lower()\n\n if language not in ['zh', 'en']:\n print(\"Error: language should be 'zh' for Chinese or 'en' for English.\")\n sys.exit(1)\n\n main(language)\n"}, "1c065ebdb5948d970024f26876f82a134d7dcdad": {"t": 1785765051.3443863, "n": 5232, "h": "46f14e77544ab7bd", "c": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\nCall GPT-4o on every answer‑pair record and save full model responses to Excel, including\nan extra column with the last line of each model response.\n\nUsage\n-----\npython evaluate_pairs_to_xlsx_v2.py <language>\n\"\"\"\n\nimport json\nimport sys\nfrom pathlib import Path\nfrom typing import Tuple\n\nimport openai\nfrom openpyxl import Workbook\n\n# openai.api_key = \"your_api_key\" # TODO: replace with real key or env var\n# openai.base_url = \"your_base_url\" # openai==1.x\nopenai.api_key = \"sk-fa663c1cb5594070b258310e4ba99d40\"\nopenai.base_url = \"https://api.deepseek.com\" \nMODEL_NAME = \"deepseek-reasoner\"\n\n\ndef build_prompt(\n question: str,\n reference: str,\n ans1: str,\n ans2: str,\n language: str\n) -> str:\n \"\"\"Construct the evaluation prompt fed to the GPT model based on the language.\"\"\"\n if language == 'zh':\n return f\"\"\"现有如下问题：\n\n问题：{question}\n该问题的**标准答案**如下：{reference}\n\n我们现在得到了两份学生对该问题的回答。\n回答一的内容如下：{ans1}\n回答二的内容如下：{ans2}\n\n现在，你应当以该问题的内容为上下文背景，必须以标准答案为判决依据，判决答案一和答案二中哪份更贴近标准答案的内容。\n现在，请你先展开简单的分析，随后给出结论：回答一或回答二更优秀\n有时，回答一与回答二和标准答案相比也有可能不相上下。\n\n你的回答必须在最后一行以如下形式给出：\n因此，【回答一】更优秀 或 因此，【回答二】更优秀\n若你认为这两个回答不相上下，你应当在最后一行以如下形式给出：\n因此，【两个回答相同】\n\"\"\"\n else:\n # Default to English prompt\n return f\"\"\"We have the following question:\n\nQuestion: {question}\nThe reference (standard) answer to this question is as follows: {reference}\n\nWe now have two student responses to this question.\nResponse 1 is as follows: {ans1}\nResponse 2 is as follows: {ans2}\n\nNow, you should evaluate the two responses based on the content of the question, using the standard answer as the sole basis for judgment.\nDetermine which of the two—Response 1 or Response 2—is closer to the reference answer in terms of content.\n\nPlease begin with a brief analysis, and then provide your final judgment in one of the following forms:\n\nIf one response is better:\n\"Therefore, [Response 1] is better.\" or \"Therefore, [Response 2] is better.\"\n\nIf the two responses are roughly equal in quality:\n\"Therefore, [Both responses are equal].\"\"\" # noqa: E501\n\n\ndef call_gpt(prompt: str) -> str:\n \"\"\"Send the prompt to the GPT model and return its reply.\"\"\"\n response = openai.chat.completions.create(\n model=MODEL_NAME,\n messages=[{\"role\": \"user\", \"content\": prompt}],\n temperature=0,\n )\n return response.choices[0].message.content.strip()\n\n\nPAIR_KEYS = (\n \"student_answer_a_vs_student_answer_b\",\n \"student_answer_a_vs_model_answer_a\",\n \"student_answer_a_vs_model_answer_b\",\n \"student_answer_b_vs_model_answer_a\",\n \"student_answer_b_vs_model_answer_b\",\n \"model_answer_a_vs_model_answer_b\",\n)\n\n\ndef extract_pair_and_fields(item: dict) -> Tuple[str, str, str]:\n \"\"\"Return the pair key and the corresponding field names for the left and right answers.\"\"\"\n for key in PAIR_KEYS:\n if key in item:\n left_base, right_base = key.split(\"_vs_\")\n return key, f\"{left_base}\", f\"{right_base}\"\n raise ValueError(\"cannot find pair key!\")\n\n\ndef main(language: str) -> None:\n # Determine the input JSON and output Excel paths based on the language\n if language == 'zh':\n input_json = Path(\"../data/LFQA-E-zh.json\").resolve()\n output_xlsx = Path(f\"../results/{MODEL_NAME}_zh.xlsx\").resolve()\n else:\n input_json = Path(\"../data/LFQA-E-en.json\").resolve()\n output_xlsx = Path(f\"../results/{MODEL_NAME}_en.xlsx\").resolve()\n\n items = json.loads(input_json.read_text(encoding=\"utf-8\"))\n\n wb = Workbook()\n ws = wb.active\n ws.title = \"full_model_responses\"\n # ➕ Add an extra column header for the last line of the model response\n ws.append([\n \"pair_type\",\n \"model_response\",\n \"human_label_raw\",\n \"model_response_verdict\",\n ])\n\n for idx, item in enumerate(items, 1):\n pair_key, left_field, right_field = extract_pair_and_fields(item)\n\n question = item[\"question\"]\n reference = item.get(\"Concise_Reference\") or item.get(\"Concise_Reference\", \"\")\n ans1 = item.get(left_field, \"\")\n ans2 = item.get(right_field, \"\")\n human_raw = item[pair_key]\n\n prompt = build_prompt(question, reference, ans1, ans2, language)\n\n try:\n gpt_reply = call_gpt(prompt)\n except Exception as e:\n print(f\"[{idx}] invoke failed: {e}\")\n gpt_reply = f\"ERROR: {e}\"\n\n # Extract the last non‑empty line from the model response\n last_line = gpt_reply.splitlines()[-1].strip() if gpt_reply else \"\"\n\n # Write a row including the extra column\n ws.append([pair_key, gpt_reply, human_raw, last_line])\n print(f\"[{idx:>4}/{len(items)}] {pair_key} → finished\")\n\n wb.save(output_xlsx)\n print(f\"\\nAll finished, results saved to {output_xlsx}\")\n\n\nif __name__ == \"__main__\":\n if len(sys.argv) != 2:\n print(\"usage: python evaluate_pairs_to_xlsx_v2.py <language>\")\n print(\"language should be 'zh' for Chinese or 'en' for English.\")\n sys.exit(1)\n\n language = sys.argv[1].lower()\n\n if language not in ['zh', 'en']:\n print(\"Error: language should be 'zh' for Chinese or 'en' for English.\")\n sys.exit(1)\n\n main(language)\n"}}}} |