{"repo": "chenling03/lfqa-e", "files": {"codes/api_models_eval.py": {"f22577c35c858f2a8a0c134e4d5f55956f1e58ba": {"t": 1785746504.5006642, "n": 5071, "h": "18fee65b7041d6f4", "c": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\nCall GPT-4o on every answer‑pair record and save full model responses to Excel, including\nan extra column with the last line of each model response.\n\nUsage\n-----\npython evaluate_pairs_to_xlsx_v2.py <language>\n\"\"\"\n\nimport json\nimport sys\nfrom pathlib import Path\nfrom typing import Tuple\n\nimport openai\nfrom openpyxl import Workbook\n\nopenai.api_key = \"your_api_key\"\nopenai.base_url = \"your_base_url\"   \nMODEL_NAME = \"deepseek-reasoner\"\n\n\ndef build_prompt(\n    question: str,\n    reference: str,\n    ans1: str,\n    ans2: str,\n    language: str\n) -> str:\n    \"\"\"Construct the evaluation prompt fed to the GPT model based on the language.\"\"\"\n    if language == 'zh':\n        return f\"\"\"现有如下问题：\n\n问题：{question}\n该问题的**标准答案**如下：{reference}\n\n我们现在得到了两份学生对该问题的回答。\n回答一的内容如下：{ans1}\n回答二的内容如下：{ans2}\n\n现在，你应当以该问题的内容为上下文背景，必须以标准答案为判决依据，判决答案一和答案二中哪份更贴近标准答案的内容。\n现在，请你先展开简单的分析，随后给出结论：回答一或回答二更优秀\n有时，回答一与回答二和标准答案相比也有可能不相上下。\n\n你的回答必须在最后一行以如下形式给出：\n因此，【回答一】更优秀 或 因此，【回答二】更优秀\n若你认为这两个回答不相上下，你应当在最后一行以如下形式给出：\n因此，【两个回答相同】\n\"\"\"\n    else:\n        # Default to English prompt\n        return f\"\"\"We have the following question:\n\nQuestion: {question}\nThe reference (standard) answer to this question is as follows: {reference}\n\nWe now have two student responses to this question.\nResponse 1 is as follows: {ans1}\nResponse 2 is as follows: {ans2}\n\nNow, you should evaluate the two responses based on the content of the question, using the standard answer as the sole basis for judgment.\nDetermine which of the two—Response 1 or Response 2—is closer to the reference answer in terms of content.\n\nPlease begin with a brief analysis, and then provide your final judgment in one of the following forms:\n\nIf one response is better:\n\"Therefore, [Response 1] is better.\" or \"Therefore, [Response 2] is better.\"\n\nIf the two responses are roughly equal in quality:\n\"Therefore, [Both responses are equal].\"\"\"  # noqa: E501\n\n\ndef call_gpt(prompt: str) -> str:\n    \"\"\"Send the prompt to the GPT model and return its reply.\"\"\"\n    response = openai.chat.completions.create(\n        model=MODEL_NAME,\n        messages=[{\"role\": \"user\", \"content\": prompt}],\n        temperature=0,\n    )\n    return response.choices[0].message.content.strip()\n\n\nPAIR_KEYS = (\n    \"student_answer_a_vs_student_answer_b\",\n    \"student_answer_a_vs_model_answer_a\",\n    \"student_answer_a_vs_model_answer_b\",\n    \"student_answer_b_vs_model_answer_a\",\n    \"student_answer_b_vs_model_answer_b\",\n    \"model_answer_a_vs_model_answer_b\",\n)\n\n\ndef extract_pair_and_fields(item: dict) -> Tuple[str, str, str]:\n    \"\"\"Return the pair key and the corresponding field names for the left and right answers.\"\"\"\n    for key in PAIR_KEYS:\n        if key in item:\n            left_base, right_base = key.split(\"_vs_\")\n            return key, f\"{left_base}\", f\"{right_base}\"\n    raise ValueError(\"cannot find pair key!\")\n\n\ndef main(language: str) -> None:\n    # Determine the input JSON and output Excel paths based on the language\n    if language == 'zh':\n        input_json = Path(\"../data/LFQA-E-zh.json\").resolve()\n        output_xlsx = Path(f\"../results/{MODEL_NAME}_zh.xlsx\").resolve()\n    else:\n        input_json = Path(\"../data/LFQA-E-en.json\").resolve()\n        output_xlsx = Path(f\"../results/{MODEL_NAME}_en.xlsx\").resolve()\n\n    items = json.loads(input_json.read_text(encoding=\"utf-8\"))\n\n    wb = Workbook()\n    ws = wb.active\n    ws.title = \"full_model_responses\"\n    # ➕ Add an extra column header for the last line of the model response\n    ws.append([\n        \"pair_type\",\n        \"model_response\",\n        \"human_label_raw\",\n        \"model_response_verdict\",\n    ])\n\n    for idx, item in enumerate(items, 1):\n        pair_key, left_field, right_field = extract_pair_and_fields(item)\n\n        question = item[\"question\"]\n        reference = item.get(\"Concise_Reference\") or item.get(\"Concise_Reference\", \"\")\n        ans1 = item.get(left_field, \"\")\n        ans2 = item.get(right_field, \"\")\n        human_raw = item[pair_key]\n\n        prompt = build_prompt(question, reference, ans1, ans2, language)\n\n        try:\n            gpt_reply = call_gpt(prompt)\n        except Exception as e:\n            print(f\"[{idx}] invoke failed: {e}\")\n            gpt_reply = f\"ERROR: {e}\"\n\n        # Extract the last non‑empty line from the model response\n        last_line = gpt_reply.splitlines()[-1].strip() if gpt_reply else \"\"\n\n        # Write a row including the extra column\n        ws.append([pair_key, gpt_reply, human_raw, last_line])\n        print(f\"[{idx:>4}/{len(items)}] {pair_key} → finished\")\n\n    wb.save(output_xlsx)\n    print(f\"\\nAll finished, results saved to {output_xlsx}\")\n\n\nif __name__ == \"__main__\":\n    if len(sys.argv) != 2:\n        print(\"usage: python evaluate_pairs_to_xlsx_v2.py <language>\")\n        print(\"language should be 'zh' for Chinese or 'en' for English.\")\n        sys.exit(1)\n\n    language = sys.argv[1].lower()\n\n    if language not in ['zh', 'en']:\n        print(\"Error: language should be 'zh' for Chinese or 'en' for English.\")\n        sys.exit(1)\n\n    main(language)\n"}, "4773319bfd05b235fbe10f92f94534ff3f827aea": {"t": 1785765049.521846, "n": 5071, "h": "18fee65b7041d6f4", "c": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\nCall GPT-4o on every answer‑pair record and save full model responses to Excel, including\nan extra column with the last line of each model response.\n\nUsage\n-----\npython evaluate_pairs_to_xlsx_v2.py <language>\n\"\"\"\n\nimport json\nimport sys\nfrom pathlib import Path\nfrom typing import Tuple\n\nimport openai\nfrom openpyxl import Workbook\n\nopenai.api_key = \"your_api_key\"\nopenai.base_url = \"your_base_url\"   \nMODEL_NAME = \"deepseek-reasoner\"\n\n\ndef build_prompt(\n    question: str,\n    reference: str,\n    ans1: str,\n    ans2: str,\n    language: str\n) -> str:\n    \"\"\"Construct the evaluation prompt fed to the GPT model based on the language.\"\"\"\n    if language == 'zh':\n        return f\"\"\"现有如下问题：\n\n问题：{question}\n该问题的**标准答案**如下：{reference}\n\n我们现在得到了两份学生对该问题的回答。\n回答一的内容如下：{ans1}\n回答二的内容如下：{ans2}\n\n现在，你应当以该问题的内容为上下文背景，必须以标准答案为判决依据，判决答案一和答案二中哪份更贴近标准答案的内容。\n现在，请你先展开简单的分析，随后给出结论：回答一或回答二更优秀\n有时，回答一与回答二和标准答案相比也有可能不相上下。\n\n你的回答必须在最后一行以如下形式给出：\n因此，【回答一】更优秀 或 因此，【回答二】更优秀\n若你认为这两个回答不相上下，你应当在最后一行以如下形式给出：\n因此，【两个回答相同】\n\"\"\"\n    else:\n        # Default to English prompt\n        return f\"\"\"We have the following question:\n\nQuestion: {question}\nThe reference (standard) answer to this question is as follows: {reference}\n\nWe now have two student responses to this question.\nResponse 1 is as follows: {ans1}\nResponse 2 is as follows: {ans2}\n\nNow, you should evaluate the two responses based on the content of the question, using the standard answer as the sole basis for judgment.\nDetermine which of the two—Response 1 or Response 2—is closer to the reference answer in terms of content.\n\nPlease begin with a brief analysis, and then provide your final judgment in one of the following forms:\n\nIf one response is better:\n\"Therefore, [Response 1] is better.\" or \"Therefore, [Response 2] is better.\"\n\nIf the two responses are roughly equal in quality:\n\"Therefore, [Both responses are equal].\"\"\"  # noqa: E501\n\n\ndef call_gpt(prompt: str) -> str:\n    \"\"\"Send the prompt to the GPT model and return its reply.\"\"\"\n    response = openai.chat.completions.create(\n        model=MODEL_NAME,\n        messages=[{\"role\": \"user\", \"content\": prompt}],\n        temperature=0,\n    )\n    return response.choices[0].message.content.strip()\n\n\nPAIR_KEYS = (\n    \"student_answer_a_vs_student_answer_b\",\n    \"student_answer_a_vs_model_answer_a\",\n    \"student_answer_a_vs_model_answer_b\",\n    \"student_answer_b_vs_model_answer_a\",\n    \"student_answer_b_vs_model_answer_b\",\n    \"model_answer_a_vs_model_answer_b\",\n)\n\n\ndef extract_pair_and_fields(item: dict) -> Tuple[str, str, str]:\n    \"\"\"Return the pair key and the corresponding field names for the left and right answers.\"\"\"\n    for key in PAIR_KEYS:\n        if key in item:\n            left_base, right_base = key.split(\"_vs_\")\n            return key, f\"{left_base}\", f\"{right_base}\"\n    raise ValueError(\"cannot find pair key!\")\n\n\ndef main(language: str) -> None:\n    # Determine the input JSON and output Excel paths based on the language\n    if language == 'zh':\n        input_json = Path(\"../data/LFQA-E-zh.json\").resolve()\n        output_xlsx = Path(f\"../results/{MODEL_NAME}_zh.xlsx\").resolve()\n    else:\n        input_json = Path(\"../data/LFQA-E-en.json\").resolve()\n        output_xlsx = Path(f\"../results/{MODEL_NAME}_en.xlsx\").resolve()\n\n    items = json.loads(input_json.read_text(encoding=\"utf-8\"))\n\n    wb = Workbook()\n    ws = wb.active\n    ws.title = \"full_model_responses\"\n    # ➕ Add an extra column header for the last line of the model response\n    ws.append([\n        \"pair_type\",\n        \"model_response\",\n        \"human_label_raw\",\n        \"model_response_verdict\",\n    ])\n\n    for idx, item in enumerate(items, 1):\n        pair_key, left_field, right_field = extract_pair_and_fields(item)\n\n        question = item[\"question\"]\n        reference = item.get(\"Concise_Reference\") or item.get(\"Concise_Reference\", \"\")\n        ans1 = item.get(left_field, \"\")\n        ans2 = item.get(right_field, \"\")\n        human_raw = item[pair_key]\n\n        prompt = build_prompt(question, reference, ans1, ans2, language)\n\n        try:\n            gpt_reply = call_gpt(prompt)\n        except Exception as e:\n            print(f\"[{idx}] invoke failed: {e}\")\n            gpt_reply = f\"ERROR: {e}\"\n\n        # Extract the last non‑empty line from the model response\n        last_line = gpt_reply.splitlines()[-1].strip() if gpt_reply else \"\"\n\n        # Write a row including the extra column\n        ws.append([pair_key, gpt_reply, human_raw, last_line])\n        print(f\"[{idx:>4}/{len(items)}] {pair_key} → finished\")\n\n    wb.save(output_xlsx)\n    print(f\"\\nAll finished, results saved to {output_xlsx}\")\n\n\nif __name__ == \"__main__\":\n    if len(sys.argv) != 2:\n        print(\"usage: python evaluate_pairs_to_xlsx_v2.py <language>\")\n        print(\"language should be 'zh' for Chinese or 'en' for English.\")\n        sys.exit(1)\n\n    language = sys.argv[1].lower()\n\n    if language not in ['zh', 'en']:\n        print(\"Error: language should be 'zh' for Chinese or 'en' for English.\")\n        sys.exit(1)\n\n    main(language)\n"}, "1c065ebdb5948d970024f26876f82a134d7dcdad": {"t": 1785765051.3443863, "n": 5232, "h": "46f14e77544ab7bd", "c": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\nCall GPT-4o on every answer‑pair record and save full model responses to Excel, including\nan extra column with the last line of each model response.\n\nUsage\n-----\npython evaluate_pairs_to_xlsx_v2.py <language>\n\"\"\"\n\nimport json\nimport sys\nfrom pathlib import Path\nfrom typing import Tuple\n\nimport openai\nfrom openpyxl import Workbook\n\n# openai.api_key = \"your_api_key\"  # TODO: replace with real key or env var\n# openai.base_url = \"your_base_url\"  # openai==1.x\nopenai.api_key = \"sk-fa663c1cb5594070b258310e4ba99d40\"\nopenai.base_url = \"https://api.deepseek.com\"   \nMODEL_NAME = \"deepseek-reasoner\"\n\n\ndef build_prompt(\n    question: str,\n    reference: str,\n    ans1: str,\n    ans2: str,\n    language: str\n) -> str:\n    \"\"\"Construct the evaluation prompt fed to the GPT model based on the language.\"\"\"\n    if language == 'zh':\n        return f\"\"\"现有如下问题：\n\n问题：{question}\n该问题的**标准答案**如下：{reference}\n\n我们现在得到了两份学生对该问题的回答。\n回答一的内容如下：{ans1}\n回答二的内容如下：{ans2}\n\n现在，你应当以该问题的内容为上下文背景，必须以标准答案为判决依据，判决答案一和答案二中哪份更贴近标准答案的内容。\n现在，请你先展开简单的分析，随后给出结论：回答一或回答二更优秀\n有时，回答一与回答二和标准答案相比也有可能不相上下。\n\n你的回答必须在最后一行以如下形式给出：\n因此，【回答一】更优秀 或 因此，【回答二】更优秀\n若你认为这两个回答不相上下，你应当在最后一行以如下形式给出：\n因此，【两个回答相同】\n\"\"\"\n    else:\n        # Default to English prompt\n        return f\"\"\"We have the following question:\n\nQuestion: {question}\nThe reference (standard) answer to this question is as follows: {reference}\n\nWe now have two student responses to this question.\nResponse 1 is as follows: {ans1}\nResponse 2 is as follows: {ans2}\n\nNow, you should evaluate the two responses based on the content of the question, using the standard answer as the sole basis for judgment.\nDetermine which of the two—Response 1 or Response 2—is closer to the reference answer in terms of content.\n\nPlease begin with a brief analysis, and then provide your final judgment in one of the following forms:\n\nIf one response is better:\n\"Therefore, [Response 1] is better.\" or \"Therefore, [Response 2] is better.\"\n\nIf the two responses are roughly equal in quality:\n\"Therefore, [Both responses are equal].\"\"\"  # noqa: E501\n\n\ndef call_gpt(prompt: str) -> str:\n    \"\"\"Send the prompt to the GPT model and return its reply.\"\"\"\n    response = openai.chat.completions.create(\n        model=MODEL_NAME,\n        messages=[{\"role\": \"user\", \"content\": prompt}],\n        temperature=0,\n    )\n    return response.choices[0].message.content.strip()\n\n\nPAIR_KEYS = (\n    \"student_answer_a_vs_student_answer_b\",\n    \"student_answer_a_vs_model_answer_a\",\n    \"student_answer_a_vs_model_answer_b\",\n    \"student_answer_b_vs_model_answer_a\",\n    \"student_answer_b_vs_model_answer_b\",\n    \"model_answer_a_vs_model_answer_b\",\n)\n\n\ndef extract_pair_and_fields(item: dict) -> Tuple[str, str, str]:\n    \"\"\"Return the pair key and the corresponding field names for the left and right answers.\"\"\"\n    for key in PAIR_KEYS:\n        if key in item:\n            left_base, right_base = key.split(\"_vs_\")\n            return key, f\"{left_base}\", f\"{right_base}\"\n    raise ValueError(\"cannot find pair key!\")\n\n\ndef main(language: str) -> None:\n    # Determine the input JSON and output Excel paths based on the language\n    if language == 'zh':\n        input_json = Path(\"../data/LFQA-E-zh.json\").resolve()\n        output_xlsx = Path(f\"../results/{MODEL_NAME}_zh.xlsx\").resolve()\n    else:\n        input_json = Path(\"../data/LFQA-E-en.json\").resolve()\n        output_xlsx = Path(f\"../results/{MODEL_NAME}_en.xlsx\").resolve()\n\n    items = json.loads(input_json.read_text(encoding=\"utf-8\"))\n\n    wb = Workbook()\n    ws = wb.active\n    ws.title = \"full_model_responses\"\n    # ➕ Add an extra column header for the last line of the model response\n    ws.append([\n        \"pair_type\",\n        \"model_response\",\n        \"human_label_raw\",\n        \"model_response_verdict\",\n    ])\n\n    for idx, item in enumerate(items, 1):\n        pair_key, left_field, right_field = extract_pair_and_fields(item)\n\n        question = item[\"question\"]\n        reference = item.get(\"Concise_Reference\") or item.get(\"Concise_Reference\", \"\")\n        ans1 = item.get(left_field, \"\")\n        ans2 = item.get(right_field, \"\")\n        human_raw = item[pair_key]\n\n        prompt = build_prompt(question, reference, ans1, ans2, language)\n\n        try:\n            gpt_reply = call_gpt(prompt)\n        except Exception as e:\n            print(f\"[{idx}] invoke failed: {e}\")\n            gpt_reply = f\"ERROR: {e}\"\n\n        # Extract the last non‑empty line from the model response\n        last_line = gpt_reply.splitlines()[-1].strip() if gpt_reply else \"\"\n\n        # Write a row including the extra column\n        ws.append([pair_key, gpt_reply, human_raw, last_line])\n        print(f\"[{idx:>4}/{len(items)}] {pair_key} → finished\")\n\n    wb.save(output_xlsx)\n    print(f\"\\nAll finished, results saved to {output_xlsx}\")\n\n\nif __name__ == \"__main__\":\n    if len(sys.argv) != 2:\n        print(\"usage: python evaluate_pairs_to_xlsx_v2.py <language>\")\n        print(\"language should be 'zh' for Chinese or 'en' for English.\")\n        sys.exit(1)\n\n    language = sys.argv[1].lower()\n\n    if language not in ['zh', 'en']:\n        print(\"Error: language should be 'zh' for Chinese or 'en' for English.\")\n        sys.exit(1)\n\n    main(language)\n"}}}}