1 line
5.8 KiB
JSON
1 line
5.8 KiB
JSON
{"repo": "antonio-jtzhou/nlp-hw3", "files": {"task2/task2_naive.py": {"bf7836413c95d1efc51ae9c768de52c5cdde3d1e": {"t": 1785747519.5715036, "n": 2399, "h": "98cf4c624afe19be", "c": "import openai\nfrom datasets import load_dataset\nfrom tqdm import tqdm\nimport re\nimport json\n\n# 配置 OpenAI API\nopenai.api_key = \"sk-9e662dcc789a4927bcd4289c1216316a\"\nopenai.api_base = \"https://api.deepseek.com\"\nMODEL = \"deepseek-chat\"\n\n# 答案提取函数\ndef extract_answer(answer_text):\n \"\"\"\n 从 GSM8K 的答案字段中提取数值答案。\n 答案位于连续的 '####' 之后。\n \"\"\"\n match = re.search(r\"####\\s*([\\d\\.\\-eE]+)\", answer_text)\n if match:\n return match.group(1)\n return None\n\n# 模型调用函数\ndef query_model(question, model=MODEL):\n \"\"\"\n 使用给定问题调用模型，并要求其输出符合格式。\n \"\"\"\n prompt = [\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n {\"role\": \"user\", \"content\": f\"For each question, show the answer started with \\\"####\\\" Make sure that the answer is only a number without any other words!\\n Question: \\n {question}\\n\"\n }\n ]\n \n try:\n response = openai.ChatCompletion.create(\n model=model,\n messages=prompt,\n temperature=0.5 # 控制回答的确定性\n )\n return response['choices'][0]['message']['content']\n except Exception as e:\n print(f\"模型调用错误: {e}\")\n return None\n\n# 正确性判定函数\ndef is_correct(predicted, ground_truth):\n \"\"\"\n 判断模型预测是否正确。\n \"\"\"\n try:\n predicted_value = float(extract_answer(predicted)) # 提取数值\n ground_truth_value = float(extract_answer(ground_truth)) # 提取标准答案\n return abs(predicted_value - ground_truth_value) < 1e-3 # 考虑浮点误差\n except:\n return False\n\n# 加载 GSM8K 数据集\nprint(\"加载 GSM8K 数据集...\")\ndataset = load_dataset(\"gsm8k\",\"main\", split=\"test\")\n\n# 评估过程\nprint(\"开始评估...\")\ncorrect = 0\ntotal = len(dataset)\nnums = 0\npredictions=[]\nwith open(\"check_naive.json\",\"w\") as f:\n for item in tqdm(dataset, desc=\"Evaluating GSM8K\"):\n question = item['question']\n ground_truth = item['answer']\n prediction = query_model(question)\n\n \n if prediction and is_correct(prediction, ground_truth):\n correct += 1\n else:\n result = {\n \"index\": nums,\n \"question\": question,\n \"ground_truth\": ground_truth,\n \"prediction\": prediction,\n }\n predictions.append(result)\n f.write(json.dumps(result, ensure_ascii=False) + \"\\n\")\n\n# 输出正确率\naccuracy = correct / total * 100\nprint(f\"GSM8K 正确率: {accuracy:.2f}% ({correct}/{total})\")\n"}, "894bbcb4de25fea49a833b34b3d0b7fdecb1bfea": {"t": 1785773712.6008759, "n": 2399, "h": "98cf4c624afe19be", "c": "import openai\nfrom datasets import load_dataset\nfrom tqdm import tqdm\nimport re\nimport json\n\n# 配置 OpenAI API\nopenai.api_key = \"sk-9e662dcc789a4927bcd4289c1216316a\"\nopenai.api_base = \"https://api.deepseek.com\"\nMODEL = \"deepseek-chat\"\n\n# 答案提取函数\ndef extract_answer(answer_text):\n \"\"\"\n 从 GSM8K 的答案字段中提取数值答案。\n 答案位于连续的 '####' 之后。\n \"\"\"\n match = re.search(r\"####\\s*([\\d\\.\\-eE]+)\", answer_text)\n if match:\n return match.group(1)\n return None\n\n# 模型调用函数\ndef query_model(question, model=MODEL):\n \"\"\"\n 使用给定问题调用模型，并要求其输出符合格式。\n \"\"\"\n prompt = [\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n {\"role\": \"user\", \"content\": f\"For each question, show the answer started with \\\"####\\\" Make sure that the answer is only a number without any other words!\\n Question: \\n {question}\\n\"\n }\n ]\n \n try:\n response = openai.ChatCompletion.create(\n model=model,\n messages=prompt,\n temperature=0.5 # 控制回答的确定性\n )\n return response['choices'][0]['message']['content']\n except Exception as e:\n print(f\"模型调用错误: {e}\")\n return None\n\n# 正确性判定函数\ndef is_correct(predicted, ground_truth):\n \"\"\"\n 判断模型预测是否正确。\n \"\"\"\n try:\n predicted_value = float(extract_answer(predicted)) # 提取数值\n ground_truth_value = float(extract_answer(ground_truth)) # 提取标准答案\n return abs(predicted_value - ground_truth_value) < 1e-3 # 考虑浮点误差\n except:\n return False\n\n# 加载 GSM8K 数据集\nprint(\"加载 GSM8K 数据集...\")\ndataset = load_dataset(\"gsm8k\",\"main\", split=\"test\")\n\n# 评估过程\nprint(\"开始评估...\")\ncorrect = 0\ntotal = len(dataset)\nnums = 0\npredictions=[]\nwith open(\"check_naive.json\",\"w\") as f:\n for item in tqdm(dataset, desc=\"Evaluating GSM8K\"):\n question = item['question']\n ground_truth = item['answer']\n prediction = query_model(question)\n\n \n if prediction and is_correct(prediction, ground_truth):\n correct += 1\n else:\n result = {\n \"index\": nums,\n \"question\": question,\n \"ground_truth\": ground_truth,\n \"prediction\": prediction,\n }\n predictions.append(result)\n f.write(json.dumps(result, ensure_ascii=False) + \"\\n\")\n\n# 输出正确率\naccuracy = correct / total * 100\nprint(f\"GSM8K 正确率: {accuracy:.2f}% ({correct}/{total})\")\n"}}}} |