1 line
21 KiB
JSON
1 line
21 KiB
JSON
{"repo": "a-chicharito-s/opinsummeval", "files": {"metric_evaluation/get_gpt_based_scores/get_Geval_chatgpt.py": {"01fc980f416766965ca9b04b4bdad4a6f926999c": {"t": 1785768943.5758479, "n": 5250, "h": "b9a2a4873fb7d791", "c": "import time\nimport sys\nimport openai\nimport os\nfrom tqdm import tqdm\nimport json\n\n# openai.api_base = 'https://api.openai-asia.com/v1'\n# openai.api_key = 'sk-437Tr7gfq8RXS7cnQQ6Vky5pXeHFqAYpWaBxLNAjbD17qcVU'\n# # \"https://closeai.deno.dev/v1\"\n\nopenai.api_base = 'https://openai.api2d.net/v1'\nopenai.api_key = 'fk205563-yEH2DxljhJV1dV1qa518UGTgIeKqYl4H'\n\n\nclass GPT3Model(object):\n\n def __init__(self, model_name, api_key):\n self.model_name = model_name\n # try:\n # openai.api_key = api_key\n # except Exception:\n # pass\n self.dimensions = ['Aspect Relevance', 'Self-Coherence', 'Sentiment Consistency', 'Readability']\n f = open('geval_instructions_mod.jsonl')\n self.CoT = json.loads(f.readline())\n\n def get_GEval_score(self):\n processed_cases = 0\n while processed_cases < 100:\n try:\n # ------ data preparation ------\n if os.path.exists('chatgpt_CoT_geval_score_mod.jsonl'):\n f = open('chatgpt_CoT_geval_score_mod.jsonl',\n 'r') # in case the connection is lost, and you need to run it more than once\n processed_cases = len(f.readlines())\n # print(processed_cases)\n f.close()\n\n model_f = open('../14model_outputs.jsonl', 'r')\n\n f = open('chatgpt_CoT_geval_score_mod.jsonl', 'a')\n print('processed case is {}'.format(processed_cases))\n for i, line in tqdm(enumerate(model_f)):\n if i < processed_cases:\n continue\n inst = json.loads(line)\n case = inst['case']\n model_names = list(inst['model_output'].keys())\n model_names.sort()\n assert len(model_names) == 14\n input_reviews = 'Reviews:\\n1.{}\\n2.{}\\n3.{}\\n4.{}\\n5.{}\\n6.{}\\n7.{}\\n8.{}\\n'.format(\n inst['revs']['rev1'],\n inst['revs']['rev2'],\n inst['revs']['rev3'],\n inst['revs']['rev4'],\n inst['revs']['rev5'],\n inst['revs']['rev6'],\n inst['revs']['rev7'],\n inst['revs']['rev8'])\n gpt_reply = {model_name: {dim: None for dim in self.dimensions} for model_name in model_names}\n for model_idx in range(14):\n input_summs = 'Summary: {}\\n'.format(inst['model_output'][model_names[model_idx]])\n for dim in self.dimensions:\n instruction = self.CoT[dim]\n prompt = instruction + '\\n' + input_reviews + '\\n' + input_summs\n # print('processing summ {} dim {}'.format(model_idx+1, dim))\n chatgpt_CoT_score = self.do_inference(raw_prompt=prompt, dimension=dim)\n\n gpt_reply[model_names[model_idx]][dim] = chatgpt_CoT_score\n\n inst_dict = {'gpt_reply': gpt_reply, 'case': int(i + 1)}\n f.write(json.dumps(inst_dict) + '\\n')\n print('instance {} is written'.format(i+1))\n # if i == 0:\n # print('case 1 is finished')\n # return 0\n time.sleep(3)\n\n f.close()\n processed_cases = case # should be 100 to end the while loop; if < 100, the program will continue to try\n break\n except Exception:\n time.sleep(3)\n\n def do_inference(self, raw_prompt, dimension):\n sys_message = {'role': 'system', 'content': 'You are a helpful assistant'}\n user_message = {'role': 'user', 'content': '{}'.format(raw_prompt)}\n prompt = [sys_message, user_message]\n response = self.gpt3(prompt)\n out = response.choices[0].message\n # print(out)\n\n # assert prompt == out[\"text\"]\n\n # score_log_prob = out['logprobs'][\"token_logprobs\"][-1]\n # assert str(score) in out['logprobs'][\"tokens\"][-1]\n\n # return score_log_prob\n return out\n\n def gpt3(self, prompt):\n response = None\n received = False\n while not received:\n try:\n # print('getting response...')\n response = openai.ChatCompletion.create(\n model=\"gpt-3.5-turbo\",\n messages=prompt,\n temperature=0.0\n )\n # print('prompt: ', prompt)\n received = True\n # print('received')\n except:\n error = sys.exc_info()[0]\n if error == openai.error.InvalidRequestError:\n # something is wrong: e.g. prompt too long\n print(f\"InvalidRequestError\\nPrompt passed in:\\n\\n{prompt}\\n\\n\")\n assert False\n print(\"API error:\", error)\n time.sleep(1)\n return response\n\n\nif __name__ == \"__main__\":\n M = GPT3Model(model_name='text-ada-001', api_key=None)\n M.get_GEval_score()\n"}}, "metric_evaluation/get_gpt_based_scores/get_chatgpt_handcraft_unstru.py": {"01fc980f416766965ca9b04b4bdad4a6f926999c": {"t": 1785768944.3643255, "n": 9306, "h": "2ae74273b633755f", "c": "import time\nimport openai\nimport re\nimport pandas as pd\nimport json\nfrom tqdm import tqdm\nimport os\nimport sys\n\n# openai.api_base = \"https://closeai.deno.dev/v1\"\n# # openai.api_key = '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' # mine\n# openai.api_key = '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' # Mason's\n\nopenai.api_base = 'https://openai.api2d.net/v1'\nopenai.api_key = 'fk205563-yEH2DxljhJV1dV1qa518UGTgIeKqYl4H'\n\n\nprocessed_cases = 0\n\n\n# def get_gpt_reply(inst):\n# instruction = 'Evaluate the quality of summaries written for 8 reviews. Rate each summary on 4 dimensions:\\n' \\\n# '1. aspect-relevance: Are the mainly discussed aspects in the reviews covered exactly by the summary?\\n' \\\n# '2. self-coherence: Is the summary consistent within itself in terms of sentiments and aspects?\\n' \\\n# '3. sentiment-consistence: Is the summary consistent with the reviews in terms of sentiments on each aspect?\\n' \\\n# '4. readability: Is the summary fluent and informative?\\n' \\\n# 'You should rate on a scale from 1 (worst) to 5 (best) and reply in separate lines.\\n' \\\n# 'For each summary, you should begin the reply with the number of the summary, e.g., summary 1\\n' \\\n# 'For each dimension, you should reply in the format \\\"dimension: score\\\", e.g., aspect-relevance: 4\\n'\n#\n# exemplary_reply = 'An exemplary reply evaluating 3 summaries:\\n' \\\n# 'summary 1:\\naspect-relevance: 3\\nself-coherence: 2\\nsentiment-consistence: 3\\nreadability: 3\\n' \\\n# 'summary 2:\\naspect-relevance: 4\\nself-coherence: 5\\nsentiment-consistence: 4\\nreadability: 4\\n' \\\n# 'summary 3:\\naspect-relevance: 2\\nself-coherence: 3\\nsentiment-consistence: 1\\nreadability: 2\\n\\n'\n# # system, user, or assistant\n# sys_message = {'role': 'system', 'content': 'You are a helpful assistant'}\n#\n# user_message1 = {'role': 'user',\n# 'content': 'Please help me to evaluate 14 summaries over 4 dimensions: '\n# '1. aspect-relevance; 2. self-coherence;3. sentiment-consistence; 4. readability.\\n'\n# 'You should rate on a scale from 1 (worst) to 5 (best) and reply in separate lines.\\n' \\\n# 'For each summary, you should begin the reply with the number of the summary, e.g., summary 7\\n' \\\n# 'For each dimension, you should reply in the format \\\"dimension: score\\\", e.g., aspect-relevance: 1\\n{}'\n# 'if you understand the task, please reply \\\"Yes, I am happy to help\\\"'.format(\n# exemplary_reply)} # TODO: insert the template here\n#\n# input_reviews = 'Reviews:\\n1.{}\\n2.{}\\n3.{}\\n4.{}\\n5.{}\\n6.{}\\n7.{}\\n8.{}\\n\\n'.format(inst['revs']['rev1'],\n# inst['revs']['rev2'],\n# inst['revs']['rev3'],\n# inst['revs']['rev4'],\n# inst['revs']['rev5'],\n# inst['revs']['rev6'],\n# inst['revs']['rev7'],\n# inst['revs']['rev8'])\n# assist_message1 = {'role': 'user', 'content': 'Yes, I am happy to help'}\n#\n# input_summs = 'Summaries:\\n'\n# model_names = list(inst['model_output'].keys())\n# model_names.sort()\n# assert len(model_names) == 14\n# for model_idx in range(14):\n# input_summs = input_summs + '{}.{}\\n'.format(str(model_idx + 1),\n# inst['model_output'][model_names[model_idx]])\n#\n# user_message2 = {'role': 'user', 'content': '{}'.format(instruction + input_reviews + input_summs)}\n#\n# input_message = [sys_message, user_message1, assist_message1, user_message2]\n#\n# completion = openai.ChatCompletion.create(\n# model=\"gpt-3.5-turbo\",\n# messages=input_message,\n# temperature=0.0\n# )\n# return completion.choices[0].message\n\n\ndef get_gpt_reply_onebyone(inst):\n instruction = 'Please help me to evaluate the quality of one summary written for 8 reviews. Rate the summary independently on 4 dimensions:\\n' \\\n '1. relevance: Are the mainly discussed aspects in the reviews covered exactly by the summary?\\n' \\\n '2. coherence: Is the summary consistent within itself in terms of sentiments and aspects?\\n' \\\n '3. consistency: Is the summary consistent with the reviews in terms of sentiments for each aspect?\\n' \\\n '4. readability: Is the summary fluent and informative?\\n' \\\n 'You should rate on a scale from 1 (worst) to 5 (best) and evaluate each dimension independently.'\n case = inst['case']\n # system, user, or assistant\n sys_message = {'role': 'system', 'content': 'You are a helpful assistant'}\n\n user_message1 = {'role': 'user',\n 'content': '{}\\n\\nif you understand the task, please reply \\\"Yes, I am happy to help\\\"'.format(instruction)} # TODO: insert the template here\n\n input_reviews = 'Reviews:\\n1.{}\\n2.{}\\n3.{}\\n4.{}\\n5.{}\\n6.{}\\n7.{}\\n8.{}\\n\\n'.format(inst['revs']['rev1'],\n inst['revs']['rev2'],\n inst['revs']['rev3'],\n inst['revs']['rev4'],\n inst['revs']['rev5'],\n inst['revs']['rev6'],\n inst['revs']['rev7'],\n inst['revs']['rev8'])\n assist_message1 = {'role': 'user', 'content': 'Yes, I am happy to help'}\n\n model_names = list(inst['model_output'].keys())\n model_names.sort()\n assert len(model_names) == 14\n\n replies = {}\n for model_idx in range(14):\n model_name = model_names[model_idx]\n input_summs = 'Summary: {}\\n'.format(inst['model_output'][model_name])\n user_message = {'role': 'user', 'content': '{}'.format(input_reviews + input_summs)}\n # TODO: another version could be removing the \"if-else\" structure and treat every reviews as an individual task\n\n input_message = [sys_message, user_message1, assist_message1, user_message]\n received = False\n while not received:\n try:\n completion = openai.ChatCompletion.create(\n model=\"gpt-3.5-turbo\",\n messages=input_message,\n temperature=0.0\n )\n time.sleep(1)\n\n message = completion.choices[0].message\n if model_name not in replies.keys():\n replies[model_name] = message['content']\n\n received = True\n\n except:\n error = sys.exc_info()[0]\n if error == openai.error.InvalidRequestError:\n # something is wrong: e.g. prompt too long\n print(f\"InvalidRequestError\\nInstance & model:\\n\\n{case+'_'+model_name}\\n\\n\")\n assert False\n print(\"API error:\", error)\n time.sleep(1)\n\n return replies\n\n\nwhile processed_cases < 100:\n try:\n # ------ data preparation ------\n if os.path.exists('chatgpt_handcraft_score_14_onebyone_unstru.jsonl'):\n f = open('chatgpt_handcraft_score_14_onebyone_unstru.jsonl',\n 'r') # in case the connection is lost, and you need to run it more than once\n processed_cases = len(f.readlines())\n print(processed_cases)\n f.close()\n\n model_f = open('../14model_outputs.jsonl', 'r')\n\n f = open('chatgpt_handcraft_score_14_onebyone_unstru.jsonl', 'a')\n\n for i, line in tqdm(enumerate(model_f)):\n if i < processed_cases:\n continue\n inst = json.loads(line)\n case = inst['case']\n gpt_reply = get_gpt_reply_onebyone(inst=inst)\n inst_dict = {'gpt_reply': gpt_reply, 'case': int(i + 1)}\n f.write(json.dumps(inst_dict) + '\\n')\n print('case {} written'.format(case))\n time.sleep(3)\n\n f.close()\n processed_cases = case # should be 100 to end the while loop; if < 100, the program will continue to try\n break\n except Exception:\n time.sleep(3)\n"}}, "metric_evaluation/get_gpt_based_scores/get_GEval_score.py": {"01fc980f416766965ca9b04b4bdad4a6f926999c": {"t": 1785768944.5115643, "n": 6098, "h": "105754c9fdbc2ae9", "c": "import time\nimport sys\nimport openai\nimport os\nfrom tqdm import tqdm\nimport json\n\n# openai.api_base = 'https://api.openai-asia.com/v1'\n# openai.api_key = 'sk-437Tr7gfq8RXS7cnQQ6Vky5pXeHFqAYpWaBxLNAjbD17qcVU'\n# # \"https://closeai.deno.dev/v1\"\n\nopenai.api_base = 'https://openai.api2d.net/v1'\nopenai.api_key = 'fk205563-yEH2DxljhJV1dV1qa518UGTgIeKqYl4H'\n\n\nclass GPT3Model(object):\n\n def __init__(self, model_name, api_key):\n self.model_name = model_name\n # try:\n # openai.api_key = api_key\n # except Exception:\n # pass\n self.dimensions = ['Aspect Relevance', 'Self-Coherence', 'Sentiment Consistency', 'Readability']\n f = open('geval_instructions_mod.jsonl')\n self.CoT = json.loads(f.readline())\n\n def get_GEval_score(self):\n processed_cases = 0\n while processed_cases < 100:\n try:\n # ------ data preparation ------\n if os.path.exists('text_ada_001_geval_score_mod.jsonl'):\n f = open('text_ada_001_geval_score_mod.jsonl',\n 'r') # in case the connection is lost, and you need to run it more than once\n processed_cases = len(f.readlines())\n # print(processed_cases)\n f.close()\n\n model_f = open('../14model_outputs.jsonl', 'r')\n\n f = open('text_ada_001_geval_score_mod.jsonl', 'a')\n print('processed cases: {}'.format(processed_cases))\n\n for i, line in tqdm(enumerate(model_f)):\n if i < processed_cases:\n continue\n inst = json.loads(line)\n case = inst['case']\n model_names = list(inst['model_output'].keys())\n model_names.sort()\n assert len(model_names) == 14\n input_reviews = 'Reviews:\\n1.{}\\n2.{}\\n3.{}\\n4.{}\\n5.{}\\n6.{}\\n7.{}\\n8.{}\\n'.format(\n inst['revs']['rev1'],\n inst['revs']['rev2'],\n inst['revs']['rev3'],\n inst['revs']['rev4'],\n inst['revs']['rev5'],\n inst['revs']['rev6'],\n inst['revs']['rev7'],\n inst['revs']['rev8'])\n gpt_reply = {model_name: {dim: None for dim in self.dimensions} for model_name in model_names}\n for model_idx in range(14):\n input_summs = 'Summary: {}\\n'.format(inst['model_output'][model_names[model_idx]])\n for dim in self.dimensions:\n instruction = self.CoT[dim]\n prompt = instruction + '\\n' + input_reviews + '\\n' + input_summs\n # print('processing summ {} dim {}'.format(model_idx+1, dim))\n score_log_prob_dict = self.get_weighted_score(raw_prompt=prompt, dimension=dim)\n\n gpt_reply[model_names[model_idx]][dim] = score_log_prob_dict\n\n inst_dict = {'gpt_reply': gpt_reply, 'case': int(i + 1)}\n f.write(json.dumps(inst_dict) + '\\n')\n print('instance {} is written'.format(i+1))\n # if i == 0:\n # print('case 1 is finished')\n # return 0\n time.sleep(20)\n\n f.close()\n processed_cases = case # should be 100 to end the while loop; if < 100, the program will continue to try\n break\n except Exception:\n time.sleep(3)\n\n def get_weighted_score(self, raw_prompt, dimension):\n score_log_prob_dict = {}\n for score in [1, 2, 3, 4, 5]:\n # print('processing score: {}'.format(score))\n score_log_prob = self.do_inference(raw_prompt=raw_prompt, dimension=dimension, score=score)\n score_log_prob_dict[score] = score_log_prob\n return score_log_prob_dict\n\n def do_inference(self, raw_prompt, dimension, score):\n score_prompt = 'the score for \\\"{}\\\" is {}'.format(dimension, score)\n prompt = raw_prompt + score_prompt\n response = self.gpt3(prompt)\n out = response[\"choices\"][0]\n # print(out)\n\n assert prompt == out[\"text\"]\n\n # score_log_prob = out['logprobs'][\"token_logprobs\"][-1]\n # assert str(score) in out['logprobs'][\"tokens\"][-1]\n\n # return score_log_prob\n return out\n\n def gpt3(self, prompt, max_len=0, temp=0, num_log_probs=0, echo=True, n=None):\n response = None\n received = False\n while not received:\n try:\n # print('getting response...')\n response = openai.Completion.create(model=self.model_name,\n prompt=prompt,\n max_tokens=max_len,\n # setting this to 0 guarantees that the GPT model will not generate anything\n temperature=temp,\n logprobs=num_log_probs,\n echo=echo,\n stop='\\n',\n n=n)\n # print('prompt: ', prompt)\n received = True\n # print('received')\n except:\n error = sys.exc_info()[0]\n if error == openai.error.InvalidRequestError:\n # something is wrong: e.g. prompt too long\n print(f\"InvalidRequestError\\nPrompt passed in:\\n\\n{prompt}\\n\\n\")\n assert False\n print(\"API error:\", error)\n time.sleep(1)\n return response\n\n\nif __name__ == \"__main__\":\n M = GPT3Model(model_name='text-ada-001', api_key=None)\n M.get_GEval_score()\n"}}}} |