1 line
41 KiB
JSON
1 line
41 KiB
JSON
{"repo": "213214/agent", "files": {"utils.py": {"123444b7690aedbe895ec68c0b69b6fe65d2a22a": {"t": 1785746634.804532, "n": 16770, "h": "c4321091a4ef0983", "c": "import os, re\nimport shutil\nimport time\nimport tiktoken, openai\nimport subprocess, string\nfrom openai import OpenAI\nimport google.generativeai as genai\nfrom huggingface_hub import InferenceClient\n\n\ndef query_deepseekv3(prompt, system, api_key, attempt=0, temperature=0.0):\n try:\n client = OpenAI(api_key=api_key, base_url=\"https://api.deepseek.com\")\n response = client.chat.completions.create(\n model=\"deepseek-chat\",\n messages=[\n {\"role\": \"system\", \"content\": system},\n {\"role\": \"user\", \"content\": prompt},\n ],\n stream=False, temperature=temperature,\n )\n return response.choices[0].message.content\n except Exception as e:\n print(f\"Query qwen error: {e}\")\n if attempt >= 10: return f\"Your attempt to query deepseekv3 failed: {e}\"\n return query_deepseekv3(prompt, system, attempt+1)\n\n\ndef query_qwen(prompt, system, api_key, attempt=0, temperature=0.0):\n try:\n client = InferenceClient(api_key=api_key)\n if system is not None:\n messages = [\n {\"role\": \"system\", \"content\": system},\n {\"role\": \"user\", \"content\": prompt}]\n else:\n messages = [\n {\"role\": \"user\", \"content\": prompt}]\n\n completion = client.chat.completions.create(\n model=\"Qwen/QwQ-32B\",\n messages=messages,\n max_tokens=500,\n temperature=temperature\n )\n return completion.choices[0].message.content.strip()\n except Exception as e:\n print(f\"Query qwen error: {e}\")\n if attempt >= 10: return f\"Your attempt to inference gemini failed: {e}\"\n return query_qwen(prompt, system, attempt+1)\n\n\ndef query_gpt4omini(prompt, system, api_key, attempt=0, temperature=0.0):\n try:\n openai_api_key = api_key\n openai.api_key = openai_api_key\n os.environ[\"OPENAI_API_KEY\"] = openai_api_key\n if system is not None:\n messages = [\n {\"role\": \"system\", \"content\": system},\n {\"role\": \"user\", \"content\": prompt}]\n else:\n messages = [\n {\"role\": \"user\", \"content\": prompt}]\n client = OpenAI()\n response = client.chat.completions.create(\n model=\"gpt-4o-mini\", messages=messages, temperature=temperature).choices[0].message.content.strip()\n return response\n except Exception as e:\n print(f\"Query 4o-mini error: {e}\")\n if attempt >= 10: return f\"Your attempt to inference gemini failed: {e}\"\n return query_gpt4omini(prompt, system, attempt+1)\n\n\n\ndef query_gpt4o(prompt, system, api_key, attempt=0, temperature=0.0):\n try:\n openai_api_key = api_key\n openai.api_key = openai_api_key\n os.environ[\"OPENAI_API_KEY\"] = openai_api_key\n if system is not None:\n messages = [\n {\"role\": \"user\", \"content\":system + prompt}]\n else:\n messages = [\n {\"role\": \"user\", \"content\": prompt}]\n client = OpenAI()\n response = client.chat.completions.create(\n model=\"gpt-4o\", messages=messages, temperature=temperature).choices[0].message.content.strip()\n return response\n except Exception as e:\n print(f\"Query gpr-4o error: {e}\")\n if attempt >= 10: return f\"Your attempt to inference gemini failed: {e}\"\n return query_gpt4o(prompt, system, attempt+1)\n\n\n\ndef query_gemini(prompt, system, api_key, attempt=0, temperature=0.0):\n try:\n genai.configure(api_key=api_key)\n model = genai.GenerativeModel(model_name=\"gemini-1.5-pro\", system_instruction=system)\n response = model.generate_content(prompt, generation_config=genai.types.GenerationConfig(temperature=temperature)).text.strip()\n time.sleep(1)\n return response\n except Exception as e:\n print(f\"Gemini error: {e}\")\n if attempt >= 10: return f\"Your attempt to inference gemini failed: {e}\"\n time.sleep(1)\n return query_gemini(prompt, system, attempt+1)\n\n\n\ndef query_gemini2p0(prompt, system, api_key, attempt=0, temperature=0.0,):\n try:\n genai.configure(api_key=api_key)\n model = genai.GenerativeModel(model_name=\"gemini-2.0-flash\", system_instruction=system)\n response = model.generate_content(prompt, generation_config=genai.types.GenerationConfig(temperature=temperature)).text.strip()\n time.sleep(1)\n return response\n except Exception as e:\n print(f\"Gemini error: {e}\")\n if attempt >= 10: return f\"Your attempt to inference gemini failed: {e}\"\n time.sleep(1)\n return query_gemini2p0(prompt, system, attempt+1)\n\n\ndef compile_latex(latex_code, output_path, compile=True, timeout=30):\n latex_code = latex_code.replace(\n r\"\\documentclass{article}\",\n \"\\\\documentclass{article}\\n\\\\usepackage{amsmath}\\n\\\\usepackage{amssymb}\\n\\\\usepackage{array}\\n\\\\usepackage{algorithm}\\n\\\\usepackage{algorithmicx}\\n\\\\usepackage{algpseudocode}\\n\\\\usepackage{booktabs}\\n\\\\usepackage{colortbl}\\n\\\\usepackage{color}\\n\\\\usepackage{enumitem}\\n\\\\usepackage{fontawesome5}\\n\\\\usepackage{float}\\n\\\\usepackage{graphicx}\\n\\\\usepackage{hyperref}\\n\\\\usepackage{listings}\\n\\\\usepackage{makecell}\\n\\\\usepackage{multicol}\\n\\\\usepackage{multirow}\\n\\\\usepackage{pgffor}\\n\\\\usepackage{pifont}\\n\\\\usepackage{soul}\\n\\\\usepackage{sidecap}\\n\\\\usepackage{subcaption}\\n\\\\usepackage{titletoc}\\n\\\\usepackage[symbol]{footmisc}\\n\\\\usepackage{url}\\n\\\\usepackage{wrapfig}\\n\\\\usepackage{xcolor}\\n\\\\usepackage{xspace}\")\n #print(latex_code)\n dir_path = f\"{output_path}/tex\"\n tex_file_path = os.path.join(dir_path, \"temp.tex\")\n # Write the LaTeX code to the .tex file in the specified directory\n with open(tex_file_path, \"w\") as f:\n f.write(latex_code)\n\n if not compile:\n return f\"Compilation successful\"\n\n # Compiling the LaTeX code using pdflatex with non-interactive mode and timeout\n try:\n result = subprocess.run(\n [\"pdflatex\", \"-interaction=nonstopmode\", \"temp.tex\"],\n check=True, # Raises a CalledProcessError on non-zero exit codes\n stdout=subprocess.PIPE, # Capture standard output\n stderr=subprocess.PIPE, # Capture standard error\n timeout=timeout, # Timeout for the process\n cwd=dir_path\n )\n\n # If compilation is successful, return the success message\n return f\"Compilation successful: {result.stdout.decode('utf-8')}\"\n\n except subprocess.TimeoutExpired:\n # If the compilation takes too long, return a timeout message\n return \"[CODE EXECUTION ERROR]: Compilation timed out after {} seconds\".format(timeout)\n except subprocess.CalledProcessError as e:\n # If there is an error during LaTeX compilation, return the error message\n return f\"[CODE EXECUTION ERROR]: Compilation failed. There was an error in your latex.\"\n\n\ndef count_tokens(messages, model=\"gpt-4\"):\n enc = tiktoken.encoding_for_model(model)\n num_tokens = sum([len(enc.encode(message[\"content\"])) for message in messages])\n return num_tokens\n\ndef remove_figures():\n \"\"\"Remove a directory if it exists.\"\"\"\n for _file in os.listdir(\".\"):\n if \"Figure_\" in _file and \".png\" in _file:\n os.remove(_file)\n\ndef remove_directory(dir_path):\n \"\"\"Remove a directory if it exists.\"\"\"\n if os.path.exists(dir_path) and os.path.isdir(dir_path):\n try:\n shutil.rmtree(dir_path)\n print(f\"Directory {dir_path} removed successfully.\")\n except Exception as e:\n print(f\"Error removing directory {dir_path}: {e}\")\n else:\n print(f\"Directory {dir_path} does not exist or is not a directory.\")\n\n\ndef save_to_file(location, filename, data):\n \"\"\"Utility function to save data as plain text.\"\"\"\n filepath = os.path.join(location, filename)\n try:\n with open(filepath, 'w') as f:\n f.write(data) # Write the raw string instead of using json.dump\n print(f\"Data successfully saved to {filepath}\")\n except Exception as e:\n print(f\"Error saving file {filename}: {e}\")\n\n\ndef clip_tokens(messages, model=\"gpt-4\", max_tokens=100000):\n enc = tiktoken.encoding_for_model(model)\n total_tokens = sum([len(enc.encode(message[\"content\"])) for message in messages])\n\n if total_tokens <= max_tokens:\n return messages # No need to clip if under the limit\n\n # Start removing tokens from the beginning\n tokenized_messages = []\n for message in messages:\n tokenized_content = enc.encode(message[\"content\"])\n tokenized_messages.append({\"role\": message[\"role\"], \"content\": tokenized_content})\n\n # Flatten all tokens\n all_tokens = [token for message in tokenized_messages for token in message[\"content\"]]\n\n # Remove tokens from the beginning\n clipped_tokens = all_tokens[total_tokens - max_tokens:]\n\n # Rebuild the clipped messages\n clipped_messages = []\n current_idx = 0\n for message in tokenized_messages:\n message_token_count = len(message[\"content\"])\n if current_idx + message_token_count > len(clipped_tokens):\n clipped_message_content = clipped_tokens[current_idx:]\n clipped_message = enc.decode(clipped_message_content)\n clipped_messages.append({\"role\": message[\"role\"], \"content\": clipped_message})\n break\n else:\n clipped_message_content = clipped_tokens[current_idx:current_idx + message_token_count]\n clipped_message = enc.decode(clipped_message_content)\n clipped_messages.append({\"role\": message[\"role\"], \"content\": clipped_message})\n current_idx += message_token_count\n return clipped_messages\n\n\n\ndef extract_prompt(text, word):\n code_block_pattern = rf\"```{word}(.*?)```\"\n code_blocks = re.findall(code_block_pattern, text, re.DOTALL)\n extracted_code = \"\\n\".join(code_blocks).strip()\n return extracted_code\n\nfrom typing import Dict, List\n\nimport datasets\n\n\ndef process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": remove_boxed(last_boxed_only_string(doc[\"solution\"])),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n\n\ndef process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n retval = 0\n indices = [pos for pos, char in enumerate(results[0]) if char == \"$\"]\n if len(indices) <= 1:\n answer = results[0]\n else:\n answer = results[0][indices[0] + 1 : indices[-1]]\n\n if is_equiv(answer, remove_boxed(last_boxed_only_string(doc[\"solution\"]))):\n retval = 1\n\n results = {\n \"exact_match\": retval,\n }\n return results\n\n\n# string normalization from https://github.com/EleutherAI/lm-evaluation-harness/blob/master/lm_eval/tasks/hendrycks_math.py\ndef is_equiv(str1, str2, verbose=False):\n if str1 is None and str2 is None:\n print(\"WARNING: Both None\")\n return True\n if str1 is None or str2 is None:\n return False\n\n try:\n ss1 = strip_string(str1)\n ss2 = strip_string(str2)\n if verbose:\n print(ss1, ss2)\n return ss1 == ss2\n except Exception:\n return str1 == str2\n\n\ndef clean_answer(s):\n s = s.replace(\"\\\\dfrac\", \"\\\\frac\") # makes no difference but can lead to errors\n s = s.replace(\"x \\\\in\", \"\")\n return s\n\ndef remove_boxed(s):\n if \"\\\\boxed \" in s:\n left = \"\\\\boxed \"\n assert s[: len(left)] == left\n return s[len(left) :]\n\n left = \"\\\\boxed{\"\n\n assert s[: len(left)] == left\n assert s[-1] == \"}\"\n\n return clean_answer(s[len(left) : -1])\n\n\ndef last_boxed_only_string(string):\n idx = string.rfind(\"\\\\boxed\")\n if \"\\\\boxed \" in string:\n return \"\\\\boxed \" + string.split(\"\\\\boxed \")[-1].split(\"$\")[0]\n if idx < 0:\n idx = string.rfind(\"\\\\fbox\")\n if idx < 0:\n return None\n\n i = idx\n right_brace_idx = None\n num_left_braces_open = 0\n while i < len(string):\n if string[i] == \"{\":\n num_left_braces_open += 1\n if string[i] == \"}\":\n num_left_braces_open -= 1\n if num_left_braces_open == 0:\n right_brace_idx = i\n break\n i += 1\n\n if right_brace_idx is None:\n retval = None\n else:\n retval = string[idx : right_brace_idx + 1]\n\n return retval\n\n\ndef fix_fracs(string):\n substrs = string.split(\"\\\\frac\")\n new_str = substrs[0]\n if len(substrs) > 1:\n substrs = substrs[1:]\n for substr in substrs:\n new_str += \"\\\\frac\"\n if substr[0] == \"{\":\n new_str += substr\n else:\n try:\n assert len(substr) >= 2\n except AssertionError:\n return string\n a = substr[0]\n b = substr[1]\n if b != \"{\":\n if len(substr) > 2:\n post_substr = substr[2:]\n new_str += \"{\" + a + \"}{\" + b + \"}\" + post_substr\n else:\n new_str += \"{\" + a + \"}{\" + b + \"}\"\n else:\n if len(substr) > 2:\n post_substr = substr[2:]\n new_str += \"{\" + a + \"}\" + b + post_substr\n else:\n new_str += \"{\" + a + \"}\" + b\n string = new_str\n return string\n\n\ndef fix_a_slash_b(string):\n if len(string.split(\"/\")) != 2:\n return string\n a = string.split(\"/\")[0]\n b = string.split(\"/\")[1]\n try:\n a = int(a)\n b = int(b)\n assert string == \"{}/{}\".format(a, b)\n new_string = \"\\\\frac{\" + str(a) + \"}{\" + str(b) + \"}\"\n return new_string\n except AssertionError:\n return string\n\n\ndef remove_right_units(string):\n # \"\\\\text{ \" only ever occurs (at least in the val set) when describing units\n if \"\\\\text{ \" in string:\n splits = string.split(\"\\\\text{ \")\n assert len(splits) == 2\n return splits[0]\n else:\n return string\n\n\ndef fix_sqrt(string):\n if \"\\\\sqrt\" not in string:\n return string\n splits = string.split(\"\\\\sqrt\")\n new_string = splits[0]\n for split in splits[1:]:\n if split[0] != \"{\":\n a = split[0]\n new_substr = \"\\\\sqrt{\" + a + \"}\" + split[1:]\n else:\n new_substr = \"\\\\sqrt\" + split\n new_string += new_substr\n return new_string\n\n\ndef strip_string(string):\n # linebreaks\n string = string.replace(\"\\n\", \"\")\n\n # remove inverse spaces\n string = string.replace(\"\\\\!\", \"\")\n\n # replace \\\\ with \\\n string = string.replace(\"\\\\\\\\\", \"\\\\\")\n\n # replace tfrac and dfrac with frac\n string = string.replace(\"tfrac\", \"frac\")\n string = string.replace(\"dfrac\", \"frac\")\n\n # remove \\left and \\right\n string = string.replace(\"\\\\left\", \"\")\n string = string.replace(\"\\\\right\", \"\")\n\n # Remove circ (degrees)\n string = string.replace(\"^{\\\\circ}\", \"\")\n string = string.replace(\"^\\\\circ\", \"\")\n\n # remove dollar signs\n string = string.replace(\"\\\\$\", \"\")\n\n # remove units (on the right)\n string = remove_right_units(string)\n\n # remove percentage\n string = string.replace(\"\\\\%\", \"\")\n string = string.replace(\"\\%\", \"\") # noqa: W605\n\n # \" 0.\" equivalent to \" .\" and \"{0.\" equivalent to \"{.\" Alternatively, add \"0\" if \".\" is the start of the string\n string = string.replace(\" .\", \" 0.\")\n string = string.replace(\"{.\", \"{0.\")\n # if empty, return empty string\n if len(string) == 0:\n return string\n if string[0] == \".\":\n string = \"0\" + string\n\n # to consider: get rid of e.g. \"k = \" or \"q = \" at beginning\n if len(string.split(\"=\")) == 2:\n if len(string.split(\"=\")[0]) <= 2:\n string = string.split(\"=\")[1]\n\n # fix sqrt3 --> sqrt{3}\n string = fix_sqrt(string)\n\n # remove spaces\n string = string.replace(\" \", \"\")\n\n # \\frac1b or \\frac12 --> \\frac{1}{b} and \\frac{1}{2}, etc. Even works with \\frac1{72} (but not \\frac{72}1). Also does a/b --> \\\\frac{a}{b}\n string = fix_fracs(string)\n\n # manually change 0.5 --> \\frac{1}{2}\n if string == \"0.5\":\n string = \"\\\\frac{1}{2}\"\n if string == \"5.5\":\n string = \"\\\\frac{11}{2}\"\n if \"(x - 3)(x + 3)\" in string:\n string = string.replace(\"(x - 3)(x + 3)\", \"(x+3)(x-3)\")\n\n # NOTE: X/Y changed to \\frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y\n string = fix_a_slash_b(string)\n\n return string\n"}, "e1d6d8bb9299791a93948574ff5eda366405973f": {"t": 1785766438.0194402, "n": 16770, "h": "c4321091a4ef0983", "c": "import os, re\nimport shutil\nimport time\nimport tiktoken, openai\nimport subprocess, string\nfrom openai import OpenAI\nimport google.generativeai as genai\nfrom huggingface_hub import InferenceClient\n\n\ndef query_deepseekv3(prompt, system, api_key, attempt=0, temperature=0.0):\n try:\n client = OpenAI(api_key=api_key, base_url=\"https://api.deepseek.com\")\n response = client.chat.completions.create(\n model=\"deepseek-chat\",\n messages=[\n {\"role\": \"system\", \"content\": system},\n {\"role\": \"user\", \"content\": prompt},\n ],\n stream=False, temperature=temperature,\n )\n return response.choices[0].message.content\n except Exception as e:\n print(f\"Query qwen error: {e}\")\n if attempt >= 10: return f\"Your attempt to query deepseekv3 failed: {e}\"\n return query_deepseekv3(prompt, system, attempt+1)\n\n\ndef query_qwen(prompt, system, api_key, attempt=0, temperature=0.0):\n try:\n client = InferenceClient(api_key=api_key)\n if system is not None:\n messages = [\n {\"role\": \"system\", \"content\": system},\n {\"role\": \"user\", \"content\": prompt}]\n else:\n messages = [\n {\"role\": \"user\", \"content\": prompt}]\n\n completion = client.chat.completions.create(\n model=\"Qwen/QwQ-32B\",\n messages=messages,\n max_tokens=500,\n temperature=temperature\n )\n return completion.choices[0].message.content.strip()\n except Exception as e:\n print(f\"Query qwen error: {e}\")\n if attempt >= 10: return f\"Your attempt to inference gemini failed: {e}\"\n return query_qwen(prompt, system, attempt+1)\n\n\ndef query_gpt4omini(prompt, system, api_key, attempt=0, temperature=0.0):\n try:\n openai_api_key = api_key\n openai.api_key = openai_api_key\n os.environ[\"OPENAI_API_KEY\"] = openai_api_key\n if system is not None:\n messages = [\n {\"role\": \"system\", \"content\": system},\n {\"role\": \"user\", \"content\": prompt}]\n else:\n messages = [\n {\"role\": \"user\", \"content\": prompt}]\n client = OpenAI()\n response = client.chat.completions.create(\n model=\"gpt-4o-mini\", messages=messages, temperature=temperature).choices[0].message.content.strip()\n return response\n except Exception as e:\n print(f\"Query 4o-mini error: {e}\")\n if attempt >= 10: return f\"Your attempt to inference gemini failed: {e}\"\n return query_gpt4omini(prompt, system, attempt+1)\n\n\n\ndef query_gpt4o(prompt, system, api_key, attempt=0, temperature=0.0):\n try:\n openai_api_key = api_key\n openai.api_key = openai_api_key\n os.environ[\"OPENAI_API_KEY\"] = openai_api_key\n if system is not None:\n messages = [\n {\"role\": \"user\", \"content\":system + prompt}]\n else:\n messages = [\n {\"role\": \"user\", \"content\": prompt}]\n client = OpenAI()\n response = client.chat.completions.create(\n model=\"gpt-4o\", messages=messages, temperature=temperature).choices[0].message.content.strip()\n return response\n except Exception as e:\n print(f\"Query gpr-4o error: {e}\")\n if attempt >= 10: return f\"Your attempt to inference gemini failed: {e}\"\n return query_gpt4o(prompt, system, attempt+1)\n\n\n\ndef query_gemini(prompt, system, api_key, attempt=0, temperature=0.0):\n try:\n genai.configure(api_key=api_key)\n model = genai.GenerativeModel(model_name=\"gemini-1.5-pro\", system_instruction=system)\n response = model.generate_content(prompt, generation_config=genai.types.GenerationConfig(temperature=temperature)).text.strip()\n time.sleep(1)\n return response\n except Exception as e:\n print(f\"Gemini error: {e}\")\n if attempt >= 10: return f\"Your attempt to inference gemini failed: {e}\"\n time.sleep(1)\n return query_gemini(prompt, system, attempt+1)\n\n\n\ndef query_gemini2p0(prompt, system, api_key, attempt=0, temperature=0.0,):\n try:\n genai.configure(api_key=api_key)\n model = genai.GenerativeModel(model_name=\"gemini-2.0-flash\", system_instruction=system)\n response = model.generate_content(prompt, generation_config=genai.types.GenerationConfig(temperature=temperature)).text.strip()\n time.sleep(1)\n return response\n except Exception as e:\n print(f\"Gemini error: {e}\")\n if attempt >= 10: return f\"Your attempt to inference gemini failed: {e}\"\n time.sleep(1)\n return query_gemini2p0(prompt, system, attempt+1)\n\n\ndef compile_latex(latex_code, output_path, compile=True, timeout=30):\n latex_code = latex_code.replace(\n r\"\\documentclass{article}\",\n \"\\\\documentclass{article}\\n\\\\usepackage{amsmath}\\n\\\\usepackage{amssymb}\\n\\\\usepackage{array}\\n\\\\usepackage{algorithm}\\n\\\\usepackage{algorithmicx}\\n\\\\usepackage{algpseudocode}\\n\\\\usepackage{booktabs}\\n\\\\usepackage{colortbl}\\n\\\\usepackage{color}\\n\\\\usepackage{enumitem}\\n\\\\usepackage{fontawesome5}\\n\\\\usepackage{float}\\n\\\\usepackage{graphicx}\\n\\\\usepackage{hyperref}\\n\\\\usepackage{listings}\\n\\\\usepackage{makecell}\\n\\\\usepackage{multicol}\\n\\\\usepackage{multirow}\\n\\\\usepackage{pgffor}\\n\\\\usepackage{pifont}\\n\\\\usepackage{soul}\\n\\\\usepackage{sidecap}\\n\\\\usepackage{subcaption}\\n\\\\usepackage{titletoc}\\n\\\\usepackage[symbol]{footmisc}\\n\\\\usepackage{url}\\n\\\\usepackage{wrapfig}\\n\\\\usepackage{xcolor}\\n\\\\usepackage{xspace}\")\n #print(latex_code)\n dir_path = f\"{output_path}/tex\"\n tex_file_path = os.path.join(dir_path, \"temp.tex\")\n # Write the LaTeX code to the .tex file in the specified directory\n with open(tex_file_path, \"w\") as f:\n f.write(latex_code)\n\n if not compile:\n return f\"Compilation successful\"\n\n # Compiling the LaTeX code using pdflatex with non-interactive mode and timeout\n try:\n result = subprocess.run(\n [\"pdflatex\", \"-interaction=nonstopmode\", \"temp.tex\"],\n check=True, # Raises a CalledProcessError on non-zero exit codes\n stdout=subprocess.PIPE, # Capture standard output\n stderr=subprocess.PIPE, # Capture standard error\n timeout=timeout, # Timeout for the process\n cwd=dir_path\n )\n\n # If compilation is successful, return the success message\n return f\"Compilation successful: {result.stdout.decode('utf-8')}\"\n\n except subprocess.TimeoutExpired:\n # If the compilation takes too long, return a timeout message\n return \"[CODE EXECUTION ERROR]: Compilation timed out after {} seconds\".format(timeout)\n except subprocess.CalledProcessError as e:\n # If there is an error during LaTeX compilation, return the error message\n return f\"[CODE EXECUTION ERROR]: Compilation failed. There was an error in your latex.\"\n\n\ndef count_tokens(messages, model=\"gpt-4\"):\n enc = tiktoken.encoding_for_model(model)\n num_tokens = sum([len(enc.encode(message[\"content\"])) for message in messages])\n return num_tokens\n\ndef remove_figures():\n \"\"\"Remove a directory if it exists.\"\"\"\n for _file in os.listdir(\".\"):\n if \"Figure_\" in _file and \".png\" in _file:\n os.remove(_file)\n\ndef remove_directory(dir_path):\n \"\"\"Remove a directory if it exists.\"\"\"\n if os.path.exists(dir_path) and os.path.isdir(dir_path):\n try:\n shutil.rmtree(dir_path)\n print(f\"Directory {dir_path} removed successfully.\")\n except Exception as e:\n print(f\"Error removing directory {dir_path}: {e}\")\n else:\n print(f\"Directory {dir_path} does not exist or is not a directory.\")\n\n\ndef save_to_file(location, filename, data):\n \"\"\"Utility function to save data as plain text.\"\"\"\n filepath = os.path.join(location, filename)\n try:\n with open(filepath, 'w') as f:\n f.write(data) # Write the raw string instead of using json.dump\n print(f\"Data successfully saved to {filepath}\")\n except Exception as e:\n print(f\"Error saving file {filename}: {e}\")\n\n\ndef clip_tokens(messages, model=\"gpt-4\", max_tokens=100000):\n enc = tiktoken.encoding_for_model(model)\n total_tokens = sum([len(enc.encode(message[\"content\"])) for message in messages])\n\n if total_tokens <= max_tokens:\n return messages # No need to clip if under the limit\n\n # Start removing tokens from the beginning\n tokenized_messages = []\n for message in messages:\n tokenized_content = enc.encode(message[\"content\"])\n tokenized_messages.append({\"role\": message[\"role\"], \"content\": tokenized_content})\n\n # Flatten all tokens\n all_tokens = [token for message in tokenized_messages for token in message[\"content\"]]\n\n # Remove tokens from the beginning\n clipped_tokens = all_tokens[total_tokens - max_tokens:]\n\n # Rebuild the clipped messages\n clipped_messages = []\n current_idx = 0\n for message in tokenized_messages:\n message_token_count = len(message[\"content\"])\n if current_idx + message_token_count > len(clipped_tokens):\n clipped_message_content = clipped_tokens[current_idx:]\n clipped_message = enc.decode(clipped_message_content)\n clipped_messages.append({\"role\": message[\"role\"], \"content\": clipped_message})\n break\n else:\n clipped_message_content = clipped_tokens[current_idx:current_idx + message_token_count]\n clipped_message = enc.decode(clipped_message_content)\n clipped_messages.append({\"role\": message[\"role\"], \"content\": clipped_message})\n current_idx += message_token_count\n return clipped_messages\n\n\n\ndef extract_prompt(text, word):\n code_block_pattern = rf\"```{word}(.*?)```\"\n code_blocks = re.findall(code_block_pattern, text, re.DOTALL)\n extracted_code = \"\\n\".join(code_blocks).strip()\n return extracted_code\n\nfrom typing import Dict, List\n\nimport datasets\n\n\ndef process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc: dict) -> dict:\n out_doc = {\n \"problem\": doc[\"problem\"],\n \"solution\": doc[\"solution\"],\n \"answer\": remove_boxed(last_boxed_only_string(doc[\"solution\"])),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n\n\ndef process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n retval = 0\n indices = [pos for pos, char in enumerate(results[0]) if char == \"$\"]\n if len(indices) <= 1:\n answer = results[0]\n else:\n answer = results[0][indices[0] + 1 : indices[-1]]\n\n if is_equiv(answer, remove_boxed(last_boxed_only_string(doc[\"solution\"]))):\n retval = 1\n\n results = {\n \"exact_match\": retval,\n }\n return results\n\n\n# string normalization from https://github.com/EleutherAI/lm-evaluation-harness/blob/master/lm_eval/tasks/hendrycks_math.py\ndef is_equiv(str1, str2, verbose=False):\n if str1 is None and str2 is None:\n print(\"WARNING: Both None\")\n return True\n if str1 is None or str2 is None:\n return False\n\n try:\n ss1 = strip_string(str1)\n ss2 = strip_string(str2)\n if verbose:\n print(ss1, ss2)\n return ss1 == ss2\n except Exception:\n return str1 == str2\n\n\ndef clean_answer(s):\n s = s.replace(\"\\\\dfrac\", \"\\\\frac\") # makes no difference but can lead to errors\n s = s.replace(\"x \\\\in\", \"\")\n return s\n\ndef remove_boxed(s):\n if \"\\\\boxed \" in s:\n left = \"\\\\boxed \"\n assert s[: len(left)] == left\n return s[len(left) :]\n\n left = \"\\\\boxed{\"\n\n assert s[: len(left)] == left\n assert s[-1] == \"}\"\n\n return clean_answer(s[len(left) : -1])\n\n\ndef last_boxed_only_string(string):\n idx = string.rfind(\"\\\\boxed\")\n if \"\\\\boxed \" in string:\n return \"\\\\boxed \" + string.split(\"\\\\boxed \")[-1].split(\"$\")[0]\n if idx < 0:\n idx = string.rfind(\"\\\\fbox\")\n if idx < 0:\n return None\n\n i = idx\n right_brace_idx = None\n num_left_braces_open = 0\n while i < len(string):\n if string[i] == \"{\":\n num_left_braces_open += 1\n if string[i] == \"}\":\n num_left_braces_open -= 1\n if num_left_braces_open == 0:\n right_brace_idx = i\n break\n i += 1\n\n if right_brace_idx is None:\n retval = None\n else:\n retval = string[idx : right_brace_idx + 1]\n\n return retval\n\n\ndef fix_fracs(string):\n substrs = string.split(\"\\\\frac\")\n new_str = substrs[0]\n if len(substrs) > 1:\n substrs = substrs[1:]\n for substr in substrs:\n new_str += \"\\\\frac\"\n if substr[0] == \"{\":\n new_str += substr\n else:\n try:\n assert len(substr) >= 2\n except AssertionError:\n return string\n a = substr[0]\n b = substr[1]\n if b != \"{\":\n if len(substr) > 2:\n post_substr = substr[2:]\n new_str += \"{\" + a + \"}{\" + b + \"}\" + post_substr\n else:\n new_str += \"{\" + a + \"}{\" + b + \"}\"\n else:\n if len(substr) > 2:\n post_substr = substr[2:]\n new_str += \"{\" + a + \"}\" + b + post_substr\n else:\n new_str += \"{\" + a + \"}\" + b\n string = new_str\n return string\n\n\ndef fix_a_slash_b(string):\n if len(string.split(\"/\")) != 2:\n return string\n a = string.split(\"/\")[0]\n b = string.split(\"/\")[1]\n try:\n a = int(a)\n b = int(b)\n assert string == \"{}/{}\".format(a, b)\n new_string = \"\\\\frac{\" + str(a) + \"}{\" + str(b) + \"}\"\n return new_string\n except AssertionError:\n return string\n\n\ndef remove_right_units(string):\n # \"\\\\text{ \" only ever occurs (at least in the val set) when describing units\n if \"\\\\text{ \" in string:\n splits = string.split(\"\\\\text{ \")\n assert len(splits) == 2\n return splits[0]\n else:\n return string\n\n\ndef fix_sqrt(string):\n if \"\\\\sqrt\" not in string:\n return string\n splits = string.split(\"\\\\sqrt\")\n new_string = splits[0]\n for split in splits[1:]:\n if split[0] != \"{\":\n a = split[0]\n new_substr = \"\\\\sqrt{\" + a + \"}\" + split[1:]\n else:\n new_substr = \"\\\\sqrt\" + split\n new_string += new_substr\n return new_string\n\n\ndef strip_string(string):\n # linebreaks\n string = string.replace(\"\\n\", \"\")\n\n # remove inverse spaces\n string = string.replace(\"\\\\!\", \"\")\n\n # replace \\\\ with \\\n string = string.replace(\"\\\\\\\\\", \"\\\\\")\n\n # replace tfrac and dfrac with frac\n string = string.replace(\"tfrac\", \"frac\")\n string = string.replace(\"dfrac\", \"frac\")\n\n # remove \\left and \\right\n string = string.replace(\"\\\\left\", \"\")\n string = string.replace(\"\\\\right\", \"\")\n\n # Remove circ (degrees)\n string = string.replace(\"^{\\\\circ}\", \"\")\n string = string.replace(\"^\\\\circ\", \"\")\n\n # remove dollar signs\n string = string.replace(\"\\\\$\", \"\")\n\n # remove units (on the right)\n string = remove_right_units(string)\n\n # remove percentage\n string = string.replace(\"\\\\%\", \"\")\n string = string.replace(\"\\%\", \"\") # noqa: W605\n\n # \" 0.\" equivalent to \" .\" and \"{0.\" equivalent to \"{.\" Alternatively, add \"0\" if \".\" is the start of the string\n string = string.replace(\" .\", \" 0.\")\n string = string.replace(\"{.\", \"{0.\")\n # if empty, return empty string\n if len(string) == 0:\n return string\n if string[0] == \".\":\n string = \"0\" + string\n\n # to consider: get rid of e.g. \"k = \" or \"q = \" at beginning\n if len(string.split(\"=\")) == 2:\n if len(string.split(\"=\")[0]) <= 2:\n string = string.split(\"=\")[1]\n\n # fix sqrt3 --> sqrt{3}\n string = fix_sqrt(string)\n\n # remove spaces\n string = string.replace(\" \", \"\")\n\n # \\frac1b or \\frac12 --> \\frac{1}{b} and \\frac{1}{2}, etc. Even works with \\frac1{72} (but not \\frac{72}1). Also does a/b --> \\\\frac{a}{b}\n string = fix_fracs(string)\n\n # manually change 0.5 --> \\frac{1}{2}\n if string == \"0.5\":\n string = \"\\\\frac{1}{2}\"\n if string == \"5.5\":\n string = \"\\\\frac{11}{2}\"\n if \"(x - 3)(x + 3)\" in string:\n string = string.replace(\"(x - 3)(x + 3)\", \"(x+3)(x-3)\")\n\n # NOTE: X/Y changed to \\frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y\n string = fix_a_slash_b(string)\n\n return string\n"}, "c76a8011765117e8292f1261ae1368189cf949e7": {"t": 1785766439.5322118, "n": 5459, "h": "ea3306de4a9bae03", "c": "import os, re\nimport shutil\nimport tiktoken\nimport subprocess\n\n\ndef compile_latex(latex_code, compile=True, output_filename=\"output.pdf\", timeout=30):\n latex_code = latex_code.replace(\n r\"\\documentclass{article}\",\n \"\\\\documentclass{article}\\n\\\\usepackage{amsmath}\\n\\\\usepackage{amssymb}\\n\\\\usepackage{array}\\n\\\\usepackage{algorithm}\\n\\\\usepackage{algorithmicx}\\n\\\\usepackage{algpseudocode}\\n\\\\usepackage{booktabs}\\n\\\\usepackage{colortbl}\\n\\\\usepackage{color}\\n\\\\usepackage{enumitem}\\n\\\\usepackage{fontawesome5}\\n\\\\usepackage{float}\\n\\\\usepackage{graphicx}\\n\\\\usepackage{hyperref}\\n\\\\usepackage{listings}\\n\\\\usepackage{makecell}\\n\\\\usepackage{multicol}\\n\\\\usepackage{multirow}\\n\\\\usepackage{pgffor}\\n\\\\usepackage{pifont}\\n\\\\usepackage{soul}\\n\\\\usepackage{sidecap}\\n\\\\usepackage{subcaption}\\n\\\\usepackage{titletoc}\\n\\\\usepackage[symbol]{footmisc}\\n\\\\usepackage{url}\\n\\\\usepackage{wrapfig}\\n\\\\usepackage{xcolor}\\n\\\\usepackage{xspace}\")\n #print(latex_code)\n dir_path = \"research_dir/tex\"\n tex_file_path = os.path.join(dir_path, \"temp.tex\")\n # Write the LaTeX code to the .tex file in the specified directory\n with open(tex_file_path, \"w\") as f:\n f.write(latex_code)\n\n if not compile:\n return f\"Compilation successful\"\n\n # Compiling the LaTeX code using pdflatex with non-interactive mode and timeout\n try:\n result = subprocess.run(\n [\"pdflatex\", \"-interaction=nonstopmode\", \"temp.tex\"],\n check=True, # Raises a CalledProcessError on non-zero exit codes\n stdout=subprocess.PIPE, # Capture standard output\n stderr=subprocess.PIPE, # Capture standard error\n timeout=timeout, # Timeout for the process\n cwd=dir_path\n )\n\n # If compilation is successful, return the success message\n return f\"Compilation successful: {result.stdout.decode('utf-8')}\"\n\n except subprocess.TimeoutExpired:\n # If the compilation takes too long, return a timeout message\n return \"[CODE EXECUTION ERROR]: Compilation timed out after {} seconds\".format(timeout)\n except subprocess.CalledProcessError as e:\n # If there is an error during LaTeX compilation, return the error message\n return f\"[CODE EXECUTION ERROR]: Compilation failed: {e.stderr.decode('utf-8')} {e.output.decode('utf-8')}. There was an error in your latex.\"\n\n\ndef count_tokens(messages, model=\"gpt-4\"):\n enc = tiktoken.encoding_for_model(model)\n num_tokens = sum([len(enc.encode(message[\"content\"])) for message in messages])\n return num_tokens\n\ndef remove_figures():\n \"\"\"Remove a directory if it exists.\"\"\"\n for _file in os.listdir(\".\"):\n if \"Figure_\" in _file and \".png\" in _file:\n os.remove(_file)\n\ndef remove_directory(dir_path):\n \"\"\"Remove a directory if it exists.\"\"\"\n if os.path.exists(dir_path) and os.path.isdir(dir_path):\n try:\n shutil.rmtree(dir_path)\n print(f\"Directory {dir_path} removed successfully.\")\n except Exception as e:\n print(f\"Error removing directory {dir_path}: {e}\")\n else:\n print(f\"Directory {dir_path} does not exist or is not a directory.\")\n\n\ndef save_to_file(location, filename, data):\n \"\"\"Utility function to save data as plain text.\"\"\"\n filepath = os.path.join(location, filename)\n try:\n with open(filepath, 'w') as f:\n f.write(data) # Write the raw string instead of using json.dump\n print(f\"Data successfully saved to {filepath}\")\n except Exception as e:\n print(f\"Error saving file {filename}: {e}\")\n\n\ndef clip_tokens(messages, model=\"gpt-4\", max_tokens=100000):\n enc = tiktoken.encoding_for_model(model)\n total_tokens = sum([len(enc.encode(message[\"content\"])) for message in messages])\n\n if total_tokens <= max_tokens:\n return messages # No need to clip if under the limit\n\n # Start removing tokens from the beginning\n tokenized_messages = []\n for message in messages:\n tokenized_content = enc.encode(message[\"content\"])\n tokenized_messages.append({\"role\": message[\"role\"], \"content\": tokenized_content})\n\n # Flatten all tokens\n all_tokens = [token for message in tokenized_messages for token in message[\"content\"]]\n\n # Remove tokens from the beginning\n clipped_tokens = all_tokens[total_tokens - max_tokens:]\n\n # Rebuild the clipped messages\n clipped_messages = []\n current_idx = 0\n for message in tokenized_messages:\n message_token_count = len(message[\"content\"])\n if current_idx + message_token_count > len(clipped_tokens):\n clipped_message_content = clipped_tokens[current_idx:]\n clipped_message = enc.decode(clipped_message_content)\n clipped_messages.append({\"role\": message[\"role\"], \"content\": clipped_message})\n break\n else:\n clipped_message_content = clipped_tokens[current_idx:current_idx + message_token_count]\n clipped_message = enc.decode(clipped_message_content)\n clipped_messages.append({\"role\": message[\"role\"], \"content\": clipped_message})\n current_idx += message_token_count\n return clipped_messages\n\n\n\ndef extract_prompt(text, word):\n code_block_pattern = rf\"```{word}(.*?)```\"\n code_blocks = re.findall(code_block_pattern, text, re.DOTALL)\n extracted_code = \"\\n\".join(code_blocks).strip()\n return extracted_code\n\n\n"}}}} |