1 line
40 KiB
JSON
1 line
40 KiB
JSON
{"repo": "archiki/utgendebug", "files": {"utils.py": {"03bc690c28b6331e7ff0131cb001dab66f4e546f": {"t": 1785746632.8403914, "n": 19529, "h": "31ce712e3623be93", "c": "import os\nimport re\nimport time\nimport openai\nimport argparse\nimport traceback\nimport fire\nimport random\nimport pandas as pd\nfrom tqdm import tqdm\nfrom typing import List\nfrom datasets import Dataset, load_dataset\nfrom termcolor import colored\nimport sys\nimport ast\nimport timeout_decorator\nimport numpy as np\nimport json\nimport subprocess\nfrom vllm import LLM, SamplingParams\nimport torch\nfrom transformers import AutoTokenizer\nfrom tenacity import (\n retry,\n retry_if_not_exception_type,\n stop_after_attempt, # type: ignore\n wait_random_exponential, # type: ignore\n)\nfrom multiprocessing import Array, Value, Process, Queue \nfrom collections import defaultdict, OrderedDict\nimport multiprocessing \nimport time \nfrom typing import List, Tuple, Any\n\nimport signal \nimport resource \nfrom contextlib import contextmanager\n\nHF_DATASET = {\n 'mbpp_plus_fix': 'datasets/MBPP+Fix_test.jsonl',\n 'mbpp_plus_fix_hard': 'datasets/MBPP+Fix_hard_test.jsonl',\n 'humaneval_plus_fix': 'datasets/HE+Fix_test.jsonl',\n }\n\n\nSTOP_WORDS = {\n \"llama3\": [\"<|eot_id|>\", \"<|end_of_text|>\"],\n \"llama3.1\": [\"<|eot_id|>\", \"<|end_of_text|>\"],\n \"tulu3\": [\"<|user|>\", \"<|assistant|>\", \"<|end_of_text|>\"],\n \"llama3_70b\": [\"<|eot_id|>\", \"<|end_of_text|>\"],\n \"gemma\": [\"<end_of_turn>\"],\n \"eurus\": [],\n \"qwen\": [\"<|endoftext|>\", \"<|im_end|>\"],\n \"big_qwen\": [\"<|endoftext|>\", \"<|im_end|>\"],\n \"deepseek\": [\"<｜end▁of▁sentence｜>\", \"### Response\", \"### Instruction:\"]\n}\n\nPASS = \"pass\"\nFAIL = \"fail\"\nTIMEOUT = \"timeout\"\n\n_SUCCESS = 0\n_FAILED = 1\n_TIMEOUT = 2\n_UNKNOWN = 3\n\n_mapping = {_SUCCESS: PASS, _FAILED: FAIL, _TIMEOUT: TIMEOUT, _UNKNOWN: None}\n\nHF_MODEL_NAMES = {\n \"llama3\": \"meta-llama/Meta-Llama-3-8B-Instruct\",\n \"llama3.1\": \"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n \"gemma\": \"google/gemma-7b-it\",\n \"deepseek\": \"deepseek-ai/deepseek-coder-7b-instruct-v1.5\",\n \"qwen\": \"Qwen/Qwen2.5-Coder-7B-Instruct\",\n \"tulu3\": \"allenai/Llama-3.1-Tulu-3-8B-SFT\",\n}\n\n\nimport time\nfrom copy import deepcopy\n\nimport contextlib\nimport faulthandler\nimport io\nimport os\nimport platform\nimport signal\nimport tempfile\nfrom typing import Optional\n\nclass TimeoutException(Exception):\n pass\n\n\nclass WriteOnlyStringIO(io.StringIO):\n \"\"\"StringIO that throws an exception when it's read from\"\"\"\n\n def read(self, *args, **kwargs):\n raise IOError\n\n def readline(self, *args, **kwargs):\n raise IOError\n\n def readlines(self, *args, **kwargs):\n raise IOError\n\n def readable(self, *args, **kwargs):\n \"\"\"Returns True if the IO object can be read.\"\"\"\n return False\n\nclass redirect_stdin(contextlib._RedirectStream): # type: ignore\n _stream = \"stdin\"\n\n@contextlib.contextmanager\ndef swallow_io():\n stream = WriteOnlyStringIO()\n with contextlib.redirect_stdout(stream):\n with contextlib.redirect_stderr(stream):\n with redirect_stdin(stream):\n yield\n\n\n@contextlib.contextmanager\ndef time_limit(seconds: float):\n def signal_handler(signum, frame):\n raise TimeoutException(\"Timed out!\")\n\n signal.setitimer(signal.ITIMER_REAL, seconds)\n signal.signal(signal.SIGALRM, signal_handler)\n try:\n yield\n finally:\n signal.setitimer(signal.ITIMER_REAL, 0)\n\n\ndef trusted_exec(code, inputs, entry_point, record_time=False, output_not_none=False):\n \"\"\"Execute trusted code in place.\"\"\"\n exec_globals = {}\n exec(code, exec_globals)\n fn = exec_globals[entry_point]\n # print(fn)\n rtime = []\n ret = []\n for inp in inputs:\n inp = deepcopy(inp)\n # print(inp)\n if record_time:\n start = time.time()\n # print(type(fn))\n ret.append(fn(*inp))\n rtime.append(time.time() - start)\n else:\n ret.append(fn(*inp))\n\n if output_not_none:\n ret = [i is not None for i in ret]\n\n if record_time:\n return ret, rtime\n else:\n return ret\n\n\ndef trusted_check_exec(code, inputs, entry_point):\n \"\"\"Check trusted_exec success.\"\"\"\n try:\n with time_limit(seconds=1.0):\n trusted_exec(code, inputs, entry_point)\n except Exception:\n return False\n return True\n\n\n\ndef extract_arguments(response, func_name, return_str=False):\n if 'rgument' in response: \n response = response.split('rgument', 1)[-1]\n arguments_str = response.split(f'{func_name}(', 1)[-1].split('\\n', 1)[0].rsplit(')', 1)[0]\n else: arguments_str = response.rsplit(f'{func_name}(', 1)[-1].split('\\n', 1)[0].rsplit(')', 1)[0]\n try: \n # Evaluate the arguments string to get the actual arguments \n # WARNING: Using eval() can be a security risk if the input is not trusted. \n # In a production environment, it's better to parse the string manually \n # or use a safe evaluation method. \n arguments = eval(arguments_str) \n if isinstance(arguments, tuple):\n if return_str: return list(arguments), arguments_str\n return list(arguments)\n if return_str: return [arguments], arguments_str\n return [arguments]\n except Exception as e: \n if return_str: return None, None\n return None \n\n\ndef he_extract_description(example):\n description = \"\\n\".join(\n [\n line.strip()\n for line in re.search(\n rf\"(?:\\\"\\\"\\\"|''')(.*?)(?:\\\"\\\"\\\"|''')\", example[\"prompt\"], re.DOTALL\n )\n .group(1)\n .split(\"\\n\")\n ]\n )\n return description\n\n\n\ndef extract_output(text):\n if 'utput' in text:\n text = text.rsplit('utput', 1)[-1]\n pattern = r'```(.*?)```'\n sol = re.findall(pattern, text, re.DOTALL)\n if len(sol) > 0:\n return sol[-1].strip('\\n')\n pattern = r'`(.*?)`'\n sol = re.findall(pattern, text, re.DOTALL)\n if len(sol) > 0:\n return sol[-1].strip('\\n').lstrip(' ')\n return None\n\n\ninput_completion_prompt = '''\nGiven a Python function `{signature}` to solve the following task:\n{description}\n\nWrite a valid input based on this task description, i.e., an acceptable input consistent with task description that a correct program should be able to execute.\nProvide a reasoning for your answer and present your response in the format below:\n```\n<reasoning>\n\nArguments: {entry_point}(<all arguments>)\n```\nNote that you MUST directly write ALL input arguments of the function in the correct order. Skip writing any names of arguments.\n'''\n\njoint_completion_prompt = '''\nYou are given a Python function `{signature}` to solve the following task:\n\n## Task\n{description}\n\nBased on the task description above, write a unit test that \n1. Is **valid** input based on the task description, i.e., an acceptable input consistent with task description that a correct program should be able to execute.\n2. The output enclosed in ```.``` and is **faithful** to the task description, i.e., the output of the unit test is consistent with what a correct program would yield.\n\nNote:\n- that you MUST directly write ALL input arguments of the function in the correct order. Skip writing any names of arguments.\n- you MUST enclose the unit test inputs and outputs in ```.```\n\nRespond in the format below:\n\n## Unit Test\n\n### Input Arguments\n< step-by-step reasoning for constructing a unit test that is valid as per the task description>\nArguments: ```{entry_point}(<all arguments>)```\n\n### Output\n< step-by-step reasoning for what a **correct** {entry_point} would execute to based on the task description and your input above. Make sure your data type of the final answer matches the expected output type of the function. >\nOutput: ```<your final answer>```\n'''\n\nseparate_unit_output_prompt = '''\nQuestion: \n\nAssume there exists a python function `{signature}` to solve the task: {description}\n\nA user calls this functions with the input: {entry_point}({unit_input}).\nBased on the task objective of the function and the user's input, what is the output of the function if implemented **correctly**?\n\nMake sure your data type of the final answer matches the expected output type of the function.\nFirst explain your reasoning and in the end format your final answer as:\n\nOutput: ```<your final answer>```\n'''\n\ndef make_signature(code, test_case): \n # Parse the code to get an Abstract Syntax Tree (AST) \n parsed_code = ast.parse(code) \n \n # A dictionary to hold function names and their signatures \n function_signatures = {} \n \n # Walk through the AST to find all function definitions \n for node in ast.walk(parsed_code): \n if isinstance(node, ast.FunctionDef): \n # Construct the function signature \n function_name = node.name \n args = [] \n for arg in node.args.args: \n # Check if the argument has a type hint \n if arg.annotation: \n # The annotation can be a Name (e.g., int) or more complex (e.g., List[int]). \n # We use ast.get_source_segment to extract the original source code for the annotation. \n arg_type = ': ' + ast.get_source_segment(code, arg.annotation) \n else: \n arg_type = '' \n args.append(f\"{arg.arg}{arg_type}\") \n signature = f\"{function_name}({', '.join(args)})\" \n function_signatures[function_name] = signature \n \n # Use regex to extract the function name from the test case\n for sign in function_signatures:\n if sign in test_case:\n break\n return function_signatures.get(sign)\n\n\nimport numpy as np\nfrom math import inf\n\ndef is_floats(x) -> bool:\n # check if it is float; List[float]; Tuple[float]\n if isinstance(x, float):\n return True\n if isinstance(x, (list, tuple)):\n return all(isinstance(i, float) for i in x)\n if isinstance(x, np.ndarray):\n return x.dtype == np.float64 or x.dtype == np.float32\n return False\n\n\ndef eval_exact_match(out, exp, atol=0, use_set = False):\n if isinstance(out, str) and isinstance(exp, str):\n if 'error' in out.lower() and 'error' in exp.lower: return True\n if atol == 0 and is_floats(exp):\n atol = 1e-6\n if use_set:\n try:\n out = set(out)\n exp = set(exp)\n except: pass\n if isinstance(out, str) and isinstance(exp, str) and 'Error' in out and \"Error\" in exp: return True\n elif out != exp and atol != 0:\n return np.allclose(out, exp, rtol=1e-07, atol=atol)\n else:\n return out == exp\n\n\nIMPORT_HELPER = {\n \"python\": [\n \"import math\",\n \"import re\",\n \"import sys\",\n \"import copy\",\n \"import datetime\",\n \"import itertools\",\n \"import collections\",\n \"import heapq\",\n \"import functools\",\n \"import hashlib\",\n \"import numpy\",\n \"import numpy as np\",\n \"import string\",\n \"from typing import *\",\n \"from collections import *\",\n \"from functools import *\"\n ]}\n\n\n\n@contextmanager \ndef memory_limit(max_mem): \n # Save the original memory limits \n soft, hard = resource.getrlimit(resource.RLIMIT_AS) \n # Set the new memory limit \n resource.setrlimit(resource.RLIMIT_AS, (max_mem, hard)) \n try: \n yield \n finally: \n # Restore the original memory limits \n resource.setrlimit(resource.RLIMIT_AS, (soft, hard)) \n\ndef unsafe_execute(\n dataset: str,\n entry_point: str,\n code: str,\n inputs: List,\n expected: List,\n verbose: bool = False,\n use_set: bool = False,\n):\n \n exec_globals = {}\n pass_rate = []\n exec_out = []\n try:\n with time_limit(2.0), memory_limit(256 * 1024 * 1024):\n with swallow_io():\n exec(code, exec_globals)\n if len(entry_point):\n fn = exec_globals[entry_point]\n except Exception as e:\n if not len(inputs) or not len(entry_point):\n if verbose: return _mapping[_TIMEOUT], 'Error: ' + str(e)\n return _mapping[_TIMEOUT]\n # time out there is some error in the code itself\n if verbose: return [_mapping[_TIMEOUT]]*len(inputs), ['Error: ' + str(e)]*len(inputs)\n return [_mapping[_TIMEOUT]]*len(inputs)\n if not len(inputs) or not len(entry_point):\n if verbose: return _mapping[_SUCCESS], _mapping[_SUCCESS]\n return _mapping[_SUCCESS]\n\n for i, inp in enumerate(inputs):\n try:\n with time_limit(2.0), memory_limit(256 * 1024 * 1024):\n with swallow_io():\n out = fn(*inp)\n exp = expected[i]\n exact_match = eval_exact_match(out, exp, use_set=use_set)\n exec_out.append(out)\n except Exception as e:\n pass_rate.append(_mapping[_TIMEOUT])\n exec_out.append('Error: ' + str(e))\n continue\n if not isinstance(exact_match, bool):\n try:\n exact_match = exact_match.all()\n except:\n pass_rate.append(_mapping[_FAILED])\n continue\n if exact_match:\n pass_rate.append(_mapping[_SUCCESS])\n else:\n pass_rate.append(_mapping[_FAILED])\n if verbose: \n assert len(exec_out) == len(pass_rate), print('Look: ', len(pass_rate), len(exec_out)) \n return pass_rate, exec_out\n return pass_rate \n\nhe_code_generation_prompt = '''\nWrite Python code to solve the task.\nWrite a Python function `{signature}` to solve the following problem: Present code in ```python```\n```python\n{prompt}\n```\n'''\n\ncode_generation_prompt = '''\nWrite Python code to solve the task.\nWrite a Python function `{signature}` to solve the following problem: Present code in ```python```\n```python\n\"\"\"\n{prompt}\n\"\"\"\n```\n'''\ndef extract_code(text):\n pattern = r'```python(.*?)```'\n sol = re.findall(pattern, text, re.DOTALL)\n if len(sol) > 0:\n return sol[0]\n \n pattern = r'```(.*?)```'\n sol = re.findall(pattern, text, re.DOTALL)\n if len(sol) > 0:\n return sol[0]\n \n return text.split('```')[0]\n\n\njoint_adv_prompt = '''\nYou are given a Python function `{signature}` to solve the following task:\n\n## Task\n{description}\n\n## Code Solution:\n```\n{code}\n```\nThe code solution I have provided to you is **incorrect**. Your job is to give feedback by generating a unit test that \n1. Is **valid** input based on the task description, i.e., an acceptable input consistent with task description that a correct program should be able to execute.\n2. The output enclosed in ```.``` and is **faithful** to the task description, i.e., the output of the unit test is consistent with what a correct program would return.\n3. **Breaks** the given code, i.e., does **not** execute to the **correct** output and brings out its mistakes and vulnerabilities.\n\nProvide a reasoning for your answer and identify a general hypothesis or rationale identifying the cause of error. Then provide input and output of the unit test consistent with the pattern (hypotheis) you have identified.\nNote:\n- that you MUST directly write ALL input arguments of the function in the correct order. Skip writing any names of arguments.\n- you MUST enclose the unit test inputs and outputs in ```.```\n\nRespond in the format below:\n\n## Hypothesis\n< step-by-step reasoning >\n\nError Pattern: <an identified pattern of inputs that yields erroneous or incorrect outputs>\n\n## Unit Test\n\n### Input Arguments\n< step-by-step reasoning for constructing a unit test that fits the error pattern identified above and is valid as per the task description>\nArguments: ```{entry_point}(<all arguments>)```\n\n### Output\n< step-by-step reasoning for what a **correct** {entry_point} would execute to based on the task description and your input above. Make sure your data type of the final answer matches the expected output type of the function. >\nOutput: ```<your final answer>```\n'''\n\nadv_input_completion_prompt = '''\nGiven a Python function `{signature}` to solve the following task:\n{description}\n\nCode Solution:\n```\n{code}\n```\n\nThe code solution I have provided to you is incorrect. Your job is to give feedback by generating a unit test input that \n1. Valid input based on the task description, i.e., an acceptable input consistent with task description that a correct program should be able to execute.\n2. Given code will NOT be able to solve and brings out its mistakes and vulnerabilities.\n\nProvide a reasoning for your answer and present your response in the format below:\n```\n<reasoning>\n\nArguments: {entry_point}(<all arguments>)\n```\nNote that you MUST directly write ALL input arguments of the function in the correct order. Skip writing any names of arguments.\n'''\n\ndef input_acceptance(code, test_input, func_name):\n test_setup = \"\\n\".join(IMPORT_HELPER[\"python\"]) + \"\\n\"\n snippet = test_setup + '\\n' + code \n return trusted_check_exec(snippet, test_input, func_name)\n\ndef check_attack_success(inst, code, unit_input):\n task_setup = \"\\n\".join(IMPORT_HELPER[\"python\"])\n decision = input_acceptance(task_setup + '\\n' + inst['canonical_solution'], [unit_input], inst['entry_point'])\n if decision:\n unit_out = trusted_exec(task_setup + '\\n' + inst[\"canonical_solution\"], [unit_input], inst['entry_point'])\n [attack_succ], [func_out] = unsafe_execute('mbpp', inst['entry_point'], task_setup + \"\\n\" + code, [unit_input], unit_out, use_set=' set(' in inst['test'], verbose=True )\n return attack_succ != \"pass\", unit_out[0], func_out\n else:\n return False, None, None\n \n\n@retry(wait=wait_random_exponential(min=1, max=30), stop=stop_after_attempt(15)) # retry=retry_if_not_exception_type(InvalidRequestError)\ndef deepseek_call(prompt):\n client = OpenAI(api_key=\"DEEP SEEK KEY\", base_url=\"https://api.deepseek.com\")\n if args.use_temp:\n temp = 0.8\n top_p = 0.9\n else:\n temp = 0\n top_p = 1\n\n messages = [\n {\"role\": \"system\", \"content\": \"You are a helpful coding assistant\"},\n {\"role\": \"user\", \"content\": prompt}\n ] \n response = client.chat.completions.create(\n model=\"deepseek-chat\",\n messages=messages,\n temperature=temp,\n max_tokens=2000,\n top_p=top_p,\n frequency_penalty=0.0,\n presence_penalty=0.0,\n timeout=40,\n stream=False\n )\n return response.choices[0].message.content\n\n@retry(wait=wait_random_exponential(min=1, max=30), stop=stop_after_attempt(25)) # retry=retry_if_not_exception_type(InvalidRequestError)\ndef llm_call(model, prompt):\n if args.use_temp:\n temp = 0.8\n top_p = 0.9\n else:\n temp = 0\n top_p = 1\n\n openai.api_key = \"YOUR OPENAI KEY\"\n ## If Using OpenAI via Azure\n openai.api_base = \"\" \n openai.api_type = \"azure\"\n openai.api_version = \"\"\n deployment_name = model\n\n messages = [\n {\"role\": \"user\", \"content\": prompt}\n ] \n response = openai.ChatCompletion.create(\n engine=deployment_name,\n messages=messages,\n temperature=temp,\n max_tokens=2000,\n top_p=top_p,\n frequency_penalty=0.0,\n presence_penalty=0.0,\n timeout=40,\n )\n choices = response[\"choices\"]\n completion_objs = [choice.message for choice in choices]\n completions = [completion.content for completion in completion_objs]\n return completions[0]"}, "34aa367f5f469eabcf4e330cc5ee10e90181c2c5": {"t": 1785766424.7152102, "n": 19529, "h": "31ce712e3623be93", "c": "import os\nimport re\nimport time\nimport openai\nimport argparse\nimport traceback\nimport fire\nimport random\nimport pandas as pd\nfrom tqdm import tqdm\nfrom typing import List\nfrom datasets import Dataset, load_dataset\nfrom termcolor import colored\nimport sys\nimport ast\nimport timeout_decorator\nimport numpy as np\nimport json\nimport subprocess\nfrom vllm import LLM, SamplingParams\nimport torch\nfrom transformers import AutoTokenizer\nfrom tenacity import (\n retry,\n retry_if_not_exception_type,\n stop_after_attempt, # type: ignore\n wait_random_exponential, # type: ignore\n)\nfrom multiprocessing import Array, Value, Process, Queue \nfrom collections import defaultdict, OrderedDict\nimport multiprocessing \nimport time \nfrom typing import List, Tuple, Any\n\nimport signal \nimport resource \nfrom contextlib import contextmanager\n\nHF_DATASET = {\n 'mbpp_plus_fix': 'datasets/MBPP+Fix_test.jsonl',\n 'mbpp_plus_fix_hard': 'datasets/MBPP+Fix_hard_test.jsonl',\n 'humaneval_plus_fix': 'datasets/HE+Fix_test.jsonl',\n }\n\n\nSTOP_WORDS = {\n \"llama3\": [\"<|eot_id|>\", \"<|end_of_text|>\"],\n \"llama3.1\": [\"<|eot_id|>\", \"<|end_of_text|>\"],\n \"tulu3\": [\"<|user|>\", \"<|assistant|>\", \"<|end_of_text|>\"],\n \"llama3_70b\": [\"<|eot_id|>\", \"<|end_of_text|>\"],\n \"gemma\": [\"<end_of_turn>\"],\n \"eurus\": [],\n \"qwen\": [\"<|endoftext|>\", \"<|im_end|>\"],\n \"big_qwen\": [\"<|endoftext|>\", \"<|im_end|>\"],\n \"deepseek\": [\"<｜end▁of▁sentence｜>\", \"### Response\", \"### Instruction:\"]\n}\n\nPASS = \"pass\"\nFAIL = \"fail\"\nTIMEOUT = \"timeout\"\n\n_SUCCESS = 0\n_FAILED = 1\n_TIMEOUT = 2\n_UNKNOWN = 3\n\n_mapping = {_SUCCESS: PASS, _FAILED: FAIL, _TIMEOUT: TIMEOUT, _UNKNOWN: None}\n\nHF_MODEL_NAMES = {\n \"llama3\": \"meta-llama/Meta-Llama-3-8B-Instruct\",\n \"llama3.1\": \"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n \"gemma\": \"google/gemma-7b-it\",\n \"deepseek\": \"deepseek-ai/deepseek-coder-7b-instruct-v1.5\",\n \"qwen\": \"Qwen/Qwen2.5-Coder-7B-Instruct\",\n \"tulu3\": \"allenai/Llama-3.1-Tulu-3-8B-SFT\",\n}\n\n\nimport time\nfrom copy import deepcopy\n\nimport contextlib\nimport faulthandler\nimport io\nimport os\nimport platform\nimport signal\nimport tempfile\nfrom typing import Optional\n\nclass TimeoutException(Exception):\n pass\n\n\nclass WriteOnlyStringIO(io.StringIO):\n \"\"\"StringIO that throws an exception when it's read from\"\"\"\n\n def read(self, *args, **kwargs):\n raise IOError\n\n def readline(self, *args, **kwargs):\n raise IOError\n\n def readlines(self, *args, **kwargs):\n raise IOError\n\n def readable(self, *args, **kwargs):\n \"\"\"Returns True if the IO object can be read.\"\"\"\n return False\n\nclass redirect_stdin(contextlib._RedirectStream): # type: ignore\n _stream = \"stdin\"\n\n@contextlib.contextmanager\ndef swallow_io():\n stream = WriteOnlyStringIO()\n with contextlib.redirect_stdout(stream):\n with contextlib.redirect_stderr(stream):\n with redirect_stdin(stream):\n yield\n\n\n@contextlib.contextmanager\ndef time_limit(seconds: float):\n def signal_handler(signum, frame):\n raise TimeoutException(\"Timed out!\")\n\n signal.setitimer(signal.ITIMER_REAL, seconds)\n signal.signal(signal.SIGALRM, signal_handler)\n try:\n yield\n finally:\n signal.setitimer(signal.ITIMER_REAL, 0)\n\n\ndef trusted_exec(code, inputs, entry_point, record_time=False, output_not_none=False):\n \"\"\"Execute trusted code in place.\"\"\"\n exec_globals = {}\n exec(code, exec_globals)\n fn = exec_globals[entry_point]\n # print(fn)\n rtime = []\n ret = []\n for inp in inputs:\n inp = deepcopy(inp)\n # print(inp)\n if record_time:\n start = time.time()\n # print(type(fn))\n ret.append(fn(*inp))\n rtime.append(time.time() - start)\n else:\n ret.append(fn(*inp))\n\n if output_not_none:\n ret = [i is not None for i in ret]\n\n if record_time:\n return ret, rtime\n else:\n return ret\n\n\ndef trusted_check_exec(code, inputs, entry_point):\n \"\"\"Check trusted_exec success.\"\"\"\n try:\n with time_limit(seconds=1.0):\n trusted_exec(code, inputs, entry_point)\n except Exception:\n return False\n return True\n\n\n\ndef extract_arguments(response, func_name, return_str=False):\n if 'rgument' in response: \n response = response.split('rgument', 1)[-1]\n arguments_str = response.split(f'{func_name}(', 1)[-1].split('\\n', 1)[0].rsplit(')', 1)[0]\n else: arguments_str = response.rsplit(f'{func_name}(', 1)[-1].split('\\n', 1)[0].rsplit(')', 1)[0]\n try: \n # Evaluate the arguments string to get the actual arguments \n # WARNING: Using eval() can be a security risk if the input is not trusted. \n # In a production environment, it's better to parse the string manually \n # or use a safe evaluation method. \n arguments = eval(arguments_str) \n if isinstance(arguments, tuple):\n if return_str: return list(arguments), arguments_str\n return list(arguments)\n if return_str: return [arguments], arguments_str\n return [arguments]\n except Exception as e: \n if return_str: return None, None\n return None \n\n\ndef he_extract_description(example):\n description = \"\\n\".join(\n [\n line.strip()\n for line in re.search(\n rf\"(?:\\\"\\\"\\\"|''')(.*?)(?:\\\"\\\"\\\"|''')\", example[\"prompt\"], re.DOTALL\n )\n .group(1)\n .split(\"\\n\")\n ]\n )\n return description\n\n\n\ndef extract_output(text):\n if 'utput' in text:\n text = text.rsplit('utput', 1)[-1]\n pattern = r'```(.*?)```'\n sol = re.findall(pattern, text, re.DOTALL)\n if len(sol) > 0:\n return sol[-1].strip('\\n')\n pattern = r'`(.*?)`'\n sol = re.findall(pattern, text, re.DOTALL)\n if len(sol) > 0:\n return sol[-1].strip('\\n').lstrip(' ')\n return None\n\n\ninput_completion_prompt = '''\nGiven a Python function `{signature}` to solve the following task:\n{description}\n\nWrite a valid input based on this task description, i.e., an acceptable input consistent with task description that a correct program should be able to execute.\nProvide a reasoning for your answer and present your response in the format below:\n```\n<reasoning>\n\nArguments: {entry_point}(<all arguments>)\n```\nNote that you MUST directly write ALL input arguments of the function in the correct order. Skip writing any names of arguments.\n'''\n\njoint_completion_prompt = '''\nYou are given a Python function `{signature}` to solve the following task:\n\n## Task\n{description}\n\nBased on the task description above, write a unit test that \n1. Is **valid** input based on the task description, i.e., an acceptable input consistent with task description that a correct program should be able to execute.\n2. The output enclosed in ```.``` and is **faithful** to the task description, i.e., the output of the unit test is consistent with what a correct program would yield.\n\nNote:\n- that you MUST directly write ALL input arguments of the function in the correct order. Skip writing any names of arguments.\n- you MUST enclose the unit test inputs and outputs in ```.```\n\nRespond in the format below:\n\n## Unit Test\n\n### Input Arguments\n< step-by-step reasoning for constructing a unit test that is valid as per the task description>\nArguments: ```{entry_point}(<all arguments>)```\n\n### Output\n< step-by-step reasoning for what a **correct** {entry_point} would execute to based on the task description and your input above. Make sure your data type of the final answer matches the expected output type of the function. >\nOutput: ```<your final answer>```\n'''\n\nseparate_unit_output_prompt = '''\nQuestion: \n\nAssume there exists a python function `{signature}` to solve the task: {description}\n\nA user calls this functions with the input: {entry_point}({unit_input}).\nBased on the task objective of the function and the user's input, what is the output of the function if implemented **correctly**?\n\nMake sure your data type of the final answer matches the expected output type of the function.\nFirst explain your reasoning and in the end format your final answer as:\n\nOutput: ```<your final answer>```\n'''\n\ndef make_signature(code, test_case): \n # Parse the code to get an Abstract Syntax Tree (AST) \n parsed_code = ast.parse(code) \n \n # A dictionary to hold function names and their signatures \n function_signatures = {} \n \n # Walk through the AST to find all function definitions \n for node in ast.walk(parsed_code): \n if isinstance(node, ast.FunctionDef): \n # Construct the function signature \n function_name = node.name \n args = [] \n for arg in node.args.args: \n # Check if the argument has a type hint \n if arg.annotation: \n # The annotation can be a Name (e.g., int) or more complex (e.g., List[int]). \n # We use ast.get_source_segment to extract the original source code for the annotation. \n arg_type = ': ' + ast.get_source_segment(code, arg.annotation) \n else: \n arg_type = '' \n args.append(f\"{arg.arg}{arg_type}\") \n signature = f\"{function_name}({', '.join(args)})\" \n function_signatures[function_name] = signature \n \n # Use regex to extract the function name from the test case\n for sign in function_signatures:\n if sign in test_case:\n break\n return function_signatures.get(sign)\n\n\nimport numpy as np\nfrom math import inf\n\ndef is_floats(x) -> bool:\n # check if it is float; List[float]; Tuple[float]\n if isinstance(x, float):\n return True\n if isinstance(x, (list, tuple)):\n return all(isinstance(i, float) for i in x)\n if isinstance(x, np.ndarray):\n return x.dtype == np.float64 or x.dtype == np.float32\n return False\n\n\ndef eval_exact_match(out, exp, atol=0, use_set = False):\n if isinstance(out, str) and isinstance(exp, str):\n if 'error' in out.lower() and 'error' in exp.lower: return True\n if atol == 0 and is_floats(exp):\n atol = 1e-6\n if use_set:\n try:\n out = set(out)\n exp = set(exp)\n except: pass\n if isinstance(out, str) and isinstance(exp, str) and 'Error' in out and \"Error\" in exp: return True\n elif out != exp and atol != 0:\n return np.allclose(out, exp, rtol=1e-07, atol=atol)\n else:\n return out == exp\n\n\nIMPORT_HELPER = {\n \"python\": [\n \"import math\",\n \"import re\",\n \"import sys\",\n \"import copy\",\n \"import datetime\",\n \"import itertools\",\n \"import collections\",\n \"import heapq\",\n \"import functools\",\n \"import hashlib\",\n \"import numpy\",\n \"import numpy as np\",\n \"import string\",\n \"from typing import *\",\n \"from collections import *\",\n \"from functools import *\"\n ]}\n\n\n\n@contextmanager \ndef memory_limit(max_mem): \n # Save the original memory limits \n soft, hard = resource.getrlimit(resource.RLIMIT_AS) \n # Set the new memory limit \n resource.setrlimit(resource.RLIMIT_AS, (max_mem, hard)) \n try: \n yield \n finally: \n # Restore the original memory limits \n resource.setrlimit(resource.RLIMIT_AS, (soft, hard)) \n\ndef unsafe_execute(\n dataset: str,\n entry_point: str,\n code: str,\n inputs: List,\n expected: List,\n verbose: bool = False,\n use_set: bool = False,\n):\n \n exec_globals = {}\n pass_rate = []\n exec_out = []\n try:\n with time_limit(2.0), memory_limit(256 * 1024 * 1024):\n with swallow_io():\n exec(code, exec_globals)\n if len(entry_point):\n fn = exec_globals[entry_point]\n except Exception as e:\n if not len(inputs) or not len(entry_point):\n if verbose: return _mapping[_TIMEOUT], 'Error: ' + str(e)\n return _mapping[_TIMEOUT]\n # time out there is some error in the code itself\n if verbose: return [_mapping[_TIMEOUT]]*len(inputs), ['Error: ' + str(e)]*len(inputs)\n return [_mapping[_TIMEOUT]]*len(inputs)\n if not len(inputs) or not len(entry_point):\n if verbose: return _mapping[_SUCCESS], _mapping[_SUCCESS]\n return _mapping[_SUCCESS]\n\n for i, inp in enumerate(inputs):\n try:\n with time_limit(2.0), memory_limit(256 * 1024 * 1024):\n with swallow_io():\n out = fn(*inp)\n exp = expected[i]\n exact_match = eval_exact_match(out, exp, use_set=use_set)\n exec_out.append(out)\n except Exception as e:\n pass_rate.append(_mapping[_TIMEOUT])\n exec_out.append('Error: ' + str(e))\n continue\n if not isinstance(exact_match, bool):\n try:\n exact_match = exact_match.all()\n except:\n pass_rate.append(_mapping[_FAILED])\n continue\n if exact_match:\n pass_rate.append(_mapping[_SUCCESS])\n else:\n pass_rate.append(_mapping[_FAILED])\n if verbose: \n assert len(exec_out) == len(pass_rate), print('Look: ', len(pass_rate), len(exec_out)) \n return pass_rate, exec_out\n return pass_rate \n\nhe_code_generation_prompt = '''\nWrite Python code to solve the task.\nWrite a Python function `{signature}` to solve the following problem: Present code in ```python```\n```python\n{prompt}\n```\n'''\n\ncode_generation_prompt = '''\nWrite Python code to solve the task.\nWrite a Python function `{signature}` to solve the following problem: Present code in ```python```\n```python\n\"\"\"\n{prompt}\n\"\"\"\n```\n'''\ndef extract_code(text):\n pattern = r'```python(.*?)```'\n sol = re.findall(pattern, text, re.DOTALL)\n if len(sol) > 0:\n return sol[0]\n \n pattern = r'```(.*?)```'\n sol = re.findall(pattern, text, re.DOTALL)\n if len(sol) > 0:\n return sol[0]\n \n return text.split('```')[0]\n\n\njoint_adv_prompt = '''\nYou are given a Python function `{signature}` to solve the following task:\n\n## Task\n{description}\n\n## Code Solution:\n```\n{code}\n```\nThe code solution I have provided to you is **incorrect**. Your job is to give feedback by generating a unit test that \n1. Is **valid** input based on the task description, i.e., an acceptable input consistent with task description that a correct program should be able to execute.\n2. The output enclosed in ```.``` and is **faithful** to the task description, i.e., the output of the unit test is consistent with what a correct program would return.\n3. **Breaks** the given code, i.e., does **not** execute to the **correct** output and brings out its mistakes and vulnerabilities.\n\nProvide a reasoning for your answer and identify a general hypothesis or rationale identifying the cause of error. Then provide input and output of the unit test consistent with the pattern (hypotheis) you have identified.\nNote:\n- that you MUST directly write ALL input arguments of the function in the correct order. Skip writing any names of arguments.\n- you MUST enclose the unit test inputs and outputs in ```.```\n\nRespond in the format below:\n\n## Hypothesis\n< step-by-step reasoning >\n\nError Pattern: <an identified pattern of inputs that yields erroneous or incorrect outputs>\n\n## Unit Test\n\n### Input Arguments\n< step-by-step reasoning for constructing a unit test that fits the error pattern identified above and is valid as per the task description>\nArguments: ```{entry_point}(<all arguments>)```\n\n### Output\n< step-by-step reasoning for what a **correct** {entry_point} would execute to based on the task description and your input above. Make sure your data type of the final answer matches the expected output type of the function. >\nOutput: ```<your final answer>```\n'''\n\nadv_input_completion_prompt = '''\nGiven a Python function `{signature}` to solve the following task:\n{description}\n\nCode Solution:\n```\n{code}\n```\n\nThe code solution I have provided to you is incorrect. Your job is to give feedback by generating a unit test input that \n1. Valid input based on the task description, i.e., an acceptable input consistent with task description that a correct program should be able to execute.\n2. Given code will NOT be able to solve and brings out its mistakes and vulnerabilities.\n\nProvide a reasoning for your answer and present your response in the format below:\n```\n<reasoning>\n\nArguments: {entry_point}(<all arguments>)\n```\nNote that you MUST directly write ALL input arguments of the function in the correct order. Skip writing any names of arguments.\n'''\n\ndef input_acceptance(code, test_input, func_name):\n test_setup = \"\\n\".join(IMPORT_HELPER[\"python\"]) + \"\\n\"\n snippet = test_setup + '\\n' + code \n return trusted_check_exec(snippet, test_input, func_name)\n\ndef check_attack_success(inst, code, unit_input):\n task_setup = \"\\n\".join(IMPORT_HELPER[\"python\"])\n decision = input_acceptance(task_setup + '\\n' + inst['canonical_solution'], [unit_input], inst['entry_point'])\n if decision:\n unit_out = trusted_exec(task_setup + '\\n' + inst[\"canonical_solution\"], [unit_input], inst['entry_point'])\n [attack_succ], [func_out] = unsafe_execute('mbpp', inst['entry_point'], task_setup + \"\\n\" + code, [unit_input], unit_out, use_set=' set(' in inst['test'], verbose=True )\n return attack_succ != \"pass\", unit_out[0], func_out\n else:\n return False, None, None\n \n\n@retry(wait=wait_random_exponential(min=1, max=30), stop=stop_after_attempt(15)) # retry=retry_if_not_exception_type(InvalidRequestError)\ndef deepseek_call(prompt):\n client = OpenAI(api_key=\"DEEP SEEK KEY\", base_url=\"https://api.deepseek.com\")\n if args.use_temp:\n temp = 0.8\n top_p = 0.9\n else:\n temp = 0\n top_p = 1\n\n messages = [\n {\"role\": \"system\", \"content\": \"You are a helpful coding assistant\"},\n {\"role\": \"user\", \"content\": prompt}\n ] \n response = client.chat.completions.create(\n model=\"deepseek-chat\",\n messages=messages,\n temperature=temp,\n max_tokens=2000,\n top_p=top_p,\n frequency_penalty=0.0,\n presence_penalty=0.0,\n timeout=40,\n stream=False\n )\n return response.choices[0].message.content\n\n@retry(wait=wait_random_exponential(min=1, max=30), stop=stop_after_attempt(25)) # retry=retry_if_not_exception_type(InvalidRequestError)\ndef llm_call(model, prompt):\n if args.use_temp:\n temp = 0.8\n top_p = 0.9\n else:\n temp = 0\n top_p = 1\n\n openai.api_key = \"YOUR OPENAI KEY\"\n ## If Using OpenAI via Azure\n openai.api_base = \"\" \n openai.api_type = \"azure\"\n openai.api_version = \"\"\n deployment_name = model\n\n messages = [\n {\"role\": \"user\", \"content\": prompt}\n ] \n response = openai.ChatCompletion.create(\n engine=deployment_name,\n messages=messages,\n temperature=temp,\n max_tokens=2000,\n top_p=top_p,\n frequency_penalty=0.0,\n presence_penalty=0.0,\n timeout=40,\n )\n choices = response[\"choices\"]\n completion_objs = [choice.message for choice in choices]\n completions = [completion.content for completion in completion_objs]\n return completions[0]"}}}} |