1 line
22 KiB
JSON
1 line
22 KiB
JSON
{"repo": "anitarau/surgbenchkit", "files": {"vlmeval/api/gpt.py": {"29d17c9d279bf1f96c054e0cb9b9cdcdd378fc76": {"t": 1785674099.9888253, "n": 10551, "h": "d507ef813166796b", "c": "from ..smp import *\nimport os\nimport sys\nfrom .base import BaseAPI\n\nAPIBASES = {\n 'OFFICIAL': 'https://api.openai.com/v1/chat/completions',\n}\n\n\ndef GPT_context_window(model):\n length_map = {\n 'gpt-4': 8192,\n 'gpt-4-0613': 8192,\n 'gpt-4-turbo-preview': 128000,\n 'gpt-4-1106-preview': 128000,\n 'gpt-4-0125-preview': 128000,\n 'gpt-4-vision-preview': 128000,\n 'gpt-4-turbo': 128000,\n 'gpt-4-turbo-2024-04-09': 128000,\n 'gpt-3.5-turbo': 16385,\n 'gpt-3.5-turbo-0125': 16385,\n 'gpt-3.5-turbo-1106': 16385,\n 'gpt-3.5-turbo-instruct': 4096,\n }\n if model in length_map:\n return length_map[model]\n else:\n return 128000\n\n\nclass OpenAIWrapper(BaseAPI):\n\n is_api: bool = True\n\n def __init__(self,\n model: str = 'gpt-3.5-turbo-0613',\n retry: int = 5,\n wait: int = 5,\n key: str = None,\n verbose: bool = False,\n system_prompt: str = None,\n temperature: float = 0,\n timeout: int = 60,\n api_base: str = None,\n max_tokens: int = 2048,\n img_size: int = 512,\n img_detail: str = 'low',\n use_azure: bool = False,\n **kwargs):\n\n self.model = model\n self.cur_idx = 0\n self.fail_msg = 'Failed to obtain answer via API. '\n self.max_tokens = max_tokens\n self.temperature = temperature\n self.use_azure = use_azure\n\n if 'step' in model:\n env_key = os.environ.get('STEPAI_API_KEY', '')\n if key is None:\n key = env_key\n elif 'yi-vision' in model:\n env_key = os.environ.get('YI_API_KEY', '')\n if key is None:\n key = env_key\n elif 'internvl2-pro' in model:\n env_key = os.environ.get('InternVL2_PRO_KEY', '')\n if key is None:\n key = env_key\n elif 'abab' in model:\n env_key = os.environ.get('MiniMax_API_KEY', '')\n if key is None:\n key = env_key\n elif 'moonshot' in model:\n env_key = os.environ.get('MOONSHOT_API_KEY', '')\n if key is None:\n key = env_key\n elif 'grok' in model:\n env_key = os.environ.get('XAI_API_KEY', '')\n if key is None:\n key = env_key\n else:\n if use_azure:\n env_key = os.environ.get('AZURE_OPENAI_API_KEY', None)\n assert env_key is not None, 'Please set the environment variable AZURE_OPENAI_API_KEY. '\n\n if key is None:\n key = env_key\n assert isinstance(key, str), (\n 'Please set the environment variable AZURE_OPENAI_API_KEY to your openai key. '\n )\n else:\n env_key = os.environ.get('OPENAI_API_KEY', '')\n if key is None:\n key = env_key\n assert isinstance(key, str) and key.startswith('sk-'), (\n f'Illegal openai_key {key}. '\n 'Please set the environment variable OPENAI_API_KEY to your openai key. '\n )\n\n self.key = key\n assert img_size > 0 or img_size == -1\n self.img_size = img_size\n assert img_detail in ['high', 'low']\n self.img_detail = img_detail\n self.timeout = timeout\n self.o1_model = 'o1' in model or 'o3' in model\n\n super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs)\n\n if use_azure:\n api_base_template = (\n '{endpoint}openai/deployments/{deployment_name}/chat/completions?api-version={api_version}'\n )\n endpoint = os.getenv('AZURE_OPENAI_ENDPOINT', None)\n assert endpoint is not None, 'Please set the environment variable AZURE_OPENAI_ENDPOINT. '\n deployment_name = os.getenv('AZURE_OPENAI_DEPLOYMENT_NAME', None)\n assert deployment_name is not None, 'Please set the environment variable AZURE_OPENAI_DEPLOYMENT_NAME. '\n api_version = os.getenv('OPENAI_API_VERSION', None)\n assert api_version is not None, 'Please set the environment variable OPENAI_API_VERSION. '\n\n self.api_base = api_base_template.format(\n endpoint=os.getenv('AZURE_OPENAI_ENDPOINT'),\n deployment_name=os.getenv('AZURE_OPENAI_DEPLOYMENT_NAME'),\n api_version=os.getenv('OPENAI_API_VERSION')\n )\n else:\n if api_base is None:\n if 'OPENAI_API_BASE' in os.environ and os.environ['OPENAI_API_BASE'] != '':\n self.logger.info('Environment variable OPENAI_API_BASE is set. Will use it as api_base. ')\n api_base = os.environ['OPENAI_API_BASE']\n else:\n api_base = 'OFFICIAL'\n\n assert api_base is not None\n\n if api_base in APIBASES:\n self.api_base = APIBASES[api_base]\n elif api_base.startswith('http'):\n self.api_base = api_base\n else:\n self.logger.error('Unknown API Base. ')\n raise NotImplementedError\n\n self.logger.info(f'Using API Base: {self.api_base}; API Key: {self.key}')\n\n # inputs can be a lvl-2 nested list: [content1, content2, content3, ...]\n # content can be a string or a list of image & text\n def prepare_itlist(self, inputs):\n assert np.all([isinstance(x, dict) for x in inputs])\n has_images = np.sum([x['type'] == 'image' for x in inputs])\n if has_images:\n content_list = []\n for msg in inputs:\n if msg['type'] == 'text':\n content_list.append(dict(type='text', text=msg['value']))\n elif msg['type'] == 'image':\n from PIL import Image\n img = Image.open(msg['value'])\n b64 = encode_image_to_base64(img, target_size=self.img_size)\n img_struct = dict(url=f'data:image/jpeg;base64,{b64}', detail=self.img_detail)\n content_list.append(dict(type='image_url', image_url=img_struct))\n else:\n assert all([x['type'] == 'text' for x in inputs])\n text = '\\n'.join([x['value'] for x in inputs])\n content_list = [dict(type='text', text=text)]\n return content_list\n\n def prepare_inputs(self, inputs):\n input_msgs = []\n if self.system_prompt is not None:\n input_msgs.append(dict(role='system', content=self.system_prompt))\n assert isinstance(inputs, list) and isinstance(inputs[0], dict)\n assert np.all(['type' in x for x in inputs]) or np.all(['role' in x for x in inputs]), inputs\n if 'role' in inputs[0]:\n assert inputs[-1]['role'] == 'user', inputs[-1]\n for item in inputs:\n input_msgs.append(dict(role=item['role'], content=self.prepare_itlist(item['content'])))\n else:\n input_msgs.append(dict(role='user', content=self.prepare_itlist(inputs)))\n return input_msgs\n\n def generate_inner(self, inputs, **kwargs) -> str:\n input_msgs = self.prepare_inputs(inputs)\n temperature = kwargs.pop('temperature', self.temperature)\n max_tokens = kwargs.pop('max_tokens', self.max_tokens)\n\n # Will send request if use Azure, dk how to use openai client for it\n if self.use_azure:\n headers = {'Content-Type': 'application/json', 'api-key': self.key}\n elif 'internvl2-pro' in self.model:\n headers = {'Content-Type': 'application/json', 'Authorization': self.key}\n else:\n headers = {'Content-Type': 'application/json', 'Authorization': f'Bearer {self.key}'}\n payload = dict(\n model=self.model,\n messages=input_msgs,\n # max_tokens=max_tokens,\n n=1,\n temperature=temperature,\n **kwargs)\n if self.o1_model:\n payload['max_completion_tokens'] = max_tokens\n payload.pop('temperature')\n else:\n payload['max_tokens'] = max_tokens\n \n response = requests.post(\n self.api_base,\n headers=headers, data=json.dumps(payload), timeout=self.timeout * 1.1)\n ret_code = response.status_code\n ret_code = 0 if (200 <= int(ret_code) < 300) else ret_code\n answer = self.fail_msg\n try:\n resp_struct = json.loads(response.text)\n answer = resp_struct['choices'][0]['message']['content'].strip()\n except Exception as err:\n if self.verbose:\n self.logger.error(f'{type(err)}: {err}')\n self.logger.error(response.text if hasattr(response, 'text') else response)\n\n return ret_code, answer, response\n\n def get_image_token_len(self, img_path, detail='low'):\n import math\n if detail == 'low':\n return 85\n\n im = Image.open(img_path)\n height, width = im.size\n if width > 1024 or height > 1024:\n if width > height:\n height = int(height * 1024 / width)\n width = 1024\n else:\n width = int(width * 1024 / height)\n height = 1024\n\n h = math.ceil(height / 512)\n w = math.ceil(width / 512)\n total = 85 + 170 * h * w\n return total\n\n def get_token_len(self, inputs) -> int:\n import tiktoken\n try:\n enc = tiktoken.encoding_for_model(self.model)\n except Exception as err:\n if 'gpt' in self.model.lower():\n if self.verbose:\n self.logger.warning(f'{type(err)}: {err}')\n enc = tiktoken.encoding_for_model('gpt-4')\n else:\n return 0\n assert isinstance(inputs, list)\n tot = 0\n for item in inputs:\n if 'role' in item:\n tot += self.get_token_len(item['content'])\n elif item['type'] == 'text':\n tot += len(enc.encode(item['value']))\n elif item['type'] == 'image':\n tot += self.get_image_token_len(item['value'], detail=self.img_detail)\n return tot\n\n\nclass GPT4V(OpenAIWrapper):\n\n def generate(self, message, dataset=None):\n return super(GPT4V, self).generate(message)\n"}}, "vlmeval/api/sensechat_vision.py": {"29d17c9d279bf1f96c054e0cb9b9cdcdd378fc76": {"t": 1785809863.9753602, "n": 10768, "h": "cacc51441695224f", "c": "from vlmeval.smp import *\nfrom vlmeval.api.base import BaseAPI\nfrom vlmeval.dataset import img_root_map\nfrom vlmeval.dataset import DATASET_TYPE\n\n\nclass SenseChatVisionWrapper(BaseAPI):\n\n is_api: bool = True\n\n def __init__(self,\n model: str = 'SenseChat-5-Vision',\n retry: int = 5,\n wait: int = 5,\n ak: str = None,\n sk: str = None,\n verbose: bool = True,\n system_prompt: str = None,\n max_tokens: int = 2048,\n proxy: str = None,\n **kwargs):\n\n self.model = model\n self.fail_msg = 'Failed to obtain answer via API. '\n self.ak = os.environ.get('SENSECHAT_AK', None) if ak is None else ak\n self.sk = os.environ.get('SENSECHAT_SK', None) if sk is None else sk\n assert self.ak is not None and self.sk is not None\n self.max_new_tokens = max_tokens\n super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs)\n\n def dump_image(self, line, dataset):\n \"\"\"Dump the image(s) of the input line to the corresponding dataset folder.\n\n Args:\n line (line of pd.DataFrame): The raw input line.\n dataset (str): The name of the dataset.\n\n Returns:\n str | list[str]: The paths of the dumped images.\n \"\"\"\n ROOT = LMUDataRoot()\n assert isinstance(dataset, str)\n img_root = osp.join(ROOT, 'images', img_root_map(dataset))\n os.makedirs(img_root, exist_ok=True)\n if 'image' in line:\n if isinstance(line['image'], list):\n tgt_path = []\n assert 'image_path' in line\n for img, im_name in zip(line['image'], line['image_path']):\n path = osp.join(img_root, im_name)\n if not read_ok(path):\n decode_base64_to_image_file(img, path)\n tgt_path.append(path)\n else:\n tgt_path = osp.join(img_root, f\"{line['index']}.jpg\")\n if not read_ok(tgt_path):\n decode_base64_to_image_file(line['image'], tgt_path)\n tgt_path = [tgt_path]\n else:\n assert 'image_path' in line\n tgt_path = toliststr(line['image_path'])\n\n return tgt_path\n\n def image_to_base64(self, image_path):\n import base64\n with open(image_path, 'rb') as image_file:\n encoded_string = base64.b64encode(image_file.read())\n return encoded_string.decode('utf-8')\n\n def encode_jwt_token(self, ak, sk):\n import jwt\n headers = {'alg': 'HS256', 'typ': 'JWT'}\n payload = {\n 'iss': ak,\n 'exp': int(time.time())\n + 1800, # 填写您期望的有效时间，此处示例代表当前时间+30分钟\n 'nbf': int(time.time()) - 5, # 填写您期望的生效时间，此处示例代表当前时间-5秒\n }\n token = jwt.encode(payload, sk, headers=headers)\n return token\n\n def use_custom_prompt(self, dataset):\n return True\n\n def build_multi_choice_prompt(self, line, dataset=None):\n question = line['question']\n hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None\n if hint is not None:\n question = hint + '\\n' + question\n\n options = {\n cand: line[cand]\n for cand in string.ascii_uppercase\n if cand in line and not pd.isna(line[cand])\n }\n for key, item in options.items():\n question += f'\\n{key}. {item}'\n prompt = question\n\n if len(options):\n prompt += '\\n请直接回答选项字母。' if cn_string(\n prompt) else \"\\nAnswer with the option's letter from the given choices directly.\"\n else:\n prompt += '\\n请直接回答问题。' if cn_string(prompt) else '\\nAnswer the question directly.'\n\n return prompt\n\n def build_prompt(self, line, dataset=None):\n assert self.use_custom_prompt(dataset)\n assert dataset is None or isinstance(dataset, str)\n\n tgt_path = self.dump_image(line, dataset)\n\n if dataset is not None and listinstr(['MME'], dataset):\n question = line['question']\n prompt = question + ' Answer the question using a single word or phrase.'\n elif dataset is not None and listinstr(['HallusionBench'], dataset):\n question = line['question']\n prompt = question + ' Please answer yes or no. Answer the question using a single word or phrase.'\n elif dataset is not None and DATASET_TYPE(dataset) == 'MCQ' and 'MMMU' not in dataset:\n prompt = self.build_multi_choice_prompt(line, dataset)\n elif dataset is not None and DATASET_TYPE(dataset) == 'VQA':\n if 'MathVista' in dataset:\n prompt = line['question']\n elif listinstr(['LLaVABench'], dataset):\n question = line['question']\n prompt = question + '\\nAnswer this question in detail.'\n elif listinstr(['MMVet'], dataset):\n prompt = line['question']\n else:\n question = line['question']\n prompt = question + '\\nAnswer the question using a single word or phrase.'\n elif dataset is not None and 'MMMU' in dataset:\n question = line['question']\n options = {\n cand: line[cand]\n for cand in string.ascii_uppercase\n if cand in line and not pd.isna(line[cand])\n }\n for key, item in options.items():\n question += f'\\n{key}. {item}'\n prompt = {\n 'multiple-choice': 'Answer with carefully thought step by step. Apply the thinking process recursively at both macro and micro levels. Verify consistency of reasoning and look for potential flaws or gaps during thinking. When realize mistakes, explain why the previous thinking was incorrect, fix it and then continue thinking.\\n\\n', # noqa\n 'open': 'Answer with carefully thought step by step. Apply the thinking process recursively at both macro and micro levels. Verify consistency of reasoning and look for potential flaws or gaps during thinking. When realize mistakes, explain why the previous thinking was incorrect, fix it and then continue thinking.\\n\\n' # noqa\n }\n subject = '_'.join(line['id'].split('_')[1:-1])\n prompt = prompt[line['question_type']].format(subject, subject) + '\\n' + question\n else:\n prompt = line['question']\n\n message = [dict(type='text', value=prompt)]\n message.extend([dict(type='image', value=s) for s in tgt_path])\n\n return message\n\n def message_to_promptimg(self, message, dataset=None):\n if dataset is None or listinstr(['MMMU', 'BLINK'], dataset):\n prompt = '\\n'.join([x['value'] for x in message if x['type'] == 'text'])\n image = [[x['value'] for x in message if x['type'] == 'image'][0]]\n else:\n prompt = '\\n'.join([x['value'] for x in message if x['type'] == 'text'])\n image = [x['value'] for x in message if x['type'] == 'image']\n return prompt, image\n\n def generate_inner(self, inputs, **kwargs) -> str:\n assert isinstance(inputs, str) or isinstance(inputs, list)\n inputs = [inputs] if isinstance(inputs, str) else inputs\n dataset = kwargs.get('dataset', None)\n\n if dataset is not None and listinstr(['ChartQA_TEST','MathVista_MINI'], dataset):\n self.max_num = 12\n elif dataset is not None and listinstr(['DocVQA_VAL', 'DocVQA_TEST'], dataset):\n self.max_num = 18\n elif dataset is not None and listinstr(['InfoVQA_VAL', 'InfoVQA_TEST', 'OCRBench'], dataset):\n self.max_num = 24\n else:\n self.max_num = 6\n\n if dataset is None:\n pass\n elif listinstr(['AI2D_TEST'], dataset):\n self.max_new_tokens = 10\n elif 'MMMU' in dataset:\n self.max_new_tokens = 4096 # 1024\n elif 'MMBench' in dataset:\n self.max_new_tokens = 100\n elif 'MathVista_MINI' in dataset:\n self.max_new_tokens = 4096\n\n prompt, image = self.message_to_promptimg(message=inputs, dataset=dataset)\n\n url = 'https://api.sensenova.cn/v1/llm/chat-completions'\n api_secret_key = self.encode_jwt_token(self.ak, self.sk)\n\n content = [{\n 'image_base64': self.image_to_base64(item),\n 'image_file_id': '',\n 'image_url': '',\n 'text': '',\n 'text': '',\n 'type': 'image_base64'\n } for item in image]\n\n content.append({\n 'image_base64': '',\n 'image_file_id': '',\n 'image_url': '',\n 'text': prompt,\n 'type': 'text'\n })\n\n message = [{'content': content, 'role': 'user'}]\n\n data = {\n 'messages': message,\n 'max_new_tokens': self.max_new_tokens, # 1024\n 'temperature': 0,\n \"top_k\": 0,\n \"top_p\": 0.99,\n 'repetition_penalty': 1.05,\n 'model': self.model,\n 'stream': False,\n }\n headers = {\n 'Content-type': 'application/json',\n 'Authorization': 'Bearer ' + api_secret_key\n }\n\n response = requests.post(\n url,\n headers=headers,\n json=data,\n )\n request_id = response.headers['x-request-id']\n\n time.sleep(1)\n try:\n assert response.status_code == 200\n response = response.json()['data']['choices'][0]['message'].strip()\n if self.verbose:\n self.logger.info(f'inputs: {inputs}\\nanswer: {response}')\n return 0, response, 'Succeeded! '\n except Exception as err:\n if self.verbose:\n self.logger.error('---------------------------ERROR---------------------------')\n self.logger.error(response.json())\n self.logger.error(err)\n self.logger.error('---------------------------request_id---------------------------' + request_id)\n self.logger.error(\n 'api error' + response.json()['error']['message']\n + str([input['value'] if input['type'] == 'image' else None for input in inputs])\n )\n self.logger.error(f'The input messages are {inputs}.')\n return -1, response.json()['error']['message'], ''\n\n\nclass SenseChatVisionAPI(SenseChatVisionWrapper):\n\n def generate(self, message, dataset=None):\n return super(SenseChatVisionAPI, self).generate(message, dataset=dataset)\n"}}}} |