1 line
11 KiB
JSON
1 line
11 KiB
JSON
{"repo": "chenyu-wang567/all-angles-bench", "files": {"VLMEvalkit/vlmeval/api/sensechat_vision.py": {"33b952312bcc5e823e866e0cfed10778a13b3d29": {"t": 1785768241.874868, "n": 10768, "h": "f65581e3de3c62fb", "c": "from vlmeval.smp import *\nfrom vlmeval.api.base import BaseAPI\nfrom vlmeval.dataset import img_root_map\nfrom vlmeval.dataset import DATASET_TYPE\n\n\nclass SenseChatVisionWrapper(BaseAPI):\n\n is_api: bool = True\n\n def __init__(self,\n model: str = 'SenseChat-5-Vision',\n retry: int = 5,\n wait: int = 5,\n ak: str = None,\n sk: str = None,\n verbose: bool = True,\n system_prompt: str = None,\n max_tokens: int = 1024,\n proxy: str = None,\n **kwargs):\n\n self.model = model\n self.fail_msg = 'Failed to obtain answer via API. '\n self.ak = os.environ.get('SENSECHAT_AK', None) if ak is None else ak\n self.sk = os.environ.get('SENSECHAT_SK', None) if sk is None else sk\n assert self.ak is not None and self.sk is not None\n self.max_new_tokens = max_tokens\n super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs)\n\n def dump_image(self, line, dataset):\n \"\"\"Dump the image(s) of the input line to the corresponding dataset folder.\n\n Args:\n line (line of pd.DataFrame): The raw input line.\n dataset (str): The name of the dataset.\n\n Returns:\n str | list[str]: The paths of the dumped images.\n \"\"\"\n ROOT = LMUDataRoot()\n assert isinstance(dataset, str)\n img_root = osp.join(ROOT, 'images', img_root_map(dataset))\n os.makedirs(img_root, exist_ok=True)\n if 'image' in line:\n if isinstance(line['image'], list):\n tgt_path = []\n assert 'image_path' in line\n for img, im_name in zip(line['image'], line['image_path']):\n path = osp.join(img_root, im_name)\n if not read_ok(path):\n decode_base64_to_image_file(img, path)\n tgt_path.append(path)\n else:\n tgt_path = osp.join(img_root, f\"{line['index']}.jpg\")\n if not read_ok(tgt_path):\n decode_base64_to_image_file(line['image'], tgt_path)\n tgt_path = [tgt_path]\n else:\n assert 'image_path' in line\n tgt_path = toliststr(line['image_path'])\n\n return tgt_path\n\n def image_to_base64(self, image_path):\n import base64\n with open(image_path, 'rb') as image_file:\n encoded_string = base64.b64encode(image_file.read())\n return encoded_string.decode('utf-8')\n\n def encode_jwt_token(self, ak, sk):\n import jwt\n headers = {'alg': 'HS256', 'typ': 'JWT'}\n payload = {\n 'iss': ak,\n 'exp': int(time.time())\n + 1800, # 填写您期望的有效时间，此处示例代表当前时间+30分钟\n 'nbf': int(time.time()) - 5, # 填写您期望的生效时间，此处示例代表当前时间-5秒\n }\n token = jwt.encode(payload, sk, headers=headers)\n return token\n\n def use_custom_prompt(self, dataset):\n return True\n\n def build_multi_choice_prompt(self, line, dataset=None):\n question = line['question']\n hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None\n if hint is not None:\n question = hint + '\\n' + question\n\n options = {\n cand: line[cand]\n for cand in string.ascii_uppercase\n if cand in line and not pd.isna(line[cand])\n }\n for key, item in options.items():\n question += f'\\n{key}. {item}'\n prompt = question\n\n if len(options):\n prompt += '\\n请直接回答选项字母。' if cn_string(\n prompt) else \"\\nAnswer with the option's letter from the given choices directly.\"\n else:\n prompt += '\\n请直接回答问题。' if cn_string(prompt) else '\\nAnswer the question directly.'\n\n return prompt\n\n def build_prompt(self, line, dataset=None):\n assert self.use_custom_prompt(dataset)\n assert dataset is None or isinstance(dataset, str)\n\n tgt_path = self.dump_image(line, dataset)\n\n if dataset is not None and listinstr(['MME'], dataset):\n question = line['question']\n prompt = question + ' Answer the question using a single word or phrase.'\n elif dataset is not None and listinstr(['HallusionBench'], dataset):\n question = line['question']\n prompt = question + ' Please answer yes or no. Answer the question using a single word or phrase.'\n elif dataset is not None and DATASET_TYPE(dataset) == 'MCQ' and 'MMMU' not in dataset:\n prompt = self.build_multi_choice_prompt(line, dataset)\n elif dataset is not None and DATASET_TYPE(dataset) == 'VQA':\n if 'MathVista' in dataset:\n prompt = line['question']\n elif listinstr(['LLaVABench'], dataset):\n question = line['question']\n prompt = question + '\\nAnswer this question in detail.'\n elif listinstr(['MMVet'], dataset):\n prompt = line['question']\n else:\n question = line['question']\n prompt = question + '\\nAnswer the question using a single word or phrase.'\n elif dataset is not None and 'MMMU' in dataset:\n question = line['question']\n options = {\n cand: line[cand]\n for cand in string.ascii_uppercase\n if cand in line and not pd.isna(line[cand])\n }\n for key, item in options.items():\n question += f'\\n{key}. {item}'\n prompt = {\n 'multiple-choice': 'Answer with carefully thought step by step. Apply the thinking process recursively at both macro and micro levels. Verify consistency of reasoning and look for potential flaws or gaps during thinking. When realize mistakes, explain why the previous thinking was incorrect, fix it and then continue thinking.\\n\\n', # noqa\n 'open': 'Answer with carefully thought step by step. Apply the thinking process recursively at both macro and micro levels. Verify consistency of reasoning and look for potential flaws or gaps during thinking. When realize mistakes, explain why the previous thinking was incorrect, fix it and then continue thinking.\\n\\n' # noqa\n }\n subject = '_'.join(line['id'].split('_')[1:-1])\n prompt = prompt[line['question_type']].format(subject, subject) + '\\n' + question\n else:\n prompt = line['question']\n\n message = [dict(type='text', value=prompt)]\n message.extend([dict(type='image', value=s) for s in tgt_path])\n\n return message\n\n def message_to_promptimg(self, message, dataset=None):\n if dataset is None or listinstr(['MMMU', 'BLINK'], dataset):\n prompt = '\\n'.join([x['value'] for x in message if x['type'] == 'text'])\n image = [[x['value'] for x in message if x['type'] == 'image'][0]]\n else:\n prompt = '\\n'.join([x['value'] for x in message if x['type'] == 'text'])\n image = [x['value'] for x in message if x['type'] == 'image']\n return prompt, image\n\n def generate_inner(self, inputs, **kwargs) -> str:\n assert isinstance(inputs, str) or isinstance(inputs, list)\n inputs = [inputs] if isinstance(inputs, str) else inputs\n dataset = kwargs.get('dataset', None)\n\n if dataset is not None and listinstr(['ChartQA_TEST','MathVista_MINI'], dataset):\n self.max_num = 12\n elif dataset is not None and listinstr(['DocVQA_VAL', 'DocVQA_TEST'], dataset):\n self.max_num = 18\n elif dataset is not None and listinstr(['InfoVQA_VAL', 'InfoVQA_TEST', 'OCRBench'], dataset):\n self.max_num = 24\n else:\n self.max_num = 6\n\n if dataset is None:\n pass\n elif listinstr(['AI2D_TEST'], dataset):\n self.max_new_tokens = 10\n elif 'MMMU' in dataset:\n self.max_new_tokens = 4096 # 1024\n elif 'MMBench' in dataset:\n self.max_new_tokens = 100\n elif 'MathVista_MINI' in dataset:\n self.max_new_tokens = 4096\n\n prompt, image = self.message_to_promptimg(message=inputs, dataset=dataset)\n\n url = 'https://api.sensenova.cn/v1/llm/chat-completions'\n api_secret_key = self.encode_jwt_token(self.ak, self.sk)\n\n content = [{\n 'image_base64': self.image_to_base64(item),\n 'image_file_id': '',\n 'image_url': '',\n 'text': '',\n 'text': '',\n 'type': 'image_base64'\n } for item in image]\n\n content.append({\n 'image_base64': '',\n 'image_file_id': '',\n 'image_url': '',\n 'text': prompt,\n 'type': 'text'\n })\n\n message = [{'content': content, 'role': 'user'}]\n\n data = {\n 'messages': message,\n 'max_new_tokens': self.max_new_tokens, # 1024\n 'temperature': 0,\n \"top_k\": 0,\n \"top_p\": 0.99,\n 'repetition_penalty': 1.05,\n 'model': self.model,\n 'stream': False,\n }\n headers = {\n 'Content-type': 'application/json',\n 'Authorization': 'Bearer ' + api_secret_key\n }\n\n response = requests.post(\n url,\n headers=headers,\n json=data,\n )\n request_id = response.headers['x-request-id']\n\n time.sleep(1)\n try:\n assert response.status_code == 200\n response = response.json()['data']['choices'][0]['message'].strip()\n if self.verbose:\n self.logger.info(f'inputs: {inputs}\\nanswer: {response}')\n return 0, response, 'Succeeded! '\n except Exception as err:\n if self.verbose:\n self.logger.error('---------------------------ERROR---------------------------')\n self.logger.error(response.json())\n self.logger.error(err)\n self.logger.error('---------------------------request_id---------------------------' + request_id)\n self.logger.error(\n 'api error' + response.json()['error']['message']\n + str([input['value'] if input['type'] == 'image' else None for input in inputs])\n )\n self.logger.error(f'The input messages are {inputs}.')\n return -1, response.json()['error']['message'], ''\n\n\nclass SenseChatVisionAPI(SenseChatVisionWrapper):\n\n def generate(self, message, dataset=None):\n return super(SenseChatVisionAPI, self).generate(message, dataset=dataset)\n"}}}} |