1 line
11 KiB
JSON
1 line
11 KiB
JSON
{"repo": "bytedance/sa2va", "files": {"sa2va_eval/vlmeval/api/sensechat_vision.py": {"91ff6237d5bfb80a8fe09f251505562e2ffab7c9": {"t": 1785768236.623487, "n": 10806, "h": "7a999752c14cf171", "c": "from vlmeval.smp import *\nfrom vlmeval.api.base import BaseAPI\nfrom vlmeval.dataset import img_root_map\nfrom vlmeval.dataset import DATASET_TYPE\n\n\nclass SenseChatVisionWrapper(BaseAPI):\n\n is_api: bool = True\n\n def __init__(self,\n model: str = 'SenseChat-5-Vision',\n retry: int = 5,\n wait: int = 5,\n ak: str = None,\n sk: str = None,\n verbose: bool = True,\n system_prompt: str = None,\n max_tokens: int = 1024,\n proxy: str = None,\n **kwargs):\n\n self.model = model\n self.fail_msg = 'Failed to obtain answer via API. '\n self.ak = os.environ.get('SENSECHAT_AK', None) if ak is None else ak\n self.sk = os.environ.get('SENSECHAT_SK', None) if sk is None else sk\n assert self.ak is not None and self.sk is not None\n self.max_new_tokens = max_tokens\n super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs)\n\n def dump_image(self, line, dataset):\n \"\"\"Dump the image(s) of the input line to the corresponding dataset folder.\n\n Args:\n line (line of pd.DataFrame): The raw input line.\n dataset (str): The name of the dataset.\n\n Returns:\n str | list[str]: The paths of the dumped images.\n \"\"\"\n ROOT = LMUDataRoot()\n assert isinstance(dataset, str)\n img_root = osp.join(ROOT, 'images', img_root_map[dataset] if dataset in img_root_map else dataset)\n os.makedirs(img_root, exist_ok=True)\n if 'image' in line:\n if isinstance(line['image'], list):\n tgt_path = []\n assert 'image_path' in line\n for img, im_name in zip(line['image'], line['image_path']):\n path = osp.join(img_root, im_name)\n if not read_ok(path):\n decode_base64_to_image_file(img, path)\n tgt_path.append(path)\n else:\n tgt_path = osp.join(img_root, f\"{line['index']}.jpg\")\n if not read_ok(tgt_path):\n decode_base64_to_image_file(line['image'], tgt_path)\n tgt_path = [tgt_path]\n else:\n assert 'image_path' in line\n tgt_path = toliststr(line['image_path'])\n\n return tgt_path\n\n def image_to_base64(self, image_path):\n import base64\n with open(image_path, 'rb') as image_file:\n encoded_string = base64.b64encode(image_file.read())\n return encoded_string.decode('utf-8')\n\n def encode_jwt_token(self, ak, sk):\n import jwt\n headers = {'alg': 'HS256', 'typ': 'JWT'}\n payload = {\n 'iss': ak,\n 'exp': int(time.time())\n + 1800, # 填写您期望的有效时间，此处示例代表当前时间+30分钟\n 'nbf': int(time.time()) - 5, # 填写您期望的生效时间，此处示例代表当前时间-5秒\n }\n token = jwt.encode(payload, sk, headers=headers)\n return token\n\n def use_custom_prompt(self, dataset):\n return True\n\n def build_multi_choice_prompt(self, line, dataset=None):\n question = line['question']\n hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None\n if hint is not None:\n question = hint + '\\n' + question\n\n options = {\n cand: line[cand]\n for cand in string.ascii_uppercase\n if cand in line and not pd.isna(line[cand])\n }\n for key, item in options.items():\n question += f'\\n{key}. {item}'\n prompt = question\n\n if len(options):\n prompt += '\\n请直接回答选项字母。' if cn_string(\n prompt) else \"\\nAnswer with the option's letter from the given choices directly.\"\n else:\n prompt += '\\n请直接回答问题。' if cn_string(prompt) else '\\nAnswer the question directly.'\n\n return prompt\n\n def build_prompt(self, line, dataset=None):\n assert self.use_custom_prompt(dataset)\n assert dataset is None or isinstance(dataset, str)\n\n tgt_path = self.dump_image(line, dataset)\n\n if dataset is not None and listinstr(['MME'], dataset):\n question = line['question']\n prompt = question + ' Answer the question using a single word or phrase.'\n elif dataset is not None and listinstr(['HallusionBench'], dataset):\n question = line['question']\n prompt = question + ' Please answer yes or no. Answer the question using a single word or phrase.'\n elif dataset is not None and DATASET_TYPE(dataset) == 'MCQ' and 'MMMU' not in dataset:\n prompt = self.build_multi_choice_prompt(line, dataset)\n elif dataset is not None and DATASET_TYPE(dataset) == 'VQA':\n if 'MathVista' in dataset:\n prompt = line['question']\n elif listinstr(['LLaVABench'], dataset):\n question = line['question']\n prompt = question + '\\nAnswer this question in detail.'\n elif listinstr(['MMVet'], dataset):\n prompt = line['question']\n else:\n question = line['question']\n prompt = question + '\\nAnswer the question using a single word or phrase.'\n elif dataset is not None and 'MMMU' in dataset:\n question = line['question']\n options = {\n cand: line[cand]\n for cand in string.ascii_uppercase\n if cand in line and not pd.isna(line[cand])\n }\n for key, item in options.items():\n question += f'\\n{key}. {item}'\n prompt = {\n 'multiple-choice': 'You are an expert in {}. Please solve the university-level {} examination question, which includes interleaved images and text. Your output should be divided into two parts: First, reason about the correct answer. Then write the answer in the following format where X is exactly one of the choices given by the problem: \"ANSWER: X\". If you are uncertain of the correct answer, guess the most likely one.', # noqa: E501\n 'open': 'You are an expert in {}. Please solve the university-level {} examination question, which includes interleaved images and text. Your output should be divided into two parts: First, reason about the correct answer. Then write the answer in the following format where X is only the answer and nothing else: \"ANSWER: X\"' # noqa: E501\n }\n subject = '_'.join(line['id'].split('_')[1:-1])\n prompt = prompt[line['question_type']].format(subject, subject) + '\\n' + question\n else:\n prompt = line['question']\n\n message = [dict(type='text', value=prompt)]\n message.extend([dict(type='image', value=s) for s in tgt_path])\n\n return message\n\n def message_to_promptimg(self, message, dataset=None):\n if dataset is None or listinstr(['MMMU', 'BLINK'], dataset):\n prompt = '\\n'.join([x['value'] for x in message if x['type'] == 'text'])\n image = [[x['value'] for x in message if x['type'] == 'image'][0]]\n else:\n prompt = '\\n'.join([x['value'] for x in message if x['type'] == 'text'])\n image = [x['value'] for x in message if x['type'] == 'image']\n return prompt, image\n\n def generate_inner(self, inputs, **kwargs) -> str:\n assert isinstance(inputs, str) or isinstance(inputs, list)\n inputs = [inputs] if isinstance(inputs, str) else inputs\n dataset = kwargs.get('dataset', None)\n\n if dataset is not None and listinstr(['ChartQA_TEST'], dataset):\n self.max_num = 12\n elif dataset is not None and listinstr(['DocVQA_VAL', 'DocVQA_TEST'], dataset):\n self.max_num = 18\n elif dataset is not None and listinstr(['InfoVQA_VAL', 'InfoVQA_TEST', 'OCRBench'], dataset):\n self.max_num = 24\n else:\n self.max_num = 6\n\n if dataset is None:\n pass\n elif listinstr(['AI2D_TEST'], dataset):\n self.max_new_tokens = 10\n elif 'MMMU' in dataset:\n self.max_new_tokens = 1024\n elif 'MMBench' in dataset:\n self.max_new_tokens = 100\n\n prompt, image = self.message_to_promptimg(message=inputs, dataset=dataset)\n\n url = 'https://api.sensenova.cn/v1/llm/chat-completions'\n api_secret_key = self.encode_jwt_token(self.ak, self.sk)\n\n content = [{\n 'image_base64': self.image_to_base64(item),\n 'image_file_id': '',\n 'image_url': '',\n 'text': '',\n 'text': '',\n 'type': 'image_base64'\n } for item in image]\n\n content.append({\n 'image_base64': '',\n 'image_file_id': '',\n 'image_url': '',\n 'text': prompt,\n 'type': 'text'\n })\n\n message = [{'content': content, 'role': 'user'}]\n\n data = {\n 'messages': message,\n 'max_new_tokens': self.max_new_tokens,\n 'model': self.model,\n 'stream': False,\n }\n headers = {\n 'Content-type': 'application/json',\n 'Authorization': 'Bearer ' + api_secret_key\n }\n\n response = requests.post(\n url,\n headers=headers,\n json=data,\n )\n request_id = response.headers['x-request-id']\n\n time.sleep(1)\n try:\n assert response.status_code == 200\n response = response.json()['data']['choices'][0]['message'].strip()\n if dataset is not None and 'MMMU' in dataset:\n response = response.split('ANSWER: ')[-1].strip()\n if self.verbose:\n self.logger.info(f'inputs: {inputs}\\nanswer: {response}')\n return 0, response, 'Succeeded! '\n except Exception as err:\n if self.verbose:\n self.logger.error('---------------------------ERROR---------------------------')\n self.logger.error(response.json())\n self.logger.error(err)\n self.logger.error('---------------------------request_id---------------------------' + request_id)\n self.logger.error(\n 'api error' + response.json()['error']['message']\n + str([input['value'] if input['type'] == 'image' else None for input in inputs])\n )\n self.logger.error(f'The input messages are {inputs}.')\n return -1, response.json()['error']['message'], ''\n\n\nclass SenseChatVisionAPI(SenseChatVisionWrapper):\n\n def generate(self, message, dataset=None):\n return super(SenseChatVisionAPI, self).generate(message, dataset=dataset)\n"}}}} |