1 line
12 KiB
JSON
1 line
12 KiB
JSON
{"repo": "activesky/mquant", "files": {"third/VLMEvalKit/vlmeval/api/sensechat_vision.py": {"d6d51d54bef42c4a35dfc279bd666bcff5d3a7d7": {"t": 1785768274.8533888, "n": 11520, "h": "3cd694831b18e01c", "c": "from vlmeval.smp import *\nfrom vlmeval.api.base import BaseAPI\nfrom vlmeval.dataset import img_root_map\nfrom vlmeval.dataset import DATASET_TYPE\n\n\nclass SenseChatVisionWrapper(BaseAPI):\n\n is_api: bool = True\n\n def __init__(\n self,\n model: str = \"SenseChat-5-Vision\",\n retry: int = 5,\n wait: int = 5,\n ak: str = None,\n sk: str = None,\n verbose: bool = True,\n system_prompt: str = None,\n max_tokens: int = 1024,\n proxy: str = None,\n **kwargs,\n ):\n\n self.model = model\n self.fail_msg = \"Failed to obtain answer via API. \"\n self.ak = os.environ.get(\"SENSECHAT_AK\", None) if ak is None else ak\n self.sk = os.environ.get(\"SENSECHAT_SK\", None) if sk is None else sk\n assert self.ak is not None and self.sk is not None\n self.max_new_tokens = max_tokens\n super().__init__(\n wait=wait,\n retry=retry,\n system_prompt=system_prompt,\n verbose=verbose,\n **kwargs,\n )\n\n def dump_image(self, line, dataset):\n \"\"\"Dump the image(s) of the input line to the corresponding dataset folder.\n\n Args:\n line (line of pd.DataFrame): The raw input line.\n dataset (str): The name of the dataset.\n\n Returns:\n str | list[str]: The paths of the dumped images.\n \"\"\"\n ROOT = LMUDataRoot()\n assert isinstance(dataset, str)\n img_root = osp.join(\n ROOT,\n \"images\",\n img_root_map[dataset] if dataset in img_root_map else dataset,\n )\n os.makedirs(img_root, exist_ok=True)\n if \"image\" in line:\n if isinstance(line[\"image\"], list):\n tgt_path = []\n assert \"image_path\" in line\n for img, im_name in zip(line[\"image\"], line[\"image_path\"]):\n path = osp.join(img_root, im_name)\n if not read_ok(path):\n decode_base64_to_image_file(img, path)\n tgt_path.append(path)\n else:\n tgt_path = osp.join(img_root, f\"{line['index']}.jpg\")\n if not read_ok(tgt_path):\n decode_base64_to_image_file(line[\"image\"], tgt_path)\n tgt_path = [tgt_path]\n else:\n assert \"image_path\" in line\n tgt_path = toliststr(line[\"image_path\"])\n\n return tgt_path\n\n def image_to_base64(self, image_path):\n import base64\n\n with open(image_path, \"rb\") as image_file:\n encoded_string = base64.b64encode(image_file.read())\n return encoded_string.decode(\"utf-8\")\n\n def encode_jwt_token(self, ak, sk):\n import jwt\n\n headers = {\"alg\": \"HS256\", \"typ\": \"JWT\"}\n payload = {\n \"iss\": ak,\n \"exp\": int(time.time())\n + 1800, # 填写您期望的有效时间，此处示例代表当前时间+30分钟\n \"nbf\": int(time.time())\n - 5, # 填写您期望的生效时间，此处示例代表当前时间-5秒\n }\n token = jwt.encode(payload, sk, headers=headers)\n return token\n\n def use_custom_prompt(self, dataset):\n return True\n\n def build_multi_choice_prompt(self, line, dataset=None):\n question = line[\"question\"]\n hint = line[\"hint\"] if (\"hint\" in line and not pd.isna(line[\"hint\"])) else None\n if hint is not None:\n question = hint + \"\\n\" + question\n\n options = {\n cand: line[cand]\n for cand in string.ascii_uppercase\n if cand in line and not pd.isna(line[cand])\n }\n for key, item in options.items():\n question += f\"\\n{key}. {item}\"\n prompt = question\n\n if len(options):\n prompt += (\n \"\\n请直接回答选项字母。\"\n if cn_string(prompt)\n else \"\\nAnswer with the option's letter from the given choices directly.\"\n )\n else:\n prompt += (\n \"\\n请直接回答问题。\"\n if cn_string(prompt)\n else \"\\nAnswer the question directly.\"\n )\n\n return prompt\n\n def build_prompt(self, line, dataset=None):\n assert self.use_custom_prompt(dataset)\n assert dataset is None or isinstance(dataset, str)\n\n tgt_path = self.dump_image(line, dataset)\n\n if dataset is not None and listinstr([\"MME\"], dataset):\n question = line[\"question\"]\n prompt = question + \" Answer the question using a single word or phrase.\"\n elif dataset is not None and listinstr([\"HallusionBench\"], dataset):\n question = line[\"question\"]\n prompt = (\n question\n + \" Please answer yes or no. Answer the question using a single word or phrase.\"\n )\n elif (\n dataset is not None\n and DATASET_TYPE(dataset) == \"MCQ\"\n and \"MMMU\" not in dataset\n ):\n prompt = self.build_multi_choice_prompt(line, dataset)\n elif dataset is not None and DATASET_TYPE(dataset) == \"VQA\":\n if \"MathVista\" in dataset:\n prompt = line[\"question\"]\n elif listinstr([\"LLaVABench\"], dataset):\n question = line[\"question\"]\n prompt = question + \"\\nAnswer this question in detail.\"\n elif listinstr([\"MMVet\"], dataset):\n prompt = line[\"question\"]\n else:\n question = line[\"question\"]\n prompt = (\n question + \"\\nAnswer the question using a single word or phrase.\"\n )\n elif dataset is not None and \"MMMU\" in dataset:\n question = line[\"question\"]\n options = {\n cand: line[cand]\n for cand in string.ascii_uppercase\n if cand in line and not pd.isna(line[cand])\n }\n for key, item in options.items():\n question += f\"\\n{key}. {item}\"\n prompt = {\n \"multiple-choice\": 'You are an expert in {}. Please solve the university-level {} examination question, which includes interleaved images and text. Your output should be divided into two parts: First, reason about the correct answer. Then write the answer in the following format where X is exactly one of the choices given by the problem: \"ANSWER: X\". If you are uncertain of the correct answer, guess the most likely one.', # noqa: E501\n \"open\": 'You are an expert in {}. Please solve the university-level {} examination question, which includes interleaved images and text. Your output should be divided into two parts: First, reason about the correct answer. Then write the answer in the following format where X is only the answer and nothing else: \"ANSWER: X\"', # noqa: E501\n }\n subject = \"_\".join(line[\"id\"].split(\"_\")[1:-1])\n prompt = (\n prompt[line[\"question_type\"]].format(subject, subject) + \"\\n\" + question\n )\n else:\n prompt = line[\"question\"]\n\n message = [dict(type=\"text\", value=prompt)]\n message.extend([dict(type=\"image\", value=s) for s in tgt_path])\n\n return message\n\n def message_to_promptimg(self, message, dataset=None):\n if dataset is None or listinstr([\"MMMU\", \"BLINK\"], dataset):\n prompt = \"\\n\".join([x[\"value\"] for x in message if x[\"type\"] == \"text\"])\n image = [[x[\"value\"] for x in message if x[\"type\"] == \"image\"][0]]\n else:\n prompt = \"\\n\".join([x[\"value\"] for x in message if x[\"type\"] == \"text\"])\n image = [x[\"value\"] for x in message if x[\"type\"] == \"image\"]\n return prompt, image\n\n def generate_inner(self, inputs, **kwargs) -> str:\n assert isinstance(inputs, str) or isinstance(inputs, list)\n inputs = [inputs] if isinstance(inputs, str) else inputs\n dataset = kwargs.get(\"dataset\", None)\n\n if dataset is not None and listinstr([\"ChartQA_TEST\"], dataset):\n self.max_num = 12\n elif dataset is not None and listinstr([\"DocVQA_VAL\", \"DocVQA_TEST\"], dataset):\n self.max_num = 18\n elif dataset is not None and listinstr(\n [\"InfoVQA_VAL\", \"InfoVQA_TEST\", \"OCRBench\"], dataset\n ):\n self.max_num = 24\n else:\n self.max_num = 6\n\n if dataset is None:\n pass\n elif listinstr([\"AI2D_TEST\"], dataset):\n self.max_new_tokens = 10\n elif \"MMMU\" in dataset:\n self.max_new_tokens = 1024\n elif \"MMBench\" in dataset:\n self.max_new_tokens = 100\n\n prompt, image = self.message_to_promptimg(message=inputs, dataset=dataset)\n\n url = \"https://api.sensenova.cn/v1/llm/chat-completions\"\n api_secret_key = self.encode_jwt_token(self.ak, self.sk)\n\n content = [\n {\n \"image_base64\": self.image_to_base64(item),\n \"image_file_id\": \"\",\n \"image_url\": \"\",\n \"text\": \"\",\n \"text\": \"\",\n \"type\": \"image_base64\",\n }\n for item in image\n ]\n\n content.append(\n {\n \"image_base64\": \"\",\n \"image_file_id\": \"\",\n \"image_url\": \"\",\n \"text\": prompt,\n \"type\": \"text\",\n }\n )\n\n message = [{\"content\": content, \"role\": \"user\"}]\n\n data = {\n \"messages\": message,\n \"max_new_tokens\": self.max_new_tokens,\n \"model\": self.model,\n \"stream\": False,\n }\n headers = {\n \"Content-type\": \"application/json\",\n \"Authorization\": \"Bearer \" + api_secret_key,\n }\n\n response = requests.post(\n url,\n headers=headers,\n json=data,\n )\n request_id = response.headers[\"x-request-id\"]\n\n time.sleep(1)\n try:\n assert response.status_code == 200\n response = response.json()[\"data\"][\"choices\"][0][\"message\"].strip()\n if dataset is not None and \"MMMU\" in dataset:\n response = response.split(\"ANSWER: \")[-1].strip()\n if self.verbose:\n self.logger.info(f\"inputs: {inputs}\\nanswer: {response}\")\n return 0, response, \"Succeeded! \"\n except Exception as err:\n if self.verbose:\n self.logger.error(\n \"---------------------------ERROR---------------------------\"\n )\n self.logger.error(response.json())\n self.logger.error(err)\n self.logger.error(\n \"---------------------------request_id---------------------------\"\n + request_id\n )\n self.logger.error(\n \"api error\"\n + response.json()[\"error\"][\"message\"]\n + str(\n [\n input[\"value\"] if input[\"type\"] == \"image\" else None\n for input in inputs\n ]\n )\n )\n self.logger.error(f\"The input messages are {inputs}.\")\n return -1, response.json()[\"error\"][\"message\"], \"\"\n\n\nclass SenseChatVisionAPI(SenseChatVisionWrapper):\n\n def generate(self, message, dataset=None):\n return super(SenseChatVisionAPI, self).generate(message, dataset=dataset)\n"}}}} |