{"repo": "ai9stars/cheers", "files": {"Evaluation_Understanding/vlmeval/api/sensechat_vision.py": {"da1cb77e4d102820ccc496fe1f3a72dfd400cd96": {"t": 1785768239.8684523, "n": 11609, "h": "d9a350f5241c67c4", "c": "import os\nimport string\nimport time\nfrom typing import Optional\nimport pandas as pd\nimport requests\nfrom vlmeval.smp import (\n    LMUDataRoot,\n    osp,\n    read_ok,\n    decode_base64_to_image_file,\n    toliststr,\n    listinstr,\n    cn_string,\n)\nfrom vlmeval.api.base import BaseAPI\nfrom vlmeval.dataset import img_root_map\nfrom vlmeval.dataset import DATASET_TYPE\n\n\nclass SenseChatVisionWrapper(BaseAPI):\n    is_api: bool = True\n\n    def __init__(\n        self,\n        base_url: str = \"https://api.sensenova.cn/v1/llm/chat-completions\",\n        api_key: str = None,\n        model: str = \"SenseNova-V6-Pro\",\n        retry: int = 5,\n        wait: int = 5,\n        verbose: bool = True,\n        system_prompt: str = None,\n        max_tokens: int = 8192,\n        **kwargs,\n    ):\n        self.base_url = base_url\n        self.model = model\n        self.fail_msg = \"Failed to obtain answer via API. \"\n        self.api_key = os.getenv(\"SENSENOVA_API_KEY\", api_key)\n        assert self.api_key is not None, (\n            \"Please set the `SENSENOVA_API_KEY` environment variable or pass `api_key` in the config.json.\"\n        )\n        self.max_new_tokens = max_tokens\n        super().__init__(\n            wait=wait,\n            retry=retry,\n            system_prompt=system_prompt,\n            verbose=verbose,\n            **kwargs,\n        )\n\n    def dump_image(self, line, dataset):\n        \"\"\"Dump the image(s) of the input line to the corresponding dataset folder.\n\n        Args:\n            line (line of pd.DataFrame): The raw input line.\n            dataset (str): The name of the dataset.\n\n        Returns:\n            str | list[str]: The paths of the dumped images.\n        \"\"\"\n        ROOT = LMUDataRoot()\n        assert isinstance(dataset, str)\n        img_root = osp.join(ROOT, \"images\", img_root_map(dataset))\n        os.makedirs(img_root, exist_ok=True)\n        if \"image\" in line:\n            if isinstance(line[\"image\"], list):\n                tgt_path = []\n                assert \"image_path\" in line\n                for img, im_name in zip(line[\"image\"], line[\"image_path\"]):\n                    path = osp.join(img_root, im_name)\n                    if not read_ok(path):\n                        decode_base64_to_image_file(img, path)\n                    tgt_path.append(path)\n            else:\n                tgt_path = osp.join(img_root, f\"{line['index']}.jpg\")\n                if not read_ok(tgt_path):\n                    decode_base64_to_image_file(line[\"image\"], tgt_path)\n                tgt_path = [tgt_path]\n        else:\n            assert \"image_path\" in line\n            tgt_path = toliststr(line[\"image_path\"])\n\n        return tgt_path\n\n    def image_to_base64(self, image_path):\n        import base64\n\n        with open(image_path, \"rb\") as image_file:\n            encoded_string = base64.b64encode(image_file.read())\n            return encoded_string.decode(\"utf-8\")\n\n    def use_custom_prompt(self, *args, **kwargs):\n        \"\"\"Check if the prompt is customized.\"\"\"\n        return True\n\n    def build_multi_choice_prompt(self, line, dataset=None):\n        question = line[\"question\"]\n        hint = line[\"hint\"] if (\"hint\" in line and not pd.isna(line[\"hint\"])) else None\n        if hint is not None:\n            question = hint + \"\\n\" + question\n\n        options = {\n            cand: line[cand]\n            for cand in string.ascii_uppercase\n            if cand in line and not pd.isna(line[cand])\n        }\n        for key, item in options.items():\n            question += f\"\\n{key}. {item}\"\n        prompt = question\n\n        if len(options):\n            prompt += (\n                \"\\n请直接回答选项字母。\"\n                if cn_string(prompt)\n                else \"\\nAnswer with the option's letter from the given choices directly.\"\n            )\n        else:\n            prompt += (\n                \"\\n请直接回答问题。\"\n                if cn_string(prompt)\n                else \"\\nAnswer the question directly.\"\n            )\n\n        return prompt\n\n    def build_mcq_cot_prompt(self, line, prompt):\n        question = line[\"question\"]\n        options = {\n            cand: line[cand]\n            for cand in string.ascii_uppercase\n            if cand in line and not pd.isna(line[cand])\n        }\n        for key, item in options.items():\n            question += f'\\n{key}. {item}'\n        prompt = {\n'multiple-choice': \"You are an expert in {}. Please solve the university-level {} examination question, which includes interleaved images and text. Answer the preceding multiple choice question. The last line of your response should follow this format: 'Answer: \\\\boxed LETTER', where LETTER is one of the options. If you are uncertain or the problem is too complex, make a reasoned guess based on the information provided. Avoid repeating steps indefinitely—provide your best guess even if unsure. Think step by step logically, considering all relevant information before answering.\",  # noqa: E122, E501\n'open': 'You are an expert in {}. Please solve the university-level {} examination question, which includes interleaved images and text. Your output should be divided into two parts: First, reason about the correct answer. Then write the answer in the following format where X is only the answer and nothing else: \"ANSWER: X\"'  # noqa: E122, E501\n        }\n        subject = '_'.join(line['id'].split('_')[1:-1])\n        prompt = prompt[line['question_type']].format(subject, subject) + '\\n' + question\n        return prompt\n\n    def build_prompt(self, line, dataset=None):\n        assert self.use_custom_prompt(dataset)\n        assert dataset is None or isinstance(dataset, str)\n\n        tgt_path = self.dump_image(line, dataset)\n\n        if dataset is not None and listinstr([\"MME\"], dataset):\n            question = line[\"question\"]\n            prompt = question + \" Answer the question using a single word or phrase.\"\n        elif dataset is not None and listinstr([\"HallusionBench\"], dataset):\n            question = line[\"question\"]\n            prompt = (\n                question\n                + \" Please answer yes or no. Answer the question using a single word or phrase.\"\n            )\n        elif dataset is not None and DATASET_TYPE(dataset) == \"MCQ\":\n            prompt = self.build_multi_choice_prompt(line, dataset)\n            if \"MMMU\" in dataset:\n                prompt = self.build_mcq_cot_prompt(line, prompt)\n        elif dataset is not None and DATASET_TYPE(dataset) == \"VQA\":\n            if \"MathVista\" in dataset:\n                prompt = line[\"question\"]\n            elif listinstr([\"LLaVABench\"], dataset):\n                question = line[\"question\"]\n                prompt = question + \"\\nAnswer this question in detail.\"\n            elif listinstr([\"MMVet\"], dataset):\n                prompt = line[\"question\"]\n            else:\n                question = line[\"question\"]\n                prompt = (\n                    question\n                    + \"\\nPlease reason step by step, and put your final answer within \\\\boxed{}.\"\n                )\n        else:\n            prompt = line[\"question\"]\n\n        message = [dict(type=\"text\", value=prompt)]\n        message.extend([dict(type=\"image\", value=s) for s in tgt_path])\n\n        return message\n\n    def message_to_promptimg(self, message, dataset=None):\n        if dataset is None or listinstr([\"MMMU\", \"BLINK\"], dataset):\n            prompt = \"\\n\".join([x[\"value\"] for x in message if x[\"type\"] == \"text\"])\n            image = [[x[\"value\"] for x in message if x[\"type\"] == \"image\"][0]]\n        else:\n            prompt = \"\\n\".join([x[\"value\"] for x in message if x[\"type\"] == \"text\"])\n            image = [x[\"value\"] for x in message if x[\"type\"] == \"image\"]\n        return prompt, image\n\n    def set_max_num(self, dataset: Optional[str] = None) -> None:\n        \"\"\"Set the max_num based on the dataset.\"\"\"\n        if dataset is not None and listinstr(\n            [\n                \"ChartQA_TEST\",\n                \"MMMU_DEV_VAL\",\n                \"MMMU_TEST\",\n                \"MME-RealWorld\",\n                \"VCR_EN\",\n                \"VCR_ZH\",\n                \"OCRVQA\",\n            ],\n            dataset,\n        ):\n            self.max_num = 12\n        elif dataset is not None and listinstr(\n            [\"DocVQA_VAL\", \"DocVQA_TEST\", \"DUDE\", \"MMLongBench_DOC\", \"SLIDEVQA\"],\n            dataset,\n        ):\n            self.max_num = 18\n        elif dataset is not None and listinstr(\n            [\"InfoVQA_VAL\", \"InfoVQA_TEST\", \"OCRBench\", \"HRBench4K\", \"HRBench8K\"],\n            dataset,\n        ):\n            self.max_num = 24\n        else:\n            self.max_num = 6\n\n    def generate_inner(self, inputs, **kwargs) -> str:\n        assert isinstance(inputs, str) or isinstance(inputs, list)\n        inputs = [inputs] if isinstance(inputs, str) else inputs\n        dataset = kwargs.get(\"dataset\", None)\n\n        self.set_max_num(dataset=dataset)\n\n        prompt, image = self.message_to_promptimg(message=inputs, dataset=dataset)\n        content = [\n            {\n                \"image_base64\": self.image_to_base64(item),\n                \"type\": \"image_base64\",\n            }\n            for item in image\n        ]\n\n        content.append(\n            {\n                \"text\": prompt,\n                \"type\": \"text\",\n            }\n        )\n        message = [{\n            \"role\": \"system\",\n            \"content\": [{\"text\": \" \", \"type\": \"text\"}]\n        }]\n        message.append({\"content\": content, \"role\": \"user\"})\n        data = {\n            \"messages\": message,\n            \"max_new_tokens\": self.max_new_tokens,  # 1024\n            \"temperature\": 1e-5,\n            \"top_k\": 1,\n            \"top_p\": 0.99,\n            \"repetition_penalty\": 1.05,\n            \"model\": self.model,\n            \"stream\": False,\n            \"image_split_count\": self.max_num,\n        }\n        headers = {\n            \"Content-type\": \"application/json\",\n            \"Authorization\": self.api_key,\n        }\n\n        response = requests.post(\n            self.base_url,\n            headers=headers,\n            json=data,\n        )\n        request_id = response.headers[\"x-request-id\"]\n        self.logger.info(f\"Request-id: {request_id}\")\n\n        time.sleep(1)\n        try:\n            assert response.status_code == 200\n            response = response.json()[\"data\"][\"choices\"][0][\"message\"].strip()\n            if self.verbose:\n                self.logger.info(f\"inputs: {inputs}\\nanswer: {response}\")\n            return 0, response, \"Succeeded! \"\n        except Exception as err:\n            if self.verbose:\n                self.logger.error(\n                    \"---------------------------ERROR---------------------------\"\n                )\n                self.logger.error(response.json())\n                self.logger.error(err)\n                self.logger.error(\n                    \"---------------------------request_id---------------------------\"\n                    + request_id\n                )\n                self.logger.error(\n                    \"api error\"\n                    + response.json()[\"error\"][\"message\"]\n                    + str(\n                        [\n                            input[\"value\"] if input[\"type\"] == \"image\" else None\n                            for input in inputs\n                        ]\n                    )\n                )\n                self.logger.error(f\"The input messages are {inputs}.\")\n            return -1, response.json()[\"error\"][\"message\"], \"\"\n\n\nclass SenseChatVisionAPI(SenseChatVisionWrapper):\n    def generate(self, message, dataset=None):\n        return super(SenseChatVisionAPI, self).generate(message, dataset=dataset)\n"}}}}