1 line
66 KiB
JSON
1 line
66 KiB
JSON
{"repo": "alpha-vllm/lumina-dimoo", "files": {"VLMEvalKit/vlmeval/config.py": {"81344e60545ae8d70f11039d3450717d974f831b": {"t": 1785674030.7896903, "n": 50756, "h": "1d8685a9fbc3a176", "c": "from vlmeval.vlm import *\nfrom vlmeval.api import *\nfrom functools import partial\nimport os\n\nPandaGPT_ROOT = None\nMiniGPT4_ROOT = None\nTransCore_ROOT = None\nYi_ROOT = None\nOmniLMM_ROOT = None\nMini_Gemini_ROOT = None\nVXVERSE_ROOT = None\nVideoChat2_ROOT = None\nVideoChatGPT_ROOT = None\nPLLaVA_ROOT = None\nRBDash_ROOT = None\nVITA_ROOT = None\nLLAVA_V1_7B_MODEL_PTH = \"Please set your local path to LLaVA-7B-v1.1 here, the model weight is obtained by merging LLaVA delta weight based on vicuna-7b-v1.1 in https://github.com/haotian-liu/LLaVA/blob/main/docs/MODEL_ZOO.md with vicuna-7b-v1.1. \"\n\nvideo_models = {\n \"Video-LLaVA-7B\": partial(VideoLLaVA, model_path=\"LanguageBind/Video-LLaVA-7B\"),\n \"Video-LLaVA-7B-HF\": partial(\n VideoLLaVA_HF, model_path=\"LanguageBind/Video-LLaVA-7B-hf\"\n ),\n \"VideoChat2-HD\": partial(\n VideoChat2_HD,\n model_path=\"OpenGVLab/VideoChat2_HD_stage4_Mistral_7B\",\n root=VideoChat2_ROOT,\n config_file=\"./vlmeval/vlm/video_llm/configs/videochat2_hd.json\",\n ),\n \"Chat-UniVi-7B\": partial(Chatunivi, model_path=\"Chat-UniVi/Chat-UniVi\"),\n \"Chat-UniVi-7B-v1.5\": partial(\n Chatunivi, model_path=\"Chat-UniVi/Chat-UniVi-7B-v1.5\"\n ),\n \"LLaMA-VID-7B\": partial(\n LLaMAVID, model_path=\"YanweiLi/llama-vid-7b-full-224-video-fps-1\"\n ),\n \"Video-ChatGPT\": partial(\n VideoChatGPT, model_path=\"MBZUAI/Video-ChatGPT-7B\", dir_root=VideoChatGPT_ROOT\n ),\n \"PLLaVA-7B\": partial(PLLaVA, model_path=\"ermu2001/pllava-7b\", dir_root=PLLaVA_ROOT),\n \"PLLaVA-13B\": partial(\n PLLaVA, model_path=\"ermu2001/pllava-13b\", dir_root=PLLaVA_ROOT\n ),\n \"PLLaVA-34B\": partial(\n PLLaVA, model_path=\"ermu2001/pllava-34b\", dir_root=PLLaVA_ROOT\n ),\n}\n\nungrouped = {\n \"AKI\": partial(AKI, name=\"AKI\", ckpt_pth=\"Sony/AKI-4B-phi-3.5-mini\"),\n \"TransCore_M\": partial(TransCoreM, root=TransCore_ROOT),\n \"PandaGPT_13B\": partial(PandaGPT, name=\"PandaGPT_13B\", root=PandaGPT_ROOT),\n \"flamingov2\": partial(\n OpenFlamingo,\n name=\"v2\",\n mpt_pth=\"anas-awadalla/mpt-7b\",\n ckpt_pth=\"openflamingo/OpenFlamingo-9B-vitl-mpt7b\",\n ),\n \"VisualGLM_6b\": partial(VisualGLM, model_path=\"THUDM/visualglm-6b\"),\n \"mPLUG-Owl2\": partial(mPLUG_Owl2, model_path=\"MAGAer13/mplug-owl2-llama2-7b\"),\n \"mPLUG-Owl3\": partial(mPLUG_Owl3, model_path=\"mPLUG/mPLUG-Owl3-7B-240728\"),\n \"OmniLMM_12B\": partial(\n OmniLMM12B, model_path=\"openbmb/OmniLMM-12B\", root=OmniLMM_ROOT\n ),\n \"MGM_7B\": partial(\n Mini_Gemini, model_path=\"YanweiLi/MGM-7B-HD\", root=Mini_Gemini_ROOT\n ),\n \"Bunny-llama3-8B\": partial(BunnyLLama3, model_path=\"BAAI/Bunny-v1_1-Llama-3-8B-V\"),\n \"VXVERSE\": partial(VXVERSE, model_name=\"XVERSE-V-13B\", root=VXVERSE_ROOT),\n \"360VL-70B\": partial(QH_360VL, model_path=\"qihoo360/360VL-70B\"),\n \"Llama-3-MixSenseV1_1\": partial(\n LLama3Mixsense, model_path=\"Zero-Vision/Llama-3-MixSenseV1_1\"\n ),\n \"Parrot\": partial(Parrot, model_path=\"AIDC-AI/Parrot-7B\"),\n \"OmChat\": partial(OmChat, model_path=\"omlab/omchat-v2.0-13B-single-beta_hf\"),\n \"RBDash_72b\": partial(\n RBDash, model_path=\"RBDash-Team/RBDash-v1.5\", root=RBDash_ROOT\n ),\n \"Pixtral-12B\": partial(Pixtral, model_path=\"mistralai/Pixtral-12B-2409\"),\n \"Falcon2-VLM-11B\": partial(Falcon2VLM, model_path=\"tiiuae/falcon-11B-vlm\"),\n}\n\no1_key = os.environ.get('O1_API_KEY', None)\no1_base = os.environ.get('O1_API_BASE', None)\no1_apis = {\n 'o1': partial(\n GPT4V,\n model=\"o1-2024-12-17\",\n key=o1_key,\n api_base=o1_base, \n temperature=0,\n img_detail='high',\n retry=3,\n timeout=1800, \n max_tokens=16384,\n verbose=False,\n\n ),\n 'o3': partial(\n GPT4V, \n model=\"o3-2025-04-16\",\n key=o1_key,\n api_base=o1_base, \n temperature=0,\n img_detail='high',\n retry=3,\n timeout=1800, \n max_tokens=16384, \n verbose=False,\n ),\n 'o4-mini': partial(\n GPT4V, \n model=\"o4-mini-2025-04-16\",\n key=o1_key,\n api_base=o1_base, \n temperature=0,\n img_detail='high',\n retry=3,\n timeout=1800,\n max_tokens=16384,\n verbose=False,\n ),\n}\n\napi_models = {\n # GPT\n \"GPT4V\": partial(\n GPT4V,\n model=\"gpt-4-1106-vision-preview\",\n temperature=0,\n img_size=512,\n img_detail=\"low\",\n retry=10,\n verbose=False,\n ),\n \"GPT4V_HIGH\": partial(\n GPT4V,\n model=\"gpt-4-1106-vision-preview\",\n temperature=0,\n img_size=-1,\n img_detail=\"high\",\n retry=10,\n verbose=False,\n ),\n \"GPT4V_20240409\": partial(\n GPT4V,\n model=\"gpt-4-turbo-2024-04-09\",\n temperature=0,\n img_size=512,\n img_detail=\"low\",\n retry=10,\n verbose=False,\n ),\n \"GPT4V_20240409_HIGH\": partial(\n GPT4V,\n model=\"gpt-4-turbo-2024-04-09\",\n temperature=0,\n img_size=-1,\n img_detail=\"high\",\n retry=10,\n verbose=False,\n ),\n \"GPT4o\": partial(\n GPT4V,\n model=\"gpt-4o-2024-05-13\",\n temperature=0,\n img_size=512,\n img_detail=\"low\",\n retry=10,\n verbose=False,\n ),\n \"GPT4o_HIGH\": partial(\n GPT4V,\n model=\"gpt-4o-2024-05-13\",\n temperature=0,\n img_size=-1,\n img_detail=\"high\",\n retry=10,\n verbose=False,\n ),\n \"GPT4o_20240806\": partial(\n GPT4V,\n model=\"gpt-4o-2024-08-06\",\n temperature=0,\n img_size=-1,\n img_detail=\"high\",\n retry=10,\n verbose=False,\n ),\n \"GPT4o_20241120\": partial(\n GPT4V,\n model=\"gpt-4o-2024-11-20\",\n temperature=0,\n img_size=-1,\n img_detail=\"high\",\n retry=10,\n verbose=False,\n ),\n \"ChatGPT4o\": partial(\n GPT4V,\n model=\"chatgpt-4o-latest\",\n temperature=0,\n img_size=-1,\n img_detail=\"high\",\n retry=10,\n verbose=False,\n ),\n \"GPT4o_MINI\": partial(\n GPT4V,\n model=\"gpt-4o-mini-2024-07-18\",\n temperature=0,\n img_size=-1,\n img_detail=\"high\",\n retry=10,\n verbose=False,\n ),\n \"GPT4.5\": partial(\n GPT4V, \n model='gpt-4.5-preview-2025-02-27',\n temperature=0, \n timeout=600,\n img_size=-1, \n img_detail='high', \n retry=10, \n verbose=False,\n ),\n \"gpt-4.1-2025-04-14\": partial(\n GPT4V,\n model=\"gpt-4.1-2025-04-14\",\n temperature=0,\n img_size=-1,\n img_detail=\"high\",\n retry=10,\n verbose=False,\n ),\n \"gpt-4.1-mini-2025-04-14\": partial(\n GPT4V,\n model=\"gpt-4.1-mini-2025-04-14\",\n temperature=0,\n img_size=-1,\n img_detail=\"high\",\n retry=10,\n verbose=False,\n ),\n \"gpt-4.1-nano-2025-04-14\": partial(\n GPT4V,\n model=\"gpt-4.1-nano-2025-04-14\",\n temperature=0,\n img_size=-1,\n img_detail=\"high\",\n retry=10,\n verbose=False,\n ),\n # Gemini\n \"GeminiPro1-0\": partial(\n Gemini, model=\"gemini-1.0-pro\", temperature=0, retry=10\n ), # now GeminiPro1-0 is only supported by vertex backend\n \"GeminiPro1-5\": partial(\n Gemini, model=\"gemini-1.5-pro\", temperature=0, retry=10\n ),\n \"GeminiFlash1-5\": partial(\n Gemini, model=\"gemini-1.5-flash\", temperature=0, retry=10\n ),\n \"GeminiPro1-5-002\": partial(\n GPT4V, model=\"gemini-1.5-pro-002\", temperature=0, retry=10\n ), # Internal Use Only\n \"GeminiFlash1-5-002\": partial(\n GPT4V, model=\"gemini-1.5-flash-002\", temperature=0, retry=10\n ), # Internal Use Only\n \"GeminiFlash2-0\": partial(\n Gemini, model=\"gemini-2.0-flash\", temperature=0, retry=10\n ),\n \"GeminiFlashLite2-0\": partial(\n Gemini, model=\"gemini-2.0-flash-lite\", temperature=0, retry=10\n ),\n \"GeminiFlash2-5\": partial(\n GPT4V, model=\"gemini-2.5-flash\", temperature=0, retry=10, timeout=1800\n ),\n \"GeminiPro2-5\": partial(\n GPT4V, model=\"gemini-2.5-pro\", temperature=0, retry=10, timeout=1800\n ),\n \n # Qwen-VL\n \"QwenVLPlus\": partial(QwenVLAPI, model=\"qwen-vl-plus\", temperature=0, retry=10),\n \"QwenVLMax\": partial(QwenVLAPI, model=\"qwen-vl-max\", temperature=0, retry=10),\n \"QwenVLMax-250408\": partial(QwenVLAPI, model=\"qwen-vl-max-2025-04-08\", temperature=0, retry=10),\n\n # Reka\n \"RekaEdge\": partial(Reka, model=\"reka-edge-20240208\"),\n \"RekaFlash\": partial(Reka, model=\"reka-flash-20240226\"),\n \"RekaCore\": partial(Reka, model=\"reka-core-20240415\"),\n # Step1V\n \"Step1V\": partial(\n GPT4V,\n model=\"step-1v-32k\",\n api_base=\"https://api.stepfun.com/v1/chat/completions\",\n temperature=0,\n retry=10,\n img_size=-1,\n img_detail=\"high\",\n ),\n \"Step1.5V-mini\": partial(\n GPT4V,\n model=\"step-1.5v-mini\",\n api_base=\"https://api.stepfun.com/v1/chat/completions\",\n temperature=0,\n retry=10,\n img_size=-1,\n img_detail=\"high\",\n ),\n \"Step1o\": partial(\n GPT4V,\n model=\"step-1o-vision-32k\",\n api_base=\"https://api.stepfun.com/v1/chat/completions\",\n temperature=0,\n retry=10,\n img_size=-1,\n img_detail=\"high\",\n ),\n # Yi-Vision\n \"Yi-Vision\": partial(\n GPT4V,\n model=\"yi-vision\",\n api_base=\"https://api.lingyiwanwu.com/v1/chat/completions\",\n temperature=0,\n retry=10,\n ),\n # Claude\n \"Claude3V_Opus\": partial(\n Claude3V, model=\"claude-3-opus-20240229\", temperature=0, retry=10, verbose=False\n ),\n \"Claude3V_Sonnet\": partial(\n Claude3V,\n model=\"claude-3-sonnet-20240229\",\n temperature=0,\n retry=10,\n verbose=False,\n ),\n \"Claude3V_Haiku\": partial(\n Claude3V,\n model=\"claude-3-haiku-20240307\",\n temperature=0,\n retry=10,\n verbose=False,\n ),\n \"Claude3-5V_Sonnet\": partial(\n Claude3V,\n model=\"claude-3-5-sonnet-20240620\",\n temperature=0,\n retry=10,\n verbose=False,\n ),\n \"Claude3-5V_Sonnet_20241022\": partial(\n Claude3V,\n model=\"claude-3-5-sonnet-20241022\",\n temperature=0,\n retry=10,\n verbose=False,\n ),\n \"Claude3-7V_Sonnet\": partial(\n Claude3V,\n model=\"claude-3-7-sonnet-20250219\",\n temperature=0,\n retry=10,\n verbose=False,\n ),\n \"Claude4_Opus\": partial(\n Claude3V,\n model=\"claude-4-opus-20250514\",\n temperature=0,\n retry=10,\n verbose=False,\n timeout=1800\n ),\n \"Claude4_Sonnet\": partial(\n Claude3V,\n model=\"claude-4-sonnet-20250514\",\n temperature=0,\n retry=10,\n verbose=False,\n timeout=1800\n ),\n # GLM4V\n \"GLM4V\": partial(GLMVisionAPI, model=\"glm4v-biz-eval\", temperature=0, retry=10),\n \"GLM4V_PLUS\": partial(GLMVisionAPI, model=\"glm-4v-plus\", temperature=0, retry=10),\n \"GLM4V_PLUS_20250111\": partial(\n GLMVisionAPI, model=\"glm-4v-plus-0111\", temperature=0, retry=10\n ),\n # MiniMax abab\n \"abab6.5s\": partial(\n GPT4V,\n model=\"abab6.5s-chat\",\n api_base=\"https://api.minimax.chat/v1/chat/completions\",\n temperature=0,\n retry=10,\n ),\n \"abab7-preview\": partial(\n GPT4V,\n model=\"abab7-chat-preview\",\n api_base=\"https://api.minimax.chat/v1/chat/completions\",\n temperature=0,\n retry=10,\n ),\n # CongRong\n \"CongRong-v1.5\": partial(CWWrapper, model=\"cw-congrong-v1.5\", temperature=0, retry=10),\n \"CongRong-v2.0\": partial(CWWrapper, model=\"cw-congrong-v2.0\", temperature=0, retry=10),\n # SenseNova\n \"SenseNova-V6-Pro\": partial(\n SenseChatVisionAPI, model=\"SenseNova-V6-Pro\", temperature=0, retry=10\n ),\n \"SenseNova-V6-Reasoner\": partial(\n SenseChatVisionAPI, model=\"SenseNova-V6-Reasoner\", temperature=0, retry=10\n ),\n \"HunYuan-Vision\": partial(\n HunyuanVision, model=\"hunyuan-vision\", temperature=0, retry=10\n ),\n \"HunYuan-Standard-Vision\": partial(\n HunyuanVision, model=\"hunyuan-standard-vision\", temperature=0, retry=10\n ),\n \"HunYuan-Large-Vision\": partial(\n HunyuanVision, model=\"hunyuan-large-vision\", temperature=0, retry=10\n ),\n \"BailingMM-Lite-1203\": partial(\n bailingMMAPI, model=\"BailingMM-Lite-1203\", temperature=0, retry=10\n ),\n \"BailingMM-Pro-0120\": partial(\n bailingMMAPI, model=\"BailingMM-Pro-0120\", temperature=0, retry=10\n ),\n # BlueLM-V\n \"BlueLM_V\": partial(BlueLM_V_API, model=\"BlueLM-VL-v3.0\", temperature=0, retry=10),\n # JiuTian-VL\n \"JTVL\": partial(JTVLChatAPI, model=\"jt-vl-chat\", temperature=0, retry=10),\n \"Taiyi\": partial(TaiyiAPI, model=\"taiyi\", temperature=0, retry=10),\n # TeleMM\n \"TeleMM\": partial(TeleMMAPI, model=\"TeleAI/TeleMM\", temperature=0, retry=10),\n \"Qwen2.5-VL-32B-Instruct-SiliconFlow\": partial(\n SiliconFlowAPI, model=\"Qwen/Qwen2.5-VL-32B-Instruct\", temperature=0, retry=10),\n # lmdeploy api\n \"lmdeploy\": partial(\n LMDeployAPI,\n api_base=\"http://0.0.0.0:23333/v1/chat/completions\",\n temperature=0,\n retry=10,\n ),\n \"lmdeploy_internvl_78B_MPO\": partial(\n LMDeployAPI,\n api_base=\"http://0.0.0.0:23333/v1/chat/completions\",\n temperature=0,\n retry=10,\n timeout=100,\n ),\n \"lmdeploy_qvq_72B_preview\": partial(\n LMDeployAPI,\n api_base=\"http://0.0.0.0:23333/v1/chat/completions\",\n temperature=0,\n retry=10,\n timeout=300,\n ),\n 'Taichu-VLR-3B': partial(\n TaichuVLRAPI, \n model='taichu_vlr_3b', \n url=\"https://platform.wair.ac.cn/maas/v1/chat/completions\"\n ),\n 'Taichu-VLR-7B': partial(\n TaichuVLRAPI, \n model='taichu_vlr_7b', \n url=\"https://platform.wair.ac.cn/maas/v1/chat/completions\"\n ),\n # doubao_vl\n \"DoubaoVL\": partial(\n DoubaoVL, model=\"Doubao-1.5-vision-pro\", temperature=0, retry=10, verbose=False\n ),\n \"Seed1.5-VL\": partial(\n DoubaoVL, \n model=\"doubao-1-5-thinking-vision-pro-250428\", \n temperature=0,\n retry=10, \n verbose=False, \n max_tokens=16384,\n ),\n \"Seed1.6\": partial(\n DoubaoVL, \n model=\"doubao-seed-1.6-250615\", \n temperature=0,\n retry=10, \n verbose=False, \n max_tokens=16384,\n ),\n \"Seed1.6-Flash\": partial(\n DoubaoVL, \n model=\"doubao-seed-1.6-flash-250615\", \n temperature=0,\n retry=10, \n verbose=False, \n max_tokens=16384,\n ),\n \"Seed1.6-Thinking\": partial(\n DoubaoVL, \n model=\"doubao-seed-1.6-thinking-250615\", \n temperature=0,\n retry=10, \n verbose=False, \n max_tokens=16384,\n ),\n # Shopee MUG-U\n 'MUG-U-7B': partial(\n MUGUAPI, \n model='MUG-U', \n temperature=0, \n retry=10, \n verbose=False, \n timeout=300),\n # grok\n \"grok-vision-beta\": partial(\n GPT4V,\n model=\"grok-vision-beta\",\n api_base=\"https://api.x.ai/v1/chat/completions\",\n temperature=0,\n retry=10,\n ),\n \"grok-2-vision-1212\": partial(\n GPT4V,\n model=\"grok-2-vision\",\n api_base=\"https://api.x.ai/v1/chat/completions\",\n temperature=0,\n retry=10,\n ),\n # kimi\n \"moonshot-v1-8k\": partial(\n GPT4V,\n model=\"moonshot-v1-8k-vision-preview\",\n api_base=\"https://api.moonshot.cn/v1/chat/completions\",\n temperature=0,\n retry=10,\n ),\n \"moonshot-v1-32k\": partial(\n GPT4V,\n model=\"moonshot-v1-32k-vision-preview\",\n api_base=\"https://api.moonshot.cn/v1/chat/completions\",\n temperature=0,\n retry=10,\n ),\n \"moonshot-v1-128k\": partial(\n GPT4V,\n model=\"moonshot-v1-128k-vision-preview\",\n api_base=\"https://api.moonshot.cn/v1/chat/completions\",\n temperature=0,\n retry=10,\n ),\n}\n\nemu_series = {\n \"emu2_chat\": partial(Emu, model_path=\"BAAI/Emu2-Chat\"),\n \"emu3_chat\": partial(Emu3_chat, model_path=\"BAAI/Emu3-Chat\"),\n \"emu3_gen\": partial(Emu3_gen, model_path=\"BAAI/Emu3-Gen\"),\n}\n\ngranite_vision_series = {\n 'granite_vision_3.1_2b_preview': partial(GraniteVision3, model_path=\"ibm-granite/granite-vision-3.1-2b-preview\"),\n 'granite_vision_3.2_2b': partial(GraniteVision3, model_path=\"ibm-granite/granite-vision-3.2-2b\"),\n 'granite_vision_3.3_2b': partial(GraniteVision3, model_path=\"ibm-granite/granite-vision-3.3-2b\"),\n}\n\nmmalaya_series = {\n \"MMAlaya\": partial(MMAlaya, model_path=\"DataCanvas/MMAlaya\"),\n \"MMAlaya2\": partial(MMAlaya2, model_path=\"DataCanvas/MMAlaya2\"),\n}\n\nminicpm_series = {\n \"MiniCPM-V\": partial(MiniCPM_V, model_path=\"openbmb/MiniCPM-V\"),\n \"MiniCPM-V-2\": partial(MiniCPM_V, model_path=\"openbmb/MiniCPM-V-2\"),\n \"MiniCPM-Llama3-V-2_5\": partial(\n MiniCPM_Llama3_V, model_path=\"openbmb/MiniCPM-Llama3-V-2_5\"\n ),\n \"MiniCPM-V-2_6\": partial(MiniCPM_V_2_6, model_path=\"openbmb/MiniCPM-V-2_6\"),\n \"MiniCPM-o-2_6\": partial(MiniCPM_o_2_6, model_path=\"openbmb/MiniCPM-o-2_6\"),\n}\n\nxtuner_series = {\n \"llava-internlm2-7b\": partial(\n LLaVA_XTuner,\n llm_path=\"internlm/internlm2-chat-7b\",\n llava_path=\"xtuner/llava-internlm2-7b\",\n visual_select_layer=-2,\n prompt_template=\"internlm2_chat\",\n ),\n \"llava-internlm2-20b\": partial(\n LLaVA_XTuner,\n llm_path=\"internlm/internlm2-chat-20b\",\n llava_path=\"xtuner/llava-internlm2-20b\",\n visual_select_layer=-2,\n prompt_template=\"internlm2_chat\",\n ),\n \"llava-internlm-7b\": partial(\n LLaVA_XTuner,\n llm_path=\"internlm/internlm-chat-7b\",\n llava_path=\"xtuner/llava-internlm-7b\",\n visual_select_layer=-2,\n prompt_template=\"internlm_chat\",\n ),\n \"llava-v1.5-7b-xtuner\": partial(\n LLaVA_XTuner,\n llm_path=\"lmsys/vicuna-7b-v1.5\",\n llava_path=\"xtuner/llava-v1.5-7b-xtuner\",\n visual_select_layer=-2,\n prompt_template=\"vicuna\",\n ),\n \"llava-v1.5-13b-xtuner\": partial(\n LLaVA_XTuner,\n llm_path=\"lmsys/vicuna-13b-v1.5\",\n llava_path=\"xtuner/llava-v1.5-13b-xtuner\",\n visual_select_layer=-2,\n prompt_template=\"vicuna\",\n ),\n \"llava-llama-3-8b\": partial(\n LLaVA_XTuner,\n llm_path=\"xtuner/llava-llama-3-8b-v1_1\",\n llava_path=\"xtuner/llava-llama-3-8b-v1_1\",\n visual_select_layer=-2,\n prompt_template=\"llama3_chat\",\n ),\n}\n\nqwen_series = {\n \"qwen_base\": partial(QwenVL, model_path=\"Qwen/Qwen-VL\"),\n \"qwen_chat\": partial(QwenVLChat, model_path=\"Qwen/Qwen-VL-Chat\"),\n \"monkey\": partial(Monkey, model_path=\"echo840/Monkey\"),\n \"monkey-chat\": partial(MonkeyChat, model_path=\"echo840/Monkey-Chat\"),\n \"minimonkey\": partial(MiniMonkey, model_path=\"mx262/MiniMonkey\"),\n}\n\nllava_series = {\n \"llava_v1.5_7b\": partial(LLaVA, model_path=\"liuhaotian/llava-v1.5-7b\"),\n \"llava_v1.5_13b\": partial(LLaVA, model_path=\"liuhaotian/llava-v1.5-13b\"),\n \"llava_v1_7b\": partial(LLaVA, model_path=LLAVA_V1_7B_MODEL_PTH),\n \"sharegpt4v_7b\": partial(LLaVA, model_path=\"Lin-Chen/ShareGPT4V-7B\"),\n \"sharegpt4v_13b\": partial(LLaVA, model_path=\"Lin-Chen/ShareGPT4V-13B\"),\n \"llava_next_vicuna_7b\": partial(\n LLaVA_Next, model_path=\"llava-hf/llava-v1.6-vicuna-7b-hf\"\n ),\n \"llava_next_vicuna_13b\": partial(\n LLaVA_Next, model_path=\"llava-hf/llava-v1.6-vicuna-13b-hf\"\n ),\n \"llava_next_mistral_7b\": partial(\n LLaVA_Next, model_path=\"llava-hf/llava-v1.6-mistral-7b-hf\"\n ),\n \"llava_next_yi_34b\": partial(LLaVA_Next, model_path=\"llava-hf/llava-v1.6-34b-hf\"),\n \"llava_next_llama3\": partial(\n LLaVA_Next, model_path=\"llava-hf/llama3-llava-next-8b-hf\"\n ),\n \"llava_next_72b\": partial(LLaVA_Next, model_path=\"llava-hf/llava-next-72b-hf\"),\n \"llava_next_110b\": partial(LLaVA_Next, model_path=\"llava-hf/llava-next-110b-hf\"),\n \"llava_next_qwen_32b\": partial(\n LLaVA_Next2, model_path=\"lmms-lab/llava-next-qwen-32b\"\n ),\n \"llava_next_interleave_7b\": partial(\n LLaVA_Next, model_path=\"llava-hf/llava-interleave-qwen-7b-hf\"\n ),\n \"llava_next_interleave_7b_dpo\": partial(\n LLaVA_Next, model_path=\"llava-hf/llava-interleave-qwen-7b-dpo-hf\"\n ),\n \"llava-onevision-qwen2-0.5b-ov-hf\": partial(\n LLaVA_OneVision_HF, model_path=\"llava-hf/llava-onevision-qwen2-0.5b-ov-hf\"\n ),\n \"llava-onevision-qwen2-0.5b-si-hf\": partial(\n LLaVA_OneVision_HF, model_path=\"llava-hf/llava-onevision-qwen2-0.5b-si-hf\"\n ),\n \"llava-onevision-qwen2-7b-ov-hf\": partial(\n LLaVA_OneVision_HF, model_path=\"llava-hf/llava-onevision-qwen2-7b-ov-hf\"\n ),\n \"llava-onevision-qwen2-7b-si-hf\": partial(\n LLaVA_OneVision_HF, model_path=\"llava-hf/llava-onevision-qwen2-7b-si-hf\"\n ),\n \"llava_onevision_qwen2_0.5b_si\": partial(\n LLaVA_OneVision, model_path=\"lmms-lab/llava-onevision-qwen2-0.5b-si\"\n ),\n \"llava_onevision_qwen2_7b_si\": partial(\n LLaVA_OneVision, model_path=\"lmms-lab/llava-onevision-qwen2-7b-si\"\n ),\n \"llava_onevision_qwen2_72b_si\": partial(\n LLaVA_OneVision, model_path=\"lmms-lab/llava-onevision-qwen2-72b-si\"\n ),\n \"llava_onevision_qwen2_0.5b_ov\": partial(\n LLaVA_OneVision, model_path=\"lmms-lab/llava-onevision-qwen2-0.5b-ov\"\n ),\n \"llava_onevision_qwen2_7b_ov\": partial(\n LLaVA_OneVision, model_path=\"lmms-lab/llava-onevision-qwen2-7b-ov\"\n ),\n \"llava_onevision_qwen2_72b_ov\": partial(\n LLaVA_OneVision, model_path=\"lmms-lab/llava-onevision-qwen2-72b-ov-sft\"\n ),\n \"Aquila-VL-2B\": partial(LLaVA_OneVision, model_path=\"BAAI/Aquila-VL-2B-llava-qwen\"),\n \"llava_video_qwen2_7b\": partial(\n LLaVA_OneVision, model_path=\"lmms-lab/LLaVA-Video-7B-Qwen2\"\n ),\n \"llava_video_qwen2_72b\": partial(\n LLaVA_OneVision, model_path=\"lmms-lab/LLaVA-Video-72B-Qwen2\"\n ),\n \"varco-vision-hf\": partial(\n LLaVA_OneVision_HF, model_path=\"NCSOFT/VARCO-VISION-14B-HF\"\n ),\n}\n\nvita_series = {\n \"vita\": partial(VITA, model_path=\"VITA-MLLM/VITA\", root=VITA_ROOT),\n \"vita_qwen2\": partial(VITAQwen2, model_path=\"VITA-MLLM/VITA-1.5\", root=VITA_ROOT),\n}\n\nlong_vita_series = {\n \"Long-VITA-16K\": partial(\n LongVITA, model_path=\"VITA-MLLM/Long-VITA-16K_HF\", max_num_frame=128\n ),\n \"Long-VITA-128K\": partial(\n LongVITA, model_path=\"VITA-MLLM/Long-VITA-128K_HF\", max_num_frame=256\n ),\n \"Long-VITA-1M\": partial(\n LongVITA, model_path=\"VITA-MLLM/Long-VITA-1M_HF\", max_num_frame=256\n ),\n}\n\ninternvl = {\n \"InternVL-Chat-V1-1\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL-Chat-V1-1\", version=\"V1.1\"\n ),\n \"InternVL-Chat-V1-2\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL-Chat-V1-2\", version=\"V1.2\"\n ),\n \"InternVL-Chat-V1-2-Plus\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL-Chat-V1-2-Plus\", version=\"V1.2\"\n ),\n \"InternVL-Chat-V1-5\": partial(\n InternVLChat,\n model_path=\"OpenGVLab/InternVL-Chat-V1-5\",\n version=\"V1.5\",\n )\n}\n\nmini_internvl = {\n \"Mini-InternVL-Chat-2B-V1-5\": partial(\n InternVLChat, model_path=\"OpenGVLab/Mini-InternVL-Chat-2B-V1-5\", version=\"V1.5\"\n ),\n \"Mini-InternVL-Chat-4B-V1-5\": partial(\n InternVLChat, model_path=\"OpenGVLab/Mini-InternVL-Chat-4B-V1-5\", version=\"V1.5\"\n ),\n}\n\ninternvl2 = {\n \"InternVL2-1B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL2-1B\", version=\"V2.0\"\n ),\n \"InternVL2-2B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL2-2B\", version=\"V2.0\"\n ),\n \"InternVL2-4B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL2-4B\", version=\"V2.0\"\n ),\n \"InternVL2-8B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL2-8B\", version=\"V2.0\"\n ),\n \"InternVL2-26B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL2-26B\", version=\"V2.0\"\n ),\n \"InternVL2-40B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL2-40B\", version=\"V2.0\"\n ),\n \"InternVL2-76B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL2-Llama3-76B\", version=\"V2.0\"\n ),\n \"InternVL2-8B-MPO\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL2-8B-MPO\", version=\"V2.0\"\n ),\n \"InternVL2-8B-MPO-CoT\": partial(\n InternVLChat,\n model_path=\"OpenGVLab/InternVL2-8B-MPO\",\n version=\"V2.0\",\n use_mpo_prompt=True,\n ),\n}\n\ninternvl2_5 = {\n \"InternVL2_5-1B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL2_5-1B\", version=\"V2.0\"\n ),\n \"InternVL2_5-2B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL2_5-2B\", version=\"V2.0\"\n ),\n \"QTuneVL1-2B\": partial(\n InternVLChat, model_path=\"hanchaow/QTuneVL1-2B\", version=\"V2.0\"\n ),\n \"InternVL2_5-4B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL2_5-4B\", version=\"V2.0\"\n ),\n \"InternVL2_5-8B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL2_5-8B\", version=\"V2.0\"\n ),\n \"InternVL2_5-26B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL2_5-26B\", version=\"V2.0\"\n ),\n \"InternVL2_5-38B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL2_5-38B\", version=\"V2.0\"\n ),\n \"InternVL2_5-78B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL2_5-78B\", version=\"V2.0\"\n ),\n # InternVL2.5 series with Best-of-N evaluation\n \"InternVL2_5-8B-BoN-8\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL2_5-8B\", version=\"V2.0\",\n best_of_n=8, reward_model_path=\"OpenGVLab/VisualPRM-8B\",\n ),\n}\n\ninternvl2_5_mpo = {\n \"InternVL2_5-1B-MPO\": partial(\n InternVLChat,\n model_path=\"OpenGVLab/InternVL2_5-1B-MPO\",\n version=\"V2.0\",\n use_mpo_prompt=True,\n ),\n \"InternVL2_5-2B-MPO\": partial(\n InternVLChat,\n model_path=\"OpenGVLab/InternVL2_5-2B-MPO\",\n version=\"V2.0\",\n use_mpo_prompt=True,\n ),\n \"InternVL2_5-4B-MPO\": partial(\n InternVLChat,\n model_path=\"OpenGVLab/InternVL2_5-4B-MPO\",\n version=\"V2.0\",\n use_mpo_prompt=True,\n ),\n \"InternVL2_5-8B-MPO\": partial(\n InternVLChat,\n model_path=\"OpenGVLab/InternVL2_5-8B-MPO\",\n version=\"V2.0\",\n use_mpo_prompt=True,\n ),\n \"InternVL2_5-26B-MPO\": partial(\n InternVLChat,\n model_path=\"OpenGVLab/InternVL2_5-26B-MPO\",\n version=\"V2.0\",\n use_mpo_prompt=True,\n ),\n \"InternVL2_5-38B-MPO\": partial(\n InternVLChat,\n model_path=\"OpenGVLab/InternVL2_5-38B-MPO\",\n version=\"V2.0\",\n use_mpo_prompt=True,\n ),\n \"InternVL2_5-78B-MPO\": partial(\n InternVLChat,\n model_path=\"OpenGVLab/InternVL2_5-78B-MPO\",\n version=\"V2.0\",\n use_mpo_prompt=True,\n ),\n \"InternVL2_5-8B-GUI\": partial(\n InternVLChat,\n model_path=\"/fs-computility/mllm1/shared/zhaoxiangyu/models/internvl2_5_8b_internlm2_5_7b_dynamic_res_stage1\", \n version=\"V2.0\", \n max_new_tokens=512,\n screen_parse=False,\n ),\n \"InternVL3-7B-GUI\": partial(\n InternVLChat,\n model_path=\"/fs-computility/mllm1/shared/zhaoxiangyu/GUI/checkpoints/internvl3_7b_dynamic_res_stage1_56/\", \n version=\"V2.0\", \n max_new_tokens=512,\n screen_parse=False,\n ),\n}\n\ninternvl3 = {\n \"InternVL3-1B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL3-1B\", version=\"V2.0\"\n ),\n \"InternVL3-2B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL3-2B\", version=\"V2.0\"\n ),\n \"InternVL3-8B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL3-8B\", version=\"V2.0\",\n ),\n \"InternVL3-9B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL3-9B\", version=\"V2.0\"\n ),\n \"InternVL3-14B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL3-14B\", version=\"V2.0\"\n ),\n \"InternVL3-38B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL3-38B\", version=\"V2.0\"\n ),\n \"InternVL3-78B\": partial(\n InternVLChat, model_path=\"OpenGVLab/InternVL3-78B\", version=\"V2.0\"\n ),\n}\n\nsail_series = {\n \"SAIL-VL-2B\": partial(SailVL, model_path=\"BytedanceDouyinContent/SAIL-VL-2B\"),\n \"SAIL-VL-1.5-2B\": partial(SailVL, model_path=\"BytedanceDouyinContent/SAIL-VL-1d5-2B\", use_msac = True),\n \"SAIL-VL-1.5-8B\": partial(SailVL, model_path=\"BytedanceDouyinContent/SAIL-VL-1d5-8B\", use_msac = True),\n \"SAIL-VL-1.6-8B\": partial(SailVL, model_path=\"BytedanceDouyinContent/SAIL-VL-1d6-8B\", use_msac = True)\n}\n\nristretto_series = {\n \"Ristretto-3B\": partial(Ristretto, model_path=\"LiAutoAD/Ristretto-3B\"),\n}\n\nyivl_series = {\n \"Yi_VL_6B\": partial(Yi_VL, model_path=\"01-ai/Yi-VL-6B\", root=Yi_ROOT),\n \"Yi_VL_34B\": partial(Yi_VL, model_path=\"01-ai/Yi-VL-34B\", root=Yi_ROOT),\n}\n\nxcomposer_series = {\n \"XComposer\": partial(XComposer, model_path=\"internlm/internlm-xcomposer-vl-7b\"),\n \"sharecaptioner\": partial(ShareCaptioner, model_path=\"Lin-Chen/ShareCaptioner\"),\n \"XComposer2\": partial(XComposer2, model_path=\"internlm/internlm-xcomposer2-vl-7b\"),\n \"XComposer2_1.8b\": partial(\n XComposer2, model_path=\"internlm/internlm-xcomposer2-vl-1_8b\"\n ),\n \"XComposer2_4KHD\": partial(\n XComposer2_4KHD, model_path=\"internlm/internlm-xcomposer2-4khd-7b\"\n ),\n \"XComposer2d5\": partial(\n XComposer2d5, model_path=\"internlm/internlm-xcomposer2d5-7b\"\n ),\n}\n\nminigpt4_series = {\n \"MiniGPT-4-v2\": partial(MiniGPT4, mode=\"v2\", root=MiniGPT4_ROOT),\n \"MiniGPT-4-v1-7B\": partial(MiniGPT4, mode=\"v1_7b\", root=MiniGPT4_ROOT),\n \"MiniGPT-4-v1-13B\": partial(MiniGPT4, mode=\"v1_13b\", root=MiniGPT4_ROOT),\n}\n\nidefics_series = {\n \"idefics_9b_instruct\": partial(\n IDEFICS, model_path=\"HuggingFaceM4/idefics-9b-instruct\"\n ),\n \"idefics_80b_instruct\": partial(\n IDEFICS, model_path=\"HuggingFaceM4/idefics-80b-instruct\"\n ),\n \"idefics2_8b\": partial(IDEFICS2, model_path=\"HuggingFaceM4/idefics2-8b\"),\n # Idefics3 follows Idefics2 Pattern\n \"Idefics3-8B-Llama3\": partial(\n IDEFICS2, model_path=\"HuggingFaceM4/Idefics3-8B-Llama3\"\n ),\n}\n\nsmolvlm_series = {\n \"SmolVLM-256M\": partial(SmolVLM, model_path=\"HuggingFaceTB/SmolVLM-256M-Instruct\"),\n \"SmolVLM-500M\": partial(SmolVLM, model_path=\"HuggingFaceTB/SmolVLM-500M-Instruct\"),\n \"SmolVLM\": partial(SmolVLM, model_path=\"HuggingFaceTB/SmolVLM-Instruct\"),\n \"SmolVLM-DPO\": partial(SmolVLM, model_path=\"HuggingFaceTB/SmolVLM-Instruct-DPO\"),\n \"SmolVLM-Synthetic\": partial(SmolVLM, model_path=\"HuggingFaceTB/SmolVLM-Synthetic\"),\n \"SmolVLM2-256M\": partial(\n SmolVLM2, model_path=\"HuggingFaceTB/SmolVLM2-256M-Video-Instruct\"\n ),\n \"SmolVLM2-500M\": partial(\n SmolVLM2, model_path=\"HuggingFaceTB/SmolVLM2-500M-Video-Instruct\"\n ),\n \"SmolVLM2\": partial(SmolVLM2, model_path=\"HuggingFaceTB/SmolVLM2-2.2B-Instruct\"),\n}\n\ninstructblip_series = {\n \"instructblip_7b\": partial(InstructBLIP, name=\"instructblip_7b\"),\n \"instructblip_13b\": partial(InstructBLIP, name=\"instructblip_13b\"),\n}\n\ndeepseekvl_series = {\n \"deepseek_vl_7b\": partial(DeepSeekVL, model_path=\"deepseek-ai/deepseek-vl-7b-chat\"),\n \"deepseek_vl_1.3b\": partial(\n DeepSeekVL, model_path=\"deepseek-ai/deepseek-vl-1.3b-chat\"\n ),\n}\n\ndeepseekvl2_series = {\n \"deepseek_vl2_tiny\": partial(\n DeepSeekVL2, model_path=\"deepseek-ai/deepseek-vl2-tiny\"\n ),\n \"deepseek_vl2_small\": partial(\n DeepSeekVL2, model_path=\"deepseek-ai/deepseek-vl2-small\"\n ),\n \"deepseek_vl2\": partial(DeepSeekVL2, model_path=\"deepseek-ai/deepseek-vl2\"),\n}\n\njanus_series = {\n \"Janus-1.3B\": partial(Janus, model_path=\"deepseek-ai/Janus-1.3B\"),\n \"Janus-Pro-1B\": partial(Janus, model_path=\"deepseek-ai/Janus-Pro-1B\"),\n \"Janus-Pro-7B\": partial(Janus, model_path=\"deepseek-ai/Janus-Pro-7B\"),\n}\n\ncogvlm_series = {\n \"cogvlm-grounding-generalist\": partial(\n CogVlm,\n model_path=\"THUDM/cogvlm-grounding-generalist-hf\",\n tokenizer_name=\"lmsys/vicuna-7b-v1.5\",\n ),\n \"cogvlm-chat\": partial(\n CogVlm, model_path=\"THUDM/cogvlm-chat-hf\", tokenizer_name=\"lmsys/vicuna-7b-v1.5\"\n ),\n \"cogvlm2-llama3-chat-19B\": partial(\n CogVlm, model_path=\"THUDM/cogvlm2-llama3-chat-19B\"\n ),\n \"glm-4v-9b\": partial(GLM4v, model_path=\"THUDM/glm-4v-9b\"),\n}\n\nwemm_series = {\n \"WeMM\": partial(WeMM, model_path=\"feipengma/WeMM\"),\n}\n\ncambrian_series = {\n \"cambrian_8b\": partial(Cambrian, model_path=\"nyu-visionx/cambrian-8b\"),\n \"cambrian_13b\": partial(Cambrian, model_path=\"nyu-visionx/cambrian-13b\"),\n \"cambrian_34b\": partial(Cambrian, model_path=\"nyu-visionx/cambrian-34b\"),\n}\n\nchameleon_series = {\n \"chameleon_7b\": partial(Chameleon, model_path=\"facebook/chameleon-7b\"),\n \"chameleon_30b\": partial(Chameleon, model_path=\"facebook/chameleon-30b\"),\n}\n\nvila_series = {\n \"VILA1.5-3b\": partial(VILA, model_path=\"Efficient-Large-Model/VILA1.5-3b\"),\n \"Llama-3-VILA1.5-8b\": partial(\n VILA, model_path=\"Efficient-Large-Model/Llama-3-VILA1.5-8b\"\n ),\n \"VILA1.5-13b\": partial(VILA, model_path=\"Efficient-Large-Model/VILA1.5-13b\"),\n \"VILA1.5-40b\": partial(VILA, model_path=\"Efficient-Large-Model/VILA1.5-40b\"),\n \"NVILA-8B\": partial(NVILA, model_path=\"Efficient-Large-Model/NVILA-8B\"),\n \"NVILA-15B\": partial(NVILA, model_path=\"Efficient-Large-Model/NVILA-15B\"),\n}\n\novis_series = {\n \"Ovis1.5-Llama3-8B\": partial(Ovis, model_path=\"AIDC-AI/Ovis1.5-Llama3-8B\"),\n \"Ovis1.5-Gemma2-9B\": partial(Ovis, model_path=\"AIDC-AI/Ovis1.5-Gemma2-9B\"),\n \"Ovis1.6-Gemma2-9B\": partial(Ovis1_6, model_path=\"AIDC-AI/Ovis1.6-Gemma2-9B\"),\n \"Ovis1.6-Llama3.2-3B\": partial(Ovis1_6, model_path=\"AIDC-AI/Ovis1.6-Llama3.2-3B\"),\n \"Ovis1.6-Gemma2-27B\": partial(\n Ovis1_6_Plus, model_path=\"AIDC-AI/Ovis1.6-Gemma2-27B\"\n ),\n \"Ovis2-1B\": partial(Ovis2, model_path=\"AIDC-AI/Ovis2-1B\"),\n \"Ovis2-2B\": partial(Ovis2, model_path=\"AIDC-AI/Ovis2-2B\"),\n \"Ovis2-4B\": partial(Ovis2, model_path=\"AIDC-AI/Ovis2-4B\"),\n \"Ovis2-8B\": partial(Ovis2, model_path=\"AIDC-AI/Ovis2-8B\"),\n \"Ovis2-16B\": partial(Ovis2, model_path=\"AIDC-AI/Ovis2-16B\"),\n \"Ovis2-34B\": partial(Ovis2, model_path=\"AIDC-AI/Ovis2-34B\"),\n \"Ovis-U1-3B\": partial(OvisU1, model_path=\"AIDC-AI/Ovis-U1-3B\"),\n}\n\nmantis_series = {\n \"Mantis-8B-siglip-llama3\": partial(\n Mantis, model_path=\"TIGER-Lab/Mantis-8B-siglip-llama3\"\n ),\n \"Mantis-8B-clip-llama3\": partial(\n Mantis, model_path=\"TIGER-Lab/Mantis-8B-clip-llama3\"\n ),\n \"Mantis-8B-Idefics2\": partial(Mantis, model_path=\"TIGER-Lab/Mantis-8B-Idefics2\"),\n \"Mantis-8B-Fuyu\": partial(Mantis, model_path=\"TIGER-Lab/Mantis-8B-Fuyu\"),\n}\n\nphi3_series = {\n \"Phi-3-Vision\": partial(\n Phi3Vision, model_path=\"microsoft/Phi-3-vision-128k-instruct\"\n ),\n \"Phi-3.5-Vision\": partial(\n Phi3_5Vision, model_path=\"microsoft/Phi-3.5-vision-instruct\"\n ),\n}\n\nphi4_series = {\n 'Phi-4-Vision': partial(Phi4Multimodal, model_path='microsoft/Phi-4-multimodal-instruct'),\n}\n\nxgen_mm_series = {\n \"xgen-mm-phi3-interleave-r-v1.5\": partial(\n XGenMM, model_path=\"Salesforce/xgen-mm-phi3-mini-instruct-interleave-r-v1.5\"\n ),\n \"xgen-mm-phi3-dpo-r-v1.5\": partial(\n XGenMM, model_path=\"Salesforce/xgen-mm-phi3-mini-instruct-dpo-r-v1.5\"\n ),\n}\n\nhawkvl_series = {\n \"HawkVL-2B\": partial(\n HawkVL,\n model_path=\"xjtupanda/HawkVL-2B\",\n min_pixels=4 * 28 * 28,\n max_pixels=6800 * 28 * 28,\n use_custom_prompt=True\n )\n}\n\nqwen2vl_series = {\n \"Qwen-VL-Max-0809\": partial(\n Qwen2VLAPI,\n model=\"qwen-vl-max-0809\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n ),\n \"Qwen-VL-Plus-0809\": partial(\n Qwen2VLAPI,\n model=\"qwen-vl-plus-0809\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n ),\n \"QVQ-72B-Preview\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/QVQ-72B-Preview\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n system_prompt=\"You are a helpful and harmless assistant. You are Qwen developed by Alibaba. You should think step-by-step.\",\n max_new_tokens=8192,\n post_process=False,\n ),\n \"Qwen2-VL-72B-Instruct\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2-VL-72B-Instruct\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n ),\n \"Qwen2-VL-7B-Instruct\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2-VL-7B-Instruct\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n ),\n \"Qwen2-VL-7B-Instruct-AWQ\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2-VL-7B-Instruct-AWQ\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n ),\n \"Qwen2-VL-7B-Instruct-GPTQ-Int4\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int4\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n ),\n \"Qwen2-VL-7B-Instruct-GPTQ-Int8\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int8\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n ),\n \"Qwen2-VL-2B-Instruct\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2-VL-2B-Instruct\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n ),\n \"Qwen2-VL-2B-Instruct-AWQ\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2-VL-2B-Instruct-AWQ\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n ),\n \"Qwen2-VL-2B-Instruct-GPTQ-Int4\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int4\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n ),\n \"Qwen2-VL-2B-Instruct-GPTQ-Int8\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int8\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n ),\n \"XinYuan-VL-2B-Instruct\": partial(\n Qwen2VLChat,\n model_path=\"Cylingo/Xinyuan-VL-2B\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n ),\n \"Qwen2.5-VL-3B-Instruct\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2.5-VL-3B-Instruct\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n use_custom_prompt=False,\n ),\n \"Qwen2.5-VL-3B-Instruct-AWQ\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2.5-VL-3B-Instruct-AWQ\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n use_custom_prompt=False,\n ),\n \"Qwen2.5-VL-7B-Instruct\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2.5-VL-7B-Instruct\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n use_custom_prompt=False,\n ),\n \"Qwen2.5-VL-7B-Instruct-ForVideo\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2.5-VL-7B-Instruct\",\n min_pixels=128 * 28 * 28,\n max_pixels=768 * 28 * 28,\n total_pixels=24576 * 28 * 28,\n use_custom_prompt=False,\n ),\n \"Qwen2.5-VL-7B-Instruct-AWQ\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2.5-VL-7B-Instruct-AWQ\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n use_custom_prompt=False,\n ),\n \"Qwen2.5-VL-32B-Instruct\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2.5-VL-32B-Instruct\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n use_custom_prompt=False,\n ),\n \"Qwen2.5-VL-72B-Instruct\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2.5-VL-72B-Instruct\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n use_custom_prompt=False,\n ),\n \"MiMo-VL-7B-SFT\": partial(\n Qwen2VLChat,\n model_path=\"XiaomiMiMo/MiMo-VL-7B-SFT\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n use_custom_prompt=False,\n use_lmdeploy=True\n ),\n \"MiMo-VL-7B-RL\": partial(\n Qwen2VLChat,\n model_path=\"XiaomiMiMo/MiMo-VL-7B-RL\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n use_custom_prompt=False,\n use_lmdeploy=True\n ),\n \"Qwen2.5-VL-72B-Instruct-ForVideo\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2.5-VL-72B-Instruct\",\n min_pixels=128 * 28 * 28,\n max_pixels=768 * 28 * 28,\n total_pixels=24576 * 28 * 28,\n use_custom_prompt=False,\n ),\n \"Qwen2.5-VL-72B-Instruct-AWQ\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2.5-VL-72B-Instruct-AWQ\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n use_custom_prompt=False,\n ),\n \"Qwen2.5-Omni-7B-ForVideo\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2.5-Omni-7B\",\n min_pixels=128 * 28 * 28,\n max_pixels=768 * 28 * 28,\n total_pixels=24576 * 28 * 28,\n use_custom_prompt=False,\n use_audio_in_video=True, # set use audio in video\n ),\n \"Qwen2.5-Omni-7B\": partial(\n Qwen2VLChat,\n model_path=\"Qwen/Qwen2.5-Omni-7B\",\n min_pixels=1280 * 28 * 28,\n max_pixels=16384 * 28 * 28,\n use_custom_prompt=False,\n ),\n 'VLM-R1': partial(\n VLMR1Chat, \n model_path='omlab/VLM-R1-Qwen2.5VL-3B-Math-0305', \n min_pixels=1280*28*28, \n max_pixels=16384*28*28, \n use_custom_prompt=False),\n 'VLAA-Thinker-Qwen2.5VL-3B': partial(\n VLAAThinkerChat, \n model_path='UCSC-VLAA/VLAA-Thinker-Qwen2.5VL-3B', \n min_pixels=1280*28*28, \n max_pixels=16384*28*28, \n use_custom_prompt=False,\n post_process=True, # post processing for evaluation\n system_prompt=(''\n \"You are VL-Thinking🤔, a helpful assistant with excellent reasoning ability.\"\n \" A user asks you a question, and you should try to solve it.\"\n \" You should first think about the reasoning process in the mind and then provides the user with the answer.\"\n \" The reasoning process and answer are enclosed within <think> </think> and\"\n \"<answer> </answer> tags, respectively, i.e., <think> reasoning process here </think>\"\n \"<answer> answer here </answer>\"\n ),\n ),\n 'VLAA-Thinker-Qwen2.5VL-7B': partial(\n VLAAThinkerChat, \n model_path='UCSC-VLAA/VLAA-Thinker-Qwen2.5VL-7B', \n min_pixels=1280*28*28, \n max_pixels=16384*28*28, \n use_custom_prompt=False,\n post_process=True, # post processing for evaluation\n system_prompt=(''\n \"You are VL-Thinking🤔, a helpful assistant with excellent reasoning ability.\"\n \" A user asks you a question, and you should try to solve it.\"\n \" You should first think about the reasoning process in the mind and then provides the user with the answer.\"\n \" The reasoning process and answer are enclosed within <think> </think> and\"\n \"<answer> </answer> tags, respectively, i.e., <think> reasoning process here </think>\"\n \"<answer> answer here </answer>\"\n ),\n ),\n 'WeThink-Qwen2.5VL-7B': partial(\n WeThinkVL, \n model_path='yangjie-cv/WeThink-Qwen2.5VL-7B', \n min_pixels=1280*28*28, \n max_pixels=16384*28*28, \n use_custom_prompt=False,\n system_prompt=(\"You FIRST think about the reasoning process as an internal monologue and then provide the final answer.\\nThe reasoning process MUST BE enclosed within <think> </think> tags. The final answer MUST BE enclosed within <answer> </answer> tags.\"\n ),\n ),\n}\n\nslime_series = {\n \"Slime-7B\": partial(SliME, model_path=\"yifanzhang114/SliME-vicuna-7B\"),\n \"Slime-8B\": partial(SliME, model_path=\"yifanzhang114/SliME-Llama3-8B\"),\n \"Slime-13B\": partial(SliME, model_path=\"yifanzhang114/SliME-vicuna-13B\"),\n}\n\neagle_series = {\n \"Eagle-X4-8B-Plus\": partial(Eagle, model_path=\"NVEagle/Eagle-X4-8B-Plus\"),\n \"Eagle-X4-13B-Plus\": partial(Eagle, model_path=\"NVEagle/Eagle-X4-13B-Plus\"),\n \"Eagle-X5-7B\": partial(Eagle, model_path=\"NVEagle/Eagle-X5-7B\"),\n \"Eagle-X5-13B\": partial(Eagle, model_path=\"NVEagle/Eagle-X5-13B\"),\n \"Eagle-X5-13B-Chat\": partial(Eagle, model_path=\"NVEagle/Eagle-X5-13B-Chat\"),\n \"Eagle-X5-34B-Chat\": partial(Eagle, model_path=\"NVEagle/Eagle-X5-34B-Chat\"),\n \"Eagle-X5-34B-Plus\": partial(Eagle, model_path=\"NVEagle/Eagle-X5-34B-Plus\"),\n}\n\nmoondream_series = {\n \"Moondream1\": partial(Moondream1, model_path=\"vikhyatk/moondream1\"),\n \"Moondream2\": partial(Moondream2, model_path=\"vikhyatk/moondream2\"),\n}\n\nllama_series = {\n \"Llama-3.2-11B-Vision-Instruct\": partial(\n llama_vision, model_path=\"meta-llama/Llama-3.2-11B-Vision-Instruct\"\n ),\n \"LLaVA-CoT\": partial(llama_vision, model_path=\"Xkev/Llama-3.2V-11B-cot\"),\n \"Llama-3.2-90B-Vision-Instruct\": partial(\n llama_vision, model_path=\"meta-llama/Llama-3.2-90B-Vision-Instruct\"\n ),\n \"Llama-4-Scout-17B-16E-Instruct\": partial(\n llama4, model_path=\"meta-llama/Llama-4-Scout-17B-16E-Instruct\", use_vllm=True\n ),\n}\n\nmolmo_series = {\n \"molmoE-1B-0924\": partial(molmo, model_path=\"allenai/MolmoE-1B-0924\"),\n \"molmo-7B-D-0924\": partial(molmo, model_path=\"allenai/Molmo-7B-D-0924\"),\n \"molmo-7B-O-0924\": partial(molmo, model_path=\"allenai/Molmo-7B-O-0924\"),\n \"molmo-72B-0924\": partial(molmo, model_path=\"allenai/Molmo-72B-0924\"),\n}\n\nkosmos_series = {\n \"Kosmos2\": partial(Kosmos2, model_path=\"microsoft/kosmos-2-patch14-224\")\n}\n\npoints_series = {\n \"POINTS-Yi-1.5-9B-Chat\": partial(\n POINTS, model_path=\"WePOINTS/POINTS-Yi-1-5-9B-Chat\"\n ),\n \"POINTS-Qwen-2.5-7B-Chat\": partial(\n POINTS, model_path=\"WePOINTS/POINTS-Qwen-2-5-7B-Chat\"\n ),\n \"POINTSV15-Qwen-2.5-7B-Chat\": partial(\n POINTSV15, model_path=\"WePOINTS/POINTS-1-5-Qwen-2-5-7B-Chat\"\n ),\n}\n\nnvlm_series = {\n \"NVLM\": partial(NVLM, model_path=\"nvidia/NVLM-D-72B\"),\n}\n\nvintern_series = {\n \"Vintern-3B-beta\": partial(VinternChat, model_path=\"5CD-AI/Vintern-3B-beta\"),\n \"Vintern-1B-v2\": partial(VinternChat, model_path=\"5CD-AI/Vintern-1B-v2\"),\n}\n\naria_series = {\"Aria\": partial(Aria, model_path=\"rhymes-ai/Aria\")}\n\nh2ovl_series = {\n \"h2ovl-mississippi-2b\": partial(H2OVLChat, model_path=\"h2oai/h2ovl-mississippi-2b\"),\n \"h2ovl-mississippi-1b\": partial(\n H2OVLChat, model_path=\"h2oai/h2ovl-mississippi-800m\"\n ),\n}\n\nvalley_series = {\n \"valley2\": partial(\n Valley2Chat, model_path=\"bytedance-research/Valley-Eagle-7B\"\n ),\n \"valley2_dpo\": partial(\n Valley2Chat, model_path=\"bytedance-research/Valley2-DPO\"\n ),\n}\n\nola_series = {\n \"ola\": partial(Ola, model_path=\"THUdyh/Ola-7b\"),\n}\n\nross_series = {\n \"ross-qwen2-7b\": partial(Ross, model_path=\"HaochenWang/ross-qwen2-7b\"),\n}\n\nursa_series = {\n \"URSA-8B\": partial(UrsaChat, model_path=\"URSA-MATH/URSA-8B\"),\n \"URSA-8B-PS-GRPO\": partial(UrsaChat, model_path=\"URSA-MATH/URSA-8B-PS-GRPO\") \n}\n\ngemma_series = {\n \"paligemma-3b-mix-448\": partial(\n PaliGemma, model_path=\"google/paligemma-3b-mix-448\"\n ),\n 'Gemma3-4B': partial(Gemma3, model_path='google/gemma-3-4b-it'),\n 'Gemma3-12B': partial(Gemma3, model_path='google/gemma-3-12b-it'),\n 'Gemma3-27B': partial(Gemma3, model_path='google/gemma-3-27b-it')\n}\n\nlumina_series = {\n \"Lumina_DiMOO\": partial(\n Lumina_DiMOO, \n model_path='Alpha-VLLM/Lumina-DiMOO',\n vae_ckpt='Alpha-VLLM/Lumina-DiMOO',\n use_custom_prompt=False,\n verbose=True,\n gen_length=1024,\n block_length=1024,\n steps=1,\n ),\n}\n\naguvis_series = {\n \"aguvis_7b\": partial(\n Qwen2VLChatAguvis,\n model_path=os.getenv(\n \"EVAL_MODEL\",\n \"xlangai/Aguvis-7B-720P\",\n ),\n min_pixels=256 * 28 * 28,\n max_pixels=46 * 26 * 28 * 28,\n use_custom_prompt=False,\n mode='grounding',\n )\n}\n\nkimi_series = {\n 'Kimi-VL-A3B-Thinking': partial(KimiVL, model_path='moonshotai/Kimi-VL-A3B-Thinking'),\n 'Kimi-VL-A3B-Instruct': partial(KimiVL, model_path='moonshotai/Kimi-VL-A3B-Instruct'),\n 'Kimi-VL-A3B-Thinking-2506': partial(KimiVL, model_path='moonshotai/Kimi-VL-A3B-Thinking-2506', temperature=0.8, max_tokens=32768, extract_summary=True)\n}\n\nflash_vl = {\n 'Flash-VL-2B-Dynamic-ISS': partial(FlashVL, model_path='FlashVL/FlashVL-2B-Dynamic-ISS')\n}\n\n\noryx_series = {\n 'oryx': partial(Oryx, model_path=\"THUdyh/Oryx-1.5-7B\"),\n}\n\n# recommend: vllm serve moonshotai/Kimi-VL-A3B-Thinking-2506 \n# --served-model-name api-kimi-vl-thinking-2506 --trust-remote-code\n# --tensor-parallel-size 2 --max-num-batched-tokens 131072 \n# --max-model-len 131072 --limit-mm-per-prompt image=256\nkimi_vllm_series = {\n \"api-kimi-vl-thinking-2506\": partial(\n KimiVLAPI,\n model=\"api-kimi-vl-thinking-2506\",\n ),\n \"api-kimi-vl-thinking\": partial(\n KimiVLAPI,\n model=\"api-kimi-vl-thinking\",\n ),\n \"api-kimi-vl\": partial(\n KimiVLAPI,\n model=\"api-kimi-vl\",\n max_new_tokens=2048,\n temperature=0,\n ),\n}\n\n\ninternvl_groups = [\n internvl, internvl2, internvl2_5, mini_internvl, internvl2_5_mpo, \n internvl3,\n]\ninternvl_series = {}\nfor group in internvl_groups:\n internvl_series.update(group)\n\nsupported_VLM = {}\n\nmodel_groups = [\n ungrouped, o1_apis, api_models, xtuner_series, qwen_series, llava_series, granite_vision_series,\n internvl_series, yivl_series, xcomposer_series, minigpt4_series, \n idefics_series, instructblip_series, deepseekvl_series, deepseekvl2_series, \n janus_series, minicpm_series, cogvlm_series, wemm_series, cambrian_series, \n chameleon_series, video_models, ovis_series, vila_series, mantis_series,\n mmalaya_series, phi3_series, phi4_series, xgen_mm_series, qwen2vl_series,\n slime_series, eagle_series, moondream_series, llama_series, molmo_series,\n kosmos_series, points_series, nvlm_series, vintern_series, h2ovl_series,\n aria_series, smolvlm_series, sail_series, valley_series, vita_series,\n ross_series, emu_series, ola_series, ursa_series, gemma_series,\n long_vita_series, ristretto_series, kimi_series, aguvis_series, hawkvl_series, \n flash_vl, kimi_vllm_series, oryx_series, lumina_series\n]\n\nfor grp in model_groups:\n supported_VLM.update(grp)\n"}}, "VLMEvalKit/.env": {"81344e60545ae8d70f11039d3450717d974f831b": {"t": 1785674201.7747347, "n": 476, "h": "13f2939508711ae9", "c": "# .env 文件，将其放置在 $VLMEvalKit 下\n# 专有 VLMs 的 API 密钥\n# QwenVL APIs\nDASHSCOPE_API_KEY=\n# Gemini w. Google Cloud Backends\nGOOGLE_API_KEY=\n# OpenAI API\nOPENAI_API_KEY=\nOPENAI_API_BASE=\n# StepAI API\nSTEPAI_API_KEY=\n# REKA API\nREKA_API_KEY=\n# GLMV API\nGLMV_API_KEY=\n# CongRong API\nCW_API_BASE=\nCW_API_KEY=\n# SenseNova API\nSENSENOVA_API_KEY=\n# Hunyuan-Vision API\nHUNYUAN_SECRET_KEY=\nHUNYUAN_SECRET_ID=\n# LMDeploy API\nLMDEPLOY_API_BASE=\n# 你可以设置一个评估时代理，评估阶段产生的 API 调用将通过这个代理进行\nEVAL_PROXY="}}, "VLMEvalKit/vlmeval/api/sensechat_vision.py": {"81344e60545ae8d70f11039d3450717d974f831b": {"t": 1785809856.845995, "n": 11609, "h": "d9a350f5241c67c4", "c": "import os\nimport string\nimport time\nfrom typing import Optional\nimport pandas as pd\nimport requests\nfrom vlmeval.smp import (\n LMUDataRoot,\n osp,\n read_ok,\n decode_base64_to_image_file,\n toliststr,\n listinstr,\n cn_string,\n)\nfrom vlmeval.api.base import BaseAPI\nfrom vlmeval.dataset import img_root_map\nfrom vlmeval.dataset import DATASET_TYPE\n\n\nclass SenseChatVisionWrapper(BaseAPI):\n is_api: bool = True\n\n def __init__(\n self,\n base_url: str = \"https://api.sensenova.cn/v1/llm/chat-completions\",\n api_key: str = None,\n model: str = \"SenseNova-V6-Pro\",\n retry: int = 5,\n wait: int = 5,\n verbose: bool = True,\n system_prompt: str = None,\n max_tokens: int = 8192,\n **kwargs,\n ):\n self.base_url = base_url\n self.model = model\n self.fail_msg = \"Failed to obtain answer via API. \"\n self.api_key = os.getenv(\"SENSENOVA_API_KEY\", api_key)\n assert self.api_key is not None, (\n \"Please set the `SENSENOVA_API_KEY` environment variable or pass `api_key` in the config.json.\"\n )\n self.max_new_tokens = max_tokens\n super().__init__(\n wait=wait,\n retry=retry,\n system_prompt=system_prompt,\n verbose=verbose,\n **kwargs,\n )\n\n def dump_image(self, line, dataset):\n \"\"\"Dump the image(s) of the input line to the corresponding dataset folder.\n\n Args:\n line (line of pd.DataFrame): The raw input line.\n dataset (str): The name of the dataset.\n\n Returns:\n str | list[str]: The paths of the dumped images.\n \"\"\"\n ROOT = LMUDataRoot()\n assert isinstance(dataset, str)\n img_root = osp.join(ROOT, \"images\", img_root_map(dataset))\n os.makedirs(img_root, exist_ok=True)\n if \"image\" in line:\n if isinstance(line[\"image\"], list):\n tgt_path = []\n assert \"image_path\" in line\n for img, im_name in zip(line[\"image\"], line[\"image_path\"]):\n path = osp.join(img_root, im_name)\n if not read_ok(path):\n decode_base64_to_image_file(img, path)\n tgt_path.append(path)\n else:\n tgt_path = osp.join(img_root, f\"{line['index']}.jpg\")\n if not read_ok(tgt_path):\n decode_base64_to_image_file(line[\"image\"], tgt_path)\n tgt_path = [tgt_path]\n else:\n assert \"image_path\" in line\n tgt_path = toliststr(line[\"image_path\"])\n\n return tgt_path\n\n def image_to_base64(self, image_path):\n import base64\n\n with open(image_path, \"rb\") as image_file:\n encoded_string = base64.b64encode(image_file.read())\n return encoded_string.decode(\"utf-8\")\n\n def use_custom_prompt(self, *args, **kwargs):\n \"\"\"Check if the prompt is customized.\"\"\"\n return True\n\n def build_multi_choice_prompt(self, line, dataset=None):\n question = line[\"question\"]\n hint = line[\"hint\"] if (\"hint\" in line and not pd.isna(line[\"hint\"])) else None\n if hint is not None:\n question = hint + \"\\n\" + question\n\n options = {\n cand: line[cand]\n for cand in string.ascii_uppercase\n if cand in line and not pd.isna(line[cand])\n }\n for key, item in options.items():\n question += f\"\\n{key}. {item}\"\n prompt = question\n\n if len(options):\n prompt += (\n \"\\n请直接回答选项字母。\"\n if cn_string(prompt)\n else \"\\nAnswer with the option's letter from the given choices directly.\"\n )\n else:\n prompt += (\n \"\\n请直接回答问题。\"\n if cn_string(prompt)\n else \"\\nAnswer the question directly.\"\n )\n\n return prompt\n\n def build_mcq_cot_prompt(self, line, prompt):\n question = line[\"question\"]\n options = {\n cand: line[cand]\n for cand in string.ascii_uppercase\n if cand in line and not pd.isna(line[cand])\n }\n for key, item in options.items():\n question += f'\\n{key}. {item}'\n prompt = {\n'multiple-choice': \"You are an expert in {}. Please solve the university-level {} examination question, which includes interleaved images and text. Answer the preceding multiple choice question. The last line of your response should follow this format: 'Answer: \\\\boxed LETTER', where LETTER is one of the options. If you are uncertain or the problem is too complex, make a reasoned guess based on the information provided. Avoid repeating steps indefinitely—provide your best guess even if unsure. Think step by step logically, considering all relevant information before answering.\", # noqa: E122, E501\n'open': 'You are an expert in {}. Please solve the university-level {} examination question, which includes interleaved images and text. Your output should be divided into two parts: First, reason about the correct answer. Then write the answer in the following format where X is only the answer and nothing else: \"ANSWER: X\"' # noqa: E122, E501\n }\n subject = '_'.join(line['id'].split('_')[1:-1])\n prompt = prompt[line['question_type']].format(subject, subject) + '\\n' + question\n return prompt\n\n def build_prompt(self, line, dataset=None):\n assert self.use_custom_prompt(dataset)\n assert dataset is None or isinstance(dataset, str)\n\n tgt_path = self.dump_image(line, dataset)\n\n if dataset is not None and listinstr([\"MME\"], dataset):\n question = line[\"question\"]\n prompt = question + \" Answer the question using a single word or phrase.\"\n elif dataset is not None and listinstr([\"HallusionBench\"], dataset):\n question = line[\"question\"]\n prompt = (\n question\n + \" Please answer yes or no. Answer the question using a single word or phrase.\"\n )\n elif dataset is not None and DATASET_TYPE(dataset) == \"MCQ\":\n prompt = self.build_multi_choice_prompt(line, dataset)\n if \"MMMU\" in dataset:\n prompt = self.build_mcq_cot_prompt(line, prompt)\n elif dataset is not None and DATASET_TYPE(dataset) == \"VQA\":\n if \"MathVista\" in dataset:\n prompt = line[\"question\"]\n elif listinstr([\"LLaVABench\"], dataset):\n question = line[\"question\"]\n prompt = question + \"\\nAnswer this question in detail.\"\n elif listinstr([\"MMVet\"], dataset):\n prompt = line[\"question\"]\n else:\n question = line[\"question\"]\n prompt = (\n question\n + \"\\nPlease reason step by step, and put your final answer within \\\\boxed{}.\"\n )\n else:\n prompt = line[\"question\"]\n\n message = [dict(type=\"text\", value=prompt)]\n message.extend([dict(type=\"image\", value=s) for s in tgt_path])\n\n return message\n\n def message_to_promptimg(self, message, dataset=None):\n if dataset is None or listinstr([\"MMMU\", \"BLINK\"], dataset):\n prompt = \"\\n\".join([x[\"value\"] for x in message if x[\"type\"] == \"text\"])\n image = [[x[\"value\"] for x in message if x[\"type\"] == \"image\"][0]]\n else:\n prompt = \"\\n\".join([x[\"value\"] for x in message if x[\"type\"] == \"text\"])\n image = [x[\"value\"] for x in message if x[\"type\"] == \"image\"]\n return prompt, image\n\n def set_max_num(self, dataset: Optional[str] = None) -> None:\n \"\"\"Set the max_num based on the dataset.\"\"\"\n if dataset is not None and listinstr(\n [\n \"ChartQA_TEST\",\n \"MMMU_DEV_VAL\",\n \"MMMU_TEST\",\n \"MME-RealWorld\",\n \"VCR_EN\",\n \"VCR_ZH\",\n \"OCRVQA\",\n ],\n dataset,\n ):\n self.max_num = 12\n elif dataset is not None and listinstr(\n [\"DocVQA_VAL\", \"DocVQA_TEST\", \"DUDE\", \"MMLongBench_DOC\", \"SLIDEVQA\"],\n dataset,\n ):\n self.max_num = 18\n elif dataset is not None and listinstr(\n [\"InfoVQA_VAL\", \"InfoVQA_TEST\", \"OCRBench\", \"HRBench4K\", \"HRBench8K\"],\n dataset,\n ):\n self.max_num = 24\n else:\n self.max_num = 6\n\n def generate_inner(self, inputs, **kwargs) -> str:\n assert isinstance(inputs, str) or isinstance(inputs, list)\n inputs = [inputs] if isinstance(inputs, str) else inputs\n dataset = kwargs.get(\"dataset\", None)\n\n self.set_max_num(dataset=dataset)\n\n prompt, image = self.message_to_promptimg(message=inputs, dataset=dataset)\n content = [\n {\n \"image_base64\": self.image_to_base64(item),\n \"type\": \"image_base64\",\n }\n for item in image\n ]\n\n content.append(\n {\n \"text\": prompt,\n \"type\": \"text\",\n }\n )\n message = [{\n \"role\": \"system\",\n \"content\": [{\"text\": \" \", \"type\": \"text\"}]\n }]\n message.append({\"content\": content, \"role\": \"user\"})\n data = {\n \"messages\": message,\n \"max_new_tokens\": self.max_new_tokens, # 1024\n \"temperature\": 1e-5,\n \"top_k\": 1,\n \"top_p\": 0.99,\n \"repetition_penalty\": 1.05,\n \"model\": self.model,\n \"stream\": False,\n \"image_split_count\": self.max_num,\n }\n headers = {\n \"Content-type\": \"application/json\",\n \"Authorization\": self.api_key,\n }\n\n response = requests.post(\n self.base_url,\n headers=headers,\n json=data,\n )\n request_id = response.headers[\"x-request-id\"]\n self.logger.info(f\"Request-id: {request_id}\")\n\n time.sleep(1)\n try:\n assert response.status_code == 200\n response = response.json()[\"data\"][\"choices\"][0][\"message\"].strip()\n if self.verbose:\n self.logger.info(f\"inputs: {inputs}\\nanswer: {response}\")\n return 0, response, \"Succeeded! \"\n except Exception as err:\n if self.verbose:\n self.logger.error(\n \"---------------------------ERROR---------------------------\"\n )\n self.logger.error(response.json())\n self.logger.error(err)\n self.logger.error(\n \"---------------------------request_id---------------------------\"\n + request_id\n )\n self.logger.error(\n \"api error\"\n + response.json()[\"error\"][\"message\"]\n + str(\n [\n input[\"value\"] if input[\"type\"] == \"image\" else None\n for input in inputs\n ]\n )\n )\n self.logger.error(f\"The input messages are {inputs}.\")\n return -1, response.json()[\"error\"][\"message\"], \"\"\n\n\nclass SenseChatVisionAPI(SenseChatVisionWrapper):\n def generate(self, message, dataset=None):\n return super(SenseChatVisionAPI, self).generate(message, dataset=dataset)\n"}}}} |