{"repo": "aochong-li/synthetic-mid-training", "files": {"eval/tasks/SuperGPQA/infer/models/__init__.py": {"78b887402660946df2a9d312f074210706ccaeb3": {"t": 1785746509.10328, "n": 19961, "h": "a359592f5d17c9ae", "c": "import importlib\n\nclass ModelLoader:\n    def __init__(self, model_name, config, use_accel=False):\n        self.model_name = model_name\n        self.config = config\n        self.use_accel = use_accel\n        self._model = None\n\n    def _lazy_import(self, module_name, func_name):\n        if module_name.startswith('.'):\n            module_name = __package__ + module_name\n        module = importlib.import_module(module_name)\n        return getattr(module, func_name)\n\n    def load_model(self):\n        if self._model is None:\n            load_func = self._lazy_import(self.config['load'][0], self.config['load'][1])\n            if 'api' in self.config.get('call_type'):\n                self._model = load_func(\n                    self.config['model_path_or_name'], \n                    self.config['base_url'], \n                    self.config['api_key'], \n                    self.config['model'],\n                    self.config['call_type']\n                )\n            else:\n                self._model = load_func(self.model_name, self.config, self.use_accel)\n        return self._model\n\n    @property\n    def model(self):\n        return self.load_model()\n\n    @property\n    def infer(self):\n        return self._lazy_import(self.config['infer'][0], self.config['infer'][1])\n\nclass ModelRegistry:\n    def __init__(self):\n        self.models = {}\n\n    def register_model(self, name, config):\n        \"\"\"Register a model configuration.\"\"\"\n        self.models[name] = ModelLoader(name, config, use_accel=False)\n\n    def load_model(self, choice, use_accel=False):\n        \"\"\"Load a model based on the choice.\"\"\"\n        if choice in self.models:\n            self.models[choice].use_accel = use_accel\n            return self.models[choice].model\n        else:\n            raise ValueError(f\"Model choice '{choice}' is not supported.\")\n\n    def infer(self, choice):\n        \"\"\"Get the inference function for a given model.\"\"\"\n        if choice in self.models:\n            return self.models[choice].infer\n        else:\n            raise ValueError(f\"Inference choice '{choice}' is not supported.\")\n\n# Initialize model registry\nmodel_registry = ModelRegistry()\n\n# Configuration of models\nmodel_configs = {\n    ####### APi models #######\n    'gpt-4o-2024-11-20': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'GPT4o',\n        'base_url': '',\n        'api_key': '',\n        'model': 'gpt-4o-2024-11-20',\n        'call_type': 'api_chat'\n    },\n    'gpt-4o-2024-08-06': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'GPT4o-2024-08-06',\n        'base_url': '',\n        'api_key': '',\n        'model': 'gpt-4o-2024-08-06',\n        'call_type': 'api_chat'\n    },\n    'claude-3-5-sonnet-20241022': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'Claude-3-5-Sonnet-20241022',\n        'base_url': '',\n        'api_key': '',\n        'model': 'claude-3-5-sonnet-20241022',\n        'call_type': 'api_chat'\n    },\n    'o1-mini': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'o1-mini',\n        'base_url': '',\n        'api_key': '',\n        'model': 'o1-mini',\n        'call_type': 'api_chat'\n    },\n    'gemini-1.5-pro-002': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'Gemini-1.5-pro-002',\n        'base_url': '',\n        'api_key': '',\n        'model': 'gemini-1.5-pro-002',\n        'call_type': 'api_chat'\n    },\n    'gpt-4-turbo-2024-04-09': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'GPT4',\n        'base_url': '',\n        'api_key': '',\n        'model': 'gpt-4-turbo',\n        'call_type': 'api_chat'\n    },\n    'claude-3-5-sonnet-20240620': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'Claude-3-5-Sonnet-20240620',\n        'base_url': '',\n        'api_key': '',\n        'model': 'claude-3-5-sonnet-20240620',\n        'call_type': 'api_chat'\n    },\n    'gemini-2.0-flash-exp': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'Gemini-2.0-Flash-Exp',\n        'base_url': '',\n        'api_key': '',\n        'model': 'gemini-2.0-flash-exp',\n        'call_type': 'api_chat'\n    },\n    'Llama-3.1-405B': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'meta-llama/Llama-3.1-405B',\n        'base_url': 'http://127.0.0.1:8000/v1',\n        'api_key': 'none',\n        'model': 'meta-llama/Llama-3.1-405B',\n        'call_type': 'api_base'\n    },\n    'Llama-3.1-405B-Instruct': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'meta-llama/Llama-3.1-405B-Instruct',\n        'base_url': 'http://127.0.0.1:8000/v1',\n        'api_key': 'none',\n        'model': 'meta-llama/Llama-3.1-405B-Instruct',\n        'call_type': 'api_chat'\n    },\n    'o3-mini-2025-01-31': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'o3-mini-2025-01-31',\n        'base_url': '',\n        'api_key': '',\n        'model': 'o3-mini-2025-01-31',\n        'call_type': 'api_chat'\n    },\n    'Doubao-1.5-pro-32k-250115': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'Doubao-1.5-pro-32k-250115',\n        'base_url': \"\", \n        'api_key': \"\",\n        'model': \"\",\n        'call_type': 'api_chat'\n    },\n    'DeepSeek-V3': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'DeepSeek-V3',\n        'base_url': '',\n        'api_key': '',\n        'model': 'deepseek-chat',\n        'call_type': 'api_chat'\n    },\n    'DeepSeek-R1': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'DeepSeek-R1',\n        'base_url': '',\n        'api_key': '',\n        'model': 'deepseek-reasoner',\n        'call_type': 'api_chat'\n    },\n    'claude-3-7-sonnet-20250219': {\n        'load': ('.anthropic_api', 'load_model'),\n        'infer': ('.anthropic_api', 'infer'),\n        'model_path_or_name': 'claude-3-7-sonnet-20250219',\n        'base_url': '',\n        'api_key': '',\n        'model': 'claude-3-7-sonnet-20250219',\n        'call_type': 'api_chat'\n    },\n\n    ####### Local Language Aligned models #######\n    'phi-4': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'microsoft/phi-4',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'granite-3.1-8b-instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'ibm-granite/granite-3.1-8b-instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'granite-3.1-2b-instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'ibm-granite/granite-3.1-2b-instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'QwQ-32B-Preview': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'Qwen/QwQ-32B-Preview',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Qwen2.5-0.5B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-0.5B-Instruct',\n        'call_type': 'local',\n        'tp': 2\n    },\n    'Qwen2.5-1.5B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-1.5B-Instruct',\n        'call_type': 'local',\n        'tp': 4\n    },\n    'Qwen2.5-3B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-3B-Instruct',\n        'call_type': 'local',\n        'tp': 4\n    },\n    'Qwen2.5-7B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-7B-Instruct',\n        'call_type': 'local',\n        'tp': 4\n    },\n    'Qwen2.5-14B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-14B-Instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Qwen2.5-32B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-32B-Instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Qwen2.5-72B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-72B-Instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'K2-Chat': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'LLM360/K2-Chat',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'gemma-2-2b-it': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'google/gemma-2-2b-it',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'gemma-2-9b-it': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'google/gemma-2-9b-it',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'gemma-2-27b-it': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'google/gemma-2-27b-it',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Llama-3.1-8B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'meta-llama/Llama-3.1-8B-Instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Llama-3.1-70B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'meta-llama/Llama-3.1-70B-Instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Llama-3.3-70B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'meta-llama/Llama-3.3-70B-Instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Yi-1.5-6B-Chat': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': '01-ai/Yi-1.5-6B-Chat',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Yi-1.5-9B-Chat': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': '01-ai/Yi-1.5-9B-Chat',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Yi-1.5-34B-Chat': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': '01-ai/Yi-1.5-34B-Chat',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'MAP-Neo-7B-Instruct-v0.1': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'm-a-p/neo_7b_instruct_v0.1',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Mistral-7B-Instruct-v0.3': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'mistralai/Mistral-7B-Instruct-v0.3',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Mistral-Large-Instruct-2411': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'mistralai/Mistral-Large-Instruct-2411',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Mistral-Small-Instruct-2409': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'mistralai/Mistral-Small-Instruct-2409',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Mixtral-8x22B-Instruct-v0.1': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'mistralai/Mixtral-8x22B-Instruct-v0.1',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Mixtral-8x7B-Instruct-v0.1': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'mistralai/Mixtral-8x7B-Instruct-v0.1',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'OLMo-2-1124-13B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'allenai/OLMo-2-1124-13B-Instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'OLMo-2-1124-7B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'allenai/OLMo-2-1124-7B-Instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n\n    ####### Local Language Base models #######\n    'Qwen2.5-0.5B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-0.5B',\n        'call_type': 'local',\n        'tp': 2\n    },\n    'Qwen2.5-1.5B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-1.5B',\n        'call_type': 'local',\n        'tp': 4\n    },\n    'Qwen2.5-3B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-3B',\n        'call_type': 'local',\n        'tp': 4\n    },\n    'Qwen2.5-7B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-7B',\n        'call_type': 'local',\n        'tp': 4\n    },\n    'Qwen2.5-14B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-14B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Qwen2.5-32B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-32B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Qwen2.5-72B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-72B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'K2': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'LLM360/K2',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'gemma-2-2b': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'google/gemma-2-2b',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'gemma-2-9b': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'google/gemma-2-9b',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'gemma-2-27b': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'google/gemma-2-27b',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Llama-3.1-8B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'meta-llama/Llama-3.1-8B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Llama-3.1-70B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'meta-llama/Llama-3.1-70B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Yi-1.5-6B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': '01-ai/Yi-1.5-6B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Yi-1.5-9B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': '01-ai/Yi-1.5-9B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Yi-1.5-34B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': '01-ai/Yi-1.5-34B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'MAP-Neo-7B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'm-a-p/neo_7b',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Mistral-7B-v0.3': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'mistralai/Mistral-7B-v0.3',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Mixtral-8x22B-v0.1': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'mistralai/Mixtral-8x22B-v0.1',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Mixtral-8x7B-v0.1': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'mistralai/Mixtral-8x7B-v0.1',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'OLMo-2-1124-13B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'allenai/OLMo-2-1124-13B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'OLMo-2-1124-7B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'allenai/OLMo-2-1124-7B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'granite-3.1-2b-base': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'ibm-granite/granite-3.1-2b-base',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'granite-3.1-8b-base': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'ibm-granite/granite-3.1-8b-base',\n        'call_type': 'local',\n        'tp': 8\n    },\n}\n\n# # Register all models\n# for model_name, config in model_configs.items():\n#     model_registry.register_model(model_name, config)\n\ndef load_model(choice, use_accel=False):\n    \"\"\"Load a specific model based on the choice.\"\"\"\n    model_registry.register_model(choice, model_configs[choice])\n    return model_registry.load_model(choice, use_accel)\n\ndef infer(choice):\n    \"\"\"Get the inference function for a specific model.\"\"\"\n    return model_registry.infer(choice)\n\n"}, "9df7b97b8ff4bd9b052554404f9c55b874babe53": {"t": 1785765088.2473161, "n": 19961, "h": "a359592f5d17c9ae", "c": "import importlib\n\nclass ModelLoader:\n    def __init__(self, model_name, config, use_accel=False):\n        self.model_name = model_name\n        self.config = config\n        self.use_accel = use_accel\n        self._model = None\n\n    def _lazy_import(self, module_name, func_name):\n        if module_name.startswith('.'):\n            module_name = __package__ + module_name\n        module = importlib.import_module(module_name)\n        return getattr(module, func_name)\n\n    def load_model(self):\n        if self._model is None:\n            load_func = self._lazy_import(self.config['load'][0], self.config['load'][1])\n            if 'api' in self.config.get('call_type'):\n                self._model = load_func(\n                    self.config['model_path_or_name'], \n                    self.config['base_url'], \n                    self.config['api_key'], \n                    self.config['model'],\n                    self.config['call_type']\n                )\n            else:\n                self._model = load_func(self.model_name, self.config, self.use_accel)\n        return self._model\n\n    @property\n    def model(self):\n        return self.load_model()\n\n    @property\n    def infer(self):\n        return self._lazy_import(self.config['infer'][0], self.config['infer'][1])\n\nclass ModelRegistry:\n    def __init__(self):\n        self.models = {}\n\n    def register_model(self, name, config):\n        \"\"\"Register a model configuration.\"\"\"\n        self.models[name] = ModelLoader(name, config, use_accel=False)\n\n    def load_model(self, choice, use_accel=False):\n        \"\"\"Load a model based on the choice.\"\"\"\n        if choice in self.models:\n            self.models[choice].use_accel = use_accel\n            return self.models[choice].model\n        else:\n            raise ValueError(f\"Model choice '{choice}' is not supported.\")\n\n    def infer(self, choice):\n        \"\"\"Get the inference function for a given model.\"\"\"\n        if choice in self.models:\n            return self.models[choice].infer\n        else:\n            raise ValueError(f\"Inference choice '{choice}' is not supported.\")\n\n# Initialize model registry\nmodel_registry = ModelRegistry()\n\n# Configuration of models\nmodel_configs = {\n    ####### APi models #######\n    'gpt-4o-2024-11-20': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'GPT4o',\n        'base_url': '',\n        'api_key': '',\n        'model': 'gpt-4o-2024-11-20',\n        'call_type': 'api_chat'\n    },\n    'gpt-4o-2024-08-06': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'GPT4o-2024-08-06',\n        'base_url': '',\n        'api_key': '',\n        'model': 'gpt-4o-2024-08-06',\n        'call_type': 'api_chat'\n    },\n    'claude-3-5-sonnet-20241022': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'Claude-3-5-Sonnet-20241022',\n        'base_url': '',\n        'api_key': '',\n        'model': 'claude-3-5-sonnet-20241022',\n        'call_type': 'api_chat'\n    },\n    'o1-mini': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'o1-mini',\n        'base_url': '',\n        'api_key': '',\n        'model': 'o1-mini',\n        'call_type': 'api_chat'\n    },\n    'gemini-1.5-pro-002': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'Gemini-1.5-pro-002',\n        'base_url': '',\n        'api_key': '',\n        'model': 'gemini-1.5-pro-002',\n        'call_type': 'api_chat'\n    },\n    'gpt-4-turbo-2024-04-09': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'GPT4',\n        'base_url': '',\n        'api_key': '',\n        'model': 'gpt-4-turbo',\n        'call_type': 'api_chat'\n    },\n    'claude-3-5-sonnet-20240620': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'Claude-3-5-Sonnet-20240620',\n        'base_url': '',\n        'api_key': '',\n        'model': 'claude-3-5-sonnet-20240620',\n        'call_type': 'api_chat'\n    },\n    'gemini-2.0-flash-exp': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'Gemini-2.0-Flash-Exp',\n        'base_url': '',\n        'api_key': '',\n        'model': 'gemini-2.0-flash-exp',\n        'call_type': 'api_chat'\n    },\n    'Llama-3.1-405B': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'meta-llama/Llama-3.1-405B',\n        'base_url': 'http://127.0.0.1:8000/v1',\n        'api_key': 'none',\n        'model': 'meta-llama/Llama-3.1-405B',\n        'call_type': 'api_base'\n    },\n    'Llama-3.1-405B-Instruct': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'meta-llama/Llama-3.1-405B-Instruct',\n        'base_url': 'http://127.0.0.1:8000/v1',\n        'api_key': 'none',\n        'model': 'meta-llama/Llama-3.1-405B-Instruct',\n        'call_type': 'api_chat'\n    },\n    'o3-mini-2025-01-31': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'o3-mini-2025-01-31',\n        'base_url': '',\n        'api_key': '',\n        'model': 'o3-mini-2025-01-31',\n        'call_type': 'api_chat'\n    },\n    'Doubao-1.5-pro-32k-250115': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'Doubao-1.5-pro-32k-250115',\n        'base_url': \"\", \n        'api_key': \"\",\n        'model': \"\",\n        'call_type': 'api_chat'\n    },\n    'DeepSeek-V3': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'DeepSeek-V3',\n        'base_url': '',\n        'api_key': '',\n        'model': 'deepseek-chat',\n        'call_type': 'api_chat'\n    },\n    'DeepSeek-R1': {\n        'load': ('.openai_api', 'load_model'),\n        'infer': ('.openai_api', 'infer'),\n        'model_path_or_name': 'DeepSeek-R1',\n        'base_url': '',\n        'api_key': '',\n        'model': 'deepseek-reasoner',\n        'call_type': 'api_chat'\n    },\n    'claude-3-7-sonnet-20250219': {\n        'load': ('.anthropic_api', 'load_model'),\n        'infer': ('.anthropic_api', 'infer'),\n        'model_path_or_name': 'claude-3-7-sonnet-20250219',\n        'base_url': '',\n        'api_key': '',\n        'model': 'claude-3-7-sonnet-20250219',\n        'call_type': 'api_chat'\n    },\n\n    ####### Local Language Aligned models #######\n    'phi-4': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'microsoft/phi-4',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'granite-3.1-8b-instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'ibm-granite/granite-3.1-8b-instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'granite-3.1-2b-instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'ibm-granite/granite-3.1-2b-instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'QwQ-32B-Preview': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'Qwen/QwQ-32B-Preview',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Qwen2.5-0.5B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-0.5B-Instruct',\n        'call_type': 'local',\n        'tp': 2\n    },\n    'Qwen2.5-1.5B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-1.5B-Instruct',\n        'call_type': 'local',\n        'tp': 4\n    },\n    'Qwen2.5-3B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-3B-Instruct',\n        'call_type': 'local',\n        'tp': 4\n    },\n    'Qwen2.5-7B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-7B-Instruct',\n        'call_type': 'local',\n        'tp': 4\n    },\n    'Qwen2.5-14B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-14B-Instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Qwen2.5-32B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-32B-Instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Qwen2.5-72B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-72B-Instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'K2-Chat': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'LLM360/K2-Chat',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'gemma-2-2b-it': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'google/gemma-2-2b-it',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'gemma-2-9b-it': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'google/gemma-2-9b-it',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'gemma-2-27b-it': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'google/gemma-2-27b-it',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Llama-3.1-8B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'meta-llama/Llama-3.1-8B-Instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Llama-3.1-70B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'meta-llama/Llama-3.1-70B-Instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Llama-3.3-70B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'meta-llama/Llama-3.3-70B-Instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Yi-1.5-6B-Chat': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': '01-ai/Yi-1.5-6B-Chat',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Yi-1.5-9B-Chat': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': '01-ai/Yi-1.5-9B-Chat',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Yi-1.5-34B-Chat': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': '01-ai/Yi-1.5-34B-Chat',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'MAP-Neo-7B-Instruct-v0.1': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'm-a-p/neo_7b_instruct_v0.1',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Mistral-7B-Instruct-v0.3': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'mistralai/Mistral-7B-Instruct-v0.3',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Mistral-Large-Instruct-2411': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'mistralai/Mistral-Large-Instruct-2411',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Mistral-Small-Instruct-2409': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'mistralai/Mistral-Small-Instruct-2409',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Mixtral-8x22B-Instruct-v0.1': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'mistralai/Mixtral-8x22B-Instruct-v0.1',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Mixtral-8x7B-Instruct-v0.1': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'mistralai/Mixtral-8x7B-Instruct-v0.1',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'OLMo-2-1124-13B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'allenai/OLMo-2-1124-13B-Instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'OLMo-2-1124-7B-Instruct': {\n        'load': ('.hf_causallm_chat', 'load_model'),\n        'infer': ('.hf_causallm_chat', 'infer'),\n        'model_path_or_name': 'allenai/OLMo-2-1124-7B-Instruct',\n        'call_type': 'local',\n        'tp': 8\n    },\n\n    ####### Local Language Base models #######\n    'Qwen2.5-0.5B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-0.5B',\n        'call_type': 'local',\n        'tp': 2\n    },\n    'Qwen2.5-1.5B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-1.5B',\n        'call_type': 'local',\n        'tp': 4\n    },\n    'Qwen2.5-3B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-3B',\n        'call_type': 'local',\n        'tp': 4\n    },\n    'Qwen2.5-7B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-7B',\n        'call_type': 'local',\n        'tp': 4\n    },\n    'Qwen2.5-14B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-14B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Qwen2.5-32B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-32B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Qwen2.5-72B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'Qwen/Qwen2.5-72B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'K2': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'LLM360/K2',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'gemma-2-2b': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'google/gemma-2-2b',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'gemma-2-9b': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'google/gemma-2-9b',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'gemma-2-27b': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'google/gemma-2-27b',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Llama-3.1-8B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'meta-llama/Llama-3.1-8B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Llama-3.1-70B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'meta-llama/Llama-3.1-70B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Yi-1.5-6B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': '01-ai/Yi-1.5-6B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Yi-1.5-9B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': '01-ai/Yi-1.5-9B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Yi-1.5-34B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': '01-ai/Yi-1.5-34B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'MAP-Neo-7B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'm-a-p/neo_7b',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Mistral-7B-v0.3': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'mistralai/Mistral-7B-v0.3',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Mixtral-8x22B-v0.1': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'mistralai/Mixtral-8x22B-v0.1',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'Mixtral-8x7B-v0.1': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'mistralai/Mixtral-8x7B-v0.1',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'OLMo-2-1124-13B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'allenai/OLMo-2-1124-13B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'OLMo-2-1124-7B': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'allenai/OLMo-2-1124-7B',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'granite-3.1-2b-base': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'ibm-granite/granite-3.1-2b-base',\n        'call_type': 'local',\n        'tp': 8\n    },\n    'granite-3.1-8b-base': {\n        'load': ('.hf_causallm_base', 'load_model'),\n        'infer': ('.hf_causallm_base', 'infer'),\n        'model_path_or_name': 'ibm-granite/granite-3.1-8b-base',\n        'call_type': 'local',\n        'tp': 8\n    },\n}\n\n# # Register all models\n# for model_name, config in model_configs.items():\n#     model_registry.register_model(model_name, config)\n\ndef load_model(choice, use_accel=False):\n    \"\"\"Load a specific model based on the choice.\"\"\"\n    model_registry.register_model(choice, model_configs[choice])\n    return model_registry.load_model(choice, use_accel)\n\ndef infer(choice):\n    \"\"\"Get the inference function for a specific model.\"\"\"\n    return model_registry.infer(choice)\n\n"}}}}