{"repo": "boden-c/benchmarks", "files": {"benchmark/execution/llm/factory.py": {"3a482f70ab441dc5b443107359916869f2399504": {"t": 1785746479.1060321, "n": 4887, "h": "123a06a6584dbf92", "c": "\"\"\"\nModel configuration and provider factory.\n\nCentralized model registry and factory for creating configured LLM providers.\n\"\"\"\n\nimport os\nimport logging\nfrom typing import Any\n\nfrom benchmark.models import ModelConfig\nfrom benchmark.execution.llm.provider import LLMProvider\nfrom benchmark.execution.llm.providers.openai import create_openai_provider\nfrom benchmark.execution.llm.providers.openrouter import create_openrouter_provider\n\nlogger = logging.getLogger(__name__)\n\n\nclass LLMFactory:\n    \"\"\"Factory for creating LLM providers.\"\"\"\n    \n    @staticmethod\n    def get_model_configs() -> dict[str, ModelConfig]:\n        \"\"\"\n        Get comprehensive model registry.\n        \n        Returns:\n            Dictionary mapping model names to configurations\n        \"\"\"\n        azure_endpoint = os.getenv(\"AZURE_OPENAI_ENDPOINT\", \"\")\n        azure_key = os.getenv(\"AZURE_OPENAI_API_KEY\", \"\")\n        openrouter_key = os.getenv(\"OPENROUTER_API_KEY\", \"\")\n        \n        configs = {\n            # Azure OpenAI models\n            \"gpt-4o\": ModelConfig(\n                name=\"gpt-4o\",\n                provider=\"azure\",\n                api_key=azure_key,\n                base_url=azure_endpoint,\n            ),\n            \"gpt-4o-mini\": ModelConfig(\n                name=\"gpt-4o-mini\",\n                provider=\"azure\",\n                api_key=azure_key,\n                base_url=azure_endpoint,\n            ),\n            \"o1\": ModelConfig(\n                name=\"o1\",\n                provider=\"azure\",\n                api_key=azure_key,\n                base_url=azure_endpoint,\n            ),\n            \"o1-mini\": ModelConfig(\n                name=\"o1-mini\",\n                provider=\"azure\",\n                api_key=azure_key,\n                base_url=azure_endpoint,\n            ),\n            \"o4-mini\": ModelConfig(\n                name=\"o4-mini\",\n                provider=\"azure\",\n                api_key=azure_key,\n                base_url=azure_endpoint,\n            ),\n            \n            # OpenRouter models\n            \"claude-sonnet-4\": ModelConfig(\n                name=\"anthropic/claude-sonnet-4\",\n                provider=\"openrouter\",\n                api_key=openrouter_key,\n            ),\n            \"claude-opus-4\": ModelConfig(\n                name=\"anthropic/claude-opus-4\",\n                provider=\"openrouter\",\n                api_key=openrouter_key,\n            ),\n            \"gemini-2.0-flash-exp\": ModelConfig(\n                name=\"google/gemini-2.0-flash-exp:free\",\n                provider=\"openrouter\",\n                api_key=openrouter_key,\n            ),\n            \"gemini-pro-1.5\": ModelConfig(\n                name=\"google/gemini-pro-1.5\",\n                provider=\"openrouter\",\n                api_key=openrouter_key,\n            ),\n            \"deepseek-chat\": ModelConfig(\n                name=\"deepseek/deepseek-chat\",\n                provider=\"openrouter\",\n                api_key=openrouter_key,\n            ),\n            \"qwen-2.5-72b\": ModelConfig(\n                name=\"qwen/qwen-2.5-72b-instruct\",\n                provider=\"openrouter\",\n                api_key=openrouter_key,\n            ),\n        }\n        \n        # Add custom models from environment\n        for key, value in os.environ.items():\n            if key.startswith(\"CUSTOM_MODEL_\"):\n                model_name = key[13:].lower().replace(\"_\", \"-\")\n                configs[model_name] = ModelConfig(\n                    name=model_name,\n                    provider=\"custom\",\n                    api_key=os.getenv(f\"{key}_API_KEY\", \"\"),\n                    base_url=os.getenv(f\"{key}_BASE_URL\", \"\"),\n                )\n        \n        return configs\n    \n    @staticmethod\n    async def create_llm_provider(model_config: ModelConfig) -> LLMProvider:\n        \"\"\"\n        Factory method creating configured provider instances.\n        \n        Args:\n            model_config: Model configuration\n        \n        Returns:\n            Configured LLMProvider instance\n        \n        Raises:\n            ValueError: If provider type unknown or configuration invalid\n        \"\"\"\n        provider_type = model_config.provider.lower()\n        \n        logger.debug(f\"Creating provider for {model_config.name} ({provider_type})\")\n        \n        if provider_type == \"azure\":\n            return await create_openai_provider(model_config, provider_type=\"azure\")\n        elif provider_type == \"openai\":\n            return await create_openai_provider(model_config, provider_type=\"openai\")\n        elif provider_type == \"openrouter\":\n            return await create_openrouter_provider(model_config)\n        elif provider_type == \"custom\":\n            # Use OpenAI-compatible client for custom endpoints\n            return await create_openai_provider(model_config, provider_type=\"custom\")\n        else:\n            raise ValueError(f\"Unknown provider type: {provider_type}\")\n"}, "752029e6529814cd7bc5e6587a25102be8ec5033": {"t": 1785764778.6962848, "n": 4887, "h": "123a06a6584dbf92", "c": "\"\"\"\nModel configuration and provider factory.\n\nCentralized model registry and factory for creating configured LLM providers.\n\"\"\"\n\nimport os\nimport logging\nfrom typing import Any\n\nfrom benchmark.models import ModelConfig\nfrom benchmark.execution.llm.provider import LLMProvider\nfrom benchmark.execution.llm.providers.openai import create_openai_provider\nfrom benchmark.execution.llm.providers.openrouter import create_openrouter_provider\n\nlogger = logging.getLogger(__name__)\n\n\nclass LLMFactory:\n    \"\"\"Factory for creating LLM providers.\"\"\"\n    \n    @staticmethod\n    def get_model_configs() -> dict[str, ModelConfig]:\n        \"\"\"\n        Get comprehensive model registry.\n        \n        Returns:\n            Dictionary mapping model names to configurations\n        \"\"\"\n        azure_endpoint = os.getenv(\"AZURE_OPENAI_ENDPOINT\", \"\")\n        azure_key = os.getenv(\"AZURE_OPENAI_API_KEY\", \"\")\n        openrouter_key = os.getenv(\"OPENROUTER_API_KEY\", \"\")\n        \n        configs = {\n            # Azure OpenAI models\n            \"gpt-4o\": ModelConfig(\n                name=\"gpt-4o\",\n                provider=\"azure\",\n                api_key=azure_key,\n                base_url=azure_endpoint,\n            ),\n            \"gpt-4o-mini\": ModelConfig(\n                name=\"gpt-4o-mini\",\n                provider=\"azure\",\n                api_key=azure_key,\n                base_url=azure_endpoint,\n            ),\n            \"o1\": ModelConfig(\n                name=\"o1\",\n                provider=\"azure\",\n                api_key=azure_key,\n                base_url=azure_endpoint,\n            ),\n            \"o1-mini\": ModelConfig(\n                name=\"o1-mini\",\n                provider=\"azure\",\n                api_key=azure_key,\n                base_url=azure_endpoint,\n            ),\n            \"o4-mini\": ModelConfig(\n                name=\"o4-mini\",\n                provider=\"azure\",\n                api_key=azure_key,\n                base_url=azure_endpoint,\n            ),\n            \n            # OpenRouter models\n            \"claude-sonnet-4\": ModelConfig(\n                name=\"anthropic/claude-sonnet-4\",\n                provider=\"openrouter\",\n                api_key=openrouter_key,\n            ),\n            \"claude-opus-4\": ModelConfig(\n                name=\"anthropic/claude-opus-4\",\n                provider=\"openrouter\",\n                api_key=openrouter_key,\n            ),\n            \"gemini-2.0-flash-exp\": ModelConfig(\n                name=\"google/gemini-2.0-flash-exp:free\",\n                provider=\"openrouter\",\n                api_key=openrouter_key,\n            ),\n            \"gemini-pro-1.5\": ModelConfig(\n                name=\"google/gemini-pro-1.5\",\n                provider=\"openrouter\",\n                api_key=openrouter_key,\n            ),\n            \"deepseek-chat\": ModelConfig(\n                name=\"deepseek/deepseek-chat\",\n                provider=\"openrouter\",\n                api_key=openrouter_key,\n            ),\n            \"qwen-2.5-72b\": ModelConfig(\n                name=\"qwen/qwen-2.5-72b-instruct\",\n                provider=\"openrouter\",\n                api_key=openrouter_key,\n            ),\n        }\n        \n        # Add custom models from environment\n        for key, value in os.environ.items():\n            if key.startswith(\"CUSTOM_MODEL_\"):\n                model_name = key[13:].lower().replace(\"_\", \"-\")\n                configs[model_name] = ModelConfig(\n                    name=model_name,\n                    provider=\"custom\",\n                    api_key=os.getenv(f\"{key}_API_KEY\", \"\"),\n                    base_url=os.getenv(f\"{key}_BASE_URL\", \"\"),\n                )\n        \n        return configs\n    \n    @staticmethod\n    async def create_llm_provider(model_config: ModelConfig) -> LLMProvider:\n        \"\"\"\n        Factory method creating configured provider instances.\n        \n        Args:\n            model_config: Model configuration\n        \n        Returns:\n            Configured LLMProvider instance\n        \n        Raises:\n            ValueError: If provider type unknown or configuration invalid\n        \"\"\"\n        provider_type = model_config.provider.lower()\n        \n        logger.debug(f\"Creating provider for {model_config.name} ({provider_type})\")\n        \n        if provider_type == \"azure\":\n            return await create_openai_provider(model_config, provider_type=\"azure\")\n        elif provider_type == \"openai\":\n            return await create_openai_provider(model_config, provider_type=\"openai\")\n        elif provider_type == \"openrouter\":\n            return await create_openrouter_provider(model_config)\n        elif provider_type == \"custom\":\n            # Use OpenAI-compatible client for custom endpoints\n            return await create_openai_provider(model_config, provider_type=\"custom\")\n        else:\n            raise ValueError(f\"Unknown provider type: {provider_type}\")\n"}}}}