1 line
41 KiB
JSON
1 line
41 KiB
JSON
{"repo": "akmessi/rlm-paper-implementation", "files": {"core/llm_client.py": {"16e78fa76a9791fd43108241d0e36ccdefd98213": {"t": 1785674095.1867564, "n": 39522, "h": "0816a739254cc044", "c": "\"\"\"\nLLM Client for RLM - ULTIMATE VERSION\n======================================\n\nUnified interface for 15+ LLM providers with ALL latest 2025 models:\n- OpenAI (GPT-5, GPT-4.5, o3, o4-mini, o3-pro, o1)\n- Anthropic (Claude Opus 4.6, Sonnet 4.5, Haiku 4.5, Claude 3.5)\n- Google (Gemini 2.5/3.0 Pro/Flash, 1.5 Pro/Flash)\n- Moonshot AI (Kimi K2.5, K3, k1.5)\n- Groq (Llama 4, 3.1, Mixtral)\n- Together AI (Llama 4, Qwen 3, DeepSeek V3)\n- And more...\n\nAll models updated as of February 2026\n\"\"\"\n\nimport os\nimport asyncio\nfrom typing import Any, Dict, List, Optional, Union\nfrom abc import ABC, abstractmethod\nimport logging\n\nimport httpx\nfrom openai import AsyncOpenAI\n\nlogger = logging.getLogger(__name__)\n\n\nclass BaseLLMClient(ABC):\n \"\"\"Abstract base class for LLM clients.\"\"\"\n \n def __init__(self, model: str, api_key: Optional[str] = None, **kwargs):\n self.model = model\n self.api_key = api_key\n self.extra_params = kwargs\n \n @abstractmethod\n async def complete(\n self,\n prompt: Optional[str] = None,\n messages: Optional[List[Dict[str, str]]] = None,\n system: Optional[str] = None,\n max_tokens: int = 4000,\n temperature: float = 0.7,\n **kwargs\n ) -> str:\n \"\"\"Get completion from the LLM.\"\"\"\n pass\n\n\nclass OpenAIClient(BaseLLMClient):\n \"\"\"Client for OpenAI API - Includes ALL latest 2025 models.\"\"\"\n \n # Latest OpenAI models as of February 2026\n AVAILABLE_MODELS = [\n # GPT-5 Series (Latest)\n \"gpt-5\",\n \"gpt-5-mini\",\n \"gpt-5-nano\",\n # GPT-4.5 Series\n \"gpt-4.5-preview\",\n \"gpt-4.5\",\n # o3/o4 Reasoning Series (Latest)\n \"o3\",\n \"o3-mini\",\n \"o3-pro\",\n \"o4-mini\",\n \"o4-mini-high\",\n \"o1\",\n \"o1-mini\",\n \"o1-pro\",\n # GPT-4o Series\n \"gpt-4o\",\n \"gpt-4o-latest\",\n \"gpt-4o-mini\",\n \"gpt-4o-mini-latest\",\n # Legacy\n \"gpt-4-turbo\",\n \"gpt-4-turbo-preview\",\n \"gpt-4\",\n \"gpt-3.5-turbo\",\n ]\n \n def __init__(\n self, \n model: str = \"gpt-4o-mini\",\n api_key: Optional[str] = None,\n base_url: Optional[str] = None,\n **kwargs\n ):\n super().__init__(model, api_key, **kwargs)\n self.client = AsyncOpenAI(\n api_key=api_key or os.getenv(\"OPENAI_API_KEY\"),\n base_url=base_url,\n timeout=httpx.Timeout(120.0)\n )\n \n async def complete(\n self,\n prompt: Optional[str] = None,\n messages: Optional[List[Dict[str, str]]] = None,\n system: Optional[str] = None,\n max_tokens: int = 4000,\n temperature: float = 0.7,\n **kwargs\n ) -> str:\n \"\"\"Get completion from OpenAI.\"\"\"\n \n if messages is None:\n messages = []\n if system:\n messages.append({\"role\": \"system\", \"content\": system})\n if prompt:\n messages.append({\"role\": \"user\", \"content\": prompt})\n elif system:\n messages = [{\"role\": \"system\", \"content\": system}] + messages\n \n try:\n response = await self.client.chat.completions.create(\n model=self.model,\n messages=messages,\n max_tokens=max_tokens,\n temperature=temperature,\n **{**self.extra_params, **kwargs}\n )\n return response.choices[0].message.content\n except Exception as e:\n logger.error(f\"OpenAI API error: {str(e)}\")\n raise\n\n\nclass AnthropicClient(BaseLLMClient):\n \"\"\"Client for Anthropic Claude API - ALL latest 2025 models.\"\"\"\n \n # Latest Anthropic models as of February 2026\n AVAILABLE_MODELS = [\n # Claude 4.6 Series (LATEST)\n \"claude-opus-4-6-20251101\",\n \"claude-opus-4-6\",\n # Claude 4.5 Series\n \"claude-opus-4-5-20251101\",\n \"claude-opus-4-5\",\n \"claude-sonnet-4-5-20251101\",\n \"claude-sonnet-4-5\",\n \"claude-haiku-4-5-20251101\",\n \"claude-haiku-4-5\",\n # Claude 3.5 Series (Current Gen)\n \"claude-3-5-sonnet-20241022\",\n \"claude-3-5-sonnet-latest\",\n \"claude-3-5-haiku-20241022\",\n \"claude-3-5-haiku-latest\",\n # Claude 3 Series\n \"claude-3-opus-20240229\",\n \"claude-3-sonnet-20240229\",\n \"claude-3-haiku-20240307\",\n ]\n \n def __init__(\n self, \n model: str = \"claude-3-5-sonnet-20241022\",\n api_key: Optional[str] = None,\n **kwargs\n ):\n super().__init__(model, api_key, **kwargs)\n self.api_key = api_key or os.getenv(\"ANTHROPIC_API_KEY\")\n self.base_url = \"https://api.anthropic.com/v1/messages\"\n \n async def complete(\n self,\n prompt: Optional[str] = None,\n messages: Optional[List[Dict[str, str]]] = None,\n system: Optional[str] = None,\n max_tokens: int = 4000,\n temperature: float = 0.7,\n **kwargs\n ) -> str:\n \"\"\"Get completion from Anthropic Claude.\"\"\"\n \n if messages is None:\n messages = []\n if prompt:\n messages.append({\"role\": \"user\", \"content\": prompt})\n \n anthropic_messages = []\n for msg in messages:\n if msg[\"role\"] != \"system\":\n anthropic_messages.append(msg)\n \n headers = {\n \"x-api-key\": self.api_key,\n \"anthropic-version\": \"2023-06-01\",\n \"content-type\": \"application/json\"\n }\n \n payload = {\n \"model\": self.model,\n \"messages\": anthropic_messages,\n \"max_tokens\": max_tokens,\n \"temperature\": temperature,\n **{**self.extra_params, **kwargs}\n }\n \n if system:\n payload[\"system\"] = system\n \n try:\n async with httpx.AsyncClient(timeout=120.0) as client:\n response = await client.post(\n self.base_url,\n headers=headers,\n json=payload\n )\n response.raise_for_status()\n data = response.json()\n return data[\"content\"][0][\"text\"]\n except Exception as e:\n logger.error(f\"Anthropic API error: {str(e)}\")\n raise\n\n\nclass GeminiClient(BaseLLMClient):\n \"\"\"Client for Google Gemini API - ALL latest 2025 models.\"\"\"\n \n # Latest Gemini models as of February 2026\n AVAILABLE_MODELS = [\n # Gemini 3.0 Series (LATEST)\n \"gemini-3-pro-preview\",\n \"gemini-3-flash-preview\",\n # Gemini 2.5 Series\n \"gemini-2.5-pro\",\n \"gemini-2.5-pro-preview\",\n \"gemini-2.5-flash\",\n \"gemini-2.5-flash-preview\",\n \"gemini-2.5-flash-lite\",\n \"gemini-2.5-flash-lite-preview\",\n # Gemini 2.0 Series\n \"gemini-2.0-flash\",\n \"gemini-2.0-flash-lite\",\n \"gemini-2.0-pro\",\n \"gemini-2.0-flash-exp\",\n # Gemini 1.5 Series\n \"gemini-1.5-pro\",\n \"gemini-1.5-pro-latest\",\n \"gemini-1.5-flash\",\n \"gemini-1.5-flash-latest\",\n \"gemini-1.5-flash-8b\",\n # Legacy\n \"gemini-1.0-pro\",\n \"gemini-pro\",\n ]\n \n def __init__(\n self,\n model: str = \"gemini-2.5-flash\",\n api_key: Optional[str] = None,\n **kwargs\n ):\n super().__init__(model, api_key, **kwargs)\n self.api_key = api_key or os.getenv(\"GOOGLE_API_KEY\")\n self.base_url = f\"https://generativelanguage.googleapis.com/v1beta/models/{model}\"\n \n async def complete(\n self,\n prompt: Optional[str] = None,\n messages: Optional[List[Dict[str, str]]] = None,\n system: Optional[str] = None,\n max_tokens: int = 4000,\n temperature: float = 0.7,\n **kwargs\n ) -> str:\n \"\"\"Get completion from Google Gemini with rate limiting.\"\"\"\n import asyncio\n import random\n \n contents = []\n if messages:\n for msg in messages:\n role = \"user\" if msg[\"role\"] == \"user\" else \"model\"\n contents.append({\"role\": role, \"parts\": [{\"text\": msg[\"content\"]}]})\n elif prompt:\n contents.append({\"role\": \"user\", \"parts\": [{\"text\": prompt}]})\n \n payload = {\n \"contents\": contents,\n \"generationConfig\": {\n \"maxOutputTokens\": max_tokens,\n \"temperature\": temperature,\n }\n }\n \n if system:\n payload[\"systemInstruction\"] = {\"parts\": [{\"text\": system}]}\n \n # Retry with exponential backoff for rate limiting\n max_retries = 5\n base_delay = 2\n \n for attempt in range(max_retries):\n try:\n async with httpx.AsyncClient(timeout=120.0) as client:\n response = await client.post(\n f\"{self.base_url}:generateContent?key={self.api_key}\",\n json=payload\n )\n \n # Handle rate limiting\n if response.status_code == 429:\n delay = base_delay * (2 ** attempt) + random.uniform(0, 1)\n logger.warning(f\"Gemini rate limit hit, retrying in {delay:.1f}s (attempt {attempt + 1}/{max_retries})\")\n await asyncio.sleep(delay)\n continue\n \n response.raise_for_status()\n data = response.json()\n return data[\"candidates\"][0][\"content\"][\"parts\"][0][\"text\"]\n \n except httpx.HTTPStatusError as e:\n if e.response.status_code == 429 and attempt < max_retries - 1:\n delay = base_delay * (2 ** attempt) + random.uniform(0, 1)\n logger.warning(f\"Gemini rate limit hit, retrying in {delay:.1f}s (attempt {attempt + 1}/{max_retries})\")\n await asyncio.sleep(delay)\n continue\n logger.error(f\"Gemini API error: {str(e)}\")\n raise\n except Exception as e:\n logger.error(f\"Gemini API error: {str(e)}\")\n raise\n \n raise Exception(\"Max retries exceeded for Gemini API rate limiting\")\n\n\nclass KimiClient(BaseLLMClient):\n \"\"\"\n Client for Moonshot AI (Kimi) API.\n Supports Kimi K2.5, K3, k1.5 and all latest models.\n \"\"\"\n \n # Latest Kimi models as of February 2026\n AVAILABLE_MODELS = [\n # Kimi K3 Series (LATEST)\n \"kimi-k3\",\n \"kimi-k3-0325\",\n # Kimi K2.5 Series\n \"kimi-k2.5\",\n \"kimi-k2.5-0120\",\n \"kimi-k2.5-long\",\n # Kimi k1.5 Series (Reasoning)\n \"kimi-k1.5\",\n \"kimi-k1.5-long\",\n \"kimi-k1.5-short\",\n # Legacy\n \"kimi-latest\",\n \"moonshot-v1-8k\",\n \"moonshot-v1-32k\",\n \"moonshot-v1-128k\",\n ]\n \n def __init__(\n self,\n model: str = \"kimi-k2.5\",\n api_key: Optional[str] = None,\n **kwargs\n ):\n super().__init__(model, api_key, **kwargs)\n self.api_key = api_key or os.getenv(\"KIMI_API_KEY\") or os.getenv(\"MOONSHOT_API_KEY\")\n self.base_url = \"https://api.moonshot.cn/v1\"\n \n self.client = AsyncOpenAI(\n api_key=self.api_key,\n base_url=self.base_url,\n timeout=httpx.Timeout(120.0)\n )\n \n async def complete(\n self,\n prompt: Optional[str] = None,\n messages: Optional[List[Dict[str, str]]] = None,\n system: Optional[str] = None,\n max_tokens: int = 4000,\n temperature: float = 0.7,\n **kwargs\n ) -> str:\n \"\"\"Get completion from Kimi.\"\"\"\n \n if messages is None:\n messages = []\n if system:\n messages.append({\"role\": \"system\", \"content\": system})\n if prompt:\n messages.append({\"role\": \"user\", \"content\": prompt})\n elif system:\n messages = [{\"role\": \"system\", \"content\": system}] + messages\n \n try:\n response = await self.client.chat.completions.create(\n model=self.model,\n messages=messages,\n max_tokens=max_tokens,\n temperature=temperature,\n **{**self.extra_params, **kwargs}\n )\n return response.choices[0].message.content\n except Exception as e:\n logger.error(f\"Kimi API error: {str(e)}\")\n raise\n\n\nclass GroqClient(BaseLLMClient):\n \"\"\"\n Client for Groq API.\n Fast inference for open source models.\n \"\"\"\n \n # Latest Groq models as of February 2026\n AVAILABLE_MODELS = [\n # Llama 4 Series (LATEST)\n \"meta-llama/llama-4-maverick-17b-128e-instruct\",\n \"meta-llama/llama-4-scout-17b-16e-instruct\",\n # Llama 3.3/3.1 Series\n \"llama-3.3-70b-versatile\",\n \"llama-3.1-405b-reasoning\",\n \"llama-3.1-70b-versatile\",\n \"llama-3.1-8b-instant\",\n # Legacy\n \"llama3-70b-8192\",\n \"llama3-8b-8192\",\n \"mixtral-8x7b-32768\",\n \"gemma-2-9b-it\",\n \"gemma-7b-it\",\n ]\n \n def __init__(\n self,\n model: str = \"llama-3.3-70b-versatile\",\n api_key: Optional[str] = None,\n **kwargs\n ):\n super().__init__(model, api_key, **kwargs)\n self.api_key = api_key or os.getenv(\"GROQ_API_KEY\")\n self.base_url = \"https://api.groq.com/openai/v1\"\n \n self.client = AsyncOpenAI(\n api_key=self.api_key,\n base_url=self.base_url,\n timeout=httpx.Timeout(60.0)\n )\n \n async def complete(\n self,\n prompt: Optional[str] = None,\n messages: Optional[List[Dict[str, str]]] = None,\n system: Optional[str] = None,\n max_tokens: int = 4000,\n temperature: float = 0.7,\n **kwargs\n ) -> str:\n \"\"\"Get completion from Groq.\"\"\"\n \n if messages is None:\n messages = []\n if system:\n messages.append({\"role\": \"system\", \"content\": system})\n if prompt:\n messages.append({\"role\": \"user\", \"content\": prompt})\n elif system:\n messages = [{\"role\": \"system\", \"content\": system}] + messages\n \n try:\n response = await self.client.chat.completions.create(\n model=self.model,\n messages=messages,\n max_tokens=max_tokens,\n temperature=temperature,\n **{**self.extra_params, **kwargs}\n )\n return response.choices[0].message.content\n except Exception as e:\n logger.error(f\"Groq API error: {str(e)}\")\n raise\n\n\nclass TogetherClient(BaseLLMClient):\n \"\"\"\n Client for Together AI API.\n Great for open source models.\n \"\"\"\n \n # Latest Together AI models as of February 2026\n AVAILABLE_MODELS = [\n # Llama 4 Series\n \"meta-llama/Llama-4-Maverick-17B-128E-Instruct\",\n \"meta-llama/Llama-4-Scout-17B-16E-Instruct\",\n # Llama 3.3 Series\n \"meta-llama/Llama-3.3-70B-Instruct-Turbo\",\n \"meta-llama/Llama-3.1-405B-Instruct-Turbo\",\n \"meta-llama/Llama-3.1-70B-Instruct-Turbo\",\n \"meta-llama/Llama-3.1-8B-Instruct-Turbo\",\n # DeepSeek\n \"deepseek-ai/DeepSeek-V3\",\n \"deepseek-ai/DeepSeek-R1\",\n \"deepseek-ai/DeepSeek-Coder-V2\",\n # Qwen\n \"Qwen/Qwen2.5-72B-Instruct-Turbo\",\n \"Qwen/Qwen2.5-7B-Instruct-Turbo\",\n # Mistral\n \"mistralai/Mistral-Large-2\",\n \"mistralai/Mixtral-8x22B-Instruct-v0.1\",\n # Other\n \"google/gemma-2-27b-it\",\n \"databricks/dbrx-instruct\",\n ]\n \n def __init__(\n self,\n model: str = \"meta-llama/Llama-3.3-70B-Instruct-Turbo\",\n api_key: Optional[str] = None,\n **kwargs\n ):\n super().__init__(model, api_key, **kwargs)\n self.api_key = api_key or os.getenv(\"TOGETHER_API_KEY\")\n self.base_url = \"https://api.together.xyz/v1\"\n \n self.client = AsyncOpenAI(\n api_key=self.api_key,\n base_url=self.base_url,\n timeout=httpx.Timeout(120.0)\n )\n \n async def complete(\n self,\n prompt: Optional[str] = None,\n messages: Optional[List[Dict[str, str]]] = None,\n system: Optional[str] = None,\n max_tokens: int = 4000,\n temperature: float = 0.7,\n **kwargs\n ) -> str:\n \"\"\"Get completion from Together AI.\"\"\"\n \n if messages is None:\n messages = []\n if system:\n messages.append({\"role\": \"system\", \"content\": system})\n if prompt:\n messages.append({\"role\": \"user\", \"content\": prompt})\n elif system:\n messages = [{\"role\": \"system\", \"content\": system}] + messages\n \n try:\n response = await self.client.chat.completions.create(\n model=self.model,\n messages=messages,\n max_tokens=max_tokens,\n temperature=temperature,\n **{**self.extra_params, **kwargs}\n )\n return response.choices[0].message.content\n except Exception as e:\n logger.error(f\"Together AI API error: {str(e)}\")\n raise\n\n\nclass MistralClient(BaseLLMClient):\n \"\"\"Client for Mistral AI API.\"\"\"\n \n AVAILABLE_MODELS = [\n \"mistral-large-latest\",\n \"mistral-medium-latest\",\n \"mistral-small-latest\",\n \"codestral-latest\",\n \"mistral-embed\",\n ]\n \n def __init__(\n self,\n model: str = \"mistral-large-latest\",\n api_key: Optional[str] = None,\n **kwargs\n ):\n super().__init__(model, api_key, **kwargs)\n self.api_key = api_key or os.getenv(\"MISTRAL_API_KEY\")\n self.base_url = \"https://api.mistral.ai/v1\"\n \n self.client = AsyncOpenAI(\n api_key=self.api_key,\n base_url=self.base_url,\n timeout=httpx.Timeout(120.0)\n )\n \n async def complete(\n self,\n prompt: Optional[str] = None,\n messages: Optional[List[Dict[str, str]]] = None,\n system: Optional[str] = None,\n max_tokens: int = 4000,\n temperature: float = 0.7,\n **kwargs\n ) -> str:\n \"\"\"Get completion from Mistral.\"\"\"\n \n if messages is None:\n messages = []\n if system:\n messages.append({\"role\": \"system\", \"content\": system})\n if prompt:\n messages.append({\"role\": \"user\", \"content\": prompt})\n elif system:\n messages = [{\"role\": \"system\", \"content\": system}] + messages\n \n try:\n response = await self.client.chat.completions.create(\n model=self.model,\n messages=messages,\n max_tokens=max_tokens,\n temperature=temperature,\n **{**self.extra_params, **kwargs}\n )\n return response.choices[0].message.content\n except Exception as e:\n logger.error(f\"Mistral API error: {str(e)}\")\n raise\n\n\nclass DeepSeekClient(BaseLLMClient):\n \"\"\"Client for DeepSeek API.\"\"\"\n \n AVAILABLE_MODELS = [\n \"deepseek-chat\",\n \"deepseek-coder\",\n \"deepseek-reasoner\",\n ]\n \n def __init__(\n self,\n model: str = \"deepseek-chat\",\n api_key: Optional[str] = None,\n **kwargs\n ):\n super().__init__(model, api_key, **kwargs)\n self.api_key = api_key or os.getenv(\"DEEPSEEK_API_KEY\")\n self.base_url = \"https://api.deepseek.com/v1\"\n \n self.client = AsyncOpenAI(\n api_key=self.api_key,\n base_url=self.base_url,\n timeout=httpx.Timeout(120.0)\n )\n \n async def complete(\n self,\n prompt: Optional[str] = None,\n messages: Optional[List[Dict[str, str]]] = None,\n system: Optional[str] = None,\n max_tokens: int = 4000,\n temperature: float = 0.7,\n **kwargs\n ) -> str:\n \"\"\"Get completion from DeepSeek.\"\"\"\n \n if messages is None:\n messages = []\n if system:\n messages.append({\"role\": \"system\", \"content\": system})\n if prompt:\n messages.append({\"role\": \"user\", \"content\": prompt})\n elif system:\n messages = [{\"role\": \"system\", \"content\": system}] + messages\n \n try:\n response = await self.client.chat.completions.create(\n model=self.model,\n messages=messages,\n max_tokens=max_tokens,\n temperature=temperature,\n **{**self.extra_params, **kwargs}\n )\n return response.choices[0].message.content\n except Exception as e:\n logger.error(f\"DeepSeek API error: {str(e)}\")\n raise\n\n\nclass AzureOpenAIClient(BaseLLMClient):\n \"\"\"Client for Azure OpenAI API.\"\"\"\n \n def __init__(\n self,\n model: str = \"gpt-4o\",\n api_key: Optional[str] = None,\n azure_endpoint: Optional[str] = None,\n api_version: str = \"2024-02-01\",\n **kwargs\n ):\n super().__init__(model, api_key, **kwargs)\n self.azure_endpoint = azure_endpoint or os.getenv(\"AZURE_OPENAI_ENDPOINT\")\n self.api_version = api_version\n \n self.client = AsyncOpenAI(\n api_key=api_key or os.getenv(\"AZURE_OPENAI_KEY\"),\n base_url=f\"{self.azure_endpoint}/openai/deployments/{model}\",\n default_query={\"api-version\": api_version},\n timeout=httpx.Timeout(120.0)\n )\n \n async def complete(\n self,\n prompt: Optional[str] = None,\n messages: Optional[List[Dict[str, str]]] = None,\n system: Optional[str] = None,\n max_tokens: int = 4000,\n temperature: float = 0.7,\n **kwargs\n ) -> str:\n \"\"\"Get completion from Azure OpenAI.\"\"\"\n \n if messages is None:\n messages = []\n if system:\n messages.append({\"role\": \"system\", \"content\": system})\n if prompt:\n messages.append({\"role\": \"user\", \"content\": prompt})\n elif system:\n messages = [{\"role\": \"system\", \"content\": system}] + messages\n \n try:\n response = await self.client.chat.completions.create(\n model=self.model,\n messages=messages,\n max_tokens=max_tokens,\n temperature=temperature,\n **{**self.extra_params, **kwargs}\n )\n return response.choices[0].message.content\n except Exception as e:\n logger.error(f\"Azure OpenAI API error: {str(e)}\")\n raise\n\n\nclass CohereClient(BaseLLMClient):\n \"\"\"Client for Cohere API.\"\"\"\n \n AVAILABLE_MODELS = [\n \"command-r-plus\",\n \"command-r\",\n \"command\",\n \"command-light\",\n ]\n \n def __init__(\n self,\n model: str = \"command-r-plus\",\n api_key: Optional[str] = None,\n **kwargs\n ):\n super().__init__(model, api_key, **kwargs)\n self.api_key = api_key or os.getenv(\"COHERE_API_KEY\")\n self.base_url = \"https://api.cohere.ai/v1\"\n \n async def complete(\n self,\n prompt: Optional[str] = None,\n messages: Optional[List[Dict[str, str]]] = None,\n system: Optional[str] = None,\n max_tokens: int = 4000,\n temperature: float = 0.7,\n **kwargs\n ) -> str:\n \"\"\"Get completion from Cohere.\"\"\"\n \n chat_history = []\n message = prompt or \"\"\n \n if messages:\n for i, msg in enumerate(messages[:-1] if len(messages) > 1 else []):\n if msg[\"role\"] == \"user\":\n chat_history.append({\"role\": \"USER\", \"message\": msg[\"content\"]})\n elif msg[\"role\"] == \"assistant\":\n chat_history.append({\"role\": \"CHATBOT\", \"message\": msg[\"content\"]})\n \n if messages:\n last_msg = messages[-1]\n if last_msg[\"role\"] == \"user\":\n message = last_msg[\"content\"]\n \n payload = {\n \"model\": self.model,\n \"message\": message,\n \"max_tokens\": max_tokens,\n \"temperature\": temperature,\n \"chat_history\": chat_history,\n **self.extra_params\n }\n \n if system:\n payload[\"preamble\"] = system\n \n headers = {\n \"Authorization\": f\"Bearer {self.api_key}\",\n \"Content-Type\": \"application/json\"\n }\n \n try:\n async with httpx.AsyncClient(timeout=120.0) as client:\n response = await client.post(\n f\"{self.base_url}/chat\",\n headers=headers,\n json=payload\n )\n response.raise_for_status()\n data = response.json()\n return data[\"text\"]\n except Exception as e:\n logger.error(f\"Cohere API error: {str(e)}\")\n raise\n\n\nclass PerplexityClient(BaseLLMClient):\n \"\"\"\n Client for Perplexity API.\n Great for search-augmented responses.\n \"\"\"\n \n AVAILABLE_MODELS = [\n \"sonar-reasoning-pro\",\n \"sonar-reasoning\",\n \"sonar-pro\",\n \"sonar\",\n \"llama-3.1-sonar-large-128k-online\",\n \"llama-3.1-sonar-small-128k-online\",\n ]\n \n def __init__(\n self,\n model: str = \"sonar-pro\",\n api_key: Optional[str] = None,\n **kwargs\n ):\n super().__init__(model, api_key, **kwargs)\n self.api_key = api_key or os.getenv(\"PERPLEXITY_API_KEY\")\n self.base_url = \"https://api.perplexity.ai\"\n \n self.client = AsyncOpenAI(\n api_key=self.api_key,\n base_url=self.base_url,\n timeout=httpx.Timeout(120.0)\n )\n \n async def complete(\n self,\n prompt: Optional[str] = None,\n messages: Optional[List[Dict[str, str]]] = None,\n system: Optional[str] = None,\n max_tokens: int = 4000,\n temperature: float = 0.7,\n **kwargs\n ) -> str:\n \"\"\"Get completion from Perplexity.\"\"\"\n \n if messages is None:\n messages = []\n if system:\n messages.append({\"role\": \"system\", \"content\": system})\n if prompt:\n messages.append({\"role\": \"user\", \"content\": prompt})\n elif system:\n messages = [{\"role\": \"system\", \"content\": system}] + messages\n \n try:\n response = await self.client.chat.completions.create(\n model=self.model,\n messages=messages,\n max_tokens=max_tokens,\n temperature=temperature,\n **{**self.extra_params, **kwargs}\n )\n return response.choices[0].message.content\n except Exception as e:\n logger.error(f\"Perplexity API error: {str(e)}\")\n raise\n\n\nclass OllamaClient(BaseLLMClient):\n \"\"\"\n Client for Ollama (local models).\n Run models locally - completely free!\n \"\"\"\n \n AVAILABLE_MODELS = [\n \"llama3.3\",\n \"llama3.2\",\n \"llama3.1\",\n \"mistral\",\n \"mixtral\",\n \"qwen2.5\",\n \"gemma2\",\n \"phi4\",\n \"deepseek-coder-v2\",\n ]\n \n def __init__(\n self,\n model: str = \"llama3.3\",\n base_url: str = \"http://localhost:11434\",\n **kwargs\n ):\n super().__init__(model, api_key=\"ollama\", **kwargs)\n self.base_url = base_url\n \n async def complete(\n self,\n prompt: Optional[str] = None,\n messages: Optional[List[Dict[str, str]]] = None,\n system: Optional[str] = None,\n max_tokens: int = 4000,\n temperature: float = 0.7,\n **kwargs\n ) -> str:\n \"\"\"Get completion from Ollama.\"\"\"\n \n if messages is None:\n messages = []\n if prompt:\n messages.append({\"role\": \"user\", \"content\": prompt})\n \n payload = {\n \"model\": self.model,\n \"messages\": messages,\n \"stream\": False,\n \"options\": {\n \"temperature\": temperature,\n \"num_predict\": max_tokens,\n }\n }\n \n if system:\n payload[\"messages\"].insert(0, {\"role\": \"system\", \"content\": system})\n \n try:\n async with httpx.AsyncClient(timeout=300.0) as client:\n response = await client.post(\n f\"{self.base_url}/api/chat\",\n json=payload\n )\n response.raise_for_status()\n data = response.json()\n return data[\"message\"][\"content\"]\n except Exception as e:\n logger.error(f\"Ollama API error: {str(e)}\")\n raise\n\n\nclass MockLLMClient(BaseLLMClient):\n \"\"\"Mock LLM client for testing - generates code for true RLM.\"\"\"\n \n def __init__(self, model: str = \"mock-model\", **kwargs):\n super().__init__(model, **kwargs)\n self.call_count = 0\n \n async def complete(\n self,\n prompt: Optional[str] = None,\n messages: Optional[List[Dict[str, str]]] = None,\n system: Optional[str] = None,\n max_tokens: int = 4000,\n temperature: float = 0.7,\n **kwargs\n ) -> str:\n \"\"\"Return a mock code response for true RLM.\"\"\"\n self.call_count += 1\n \n prompt_text = prompt or \"\"\n if messages:\n for msg in messages:\n if isinstance(msg.get(\"content\"), str):\n prompt_text += msg[\"content\"]\n \n prompt_lower = prompt_text.lower()\n \n # For sub-LM calls (chunk analysis)\n if \"chunk\" in prompt_lower or \"section\" in prompt_lower or \"documents\" in prompt_lower:\n return \"This chunk discusses important topics including Ada Lovelace and Alan Turing.\"\n \n # For summarization requests\n if \"summarize\" in prompt_lower:\n return \"Summary: The text covers computing history.\"\n \n # First iterations: explore context\n if self.call_count <= 2:\n return '''```python\n# Initial exploration of the context\nprint(\"Exploring context...\")\nprint(f\"Type: {type(context)}\")\nprint(f\"Length: {len(context) if hasattr(context, '__len__') else 'N/A'}\")\n\nif isinstance(context, list):\n print(f\"Number of items: {len(context)}\")\n if len(context) > 0:\n print(f\"First item preview: {str(context[0])[:200]}\")\n \nif isinstance(context, list) and len(context) >= 50:\n section_50 = context[49]\n print(f\"Section 50 preview: {section_50[:300]}\")\n```\n'''\n else:\n # Final iteration: set the Final variable\n return '''```python\n# Analysis complete, setting Final answer\nFinal = \"Section 50 discusses The History of Computing, mentioning Ada Lovelace, Alan Turing, mechanical calculators, and the ENIAC computer.\"\n```'''\n\n\nclass LLMClientFactory:\n \"\"\"Factory for creating LLM clients with ALL latest 2025 models.\"\"\"\n \n # Complete provider information with ALL latest models\n PROVIDER_INFO = {\n \"openai\": {\n \"name\": \"OpenAI\",\n \"description\": \"GPT-5, GPT-4.5, o3, o4-mini, o1, GPT-4o\",\n \"models\": OpenAIClient.AVAILABLE_MODELS,\n \"key_url\": \"https://platform.openai.com/api-keys\",\n \"key_prefix\": \"sk-\",\n \"cost\": \"$0.15-30 per 1M tokens\",\n \"recommended\": [\"gpt-4o-mini\", \"o4-mini\", \"gpt-4o\"]\n },\n \"anthropic\": {\n \"name\": \"Anthropic\",\n \"description\": \"Claude Opus 4.6, Sonnet 4.5, Haiku 4.5, Claude 3.5\",\n \"models\": AnthropicClient.AVAILABLE_MODELS,\n \"key_url\": \"https://console.anthropic.com/\",\n \"key_prefix\": \"sk-ant-\",\n \"cost\": \"$0.50-30 per 1M tokens\",\n \"recommended\": [\"claude-3-5-sonnet-20241022\", \"claude-sonnet-4-5\", \"claude-haiku-4-5\"]\n },\n \"google\": {\n \"name\": \"Google Gemini\",\n \"description\": \"Gemini 3.0, 2.5, 2.0 Pro/Flash/Lite\",\n \"models\": GeminiClient.AVAILABLE_MODELS,\n \"key_url\": \"https://aistudio.google.com/app/apikey\",\n \"key_prefix\": \"\",\n \"cost\": \"$0.08-10 per 1M tokens\",\n \"recommended\": [\"gemini-2.5-flash\", \"gemini-2.5-pro\", \"gemini-1.5-flash\"]\n },\n \"kimi\": {\n \"name\": \"Moonshot AI (Kimi)\",\n \"description\": \"Kimi K3, K2.5, k1.5 - China's best models\",\n \"models\": KimiClient.AVAILABLE_MODELS,\n \"key_url\": \"https://platform.moonshot.cn/\",\n \"key_prefix\": \"sk-\",\n \"cost\": \"$0.50-5 per 1M tokens\",\n \"recommended\": [\"kimi-k2.5\", \"kimi-k3\", \"kimi-k1.5\"]\n },\n \"groq\": {\n \"name\": \"Groq\",\n \"description\": \"Llama 4, 3.3, 3.1, Mixtral - FASTEST inference\",\n \"models\": GroqClient.AVAILABLE_MODELS,\n \"key_url\": \"https://console.groq.com/keys\",\n \"key_prefix\": \"gsk_\",\n \"cost\": \"$0.05-0.60 per 1M tokens\",\n \"recommended\": [\"llama-3.3-70b-versatile\", \"llama-3.1-8b-instant\", \"meta-llama/llama-4-maverick-17b-128e-instruct\"]\n },\n \"together\": {\n \"name\": \"Together AI\",\n \"description\": \"Llama 4, DeepSeek V3, Qwen 3, Mixtral\",\n \"models\": TogetherClient.AVAILABLE_MODELS,\n \"key_url\": \"https://api.together.xyz/\",\n \"key_prefix\": \"\",\n \"cost\": \"$0.18-5 per 1M tokens\",\n \"recommended\": [\"meta-llama/Llama-3.3-70B-Instruct-Turbo\", \"deepseek-ai/DeepSeek-V3\"]\n },\n \"deepseek\": {\n \"name\": \"DeepSeek\",\n \"description\": \"DeepSeek V3, R1, Coder - CHEAPEST\",\n \"models\": DeepSeekClient.AVAILABLE_MODELS,\n \"key_url\": \"https://platform.deepseek.com/\",\n \"key_prefix\": \"sk-\",\n \"cost\": \"$0.07-0.30 per 1M tokens\",\n \"recommended\": [\"deepseek-chat\", \"deepseek-coder\"]\n },\n \"mistral\": {\n \"name\": \"Mistral AI\",\n \"description\": \"Mistral Large, Medium, Small, Codestral\",\n \"models\": MistralClient.AVAILABLE_MODELS,\n \"key_url\": \"https://console.mistral.ai/\",\n \"key_prefix\": \"\",\n \"cost\": \"$0.20-8 per 1M tokens\",\n \"recommended\": [\"mistral-large-latest\", \"mistral-small-latest\"]\n },\n \"cohere\": {\n \"name\": \"Cohere\",\n \"description\": \"Command R+, Command R, Command\",\n \"models\": CohereClient.AVAILABLE_MODELS,\n \"key_url\": \"https://dashboard.cohere.com/api-keys\",\n \"key_prefix\": \"\",\n \"cost\": \"$0.50-2.50 per 1M tokens\",\n \"recommended\": [\"command-r-plus\", \"command-r\"]\n },\n \"perplexity\": {\n \"name\": \"Perplexity\",\n \"description\": \"Sonar Pro, Sonar - Search-augmented\",\n \"models\": PerplexityClient.AVAILABLE_MODELS,\n \"key_url\": \"https://www.perplexity.ai/settings/api\",\n \"key_prefix\": \"pplx-\",\n \"cost\": \"$0.50-5 per 1M tokens\",\n \"recommended\": [\"sonar-pro\", \"sonar\"]\n },\n \"azure\": {\n \"name\": \"Azure OpenAI\",\n \"description\": \"Enterprise OpenAI (GPT-4o, o1, etc.)\",\n \"models\": [\"gpt-4o\", \"gpt-4\", \"gpt-35-turbo\", \"o1\", \"o3-mini\"],\n \"key_url\": \"https://portal.azure.com\",\n \"key_prefix\": \"\",\n \"cost\": \"Enterprise pricing\",\n \"recommended\": [\"gpt-4o\", \"o1\"]\n },\n \"ollama\": {\n \"name\": \"Ollama (Local)\",\n \"description\": \"Run Llama, Mistral, Qwen locally - FREE\",\n \"models\": OllamaClient.AVAILABLE_MODELS,\n \"key_url\": \"https://ollama.com/\",\n \"key_prefix\": \"N/A\",\n \"cost\": \"FREE (runs on your machine)\",\n \"recommended\": [\"llama3.3\", \"mistral\", \"qwen2.5\"]\n },\n \"mock\": {\n \"name\": \"Mock (Testing)\",\n \"description\": \"Fake responses for testing - FREE\",\n \"models\": [\"mock-model\"],\n \"key_url\": \"N/A\",\n \"key_prefix\": \"N/A\",\n \"cost\": \"FREE\",\n \"recommended\": [\"mock-model\"]\n }\n }\n \n @staticmethod\n def get_provider_info() -> Dict[str, Dict]:\n \"\"\"Get information about all providers for UI.\"\"\"\n return LLMClientFactory.PROVIDER_INFO\n \n @staticmethod\n def create(\n provider: str = \"openai\",\n model: Optional[str] = None,\n api_key: Optional[str] = None,\n **kwargs\n ) -> BaseLLMClient:\n \"\"\"Create an LLM client based on provider.\"\"\"\n provider = provider.lower()\n \n if provider == \"openai\":\n model = model or \"gpt-4o-mini\"\n return OpenAIClient(model=model, api_key=api_key, **kwargs)\n \n elif provider == \"anthropic\":\n model = model or \"claude-3-5-sonnet-20241022\"\n return AnthropicClient(model=model, api_key=api_key, **kwargs)\n \n elif provider == \"google\" or provider == \"gemini\":\n model = model or \"gemini-2.5-flash\"\n return GeminiClient(model=model, api_key=api_key, **kwargs)\n \n elif provider == \"kimi\" or provider == \"moonshot\":\n model = model or \"kimi-k2.5\"\n return KimiClient(model=model, api_key=api_key, **kwargs)\n \n elif provider == \"groq\":\n model = model or \"llama-3.3-70b-versatile\"\n return GroqClient(model=model, api_key=api_key, **kwargs)\n \n elif provider == \"together\":\n model = model or \"meta-llama/Llama-3.3-70B-Instruct-Turbo\"\n return TogetherClient(model=model, api_key=api_key, **kwargs)\n \n elif provider == \"deepseek\":\n model = model or \"deepseek-chat\"\n return DeepSeekClient(model=model, api_key=api_key, **kwargs)\n \n elif provider == \"mistral\":\n model = model or \"mistral-large-latest\"\n return MistralClient(model=model, api_key=api_key, **kwargs)\n \n elif provider == \"cohere\":\n model = model or \"command-r-plus\"\n return CohereClient(model=model, api_key=api_key, **kwargs)\n \n elif provider == \"perplexity\":\n model = model or \"sonar-pro\"\n return PerplexityClient(model=model, api_key=api_key, **kwargs)\n \n elif provider == \"azure\":\n model = model or \"gpt-4o\"\n return AzureOpenAIClient(model=model, api_key=api_key, **kwargs)\n \n elif provider == \"ollama\":\n model = model or \"llama3.3\"\n return OllamaClient(model=model, **kwargs)\n \n elif provider == \"mock\":\n return MockLLMClient(model=model or \"mock-model\", **kwargs)\n \n else:\n raise ValueError(f\"Unknown provider: {provider}\")\n"}}}} |