1 line
48 KiB
JSON
1 line
48 KiB
JSON
{"repo": "beevr-labs/kite", "files": {"kite/llm_providers.py": {"257a0accb729919f1c33fc2948e97feee7b1929e": {"t": 1785674495.4848492, "n": 47103, "h": "c67bc2cdd279264e", "c": "\"\"\"\nLLM Provider Abstraction Layer\nSupports multiple LLM providers with priority on open source.\n\nSupported Providers:\n- Ollama (Local, Free) - PRIORITY\n- LM Studio (Local, Free)\n- vLLM (Local, Free)\n- Anthropic Claude\n- OpenAI GPT\n- Google Gemini\n- Mistral AI\n- Groq (Fast inference)\n- Together AI\n- Replicate\n\"\"\"\n\nfrom typing import Dict, List, Optional, Any, Tuple\nfrom abc import ABC, abstractmethod\nimport os\nimport logging\nimport json\nimport asyncio\nimport time\nfrom tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type\n\n\nclass BaseLLMProvider(ABC):\n \"\"\"Base class for all LLM providers.\"\"\"\n \n @abstractmethod\n def complete(self, prompt: str, **kwargs) -> str:\n \"\"\"\n Generate completion.\n \n Args:\n prompt: Input text.\n response_schema: Optional Dict/Schema to enforce JSON output.\n \"\"\"\n pass\n \n @abstractmethod\n def chat(self, messages: List[Dict], **kwargs) -> str:\n \"\"\"Chat completion.\"\"\"\n pass\n \n @abstractmethod\n def embed(self, text: str) -> List[float]:\n \"\"\"Generate embeddings.\"\"\"\n pass\n \n @abstractmethod\n async def stream_complete(self, prompt: str, **kwargs):\n \"\"\"Stream completion.\"\"\"\n pass\n \n @abstractmethod\n async def stream_chat(self, messages: List[Dict], **kwargs):\n \"\"\"Stream chat completion.\"\"\"\n pass\n \n @property\n @abstractmethod\n def name(self) -> str:\n \"\"\"Provider name.\"\"\"\n pass\n \n async def complete_async(self, prompt: str, **kwargs) -> str:\n \"\"\"Async version of complete.\"\"\"\n import asyncio\n return await asyncio.to_thread(self.complete, prompt, **kwargs)\n \n async def chat_async(self, messages: List[Dict], **kwargs) -> str:\n \"\"\"Async version of chat.\"\"\"\n import asyncio\n return await asyncio.to_thread(self.chat, messages, **kwargs)\n\n\n# ============================================================================\n# LOCAL / OPENSOURCE PROVIDERS (PRIORITY)\n# ============================================================================\n\nclass OllamaProvider(BaseLLMProvider):\n \"\"\"\n Ollama - Run LLMs locally (FREE, OPENSOURCE)\n \n Models: llama3, mistral, codellama, phi, gemma, etc.\n Installation: curl -fsSL https://ollama.com/install.sh | sh\n \"\"\"\n \n def __init__(self, \n model: str = \"llama3\",\n base_url: str = \"http://localhost:11434\",\n timeout: float = 600.0,\n **kwargs):\n self.model = model\n self.base_url = base_url\n self.timeout = timeout\n self.logger = logging.getLogger(\"Ollama\")\n \n try:\n import httpx\n self._test_connection()\n except ImportError:\n raise ImportError(\"pip install httpx\")\n \n def _test_connection(self):\n \"\"\"Test Ollama connection.\"\"\"\n import httpx\n try:\n with httpx.Client(timeout=2.0) as client:\n response = client.get(f\"{self.base_url}/api/tags\")\n if response.status_code == 200:\n self.logger.info(f\"[OK] Ollama connected: {self.model}\")\n else:\n raise ConnectionError(f\"Ollama returned {response.status_code}\")\n except Exception as e:\n raise ConnectionError(f\"Ollama not running at {self.base_url}: {e}\")\n \n def _sanitize_ollama_params(self, kwargs: Dict) -> Dict:\n \"\"\"Helper to ensure only valid Ollama parameters are sent.\"\"\"\n valid_top_level = {'model', 'prompt', 'messages', 'stream', 'format', 'options', 'keep_alive', 'tools'}\n # Common model options that should go into 'options' dict\n valid_options = {\n 'num_keep', 'seed', 'num_predict', 'top_k', 'top_p', 'tfs_z', \n 'typical_p', 'repeat_last_n', 'temperature', 'repeat_penalty', \n 'presence_penalty', 'frequency_penalty', 'mixtral_mi', 'mixtral_m', \n 'mixtral_s', 'num_ctx', 'num_batch', 'num_gqa', 'num_gpu', \n 'main_gpu', 'low_vram', 'f16_kv', 'logits_all', 'vocab_only', \n 'use_mmap', 'use_mlock', 'num_thread'\n }\n \n sanitized = {}\n options = kwargs.get('options', {})\n \n for k, v in kwargs.items():\n if k in valid_top_level:\n sanitized[k] = v\n elif k in valid_options:\n options[k] = v\n \n if options:\n sanitized['options'] = options\n \n # Handle Structured Output (JSON Schema)\n if 'response_schema' in kwargs:\n sanitized['format'] = 'json'\n # We trust the caller to put instructions in the prompt.\n pass\n\n return sanitized\n\n def complete(self, prompt: str, **kwargs) -> str:\n \"\"\"Generate completion (sync).\"\"\"\n import httpx\n import threading\n \n params = self._sanitize_ollama_params({\n \"model\": self.model,\n \"prompt\": prompt,\n \"stream\": False,\n **kwargs\n })\n \n # Simple heartbeat thread\n stop_heartbeat = threading.Event()\n def heartbeat():\n start = time.time()\n while not stop_heartbeat.wait(30):\n self.logger.info(f\"Ollama is still thinking... ({int(time.time() - start)}s elapsed)\")\n \n h_thread = threading.Thread(target=heartbeat, daemon=True)\n h_thread.start()\n \n try:\n timeout = httpx.Timeout(self.timeout, read=None)\n with httpx.Client(timeout=timeout) as client:\n response = client.post(\n f\"{self.base_url}/api/generate\",\n json=params\n )\n data = response.json()\n if \"response\" in data:\n if kwargs.get(\"metrics\"):\n kwargs[\"metrics\"].record_llm_usage(\n provider=\"ollama\",\n model=self.model,\n prompt_tokens=data.get(\"prompt_eval_count\", 0),\n completion_tokens=data.get(\"eval_count\", 0)\n )\n return data[\"response\"]\n elif \"message\" in data and \"content\" in data[\"message\"]:\n return data[\"message\"][\"content\"]\n raise KeyError(f\"Unexpected Ollama response format: {data}\")\n finally:\n stop_heartbeat.set()\n\n async def complete_async(self, prompt: str, **kwargs) -> str:\n \"\"\"Native async complete.\"\"\"\n import httpx\n \n params = self._sanitize_ollama_params({\n \"model\": self.model,\n \"prompt\": prompt,\n \"stream\": False,\n **kwargs\n })\n \n async def heartbeat(task):\n start = time.time()\n while not task.done():\n await asyncio.sleep(30)\n if not task.done():\n self.logger.info(f\"Ollama is still thinking... ({int(time.time() - start)}s elapsed)\")\n\n timeout = httpx.Timeout(self.timeout, read=None)\n async with httpx.AsyncClient(timeout=timeout) as client:\n request_task = asyncio.create_task(client.post(\n f\"{self.base_url}/api/generate\",\n json=params\n ))\n heartbeat_task = asyncio.create_task(heartbeat(request_task))\n \n try:\n response = await request_task\n response.raise_for_status()\n data = response.json()\n if \"response\" in data:\n res = data[\"response\"]\n elif \"message\" in data and \"content\" in data[\"message\"]:\n res = data[\"message\"][\"content\"]\n else:\n raise KeyError(f\"Unexpected Ollama response format: {data}\")\n \n if not res and data.get(\"done\") and data.get(\"done_reason\") == \"load\":\n self.logger.warning(f\"Ollama returned 'load' reason. Retrying in 2s...\")\n await asyncio.sleep(2)\n return await self.complete_async(prompt, **kwargs)\n\n if not res or res.strip() == \"\":\n raise ValueError(f\"Ollama returned empty response for model {self.model}\")\n return res\n finally:\n heartbeat_task.cancel()\n \n def chat(self, messages: List[Dict], **kwargs) -> str:\n \"\"\"Chat completion (sync).\"\"\"\n import httpx\n import threading\n \n params = self._sanitize_ollama_params({\n \"model\": self.model,\n \"messages\": messages,\n \"stream\": False,\n **kwargs\n })\n \n # Inject Schema into System Prompt if provided and format is JSON\n if kwargs.get(\"response_schema\") and params.get(\"format\") == \"json\":\n schema_str = json.dumps(kwargs[\"response_schema\"], indent=2)\n sys_msg = f\"\\n\\nIMPORTANT: Output data MUST be valid JSON matching this schema:\\n{schema_str}\"\n if params[\"messages\"] and params[\"messages\"][0][\"role\"] == \"system\":\n params[\"messages\"][0][\"content\"] += sys_msg\n else:\n params[\"messages\"].insert(0, {\"role\": \"system\", \"content\": f\"You are a helpful assistant.{sys_msg}\"})\n \n stop_heartbeat = threading.Event()\n def heartbeat():\n start = time.time()\n while not stop_heartbeat.wait(30):\n self.logger.info(f\"Ollama is still thinking... ({int(time.time() - start)}s elapsed)\")\n \n h_thread = threading.Thread(target=heartbeat, daemon=True)\n h_thread.start()\n \n try:\n timeout = httpx.Timeout(self.timeout, read=None)\n with httpx.Client(timeout=timeout) as client:\n response = client.post(\n f\"{self.base_url}/api/chat\",\n json=params\n )\n if response.status_code != 200:\n self.logger.error(f\"Ollama Chat Error {response.status_code}: {response.text}\")\n \n data = response.json()\n if \"message\" in data:\n if kwargs.get(\"metrics\"):\n kwargs[\"metrics\"].record_llm_usage(\n provider=\"ollama\",\n model=self.model,\n prompt_tokens=data.get(\"prompt_eval_count\", 0),\n completion_tokens=data.get(\"eval_count\", 0)\n )\n msg = data[\"message\"]\n if msg.get(\"tool_calls\"):\n return {\"content\": msg.get(\"content\"), \"tool_calls\": msg[\"tool_calls\"]}\n return msg.get(\"content\", \"\")\n elif \"response\" in data:\n return data[\"response\"]\n raise KeyError(f\"Unexpected Ollama chat response format: {data}\")\n finally:\n stop_heartbeat.set()\n\n async def chat_async(self, messages: List[Dict], **kwargs) -> str:\n \"\"\"Native async chat.\"\"\"\n import httpx\n \n params = self._sanitize_ollama_params({\n \"model\": self.model,\n \"messages\": messages,\n \"stream\": False,\n **kwargs\n })\n\n if kwargs.get(\"response_schema\") and params.get(\"format\") == \"json\":\n schema_str = json.dumps(kwargs[\"response_schema\"], indent=2)\n sys_msg = f\"\\n\\nIMPORTANT: Output data MUST be valid JSON matching this schema:\\n{schema_str}\"\n if params[\"messages\"] and params[\"messages\"][0][\"role\"] == \"system\":\n params[\"messages\"][0][\"content\"] += sys_msg\n else:\n params[\"messages\"].insert(0, {\"role\": \"system\", \"content\": f\"You are a helpful assistant.{sys_msg}\"})\n \n async def heartbeat(task):\n start = time.time()\n while not task.done():\n await asyncio.sleep(30)\n if not task.done():\n self.logger.info(f\"Ollama is still thinking... ({int(time.time() - start)}s elapsed)\")\n\n timeout = httpx.Timeout(self.timeout, read=None)\n async with httpx.AsyncClient(timeout=timeout) as client:\n request_task = asyncio.create_task(client.post(\n f\"{self.base_url}/api/chat\",\n json=params\n ))\n heartbeat_task = asyncio.create_task(heartbeat(request_task))\n \n try:\n response = await request_task\n if response.status_code != 200:\n error_text = response.text\n self.logger.error(f\"Ollama Chat Async Error {response.status_code}: {error_text}\")\n response.raise_for_status()\n \n data = response.json()\n res = None\n if \"message\" in data:\n if kwargs.get(\"metrics\"):\n kwargs[\"metrics\"].record_llm_usage(\n provider=\"ollama\",\n model=self.model,\n prompt_tokens=data.get(\"prompt_eval_count\", 0),\n completion_tokens=data.get(\"eval_count\", 0)\n )\n msg = data[\"message\"]\n if msg.get(\"tool_calls\"):\n res = {\"content\": msg.get(\"content\"), \"tool_calls\": msg[\"tool_calls\"]}\n else:\n res = msg.get(\"content\")\n elif \"response\" in data:\n res = data[\"response\"]\n \n if not res and data.get(\"done\") and data.get(\"done_reason\") == \"load\":\n self.logger.warning(f\"Ollama returned 'load' reason in chat. Retrying in 2s...\")\n await asyncio.sleep(2)\n return await self.chat_async(messages, **kwargs)\n\n if res is None:\n raise ValueError(f\"Ollama returned empty response for chat model {self.model}\")\n return res\n finally:\n heartbeat_task.cancel()\n \n def embed(self, text: str) -> List[float]:\n \"\"\"Generate embeddings.\"\"\"\n import httpx\n with httpx.Client(timeout=self.timeout) as client:\n response = client.post(\n f\"{self.base_url}/api/embeddings\",\n json={\n \"model\": self.model,\n \"prompt\": text\n }\n )\n return response.json()[\"embedding\"]\n \n async def stream_complete(self, prompt: str, **kwargs):\n \"\"\"Stream completion.\"\"\"\n import httpx\n import json\n async with httpx.AsyncClient(timeout=self.timeout) as client:\n async with client.stream(\n \"POST\",\n f\"{self.base_url}/api/generate\",\n json={\n \"model\": self.model,\n \"prompt\": prompt,\n \"stream\": True,\n **kwargs\n }\n ) as response:\n async for line in response.aiter_lines():\n if line:\n chunk = json.loads(line)\n yield chunk.get(\"response\", \"\")\n if chunk.get(\"done\"):\n break\n\n async def stream_chat(self, messages: List[Dict], **kwargs):\n \"\"\"Stream chat completion.\"\"\"\n import httpx\n import json\n async with httpx.AsyncClient(timeout=self.timeout) as client:\n async with client.stream(\n \"POST\",\n f\"{self.base_url}/api/chat\",\n json={\n \"model\": self.model,\n \"messages\": messages,\n \"stream\": True,\n **kwargs\n }\n ) as response:\n async for line in response.aiter_lines():\n if line:\n chunk = json.loads(line)\n yield chunk.get(\"message\", {}).get(\"content\", \"\")\n if chunk.get(\"done\"):\n break\n \n @property\n def name(self) -> str:\n return f\"Ollama/{self.model}\"\n\n\nclass LMStudioProvider(BaseLLMProvider):\n \"\"\"\n LM Studio - Local LLM with GUI (FREE, OPENSOURCE)\n \n Download: https://lmstudio.ai/\n Compatible with OpenAI API format.\n \"\"\"\n \n def __init__(self, \n model: str = \"local-model\",\n base_url: str = \"http://localhost:1234/v1\",\n **kwargs):\n self.model = model\n self.base_url = base_url\n self.logger = logging.getLogger(\"LMStudio\")\n \n try:\n import openai\n import requests\n import httpx\n self.requests = requests\n self.client = openai.OpenAI(\n base_url=base_url,\n api_key=\"lm-studio\"\n )\n self.async_client = openai.AsyncOpenAI(\n base_url=base_url,\n api_key=\"lm-studio\"\n )\n self._test_connection()\n self.logger.info(f\"[OK] LM Studio connected: {model}\")\n except Exception as e:\n raise ConnectionError(f\"LM Studio not found: {e}\")\n \n def _test_connection(self):\n \"\"\"Test LM Studio connection.\"\"\"\n try:\n response = self.requests.get(f\"{self.base_url}/models\", timeout=1)\n if response.status_code != 200:\n raise ConnectionError(f\"LM Studio returned {response.status_code}\")\n except Exception as e:\n raise ConnectionError(f\"LM Studio not running at {self.base_url}: {e}\")\n \n def _sanitize_params(self, kwargs: Dict) -> Dict:\n \"\"\"Translate Kite params (format='json') to LM Studio (response_format).\"\"\"\n clean = {}\n # LM Studio is OpenAI compatible\n valid = {'temperature', 'max_tokens', 'top_p', 'stream', 'stop', 'response_format', 'seed'}\n for k, v in kwargs.items():\n if k == 'format' and v == 'json':\n clean['response_format'] = {\"type\": \"json_object\"}\n elif k in valid:\n clean[k] = v\n return clean\n\n def complete(self, prompt: str, **kwargs) -> str:\n \"\"\"Generate completion.\"\"\"\n response = self.client.completions.create(\n model=self.model,\n prompt=prompt,\n **kwargs\n )\n return response.choices[0].text\n \n async def complete_async(self, prompt: str, **kwargs) -> str:\n \"\"\"Async completion.\"\"\"\n response = await self.async_client.completions.create(\n model=self.model,\n prompt=prompt,\n **kwargs\n )\n return response.choices[0].text\n\n async def chat_async(self, messages: List[Dict], **kwargs) -> str:\n \"\"\"Async chat completion.\"\"\"\n response = await self.async_client.chat.completions.create(\n model=self.model,\n messages=messages,\n **kwargs\n )\n return response.choices[0].message.content\n\n def chat(self, messages: List[Dict], **kwargs) -> str:\n \"\"\"Chat completion.\"\"\"\n params = self._sanitize_params(kwargs)\n response = self.client.chat.completions.create(\n model=self.model,\n messages=messages,\n **params\n )\n return response.choices[0].message.content\n \n def embed(self, text: str) -> List[float]:\n \"\"\"Generate embeddings.\"\"\"\n response = self.client.embeddings.create(\n model=self.model,\n input=text\n )\n return response.data[0].embedding\n \n async def stream_complete(self, prompt: str, **kwargs):\n \"\"\"Stream completion.\"\"\"\n params = self._sanitize_params(kwargs)\n stream = await self.async_client.completions.create(\n model=self.model,\n prompt=prompt,\n stream=True,\n **params\n )\n async for chunk in stream:\n if hasattr(chunk, 'choices') and chunk.choices and chunk.choices[0].text:\n yield chunk.choices[0].text\n\n async def stream_chat(self, messages: List[Dict], **kwargs):\n \"\"\"Stream chat completion.\"\"\"\n params = self._sanitize_params(kwargs)\n stream = await self.async_client.chat.completions.create(\n model=self.model,\n messages=messages,\n stream=True,\n **params\n )\n async for chunk in stream:\n if hasattr(chunk, 'choices') and chunk.choices and chunk.choices[0].delta.content:\n yield chunk.choices[0].delta.content\n \n @property\n def name(self) -> str:\n return f\"LMStudio/{self.model}\"\n\n\nclass VLLMProvider(BaseLLMProvider):\n \"\"\"\n vLLM - Fast inference server (FREE, OPENSOURCE)\n \"\"\"\n \n def __init__(self, \n model: str = \"meta-llama/Llama-2-7b-hf\",\n base_url: str = \"http://localhost:8000\",\n **kwargs):\n self.model = model\n self.base_url = base_url\n self.logger = logging.getLogger(\"vLLM\")\n \n try:\n import requests\n self.requests = requests\n self._test_connection()\n self.logger.info(f\"[OK] vLLM connected: {model}\")\n except ImportError:\n raise ImportError(\"pip install requests\")\n except Exception as e:\n raise ConnectionError(f\"vLLM server not found at {base_url}: {e}\")\n \n def _test_connection(self):\n try:\n response = self.requests.get(f\"{self.base_url}/models\", timeout=1)\n if response.status_code != 200:\n response = self.requests.get(f\"{self.base_url}/health\", timeout=1)\n if response.status_code != 200:\n raise ConnectionError(f\"vLLM returned {response.status_code}\")\n except Exception as e:\n raise ConnectionError(f\"vLLM not running at {self.base_url}\")\n \n async def complete_async(self, prompt: str, **kwargs) -> str:\n import httpx\n params = {\n \"prompt\": prompt,\n \"max_tokens\": kwargs.get(\"max_tokens\", 512),\n **kwargs\n }\n async with httpx.AsyncClient(timeout=30.0) as client:\n response = await client.post(\n f\"{self.base_url}/generate\",\n json=params\n )\n return response.json()[\"text\"][0]\n\n async def chat_async(self, messages: List[Dict], **kwargs) -> str:\n prompt = \"\\n\".join([f\"{m['role']}: {m['content']}\" for m in messages])\n return await self.complete_async(prompt, **kwargs)\n\n def complete(self, prompt: str, **kwargs) -> str:\n response = self.requests.post(\n f\"{self.base_url}/generate\",\n json={\n \"prompt\": prompt,\n \"max_tokens\": kwargs.get(\"max_tokens\", 512),\n **kwargs\n }\n )\n return response.json()[\"text\"][0]\n\n def chat(self, messages: List[Dict], **kwargs) -> str:\n prompt = \"\\n\".join([f\"{m['role']}: {m['content']}\" for m in messages])\n return self.complete(prompt, **kwargs)\n \n async def stream_complete(self, prompt: str, **kwargs):\n import httpx\n import json\n params = {\n \"prompt\": prompt,\n \"stream\": True,\n \"max_tokens\": kwargs.get(\"max_tokens\", 512),\n **kwargs\n }\n async with httpx.AsyncClient(timeout=60.0) as client:\n async with client.stream(\"POST\", f\"{self.base_url}/generate\", json=params) as response:\n async for line in response.aiter_lines():\n if line:\n chunk = json.loads(line)\n yield chunk.get(\"text\", [\"\"])[0]\n\n async def stream_chat(self, messages: List[Dict], **kwargs):\n prompt = \"\\n\".join([f\"{m['role']}: {m['content']}\" for m in messages])\n async for chunk in self.stream_complete(prompt, **kwargs):\n yield chunk\n \n def embed(self, text: str) -> List[float]:\n from .embedding_providers import EmbeddingFactory\n if not hasattr(self, '_embedding_fallback'):\n self._embedding_fallback = EmbeddingFactory.auto_detect()\n return self._embedding_fallback.embed(text)\n \n @property\n def name(self) -> str:\n return f\"vLLM/{self.model}\"\n\n\nclass MockLLMProvider(BaseLLMProvider):\n \"\"\"Mock LLM Provider for testing.\"\"\"\n \n def __init__(self, model: str = \"mock-model\", **kwargs):\n self.model = model\n \n def complete(self, prompt: str, **kwargs) -> str:\n return f\"Mock response to: {prompt[:50]}...\"\n \n def chat(self, messages: List[Dict], **kwargs) -> str:\n last_msg = messages[-1][\"content\"]\n if \"ORD-001\" in last_msg and kwargs.get(\"tools\"):\n return {\n \"content\": \"\",\n \"tool_calls\": [{\n \"function\": {\n \"name\": \"search_order\",\n \"arguments\": '{\"order_id\": \"ORD-001\"}'\n }\n }]\n }\n \n return \"Mock chat response: I'm here to help with your agentic tasks!\"\n \n def embed(self, text: str) -> List[float]:\n import random\n return [random.random() for _ in range(1536)]\n \n async def chat_async(self, messages: List[Dict], **kwargs):\n return self.chat(messages, **kwargs)\n \n async def stream_complete(self, prompt: str, **kwargs):\n yield f\"Mock stream response to: {prompt[:20]}...\"\n \n async def stream_chat(self, messages: List[Dict], **kwargs):\n yield \"Mock chat stream: \"\n yield \"I'm \"\n yield \"helping!\"\n \n @property\n def name(self) -> str:\n return \"Mock/LLM\"\n\n\n# ============================================================================\n# CLOUD OPENSOURCE PROVIDERS\n# ============================================================================\n\nclass GroqProvider(BaseLLMProvider):\n \"\"\"\n Groq - Ultra-fast inference (FREE tier, OPENSOURCE models)\n \"\"\"\n \n def __init__(self, \n model: str = \"llama3-70b-8192\",\n api_key: Optional[str] = None,\n **kwargs):\n self.model = model\n self.api_key = api_key or os.getenv(\"GROQ_API_KEY\")\n self.logger = logging.getLogger(\"Groq\")\n \n if not self.api_key:\n raise ValueError(\"GROQ_API_KEY required\")\n \n try:\n from groq import Groq, AsyncGroq\n self.client = Groq(api_key=self.api_key)\n self.async_client = AsyncGroq(api_key=self.api_key)\n self.logger.info(f\"[OK] Groq connected: {model}\")\n except ImportError:\n raise ImportError(\"pip install groq\")\n \n def _sanitize_params(self, kwargs: Dict) -> Dict:\n \"\"\"Translate Kite params (format='json') to Groq/OpenAI (response_format).\"\"\"\n clean = {}\n # Valid Groq params\n valid = {'temperature', 'max_tokens', 'top_p', 'stream', 'stop', 'response_format', 'seed', 'tools', 'tool_choice'}\n \n for k, v in kwargs.items():\n if k == 'format' and v == 'json':\n clean['response_format'] = {\"type\": \"json_object\"}\n elif k in valid:\n clean[k] = v\n return clean\n \n async def complete_async(self, prompt: str, **kwargs) -> str:\n \"\"\"Async completion.\"\"\"\n return await self.chat_async([{\"role\": \"user\", \"content\": prompt}], **kwargs)\n\n @retry(wait=wait_exponential(multiplier=1, min=4, max=60), stop=stop_after_attempt(5), reraise=True)\n async def chat_async(self, messages: List[Dict], **kwargs) -> str:\n \"\"\"Async chat completion.\"\"\"\n params = self._sanitize_params(kwargs)\n try:\n response = await self.async_client.chat.completions.create(\n model=self.model,\n messages=messages,\n **params\n )\n except Exception as e:\n self.logger.error(f\"Groq Chat Async Error: {e}\")\n raise\n \n msg = response.choices[0].message\n if msg.tool_calls:\n # Convert to dicts same as sync version\n tool_calls = []\n for tc in msg.tool_calls:\n tool_calls.append({\n \"id\": tc.id,\n \"type\": tc.type,\n \"function\": {\n \"name\": tc.function.name,\n \"arguments\": tc.function.arguments\n }\n })\n return {\"content\": msg.content, \"tool_calls\": tool_calls}\n \n return msg.content\n\n def complete(self, prompt: str, **kwargs) -> str:\n \"\"\"Generate completion.\"\"\"\n return self.chat([{\"role\": \"user\", \"content\": prompt}], **kwargs)\n \n @retry(wait=wait_exponential(multiplier=1, min=4, max=60), stop=stop_after_attempt(5), reraise=True)\n def chat(self, messages: List[Dict], **kwargs) -> str:\n \"\"\"Chat completion.\"\"\"\n params = self._sanitize_params(kwargs)\n try:\n response = self.client.chat.completions.create(\n model=self.model,\n messages=messages,\n **params\n )\n except Exception as e:\n self.logger.error(f\"Groq Chat Error: {e}\")\n raise\n \n msg = response.choices[0].message\n if msg.tool_calls:\n # Convert objects to dicts for Agent.run\n tool_calls = []\n for tc in msg.tool_calls:\n tool_calls.append({\n \"id\": tc.id,\n \"type\": tc.type,\n \"function\": {\n \"name\": tc.function.name,\n \"arguments\": tc.function.arguments\n }\n })\n return {\"content\": msg.content, \"tool_calls\": tool_calls}\n \n return msg.content\n \n async def stream_complete(self, prompt: str, **kwargs):\n \"\"\"Stream completion.\"\"\"\n params = self._sanitize_params(kwargs)\n stream = await self.async_client.chat.completions.create(\n model=self.model,\n messages=[{\"role\": \"user\", \"content\": prompt}],\n stream=True,\n **params\n )\n async for chunk in stream:\n if chunk.choices[0].delta.content:\n yield chunk.choices[0].delta.content\n\n async def stream_chat(self, messages: List[Dict], **kwargs):\n \"\"\"Stream chat completion.\"\"\"\n params = self._sanitize_params(kwargs)\n stream = await self.async_client.chat.completions.create(\n model=self.model,\n messages=messages,\n stream=True,\n **params\n )\n async for chunk in stream:\n if chunk.choices[0].delta.content:\n yield chunk.choices[0].delta.content\n \n def embed(self, text: str) -> List[float]:\n \"\"\"Generate embeddings (fallback).\"\"\"\n from .embedding_providers import EmbeddingFactory\n if not hasattr(self, '_embedding_fallback'):\n self._embedding_fallback = EmbeddingFactory.auto_detect()\n return self._embedding_fallback.embed(text)\n \n @property\n def name(self) -> str:\n return f\"Groq/{self.model}\"\n\n\nclass TogetherProvider(BaseLLMProvider):\n \"\"\"\n Together AI - Opensource models (PAID, but cheap)\n \"\"\"\n \n def __init__(self, \n model: str = \"mistralai/Mixtral-8x7B-Instruct-v0.1\",\n api_key: Optional[str] = None,\n **kwargs):\n self.model = model\n self.api_key = api_key or os.getenv(\"TOGETHER_API_KEY\")\n self.logger = logging.getLogger(\"Together\")\n \n if not self.api_key:\n raise ValueError(\"TOGETHER_API_KEY required\")\n \n try:\n import together\n together.api_key = self.api_key\n self.together = together\n self.logger.info(f\"[OK] Together AI connected: {model}\")\n except ImportError:\n raise ImportError(\"pip install together\")\n \n async def complete_async(self, prompt: str, **kwargs) -> str:\n import httpx\n params = {\n \"model\": self.model,\n \"prompt\": prompt,\n **kwargs\n }\n headers = {\"Authorization\": f\"Bearer {self.api_key}\"}\n async with httpx.AsyncClient() as client:\n response = await client.post(\n \"https://api.together.xyz/v1/completions\",\n json=params,\n headers=headers\n )\n data = response.json()\n return data['choices'][0]['text']\n\n async def chat_async(self, messages: List[Dict], **kwargs) -> str:\n prompt = \"\\n\".join([f\"{m['role']}: {m['content']}\" for m in messages])\n return await self.complete_async(prompt, **kwargs)\n\n def complete(self, prompt: str, **kwargs) -> str:\n response = self.together.Complete.create(\n prompt=prompt,\n model=self.model,\n **kwargs\n )\n return response['output']['choices'][0]['text']\n\n def chat(self, messages: List[Dict], **kwargs) -> str:\n prompt = \"\\n\".join([f\"{m['role']}: {m['content']}\" for m in messages])\n return self.complete(prompt, **kwargs)\n \n def embed(self, text: str) -> List[float]:\n response = self.together.Embeddings.create(\n input=text,\n model=\"togethercomputer/m2-bert-80M-8k-retrieval\"\n )\n return response['data'][0]['embedding']\n \n async def stream_complete(self, prompt: str, **kwargs):\n import httpx\n import json\n params = {\n \"model\": self.model,\n \"prompt\": prompt,\n \"stream\": True,\n **kwargs\n }\n headers = {\"Authorization\": f\"Bearer {self.api_key}\"}\n async with httpx.AsyncClient(timeout=60.0) as client:\n async with client.stream(\n \"POST\", \n \"https://api.together.xyz/v1/completions\", \n json=params, \n headers=headers\n ) as response:\n async for line in response.aiter_lines():\n if line.startswith(\"data: \"):\n line = line[6:]\n if not line or line == \"[DONE]\":\n continue\n try:\n chunk = json.loads(line)\n yield chunk['choices'][0]['text']\n except:\n continue\n\n async def stream_chat(self, messages: List[Dict], **kwargs):\n prompt = \"\\n\".join([f\"{m['role']}: {m['content']}\" for m in messages])\n async for chunk in self.stream_complete(prompt, **kwargs):\n yield chunk\n \n @property\n def name(self) -> str:\n return f\"Together/{self.model}\"\n\n\n# ============================================================================\n# COMMERCIAL PROVIDERS (Secondary)\n# ============================================================================\n\nclass AnthropicProvider(BaseLLMProvider):\n \"\"\"Anthropic Claude (PAID).\"\"\"\n \n def __init__(self, \n model: str = \"claude-3-sonnet-20240229\",\n api_key: Optional[str] = None,\n **kwargs):\n self.model = model\n self.api_key = api_key or os.getenv(\"ANTHROPIC_API_KEY\")\n \n try:\n import anthropic\n self.client = anthropic.Anthropic(api_key=self.api_key)\n self.async_client = anthropic.AsyncAnthropic(api_key=self.api_key)\n except ImportError:\n raise ImportError(\"pip install anthropic\")\n \n def _prepare_messages(self, messages: List[Dict]) -> Tuple[str, List[Dict]]:\n \"\"\"Extract system prompt and filter messages.\"\"\"\n system_prompt = \"\"\n filtered_messages = []\n for msg in messages:\n if msg[\"role\"] == \"system\":\n system_prompt += msg[\"content\"] + \"\\n\"\n else:\n filtered_messages.append(msg)\n return system_prompt.strip(), filtered_messages\n \n def complete(self, prompt: str, **kwargs) -> str:\n return self.chat([{\"role\": \"user\", \"content\": prompt}], **kwargs)\n \n def chat(self, messages: List[Dict], **kwargs) -> str:\n system, filtered_msgs = self._prepare_messages(messages)\n \n params = {\n \"model\": self.model,\n \"messages\": filtered_msgs,\n \"max_tokens\": kwargs.get(\"max_tokens\", 1024),\n **kwargs\n }\n if system:\n params[\"system\"] = system\n \n response = self.client.messages.create(**params)\n return response.content[0].text\n \n async def complete_async(self, prompt: str, **kwargs) -> str:\n return await self.chat_async([{\"role\": \"user\", \"content\": prompt}], **kwargs)\n\n async def chat_async(self, messages: List[Dict], **kwargs) -> str:\n system, filtered_msgs = self._prepare_messages(messages)\n \n params = {\n \"model\": self.model,\n \"messages\": filtered_msgs,\n \"max_tokens\": kwargs.get(\"max_tokens\", 1024),\n **kwargs\n }\n if system:\n params[\"system\"] = system\n\n response = await self.async_client.messages.create(**params)\n return response.content[0].text\n\n async def stream_complete(self, prompt: str, **kwargs):\n async for chunk in self.stream_chat([{\"role\": \"user\", \"content\": prompt}], **kwargs):\n yield chunk\n\n async def stream_chat(self, messages: List[Dict], **kwargs):\n system, filtered_msgs = self._prepare_messages(messages)\n \n params = {\n \"model\": self.model,\n \"messages\": filtered_msgs,\n \"max_tokens\": kwargs.get(\"max_tokens\", 1024),\n **kwargs\n }\n if system:\n params[\"system\"] = system\n\n async with self.async_client.messages.stream(**params) as stream:\n async for text in stream.text_stream:\n yield text\n \n def embed(self, text: str) -> List[float]:\n from .embedding_providers import EmbeddingFactory\n if not hasattr(self, '_embedding_fallback'):\n self._embedding_fallback = EmbeddingFactory.auto_detect()\n return self._embedding_fallback.embed(text)\n \n @property\n def name(self) -> str:\n return f\"Anthropic/{self.model}\"\n\nclass OpenAIProvider(BaseLLMProvider):\n \"\"\"OpenAI GPT (PAID).\"\"\"\n \n def __init__(self, \n model: str = \"gpt-4-turbo-preview\",\n api_key: Optional[str] = None,\n **kwargs):\n self.model = model\n self.api_key = api_key or os.getenv(\"OPENAI_API_KEY\")\n \n try:\n import openai\n self.client = openai.OpenAI(api_key=self.api_key)\n self._test_connection()\n except Exception as e:\n raise ConnectionError(f\"OpenAI failed: {e}\")\n\n def _sanitize_params(self, kwargs: Dict) -> Dict:\n \"\"\"Translate Kite params (format='json') to OpenAI (response_format).\"\"\"\n clean = {}\n # Valid OpenAI params\n valid = {'temperature', 'max_tokens', 'top_p', 'stream', 'stop', 'response_format', 'seed', 'n', 'presence_penalty', 'frequency_penalty'}\n \n for k, v in kwargs.items():\n if k == 'format' and v == 'json':\n clean['response_format'] = {\"type\": \"json_object\"}\n elif k in valid:\n clean[k] = v\n return clean\n \n def _test_connection(self):\n \"\"\"Test OpenAI connection (auth check).\"\"\"\n if not self.api_key or self.api_key.startswith(\"sk-...\"):\n raise ValueError(\"OpenAI API key is invalid or placeholder\")\n try:\n self.client.models.list()\n except Exception as e:\n raise ConnectionError(f\"OpenAI auth failed: {e}\")\n \n async def chat_async(self, messages: List[Dict], **kwargs) -> str:\n \"\"\"True async chat for OpenAI.\"\"\"\n if not hasattr(self, 'async_client'):\n import openai\n self.async_client = openai.AsyncOpenAI(api_key=self.api_key)\n \n params = self._sanitize_params(kwargs)\n response = await self.async_client.chat.completions.create(\n model=self.model,\n messages=messages,\n **params\n )\n return response.choices[0].message.content\n\n async def complete_async(self, prompt: str, **kwargs) -> str:\n \"\"\"True async complete for OpenAI.\"\"\"\n return await self.chat_async([{\"role\": \"user\", \"content\": prompt}], **kwargs)\n \n def complete(self, prompt: str, **kwargs) -> str:\n return self.chat([{\"role\": \"user\", \"content\": prompt}], **kwargs)\n \n def chat(self, messages: List[Dict], **kwargs) -> str:\n params = self._sanitize_params(kwargs)\n response = self.client.chat.completions.create(\n model=self.model,\n messages=messages,\n **params\n )\n return response.choices[0].message.content\n \n def embed(self, text: str) -> List[float]:\n response = self.client.embeddings.create(\n model=\"text-embedding-3-small\",\n input=text\n )\n return response.data[0].embedding\n \n async def stream_complete(self, prompt: str, **kwargs):\n \"\"\"Stream completion.\"\"\"\n # Ensure async_client is initialized\n if not hasattr(self, 'async_client'):\n import openai\n self.async_client = openai.AsyncOpenAI(api_key=self.api_key)\n \n params = self._sanitize_params(kwargs)\n stream = await self.async_client.completions.create(\n model=self.model,\n prompt=prompt,\n stream=True,\n **params\n )\n async for chunk in stream:\n if chunk.choices[0].text:\n yield chunk.choices[0].text\n\n async def stream_chat(self, messages: List[Dict], **kwargs):\n \"\"\"Stream chat completion.\"\"\"\n # Ensure async_client is initialized\n if not hasattr(self, 'async_client'):\n import openai\n self.async_client = openai.AsyncOpenAI(api_key=self.api_key)\n\n params = self._sanitize_params(kwargs)\n stream = await self.async_client.chat.completions.create(\n model=self.model,\n messages=messages,\n stream=True,\n **params\n )\n async for chunk in stream:\n if chunk.choices[0].delta.content:\n yield chunk.choices[0].delta.content\n \n @property\n def name(self) -> str:\n return f\"OpenAI/{self.model}\"\n\n\nclass LLMFactory:\n \"\"\"\n Factory for creating LLM providers.\n \n Priority order:\n 1. Local/Free (Ollama, LM Studio, vLLM)\n 2. Cloud Free Tier (Groq)\n 3. Cloud Opensource (Together)\n 4. Commercial (Claude, GPT)\n \"\"\"\n \n PROVIDERS = {\n # Local (Priority)\n 'ollama': OllamaProvider,\n 'lmstudio': LMStudioProvider,\n 'vllm': VLLMProvider,\n \n # Cloud Opensource\n 'groq': GroqProvider,\n 'together': TogetherProvider,\n \n # Commercial\n 'anthropic': AnthropicProvider,\n 'openai': OpenAIProvider,\n \n # Testing\n 'mock': MockLLMProvider,\n }\n \n @classmethod\n def create(cls, \n provider: str = \"ollama\",\n model: Optional[str] = None,\n **kwargs) -> BaseLLMProvider:\n \"\"\"\n Create LLM provider.\n \n Args:\n provider: Provider name\n model: Model name (optional, uses default)\n **kwargs: Provider-specific kwargs\n \"\"\"\n if provider not in cls.PROVIDERS:\n raise ValueError(f\"Unknown provider: {provider}. \"\n f\"Available: {list(cls.PROVIDERS.keys())}\")\n \n provider_class = cls.PROVIDERS[provider]\n \n # Create with model if specified\n if model:\n return provider_class(model=model, **kwargs)\n else:\n return provider_class(**kwargs)\n \n @classmethod\n def auto_detect(cls, timeout: float = 600.0) -> BaseLLMProvider:\n \"\"\"\n Auto-detect best available provider.\n \n Priority:\n 1. Try Ollama (local)\n 2. Try LM Studio (local)\n 3. Try Groq (cloud, free)\n 4. Try OpenAI (fallback)\n \"\"\"\n logger = logging.getLogger(\"LLMFactory\")\n \n # Try Ollama\n try:\n provider = cls.create(\"ollama\", timeout=timeout)\n logger.info(\"[OK] Using Ollama (local, free)\")\n return provider\n except:\n pass\n \n # Try LM Studio\n try:\n provider = cls.create(\"lmstudio\", timeout=timeout)\n logger.info(\"[OK] Using LM Studio (local, free)\")\n return provider\n except:\n pass\n \n # Try Groq\n if os.getenv(\"GROQ_API_KEY\"):\n try:\n provider = cls.create(\"groq\", timeout=timeout)\n logger.info(\"[OK] Using Groq (cloud, free tier)\")\n return provider\n except:\n pass\n \n # Fallback to OpenAI\n if os.getenv(\"OPENAI_API_KEY\"):\n try:\n provider = cls.create(\"openai\", timeout=timeout)\n logger.warning(\" Using OpenAI (commercial, paid)\")\n return provider\n except:\n pass\n \n # Try mock as ultimate fallback\n try:\n return cls.create(\"mock\", timeout=timeout)\n except:\n pass\n \n raise RuntimeError(\n \"No LLM provider available. Install Ollama or set API keys.\"\n )\n\n\nif __name__ == \"__main__\":\n print(\"LLM Provider Examples:\\n\")\n \n # Auto-detect\n print(\"1. Auto-detect:\")\n llm = LLMFactory.auto_detect()\n print(f\" Using: {llm.name}\\n\")\n \n # Specific providers\n print(\"2. Specific providers:\")\n \n providers = [\n (\"ollama\", \"llama3\"),\n (\"groq\", \"llama3-70b-8192\"),\n (\"together\", \"mistralai/Mixtral-8x7B-Instruct-v0.1\"),\n ]\n \n for provider, model in providers:\n try:\n llm = LLMFactory.create(provider, model)\n print(f\" [OK] {llm.name}\")\n except Exception as e:\n print(f\" {provider}: {e}\")\n"}}}} |