hack/tools/scripts/llm-key-hunter/results/_cache/content/abhirupbanerjee_ai-assistant.json

{"repo": "abhirupbanerjee/ai-assistant", "files": {"src/lib/provider-helpers.ts": {"8fe509670a4b0337ef34998bfce43c856fd03e67": {"t": 1785745572.2322443, "n": 2853, "h": "258951e5b1587183", "c": "/**\n * Provider Helpers\n *\n * Centralized helpers for getting API keys and configuration for LLM providers.\n * Tools should use these helpers instead of reading environment variables directly.\n *\n * Benefits:\n * - API keys configured via Admin UI take precedence\n * - Falls back to environment variables if not configured in UI\n * - Single source of truth for provider configuration\n *\n * Usage:\n * ```typescript\n * import { getApiKey, getApiBase, isProviderConfigured } from '@/lib/provider-helpers';\n *\n * // Get API key (checks DB first, then env var)\n * const openaiKey = getApiKey('openai');\n * const geminiKey = getApiKey('gemini');\n * const mistralKey = getApiKey('mistral');\n *\n * // Get API base URL (for Ollama or custom endpoints)\n * const ollamaBase = getApiBase('ollama');\n *\n * // Check if provider is properly configured\n * if (isProviderConfigured('openai')) {\n *   // Provider has API key configured\n * }\n * ```\n */\n\nimport { getProviderApiKey, getProviderApiBase } from './db/compat/llm-providers';\n\n// Re-export with cleaner names\nexport const getApiKey = getProviderApiKey;\nexport const getApiBase = getProviderApiBase;\n\n/**\n * Check if a provider is properly configured (has API key or base URL)\n */\nexport async function isProviderConfigured(providerId: string): Promise<boolean> {\n  if (providerId === 'ollama') {\n    return !!(await getApiBase('ollama'));\n  }\n  return !!(await getApiKey(providerId));\n}\n\n/**\n * Get provider configuration with both key and base URL\n */\nexport async function getProviderConfig(providerId: string): Promise<{\n  apiKey: string | null;\n  apiBase: string | null;\n  isConfigured: boolean;\n}> {\n  const apiKey = await getApiKey(providerId);\n  const apiBase = await getApiBase(providerId);\n\n  return {\n    apiKey,\n    apiBase,\n    isConfigured: providerId === 'ollama' ? !!apiBase : !!apiKey,\n  };\n}\n\n/**\n * Provider IDs\n */\nexport const PROVIDERS = {\n  OPENAI: 'openai',\n  GEMINI: 'gemini',\n  MISTRAL: 'mistral',\n  OLLAMA: 'ollama',\n  ANTHROPIC: 'anthropic',\n  DEEPSEEK: 'deepseek',\n  FIREWORKS: 'fireworks',\n  MOONSHOT: 'moonshot',\n  OLLAMA_CLOUD: 'ollama-cloud',\n} as const;\n\nexport type ProviderId = typeof PROVIDERS[keyof typeof PROVIDERS];\n\n/**\n * Environment variable names for reference\n * (These are checked as fallback if not configured in Admin UI)\n */\nexport const PROVIDER_ENV_VARS = {\n  openai: {\n    apiKey: 'OPENAI_API_KEY',\n  },\n  gemini: {\n    apiKey: 'GEMINI_API_KEY',\n  },\n  mistral: {\n    apiKey: 'MISTRAL_API_KEY',\n  },\n  ollama: {\n    apiBase: 'OLLAMA_API_BASE',\n  },\n  anthropic: {\n    apiKey: 'ANTHROPIC_API_KEY',\n  },\n  deepseek: {\n    apiKey: 'DEEPSEEK_API_KEY',\n  },\n  fireworks: {\n    apiKey: 'FIREWORKS_AI_API_KEY',\n  },\n  moonshot: {\n    apiKey: 'MOONSHOT_API_KEY',\n    apiBase: 'MOONSHOT_API_BASE',\n  },\n  'ollama-cloud': {\n    apiKey: 'OLLAMA_API_KEY',\n  },\n} as const;\n"}, "a9f4f896d2541099be8e168e9095edb75b1f04a6": {"t": 1785754358.878663, "n": 2853, "h": "258951e5b1587183", "c": "/**\n * Provider Helpers\n *\n * Centralized helpers for getting API keys and configuration for LLM providers.\n * Tools should use these helpers instead of reading environment variables directly.\n *\n * Benefits:\n * - API keys configured via Admin UI take precedence\n * - Falls back to environment variables if not configured in UI\n * - Single source of truth for provider configuration\n *\n * Usage:\n * ```typescript\n * import { getApiKey, getApiBase, isProviderConfigured } from '@/lib/provider-helpers';\n *\n * // Get API key (checks DB first, then env var)\n * const openaiKey = getApiKey('openai');\n * const geminiKey = getApiKey('gemini');\n * const mistralKey = getApiKey('mistral');\n *\n * // Get API base URL (for Ollama or custom endpoints)\n * const ollamaBase = getApiBase('ollama');\n *\n * // Check if provider is properly configured\n * if (isProviderConfigured('openai')) {\n *   // Provider has API key configured\n * }\n * ```\n */\n\nimport { getProviderApiKey, getProviderApiBase } from './db/compat/llm-providers';\n\n// Re-export with cleaner names\nexport const getApiKey = getProviderApiKey;\nexport const getApiBase = getProviderApiBase;\n\n/**\n * Check if a provider is properly configured (has API key or base URL)\n */\nexport async function isProviderConfigured(providerId: string): Promise<boolean> {\n  if (providerId === 'ollama') {\n    return !!(await getApiBase('ollama'));\n  }\n  return !!(await getApiKey(providerId));\n}\n\n/**\n * Get provider configuration with both key and base URL\n */\nexport async function getProviderConfig(providerId: string): Promise<{\n  apiKey: string | null;\n  apiBase: string | null;\n  isConfigured: boolean;\n}> {\n  const apiKey = await getApiKey(providerId);\n  const apiBase = await getApiBase(providerId);\n\n  return {\n    apiKey,\n    apiBase,\n    isConfigured: providerId === 'ollama' ? !!apiBase : !!apiKey,\n  };\n}\n\n/**\n * Provider IDs\n */\nexport const PROVIDERS = {\n  OPENAI: 'openai',\n  GEMINI: 'gemini',\n  MISTRAL: 'mistral',\n  OLLAMA: 'ollama',\n  ANTHROPIC: 'anthropic',\n  DEEPSEEK: 'deepseek',\n  FIREWORKS: 'fireworks',\n  MOONSHOT: 'moonshot',\n  OLLAMA_CLOUD: 'ollama-cloud',\n} as const;\n\nexport type ProviderId = typeof PROVIDERS[keyof typeof PROVIDERS];\n\n/**\n * Environment variable names for reference\n * (These are checked as fallback if not configured in Admin UI)\n */\nexport const PROVIDER_ENV_VARS = {\n  openai: {\n    apiKey: 'OPENAI_API_KEY',\n  },\n  gemini: {\n    apiKey: 'GEMINI_API_KEY',\n  },\n  mistral: {\n    apiKey: 'MISTRAL_API_KEY',\n  },\n  ollama: {\n    apiBase: 'OLLAMA_API_BASE',\n  },\n  anthropic: {\n    apiKey: 'ANTHROPIC_API_KEY',\n  },\n  deepseek: {\n    apiKey: 'DEEPSEEK_API_KEY',\n  },\n  fireworks: {\n    apiKey: 'FIREWORKS_AI_API_KEY',\n  },\n  moonshot: {\n    apiKey: 'MOONSHOT_API_KEY',\n    apiBase: 'MOONSHOT_API_BASE',\n  },\n  'ollama-cloud': {\n    apiKey: 'OLLAMA_API_KEY',\n  },\n} as const;\n"}, "8ba02c6c64911ba92e2571661dba2c0c1cac0dd6": {"t": 1785754360.1653857, "n": 2885, "h": "675ded663f63319a", "c": "/**\n * Provider Helpers\n *\n * Centralized helpers for getting API keys and configuration for LLM providers.\n * Tools should use these helpers instead of reading environment variables directly.\n *\n * Benefits:\n * - API keys configured via Admin UI take precedence\n * - Falls back to environment variables if not configured in UI\n * - Single source of truth for provider configuration\n *\n * Usage:\n * ```typescript\n * import { getApiKey, getApiBase, isProviderConfigured } from '@/lib/provider-helpers';\n *\n * // Get API key (checks DB first, then env var)\n * const openaiKey = getApiKey('openai');\n * const geminiKey = getApiKey('gemini');\n * const mistralKey = getApiKey('mistral');\n *\n * // Get API base URL (for Ollama or custom endpoints)\n * const ollamaBase = getApiBase('ollama');\n *\n * // Check if provider is properly configured\n * if (isProviderConfigured('openai')) {\n *   // Provider has API key configured\n * }\n * ```\n */\n\nimport { getProviderApiKey, getProviderApiBase } from './db/compat/llm-providers';\n\n// Re-export with cleaner names\nexport const getApiKey = getProviderApiKey;\nexport const getApiBase = getProviderApiBase;\n\n/**\n * Check if a provider is properly configured (has API key or base URL)\n */\nexport async function isProviderConfigured(providerId: string): Promise<boolean> {\n  if (providerId === 'ollama') {\n    return !!(await getApiBase('ollama'));\n  }\n  return !!(await getApiKey(providerId));\n}\n\n/**\n * Get provider configuration with both key and base URL\n */\nexport async function getProviderConfig(providerId: string): Promise<{\n  apiKey: string | null;\n  apiBase: string | null;\n  isConfigured: boolean;\n}> {\n  const apiKey = await getApiKey(providerId);\n  const apiBase = await getApiBase(providerId);\n\n  return {\n    apiKey,\n    apiBase,\n    isConfigured: providerId === 'ollama' ? !!apiBase : !!apiKey,\n  };\n}\n\n/**\n * Provider IDs\n */\nexport const PROVIDERS = {\n  OPENAI: 'openai',\n  GEMINI: 'gemini',\n  MISTRAL: 'mistral',\n  OLLAMA: 'ollama',\n  ANTHROPIC: 'anthropic',\n  DEEPSEEK: 'deepseek',\n  FIREWORKS: 'fireworks',\n  MOONSHOT: 'moonshot',\n  OLLAMA_CLOUD: 'ollama-cloud',\n} as const;\n\nexport type ProviderId = typeof PROVIDERS[keyof typeof PROVIDERS];\n\n/**\n * Environment variable names for reference\n * (These are checked as fallback if not configured in Admin UI)\n */\nexport const PROVIDER_ENV_VARS = {\n  openai: {\n    apiKey: 'OPENAI_API_KEY',\n    apiBase: 'OPENAI_BASE_URL',\n  },\n  gemini: {\n    apiKey: 'GEMINI_API_KEY',\n  },\n  mistral: {\n    apiKey: 'MISTRAL_API_KEY',\n  },\n  ollama: {\n    apiBase: 'OLLAMA_API_BASE',\n  },\n  anthropic: {\n    apiKey: 'ANTHROPIC_API_KEY',\n  },\n  deepseek: {\n    apiKey: 'DEEPSEEK_API_KEY',\n  },\n  fireworks: {\n    apiKey: 'FIREWORKS_AI_API_KEY',\n  },\n  moonshot: {\n    apiKey: 'MOONSHOT_API_KEY',\n    apiBase: 'MOONSHOT_API_BASE',\n  },\n  'ollama-cloud': {\n    apiKey: 'OLLAMA_API_KEY',\n  },\n} as const;\n"}, "8042badb2057d006a4cfba87238cf7514f2017d4": {"t": 1785754362.007802, "n": 2776, "h": "ed11fb01d5e2c814", "c": "/**\n * Provider Helpers\n *\n * Centralized helpers for getting API keys and configuration for LLM providers.\n * Tools should use these helpers instead of reading environment variables directly.\n *\n * Benefits:\n * - API keys configured via Admin UI take precedence\n * - Falls back to environment variables if not configured in UI\n * - Single source of truth for provider configuration\n *\n * Usage:\n * ```typescript\n * import { getApiKey, getApiBase, isProviderConfigured } from '@/lib/provider-helpers';\n *\n * // Get API key (checks DB first, then env var)\n * const openaiKey = getApiKey('openai');\n * const geminiKey = getApiKey('gemini');\n * const mistralKey = getApiKey('mistral');\n *\n * // Get API base URL (for Ollama or custom endpoints)\n * const ollamaBase = getApiBase('ollama');\n *\n * // Check if provider is properly configured\n * if (isProviderConfigured('openai')) {\n *   // Provider has API key configured\n * }\n * ```\n */\n\nimport { getProviderApiKey, getProviderApiBase } from './db/compat/llm-providers';\n\n// Re-export with cleaner names\nexport const getApiKey = getProviderApiKey;\nexport const getApiBase = getProviderApiBase;\n\n/**\n * Check if a provider is properly configured (has API key or base URL)\n */\nexport async function isProviderConfigured(providerId: string): Promise<boolean> {\n  if (providerId === 'ollama') {\n    return !!(await getApiBase('ollama'));\n  }\n  return !!(await getApiKey(providerId));\n}\n\n/**\n * Get provider configuration with both key and base URL\n */\nexport async function getProviderConfig(providerId: string): Promise<{\n  apiKey: string | null;\n  apiBase: string | null;\n  isConfigured: boolean;\n}> {\n  const apiKey = await getApiKey(providerId);\n  const apiBase = await getApiBase(providerId);\n\n  return {\n    apiKey,\n    apiBase,\n    isConfigured: providerId === 'ollama' ? !!apiBase : !!apiKey,\n  };\n}\n\n/**\n * Provider IDs\n */\nexport const PROVIDERS = {\n  OPENAI: 'openai',\n  GEMINI: 'gemini',\n  MISTRAL: 'mistral',\n  OLLAMA: 'ollama',\n  ANTHROPIC: 'anthropic',\n  DEEPSEEK: 'deepseek',\n  FIREWORKS: 'fireworks',\n  OLLAMA_CLOUD: 'ollama-cloud',\n} as const;\n\nexport type ProviderId = typeof PROVIDERS[keyof typeof PROVIDERS];\n\n/**\n * Environment variable names for reference\n * (These are checked as fallback if not configured in Admin UI)\n */\nexport const PROVIDER_ENV_VARS = {\n  openai: {\n    apiKey: 'OPENAI_API_KEY',\n    apiBase: 'OPENAI_BASE_URL',\n  },\n  gemini: {\n    apiKey: 'GEMINI_API_KEY',\n  },\n  mistral: {\n    apiKey: 'MISTRAL_API_KEY',\n  },\n  ollama: {\n    apiBase: 'OLLAMA_API_BASE',\n  },\n  anthropic: {\n    apiKey: 'ANTHROPIC_API_KEY',\n  },\n  deepseek: {\n    apiKey: 'DEEPSEEK_API_KEY',\n  },\n  fireworks: {\n    apiKey: 'FIREWORKS_AI_API_KEY',\n  },\n  'ollama-cloud': {\n    apiKey: 'OLLAMA_API_KEY',\n  },\n} as const;\n"}, "4fef8addc8f763d280833f96b6ddf7a7614a992b": {"t": 1785754363.2687063, "n": 2689, "h": "2962178eab393621", "c": "/**\n * Provider Helpers\n *\n * Centralized helpers for getting API keys and configuration for LLM providers.\n * Tools should use these helpers instead of reading environment variables directly.\n *\n * Benefits:\n * - API keys configured via Admin UI take precedence\n * - Falls back to environment variables if not configured in UI\n * - Single source of truth for provider configuration\n *\n * Usage:\n * ```typescript\n * import { getApiKey, getApiBase, isProviderConfigured } from '@/lib/provider-helpers';\n *\n * // Get API key (checks DB first, then env var)\n * const openaiKey = getApiKey('openai');\n * const geminiKey = getApiKey('gemini');\n * const mistralKey = getApiKey('mistral');\n *\n * // Get API base URL (for Ollama or custom endpoints)\n * const ollamaBase = getApiBase('ollama');\n *\n * // Check if provider is properly configured\n * if (isProviderConfigured('openai')) {\n *   // Provider has API key configured\n * }\n * ```\n */\n\nimport { getProviderApiKey, getProviderApiBase } from './db/compat/llm-providers';\n\n// Re-export with cleaner names\nexport const getApiKey = getProviderApiKey;\nexport const getApiBase = getProviderApiBase;\n\n/**\n * Check if a provider is properly configured (has API key or base URL)\n */\nexport async function isProviderConfigured(providerId: string): Promise<boolean> {\n  if (providerId === 'ollama') {\n    return !!(await getApiBase('ollama'));\n  }\n  return !!(await getApiKey(providerId));\n}\n\n/**\n * Get provider configuration with both key and base URL\n */\nexport async function getProviderConfig(providerId: string): Promise<{\n  apiKey: string | null;\n  apiBase: string | null;\n  isConfigured: boolean;\n}> {\n  const apiKey = await getApiKey(providerId);\n  const apiBase = await getApiBase(providerId);\n\n  return {\n    apiKey,\n    apiBase,\n    isConfigured: providerId === 'ollama' ? !!apiBase : !!apiKey,\n  };\n}\n\n/**\n * Provider IDs\n */\nexport const PROVIDERS = {\n  OPENAI: 'openai',\n  GEMINI: 'gemini',\n  MISTRAL: 'mistral',\n  OLLAMA: 'ollama',\n  ANTHROPIC: 'anthropic',\n  DEEPSEEK: 'deepseek',\n  FIREWORKS: 'fireworks',\n} as const;\n\nexport type ProviderId = typeof PROVIDERS[keyof typeof PROVIDERS];\n\n/**\n * Environment variable names for reference\n * (These are checked as fallback if not configured in Admin UI)\n */\nexport const PROVIDER_ENV_VARS = {\n  openai: {\n    apiKey: 'OPENAI_API_KEY',\n    apiBase: 'OPENAI_BASE_URL',\n  },\n  gemini: {\n    apiKey: 'GEMINI_API_KEY',\n  },\n  mistral: {\n    apiKey: 'MISTRAL_API_KEY',\n  },\n  ollama: {\n    apiBase: 'OLLAMA_API_BASE',\n  },\n  anthropic: {\n    apiKey: 'ANTHROPIC_API_KEY',\n  },\n  deepseek: {\n    apiKey: 'DEEPSEEK_API_KEY',\n  },\n  fireworks: {\n    apiKey: 'FIREWORKS_AI_API_KEY',\n  },\n} as const;\n"}, "4f9c52ccaed027e77ce0234767cb6197b126f706": {"t": 1785754364.5686789, "n": 2607, "h": "3e0730059982b2b7", "c": "/**\n * Provider Helpers\n *\n * Centralized helpers for getting API keys and configuration for LLM providers.\n * Tools should use these helpers instead of reading environment variables directly.\n *\n * Benefits:\n * - API keys configured via Admin UI take precedence\n * - Falls back to environment variables if not configured in UI\n * - Single source of truth for provider configuration\n *\n * Usage:\n * ```typescript\n * import { getApiKey, getApiBase, isProviderConfigured } from '@/lib/provider-helpers';\n *\n * // Get API key (checks DB first, then env var)\n * const openaiKey = getApiKey('openai');\n * const geminiKey = getApiKey('gemini');\n * const mistralKey = getApiKey('mistral');\n *\n * // Get API base URL (for Ollama or custom endpoints)\n * const ollamaBase = getApiBase('ollama');\n *\n * // Check if provider is properly configured\n * if (isProviderConfigured('openai')) {\n *   // Provider has API key configured\n * }\n * ```\n */\n\nimport { getProviderApiKey, getProviderApiBase } from './db/compat/llm-providers';\n\n// Re-export with cleaner names\nexport const getApiKey = getProviderApiKey;\nexport const getApiBase = getProviderApiBase;\n\n/**\n * Check if a provider is properly configured (has API key or base URL)\n */\nexport async function isProviderConfigured(providerId: string): Promise<boolean> {\n  if (providerId === 'ollama') {\n    return !!(await getApiBase('ollama'));\n  }\n  return !!(await getApiKey(providerId));\n}\n\n/**\n * Get provider configuration with both key and base URL\n */\nexport async function getProviderConfig(providerId: string): Promise<{\n  apiKey: string | null;\n  apiBase: string | null;\n  isConfigured: boolean;\n}> {\n  const apiKey = await getApiKey(providerId);\n  const apiBase = await getApiBase(providerId);\n\n  return {\n    apiKey,\n    apiBase,\n    isConfigured: providerId === 'ollama' ? !!apiBase : !!apiKey,\n  };\n}\n\n/**\n * Provider IDs\n */\nexport const PROVIDERS = {\n  OPENAI: 'openai',\n  GEMINI: 'gemini',\n  MISTRAL: 'mistral',\n  OLLAMA: 'ollama',\n  ANTHROPIC: 'anthropic',\n  DEEPSEEK: 'deepseek',\n} as const;\n\nexport type ProviderId = typeof PROVIDERS[keyof typeof PROVIDERS];\n\n/**\n * Environment variable names for reference\n * (These are checked as fallback if not configured in Admin UI)\n */\nexport const PROVIDER_ENV_VARS = {\n  openai: {\n    apiKey: 'OPENAI_API_KEY',\n    apiBase: 'OPENAI_BASE_URL',\n  },\n  gemini: {\n    apiKey: 'GEMINI_API_KEY',\n  },\n  mistral: {\n    apiKey: 'MISTRAL_API_KEY',\n  },\n  ollama: {\n    apiBase: 'OLLAMA_API_BASE',\n  },\n  anthropic: {\n    apiKey: 'ANTHROPIC_API_KEY',\n  },\n  deepseek: {\n    apiKey: 'DEEPSEEK_API_KEY',\n  },\n} as const;\n"}, "efe833353126e628c0ca7584a88ad5165018fb7f": {"t": 1785754366.0885248, "n": 2555, "h": "18936189bbd7e8f5", "c": "/**\n * Provider Helpers\n *\n * Centralized helpers for getting API keys and configuration for LLM providers.\n * Tools should use these helpers instead of reading environment variables directly.\n *\n * Benefits:\n * - API keys configured via Admin UI take precedence\n * - Falls back to environment variables if not configured in UI\n * - Single source of truth for provider configuration\n *\n * Usage:\n * ```typescript\n * import { getApiKey, getApiBase, isProviderConfigured } from '@/lib/provider-helpers';\n *\n * // Get API key (checks DB first, then env var)\n * const openaiKey = getApiKey('openai');\n * const geminiKey = getApiKey('gemini');\n * const mistralKey = getApiKey('mistral');\n *\n * // Get API base URL (for Ollama or custom endpoints)\n * const ollamaBase = getApiBase('ollama');\n *\n * // Check if provider is properly configured\n * if (isProviderConfigured('openai')) {\n *   // Provider has API key configured\n * }\n * ```\n */\n\nimport { getProviderApiKey, getProviderApiBase, getProvider } from './db/llm-providers';\n\n// Re-export with cleaner names\nexport const getApiKey = getProviderApiKey;\nexport const getApiBase = getProviderApiBase;\n\n/**\n * Check if a provider is properly configured (has API key or base URL)\n */\nexport function isProviderConfigured(providerId: string): boolean {\n  if (providerId === 'ollama') {\n    return !!getApiBase('ollama');\n  }\n  return !!getApiKey(providerId);\n}\n\n/**\n * Get provider configuration with both key and base URL\n */\nexport function getProviderConfig(providerId: string): {\n  apiKey: string | null;\n  apiBase: string | null;\n  isConfigured: boolean;\n} {\n  const apiKey = getApiKey(providerId);\n  const apiBase = getApiBase(providerId);\n\n  return {\n    apiKey,\n    apiBase,\n    isConfigured: providerId === 'ollama' ? !!apiBase : !!apiKey,\n  };\n}\n\n/**\n * Provider IDs\n */\nexport const PROVIDERS = {\n  OPENAI: 'openai',\n  GEMINI: 'gemini',\n  MISTRAL: 'mistral',\n  OLLAMA: 'ollama',\n  ANTHROPIC: 'anthropic',\n  DEEPSEEK: 'deepseek',\n} as const;\n\nexport type ProviderId = typeof PROVIDERS[keyof typeof PROVIDERS];\n\n/**\n * Environment variable names for reference\n * (These are checked as fallback if not configured in Admin UI)\n */\nexport const PROVIDER_ENV_VARS = {\n  openai: {\n    apiKey: 'OPENAI_API_KEY',\n    apiBase: 'OPENAI_BASE_URL',\n  },\n  gemini: {\n    apiKey: 'GEMINI_API_KEY',\n  },\n  mistral: {\n    apiKey: 'MISTRAL_API_KEY',\n  },\n  ollama: {\n    apiBase: 'OLLAMA_API_BASE',\n  },\n  anthropic: {\n    apiKey: 'ANTHROPIC_API_KEY',\n  },\n  deepseek: {\n    apiKey: 'DEEPSEEK_API_KEY',\n  },\n} as const;\n"}, "e3883a2ebcb661f615cdd1461a53ea984fd44201": {"t": 1785754367.9678564, "n": 2401, "h": "2efaf51f23d8a559", "c": "/**\n * Provider Helpers\n *\n * Centralized helpers for getting API keys and configuration for LLM providers.\n * Tools should use these helpers instead of reading environment variables directly.\n *\n * Benefits:\n * - API keys configured via Admin UI take precedence\n * - Falls back to environment variables if not configured in UI\n * - Single source of truth for provider configuration\n *\n * Usage:\n * ```typescript\n * import { getApiKey, getApiBase, isProviderConfigured } from '@/lib/provider-helpers';\n *\n * // Get API key (checks DB first, then env var)\n * const openaiKey = getApiKey('openai');\n * const geminiKey = getApiKey('gemini');\n * const mistralKey = getApiKey('mistral');\n *\n * // Get API base URL (for Ollama or custom endpoints)\n * const ollamaBase = getApiBase('ollama');\n *\n * // Check if provider is properly configured\n * if (isProviderConfigured('openai')) {\n *   // Provider has API key configured\n * }\n * ```\n */\n\nimport { getProviderApiKey, getProviderApiBase, getProvider } from './db/llm-providers';\n\n// Re-export with cleaner names\nexport const getApiKey = getProviderApiKey;\nexport const getApiBase = getProviderApiBase;\n\n/**\n * Check if a provider is properly configured (has API key or base URL)\n */\nexport function isProviderConfigured(providerId: string): boolean {\n  if (providerId === 'ollama') {\n    return !!getApiBase('ollama');\n  }\n  return !!getApiKey(providerId);\n}\n\n/**\n * Get provider configuration with both key and base URL\n */\nexport function getProviderConfig(providerId: string): {\n  apiKey: string | null;\n  apiBase: string | null;\n  isConfigured: boolean;\n} {\n  const apiKey = getApiKey(providerId);\n  const apiBase = getApiBase(providerId);\n\n  return {\n    apiKey,\n    apiBase,\n    isConfigured: providerId === 'ollama' ? !!apiBase : !!apiKey,\n  };\n}\n\n/**\n * Provider IDs\n */\nexport const PROVIDERS = {\n  OPENAI: 'openai',\n  GEMINI: 'gemini',\n  MISTRAL: 'mistral',\n  OLLAMA: 'ollama',\n} as const;\n\nexport type ProviderId = typeof PROVIDERS[keyof typeof PROVIDERS];\n\n/**\n * Environment variable names for reference\n * (These are checked as fallback if not configured in Admin UI)\n */\nexport const PROVIDER_ENV_VARS = {\n  openai: {\n    apiKey: 'OPENAI_API_KEY',\n    apiBase: 'OPENAI_BASE_URL',\n  },\n  gemini: {\n    apiKey: 'GEMINI_API_KEY',\n  },\n  mistral: {\n    apiKey: 'MISTRAL_API_KEY',\n  },\n  ollama: {\n    apiBase: 'OLLAMA_API_BASE',\n  },\n} as const;\n"}}, "src/app/api/admin/routes/health/route.ts": {"8fe509670a4b0337ef34998bfce43c856fd03e67": {"t": 1785745609.9508958, "n": 5721, "h": "18475eecdb7ac0be", "c": "/**\n * Routes Health Check API\n *\n * Returns health status for direct LLM routes:\n * - Route 2: OpenAI + Claude + Gemini + Mistral + DeepSeek + Moonshot + Fireworks\n */\n\nimport { NextResponse } from 'next/server';\nimport { getCurrentUser, requireElevated } from '@/lib/auth';\nimport { getApiBase, getApiKey } from '@/lib/provider-helpers';\nimport { getMoonshotBaseUrl } from '@/lib/moonshot-config';\n\ninterface RouteHealth {\n  route2: {\n    fireworks: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n    deepseek: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n    claude: { configured: boolean };\n    moonshot: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n    mistral: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n  };\n}\n\n/**\n * GET - Check health of routes\n */\nexport async function GET() {\n  try {\n    const user = await requireElevated().catch(() => null);\n    if (!user) {\n      return NextResponse.json({ error: 'Forbidden' }, { status: 403 });\n    }\n\n    const [fireworksHealth, deepseekHealth, claudeConfigured, moonshotHealth, mistralHealth] = await Promise.all([\n      checkFireworksHealth(),\n      checkDeepSeekHealth(),\n      checkClaudeConfigured(),\n      checkMoonshotHealth(),\n      checkMistralHealth(),\n    ]);\n\n    const health: RouteHealth = {\n      route2: {\n        fireworks: fireworksHealth,\n        deepseek: deepseekHealth,\n        claude: { configured: claudeConfigured },\n        moonshot: moonshotHealth,\n        mistral: mistralHealth,\n      },\n    };\n\n    return NextResponse.json(health);\n  } catch (error) {\n    console.error('[Routes Health API] Error:', error);\n    return NextResponse.json(\n      { error: 'Failed to check route health' },\n      { status: 500 }\n    );\n  }\n}\n\nasync function checkFireworksHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('fireworks');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'FIREWORKS_AI_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const res = await fetch('https://api.fireworks.ai/inference/v1/models', {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkDeepSeekHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('deepseek');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'DEEPSEEK_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const baseUrl = ((await getApiBase('deepseek')) || 'https://api.deepseek.com/v1').replace(/\\/+$/, '');\n    const res = await fetch(`${baseUrl}/models`, {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkClaudeConfigured(): Promise<boolean> {\n  const apiKey = await getApiKey('anthropic');\n  return !!apiKey;\n}\n\nasync function checkMoonshotHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('moonshot');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'MOONSHOT_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const baseUrl = await getMoonshotBaseUrl();\n    const res = await fetch(`${baseUrl}/models`, {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkMistralHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('mistral');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'MISTRAL_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const res = await fetch('https://api.mistral.ai/v1/models', {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n"}, "a9f4f896d2541099be8e168e9095edb75b1f04a6": {"t": 1785754358.7733674, "n": 5721, "h": "18475eecdb7ac0be", "c": "/**\n * Routes Health Check API\n *\n * Returns health status for direct LLM routes:\n * - Route 2: OpenAI + Claude + Gemini + Mistral + DeepSeek + Moonshot + Fireworks\n */\n\nimport { NextResponse } from 'next/server';\nimport { getCurrentUser, requireElevated } from '@/lib/auth';\nimport { getApiBase, getApiKey } from '@/lib/provider-helpers';\nimport { getMoonshotBaseUrl } from '@/lib/moonshot-config';\n\ninterface RouteHealth {\n  route2: {\n    fireworks: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n    deepseek: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n    claude: { configured: boolean };\n    moonshot: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n    mistral: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n  };\n}\n\n/**\n * GET - Check health of routes\n */\nexport async function GET() {\n  try {\n    const user = await requireElevated().catch(() => null);\n    if (!user) {\n      return NextResponse.json({ error: 'Forbidden' }, { status: 403 });\n    }\n\n    const [fireworksHealth, deepseekHealth, claudeConfigured, moonshotHealth, mistralHealth] = await Promise.all([\n      checkFireworksHealth(),\n      checkDeepSeekHealth(),\n      checkClaudeConfigured(),\n      checkMoonshotHealth(),\n      checkMistralHealth(),\n    ]);\n\n    const health: RouteHealth = {\n      route2: {\n        fireworks: fireworksHealth,\n        deepseek: deepseekHealth,\n        claude: { configured: claudeConfigured },\n        moonshot: moonshotHealth,\n        mistral: mistralHealth,\n      },\n    };\n\n    return NextResponse.json(health);\n  } catch (error) {\n    console.error('[Routes Health API] Error:', error);\n    return NextResponse.json(\n      { error: 'Failed to check route health' },\n      { status: 500 }\n    );\n  }\n}\n\nasync function checkFireworksHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('fireworks');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'FIREWORKS_AI_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const res = await fetch('https://api.fireworks.ai/inference/v1/models', {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkDeepSeekHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('deepseek');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'DEEPSEEK_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const baseUrl = ((await getApiBase('deepseek')) || 'https://api.deepseek.com/v1').replace(/\\/+$/, '');\n    const res = await fetch(`${baseUrl}/models`, {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkClaudeConfigured(): Promise<boolean> {\n  const apiKey = await getApiKey('anthropic');\n  return !!apiKey;\n}\n\nasync function checkMoonshotHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('moonshot');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'MOONSHOT_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const baseUrl = await getMoonshotBaseUrl();\n    const res = await fetch(`${baseUrl}/models`, {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkMistralHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('mistral');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'MISTRAL_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const res = await fetch('https://api.mistral.ai/v1/models', {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n"}, "406fbbe60567dbc485ff4c66c7f9b8bc5328ac2c": {"t": 1785754359.9243844, "n": 6892, "h": "7a84195d0d372b3f", "c": "/**\n * Routes Health Check API\n *\n * Returns health status for both LLM routes:\n * - Route 1: LiteLLM proxy health via /health/liveliness endpoint (no auth required)\n * - Route 2: Fireworks + DeepSeek + Moonshot + Mistral reachability + Anthropic API key configured\n */\n\nimport { NextResponse } from 'next/server';\nimport { getCurrentUser, requireElevated } from '@/lib/auth';\nimport { getApiBase, getApiKey } from '@/lib/provider-helpers';\nimport { getMoonshotBaseUrl } from '@/lib/moonshot-config';\n\ninterface RouteHealth {\n  route1: { healthy: boolean; latencyMs: number | null; error?: string };\n  route2: {\n    fireworks: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n    deepseek: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n    claude: { configured: boolean };\n    moonshot: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n    mistral: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n  };\n}\n\n/**\n * GET - Check health of both routes\n */\nexport async function GET() {\n  try {\n    const user = await requireElevated().catch(() => null);\n    if (!user) {\n      return NextResponse.json({ error: 'Forbidden' }, { status: 403 });\n    }\n\n    // Check both routes in parallel\n    const [route1Health, fireworksHealth, deepseekHealth, claudeConfigured, moonshotHealth, mistralHealth] = await Promise.all([\n      checkRoute1Health(),\n      checkFireworksHealth(),\n      checkDeepSeekHealth(),\n      checkClaudeConfigured(),\n      checkMoonshotHealth(),\n      checkMistralHealth(),\n    ]);\n\n    const health: RouteHealth = {\n      route1: route1Health,\n      route2: {\n        fireworks: fireworksHealth,\n        deepseek: deepseekHealth,\n        claude: { configured: claudeConfigured },\n        moonshot: moonshotHealth,\n        mistral: mistralHealth,\n      },\n    };\n\n    return NextResponse.json(health);\n  } catch (error) {\n    console.error('[Routes Health API] Error:', error);\n    return NextResponse.json(\n      { error: 'Failed to check route health' },\n      { status: 500 }\n    );\n  }\n}\n\nasync function checkRoute1Health(): Promise<{ healthy: boolean; latencyMs: number | null; error?: string }> {\n  const baseUrl = process.env.OPENAI_BASE_URL;\n  if (!baseUrl) {\n    return { healthy: false, latencyMs: null, error: 'OPENAI_BASE_URL not configured' };\n  }\n\n  // Use /health/liveliness (no auth required) instead of /health (requires LITELLM_MASTER_KEY)\n  const healthUrl = baseUrl.replace(/\\/v1\\/?$/, '') + '/health/liveliness';\n  const start = Date.now();\n\n  try {\n    const res = await fetch(healthUrl, { signal: AbortSignal.timeout(5000) });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs };\n    }\n    return { healthy: false, latencyMs, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkFireworksHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('fireworks');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'FIREWORKS_AI_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const res = await fetch('https://api.fireworks.ai/inference/v1/models', {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkDeepSeekHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('deepseek');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'DEEPSEEK_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const baseUrl = ((await getApiBase('deepseek')) || 'https://api.deepseek.com/v1').replace(/\\/+$/, '');\n    const res = await fetch(`${baseUrl}/models`, {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkClaudeConfigured(): Promise<boolean> {\n  const apiKey = await getApiKey('anthropic');\n  return !!apiKey;\n}\n\nasync function checkMoonshotHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('moonshot');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'MOONSHOT_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const baseUrl = await getMoonshotBaseUrl();\n    const res = await fetch(`${baseUrl}/models`, {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkMistralHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('mistral');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'MISTRAL_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const res = await fetch('https://api.mistral.ai/v1/models', {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n"}, "1f8f55e7501f4337ab8bc68b1edb2c1ef028b189": {"t": 1785754361.870938, "n": 5815, "h": "8d3648107383c9f2", "c": "/**\n * Routes Health Check API\n *\n * Returns health status for both LLM routes:\n * - Route 1: LiteLLM proxy health via /health/liveliness endpoint (no auth required)\n * - Route 2: Fireworks + DeepSeek + Moonshot reachability + Anthropic API key configured\n */\n\nimport { NextResponse } from 'next/server';\nimport { getCurrentUser, requireElevated } from '@/lib/auth';\nimport { getApiBase, getApiKey } from '@/lib/provider-helpers';\nimport { getMoonshotBaseUrl } from '@/lib/moonshot-config';\n\ninterface RouteHealth {\n  route1: { healthy: boolean; latencyMs: number | null; error?: string };\n  route2: {\n    fireworks: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n    deepseek: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n    claude: { configured: boolean };\n    moonshot: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n  };\n}\n\n/**\n * GET - Check health of both routes\n */\nexport async function GET() {\n  try {\n    const user = await requireElevated().catch(() => null);\n    if (!user) {\n      return NextResponse.json({ error: 'Forbidden' }, { status: 403 });\n    }\n\n    // Check both routes in parallel\n    const [route1Health, fireworksHealth, deepseekHealth, claudeConfigured, moonshotHealth] = await Promise.all([\n      checkRoute1Health(),\n      checkFireworksHealth(),\n      checkDeepSeekHealth(),\n      checkClaudeConfigured(),\n      checkMoonshotHealth(),\n    ]);\n\n    const health: RouteHealth = {\n      route1: route1Health,\n      route2: {\n        fireworks: fireworksHealth,\n        deepseek: deepseekHealth,\n        claude: { configured: claudeConfigured },\n        moonshot: moonshotHealth,\n      },\n    };\n\n    return NextResponse.json(health);\n  } catch (error) {\n    console.error('[Routes Health API] Error:', error);\n    return NextResponse.json(\n      { error: 'Failed to check route health' },\n      { status: 500 }\n    );\n  }\n}\n\nasync function checkRoute1Health(): Promise<{ healthy: boolean; latencyMs: number | null; error?: string }> {\n  const baseUrl = process.env.OPENAI_BASE_URL;\n  if (!baseUrl) {\n    return { healthy: false, latencyMs: null, error: 'OPENAI_BASE_URL not configured' };\n  }\n\n  // Use /health/liveliness (no auth required) instead of /health (requires LITELLM_MASTER_KEY)\n  const healthUrl = baseUrl.replace(/\\/v1\\/?$/, '') + '/health/liveliness';\n  const start = Date.now();\n\n  try {\n    const res = await fetch(healthUrl, { signal: AbortSignal.timeout(5000) });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs };\n    }\n    return { healthy: false, latencyMs, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkFireworksHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('fireworks');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'FIREWORKS_AI_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const res = await fetch('https://api.fireworks.ai/inference/v1/models', {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkDeepSeekHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('deepseek');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'DEEPSEEK_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const baseUrl = ((await getApiBase('deepseek')) || 'https://api.deepseek.com/v1').replace(/\\/+$/, '');\n    const res = await fetch(`${baseUrl}/models`, {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkClaudeConfigured(): Promise<boolean> {\n  const apiKey = await getApiKey('anthropic');\n  return !!apiKey;\n}\n\nasync function checkMoonshotHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('moonshot');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'MOONSHOT_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const baseUrl = await getMoonshotBaseUrl();\n    const res = await fetch(`${baseUrl}/models`, {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n"}, "3f3fee2110b5e8b6f0d959988d2ec5b70b4bb397": {"t": 1785754363.1592472, "n": 5782, "h": "4df80435cfb22af4", "c": "/**\n * Routes Health Check API\n *\n * Returns health status for both LLM routes:\n * - Route 1: LiteLLM proxy health via /health/liveliness endpoint (no auth required)\n * - Route 2: Fireworks + DeepSeek + Moonshot reachability + Anthropic API key configured\n */\n\nimport { NextResponse } from 'next/server';\nimport { getCurrentUser } from '@/lib/auth';\nimport { getApiBase, getApiKey } from '@/lib/provider-helpers';\nimport { getMoonshotBaseUrl } from '@/lib/moonshot-config';\n\ninterface RouteHealth {\n  route1: { healthy: boolean; latencyMs: number | null; error?: string };\n  route2: {\n    fireworks: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n    deepseek: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n    claude: { configured: boolean };\n    moonshot: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n  };\n}\n\n/**\n * GET - Check health of both routes\n */\nexport async function GET() {\n  try {\n    const user = await getCurrentUser();\n    if (!user) {\n      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });\n    }\n\n    // Check both routes in parallel\n    const [route1Health, fireworksHealth, deepseekHealth, claudeConfigured, moonshotHealth] = await Promise.all([\n      checkRoute1Health(),\n      checkFireworksHealth(),\n      checkDeepSeekHealth(),\n      checkClaudeConfigured(),\n      checkMoonshotHealth(),\n    ]);\n\n    const health: RouteHealth = {\n      route1: route1Health,\n      route2: {\n        fireworks: fireworksHealth,\n        deepseek: deepseekHealth,\n        claude: { configured: claudeConfigured },\n        moonshot: moonshotHealth,\n      },\n    };\n\n    return NextResponse.json(health);\n  } catch (error) {\n    console.error('[Routes Health API] Error:', error);\n    return NextResponse.json(\n      { error: 'Failed to check route health' },\n      { status: 500 }\n    );\n  }\n}\n\nasync function checkRoute1Health(): Promise<{ healthy: boolean; latencyMs: number | null; error?: string }> {\n  const baseUrl = process.env.OPENAI_BASE_URL;\n  if (!baseUrl) {\n    return { healthy: false, latencyMs: null, error: 'OPENAI_BASE_URL not configured' };\n  }\n\n  // Use /health/liveliness (no auth required) instead of /health (requires LITELLM_MASTER_KEY)\n  const healthUrl = baseUrl.replace(/\\/v1\\/?$/, '') + '/health/liveliness';\n  const start = Date.now();\n\n  try {\n    const res = await fetch(healthUrl, { signal: AbortSignal.timeout(5000) });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs };\n    }\n    return { healthy: false, latencyMs, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkFireworksHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('fireworks');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'FIREWORKS_AI_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const res = await fetch('https://api.fireworks.ai/inference/v1/models', {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkDeepSeekHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('deepseek');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'DEEPSEEK_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const baseUrl = ((await getApiBase('deepseek')) || 'https://api.deepseek.com/v1').replace(/\\/+$/, '');\n    const res = await fetch(`${baseUrl}/models`, {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkClaudeConfigured(): Promise<boolean> {\n  const apiKey = await getApiKey('anthropic');\n  return !!apiKey;\n}\n\nasync function checkMoonshotHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('moonshot');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'MOONSHOT_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const baseUrl = await getMoonshotBaseUrl();\n    const res = await fetch(`${baseUrl}/models`, {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n"}, "8ba02c6c64911ba92e2571661dba2c0c1cac0dd6": {"t": 1785754364.4826727, "n": 4615, "h": "80d03c9a59a62ac4", "c": "/**\n * Routes Health Check API\n *\n * Returns health status for both LLM routes:\n * - Route 1: LiteLLM proxy health via /health/liveliness endpoint (no auth required)\n * - Route 2: Fireworks AI reachability + Anthropic API key + Moonshot API key configured\n */\n\nimport { NextResponse } from 'next/server';\nimport { getCurrentUser } from '@/lib/auth';\nimport { getApiKey } from '@/lib/provider-helpers';\nimport { getMoonshotBaseUrl } from '@/lib/agent-swarm/moonshot-config';\n\ninterface RouteHealth {\n  route1: { healthy: boolean; latencyMs: number | null; error?: string };\n  route2: {\n    fireworks: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n    claude: { configured: boolean };\n    moonshot: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n  };\n}\n\n/**\n * GET - Check health of both routes\n */\nexport async function GET() {\n  try {\n    const user = await getCurrentUser();\n    if (!user) {\n      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });\n    }\n\n    // Check both routes in parallel\n    const [route1Health, fireworksHealth, claudeConfigured, moonshotHealth] = await Promise.all([\n      checkRoute1Health(),\n      checkFireworksHealth(),\n      checkClaudeConfigured(),\n      checkMoonshotHealth(),\n    ]);\n\n    const health: RouteHealth = {\n      route1: route1Health,\n      route2: {\n        fireworks: fireworksHealth,\n        claude: { configured: claudeConfigured },\n        moonshot: moonshotHealth,\n      },\n    };\n\n    return NextResponse.json(health);\n  } catch (error) {\n    console.error('[Routes Health API] Error:', error);\n    return NextResponse.json(\n      { error: 'Failed to check route health' },\n      { status: 500 }\n    );\n  }\n}\n\nasync function checkRoute1Health(): Promise<{ healthy: boolean; latencyMs: number | null; error?: string }> {\n  const baseUrl = process.env.OPENAI_BASE_URL;\n  if (!baseUrl) {\n    return { healthy: false, latencyMs: null, error: 'OPENAI_BASE_URL not configured' };\n  }\n\n  // Use /health/liveliness (no auth required) instead of /health (requires LITELLM_MASTER_KEY)\n  const healthUrl = baseUrl.replace(/\\/v1\\/?$/, '') + '/health/liveliness';\n  const start = Date.now();\n\n  try {\n    const res = await fetch(healthUrl, { signal: AbortSignal.timeout(5000) });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs };\n    }\n    return { healthy: false, latencyMs, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkFireworksHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('fireworks');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'FIREWORKS_AI_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const res = await fetch('https://api.fireworks.ai/inference/v1/models', {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkClaudeConfigured(): Promise<boolean> {\n  const apiKey = await getApiKey('anthropic');\n  return !!apiKey;\n}\n\nasync function checkMoonshotHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('moonshot');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'MOONSHOT_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const baseUrl = await getMoonshotBaseUrl();\n    const res = await fetch(`${baseUrl}/models`, {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n"}, "1efa77f79ea5b9aba53577768660d83736049547": {"t": 1785754365.9526181, "n": 4511, "h": "c9619bc7d33ad180", "c": "/**\n * Routes Health Check API\n *\n * Returns health status for both LLM routes:\n * - Route 1: LiteLLM proxy health via /health/liveliness endpoint (no auth required)\n * - Route 2: Fireworks AI reachability + Anthropic API key + Moonshot API key configured\n */\n\nimport { NextResponse } from 'next/server';\nimport { getCurrentUser } from '@/lib/auth';\nimport { getApiKey } from '@/lib/provider-helpers';\n\ninterface RouteHealth {\n  route1: { healthy: boolean; latencyMs: number | null; error?: string };\n  route2: {\n    fireworks: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n    claude: { configured: boolean };\n    moonshot: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n  };\n}\n\n/**\n * GET - Check health of both routes\n */\nexport async function GET() {\n  try {\n    const user = await getCurrentUser();\n    if (!user) {\n      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });\n    }\n\n    // Check both routes in parallel\n    const [route1Health, fireworksHealth, claudeConfigured, moonshotHealth] = await Promise.all([\n      checkRoute1Health(),\n      checkFireworksHealth(),\n      checkClaudeConfigured(),\n      checkMoonshotHealth(),\n    ]);\n\n    const health: RouteHealth = {\n      route1: route1Health,\n      route2: {\n        fireworks: fireworksHealth,\n        claude: { configured: claudeConfigured },\n        moonshot: moonshotHealth,\n      },\n    };\n\n    return NextResponse.json(health);\n  } catch (error) {\n    console.error('[Routes Health API] Error:', error);\n    return NextResponse.json(\n      { error: 'Failed to check route health' },\n      { status: 500 }\n    );\n  }\n}\n\nasync function checkRoute1Health(): Promise<{ healthy: boolean; latencyMs: number | null; error?: string }> {\n  const baseUrl = process.env.OPENAI_BASE_URL;\n  if (!baseUrl) {\n    return { healthy: false, latencyMs: null, error: 'OPENAI_BASE_URL not configured' };\n  }\n\n  // Use /health/liveliness (no auth required) instead of /health (requires LITELLM_MASTER_KEY)\n  const healthUrl = baseUrl.replace(/\\/v1\\/?$/, '') + '/health/liveliness';\n  const start = Date.now();\n\n  try {\n    const res = await fetch(healthUrl, { signal: AbortSignal.timeout(5000) });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs };\n    }\n    return { healthy: false, latencyMs, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkFireworksHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('fireworks');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'FIREWORKS_AI_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const res = await fetch('https://api.fireworks.ai/inference/v1/models', {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkClaudeConfigured(): Promise<boolean> {\n  const apiKey = await getApiKey('anthropic');\n  return !!apiKey;\n}\n\nasync function checkMoonshotHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('moonshot');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'MOONSHOT_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const res = await fetch('https://api.moonshot.cn/v1/models', {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n"}, "235fece261c2990d381933b2fd0cc2d43656028b": {"t": 1785754367.7710493, "n": 3416, "h": "a35c1dbddbbb8fd7", "c": "/**\n * Routes Health Check API\n *\n * Returns health status for both LLM routes:\n * - Route 1: LiteLLM proxy health via /health/liveliness endpoint (no auth required)\n * - Route 2: Fireworks AI reachability + Anthropic API key configured\n */\n\nimport { NextResponse } from 'next/server';\nimport { getCurrentUser } from '@/lib/auth';\nimport { getApiKey } from '@/lib/provider-helpers';\n\ninterface RouteHealth {\n  route1: { healthy: boolean; latencyMs: number | null; error?: string };\n  route2: {\n    fireworks: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n    claude: { configured: boolean };\n  };\n}\n\n/**\n * GET - Check health of both routes\n */\nexport async function GET() {\n  try {\n    const user = await getCurrentUser();\n    if (!user) {\n      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });\n    }\n\n    // Check both routes in parallel\n    const [route1Health, fireworksHealth, claudeConfigured] = await Promise.all([\n      checkRoute1Health(),\n      checkFireworksHealth(),\n      checkClaudeConfigured(),\n    ]);\n\n    const health: RouteHealth = {\n      route1: route1Health,\n      route2: {\n        fireworks: fireworksHealth,\n        claude: { configured: claudeConfigured },\n      },\n    };\n\n    return NextResponse.json(health);\n  } catch (error) {\n    console.error('[Routes Health API] Error:', error);\n    return NextResponse.json(\n      { error: 'Failed to check route health' },\n      { status: 500 }\n    );\n  }\n}\n\nasync function checkRoute1Health(): Promise<{ healthy: boolean; latencyMs: number | null; error?: string }> {\n  const baseUrl = process.env.OPENAI_BASE_URL;\n  if (!baseUrl) {\n    return { healthy: false, latencyMs: null, error: 'OPENAI_BASE_URL not configured' };\n  }\n\n  // Use /health/liveliness (no auth required) instead of /health (requires LITELLM_MASTER_KEY)\n  const healthUrl = baseUrl.replace(/\\/v1\\/?$/, '') + '/health/liveliness';\n  const start = Date.now();\n\n  try {\n    const res = await fetch(healthUrl, { signal: AbortSignal.timeout(5000) });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs };\n    }\n    return { healthy: false, latencyMs, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkFireworksHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('fireworks');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'FIREWORKS_AI_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const res = await fetch('https://api.fireworks.ai/inference/v1/models', {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkClaudeConfigured(): Promise<boolean> {\n  const apiKey = await getApiKey('anthropic');\n  return !!apiKey;\n}\n"}, "c8a67ef389aedaea6adf0d909e5661190dc5fa2b": {"t": 1785754369.4745688, "n": 3279, "h": "364b7d6cf813570c", "c": "/**\n * Routes Health Check API\n *\n * Returns health status for both LLM routes:\n * - Route 1: LiteLLM proxy health via /health endpoint\n * - Route 2: Fireworks AI reachability + Anthropic API key configured\n */\n\nimport { NextResponse } from 'next/server';\nimport { getCurrentUser } from '@/lib/auth';\nimport { getApiKey } from '@/lib/provider-helpers';\n\ninterface RouteHealth {\n  route1: { healthy: boolean; latencyMs: number | null; error?: string };\n  route2: {\n    fireworks: { healthy: boolean; latencyMs: number | null; configured: boolean; error?: string };\n    claude: { configured: boolean };\n  };\n}\n\n/**\n * GET - Check health of both routes\n */\nexport async function GET() {\n  try {\n    const user = await getCurrentUser();\n    if (!user) {\n      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });\n    }\n\n    // Check both routes in parallel\n    const [route1Health, fireworksHealth, claudeConfigured] = await Promise.all([\n      checkRoute1Health(),\n      checkFireworksHealth(),\n      checkClaudeConfigured(),\n    ]);\n\n    const health: RouteHealth = {\n      route1: route1Health,\n      route2: {\n        fireworks: fireworksHealth,\n        claude: { configured: claudeConfigured },\n      },\n    };\n\n    return NextResponse.json(health);\n  } catch (error) {\n    console.error('[Routes Health API] Error:', error);\n    return NextResponse.json(\n      { error: 'Failed to check route health' },\n      { status: 500 }\n    );\n  }\n}\n\nasync function checkRoute1Health(): Promise<{ healthy: boolean; latencyMs: number | null; error?: string }> {\n  const baseUrl = process.env.OPENAI_BASE_URL;\n  if (!baseUrl) {\n    return { healthy: false, latencyMs: null, error: 'OPENAI_BASE_URL not configured' };\n  }\n\n  const healthUrl = baseUrl.replace(/\\/v1\\/?$/, '') + '/health';\n  const start = Date.now();\n\n  try {\n    const res = await fetch(healthUrl, { signal: AbortSignal.timeout(5000) });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs };\n    }\n    return { healthy: false, latencyMs, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkFireworksHealth(): Promise<{ healthy: boolean; latencyMs: number | null; configured: boolean; error?: string }> {\n  const apiKey = await getApiKey('fireworks');\n  if (!apiKey) {\n    return { healthy: false, latencyMs: null, configured: false, error: 'FIREWORKS_AI_API_KEY not configured' };\n  }\n\n  const start = Date.now();\n  try {\n    const res = await fetch('https://api.fireworks.ai/inference/v1/models', {\n      headers: { Authorization: `Bearer ${apiKey}` },\n      signal: AbortSignal.timeout(5000),\n    });\n    const latencyMs = Date.now() - start;\n    if (res.ok) {\n      return { healthy: true, latencyMs, configured: true };\n    }\n    return { healthy: false, latencyMs, configured: true, error: `HTTP ${res.status}` };\n  } catch (err) {\n    return { healthy: false, latencyMs: Date.now() - start, configured: true, error: err instanceof Error ? err.message : 'Unknown error' };\n  }\n}\n\nasync function checkClaudeConfigured(): Promise<boolean> {\n  const apiKey = await getApiKey('anthropic');\n  return !!apiKey;\n}\n"}}, "docs/tech/liteLLM-implementation-guide.md": {"8fe509670a4b0337ef34998bfce43c856fd03e67": {"t": 1785746566.0664334, "n": 39004, "h": "e6f36b50de7853c4", "c": "# LiteLLM Implementation Guide\n\n> **⚠️ STALE DOCUMENT — June 2026:** LiteLLM proxy has been **removed** from AI Assistant. All providers (OpenAI, Gemini, Mistral, Anthropic, DeepSeek, Moonshot, Ollama, Azure Foundry, Fireworks, Ollama Cloud) now use direct native SDKs/APIs. Routes 1 and 4 no longer exist. See [`docs/features/LLM.md`](../features/LLM.md) for the authoritative current architecture. This document is retained for historical reference and LiteLLM setup instructions that may apply to external consumers.\n\n> Customized for multi-provider LLM abstraction with proxy approach, embeddings routing, function calling support, and audio transcription.\n\n---\n\n## LLM Service Routing Overview\n\nAI Assistant uses a **hybrid architecture**: most chat services route through LiteLLM proxy for unified model management, while **Anthropic Claude models bypass LiteLLM entirely** using the `@anthropic-ai/sdk` for direct API access (eliminating tool-calling JSON assembly issues), and specialized services (images, audio, document processing) call provider APIs directly or use local parsers.\n\n### Service Routing Table\n\n| Service | AI Assistant Feature | Routes Through | Provider(s) | Notes |\n|---------|-------------------|----------------|-------------|-------|\n| **Chat Completions** | Main chat, RAG responses | ✅ LiteLLM / ⚡ Anthropic Direct | OpenAI, Gemini, Mistral, DeepSeek, Fireworks AI, Ollama via LiteLLM; **Anthropic Claude via direct SDK** | Claude uses `@anthropic-ai/sdk` for reliable tool calling |\n| **Embeddings** | Document indexing, search | ✅ LiteLLM | OpenAI, Mistral, Gemini, Ollama, Fireworks | `text-embedding-3-large` default |\n| **Diagram Generation** | `diagram_gen` tool | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Generates Mermaid syntax |\n| **Summarization** | Message compression | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic, DeepSeek | Long conversation handling |\n| **Memory Extraction** | User fact storage | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Per-category memory |\n| **Prompt Optimization** | Query refinement | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Pre-RAG processing |\n| **Compliance Checks** | HITL clarification | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Skill compliance |\n| **Translation** | Multi-language support | ✅ LiteLLM / ❌ Direct | OpenAI (proxy), Gemini/Mistral (direct) | Provider-dependent |\n| **Audio Transcription** | Voice input | ✅ LiteLLM | OpenAI Whisper, Mistral Voxtral | `transcribeAudio()` uses `getOpenAI()` client |\n| **Image Generation** | `image_gen` tool | ❌ Direct | Gemini Nano Banana, Imagen 4 | Specialized APIs |\n| **Podcast Generation** | `podcast_gen` tool | ❌ Direct | OpenAI TTS, Gemini TTS | Multi-voice audio synthesis |\n| **Document Processing** | Document text extraction | ❌ Direct | mammoth/exceljs/officeparser (local), Mistral OCR, Azure DI, pdf-parse | Tiered fallback: local parsers first, then API providers |\n| **Reranking** | Search result scoring | ❌ Direct | Fireworks AI | Direct HTTP to `api.fireworks.ai/inference/v1/rerank` |\n| **PPTX Generation** | `pptx_gen` tool | N/A | None | Template-based (no LLM) |\n| **XLSX Generation** | `xlsx_gen` tool | N/A | None | ExcelJS-based (no LLM) |\n| **Chart Generation** | `chart_gen` tool | N/A | None | Client-side rendering |\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ LiteLLM | Routes through LiteLLM proxy (port 4000) |\n| ⚡ Anthropic Direct | Calls Anthropic API directly via `@anthropic-ai/sdk` (bypasses LiteLLM) |\n| ❌ Direct | Calls provider API directly (bypasses LiteLLM) |\n| N/A | No LLM calls required |\n\n### Why Some Services Bypass LiteLLM\n\n| Service | Reason |\n|---------|--------|\n| **Anthropic Claude** | LiteLLM's Anthropic→OpenAI streaming translation produces malformed tool-calling JSON. Direct `@anthropic-ai/sdk` provides native, reliable tool call parsing |\n| **Image Generation** | Gemini and Imagen APIs not supported by LiteLLM proxy |\n| **Podcast TTS** | OpenAI TTS and Gemini TTS are audio generation APIs, not chat completions |\n| **OCR** | Mistral OCR is a specialized document API with unique request format |\n| **Reranking** | Fireworks reranker is a specialized search API (`/inference/v1/rerank`), not a chat model |\n\n> **Note**: Audio transcription (Whisper, Voxtral) routes **through LiteLLM** — `transcribeAudio()` in `openai.ts` uses the same `getOpenAI()` client as chat completions. Both `whisper-1` and `voxtral-mini` are registered in `litellm_config.yaml`.\n\n### Proxy Detection Logic\n\nThe actual detection in `openai.ts` checks for the `OPENAI_BASE_URL` environment variable:\n\n```typescript\n// src/lib/openai.ts — getOpenAI()\nasync function getOpenAI(): Promise<OpenAI> {\n  if (!openaiClient) {\n    // When OPENAI_BASE_URL is set, route through LiteLLM proxy\n    const apiKey = process.env.OPENAI_BASE_URL\n      ? (process.env.LITELLM_MASTER_KEY || await getApiKey('openai'))\n      : await getApiKey('openai');\n\n    openaiClient = new OpenAI({\n      apiKey: apiKey || undefined,\n      baseURL: process.env.OPENAI_BASE_URL || undefined,\n      timeout: 300 * 1000,\n    });\n  }\n  return openaiClient;\n}\n```\n\nWhen `OPENAI_BASE_URL` points to the LiteLLM container (e.g. `http://litellm:4000/v1`), embeddings, transcription, and non-Claude chat completions are routed through a single OpenAI SDK client.\n\n### Claude Direct SDK Detection\n\nFor Anthropic Claude models, `generateResponseWithTools()` bypasses LiteLLM entirely:\n\n```typescript\n// src/lib/openai.ts — isClaudeModel() + routing\nfunction isClaudeModel(model: string): boolean {\n  return model.startsWith('anthropic/') || model.startsWith('claude-');\n}\n\n// In generateResponseWithTools():\nconst useAnthropicDirect = isClaudeModel(effectiveModel);\nconst openai = useAnthropicDirect ? null : await getOpenAI();          // LiteLLM path\nconst anthropicClient = useAnthropicDirect ? await getAnthropicClient() : null;  // Direct path\n```\n\n**Why**: LiteLLM translates Anthropic's native streaming into OpenAI-compatible `delta.tool_calls` format, but the tool call JSON assembly is unreliable — producing malformed arguments that fail `executeTool()`. The direct `@anthropic-ai/sdk` receives pre-parsed tool inputs via `stream.finalMessage()`, eliminating this class of errors.\n\n---\n\n## AI Assistant Integration\n\nThis guide is configured for the AI Assistant RAG application. Default model preset:\n\n| Setting | Value |\n|---------|-------|\n| Default Model | `gpt-4.1-mini` |\n| Temperature | `0.2` |\n| Max Tokens | `2000` |\n| Embedding Model | `text-embedding-3-large` |\n| Embedding Dimensions | `3072` |\n\nAvailable model presets in AI Assistant (via `config/defaults.json`):\n- **gpt-4.1** - High Performance (1M context)\n- **gpt-4.1-mini** - Balanced (default)\n- **gpt-4.1-nano** - Cost-Effective\n- **mistral-large-3** - Mistral Flagship (256K context)\n- **mistral-small-3.2** - Mistral Cost-Effective\n- **ministral-8b** - Mistral Ultra Cost-Effective\n- **gemini-2.5-pro** - Google Flagship Reasoning (1M context)\n- **gemini-2.5-flash** - Google Balanced (1M context)\n- **gemini-2.5-flash-lite** - Google Cost-Effective (1M context)\n- **claude-opus-4-6** - Anthropic Flagship\n- **claude-sonnet-4-6** - Anthropic Balanced\n- **deepseek-chat** - DeepSeek V3 (cost-effective)\n- **deepseek-reasoner** - DeepSeek R1 (thinking model)\n- **fw/kimi-k2-instruct** - Fireworks Kimi K2 (dev/test)\n- **ollama-llama3.2** - Local (no API cost)\n- **ollama-qwen2.5** - Local with excellent reasoning\n\n---\n\n## Architecture Overview\n\nAI Assistant uses a **four-tier hybrid architecture**. LiteLLM handles most chat, embeddings, and transcription. **Anthropic Claude models bypass LiteLLM** via the native `@anthropic-ai/sdk` for reliable tool calling. Specialized services use direct API calls.\n\n```\n┌───────────────────────────────────────────────────────────────────────┐\n│                         AI Assistant Application                         │\n│              (Next.js — openai.ts, tools, translation)                │\n└──┬──────────────────┬──────────────────┬──────────────────┬───────────┘\n   │                  │                  │                  │\n TIER 1            TIER 1b            TIER 2             TIER 3\n LiteLLM Proxy     Anthropic Direct   Direct Provider    Direct Google\n (Chat, Embed,     SDK (@anthropic-   APIs (Non-Chat)    GenAI SDK\n  Transcription)    ai/sdk)                              (Image/TTS)\n   │                  │                  │                  │\n   ▼                  ▼                  ▼                  ▼\n┌────────────────┐ ┌────────────────┐ ┌────────────────┐ ┌────────────────┐\n│  Port 4000     │ │ api.anthropic  │ │                │ │                │\n│                │ │ .com           │ │ Fireworks AI   │ │ Gemini Imagen  │\n│ ┌────────────┐ │ │                │ │  (reranking)   │ │  (image_gen)   │\n│ │ OpenAI     │ │ │ Claude chat +  │ │                │ │  via REST API  │\n│ │ Gemini     │ │ │ tool calling   │ │ Tavily         │ │                │\n│ │ Mistral    │ │ │ via native     │ │  (web_search)  │ │ Gemini TTS     │\n│ │ DeepSeek   │ │ │ streaming      │ │                │ │  (podcast_gen) │\n│ │ Fireworks* │ │ │                │ │ Gemini/Mistral │ │  via @google/  │\n│ │ Ollama*    │ │ │ Models:        │ │  (translation) │ │  genai SDK     │\n│ └────────────┘ │ │ claude-opus-*  │ │                │ │                │\n│                │ │ claude-sonnet-*│ │ OpenAI TTS     │ │ Imagen 4       │\n│ * YAML-only,   │ │ claude-haiku-* │ │  (podcast_gen) │ │  (image_gen)   │\n│   not dynamic  │ │                │ │  hardcoded to  │ │  via @google/  │\n│   sync         │ │ Why: LiteLLM   │ │  api.openai.com│ │  genai SDK     │\n│                │ │ breaks tool    │ │                │ │                │\n│                │ │ call JSON      │ │                │ │                │\n└────────────────┘ └────────────────┘ └────────────────┘ └────────────────┘\n```\n\n### Model Capability Detection\n\nCapabilities are auto-detected via regex patterns in `src/lib/services/model-discovery.ts` and stored in the `enabled_models` table. Admins can override via the LLM Settings UI.\n\n| Capability | DB Column | Detection Function | Used By |\n|------------|-----------|-------------------|---------|\n| Tool calling | `tool_capable` | `isToolCapable(modelId)` | `openai.ts` — whether to send tool definitions |\n| Vision/multimodal | `vision_capable` | `isVisionCapable(modelId)` | `config-capability-checker.ts` — image upload gating |\n| Parallel tool calls | `parallel_tool_capable` | `isParallelToolCapable(modelId)` | `openai.ts` — sequential vs `Promise.allSettled` execution |\n| Thinking/reasoning | `thinking_capable` | `isThinkingCapable(modelId)` | UI — thinking content display |\n\n### Model Registration: Dynamic vs Static\n\nAI Assistant uses **two registration paths** for LiteLLM models:\n\n| Path | Providers | How |\n|------|-----------|-----|\n| **Dynamic** (via `/model/new` API) | OpenAI, Anthropic, Gemini, Mistral, DeepSeek | `litellm-sync.ts` reads `enabled_models` table and registers via API |\n| **Static** (YAML only) | Fireworks, Ollama, Embeddings, Audio | Defined in `litellm_config.yaml`, skipped by sync service |\n| **Direct SDK** (bypasses LiteLLM) | Anthropic Claude | `isClaudeModel()` in `openai.ts` detects `anthropic/` or `claude-` prefix → routes to `@anthropic-ai/sdk` |\n\n> Dynamic models (5 providers) are NOT in the YAML — they are registered at startup by the app. Only Fireworks, Ollama, embedding, and audio models are declared in `litellm_config.yaml`.\n>\n> **Note**: Anthropic models are still registered dynamically with LiteLLM (for embeddings/non-chat use), but **chat completions with tool calling** bypass LiteLLM entirely via the Anthropic SDK. This is controlled by `isClaudeModel()` in `openai.ts`.\n\n---\n\n## Decision Summary\n\n| Component | Decision |\n|-----------|----------|\n| **Approach** | LiteLLM Proxy (Docker Compose) + Anthropic Direct SDK |\n| **Chat Completions** | Route through LiteLLM (except Claude → `@anthropic-ai/sdk` direct) |\n| **Embeddings** | Route through LiteLLM (per-provider config) |\n| **Function Calling** | Gated by `toolCapable` flag from `litellm_config.yaml` via DB sync |\n| **Audio Transcription** | Route through LiteLLM (Whisper, Voxtral) |\n| **Deployment** | Docker Compose |\n\n---\n\n## Project Structure\n\n```\nlitellm-proxy/\n├── docker-compose.yml\n├── litellm_config.yaml\n├── .env\n└── README.md\n```\n\n---\n\n## Step 1: Environment Variables\n\nCreate `.env` file:\n\n```bash\n# ===================\n# LiteLLM Proxy Keys\n# ===================\nLITELLM_MASTER_KEY=sk-litellm-master-change-this\nLITELLM_SALT_KEY=sk-litellm-salt-change-this\n\n# ===================\n# OpenAI\n# ===================\nOPENAI_API_KEY=sk-...\n\n# ===================\n# Azure OpenAI\n# ===================\nAZURE_API_KEY=...\nAZURE_API_BASE=https://your-resource.openai.azure.com/\nAZURE_API_VERSION=2024-02-15-preview\nAZURE_CHAT_DEPLOYMENT=gpt-4-deployment\nAZURE_EMBEDDING_DEPLOYMENT=text-embedding-ada-002\n\n# ===================\n# Mistral\n# ===================\nMISTRAL_API_KEY=...\n\n# ===================\n# Google Gemini\n# ===================\nGEMINI_API_KEY=...\n\n# ===================\n# Anthropic\n# ===================\nANTHROPIC_API_KEY=...\n\n# ===================\n# DeepSeek\n# ===================\nDEEPSEEK_API_KEY=...\n\n# ===================\n# Fireworks AI (dev/test open-source models)\n# ===================\nFIREWORKS_AI_API_KEY=fw-...\n\n# ===================\n# Ollama (Local)\n# ===================\nOLLAMA_API_BASE=http://host.docker.internal:11434\n# Use 'http://localhost:11434' if running outside Docker\n```\n\n---\n\n## Step 2: LiteLLM Configuration\n\n> **Important**: The reference YAML below is a **complete template** listing all models statically. In actual AI Assistant deployment, chat models for OpenAI, Anthropic, Gemini, Mistral, and DeepSeek are **registered dynamically** via `litellm-sync.ts` (see Architecture Overview above). Only Fireworks, Ollama, embedding, and audio models are declared in the YAML. See the actual config at `litellm-proxy/litellm_config.yaml`.\n\nCreate `litellm_config.yaml`:\n\n```yaml\n# =============================================================================\n# LITELLM PROXY CONFIGURATION\n# Multi-provider setup: OpenAI, Azure, Mistral, Gemini, Anthropic, DeepSeek, Fireworks AI, Ollama\n# Updated: March 2026\n# =============================================================================\n\nmodel_list:\n\n  # ===========================================================================\n  # CHAT COMPLETION MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Models - GPT-4.1 Family (Latest - April 2025+)\n  # ---------------------------------------------------------------------------\n  - model_name: openai-gpt41\n    litellm_params:\n      model: gpt-4.1\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-mini\n    litellm_params:\n      model: gpt-4.1-mini\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-nano\n    litellm_params:\n      model: gpt-4.1-nano\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt35\n    litellm_params:\n      model: gpt-3.5-turbo\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Models - GPT-4.1 Family\n  # ---------------------------------------------------------------------------\n  - model_name: azure-gpt41\n    litellm_params:\n      model: azure/gpt-41-deployment  # Your deployment name\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: azure-gpt41-mini\n    litellm_params:\n      model: azure/gpt-41-mini-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Mistral AI Models - Mistral 3 Family (Latest - December 2025)\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-large-3\n    litellm_params:\n      model: mistral/mistral-large-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 256000\n\n  - model_name: mistral-medium-31\n    litellm_params:\n      model: mistral/mistral-medium-2508\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      supports_vision: true\n\n  - model_name: mistral-small-32\n    litellm_params:\n      model: mistral/mistral-small-2506\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ministral-8b\n    litellm_params:\n      model: mistral/ministral-8b-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Ollama Models (Local) - With Full Tool Support\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-llama32\n    litellm_params:\n      model: ollama/llama3.2\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # Fine-tuned for function calling\n\n  - model_name: ollama-llama31-8b\n    litellm_params:\n      model: ollama/llama3.1:8b\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-mistral\n    litellm_params:\n      model: ollama/mistral\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # v0.3+ supports tools\n\n  - model_name: ollama-qwen25\n    litellm_params:\n      model: ollama/qwen2.5\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-phi4\n    litellm_params:\n      model: ollama/phi4\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: false  # Limited support\n\n  # ===========================================================================\n  # EMBEDDING MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: openai-embedding-large\n    litellm_params:\n      model: text-embedding-3-large\n      api_key: os.environ/OPENAI_API_KEY\n\n  - model_name: openai-embedding-small\n    litellm_params:\n      model: text-embedding-3-small\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: azure-embedding\n    litellm_params:\n      model: azure/text-embedding-ada-002\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  - model_name: azure-embedding-3-large\n    litellm_params:\n      model: azure/text-embedding-3-large-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-embedding\n    litellm_params:\n      model: mistral/mistral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: codestral-embed\n    litellm_params:\n      model: mistral/codestral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Ollama Embeddings (Local)\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-embedding\n    litellm_params:\n      model: ollama/nomic-embed-text\n      api_base: os.environ/OLLAMA_API_BASE\n\n  - model_name: ollama-mxbai-embed\n    litellm_params:\n      model: ollama/mxbai-embed-large\n      api_base: os.environ/OLLAMA_API_BASE\n\n  # ===========================================================================\n  # AUDIO MODELS (Speech-to-Text / Transcription)\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Whisper\n  # ---------------------------------------------------------------------------\n  - model_name: openai-whisper\n    litellm_params:\n      model: whisper-1\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure Whisper (if deployed)\n  # ---------------------------------------------------------------------------\n  - model_name: azure-whisper\n    litellm_params:\n      model: azure/whisper-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Voxtral (Released July 2025 - Beats Whisper at half cost)\n  # ---------------------------------------------------------------------------\n  - model_name: voxtral-small\n    litellm_params:\n      model: mistral/voxtral-small-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: voxtral-mini\n    litellm_params:\n      model: mistral/voxtral-mini-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n# =============================================================================\n# LITELLM SETTINGS\n# =============================================================================\n\nlitellm_settings:\n  # Drop unsupported params instead of erroring\n  drop_params: true\n  \n  # Enable detailed logging\n  set_verbose: false\n\n  # Request timeout\n  request_timeout: 120\n\n# =============================================================================\n# GENERAL SETTINGS\n# =============================================================================\n\ngeneral_settings:\n  # Master key for proxy authentication\n  master_key: os.environ/LITELLM_MASTER_KEY\n```\n\n---\n\n## Step 3: Docker Compose\n\n> **Note**: In the actual AI Assistant deployment, LiteLLM runs as part of the full stack in `docker-compose.local.yml` alongside Postgres, Qdrant, and Redis. The standalone example below is for reference.\n\nCreate `docker-compose.yml`:\n\n```yaml\nversion: \"3.9\"\n\nservices:\n  litellm:\n    image: ghcr.io/berriai/litellm:v1.82.3-stable.patch.2\n    container_name: litellm-proxy\n    ports:\n      - \"4000:4000\"\n    volumes:\n      - ./litellm_config.yaml:/app/config.yaml\n    env_file:\n      - .env\n    environment:\n      - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY}\n      - LITELLM_SALT_KEY=${LITELLM_SALT_KEY}\n    command: --config /app/config.yaml --port 4000 --detailed_debug\n    restart: unless-stopped\n    extra_hosts:\n      - \"host.docker.internal:host-gateway\"\n    healthcheck:\n      test: [\"CMD\", \"curl\", \"-f\", \"http://localhost:4000/health\"]\n      interval: 30s\n      timeout: 10s\n      retries: 3\n\n  # Optional: Ollama service (if not running separately)\n  # ollama:\n  #   image: ollama/ollama:latest\n  #   container_name: ollama\n  #   ports:\n  #     - \"11434:11434\"\n  #   volumes:\n  #     - ollama_data:/root/.ollama\n  #   restart: unless-stopped\n\n# volumes:\n#   ollama_data:\n```\n\n---\n\n## Step 4: Application Code Changes\n\n### Minimal Change Required\n\nAI Assistant is a Next.js (TypeScript) application. The proxy switch is controlled entirely by environment variables — no code changes needed:\n\n**Environment (`.env`):**\n```bash\n# Set this to route all chat/embeddings/transcription through LiteLLM\nOPENAI_BASE_URL=http://litellm:4000/v1\nLITELLM_MASTER_KEY=sk-litellm-master-change-this\n```\n\n**How it works (`src/lib/openai.ts`):**\n```typescript\n// When OPENAI_BASE_URL is set → uses LiteLLM proxy with LITELLM_MASTER_KEY\n// When OPENAI_BASE_URL is unset → calls OpenAI directly with OPENAI_API_KEY\nconst apiKey = process.env.OPENAI_BASE_URL\n  ? (process.env.LITELLM_MASTER_KEY || await getApiKey('openai'))\n  : await getApiKey('openai');\n\nopenaiClient = new OpenAI({\n  apiKey: apiKey || undefined,\n  baseURL: process.env.OPENAI_BASE_URL || undefined,\n});\n```\n\nFor non-Claude models, that's it — the OpenAI SDK handles the routing transparently.\n\n**Claude models** are automatically detected and routed to the Anthropic SDK:\n```typescript\n// Automatic — no env var needed, just ANTHROPIC_API_KEY in DB or env\n// isClaudeModel() detects 'anthropic/' or 'claude-' prefix\n// Uses @anthropic-ai/sdk directly for chat + tool calling\n// Embeddings/transcription still route through LiteLLM\n```\n\n---\n\n## Step 5: Function Calling Handling\n\nSince your `generateResponseWithTools()` uses function calling, here's how to handle provider capabilities:\n\n```python\n# Provider capability mapping (Updated December 2025)\n\n# Full tool/function calling support\n# Matches AI Assistant config/defaults.json toolCapable list\nTOOL_CAPABLE_MODELS = [\n    # OpenAI\n    \"gpt-4.1\", \"gpt-4.1-mini\", \"gpt-4.1-nano\", \"gpt-3.5-turbo\",\n    # Mistral\n    \"mistral-large-3\", \"mistral-medium-3.1\", \"mistral-small-3.2\",\n    \"ministral-8b\", \"ministral-3b\",\n    # Ollama (Now with full tool support!)\n    \"ollama-llama3.2\", \"ollama-llama3.1\", \"ollama-mistral\", \"ollama-qwen2.5\",\n]\n\n# Limited or no tool support (show warning)\nTOOL_LIMITED_MODELS = [\n    \"ollama-phi4\",\n]\n\ndef generate_response_with_tools(model: str, messages: list, tools: list):\n    \"\"\"\n    Generate response with function calling support.\n    Warns and disables tools for unsupported providers.\n    \"\"\"\n    \n    if model in TOOL_LIMITED_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' has limited function calling support.\")\n        print(f\"   Tools will be disabled. Consider using: {', '.join(TOOL_CAPABLE_MODELS[:5])}\")\n        # Call without tools\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages\n        )\n    elif model not in TOOL_CAPABLE_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' tool support unknown. Attempting with tools...\")\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    else:\n        # Full tool support\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    \n    return response\n```\n\n> **Good News (2025 Update)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n---\n\n## Step 6: Embeddings Usage\n\n```python\n# Embeddings via LiteLLM Proxy\n\n# OpenAI embeddings (recommended for RAG quality)\nresponse = client.embeddings.create(\n    model=\"openai-embedding-large\",\n    input=\"Your text here\"\n)\n\n# Azure embeddings\nresponse = client.embeddings.create(\n    model=\"azure-embedding\",\n    input=\"Your text here\"\n)\n\n# Mistral embeddings\nresponse = client.embeddings.create(\n    model=\"mistral-embedding\",\n    input=\"Your text here\"\n)\n\n# Ollama embeddings (local)\nresponse = client.embeddings.create(\n    model=\"ollama-embedding\",\n    input=\"Your text here\"\n)\n\nembedding = response.data[0].embedding\n```\n\n### Important: RAG Consistency\n\n> ⚠️ **Warning**: Don't mix embedding providers for the same vector store. If your RAG was built with `text-embedding-3-large`, query with the same model.\n\n```python\n# Configuration approach\nEMBEDDING_CONFIG = {\n    \"openai\": \"openai-embedding-large\",\n    \"azure\": \"azure-embedding\",\n    \"mistral\": \"mistral-embedding\",\n    \"ollama\": \"ollama-embedding\"\n}\n\n# Use consistently\ncurrent_provider = \"openai\"\nembedding_model = EMBEDDING_CONFIG[current_provider]\n```\n\n---\n\n## Step 7: Audio Transcription (Whisper)\n\n```python\n# Audio transcription via LiteLLM Proxy\n\naudio_file = open(\"audio.mp3\", \"rb\")\n\nresponse = client.audio.transcriptions.create(\n    model=\"openai-whisper\",\n    file=audio_file\n)\n\nprint(response.text)\n```\n\n---\n\n## Deployment Commands\n\n```bash\n# Start the proxy\ndocker compose up -d\n\n# View logs\ndocker compose logs -f litellm\n\n# Stop\ndocker compose down\n\n# Restart after config changes\ndocker compose restart litellm\n```\n\n---\n\n## Testing the Setup\n\n### 1. Health Check\n\n```bash\ncurl http://localhost:4000/health\n```\n\n### 2. List Available Models\n\n```bash\ncurl http://localhost:4000/models \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\"\n```\n\n### 3. Test Chat Completion\n\n```bash\ncurl -X POST http://localhost:4000/chat/completions \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-gpt35\",\n    \"messages\": [{\"role\": \"user\", \"content\": \"Hello, test!\"}]\n  }'\n```\n\n### 4. Test Embeddings\n\n```bash\ncurl -X POST http://localhost:4000/embeddings \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-embedding-large\",\n    \"input\": \"Test embedding\"\n  }'\n```\n\n---\n\n## Model Quick Reference\n\n*Updated: March 2026*\n\n### OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `openai-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Latest flagship, best for coding |\n| `openai-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Faster, 83% cheaper than GPT-4o |\n| `openai-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Fastest, cheapest |\n| `openai-gpt35` | `gpt-3.5-turbo` | ✅ | - | ✅ | - | 16K | Legacy, cost-effective |\n| `openai-embedding-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality embeddings |\n| `openai-embedding-small` | `text-embedding-3-small` | - | ✅ | - | - | 8K | Faster, cheaper |\n| `openai-whisper` | `whisper-1` | - | - | - | ✅ | - | $0.006/min transcription |\n\n### Azure OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `azure-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-embedding` | `text-embedding-ada-002` | - | ✅ | - | - | 8K | Standard embedding |\n| `azure-embedding-3-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality |\n| `azure-whisper` | `whisper` | - | - | - | ✅ | - | Speech-to-text |\n\n### Mistral AI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `mistral-large-3` | `mistral-large-latest` | ✅ | - | ✅ | - | 256K | Flagship MoE (675B total params) |\n| `mistral-medium-31` | `mistral-medium-2508` | ✅ | - | ✅ | - | 128K | Frontier multimodal |\n| `mistral-small-32` | `mistral-small-2506` | ✅ | - | ✅ | - | 128K | Fast, efficient |\n| `ministral-14b` | `ministral-3-14b` | ✅ | - | ✅ | - | 128K | Best small model |\n| `ministral-8b` | `ministral-3-8b` | ✅ | - | ✅ | - | 128K | Edge deployment |\n| `ministral-3b` | `ministral-3-3b` | ✅ | - | ✅ | - | 128K | Ultra-light |\n| `mistral-embedding` | `mistral-embed` | - | ✅ | - | - | 8K | Text embeddings |\n| `codestral-embed` | `codestral-embed` | - | ✅ | - | - | - | Code embeddings |\n| `voxtral-small` | `voxtral-small-latest` | - | - | - | ✅ | 32K | 24B, best accuracy, $0.001/min |\n| `voxtral-mini` | `voxtral-mini-latest` | - | - | - | ✅ | 32K | 3B, edge/local deployment |\n| `voxtral-transcribe` | `voxtral-mini-latest` (via /audio/transcriptions) | - | - | - | ✅ | - | Transcription-only, beats Whisper |\n\n> **Note**: Voxtral (released July 2025) outperforms OpenAI Whisper at half the cost ($0.001/min vs $0.006/min). Supports up to 30 min audio for transcription, 40 min for understanding.\n\n### Google Gemini Models\n\n| Model Name | API Model ID | Chat | Tools | Context | Notes |\n|------------|--------------|------|-------|---------|-------|\n| `gemini-2.5-pro` | `gemini/gemini-2.5-pro` | ✅ | ✅ | 1M | Flagship reasoning model |\n| `gemini-2.5-flash` | `gemini/gemini-2.5-flash` | ✅ | ✅ | 1M | Fast, balanced performance |\n| `gemini-2.5-flash-lite` | `gemini/gemini-2.5-flash-lite` | ✅ | ✅ | 1M | Lowest cost option |\n\n**Pricing (per 1M tokens):**\n- Gemini 2.5 Pro: $1.25 input / $10.00 output (≤200k context)\n- Gemini 2.5 Flash: $0.30 input / $2.50 output\n- Gemini 2.5 Flash-Lite: $0.10 input / $0.40 output\n\n**Features:**\n- All models support function calling\n- Built-in \"thinking\" capabilities with controllable budgets\n- Free tier available for development/testing\n- Native multimodal support (text, image, audio, video)\n\n### Ollama Local Models\n\n| Model Name | Ollama Model | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `ollama-llama32` | `llama3.2` | ✅ | - | ✅ | - | 128K | **Full tool support** (fine-tuned for function calling) |\n| `ollama-llama31-8b` | `llama3.1:8b` | ✅ | - | ✅ | - | 128K | **Full tool support**, best overall |\n| `ollama-llama31-70b` | `llama3.1:70b` | ✅ | - | ✅ | - | 128K | **Full tool support**, highest accuracy |\n| `ollama-mistral` | `mistral` | ✅ | - | ✅ | - | 32K | **Supports tools** (v0.3+) |\n| `ollama-mixtral` | `mixtral:8x7b` | ✅ | - | ✅ | - | 32K | MoE, good multi-domain |\n| `ollama-qwen25` | `qwen2.5` | ✅ | - | ✅ | - | 128K | Strong tool support |\n| `ollama-phi4` | `phi4` | ✅ | - | ⚠️ | - | 16K | Limited tool support |\n| `ollama-embedding` | `nomic-embed-text` | - | ✅ | - | - | 8K | Best local embedding |\n| `ollama-mxbai-embed` | `mxbai-embed-large` | - | ✅ | - | - | 512 | Alternative embedding |\n\n> **Tool Calling Update (2025)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), Mixtral, and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ | Fully Supported |\n| ⚠️ | Limited/Experimental (may need prompt adjustments) |\n| - | Not applicable |\n\n### Recommendations by Use Case\n\n| Use Case | Recommended Model | Why |\n|----------|-------------------|-----|\n| **Production Chat** | `gpt-4.1` or `gemini-2.5-pro` | Best instruction following, 1M context |\n| **Balanced (Default)** | `gpt-4.1-mini` or `gemini-2.5-flash` | Good quality at lower cost (AI Assistant default) |\n| **Budget Chat** | `gpt-4.1-nano` or `gemini-2.5-flash-lite` | Cost-effective for simple queries |\n| **Deep Reasoning** | `gemini-2.5-pro` | Built-in thinking capabilities |\n| **Local/Offline Chat** | `ollama-llama3.2` or `ollama-qwen2.5` | Full tool support, runs locally |\n| **RAG Embeddings** | `text-embedding-3-large` | Best quality for retrieval (AI Assistant default) |\n| **Local Embeddings** | `ollama-embedding` (nomic) | Good quality, no API cost |\n| **Audio Transcription** | `voxtral-transcribe` | Beats Whisper, half the cost |\n| **Budget Transcription** | `whisper-1` | Well-established, $0.006/min |\n| **Function Calling** | `gpt-4.1-mini`, `gemini-2.5-flash`, `mistral-large-3` | Reliable tool use |\n\n---\n\n## Switching Providers\n\nTo switch your entire application from OpenAI to Ollama:\n\n1. **No code changes needed**\n2. Update model name in your application config:\n\n```python\n# Before (OpenAI)\nMODEL = \"openai-gpt41\"\nEMBEDDING_MODEL = \"openai-embedding-large\"\nAUDIO_MODEL = \"openai-whisper\"\n\n# After (Ollama + Mistral Voxtral for audio)\nMODEL = \"ollama-llama32\"\nEMBEDDING_MODEL = \"ollama-embedding\"\nAUDIO_MODEL = \"voxtral-mini\"  # Or keep openai-whisper\n\n# Or fully Mistral\nMODEL = \"mistral-large-3\"\nEMBEDDING_MODEL = \"mistral-embedding\"\nAUDIO_MODEL = \"voxtral-small\"\n```\n\n---\n\n## Troubleshooting\n\n| Issue | Solution |\n|-------|----------|\n| Connection refused | Check if Docker is running, verify port 4000 |\n| Ollama not reachable | Use `host.docker.internal` in Docker, check Ollama is running |\n| API key errors | Verify `.env` file is loaded, check key format |\n| Model not found | Run `/models` endpoint to list available models |\n| Timeout errors | Increase `request_timeout` in config |\n\n---\n\n## Known Limitations\n\n| # | Limitation | Impact | Status | File |\n|---|-----------|--------|--------|------|\n| 1 | ~~Static YAML cache never invalidated~~ | ~~Runtime model additions invisible until restart~~ | **FIXED** — `clearLiteLLMCache()` now called after `syncAllModelsToLiteLLM()` completes | `litellm-sync.ts` |\n| 2 | ~~Claude tool-calling streaming mismatch~~ | ~~Anthropic models routed through LiteLLM fail tool calling~~ | **FIXED** — Claude models now bypass LiteLLM via `@anthropic-ai/sdk` direct. `isClaudeModel()` routes to `streamAnthropicCompletion()` which receives pre-parsed tool inputs. | `openai.ts` |\n| 3 | **No circuit breaker for chat completions** | If LiteLLM proxy is unreachable, non-Claude chat completions fail. Claude models are unaffected (direct SDK). Embeddings have a fallback model path, but chat/transcription do not. | Open | `openai.ts` |\n| 4 | **Prefix-based provider detection only** | `getProviderFromModelPath()` defaults to `openai` for unrecognized prefixes. Custom/vLLM models without a prefix will be misidentified. No `deepseek/` prefix exists — DeepSeek models are synced dynamically with prefix by `litellm-sync.ts`. | Open | `litellm-validator.ts:282-294` |\n| 5 | **No TTS or image generation via proxy** | `podcast_gen` and `image_gen` must bypass LiteLLM entirely | By design | `podcast-gen.ts`, `image-gen/` |\n\n---\n\n## Resources\n\n- **LiteLLM Docs**: https://docs.litellm.ai/docs/\n- **Supported Providers**: https://docs.litellm.ai/docs/providers\n- **GitHub**: https://github.com/BerriAI/litellm\n- **Ollama Models**: https://ollama.ai/library\n\n---\n\n*Last updated: March 2026 — Added Anthropic Direct SDK (Tier 1b) for Claude chat + tool calling, bypassing LiteLLM. Updated architecture to four-tier hybrid. Fixed Known Limitations #1 (cache invalidation) and #2 (Claude tool-calling). Added `isClaudeModel()` detection logic documentation.*"}, "1961976644bfd91d70c096fc9b427bc3476146f8": {"t": 1785754358.2056017, "n": 39004, "h": "e6f36b50de7853c4", "c": "# LiteLLM Implementation Guide\n\n> **⚠️ STALE DOCUMENT — June 2026:** LiteLLM proxy has been **removed** from AI Assistant. All providers (OpenAI, Gemini, Mistral, Anthropic, DeepSeek, Moonshot, Ollama, Azure Foundry, Fireworks, Ollama Cloud) now use direct native SDKs/APIs. Routes 1 and 4 no longer exist. See [`docs/features/LLM.md`](../features/LLM.md) for the authoritative current architecture. This document is retained for historical reference and LiteLLM setup instructions that may apply to external consumers.\n\n> Customized for multi-provider LLM abstraction with proxy approach, embeddings routing, function calling support, and audio transcription.\n\n---\n\n## LLM Service Routing Overview\n\nAI Assistant uses a **hybrid architecture**: most chat services route through LiteLLM proxy for unified model management, while **Anthropic Claude models bypass LiteLLM entirely** using the `@anthropic-ai/sdk` for direct API access (eliminating tool-calling JSON assembly issues), and specialized services (images, audio, document processing) call provider APIs directly or use local parsers.\n\n### Service Routing Table\n\n| Service | AI Assistant Feature | Routes Through | Provider(s) | Notes |\n|---------|-------------------|----------------|-------------|-------|\n| **Chat Completions** | Main chat, RAG responses | ✅ LiteLLM / ⚡ Anthropic Direct | OpenAI, Gemini, Mistral, DeepSeek, Fireworks AI, Ollama via LiteLLM; **Anthropic Claude via direct SDK** | Claude uses `@anthropic-ai/sdk` for reliable tool calling |\n| **Embeddings** | Document indexing, search | ✅ LiteLLM | OpenAI, Mistral, Gemini, Ollama, Fireworks | `text-embedding-3-large` default |\n| **Diagram Generation** | `diagram_gen` tool | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Generates Mermaid syntax |\n| **Summarization** | Message compression | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic, DeepSeek | Long conversation handling |\n| **Memory Extraction** | User fact storage | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Per-category memory |\n| **Prompt Optimization** | Query refinement | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Pre-RAG processing |\n| **Compliance Checks** | HITL clarification | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Skill compliance |\n| **Translation** | Multi-language support | ✅ LiteLLM / ❌ Direct | OpenAI (proxy), Gemini/Mistral (direct) | Provider-dependent |\n| **Audio Transcription** | Voice input | ✅ LiteLLM | OpenAI Whisper, Mistral Voxtral | `transcribeAudio()` uses `getOpenAI()` client |\n| **Image Generation** | `image_gen` tool | ❌ Direct | Gemini Nano Banana, Imagen 4 | Specialized APIs |\n| **Podcast Generation** | `podcast_gen` tool | ❌ Direct | OpenAI TTS, Gemini TTS | Multi-voice audio synthesis |\n| **Document Processing** | Document text extraction | ❌ Direct | mammoth/exceljs/officeparser (local), Mistral OCR, Azure DI, pdf-parse | Tiered fallback: local parsers first, then API providers |\n| **Reranking** | Search result scoring | ❌ Direct | Fireworks AI | Direct HTTP to `api.fireworks.ai/inference/v1/rerank` |\n| **PPTX Generation** | `pptx_gen` tool | N/A | None | Template-based (no LLM) |\n| **XLSX Generation** | `xlsx_gen` tool | N/A | None | ExcelJS-based (no LLM) |\n| **Chart Generation** | `chart_gen` tool | N/A | None | Client-side rendering |\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ LiteLLM | Routes through LiteLLM proxy (port 4000) |\n| ⚡ Anthropic Direct | Calls Anthropic API directly via `@anthropic-ai/sdk` (bypasses LiteLLM) |\n| ❌ Direct | Calls provider API directly (bypasses LiteLLM) |\n| N/A | No LLM calls required |\n\n### Why Some Services Bypass LiteLLM\n\n| Service | Reason |\n|---------|--------|\n| **Anthropic Claude** | LiteLLM's Anthropic→OpenAI streaming translation produces malformed tool-calling JSON. Direct `@anthropic-ai/sdk` provides native, reliable tool call parsing |\n| **Image Generation** | Gemini and Imagen APIs not supported by LiteLLM proxy |\n| **Podcast TTS** | OpenAI TTS and Gemini TTS are audio generation APIs, not chat completions |\n| **OCR** | Mistral OCR is a specialized document API with unique request format |\n| **Reranking** | Fireworks reranker is a specialized search API (`/inference/v1/rerank`), not a chat model |\n\n> **Note**: Audio transcription (Whisper, Voxtral) routes **through LiteLLM** — `transcribeAudio()` in `openai.ts` uses the same `getOpenAI()` client as chat completions. Both `whisper-1` and `voxtral-mini` are registered in `litellm_config.yaml`.\n\n### Proxy Detection Logic\n\nThe actual detection in `openai.ts` checks for the `OPENAI_BASE_URL` environment variable:\n\n```typescript\n// src/lib/openai.ts — getOpenAI()\nasync function getOpenAI(): Promise<OpenAI> {\n  if (!openaiClient) {\n    // When OPENAI_BASE_URL is set, route through LiteLLM proxy\n    const apiKey = process.env.OPENAI_BASE_URL\n      ? (process.env.LITELLM_MASTER_KEY || await getApiKey('openai'))\n      : await getApiKey('openai');\n\n    openaiClient = new OpenAI({\n      apiKey: apiKey || undefined,\n      baseURL: process.env.OPENAI_BASE_URL || undefined,\n      timeout: 300 * 1000,\n    });\n  }\n  return openaiClient;\n}\n```\n\nWhen `OPENAI_BASE_URL` points to the LiteLLM container (e.g. `http://litellm:4000/v1`), embeddings, transcription, and non-Claude chat completions are routed through a single OpenAI SDK client.\n\n### Claude Direct SDK Detection\n\nFor Anthropic Claude models, `generateResponseWithTools()` bypasses LiteLLM entirely:\n\n```typescript\n// src/lib/openai.ts — isClaudeModel() + routing\nfunction isClaudeModel(model: string): boolean {\n  return model.startsWith('anthropic/') || model.startsWith('claude-');\n}\n\n// In generateResponseWithTools():\nconst useAnthropicDirect = isClaudeModel(effectiveModel);\nconst openai = useAnthropicDirect ? null : await getOpenAI();          // LiteLLM path\nconst anthropicClient = useAnthropicDirect ? await getAnthropicClient() : null;  // Direct path\n```\n\n**Why**: LiteLLM translates Anthropic's native streaming into OpenAI-compatible `delta.tool_calls` format, but the tool call JSON assembly is unreliable — producing malformed arguments that fail `executeTool()`. The direct `@anthropic-ai/sdk` receives pre-parsed tool inputs via `stream.finalMessage()`, eliminating this class of errors.\n\n---\n\n## AI Assistant Integration\n\nThis guide is configured for the AI Assistant RAG application. Default model preset:\n\n| Setting | Value |\n|---------|-------|\n| Default Model | `gpt-4.1-mini` |\n| Temperature | `0.2` |\n| Max Tokens | `2000` |\n| Embedding Model | `text-embedding-3-large` |\n| Embedding Dimensions | `3072` |\n\nAvailable model presets in AI Assistant (via `config/defaults.json`):\n- **gpt-4.1** - High Performance (1M context)\n- **gpt-4.1-mini** - Balanced (default)\n- **gpt-4.1-nano** - Cost-Effective\n- **mistral-large-3** - Mistral Flagship (256K context)\n- **mistral-small-3.2** - Mistral Cost-Effective\n- **ministral-8b** - Mistral Ultra Cost-Effective\n- **gemini-2.5-pro** - Google Flagship Reasoning (1M context)\n- **gemini-2.5-flash** - Google Balanced (1M context)\n- **gemini-2.5-flash-lite** - Google Cost-Effective (1M context)\n- **claude-opus-4-6** - Anthropic Flagship\n- **claude-sonnet-4-6** - Anthropic Balanced\n- **deepseek-chat** - DeepSeek V3 (cost-effective)\n- **deepseek-reasoner** - DeepSeek R1 (thinking model)\n- **fw/kimi-k2-instruct** - Fireworks Kimi K2 (dev/test)\n- **ollama-llama3.2** - Local (no API cost)\n- **ollama-qwen2.5** - Local with excellent reasoning\n\n---\n\n## Architecture Overview\n\nAI Assistant uses a **four-tier hybrid architecture**. LiteLLM handles most chat, embeddings, and transcription. **Anthropic Claude models bypass LiteLLM** via the native `@anthropic-ai/sdk` for reliable tool calling. Specialized services use direct API calls.\n\n```\n┌───────────────────────────────────────────────────────────────────────┐\n│                         AI Assistant Application                         │\n│              (Next.js — openai.ts, tools, translation)                │\n└──┬──────────────────┬──────────────────┬──────────────────┬───────────┘\n   │                  │                  │                  │\n TIER 1            TIER 1b            TIER 2             TIER 3\n LiteLLM Proxy     Anthropic Direct   Direct Provider    Direct Google\n (Chat, Embed,     SDK (@anthropic-   APIs (Non-Chat)    GenAI SDK\n  Transcription)    ai/sdk)                              (Image/TTS)\n   │                  │                  │                  │\n   ▼                  ▼                  ▼                  ▼\n┌────────────────┐ ┌────────────────┐ ┌────────────────┐ ┌────────────────┐\n│  Port 4000     │ │ api.anthropic  │ │                │ │                │\n│                │ │ .com           │ │ Fireworks AI   │ │ Gemini Imagen  │\n│ ┌────────────┐ │ │                │ │  (reranking)   │ │  (image_gen)   │\n│ │ OpenAI     │ │ │ Claude chat +  │ │                │ │  via REST API  │\n│ │ Gemini     │ │ │ tool calling   │ │ Tavily         │ │                │\n│ │ Mistral    │ │ │ via native     │ │  (web_search)  │ │ Gemini TTS     │\n│ │ DeepSeek   │ │ │ streaming      │ │                │ │  (podcast_gen) │\n│ │ Fireworks* │ │ │                │ │ Gemini/Mistral │ │  via @google/  │\n│ │ Ollama*    │ │ │ Models:        │ │  (translation) │ │  genai SDK     │\n│ └────────────┘ │ │ claude-opus-*  │ │                │ │                │\n│                │ │ claude-sonnet-*│ │ OpenAI TTS     │ │ Imagen 4       │\n│ * YAML-only,   │ │ claude-haiku-* │ │  (podcast_gen) │ │  (image_gen)   │\n│   not dynamic  │ │                │ │  hardcoded to  │ │  via @google/  │\n│   sync         │ │ Why: LiteLLM   │ │  api.openai.com│ │  genai SDK     │\n│                │ │ breaks tool    │ │                │ │                │\n│                │ │ call JSON      │ │                │ │                │\n└────────────────┘ └────────────────┘ └────────────────┘ └────────────────┘\n```\n\n### Model Capability Detection\n\nCapabilities are auto-detected via regex patterns in `src/lib/services/model-discovery.ts` and stored in the `enabled_models` table. Admins can override via the LLM Settings UI.\n\n| Capability | DB Column | Detection Function | Used By |\n|------------|-----------|-------------------|---------|\n| Tool calling | `tool_capable` | `isToolCapable(modelId)` | `openai.ts` — whether to send tool definitions |\n| Vision/multimodal | `vision_capable` | `isVisionCapable(modelId)` | `config-capability-checker.ts` — image upload gating |\n| Parallel tool calls | `parallel_tool_capable` | `isParallelToolCapable(modelId)` | `openai.ts` — sequential vs `Promise.allSettled` execution |\n| Thinking/reasoning | `thinking_capable` | `isThinkingCapable(modelId)` | UI — thinking content display |\n\n### Model Registration: Dynamic vs Static\n\nAI Assistant uses **two registration paths** for LiteLLM models:\n\n| Path | Providers | How |\n|------|-----------|-----|\n| **Dynamic** (via `/model/new` API) | OpenAI, Anthropic, Gemini, Mistral, DeepSeek | `litellm-sync.ts` reads `enabled_models` table and registers via API |\n| **Static** (YAML only) | Fireworks, Ollama, Embeddings, Audio | Defined in `litellm_config.yaml`, skipped by sync service |\n| **Direct SDK** (bypasses LiteLLM) | Anthropic Claude | `isClaudeModel()` in `openai.ts` detects `anthropic/` or `claude-` prefix → routes to `@anthropic-ai/sdk` |\n\n> Dynamic models (5 providers) are NOT in the YAML — they are registered at startup by the app. Only Fireworks, Ollama, embedding, and audio models are declared in `litellm_config.yaml`.\n>\n> **Note**: Anthropic models are still registered dynamically with LiteLLM (for embeddings/non-chat use), but **chat completions with tool calling** bypass LiteLLM entirely via the Anthropic SDK. This is controlled by `isClaudeModel()` in `openai.ts`.\n\n---\n\n## Decision Summary\n\n| Component | Decision |\n|-----------|----------|\n| **Approach** | LiteLLM Proxy (Docker Compose) + Anthropic Direct SDK |\n| **Chat Completions** | Route through LiteLLM (except Claude → `@anthropic-ai/sdk` direct) |\n| **Embeddings** | Route through LiteLLM (per-provider config) |\n| **Function Calling** | Gated by `toolCapable` flag from `litellm_config.yaml` via DB sync |\n| **Audio Transcription** | Route through LiteLLM (Whisper, Voxtral) |\n| **Deployment** | Docker Compose |\n\n---\n\n## Project Structure\n\n```\nlitellm-proxy/\n├── docker-compose.yml\n├── litellm_config.yaml\n├── .env\n└── README.md\n```\n\n---\n\n## Step 1: Environment Variables\n\nCreate `.env` file:\n\n```bash\n# ===================\n# LiteLLM Proxy Keys\n# ===================\nLITELLM_MASTER_KEY=sk-litellm-master-change-this\nLITELLM_SALT_KEY=sk-litellm-salt-change-this\n\n# ===================\n# OpenAI\n# ===================\nOPENAI_API_KEY=sk-...\n\n# ===================\n# Azure OpenAI\n# ===================\nAZURE_API_KEY=...\nAZURE_API_BASE=https://your-resource.openai.azure.com/\nAZURE_API_VERSION=2024-02-15-preview\nAZURE_CHAT_DEPLOYMENT=gpt-4-deployment\nAZURE_EMBEDDING_DEPLOYMENT=text-embedding-ada-002\n\n# ===================\n# Mistral\n# ===================\nMISTRAL_API_KEY=...\n\n# ===================\n# Google Gemini\n# ===================\nGEMINI_API_KEY=...\n\n# ===================\n# Anthropic\n# ===================\nANTHROPIC_API_KEY=...\n\n# ===================\n# DeepSeek\n# ===================\nDEEPSEEK_API_KEY=...\n\n# ===================\n# Fireworks AI (dev/test open-source models)\n# ===================\nFIREWORKS_AI_API_KEY=fw-...\n\n# ===================\n# Ollama (Local)\n# ===================\nOLLAMA_API_BASE=http://host.docker.internal:11434\n# Use 'http://localhost:11434' if running outside Docker\n```\n\n---\n\n## Step 2: LiteLLM Configuration\n\n> **Important**: The reference YAML below is a **complete template** listing all models statically. In actual AI Assistant deployment, chat models for OpenAI, Anthropic, Gemini, Mistral, and DeepSeek are **registered dynamically** via `litellm-sync.ts` (see Architecture Overview above). Only Fireworks, Ollama, embedding, and audio models are declared in the YAML. See the actual config at `litellm-proxy/litellm_config.yaml`.\n\nCreate `litellm_config.yaml`:\n\n```yaml\n# =============================================================================\n# LITELLM PROXY CONFIGURATION\n# Multi-provider setup: OpenAI, Azure, Mistral, Gemini, Anthropic, DeepSeek, Fireworks AI, Ollama\n# Updated: March 2026\n# =============================================================================\n\nmodel_list:\n\n  # ===========================================================================\n  # CHAT COMPLETION MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Models - GPT-4.1 Family (Latest - April 2025+)\n  # ---------------------------------------------------------------------------\n  - model_name: openai-gpt41\n    litellm_params:\n      model: gpt-4.1\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-mini\n    litellm_params:\n      model: gpt-4.1-mini\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-nano\n    litellm_params:\n      model: gpt-4.1-nano\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt35\n    litellm_params:\n      model: gpt-3.5-turbo\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Models - GPT-4.1 Family\n  # ---------------------------------------------------------------------------\n  - model_name: azure-gpt41\n    litellm_params:\n      model: azure/gpt-41-deployment  # Your deployment name\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: azure-gpt41-mini\n    litellm_params:\n      model: azure/gpt-41-mini-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Mistral AI Models - Mistral 3 Family (Latest - December 2025)\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-large-3\n    litellm_params:\n      model: mistral/mistral-large-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 256000\n\n  - model_name: mistral-medium-31\n    litellm_params:\n      model: mistral/mistral-medium-2508\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      supports_vision: true\n\n  - model_name: mistral-small-32\n    litellm_params:\n      model: mistral/mistral-small-2506\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ministral-8b\n    litellm_params:\n      model: mistral/ministral-8b-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Ollama Models (Local) - With Full Tool Support\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-llama32\n    litellm_params:\n      model: ollama/llama3.2\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # Fine-tuned for function calling\n\n  - model_name: ollama-llama31-8b\n    litellm_params:\n      model: ollama/llama3.1:8b\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-mistral\n    litellm_params:\n      model: ollama/mistral\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # v0.3+ supports tools\n\n  - model_name: ollama-qwen25\n    litellm_params:\n      model: ollama/qwen2.5\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-phi4\n    litellm_params:\n      model: ollama/phi4\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: false  # Limited support\n\n  # ===========================================================================\n  # EMBEDDING MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: openai-embedding-large\n    litellm_params:\n      model: text-embedding-3-large\n      api_key: os.environ/OPENAI_API_KEY\n\n  - model_name: openai-embedding-small\n    litellm_params:\n      model: text-embedding-3-small\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: azure-embedding\n    litellm_params:\n      model: azure/text-embedding-ada-002\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  - model_name: azure-embedding-3-large\n    litellm_params:\n      model: azure/text-embedding-3-large-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-embedding\n    litellm_params:\n      model: mistral/mistral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: codestral-embed\n    litellm_params:\n      model: mistral/codestral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Ollama Embeddings (Local)\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-embedding\n    litellm_params:\n      model: ollama/nomic-embed-text\n      api_base: os.environ/OLLAMA_API_BASE\n\n  - model_name: ollama-mxbai-embed\n    litellm_params:\n      model: ollama/mxbai-embed-large\n      api_base: os.environ/OLLAMA_API_BASE\n\n  # ===========================================================================\n  # AUDIO MODELS (Speech-to-Text / Transcription)\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Whisper\n  # ---------------------------------------------------------------------------\n  - model_name: openai-whisper\n    litellm_params:\n      model: whisper-1\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure Whisper (if deployed)\n  # ---------------------------------------------------------------------------\n  - model_name: azure-whisper\n    litellm_params:\n      model: azure/whisper-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Voxtral (Released July 2025 - Beats Whisper at half cost)\n  # ---------------------------------------------------------------------------\n  - model_name: voxtral-small\n    litellm_params:\n      model: mistral/voxtral-small-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: voxtral-mini\n    litellm_params:\n      model: mistral/voxtral-mini-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n# =============================================================================\n# LITELLM SETTINGS\n# =============================================================================\n\nlitellm_settings:\n  # Drop unsupported params instead of erroring\n  drop_params: true\n  \n  # Enable detailed logging\n  set_verbose: false\n\n  # Request timeout\n  request_timeout: 120\n\n# =============================================================================\n# GENERAL SETTINGS\n# =============================================================================\n\ngeneral_settings:\n  # Master key for proxy authentication\n  master_key: os.environ/LITELLM_MASTER_KEY\n```\n\n---\n\n## Step 3: Docker Compose\n\n> **Note**: In the actual AI Assistant deployment, LiteLLM runs as part of the full stack in `docker-compose.local.yml` alongside Postgres, Qdrant, and Redis. The standalone example below is for reference.\n\nCreate `docker-compose.yml`:\n\n```yaml\nversion: \"3.9\"\n\nservices:\n  litellm:\n    image: ghcr.io/berriai/litellm:v1.82.3-stable.patch.2\n    container_name: litellm-proxy\n    ports:\n      - \"4000:4000\"\n    volumes:\n      - ./litellm_config.yaml:/app/config.yaml\n    env_file:\n      - .env\n    environment:\n      - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY}\n      - LITELLM_SALT_KEY=${LITELLM_SALT_KEY}\n    command: --config /app/config.yaml --port 4000 --detailed_debug\n    restart: unless-stopped\n    extra_hosts:\n      - \"host.docker.internal:host-gateway\"\n    healthcheck:\n      test: [\"CMD\", \"curl\", \"-f\", \"http://localhost:4000/health\"]\n      interval: 30s\n      timeout: 10s\n      retries: 3\n\n  # Optional: Ollama service (if not running separately)\n  # ollama:\n  #   image: ollama/ollama:latest\n  #   container_name: ollama\n  #   ports:\n  #     - \"11434:11434\"\n  #   volumes:\n  #     - ollama_data:/root/.ollama\n  #   restart: unless-stopped\n\n# volumes:\n#   ollama_data:\n```\n\n---\n\n## Step 4: Application Code Changes\n\n### Minimal Change Required\n\nAI Assistant is a Next.js (TypeScript) application. The proxy switch is controlled entirely by environment variables — no code changes needed:\n\n**Environment (`.env`):**\n```bash\n# Set this to route all chat/embeddings/transcription through LiteLLM\nOPENAI_BASE_URL=http://litellm:4000/v1\nLITELLM_MASTER_KEY=sk-litellm-master-change-this\n```\n\n**How it works (`src/lib/openai.ts`):**\n```typescript\n// When OPENAI_BASE_URL is set → uses LiteLLM proxy with LITELLM_MASTER_KEY\n// When OPENAI_BASE_URL is unset → calls OpenAI directly with OPENAI_API_KEY\nconst apiKey = process.env.OPENAI_BASE_URL\n  ? (process.env.LITELLM_MASTER_KEY || await getApiKey('openai'))\n  : await getApiKey('openai');\n\nopenaiClient = new OpenAI({\n  apiKey: apiKey || undefined,\n  baseURL: process.env.OPENAI_BASE_URL || undefined,\n});\n```\n\nFor non-Claude models, that's it — the OpenAI SDK handles the routing transparently.\n\n**Claude models** are automatically detected and routed to the Anthropic SDK:\n```typescript\n// Automatic — no env var needed, just ANTHROPIC_API_KEY in DB or env\n// isClaudeModel() detects 'anthropic/' or 'claude-' prefix\n// Uses @anthropic-ai/sdk directly for chat + tool calling\n// Embeddings/transcription still route through LiteLLM\n```\n\n---\n\n## Step 5: Function Calling Handling\n\nSince your `generateResponseWithTools()` uses function calling, here's how to handle provider capabilities:\n\n```python\n# Provider capability mapping (Updated December 2025)\n\n# Full tool/function calling support\n# Matches AI Assistant config/defaults.json toolCapable list\nTOOL_CAPABLE_MODELS = [\n    # OpenAI\n    \"gpt-4.1\", \"gpt-4.1-mini\", \"gpt-4.1-nano\", \"gpt-3.5-turbo\",\n    # Mistral\n    \"mistral-large-3\", \"mistral-medium-3.1\", \"mistral-small-3.2\",\n    \"ministral-8b\", \"ministral-3b\",\n    # Ollama (Now with full tool support!)\n    \"ollama-llama3.2\", \"ollama-llama3.1\", \"ollama-mistral\", \"ollama-qwen2.5\",\n]\n\n# Limited or no tool support (show warning)\nTOOL_LIMITED_MODELS = [\n    \"ollama-phi4\",\n]\n\ndef generate_response_with_tools(model: str, messages: list, tools: list):\n    \"\"\"\n    Generate response with function calling support.\n    Warns and disables tools for unsupported providers.\n    \"\"\"\n    \n    if model in TOOL_LIMITED_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' has limited function calling support.\")\n        print(f\"   Tools will be disabled. Consider using: {', '.join(TOOL_CAPABLE_MODELS[:5])}\")\n        # Call without tools\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages\n        )\n    elif model not in TOOL_CAPABLE_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' tool support unknown. Attempting with tools...\")\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    else:\n        # Full tool support\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    \n    return response\n```\n\n> **Good News (2025 Update)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n---\n\n## Step 6: Embeddings Usage\n\n```python\n# Embeddings via LiteLLM Proxy\n\n# OpenAI embeddings (recommended for RAG quality)\nresponse = client.embeddings.create(\n    model=\"openai-embedding-large\",\n    input=\"Your text here\"\n)\n\n# Azure embeddings\nresponse = client.embeddings.create(\n    model=\"azure-embedding\",\n    input=\"Your text here\"\n)\n\n# Mistral embeddings\nresponse = client.embeddings.create(\n    model=\"mistral-embedding\",\n    input=\"Your text here\"\n)\n\n# Ollama embeddings (local)\nresponse = client.embeddings.create(\n    model=\"ollama-embedding\",\n    input=\"Your text here\"\n)\n\nembedding = response.data[0].embedding\n```\n\n### Important: RAG Consistency\n\n> ⚠️ **Warning**: Don't mix embedding providers for the same vector store. If your RAG was built with `text-embedding-3-large`, query with the same model.\n\n```python\n# Configuration approach\nEMBEDDING_CONFIG = {\n    \"openai\": \"openai-embedding-large\",\n    \"azure\": \"azure-embedding\",\n    \"mistral\": \"mistral-embedding\",\n    \"ollama\": \"ollama-embedding\"\n}\n\n# Use consistently\ncurrent_provider = \"openai\"\nembedding_model = EMBEDDING_CONFIG[current_provider]\n```\n\n---\n\n## Step 7: Audio Transcription (Whisper)\n\n```python\n# Audio transcription via LiteLLM Proxy\n\naudio_file = open(\"audio.mp3\", \"rb\")\n\nresponse = client.audio.transcriptions.create(\n    model=\"openai-whisper\",\n    file=audio_file\n)\n\nprint(response.text)\n```\n\n---\n\n## Deployment Commands\n\n```bash\n# Start the proxy\ndocker compose up -d\n\n# View logs\ndocker compose logs -f litellm\n\n# Stop\ndocker compose down\n\n# Restart after config changes\ndocker compose restart litellm\n```\n\n---\n\n## Testing the Setup\n\n### 1. Health Check\n\n```bash\ncurl http://localhost:4000/health\n```\n\n### 2. List Available Models\n\n```bash\ncurl http://localhost:4000/models \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\"\n```\n\n### 3. Test Chat Completion\n\n```bash\ncurl -X POST http://localhost:4000/chat/completions \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-gpt35\",\n    \"messages\": [{\"role\": \"user\", \"content\": \"Hello, test!\"}]\n  }'\n```\n\n### 4. Test Embeddings\n\n```bash\ncurl -X POST http://localhost:4000/embeddings \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-embedding-large\",\n    \"input\": \"Test embedding\"\n  }'\n```\n\n---\n\n## Model Quick Reference\n\n*Updated: March 2026*\n\n### OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `openai-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Latest flagship, best for coding |\n| `openai-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Faster, 83% cheaper than GPT-4o |\n| `openai-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Fastest, cheapest |\n| `openai-gpt35` | `gpt-3.5-turbo` | ✅ | - | ✅ | - | 16K | Legacy, cost-effective |\n| `openai-embedding-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality embeddings |\n| `openai-embedding-small` | `text-embedding-3-small` | - | ✅ | - | - | 8K | Faster, cheaper |\n| `openai-whisper` | `whisper-1` | - | - | - | ✅ | - | $0.006/min transcription |\n\n### Azure OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `azure-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-embedding` | `text-embedding-ada-002` | - | ✅ | - | - | 8K | Standard embedding |\n| `azure-embedding-3-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality |\n| `azure-whisper` | `whisper` | - | - | - | ✅ | - | Speech-to-text |\n\n### Mistral AI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `mistral-large-3` | `mistral-large-latest` | ✅ | - | ✅ | - | 256K | Flagship MoE (675B total params) |\n| `mistral-medium-31` | `mistral-medium-2508` | ✅ | - | ✅ | - | 128K | Frontier multimodal |\n| `mistral-small-32` | `mistral-small-2506` | ✅ | - | ✅ | - | 128K | Fast, efficient |\n| `ministral-14b` | `ministral-3-14b` | ✅ | - | ✅ | - | 128K | Best small model |\n| `ministral-8b` | `ministral-3-8b` | ✅ | - | ✅ | - | 128K | Edge deployment |\n| `ministral-3b` | `ministral-3-3b` | ✅ | - | ✅ | - | 128K | Ultra-light |\n| `mistral-embedding` | `mistral-embed` | - | ✅ | - | - | 8K | Text embeddings |\n| `codestral-embed` | `codestral-embed` | - | ✅ | - | - | - | Code embeddings |\n| `voxtral-small` | `voxtral-small-latest` | - | - | - | ✅ | 32K | 24B, best accuracy, $0.001/min |\n| `voxtral-mini` | `voxtral-mini-latest` | - | - | - | ✅ | 32K | 3B, edge/local deployment |\n| `voxtral-transcribe` | `voxtral-mini-latest` (via /audio/transcriptions) | - | - | - | ✅ | - | Transcription-only, beats Whisper |\n\n> **Note**: Voxtral (released July 2025) outperforms OpenAI Whisper at half the cost ($0.001/min vs $0.006/min). Supports up to 30 min audio for transcription, 40 min for understanding.\n\n### Google Gemini Models\n\n| Model Name | API Model ID | Chat | Tools | Context | Notes |\n|------------|--------------|------|-------|---------|-------|\n| `gemini-2.5-pro` | `gemini/gemini-2.5-pro` | ✅ | ✅ | 1M | Flagship reasoning model |\n| `gemini-2.5-flash` | `gemini/gemini-2.5-flash` | ✅ | ✅ | 1M | Fast, balanced performance |\n| `gemini-2.5-flash-lite` | `gemini/gemini-2.5-flash-lite` | ✅ | ✅ | 1M | Lowest cost option |\n\n**Pricing (per 1M tokens):**\n- Gemini 2.5 Pro: $1.25 input / $10.00 output (≤200k context)\n- Gemini 2.5 Flash: $0.30 input / $2.50 output\n- Gemini 2.5 Flash-Lite: $0.10 input / $0.40 output\n\n**Features:**\n- All models support function calling\n- Built-in \"thinking\" capabilities with controllable budgets\n- Free tier available for development/testing\n- Native multimodal support (text, image, audio, video)\n\n### Ollama Local Models\n\n| Model Name | Ollama Model | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `ollama-llama32` | `llama3.2` | ✅ | - | ✅ | - | 128K | **Full tool support** (fine-tuned for function calling) |\n| `ollama-llama31-8b` | `llama3.1:8b` | ✅ | - | ✅ | - | 128K | **Full tool support**, best overall |\n| `ollama-llama31-70b` | `llama3.1:70b` | ✅ | - | ✅ | - | 128K | **Full tool support**, highest accuracy |\n| `ollama-mistral` | `mistral` | ✅ | - | ✅ | - | 32K | **Supports tools** (v0.3+) |\n| `ollama-mixtral` | `mixtral:8x7b` | ✅ | - | ✅ | - | 32K | MoE, good multi-domain |\n| `ollama-qwen25` | `qwen2.5` | ✅ | - | ✅ | - | 128K | Strong tool support |\n| `ollama-phi4` | `phi4` | ✅ | - | ⚠️ | - | 16K | Limited tool support |\n| `ollama-embedding` | `nomic-embed-text` | - | ✅ | - | - | 8K | Best local embedding |\n| `ollama-mxbai-embed` | `mxbai-embed-large` | - | ✅ | - | - | 512 | Alternative embedding |\n\n> **Tool Calling Update (2025)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), Mixtral, and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ | Fully Supported |\n| ⚠️ | Limited/Experimental (may need prompt adjustments) |\n| - | Not applicable |\n\n### Recommendations by Use Case\n\n| Use Case | Recommended Model | Why |\n|----------|-------------------|-----|\n| **Production Chat** | `gpt-4.1` or `gemini-2.5-pro` | Best instruction following, 1M context |\n| **Balanced (Default)** | `gpt-4.1-mini` or `gemini-2.5-flash` | Good quality at lower cost (AI Assistant default) |\n| **Budget Chat** | `gpt-4.1-nano` or `gemini-2.5-flash-lite` | Cost-effective for simple queries |\n| **Deep Reasoning** | `gemini-2.5-pro` | Built-in thinking capabilities |\n| **Local/Offline Chat** | `ollama-llama3.2` or `ollama-qwen2.5` | Full tool support, runs locally |\n| **RAG Embeddings** | `text-embedding-3-large` | Best quality for retrieval (AI Assistant default) |\n| **Local Embeddings** | `ollama-embedding` (nomic) | Good quality, no API cost |\n| **Audio Transcription** | `voxtral-transcribe` | Beats Whisper, half the cost |\n| **Budget Transcription** | `whisper-1` | Well-established, $0.006/min |\n| **Function Calling** | `gpt-4.1-mini`, `gemini-2.5-flash`, `mistral-large-3` | Reliable tool use |\n\n---\n\n## Switching Providers\n\nTo switch your entire application from OpenAI to Ollama:\n\n1. **No code changes needed**\n2. Update model name in your application config:\n\n```python\n# Before (OpenAI)\nMODEL = \"openai-gpt41\"\nEMBEDDING_MODEL = \"openai-embedding-large\"\nAUDIO_MODEL = \"openai-whisper\"\n\n# After (Ollama + Mistral Voxtral for audio)\nMODEL = \"ollama-llama32\"\nEMBEDDING_MODEL = \"ollama-embedding\"\nAUDIO_MODEL = \"voxtral-mini\"  # Or keep openai-whisper\n\n# Or fully Mistral\nMODEL = \"mistral-large-3\"\nEMBEDDING_MODEL = \"mistral-embedding\"\nAUDIO_MODEL = \"voxtral-small\"\n```\n\n---\n\n## Troubleshooting\n\n| Issue | Solution |\n|-------|----------|\n| Connection refused | Check if Docker is running, verify port 4000 |\n| Ollama not reachable | Use `host.docker.internal` in Docker, check Ollama is running |\n| API key errors | Verify `.env` file is loaded, check key format |\n| Model not found | Run `/models` endpoint to list available models |\n| Timeout errors | Increase `request_timeout` in config |\n\n---\n\n## Known Limitations\n\n| # | Limitation | Impact | Status | File |\n|---|-----------|--------|--------|------|\n| 1 | ~~Static YAML cache never invalidated~~ | ~~Runtime model additions invisible until restart~~ | **FIXED** — `clearLiteLLMCache()` now called after `syncAllModelsToLiteLLM()` completes | `litellm-sync.ts` |\n| 2 | ~~Claude tool-calling streaming mismatch~~ | ~~Anthropic models routed through LiteLLM fail tool calling~~ | **FIXED** — Claude models now bypass LiteLLM via `@anthropic-ai/sdk` direct. `isClaudeModel()` routes to `streamAnthropicCompletion()` which receives pre-parsed tool inputs. | `openai.ts` |\n| 3 | **No circuit breaker for chat completions** | If LiteLLM proxy is unreachable, non-Claude chat completions fail. Claude models are unaffected (direct SDK). Embeddings have a fallback model path, but chat/transcription do not. | Open | `openai.ts` |\n| 4 | **Prefix-based provider detection only** | `getProviderFromModelPath()` defaults to `openai` for unrecognized prefixes. Custom/vLLM models without a prefix will be misidentified. No `deepseek/` prefix exists — DeepSeek models are synced dynamically with prefix by `litellm-sync.ts`. | Open | `litellm-validator.ts:282-294` |\n| 5 | **No TTS or image generation via proxy** | `podcast_gen` and `image_gen` must bypass LiteLLM entirely | By design | `podcast-gen.ts`, `image-gen/` |\n\n---\n\n## Resources\n\n- **LiteLLM Docs**: https://docs.litellm.ai/docs/\n- **Supported Providers**: https://docs.litellm.ai/docs/providers\n- **GitHub**: https://github.com/BerriAI/litellm\n- **Ollama Models**: https://ollama.ai/library\n\n---\n\n*Last updated: March 2026 — Added Anthropic Direct SDK (Tier 1b) for Claude chat + tool calling, bypassing LiteLLM. Updated architecture to four-tier hybrid. Fixed Known Limitations #1 (cache invalidation) and #2 (Claude tool-calling). Added `isClaudeModel()` detection logic documentation.*"}, "ed9250b8d405f019371a97b2afba19b4ddfb1af1": {"t": 1785754359.7214315, "n": 38968, "h": "da87126ead045688", "c": "# LiteLLM Implementation Guide\n\n> **⚠️ STALE DOCUMENT — June 2026:** LiteLLM proxy has been **removed** from Policy Bot. All providers (OpenAI, Gemini, Mistral, Anthropic, DeepSeek, Moonshot, Ollama, Azure Foundry, Fireworks, Ollama Cloud) now use direct native SDKs/APIs. Routes 1 and 4 no longer exist. See [`docs/features/LLM.md`](../features/LLM.md) for the authoritative current architecture. This document is retained for historical reference and LiteLLM setup instructions that may apply to external consumers.\n\n> Customized for multi-provider LLM abstraction with proxy approach, embeddings routing, function calling support, and audio transcription.\n\n---\n\n## LLM Service Routing Overview\n\nPolicy Bot uses a **hybrid architecture**: most chat services route through LiteLLM proxy for unified model management, while **Anthropic Claude models bypass LiteLLM entirely** using the `@anthropic-ai/sdk` for direct API access (eliminating tool-calling JSON assembly issues), and specialized services (images, audio, document processing) call provider APIs directly or use local parsers.\n\n### Service Routing Table\n\n| Service | Policy Bot Feature | Routes Through | Provider(s) | Notes |\n|---------|-------------------|----------------|-------------|-------|\n| **Chat Completions** | Main chat, RAG responses | ✅ LiteLLM / ⚡ Anthropic Direct | OpenAI, Gemini, Mistral, DeepSeek, Fireworks AI, Ollama via LiteLLM; **Anthropic Claude via direct SDK** | Claude uses `@anthropic-ai/sdk` for reliable tool calling |\n| **Embeddings** | Document indexing, search | ✅ LiteLLM | OpenAI, Mistral, Gemini, Ollama, Fireworks | `text-embedding-3-large` default |\n| **Diagram Generation** | `diagram_gen` tool | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Generates Mermaid syntax |\n| **Summarization** | Message compression | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic, DeepSeek | Long conversation handling |\n| **Memory Extraction** | User fact storage | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Per-category memory |\n| **Prompt Optimization** | Query refinement | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Pre-RAG processing |\n| **Compliance Checks** | HITL clarification | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Skill compliance |\n| **Translation** | Multi-language support | ✅ LiteLLM / ❌ Direct | OpenAI (proxy), Gemini/Mistral (direct) | Provider-dependent |\n| **Audio Transcription** | Voice input | ✅ LiteLLM | OpenAI Whisper, Mistral Voxtral | `transcribeAudio()` uses `getOpenAI()` client |\n| **Image Generation** | `image_gen` tool | ❌ Direct | Gemini Nano Banana, Imagen 4 | Specialized APIs |\n| **Podcast Generation** | `podcast_gen` tool | ❌ Direct | OpenAI TTS, Gemini TTS | Multi-voice audio synthesis |\n| **Document Processing** | Document text extraction | ❌ Direct | mammoth/exceljs/officeparser (local), Mistral OCR, Azure DI, pdf-parse | Tiered fallback: local parsers first, then API providers |\n| **Reranking** | Search result scoring | ❌ Direct | Fireworks AI | Direct HTTP to `api.fireworks.ai/inference/v1/rerank` |\n| **PPTX Generation** | `pptx_gen` tool | N/A | None | Template-based (no LLM) |\n| **XLSX Generation** | `xlsx_gen` tool | N/A | None | ExcelJS-based (no LLM) |\n| **Chart Generation** | `chart_gen` tool | N/A | None | Client-side rendering |\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ LiteLLM | Routes through LiteLLM proxy (port 4000) |\n| ⚡ Anthropic Direct | Calls Anthropic API directly via `@anthropic-ai/sdk` (bypasses LiteLLM) |\n| ❌ Direct | Calls provider API directly (bypasses LiteLLM) |\n| N/A | No LLM calls required |\n\n### Why Some Services Bypass LiteLLM\n\n| Service | Reason |\n|---------|--------|\n| **Anthropic Claude** | LiteLLM's Anthropic→OpenAI streaming translation produces malformed tool-calling JSON. Direct `@anthropic-ai/sdk` provides native, reliable tool call parsing |\n| **Image Generation** | Gemini and Imagen APIs not supported by LiteLLM proxy |\n| **Podcast TTS** | OpenAI TTS and Gemini TTS are audio generation APIs, not chat completions |\n| **OCR** | Mistral OCR is a specialized document API with unique request format |\n| **Reranking** | Fireworks reranker is a specialized search API (`/inference/v1/rerank`), not a chat model |\n\n> **Note**: Audio transcription (Whisper, Voxtral) routes **through LiteLLM** — `transcribeAudio()` in `openai.ts` uses the same `getOpenAI()` client as chat completions. Both `whisper-1` and `voxtral-mini` are registered in `litellm_config.yaml`.\n\n### Proxy Detection Logic\n\nThe actual detection in `openai.ts` checks for the `OPENAI_BASE_URL` environment variable:\n\n```typescript\n// src/lib/openai.ts — getOpenAI()\nasync function getOpenAI(): Promise<OpenAI> {\n  if (!openaiClient) {\n    // When OPENAI_BASE_URL is set, route through LiteLLM proxy\n    const apiKey = process.env.OPENAI_BASE_URL\n      ? (process.env.LITELLM_MASTER_KEY || await getApiKey('openai'))\n      : await getApiKey('openai');\n\n    openaiClient = new OpenAI({\n      apiKey: apiKey || undefined,\n      baseURL: process.env.OPENAI_BASE_URL || undefined,\n      timeout: 300 * 1000,\n    });\n  }\n  return openaiClient;\n}\n```\n\nWhen `OPENAI_BASE_URL` points to the LiteLLM container (e.g. `http://litellm:4000/v1`), embeddings, transcription, and non-Claude chat completions are routed through a single OpenAI SDK client.\n\n### Claude Direct SDK Detection\n\nFor Anthropic Claude models, `generateResponseWithTools()` bypasses LiteLLM entirely:\n\n```typescript\n// src/lib/openai.ts — isClaudeModel() + routing\nfunction isClaudeModel(model: string): boolean {\n  return model.startsWith('anthropic/') || model.startsWith('claude-');\n}\n\n// In generateResponseWithTools():\nconst useAnthropicDirect = isClaudeModel(effectiveModel);\nconst openai = useAnthropicDirect ? null : await getOpenAI();          // LiteLLM path\nconst anthropicClient = useAnthropicDirect ? await getAnthropicClient() : null;  // Direct path\n```\n\n**Why**: LiteLLM translates Anthropic's native streaming into OpenAI-compatible `delta.tool_calls` format, but the tool call JSON assembly is unreliable — producing malformed arguments that fail `executeTool()`. The direct `@anthropic-ai/sdk` receives pre-parsed tool inputs via `stream.finalMessage()`, eliminating this class of errors.\n\n---\n\n## Policy Bot Integration\n\nThis guide is configured for the Policy Bot RAG application. Default model preset:\n\n| Setting | Value |\n|---------|-------|\n| Default Model | `gpt-4.1-mini` |\n| Temperature | `0.2` |\n| Max Tokens | `2000` |\n| Embedding Model | `text-embedding-3-large` |\n| Embedding Dimensions | `3072` |\n\nAvailable model presets in Policy Bot (via `config/defaults.json`):\n- **gpt-4.1** - High Performance (1M context)\n- **gpt-4.1-mini** - Balanced (default)\n- **gpt-4.1-nano** - Cost-Effective\n- **mistral-large-3** - Mistral Flagship (256K context)\n- **mistral-small-3.2** - Mistral Cost-Effective\n- **ministral-8b** - Mistral Ultra Cost-Effective\n- **gemini-2.5-pro** - Google Flagship Reasoning (1M context)\n- **gemini-2.5-flash** - Google Balanced (1M context)\n- **gemini-2.5-flash-lite** - Google Cost-Effective (1M context)\n- **claude-opus-4-6** - Anthropic Flagship\n- **claude-sonnet-4-6** - Anthropic Balanced\n- **deepseek-chat** - DeepSeek V3 (cost-effective)\n- **deepseek-reasoner** - DeepSeek R1 (thinking model)\n- **fw/kimi-k2-instruct** - Fireworks Kimi K2 (dev/test)\n- **ollama-llama3.2** - Local (no API cost)\n- **ollama-qwen2.5** - Local with excellent reasoning\n\n---\n\n## Architecture Overview\n\nPolicyBot uses a **four-tier hybrid architecture**. LiteLLM handles most chat, embeddings, and transcription. **Anthropic Claude models bypass LiteLLM** via the native `@anthropic-ai/sdk` for reliable tool calling. Specialized services use direct API calls.\n\n```\n┌───────────────────────────────────────────────────────────────────────┐\n│                         PolicyBot Application                         │\n│              (Next.js — openai.ts, tools, translation)                │\n└──┬──────────────────┬──────────────────┬──────────────────┬───────────┘\n   │                  │                  │                  │\n TIER 1            TIER 1b            TIER 2             TIER 3\n LiteLLM Proxy     Anthropic Direct   Direct Provider    Direct Google\n (Chat, Embed,     SDK (@anthropic-   APIs (Non-Chat)    GenAI SDK\n  Transcription)    ai/sdk)                              (Image/TTS)\n   │                  │                  │                  │\n   ▼                  ▼                  ▼                  ▼\n┌────────────────┐ ┌────────────────┐ ┌────────────────┐ ┌────────────────┐\n│  Port 4000     │ │ api.anthropic  │ │                │ │                │\n│                │ │ .com           │ │ Fireworks AI   │ │ Gemini Imagen  │\n│ ┌────────────┐ │ │                │ │  (reranking)   │ │  (image_gen)   │\n│ │ OpenAI     │ │ │ Claude chat +  │ │                │ │  via REST API  │\n│ │ Gemini     │ │ │ tool calling   │ │ Tavily         │ │                │\n│ │ Mistral    │ │ │ via native     │ │  (web_search)  │ │ Gemini TTS     │\n│ │ DeepSeek   │ │ │ streaming      │ │                │ │  (podcast_gen) │\n│ │ Fireworks* │ │ │                │ │ Gemini/Mistral │ │  via @google/  │\n│ │ Ollama*    │ │ │ Models:        │ │  (translation) │ │  genai SDK     │\n│ └────────────┘ │ │ claude-opus-*  │ │                │ │                │\n│                │ │ claude-sonnet-*│ │ OpenAI TTS     │ │ Imagen 4       │\n│ * YAML-only,   │ │ claude-haiku-* │ │  (podcast_gen) │ │  (image_gen)   │\n│   not dynamic  │ │                │ │  hardcoded to  │ │  via @google/  │\n│   sync         │ │ Why: LiteLLM   │ │  api.openai.com│ │  genai SDK     │\n│                │ │ breaks tool    │ │                │ │                │\n│                │ │ call JSON      │ │                │ │                │\n└────────────────┘ └────────────────┘ └────────────────┘ └────────────────┘\n```\n\n### Model Capability Detection\n\nCapabilities are auto-detected via regex patterns in `src/lib/services/model-discovery.ts` and stored in the `enabled_models` table. Admins can override via the LLM Settings UI.\n\n| Capability | DB Column | Detection Function | Used By |\n|------------|-----------|-------------------|---------|\n| Tool calling | `tool_capable` | `isToolCapable(modelId)` | `openai.ts` — whether to send tool definitions |\n| Vision/multimodal | `vision_capable` | `isVisionCapable(modelId)` | `config-capability-checker.ts` — image upload gating |\n| Parallel tool calls | `parallel_tool_capable` | `isParallelToolCapable(modelId)` | `openai.ts` — sequential vs `Promise.allSettled` execution |\n| Thinking/reasoning | `thinking_capable` | `isThinkingCapable(modelId)` | UI — thinking content display |\n\n### Model Registration: Dynamic vs Static\n\nPolicyBot uses **two registration paths** for LiteLLM models:\n\n| Path | Providers | How |\n|------|-----------|-----|\n| **Dynamic** (via `/model/new` API) | OpenAI, Anthropic, Gemini, Mistral, DeepSeek | `litellm-sync.ts` reads `enabled_models` table and registers via API |\n| **Static** (YAML only) | Fireworks, Ollama, Embeddings, Audio | Defined in `litellm_config.yaml`, skipped by sync service |\n| **Direct SDK** (bypasses LiteLLM) | Anthropic Claude | `isClaudeModel()` in `openai.ts` detects `anthropic/` or `claude-` prefix → routes to `@anthropic-ai/sdk` |\n\n> Dynamic models (5 providers) are NOT in the YAML — they are registered at startup by the app. Only Fireworks, Ollama, embedding, and audio models are declared in `litellm_config.yaml`.\n>\n> **Note**: Anthropic models are still registered dynamically with LiteLLM (for embeddings/non-chat use), but **chat completions with tool calling** bypass LiteLLM entirely via the Anthropic SDK. This is controlled by `isClaudeModel()` in `openai.ts`.\n\n---\n\n## Decision Summary\n\n| Component | Decision |\n|-----------|----------|\n| **Approach** | LiteLLM Proxy (Docker Compose) + Anthropic Direct SDK |\n| **Chat Completions** | Route through LiteLLM (except Claude → `@anthropic-ai/sdk` direct) |\n| **Embeddings** | Route through LiteLLM (per-provider config) |\n| **Function Calling** | Gated by `toolCapable` flag from `litellm_config.yaml` via DB sync |\n| **Audio Transcription** | Route through LiteLLM (Whisper, Voxtral) |\n| **Deployment** | Docker Compose |\n\n---\n\n## Project Structure\n\n```\nlitellm-proxy/\n├── docker-compose.yml\n├── litellm_config.yaml\n├── .env\n└── README.md\n```\n\n---\n\n## Step 1: Environment Variables\n\nCreate `.env` file:\n\n```bash\n# ===================\n# LiteLLM Proxy Keys\n# ===================\nLITELLM_MASTER_KEY=sk-litellm-master-change-this\nLITELLM_SALT_KEY=sk-litellm-salt-change-this\n\n# ===================\n# OpenAI\n# ===================\nOPENAI_API_KEY=sk-...\n\n# ===================\n# Azure OpenAI\n# ===================\nAZURE_API_KEY=...\nAZURE_API_BASE=https://your-resource.openai.azure.com/\nAZURE_API_VERSION=2024-02-15-preview\nAZURE_CHAT_DEPLOYMENT=gpt-4-deployment\nAZURE_EMBEDDING_DEPLOYMENT=text-embedding-ada-002\n\n# ===================\n# Mistral\n# ===================\nMISTRAL_API_KEY=...\n\n# ===================\n# Google Gemini\n# ===================\nGEMINI_API_KEY=...\n\n# ===================\n# Anthropic\n# ===================\nANTHROPIC_API_KEY=...\n\n# ===================\n# DeepSeek\n# ===================\nDEEPSEEK_API_KEY=...\n\n# ===================\n# Fireworks AI (dev/test open-source models)\n# ===================\nFIREWORKS_AI_API_KEY=fw-...\n\n# ===================\n# Ollama (Local)\n# ===================\nOLLAMA_API_BASE=http://host.docker.internal:11434\n# Use 'http://localhost:11434' if running outside Docker\n```\n\n---\n\n## Step 2: LiteLLM Configuration\n\n> **Important**: The reference YAML below is a **complete template** listing all models statically. In actual PolicyBot deployment, chat models for OpenAI, Anthropic, Gemini, Mistral, and DeepSeek are **registered dynamically** via `litellm-sync.ts` (see Architecture Overview above). Only Fireworks, Ollama, embedding, and audio models are declared in the YAML. See the actual config at `litellm-proxy/litellm_config.yaml`.\n\nCreate `litellm_config.yaml`:\n\n```yaml\n# =============================================================================\n# LITELLM PROXY CONFIGURATION\n# Multi-provider setup: OpenAI, Azure, Mistral, Gemini, Anthropic, DeepSeek, Fireworks AI, Ollama\n# Updated: March 2026\n# =============================================================================\n\nmodel_list:\n\n  # ===========================================================================\n  # CHAT COMPLETION MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Models - GPT-4.1 Family (Latest - April 2025+)\n  # ---------------------------------------------------------------------------\n  - model_name: openai-gpt41\n    litellm_params:\n      model: gpt-4.1\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-mini\n    litellm_params:\n      model: gpt-4.1-mini\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-nano\n    litellm_params:\n      model: gpt-4.1-nano\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt35\n    litellm_params:\n      model: gpt-3.5-turbo\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Models - GPT-4.1 Family\n  # ---------------------------------------------------------------------------\n  - model_name: azure-gpt41\n    litellm_params:\n      model: azure/gpt-41-deployment  # Your deployment name\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: azure-gpt41-mini\n    litellm_params:\n      model: azure/gpt-41-mini-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Mistral AI Models - Mistral 3 Family (Latest - December 2025)\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-large-3\n    litellm_params:\n      model: mistral/mistral-large-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 256000\n\n  - model_name: mistral-medium-31\n    litellm_params:\n      model: mistral/mistral-medium-2508\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      supports_vision: true\n\n  - model_name: mistral-small-32\n    litellm_params:\n      model: mistral/mistral-small-2506\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ministral-8b\n    litellm_params:\n      model: mistral/ministral-8b-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Ollama Models (Local) - With Full Tool Support\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-llama32\n    litellm_params:\n      model: ollama/llama3.2\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # Fine-tuned for function calling\n\n  - model_name: ollama-llama31-8b\n    litellm_params:\n      model: ollama/llama3.1:8b\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-mistral\n    litellm_params:\n      model: ollama/mistral\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # v0.3+ supports tools\n\n  - model_name: ollama-qwen25\n    litellm_params:\n      model: ollama/qwen2.5\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-phi4\n    litellm_params:\n      model: ollama/phi4\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: false  # Limited support\n\n  # ===========================================================================\n  # EMBEDDING MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: openai-embedding-large\n    litellm_params:\n      model: text-embedding-3-large\n      api_key: os.environ/OPENAI_API_KEY\n\n  - model_name: openai-embedding-small\n    litellm_params:\n      model: text-embedding-3-small\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: azure-embedding\n    litellm_params:\n      model: azure/text-embedding-ada-002\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  - model_name: azure-embedding-3-large\n    litellm_params:\n      model: azure/text-embedding-3-large-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-embedding\n    litellm_params:\n      model: mistral/mistral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: codestral-embed\n    litellm_params:\n      model: mistral/codestral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Ollama Embeddings (Local)\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-embedding\n    litellm_params:\n      model: ollama/nomic-embed-text\n      api_base: os.environ/OLLAMA_API_BASE\n\n  - model_name: ollama-mxbai-embed\n    litellm_params:\n      model: ollama/mxbai-embed-large\n      api_base: os.environ/OLLAMA_API_BASE\n\n  # ===========================================================================\n  # AUDIO MODELS (Speech-to-Text / Transcription)\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Whisper\n  # ---------------------------------------------------------------------------\n  - model_name: openai-whisper\n    litellm_params:\n      model: whisper-1\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure Whisper (if deployed)\n  # ---------------------------------------------------------------------------\n  - model_name: azure-whisper\n    litellm_params:\n      model: azure/whisper-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Voxtral (Released July 2025 - Beats Whisper at half cost)\n  # ---------------------------------------------------------------------------\n  - model_name: voxtral-small\n    litellm_params:\n      model: mistral/voxtral-small-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: voxtral-mini\n    litellm_params:\n      model: mistral/voxtral-mini-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n# =============================================================================\n# LITELLM SETTINGS\n# =============================================================================\n\nlitellm_settings:\n  # Drop unsupported params instead of erroring\n  drop_params: true\n  \n  # Enable detailed logging\n  set_verbose: false\n\n  # Request timeout\n  request_timeout: 120\n\n# =============================================================================\n# GENERAL SETTINGS\n# =============================================================================\n\ngeneral_settings:\n  # Master key for proxy authentication\n  master_key: os.environ/LITELLM_MASTER_KEY\n```\n\n---\n\n## Step 3: Docker Compose\n\n> **Note**: In the actual PolicyBot deployment, LiteLLM runs as part of the full stack in `docker-compose.local.yml` alongside Postgres, Qdrant, and Redis. The standalone example below is for reference.\n\nCreate `docker-compose.yml`:\n\n```yaml\nversion: \"3.9\"\n\nservices:\n  litellm:\n    image: ghcr.io/berriai/litellm:v1.82.3-stable.patch.2\n    container_name: litellm-proxy\n    ports:\n      - \"4000:4000\"\n    volumes:\n      - ./litellm_config.yaml:/app/config.yaml\n    env_file:\n      - .env\n    environment:\n      - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY}\n      - LITELLM_SALT_KEY=${LITELLM_SALT_KEY}\n    command: --config /app/config.yaml --port 4000 --detailed_debug\n    restart: unless-stopped\n    extra_hosts:\n      - \"host.docker.internal:host-gateway\"\n    healthcheck:\n      test: [\"CMD\", \"curl\", \"-f\", \"http://localhost:4000/health\"]\n      interval: 30s\n      timeout: 10s\n      retries: 3\n\n  # Optional: Ollama service (if not running separately)\n  # ollama:\n  #   image: ollama/ollama:latest\n  #   container_name: ollama\n  #   ports:\n  #     - \"11434:11434\"\n  #   volumes:\n  #     - ollama_data:/root/.ollama\n  #   restart: unless-stopped\n\n# volumes:\n#   ollama_data:\n```\n\n---\n\n## Step 4: Application Code Changes\n\n### Minimal Change Required\n\nPolicyBot is a Next.js (TypeScript) application. The proxy switch is controlled entirely by environment variables — no code changes needed:\n\n**Environment (`.env`):**\n```bash\n# Set this to route all chat/embeddings/transcription through LiteLLM\nOPENAI_BASE_URL=http://litellm:4000/v1\nLITELLM_MASTER_KEY=sk-litellm-master-change-this\n```\n\n**How it works (`src/lib/openai.ts`):**\n```typescript\n// When OPENAI_BASE_URL is set → uses LiteLLM proxy with LITELLM_MASTER_KEY\n// When OPENAI_BASE_URL is unset → calls OpenAI directly with OPENAI_API_KEY\nconst apiKey = process.env.OPENAI_BASE_URL\n  ? (process.env.LITELLM_MASTER_KEY || await getApiKey('openai'))\n  : await getApiKey('openai');\n\nopenaiClient = new OpenAI({\n  apiKey: apiKey || undefined,\n  baseURL: process.env.OPENAI_BASE_URL || undefined,\n});\n```\n\nFor non-Claude models, that's it — the OpenAI SDK handles the routing transparently.\n\n**Claude models** are automatically detected and routed to the Anthropic SDK:\n```typescript\n// Automatic — no env var needed, just ANTHROPIC_API_KEY in DB or env\n// isClaudeModel() detects 'anthropic/' or 'claude-' prefix\n// Uses @anthropic-ai/sdk directly for chat + tool calling\n// Embeddings/transcription still route through LiteLLM\n```\n\n---\n\n## Step 5: Function Calling Handling\n\nSince your `generateResponseWithTools()` uses function calling, here's how to handle provider capabilities:\n\n```python\n# Provider capability mapping (Updated December 2025)\n\n# Full tool/function calling support\n# Matches Policy Bot config/defaults.json toolCapable list\nTOOL_CAPABLE_MODELS = [\n    # OpenAI\n    \"gpt-4.1\", \"gpt-4.1-mini\", \"gpt-4.1-nano\", \"gpt-3.5-turbo\",\n    # Mistral\n    \"mistral-large-3\", \"mistral-medium-3.1\", \"mistral-small-3.2\",\n    \"ministral-8b\", \"ministral-3b\",\n    # Ollama (Now with full tool support!)\n    \"ollama-llama3.2\", \"ollama-llama3.1\", \"ollama-mistral\", \"ollama-qwen2.5\",\n]\n\n# Limited or no tool support (show warning)\nTOOL_LIMITED_MODELS = [\n    \"ollama-phi4\",\n]\n\ndef generate_response_with_tools(model: str, messages: list, tools: list):\n    \"\"\"\n    Generate response with function calling support.\n    Warns and disables tools for unsupported providers.\n    \"\"\"\n    \n    if model in TOOL_LIMITED_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' has limited function calling support.\")\n        print(f\"   Tools will be disabled. Consider using: {', '.join(TOOL_CAPABLE_MODELS[:5])}\")\n        # Call without tools\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages\n        )\n    elif model not in TOOL_CAPABLE_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' tool support unknown. Attempting with tools...\")\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    else:\n        # Full tool support\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    \n    return response\n```\n\n> **Good News (2025 Update)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n---\n\n## Step 6: Embeddings Usage\n\n```python\n# Embeddings via LiteLLM Proxy\n\n# OpenAI embeddings (recommended for RAG quality)\nresponse = client.embeddings.create(\n    model=\"openai-embedding-large\",\n    input=\"Your text here\"\n)\n\n# Azure embeddings\nresponse = client.embeddings.create(\n    model=\"azure-embedding\",\n    input=\"Your text here\"\n)\n\n# Mistral embeddings\nresponse = client.embeddings.create(\n    model=\"mistral-embedding\",\n    input=\"Your text here\"\n)\n\n# Ollama embeddings (local)\nresponse = client.embeddings.create(\n    model=\"ollama-embedding\",\n    input=\"Your text here\"\n)\n\nembedding = response.data[0].embedding\n```\n\n### Important: RAG Consistency\n\n> ⚠️ **Warning**: Don't mix embedding providers for the same vector store. If your RAG was built with `text-embedding-3-large`, query with the same model.\n\n```python\n# Configuration approach\nEMBEDDING_CONFIG = {\n    \"openai\": \"openai-embedding-large\",\n    \"azure\": \"azure-embedding\",\n    \"mistral\": \"mistral-embedding\",\n    \"ollama\": \"ollama-embedding\"\n}\n\n# Use consistently\ncurrent_provider = \"openai\"\nembedding_model = EMBEDDING_CONFIG[current_provider]\n```\n\n---\n\n## Step 7: Audio Transcription (Whisper)\n\n```python\n# Audio transcription via LiteLLM Proxy\n\naudio_file = open(\"audio.mp3\", \"rb\")\n\nresponse = client.audio.transcriptions.create(\n    model=\"openai-whisper\",\n    file=audio_file\n)\n\nprint(response.text)\n```\n\n---\n\n## Deployment Commands\n\n```bash\n# Start the proxy\ndocker compose up -d\n\n# View logs\ndocker compose logs -f litellm\n\n# Stop\ndocker compose down\n\n# Restart after config changes\ndocker compose restart litellm\n```\n\n---\n\n## Testing the Setup\n\n### 1. Health Check\n\n```bash\ncurl http://localhost:4000/health\n```\n\n### 2. List Available Models\n\n```bash\ncurl http://localhost:4000/models \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\"\n```\n\n### 3. Test Chat Completion\n\n```bash\ncurl -X POST http://localhost:4000/chat/completions \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-gpt35\",\n    \"messages\": [{\"role\": \"user\", \"content\": \"Hello, test!\"}]\n  }'\n```\n\n### 4. Test Embeddings\n\n```bash\ncurl -X POST http://localhost:4000/embeddings \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-embedding-large\",\n    \"input\": \"Test embedding\"\n  }'\n```\n\n---\n\n## Model Quick Reference\n\n*Updated: March 2026*\n\n### OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `openai-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Latest flagship, best for coding |\n| `openai-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Faster, 83% cheaper than GPT-4o |\n| `openai-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Fastest, cheapest |\n| `openai-gpt35` | `gpt-3.5-turbo` | ✅ | - | ✅ | - | 16K | Legacy, cost-effective |\n| `openai-embedding-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality embeddings |\n| `openai-embedding-small` | `text-embedding-3-small` | - | ✅ | - | - | 8K | Faster, cheaper |\n| `openai-whisper` | `whisper-1` | - | - | - | ✅ | - | $0.006/min transcription |\n\n### Azure OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `azure-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-embedding` | `text-embedding-ada-002` | - | ✅ | - | - | 8K | Standard embedding |\n| `azure-embedding-3-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality |\n| `azure-whisper` | `whisper` | - | - | - | ✅ | - | Speech-to-text |\n\n### Mistral AI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `mistral-large-3` | `mistral-large-latest` | ✅ | - | ✅ | - | 256K | Flagship MoE (675B total params) |\n| `mistral-medium-31` | `mistral-medium-2508` | ✅ | - | ✅ | - | 128K | Frontier multimodal |\n| `mistral-small-32` | `mistral-small-2506` | ✅ | - | ✅ | - | 128K | Fast, efficient |\n| `ministral-14b` | `ministral-3-14b` | ✅ | - | ✅ | - | 128K | Best small model |\n| `ministral-8b` | `ministral-3-8b` | ✅ | - | ✅ | - | 128K | Edge deployment |\n| `ministral-3b` | `ministral-3-3b` | ✅ | - | ✅ | - | 128K | Ultra-light |\n| `mistral-embedding` | `mistral-embed` | - | ✅ | - | - | 8K | Text embeddings |\n| `codestral-embed` | `codestral-embed` | - | ✅ | - | - | - | Code embeddings |\n| `voxtral-small` | `voxtral-small-latest` | - | - | - | ✅ | 32K | 24B, best accuracy, $0.001/min |\n| `voxtral-mini` | `voxtral-mini-latest` | - | - | - | ✅ | 32K | 3B, edge/local deployment |\n| `voxtral-transcribe` | `voxtral-mini-latest` (via /audio/transcriptions) | - | - | - | ✅ | - | Transcription-only, beats Whisper |\n\n> **Note**: Voxtral (released July 2025) outperforms OpenAI Whisper at half the cost ($0.001/min vs $0.006/min). Supports up to 30 min audio for transcription, 40 min for understanding.\n\n### Google Gemini Models\n\n| Model Name | API Model ID | Chat | Tools | Context | Notes |\n|------------|--------------|------|-------|---------|-------|\n| `gemini-2.5-pro` | `gemini/gemini-2.5-pro` | ✅ | ✅ | 1M | Flagship reasoning model |\n| `gemini-2.5-flash` | `gemini/gemini-2.5-flash` | ✅ | ✅ | 1M | Fast, balanced performance |\n| `gemini-2.5-flash-lite` | `gemini/gemini-2.5-flash-lite` | ✅ | ✅ | 1M | Lowest cost option |\n\n**Pricing (per 1M tokens):**\n- Gemini 2.5 Pro: $1.25 input / $10.00 output (≤200k context)\n- Gemini 2.5 Flash: $0.30 input / $2.50 output\n- Gemini 2.5 Flash-Lite: $0.10 input / $0.40 output\n\n**Features:**\n- All models support function calling\n- Built-in \"thinking\" capabilities with controllable budgets\n- Free tier available for development/testing\n- Native multimodal support (text, image, audio, video)\n\n### Ollama Local Models\n\n| Model Name | Ollama Model | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `ollama-llama32` | `llama3.2` | ✅ | - | ✅ | - | 128K | **Full tool support** (fine-tuned for function calling) |\n| `ollama-llama31-8b` | `llama3.1:8b` | ✅ | - | ✅ | - | 128K | **Full tool support**, best overall |\n| `ollama-llama31-70b` | `llama3.1:70b` | ✅ | - | ✅ | - | 128K | **Full tool support**, highest accuracy |\n| `ollama-mistral` | `mistral` | ✅ | - | ✅ | - | 32K | **Supports tools** (v0.3+) |\n| `ollama-mixtral` | `mixtral:8x7b` | ✅ | - | ✅ | - | 32K | MoE, good multi-domain |\n| `ollama-qwen25` | `qwen2.5` | ✅ | - | ✅ | - | 128K | Strong tool support |\n| `ollama-phi4` | `phi4` | ✅ | - | ⚠️ | - | 16K | Limited tool support |\n| `ollama-embedding` | `nomic-embed-text` | - | ✅ | - | - | 8K | Best local embedding |\n| `ollama-mxbai-embed` | `mxbai-embed-large` | - | ✅ | - | - | 512 | Alternative embedding |\n\n> **Tool Calling Update (2025)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), Mixtral, and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ | Fully Supported |\n| ⚠️ | Limited/Experimental (may need prompt adjustments) |\n| - | Not applicable |\n\n### Recommendations by Use Case\n\n| Use Case | Recommended Model | Why |\n|----------|-------------------|-----|\n| **Production Chat** | `gpt-4.1` or `gemini-2.5-pro` | Best instruction following, 1M context |\n| **Balanced (Default)** | `gpt-4.1-mini` or `gemini-2.5-flash` | Good quality at lower cost (Policy Bot default) |\n| **Budget Chat** | `gpt-4.1-nano` or `gemini-2.5-flash-lite` | Cost-effective for simple queries |\n| **Deep Reasoning** | `gemini-2.5-pro` | Built-in thinking capabilities |\n| **Local/Offline Chat** | `ollama-llama3.2` or `ollama-qwen2.5` | Full tool support, runs locally |\n| **RAG Embeddings** | `text-embedding-3-large` | Best quality for retrieval (Policy Bot default) |\n| **Local Embeddings** | `ollama-embedding` (nomic) | Good quality, no API cost |\n| **Audio Transcription** | `voxtral-transcribe` | Beats Whisper, half the cost |\n| **Budget Transcription** | `whisper-1` | Well-established, $0.006/min |\n| **Function Calling** | `gpt-4.1-mini`, `gemini-2.5-flash`, `mistral-large-3` | Reliable tool use |\n\n---\n\n## Switching Providers\n\nTo switch your entire application from OpenAI to Ollama:\n\n1. **No code changes needed**\n2. Update model name in your application config:\n\n```python\n# Before (OpenAI)\nMODEL = \"openai-gpt41\"\nEMBEDDING_MODEL = \"openai-embedding-large\"\nAUDIO_MODEL = \"openai-whisper\"\n\n# After (Ollama + Mistral Voxtral for audio)\nMODEL = \"ollama-llama32\"\nEMBEDDING_MODEL = \"ollama-embedding\"\nAUDIO_MODEL = \"voxtral-mini\"  # Or keep openai-whisper\n\n# Or fully Mistral\nMODEL = \"mistral-large-3\"\nEMBEDDING_MODEL = \"mistral-embedding\"\nAUDIO_MODEL = \"voxtral-small\"\n```\n\n---\n\n## Troubleshooting\n\n| Issue | Solution |\n|-------|----------|\n| Connection refused | Check if Docker is running, verify port 4000 |\n| Ollama not reachable | Use `host.docker.internal` in Docker, check Ollama is running |\n| API key errors | Verify `.env` file is loaded, check key format |\n| Model not found | Run `/models` endpoint to list available models |\n| Timeout errors | Increase `request_timeout` in config |\n\n---\n\n## Known Limitations\n\n| # | Limitation | Impact | Status | File |\n|---|-----------|--------|--------|------|\n| 1 | ~~Static YAML cache never invalidated~~ | ~~Runtime model additions invisible until restart~~ | **FIXED** — `clearLiteLLMCache()` now called after `syncAllModelsToLiteLLM()` completes | `litellm-sync.ts` |\n| 2 | ~~Claude tool-calling streaming mismatch~~ | ~~Anthropic models routed through LiteLLM fail tool calling~~ | **FIXED** — Claude models now bypass LiteLLM via `@anthropic-ai/sdk` direct. `isClaudeModel()` routes to `streamAnthropicCompletion()` which receives pre-parsed tool inputs. | `openai.ts` |\n| 3 | **No circuit breaker for chat completions** | If LiteLLM proxy is unreachable, non-Claude chat completions fail. Claude models are unaffected (direct SDK). Embeddings have a fallback model path, but chat/transcription do not. | Open | `openai.ts` |\n| 4 | **Prefix-based provider detection only** | `getProviderFromModelPath()` defaults to `openai` for unrecognized prefixes. Custom/vLLM models without a prefix will be misidentified. No `deepseek/` prefix exists — DeepSeek models are synced dynamically with prefix by `litellm-sync.ts`. | Open | `litellm-validator.ts:282-294` |\n| 5 | **No TTS or image generation via proxy** | `podcast_gen` and `image_gen` must bypass LiteLLM entirely | By design | `podcast-gen.ts`, `image-gen/` |\n\n---\n\n## Resources\n\n- **LiteLLM Docs**: https://docs.litellm.ai/docs/\n- **Supported Providers**: https://docs.litellm.ai/docs/providers\n- **GitHub**: https://github.com/BerriAI/litellm\n- **Ollama Models**: https://ollama.ai/library\n\n---\n\n*Last updated: March 2026 — Added Anthropic Direct SDK (Tier 1b) for Claude chat + tool calling, bypassing LiteLLM. Updated architecture to four-tier hybrid. Fixed Known Limitations #1 (cache invalidation) and #2 (Claude tool-calling). Added `isClaudeModel()` detection logic documentation.*"}, "c06e422e3283ea000f3811be3f03163e7aab4fe9": {"t": 1785754361.9347734, "n": 38481, "h": "305749f65505ca47", "c": "# LiteLLM Implementation Guide\n\n> Customized for multi-provider LLM abstraction with proxy approach, embeddings routing, function calling support, and audio transcription.\n\n---\n\n## LLM Service Routing Overview\n\nPolicy Bot uses a **hybrid architecture**: most chat services route through LiteLLM proxy for unified model management, while **Anthropic Claude models bypass LiteLLM entirely** using the `@anthropic-ai/sdk` for direct API access (eliminating tool-calling JSON assembly issues), and specialized services (images, audio, document processing) call provider APIs directly or use local parsers.\n\n### Service Routing Table\n\n| Service | Policy Bot Feature | Routes Through | Provider(s) | Notes |\n|---------|-------------------|----------------|-------------|-------|\n| **Chat Completions** | Main chat, RAG responses | ✅ LiteLLM / ⚡ Anthropic Direct | OpenAI, Gemini, Mistral, DeepSeek, Fireworks AI, Ollama via LiteLLM; **Anthropic Claude via direct SDK** | Claude uses `@anthropic-ai/sdk` for reliable tool calling |\n| **Embeddings** | Document indexing, search | ✅ LiteLLM | OpenAI, Mistral, Gemini, Ollama, Fireworks | `text-embedding-3-large` default |\n| **Diagram Generation** | `diagram_gen` tool | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Generates Mermaid syntax |\n| **Summarization** | Message compression | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic, DeepSeek | Long conversation handling |\n| **Memory Extraction** | User fact storage | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Per-category memory |\n| **Prompt Optimization** | Query refinement | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Pre-RAG processing |\n| **Compliance Checks** | HITL clarification | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Skill compliance |\n| **Translation** | Multi-language support | ✅ LiteLLM / ❌ Direct | OpenAI (proxy), Gemini/Mistral (direct) | Provider-dependent |\n| **Audio Transcription** | Voice input | ✅ LiteLLM | OpenAI Whisper, Mistral Voxtral | `transcribeAudio()` uses `getOpenAI()` client |\n| **Image Generation** | `image_gen` tool | ❌ Direct | Gemini Nano Banana, Imagen 4 | Specialized APIs |\n| **Podcast Generation** | `podcast_gen` tool | ❌ Direct | OpenAI TTS, Gemini TTS | Multi-voice audio synthesis |\n| **Document Processing** | Document text extraction | ❌ Direct | mammoth/exceljs/officeparser (local), Mistral OCR, Azure DI, pdf-parse | Tiered fallback: local parsers first, then API providers |\n| **Reranking** | Search result scoring | ❌ Direct | Fireworks AI | Direct HTTP to `api.fireworks.ai/inference/v1/rerank` |\n| **PPTX Generation** | `pptx_gen` tool | N/A | None | Template-based (no LLM) |\n| **XLSX Generation** | `xlsx_gen` tool | N/A | None | ExcelJS-based (no LLM) |\n| **Chart Generation** | `chart_gen` tool | N/A | None | Client-side rendering |\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ LiteLLM | Routes through LiteLLM proxy (port 4000) |\n| ⚡ Anthropic Direct | Calls Anthropic API directly via `@anthropic-ai/sdk` (bypasses LiteLLM) |\n| ❌ Direct | Calls provider API directly (bypasses LiteLLM) |\n| N/A | No LLM calls required |\n\n### Why Some Services Bypass LiteLLM\n\n| Service | Reason |\n|---------|--------|\n| **Anthropic Claude** | LiteLLM's Anthropic→OpenAI streaming translation produces malformed tool-calling JSON. Direct `@anthropic-ai/sdk` provides native, reliable tool call parsing |\n| **Image Generation** | Gemini and Imagen APIs not supported by LiteLLM proxy |\n| **Podcast TTS** | OpenAI TTS and Gemini TTS are audio generation APIs, not chat completions |\n| **OCR** | Mistral OCR is a specialized document API with unique request format |\n| **Reranking** | Fireworks reranker is a specialized search API (`/inference/v1/rerank`), not a chat model |\n\n> **Note**: Audio transcription (Whisper, Voxtral) routes **through LiteLLM** — `transcribeAudio()` in `openai.ts` uses the same `getOpenAI()` client as chat completions. Both `whisper-1` and `voxtral-mini` are registered in `litellm_config.yaml`.\n\n### Proxy Detection Logic\n\nThe actual detection in `openai.ts` checks for the `OPENAI_BASE_URL` environment variable:\n\n```typescript\n// src/lib/openai.ts — getOpenAI()\nasync function getOpenAI(): Promise<OpenAI> {\n  if (!openaiClient) {\n    // When OPENAI_BASE_URL is set, route through LiteLLM proxy\n    const apiKey = process.env.OPENAI_BASE_URL\n      ? (process.env.LITELLM_MASTER_KEY || await getApiKey('openai'))\n      : await getApiKey('openai');\n\n    openaiClient = new OpenAI({\n      apiKey: apiKey || undefined,\n      baseURL: process.env.OPENAI_BASE_URL || undefined,\n      timeout: 300 * 1000,\n    });\n  }\n  return openaiClient;\n}\n```\n\nWhen `OPENAI_BASE_URL` points to the LiteLLM container (e.g. `http://litellm:4000/v1`), embeddings, transcription, and non-Claude chat completions are routed through a single OpenAI SDK client.\n\n### Claude Direct SDK Detection\n\nFor Anthropic Claude models, `generateResponseWithTools()` bypasses LiteLLM entirely:\n\n```typescript\n// src/lib/openai.ts — isClaudeModel() + routing\nfunction isClaudeModel(model: string): boolean {\n  return model.startsWith('anthropic/') || model.startsWith('claude-');\n}\n\n// In generateResponseWithTools():\nconst useAnthropicDirect = isClaudeModel(effectiveModel);\nconst openai = useAnthropicDirect ? null : await getOpenAI();          // LiteLLM path\nconst anthropicClient = useAnthropicDirect ? await getAnthropicClient() : null;  // Direct path\n```\n\n**Why**: LiteLLM translates Anthropic's native streaming into OpenAI-compatible `delta.tool_calls` format, but the tool call JSON assembly is unreliable — producing malformed arguments that fail `executeTool()`. The direct `@anthropic-ai/sdk` receives pre-parsed tool inputs via `stream.finalMessage()`, eliminating this class of errors.\n\n---\n\n## Policy Bot Integration\n\nThis guide is configured for the Policy Bot RAG application. Default model preset:\n\n| Setting | Value |\n|---------|-------|\n| Default Model | `gpt-4.1-mini` |\n| Temperature | `0.2` |\n| Max Tokens | `2000` |\n| Embedding Model | `text-embedding-3-large` |\n| Embedding Dimensions | `3072` |\n\nAvailable model presets in Policy Bot (via `config/defaults.json`):\n- **gpt-4.1** - High Performance (1M context)\n- **gpt-4.1-mini** - Balanced (default)\n- **gpt-4.1-nano** - Cost-Effective\n- **mistral-large-3** - Mistral Flagship (256K context)\n- **mistral-small-3.2** - Mistral Cost-Effective\n- **ministral-8b** - Mistral Ultra Cost-Effective\n- **gemini-2.5-pro** - Google Flagship Reasoning (1M context)\n- **gemini-2.5-flash** - Google Balanced (1M context)\n- **gemini-2.5-flash-lite** - Google Cost-Effective (1M context)\n- **claude-opus-4-6** - Anthropic Flagship\n- **claude-sonnet-4-6** - Anthropic Balanced\n- **deepseek-chat** - DeepSeek V3 (cost-effective)\n- **deepseek-reasoner** - DeepSeek R1 (thinking model)\n- **fw/kimi-k2-instruct** - Fireworks Kimi K2 (dev/test)\n- **ollama-llama3.2** - Local (no API cost)\n- **ollama-qwen2.5** - Local with excellent reasoning\n\n---\n\n## Architecture Overview\n\nPolicyBot uses a **four-tier hybrid architecture**. LiteLLM handles most chat, embeddings, and transcription. **Anthropic Claude models bypass LiteLLM** via the native `@anthropic-ai/sdk` for reliable tool calling. Specialized services use direct API calls.\n\n```\n┌───────────────────────────────────────────────────────────────────────┐\n│                         PolicyBot Application                         │\n│              (Next.js — openai.ts, tools, translation)                │\n└──┬──────────────────┬──────────────────┬──────────────────┬───────────┘\n   │                  │                  │                  │\n TIER 1            TIER 1b            TIER 2             TIER 3\n LiteLLM Proxy     Anthropic Direct   Direct Provider    Direct Google\n (Chat, Embed,     SDK (@anthropic-   APIs (Non-Chat)    GenAI SDK\n  Transcription)    ai/sdk)                              (Image/TTS)\n   │                  │                  │                  │\n   ▼                  ▼                  ▼                  ▼\n┌────────────────┐ ┌────────────────┐ ┌────────────────┐ ┌────────────────┐\n│  Port 4000     │ │ api.anthropic  │ │                │ │                │\n│                │ │ .com           │ │ Fireworks AI   │ │ Gemini Imagen  │\n│ ┌────────────┐ │ │                │ │  (reranking)   │ │  (image_gen)   │\n│ │ OpenAI     │ │ │ Claude chat +  │ │                │ │  via REST API  │\n│ │ Gemini     │ │ │ tool calling   │ │ Tavily         │ │                │\n│ │ Mistral    │ │ │ via native     │ │  (web_search)  │ │ Gemini TTS     │\n│ │ DeepSeek   │ │ │ streaming      │ │                │ │  (podcast_gen) │\n│ │ Fireworks* │ │ │                │ │ Gemini/Mistral │ │  via @google/  │\n│ │ Ollama*    │ │ │ Models:        │ │  (translation) │ │  genai SDK     │\n│ └────────────┘ │ │ claude-opus-*  │ │                │ │                │\n│                │ │ claude-sonnet-*│ │ OpenAI TTS     │ │ Imagen 4       │\n│ * YAML-only,   │ │ claude-haiku-* │ │  (podcast_gen) │ │  (image_gen)   │\n│   not dynamic  │ │                │ │  hardcoded to  │ │  via @google/  │\n│   sync         │ │ Why: LiteLLM   │ │  api.openai.com│ │  genai SDK     │\n│                │ │ breaks tool    │ │                │ │                │\n│                │ │ call JSON      │ │                │ │                │\n└────────────────┘ └────────────────┘ └────────────────┘ └────────────────┘\n```\n\n### Model Capability Detection\n\nCapabilities are auto-detected via regex patterns in `src/lib/services/model-discovery.ts` and stored in the `enabled_models` table. Admins can override via the LLM Settings UI.\n\n| Capability | DB Column | Detection Function | Used By |\n|------------|-----------|-------------------|---------|\n| Tool calling | `tool_capable` | `isToolCapable(modelId)` | `openai.ts` — whether to send tool definitions |\n| Vision/multimodal | `vision_capable` | `isVisionCapable(modelId)` | `config-capability-checker.ts` — image upload gating |\n| Parallel tool calls | `parallel_tool_capable` | `isParallelToolCapable(modelId)` | `openai.ts` — sequential vs `Promise.allSettled` execution |\n| Thinking/reasoning | `thinking_capable` | `isThinkingCapable(modelId)` | UI — thinking content display |\n\n### Model Registration: Dynamic vs Static\n\nPolicyBot uses **two registration paths** for LiteLLM models:\n\n| Path | Providers | How |\n|------|-----------|-----|\n| **Dynamic** (via `/model/new` API) | OpenAI, Anthropic, Gemini, Mistral, DeepSeek | `litellm-sync.ts` reads `enabled_models` table and registers via API |\n| **Static** (YAML only) | Fireworks, Ollama, Embeddings, Audio | Defined in `litellm_config.yaml`, skipped by sync service |\n| **Direct SDK** (bypasses LiteLLM) | Anthropic Claude | `isClaudeModel()` in `openai.ts` detects `anthropic/` or `claude-` prefix → routes to `@anthropic-ai/sdk` |\n\n> Dynamic models (5 providers) are NOT in the YAML — they are registered at startup by the app. Only Fireworks, Ollama, embedding, and audio models are declared in `litellm_config.yaml`.\n>\n> **Note**: Anthropic models are still registered dynamically with LiteLLM (for embeddings/non-chat use), but **chat completions with tool calling** bypass LiteLLM entirely via the Anthropic SDK. This is controlled by `isClaudeModel()` in `openai.ts`.\n\n---\n\n## Decision Summary\n\n| Component | Decision |\n|-----------|----------|\n| **Approach** | LiteLLM Proxy (Docker Compose) + Anthropic Direct SDK |\n| **Chat Completions** | Route through LiteLLM (except Claude → `@anthropic-ai/sdk` direct) |\n| **Embeddings** | Route through LiteLLM (per-provider config) |\n| **Function Calling** | Gated by `toolCapable` flag from `litellm_config.yaml` via DB sync |\n| **Audio Transcription** | Route through LiteLLM (Whisper, Voxtral) |\n| **Deployment** | Docker Compose |\n\n---\n\n## Project Structure\n\n```\nlitellm-proxy/\n├── docker-compose.yml\n├── litellm_config.yaml\n├── .env\n└── README.md\n```\n\n---\n\n## Step 1: Environment Variables\n\nCreate `.env` file:\n\n```bash\n# ===================\n# LiteLLM Proxy Keys\n# ===================\nLITELLM_MASTER_KEY=sk-litellm-master-change-this\nLITELLM_SALT_KEY=sk-litellm-salt-change-this\n\n# ===================\n# OpenAI\n# ===================\nOPENAI_API_KEY=sk-...\n\n# ===================\n# Azure OpenAI\n# ===================\nAZURE_API_KEY=...\nAZURE_API_BASE=https://your-resource.openai.azure.com/\nAZURE_API_VERSION=2024-02-15-preview\nAZURE_CHAT_DEPLOYMENT=gpt-4-deployment\nAZURE_EMBEDDING_DEPLOYMENT=text-embedding-ada-002\n\n# ===================\n# Mistral\n# ===================\nMISTRAL_API_KEY=...\n\n# ===================\n# Google Gemini\n# ===================\nGEMINI_API_KEY=...\n\n# ===================\n# Anthropic\n# ===================\nANTHROPIC_API_KEY=...\n\n# ===================\n# DeepSeek\n# ===================\nDEEPSEEK_API_KEY=...\n\n# ===================\n# Fireworks AI (dev/test open-source models)\n# ===================\nFIREWORKS_AI_API_KEY=fw-...\n\n# ===================\n# Ollama (Local)\n# ===================\nOLLAMA_API_BASE=http://host.docker.internal:11434\n# Use 'http://localhost:11434' if running outside Docker\n```\n\n---\n\n## Step 2: LiteLLM Configuration\n\n> **Important**: The reference YAML below is a **complete template** listing all models statically. In actual PolicyBot deployment, chat models for OpenAI, Anthropic, Gemini, Mistral, and DeepSeek are **registered dynamically** via `litellm-sync.ts` (see Architecture Overview above). Only Fireworks, Ollama, embedding, and audio models are declared in the YAML. See the actual config at `litellm-proxy/litellm_config.yaml`.\n\nCreate `litellm_config.yaml`:\n\n```yaml\n# =============================================================================\n# LITELLM PROXY CONFIGURATION\n# Multi-provider setup: OpenAI, Azure, Mistral, Gemini, Anthropic, DeepSeek, Fireworks AI, Ollama\n# Updated: March 2026\n# =============================================================================\n\nmodel_list:\n\n  # ===========================================================================\n  # CHAT COMPLETION MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Models - GPT-4.1 Family (Latest - April 2025+)\n  # ---------------------------------------------------------------------------\n  - model_name: openai-gpt41\n    litellm_params:\n      model: gpt-4.1\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-mini\n    litellm_params:\n      model: gpt-4.1-mini\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-nano\n    litellm_params:\n      model: gpt-4.1-nano\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt35\n    litellm_params:\n      model: gpt-3.5-turbo\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Models - GPT-4.1 Family\n  # ---------------------------------------------------------------------------\n  - model_name: azure-gpt41\n    litellm_params:\n      model: azure/gpt-41-deployment  # Your deployment name\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: azure-gpt41-mini\n    litellm_params:\n      model: azure/gpt-41-mini-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Mistral AI Models - Mistral 3 Family (Latest - December 2025)\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-large-3\n    litellm_params:\n      model: mistral/mistral-large-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 256000\n\n  - model_name: mistral-medium-31\n    litellm_params:\n      model: mistral/mistral-medium-2508\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      supports_vision: true\n\n  - model_name: mistral-small-32\n    litellm_params:\n      model: mistral/mistral-small-2506\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ministral-8b\n    litellm_params:\n      model: mistral/ministral-8b-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Ollama Models (Local) - With Full Tool Support\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-llama32\n    litellm_params:\n      model: ollama/llama3.2\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # Fine-tuned for function calling\n\n  - model_name: ollama-llama31-8b\n    litellm_params:\n      model: ollama/llama3.1:8b\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-mistral\n    litellm_params:\n      model: ollama/mistral\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # v0.3+ supports tools\n\n  - model_name: ollama-qwen25\n    litellm_params:\n      model: ollama/qwen2.5\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-phi4\n    litellm_params:\n      model: ollama/phi4\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: false  # Limited support\n\n  # ===========================================================================\n  # EMBEDDING MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: openai-embedding-large\n    litellm_params:\n      model: text-embedding-3-large\n      api_key: os.environ/OPENAI_API_KEY\n\n  - model_name: openai-embedding-small\n    litellm_params:\n      model: text-embedding-3-small\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: azure-embedding\n    litellm_params:\n      model: azure/text-embedding-ada-002\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  - model_name: azure-embedding-3-large\n    litellm_params:\n      model: azure/text-embedding-3-large-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-embedding\n    litellm_params:\n      model: mistral/mistral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: codestral-embed\n    litellm_params:\n      model: mistral/codestral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Ollama Embeddings (Local)\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-embedding\n    litellm_params:\n      model: ollama/nomic-embed-text\n      api_base: os.environ/OLLAMA_API_BASE\n\n  - model_name: ollama-mxbai-embed\n    litellm_params:\n      model: ollama/mxbai-embed-large\n      api_base: os.environ/OLLAMA_API_BASE\n\n  # ===========================================================================\n  # AUDIO MODELS (Speech-to-Text / Transcription)\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Whisper\n  # ---------------------------------------------------------------------------\n  - model_name: openai-whisper\n    litellm_params:\n      model: whisper-1\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure Whisper (if deployed)\n  # ---------------------------------------------------------------------------\n  - model_name: azure-whisper\n    litellm_params:\n      model: azure/whisper-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Voxtral (Released July 2025 - Beats Whisper at half cost)\n  # ---------------------------------------------------------------------------\n  - model_name: voxtral-small\n    litellm_params:\n      model: mistral/voxtral-small-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: voxtral-mini\n    litellm_params:\n      model: mistral/voxtral-mini-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n# =============================================================================\n# LITELLM SETTINGS\n# =============================================================================\n\nlitellm_settings:\n  # Drop unsupported params instead of erroring\n  drop_params: true\n  \n  # Enable detailed logging\n  set_verbose: false\n\n  # Request timeout\n  request_timeout: 120\n\n# =============================================================================\n# GENERAL SETTINGS\n# =============================================================================\n\ngeneral_settings:\n  # Master key for proxy authentication\n  master_key: os.environ/LITELLM_MASTER_KEY\n```\n\n---\n\n## Step 3: Docker Compose\n\n> **Note**: In the actual PolicyBot deployment, LiteLLM runs as part of the full stack in `docker-compose.local.yml` alongside Postgres, Qdrant, and Redis. The standalone example below is for reference.\n\nCreate `docker-compose.yml`:\n\n```yaml\nversion: \"3.9\"\n\nservices:\n  litellm:\n    image: ghcr.io/berriai/litellm:v1.82.3-stable.patch.2\n    container_name: litellm-proxy\n    ports:\n      - \"4000:4000\"\n    volumes:\n      - ./litellm_config.yaml:/app/config.yaml\n    env_file:\n      - .env\n    environment:\n      - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY}\n      - LITELLM_SALT_KEY=${LITELLM_SALT_KEY}\n    command: --config /app/config.yaml --port 4000 --detailed_debug\n    restart: unless-stopped\n    extra_hosts:\n      - \"host.docker.internal:host-gateway\"\n    healthcheck:\n      test: [\"CMD\", \"curl\", \"-f\", \"http://localhost:4000/health\"]\n      interval: 30s\n      timeout: 10s\n      retries: 3\n\n  # Optional: Ollama service (if not running separately)\n  # ollama:\n  #   image: ollama/ollama:latest\n  #   container_name: ollama\n  #   ports:\n  #     - \"11434:11434\"\n  #   volumes:\n  #     - ollama_data:/root/.ollama\n  #   restart: unless-stopped\n\n# volumes:\n#   ollama_data:\n```\n\n---\n\n## Step 4: Application Code Changes\n\n### Minimal Change Required\n\nPolicyBot is a Next.js (TypeScript) application. The proxy switch is controlled entirely by environment variables — no code changes needed:\n\n**Environment (`.env`):**\n```bash\n# Set this to route all chat/embeddings/transcription through LiteLLM\nOPENAI_BASE_URL=http://litellm:4000/v1\nLITELLM_MASTER_KEY=sk-litellm-master-change-this\n```\n\n**How it works (`src/lib/openai.ts`):**\n```typescript\n// When OPENAI_BASE_URL is set → uses LiteLLM proxy with LITELLM_MASTER_KEY\n// When OPENAI_BASE_URL is unset → calls OpenAI directly with OPENAI_API_KEY\nconst apiKey = process.env.OPENAI_BASE_URL\n  ? (process.env.LITELLM_MASTER_KEY || await getApiKey('openai'))\n  : await getApiKey('openai');\n\nopenaiClient = new OpenAI({\n  apiKey: apiKey || undefined,\n  baseURL: process.env.OPENAI_BASE_URL || undefined,\n});\n```\n\nFor non-Claude models, that's it — the OpenAI SDK handles the routing transparently.\n\n**Claude models** are automatically detected and routed to the Anthropic SDK:\n```typescript\n// Automatic — no env var needed, just ANTHROPIC_API_KEY in DB or env\n// isClaudeModel() detects 'anthropic/' or 'claude-' prefix\n// Uses @anthropic-ai/sdk directly for chat + tool calling\n// Embeddings/transcription still route through LiteLLM\n```\n\n---\n\n## Step 5: Function Calling Handling\n\nSince your `generateResponseWithTools()` uses function calling, here's how to handle provider capabilities:\n\n```python\n# Provider capability mapping (Updated December 2025)\n\n# Full tool/function calling support\n# Matches Policy Bot config/defaults.json toolCapable list\nTOOL_CAPABLE_MODELS = [\n    # OpenAI\n    \"gpt-4.1\", \"gpt-4.1-mini\", \"gpt-4.1-nano\", \"gpt-3.5-turbo\",\n    # Mistral\n    \"mistral-large-3\", \"mistral-medium-3.1\", \"mistral-small-3.2\",\n    \"ministral-8b\", \"ministral-3b\",\n    # Ollama (Now with full tool support!)\n    \"ollama-llama3.2\", \"ollama-llama3.1\", \"ollama-mistral\", \"ollama-qwen2.5\",\n]\n\n# Limited or no tool support (show warning)\nTOOL_LIMITED_MODELS = [\n    \"ollama-phi4\",\n]\n\ndef generate_response_with_tools(model: str, messages: list, tools: list):\n    \"\"\"\n    Generate response with function calling support.\n    Warns and disables tools for unsupported providers.\n    \"\"\"\n    \n    if model in TOOL_LIMITED_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' has limited function calling support.\")\n        print(f\"   Tools will be disabled. Consider using: {', '.join(TOOL_CAPABLE_MODELS[:5])}\")\n        # Call without tools\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages\n        )\n    elif model not in TOOL_CAPABLE_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' tool support unknown. Attempting with tools...\")\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    else:\n        # Full tool support\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    \n    return response\n```\n\n> **Good News (2025 Update)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n---\n\n## Step 6: Embeddings Usage\n\n```python\n# Embeddings via LiteLLM Proxy\n\n# OpenAI embeddings (recommended for RAG quality)\nresponse = client.embeddings.create(\n    model=\"openai-embedding-large\",\n    input=\"Your text here\"\n)\n\n# Azure embeddings\nresponse = client.embeddings.create(\n    model=\"azure-embedding\",\n    input=\"Your text here\"\n)\n\n# Mistral embeddings\nresponse = client.embeddings.create(\n    model=\"mistral-embedding\",\n    input=\"Your text here\"\n)\n\n# Ollama embeddings (local)\nresponse = client.embeddings.create(\n    model=\"ollama-embedding\",\n    input=\"Your text here\"\n)\n\nembedding = response.data[0].embedding\n```\n\n### Important: RAG Consistency\n\n> ⚠️ **Warning**: Don't mix embedding providers for the same vector store. If your RAG was built with `text-embedding-3-large`, query with the same model.\n\n```python\n# Configuration approach\nEMBEDDING_CONFIG = {\n    \"openai\": \"openai-embedding-large\",\n    \"azure\": \"azure-embedding\",\n    \"mistral\": \"mistral-embedding\",\n    \"ollama\": \"ollama-embedding\"\n}\n\n# Use consistently\ncurrent_provider = \"openai\"\nembedding_model = EMBEDDING_CONFIG[current_provider]\n```\n\n---\n\n## Step 7: Audio Transcription (Whisper)\n\n```python\n# Audio transcription via LiteLLM Proxy\n\naudio_file = open(\"audio.mp3\", \"rb\")\n\nresponse = client.audio.transcriptions.create(\n    model=\"openai-whisper\",\n    file=audio_file\n)\n\nprint(response.text)\n```\n\n---\n\n## Deployment Commands\n\n```bash\n# Start the proxy\ndocker compose up -d\n\n# View logs\ndocker compose logs -f litellm\n\n# Stop\ndocker compose down\n\n# Restart after config changes\ndocker compose restart litellm\n```\n\n---\n\n## Testing the Setup\n\n### 1. Health Check\n\n```bash\ncurl http://localhost:4000/health\n```\n\n### 2. List Available Models\n\n```bash\ncurl http://localhost:4000/models \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\"\n```\n\n### 3. Test Chat Completion\n\n```bash\ncurl -X POST http://localhost:4000/chat/completions \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-gpt35\",\n    \"messages\": [{\"role\": \"user\", \"content\": \"Hello, test!\"}]\n  }'\n```\n\n### 4. Test Embeddings\n\n```bash\ncurl -X POST http://localhost:4000/embeddings \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-embedding-large\",\n    \"input\": \"Test embedding\"\n  }'\n```\n\n---\n\n## Model Quick Reference\n\n*Updated: March 2026*\n\n### OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `openai-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Latest flagship, best for coding |\n| `openai-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Faster, 83% cheaper than GPT-4o |\n| `openai-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Fastest, cheapest |\n| `openai-gpt35` | `gpt-3.5-turbo` | ✅ | - | ✅ | - | 16K | Legacy, cost-effective |\n| `openai-embedding-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality embeddings |\n| `openai-embedding-small` | `text-embedding-3-small` | - | ✅ | - | - | 8K | Faster, cheaper |\n| `openai-whisper` | `whisper-1` | - | - | - | ✅ | - | $0.006/min transcription |\n\n### Azure OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `azure-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-embedding` | `text-embedding-ada-002` | - | ✅ | - | - | 8K | Standard embedding |\n| `azure-embedding-3-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality |\n| `azure-whisper` | `whisper` | - | - | - | ✅ | - | Speech-to-text |\n\n### Mistral AI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `mistral-large-3` | `mistral-large-latest` | ✅ | - | ✅ | - | 256K | Flagship MoE (675B total params) |\n| `mistral-medium-31` | `mistral-medium-2508` | ✅ | - | ✅ | - | 128K | Frontier multimodal |\n| `mistral-small-32` | `mistral-small-2506` | ✅ | - | ✅ | - | 128K | Fast, efficient |\n| `ministral-14b` | `ministral-3-14b` | ✅ | - | ✅ | - | 128K | Best small model |\n| `ministral-8b` | `ministral-3-8b` | ✅ | - | ✅ | - | 128K | Edge deployment |\n| `ministral-3b` | `ministral-3-3b` | ✅ | - | ✅ | - | 128K | Ultra-light |\n| `mistral-embedding` | `mistral-embed` | - | ✅ | - | - | 8K | Text embeddings |\n| `codestral-embed` | `codestral-embed` | - | ✅ | - | - | - | Code embeddings |\n| `voxtral-small` | `voxtral-small-latest` | - | - | - | ✅ | 32K | 24B, best accuracy, $0.001/min |\n| `voxtral-mini` | `voxtral-mini-latest` | - | - | - | ✅ | 32K | 3B, edge/local deployment |\n| `voxtral-transcribe` | `voxtral-mini-latest` (via /audio/transcriptions) | - | - | - | ✅ | - | Transcription-only, beats Whisper |\n\n> **Note**: Voxtral (released July 2025) outperforms OpenAI Whisper at half the cost ($0.001/min vs $0.006/min). Supports up to 30 min audio for transcription, 40 min for understanding.\n\n### Google Gemini Models\n\n| Model Name | API Model ID | Chat | Tools | Context | Notes |\n|------------|--------------|------|-------|---------|-------|\n| `gemini-2.5-pro` | `gemini/gemini-2.5-pro` | ✅ | ✅ | 1M | Flagship reasoning model |\n| `gemini-2.5-flash` | `gemini/gemini-2.5-flash` | ✅ | ✅ | 1M | Fast, balanced performance |\n| `gemini-2.5-flash-lite` | `gemini/gemini-2.5-flash-lite` | ✅ | ✅ | 1M | Lowest cost option |\n\n**Pricing (per 1M tokens):**\n- Gemini 2.5 Pro: $1.25 input / $10.00 output (≤200k context)\n- Gemini 2.5 Flash: $0.30 input / $2.50 output\n- Gemini 2.5 Flash-Lite: $0.10 input / $0.40 output\n\n**Features:**\n- All models support function calling\n- Built-in \"thinking\" capabilities with controllable budgets\n- Free tier available for development/testing\n- Native multimodal support (text, image, audio, video)\n\n### Ollama Local Models\n\n| Model Name | Ollama Model | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `ollama-llama32` | `llama3.2` | ✅ | - | ✅ | - | 128K | **Full tool support** (fine-tuned for function calling) |\n| `ollama-llama31-8b` | `llama3.1:8b` | ✅ | - | ✅ | - | 128K | **Full tool support**, best overall |\n| `ollama-llama31-70b` | `llama3.1:70b` | ✅ | - | ✅ | - | 128K | **Full tool support**, highest accuracy |\n| `ollama-mistral` | `mistral` | ✅ | - | ✅ | - | 32K | **Supports tools** (v0.3+) |\n| `ollama-mixtral` | `mixtral:8x7b` | ✅ | - | ✅ | - | 32K | MoE, good multi-domain |\n| `ollama-qwen25` | `qwen2.5` | ✅ | - | ✅ | - | 128K | Strong tool support |\n| `ollama-phi4` | `phi4` | ✅ | - | ⚠️ | - | 16K | Limited tool support |\n| `ollama-embedding` | `nomic-embed-text` | - | ✅ | - | - | 8K | Best local embedding |\n| `ollama-mxbai-embed` | `mxbai-embed-large` | - | ✅ | - | - | 512 | Alternative embedding |\n\n> **Tool Calling Update (2025)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), Mixtral, and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ | Fully Supported |\n| ⚠️ | Limited/Experimental (may need prompt adjustments) |\n| - | Not applicable |\n\n### Recommendations by Use Case\n\n| Use Case | Recommended Model | Why |\n|----------|-------------------|-----|\n| **Production Chat** | `gpt-4.1` or `gemini-2.5-pro` | Best instruction following, 1M context |\n| **Balanced (Default)** | `gpt-4.1-mini` or `gemini-2.5-flash` | Good quality at lower cost (Policy Bot default) |\n| **Budget Chat** | `gpt-4.1-nano` or `gemini-2.5-flash-lite` | Cost-effective for simple queries |\n| **Deep Reasoning** | `gemini-2.5-pro` | Built-in thinking capabilities |\n| **Local/Offline Chat** | `ollama-llama3.2` or `ollama-qwen2.5` | Full tool support, runs locally |\n| **RAG Embeddings** | `text-embedding-3-large` | Best quality for retrieval (Policy Bot default) |\n| **Local Embeddings** | `ollama-embedding` (nomic) | Good quality, no API cost |\n| **Audio Transcription** | `voxtral-transcribe` | Beats Whisper, half the cost |\n| **Budget Transcription** | `whisper-1` | Well-established, $0.006/min |\n| **Function Calling** | `gpt-4.1-mini`, `gemini-2.5-flash`, `mistral-large-3` | Reliable tool use |\n\n---\n\n## Switching Providers\n\nTo switch your entire application from OpenAI to Ollama:\n\n1. **No code changes needed**\n2. Update model name in your application config:\n\n```python\n# Before (OpenAI)\nMODEL = \"openai-gpt41\"\nEMBEDDING_MODEL = \"openai-embedding-large\"\nAUDIO_MODEL = \"openai-whisper\"\n\n# After (Ollama + Mistral Voxtral for audio)\nMODEL = \"ollama-llama32\"\nEMBEDDING_MODEL = \"ollama-embedding\"\nAUDIO_MODEL = \"voxtral-mini\"  # Or keep openai-whisper\n\n# Or fully Mistral\nMODEL = \"mistral-large-3\"\nEMBEDDING_MODEL = \"mistral-embedding\"\nAUDIO_MODEL = \"voxtral-small\"\n```\n\n---\n\n## Troubleshooting\n\n| Issue | Solution |\n|-------|----------|\n| Connection refused | Check if Docker is running, verify port 4000 |\n| Ollama not reachable | Use `host.docker.internal` in Docker, check Ollama is running |\n| API key errors | Verify `.env` file is loaded, check key format |\n| Model not found | Run `/models` endpoint to list available models |\n| Timeout errors | Increase `request_timeout` in config |\n\n---\n\n## Known Limitations\n\n| # | Limitation | Impact | Status | File |\n|---|-----------|--------|--------|------|\n| 1 | ~~Static YAML cache never invalidated~~ | ~~Runtime model additions invisible until restart~~ | **FIXED** — `clearLiteLLMCache()` now called after `syncAllModelsToLiteLLM()` completes | `litellm-sync.ts` |\n| 2 | ~~Claude tool-calling streaming mismatch~~ | ~~Anthropic models routed through LiteLLM fail tool calling~~ | **FIXED** — Claude models now bypass LiteLLM via `@anthropic-ai/sdk` direct. `isClaudeModel()` routes to `streamAnthropicCompletion()` which receives pre-parsed tool inputs. | `openai.ts` |\n| 3 | **No circuit breaker for chat completions** | If LiteLLM proxy is unreachable, non-Claude chat completions fail. Claude models are unaffected (direct SDK). Embeddings have a fallback model path, but chat/transcription do not. | Open | `openai.ts` |\n| 4 | **Prefix-based provider detection only** | `getProviderFromModelPath()` defaults to `openai` for unrecognized prefixes. Custom/vLLM models without a prefix will be misidentified. No `deepseek/` prefix exists — DeepSeek models are synced dynamically with prefix by `litellm-sync.ts`. | Open | `litellm-validator.ts:282-294` |\n| 5 | **No TTS or image generation via proxy** | `podcast_gen` and `image_gen` must bypass LiteLLM entirely | By design | `podcast-gen.ts`, `image-gen/` |\n\n---\n\n## Resources\n\n- **LiteLLM Docs**: https://docs.litellm.ai/docs/\n- **Supported Providers**: https://docs.litellm.ai/docs/providers\n- **GitHub**: https://github.com/BerriAI/litellm\n- **Ollama Models**: https://ollama.ai/library\n\n---\n\n*Last updated: March 2026 — Added Anthropic Direct SDK (Tier 1b) for Claude chat + tool calling, bypassing LiteLLM. Updated architecture to four-tier hybrid. Fixed Known Limitations #1 (cache invalidation) and #2 (Claude tool-calling). Added `isClaudeModel()` detection logic documentation.*"}, "690a30fda8c1fefae45995d88bb309c10a609ca2": {"t": 1785754363.3954985, "n": 38488, "h": "1ffeceb4105fa649", "c": "# LiteLLM Implementation Guide\n\n> Customized for multi-provider LLM abstraction with proxy approach, embeddings routing, function calling support, and audio transcription.\n\n---\n\n## LLM Service Routing Overview\n\nPolicy Bot uses a **hybrid architecture**: most chat services route through LiteLLM proxy for unified model management, while **Anthropic Claude models bypass LiteLLM entirely** using the `@anthropic-ai/sdk` for direct API access (eliminating tool-calling JSON assembly issues), and specialized services (images, audio, document processing) call provider APIs directly or use local parsers.\n\n### Service Routing Table\n\n| Service | Policy Bot Feature | Routes Through | Provider(s) | Notes |\n|---------|-------------------|----------------|-------------|-------|\n| **Chat Completions** | Main chat, RAG responses | ✅ LiteLLM / ⚡ Anthropic Direct | OpenAI, Gemini, Mistral, DeepSeek, Fireworks AI, Ollama via LiteLLM; **Anthropic Claude via direct SDK** | Claude uses `@anthropic-ai/sdk` for reliable tool calling |\n| **Embeddings** | Document indexing, search | ✅ LiteLLM | OpenAI, Mistral, Gemini, Ollama, Fireworks | `text-embedding-3-large` default |\n| **Diagram Generation** | `diagram_gen` tool | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Generates Mermaid syntax |\n| **Summarization** | Message compression | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic, DeepSeek | Long conversation handling |\n| **Memory Extraction** | User fact storage | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Per-category memory |\n| **Prompt Optimization** | Query refinement | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Pre-RAG processing |\n| **Compliance Checks** | HITL clarification | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Skill compliance |\n| **Translation** | Multi-language support | ✅ LiteLLM / ❌ Direct | OpenAI (proxy), Gemini/Mistral (direct) | Provider-dependent |\n| **Audio Transcription** | Voice input | ✅ LiteLLM | OpenAI Whisper, Mistral Voxtral | `transcribeAudio()` uses `getOpenAI()` client |\n| **Image Generation** | `image_gen` tool | ❌ Direct | OpenAI DALL-E, Gemini Imagen | Specialized APIs |\n| **Podcast Generation** | `podcast_gen` tool | ❌ Direct | OpenAI TTS, Gemini TTS | Multi-voice audio synthesis |\n| **Document Processing** | Document text extraction | ❌ Direct | mammoth/exceljs/officeparser (local), Mistral OCR, Azure DI, pdf-parse | Tiered fallback: local parsers first, then API providers |\n| **Reranking** | Search result scoring | ❌ Direct | Fireworks AI | Direct HTTP to `api.fireworks.ai/inference/v1/rerank` |\n| **PPTX Generation** | `pptx_gen` tool | N/A | None | Template-based (no LLM) |\n| **XLSX Generation** | `xlsx_gen` tool | N/A | None | ExcelJS-based (no LLM) |\n| **Chart Generation** | `chart_gen` tool | N/A | None | Client-side rendering |\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ LiteLLM | Routes through LiteLLM proxy (port 4000) |\n| ⚡ Anthropic Direct | Calls Anthropic API directly via `@anthropic-ai/sdk` (bypasses LiteLLM) |\n| ❌ Direct | Calls provider API directly (bypasses LiteLLM) |\n| N/A | No LLM calls required |\n\n### Why Some Services Bypass LiteLLM\n\n| Service | Reason |\n|---------|--------|\n| **Anthropic Claude** | LiteLLM's Anthropic→OpenAI streaming translation produces malformed tool-calling JSON. Direct `@anthropic-ai/sdk` provides native, reliable tool call parsing |\n| **Image Generation** | DALL-E and Gemini Imagen APIs not supported by LiteLLM proxy |\n| **Podcast TTS** | OpenAI TTS and Gemini TTS are audio generation APIs, not chat completions |\n| **OCR** | Mistral OCR is a specialized document API with unique request format |\n| **Reranking** | Fireworks reranker is a specialized search API (`/inference/v1/rerank`), not a chat model |\n\n> **Note**: Audio transcription (Whisper, Voxtral) routes **through LiteLLM** — `transcribeAudio()` in `openai.ts` uses the same `getOpenAI()` client as chat completions. Both `whisper-1` and `voxtral-mini` are registered in `litellm_config.yaml`.\n\n### Proxy Detection Logic\n\nThe actual detection in `openai.ts` checks for the `OPENAI_BASE_URL` environment variable:\n\n```typescript\n// src/lib/openai.ts — getOpenAI()\nasync function getOpenAI(): Promise<OpenAI> {\n  if (!openaiClient) {\n    // When OPENAI_BASE_URL is set, route through LiteLLM proxy\n    const apiKey = process.env.OPENAI_BASE_URL\n      ? (process.env.LITELLM_MASTER_KEY || await getApiKey('openai'))\n      : await getApiKey('openai');\n\n    openaiClient = new OpenAI({\n      apiKey: apiKey || undefined,\n      baseURL: process.env.OPENAI_BASE_URL || undefined,\n      timeout: 300 * 1000,\n    });\n  }\n  return openaiClient;\n}\n```\n\nWhen `OPENAI_BASE_URL` points to the LiteLLM container (e.g. `http://litellm:4000/v1`), embeddings, transcription, and non-Claude chat completions are routed through a single OpenAI SDK client.\n\n### Claude Direct SDK Detection\n\nFor Anthropic Claude models, `generateResponseWithTools()` bypasses LiteLLM entirely:\n\n```typescript\n// src/lib/openai.ts — isClaudeModel() + routing\nfunction isClaudeModel(model: string): boolean {\n  return model.startsWith('anthropic/') || model.startsWith('claude-');\n}\n\n// In generateResponseWithTools():\nconst useAnthropicDirect = isClaudeModel(effectiveModel);\nconst openai = useAnthropicDirect ? null : await getOpenAI();          // LiteLLM path\nconst anthropicClient = useAnthropicDirect ? await getAnthropicClient() : null;  // Direct path\n```\n\n**Why**: LiteLLM translates Anthropic's native streaming into OpenAI-compatible `delta.tool_calls` format, but the tool call JSON assembly is unreliable — producing malformed arguments that fail `executeTool()`. The direct `@anthropic-ai/sdk` receives pre-parsed tool inputs via `stream.finalMessage()`, eliminating this class of errors.\n\n---\n\n## Policy Bot Integration\n\nThis guide is configured for the Policy Bot RAG application. Default model preset:\n\n| Setting | Value |\n|---------|-------|\n| Default Model | `gpt-4.1-mini` |\n| Temperature | `0.2` |\n| Max Tokens | `2000` |\n| Embedding Model | `text-embedding-3-large` |\n| Embedding Dimensions | `3072` |\n\nAvailable model presets in Policy Bot (via `config/defaults.json`):\n- **gpt-4.1** - High Performance (1M context)\n- **gpt-4.1-mini** - Balanced (default)\n- **gpt-4.1-nano** - Cost-Effective\n- **mistral-large-3** - Mistral Flagship (256K context)\n- **mistral-small-3.2** - Mistral Cost-Effective\n- **ministral-8b** - Mistral Ultra Cost-Effective\n- **gemini-2.5-pro** - Google Flagship Reasoning (1M context)\n- **gemini-2.5-flash** - Google Balanced (1M context)\n- **gemini-2.5-flash-lite** - Google Cost-Effective (1M context)\n- **claude-opus-4-6** - Anthropic Flagship\n- **claude-sonnet-4-6** - Anthropic Balanced\n- **deepseek-chat** - DeepSeek V3 (cost-effective)\n- **deepseek-reasoner** - DeepSeek R1 (thinking model)\n- **fw/kimi-k2-instruct** - Fireworks Kimi K2 (dev/test)\n- **ollama-llama3.2** - Local (no API cost)\n- **ollama-qwen2.5** - Local with excellent reasoning\n\n---\n\n## Architecture Overview\n\nPolicyBot uses a **four-tier hybrid architecture**. LiteLLM handles most chat, embeddings, and transcription. **Anthropic Claude models bypass LiteLLM** via the native `@anthropic-ai/sdk` for reliable tool calling. Specialized services use direct API calls.\n\n```\n┌───────────────────────────────────────────────────────────────────────┐\n│                         PolicyBot Application                         │\n│              (Next.js — openai.ts, tools, translation)                │\n└──┬──────────────────┬──────────────────┬──────────────────┬───────────┘\n   │                  │                  │                  │\n TIER 1            TIER 1b            TIER 2             TIER 3\n LiteLLM Proxy     Anthropic Direct   Direct Provider    Direct Google\n (Chat, Embed,     SDK (@anthropic-   APIs (Non-Chat)    GenAI SDK\n  Transcription)    ai/sdk)                              (Image/TTS)\n   │                  │                  │                  │\n   ▼                  ▼                  ▼                  ▼\n┌────────────────┐ ┌────────────────┐ ┌────────────────┐ ┌────────────────┐\n│  Port 4000     │ │ api.anthropic  │ │                │ │                │\n│                │ │ .com           │ │ Fireworks AI   │ │ Gemini Imagen  │\n│ ┌────────────┐ │ │                │ │  (reranking)   │ │  (image_gen)   │\n│ │ OpenAI     │ │ │ Claude chat +  │ │                │ │  via REST API  │\n│ │ Gemini     │ │ │ tool calling   │ │ Tavily         │ │                │\n│ │ Mistral    │ │ │ via native     │ │  (web_search)  │ │ Gemini TTS     │\n│ │ DeepSeek   │ │ │ streaming      │ │                │ │  (podcast_gen) │\n│ │ Fireworks* │ │ │                │ │ Gemini/Mistral │ │  via @google/  │\n│ │ Ollama*    │ │ │ Models:        │ │  (translation) │ │  genai SDK     │\n│ └────────────┘ │ │ claude-opus-*  │ │                │ │                │\n│                │ │ claude-sonnet-*│ │ OpenAI TTS     │ │ DALL-E 3       │\n│ * YAML-only,   │ │ claude-haiku-* │ │  (podcast_gen) │ │  (image_gen)   │\n│   not dynamic  │ │                │ │  hardcoded to  │ │  via OpenAI SDK│\n│   sync         │ │ Why: LiteLLM   │ │  api.openai.com│ │  (no proxy)    │\n│                │ │ breaks tool    │ │                │ │                │\n│                │ │ call JSON      │ │                │ │                │\n└────────────────┘ └────────────────┘ └────────────────┘ └────────────────┘\n```\n\n### Model Capability Detection\n\nCapabilities are auto-detected via regex patterns in `src/lib/services/model-discovery.ts` and stored in the `enabled_models` table. Admins can override via the LLM Settings UI.\n\n| Capability | DB Column | Detection Function | Used By |\n|------------|-----------|-------------------|---------|\n| Tool calling | `tool_capable` | `isToolCapable(modelId)` | `openai.ts` — whether to send tool definitions |\n| Vision/multimodal | `vision_capable` | `isVisionCapable(modelId)` | `config-capability-checker.ts` — image upload gating |\n| Parallel tool calls | `parallel_tool_capable` | `isParallelToolCapable(modelId)` | `openai.ts` — sequential vs `Promise.allSettled` execution |\n| Thinking/reasoning | `thinking_capable` | `isThinkingCapable(modelId)` | UI — thinking content display |\n\n### Model Registration: Dynamic vs Static\n\nPolicyBot uses **two registration paths** for LiteLLM models:\n\n| Path | Providers | How |\n|------|-----------|-----|\n| **Dynamic** (via `/model/new` API) | OpenAI, Anthropic, Gemini, Mistral, DeepSeek | `litellm-sync.ts` reads `enabled_models` table and registers via API |\n| **Static** (YAML only) | Fireworks, Ollama, Embeddings, Audio | Defined in `litellm_config.yaml`, skipped by sync service |\n| **Direct SDK** (bypasses LiteLLM) | Anthropic Claude | `isClaudeModel()` in `openai.ts` detects `anthropic/` or `claude-` prefix → routes to `@anthropic-ai/sdk` |\n\n> Dynamic models (5 providers) are NOT in the YAML — they are registered at startup by the app. Only Fireworks, Ollama, embedding, and audio models are declared in `litellm_config.yaml`.\n>\n> **Note**: Anthropic models are still registered dynamically with LiteLLM (for embeddings/non-chat use), but **chat completions with tool calling** bypass LiteLLM entirely via the Anthropic SDK. This is controlled by `isClaudeModel()` in `openai.ts`.\n\n---\n\n## Decision Summary\n\n| Component | Decision |\n|-----------|----------|\n| **Approach** | LiteLLM Proxy (Docker Compose) + Anthropic Direct SDK |\n| **Chat Completions** | Route through LiteLLM (except Claude → `@anthropic-ai/sdk` direct) |\n| **Embeddings** | Route through LiteLLM (per-provider config) |\n| **Function Calling** | Gated by `toolCapable` flag from `litellm_config.yaml` via DB sync |\n| **Audio Transcription** | Route through LiteLLM (Whisper, Voxtral) |\n| **Deployment** | Docker Compose |\n\n---\n\n## Project Structure\n\n```\nlitellm-proxy/\n├── docker-compose.yml\n├── litellm_config.yaml\n├── .env\n└── README.md\n```\n\n---\n\n## Step 1: Environment Variables\n\nCreate `.env` file:\n\n```bash\n# ===================\n# LiteLLM Proxy Keys\n# ===================\nLITELLM_MASTER_KEY=sk-litellm-master-change-this\nLITELLM_SALT_KEY=sk-litellm-salt-change-this\n\n# ===================\n# OpenAI\n# ===================\nOPENAI_API_KEY=sk-...\n\n# ===================\n# Azure OpenAI\n# ===================\nAZURE_API_KEY=...\nAZURE_API_BASE=https://your-resource.openai.azure.com/\nAZURE_API_VERSION=2024-02-15-preview\nAZURE_CHAT_DEPLOYMENT=gpt-4-deployment\nAZURE_EMBEDDING_DEPLOYMENT=text-embedding-ada-002\n\n# ===================\n# Mistral\n# ===================\nMISTRAL_API_KEY=...\n\n# ===================\n# Google Gemini\n# ===================\nGEMINI_API_KEY=...\n\n# ===================\n# Anthropic\n# ===================\nANTHROPIC_API_KEY=...\n\n# ===================\n# DeepSeek\n# ===================\nDEEPSEEK_API_KEY=...\n\n# ===================\n# Fireworks AI (dev/test open-source models)\n# ===================\nFIREWORKS_AI_API_KEY=fw-...\n\n# ===================\n# Ollama (Local)\n# ===================\nOLLAMA_API_BASE=http://host.docker.internal:11434\n# Use 'http://localhost:11434' if running outside Docker\n```\n\n---\n\n## Step 2: LiteLLM Configuration\n\n> **Important**: The reference YAML below is a **complete template** listing all models statically. In actual PolicyBot deployment, chat models for OpenAI, Anthropic, Gemini, Mistral, and DeepSeek are **registered dynamically** via `litellm-sync.ts` (see Architecture Overview above). Only Fireworks, Ollama, embedding, and audio models are declared in the YAML. See the actual config at `litellm-proxy/litellm_config.yaml`.\n\nCreate `litellm_config.yaml`:\n\n```yaml\n# =============================================================================\n# LITELLM PROXY CONFIGURATION\n# Multi-provider setup: OpenAI, Azure, Mistral, Gemini, Anthropic, DeepSeek, Fireworks AI, Ollama\n# Updated: March 2026\n# =============================================================================\n\nmodel_list:\n\n  # ===========================================================================\n  # CHAT COMPLETION MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Models - GPT-4.1 Family (Latest - April 2025+)\n  # ---------------------------------------------------------------------------\n  - model_name: openai-gpt41\n    litellm_params:\n      model: gpt-4.1\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-mini\n    litellm_params:\n      model: gpt-4.1-mini\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-nano\n    litellm_params:\n      model: gpt-4.1-nano\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt35\n    litellm_params:\n      model: gpt-3.5-turbo\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Models - GPT-4.1 Family\n  # ---------------------------------------------------------------------------\n  - model_name: azure-gpt41\n    litellm_params:\n      model: azure/gpt-41-deployment  # Your deployment name\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: azure-gpt41-mini\n    litellm_params:\n      model: azure/gpt-41-mini-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Mistral AI Models - Mistral 3 Family (Latest - December 2025)\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-large-3\n    litellm_params:\n      model: mistral/mistral-large-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 256000\n\n  - model_name: mistral-medium-31\n    litellm_params:\n      model: mistral/mistral-medium-2508\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      supports_vision: true\n\n  - model_name: mistral-small-32\n    litellm_params:\n      model: mistral/mistral-small-2506\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ministral-8b\n    litellm_params:\n      model: mistral/ministral-8b-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Ollama Models (Local) - With Full Tool Support\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-llama32\n    litellm_params:\n      model: ollama/llama3.2\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # Fine-tuned for function calling\n\n  - model_name: ollama-llama31-8b\n    litellm_params:\n      model: ollama/llama3.1:8b\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-mistral\n    litellm_params:\n      model: ollama/mistral\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # v0.3+ supports tools\n\n  - model_name: ollama-qwen25\n    litellm_params:\n      model: ollama/qwen2.5\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-phi4\n    litellm_params:\n      model: ollama/phi4\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: false  # Limited support\n\n  # ===========================================================================\n  # EMBEDDING MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: openai-embedding-large\n    litellm_params:\n      model: text-embedding-3-large\n      api_key: os.environ/OPENAI_API_KEY\n\n  - model_name: openai-embedding-small\n    litellm_params:\n      model: text-embedding-3-small\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: azure-embedding\n    litellm_params:\n      model: azure/text-embedding-ada-002\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  - model_name: azure-embedding-3-large\n    litellm_params:\n      model: azure/text-embedding-3-large-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-embedding\n    litellm_params:\n      model: mistral/mistral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: codestral-embed\n    litellm_params:\n      model: mistral/codestral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Ollama Embeddings (Local)\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-embedding\n    litellm_params:\n      model: ollama/nomic-embed-text\n      api_base: os.environ/OLLAMA_API_BASE\n\n  - model_name: ollama-mxbai-embed\n    litellm_params:\n      model: ollama/mxbai-embed-large\n      api_base: os.environ/OLLAMA_API_BASE\n\n  # ===========================================================================\n  # AUDIO MODELS (Speech-to-Text / Transcription)\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Whisper\n  # ---------------------------------------------------------------------------\n  - model_name: openai-whisper\n    litellm_params:\n      model: whisper-1\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure Whisper (if deployed)\n  # ---------------------------------------------------------------------------\n  - model_name: azure-whisper\n    litellm_params:\n      model: azure/whisper-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Voxtral (Released July 2025 - Beats Whisper at half cost)\n  # ---------------------------------------------------------------------------\n  - model_name: voxtral-small\n    litellm_params:\n      model: mistral/voxtral-small-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: voxtral-mini\n    litellm_params:\n      model: mistral/voxtral-mini-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n# =============================================================================\n# LITELLM SETTINGS\n# =============================================================================\n\nlitellm_settings:\n  # Drop unsupported params instead of erroring\n  drop_params: true\n  \n  # Enable detailed logging\n  set_verbose: false\n\n  # Request timeout\n  request_timeout: 120\n\n# =============================================================================\n# GENERAL SETTINGS\n# =============================================================================\n\ngeneral_settings:\n  # Master key for proxy authentication\n  master_key: os.environ/LITELLM_MASTER_KEY\n```\n\n---\n\n## Step 3: Docker Compose\n\n> **Note**: In the actual PolicyBot deployment, LiteLLM runs as part of the full stack in `docker-compose.local.yml` alongside Postgres, Qdrant, and Redis. The standalone example below is for reference.\n\nCreate `docker-compose.yml`:\n\n```yaml\nversion: \"3.9\"\n\nservices:\n  litellm:\n    image: ghcr.io/berriai/litellm:v1.82.3-stable.patch.2\n    container_name: litellm-proxy\n    ports:\n      - \"4000:4000\"\n    volumes:\n      - ./litellm_config.yaml:/app/config.yaml\n    env_file:\n      - .env\n    environment:\n      - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY}\n      - LITELLM_SALT_KEY=${LITELLM_SALT_KEY}\n    command: --config /app/config.yaml --port 4000 --detailed_debug\n    restart: unless-stopped\n    extra_hosts:\n      - \"host.docker.internal:host-gateway\"\n    healthcheck:\n      test: [\"CMD\", \"curl\", \"-f\", \"http://localhost:4000/health\"]\n      interval: 30s\n      timeout: 10s\n      retries: 3\n\n  # Optional: Ollama service (if not running separately)\n  # ollama:\n  #   image: ollama/ollama:latest\n  #   container_name: ollama\n  #   ports:\n  #     - \"11434:11434\"\n  #   volumes:\n  #     - ollama_data:/root/.ollama\n  #   restart: unless-stopped\n\n# volumes:\n#   ollama_data:\n```\n\n---\n\n## Step 4: Application Code Changes\n\n### Minimal Change Required\n\nPolicyBot is a Next.js (TypeScript) application. The proxy switch is controlled entirely by environment variables — no code changes needed:\n\n**Environment (`.env`):**\n```bash\n# Set this to route all chat/embeddings/transcription through LiteLLM\nOPENAI_BASE_URL=http://litellm:4000/v1\nLITELLM_MASTER_KEY=sk-litellm-master-change-this\n```\n\n**How it works (`src/lib/openai.ts`):**\n```typescript\n// When OPENAI_BASE_URL is set → uses LiteLLM proxy with LITELLM_MASTER_KEY\n// When OPENAI_BASE_URL is unset → calls OpenAI directly with OPENAI_API_KEY\nconst apiKey = process.env.OPENAI_BASE_URL\n  ? (process.env.LITELLM_MASTER_KEY || await getApiKey('openai'))\n  : await getApiKey('openai');\n\nopenaiClient = new OpenAI({\n  apiKey: apiKey || undefined,\n  baseURL: process.env.OPENAI_BASE_URL || undefined,\n});\n```\n\nFor non-Claude models, that's it — the OpenAI SDK handles the routing transparently.\n\n**Claude models** are automatically detected and routed to the Anthropic SDK:\n```typescript\n// Automatic — no env var needed, just ANTHROPIC_API_KEY in DB or env\n// isClaudeModel() detects 'anthropic/' or 'claude-' prefix\n// Uses @anthropic-ai/sdk directly for chat + tool calling\n// Embeddings/transcription still route through LiteLLM\n```\n\n---\n\n## Step 5: Function Calling Handling\n\nSince your `generateResponseWithTools()` uses function calling, here's how to handle provider capabilities:\n\n```python\n# Provider capability mapping (Updated December 2025)\n\n# Full tool/function calling support\n# Matches Policy Bot config/defaults.json toolCapable list\nTOOL_CAPABLE_MODELS = [\n    # OpenAI\n    \"gpt-4.1\", \"gpt-4.1-mini\", \"gpt-4.1-nano\", \"gpt-3.5-turbo\",\n    # Mistral\n    \"mistral-large-3\", \"mistral-medium-3.1\", \"mistral-small-3.2\",\n    \"ministral-8b\", \"ministral-3b\",\n    # Ollama (Now with full tool support!)\n    \"ollama-llama3.2\", \"ollama-llama3.1\", \"ollama-mistral\", \"ollama-qwen2.5\",\n]\n\n# Limited or no tool support (show warning)\nTOOL_LIMITED_MODELS = [\n    \"ollama-phi4\",\n]\n\ndef generate_response_with_tools(model: str, messages: list, tools: list):\n    \"\"\"\n    Generate response with function calling support.\n    Warns and disables tools for unsupported providers.\n    \"\"\"\n    \n    if model in TOOL_LIMITED_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' has limited function calling support.\")\n        print(f\"   Tools will be disabled. Consider using: {', '.join(TOOL_CAPABLE_MODELS[:5])}\")\n        # Call without tools\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages\n        )\n    elif model not in TOOL_CAPABLE_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' tool support unknown. Attempting with tools...\")\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    else:\n        # Full tool support\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    \n    return response\n```\n\n> **Good News (2025 Update)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n---\n\n## Step 6: Embeddings Usage\n\n```python\n# Embeddings via LiteLLM Proxy\n\n# OpenAI embeddings (recommended for RAG quality)\nresponse = client.embeddings.create(\n    model=\"openai-embedding-large\",\n    input=\"Your text here\"\n)\n\n# Azure embeddings\nresponse = client.embeddings.create(\n    model=\"azure-embedding\",\n    input=\"Your text here\"\n)\n\n# Mistral embeddings\nresponse = client.embeddings.create(\n    model=\"mistral-embedding\",\n    input=\"Your text here\"\n)\n\n# Ollama embeddings (local)\nresponse = client.embeddings.create(\n    model=\"ollama-embedding\",\n    input=\"Your text here\"\n)\n\nembedding = response.data[0].embedding\n```\n\n### Important: RAG Consistency\n\n> ⚠️ **Warning**: Don't mix embedding providers for the same vector store. If your RAG was built with `text-embedding-3-large`, query with the same model.\n\n```python\n# Configuration approach\nEMBEDDING_CONFIG = {\n    \"openai\": \"openai-embedding-large\",\n    \"azure\": \"azure-embedding\",\n    \"mistral\": \"mistral-embedding\",\n    \"ollama\": \"ollama-embedding\"\n}\n\n# Use consistently\ncurrent_provider = \"openai\"\nembedding_model = EMBEDDING_CONFIG[current_provider]\n```\n\n---\n\n## Step 7: Audio Transcription (Whisper)\n\n```python\n# Audio transcription via LiteLLM Proxy\n\naudio_file = open(\"audio.mp3\", \"rb\")\n\nresponse = client.audio.transcriptions.create(\n    model=\"openai-whisper\",\n    file=audio_file\n)\n\nprint(response.text)\n```\n\n---\n\n## Deployment Commands\n\n```bash\n# Start the proxy\ndocker compose up -d\n\n# View logs\ndocker compose logs -f litellm\n\n# Stop\ndocker compose down\n\n# Restart after config changes\ndocker compose restart litellm\n```\n\n---\n\n## Testing the Setup\n\n### 1. Health Check\n\n```bash\ncurl http://localhost:4000/health\n```\n\n### 2. List Available Models\n\n```bash\ncurl http://localhost:4000/models \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\"\n```\n\n### 3. Test Chat Completion\n\n```bash\ncurl -X POST http://localhost:4000/chat/completions \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-gpt35\",\n    \"messages\": [{\"role\": \"user\", \"content\": \"Hello, test!\"}]\n  }'\n```\n\n### 4. Test Embeddings\n\n```bash\ncurl -X POST http://localhost:4000/embeddings \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-embedding-large\",\n    \"input\": \"Test embedding\"\n  }'\n```\n\n---\n\n## Model Quick Reference\n\n*Updated: March 2026*\n\n### OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `openai-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Latest flagship, best for coding |\n| `openai-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Faster, 83% cheaper than GPT-4o |\n| `openai-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Fastest, cheapest |\n| `openai-gpt35` | `gpt-3.5-turbo` | ✅ | - | ✅ | - | 16K | Legacy, cost-effective |\n| `openai-embedding-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality embeddings |\n| `openai-embedding-small` | `text-embedding-3-small` | - | ✅ | - | - | 8K | Faster, cheaper |\n| `openai-whisper` | `whisper-1` | - | - | - | ✅ | - | $0.006/min transcription |\n\n### Azure OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `azure-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-embedding` | `text-embedding-ada-002` | - | ✅ | - | - | 8K | Standard embedding |\n| `azure-embedding-3-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality |\n| `azure-whisper` | `whisper` | - | - | - | ✅ | - | Speech-to-text |\n\n### Mistral AI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `mistral-large-3` | `mistral-large-latest` | ✅ | - | ✅ | - | 256K | Flagship MoE (675B total params) |\n| `mistral-medium-31` | `mistral-medium-2508` | ✅ | - | ✅ | - | 128K | Frontier multimodal |\n| `mistral-small-32` | `mistral-small-2506` | ✅ | - | ✅ | - | 128K | Fast, efficient |\n| `ministral-14b` | `ministral-3-14b` | ✅ | - | ✅ | - | 128K | Best small model |\n| `ministral-8b` | `ministral-3-8b` | ✅ | - | ✅ | - | 128K | Edge deployment |\n| `ministral-3b` | `ministral-3-3b` | ✅ | - | ✅ | - | 128K | Ultra-light |\n| `mistral-embedding` | `mistral-embed` | - | ✅ | - | - | 8K | Text embeddings |\n| `codestral-embed` | `codestral-embed` | - | ✅ | - | - | - | Code embeddings |\n| `voxtral-small` | `voxtral-small-latest` | - | - | - | ✅ | 32K | 24B, best accuracy, $0.001/min |\n| `voxtral-mini` | `voxtral-mini-latest` | - | - | - | ✅ | 32K | 3B, edge/local deployment |\n| `voxtral-transcribe` | `voxtral-mini-latest` (via /audio/transcriptions) | - | - | - | ✅ | - | Transcription-only, beats Whisper |\n\n> **Note**: Voxtral (released July 2025) outperforms OpenAI Whisper at half the cost ($0.001/min vs $0.006/min). Supports up to 30 min audio for transcription, 40 min for understanding.\n\n### Google Gemini Models\n\n| Model Name | API Model ID | Chat | Tools | Context | Notes |\n|------------|--------------|------|-------|---------|-------|\n| `gemini-2.5-pro` | `gemini/gemini-2.5-pro` | ✅ | ✅ | 1M | Flagship reasoning model |\n| `gemini-2.5-flash` | `gemini/gemini-2.5-flash` | ✅ | ✅ | 1M | Fast, balanced performance |\n| `gemini-2.5-flash-lite` | `gemini/gemini-2.5-flash-lite` | ✅ | ✅ | 1M | Lowest cost option |\n\n**Pricing (per 1M tokens):**\n- Gemini 2.5 Pro: $1.25 input / $10.00 output (≤200k context)\n- Gemini 2.5 Flash: $0.30 input / $2.50 output\n- Gemini 2.5 Flash-Lite: $0.10 input / $0.40 output\n\n**Features:**\n- All models support function calling\n- Built-in \"thinking\" capabilities with controllable budgets\n- Free tier available for development/testing\n- Native multimodal support (text, image, audio, video)\n\n### Ollama Local Models\n\n| Model Name | Ollama Model | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `ollama-llama32` | `llama3.2` | ✅ | - | ✅ | - | 128K | **Full tool support** (fine-tuned for function calling) |\n| `ollama-llama31-8b` | `llama3.1:8b` | ✅ | - | ✅ | - | 128K | **Full tool support**, best overall |\n| `ollama-llama31-70b` | `llama3.1:70b` | ✅ | - | ✅ | - | 128K | **Full tool support**, highest accuracy |\n| `ollama-mistral` | `mistral` | ✅ | - | ✅ | - | 32K | **Supports tools** (v0.3+) |\n| `ollama-mixtral` | `mixtral:8x7b` | ✅ | - | ✅ | - | 32K | MoE, good multi-domain |\n| `ollama-qwen25` | `qwen2.5` | ✅ | - | ✅ | - | 128K | Strong tool support |\n| `ollama-phi4` | `phi4` | ✅ | - | ⚠️ | - | 16K | Limited tool support |\n| `ollama-embedding` | `nomic-embed-text` | - | ✅ | - | - | 8K | Best local embedding |\n| `ollama-mxbai-embed` | `mxbai-embed-large` | - | ✅ | - | - | 512 | Alternative embedding |\n\n> **Tool Calling Update (2025)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), Mixtral, and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ | Fully Supported |\n| ⚠️ | Limited/Experimental (may need prompt adjustments) |\n| - | Not applicable |\n\n### Recommendations by Use Case\n\n| Use Case | Recommended Model | Why |\n|----------|-------------------|-----|\n| **Production Chat** | `gpt-4.1` or `gemini-2.5-pro` | Best instruction following, 1M context |\n| **Balanced (Default)** | `gpt-4.1-mini` or `gemini-2.5-flash` | Good quality at lower cost (Policy Bot default) |\n| **Budget Chat** | `gpt-4.1-nano` or `gemini-2.5-flash-lite` | Cost-effective for simple queries |\n| **Deep Reasoning** | `gemini-2.5-pro` | Built-in thinking capabilities |\n| **Local/Offline Chat** | `ollama-llama3.2` or `ollama-qwen2.5` | Full tool support, runs locally |\n| **RAG Embeddings** | `text-embedding-3-large` | Best quality for retrieval (Policy Bot default) |\n| **Local Embeddings** | `ollama-embedding` (nomic) | Good quality, no API cost |\n| **Audio Transcription** | `voxtral-transcribe` | Beats Whisper, half the cost |\n| **Budget Transcription** | `whisper-1` | Well-established, $0.006/min |\n| **Function Calling** | `gpt-4.1-mini`, `gemini-2.5-flash`, `mistral-large-3` | Reliable tool use |\n\n---\n\n## Switching Providers\n\nTo switch your entire application from OpenAI to Ollama:\n\n1. **No code changes needed**\n2. Update model name in your application config:\n\n```python\n# Before (OpenAI)\nMODEL = \"openai-gpt41\"\nEMBEDDING_MODEL = \"openai-embedding-large\"\nAUDIO_MODEL = \"openai-whisper\"\n\n# After (Ollama + Mistral Voxtral for audio)\nMODEL = \"ollama-llama32\"\nEMBEDDING_MODEL = \"ollama-embedding\"\nAUDIO_MODEL = \"voxtral-mini\"  # Or keep openai-whisper\n\n# Or fully Mistral\nMODEL = \"mistral-large-3\"\nEMBEDDING_MODEL = \"mistral-embedding\"\nAUDIO_MODEL = \"voxtral-small\"\n```\n\n---\n\n## Troubleshooting\n\n| Issue | Solution |\n|-------|----------|\n| Connection refused | Check if Docker is running, verify port 4000 |\n| Ollama not reachable | Use `host.docker.internal` in Docker, check Ollama is running |\n| API key errors | Verify `.env` file is loaded, check key format |\n| Model not found | Run `/models` endpoint to list available models |\n| Timeout errors | Increase `request_timeout` in config |\n\n---\n\n## Known Limitations\n\n| # | Limitation | Impact | Status | File |\n|---|-----------|--------|--------|------|\n| 1 | ~~Static YAML cache never invalidated~~ | ~~Runtime model additions invisible until restart~~ | **FIXED** — `clearLiteLLMCache()` now called after `syncAllModelsToLiteLLM()` completes | `litellm-sync.ts` |\n| 2 | ~~Claude tool-calling streaming mismatch~~ | ~~Anthropic models routed through LiteLLM fail tool calling~~ | **FIXED** — Claude models now bypass LiteLLM via `@anthropic-ai/sdk` direct. `isClaudeModel()` routes to `streamAnthropicCompletion()` which receives pre-parsed tool inputs. | `openai.ts` |\n| 3 | **No circuit breaker for chat completions** | If LiteLLM proxy is unreachable, non-Claude chat completions fail. Claude models are unaffected (direct SDK). Embeddings have a fallback model path, but chat/transcription do not. | Open | `openai.ts` |\n| 4 | **Prefix-based provider detection only** | `getProviderFromModelPath()` defaults to `openai` for unrecognized prefixes. Custom/vLLM models without a prefix will be misidentified. No `deepseek/` prefix exists — DeepSeek models are synced dynamically with prefix by `litellm-sync.ts`. | Open | `litellm-validator.ts:282-294` |\n| 5 | **No TTS or image generation via proxy** | `podcast_gen` and `image_gen` must bypass LiteLLM entirely | By design | `podcast-gen.ts`, `image-gen/` |\n\n---\n\n## Resources\n\n- **LiteLLM Docs**: https://docs.litellm.ai/docs/\n- **Supported Providers**: https://docs.litellm.ai/docs/providers\n- **GitHub**: https://github.com/BerriAI/litellm\n- **Ollama Models**: https://ollama.ai/library\n\n---\n\n*Last updated: March 2026 — Added Anthropic Direct SDK (Tier 1b) for Claude chat + tool calling, bypassing LiteLLM. Updated architecture to four-tier hybrid. Fixed Known Limitations #1 (cache invalidation) and #2 (Claude tool-calling). Added `isClaudeModel()` detection logic documentation.*"}, "43778749972440cbd293401f876ee6841d67b862": {"t": 1785754365.0532944, "n": 37673, "h": "3d0f1f61d092d0f8", "c": "# LiteLLM Implementation Guide\n\n> Customized for multi-provider LLM abstraction with proxy approach, embeddings routing, function calling support, and audio transcription.\n\n---\n\n## LLM Service Routing Overview\n\nPolicy Bot uses a **hybrid architecture**: most chat services route through LiteLLM proxy for unified model management, while **Anthropic Claude models bypass LiteLLM entirely** using the `@anthropic-ai/sdk` for direct API access (eliminating tool-calling JSON assembly issues), and specialized services (images, audio, document processing) call provider APIs directly or use local parsers.\n\n### Service Routing Table\n\n| Service | Policy Bot Feature | Routes Through | Provider(s) | Notes |\n|---------|-------------------|----------------|-------------|-------|\n| **Chat Completions** | Main chat, RAG responses | ✅ LiteLLM / ⚡ Anthropic Direct | OpenAI, Gemini, Mistral, DeepSeek, Fireworks AI, Ollama via LiteLLM; **Anthropic Claude via direct SDK** | Claude uses `@anthropic-ai/sdk` for reliable tool calling |\n| **Embeddings** | Document indexing, search | ✅ LiteLLM | OpenAI, Mistral, Gemini, Ollama, Fireworks | `text-embedding-3-large` default |\n| **Diagram Generation** | `diagram_gen` tool | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Generates Mermaid syntax |\n| **Summarization** | Message compression | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic, DeepSeek | Long conversation handling |\n| **Memory Extraction** | User fact storage | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Per-category memory |\n| **Prompt Optimization** | Query refinement | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Pre-RAG processing |\n| **Compliance Checks** | HITL clarification | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Skill compliance |\n| **Translation** | Multi-language support | ✅ LiteLLM / ❌ Direct | OpenAI (proxy), Gemini/Mistral (direct) | Provider-dependent |\n| **Audio Transcription** | Voice input | ✅ LiteLLM | OpenAI Whisper, Mistral Voxtral | `transcribeAudio()` uses `getOpenAI()` client |\n| **Image Generation** | `image_gen` tool | ❌ Direct | OpenAI DALL-E, Gemini Imagen | Specialized APIs |\n| **Podcast Generation** | `podcast_gen` tool | ❌ Direct | OpenAI TTS, Gemini TTS | Multi-voice audio synthesis |\n| **Document Processing** | Document text extraction | ❌ Direct | mammoth/exceljs/officeparser (local), Mistral OCR, Azure DI, pdf-parse | Tiered fallback: local parsers first, then API providers |\n| **Reranking** | Search result scoring | ❌ Direct | Fireworks AI | Direct HTTP to `api.fireworks.ai/inference/v1/rerank` |\n| **PPTX Generation** | `pptx_gen` tool | N/A | None | Template-based (no LLM) |\n| **XLSX Generation** | `xlsx_gen` tool | N/A | None | ExcelJS-based (no LLM) |\n| **Chart Generation** | `chart_gen` tool | N/A | None | Client-side rendering |\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ LiteLLM | Routes through LiteLLM proxy (port 4000) |\n| ⚡ Anthropic Direct | Calls Anthropic API directly via `@anthropic-ai/sdk` (bypasses LiteLLM) |\n| ❌ Direct | Calls provider API directly (bypasses LiteLLM) |\n| N/A | No LLM calls required |\n\n### Why Some Services Bypass LiteLLM\n\n| Service | Reason |\n|---------|--------|\n| **Anthropic Claude** | LiteLLM's Anthropic→OpenAI streaming translation produces malformed tool-calling JSON. Direct `@anthropic-ai/sdk` provides native, reliable tool call parsing |\n| **Image Generation** | DALL-E and Gemini Imagen APIs not supported by LiteLLM proxy |\n| **Podcast TTS** | OpenAI TTS and Gemini TTS are audio generation APIs, not chat completions |\n| **OCR** | Mistral OCR is a specialized document API with unique request format |\n| **Reranking** | Fireworks reranker is a specialized search API (`/inference/v1/rerank`), not a chat model |\n\n> **Note**: Audio transcription (Whisper, Voxtral) routes **through LiteLLM** — `transcribeAudio()` in `openai.ts` uses the same `getOpenAI()` client as chat completions. Both `whisper-1` and `voxtral-mini` are registered in `litellm_config.yaml`.\n\n### Proxy Detection Logic\n\nThe actual detection in `openai.ts` checks for the `OPENAI_BASE_URL` environment variable:\n\n```typescript\n// src/lib/openai.ts — getOpenAI()\nasync function getOpenAI(): Promise<OpenAI> {\n  if (!openaiClient) {\n    // When OPENAI_BASE_URL is set, route through LiteLLM proxy\n    const apiKey = process.env.OPENAI_BASE_URL\n      ? (process.env.LITELLM_MASTER_KEY || await getApiKey('openai'))\n      : await getApiKey('openai');\n\n    openaiClient = new OpenAI({\n      apiKey: apiKey || undefined,\n      baseURL: process.env.OPENAI_BASE_URL || undefined,\n      timeout: 300 * 1000,\n    });\n  }\n  return openaiClient;\n}\n```\n\nWhen `OPENAI_BASE_URL` points to the LiteLLM container (e.g. `http://litellm:4000/v1`), embeddings, transcription, and non-Claude chat completions are routed through a single OpenAI SDK client.\n\n### Claude Direct SDK Detection\n\nFor Anthropic Claude models, `generateResponseWithTools()` bypasses LiteLLM entirely:\n\n```typescript\n// src/lib/openai.ts — isClaudeModel() + routing\nfunction isClaudeModel(model: string): boolean {\n  return model.startsWith('anthropic/') || model.startsWith('claude-');\n}\n\n// In generateResponseWithTools():\nconst useAnthropicDirect = isClaudeModel(effectiveModel);\nconst openai = useAnthropicDirect ? null : await getOpenAI();          // LiteLLM path\nconst anthropicClient = useAnthropicDirect ? await getAnthropicClient() : null;  // Direct path\n```\n\n**Why**: LiteLLM translates Anthropic's native streaming into OpenAI-compatible `delta.tool_calls` format, but the tool call JSON assembly is unreliable — producing malformed arguments that fail `executeTool()`. The direct `@anthropic-ai/sdk` receives pre-parsed tool inputs via `stream.finalMessage()`, eliminating this class of errors.\n\n---\n\n## Policy Bot Integration\n\nThis guide is configured for the Policy Bot RAG application. Default model preset:\n\n| Setting | Value |\n|---------|-------|\n| Default Model | `gpt-4.1-mini` |\n| Temperature | `0.2` |\n| Max Tokens | `2000` |\n| Embedding Model | `text-embedding-3-large` |\n| Embedding Dimensions | `3072` |\n\nAvailable model presets in Policy Bot (via `config/defaults.json`):\n- **gpt-4.1** - High Performance (1M context)\n- **gpt-4.1-mini** - Balanced (default)\n- **gpt-4.1-nano** - Cost-Effective\n- **mistral-large-3** - Mistral Flagship (256K context)\n- **mistral-small-3.2** - Mistral Cost-Effective\n- **ministral-8b** - Mistral Ultra Cost-Effective\n- **gemini-2.5-pro** - Google Flagship Reasoning (1M context)\n- **gemini-2.5-flash** - Google Balanced (1M context)\n- **gemini-2.5-flash-lite** - Google Cost-Effective (1M context)\n- **claude-opus-4-6** - Anthropic Flagship\n- **claude-sonnet-4-6** - Anthropic Balanced\n- **deepseek-chat** - DeepSeek V3 (cost-effective)\n- **deepseek-reasoner** - DeepSeek R1 (thinking model)\n- **fw/kimi-k2-instruct** - Fireworks Kimi K2 (dev/test)\n- **ollama-llama3.2** - Local (no API cost)\n- **ollama-qwen2.5** - Local with excellent reasoning\n\n---\n\n## Architecture Overview\n\nPolicyBot uses a **four-tier hybrid architecture**. LiteLLM handles most chat, embeddings, and transcription. **Anthropic Claude models bypass LiteLLM** via the native `@anthropic-ai/sdk` for reliable tool calling. Specialized services use direct API calls.\n\n```\n┌───────────────────────────────────────────────────────────────────────┐\n│                         PolicyBot Application                         │\n│              (Next.js — openai.ts, tools, translation)                │\n└──┬──────────────────┬──────────────────┬──────────────────┬───────────┘\n   │                  │                  │                  │\n TIER 1            TIER 1b            TIER 2             TIER 3\n LiteLLM Proxy     Anthropic Direct   Direct Provider    Direct Google\n (Chat, Embed,     SDK (@anthropic-   APIs (Non-Chat)    GenAI SDK\n  Transcription)    ai/sdk)                              (Image/TTS)\n   │                  │                  │                  │\n   ▼                  ▼                  ▼                  ▼\n┌────────────────┐ ┌────────────────┐ ┌────────────────┐ ┌────────────────┐\n│  Port 4000     │ │ api.anthropic  │ │                │ │                │\n│                │ │ .com           │ │ Fireworks AI   │ │ Gemini Imagen  │\n│ ┌────────────┐ │ │                │ │  (reranking)   │ │  (image_gen)   │\n│ │ OpenAI     │ │ │ Claude chat +  │ │                │ │  via REST API  │\n│ │ Gemini     │ │ │ tool calling   │ │ Tavily         │ │                │\n│ │ Mistral    │ │ │ via native     │ │  (web_search)  │ │ Gemini TTS     │\n│ │ DeepSeek   │ │ │ streaming      │ │                │ │  (podcast_gen) │\n│ │ Fireworks* │ │ │                │ │ Gemini/Mistral │ │  via @google/  │\n│ │ Ollama*    │ │ │ Models:        │ │  (translation) │ │  genai SDK     │\n│ └────────────┘ │ │ claude-opus-*  │ │                │ │                │\n│                │ │ claude-sonnet-*│ │ OpenAI TTS     │ │ DALL-E 3       │\n│ * YAML-only,   │ │ claude-haiku-* │ │  (podcast_gen) │ │  (image_gen)   │\n│   not dynamic  │ │                │ │  hardcoded to  │ │  via OpenAI SDK│\n│   sync         │ │ Why: LiteLLM   │ │  api.openai.com│ │  (no proxy)    │\n│                │ │ breaks tool    │ │                │ │                │\n│                │ │ call JSON      │ │                │ │                │\n└────────────────┘ └────────────────┘ └────────────────┘ └────────────────┘\n```\n\n### Model Registration: Dynamic vs Static\n\nPolicyBot uses **two registration paths** for LiteLLM models:\n\n| Path | Providers | How |\n|------|-----------|-----|\n| **Dynamic** (via `/model/new` API) | OpenAI, Anthropic, Gemini, Mistral, DeepSeek | `litellm-sync.ts` reads `enabled_models` table and registers via API |\n| **Static** (YAML only) | Fireworks, Ollama, Embeddings, Audio | Defined in `litellm_config.yaml`, skipped by sync service |\n| **Direct SDK** (bypasses LiteLLM) | Anthropic Claude | `isClaudeModel()` in `openai.ts` detects `anthropic/` or `claude-` prefix → routes to `@anthropic-ai/sdk` |\n\n> Dynamic models (5 providers) are NOT in the YAML — they are registered at startup by the app. Only Fireworks, Ollama, embedding, and audio models are declared in `litellm_config.yaml`.\n>\n> **Note**: Anthropic models are still registered dynamically with LiteLLM (for embeddings/non-chat use), but **chat completions with tool calling** bypass LiteLLM entirely via the Anthropic SDK. This is controlled by `isClaudeModel()` in `openai.ts`.\n\n---\n\n## Decision Summary\n\n| Component | Decision |\n|-----------|----------|\n| **Approach** | LiteLLM Proxy (Docker Compose) + Anthropic Direct SDK |\n| **Chat Completions** | Route through LiteLLM (except Claude → `@anthropic-ai/sdk` direct) |\n| **Embeddings** | Route through LiteLLM (per-provider config) |\n| **Function Calling** | Gated by `toolCapable` flag from `litellm_config.yaml` via DB sync |\n| **Audio Transcription** | Route through LiteLLM (Whisper, Voxtral) |\n| **Deployment** | Docker Compose |\n\n---\n\n## Project Structure\n\n```\nlitellm-proxy/\n├── docker-compose.yml\n├── litellm_config.yaml\n├── .env\n└── README.md\n```\n\n---\n\n## Step 1: Environment Variables\n\nCreate `.env` file:\n\n```bash\n# ===================\n# LiteLLM Proxy Keys\n# ===================\nLITELLM_MASTER_KEY=sk-litellm-master-change-this\nLITELLM_SALT_KEY=sk-litellm-salt-change-this\n\n# ===================\n# OpenAI\n# ===================\nOPENAI_API_KEY=sk-...\n\n# ===================\n# Azure OpenAI\n# ===================\nAZURE_API_KEY=...\nAZURE_API_BASE=https://your-resource.openai.azure.com/\nAZURE_API_VERSION=2024-02-15-preview\nAZURE_CHAT_DEPLOYMENT=gpt-4-deployment\nAZURE_EMBEDDING_DEPLOYMENT=text-embedding-ada-002\n\n# ===================\n# Mistral\n# ===================\nMISTRAL_API_KEY=...\n\n# ===================\n# Google Gemini\n# ===================\nGEMINI_API_KEY=...\n\n# ===================\n# Anthropic\n# ===================\nANTHROPIC_API_KEY=...\n\n# ===================\n# DeepSeek\n# ===================\nDEEPSEEK_API_KEY=...\n\n# ===================\n# Fireworks AI (dev/test open-source models)\n# ===================\nFIREWORKS_AI_API_KEY=fw-...\n\n# ===================\n# Ollama (Local)\n# ===================\nOLLAMA_API_BASE=http://host.docker.internal:11434\n# Use 'http://localhost:11434' if running outside Docker\n```\n\n---\n\n## Step 2: LiteLLM Configuration\n\n> **Important**: The reference YAML below is a **complete template** listing all models statically. In actual PolicyBot deployment, chat models for OpenAI, Anthropic, Gemini, Mistral, and DeepSeek are **registered dynamically** via `litellm-sync.ts` (see Architecture Overview above). Only Fireworks, Ollama, embedding, and audio models are declared in the YAML. See the actual config at `litellm-proxy/litellm_config.yaml`.\n\nCreate `litellm_config.yaml`:\n\n```yaml\n# =============================================================================\n# LITELLM PROXY CONFIGURATION\n# Multi-provider setup: OpenAI, Azure, Mistral, Gemini, Anthropic, DeepSeek, Fireworks AI, Ollama\n# Updated: March 2026\n# =============================================================================\n\nmodel_list:\n\n  # ===========================================================================\n  # CHAT COMPLETION MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Models - GPT-4.1 Family (Latest - April 2025+)\n  # ---------------------------------------------------------------------------\n  - model_name: openai-gpt41\n    litellm_params:\n      model: gpt-4.1\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-mini\n    litellm_params:\n      model: gpt-4.1-mini\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-nano\n    litellm_params:\n      model: gpt-4.1-nano\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt35\n    litellm_params:\n      model: gpt-3.5-turbo\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Models - GPT-4.1 Family\n  # ---------------------------------------------------------------------------\n  - model_name: azure-gpt41\n    litellm_params:\n      model: azure/gpt-41-deployment  # Your deployment name\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: azure-gpt41-mini\n    litellm_params:\n      model: azure/gpt-41-mini-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Mistral AI Models - Mistral 3 Family (Latest - December 2025)\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-large-3\n    litellm_params:\n      model: mistral/mistral-large-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 256000\n\n  - model_name: mistral-medium-31\n    litellm_params:\n      model: mistral/mistral-medium-2508\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      supports_vision: true\n\n  - model_name: mistral-small-32\n    litellm_params:\n      model: mistral/mistral-small-2506\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ministral-8b\n    litellm_params:\n      model: mistral/ministral-8b-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Ollama Models (Local) - With Full Tool Support\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-llama32\n    litellm_params:\n      model: ollama/llama3.2\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # Fine-tuned for function calling\n\n  - model_name: ollama-llama31-8b\n    litellm_params:\n      model: ollama/llama3.1:8b\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-mistral\n    litellm_params:\n      model: ollama/mistral\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # v0.3+ supports tools\n\n  - model_name: ollama-qwen25\n    litellm_params:\n      model: ollama/qwen2.5\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-phi4\n    litellm_params:\n      model: ollama/phi4\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: false  # Limited support\n\n  # ===========================================================================\n  # EMBEDDING MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: openai-embedding-large\n    litellm_params:\n      model: text-embedding-3-large\n      api_key: os.environ/OPENAI_API_KEY\n\n  - model_name: openai-embedding-small\n    litellm_params:\n      model: text-embedding-3-small\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: azure-embedding\n    litellm_params:\n      model: azure/text-embedding-ada-002\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  - model_name: azure-embedding-3-large\n    litellm_params:\n      model: azure/text-embedding-3-large-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-embedding\n    litellm_params:\n      model: mistral/mistral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: codestral-embed\n    litellm_params:\n      model: mistral/codestral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Ollama Embeddings (Local)\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-embedding\n    litellm_params:\n      model: ollama/nomic-embed-text\n      api_base: os.environ/OLLAMA_API_BASE\n\n  - model_name: ollama-mxbai-embed\n    litellm_params:\n      model: ollama/mxbai-embed-large\n      api_base: os.environ/OLLAMA_API_BASE\n\n  # ===========================================================================\n  # AUDIO MODELS (Speech-to-Text / Transcription)\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Whisper\n  # ---------------------------------------------------------------------------\n  - model_name: openai-whisper\n    litellm_params:\n      model: whisper-1\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure Whisper (if deployed)\n  # ---------------------------------------------------------------------------\n  - model_name: azure-whisper\n    litellm_params:\n      model: azure/whisper-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Voxtral (Released July 2025 - Beats Whisper at half cost)\n  # ---------------------------------------------------------------------------\n  - model_name: voxtral-small\n    litellm_params:\n      model: mistral/voxtral-small-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: voxtral-mini\n    litellm_params:\n      model: mistral/voxtral-mini-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n# =============================================================================\n# LITELLM SETTINGS\n# =============================================================================\n\nlitellm_settings:\n  # Drop unsupported params instead of erroring\n  drop_params: true\n  \n  # Enable detailed logging\n  set_verbose: false\n\n  # Request timeout\n  request_timeout: 120\n\n# =============================================================================\n# GENERAL SETTINGS\n# =============================================================================\n\ngeneral_settings:\n  # Master key for proxy authentication\n  master_key: os.environ/LITELLM_MASTER_KEY\n```\n\n---\n\n## Step 3: Docker Compose\n\n> **Note**: In the actual PolicyBot deployment, LiteLLM runs as part of the full stack in `docker-compose.local.yml` alongside Postgres, Qdrant, and Redis. The standalone example below is for reference.\n\nCreate `docker-compose.yml`:\n\n```yaml\nversion: \"3.9\"\n\nservices:\n  litellm:\n    image: ghcr.io/berriai/litellm:v1.82.3-stable.patch.2\n    container_name: litellm-proxy\n    ports:\n      - \"4000:4000\"\n    volumes:\n      - ./litellm_config.yaml:/app/config.yaml\n    env_file:\n      - .env\n    environment:\n      - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY}\n      - LITELLM_SALT_KEY=${LITELLM_SALT_KEY}\n    command: --config /app/config.yaml --port 4000 --detailed_debug\n    restart: unless-stopped\n    extra_hosts:\n      - \"host.docker.internal:host-gateway\"\n    healthcheck:\n      test: [\"CMD\", \"curl\", \"-f\", \"http://localhost:4000/health\"]\n      interval: 30s\n      timeout: 10s\n      retries: 3\n\n  # Optional: Ollama service (if not running separately)\n  # ollama:\n  #   image: ollama/ollama:latest\n  #   container_name: ollama\n  #   ports:\n  #     - \"11434:11434\"\n  #   volumes:\n  #     - ollama_data:/root/.ollama\n  #   restart: unless-stopped\n\n# volumes:\n#   ollama_data:\n```\n\n---\n\n## Step 4: Application Code Changes\n\n### Minimal Change Required\n\nPolicyBot is a Next.js (TypeScript) application. The proxy switch is controlled entirely by environment variables — no code changes needed:\n\n**Environment (`.env`):**\n```bash\n# Set this to route all chat/embeddings/transcription through LiteLLM\nOPENAI_BASE_URL=http://litellm:4000/v1\nLITELLM_MASTER_KEY=sk-litellm-master-change-this\n```\n\n**How it works (`src/lib/openai.ts`):**\n```typescript\n// When OPENAI_BASE_URL is set → uses LiteLLM proxy with LITELLM_MASTER_KEY\n// When OPENAI_BASE_URL is unset → calls OpenAI directly with OPENAI_API_KEY\nconst apiKey = process.env.OPENAI_BASE_URL\n  ? (process.env.LITELLM_MASTER_KEY || await getApiKey('openai'))\n  : await getApiKey('openai');\n\nopenaiClient = new OpenAI({\n  apiKey: apiKey || undefined,\n  baseURL: process.env.OPENAI_BASE_URL || undefined,\n});\n```\n\nFor non-Claude models, that's it — the OpenAI SDK handles the routing transparently.\n\n**Claude models** are automatically detected and routed to the Anthropic SDK:\n```typescript\n// Automatic — no env var needed, just ANTHROPIC_API_KEY in DB or env\n// isClaudeModel() detects 'anthropic/' or 'claude-' prefix\n// Uses @anthropic-ai/sdk directly for chat + tool calling\n// Embeddings/transcription still route through LiteLLM\n```\n\n---\n\n## Step 5: Function Calling Handling\n\nSince your `generateResponseWithTools()` uses function calling, here's how to handle provider capabilities:\n\n```python\n# Provider capability mapping (Updated December 2025)\n\n# Full tool/function calling support\n# Matches Policy Bot config/defaults.json toolCapable list\nTOOL_CAPABLE_MODELS = [\n    # OpenAI\n    \"gpt-4.1\", \"gpt-4.1-mini\", \"gpt-4.1-nano\", \"gpt-3.5-turbo\",\n    # Mistral\n    \"mistral-large-3\", \"mistral-medium-3.1\", \"mistral-small-3.2\",\n    \"ministral-8b\", \"ministral-3b\",\n    # Ollama (Now with full tool support!)\n    \"ollama-llama3.2\", \"ollama-llama3.1\", \"ollama-mistral\", \"ollama-qwen2.5\",\n]\n\n# Limited or no tool support (show warning)\nTOOL_LIMITED_MODELS = [\n    \"ollama-phi4\",\n]\n\ndef generate_response_with_tools(model: str, messages: list, tools: list):\n    \"\"\"\n    Generate response with function calling support.\n    Warns and disables tools for unsupported providers.\n    \"\"\"\n    \n    if model in TOOL_LIMITED_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' has limited function calling support.\")\n        print(f\"   Tools will be disabled. Consider using: {', '.join(TOOL_CAPABLE_MODELS[:5])}\")\n        # Call without tools\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages\n        )\n    elif model not in TOOL_CAPABLE_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' tool support unknown. Attempting with tools...\")\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    else:\n        # Full tool support\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    \n    return response\n```\n\n> **Good News (2025 Update)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n---\n\n## Step 6: Embeddings Usage\n\n```python\n# Embeddings via LiteLLM Proxy\n\n# OpenAI embeddings (recommended for RAG quality)\nresponse = client.embeddings.create(\n    model=\"openai-embedding-large\",\n    input=\"Your text here\"\n)\n\n# Azure embeddings\nresponse = client.embeddings.create(\n    model=\"azure-embedding\",\n    input=\"Your text here\"\n)\n\n# Mistral embeddings\nresponse = client.embeddings.create(\n    model=\"mistral-embedding\",\n    input=\"Your text here\"\n)\n\n# Ollama embeddings (local)\nresponse = client.embeddings.create(\n    model=\"ollama-embedding\",\n    input=\"Your text here\"\n)\n\nembedding = response.data[0].embedding\n```\n\n### Important: RAG Consistency\n\n> ⚠️ **Warning**: Don't mix embedding providers for the same vector store. If your RAG was built with `text-embedding-3-large`, query with the same model.\n\n```python\n# Configuration approach\nEMBEDDING_CONFIG = {\n    \"openai\": \"openai-embedding-large\",\n    \"azure\": \"azure-embedding\",\n    \"mistral\": \"mistral-embedding\",\n    \"ollama\": \"ollama-embedding\"\n}\n\n# Use consistently\ncurrent_provider = \"openai\"\nembedding_model = EMBEDDING_CONFIG[current_provider]\n```\n\n---\n\n## Step 7: Audio Transcription (Whisper)\n\n```python\n# Audio transcription via LiteLLM Proxy\n\naudio_file = open(\"audio.mp3\", \"rb\")\n\nresponse = client.audio.transcriptions.create(\n    model=\"openai-whisper\",\n    file=audio_file\n)\n\nprint(response.text)\n```\n\n---\n\n## Deployment Commands\n\n```bash\n# Start the proxy\ndocker compose up -d\n\n# View logs\ndocker compose logs -f litellm\n\n# Stop\ndocker compose down\n\n# Restart after config changes\ndocker compose restart litellm\n```\n\n---\n\n## Testing the Setup\n\n### 1. Health Check\n\n```bash\ncurl http://localhost:4000/health\n```\n\n### 2. List Available Models\n\n```bash\ncurl http://localhost:4000/models \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\"\n```\n\n### 3. Test Chat Completion\n\n```bash\ncurl -X POST http://localhost:4000/chat/completions \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-gpt35\",\n    \"messages\": [{\"role\": \"user\", \"content\": \"Hello, test!\"}]\n  }'\n```\n\n### 4. Test Embeddings\n\n```bash\ncurl -X POST http://localhost:4000/embeddings \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-embedding-large\",\n    \"input\": \"Test embedding\"\n  }'\n```\n\n---\n\n## Model Quick Reference\n\n*Updated: March 2026*\n\n### OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `openai-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Latest flagship, best for coding |\n| `openai-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Faster, 83% cheaper than GPT-4o |\n| `openai-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Fastest, cheapest |\n| `openai-gpt35` | `gpt-3.5-turbo` | ✅ | - | ✅ | - | 16K | Legacy, cost-effective |\n| `openai-embedding-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality embeddings |\n| `openai-embedding-small` | `text-embedding-3-small` | - | ✅ | - | - | 8K | Faster, cheaper |\n| `openai-whisper` | `whisper-1` | - | - | - | ✅ | - | $0.006/min transcription |\n\n### Azure OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `azure-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-embedding` | `text-embedding-ada-002` | - | ✅ | - | - | 8K | Standard embedding |\n| `azure-embedding-3-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality |\n| `azure-whisper` | `whisper` | - | - | - | ✅ | - | Speech-to-text |\n\n### Mistral AI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `mistral-large-3` | `mistral-large-latest` | ✅ | - | ✅ | - | 256K | Flagship MoE (675B total params) |\n| `mistral-medium-31` | `mistral-medium-2508` | ✅ | - | ✅ | - | 128K | Frontier multimodal |\n| `mistral-small-32` | `mistral-small-2506` | ✅ | - | ✅ | - | 128K | Fast, efficient |\n| `ministral-14b` | `ministral-3-14b` | ✅ | - | ✅ | - | 128K | Best small model |\n| `ministral-8b` | `ministral-3-8b` | ✅ | - | ✅ | - | 128K | Edge deployment |\n| `ministral-3b` | `ministral-3-3b` | ✅ | - | ✅ | - | 128K | Ultra-light |\n| `mistral-embedding` | `mistral-embed` | - | ✅ | - | - | 8K | Text embeddings |\n| `codestral-embed` | `codestral-embed` | - | ✅ | - | - | - | Code embeddings |\n| `voxtral-small` | `voxtral-small-latest` | - | - | - | ✅ | 32K | 24B, best accuracy, $0.001/min |\n| `voxtral-mini` | `voxtral-mini-latest` | - | - | - | ✅ | 32K | 3B, edge/local deployment |\n| `voxtral-transcribe` | `voxtral-mini-latest` (via /audio/transcriptions) | - | - | - | ✅ | - | Transcription-only, beats Whisper |\n\n> **Note**: Voxtral (released July 2025) outperforms OpenAI Whisper at half the cost ($0.001/min vs $0.006/min). Supports up to 30 min audio for transcription, 40 min for understanding.\n\n### Google Gemini Models\n\n| Model Name | API Model ID | Chat | Tools | Context | Notes |\n|------------|--------------|------|-------|---------|-------|\n| `gemini-2.5-pro` | `gemini/gemini-2.5-pro` | ✅ | ✅ | 1M | Flagship reasoning model |\n| `gemini-2.5-flash` | `gemini/gemini-2.5-flash` | ✅ | ✅ | 1M | Fast, balanced performance |\n| `gemini-2.5-flash-lite` | `gemini/gemini-2.5-flash-lite` | ✅ | ✅ | 1M | Lowest cost option |\n\n**Pricing (per 1M tokens):**\n- Gemini 2.5 Pro: $1.25 input / $10.00 output (≤200k context)\n- Gemini 2.5 Flash: $0.30 input / $2.50 output\n- Gemini 2.5 Flash-Lite: $0.10 input / $0.40 output\n\n**Features:**\n- All models support function calling\n- Built-in \"thinking\" capabilities with controllable budgets\n- Free tier available for development/testing\n- Native multimodal support (text, image, audio, video)\n\n### Ollama Local Models\n\n| Model Name | Ollama Model | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `ollama-llama32` | `llama3.2` | ✅ | - | ✅ | - | 128K | **Full tool support** (fine-tuned for function calling) |\n| `ollama-llama31-8b` | `llama3.1:8b` | ✅ | - | ✅ | - | 128K | **Full tool support**, best overall |\n| `ollama-llama31-70b` | `llama3.1:70b` | ✅ | - | ✅ | - | 128K | **Full tool support**, highest accuracy |\n| `ollama-mistral` | `mistral` | ✅ | - | ✅ | - | 32K | **Supports tools** (v0.3+) |\n| `ollama-mixtral` | `mixtral:8x7b` | ✅ | - | ✅ | - | 32K | MoE, good multi-domain |\n| `ollama-qwen25` | `qwen2.5` | ✅ | - | ✅ | - | 128K | Strong tool support |\n| `ollama-phi4` | `phi4` | ✅ | - | ⚠️ | - | 16K | Limited tool support |\n| `ollama-embedding` | `nomic-embed-text` | - | ✅ | - | - | 8K | Best local embedding |\n| `ollama-mxbai-embed` | `mxbai-embed-large` | - | ✅ | - | - | 512 | Alternative embedding |\n\n> **Tool Calling Update (2025)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), Mixtral, and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ | Fully Supported |\n| ⚠️ | Limited/Experimental (may need prompt adjustments) |\n| - | Not applicable |\n\n### Recommendations by Use Case\n\n| Use Case | Recommended Model | Why |\n|----------|-------------------|-----|\n| **Production Chat** | `gpt-4.1` or `gemini-2.5-pro` | Best instruction following, 1M context |\n| **Balanced (Default)** | `gpt-4.1-mini` or `gemini-2.5-flash` | Good quality at lower cost (Policy Bot default) |\n| **Budget Chat** | `gpt-4.1-nano` or `gemini-2.5-flash-lite` | Cost-effective for simple queries |\n| **Deep Reasoning** | `gemini-2.5-pro` | Built-in thinking capabilities |\n| **Local/Offline Chat** | `ollama-llama3.2` or `ollama-qwen2.5` | Full tool support, runs locally |\n| **RAG Embeddings** | `text-embedding-3-large` | Best quality for retrieval (Policy Bot default) |\n| **Local Embeddings** | `ollama-embedding` (nomic) | Good quality, no API cost |\n| **Audio Transcription** | `voxtral-transcribe` | Beats Whisper, half the cost |\n| **Budget Transcription** | `whisper-1` | Well-established, $0.006/min |\n| **Function Calling** | `gpt-4.1-mini`, `gemini-2.5-flash`, `mistral-large-3` | Reliable tool use |\n\n---\n\n## Switching Providers\n\nTo switch your entire application from OpenAI to Ollama:\n\n1. **No code changes needed**\n2. Update model name in your application config:\n\n```python\n# Before (OpenAI)\nMODEL = \"openai-gpt41\"\nEMBEDDING_MODEL = \"openai-embedding-large\"\nAUDIO_MODEL = \"openai-whisper\"\n\n# After (Ollama + Mistral Voxtral for audio)\nMODEL = \"ollama-llama32\"\nEMBEDDING_MODEL = \"ollama-embedding\"\nAUDIO_MODEL = \"voxtral-mini\"  # Or keep openai-whisper\n\n# Or fully Mistral\nMODEL = \"mistral-large-3\"\nEMBEDDING_MODEL = \"mistral-embedding\"\nAUDIO_MODEL = \"voxtral-small\"\n```\n\n---\n\n## Troubleshooting\n\n| Issue | Solution |\n|-------|----------|\n| Connection refused | Check if Docker is running, verify port 4000 |\n| Ollama not reachable | Use `host.docker.internal` in Docker, check Ollama is running |\n| API key errors | Verify `.env` file is loaded, check key format |\n| Model not found | Run `/models` endpoint to list available models |\n| Timeout errors | Increase `request_timeout` in config |\n\n---\n\n## Known Limitations\n\n| # | Limitation | Impact | Status | File |\n|---|-----------|--------|--------|------|\n| 1 | ~~Static YAML cache never invalidated~~ | ~~Runtime model additions invisible until restart~~ | **FIXED** — `clearLiteLLMCache()` now called after `syncAllModelsToLiteLLM()` completes | `litellm-sync.ts` |\n| 2 | ~~Claude tool-calling streaming mismatch~~ | ~~Anthropic models routed through LiteLLM fail tool calling~~ | **FIXED** — Claude models now bypass LiteLLM via `@anthropic-ai/sdk` direct. `isClaudeModel()` routes to `streamAnthropicCompletion()` which receives pre-parsed tool inputs. | `openai.ts` |\n| 3 | **No circuit breaker for chat completions** | If LiteLLM proxy is unreachable, non-Claude chat completions fail. Claude models are unaffected (direct SDK). Embeddings have a fallback model path, but chat/transcription do not. | Open | `openai.ts` |\n| 4 | **Prefix-based provider detection only** | `getProviderFromModelPath()` defaults to `openai` for unrecognized prefixes. Custom/vLLM models without a prefix will be misidentified. No `deepseek/` prefix exists — DeepSeek models are synced dynamically with prefix by `litellm-sync.ts`. | Open | `litellm-validator.ts:282-294` |\n| 5 | **No TTS or image generation via proxy** | `podcast_gen` and `image_gen` must bypass LiteLLM entirely | By design | `podcast-gen.ts`, `image-gen/` |\n\n---\n\n## Resources\n\n- **LiteLLM Docs**: https://docs.litellm.ai/docs/\n- **Supported Providers**: https://docs.litellm.ai/docs/providers\n- **GitHub**: https://github.com/BerriAI/litellm\n- **Ollama Models**: https://ollama.ai/library\n\n---\n\n*Last updated: March 2026 — Added Anthropic Direct SDK (Tier 1b) for Claude chat + tool calling, bypassing LiteLLM. Updated architecture to four-tier hybrid. Fixed Known Limitations #1 (cache invalidation) and #2 (Claude tool-calling). Added `isClaudeModel()` detection logic documentation.*"}, "c4a62d2421e4876b373e1f646010c9abd4308a1d": {"t": 1785754366.7588344, "n": 34936, "h": "9cfa3a4e2cafd7e2", "c": "# LiteLLM Implementation Guide\n\n> Customized for multi-provider LLM abstraction with proxy approach, embeddings routing, function calling support, and audio transcription.\n\n---\n\n## LLM Service Routing Overview\n\nPolicy Bot uses a **hybrid architecture**: core chat services route through LiteLLM proxy for unified model management, while specialized services (images, audio, document processing) call provider APIs directly or use local parsers for capabilities not available through the proxy.\n\n### Service Routing Table\n\n| Service | Policy Bot Feature | Routes Through | Provider(s) | Notes |\n|---------|-------------------|----------------|-------------|-------|\n| **Chat Completions** | Main chat, RAG responses | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic, DeepSeek, Fireworks AI, Ollama | Primary conversation engine |\n| **Embeddings** | Document indexing, search | ✅ LiteLLM | OpenAI, Mistral, Gemini, Ollama, Fireworks | `text-embedding-3-large` default |\n| **Diagram Generation** | `diagram_gen` tool | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Generates Mermaid syntax |\n| **Summarization** | Message compression | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic, DeepSeek | Long conversation handling |\n| **Memory Extraction** | User fact storage | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Per-category memory |\n| **Prompt Optimization** | Query refinement | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Pre-RAG processing |\n| **Compliance Checks** | HITL clarification | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Skill compliance |\n| **Translation** | Multi-language support | ✅ LiteLLM / ❌ Direct | OpenAI (proxy), Gemini/Mistral (direct) | Provider-dependent |\n| **Audio Transcription** | Voice input | ✅ LiteLLM | OpenAI Whisper, Mistral Voxtral | `transcribeAudio()` uses `getOpenAI()` client |\n| **Image Generation** | `image_gen` tool | ❌ Direct | OpenAI DALL-E, Gemini Imagen | Specialized APIs |\n| **Podcast Generation** | `podcast_gen` tool | ❌ Direct | OpenAI TTS, Gemini TTS | Multi-voice audio synthesis |\n| **Document Processing** | Document text extraction | ❌ Direct | mammoth/exceljs/officeparser (local), Mistral OCR, Azure DI, pdf-parse | Tiered fallback: local parsers first, then API providers |\n| **Reranking** | Search result scoring | ❌ Direct | Fireworks AI | Direct HTTP to `api.fireworks.ai/inference/v1/rerank` |\n| **PPTX Generation** | `pptx_gen` tool | N/A | None | Template-based (no LLM) |\n| **XLSX Generation** | `xlsx_gen` tool | N/A | None | ExcelJS-based (no LLM) |\n| **Chart Generation** | `chart_gen` tool | N/A | None | Client-side rendering |\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ LiteLLM | Routes through LiteLLM proxy (port 4000) |\n| ❌ Direct | Calls provider API directly (bypasses LiteLLM) |\n| N/A | No LLM calls required |\n\n### Why Some Services Bypass LiteLLM\n\n| Service | Reason |\n|---------|--------|\n| **Image Generation** | DALL-E and Gemini Imagen APIs not supported by LiteLLM proxy |\n| **Podcast TTS** | OpenAI TTS and Gemini TTS are audio generation APIs, not chat completions |\n| **OCR** | Mistral OCR is a specialized document API with unique request format |\n| **Reranking** | Fireworks reranker is a specialized search API (`/inference/v1/rerank`), not a chat model |\n\n> **Note**: Audio transcription (Whisper, Voxtral) routes **through LiteLLM** — `transcribeAudio()` in `openai.ts` uses the same `getOpenAI()` client as chat completions. Both `whisper-1` and `voxtral-mini` are registered in `litellm_config.yaml`.\n\n### Proxy Detection Logic\n\nThe actual detection in `openai.ts` checks for the `OPENAI_BASE_URL` environment variable:\n\n```typescript\n// src/lib/openai.ts — getOpenAI()\nasync function getOpenAI(): Promise<OpenAI> {\n  if (!openaiClient) {\n    // When OPENAI_BASE_URL is set, route through LiteLLM proxy\n    const apiKey = process.env.OPENAI_BASE_URL\n      ? (process.env.LITELLM_MASTER_KEY || await getApiKey('openai'))\n      : await getApiKey('openai');\n\n    openaiClient = new OpenAI({\n      apiKey: apiKey || undefined,\n      baseURL: process.env.OPENAI_BASE_URL || undefined,\n      timeout: 300 * 1000,\n    });\n  }\n  return openaiClient;\n}\n```\n\nWhen `OPENAI_BASE_URL` points to the LiteLLM container (e.g. `http://litellm:4000/v1`), all chat completions, embeddings, and transcription are routed through a single OpenAI SDK client.\n\n---\n\n## Policy Bot Integration\n\nThis guide is configured for the Policy Bot RAG application. Default model preset:\n\n| Setting | Value |\n|---------|-------|\n| Default Model | `gpt-4.1-mini` |\n| Temperature | `0.2` |\n| Max Tokens | `2000` |\n| Embedding Model | `text-embedding-3-large` |\n| Embedding Dimensions | `3072` |\n\nAvailable model presets in Policy Bot (via `config/defaults.json`):\n- **gpt-4.1** - High Performance (1M context)\n- **gpt-4.1-mini** - Balanced (default)\n- **gpt-4.1-nano** - Cost-Effective\n- **mistral-large-3** - Mistral Flagship (256K context)\n- **mistral-small-3.2** - Mistral Cost-Effective\n- **ministral-8b** - Mistral Ultra Cost-Effective\n- **gemini-2.5-pro** - Google Flagship Reasoning (1M context)\n- **gemini-2.5-flash** - Google Balanced (1M context)\n- **gemini-2.5-flash-lite** - Google Cost-Effective (1M context)\n- **claude-opus-4-6** - Anthropic Flagship\n- **claude-sonnet-4-6** - Anthropic Balanced\n- **deepseek-chat** - DeepSeek V3 (cost-effective)\n- **deepseek-reasoner** - DeepSeek R1 (thinking model)\n- **fw/kimi-k2-instruct** - Fireworks Kimi K2 (dev/test)\n- **ollama-llama3.2** - Local (no API cost)\n- **ollama-qwen2.5** - Local with excellent reasoning\n\n---\n\n## Architecture Overview\n\nPolicyBot uses a **three-tier hybrid architecture**. LiteLLM handles chat, embeddings, and transcription. Specialized services bypass LiteLLM via direct API calls.\n\n```\n┌───────────────────────────────────────────────────────────────────────┐\n│                         PolicyBot Application                         │\n│              (Next.js — openai.ts, tools, translation)                │\n└──────┬──────────────────────┬──────────────────────┬──────────────────┘\n       │                      │                      │\n   TIER 1                  TIER 2                 TIER 3\n   LiteLLM Proxy           Direct Provider        Direct Google\n   (Chat, Embed,           APIs (Non-Chat)        GenAI SDK\n    Transcription)                                 (Image/TTS)\n       │                      │                      │\n       ▼                      ▼                      ▼\n┌──────────────────┐  ┌──────────────────┐  ┌──────────────────┐\n│  Port 4000       │  │                  │  │                  │\n│                  │  │ Fireworks AI     │  │ Gemini Imagen    │\n│ ┌──────────────┐ │  │  (reranking)     │  │  (image_gen)     │\n│ │ OpenAI       │ │  │                  │  │  via REST API    │\n│ │ Anthropic    │ │  │ Tavily           │  │                  │\n│ │ Gemini       │ │  │  (web_search)    │  │ Gemini TTS       │\n│ │ Mistral      │ │  │                  │  │  (podcast_gen)   │\n│ │ DeepSeek     │ │  │ Gemini/Mistral   │  │  via @google/    │\n│ │ Fireworks*   │ │  │  (translation)   │  │  genai SDK       │\n│ │ Ollama*      │ │  │                  │  │                  │\n│ └──────────────┘ │  │ OpenAI TTS       │  │ DALL-E 3         │\n│                  │  │  (podcast_gen)   │  │  (image_gen)     │\n│ * YAML-only,     │  │  hardcoded to    │  │  via OpenAI SDK  │\n│   not dynamic    │  │  api.openai.com  │  │  (no proxy)      │\n│   sync           │  │                  │  │                  │\n└──────────────────┘  └──────────────────┘  └──────────────────┘\n```\n\n### Model Registration: Dynamic vs Static\n\nPolicyBot uses **two registration paths** for LiteLLM models:\n\n| Path | Providers | How |\n|------|-----------|-----|\n| **Dynamic** (via `/model/new` API) | OpenAI, Anthropic, Gemini, Mistral, DeepSeek | `litellm-sync.ts` reads `enabled_models` table and registers via API |\n| **Static** (YAML only) | Fireworks, Ollama, Embeddings, Audio | Defined in `litellm_config.yaml`, skipped by sync service |\n\n> Dynamic models (5 providers) are NOT in the YAML — they are registered at startup by the app. Only Fireworks, Ollama, embedding, and audio models are declared in `litellm_config.yaml`.\n\n---\n\n## Decision Summary\n\n| Component | Decision |\n|-----------|----------|\n| **Approach** | LiteLLM Proxy (Docker Compose) |\n| **Chat Completions** | Route through LiteLLM |\n| **Embeddings** | Route through LiteLLM (per-provider config) |\n| **Function Calling** | Gated by `toolCapable` flag from `litellm_config.yaml` via DB sync |\n| **Audio Transcription** | Route through LiteLLM (Whisper, Voxtral) |\n| **Deployment** | Docker Compose |\n\n---\n\n## Project Structure\n\n```\nlitellm-proxy/\n├── docker-compose.yml\n├── litellm_config.yaml\n├── .env\n└── README.md\n```\n\n---\n\n## Step 1: Environment Variables\n\nCreate `.env` file:\n\n```bash\n# ===================\n# LiteLLM Proxy Keys\n# ===================\nLITELLM_MASTER_KEY=sk-litellm-master-change-this\nLITELLM_SALT_KEY=sk-litellm-salt-change-this\n\n# ===================\n# OpenAI\n# ===================\nOPENAI_API_KEY=sk-...\n\n# ===================\n# Azure OpenAI\n# ===================\nAZURE_API_KEY=...\nAZURE_API_BASE=https://your-resource.openai.azure.com/\nAZURE_API_VERSION=2024-02-15-preview\nAZURE_CHAT_DEPLOYMENT=gpt-4-deployment\nAZURE_EMBEDDING_DEPLOYMENT=text-embedding-ada-002\n\n# ===================\n# Mistral\n# ===================\nMISTRAL_API_KEY=...\n\n# ===================\n# Google Gemini\n# ===================\nGEMINI_API_KEY=...\n\n# ===================\n# Anthropic\n# ===================\nANTHROPIC_API_KEY=...\n\n# ===================\n# DeepSeek\n# ===================\nDEEPSEEK_API_KEY=...\n\n# ===================\n# Fireworks AI (dev/test open-source models)\n# ===================\nFIREWORKS_AI_API_KEY=fw-...\n\n# ===================\n# Ollama (Local)\n# ===================\nOLLAMA_API_BASE=http://host.docker.internal:11434\n# Use 'http://localhost:11434' if running outside Docker\n```\n\n---\n\n## Step 2: LiteLLM Configuration\n\n> **Important**: The reference YAML below is a **complete template** listing all models statically. In actual PolicyBot deployment, chat models for OpenAI, Anthropic, Gemini, Mistral, and DeepSeek are **registered dynamically** via `litellm-sync.ts` (see Architecture Overview above). Only Fireworks, Ollama, embedding, and audio models are declared in the YAML. See the actual config at `litellm-proxy/litellm_config.yaml`.\n\nCreate `litellm_config.yaml`:\n\n```yaml\n# =============================================================================\n# LITELLM PROXY CONFIGURATION\n# Multi-provider setup: OpenAI, Azure, Mistral, Gemini, Anthropic, DeepSeek, Fireworks AI, Ollama\n# Updated: March 2026\n# =============================================================================\n\nmodel_list:\n\n  # ===========================================================================\n  # CHAT COMPLETION MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Models - GPT-4.1 Family (Latest - April 2025+)\n  # ---------------------------------------------------------------------------\n  - model_name: openai-gpt41\n    litellm_params:\n      model: gpt-4.1\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-mini\n    litellm_params:\n      model: gpt-4.1-mini\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-nano\n    litellm_params:\n      model: gpt-4.1-nano\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt35\n    litellm_params:\n      model: gpt-3.5-turbo\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Models - GPT-4.1 Family\n  # ---------------------------------------------------------------------------\n  - model_name: azure-gpt41\n    litellm_params:\n      model: azure/gpt-41-deployment  # Your deployment name\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: azure-gpt41-mini\n    litellm_params:\n      model: azure/gpt-41-mini-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Mistral AI Models - Mistral 3 Family (Latest - December 2025)\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-large-3\n    litellm_params:\n      model: mistral/mistral-large-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 256000\n\n  - model_name: mistral-medium-31\n    litellm_params:\n      model: mistral/mistral-medium-2508\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      supports_vision: true\n\n  - model_name: mistral-small-32\n    litellm_params:\n      model: mistral/mistral-small-2506\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ministral-8b\n    litellm_params:\n      model: mistral/ministral-8b-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Ollama Models (Local) - With Full Tool Support\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-llama32\n    litellm_params:\n      model: ollama/llama3.2\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # Fine-tuned for function calling\n\n  - model_name: ollama-llama31-8b\n    litellm_params:\n      model: ollama/llama3.1:8b\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-mistral\n    litellm_params:\n      model: ollama/mistral\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # v0.3+ supports tools\n\n  - model_name: ollama-qwen25\n    litellm_params:\n      model: ollama/qwen2.5\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-phi4\n    litellm_params:\n      model: ollama/phi4\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: false  # Limited support\n\n  # ===========================================================================\n  # EMBEDDING MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: openai-embedding-large\n    litellm_params:\n      model: text-embedding-3-large\n      api_key: os.environ/OPENAI_API_KEY\n\n  - model_name: openai-embedding-small\n    litellm_params:\n      model: text-embedding-3-small\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: azure-embedding\n    litellm_params:\n      model: azure/text-embedding-ada-002\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  - model_name: azure-embedding-3-large\n    litellm_params:\n      model: azure/text-embedding-3-large-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-embedding\n    litellm_params:\n      model: mistral/mistral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: codestral-embed\n    litellm_params:\n      model: mistral/codestral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Ollama Embeddings (Local)\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-embedding\n    litellm_params:\n      model: ollama/nomic-embed-text\n      api_base: os.environ/OLLAMA_API_BASE\n\n  - model_name: ollama-mxbai-embed\n    litellm_params:\n      model: ollama/mxbai-embed-large\n      api_base: os.environ/OLLAMA_API_BASE\n\n  # ===========================================================================\n  # AUDIO MODELS (Speech-to-Text / Transcription)\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Whisper\n  # ---------------------------------------------------------------------------\n  - model_name: openai-whisper\n    litellm_params:\n      model: whisper-1\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure Whisper (if deployed)\n  # ---------------------------------------------------------------------------\n  - model_name: azure-whisper\n    litellm_params:\n      model: azure/whisper-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Voxtral (Released July 2025 - Beats Whisper at half cost)\n  # ---------------------------------------------------------------------------\n  - model_name: voxtral-small\n    litellm_params:\n      model: mistral/voxtral-small-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: voxtral-mini\n    litellm_params:\n      model: mistral/voxtral-mini-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n# =============================================================================\n# LITELLM SETTINGS\n# =============================================================================\n\nlitellm_settings:\n  # Drop unsupported params instead of erroring\n  drop_params: true\n  \n  # Enable detailed logging\n  set_verbose: false\n\n  # Request timeout\n  request_timeout: 120\n\n# =============================================================================\n# GENERAL SETTINGS\n# =============================================================================\n\ngeneral_settings:\n  # Master key for proxy authentication\n  master_key: os.environ/LITELLM_MASTER_KEY\n```\n\n---\n\n## Step 3: Docker Compose\n\n> **Note**: In the actual PolicyBot deployment, LiteLLM runs as part of the full stack in `docker-compose.local.yml` alongside Postgres, Qdrant, and Redis. The standalone example below is for reference.\n\nCreate `docker-compose.yml`:\n\n```yaml\nversion: \"3.9\"\n\nservices:\n  litellm:\n    image: ghcr.io/berriai/litellm:v1.82.3-stable.patch.2\n    container_name: litellm-proxy\n    ports:\n      - \"4000:4000\"\n    volumes:\n      - ./litellm_config.yaml:/app/config.yaml\n    env_file:\n      - .env\n    environment:\n      - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY}\n      - LITELLM_SALT_KEY=${LITELLM_SALT_KEY}\n    command: --config /app/config.yaml --port 4000 --detailed_debug\n    restart: unless-stopped\n    extra_hosts:\n      - \"host.docker.internal:host-gateway\"\n    healthcheck:\n      test: [\"CMD\", \"curl\", \"-f\", \"http://localhost:4000/health\"]\n      interval: 30s\n      timeout: 10s\n      retries: 3\n\n  # Optional: Ollama service (if not running separately)\n  # ollama:\n  #   image: ollama/ollama:latest\n  #   container_name: ollama\n  #   ports:\n  #     - \"11434:11434\"\n  #   volumes:\n  #     - ollama_data:/root/.ollama\n  #   restart: unless-stopped\n\n# volumes:\n#   ollama_data:\n```\n\n---\n\n## Step 4: Application Code Changes\n\n### Minimal Change Required\n\nPolicyBot is a Next.js (TypeScript) application. The proxy switch is controlled entirely by environment variables — no code changes needed:\n\n**Environment (`.env`):**\n```bash\n# Set this to route all chat/embeddings/transcription through LiteLLM\nOPENAI_BASE_URL=http://litellm:4000/v1\nLITELLM_MASTER_KEY=sk-litellm-master-change-this\n```\n\n**How it works (`src/lib/openai.ts`):**\n```typescript\n// When OPENAI_BASE_URL is set → uses LiteLLM proxy with LITELLM_MASTER_KEY\n// When OPENAI_BASE_URL is unset → calls OpenAI directly with OPENAI_API_KEY\nconst apiKey = process.env.OPENAI_BASE_URL\n  ? (process.env.LITELLM_MASTER_KEY || await getApiKey('openai'))\n  : await getApiKey('openai');\n\nopenaiClient = new OpenAI({\n  apiKey: apiKey || undefined,\n  baseURL: process.env.OPENAI_BASE_URL || undefined,\n});\n```\n\nThat's it! All other code stays the same — the OpenAI SDK handles the routing transparently.\n\n---\n\n## Step 5: Function Calling Handling\n\nSince your `generateResponseWithTools()` uses function calling, here's how to handle provider capabilities:\n\n```python\n# Provider capability mapping (Updated December 2025)\n\n# Full tool/function calling support\n# Matches Policy Bot config/defaults.json toolCapable list\nTOOL_CAPABLE_MODELS = [\n    # OpenAI\n    \"gpt-4.1\", \"gpt-4.1-mini\", \"gpt-4.1-nano\", \"gpt-3.5-turbo\",\n    # Mistral\n    \"mistral-large-3\", \"mistral-medium-3.1\", \"mistral-small-3.2\",\n    \"ministral-8b\", \"ministral-3b\",\n    # Ollama (Now with full tool support!)\n    \"ollama-llama3.2\", \"ollama-llama3.1\", \"ollama-mistral\", \"ollama-qwen2.5\",\n]\n\n# Limited or no tool support (show warning)\nTOOL_LIMITED_MODELS = [\n    \"ollama-phi4\",\n]\n\ndef generate_response_with_tools(model: str, messages: list, tools: list):\n    \"\"\"\n    Generate response with function calling support.\n    Warns and disables tools for unsupported providers.\n    \"\"\"\n    \n    if model in TOOL_LIMITED_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' has limited function calling support.\")\n        print(f\"   Tools will be disabled. Consider using: {', '.join(TOOL_CAPABLE_MODELS[:5])}\")\n        # Call without tools\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages\n        )\n    elif model not in TOOL_CAPABLE_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' tool support unknown. Attempting with tools...\")\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    else:\n        # Full tool support\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    \n    return response\n```\n\n> **Good News (2025 Update)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n---\n\n## Step 6: Embeddings Usage\n\n```python\n# Embeddings via LiteLLM Proxy\n\n# OpenAI embeddings (recommended for RAG quality)\nresponse = client.embeddings.create(\n    model=\"openai-embedding-large\",\n    input=\"Your text here\"\n)\n\n# Azure embeddings\nresponse = client.embeddings.create(\n    model=\"azure-embedding\",\n    input=\"Your text here\"\n)\n\n# Mistral embeddings\nresponse = client.embeddings.create(\n    model=\"mistral-embedding\",\n    input=\"Your text here\"\n)\n\n# Ollama embeddings (local)\nresponse = client.embeddings.create(\n    model=\"ollama-embedding\",\n    input=\"Your text here\"\n)\n\nembedding = response.data[0].embedding\n```\n\n### Important: RAG Consistency\n\n> ⚠️ **Warning**: Don't mix embedding providers for the same vector store. If your RAG was built with `text-embedding-3-large`, query with the same model.\n\n```python\n# Configuration approach\nEMBEDDING_CONFIG = {\n    \"openai\": \"openai-embedding-large\",\n    \"azure\": \"azure-embedding\",\n    \"mistral\": \"mistral-embedding\",\n    \"ollama\": \"ollama-embedding\"\n}\n\n# Use consistently\ncurrent_provider = \"openai\"\nembedding_model = EMBEDDING_CONFIG[current_provider]\n```\n\n---\n\n## Step 7: Audio Transcription (Whisper)\n\n```python\n# Audio transcription via LiteLLM Proxy\n\naudio_file = open(\"audio.mp3\", \"rb\")\n\nresponse = client.audio.transcriptions.create(\n    model=\"openai-whisper\",\n    file=audio_file\n)\n\nprint(response.text)\n```\n\n---\n\n## Deployment Commands\n\n```bash\n# Start the proxy\ndocker compose up -d\n\n# View logs\ndocker compose logs -f litellm\n\n# Stop\ndocker compose down\n\n# Restart after config changes\ndocker compose restart litellm\n```\n\n---\n\n## Testing the Setup\n\n### 1. Health Check\n\n```bash\ncurl http://localhost:4000/health\n```\n\n### 2. List Available Models\n\n```bash\ncurl http://localhost:4000/models \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\"\n```\n\n### 3. Test Chat Completion\n\n```bash\ncurl -X POST http://localhost:4000/chat/completions \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-gpt35\",\n    \"messages\": [{\"role\": \"user\", \"content\": \"Hello, test!\"}]\n  }'\n```\n\n### 4. Test Embeddings\n\n```bash\ncurl -X POST http://localhost:4000/embeddings \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-embedding-large\",\n    \"input\": \"Test embedding\"\n  }'\n```\n\n---\n\n## Model Quick Reference\n\n*Updated: March 2026*\n\n### OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `openai-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Latest flagship, best for coding |\n| `openai-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Faster, 83% cheaper than GPT-4o |\n| `openai-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Fastest, cheapest |\n| `openai-gpt35` | `gpt-3.5-turbo` | ✅ | - | ✅ | - | 16K | Legacy, cost-effective |\n| `openai-embedding-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality embeddings |\n| `openai-embedding-small` | `text-embedding-3-small` | - | ✅ | - | - | 8K | Faster, cheaper |\n| `openai-whisper` | `whisper-1` | - | - | - | ✅ | - | $0.006/min transcription |\n\n### Azure OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `azure-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-embedding` | `text-embedding-ada-002` | - | ✅ | - | - | 8K | Standard embedding |\n| `azure-embedding-3-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality |\n| `azure-whisper` | `whisper` | - | - | - | ✅ | - | Speech-to-text |\n\n### Mistral AI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `mistral-large-3` | `mistral-large-latest` | ✅ | - | ✅ | - | 256K | Flagship MoE (675B total params) |\n| `mistral-medium-31` | `mistral-medium-2508` | ✅ | - | ✅ | - | 128K | Frontier multimodal |\n| `mistral-small-32` | `mistral-small-2506` | ✅ | - | ✅ | - | 128K | Fast, efficient |\n| `ministral-14b` | `ministral-3-14b` | ✅ | - | ✅ | - | 128K | Best small model |\n| `ministral-8b` | `ministral-3-8b` | ✅ | - | ✅ | - | 128K | Edge deployment |\n| `ministral-3b` | `ministral-3-3b` | ✅ | - | ✅ | - | 128K | Ultra-light |\n| `mistral-embedding` | `mistral-embed` | - | ✅ | - | - | 8K | Text embeddings |\n| `codestral-embed` | `codestral-embed` | - | ✅ | - | - | - | Code embeddings |\n| `voxtral-small` | `voxtral-small-latest` | - | - | - | ✅ | 32K | 24B, best accuracy, $0.001/min |\n| `voxtral-mini` | `voxtral-mini-latest` | - | - | - | ✅ | 32K | 3B, edge/local deployment |\n| `voxtral-transcribe` | `voxtral-mini-latest` (via /audio/transcriptions) | - | - | - | ✅ | - | Transcription-only, beats Whisper |\n\n> **Note**: Voxtral (released July 2025) outperforms OpenAI Whisper at half the cost ($0.001/min vs $0.006/min). Supports up to 30 min audio for transcription, 40 min for understanding.\n\n### Google Gemini Models\n\n| Model Name | API Model ID | Chat | Tools | Context | Notes |\n|------------|--------------|------|-------|---------|-------|\n| `gemini-2.5-pro` | `gemini/gemini-2.5-pro` | ✅ | ✅ | 1M | Flagship reasoning model |\n| `gemini-2.5-flash` | `gemini/gemini-2.5-flash` | ✅ | ✅ | 1M | Fast, balanced performance |\n| `gemini-2.5-flash-lite` | `gemini/gemini-2.5-flash-lite` | ✅ | ✅ | 1M | Lowest cost option |\n\n**Pricing (per 1M tokens):**\n- Gemini 2.5 Pro: $1.25 input / $10.00 output (≤200k context)\n- Gemini 2.5 Flash: $0.30 input / $2.50 output\n- Gemini 2.5 Flash-Lite: $0.10 input / $0.40 output\n\n**Features:**\n- All models support function calling\n- Built-in \"thinking\" capabilities with controllable budgets\n- Free tier available for development/testing\n- Native multimodal support (text, image, audio, video)\n\n### Ollama Local Models\n\n| Model Name | Ollama Model | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `ollama-llama32` | `llama3.2` | ✅ | - | ✅ | - | 128K | **Full tool support** (fine-tuned for function calling) |\n| `ollama-llama31-8b` | `llama3.1:8b` | ✅ | - | ✅ | - | 128K | **Full tool support**, best overall |\n| `ollama-llama31-70b` | `llama3.1:70b` | ✅ | - | ✅ | - | 128K | **Full tool support**, highest accuracy |\n| `ollama-mistral` | `mistral` | ✅ | - | ✅ | - | 32K | **Supports tools** (v0.3+) |\n| `ollama-mixtral` | `mixtral:8x7b` | ✅ | - | ✅ | - | 32K | MoE, good multi-domain |\n| `ollama-qwen25` | `qwen2.5` | ✅ | - | ✅ | - | 128K | Strong tool support |\n| `ollama-phi4` | `phi4` | ✅ | - | ⚠️ | - | 16K | Limited tool support |\n| `ollama-embedding` | `nomic-embed-text` | - | ✅ | - | - | 8K | Best local embedding |\n| `ollama-mxbai-embed` | `mxbai-embed-large` | - | ✅ | - | - | 512 | Alternative embedding |\n\n> **Tool Calling Update (2025)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), Mixtral, and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ | Fully Supported |\n| ⚠️ | Limited/Experimental (may need prompt adjustments) |\n| - | Not applicable |\n\n### Recommendations by Use Case\n\n| Use Case | Recommended Model | Why |\n|----------|-------------------|-----|\n| **Production Chat** | `gpt-4.1` or `gemini-2.5-pro` | Best instruction following, 1M context |\n| **Balanced (Default)** | `gpt-4.1-mini` or `gemini-2.5-flash` | Good quality at lower cost (Policy Bot default) |\n| **Budget Chat** | `gpt-4.1-nano` or `gemini-2.5-flash-lite` | Cost-effective for simple queries |\n| **Deep Reasoning** | `gemini-2.5-pro` | Built-in thinking capabilities |\n| **Local/Offline Chat** | `ollama-llama3.2` or `ollama-qwen2.5` | Full tool support, runs locally |\n| **RAG Embeddings** | `text-embedding-3-large` | Best quality for retrieval (Policy Bot default) |\n| **Local Embeddings** | `ollama-embedding` (nomic) | Good quality, no API cost |\n| **Audio Transcription** | `voxtral-transcribe` | Beats Whisper, half the cost |\n| **Budget Transcription** | `whisper-1` | Well-established, $0.006/min |\n| **Function Calling** | `gpt-4.1-mini`, `gemini-2.5-flash`, `mistral-large-3` | Reliable tool use |\n\n---\n\n## Switching Providers\n\nTo switch your entire application from OpenAI to Ollama:\n\n1. **No code changes needed**\n2. Update model name in your application config:\n\n```python\n# Before (OpenAI)\nMODEL = \"openai-gpt41\"\nEMBEDDING_MODEL = \"openai-embedding-large\"\nAUDIO_MODEL = \"openai-whisper\"\n\n# After (Ollama + Mistral Voxtral for audio)\nMODEL = \"ollama-llama32\"\nEMBEDDING_MODEL = \"ollama-embedding\"\nAUDIO_MODEL = \"voxtral-mini\"  # Or keep openai-whisper\n\n# Or fully Mistral\nMODEL = \"mistral-large-3\"\nEMBEDDING_MODEL = \"mistral-embedding\"\nAUDIO_MODEL = \"voxtral-small\"\n```\n\n---\n\n## Troubleshooting\n\n| Issue | Solution |\n|-------|----------|\n| Connection refused | Check if Docker is running, verify port 4000 |\n| Ollama not reachable | Use `host.docker.internal` in Docker, check Ollama is running |\n| API key errors | Verify `.env` file is loaded, check key format |\n| Model not found | Run `/models` endpoint to list available models |\n| Timeout errors | Increase `request_timeout` in config |\n\n---\n\n## Known Limitations\n\n| # | Limitation | Impact | File |\n|---|-----------|--------|------|\n| 1 | **Static YAML cache never invalidated** | `_parsedModelsCache` in `litellm-validator.ts` is set once and never cleared. `clearLiteLLMCache()` exists but has zero call sites. Runtime model additions invisible until restart. | `litellm-validator.ts:375,456` |\n| 2 | **Claude tool-calling streaming mismatch** | Anthropic models routed through LiteLLM fail tool calling — LiteLLM wraps Anthropic's streaming into OpenAI format, but tool call JSON is malformed. `streamOneCompletion()` parses OpenAI format only. | `openai.ts:421-434` |\n| 3 | **No circuit breaker for chat completions** | If LiteLLM proxy is unreachable, all chat completions fail. Embeddings have a fallback model path, but chat/transcription do not. | `openai.ts` |\n| 4 | **Prefix-based provider detection only** | `getProviderFromModelPath()` defaults to `openai` for unrecognized prefixes. Custom/vLLM models without a prefix will be misidentified. No `deepseek/` prefix exists — DeepSeek models are synced dynamically with prefix by `litellm-sync.ts`. | `litellm-validator.ts:282-294` |\n| 5 | **No TTS or image generation via proxy** | `podcast_gen` and `image_gen` must bypass LiteLLM entirely | `podcast-gen.ts`, `image-gen/` |\n\n---\n\n## Resources\n\n- **LiteLLM Docs**: https://docs.litellm.ai/docs/\n- **Supported Providers**: https://docs.litellm.ai/docs/providers\n- **GitHub**: https://github.com/BerriAI/litellm\n- **Ollama Models**: https://ollama.ai/library\n\n---\n\n*Last updated: March 2026 — Corrected architecture (three-tier hybrid), fixed routing table (transcription via LiteLLM, Fireworks reranking, multi-provider embeddings), added dynamic vs static model registration, updated proxy detection logic to match actual code.*"}, "ec4253d9e81fb6e6769c4e48c88a96121fb8f1c6": {"t": 1785754368.6160672, "n": 29806, "h": "f96a91c1ae033d45", "c": "# LiteLLM Implementation Guide\n\n> Customized for multi-provider LLM abstraction with proxy approach, embeddings routing, function calling support, and audio transcription.\n\n---\n\n## LLM Service Routing Overview\n\nPolicy Bot uses a **hybrid architecture**: core chat services route through LiteLLM proxy for unified model management, while specialized services (images, audio, document processing) call provider APIs directly or use local parsers for capabilities not available through the proxy.\n\n### Service Routing Table\n\n| Service | Policy Bot Feature | Routes Through | Provider(s) | Notes |\n|---------|-------------------|----------------|-------------|-------|\n| **Chat Completions** | Main chat, RAG responses | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic, DeepSeek, Fireworks AI, Ollama | Primary conversation engine |\n| **Embeddings** | Document indexing, search | ✅ LiteLLM | OpenAI | `text-embedding-3-large` default |\n| **Diagram Generation** | `diagram_gen` tool | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Generates Mermaid syntax |\n| **Summarization** | Message compression | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic, DeepSeek | Long conversation handling |\n| **Memory Extraction** | User fact storage | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Per-category memory |\n| **Prompt Optimization** | Query refinement | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Pre-RAG processing |\n| **Compliance Checks** | HITL clarification | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Skill compliance |\n| **Translation** | Multi-language support | ✅ LiteLLM / ❌ Direct | OpenAI (proxy), Gemini/Mistral (direct) | Provider-dependent |\n| **Audio Transcription** | Voice input | ❌ Direct | OpenAI Whisper | Not available via LiteLLM |\n| **Image Generation** | `image_gen` tool | ❌ Direct | OpenAI DALL-E, Gemini Imagen | Specialized APIs |\n| **Podcast Generation** | `podcast_gen` tool | ❌ Direct | OpenAI TTS, Gemini TTS | Multi-voice audio synthesis |\n| **Document Processing** | Document text extraction | ❌ Direct | mammoth/exceljs/officeparser (local), Mistral OCR, Azure DI, pdf-parse | Tiered fallback: local parsers first, then API providers |\n| **Reranking** | Search result scoring | ❌ Direct | Cohere | Not an LLM service |\n| **PPTX Generation** | `pptx_gen` tool | N/A | None | Template-based (no LLM) |\n| **XLSX Generation** | `xlsx_gen` tool | N/A | None | ExcelJS-based (no LLM) |\n| **Chart Generation** | `chart_gen` tool | N/A | None | Client-side rendering |\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ LiteLLM | Routes through LiteLLM proxy (port 4000) |\n| ❌ Direct | Calls provider API directly (bypasses LiteLLM) |\n| N/A | No LLM calls required |\n\n### Why Some Services Bypass LiteLLM\n\n| Service | Reason |\n|---------|--------|\n| **Image Generation** | DALL-E and Gemini Imagen APIs not supported by LiteLLM proxy |\n| **Podcast TTS** | OpenAI TTS and Gemini TTS are audio generation APIs, not chat completions |\n| **Transcription** | Whisper API uses different endpoint structure (`/audio/transcriptions`) |\n| **OCR** | Mistral OCR is a specialized document API with unique request format |\n| **Reranking** | Cohere reranker is a specialized search API, not a chat model |\n\n### Proxy Detection Logic\n\n```typescript\n// Services detect LiteLLM mode by checking the base URL\nconst isUsingProxy = baseUrl.includes('litellm') || baseUrl.includes(':4000');\n\nif (isUsingProxy) {\n  // Use LITELLM_MASTER_KEY for authentication\n  // All chat requests route through unified proxy\n} else {\n  // Use provider-specific API keys directly\n}\n```\n\n---\n\n## Policy Bot Integration\n\nThis guide is configured for the Policy Bot RAG application. Default model preset:\n\n| Setting | Value |\n|---------|-------|\n| Default Model | `gpt-4.1-mini` |\n| Temperature | `0.2` |\n| Max Tokens | `2000` |\n| Embedding Model | `text-embedding-3-large` |\n| Embedding Dimensions | `3072` |\n\nAvailable model presets in Policy Bot (via `config/defaults.json`):\n- **gpt-4.1** - High Performance (1M context)\n- **gpt-4.1-mini** - Balanced (default)\n- **gpt-4.1-nano** - Cost-Effective\n- **mistral-large-3** - Mistral Flagship (256K context)\n- **mistral-small-3.2** - Mistral Cost-Effective\n- **ministral-8b** - Mistral Ultra Cost-Effective\n- **gemini-2.5-pro** - Google Flagship Reasoning (1M context)\n- **gemini-2.5-flash** - Google Balanced (1M context)\n- **gemini-2.5-flash-lite** - Google Cost-Effective (1M context)\n- **claude-opus-4-6** - Anthropic Flagship\n- **claude-sonnet-4-6** - Anthropic Balanced\n- **deepseek-chat** - DeepSeek V3 (cost-effective)\n- **deepseek-reasoner** - DeepSeek R1 (thinking model)\n- **fw/kimi-k2-instruct** - Fireworks Kimi K2 (dev/test)\n- **ollama-llama3.2** - Local (no API cost)\n- **ollama-qwen2.5** - Local with excellent reasoning\n\n---\n\n## Architecture Overview\n\n```\n┌─────────────────────────────────────────────────────────────────┐\n│                      Your Application                            │\n│  (OpenAI SDK client pointing to http://localhost:4000)          │\n└─────────────────────┬───────────────────────────────────────────┘\n                      │\n                      ▼\n┌─────────────────────────────────────────────────────────────────┐\n│                    LiteLLM Proxy (Port 4000)                    │\n│                    Docker Compose Service                        │\n└─────┬───────┬──────┬──────┬──────┬──────┬──────┬──────────────┘\n      │       │      │      │      │      │      │\n      ▼       ▼      ▼      ▼      ▼      ▼      ▼\n  ┌──────┐ ┌──────┐ ┌─────┐ ┌─────┐ ┌──────┐ ┌──────┐ ┌──────┐\n  │OpenAI│ │Azure │ │Mistr│ │Gemini│ │Anthro│ │DeepSk│ │Ollama│\n  │      │ │OpenAI│ │  al │ │      │ │ pic  │ │      │ │(Loc.)│\n  └──────┘ └──────┘ └─────┘ └─────┘ └──────┘ └──────┘ └──────┘\n```\n\n---\n\n## Decision Summary\n\n| Component | Decision |\n|-----------|----------|\n| **Approach** | LiteLLM Proxy (Docker Compose) |\n| **Chat Completions** | Route through LiteLLM |\n| **Embeddings** | Route through LiteLLM (per-provider config) |\n| **Function Calling** | OpenAI, Claude, Mistral only; warn for others |\n| **Audio (Whisper)** | Abstract through LiteLLM |\n| **Deployment** | Docker Compose |\n\n---\n\n## Project Structure\n\n```\nlitellm-proxy/\n├── docker-compose.yml\n├── litellm_config.yaml\n├── .env\n└── README.md\n```\n\n---\n\n## Step 1: Environment Variables\n\nCreate `.env` file:\n\n```bash\n# ===================\n# LiteLLM Proxy Keys\n# ===================\nLITELLM_MASTER_KEY=sk-litellm-master-change-this\nLITELLM_SALT_KEY=sk-litellm-salt-change-this\n\n# ===================\n# OpenAI\n# ===================\nOPENAI_API_KEY=sk-...\n\n# ===================\n# Azure OpenAI\n# ===================\nAZURE_API_KEY=...\nAZURE_API_BASE=https://your-resource.openai.azure.com/\nAZURE_API_VERSION=2024-02-15-preview\nAZURE_CHAT_DEPLOYMENT=gpt-4-deployment\nAZURE_EMBEDDING_DEPLOYMENT=text-embedding-ada-002\n\n# ===================\n# Mistral\n# ===================\nMISTRAL_API_KEY=...\n\n# ===================\n# Google Gemini\n# ===================\nGEMINI_API_KEY=...\n\n# ===================\n# Anthropic\n# ===================\nANTHROPIC_API_KEY=...\n\n# ===================\n# DeepSeek\n# ===================\nDEEPSEEK_API_KEY=...\n\n# ===================\n# Fireworks AI (dev/test open-source models)\n# ===================\nFIREWORKS_AI_API_KEY=fw-...\n\n# ===================\n# Ollama (Local)\n# ===================\nOLLAMA_API_BASE=http://host.docker.internal:11434\n# Use 'http://localhost:11434' if running outside Docker\n```\n\n---\n\n## Step 2: LiteLLM Configuration\n\nCreate `litellm_config.yaml`:\n\n```yaml\n# =============================================================================\n# LITELLM PROXY CONFIGURATION\n# Multi-provider setup: OpenAI, Azure, Mistral, Gemini, Anthropic, DeepSeek, Fireworks AI, Ollama\n# Updated: March 2026\n# =============================================================================\n\nmodel_list:\n\n  # ===========================================================================\n  # CHAT COMPLETION MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Models - GPT-4.1 Family (Latest - April 2025+)\n  # ---------------------------------------------------------------------------\n  - model_name: openai-gpt41\n    litellm_params:\n      model: gpt-4.1\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-mini\n    litellm_params:\n      model: gpt-4.1-mini\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-nano\n    litellm_params:\n      model: gpt-4.1-nano\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt35\n    litellm_params:\n      model: gpt-3.5-turbo\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Models - GPT-4.1 Family\n  # ---------------------------------------------------------------------------\n  - model_name: azure-gpt41\n    litellm_params:\n      model: azure/gpt-41-deployment  # Your deployment name\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: azure-gpt41-mini\n    litellm_params:\n      model: azure/gpt-41-mini-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Mistral AI Models - Mistral 3 Family (Latest - December 2025)\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-large-3\n    litellm_params:\n      model: mistral/mistral-large-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 256000\n\n  - model_name: mistral-medium-31\n    litellm_params:\n      model: mistral/mistral-medium-2508\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      supports_vision: true\n\n  - model_name: mistral-small-32\n    litellm_params:\n      model: mistral/mistral-small-2506\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ministral-8b\n    litellm_params:\n      model: mistral/ministral-8b-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Ollama Models (Local) - With Full Tool Support\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-llama32\n    litellm_params:\n      model: ollama/llama3.2\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # Fine-tuned for function calling\n\n  - model_name: ollama-llama31-8b\n    litellm_params:\n      model: ollama/llama3.1:8b\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-mistral\n    litellm_params:\n      model: ollama/mistral\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # v0.3+ supports tools\n\n  - model_name: ollama-qwen25\n    litellm_params:\n      model: ollama/qwen2.5\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-phi4\n    litellm_params:\n      model: ollama/phi4\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: false  # Limited support\n\n  # ===========================================================================\n  # EMBEDDING MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: openai-embedding-large\n    litellm_params:\n      model: text-embedding-3-large\n      api_key: os.environ/OPENAI_API_KEY\n\n  - model_name: openai-embedding-small\n    litellm_params:\n      model: text-embedding-3-small\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: azure-embedding\n    litellm_params:\n      model: azure/text-embedding-ada-002\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  - model_name: azure-embedding-3-large\n    litellm_params:\n      model: azure/text-embedding-3-large-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-embedding\n    litellm_params:\n      model: mistral/mistral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: codestral-embed\n    litellm_params:\n      model: mistral/codestral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Ollama Embeddings (Local)\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-embedding\n    litellm_params:\n      model: ollama/nomic-embed-text\n      api_base: os.environ/OLLAMA_API_BASE\n\n  - model_name: ollama-mxbai-embed\n    litellm_params:\n      model: ollama/mxbai-embed-large\n      api_base: os.environ/OLLAMA_API_BASE\n\n  # ===========================================================================\n  # AUDIO MODELS (Speech-to-Text / Transcription)\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Whisper\n  # ---------------------------------------------------------------------------\n  - model_name: openai-whisper\n    litellm_params:\n      model: whisper-1\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure Whisper (if deployed)\n  # ---------------------------------------------------------------------------\n  - model_name: azure-whisper\n    litellm_params:\n      model: azure/whisper-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Voxtral (Released July 2025 - Beats Whisper at half cost)\n  # ---------------------------------------------------------------------------\n  - model_name: voxtral-small\n    litellm_params:\n      model: mistral/voxtral-small-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: voxtral-mini\n    litellm_params:\n      model: mistral/voxtral-mini-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n# =============================================================================\n# LITELLM SETTINGS\n# =============================================================================\n\nlitellm_settings:\n  # Drop unsupported params instead of erroring\n  drop_params: true\n  \n  # Enable detailed logging\n  set_verbose: false\n\n  # Request timeout\n  request_timeout: 120\n\n# =============================================================================\n# GENERAL SETTINGS\n# =============================================================================\n\ngeneral_settings:\n  # Master key for proxy authentication\n  master_key: os.environ/LITELLM_MASTER_KEY\n```\n\n---\n\n## Step 3: Docker Compose\n\nCreate `docker-compose.yml`:\n\n```yaml\nversion: \"3.9\"\n\nservices:\n  litellm:\n    image: ghcr.io/berriai/litellm:main-latest\n    container_name: litellm-proxy\n    ports:\n      - \"4000:4000\"\n    volumes:\n      - ./litellm_config.yaml:/app/config.yaml\n    env_file:\n      - .env\n    environment:\n      - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY}\n      - LITELLM_SALT_KEY=${LITELLM_SALT_KEY}\n    command: --config /app/config.yaml --port 4000 --detailed_debug\n    restart: unless-stopped\n    extra_hosts:\n      - \"host.docker.internal:host-gateway\"\n    healthcheck:\n      test: [\"CMD\", \"curl\", \"-f\", \"http://localhost:4000/health\"]\n      interval: 30s\n      timeout: 10s\n      retries: 3\n\n  # Optional: Ollama service (if not running separately)\n  # ollama:\n  #   image: ollama/ollama:latest\n  #   container_name: ollama\n  #   ports:\n  #     - \"11434:11434\"\n  #   volumes:\n  #     - ollama_data:/root/.ollama\n  #   restart: unless-stopped\n\n# volumes:\n#   ollama_data:\n```\n\n---\n\n## Step 4: Application Code Changes\n\n### Minimal Change Required\n\n**Before (Direct OpenAI):**\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(api_key=\"sk-...\")\n```\n\n**After (Via LiteLLM Proxy):**\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(\n    base_url=\"http://localhost:4000\",\n    api_key=\"sk-litellm-master-change-this\"  # Your LITELLM_MASTER_KEY\n)\n```\n\nThat's it! All other code stays the same.\n\n---\n\n## Step 5: Function Calling Handling\n\nSince your `generateResponseWithTools()` uses function calling, here's how to handle provider capabilities:\n\n```python\n# Provider capability mapping (Updated December 2025)\n\n# Full tool/function calling support\n# Matches Policy Bot config/defaults.json toolCapable list\nTOOL_CAPABLE_MODELS = [\n    # OpenAI\n    \"gpt-4.1\", \"gpt-4.1-mini\", \"gpt-4.1-nano\", \"gpt-3.5-turbo\",\n    # Mistral\n    \"mistral-large-3\", \"mistral-medium-3.1\", \"mistral-small-3.2\",\n    \"ministral-8b\", \"ministral-3b\",\n    # Ollama (Now with full tool support!)\n    \"ollama-llama3.2\", \"ollama-llama3.1\", \"ollama-mistral\", \"ollama-qwen2.5\",\n]\n\n# Limited or no tool support (show warning)\nTOOL_LIMITED_MODELS = [\n    \"ollama-phi4\",\n]\n\ndef generate_response_with_tools(model: str, messages: list, tools: list):\n    \"\"\"\n    Generate response with function calling support.\n    Warns and disables tools for unsupported providers.\n    \"\"\"\n    \n    if model in TOOL_LIMITED_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' has limited function calling support.\")\n        print(f\"   Tools will be disabled. Consider using: {', '.join(TOOL_CAPABLE_MODELS[:5])}\")\n        # Call without tools\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages\n        )\n    elif model not in TOOL_CAPABLE_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' tool support unknown. Attempting with tools...\")\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    else:\n        # Full tool support\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    \n    return response\n```\n\n> **Good News (2025 Update)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n---\n\n## Step 6: Embeddings Usage\n\n```python\n# Embeddings via LiteLLM Proxy\n\n# OpenAI embeddings (recommended for RAG quality)\nresponse = client.embeddings.create(\n    model=\"openai-embedding-large\",\n    input=\"Your text here\"\n)\n\n# Azure embeddings\nresponse = client.embeddings.create(\n    model=\"azure-embedding\",\n    input=\"Your text here\"\n)\n\n# Mistral embeddings\nresponse = client.embeddings.create(\n    model=\"mistral-embedding\",\n    input=\"Your text here\"\n)\n\n# Ollama embeddings (local)\nresponse = client.embeddings.create(\n    model=\"ollama-embedding\",\n    input=\"Your text here\"\n)\n\nembedding = response.data[0].embedding\n```\n\n### Important: RAG Consistency\n\n> ⚠️ **Warning**: Don't mix embedding providers for the same vector store. If your RAG was built with `text-embedding-3-large`, query with the same model.\n\n```python\n# Configuration approach\nEMBEDDING_CONFIG = {\n    \"openai\": \"openai-embedding-large\",\n    \"azure\": \"azure-embedding\",\n    \"mistral\": \"mistral-embedding\",\n    \"ollama\": \"ollama-embedding\"\n}\n\n# Use consistently\ncurrent_provider = \"openai\"\nembedding_model = EMBEDDING_CONFIG[current_provider]\n```\n\n---\n\n## Step 7: Audio Transcription (Whisper)\n\n```python\n# Audio transcription via LiteLLM Proxy\n\naudio_file = open(\"audio.mp3\", \"rb\")\n\nresponse = client.audio.transcriptions.create(\n    model=\"openai-whisper\",\n    file=audio_file\n)\n\nprint(response.text)\n```\n\n---\n\n## Deployment Commands\n\n```bash\n# Start the proxy\ndocker compose up -d\n\n# View logs\ndocker compose logs -f litellm\n\n# Stop\ndocker compose down\n\n# Restart after config changes\ndocker compose restart litellm\n```\n\n---\n\n## Testing the Setup\n\n### 1. Health Check\n\n```bash\ncurl http://localhost:4000/health\n```\n\n### 2. List Available Models\n\n```bash\ncurl http://localhost:4000/models \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\"\n```\n\n### 3. Test Chat Completion\n\n```bash\ncurl -X POST http://localhost:4000/chat/completions \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-gpt35\",\n    \"messages\": [{\"role\": \"user\", \"content\": \"Hello, test!\"}]\n  }'\n```\n\n### 4. Test Embeddings\n\n```bash\ncurl -X POST http://localhost:4000/embeddings \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-embedding-large\",\n    \"input\": \"Test embedding\"\n  }'\n```\n\n---\n\n## Model Quick Reference\n\n*Updated: December 2025*\n\n### OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `openai-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Latest flagship, best for coding |\n| `openai-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Faster, 83% cheaper than GPT-4o |\n| `openai-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Fastest, cheapest |\n| `openai-gpt35` | `gpt-3.5-turbo` | ✅ | - | ✅ | - | 16K | Legacy, cost-effective |\n| `openai-embedding-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality embeddings |\n| `openai-embedding-small` | `text-embedding-3-small` | - | ✅ | - | - | 8K | Faster, cheaper |\n| `openai-whisper` | `whisper-1` | - | - | - | ✅ | - | $0.006/min transcription |\n\n### Azure OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `azure-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-embedding` | `text-embedding-ada-002` | - | ✅ | - | - | 8K | Standard embedding |\n| `azure-embedding-3-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality |\n| `azure-whisper` | `whisper` | - | - | - | ✅ | - | Speech-to-text |\n\n### Mistral AI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `mistral-large-3` | `mistral-large-latest` | ✅ | - | ✅ | - | 256K | Flagship MoE (675B total params) |\n| `mistral-medium-31` | `mistral-medium-2508` | ✅ | - | ✅ | - | 128K | Frontier multimodal |\n| `mistral-small-32` | `mistral-small-2506` | ✅ | - | ✅ | - | 128K | Fast, efficient |\n| `ministral-14b` | `ministral-3-14b` | ✅ | - | ✅ | - | 128K | Best small model |\n| `ministral-8b` | `ministral-3-8b` | ✅ | - | ✅ | - | 128K | Edge deployment |\n| `ministral-3b` | `ministral-3-3b` | ✅ | - | ✅ | - | 128K | Ultra-light |\n| `mistral-embedding` | `mistral-embed` | - | ✅ | - | - | 8K | Text embeddings |\n| `codestral-embed` | `codestral-embed` | - | ✅ | - | - | - | Code embeddings |\n| `voxtral-small` | `voxtral-small-latest` | - | - | - | ✅ | 32K | 24B, best accuracy, $0.001/min |\n| `voxtral-mini` | `voxtral-mini-latest` | - | - | - | ✅ | 32K | 3B, edge/local deployment |\n| `voxtral-transcribe` | `voxtral-mini-latest` (via /audio/transcriptions) | - | - | - | ✅ | - | Transcription-only, beats Whisper |\n\n> **Note**: Voxtral (released July 2025) outperforms OpenAI Whisper at half the cost ($0.001/min vs $0.006/min). Supports up to 30 min audio for transcription, 40 min for understanding.\n\n### Google Gemini Models\n\n| Model Name | API Model ID | Chat | Tools | Context | Notes |\n|------------|--------------|------|-------|---------|-------|\n| `gemini-2.5-pro` | `gemini/gemini-2.5-pro` | ✅ | ✅ | 1M | Flagship reasoning model |\n| `gemini-2.5-flash` | `gemini/gemini-2.5-flash` | ✅ | ✅ | 1M | Fast, balanced performance |\n| `gemini-2.5-flash-lite` | `gemini/gemini-2.5-flash-lite` | ✅ | ✅ | 1M | Lowest cost option |\n\n**Pricing (per 1M tokens):**\n- Gemini 2.5 Pro: $1.25 input / $10.00 output (≤200k context)\n- Gemini 2.5 Flash: $0.30 input / $2.50 output\n- Gemini 2.5 Flash-Lite: $0.10 input / $0.40 output\n\n**Features:**\n- All models support function calling\n- Built-in \"thinking\" capabilities with controllable budgets\n- Free tier available for development/testing\n- Native multimodal support (text, image, audio, video)\n\n### Ollama Local Models\n\n| Model Name | Ollama Model | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `ollama-llama32` | `llama3.2` | ✅ | - | ✅ | - | 128K | **Full tool support** (fine-tuned for function calling) |\n| `ollama-llama31-8b` | `llama3.1:8b` | ✅ | - | ✅ | - | 128K | **Full tool support**, best overall |\n| `ollama-llama31-70b` | `llama3.1:70b` | ✅ | - | ✅ | - | 128K | **Full tool support**, highest accuracy |\n| `ollama-mistral` | `mistral` | ✅ | - | ✅ | - | 32K | **Supports tools** (v0.3+) |\n| `ollama-mixtral` | `mixtral:8x7b` | ✅ | - | ✅ | - | 32K | MoE, good multi-domain |\n| `ollama-qwen25` | `qwen2.5` | ✅ | - | ✅ | - | 128K | Strong tool support |\n| `ollama-phi4` | `phi4` | ✅ | - | ⚠️ | - | 16K | Limited tool support |\n| `ollama-embedding` | `nomic-embed-text` | - | ✅ | - | - | 8K | Best local embedding |\n| `ollama-mxbai-embed` | `mxbai-embed-large` | - | ✅ | - | - | 512 | Alternative embedding |\n\n> **Tool Calling Update (2025)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), Mixtral, and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ | Fully Supported |\n| ⚠️ | Limited/Experimental (may need prompt adjustments) |\n| - | Not applicable |\n\n### Recommendations by Use Case\n\n| Use Case | Recommended Model | Why |\n|----------|-------------------|-----|\n| **Production Chat** | `gpt-4.1` or `gemini-2.5-pro` | Best instruction following, 1M context |\n| **Balanced (Default)** | `gpt-4.1-mini` or `gemini-2.5-flash` | Good quality at lower cost (Policy Bot default) |\n| **Budget Chat** | `gpt-4.1-nano` or `gemini-2.5-flash-lite` | Cost-effective for simple queries |\n| **Deep Reasoning** | `gemini-2.5-pro` | Built-in thinking capabilities |\n| **Local/Offline Chat** | `ollama-llama3.2` or `ollama-qwen2.5` | Full tool support, runs locally |\n| **RAG Embeddings** | `text-embedding-3-large` | Best quality for retrieval (Policy Bot default) |\n| **Local Embeddings** | `ollama-embedding` (nomic) | Good quality, no API cost |\n| **Audio Transcription** | `voxtral-transcribe` | Beats Whisper, half the cost |\n| **Budget Transcription** | `whisper-1` | Well-established, $0.006/min |\n| **Function Calling** | `gpt-4.1-mini`, `gemini-2.5-flash`, `mistral-large-3` | Reliable tool use |\n\n---\n\n## Switching Providers\n\nTo switch your entire application from OpenAI to Ollama:\n\n1. **No code changes needed**\n2. Update model name in your application config:\n\n```python\n# Before (OpenAI)\nMODEL = \"openai-gpt41\"\nEMBEDDING_MODEL = \"openai-embedding-large\"\nAUDIO_MODEL = \"openai-whisper\"\n\n# After (Ollama + Mistral Voxtral for audio)\nMODEL = \"ollama-llama32\"\nEMBEDDING_MODEL = \"ollama-embedding\"\nAUDIO_MODEL = \"voxtral-mini\"  # Or keep openai-whisper\n\n# Or fully Mistral\nMODEL = \"mistral-large-3\"\nEMBEDDING_MODEL = \"mistral-embedding\"\nAUDIO_MODEL = \"voxtral-small\"\n```\n\n---\n\n## Troubleshooting\n\n| Issue | Solution |\n|-------|----------|\n| Connection refused | Check if Docker is running, verify port 4000 |\n| Ollama not reachable | Use `host.docker.internal` in Docker, check Ollama is running |\n| API key errors | Verify `.env` file is loaded, check key format |\n| Model not found | Run `/models` endpoint to list available models |\n| Timeout errors | Increase `request_timeout` in config |\n\n---\n\n## Resources\n\n- **LiteLLM Docs**: https://docs.litellm.ai/docs/\n- **Supported Providers**: https://docs.litellm.ai/docs/providers\n- **GitHub**: https://github.com/BerriAI/litellm\n- **Ollama Models**: https://ollama.ai/library\n\n---\n\n*Last updated: December 2025 - Includes GPT-4.1, Mistral 3, Voxtral, and Ollama tool support updates*"}, "dca46325f6de33960ebfdb8fe9a8a26408f1da3d": {"t": 1785754372.1258426, "n": 29658, "h": "e37bc079d73fc72e", "c": "# LiteLLM Implementation Guide\n\n> Customized for multi-provider LLM abstraction with proxy approach, embeddings routing, function calling support, and audio transcription.\n\n---\n\n## LLM Service Routing Overview\n\nPolicy Bot uses a **hybrid architecture**: core chat services route through LiteLLM proxy for unified model management, while specialized services (images, audio, OCR) call provider APIs directly for capabilities not available through the proxy.\n\n### Service Routing Table\n\n| Service | Policy Bot Feature | Routes Through | Provider(s) | Notes |\n|---------|-------------------|----------------|-------------|-------|\n| **Chat Completions** | Main chat, RAG responses | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic, DeepSeek, Fireworks AI, Ollama | Primary conversation engine |\n| **Embeddings** | Document indexing, search | ✅ LiteLLM | OpenAI | `text-embedding-3-large` default |\n| **Diagram Generation** | `diagram_gen` tool | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Generates Mermaid syntax |\n| **Summarization** | Message compression | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic, DeepSeek | Long conversation handling |\n| **Memory Extraction** | User fact storage | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Per-category memory |\n| **Prompt Optimization** | Query refinement | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Pre-RAG processing |\n| **Compliance Checks** | HITL clarification | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic | Skill compliance |\n| **Translation** | Multi-language support | ✅ LiteLLM / ❌ Direct | OpenAI (proxy), Gemini/Mistral (direct) | Provider-dependent |\n| **Audio Transcription** | Voice input | ❌ Direct | OpenAI Whisper | Not available via LiteLLM |\n| **Image Generation** | `image_gen` tool | ❌ Direct | OpenAI DALL-E, Gemini Imagen | Specialized APIs |\n| **Podcast Generation** | `podcast_gen` tool | ❌ Direct | OpenAI TTS, Gemini TTS | Multi-voice audio synthesis |\n| **OCR** | PDF/image text extraction | ❌ Direct | Mistral OCR | Document processing |\n| **Reranking** | Search result scoring | ❌ Direct | Cohere | Not an LLM service |\n| **PPTX Generation** | `pptx_gen` tool | N/A | None | Template-based (no LLM) |\n| **XLSX Generation** | `xlsx_gen` tool | N/A | None | ExcelJS-based (no LLM) |\n| **Chart Generation** | `chart_gen` tool | N/A | None | Client-side rendering |\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ LiteLLM | Routes through LiteLLM proxy (port 4000) |\n| ❌ Direct | Calls provider API directly (bypasses LiteLLM) |\n| N/A | No LLM calls required |\n\n### Why Some Services Bypass LiteLLM\n\n| Service | Reason |\n|---------|--------|\n| **Image Generation** | DALL-E and Gemini Imagen APIs not supported by LiteLLM proxy |\n| **Podcast TTS** | OpenAI TTS and Gemini TTS are audio generation APIs, not chat completions |\n| **Transcription** | Whisper API uses different endpoint structure (`/audio/transcriptions`) |\n| **OCR** | Mistral OCR is a specialized document API with unique request format |\n| **Reranking** | Cohere reranker is a specialized search API, not a chat model |\n\n### Proxy Detection Logic\n\n```typescript\n// Services detect LiteLLM mode by checking the base URL\nconst isUsingProxy = baseUrl.includes('litellm') || baseUrl.includes(':4000');\n\nif (isUsingProxy) {\n  // Use LITELLM_MASTER_KEY for authentication\n  // All chat requests route through unified proxy\n} else {\n  // Use provider-specific API keys directly\n}\n```\n\n---\n\n## Policy Bot Integration\n\nThis guide is configured for the Policy Bot RAG application. Default model preset:\n\n| Setting | Value |\n|---------|-------|\n| Default Model | `gpt-4.1-mini` |\n| Temperature | `0.2` |\n| Max Tokens | `2000` |\n| Embedding Model | `text-embedding-3-large` |\n| Embedding Dimensions | `3072` |\n\nAvailable model presets in Policy Bot (via `config/defaults.json`):\n- **gpt-4.1** - High Performance (1M context)\n- **gpt-4.1-mini** - Balanced (default)\n- **gpt-4.1-nano** - Cost-Effective\n- **mistral-large-3** - Mistral Flagship (256K context)\n- **mistral-small-3.2** - Mistral Cost-Effective\n- **ministral-8b** - Mistral Ultra Cost-Effective\n- **gemini-2.5-pro** - Google Flagship Reasoning (1M context)\n- **gemini-2.5-flash** - Google Balanced (1M context)\n- **gemini-2.5-flash-lite** - Google Cost-Effective (1M context)\n- **claude-opus-4-6** - Anthropic Flagship\n- **claude-sonnet-4-6** - Anthropic Balanced\n- **deepseek-chat** - DeepSeek V3 (cost-effective)\n- **deepseek-reasoner** - DeepSeek R1 (thinking model)\n- **fw/kimi-k2-instruct** - Fireworks Kimi K2 (dev/test)\n- **ollama-llama3.2** - Local (no API cost)\n- **ollama-qwen2.5** - Local with excellent reasoning\n\n---\n\n## Architecture Overview\n\n```\n┌─────────────────────────────────────────────────────────────────┐\n│                      Your Application                            │\n│  (OpenAI SDK client pointing to http://localhost:4000)          │\n└─────────────────────┬───────────────────────────────────────────┘\n                      │\n                      ▼\n┌─────────────────────────────────────────────────────────────────┐\n│                    LiteLLM Proxy (Port 4000)                    │\n│                    Docker Compose Service                        │\n└─────┬───────┬──────┬──────┬──────┬──────┬──────┬──────────────┘\n      │       │      │      │      │      │      │\n      ▼       ▼      ▼      ▼      ▼      ▼      ▼\n  ┌──────┐ ┌──────┐ ┌─────┐ ┌─────┐ ┌──────┐ ┌──────┐ ┌──────┐\n  │OpenAI│ │Azure │ │Mistr│ │Gemini│ │Anthro│ │DeepSk│ │Ollama│\n  │      │ │OpenAI│ │  al │ │      │ │ pic  │ │      │ │(Loc.)│\n  └──────┘ └──────┘ └─────┘ └─────┘ └──────┘ └──────┘ └──────┘\n```\n\n---\n\n## Decision Summary\n\n| Component | Decision |\n|-----------|----------|\n| **Approach** | LiteLLM Proxy (Docker Compose) |\n| **Chat Completions** | Route through LiteLLM |\n| **Embeddings** | Route through LiteLLM (per-provider config) |\n| **Function Calling** | OpenAI, Claude, Mistral only; warn for others |\n| **Audio (Whisper)** | Abstract through LiteLLM |\n| **Deployment** | Docker Compose |\n\n---\n\n## Project Structure\n\n```\nlitellm-proxy/\n├── docker-compose.yml\n├── litellm_config.yaml\n├── .env\n└── README.md\n```\n\n---\n\n## Step 1: Environment Variables\n\nCreate `.env` file:\n\n```bash\n# ===================\n# LiteLLM Proxy Keys\n# ===================\nLITELLM_MASTER_KEY=sk-litellm-master-change-this\nLITELLM_SALT_KEY=sk-litellm-salt-change-this\n\n# ===================\n# OpenAI\n# ===================\nOPENAI_API_KEY=sk-...\n\n# ===================\n# Azure OpenAI\n# ===================\nAZURE_API_KEY=...\nAZURE_API_BASE=https://your-resource.openai.azure.com/\nAZURE_API_VERSION=2024-02-15-preview\nAZURE_CHAT_DEPLOYMENT=gpt-4-deployment\nAZURE_EMBEDDING_DEPLOYMENT=text-embedding-ada-002\n\n# ===================\n# Mistral\n# ===================\nMISTRAL_API_KEY=...\n\n# ===================\n# Google Gemini\n# ===================\nGEMINI_API_KEY=...\n\n# ===================\n# Anthropic\n# ===================\nANTHROPIC_API_KEY=...\n\n# ===================\n# DeepSeek\n# ===================\nDEEPSEEK_API_KEY=...\n\n# ===================\n# Fireworks AI (dev/test open-source models)\n# ===================\nFIREWORKS_AI_API_KEY=fw-...\n\n# ===================\n# Ollama (Local)\n# ===================\nOLLAMA_API_BASE=http://host.docker.internal:11434\n# Use 'http://localhost:11434' if running outside Docker\n```\n\n---\n\n## Step 2: LiteLLM Configuration\n\nCreate `litellm_config.yaml`:\n\n```yaml\n# =============================================================================\n# LITELLM PROXY CONFIGURATION\n# Multi-provider setup: OpenAI, Azure, Mistral, Gemini, Anthropic, DeepSeek, Fireworks AI, Ollama\n# Updated: March 2026\n# =============================================================================\n\nmodel_list:\n\n  # ===========================================================================\n  # CHAT COMPLETION MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Models - GPT-4.1 Family (Latest - April 2025+)\n  # ---------------------------------------------------------------------------\n  - model_name: openai-gpt41\n    litellm_params:\n      model: gpt-4.1\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-mini\n    litellm_params:\n      model: gpt-4.1-mini\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-nano\n    litellm_params:\n      model: gpt-4.1-nano\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt35\n    litellm_params:\n      model: gpt-3.5-turbo\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Models - GPT-4.1 Family\n  # ---------------------------------------------------------------------------\n  - model_name: azure-gpt41\n    litellm_params:\n      model: azure/gpt-41-deployment  # Your deployment name\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: azure-gpt41-mini\n    litellm_params:\n      model: azure/gpt-41-mini-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Mistral AI Models - Mistral 3 Family (Latest - December 2025)\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-large-3\n    litellm_params:\n      model: mistral/mistral-large-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 256000\n\n  - model_name: mistral-medium-31\n    litellm_params:\n      model: mistral/mistral-medium-2508\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      supports_vision: true\n\n  - model_name: mistral-small-32\n    litellm_params:\n      model: mistral/mistral-small-2506\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ministral-8b\n    litellm_params:\n      model: mistral/ministral-8b-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Ollama Models (Local) - With Full Tool Support\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-llama32\n    litellm_params:\n      model: ollama/llama3.2\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # Fine-tuned for function calling\n\n  - model_name: ollama-llama31-8b\n    litellm_params:\n      model: ollama/llama3.1:8b\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-mistral\n    litellm_params:\n      model: ollama/mistral\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # v0.3+ supports tools\n\n  - model_name: ollama-qwen25\n    litellm_params:\n      model: ollama/qwen2.5\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-phi4\n    litellm_params:\n      model: ollama/phi4\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: false  # Limited support\n\n  # ===========================================================================\n  # EMBEDDING MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: openai-embedding-large\n    litellm_params:\n      model: text-embedding-3-large\n      api_key: os.environ/OPENAI_API_KEY\n\n  - model_name: openai-embedding-small\n    litellm_params:\n      model: text-embedding-3-small\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: azure-embedding\n    litellm_params:\n      model: azure/text-embedding-ada-002\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  - model_name: azure-embedding-3-large\n    litellm_params:\n      model: azure/text-embedding-3-large-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-embedding\n    litellm_params:\n      model: mistral/mistral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: codestral-embed\n    litellm_params:\n      model: mistral/codestral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Ollama Embeddings (Local)\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-embedding\n    litellm_params:\n      model: ollama/nomic-embed-text\n      api_base: os.environ/OLLAMA_API_BASE\n\n  - model_name: ollama-mxbai-embed\n    litellm_params:\n      model: ollama/mxbai-embed-large\n      api_base: os.environ/OLLAMA_API_BASE\n\n  # ===========================================================================\n  # AUDIO MODELS (Speech-to-Text / Transcription)\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Whisper\n  # ---------------------------------------------------------------------------\n  - model_name: openai-whisper\n    litellm_params:\n      model: whisper-1\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure Whisper (if deployed)\n  # ---------------------------------------------------------------------------\n  - model_name: azure-whisper\n    litellm_params:\n      model: azure/whisper-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Voxtral (Released July 2025 - Beats Whisper at half cost)\n  # ---------------------------------------------------------------------------\n  - model_name: voxtral-small\n    litellm_params:\n      model: mistral/voxtral-small-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: voxtral-mini\n    litellm_params:\n      model: mistral/voxtral-mini-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n# =============================================================================\n# LITELLM SETTINGS\n# =============================================================================\n\nlitellm_settings:\n  # Drop unsupported params instead of erroring\n  drop_params: true\n  \n  # Enable detailed logging\n  set_verbose: false\n\n  # Request timeout\n  request_timeout: 120\n\n# =============================================================================\n# GENERAL SETTINGS\n# =============================================================================\n\ngeneral_settings:\n  # Master key for proxy authentication\n  master_key: os.environ/LITELLM_MASTER_KEY\n```\n\n---\n\n## Step 3: Docker Compose\n\nCreate `docker-compose.yml`:\n\n```yaml\nversion: \"3.9\"\n\nservices:\n  litellm:\n    image: ghcr.io/berriai/litellm:main-latest\n    container_name: litellm-proxy\n    ports:\n      - \"4000:4000\"\n    volumes:\n      - ./litellm_config.yaml:/app/config.yaml\n    env_file:\n      - .env\n    environment:\n      - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY}\n      - LITELLM_SALT_KEY=${LITELLM_SALT_KEY}\n    command: --config /app/config.yaml --port 4000 --detailed_debug\n    restart: unless-stopped\n    extra_hosts:\n      - \"host.docker.internal:host-gateway\"\n    healthcheck:\n      test: [\"CMD\", \"curl\", \"-f\", \"http://localhost:4000/health\"]\n      interval: 30s\n      timeout: 10s\n      retries: 3\n\n  # Optional: Ollama service (if not running separately)\n  # ollama:\n  #   image: ollama/ollama:latest\n  #   container_name: ollama\n  #   ports:\n  #     - \"11434:11434\"\n  #   volumes:\n  #     - ollama_data:/root/.ollama\n  #   restart: unless-stopped\n\n# volumes:\n#   ollama_data:\n```\n\n---\n\n## Step 4: Application Code Changes\n\n### Minimal Change Required\n\n**Before (Direct OpenAI):**\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(api_key=\"sk-...\")\n```\n\n**After (Via LiteLLM Proxy):**\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(\n    base_url=\"http://localhost:4000\",\n    api_key=\"sk-litellm-master-change-this\"  # Your LITELLM_MASTER_KEY\n)\n```\n\nThat's it! All other code stays the same.\n\n---\n\n## Step 5: Function Calling Handling\n\nSince your `generateResponseWithTools()` uses function calling, here's how to handle provider capabilities:\n\n```python\n# Provider capability mapping (Updated December 2025)\n\n# Full tool/function calling support\n# Matches Policy Bot config/defaults.json toolCapable list\nTOOL_CAPABLE_MODELS = [\n    # OpenAI\n    \"gpt-4.1\", \"gpt-4.1-mini\", \"gpt-4.1-nano\", \"gpt-3.5-turbo\",\n    # Mistral\n    \"mistral-large-3\", \"mistral-medium-3.1\", \"mistral-small-3.2\",\n    \"ministral-8b\", \"ministral-3b\",\n    # Ollama (Now with full tool support!)\n    \"ollama-llama3.2\", \"ollama-llama3.1\", \"ollama-mistral\", \"ollama-qwen2.5\",\n]\n\n# Limited or no tool support (show warning)\nTOOL_LIMITED_MODELS = [\n    \"ollama-phi4\",\n]\n\ndef generate_response_with_tools(model: str, messages: list, tools: list):\n    \"\"\"\n    Generate response with function calling support.\n    Warns and disables tools for unsupported providers.\n    \"\"\"\n    \n    if model in TOOL_LIMITED_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' has limited function calling support.\")\n        print(f\"   Tools will be disabled. Consider using: {', '.join(TOOL_CAPABLE_MODELS[:5])}\")\n        # Call without tools\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages\n        )\n    elif model not in TOOL_CAPABLE_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' tool support unknown. Attempting with tools...\")\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    else:\n        # Full tool support\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    \n    return response\n```\n\n> **Good News (2025 Update)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n---\n\n## Step 6: Embeddings Usage\n\n```python\n# Embeddings via LiteLLM Proxy\n\n# OpenAI embeddings (recommended for RAG quality)\nresponse = client.embeddings.create(\n    model=\"openai-embedding-large\",\n    input=\"Your text here\"\n)\n\n# Azure embeddings\nresponse = client.embeddings.create(\n    model=\"azure-embedding\",\n    input=\"Your text here\"\n)\n\n# Mistral embeddings\nresponse = client.embeddings.create(\n    model=\"mistral-embedding\",\n    input=\"Your text here\"\n)\n\n# Ollama embeddings (local)\nresponse = client.embeddings.create(\n    model=\"ollama-embedding\",\n    input=\"Your text here\"\n)\n\nembedding = response.data[0].embedding\n```\n\n### Important: RAG Consistency\n\n> ⚠️ **Warning**: Don't mix embedding providers for the same vector store. If your RAG was built with `text-embedding-3-large`, query with the same model.\n\n```python\n# Configuration approach\nEMBEDDING_CONFIG = {\n    \"openai\": \"openai-embedding-large\",\n    \"azure\": \"azure-embedding\",\n    \"mistral\": \"mistral-embedding\",\n    \"ollama\": \"ollama-embedding\"\n}\n\n# Use consistently\ncurrent_provider = \"openai\"\nembedding_model = EMBEDDING_CONFIG[current_provider]\n```\n\n---\n\n## Step 7: Audio Transcription (Whisper)\n\n```python\n# Audio transcription via LiteLLM Proxy\n\naudio_file = open(\"audio.mp3\", \"rb\")\n\nresponse = client.audio.transcriptions.create(\n    model=\"openai-whisper\",\n    file=audio_file\n)\n\nprint(response.text)\n```\n\n---\n\n## Deployment Commands\n\n```bash\n# Start the proxy\ndocker compose up -d\n\n# View logs\ndocker compose logs -f litellm\n\n# Stop\ndocker compose down\n\n# Restart after config changes\ndocker compose restart litellm\n```\n\n---\n\n## Testing the Setup\n\n### 1. Health Check\n\n```bash\ncurl http://localhost:4000/health\n```\n\n### 2. List Available Models\n\n```bash\ncurl http://localhost:4000/models \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\"\n```\n\n### 3. Test Chat Completion\n\n```bash\ncurl -X POST http://localhost:4000/chat/completions \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-gpt35\",\n    \"messages\": [{\"role\": \"user\", \"content\": \"Hello, test!\"}]\n  }'\n```\n\n### 4. Test Embeddings\n\n```bash\ncurl -X POST http://localhost:4000/embeddings \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-embedding-large\",\n    \"input\": \"Test embedding\"\n  }'\n```\n\n---\n\n## Model Quick Reference\n\n*Updated: December 2025*\n\n### OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `openai-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Latest flagship, best for coding |\n| `openai-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Faster, 83% cheaper than GPT-4o |\n| `openai-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Fastest, cheapest |\n| `openai-gpt35` | `gpt-3.5-turbo` | ✅ | - | ✅ | - | 16K | Legacy, cost-effective |\n| `openai-embedding-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality embeddings |\n| `openai-embedding-small` | `text-embedding-3-small` | - | ✅ | - | - | 8K | Faster, cheaper |\n| `openai-whisper` | `whisper-1` | - | - | - | ✅ | - | $0.006/min transcription |\n\n### Azure OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `azure-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-embedding` | `text-embedding-ada-002` | - | ✅ | - | - | 8K | Standard embedding |\n| `azure-embedding-3-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality |\n| `azure-whisper` | `whisper` | - | - | - | ✅ | - | Speech-to-text |\n\n### Mistral AI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `mistral-large-3` | `mistral-large-latest` | ✅ | - | ✅ | - | 256K | Flagship MoE (675B total params) |\n| `mistral-medium-31` | `mistral-medium-2508` | ✅ | - | ✅ | - | 128K | Frontier multimodal |\n| `mistral-small-32` | `mistral-small-2506` | ✅ | - | ✅ | - | 128K | Fast, efficient |\n| `ministral-14b` | `ministral-3-14b` | ✅ | - | ✅ | - | 128K | Best small model |\n| `ministral-8b` | `ministral-3-8b` | ✅ | - | ✅ | - | 128K | Edge deployment |\n| `ministral-3b` | `ministral-3-3b` | ✅ | - | ✅ | - | 128K | Ultra-light |\n| `mistral-embedding` | `mistral-embed` | - | ✅ | - | - | 8K | Text embeddings |\n| `codestral-embed` | `codestral-embed` | - | ✅ | - | - | - | Code embeddings |\n| `voxtral-small` | `voxtral-small-latest` | - | - | - | ✅ | 32K | 24B, best accuracy, $0.001/min |\n| `voxtral-mini` | `voxtral-mini-latest` | - | - | - | ✅ | 32K | 3B, edge/local deployment |\n| `voxtral-transcribe` | `voxtral-mini-latest` (via /audio/transcriptions) | - | - | - | ✅ | - | Transcription-only, beats Whisper |\n\n> **Note**: Voxtral (released July 2025) outperforms OpenAI Whisper at half the cost ($0.001/min vs $0.006/min). Supports up to 30 min audio for transcription, 40 min for understanding.\n\n### Google Gemini Models\n\n| Model Name | API Model ID | Chat | Tools | Context | Notes |\n|------------|--------------|------|-------|---------|-------|\n| `gemini-2.5-pro` | `gemini/gemini-2.5-pro` | ✅ | ✅ | 1M | Flagship reasoning model |\n| `gemini-2.5-flash` | `gemini/gemini-2.5-flash` | ✅ | ✅ | 1M | Fast, balanced performance |\n| `gemini-2.5-flash-lite` | `gemini/gemini-2.5-flash-lite` | ✅ | ✅ | 1M | Lowest cost option |\n\n**Pricing (per 1M tokens):**\n- Gemini 2.5 Pro: $1.25 input / $10.00 output (≤200k context)\n- Gemini 2.5 Flash: $0.30 input / $2.50 output\n- Gemini 2.5 Flash-Lite: $0.10 input / $0.40 output\n\n**Features:**\n- All models support function calling\n- Built-in \"thinking\" capabilities with controllable budgets\n- Free tier available for development/testing\n- Native multimodal support (text, image, audio, video)\n\n### Ollama Local Models\n\n| Model Name | Ollama Model | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `ollama-llama32` | `llama3.2` | ✅ | - | ✅ | - | 128K | **Full tool support** (fine-tuned for function calling) |\n| `ollama-llama31-8b` | `llama3.1:8b` | ✅ | - | ✅ | - | 128K | **Full tool support**, best overall |\n| `ollama-llama31-70b` | `llama3.1:70b` | ✅ | - | ✅ | - | 128K | **Full tool support**, highest accuracy |\n| `ollama-mistral` | `mistral` | ✅ | - | ✅ | - | 32K | **Supports tools** (v0.3+) |\n| `ollama-mixtral` | `mixtral:8x7b` | ✅ | - | ✅ | - | 32K | MoE, good multi-domain |\n| `ollama-qwen25` | `qwen2.5` | ✅ | - | ✅ | - | 128K | Strong tool support |\n| `ollama-phi4` | `phi4` | ✅ | - | ⚠️ | - | 16K | Limited tool support |\n| `ollama-embedding` | `nomic-embed-text` | - | ✅ | - | - | 8K | Best local embedding |\n| `ollama-mxbai-embed` | `mxbai-embed-large` | - | ✅ | - | - | 512 | Alternative embedding |\n\n> **Tool Calling Update (2025)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), Mixtral, and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ | Fully Supported |\n| ⚠️ | Limited/Experimental (may need prompt adjustments) |\n| - | Not applicable |\n\n### Recommendations by Use Case\n\n| Use Case | Recommended Model | Why |\n|----------|-------------------|-----|\n| **Production Chat** | `gpt-4.1` or `gemini-2.5-pro` | Best instruction following, 1M context |\n| **Balanced (Default)** | `gpt-4.1-mini` or `gemini-2.5-flash` | Good quality at lower cost (Policy Bot default) |\n| **Budget Chat** | `gpt-4.1-nano` or `gemini-2.5-flash-lite` | Cost-effective for simple queries |\n| **Deep Reasoning** | `gemini-2.5-pro` | Built-in thinking capabilities |\n| **Local/Offline Chat** | `ollama-llama3.2` or `ollama-qwen2.5` | Full tool support, runs locally |\n| **RAG Embeddings** | `text-embedding-3-large` | Best quality for retrieval (Policy Bot default) |\n| **Local Embeddings** | `ollama-embedding` (nomic) | Good quality, no API cost |\n| **Audio Transcription** | `voxtral-transcribe` | Beats Whisper, half the cost |\n| **Budget Transcription** | `whisper-1` | Well-established, $0.006/min |\n| **Function Calling** | `gpt-4.1-mini`, `gemini-2.5-flash`, `mistral-large-3` | Reliable tool use |\n\n---\n\n## Switching Providers\n\nTo switch your entire application from OpenAI to Ollama:\n\n1. **No code changes needed**\n2. Update model name in your application config:\n\n```python\n# Before (OpenAI)\nMODEL = \"openai-gpt41\"\nEMBEDDING_MODEL = \"openai-embedding-large\"\nAUDIO_MODEL = \"openai-whisper\"\n\n# After (Ollama + Mistral Voxtral for audio)\nMODEL = \"ollama-llama32\"\nEMBEDDING_MODEL = \"ollama-embedding\"\nAUDIO_MODEL = \"voxtral-mini\"  # Or keep openai-whisper\n\n# Or fully Mistral\nMODEL = \"mistral-large-3\"\nEMBEDDING_MODEL = \"mistral-embedding\"\nAUDIO_MODEL = \"voxtral-small\"\n```\n\n---\n\n## Troubleshooting\n\n| Issue | Solution |\n|-------|----------|\n| Connection refused | Check if Docker is running, verify port 4000 |\n| Ollama not reachable | Use `host.docker.internal` in Docker, check Ollama is running |\n| API key errors | Verify `.env` file is loaded, check key format |\n| Model not found | Run `/models` endpoint to list available models |\n| Timeout errors | Increase `request_timeout` in config |\n\n---\n\n## Resources\n\n- **LiteLLM Docs**: https://docs.litellm.ai/docs/\n- **Supported Providers**: https://docs.litellm.ai/docs/providers\n- **GitHub**: https://github.com/BerriAI/litellm\n- **Ollama Models**: https://ollama.ai/library\n\n---\n\n*Last updated: December 2025 - Includes GPT-4.1, Mistral 3, Voxtral, and Ollama tool support updates*"}, "1cdd8e5ed7baf716e70b8b3eb6e938fdfed5aa54": {"t": 1785754373.616291, "n": 28954, "h": "7ed458f23b72488b", "c": "# LiteLLM Implementation Guide\n\n> Customized for multi-provider LLM abstraction with proxy approach, embeddings routing, function calling support, and audio transcription.\n\n---\n\n## LLM Service Routing Overview\n\nPolicy Bot uses a **hybrid architecture**: core chat services route through LiteLLM proxy for unified model management, while specialized services (images, audio, OCR) call provider APIs directly for capabilities not available through the proxy.\n\n### Service Routing Table\n\n| Service | Policy Bot Feature | Routes Through | Provider(s) | Notes |\n|---------|-------------------|----------------|-------------|-------|\n| **Chat Completions** | Main chat, RAG responses | ✅ LiteLLM | OpenAI, Gemini, Mistral, Anthropic, Ollama | Primary conversation engine |\n| **Embeddings** | Document indexing, search | ✅ LiteLLM | OpenAI | `text-embedding-3-large` default |\n| **Diagram Generation** | `diagram_gen` tool | ✅ LiteLLM | OpenAI, Gemini, Mistral | Generates Mermaid syntax |\n| **Summarization** | Message compression | ✅ LiteLLM | OpenAI, Gemini, Mistral | Long conversation handling |\n| **Memory Extraction** | User fact storage | ✅ LiteLLM | OpenAI, Gemini, Mistral | Per-category memory |\n| **Prompt Optimization** | Query refinement | ✅ LiteLLM | OpenAI, Gemini, Mistral | Pre-RAG processing |\n| **Compliance Checks** | HITL clarification | ✅ LiteLLM | OpenAI, Gemini, Mistral | Skill compliance |\n| **Translation** | Multi-language support | ✅ LiteLLM / ❌ Direct | OpenAI (proxy), Gemini/Mistral (direct) | Provider-dependent |\n| **Audio Transcription** | Voice input | ❌ Direct | OpenAI Whisper | Not available via LiteLLM |\n| **Image Generation** | `image_gen` tool | ❌ Direct | OpenAI DALL-E, Gemini Imagen | Specialized APIs |\n| **Podcast Generation** | `podcast_gen` tool | ❌ Direct | OpenAI TTS, Gemini TTS | Multi-voice audio synthesis |\n| **OCR** | PDF/image text extraction | ❌ Direct | Mistral OCR | Document processing |\n| **Reranking** | Search result scoring | ❌ Direct | Cohere | Not an LLM service |\n| **PPTX Generation** | `pptx_gen` tool | N/A | None | Template-based (no LLM) |\n| **XLSX Generation** | `xlsx_gen` tool | N/A | None | ExcelJS-based (no LLM) |\n| **Chart Generation** | `chart_gen` tool | N/A | None | Client-side rendering |\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ LiteLLM | Routes through LiteLLM proxy (port 4000) |\n| ❌ Direct | Calls provider API directly (bypasses LiteLLM) |\n| N/A | No LLM calls required |\n\n### Why Some Services Bypass LiteLLM\n\n| Service | Reason |\n|---------|--------|\n| **Image Generation** | DALL-E and Gemini Imagen APIs not supported by LiteLLM proxy |\n| **Podcast TTS** | OpenAI TTS and Gemini TTS are audio generation APIs, not chat completions |\n| **Transcription** | Whisper API uses different endpoint structure (`/audio/transcriptions`) |\n| **OCR** | Mistral OCR is a specialized document API with unique request format |\n| **Reranking** | Cohere reranker is a specialized search API, not a chat model |\n\n### Proxy Detection Logic\n\n```typescript\n// Services detect LiteLLM mode by checking the base URL\nconst isUsingProxy = baseUrl.includes('litellm') || baseUrl.includes(':4000');\n\nif (isUsingProxy) {\n  // Use LITELLM_MASTER_KEY for authentication\n  // All chat requests route through unified proxy\n} else {\n  // Use provider-specific API keys directly\n}\n```\n\n---\n\n## Policy Bot Integration\n\nThis guide is configured for the Policy Bot RAG application. Default model preset:\n\n| Setting | Value |\n|---------|-------|\n| Default Model | `gpt-4.1-mini` |\n| Temperature | `0.2` |\n| Max Tokens | `2000` |\n| Embedding Model | `text-embedding-3-large` |\n| Embedding Dimensions | `3072` |\n\nAvailable model presets in Policy Bot (via `config/defaults.json`):\n- **gpt-4.1** - High Performance (1M context)\n- **gpt-4.1-mini** - Balanced (default)\n- **gpt-4.1-nano** - Cost-Effective\n- **mistral-large-3** - Mistral Flagship (256K context)\n- **mistral-small-3.2** - Mistral Cost-Effective\n- **ministral-8b** - Mistral Ultra Cost-Effective\n- **gemini-2.5-pro** - Google Flagship Reasoning (1M context)\n- **gemini-2.5-flash** - Google Balanced (1M context)\n- **gemini-2.5-flash-lite** - Google Cost-Effective (1M context)\n- **ollama-llama3.2** - Local (no API cost)\n- **ollama-qwen2.5** - Local with excellent reasoning\n\n---\n\n## Architecture Overview\n\n```\n┌─────────────────────────────────────────────────────────────────┐\n│                      Your Application                            │\n│  (OpenAI SDK client pointing to http://localhost:4000)          │\n└─────────────────────┬───────────────────────────────────────────┘\n                      │\n                      ▼\n┌─────────────────────────────────────────────────────────────────┐\n│                    LiteLLM Proxy (Port 4000)                    │\n│                    Docker Compose Service                        │\n└─────────┬───────────┬───────────┬───────────┬───────────────────┘\n          │           │           │           │\n          ▼           ▼           ▼           ▼\n     ┌────────┐  ┌────────┐  ┌────────┐  ┌────────┐\n     │ OpenAI │  │ Azure  │  │Mistral │  │ Ollama │\n     │        │  │ OpenAI │  │        │  │(Local) │\n     └────────┘  └────────┘  └────────┘  └────────┘\n```\n\n---\n\n## Decision Summary\n\n| Component | Decision |\n|-----------|----------|\n| **Approach** | LiteLLM Proxy (Docker Compose) |\n| **Chat Completions** | Route through LiteLLM |\n| **Embeddings** | Route through LiteLLM (per-provider config) |\n| **Function Calling** | OpenAI, Claude, Mistral only; warn for others |\n| **Audio (Whisper)** | Abstract through LiteLLM |\n| **Deployment** | Docker Compose |\n\n---\n\n## Project Structure\n\n```\nlitellm-proxy/\n├── docker-compose.yml\n├── litellm_config.yaml\n├── .env\n└── README.md\n```\n\n---\n\n## Step 1: Environment Variables\n\nCreate `.env` file:\n\n```bash\n# ===================\n# LiteLLM Proxy Keys\n# ===================\nLITELLM_MASTER_KEY=sk-litellm-master-change-this\nLITELLM_SALT_KEY=sk-litellm-salt-change-this\n\n# ===================\n# OpenAI\n# ===================\nOPENAI_API_KEY=sk-...\n\n# ===================\n# Azure OpenAI\n# ===================\nAZURE_API_KEY=...\nAZURE_API_BASE=https://your-resource.openai.azure.com/\nAZURE_API_VERSION=2024-02-15-preview\nAZURE_CHAT_DEPLOYMENT=gpt-4-deployment\nAZURE_EMBEDDING_DEPLOYMENT=text-embedding-ada-002\n\n# ===================\n# Mistral\n# ===================\nMISTRAL_API_KEY=...\n\n# ===================\n# Google Gemini\n# ===================\nGEMINI_API_KEY=...\n\n# ===================\n# Ollama (Local)\n# ===================\nOLLAMA_API_BASE=http://host.docker.internal:11434\n# Use 'http://localhost:11434' if running outside Docker\n```\n\n---\n\n## Step 2: LiteLLM Configuration\n\nCreate `litellm_config.yaml`:\n\n```yaml\n# =============================================================================\n# LITELLM PROXY CONFIGURATION\n# Multi-provider setup: OpenAI, Azure, Mistral, Ollama\n# Updated: December 2025\n# =============================================================================\n\nmodel_list:\n\n  # ===========================================================================\n  # CHAT COMPLETION MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Models - GPT-4.1 Family (Latest - April 2025+)\n  # ---------------------------------------------------------------------------\n  - model_name: openai-gpt41\n    litellm_params:\n      model: gpt-4.1\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-mini\n    litellm_params:\n      model: gpt-4.1-mini\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-nano\n    litellm_params:\n      model: gpt-4.1-nano\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt35\n    litellm_params:\n      model: gpt-3.5-turbo\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Models - GPT-4.1 Family\n  # ---------------------------------------------------------------------------\n  - model_name: azure-gpt41\n    litellm_params:\n      model: azure/gpt-41-deployment  # Your deployment name\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: azure-gpt41-mini\n    litellm_params:\n      model: azure/gpt-41-mini-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Mistral AI Models - Mistral 3 Family (Latest - December 2025)\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-large-3\n    litellm_params:\n      model: mistral/mistral-large-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 256000\n\n  - model_name: mistral-medium-31\n    litellm_params:\n      model: mistral/mistral-medium-2508\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      supports_vision: true\n\n  - model_name: mistral-small-32\n    litellm_params:\n      model: mistral/mistral-small-2506\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ministral-8b\n    litellm_params:\n      model: mistral/ministral-8b-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Ollama Models (Local) - With Full Tool Support\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-llama32\n    litellm_params:\n      model: ollama/llama3.2\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # Fine-tuned for function calling\n\n  - model_name: ollama-llama31-8b\n    litellm_params:\n      model: ollama/llama3.1:8b\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-mistral\n    litellm_params:\n      model: ollama/mistral\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # v0.3+ supports tools\n\n  - model_name: ollama-qwen25\n    litellm_params:\n      model: ollama/qwen2.5\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-phi4\n    litellm_params:\n      model: ollama/phi4\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: false  # Limited support\n\n  # ===========================================================================\n  # EMBEDDING MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: openai-embedding-large\n    litellm_params:\n      model: text-embedding-3-large\n      api_key: os.environ/OPENAI_API_KEY\n\n  - model_name: openai-embedding-small\n    litellm_params:\n      model: text-embedding-3-small\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: azure-embedding\n    litellm_params:\n      model: azure/text-embedding-ada-002\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  - model_name: azure-embedding-3-large\n    litellm_params:\n      model: azure/text-embedding-3-large-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-embedding\n    litellm_params:\n      model: mistral/mistral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: codestral-embed\n    litellm_params:\n      model: mistral/codestral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Ollama Embeddings (Local)\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-embedding\n    litellm_params:\n      model: ollama/nomic-embed-text\n      api_base: os.environ/OLLAMA_API_BASE\n\n  - model_name: ollama-mxbai-embed\n    litellm_params:\n      model: ollama/mxbai-embed-large\n      api_base: os.environ/OLLAMA_API_BASE\n\n  # ===========================================================================\n  # AUDIO MODELS (Speech-to-Text / Transcription)\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Whisper\n  # ---------------------------------------------------------------------------\n  - model_name: openai-whisper\n    litellm_params:\n      model: whisper-1\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure Whisper (if deployed)\n  # ---------------------------------------------------------------------------\n  - model_name: azure-whisper\n    litellm_params:\n      model: azure/whisper-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Voxtral (Released July 2025 - Beats Whisper at half cost)\n  # ---------------------------------------------------------------------------\n  - model_name: voxtral-small\n    litellm_params:\n      model: mistral/voxtral-small-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: voxtral-mini\n    litellm_params:\n      model: mistral/voxtral-mini-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n# =============================================================================\n# LITELLM SETTINGS\n# =============================================================================\n\nlitellm_settings:\n  # Drop unsupported params instead of erroring\n  drop_params: true\n  \n  # Enable detailed logging\n  set_verbose: false\n\n  # Request timeout\n  request_timeout: 120\n\n# =============================================================================\n# GENERAL SETTINGS\n# =============================================================================\n\ngeneral_settings:\n  # Master key for proxy authentication\n  master_key: os.environ/LITELLM_MASTER_KEY\n```\n\n---\n\n## Step 3: Docker Compose\n\nCreate `docker-compose.yml`:\n\n```yaml\nversion: \"3.9\"\n\nservices:\n  litellm:\n    image: ghcr.io/berriai/litellm:main-latest\n    container_name: litellm-proxy\n    ports:\n      - \"4000:4000\"\n    volumes:\n      - ./litellm_config.yaml:/app/config.yaml\n    env_file:\n      - .env\n    environment:\n      - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY}\n      - LITELLM_SALT_KEY=${LITELLM_SALT_KEY}\n    command: --config /app/config.yaml --port 4000 --detailed_debug\n    restart: unless-stopped\n    extra_hosts:\n      - \"host.docker.internal:host-gateway\"\n    healthcheck:\n      test: [\"CMD\", \"curl\", \"-f\", \"http://localhost:4000/health\"]\n      interval: 30s\n      timeout: 10s\n      retries: 3\n\n  # Optional: Ollama service (if not running separately)\n  # ollama:\n  #   image: ollama/ollama:latest\n  #   container_name: ollama\n  #   ports:\n  #     - \"11434:11434\"\n  #   volumes:\n  #     - ollama_data:/root/.ollama\n  #   restart: unless-stopped\n\n# volumes:\n#   ollama_data:\n```\n\n---\n\n## Step 4: Application Code Changes\n\n### Minimal Change Required\n\n**Before (Direct OpenAI):**\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(api_key=\"sk-...\")\n```\n\n**After (Via LiteLLM Proxy):**\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(\n    base_url=\"http://localhost:4000\",\n    api_key=\"sk-litellm-master-change-this\"  # Your LITELLM_MASTER_KEY\n)\n```\n\nThat's it! All other code stays the same.\n\n---\n\n## Step 5: Function Calling Handling\n\nSince your `generateResponseWithTools()` uses function calling, here's how to handle provider capabilities:\n\n```python\n# Provider capability mapping (Updated December 2025)\n\n# Full tool/function calling support\n# Matches Policy Bot config/defaults.json toolCapable list\nTOOL_CAPABLE_MODELS = [\n    # OpenAI\n    \"gpt-4.1\", \"gpt-4.1-mini\", \"gpt-4.1-nano\", \"gpt-3.5-turbo\",\n    # Mistral\n    \"mistral-large-3\", \"mistral-medium-3.1\", \"mistral-small-3.2\",\n    \"ministral-8b\", \"ministral-3b\",\n    # Ollama (Now with full tool support!)\n    \"ollama-llama3.2\", \"ollama-llama3.1\", \"ollama-mistral\", \"ollama-qwen2.5\",\n]\n\n# Limited or no tool support (show warning)\nTOOL_LIMITED_MODELS = [\n    \"ollama-phi4\",\n]\n\ndef generate_response_with_tools(model: str, messages: list, tools: list):\n    \"\"\"\n    Generate response with function calling support.\n    Warns and disables tools for unsupported providers.\n    \"\"\"\n    \n    if model in TOOL_LIMITED_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' has limited function calling support.\")\n        print(f\"   Tools will be disabled. Consider using: {', '.join(TOOL_CAPABLE_MODELS[:5])}\")\n        # Call without tools\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages\n        )\n    elif model not in TOOL_CAPABLE_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' tool support unknown. Attempting with tools...\")\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    else:\n        # Full tool support\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    \n    return response\n```\n\n> **Good News (2025 Update)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n---\n\n## Step 6: Embeddings Usage\n\n```python\n# Embeddings via LiteLLM Proxy\n\n# OpenAI embeddings (recommended for RAG quality)\nresponse = client.embeddings.create(\n    model=\"openai-embedding-large\",\n    input=\"Your text here\"\n)\n\n# Azure embeddings\nresponse = client.embeddings.create(\n    model=\"azure-embedding\",\n    input=\"Your text here\"\n)\n\n# Mistral embeddings\nresponse = client.embeddings.create(\n    model=\"mistral-embedding\",\n    input=\"Your text here\"\n)\n\n# Ollama embeddings (local)\nresponse = client.embeddings.create(\n    model=\"ollama-embedding\",\n    input=\"Your text here\"\n)\n\nembedding = response.data[0].embedding\n```\n\n### Important: RAG Consistency\n\n> ⚠️ **Warning**: Don't mix embedding providers for the same vector store. If your RAG was built with `text-embedding-3-large`, query with the same model.\n\n```python\n# Configuration approach\nEMBEDDING_CONFIG = {\n    \"openai\": \"openai-embedding-large\",\n    \"azure\": \"azure-embedding\",\n    \"mistral\": \"mistral-embedding\",\n    \"ollama\": \"ollama-embedding\"\n}\n\n# Use consistently\ncurrent_provider = \"openai\"\nembedding_model = EMBEDDING_CONFIG[current_provider]\n```\n\n---\n\n## Step 7: Audio Transcription (Whisper)\n\n```python\n# Audio transcription via LiteLLM Proxy\n\naudio_file = open(\"audio.mp3\", \"rb\")\n\nresponse = client.audio.transcriptions.create(\n    model=\"openai-whisper\",\n    file=audio_file\n)\n\nprint(response.text)\n```\n\n---\n\n## Deployment Commands\n\n```bash\n# Start the proxy\ndocker compose up -d\n\n# View logs\ndocker compose logs -f litellm\n\n# Stop\ndocker compose down\n\n# Restart after config changes\ndocker compose restart litellm\n```\n\n---\n\n## Testing the Setup\n\n### 1. Health Check\n\n```bash\ncurl http://localhost:4000/health\n```\n\n### 2. List Available Models\n\n```bash\ncurl http://localhost:4000/models \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\"\n```\n\n### 3. Test Chat Completion\n\n```bash\ncurl -X POST http://localhost:4000/chat/completions \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-gpt35\",\n    \"messages\": [{\"role\": \"user\", \"content\": \"Hello, test!\"}]\n  }'\n```\n\n### 4. Test Embeddings\n\n```bash\ncurl -X POST http://localhost:4000/embeddings \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-embedding-large\",\n    \"input\": \"Test embedding\"\n  }'\n```\n\n---\n\n## Model Quick Reference\n\n*Updated: December 2025*\n\n### OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `openai-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Latest flagship, best for coding |\n| `openai-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Faster, 83% cheaper than GPT-4o |\n| `openai-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Fastest, cheapest |\n| `openai-gpt35` | `gpt-3.5-turbo` | ✅ | - | ✅ | - | 16K | Legacy, cost-effective |\n| `openai-embedding-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality embeddings |\n| `openai-embedding-small` | `text-embedding-3-small` | - | ✅ | - | - | 8K | Faster, cheaper |\n| `openai-whisper` | `whisper-1` | - | - | - | ✅ | - | $0.006/min transcription |\n\n### Azure OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `azure-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-embedding` | `text-embedding-ada-002` | - | ✅ | - | - | 8K | Standard embedding |\n| `azure-embedding-3-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality |\n| `azure-whisper` | `whisper` | - | - | - | ✅ | - | Speech-to-text |\n\n### Mistral AI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `mistral-large-3` | `mistral-large-latest` | ✅ | - | ✅ | - | 256K | Flagship MoE (675B total params) |\n| `mistral-medium-31` | `mistral-medium-2508` | ✅ | - | ✅ | - | 128K | Frontier multimodal |\n| `mistral-small-32` | `mistral-small-2506` | ✅ | - | ✅ | - | 128K | Fast, efficient |\n| `ministral-14b` | `ministral-3-14b` | ✅ | - | ✅ | - | 128K | Best small model |\n| `ministral-8b` | `ministral-3-8b` | ✅ | - | ✅ | - | 128K | Edge deployment |\n| `ministral-3b` | `ministral-3-3b` | ✅ | - | ✅ | - | 128K | Ultra-light |\n| `mistral-embedding` | `mistral-embed` | - | ✅ | - | - | 8K | Text embeddings |\n| `codestral-embed` | `codestral-embed` | - | ✅ | - | - | - | Code embeddings |\n| `voxtral-small` | `voxtral-small-latest` | - | - | - | ✅ | 32K | 24B, best accuracy, $0.001/min |\n| `voxtral-mini` | `voxtral-mini-latest` | - | - | - | ✅ | 32K | 3B, edge/local deployment |\n| `voxtral-transcribe` | `voxtral-mini-latest` (via /audio/transcriptions) | - | - | - | ✅ | - | Transcription-only, beats Whisper |\n\n> **Note**: Voxtral (released July 2025) outperforms OpenAI Whisper at half the cost ($0.001/min vs $0.006/min). Supports up to 30 min audio for transcription, 40 min for understanding.\n\n### Google Gemini Models\n\n| Model Name | API Model ID | Chat | Tools | Context | Notes |\n|------------|--------------|------|-------|---------|-------|\n| `gemini-2.5-pro` | `gemini/gemini-2.5-pro` | ✅ | ✅ | 1M | Flagship reasoning model |\n| `gemini-2.5-flash` | `gemini/gemini-2.5-flash` | ✅ | ✅ | 1M | Fast, balanced performance |\n| `gemini-2.5-flash-lite` | `gemini/gemini-2.5-flash-lite` | ✅ | ✅ | 1M | Lowest cost option |\n\n**Pricing (per 1M tokens):**\n- Gemini 2.5 Pro: $1.25 input / $10.00 output (≤200k context)\n- Gemini 2.5 Flash: $0.30 input / $2.50 output\n- Gemini 2.5 Flash-Lite: $0.10 input / $0.40 output\n\n**Features:**\n- All models support function calling\n- Built-in \"thinking\" capabilities with controllable budgets\n- Free tier available for development/testing\n- Native multimodal support (text, image, audio, video)\n\n### Ollama Local Models\n\n| Model Name | Ollama Model | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `ollama-llama32` | `llama3.2` | ✅ | - | ✅ | - | 128K | **Full tool support** (fine-tuned for function calling) |\n| `ollama-llama31-8b` | `llama3.1:8b` | ✅ | - | ✅ | - | 128K | **Full tool support**, best overall |\n| `ollama-llama31-70b` | `llama3.1:70b` | ✅ | - | ✅ | - | 128K | **Full tool support**, highest accuracy |\n| `ollama-mistral` | `mistral` | ✅ | - | ✅ | - | 32K | **Supports tools** (v0.3+) |\n| `ollama-mixtral` | `mixtral:8x7b` | ✅ | - | ✅ | - | 32K | MoE, good multi-domain |\n| `ollama-qwen25` | `qwen2.5` | ✅ | - | ✅ | - | 128K | Strong tool support |\n| `ollama-phi4` | `phi4` | ✅ | - | ⚠️ | - | 16K | Limited tool support |\n| `ollama-embedding` | `nomic-embed-text` | - | ✅ | - | - | 8K | Best local embedding |\n| `ollama-mxbai-embed` | `mxbai-embed-large` | - | ✅ | - | - | 512 | Alternative embedding |\n\n> **Tool Calling Update (2025)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), Mixtral, and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ | Fully Supported |\n| ⚠️ | Limited/Experimental (may need prompt adjustments) |\n| - | Not applicable |\n\n### Recommendations by Use Case\n\n| Use Case | Recommended Model | Why |\n|----------|-------------------|-----|\n| **Production Chat** | `gpt-4.1` or `gemini-2.5-pro` | Best instruction following, 1M context |\n| **Balanced (Default)** | `gpt-4.1-mini` or `gemini-2.5-flash` | Good quality at lower cost (Policy Bot default) |\n| **Budget Chat** | `gpt-4.1-nano` or `gemini-2.5-flash-lite` | Cost-effective for simple queries |\n| **Deep Reasoning** | `gemini-2.5-pro` | Built-in thinking capabilities |\n| **Local/Offline Chat** | `ollama-llama3.2` or `ollama-qwen2.5` | Full tool support, runs locally |\n| **RAG Embeddings** | `text-embedding-3-large` | Best quality for retrieval (Policy Bot default) |\n| **Local Embeddings** | `ollama-embedding` (nomic) | Good quality, no API cost |\n| **Audio Transcription** | `voxtral-transcribe` | Beats Whisper, half the cost |\n| **Budget Transcription** | `whisper-1` | Well-established, $0.006/min |\n| **Function Calling** | `gpt-4.1-mini`, `gemini-2.5-flash`, `mistral-large-3` | Reliable tool use |\n\n---\n\n## Switching Providers\n\nTo switch your entire application from OpenAI to Ollama:\n\n1. **No code changes needed**\n2. Update model name in your application config:\n\n```python\n# Before (OpenAI)\nMODEL = \"openai-gpt41\"\nEMBEDDING_MODEL = \"openai-embedding-large\"\nAUDIO_MODEL = \"openai-whisper\"\n\n# After (Ollama + Mistral Voxtral for audio)\nMODEL = \"ollama-llama32\"\nEMBEDDING_MODEL = \"ollama-embedding\"\nAUDIO_MODEL = \"voxtral-mini\"  # Or keep openai-whisper\n\n# Or fully Mistral\nMODEL = \"mistral-large-3\"\nEMBEDDING_MODEL = \"mistral-embedding\"\nAUDIO_MODEL = \"voxtral-small\"\n```\n\n---\n\n## Troubleshooting\n\n| Issue | Solution |\n|-------|----------|\n| Connection refused | Check if Docker is running, verify port 4000 |\n| Ollama not reachable | Use `host.docker.internal` in Docker, check Ollama is running |\n| API key errors | Verify `.env` file is loaded, check key format |\n| Model not found | Run `/models` endpoint to list available models |\n| Timeout errors | Increase `request_timeout` in config |\n\n---\n\n## Resources\n\n- **LiteLLM Docs**: https://docs.litellm.ai/docs/\n- **Supported Providers**: https://docs.litellm.ai/docs/providers\n- **GitHub**: https://github.com/BerriAI/litellm\n- **Ollama Models**: https://ollama.ai/library\n\n---\n\n*Last updated: December 2025 - Includes GPT-4.1, Mistral 3, Voxtral, and Ollama tool support updates*"}, "a823c770b40b0c6e74ec130a62187c7647762ae6": {"t": 1785754375.0775328, "n": 25762, "h": "e6e8d91578070b56", "c": "# LiteLLM Implementation Guide\n\n> Customized for multi-provider LLM abstraction with proxy approach, embeddings routing, function calling support, and audio transcription.\n\n## Policy Bot Integration\n\nThis guide is configured for the Policy Bot RAG application. Default model preset:\n\n| Setting | Value |\n|---------|-------|\n| Default Model | `gpt-4.1-mini` |\n| Temperature | `0.2` |\n| Max Tokens | `2000` |\n| Embedding Model | `text-embedding-3-large` |\n| Embedding Dimensions | `3072` |\n\nAvailable model presets in Policy Bot (via `config/defaults.json`):\n- **gpt-4.1** - High Performance (1M context)\n- **gpt-4.1-mini** - Balanced (default)\n- **gpt-4.1-nano** - Cost-Effective\n- **mistral-large-3** - Mistral Flagship (256K context)\n- **mistral-small-3.2** - Mistral Cost-Effective\n- **ministral-8b** - Mistral Ultra Cost-Effective\n- **gemini-2.5-pro** - Google Flagship Reasoning (1M context)\n- **gemini-2.5-flash** - Google Balanced (1M context)\n- **gemini-2.5-flash-lite** - Google Cost-Effective (1M context)\n- **ollama-llama3.2** - Local (no API cost)\n- **ollama-qwen2.5** - Local with excellent reasoning\n\n---\n\n## Architecture Overview\n\n```\n┌─────────────────────────────────────────────────────────────────┐\n│                      Your Application                            │\n│  (OpenAI SDK client pointing to http://localhost:4000)          │\n└─────────────────────┬───────────────────────────────────────────┘\n                      │\n                      ▼\n┌─────────────────────────────────────────────────────────────────┐\n│                    LiteLLM Proxy (Port 4000)                    │\n│                    Docker Compose Service                        │\n└─────────┬───────────┬───────────┬───────────┬───────────────────┘\n          │           │           │           │\n          ▼           ▼           ▼           ▼\n     ┌────────┐  ┌────────┐  ┌────────┐  ┌────────┐\n     │ OpenAI │  │ Azure  │  │Mistral │  │ Ollama │\n     │        │  │ OpenAI │  │        │  │(Local) │\n     └────────┘  └────────┘  └────────┘  └────────┘\n```\n\n---\n\n## Decision Summary\n\n| Component | Decision |\n|-----------|----------|\n| **Approach** | LiteLLM Proxy (Docker Compose) |\n| **Chat Completions** | Route through LiteLLM |\n| **Embeddings** | Route through LiteLLM (per-provider config) |\n| **Function Calling** | OpenAI, Claude, Mistral only; warn for others |\n| **Audio (Whisper)** | Abstract through LiteLLM |\n| **Deployment** | Docker Compose |\n\n---\n\n## Project Structure\n\n```\nlitellm-proxy/\n├── docker-compose.yml\n├── litellm_config.yaml\n├── .env\n└── README.md\n```\n\n---\n\n## Step 1: Environment Variables\n\nCreate `.env` file:\n\n```bash\n# ===================\n# LiteLLM Proxy Keys\n# ===================\nLITELLM_MASTER_KEY=sk-litellm-master-change-this\nLITELLM_SALT_KEY=sk-litellm-salt-change-this\n\n# ===================\n# OpenAI\n# ===================\nOPENAI_API_KEY=sk-...\n\n# ===================\n# Azure OpenAI\n# ===================\nAZURE_API_KEY=...\nAZURE_API_BASE=https://your-resource.openai.azure.com/\nAZURE_API_VERSION=2024-02-15-preview\nAZURE_CHAT_DEPLOYMENT=gpt-4-deployment\nAZURE_EMBEDDING_DEPLOYMENT=text-embedding-ada-002\n\n# ===================\n# Mistral\n# ===================\nMISTRAL_API_KEY=...\n\n# ===================\n# Google Gemini\n# ===================\nGEMINI_API_KEY=...\n\n# ===================\n# Ollama (Local)\n# ===================\nOLLAMA_API_BASE=http://host.docker.internal:11434\n# Use 'http://localhost:11434' if running outside Docker\n```\n\n---\n\n## Step 2: LiteLLM Configuration\n\nCreate `litellm_config.yaml`:\n\n```yaml\n# =============================================================================\n# LITELLM PROXY CONFIGURATION\n# Multi-provider setup: OpenAI, Azure, Mistral, Ollama\n# Updated: December 2025\n# =============================================================================\n\nmodel_list:\n\n  # ===========================================================================\n  # CHAT COMPLETION MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Models - GPT-4.1 Family (Latest - April 2025+)\n  # ---------------------------------------------------------------------------\n  - model_name: openai-gpt41\n    litellm_params:\n      model: gpt-4.1\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-mini\n    litellm_params:\n      model: gpt-4.1-mini\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt41-nano\n    litellm_params:\n      model: gpt-4.1-nano\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: openai-gpt35\n    litellm_params:\n      model: gpt-3.5-turbo\n      api_key: os.environ/OPENAI_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Models - GPT-4.1 Family\n  # ---------------------------------------------------------------------------\n  - model_name: azure-gpt41\n    litellm_params:\n      model: azure/gpt-41-deployment  # Your deployment name\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 1000000\n\n  - model_name: azure-gpt41-mini\n    litellm_params:\n      model: azure/gpt-41-mini-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Mistral AI Models - Mistral 3 Family (Latest - December 2025)\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-large-3\n    litellm_params:\n      model: mistral/mistral-large-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      max_input_tokens: 256000\n\n  - model_name: mistral-medium-31\n    litellm_params:\n      model: mistral/mistral-medium-2508\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n      supports_vision: true\n\n  - model_name: mistral-small-32\n    litellm_params:\n      model: mistral/mistral-small-2506\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ministral-8b\n    litellm_params:\n      model: mistral/ministral-8b-latest\n      api_key: os.environ/MISTRAL_API_KEY\n    model_info:\n      supports_function_calling: true\n\n  # ---------------------------------------------------------------------------\n  # Ollama Models (Local) - With Full Tool Support\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-llama32\n    litellm_params:\n      model: ollama/llama3.2\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # Fine-tuned for function calling\n\n  - model_name: ollama-llama31-8b\n    litellm_params:\n      model: ollama/llama3.1:8b\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-mistral\n    litellm_params:\n      model: ollama/mistral\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true  # v0.3+ supports tools\n\n  - model_name: ollama-qwen25\n    litellm_params:\n      model: ollama/qwen2.5\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: true\n\n  - model_name: ollama-phi4\n    litellm_params:\n      model: ollama/phi4\n      api_base: os.environ/OLLAMA_API_BASE\n    model_info:\n      supports_function_calling: false  # Limited support\n\n  # ===========================================================================\n  # EMBEDDING MODELS\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: openai-embedding-large\n    litellm_params:\n      model: text-embedding-3-large\n      api_key: os.environ/OPENAI_API_KEY\n\n  - model_name: openai-embedding-small\n    litellm_params:\n      model: text-embedding-3-small\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure OpenAI Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: azure-embedding\n    litellm_params:\n      model: azure/text-embedding-ada-002\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  - model_name: azure-embedding-3-large\n    litellm_params:\n      model: azure/text-embedding-3-large-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Embeddings\n  # ---------------------------------------------------------------------------\n  - model_name: mistral-embedding\n    litellm_params:\n      model: mistral/mistral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: codestral-embed\n    litellm_params:\n      model: mistral/codestral-embed\n      api_key: os.environ/MISTRAL_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Ollama Embeddings (Local)\n  # ---------------------------------------------------------------------------\n  - model_name: ollama-embedding\n    litellm_params:\n      model: ollama/nomic-embed-text\n      api_base: os.environ/OLLAMA_API_BASE\n\n  - model_name: ollama-mxbai-embed\n    litellm_params:\n      model: ollama/mxbai-embed-large\n      api_base: os.environ/OLLAMA_API_BASE\n\n  # ===========================================================================\n  # AUDIO MODELS (Speech-to-Text / Transcription)\n  # ===========================================================================\n\n  # ---------------------------------------------------------------------------\n  # OpenAI Whisper\n  # ---------------------------------------------------------------------------\n  - model_name: openai-whisper\n    litellm_params:\n      model: whisper-1\n      api_key: os.environ/OPENAI_API_KEY\n\n  # ---------------------------------------------------------------------------\n  # Azure Whisper (if deployed)\n  # ---------------------------------------------------------------------------\n  - model_name: azure-whisper\n    litellm_params:\n      model: azure/whisper-deployment\n      api_base: os.environ/AZURE_API_BASE\n      api_key: os.environ/AZURE_API_KEY\n      api_version: os.environ/AZURE_API_VERSION\n\n  # ---------------------------------------------------------------------------\n  # Mistral Voxtral (Released July 2025 - Beats Whisper at half cost)\n  # ---------------------------------------------------------------------------\n  - model_name: voxtral-small\n    litellm_params:\n      model: mistral/voxtral-small-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n  - model_name: voxtral-mini\n    litellm_params:\n      model: mistral/voxtral-mini-latest\n      api_key: os.environ/MISTRAL_API_KEY\n\n# =============================================================================\n# LITELLM SETTINGS\n# =============================================================================\n\nlitellm_settings:\n  # Drop unsupported params instead of erroring\n  drop_params: true\n  \n  # Enable detailed logging\n  set_verbose: false\n\n  # Request timeout\n  request_timeout: 120\n\n# =============================================================================\n# GENERAL SETTINGS\n# =============================================================================\n\ngeneral_settings:\n  # Master key for proxy authentication\n  master_key: os.environ/LITELLM_MASTER_KEY\n```\n\n---\n\n## Step 3: Docker Compose\n\nCreate `docker-compose.yml`:\n\n```yaml\nversion: \"3.9\"\n\nservices:\n  litellm:\n    image: ghcr.io/berriai/litellm:main-latest\n    container_name: litellm-proxy\n    ports:\n      - \"4000:4000\"\n    volumes:\n      - ./litellm_config.yaml:/app/config.yaml\n    env_file:\n      - .env\n    environment:\n      - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY}\n      - LITELLM_SALT_KEY=${LITELLM_SALT_KEY}\n    command: --config /app/config.yaml --port 4000 --detailed_debug\n    restart: unless-stopped\n    extra_hosts:\n      - \"host.docker.internal:host-gateway\"\n    healthcheck:\n      test: [\"CMD\", \"curl\", \"-f\", \"http://localhost:4000/health\"]\n      interval: 30s\n      timeout: 10s\n      retries: 3\n\n  # Optional: Ollama service (if not running separately)\n  # ollama:\n  #   image: ollama/ollama:latest\n  #   container_name: ollama\n  #   ports:\n  #     - \"11434:11434\"\n  #   volumes:\n  #     - ollama_data:/root/.ollama\n  #   restart: unless-stopped\n\n# volumes:\n#   ollama_data:\n```\n\n---\n\n## Step 4: Application Code Changes\n\n### Minimal Change Required\n\n**Before (Direct OpenAI):**\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(api_key=\"sk-...\")\n```\n\n**After (Via LiteLLM Proxy):**\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(\n    base_url=\"http://localhost:4000\",\n    api_key=\"sk-litellm-master-change-this\"  # Your LITELLM_MASTER_KEY\n)\n```\n\nThat's it! All other code stays the same.\n\n---\n\n## Step 5: Function Calling Handling\n\nSince your `generateResponseWithTools()` uses function calling, here's how to handle provider capabilities:\n\n```python\n# Provider capability mapping (Updated December 2025)\n\n# Full tool/function calling support\n# Matches Policy Bot config/defaults.json toolCapable list\nTOOL_CAPABLE_MODELS = [\n    # OpenAI\n    \"gpt-4.1\", \"gpt-4.1-mini\", \"gpt-4.1-nano\", \"gpt-3.5-turbo\",\n    # Mistral\n    \"mistral-large-3\", \"mistral-medium-3.1\", \"mistral-small-3.2\",\n    \"ministral-8b\", \"ministral-3b\",\n    # Ollama (Now with full tool support!)\n    \"ollama-llama3.2\", \"ollama-llama3.1\", \"ollama-mistral\", \"ollama-qwen2.5\",\n]\n\n# Limited or no tool support (show warning)\nTOOL_LIMITED_MODELS = [\n    \"ollama-phi4\",\n]\n\ndef generate_response_with_tools(model: str, messages: list, tools: list):\n    \"\"\"\n    Generate response with function calling support.\n    Warns and disables tools for unsupported providers.\n    \"\"\"\n    \n    if model in TOOL_LIMITED_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' has limited function calling support.\")\n        print(f\"   Tools will be disabled. Consider using: {', '.join(TOOL_CAPABLE_MODELS[:5])}\")\n        # Call without tools\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages\n        )\n    elif model not in TOOL_CAPABLE_MODELS:\n        print(f\"⚠️  WARNING: Model '{model}' tool support unknown. Attempting with tools...\")\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    else:\n        # Full tool support\n        response = client.chat.completions.create(\n            model=model,\n            messages=messages,\n            tools=tools\n        )\n    \n    return response\n```\n\n> **Good News (2025 Update)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n---\n\n## Step 6: Embeddings Usage\n\n```python\n# Embeddings via LiteLLM Proxy\n\n# OpenAI embeddings (recommended for RAG quality)\nresponse = client.embeddings.create(\n    model=\"openai-embedding-large\",\n    input=\"Your text here\"\n)\n\n# Azure embeddings\nresponse = client.embeddings.create(\n    model=\"azure-embedding\",\n    input=\"Your text here\"\n)\n\n# Mistral embeddings\nresponse = client.embeddings.create(\n    model=\"mistral-embedding\",\n    input=\"Your text here\"\n)\n\n# Ollama embeddings (local)\nresponse = client.embeddings.create(\n    model=\"ollama-embedding\",\n    input=\"Your text here\"\n)\n\nembedding = response.data[0].embedding\n```\n\n### Important: RAG Consistency\n\n> ⚠️ **Warning**: Don't mix embedding providers for the same vector store. If your RAG was built with `text-embedding-3-large`, query with the same model.\n\n```python\n# Configuration approach\nEMBEDDING_CONFIG = {\n    \"openai\": \"openai-embedding-large\",\n    \"azure\": \"azure-embedding\",\n    \"mistral\": \"mistral-embedding\",\n    \"ollama\": \"ollama-embedding\"\n}\n\n# Use consistently\ncurrent_provider = \"openai\"\nembedding_model = EMBEDDING_CONFIG[current_provider]\n```\n\n---\n\n## Step 7: Audio Transcription (Whisper)\n\n```python\n# Audio transcription via LiteLLM Proxy\n\naudio_file = open(\"audio.mp3\", \"rb\")\n\nresponse = client.audio.transcriptions.create(\n    model=\"openai-whisper\",\n    file=audio_file\n)\n\nprint(response.text)\n```\n\n---\n\n## Deployment Commands\n\n```bash\n# Start the proxy\ndocker compose up -d\n\n# View logs\ndocker compose logs -f litellm\n\n# Stop\ndocker compose down\n\n# Restart after config changes\ndocker compose restart litellm\n```\n\n---\n\n## Testing the Setup\n\n### 1. Health Check\n\n```bash\ncurl http://localhost:4000/health\n```\n\n### 2. List Available Models\n\n```bash\ncurl http://localhost:4000/models \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\"\n```\n\n### 3. Test Chat Completion\n\n```bash\ncurl -X POST http://localhost:4000/chat/completions \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-gpt35\",\n    \"messages\": [{\"role\": \"user\", \"content\": \"Hello, test!\"}]\n  }'\n```\n\n### 4. Test Embeddings\n\n```bash\ncurl -X POST http://localhost:4000/embeddings \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer sk-litellm-master-change-this\" \\\n  -d '{\n    \"model\": \"openai-embedding-large\",\n    \"input\": \"Test embedding\"\n  }'\n```\n\n---\n\n## Model Quick Reference\n\n*Updated: December 2025*\n\n### OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `openai-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Latest flagship, best for coding |\n| `openai-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Faster, 83% cheaper than GPT-4o |\n| `openai-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Fastest, cheapest |\n| `openai-gpt35` | `gpt-3.5-turbo` | ✅ | - | ✅ | - | 16K | Legacy, cost-effective |\n| `openai-embedding-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality embeddings |\n| `openai-embedding-small` | `text-embedding-3-small` | - | ✅ | - | - | 8K | Faster, cheaper |\n| `openai-whisper` | `whisper-1` | - | - | - | ✅ | - | $0.006/min transcription |\n\n### Azure OpenAI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `azure-gpt41` | `gpt-4.1` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-mini` | `gpt-4.1-mini` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-gpt41-nano` | `gpt-4.1-nano` | ✅ | - | ✅ | - | 1M | Available April 2025+ |\n| `azure-embedding` | `text-embedding-ada-002` | - | ✅ | - | - | 8K | Standard embedding |\n| `azure-embedding-3-large` | `text-embedding-3-large` | - | ✅ | - | - | 8K | Best quality |\n| `azure-whisper` | `whisper` | - | - | - | ✅ | - | Speech-to-text |\n\n### Mistral AI Models\n\n| Model Name | API Model ID | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `mistral-large-3` | `mistral-large-latest` | ✅ | - | ✅ | - | 256K | Flagship MoE (675B total params) |\n| `mistral-medium-31` | `mistral-medium-2508` | ✅ | - | ✅ | - | 128K | Frontier multimodal |\n| `mistral-small-32` | `mistral-small-2506` | ✅ | - | ✅ | - | 128K | Fast, efficient |\n| `ministral-14b` | `ministral-3-14b` | ✅ | - | ✅ | - | 128K | Best small model |\n| `ministral-8b` | `ministral-3-8b` | ✅ | - | ✅ | - | 128K | Edge deployment |\n| `ministral-3b` | `ministral-3-3b` | ✅ | - | ✅ | - | 128K | Ultra-light |\n| `mistral-embedding` | `mistral-embed` | - | ✅ | - | - | 8K | Text embeddings |\n| `codestral-embed` | `codestral-embed` | - | ✅ | - | - | - | Code embeddings |\n| `voxtral-small` | `voxtral-small-latest` | - | - | - | ✅ | 32K | 24B, best accuracy, $0.001/min |\n| `voxtral-mini` | `voxtral-mini-latest` | - | - | - | ✅ | 32K | 3B, edge/local deployment |\n| `voxtral-transcribe` | `voxtral-mini-latest` (via /audio/transcriptions) | - | - | - | ✅ | - | Transcription-only, beats Whisper |\n\n> **Note**: Voxtral (released July 2025) outperforms OpenAI Whisper at half the cost ($0.001/min vs $0.006/min). Supports up to 30 min audio for transcription, 40 min for understanding.\n\n### Google Gemini Models\n\n| Model Name | API Model ID | Chat | Tools | Context | Notes |\n|------------|--------------|------|-------|---------|-------|\n| `gemini-2.5-pro` | `gemini/gemini-2.5-pro` | ✅ | ✅ | 1M | Flagship reasoning model |\n| `gemini-2.5-flash` | `gemini/gemini-2.5-flash` | ✅ | ✅ | 1M | Fast, balanced performance |\n| `gemini-2.5-flash-lite` | `gemini/gemini-2.5-flash-lite` | ✅ | ✅ | 1M | Lowest cost option |\n\n**Pricing (per 1M tokens):**\n- Gemini 2.5 Pro: $1.25 input / $10.00 output (≤200k context)\n- Gemini 2.5 Flash: $0.30 input / $2.50 output\n- Gemini 2.5 Flash-Lite: $0.10 input / $0.40 output\n\n**Features:**\n- All models support function calling\n- Built-in \"thinking\" capabilities with controllable budgets\n- Free tier available for development/testing\n- Native multimodal support (text, image, audio, video)\n\n### Ollama Local Models\n\n| Model Name | Ollama Model | Chat | Embeddings | Tools | Audio | Context | Notes |\n|------------|--------------|------|------------|-------|-------|---------|-------|\n| `ollama-llama32` | `llama3.2` | ✅ | - | ✅ | - | 128K | **Full tool support** (fine-tuned for function calling) |\n| `ollama-llama31-8b` | `llama3.1:8b` | ✅ | - | ✅ | - | 128K | **Full tool support**, best overall |\n| `ollama-llama31-70b` | `llama3.1:70b` | ✅ | - | ✅ | - | 128K | **Full tool support**, highest accuracy |\n| `ollama-mistral` | `mistral` | ✅ | - | ✅ | - | 32K | **Supports tools** (v0.3+) |\n| `ollama-mixtral` | `mixtral:8x7b` | ✅ | - | ✅ | - | 32K | MoE, good multi-domain |\n| `ollama-qwen25` | `qwen2.5` | ✅ | - | ✅ | - | 128K | Strong tool support |\n| `ollama-phi4` | `phi4` | ✅ | - | ⚠️ | - | 16K | Limited tool support |\n| `ollama-embedding` | `nomic-embed-text` | - | ✅ | - | - | 8K | Best local embedding |\n| `ollama-mxbai-embed` | `mxbai-embed-large` | - | ✅ | - | - | 512 | Alternative embedding |\n\n> **Tool Calling Update (2025)**: Ollama now officially supports tool/function calling for Llama 3.1, Llama 3.2, Mistral (v0.3+), Mixtral, and Qwen2.5. These models can reliably determine when to call functions and generate proper JSON arguments.\n\n### Legend\n\n| Symbol | Meaning |\n|--------|---------|\n| ✅ | Fully Supported |\n| ⚠️ | Limited/Experimental (may need prompt adjustments) |\n| - | Not applicable |\n\n### Recommendations by Use Case\n\n| Use Case | Recommended Model | Why |\n|----------|-------------------|-----|\n| **Production Chat** | `gpt-4.1` or `gemini-2.5-pro` | Best instruction following, 1M context |\n| **Balanced (Default)** | `gpt-4.1-mini` or `gemini-2.5-flash` | Good quality at lower cost (Policy Bot default) |\n| **Budget Chat** | `gpt-4.1-nano` or `gemini-2.5-flash-lite` | Cost-effective for simple queries |\n| **Deep Reasoning** | `gemini-2.5-pro` | Built-in thinking capabilities |\n| **Local/Offline Chat** | `ollama-llama3.2` or `ollama-qwen2.5` | Full tool support, runs locally |\n| **RAG Embeddings** | `text-embedding-3-large` | Best quality for retrieval (Policy Bot default) |\n| **Local Embeddings** | `ollama-embedding` (nomic) | Good quality, no API cost |\n| **Audio Transcription** | `voxtral-transcribe` | Beats Whisper, half the cost |\n| **Budget Transcription** | `whisper-1` | Well-established, $0.006/min |\n| **Function Calling** | `gpt-4.1-mini`, `gemini-2.5-flash`, `mistral-large-3` | Reliable tool use |\n\n---\n\n## Switching Providers\n\nTo switch your entire application from OpenAI to Ollama:\n\n1. **No code changes needed**\n2. Update model name in your application config:\n\n```python\n# Before (OpenAI)\nMODEL = \"openai-gpt41\"\nEMBEDDING_MODEL = \"openai-embedding-large\"\nAUDIO_MODEL = \"openai-whisper\"\n\n# After (Ollama + Mistral Voxtral for audio)\nMODEL = \"ollama-llama32\"\nEMBEDDING_MODEL = \"ollama-embedding\"\nAUDIO_MODEL = \"voxtral-mini\"  # Or keep openai-whisper\n\n# Or fully Mistral\nMODEL = \"mistral-large-3\"\nEMBEDDING_MODEL = \"mistral-embedding\"\nAUDIO_MODEL = \"voxtral-small\"\n```\n\n---\n\n## Troubleshooting\n\n| Issue | Solution |\n|-------|----------|\n| Connection refused | Check if Docker is running, verify port 4000 |\n| Ollama not reachable | Use `host.docker.internal` in Docker, check Ollama is running |\n| API key errors | Verify `.env` file is loaded, check key format |\n| Model not found | Run `/models` endpoint to list available models |\n| Timeout errors | Increase `request_timeout` in config |\n\n---\n\n## Resources\n\n- **LiteLLM Docs**: https://docs.litellm.ai/docs/\n- **Supported Providers**: https://docs.litellm.ai/docs/providers\n- **GitHub**: https://github.com/BerriAI/litellm\n- **Ollama Models**: https://ollama.ai/library\n\n---\n\n*Last updated: December 2025 - Includes GPT-4.1, Mistral 3, Voxtral, and Ollama tool support updates*"}}}}