1 line
59 KiB
JSON
1 line
59 KiB
JSON
{"repo": "anomaly501/geohelper", "files": {"rwaler/hell/Lib/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py": {"e8886f3535aafdf6baeef8a354a5cadef23009bd": {"t": 1785746648.708285, "n": 28873, "h": "afdf9986553a31d3", "c": "from typing import Optional, Tuple\n\nimport httpx\n\nimport litellm\nfrom litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH\nfrom litellm.secret_managers.main import get_secret, get_secret_str\n\nfrom ..types.router import LiteLLM_Params\n\n\ndef _is_non_openai_azure_model(model: str) -> bool:\n try:\n model_name = model.split(\"/\", 1)[1]\n if (\n model_name in litellm.cohere_chat_models\n or f\"mistral/{model_name}\" in litellm.mistral_chat_models\n ):\n return True\n except Exception:\n return False\n return False\n\n\ndef handle_cohere_chat_model_custom_llm_provider(\n model: str, custom_llm_provider: Optional[str] = None\n) -> Tuple[str, Optional[str]]:\n \"\"\"\n if user sets model = \"cohere/command-r\" -> use custom_llm_provider = \"cohere_chat\"\n\n Args:\n model:\n custom_llm_provider:\n\n Returns:\n model, custom_llm_provider\n \"\"\"\n\n if custom_llm_provider:\n if custom_llm_provider == \"cohere\" and model in litellm.cohere_chat_models:\n return model, \"cohere_chat\"\n\n if \"/\" in model:\n _custom_llm_provider, _model = model.split(\"/\", 1)\n if (\n _custom_llm_provider\n and _custom_llm_provider == \"cohere\"\n and _model in litellm.cohere_chat_models\n ):\n return _model, \"cohere_chat\"\n\n return model, custom_llm_provider\n\n\ndef handle_anthropic_text_model_custom_llm_provider(\n model: str, custom_llm_provider: Optional[str] = None\n) -> Tuple[str, Optional[str]]:\n \"\"\"\n if user sets model = \"anthropic/claude-2\" -> use custom_llm_provider = \"anthropic_text\"\n\n Args:\n model:\n custom_llm_provider:\n\n Returns:\n model, custom_llm_provider\n \"\"\"\n\n if custom_llm_provider:\n if (\n custom_llm_provider == \"anthropic\"\n and litellm.AnthropicTextConfig._is_anthropic_text_model(model)\n ):\n return model, \"anthropic_text\"\n\n if \"/\" in model:\n _custom_llm_provider, _model = model.split(\"/\", 1)\n if (\n _custom_llm_provider\n and _custom_llm_provider == \"anthropic\"\n and litellm.AnthropicTextConfig._is_anthropic_text_model(_model)\n ):\n return _model, \"anthropic_text\"\n\n return model, custom_llm_provider\n\n\ndef get_llm_provider( # noqa: PLR0915\n model: str,\n custom_llm_provider: Optional[str] = None,\n api_base: Optional[str] = None,\n api_key: Optional[str] = None,\n litellm_params: Optional[LiteLLM_Params] = None,\n) -> Tuple[str, str, Optional[str], Optional[str]]:\n \"\"\"\n Returns the provider for a given model name - e.g. 'azure/chatgpt-v-2' -> 'azure'\n\n For router -> Can also give the whole litellm param dict -> this function will extract the relevant details\n\n Raises Error - if unable to map model to a provider\n\n Return model, custom_llm_provider, dynamic_api_key, api_base\n \"\"\"\n try:\n if litellm.LiteLLMProxyChatConfig._should_use_litellm_proxy_by_default(\n litellm_params=litellm_params\n ):\n return litellm.LiteLLMProxyChatConfig.litellm_proxy_get_custom_llm_provider_info(\n model=model, api_base=api_base, api_key=api_key\n )\n\n ## IF LITELLM PARAMS GIVEN ##\n if litellm_params:\n assert (\n custom_llm_provider is None and api_base is None and api_key is None\n ), \"Either pass in litellm_params or the custom_llm_provider/api_base/api_key. Otherwise, these values will be overriden.\"\n custom_llm_provider = litellm_params.custom_llm_provider\n api_base = litellm_params.api_base\n api_key = litellm_params.api_key\n\n dynamic_api_key = None\n # check if llm provider provided\n # AZURE AI-Studio Logic - Azure AI Studio supports AZURE/Cohere\n # If User passes azure/command-r-plus -> we should send it to cohere_chat/command-r-plus\n if model.split(\"/\", 1)[0] == \"azure\":\n if _is_non_openai_azure_model(model):\n custom_llm_provider = \"openai\"\n return model, custom_llm_provider, dynamic_api_key, api_base\n\n ### Handle cases when custom_llm_provider is set to cohere/command-r-plus but it should use cohere_chat route\n model, custom_llm_provider = handle_cohere_chat_model_custom_llm_provider(\n model, custom_llm_provider\n )\n\n model, custom_llm_provider = handle_anthropic_text_model_custom_llm_provider(\n model, custom_llm_provider\n )\n\n if custom_llm_provider and (\n model.split(\"/\")[0] != custom_llm_provider\n ): # handle scenario where model=\"azure/*\" and custom_llm_provider=\"azure\"\n model = custom_llm_provider + \"/\" + model\n\n if api_key and api_key.startswith(\"os.environ/\"):\n dynamic_api_key = get_secret_str(api_key)\n # check if llm provider part of model name\n\n if (\n model.split(\"/\", 1)[0] in litellm.provider_list\n and model.split(\"/\", 1)[0] not in litellm.model_list_set\n and len(model.split(\"/\"))\n > 1 # handle edge case where user passes in `litellm --model mistral` https://github.com/BerriAI/litellm/issues/1351\n ):\n return _get_openai_compatible_provider_info(\n model=model,\n api_base=api_base,\n api_key=api_key,\n dynamic_api_key=dynamic_api_key,\n )\n elif model.split(\"/\", 1)[0] in litellm.provider_list:\n custom_llm_provider = model.split(\"/\", 1)[0]\n model = model.split(\"/\", 1)[1]\n if api_base is not None and not isinstance(api_base, str):\n raise Exception(\n \"api base needs to be a string. api_base={}\".format(api_base)\n )\n if dynamic_api_key is not None and not isinstance(dynamic_api_key, str):\n raise Exception(\n \"dynamic_api_key needs to be a string. dynamic_api_key={}\".format(\n dynamic_api_key\n )\n )\n return model, custom_llm_provider, dynamic_api_key, api_base\n # check if api base is a known openai compatible endpoint\n if api_base:\n for endpoint in litellm.openai_compatible_endpoints:\n if endpoint in api_base:\n if endpoint == \"api.perplexity.ai\":\n custom_llm_provider = \"perplexity\"\n dynamic_api_key = get_secret_str(\"PERPLEXITYAI_API_KEY\")\n elif endpoint == \"api.endpoints.anyscale.com/v1\":\n custom_llm_provider = \"anyscale\"\n dynamic_api_key = get_secret_str(\"ANYSCALE_API_KEY\")\n elif endpoint == \"api.deepinfra.com/v1/openai\":\n custom_llm_provider = \"deepinfra\"\n dynamic_api_key = get_secret_str(\"DEEPINFRA_API_KEY\")\n elif endpoint == \"api.mistral.ai/v1\":\n custom_llm_provider = \"mistral\"\n dynamic_api_key = get_secret_str(\"MISTRAL_API_KEY\")\n elif endpoint == \"api.groq.com/openai/v1\":\n custom_llm_provider = \"groq\"\n dynamic_api_key = get_secret_str(\"GROQ_API_KEY\")\n elif endpoint == \"https://integrate.api.nvidia.com/v1\":\n custom_llm_provider = \"nvidia_nim\"\n dynamic_api_key = get_secret_str(\"NVIDIA_NIM_API_KEY\")\n elif endpoint == \"https://api.cerebras.ai/v1\":\n custom_llm_provider = \"cerebras\"\n dynamic_api_key = get_secret_str(\"CEREBRAS_API_KEY\")\n elif endpoint == \"https://api.sambanova.ai/v1\":\n custom_llm_provider = \"sambanova\"\n dynamic_api_key = get_secret_str(\"SAMBANOVA_API_KEY\")\n elif endpoint == \"https://api.ai21.com/studio/v1\":\n custom_llm_provider = \"ai21_chat\"\n dynamic_api_key = get_secret_str(\"AI21_API_KEY\")\n elif endpoint == \"https://codestral.mistral.ai/v1\":\n custom_llm_provider = \"codestral\"\n dynamic_api_key = get_secret_str(\"CODESTRAL_API_KEY\")\n elif endpoint == \"https://codestral.mistral.ai/v1\":\n custom_llm_provider = \"text-completion-codestral\"\n dynamic_api_key = get_secret_str(\"CODESTRAL_API_KEY\")\n elif endpoint == \"app.empower.dev/api/v1\":\n custom_llm_provider = \"empower\"\n dynamic_api_key = get_secret_str(\"EMPOWER_API_KEY\")\n elif endpoint == \"api.deepseek.com/v1\":\n custom_llm_provider = \"deepseek\"\n dynamic_api_key = get_secret_str(\"DEEPSEEK_API_KEY\")\n elif endpoint == \"https://api.friendli.ai/serverless/v1\":\n custom_llm_provider = \"friendliai\"\n dynamic_api_key = get_secret_str(\n \"FRIENDLIAI_API_KEY\"\n ) or get_secret(\"FRIENDLI_TOKEN\")\n elif endpoint == \"api.galadriel.com/v1\":\n custom_llm_provider = \"galadriel\"\n dynamic_api_key = get_secret_str(\"GALADRIEL_API_KEY\")\n elif endpoint == \"https://api.llama.com/compat/v1\":\n custom_llm_provider = \"meta_llama\"\n dynamic_api_key = api_key or get_secret_str(\"LLAMA_API_KEY\")\n elif endpoint == \"https://api.featherless.ai/v1\":\n custom_llm_provider = \"featherless_ai\"\n dynamic_api_key = get_secret_str(\"FEATHERLESS_AI_API_KEY\")\n elif endpoint == litellm.NscaleConfig.API_BASE_URL:\n custom_llm_provider = \"nscale\"\n dynamic_api_key = litellm.NscaleConfig.get_api_key()\n\n if api_base is not None and not isinstance(api_base, str):\n raise Exception(\n \"api base needs to be a string. api_base={}\".format(\n api_base\n )\n )\n if dynamic_api_key is not None and not isinstance(\n dynamic_api_key, str\n ):\n raise Exception(\n \"dynamic_api_key needs to be a string. dynamic_api_key={}\".format(\n dynamic_api_key\n )\n )\n return model, custom_llm_provider, dynamic_api_key, api_base # type: ignore\n\n # check if model in known model provider list -> for huggingface models, raise exception as they don't have a fixed provider (can be togetherai, anyscale, baseten, runpod, et.)\n ## openai - chatcompletion + text completion\n if (\n model in litellm.open_ai_chat_completion_models\n or \"ft:gpt-3.5-turbo\" in model\n or \"ft:gpt-4\" in model # catches ft:gpt-4-0613, ft:gpt-4o\n or model in litellm.openai_image_generation_models\n ):\n custom_llm_provider = \"openai\"\n elif model in litellm.open_ai_text_completion_models:\n custom_llm_provider = \"text-completion-openai\"\n ## anthropic\n elif model in litellm.anthropic_models:\n if litellm.AnthropicTextConfig._is_anthropic_text_model(model):\n custom_llm_provider = \"anthropic_text\"\n else:\n custom_llm_provider = \"anthropic\"\n ## cohere\n elif model in litellm.cohere_models or model in litellm.cohere_embedding_models:\n custom_llm_provider = \"cohere\"\n ## cohere chat models\n elif model in litellm.cohere_chat_models:\n custom_llm_provider = \"cohere_chat\"\n ## replicate\n elif model in litellm.replicate_models or (\n \":\" in model and len(model) > REPLICATE_MODEL_NAME_WITH_ID_LENGTH\n ):\n model_parts = model.split(\":\")\n if (\n len(model_parts) > 1\n and len(model_parts[1]) == REPLICATE_MODEL_NAME_WITH_ID_LENGTH\n ): ## checks if model name has a 64 digit code - e.g. \"meta/llama-2-70b-chat:02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3\"\n custom_llm_provider = \"replicate\"\n elif model in litellm.replicate_models:\n custom_llm_provider = \"replicate\"\n ## openrouter\n elif model in litellm.openrouter_models:\n custom_llm_provider = \"openrouter\"\n ## maritalk\n elif model in litellm.maritalk_models:\n custom_llm_provider = \"maritalk\"\n ## vertex - text + chat + language (gemini) models\n elif (\n model in litellm.vertex_chat_models\n or model in litellm.vertex_code_chat_models\n or model in litellm.vertex_text_models\n or model in litellm.vertex_code_text_models\n or model in litellm.vertex_language_models\n or model in litellm.vertex_embedding_models\n or model in litellm.vertex_vision_models\n or model in litellm.vertex_ai_image_models\n ):\n custom_llm_provider = \"vertex_ai\"\n ## ai21\n elif model in litellm.ai21_chat_models or model in litellm.ai21_models:\n custom_llm_provider = \"ai21_chat\"\n api_base = (\n api_base\n or get_secret(\"AI21_API_BASE\")\n or \"https://api.ai21.com/studio/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret(\"AI21_API_KEY\")\n ## aleph_alpha\n elif model in litellm.aleph_alpha_models:\n custom_llm_provider = \"aleph_alpha\"\n ## baseten\n elif model in litellm.baseten_models:\n custom_llm_provider = \"baseten\"\n ## nlp_cloud\n elif model in litellm.nlp_cloud_models:\n custom_llm_provider = \"nlp_cloud\"\n ## petals\n elif model in litellm.petals_models:\n custom_llm_provider = \"petals\"\n ## bedrock\n elif (\n model in litellm.bedrock_models\n or model in litellm.bedrock_embedding_models\n or model in litellm.bedrock_converse_models\n ):\n custom_llm_provider = \"bedrock\"\n elif model in litellm.watsonx_models:\n custom_llm_provider = \"watsonx\"\n # openai embeddings\n elif model in litellm.open_ai_embedding_models:\n custom_llm_provider = \"openai\"\n elif model in litellm.empower_models:\n custom_llm_provider = \"empower\"\n elif model == \"*\":\n custom_llm_provider = \"openai\"\n if not custom_llm_provider:\n if litellm.suppress_debug_info is False:\n print() # noqa\n print( # noqa\n \"\\033[1;31mProvider List: https://docs.litellm.ai/docs/providers\\033[0m\" # noqa\n ) # noqa\n print() # noqa\n error_str = f\"LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model={model}\\n Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers\"\n # maps to openai.NotFoundError, this is raised when openai does not recognize the llm\n raise litellm.exceptions.BadRequestError( # type: ignore\n message=error_str,\n model=model,\n response=httpx.Response(\n status_code=400,\n content=error_str,\n request=httpx.Request(method=\"completion\", url=\"https://github.com/BerriAI/litellm\"), # type: ignore\n ),\n llm_provider=\"\",\n )\n if api_base is not None and not isinstance(api_base, str):\n raise Exception(\n \"api base needs to be a string. api_base={}\".format(api_base)\n )\n if dynamic_api_key is not None and not isinstance(dynamic_api_key, str):\n raise Exception(\n \"dynamic_api_key needs to be a string. dynamic_api_key={}\".format(\n dynamic_api_key\n )\n )\n return model, custom_llm_provider, dynamic_api_key, api_base\n except Exception as e:\n if isinstance(e, litellm.exceptions.BadRequestError):\n raise e\n else:\n error_str = (\n f\"GetLLMProvider Exception - {str(e)}\\n\\noriginal model: {model}\"\n )\n raise litellm.exceptions.BadRequestError( # type: ignore\n message=f\"GetLLMProvider Exception - {str(e)}\\n\\noriginal model: {model}\",\n model=model,\n response=httpx.Response(\n status_code=400,\n content=error_str,\n request=httpx.Request(method=\"completion\", url=\"https://github.com/BerriAI/litellm\"), # type: ignore\n ),\n llm_provider=\"\",\n )\n\n\ndef _get_openai_compatible_provider_info( # noqa: PLR0915\n model: str,\n api_base: Optional[str],\n api_key: Optional[str],\n dynamic_api_key: Optional[str],\n) -> Tuple[str, str, Optional[str], Optional[str]]:\n \"\"\"\n Returns:\n Tuple[str, str, Optional[str], Optional[str]]:\n model: str\n custom_llm_provider: str\n dynamic_api_key: Optional[str]\n api_base: Optional[str]\n \"\"\"\n\n custom_llm_provider = model.split(\"/\", 1)[0]\n model = model.split(\"/\", 1)[1]\n\n if custom_llm_provider == \"perplexity\":\n # perplexity is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.perplexity.ai\n (\n api_base,\n dynamic_api_key,\n ) = litellm.PerplexityChatConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"aiohttp_openai\":\n return model, \"aiohttp_openai\", api_key, api_base\n elif custom_llm_provider == \"anyscale\":\n # anyscale is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1\n api_base = api_base or get_secret_str(\"ANYSCALE_API_BASE\") or \"https://api.endpoints.anyscale.com/v1\" # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"ANYSCALE_API_KEY\")\n elif custom_llm_provider == \"deepinfra\":\n (\n api_base,\n dynamic_api_key,\n ) = litellm.DeepInfraConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"empower\":\n api_base = (\n api_base\n or get_secret(\"EMPOWER_API_BASE\")\n or \"https://app.empower.dev/api/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"EMPOWER_API_KEY\")\n elif custom_llm_provider == \"groq\":\n (\n api_base,\n dynamic_api_key,\n ) = litellm.GroqChatConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"nvidia_nim\":\n # nvidia_nim is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1\n api_base = (\n api_base\n or get_secret(\"NVIDIA_NIM_API_BASE\")\n or \"https://integrate.api.nvidia.com/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"NVIDIA_NIM_API_KEY\")\n elif custom_llm_provider == \"cerebras\":\n api_base = (\n api_base or get_secret(\"CEREBRAS_API_BASE\") or \"https://api.cerebras.ai/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"CEREBRAS_API_KEY\")\n elif custom_llm_provider == \"sambanova\":\n api_base = (\n api_base\n or get_secret(\"SAMBANOVA_API_BASE\")\n or \"https://api.sambanova.ai/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"SAMBANOVA_API_KEY\")\n elif custom_llm_provider == \"meta_llama\":\n api_base = (\n api_base\n or get_secret(\"LLAMA_API_BASE\")\n or \"https://api.llama.com/compat/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"LLAMA_API_KEY\")\n elif custom_llm_provider == \"nebius\":\n api_base = (\n api_base\n or get_secret(\"NEBIUS_API_BASE\")\n or \"https://api.studio.nebius.ai/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"NEBIUS_API_KEY\")\n elif (custom_llm_provider == \"ai21_chat\") or (\n custom_llm_provider == \"ai21\" and model in litellm.ai21_chat_models\n ):\n api_base = (\n api_base or get_secret(\"AI21_API_BASE\") or \"https://api.ai21.com/studio/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"AI21_API_KEY\")\n custom_llm_provider = \"ai21_chat\"\n elif custom_llm_provider == \"volcengine\":\n # volcengine is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1\n api_base = (\n api_base\n or get_secret(\"VOLCENGINE_API_BASE\")\n or \"https://ark.cn-beijing.volces.com/api/v3\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"VOLCENGINE_API_KEY\")\n elif custom_llm_provider == \"codestral\":\n # codestral is openai compatible, we just need to set this to custom_openai and have the api_base be https://codestral.mistral.ai/v1\n api_base = (\n api_base\n or get_secret(\"CODESTRAL_API_BASE\")\n or \"https://codestral.mistral.ai/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"CODESTRAL_API_KEY\")\n elif custom_llm_provider == \"hosted_vllm\":\n # vllm is openai compatible, we just need to set this to custom_openai\n (\n api_base,\n dynamic_api_key,\n ) = litellm.HostedVLLMChatConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"llamafile\":\n # llamafile is OpenAI compatible.\n (\n api_base,\n dynamic_api_key,\n ) = litellm.LlamafileChatConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"datarobot\":\n # DataRobot is OpenAI compatible.\n (\n api_base,\n dynamic_api_key\n ) = litellm.DataRobotConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"lm_studio\":\n # lm_studio is openai compatible, we just need to set this to custom_openai\n (\n api_base,\n dynamic_api_key,\n ) = litellm.LMStudioChatConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"deepseek\":\n # deepseek is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.deepseek.com/v1\n api_base = (\n api_base\n or get_secret(\"DEEPSEEK_API_BASE\")\n or \"https://api.deepseek.com/beta\"\n ) # type: ignore\n\n dynamic_api_key = api_key or get_secret_str(\"DEEPSEEK_API_KEY\")\n elif custom_llm_provider == \"fireworks_ai\":\n # fireworks is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.fireworks.ai/inference/v1\n (\n api_base,\n dynamic_api_key,\n ) = litellm.FireworksAIConfig()._get_openai_compatible_provider_info(\n api_base=api_base, api_key=api_key\n )\n elif custom_llm_provider == \"azure_ai\":\n (\n api_base,\n dynamic_api_key,\n custom_llm_provider,\n ) = litellm.AzureAIStudioConfig()._get_openai_compatible_provider_info(\n model, api_base, api_key, custom_llm_provider\n )\n elif custom_llm_provider == \"github\":\n api_base = (\n api_base\n or get_secret_str(\"GITHUB_API_BASE\")\n or \"https://models.inference.ai.azure.com\" # This is github's default base url\n )\n dynamic_api_key = api_key or get_secret_str(\"GITHUB_API_KEY\")\n elif custom_llm_provider == \"litellm_proxy\":\n (\n api_base,\n dynamic_api_key,\n ) = litellm.LiteLLMProxyChatConfig()._get_openai_compatible_provider_info(\n api_base=api_base, api_key=api_key\n )\n\n elif custom_llm_provider == \"mistral\":\n (\n api_base,\n dynamic_api_key,\n ) = litellm.MistralConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"jina_ai\":\n (\n custom_llm_provider,\n api_base,\n dynamic_api_key,\n ) = litellm.JinaAIEmbeddingConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"xai\":\n (\n api_base,\n dynamic_api_key,\n ) = litellm.XAIChatConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"together_ai\":\n api_base = (\n api_base\n or get_secret_str(\"TOGETHER_AI_API_BASE\")\n or \"https://api.together.xyz/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or (\n get_secret_str(\"TOGETHER_API_KEY\")\n or get_secret_str(\"TOGETHER_AI_API_KEY\")\n or get_secret_str(\"TOGETHERAI_API_KEY\")\n or get_secret_str(\"TOGETHER_AI_TOKEN\")\n )\n elif custom_llm_provider == \"friendliai\":\n api_base = (\n api_base\n or get_secret(\"FRIENDLI_API_BASE\")\n or \"https://api.friendli.ai/serverless/v1\"\n ) # type: ignore\n dynamic_api_key = (\n api_key\n or get_secret_str(\"FRIENDLIAI_API_KEY\")\n or get_secret_str(\"FRIENDLI_TOKEN\")\n )\n elif custom_llm_provider == \"galadriel\":\n api_base = (\n api_base\n or get_secret(\"GALADRIEL_API_BASE\")\n or \"https://api.galadriel.com/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"GALADRIEL_API_KEY\")\n elif custom_llm_provider == \"github_copilot\":\n (\n api_base,\n dynamic_api_key,\n custom_llm_provider,\n ) = litellm.GithubCopilotConfig()._get_openai_compatible_provider_info(\n model, api_base, api_key, custom_llm_provider\n )\n elif custom_llm_provider == \"novita\":\n api_base = (\n api_base\n or get_secret(\"NOVITA_API_BASE\")\n or \"https://api.novita.ai/v3/openai\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"NOVITA_API_KEY\")\n elif custom_llm_provider == \"snowflake\":\n api_base = (\n api_base\n or get_secret_str(\"SNOWFLAKE_API_BASE\")\n or f\"https://{get_secret('SNOWFLAKE_ACCOUNT_ID')}.snowflakecomputing.com/api/v2/cortex/inference:complete\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"SNOWFLAKE_JWT\")\n elif custom_llm_provider == \"featherless_ai\":\n (\n api_base,\n dynamic_api_key,\n ) = litellm.FeatherlessAIConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"nscale\":\n (\n api_base,\n dynamic_api_key,\n ) = litellm.NscaleConfig()._get_openai_compatible_provider_info(\n api_base=api_base, api_key=api_key\n )\n\n if api_base is not None and not isinstance(api_base, str):\n raise Exception(\"api base needs to be a string. api_base={}\".format(api_base))\n if dynamic_api_key is not None and not isinstance(dynamic_api_key, str):\n raise Exception(\n \"dynamic_api_key needs to be a string. dynamic_api_key={}\".format(\n dynamic_api_key\n )\n )\n if dynamic_api_key is None and api_key is not None:\n dynamic_api_key = api_key\n return model, custom_llm_provider, dynamic_api_key, api_base\n"}, "0c6948afd902379f11a9cb8cd61ecaf716d35730": {"t": 1785766561.1888692, "n": 28873, "h": "afdf9986553a31d3", "c": "from typing import Optional, Tuple\n\nimport httpx\n\nimport litellm\nfrom litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH\nfrom litellm.secret_managers.main import get_secret, get_secret_str\n\nfrom ..types.router import LiteLLM_Params\n\n\ndef _is_non_openai_azure_model(model: str) -> bool:\n try:\n model_name = model.split(\"/\", 1)[1]\n if (\n model_name in litellm.cohere_chat_models\n or f\"mistral/{model_name}\" in litellm.mistral_chat_models\n ):\n return True\n except Exception:\n return False\n return False\n\n\ndef handle_cohere_chat_model_custom_llm_provider(\n model: str, custom_llm_provider: Optional[str] = None\n) -> Tuple[str, Optional[str]]:\n \"\"\"\n if user sets model = \"cohere/command-r\" -> use custom_llm_provider = \"cohere_chat\"\n\n Args:\n model:\n custom_llm_provider:\n\n Returns:\n model, custom_llm_provider\n \"\"\"\n\n if custom_llm_provider:\n if custom_llm_provider == \"cohere\" and model in litellm.cohere_chat_models:\n return model, \"cohere_chat\"\n\n if \"/\" in model:\n _custom_llm_provider, _model = model.split(\"/\", 1)\n if (\n _custom_llm_provider\n and _custom_llm_provider == \"cohere\"\n and _model in litellm.cohere_chat_models\n ):\n return _model, \"cohere_chat\"\n\n return model, custom_llm_provider\n\n\ndef handle_anthropic_text_model_custom_llm_provider(\n model: str, custom_llm_provider: Optional[str] = None\n) -> Tuple[str, Optional[str]]:\n \"\"\"\n if user sets model = \"anthropic/claude-2\" -> use custom_llm_provider = \"anthropic_text\"\n\n Args:\n model:\n custom_llm_provider:\n\n Returns:\n model, custom_llm_provider\n \"\"\"\n\n if custom_llm_provider:\n if (\n custom_llm_provider == \"anthropic\"\n and litellm.AnthropicTextConfig._is_anthropic_text_model(model)\n ):\n return model, \"anthropic_text\"\n\n if \"/\" in model:\n _custom_llm_provider, _model = model.split(\"/\", 1)\n if (\n _custom_llm_provider\n and _custom_llm_provider == \"anthropic\"\n and litellm.AnthropicTextConfig._is_anthropic_text_model(_model)\n ):\n return _model, \"anthropic_text\"\n\n return model, custom_llm_provider\n\n\ndef get_llm_provider( # noqa: PLR0915\n model: str,\n custom_llm_provider: Optional[str] = None,\n api_base: Optional[str] = None,\n api_key: Optional[str] = None,\n litellm_params: Optional[LiteLLM_Params] = None,\n) -> Tuple[str, str, Optional[str], Optional[str]]:\n \"\"\"\n Returns the provider for a given model name - e.g. 'azure/chatgpt-v-2' -> 'azure'\n\n For router -> Can also give the whole litellm param dict -> this function will extract the relevant details\n\n Raises Error - if unable to map model to a provider\n\n Return model, custom_llm_provider, dynamic_api_key, api_base\n \"\"\"\n try:\n if litellm.LiteLLMProxyChatConfig._should_use_litellm_proxy_by_default(\n litellm_params=litellm_params\n ):\n return litellm.LiteLLMProxyChatConfig.litellm_proxy_get_custom_llm_provider_info(\n model=model, api_base=api_base, api_key=api_key\n )\n\n ## IF LITELLM PARAMS GIVEN ##\n if litellm_params:\n assert (\n custom_llm_provider is None and api_base is None and api_key is None\n ), \"Either pass in litellm_params or the custom_llm_provider/api_base/api_key. Otherwise, these values will be overriden.\"\n custom_llm_provider = litellm_params.custom_llm_provider\n api_base = litellm_params.api_base\n api_key = litellm_params.api_key\n\n dynamic_api_key = None\n # check if llm provider provided\n # AZURE AI-Studio Logic - Azure AI Studio supports AZURE/Cohere\n # If User passes azure/command-r-plus -> we should send it to cohere_chat/command-r-plus\n if model.split(\"/\", 1)[0] == \"azure\":\n if _is_non_openai_azure_model(model):\n custom_llm_provider = \"openai\"\n return model, custom_llm_provider, dynamic_api_key, api_base\n\n ### Handle cases when custom_llm_provider is set to cohere/command-r-plus but it should use cohere_chat route\n model, custom_llm_provider = handle_cohere_chat_model_custom_llm_provider(\n model, custom_llm_provider\n )\n\n model, custom_llm_provider = handle_anthropic_text_model_custom_llm_provider(\n model, custom_llm_provider\n )\n\n if custom_llm_provider and (\n model.split(\"/\")[0] != custom_llm_provider\n ): # handle scenario where model=\"azure/*\" and custom_llm_provider=\"azure\"\n model = custom_llm_provider + \"/\" + model\n\n if api_key and api_key.startswith(\"os.environ/\"):\n dynamic_api_key = get_secret_str(api_key)\n # check if llm provider part of model name\n\n if (\n model.split(\"/\", 1)[0] in litellm.provider_list\n and model.split(\"/\", 1)[0] not in litellm.model_list_set\n and len(model.split(\"/\"))\n > 1 # handle edge case where user passes in `litellm --model mistral` https://github.com/BerriAI/litellm/issues/1351\n ):\n return _get_openai_compatible_provider_info(\n model=model,\n api_base=api_base,\n api_key=api_key,\n dynamic_api_key=dynamic_api_key,\n )\n elif model.split(\"/\", 1)[0] in litellm.provider_list:\n custom_llm_provider = model.split(\"/\", 1)[0]\n model = model.split(\"/\", 1)[1]\n if api_base is not None and not isinstance(api_base, str):\n raise Exception(\n \"api base needs to be a string. api_base={}\".format(api_base)\n )\n if dynamic_api_key is not None and not isinstance(dynamic_api_key, str):\n raise Exception(\n \"dynamic_api_key needs to be a string. dynamic_api_key={}\".format(\n dynamic_api_key\n )\n )\n return model, custom_llm_provider, dynamic_api_key, api_base\n # check if api base is a known openai compatible endpoint\n if api_base:\n for endpoint in litellm.openai_compatible_endpoints:\n if endpoint in api_base:\n if endpoint == \"api.perplexity.ai\":\n custom_llm_provider = \"perplexity\"\n dynamic_api_key = get_secret_str(\"PERPLEXITYAI_API_KEY\")\n elif endpoint == \"api.endpoints.anyscale.com/v1\":\n custom_llm_provider = \"anyscale\"\n dynamic_api_key = get_secret_str(\"ANYSCALE_API_KEY\")\n elif endpoint == \"api.deepinfra.com/v1/openai\":\n custom_llm_provider = \"deepinfra\"\n dynamic_api_key = get_secret_str(\"DEEPINFRA_API_KEY\")\n elif endpoint == \"api.mistral.ai/v1\":\n custom_llm_provider = \"mistral\"\n dynamic_api_key = get_secret_str(\"MISTRAL_API_KEY\")\n elif endpoint == \"api.groq.com/openai/v1\":\n custom_llm_provider = \"groq\"\n dynamic_api_key = get_secret_str(\"GROQ_API_KEY\")\n elif endpoint == \"https://integrate.api.nvidia.com/v1\":\n custom_llm_provider = \"nvidia_nim\"\n dynamic_api_key = get_secret_str(\"NVIDIA_NIM_API_KEY\")\n elif endpoint == \"https://api.cerebras.ai/v1\":\n custom_llm_provider = \"cerebras\"\n dynamic_api_key = get_secret_str(\"CEREBRAS_API_KEY\")\n elif endpoint == \"https://api.sambanova.ai/v1\":\n custom_llm_provider = \"sambanova\"\n dynamic_api_key = get_secret_str(\"SAMBANOVA_API_KEY\")\n elif endpoint == \"https://api.ai21.com/studio/v1\":\n custom_llm_provider = \"ai21_chat\"\n dynamic_api_key = get_secret_str(\"AI21_API_KEY\")\n elif endpoint == \"https://codestral.mistral.ai/v1\":\n custom_llm_provider = \"codestral\"\n dynamic_api_key = get_secret_str(\"CODESTRAL_API_KEY\")\n elif endpoint == \"https://codestral.mistral.ai/v1\":\n custom_llm_provider = \"text-completion-codestral\"\n dynamic_api_key = get_secret_str(\"CODESTRAL_API_KEY\")\n elif endpoint == \"app.empower.dev/api/v1\":\n custom_llm_provider = \"empower\"\n dynamic_api_key = get_secret_str(\"EMPOWER_API_KEY\")\n elif endpoint == \"api.deepseek.com/v1\":\n custom_llm_provider = \"deepseek\"\n dynamic_api_key = get_secret_str(\"DEEPSEEK_API_KEY\")\n elif endpoint == \"https://api.friendli.ai/serverless/v1\":\n custom_llm_provider = \"friendliai\"\n dynamic_api_key = get_secret_str(\n \"FRIENDLIAI_API_KEY\"\n ) or get_secret(\"FRIENDLI_TOKEN\")\n elif endpoint == \"api.galadriel.com/v1\":\n custom_llm_provider = \"galadriel\"\n dynamic_api_key = get_secret_str(\"GALADRIEL_API_KEY\")\n elif endpoint == \"https://api.llama.com/compat/v1\":\n custom_llm_provider = \"meta_llama\"\n dynamic_api_key = api_key or get_secret_str(\"LLAMA_API_KEY\")\n elif endpoint == \"https://api.featherless.ai/v1\":\n custom_llm_provider = \"featherless_ai\"\n dynamic_api_key = get_secret_str(\"FEATHERLESS_AI_API_KEY\")\n elif endpoint == litellm.NscaleConfig.API_BASE_URL:\n custom_llm_provider = \"nscale\"\n dynamic_api_key = litellm.NscaleConfig.get_api_key()\n\n if api_base is not None and not isinstance(api_base, str):\n raise Exception(\n \"api base needs to be a string. api_base={}\".format(\n api_base\n )\n )\n if dynamic_api_key is not None and not isinstance(\n dynamic_api_key, str\n ):\n raise Exception(\n \"dynamic_api_key needs to be a string. dynamic_api_key={}\".format(\n dynamic_api_key\n )\n )\n return model, custom_llm_provider, dynamic_api_key, api_base # type: ignore\n\n # check if model in known model provider list -> for huggingface models, raise exception as they don't have a fixed provider (can be togetherai, anyscale, baseten, runpod, et.)\n ## openai - chatcompletion + text completion\n if (\n model in litellm.open_ai_chat_completion_models\n or \"ft:gpt-3.5-turbo\" in model\n or \"ft:gpt-4\" in model # catches ft:gpt-4-0613, ft:gpt-4o\n or model in litellm.openai_image_generation_models\n ):\n custom_llm_provider = \"openai\"\n elif model in litellm.open_ai_text_completion_models:\n custom_llm_provider = \"text-completion-openai\"\n ## anthropic\n elif model in litellm.anthropic_models:\n if litellm.AnthropicTextConfig._is_anthropic_text_model(model):\n custom_llm_provider = \"anthropic_text\"\n else:\n custom_llm_provider = \"anthropic\"\n ## cohere\n elif model in litellm.cohere_models or model in litellm.cohere_embedding_models:\n custom_llm_provider = \"cohere\"\n ## cohere chat models\n elif model in litellm.cohere_chat_models:\n custom_llm_provider = \"cohere_chat\"\n ## replicate\n elif model in litellm.replicate_models or (\n \":\" in model and len(model) > REPLICATE_MODEL_NAME_WITH_ID_LENGTH\n ):\n model_parts = model.split(\":\")\n if (\n len(model_parts) > 1\n and len(model_parts[1]) == REPLICATE_MODEL_NAME_WITH_ID_LENGTH\n ): ## checks if model name has a 64 digit code - e.g. \"meta/llama-2-70b-chat:02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3\"\n custom_llm_provider = \"replicate\"\n elif model in litellm.replicate_models:\n custom_llm_provider = \"replicate\"\n ## openrouter\n elif model in litellm.openrouter_models:\n custom_llm_provider = \"openrouter\"\n ## maritalk\n elif model in litellm.maritalk_models:\n custom_llm_provider = \"maritalk\"\n ## vertex - text + chat + language (gemini) models\n elif (\n model in litellm.vertex_chat_models\n or model in litellm.vertex_code_chat_models\n or model in litellm.vertex_text_models\n or model in litellm.vertex_code_text_models\n or model in litellm.vertex_language_models\n or model in litellm.vertex_embedding_models\n or model in litellm.vertex_vision_models\n or model in litellm.vertex_ai_image_models\n ):\n custom_llm_provider = \"vertex_ai\"\n ## ai21\n elif model in litellm.ai21_chat_models or model in litellm.ai21_models:\n custom_llm_provider = \"ai21_chat\"\n api_base = (\n api_base\n or get_secret(\"AI21_API_BASE\")\n or \"https://api.ai21.com/studio/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret(\"AI21_API_KEY\")\n ## aleph_alpha\n elif model in litellm.aleph_alpha_models:\n custom_llm_provider = \"aleph_alpha\"\n ## baseten\n elif model in litellm.baseten_models:\n custom_llm_provider = \"baseten\"\n ## nlp_cloud\n elif model in litellm.nlp_cloud_models:\n custom_llm_provider = \"nlp_cloud\"\n ## petals\n elif model in litellm.petals_models:\n custom_llm_provider = \"petals\"\n ## bedrock\n elif (\n model in litellm.bedrock_models\n or model in litellm.bedrock_embedding_models\n or model in litellm.bedrock_converse_models\n ):\n custom_llm_provider = \"bedrock\"\n elif model in litellm.watsonx_models:\n custom_llm_provider = \"watsonx\"\n # openai embeddings\n elif model in litellm.open_ai_embedding_models:\n custom_llm_provider = \"openai\"\n elif model in litellm.empower_models:\n custom_llm_provider = \"empower\"\n elif model == \"*\":\n custom_llm_provider = \"openai\"\n if not custom_llm_provider:\n if litellm.suppress_debug_info is False:\n print() # noqa\n print( # noqa\n \"\\033[1;31mProvider List: https://docs.litellm.ai/docs/providers\\033[0m\" # noqa\n ) # noqa\n print() # noqa\n error_str = f\"LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model={model}\\n Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers\"\n # maps to openai.NotFoundError, this is raised when openai does not recognize the llm\n raise litellm.exceptions.BadRequestError( # type: ignore\n message=error_str,\n model=model,\n response=httpx.Response(\n status_code=400,\n content=error_str,\n request=httpx.Request(method=\"completion\", url=\"https://github.com/BerriAI/litellm\"), # type: ignore\n ),\n llm_provider=\"\",\n )\n if api_base is not None and not isinstance(api_base, str):\n raise Exception(\n \"api base needs to be a string. api_base={}\".format(api_base)\n )\n if dynamic_api_key is not None and not isinstance(dynamic_api_key, str):\n raise Exception(\n \"dynamic_api_key needs to be a string. dynamic_api_key={}\".format(\n dynamic_api_key\n )\n )\n return model, custom_llm_provider, dynamic_api_key, api_base\n except Exception as e:\n if isinstance(e, litellm.exceptions.BadRequestError):\n raise e\n else:\n error_str = (\n f\"GetLLMProvider Exception - {str(e)}\\n\\noriginal model: {model}\"\n )\n raise litellm.exceptions.BadRequestError( # type: ignore\n message=f\"GetLLMProvider Exception - {str(e)}\\n\\noriginal model: {model}\",\n model=model,\n response=httpx.Response(\n status_code=400,\n content=error_str,\n request=httpx.Request(method=\"completion\", url=\"https://github.com/BerriAI/litellm\"), # type: ignore\n ),\n llm_provider=\"\",\n )\n\n\ndef _get_openai_compatible_provider_info( # noqa: PLR0915\n model: str,\n api_base: Optional[str],\n api_key: Optional[str],\n dynamic_api_key: Optional[str],\n) -> Tuple[str, str, Optional[str], Optional[str]]:\n \"\"\"\n Returns:\n Tuple[str, str, Optional[str], Optional[str]]:\n model: str\n custom_llm_provider: str\n dynamic_api_key: Optional[str]\n api_base: Optional[str]\n \"\"\"\n\n custom_llm_provider = model.split(\"/\", 1)[0]\n model = model.split(\"/\", 1)[1]\n\n if custom_llm_provider == \"perplexity\":\n # perplexity is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.perplexity.ai\n (\n api_base,\n dynamic_api_key,\n ) = litellm.PerplexityChatConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"aiohttp_openai\":\n return model, \"aiohttp_openai\", api_key, api_base\n elif custom_llm_provider == \"anyscale\":\n # anyscale is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1\n api_base = api_base or get_secret_str(\"ANYSCALE_API_BASE\") or \"https://api.endpoints.anyscale.com/v1\" # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"ANYSCALE_API_KEY\")\n elif custom_llm_provider == \"deepinfra\":\n (\n api_base,\n dynamic_api_key,\n ) = litellm.DeepInfraConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"empower\":\n api_base = (\n api_base\n or get_secret(\"EMPOWER_API_BASE\")\n or \"https://app.empower.dev/api/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"EMPOWER_API_KEY\")\n elif custom_llm_provider == \"groq\":\n (\n api_base,\n dynamic_api_key,\n ) = litellm.GroqChatConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"nvidia_nim\":\n # nvidia_nim is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1\n api_base = (\n api_base\n or get_secret(\"NVIDIA_NIM_API_BASE\")\n or \"https://integrate.api.nvidia.com/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"NVIDIA_NIM_API_KEY\")\n elif custom_llm_provider == \"cerebras\":\n api_base = (\n api_base or get_secret(\"CEREBRAS_API_BASE\") or \"https://api.cerebras.ai/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"CEREBRAS_API_KEY\")\n elif custom_llm_provider == \"sambanova\":\n api_base = (\n api_base\n or get_secret(\"SAMBANOVA_API_BASE\")\n or \"https://api.sambanova.ai/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"SAMBANOVA_API_KEY\")\n elif custom_llm_provider == \"meta_llama\":\n api_base = (\n api_base\n or get_secret(\"LLAMA_API_BASE\")\n or \"https://api.llama.com/compat/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"LLAMA_API_KEY\")\n elif custom_llm_provider == \"nebius\":\n api_base = (\n api_base\n or get_secret(\"NEBIUS_API_BASE\")\n or \"https://api.studio.nebius.ai/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"NEBIUS_API_KEY\")\n elif (custom_llm_provider == \"ai21_chat\") or (\n custom_llm_provider == \"ai21\" and model in litellm.ai21_chat_models\n ):\n api_base = (\n api_base or get_secret(\"AI21_API_BASE\") or \"https://api.ai21.com/studio/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"AI21_API_KEY\")\n custom_llm_provider = \"ai21_chat\"\n elif custom_llm_provider == \"volcengine\":\n # volcengine is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1\n api_base = (\n api_base\n or get_secret(\"VOLCENGINE_API_BASE\")\n or \"https://ark.cn-beijing.volces.com/api/v3\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"VOLCENGINE_API_KEY\")\n elif custom_llm_provider == \"codestral\":\n # codestral is openai compatible, we just need to set this to custom_openai and have the api_base be https://codestral.mistral.ai/v1\n api_base = (\n api_base\n or get_secret(\"CODESTRAL_API_BASE\")\n or \"https://codestral.mistral.ai/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"CODESTRAL_API_KEY\")\n elif custom_llm_provider == \"hosted_vllm\":\n # vllm is openai compatible, we just need to set this to custom_openai\n (\n api_base,\n dynamic_api_key,\n ) = litellm.HostedVLLMChatConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"llamafile\":\n # llamafile is OpenAI compatible.\n (\n api_base,\n dynamic_api_key,\n ) = litellm.LlamafileChatConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"datarobot\":\n # DataRobot is OpenAI compatible.\n (\n api_base,\n dynamic_api_key\n ) = litellm.DataRobotConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"lm_studio\":\n # lm_studio is openai compatible, we just need to set this to custom_openai\n (\n api_base,\n dynamic_api_key,\n ) = litellm.LMStudioChatConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"deepseek\":\n # deepseek is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.deepseek.com/v1\n api_base = (\n api_base\n or get_secret(\"DEEPSEEK_API_BASE\")\n or \"https://api.deepseek.com/beta\"\n ) # type: ignore\n\n dynamic_api_key = api_key or get_secret_str(\"DEEPSEEK_API_KEY\")\n elif custom_llm_provider == \"fireworks_ai\":\n # fireworks is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.fireworks.ai/inference/v1\n (\n api_base,\n dynamic_api_key,\n ) = litellm.FireworksAIConfig()._get_openai_compatible_provider_info(\n api_base=api_base, api_key=api_key\n )\n elif custom_llm_provider == \"azure_ai\":\n (\n api_base,\n dynamic_api_key,\n custom_llm_provider,\n ) = litellm.AzureAIStudioConfig()._get_openai_compatible_provider_info(\n model, api_base, api_key, custom_llm_provider\n )\n elif custom_llm_provider == \"github\":\n api_base = (\n api_base\n or get_secret_str(\"GITHUB_API_BASE\")\n or \"https://models.inference.ai.azure.com\" # This is github's default base url\n )\n dynamic_api_key = api_key or get_secret_str(\"GITHUB_API_KEY\")\n elif custom_llm_provider == \"litellm_proxy\":\n (\n api_base,\n dynamic_api_key,\n ) = litellm.LiteLLMProxyChatConfig()._get_openai_compatible_provider_info(\n api_base=api_base, api_key=api_key\n )\n\n elif custom_llm_provider == \"mistral\":\n (\n api_base,\n dynamic_api_key,\n ) = litellm.MistralConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"jina_ai\":\n (\n custom_llm_provider,\n api_base,\n dynamic_api_key,\n ) = litellm.JinaAIEmbeddingConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"xai\":\n (\n api_base,\n dynamic_api_key,\n ) = litellm.XAIChatConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"together_ai\":\n api_base = (\n api_base\n or get_secret_str(\"TOGETHER_AI_API_BASE\")\n or \"https://api.together.xyz/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or (\n get_secret_str(\"TOGETHER_API_KEY\")\n or get_secret_str(\"TOGETHER_AI_API_KEY\")\n or get_secret_str(\"TOGETHERAI_API_KEY\")\n or get_secret_str(\"TOGETHER_AI_TOKEN\")\n )\n elif custom_llm_provider == \"friendliai\":\n api_base = (\n api_base\n or get_secret(\"FRIENDLI_API_BASE\")\n or \"https://api.friendli.ai/serverless/v1\"\n ) # type: ignore\n dynamic_api_key = (\n api_key\n or get_secret_str(\"FRIENDLIAI_API_KEY\")\n or get_secret_str(\"FRIENDLI_TOKEN\")\n )\n elif custom_llm_provider == \"galadriel\":\n api_base = (\n api_base\n or get_secret(\"GALADRIEL_API_BASE\")\n or \"https://api.galadriel.com/v1\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"GALADRIEL_API_KEY\")\n elif custom_llm_provider == \"github_copilot\":\n (\n api_base,\n dynamic_api_key,\n custom_llm_provider,\n ) = litellm.GithubCopilotConfig()._get_openai_compatible_provider_info(\n model, api_base, api_key, custom_llm_provider\n )\n elif custom_llm_provider == \"novita\":\n api_base = (\n api_base\n or get_secret(\"NOVITA_API_BASE\")\n or \"https://api.novita.ai/v3/openai\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"NOVITA_API_KEY\")\n elif custom_llm_provider == \"snowflake\":\n api_base = (\n api_base\n or get_secret_str(\"SNOWFLAKE_API_BASE\")\n or f\"https://{get_secret('SNOWFLAKE_ACCOUNT_ID')}.snowflakecomputing.com/api/v2/cortex/inference:complete\"\n ) # type: ignore\n dynamic_api_key = api_key or get_secret_str(\"SNOWFLAKE_JWT\")\n elif custom_llm_provider == \"featherless_ai\":\n (\n api_base,\n dynamic_api_key,\n ) = litellm.FeatherlessAIConfig()._get_openai_compatible_provider_info(\n api_base, api_key\n )\n elif custom_llm_provider == \"nscale\":\n (\n api_base,\n dynamic_api_key,\n ) = litellm.NscaleConfig()._get_openai_compatible_provider_info(\n api_base=api_base, api_key=api_key\n )\n\n if api_base is not None and not isinstance(api_base, str):\n raise Exception(\"api base needs to be a string. api_base={}\".format(api_base))\n if dynamic_api_key is not None and not isinstance(dynamic_api_key, str):\n raise Exception(\n \"dynamic_api_key needs to be a string. dynamic_api_key={}\".format(\n dynamic_api_key\n )\n )\n if dynamic_api_key is None and api_key is not None:\n dynamic_api_key = api_key\n return model, custom_llm_provider, dynamic_api_key, api_base\n"}}}} |