1 line
105 KiB
JSON
1 line
105 KiB
JSON
{"repo": "btcfoxman/litellm", "files": {"tests/local_testing/test_exceptions.py": {"caf0ebe3554c939fb0d628e22898fc40f2e08b0f": {"t": 1785674502.6944985, "n": 50646, "h": "eb7c09ab57d9c80c", "c": "import asyncio\nimport os\nimport subprocess\nimport sys\nimport traceback\nfrom typing import Any\n\nfrom openai import AuthenticationError, BadRequestError, OpenAIError, RateLimitError\n\nfrom litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler\n\nsys.path.insert(\n 0, os.path.abspath(\"../..\")\n) # Adds the parent directory to the system path\nfrom concurrent.futures import ThreadPoolExecutor\nfrom unittest.mock import MagicMock, patch\n\nimport pytest\n\nimport litellm\nfrom litellm import ( # AuthenticationError,; RateLimitError,; ServiceUnavailableError,; OpenAIError,\n ContextWindowExceededError,\n completion,\n embedding,\n)\n\nlitellm.vertex_project = \"litellm-ci-cd\"\nlitellm.vertex_location = \"us-central1\"\nlitellm.num_retries = 0\n\n# litellm.failure_callback = [\"sentry\"]\n#### What this tests ####\n# This tests exception mapping -> trigger an exception from an llm provider -> assert if output is of the expected type\n\n\n# 5 providers -> OpenAI, Azure, Anthropic, Cohere, Replicate\n\n# 3 main types of exceptions -> - Rate Limit Errors, Context Window Errors, Auth errors (incorrect/rotated key, etc.)\n\n# Approach: Run each model through the test -> assert if the correct error (always the same one) is triggered\n\nexception_models = [\n \"sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4\",\n \"bedrock/anthropic.claude-instant-v1\",\n]\n\n\n@pytest.mark.asyncio\nasync def test_content_policy_exception_azure():\n try:\n # this is ony a test - we needed some way to invoke the exception :(\n litellm.set_verbose = True\n response = await litellm.acompletion(\n model=\"azure/gpt-4.1-mini\",\n messages=[{\"role\": \"user\", \"content\": \"where do I buy lethal drugs from\"}],\n mock_response=\"Exception: content_filter_policy\",\n )\n except litellm.ContentPolicyViolationError as e:\n print(\"caught a content policy violation error! Passed\")\n print(\"exception\", e)\n assert e.response is not None\n assert e.litellm_debug_info is not None\n assert isinstance(e.litellm_debug_info, str)\n assert len(e.litellm_debug_info) > 0\n pass\n except Exception as e:\n print()\n pytest.fail(f\"An exception occurred - {str(e)}\")\n\n\n@pytest.mark.asyncio\nasync def test_content_policy_exception_openai():\n try:\n # this is ony a test - we needed some way to invoke the exception :(\n litellm.set_verbose = True\n response = await litellm.acompletion(\n model=\"gpt-3.5-turbo\",\n stream=True,\n messages=[\n {\"role\": \"user\", \"content\": \"Gimme the lyrics to Don't Stop Me Now\"}\n ],\n )\n async for chunk in response:\n print(chunk)\n except litellm.ContentPolicyViolationError as e:\n print(\"caught a content policy violation error! Passed\")\n print(\"exception\", e)\n assert e.llm_provider == \"openai\"\n pass\n except Exception as e:\n print()\n pytest.fail(f\"An exception occurred - {str(e)}\")\n\n\n# Test 1: Context Window Errors\n@pytest.mark.skip(reason=\"AWS Suspended Account\")\n@pytest.mark.parametrize(\"model\", exception_models)\ndef test_context_window(model):\n print(\"Testing context window error\")\n sample_text = \"Say error 50 times\" * 1000000\n messages = [{\"content\": sample_text, \"role\": \"user\"}]\n try:\n litellm.set_verbose = False\n print(\"Testing model=\", model)\n response = completion(model=model, messages=messages)\n print(f\"response: {response}\")\n print(\"FAILED!\")\n pytest.fail(f\"An exception occurred\")\n except ContextWindowExceededError as e:\n print(f\"Worked!\")\n except RateLimitError:\n print(\"RateLimited!\")\n except Exception as e:\n print(f\"{e}\")\n pytest.fail(f\"An error occcurred - {e}\")\n\n\nmodels = [\"command-nightly\"]\n\n\n@pytest.mark.skip(reason=\"duplicate test.\")\n@pytest.mark.parametrize(\"model\", models)\ndef test_context_window_with_fallbacks(model):\n ctx_window_fallback_dict = {\n \"command-nightly\": \"claude-2.1\",\n \"gpt-3.5-turbo-instruct\": \"gpt-3.5-turbo-16k\",\n \"azure/gpt-4.1-mini\": \"gpt-3.5-turbo-16k\",\n }\n sample_text = \"how does a court case get to the Supreme Court?\" * 1000\n messages = [{\"content\": sample_text, \"role\": \"user\"}]\n\n try:\n completion(\n model=model,\n messages=messages,\n context_window_fallback_dict=ctx_window_fallback_dict,\n )\n except litellm.ServiceUnavailableError as e:\n pass\n except litellm.APIConnectionError as e:\n pass\n\n\n# for model in litellm.models_by_provider[\"bedrock\"]:\n# test_context_window(model=model)\n# test_context_window(model=\"chat-bison\")\n# test_context_window_with_fallbacks(model=\"command-nightly\")\n# Test 2: InvalidAuth Errors\n@pytest.mark.parametrize(\"model\", models)\ndef invalid_auth(model): # set the model key to an invalid key, depending on the model\n messages = [{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n temporary_key = None\n try:\n if model == \"gpt-3.5-turbo\" or model == \"gpt-3.5-turbo-instruct\":\n temporary_key = os.environ[\"OPENAI_API_KEY\"]\n os.environ[\"OPENAI_API_KEY\"] = \"bad-key\"\n elif \"bedrock\" in model:\n temporary_aws_access_key = os.environ[\"AWS_ACCESS_KEY_ID\"]\n os.environ[\"AWS_ACCESS_KEY_ID\"] = \"bad-key\"\n temporary_aws_region_name = os.environ[\"AWS_REGION_NAME\"]\n os.environ[\"AWS_REGION_NAME\"] = \"bad-key\"\n temporary_secret_key = os.environ[\"AWS_SECRET_ACCESS_KEY\"]\n os.environ[\"AWS_SECRET_ACCESS_KEY\"] = \"bad-key\"\n elif model == \"azure/gpt-4.1-mini\":\n temporary_key = os.environ[\"AZURE_AI_API_KEY\"]\n os.environ[\"AZURE_AI_API_KEY\"] = \"bad-key\"\n elif model == \"claude-3-5-haiku-20241022\":\n temporary_key = os.environ[\"ANTHROPIC_API_KEY\"]\n os.environ[\"ANTHROPIC_API_KEY\"] = \"bad-key\"\n elif model == \"command-nightly\":\n temporary_key = os.environ[\"COHERE_API_KEY\"]\n os.environ[\"COHERE_API_KEY\"] = \"bad-key\"\n elif \"j2\" in model:\n temporary_key = os.environ[\"AI21_API_KEY\"]\n os.environ[\"AI21_API_KEY\"] = \"bad-key\"\n elif \"togethercomputer\" in model:\n temporary_key = os.environ[\"TOGETHERAI_API_KEY\"]\n os.environ[\"TOGETHERAI_API_KEY\"] = \"sk-test-togetherai-key-808\"\n elif model in litellm.openrouter_models:\n temporary_key = os.environ[\"OPENROUTER_API_KEY\"]\n os.environ[\"OPENROUTER_API_KEY\"] = \"bad-key\"\n elif model in litellm.aleph_alpha_models:\n temporary_key = os.environ[\"ALEPH_ALPHA_API_KEY\"]\n os.environ[\"ALEPH_ALPHA_API_KEY\"] = \"bad-key\"\n elif model in litellm.nlp_cloud_models:\n os.environ[\"NLP_CLOUD_API_KEY\"] = \"bad-key\"\n elif (\n model\n == \"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\"\n ):\n temporary_key = os.environ[\"REPLICATE_API_KEY\"]\n os.environ[\"REPLICATE_API_KEY\"] = \"bad-key\"\n print(f\"model: {model}\")\n response = completion(model=model, messages=messages)\n print(f\"response: {response}\")\n except AuthenticationError as e:\n print(f\"AuthenticationError Caught Exception - {str(e)}\")\n except (\n OpenAIError\n ) as e: # is at least an openai error -> in case of random model errors - e.g. overloaded server\n print(f\"OpenAIError Caught Exception - {e}\")\n except Exception as e:\n print(type(e))\n print(type(AuthenticationError))\n print(e.__class__.__name__)\n print(f\"Uncaught Exception - {e}\")\n pytest.fail(f\"Error occurred: {e}\")\n if temporary_key != None: # reset the key\n if model == \"gpt-3.5-turbo\":\n os.environ[\"OPENAI_API_KEY\"] = temporary_key\n elif model == \"chatgpt-test\":\n os.environ[\"AZURE_AI_API_KEY\"] = temporary_key\n azure = True\n elif model == \"claude-3-5-haiku-20241022\":\n os.environ[\"ANTHROPIC_API_KEY\"] = temporary_key\n elif model == \"command-nightly\":\n os.environ[\"COHERE_API_KEY\"] = temporary_key\n elif (\n model\n == \"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\"\n ):\n os.environ[\"REPLICATE_API_KEY\"] = temporary_key\n elif \"j2\" in model:\n os.environ[\"AI21_API_KEY\"] = temporary_key\n elif \"togethercomputer\" in model:\n os.environ[\"TOGETHERAI_API_KEY\"] = temporary_key\n elif model in litellm.aleph_alpha_models:\n os.environ[\"ALEPH_ALPHA_API_KEY\"] = temporary_key\n elif model in litellm.nlp_cloud_models:\n os.environ.pop(\"NLP_CLOUD_API_KEY\", None)\n elif \"bedrock\" in model:\n os.environ[\"AWS_ACCESS_KEY_ID\"] = temporary_aws_access_key\n os.environ[\"AWS_REGION_NAME\"] = temporary_aws_region_name\n os.environ[\"AWS_SECRET_ACCESS_KEY\"] = temporary_secret_key\n return\n\n\n# for model in litellm.models_by_provider[\"bedrock\"]:\n# invalid_auth(model=model)\n# invalid_auth(model=\"command-nightly\")\n\n\n# Test 3: Invalid Request Error\n@pytest.mark.parametrize(\"model\", models)\ndef test_invalid_request_error(model):\n messages = [{\"content\": \"hey, how's it going?\", \"role\": \"user\"}]\n\n with pytest.raises(BadRequestError):\n completion(model=model, messages=messages, max_tokens=\"hello world\")\n\n\ndef test_completion_azure_exception():\n try:\n import openai\n\n print(\"azure gpt-3.5 test\\n\\n\")\n litellm.set_verbose = True\n ## Test azure call\n old_azure_key = os.environ[\"AZURE_AI_API_KEY\"]\n os.environ[\"AZURE_AI_API_KEY\"] = \"good morning\"\n response = completion(\n model=\"azure/gpt-4.1-mini\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n os.environ[\"AZURE_AI_API_KEY\"] = old_azure_key\n print(f\"response: {response}\")\n print(response)\n except openai.AuthenticationError as e:\n os.environ[\"AZURE_AI_API_KEY\"] = old_azure_key\n print(\"good job got the correct error for azure when key not set\")\n except Exception as e:\n pytest.fail(f\"Error occurred: {e}\")\n\n\n# test_completion_azure_exception()\n\n\ndef test_azure_embedding_exceptions():\n try:\n\n response = litellm.embedding(\n model=\"azure/text-embedding-ada-002\",\n input=\"hello\",\n mock_response=\"error\",\n )\n pytest.fail(f\"Bad request this should have failed but got {response}\")\n\n except Exception as e:\n print(vars(e))\n # CRUCIAL Test - Ensures our exceptions are readable and not overly complicated. some users have complained exceptions will randomly have another exception raised in our exception mapping\n assert str(e) == \"Mock error\"\n\n\nasync def asynctest_completion_azure_exception():\n try:\n import openai\n\n import litellm\n\n print(\"azure gpt-3.5 test\\n\\n\")\n litellm.set_verbose = True\n ## Test azure call\n old_azure_key = os.environ[\"AZURE_AI_API_KEY\"]\n os.environ[\"AZURE_AI_API_KEY\"] = \"good morning\"\n response = await litellm.acompletion(\n model=\"azure/gpt-4.1-mini\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n print(f\"response: {response}\")\n print(response)\n except openai.AuthenticationError as e:\n os.environ[\"AZURE_AI_API_KEY\"] = old_azure_key\n print(\"good job got the correct error for azure when key not set\")\n print(e)\n except Exception as e:\n print(\"Got wrong exception\")\n print(\"exception\", e)\n pytest.fail(f\"Error occurred: {e}\")\n\n\n# import asyncio\n# asyncio.run(\n# asynctest_completion_azure_exception()\n# )\n\n\ndef asynctest_completion_openai_exception_bad_model():\n try:\n import asyncio\n\n import openai\n\n import litellm\n\n print(\"azure exception bad model\\n\\n\")\n litellm.set_verbose = True\n\n ## Test azure call\n async def test():\n response = await litellm.acompletion(\n model=\"openai/gpt-6\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n\n asyncio.run(test())\n except openai.NotFoundError:\n print(\"Good job this is a NotFoundError for a model that does not exist!\")\n print(\"Passed\")\n except Exception as e:\n print(\"Raised wrong type of exception\", type(e))\n assert isinstance(e, openai.BadRequestError)\n pytest.fail(f\"Error occurred: {e}\")\n\n\n# asynctest_completion_openai_exception_bad_model()\n\n\ndef asynctest_completion_azure_exception_bad_model():\n try:\n import asyncio\n\n import openai\n\n import litellm\n\n print(\"azure exception bad model\\n\\n\")\n litellm.set_verbose = True\n\n ## Test azure call\n async def test():\n response = await litellm.acompletion(\n model=\"azure/gpt-12\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n\n asyncio.run(test())\n except openai.NotFoundError:\n print(\"Good job this is a NotFoundError for a model that does not exist!\")\n print(\"Passed\")\n except Exception as e:\n print(\"Raised wrong type of exception\", type(e))\n pytest.fail(f\"Error occurred: {e}\")\n\n\n# asynctest_completion_azure_exception_bad_model()\n\n\ndef test_completion_openai_exception():\n # test if openai:gpt raises openai.AuthenticationError\n try:\n import openai\n\n print(\"openai gpt-3.5 test\\n\\n\")\n litellm.set_verbose = True\n ## Test azure call\n old_azure_key = os.environ[\"OPENAI_API_KEY\"]\n os.environ[\"OPENAI_API_KEY\"] = \"good morning\"\n response = completion(\n model=\"gpt-4\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n print(f\"response: {response}\")\n print(response)\n except openai.AuthenticationError as e:\n os.environ[\"OPENAI_API_KEY\"] = old_azure_key\n print(\"OpenAI: good job got the correct error for openai when key not set\")\n except Exception as e:\n pytest.fail(f\"Error occurred: {e}\")\n\n\n# test_completion_openai_exception()\n\n\ndef test_anthropic_openai_exception():\n # test if anthropic raises litellm.AuthenticationError\n try:\n litellm.set_verbose = True\n ## Test azure call\n old_azure_key = os.environ[\"ANTHROPIC_API_KEY\"]\n os.environ.pop(\"ANTHROPIC_API_KEY\")\n response = completion(\n model=\"anthropic/claude-3-sonnet-20240229\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n print(f\"response: {response}\")\n print(response)\n except litellm.AuthenticationError as e:\n os.environ[\"ANTHROPIC_API_KEY\"] = old_azure_key\n print(\"Exception vars=\", vars(e))\n assert (\n \"Missing Anthropic API Key - A call is being made to anthropic but no key is set either in the environment variables or via params\"\n in e.message\n )\n print(\n \"ANTHROPIC_API_KEY: good job got the correct error for ANTHROPIC_API_KEY when key not set\"\n )\n except Exception as e:\n pytest.fail(f\"Error occurred: {e}\")\n\n\ndef test_completion_mistral_exception():\n # test if mistral/mistral-tiny raises openai.AuthenticationError\n try:\n import openai\n\n print(\"Testing mistral ai exception mapping\")\n litellm.set_verbose = True\n ## Test azure call\n old_azure_key = os.environ[\"MISTRAL_API_KEY\"]\n os.environ[\"MISTRAL_API_KEY\"] = \"good morning\"\n response = completion(\n model=\"mistral/mistral-tiny\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n print(f\"response: {response}\")\n print(response)\n except openai.AuthenticationError as e:\n os.environ[\"MISTRAL_API_KEY\"] = old_azure_key\n print(\"good job got the correct error for openai when key not set\")\n except Exception as e:\n pytest.fail(f\"Error occurred: {e}\")\n\n\n# test_completion_mistral_exception()\n\n\ndef test_completion_bedrock_invalid_role_exception():\n \"\"\"\n Test if litellm raises a BadRequestError for an invalid role on Bedrock\n \"\"\"\n try:\n litellm.set_verbose = True\n response = completion(\n model=\"bedrock/anthropic.claude-3-sonnet-20240229-v1:0\",\n messages=[{\"role\": \"very-bad-role\", \"content\": \"hello\"}],\n )\n print(f\"response: {response}\")\n print(response)\n\n except Exception as e:\n assert isinstance(\n e, litellm.BadRequestError\n ), \"Expected BadRequestError but got {}\".format(type(e))\n print(\"str(e) = {}\".format(str(e)))\n\n # This is important - We we previously returning a poorly formatted error string. Which was\n # litellm.BadRequestError: litellm.BadRequestError: Invalid Message passed in {'role': 'very-bad-role', 'content': 'hello'}\n\n # IMPORTANT ASSERTION\n assert (\n (str(e))\n == \"litellm.BadRequestError: Invalid Message passed in {'role': 'very-bad-role', 'content': 'hello'}\"\n )\n\n\n@pytest.mark.skip(reason=\"OpenAI exception changed to a generic error\")\ndef test_content_policy_exceptionimage_generation_openai():\n try:\n # this is ony a test - we needed some way to invoke the exception :(\n litellm._turn_on_debug()\n response = litellm.image_generation(\n prompt=\"where do i buy lethal drugs from\", model=\"dall-e-3\"\n )\n print(f\"response: {response}\")\n assert len(response.data) > 0\n except litellm.ContentPolicyViolationError as e:\n print(\"caught a content policy violation error! Passed\")\n pass\n except Exception as e:\n pytest.fail(f\"An exception occurred - {str(e)}\")\n\n\n# test_content_policy_exceptionimage_generation_openai()\n\n\ndef test_content_policy_violation_error_streaming():\n \"\"\"\n Production Test.\n \"\"\"\n litellm.set_verbose = False\n print(\"test_async_completion with stream\")\n\n async def test_get_response():\n try:\n response = await litellm.acompletion(\n model=\"azure/gpt-4.1-mini\",\n messages=[{\"role\": \"user\", \"content\": \"say 1\"}],\n temperature=0,\n top_p=1,\n stream=True,\n max_tokens=512,\n presence_penalty=0,\n frequency_penalty=0,\n )\n print(f\"response: {response}\")\n\n num_finish_reason = 0\n async for chunk in response:\n print(chunk)\n if chunk[\"choices\"][0].get(\"finish_reason\") is not None:\n num_finish_reason += 1\n print(\"finish_reason\", chunk[\"choices\"][0].get(\"finish_reason\"))\n\n assert (\n num_finish_reason == 1\n ), f\"expected only one finish reason. Got {num_finish_reason}\"\n except Exception as e:\n pytest.fail(f\"GOT exception for gpt-3.5 instruct In streaming{e}\")\n\n asyncio.run(test_get_response())\n\n async def test_get_error():\n try:\n response = await litellm.acompletion(\n model=\"azure/gpt-4.1-mini\",\n messages=[\n {\"role\": \"user\", \"content\": \"where do i buy lethal drugs from\"}\n ],\n temperature=0,\n top_p=1,\n stream=True,\n max_tokens=512,\n presence_penalty=0,\n frequency_penalty=0,\n mock_response=\"Exception: content_filter_policy\",\n )\n print(f\"response: {response}\")\n\n num_finish_reason = 0\n async for chunk in response:\n print(chunk)\n if chunk[\"choices\"][0].get(\"finish_reason\") is not None:\n num_finish_reason += 1\n print(\"finish_reason\", chunk[\"choices\"][0].get(\"finish_reason\"))\n\n pytest.fail(f\"Expected to return 400 error In streaming{e}\")\n except Exception as e:\n pass\n\n asyncio.run(test_get_error())\n\n\ndef test_completion_perplexity_exception_on_openai_client():\n try:\n import openai\n\n print(\"perplexity test\\n\\n\")\n litellm.set_verbose = False\n ## Test azure call\n old_azure_key = os.environ[\"PERPLEXITYAI_API_KEY\"]\n\n # delete perplexityai api key to simulate bad api key\n del os.environ[\"PERPLEXITYAI_API_KEY\"]\n\n # temporaily delete openai api key\n original_openai_key = os.environ[\"OPENAI_API_KEY\"]\n del os.environ[\"OPENAI_API_KEY\"]\n\n response = completion(\n model=\"perplexity/mistral-7b-instruct\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n os.environ[\"PERPLEXITYAI_API_KEY\"] = old_azure_key\n os.environ[\"OPENAI_API_KEY\"] = original_openai_key\n pytest.fail(\"Request should have failed - bad api key\")\n except openai.AuthenticationError as e:\n os.environ[\"PERPLEXITYAI_API_KEY\"] = old_azure_key\n os.environ[\"OPENAI_API_KEY\"] = original_openai_key\n print(\"exception: \", e)\n assert (\n \"The api_key client option must be set either by passing api_key to the client or by setting the PERPLEXITY_API_KEY environment variable\"\n in str(e)\n )\n except Exception as e:\n pytest.fail(f\"Error occurred: {e}\")\n\n\n# test_completion_perplexity_exception_on_openai_client()\n\n\ndef test_completion_perplexity_exception():\n try:\n import openai\n\n print(\"perplexity test\\n\\n\")\n litellm.set_verbose = True\n ## Test azure call\n old_azure_key = os.environ[\"PERPLEXITYAI_API_KEY\"]\n os.environ[\"PERPLEXITYAI_API_KEY\"] = \"good morning\"\n response = completion(\n model=\"perplexity/mistral-7b-instruct\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n os.environ[\"PERPLEXITYAI_API_KEY\"] = old_azure_key\n pytest.fail(\"Request should have failed - bad api key\")\n except openai.AuthenticationError as e:\n os.environ[\"PERPLEXITYAI_API_KEY\"] = old_azure_key\n print(\"exception: \", e)\n assert \"PerplexityException\" in str(e)\n except Exception as e:\n pytest.fail(f\"Error occurred: {e}\")\n\n\ndef test_completion_openai_api_key_exception():\n try:\n import openai\n\n print(\"gpt-3.5 test\\n\\n\")\n litellm.set_verbose = True\n ## Test azure call\n old_azure_key = os.environ[\"OPENAI_API_KEY\"]\n os.environ[\"OPENAI_API_KEY\"] = \"good morning\"\n response = completion(\n model=\"gpt-3.5-turbo\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n os.environ[\"OPENAI_API_KEY\"] = old_azure_key\n pytest.fail(\"Request should have failed - bad api key\")\n except openai.AuthenticationError as e:\n os.environ[\"OPENAI_API_KEY\"] = old_azure_key\n print(\"exception: \", e)\n assert \"OpenAIException\" in str(e)\n except Exception as e:\n pytest.fail(f\"Error occurred: {e}\")\n\n\n# tesy_async_acompletion()\n\n\ndef test_router_completion_vertex_exception():\n try:\n import litellm\n\n litellm.set_verbose = True\n router = litellm.Router(\n model_list=[\n {\n \"model_name\": \"vertex-gemini-pro\",\n \"litellm_params\": {\n \"model\": \"vertex_ai/gemini-pro\",\n \"api_key\": \"good-morning\",\n },\n },\n ]\n )\n response = router.completion(\n model=\"vertex-gemini-pro\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n vertex_project=\"bad-project\",\n )\n pytest.fail(\"Request should have failed - bad api key\")\n except Exception as e:\n print(\"exception: \", e)\n\n\ndef test_litellm_completion_vertex_exception():\n try:\n import litellm\n\n litellm.set_verbose = True\n response = completion(\n model=\"vertex_ai/gemini-pro\",\n api_key=\"good-morning\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n vertex_project=\"bad-project\",\n )\n pytest.fail(\"Request should have failed - bad api key\")\n except Exception as e:\n print(\"exception: \", e)\n\n\ndef test_litellm_predibase_exception():\n \"\"\"\n Test - Assert that the Predibase API Key is not returned on Authentication Errors\n \"\"\"\n try:\n import litellm\n\n litellm.set_verbose = True\n response = completion(\n model=\"predibase/llama-3-8b-instruct\",\n messages=[{\"role\": \"user\", \"content\": \"What is the meaning of life?\"}],\n tenant_id=\"c4768f95\",\n api_key=\"hf-rawapikey\",\n )\n pytest.fail(\"Request should have failed - bad api key\")\n except Exception as e:\n assert \"hf-rawapikey\" not in str(e)\n print(\"exception: \", e)\n\n\n# # test_invalid_request_error(model=\"command-nightly\")\n# # Test 3: Rate Limit Errors\n# def test_model_call(model):\n# try:\n# sample_text = \"how does a court case get to the Supreme Court?\"\n# messages = [{ \"content\": sample_text,\"role\": \"user\"}]\n# print(f\"model: {model}\")\n# response = completion(model=model, messages=messages)\n# except RateLimitError as e:\n# print(f\"headers: {e.response.headers}\")\n# return True\n# # except OpenAIError: # is at least an openai error -> in case of random model errors - e.g. overloaded server\n# # return True\n# except Exception as e:\n# print(f\"Uncaught Exception {model}: {type(e).__name__} - {e}\")\n# traceback.print_exc()\n# pass\n# return False\n# # Repeat each model 500 times\n# # extended_models = [model for model in models for _ in range(250)]\n# extended_models = [\"azure/gpt-4.1-mini\" for _ in range(250)]\n\n# def worker(model):\n# return test_model_call(model)\n\n# # Create a dictionary to store the results\n# counts = {True: 0, False: 0}\n\n# # Use Thread Pool Executor\n# with ThreadPoolExecutor(max_workers=500) as executor:\n# # Use map to start the operation in thread pool\n# results = executor.map(worker, extended_models)\n\n# # Iterate over results and count True/False\n# for result in results:\n# counts[result] += 1\n\n# accuracy_score = counts[True]/(counts[True] + counts[False])\n# print(f\"accuracy_score: {accuracy_score}\")\n\n\n@pytest.mark.parametrize(\n \"provider\",\n [\n \"predibase\",\n \"vertex_ai_beta\",\n \"anthropic\",\n \"databricks\",\n \"watsonx\",\n \"fireworks_ai\",\n ],\n)\ndef test_exception_mapping(provider):\n \"\"\"\n For predibase, run through a set of mock exceptions\n\n assert that they are being mapped correctly\n \"\"\"\n litellm.set_verbose = True\n error_map = {\n 400: litellm.BadRequestError,\n 401: litellm.AuthenticationError,\n 404: litellm.NotFoundError,\n 408: litellm.Timeout,\n 429: litellm.RateLimitError,\n 500: litellm.InternalServerError,\n 503: litellm.ServiceUnavailableError,\n }\n\n for code, expected_exception in error_map.items():\n mock_response = Exception()\n setattr(mock_response, \"text\", \"This is an error message\")\n setattr(mock_response, \"llm_provider\", provider)\n setattr(mock_response, \"status_code\", code)\n\n response: Any = None\n try:\n response = completion(\n model=\"{}/test-model\".format(provider),\n messages=[{\"role\": \"user\", \"content\": \"Hey, how's it going?\"}],\n mock_response=mock_response,\n )\n except expected_exception:\n continue\n except Exception as e:\n traceback.print_exc()\n response = \"{}\".format(str(e))\n pytest.fail(\n \"Did not raise expected exception. Expected={}, Return={},\".format(\n expected_exception, response\n )\n )\n\n pass\n\n\ndef test_fireworks_ai_exception_mapping():\n \"\"\"\n Comprehensive test for Fireworks AI exception mapping, including:\n 1. Standard 429 rate limit errors\n 2. Text-based rate limit detection (the main issue fixed)\n 3. Generic 400 errors that should NOT be rate limits\n 4. ExceptionCheckers utility function\n\n Related to: https://github.com/BerriAI/litellm/pull/11455\n Based on Fireworks AI documentation: https://docs.fireworks.ai/tools-sdks/python-client/api-reference\n \"\"\"\n import litellm\n from litellm.llms.fireworks_ai.common_utils import FireworksAIException\n from litellm.litellm_core_utils.exception_mapping_utils import ExceptionCheckers\n\n # Test scenarios covering all important cases\n test_scenarios = [\n {\n \"name\": \"Standard 429 rate limit with proper status code\",\n \"status_code\": 429,\n \"message\": \"Rate limit exceeded. Please try again in 60 seconds.\",\n \"expected_exception\": litellm.RateLimitError,\n },\n {\n \"name\": \"Status 400 with rate limit text (the main issue fixed)\",\n \"status_code\": 400,\n \"message\": '{\"error\":{\"object\":\"error\",\"type\":\"invalid_request_error\",\"message\":\"rate limit exceeded, please try again later\"}}',\n \"expected_exception\": litellm.RateLimitError,\n },\n {\n \"name\": \"Status 400 with generic invalid request (should NOT be rate limit)\",\n \"status_code\": 400,\n \"message\": '{\"error\":{\"type\":\"invalid_request_error\",\"message\":\"Invalid parameter value\"}}',\n \"expected_exception\": litellm.BadRequestError,\n },\n ]\n\n # Test each scenario\n for scenario in test_scenarios:\n mock_exception = FireworksAIException(\n status_code=scenario[\"status_code\"], message=scenario[\"message\"], headers={}\n )\n\n try:\n response = litellm.completion(\n model=\"fireworks_ai/llama-v3p1-70b-instruct\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n mock_response=mock_exception,\n )\n pytest.fail(\n f\"Expected {scenario['expected_exception'].__name__} to be raised\"\n )\n except scenario[\"expected_exception\"] as e:\n if scenario[\"expected_exception\"] == litellm.RateLimitError:\n assert \"rate limit\" in str(e).lower() or \"429\" in str(e)\n except Exception as e:\n pytest.fail(\n f\"Expected {scenario['expected_exception'].__name__} but got {type(e).__name__}: {e}\"\n )\n\n # Test ExceptionCheckers.is_error_str_rate_limit() method directly\n\n # Test cases that should return True (rate limit detected)\n rate_limit_strings = [\n \"429 rate limit exceeded\",\n \"Rate limit exceeded, please try again later\",\n \"RATE LIMIT ERROR\",\n \"Error 429: rate limit\",\n '{\"error\":{\"type\":\"invalid_request_error\",\"message\":\"rate limit exceeded, please try again later\"}}',\n \"HTTP 429 Too Many Requests\",\n ]\n\n for error_str in rate_limit_strings:\n assert ExceptionCheckers.is_error_str_rate_limit(\n error_str\n ), f\"Should detect rate limit in: {error_str}\"\n\n # Test cases that should return False (not rate limit)\n non_rate_limit_strings = [\n \"400 Bad Request\",\n \"Authentication failed\",\n \"Invalid model specified\",\n \"Context window exceeded\",\n \"Internal server error\",\n \"\",\n \"Some other error message\",\n ]\n\n for error_str in non_rate_limit_strings:\n assert not ExceptionCheckers.is_error_str_rate_limit(\n error_str\n ), f\"Should NOT detect rate limit in: {error_str}\"\n\n # Test edge cases\n assert not ExceptionCheckers.is_error_str_rate_limit(None) # type: ignore\n assert not ExceptionCheckers.is_error_str_rate_limit(42) # type: ignore\n\n\ndef test_anthropic_tool_calling_exception():\n \"\"\"\n Related - https://github.com/BerriAI/litellm/issues/4348\n \"\"\"\n tools = [\n {\n \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather in a given location\",\n \"parameters\": {},\n },\n }\n ]\n try:\n litellm.completion(\n model=\"claude-haiku-4-5-20251001\",\n messages=[{\"role\": \"user\", \"content\": \"Hey, how's it going?\"}],\n tools=tools,\n )\n except litellm.BadRequestError:\n pass\n\n\nfrom typing import Optional, Union\n\nfrom openai import AsyncOpenAI, OpenAI\n\n\ndef _pre_call_utils(\n call_type: str,\n data: dict,\n client: Union[OpenAI, AsyncOpenAI],\n sync_mode: bool,\n streaming: Optional[bool],\n):\n if call_type == \"embedding\":\n data[\"input\"] = \"Hello world!\"\n mapped_target: Any = client.embeddings.with_raw_response\n if sync_mode:\n original_function = litellm.embedding\n else:\n original_function = litellm.aembedding\n elif call_type == \"chat_completion\":\n data[\"messages\"] = [{\"role\": \"user\", \"content\": \"Hello world\"}]\n if streaming is True:\n data[\"stream\"] = True\n mapped_target = client.chat.completions.with_raw_response # type: ignore\n if sync_mode:\n original_function = litellm.completion\n else:\n original_function = litellm.acompletion\n elif call_type == \"completion\":\n data[\"prompt\"] = \"Hello world\"\n if streaming is True:\n data[\"stream\"] = True\n mapped_target = client.completions.with_raw_response # type: ignore\n if sync_mode:\n original_function = litellm.text_completion\n else:\n original_function = litellm.atext_completion\n\n return data, original_function, mapped_target\n\n\ndef _pre_call_utils_httpx(\n call_type: str,\n data: dict,\n client: Union[HTTPHandler, AsyncHTTPHandler],\n sync_mode: bool,\n streaming: Optional[bool],\n):\n mapped_target: Any = client.client\n if call_type == \"embedding\":\n data[\"input\"] = \"Hello world!\"\n\n if sync_mode:\n original_function = litellm.embedding\n else:\n original_function = litellm.aembedding\n elif call_type == \"chat_completion\":\n data[\"messages\"] = [{\"role\": \"user\", \"content\": \"Hello world\"}]\n if streaming is True:\n data[\"stream\"] = True\n\n if sync_mode:\n original_function = litellm.completion\n else:\n original_function = litellm.acompletion\n elif call_type == \"completion\":\n data[\"prompt\"] = \"Hello world\"\n if streaming is True:\n data[\"stream\"] = True\n if sync_mode:\n original_function = litellm.text_completion\n else:\n original_function = litellm.atext_completion\n\n return data, original_function, mapped_target\n\n\n@pytest.mark.parametrize(\n \"sync_mode\",\n [True, False],\n)\n@pytest.mark.parametrize(\n \"provider, model, call_type, streaming\",\n [\n (\"openai\", \"text-embedding-ada-002\", \"embedding\", None),\n (\"openai\", \"gpt-3.5-turbo\", \"chat_completion\", False),\n (\"openai\", \"gpt-3.5-turbo\", \"chat_completion\", True),\n (\"openai\", \"gpt-3.5-turbo-instruct\", \"completion\", True),\n (\"azure\", \"azure/gpt-4.1-mini\", \"chat_completion\", True),\n (\"azure\", \"azure/text-embedding-ada-002\", \"embedding\", True),\n (\"azure\", \"azure_text/gpt-3.5-turbo-instruct\", \"completion\", True),\n ],\n)\n@pytest.mark.asyncio\nasync def test_exception_with_headers(sync_mode, provider, model, call_type, streaming):\n \"\"\"\n User feedback: litellm says \"No deployments available for selected model, Try again in 60 seconds\"\n but Azure says to retry in at most 9s\n\n ```\n {\"message\": \"litellm.proxy.proxy_server.embeddings(): Exception occured - No deployments available for selected model, Try again in 60 seconds. Passed model=text-embedding-ada-002. pre-call-checks=False, allowed_model_region=n/a, cooldown_list=[('b49cbc9314273db7181fe69b1b19993f04efb88f2c1819947c538bac08097e4c', {'Exception Received': 'litellm.RateLimitError: AzureException RateLimitError - Requests to the Embeddings_Create Operation under Azure OpenAI API version 2023-09-01-preview have exceeded call rate limit of your current OpenAI S0 pricing tier. Please retry after 9 seconds. Please go here: https://aka.ms/oai/quotaincrease if you would like to further increase the default rate limit.', 'Status Code': '429'})]\", \"level\": \"ERROR\", \"timestamp\": \"2024-08-22T03:25:36.900476\"}\n ```\n \"\"\"\n print(f\"Received args: {locals()}\")\n import openai\n\n if sync_mode:\n if provider == \"openai\":\n openai_client = openai.OpenAI(api_key=\"\")\n elif provider == \"azure\":\n openai_client = openai.AzureOpenAI(\n api_key=\"\", base_url=\"\", api_version=litellm.AZURE_DEFAULT_API_VERSION\n )\n else:\n if provider == \"openai\":\n openai_client = openai.AsyncOpenAI(api_key=\"\")\n elif provider == \"azure\":\n openai_client = openai.AsyncAzureOpenAI(\n api_key=\"\", base_url=\"\", api_version=litellm.AZURE_DEFAULT_API_VERSION\n )\n\n data = {\"model\": model}\n data, original_function, mapped_target = _pre_call_utils(\n call_type=call_type,\n data=data,\n client=openai_client,\n sync_mode=sync_mode,\n streaming=streaming,\n )\n\n cooldown_time = 30.0\n\n def _return_exception(*args, **kwargs):\n import datetime\n\n from httpx import Headers, Request, Response\n\n kwargs = {\n \"request\": Request(\"POST\", \"https://www.google.com\"),\n \"message\": \"Error code: 429 - Rate Limit Error!\",\n \"body\": {\"detail\": \"Rate Limit Error!\"},\n \"code\": None,\n \"param\": None,\n \"type\": None,\n \"response\": Response(\n status_code=429,\n headers=Headers(\n {\n \"date\": \"Sat, 21 Sep 2024 22:56:53 GMT\",\n \"server\": \"uvicorn\",\n \"retry-after\": \"30\",\n \"content-length\": \"30\",\n \"content-type\": \"application/json\",\n }\n ),\n request=Request(\"POST\", \"http://0.0.0.0:9000/chat/completions\"),\n ),\n \"status_code\": 429,\n \"request_id\": None,\n }\n\n exception = Exception()\n for k, v in kwargs.items():\n setattr(exception, k, v)\n raise exception\n\n with patch.object(\n mapped_target,\n \"create\",\n side_effect=_return_exception,\n ):\n new_retry_after_mock_client = MagicMock(return_value=-1)\n\n litellm.utils._get_retry_after_from_exception_header = (\n new_retry_after_mock_client\n )\n\n exception_raised = False\n try:\n if sync_mode:\n resp = original_function(**data, client=openai_client)\n if streaming:\n for chunk in resp:\n continue\n else:\n resp = await original_function(**data, client=openai_client)\n\n if streaming:\n async for chunk in resp:\n continue\n\n except litellm.RateLimitError as e:\n exception_raised = True\n assert e.litellm_response_headers is not None\n assert int(e.litellm_response_headers[\"retry-after\"]) == cooldown_time\n\n if exception_raised is False:\n print(resp)\n assert exception_raised\n\n\ndef test_openai_gateway_timeout_error():\n \"\"\"\n Test that the OpenAI gateway timeout error is raised\n \"\"\"\n openai_client = OpenAI()\n mapped_target = openai_client.chat.completions.with_raw_response # type: ignore\n\n def _return_exception(*args, **kwargs):\n import datetime\n\n from httpx import Headers, Request, Response\n\n kwargs = {\n \"request\": Request(\"POST\", \"https://www.google.com\"),\n \"message\": \"Error code: 504 - Gateway Timeout Error!\",\n \"body\": {\"detail\": \"Gateway Timeout Error!\"},\n \"code\": None,\n \"param\": None,\n \"type\": None,\n \"response\": Response(\n status_code=504,\n headers=Headers(\n {\n \"date\": \"Sat, 21 Sep 2024 22:56:53 GMT\",\n \"server\": \"uvicorn\",\n \"content-length\": \"30\",\n \"content-type\": \"application/json\",\n }\n ),\n request=Request(\"POST\", \"http://0.0.0.0:9000/chat/completions\"),\n ),\n \"status_code\": 504,\n \"request_id\": None,\n }\n\n exception = Exception()\n for k, v in kwargs.items():\n setattr(exception, k, v)\n raise exception\n\n try:\n with patch.object(\n mapped_target,\n \"create\",\n side_effect=_return_exception,\n ):\n litellm.completion(\n model=\"openai/gpt-3.5-turbo\",\n messages=[{\"role\": \"user\", \"content\": \"Hello world\"}],\n client=openai_client,\n )\n pytest.fail(\"Expected to raise Timeout\")\n except litellm.Timeout as e:\n assert e.status_code == 504\n\n\n@pytest.mark.parametrize(\n \"sync_mode\",\n [True, False],\n)\n@pytest.mark.parametrize(\"streaming\", [True, False])\n@pytest.mark.parametrize(\n \"provider, model, call_type\",\n [\n (\"anthropic\", \"claude-3-haiku-20240307\", \"chat_completion\"),\n ],\n)\n@pytest.mark.asyncio\nasync def test_exception_with_headers_httpx(\n sync_mode, provider, model, call_type, streaming\n):\n \"\"\"\n User feedback: litellm says \"No deployments available for selected model, Try again in 60 seconds\"\n but Azure says to retry in at most 9s\n\n ```\n {\"message\": \"litellm.proxy.proxy_server.embeddings(): Exception occured - No deployments available for selected model, Try again in 60 seconds. Passed model=text-embedding-ada-002. pre-call-checks=False, allowed_model_region=n/a, cooldown_list=[('b49cbc9314273db7181fe69b1b19993f04efb88f2c1819947c538bac08097e4c', {'Exception Received': 'litellm.RateLimitError: AzureException RateLimitError - Requests to the Embeddings_Create Operation under Azure OpenAI API version 2023-09-01-preview have exceeded call rate limit of your current OpenAI S0 pricing tier. Please retry after 9 seconds. Please go here: https://aka.ms/oai/quotaincrease if you would like to further increase the default rate limit.', 'Status Code': '429'})]\", \"level\": \"ERROR\", \"timestamp\": \"2024-08-22T03:25:36.900476\"}\n ```\n \"\"\"\n print(f\"Received args: {locals()}\")\n import openai\n\n if sync_mode:\n client = HTTPHandler()\n else:\n client = AsyncHTTPHandler()\n\n data = {\"model\": model}\n data, original_function, mapped_target = _pre_call_utils_httpx(\n call_type=call_type,\n data=data,\n client=client,\n sync_mode=sync_mode,\n streaming=streaming,\n )\n\n cooldown_time = 30.0\n\n def _return_exception(*args, **kwargs):\n import datetime\n\n from httpx import Headers, HTTPStatusError, Request, Response\n\n # Create the Request object\n request = Request(\"POST\", \"http://0.0.0.0:9000/chat/completions\")\n\n # Create the Response object with the necessary headers and status code\n response = Response(\n status_code=429,\n headers=Headers(\n {\n \"date\": \"Sat, 21 Sep 2024 22:56:53 GMT\",\n \"server\": \"uvicorn\",\n \"retry-after\": \"30\",\n \"content-length\": \"30\",\n \"content-type\": \"application/json\",\n }\n ),\n request=request,\n )\n\n # Create and raise the HTTPStatusError exception\n raise HTTPStatusError(\n message=\"Error code: 429 - Rate Limit Error!\",\n request=request,\n response=response,\n )\n\n with patch.object(\n mapped_target,\n \"send\",\n side_effect=_return_exception,\n ):\n new_retry_after_mock_client = MagicMock(return_value=-1)\n\n litellm.utils._get_retry_after_from_exception_header = (\n new_retry_after_mock_client\n )\n\n exception_raised = False\n try:\n if sync_mode:\n resp = original_function(**data, client=client)\n if streaming:\n for chunk in resp:\n continue\n else:\n resp = await original_function(**data, client=client)\n\n if streaming:\n async for chunk in resp:\n continue\n\n except litellm.RateLimitError as e:\n exception_raised = True\n assert (\n e.litellm_response_headers is not None\n ), \"litellm_response_headers is None\"\n print(\"e.litellm_response_headers\", e.litellm_response_headers)\n assert int(e.litellm_response_headers[\"retry-after\"]) == cooldown_time\n\n if exception_raised is False:\n print(resp)\n assert exception_raised\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"model\", [\"azure/gpt-4.1-mini\", \"openai/gpt-3.5-turbo\"])\nasync def test_bad_request_error_contains_httpx_response(model):\n \"\"\"\n Test that the BadRequestError contains the httpx response\n\n Relevant issue: https://github.com/BerriAI/litellm/issues/6732\n \"\"\"\n try:\n await litellm.acompletion(\n model=model,\n messages=[{\"role\": \"user\", \"content\": \"Hello world\"}],\n bad_arg=\"bad_arg\",\n )\n pytest.fail(\"Expected to raise BadRequestError\")\n except litellm.BadRequestError as e:\n print(\"e.response\", e.response)\n print(\"vars(e.response)\", vars(e.response))\n assert e.response is not None\n\n\ndef test_exceptions_base_class():\n try:\n raise litellm.RateLimitError(\n message=\"BedrockException: Rate Limit Error\",\n model=\"model\",\n llm_provider=\"bedrock\",\n )\n except litellm.RateLimitError as e:\n assert isinstance(e, litellm.RateLimitError)\n assert e.code == \"429\"\n assert e.type == \"throttling_error\"\n\n\ndef test_context_window_exceeded_error_from_litellm_proxy():\n from httpx import Response\n from litellm.litellm_core_utils.exception_mapping_utils import (\n extract_and_raise_litellm_exception,\n )\n\n args = {\n \"response\": Response(status_code=400, text=\"Bad Request\"),\n \"error_str\": \"Error code: 400 - {'error': {'message': \\\"litellm.ContextWindowExceededError: litellm.BadRequestError: this is a mock context window exceeded error\\\\nmodel=gpt-3.5-turbo. context_window_fallbacks=None. fallbacks=None.\\\\n\\\\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks\\\\nReceived Model Group=gpt-3.5-turbo\\\\nAvailable Model Group Fallbacks=None\\\", 'type': None, 'param': None, 'code': '400'}}\",\n \"model\": \"gpt-3.5-turbo\",\n \"custom_llm_provider\": \"litellm_proxy\",\n }\n with pytest.raises(litellm.ContextWindowExceededError):\n extract_and_raise_litellm_exception(**args)\n\n\ndef test_bad_request_error_with_response_without_request():\n \"\"\"\n Test that BadRequestError handles Response objects without a request attribute.\n\n This simulates a real scenario where a Response is created without a request\n (e.g., in tests or when manually creating error responses), and we need to\n ensure it doesn't raise RuntimeError when the exception is created.\n \"\"\"\n from httpx import Response\n from litellm.litellm_core_utils.exception_mapping_utils import (\n extract_and_raise_litellm_exception,\n )\n\n # Create a Response without a request (simulates the scenario that was failing)\n response_without_request = Response(status_code=400, text=\"Bad Request\")\n\n # Test that extract_and_raise_litellm_exception can handle this\n args = {\n \"response\": response_without_request,\n \"error_str\": \"Error code: 400 - {'error': {'message': 'litellm.BadRequestError: Invalid request parameters', 'type': None, 'param': None, 'code': '400'}}\",\n \"model\": \"gpt-3.5-turbo\",\n \"custom_llm_provider\": \"openai\",\n }\n\n # This should raise BadRequestError without RuntimeError\n with pytest.raises(litellm.BadRequestError) as exc_info:\n extract_and_raise_litellm_exception(**args)\n\n # Verify the exception was created successfully\n error = exc_info.value\n assert error is not None\n assert error.model == \"gpt-3.5-turbo\"\n assert error.llm_provider == \"openai\"\n\n # Verify the exception has a response (should be minimal error response)\n assert error.response is not None\n # The response should have a request (minimal error response has one)\n assert getattr(error.response, \"_request\", None) is not None\n # Should be able to access request property without RuntimeError\n assert error.response.request is not None\n\n\n@pytest.mark.parametrize(\"sync_mode\", [True, False])\n@pytest.mark.parametrize(\"stream_mode\", [True, False])\n@pytest.mark.parametrize(\"model\", [\"gpt-4.1-nano\"]) # \"gpt-4o-mini\",\n@pytest.mark.asyncio\nasync def test_exception_bubbling_up(sync_mode, stream_mode, model):\n \"\"\"\n make sure code, param, and type are bubbled up\n \"\"\"\n import litellm\n\n litellm.set_verbose = True\n with pytest.raises(Exception) as exc_info:\n if sync_mode:\n litellm.completion(\n model=model,\n messages=[{\"role\": \"usera\", \"content\": \"hi\"}],\n stream=stream_mode,\n sync_stream=sync_mode,\n )\n else:\n await litellm.acompletion(\n model=model,\n messages=[{\"role\": \"usera\", \"content\": \"hi\"}],\n stream=stream_mode,\n sync_stream=sync_mode,\n )\n\n assert exc_info.value.code == \"invalid_value\"\n assert exc_info.value.param is not None\n assert exc_info.value.type == \"invalid_request_error\"\n"}, "fc16a8cc0e13f8d9ade7e6c180f4cd8e841ad7c5": {"t": 1785846887.2155786, "n": 50646, "h": "eb7c09ab57d9c80c", "c": "import asyncio\nimport os\nimport subprocess\nimport sys\nimport traceback\nfrom typing import Any\n\nfrom openai import AuthenticationError, BadRequestError, OpenAIError, RateLimitError\n\nfrom litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler\n\nsys.path.insert(\n 0, os.path.abspath(\"../..\")\n) # Adds the parent directory to the system path\nfrom concurrent.futures import ThreadPoolExecutor\nfrom unittest.mock import MagicMock, patch\n\nimport pytest\n\nimport litellm\nfrom litellm import ( # AuthenticationError,; RateLimitError,; ServiceUnavailableError,; OpenAIError,\n ContextWindowExceededError,\n completion,\n embedding,\n)\n\nlitellm.vertex_project = \"litellm-ci-cd\"\nlitellm.vertex_location = \"us-central1\"\nlitellm.num_retries = 0\n\n# litellm.failure_callback = [\"sentry\"]\n#### What this tests ####\n# This tests exception mapping -> trigger an exception from an llm provider -> assert if output is of the expected type\n\n\n# 5 providers -> OpenAI, Azure, Anthropic, Cohere, Replicate\n\n# 3 main types of exceptions -> - Rate Limit Errors, Context Window Errors, Auth errors (incorrect/rotated key, etc.)\n\n# Approach: Run each model through the test -> assert if the correct error (always the same one) is triggered\n\nexception_models = [\n \"sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4\",\n \"bedrock/anthropic.claude-instant-v1\",\n]\n\n\n@pytest.mark.asyncio\nasync def test_content_policy_exception_azure():\n try:\n # this is ony a test - we needed some way to invoke the exception :(\n litellm.set_verbose = True\n response = await litellm.acompletion(\n model=\"azure/gpt-4.1-mini\",\n messages=[{\"role\": \"user\", \"content\": \"where do I buy lethal drugs from\"}],\n mock_response=\"Exception: content_filter_policy\",\n )\n except litellm.ContentPolicyViolationError as e:\n print(\"caught a content policy violation error! Passed\")\n print(\"exception\", e)\n assert e.response is not None\n assert e.litellm_debug_info is not None\n assert isinstance(e.litellm_debug_info, str)\n assert len(e.litellm_debug_info) > 0\n pass\n except Exception as e:\n print()\n pytest.fail(f\"An exception occurred - {str(e)}\")\n\n\n@pytest.mark.asyncio\nasync def test_content_policy_exception_openai():\n try:\n # this is ony a test - we needed some way to invoke the exception :(\n litellm.set_verbose = True\n response = await litellm.acompletion(\n model=\"gpt-3.5-turbo\",\n stream=True,\n messages=[\n {\"role\": \"user\", \"content\": \"Gimme the lyrics to Don't Stop Me Now\"}\n ],\n )\n async for chunk in response:\n print(chunk)\n except litellm.ContentPolicyViolationError as e:\n print(\"caught a content policy violation error! Passed\")\n print(\"exception\", e)\n assert e.llm_provider == \"openai\"\n pass\n except Exception as e:\n print()\n pytest.fail(f\"An exception occurred - {str(e)}\")\n\n\n# Test 1: Context Window Errors\n@pytest.mark.skip(reason=\"AWS Suspended Account\")\n@pytest.mark.parametrize(\"model\", exception_models)\ndef test_context_window(model):\n print(\"Testing context window error\")\n sample_text = \"Say error 50 times\" * 1000000\n messages = [{\"content\": sample_text, \"role\": \"user\"}]\n try:\n litellm.set_verbose = False\n print(\"Testing model=\", model)\n response = completion(model=model, messages=messages)\n print(f\"response: {response}\")\n print(\"FAILED!\")\n pytest.fail(f\"An exception occurred\")\n except ContextWindowExceededError as e:\n print(f\"Worked!\")\n except RateLimitError:\n print(\"RateLimited!\")\n except Exception as e:\n print(f\"{e}\")\n pytest.fail(f\"An error occcurred - {e}\")\n\n\nmodels = [\"command-nightly\"]\n\n\n@pytest.mark.skip(reason=\"duplicate test.\")\n@pytest.mark.parametrize(\"model\", models)\ndef test_context_window_with_fallbacks(model):\n ctx_window_fallback_dict = {\n \"command-nightly\": \"claude-2.1\",\n \"gpt-3.5-turbo-instruct\": \"gpt-3.5-turbo-16k\",\n \"azure/gpt-4.1-mini\": \"gpt-3.5-turbo-16k\",\n }\n sample_text = \"how does a court case get to the Supreme Court?\" * 1000\n messages = [{\"content\": sample_text, \"role\": \"user\"}]\n\n try:\n completion(\n model=model,\n messages=messages,\n context_window_fallback_dict=ctx_window_fallback_dict,\n )\n except litellm.ServiceUnavailableError as e:\n pass\n except litellm.APIConnectionError as e:\n pass\n\n\n# for model in litellm.models_by_provider[\"bedrock\"]:\n# test_context_window(model=model)\n# test_context_window(model=\"chat-bison\")\n# test_context_window_with_fallbacks(model=\"command-nightly\")\n# Test 2: InvalidAuth Errors\n@pytest.mark.parametrize(\"model\", models)\ndef invalid_auth(model): # set the model key to an invalid key, depending on the model\n messages = [{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n temporary_key = None\n try:\n if model == \"gpt-3.5-turbo\" or model == \"gpt-3.5-turbo-instruct\":\n temporary_key = os.environ[\"OPENAI_API_KEY\"]\n os.environ[\"OPENAI_API_KEY\"] = \"bad-key\"\n elif \"bedrock\" in model:\n temporary_aws_access_key = os.environ[\"AWS_ACCESS_KEY_ID\"]\n os.environ[\"AWS_ACCESS_KEY_ID\"] = \"bad-key\"\n temporary_aws_region_name = os.environ[\"AWS_REGION_NAME\"]\n os.environ[\"AWS_REGION_NAME\"] = \"bad-key\"\n temporary_secret_key = os.environ[\"AWS_SECRET_ACCESS_KEY\"]\n os.environ[\"AWS_SECRET_ACCESS_KEY\"] = \"bad-key\"\n elif model == \"azure/gpt-4.1-mini\":\n temporary_key = os.environ[\"AZURE_AI_API_KEY\"]\n os.environ[\"AZURE_AI_API_KEY\"] = \"bad-key\"\n elif model == \"claude-3-5-haiku-20241022\":\n temporary_key = os.environ[\"ANTHROPIC_API_KEY\"]\n os.environ[\"ANTHROPIC_API_KEY\"] = \"bad-key\"\n elif model == \"command-nightly\":\n temporary_key = os.environ[\"COHERE_API_KEY\"]\n os.environ[\"COHERE_API_KEY\"] = \"bad-key\"\n elif \"j2\" in model:\n temporary_key = os.environ[\"AI21_API_KEY\"]\n os.environ[\"AI21_API_KEY\"] = \"bad-key\"\n elif \"togethercomputer\" in model:\n temporary_key = os.environ[\"TOGETHERAI_API_KEY\"]\n os.environ[\"TOGETHERAI_API_KEY\"] = \"sk-test-togetherai-key-808\"\n elif model in litellm.openrouter_models:\n temporary_key = os.environ[\"OPENROUTER_API_KEY\"]\n os.environ[\"OPENROUTER_API_KEY\"] = \"bad-key\"\n elif model in litellm.aleph_alpha_models:\n temporary_key = os.environ[\"ALEPH_ALPHA_API_KEY\"]\n os.environ[\"ALEPH_ALPHA_API_KEY\"] = \"bad-key\"\n elif model in litellm.nlp_cloud_models:\n os.environ[\"NLP_CLOUD_API_KEY\"] = \"bad-key\"\n elif (\n model\n == \"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\"\n ):\n temporary_key = os.environ[\"REPLICATE_API_KEY\"]\n os.environ[\"REPLICATE_API_KEY\"] = \"bad-key\"\n print(f\"model: {model}\")\n response = completion(model=model, messages=messages)\n print(f\"response: {response}\")\n except AuthenticationError as e:\n print(f\"AuthenticationError Caught Exception - {str(e)}\")\n except (\n OpenAIError\n ) as e: # is at least an openai error -> in case of random model errors - e.g. overloaded server\n print(f\"OpenAIError Caught Exception - {e}\")\n except Exception as e:\n print(type(e))\n print(type(AuthenticationError))\n print(e.__class__.__name__)\n print(f\"Uncaught Exception - {e}\")\n pytest.fail(f\"Error occurred: {e}\")\n if temporary_key != None: # reset the key\n if model == \"gpt-3.5-turbo\":\n os.environ[\"OPENAI_API_KEY\"] = temporary_key\n elif model == \"chatgpt-test\":\n os.environ[\"AZURE_AI_API_KEY\"] = temporary_key\n azure = True\n elif model == \"claude-3-5-haiku-20241022\":\n os.environ[\"ANTHROPIC_API_KEY\"] = temporary_key\n elif model == \"command-nightly\":\n os.environ[\"COHERE_API_KEY\"] = temporary_key\n elif (\n model\n == \"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\"\n ):\n os.environ[\"REPLICATE_API_KEY\"] = temporary_key\n elif \"j2\" in model:\n os.environ[\"AI21_API_KEY\"] = temporary_key\n elif \"togethercomputer\" in model:\n os.environ[\"TOGETHERAI_API_KEY\"] = temporary_key\n elif model in litellm.aleph_alpha_models:\n os.environ[\"ALEPH_ALPHA_API_KEY\"] = temporary_key\n elif model in litellm.nlp_cloud_models:\n os.environ.pop(\"NLP_CLOUD_API_KEY\", None)\n elif \"bedrock\" in model:\n os.environ[\"AWS_ACCESS_KEY_ID\"] = temporary_aws_access_key\n os.environ[\"AWS_REGION_NAME\"] = temporary_aws_region_name\n os.environ[\"AWS_SECRET_ACCESS_KEY\"] = temporary_secret_key\n return\n\n\n# for model in litellm.models_by_provider[\"bedrock\"]:\n# invalid_auth(model=model)\n# invalid_auth(model=\"command-nightly\")\n\n\n# Test 3: Invalid Request Error\n@pytest.mark.parametrize(\"model\", models)\ndef test_invalid_request_error(model):\n messages = [{\"content\": \"hey, how's it going?\", \"role\": \"user\"}]\n\n with pytest.raises(BadRequestError):\n completion(model=model, messages=messages, max_tokens=\"hello world\")\n\n\ndef test_completion_azure_exception():\n try:\n import openai\n\n print(\"azure gpt-3.5 test\\n\\n\")\n litellm.set_verbose = True\n ## Test azure call\n old_azure_key = os.environ[\"AZURE_AI_API_KEY\"]\n os.environ[\"AZURE_AI_API_KEY\"] = \"good morning\"\n response = completion(\n model=\"azure/gpt-4.1-mini\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n os.environ[\"AZURE_AI_API_KEY\"] = old_azure_key\n print(f\"response: {response}\")\n print(response)\n except openai.AuthenticationError as e:\n os.environ[\"AZURE_AI_API_KEY\"] = old_azure_key\n print(\"good job got the correct error for azure when key not set\")\n except Exception as e:\n pytest.fail(f\"Error occurred: {e}\")\n\n\n# test_completion_azure_exception()\n\n\ndef test_azure_embedding_exceptions():\n try:\n\n response = litellm.embedding(\n model=\"azure/text-embedding-ada-002\",\n input=\"hello\",\n mock_response=\"error\",\n )\n pytest.fail(f\"Bad request this should have failed but got {response}\")\n\n except Exception as e:\n print(vars(e))\n # CRUCIAL Test - Ensures our exceptions are readable and not overly complicated. some users have complained exceptions will randomly have another exception raised in our exception mapping\n assert str(e) == \"Mock error\"\n\n\nasync def asynctest_completion_azure_exception():\n try:\n import openai\n\n import litellm\n\n print(\"azure gpt-3.5 test\\n\\n\")\n litellm.set_verbose = True\n ## Test azure call\n old_azure_key = os.environ[\"AZURE_AI_API_KEY\"]\n os.environ[\"AZURE_AI_API_KEY\"] = \"good morning\"\n response = await litellm.acompletion(\n model=\"azure/gpt-4.1-mini\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n print(f\"response: {response}\")\n print(response)\n except openai.AuthenticationError as e:\n os.environ[\"AZURE_AI_API_KEY\"] = old_azure_key\n print(\"good job got the correct error for azure when key not set\")\n print(e)\n except Exception as e:\n print(\"Got wrong exception\")\n print(\"exception\", e)\n pytest.fail(f\"Error occurred: {e}\")\n\n\n# import asyncio\n# asyncio.run(\n# asynctest_completion_azure_exception()\n# )\n\n\ndef asynctest_completion_openai_exception_bad_model():\n try:\n import asyncio\n\n import openai\n\n import litellm\n\n print(\"azure exception bad model\\n\\n\")\n litellm.set_verbose = True\n\n ## Test azure call\n async def test():\n response = await litellm.acompletion(\n model=\"openai/gpt-6\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n\n asyncio.run(test())\n except openai.NotFoundError:\n print(\"Good job this is a NotFoundError for a model that does not exist!\")\n print(\"Passed\")\n except Exception as e:\n print(\"Raised wrong type of exception\", type(e))\n assert isinstance(e, openai.BadRequestError)\n pytest.fail(f\"Error occurred: {e}\")\n\n\n# asynctest_completion_openai_exception_bad_model()\n\n\ndef asynctest_completion_azure_exception_bad_model():\n try:\n import asyncio\n\n import openai\n\n import litellm\n\n print(\"azure exception bad model\\n\\n\")\n litellm.set_verbose = True\n\n ## Test azure call\n async def test():\n response = await litellm.acompletion(\n model=\"azure/gpt-12\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n\n asyncio.run(test())\n except openai.NotFoundError:\n print(\"Good job this is a NotFoundError for a model that does not exist!\")\n print(\"Passed\")\n except Exception as e:\n print(\"Raised wrong type of exception\", type(e))\n pytest.fail(f\"Error occurred: {e}\")\n\n\n# asynctest_completion_azure_exception_bad_model()\n\n\ndef test_completion_openai_exception():\n # test if openai:gpt raises openai.AuthenticationError\n try:\n import openai\n\n print(\"openai gpt-3.5 test\\n\\n\")\n litellm.set_verbose = True\n ## Test azure call\n old_azure_key = os.environ[\"OPENAI_API_KEY\"]\n os.environ[\"OPENAI_API_KEY\"] = \"good morning\"\n response = completion(\n model=\"gpt-4\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n print(f\"response: {response}\")\n print(response)\n except openai.AuthenticationError as e:\n os.environ[\"OPENAI_API_KEY\"] = old_azure_key\n print(\"OpenAI: good job got the correct error for openai when key not set\")\n except Exception as e:\n pytest.fail(f\"Error occurred: {e}\")\n\n\n# test_completion_openai_exception()\n\n\ndef test_anthropic_openai_exception():\n # test if anthropic raises litellm.AuthenticationError\n try:\n litellm.set_verbose = True\n ## Test azure call\n old_azure_key = os.environ[\"ANTHROPIC_API_KEY\"]\n os.environ.pop(\"ANTHROPIC_API_KEY\")\n response = completion(\n model=\"anthropic/claude-3-sonnet-20240229\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n print(f\"response: {response}\")\n print(response)\n except litellm.AuthenticationError as e:\n os.environ[\"ANTHROPIC_API_KEY\"] = old_azure_key\n print(\"Exception vars=\", vars(e))\n assert (\n \"Missing Anthropic API Key - A call is being made to anthropic but no key is set either in the environment variables or via params\"\n in e.message\n )\n print(\n \"ANTHROPIC_API_KEY: good job got the correct error for ANTHROPIC_API_KEY when key not set\"\n )\n except Exception as e:\n pytest.fail(f\"Error occurred: {e}\")\n\n\ndef test_completion_mistral_exception():\n # test if mistral/mistral-tiny raises openai.AuthenticationError\n try:\n import openai\n\n print(\"Testing mistral ai exception mapping\")\n litellm.set_verbose = True\n ## Test azure call\n old_azure_key = os.environ[\"MISTRAL_API_KEY\"]\n os.environ[\"MISTRAL_API_KEY\"] = \"good morning\"\n response = completion(\n model=\"mistral/mistral-tiny\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n print(f\"response: {response}\")\n print(response)\n except openai.AuthenticationError as e:\n os.environ[\"MISTRAL_API_KEY\"] = old_azure_key\n print(\"good job got the correct error for openai when key not set\")\n except Exception as e:\n pytest.fail(f\"Error occurred: {e}\")\n\n\n# test_completion_mistral_exception()\n\n\ndef test_completion_bedrock_invalid_role_exception():\n \"\"\"\n Test if litellm raises a BadRequestError for an invalid role on Bedrock\n \"\"\"\n try:\n litellm.set_verbose = True\n response = completion(\n model=\"bedrock/anthropic.claude-3-sonnet-20240229-v1:0\",\n messages=[{\"role\": \"very-bad-role\", \"content\": \"hello\"}],\n )\n print(f\"response: {response}\")\n print(response)\n\n except Exception as e:\n assert isinstance(\n e, litellm.BadRequestError\n ), \"Expected BadRequestError but got {}\".format(type(e))\n print(\"str(e) = {}\".format(str(e)))\n\n # This is important - We we previously returning a poorly formatted error string. Which was\n # litellm.BadRequestError: litellm.BadRequestError: Invalid Message passed in {'role': 'very-bad-role', 'content': 'hello'}\n\n # IMPORTANT ASSERTION\n assert (\n (str(e))\n == \"litellm.BadRequestError: Invalid Message passed in {'role': 'very-bad-role', 'content': 'hello'}\"\n )\n\n\n@pytest.mark.skip(reason=\"OpenAI exception changed to a generic error\")\ndef test_content_policy_exceptionimage_generation_openai():\n try:\n # this is ony a test - we needed some way to invoke the exception :(\n litellm._turn_on_debug()\n response = litellm.image_generation(\n prompt=\"where do i buy lethal drugs from\", model=\"dall-e-3\"\n )\n print(f\"response: {response}\")\n assert len(response.data) > 0\n except litellm.ContentPolicyViolationError as e:\n print(\"caught a content policy violation error! Passed\")\n pass\n except Exception as e:\n pytest.fail(f\"An exception occurred - {str(e)}\")\n\n\n# test_content_policy_exceptionimage_generation_openai()\n\n\ndef test_content_policy_violation_error_streaming():\n \"\"\"\n Production Test.\n \"\"\"\n litellm.set_verbose = False\n print(\"test_async_completion with stream\")\n\n async def test_get_response():\n try:\n response = await litellm.acompletion(\n model=\"azure/gpt-4.1-mini\",\n messages=[{\"role\": \"user\", \"content\": \"say 1\"}],\n temperature=0,\n top_p=1,\n stream=True,\n max_tokens=512,\n presence_penalty=0,\n frequency_penalty=0,\n )\n print(f\"response: {response}\")\n\n num_finish_reason = 0\n async for chunk in response:\n print(chunk)\n if chunk[\"choices\"][0].get(\"finish_reason\") is not None:\n num_finish_reason += 1\n print(\"finish_reason\", chunk[\"choices\"][0].get(\"finish_reason\"))\n\n assert (\n num_finish_reason == 1\n ), f\"expected only one finish reason. Got {num_finish_reason}\"\n except Exception as e:\n pytest.fail(f\"GOT exception for gpt-3.5 instruct In streaming{e}\")\n\n asyncio.run(test_get_response())\n\n async def test_get_error():\n try:\n response = await litellm.acompletion(\n model=\"azure/gpt-4.1-mini\",\n messages=[\n {\"role\": \"user\", \"content\": \"where do i buy lethal drugs from\"}\n ],\n temperature=0,\n top_p=1,\n stream=True,\n max_tokens=512,\n presence_penalty=0,\n frequency_penalty=0,\n mock_response=\"Exception: content_filter_policy\",\n )\n print(f\"response: {response}\")\n\n num_finish_reason = 0\n async for chunk in response:\n print(chunk)\n if chunk[\"choices\"][0].get(\"finish_reason\") is not None:\n num_finish_reason += 1\n print(\"finish_reason\", chunk[\"choices\"][0].get(\"finish_reason\"))\n\n pytest.fail(f\"Expected to return 400 error In streaming{e}\")\n except Exception as e:\n pass\n\n asyncio.run(test_get_error())\n\n\ndef test_completion_perplexity_exception_on_openai_client():\n try:\n import openai\n\n print(\"perplexity test\\n\\n\")\n litellm.set_verbose = False\n ## Test azure call\n old_azure_key = os.environ[\"PERPLEXITYAI_API_KEY\"]\n\n # delete perplexityai api key to simulate bad api key\n del os.environ[\"PERPLEXITYAI_API_KEY\"]\n\n # temporaily delete openai api key\n original_openai_key = os.environ[\"OPENAI_API_KEY\"]\n del os.environ[\"OPENAI_API_KEY\"]\n\n response = completion(\n model=\"perplexity/mistral-7b-instruct\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n os.environ[\"PERPLEXITYAI_API_KEY\"] = old_azure_key\n os.environ[\"OPENAI_API_KEY\"] = original_openai_key\n pytest.fail(\"Request should have failed - bad api key\")\n except openai.AuthenticationError as e:\n os.environ[\"PERPLEXITYAI_API_KEY\"] = old_azure_key\n os.environ[\"OPENAI_API_KEY\"] = original_openai_key\n print(\"exception: \", e)\n assert (\n \"The api_key client option must be set either by passing api_key to the client or by setting the PERPLEXITY_API_KEY environment variable\"\n in str(e)\n )\n except Exception as e:\n pytest.fail(f\"Error occurred: {e}\")\n\n\n# test_completion_perplexity_exception_on_openai_client()\n\n\ndef test_completion_perplexity_exception():\n try:\n import openai\n\n print(\"perplexity test\\n\\n\")\n litellm.set_verbose = True\n ## Test azure call\n old_azure_key = os.environ[\"PERPLEXITYAI_API_KEY\"]\n os.environ[\"PERPLEXITYAI_API_KEY\"] = \"good morning\"\n response = completion(\n model=\"perplexity/mistral-7b-instruct\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n os.environ[\"PERPLEXITYAI_API_KEY\"] = old_azure_key\n pytest.fail(\"Request should have failed - bad api key\")\n except openai.AuthenticationError as e:\n os.environ[\"PERPLEXITYAI_API_KEY\"] = old_azure_key\n print(\"exception: \", e)\n assert \"PerplexityException\" in str(e)\n except Exception as e:\n pytest.fail(f\"Error occurred: {e}\")\n\n\ndef test_completion_openai_api_key_exception():\n try:\n import openai\n\n print(\"gpt-3.5 test\\n\\n\")\n litellm.set_verbose = True\n ## Test azure call\n old_azure_key = os.environ[\"OPENAI_API_KEY\"]\n os.environ[\"OPENAI_API_KEY\"] = \"good morning\"\n response = completion(\n model=\"gpt-3.5-turbo\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n )\n os.environ[\"OPENAI_API_KEY\"] = old_azure_key\n pytest.fail(\"Request should have failed - bad api key\")\n except openai.AuthenticationError as e:\n os.environ[\"OPENAI_API_KEY\"] = old_azure_key\n print(\"exception: \", e)\n assert \"OpenAIException\" in str(e)\n except Exception as e:\n pytest.fail(f\"Error occurred: {e}\")\n\n\n# tesy_async_acompletion()\n\n\ndef test_router_completion_vertex_exception():\n try:\n import litellm\n\n litellm.set_verbose = True\n router = litellm.Router(\n model_list=[\n {\n \"model_name\": \"vertex-gemini-pro\",\n \"litellm_params\": {\n \"model\": \"vertex_ai/gemini-pro\",\n \"api_key\": \"good-morning\",\n },\n },\n ]\n )\n response = router.completion(\n model=\"vertex-gemini-pro\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n vertex_project=\"bad-project\",\n )\n pytest.fail(\"Request should have failed - bad api key\")\n except Exception as e:\n print(\"exception: \", e)\n\n\ndef test_litellm_completion_vertex_exception():\n try:\n import litellm\n\n litellm.set_verbose = True\n response = completion(\n model=\"vertex_ai/gemini-pro\",\n api_key=\"good-morning\",\n messages=[{\"role\": \"user\", \"content\": \"hello\"}],\n vertex_project=\"bad-project\",\n )\n pytest.fail(\"Request should have failed - bad api key\")\n except Exception as e:\n print(\"exception: \", e)\n\n\ndef test_litellm_predibase_exception():\n \"\"\"\n Test - Assert that the Predibase API Key is not returned on Authentication Errors\n \"\"\"\n try:\n import litellm\n\n litellm.set_verbose = True\n response = completion(\n model=\"predibase/llama-3-8b-instruct\",\n messages=[{\"role\": \"user\", \"content\": \"What is the meaning of life?\"}],\n tenant_id=\"c4768f95\",\n api_key=\"hf-rawapikey\",\n )\n pytest.fail(\"Request should have failed - bad api key\")\n except Exception as e:\n assert \"hf-rawapikey\" not in str(e)\n print(\"exception: \", e)\n\n\n# # test_invalid_request_error(model=\"command-nightly\")\n# # Test 3: Rate Limit Errors\n# def test_model_call(model):\n# try:\n# sample_text = \"how does a court case get to the Supreme Court?\"\n# messages = [{ \"content\": sample_text,\"role\": \"user\"}]\n# print(f\"model: {model}\")\n# response = completion(model=model, messages=messages)\n# except RateLimitError as e:\n# print(f\"headers: {e.response.headers}\")\n# return True\n# # except OpenAIError: # is at least an openai error -> in case of random model errors - e.g. overloaded server\n# # return True\n# except Exception as e:\n# print(f\"Uncaught Exception {model}: {type(e).__name__} - {e}\")\n# traceback.print_exc()\n# pass\n# return False\n# # Repeat each model 500 times\n# # extended_models = [model for model in models for _ in range(250)]\n# extended_models = [\"azure/gpt-4.1-mini\" for _ in range(250)]\n\n# def worker(model):\n# return test_model_call(model)\n\n# # Create a dictionary to store the results\n# counts = {True: 0, False: 0}\n\n# # Use Thread Pool Executor\n# with ThreadPoolExecutor(max_workers=500) as executor:\n# # Use map to start the operation in thread pool\n# results = executor.map(worker, extended_models)\n\n# # Iterate over results and count True/False\n# for result in results:\n# counts[result] += 1\n\n# accuracy_score = counts[True]/(counts[True] + counts[False])\n# print(f\"accuracy_score: {accuracy_score}\")\n\n\n@pytest.mark.parametrize(\n \"provider\",\n [\n \"predibase\",\n \"vertex_ai_beta\",\n \"anthropic\",\n \"databricks\",\n \"watsonx\",\n \"fireworks_ai\",\n ],\n)\ndef test_exception_mapping(provider):\n \"\"\"\n For predibase, run through a set of mock exceptions\n\n assert that they are being mapped correctly\n \"\"\"\n litellm.set_verbose = True\n error_map = {\n 400: litellm.BadRequestError,\n 401: litellm.AuthenticationError,\n 404: litellm.NotFoundError,\n 408: litellm.Timeout,\n 429: litellm.RateLimitError,\n 500: litellm.InternalServerError,\n 503: litellm.ServiceUnavailableError,\n }\n\n for code, expected_exception in error_map.items():\n mock_response = Exception()\n setattr(mock_response, \"text\", \"This is an error message\")\n setattr(mock_response, \"llm_provider\", provider)\n setattr(mock_response, \"status_code\", code)\n\n response: Any = None\n try:\n response = completion(\n model=\"{}/test-model\".format(provider),\n messages=[{\"role\": \"user\", \"content\": \"Hey, how's it going?\"}],\n mock_response=mock_response,\n )\n except expected_exception:\n continue\n except Exception as e:\n traceback.print_exc()\n response = \"{}\".format(str(e))\n pytest.fail(\n \"Did not raise expected exception. Expected={}, Return={},\".format(\n expected_exception, response\n )\n )\n\n pass\n\n\ndef test_fireworks_ai_exception_mapping():\n \"\"\"\n Comprehensive test for Fireworks AI exception mapping, including:\n 1. Standard 429 rate limit errors\n 2. Text-based rate limit detection (the main issue fixed)\n 3. Generic 400 errors that should NOT be rate limits\n 4. ExceptionCheckers utility function\n\n Related to: https://github.com/BerriAI/litellm/pull/11455\n Based on Fireworks AI documentation: https://docs.fireworks.ai/tools-sdks/python-client/api-reference\n \"\"\"\n import litellm\n from litellm.llms.fireworks_ai.common_utils import FireworksAIException\n from litellm.litellm_core_utils.exception_mapping_utils import ExceptionCheckers\n\n # Test scenarios covering all important cases\n test_scenarios = [\n {\n \"name\": \"Standard 429 rate limit with proper status code\",\n \"status_code\": 429,\n \"message\": \"Rate limit exceeded. Please try again in 60 seconds.\",\n \"expected_exception\": litellm.RateLimitError,\n },\n {\n \"name\": \"Status 400 with rate limit text (the main issue fixed)\",\n \"status_code\": 400,\n \"message\": '{\"error\":{\"object\":\"error\",\"type\":\"invalid_request_error\",\"message\":\"rate limit exceeded, please try again later\"}}',\n \"expected_exception\": litellm.RateLimitError,\n },\n {\n \"name\": \"Status 400 with generic invalid request (should NOT be rate limit)\",\n \"status_code\": 400,\n \"message\": '{\"error\":{\"type\":\"invalid_request_error\",\"message\":\"Invalid parameter value\"}}',\n \"expected_exception\": litellm.BadRequestError,\n },\n ]\n\n # Test each scenario\n for scenario in test_scenarios:\n mock_exception = FireworksAIException(\n status_code=scenario[\"status_code\"], message=scenario[\"message\"], headers={}\n )\n\n try:\n response = litellm.completion(\n model=\"fireworks_ai/llama-v3p1-70b-instruct\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n mock_response=mock_exception,\n )\n pytest.fail(\n f\"Expected {scenario['expected_exception'].__name__} to be raised\"\n )\n except scenario[\"expected_exception\"] as e:\n if scenario[\"expected_exception\"] == litellm.RateLimitError:\n assert \"rate limit\" in str(e).lower() or \"429\" in str(e)\n except Exception as e:\n pytest.fail(\n f\"Expected {scenario['expected_exception'].__name__} but got {type(e).__name__}: {e}\"\n )\n\n # Test ExceptionCheckers.is_error_str_rate_limit() method directly\n\n # Test cases that should return True (rate limit detected)\n rate_limit_strings = [\n \"429 rate limit exceeded\",\n \"Rate limit exceeded, please try again later\",\n \"RATE LIMIT ERROR\",\n \"Error 429: rate limit\",\n '{\"error\":{\"type\":\"invalid_request_error\",\"message\":\"rate limit exceeded, please try again later\"}}',\n \"HTTP 429 Too Many Requests\",\n ]\n\n for error_str in rate_limit_strings:\n assert ExceptionCheckers.is_error_str_rate_limit(\n error_str\n ), f\"Should detect rate limit in: {error_str}\"\n\n # Test cases that should return False (not rate limit)\n non_rate_limit_strings = [\n \"400 Bad Request\",\n \"Authentication failed\",\n \"Invalid model specified\",\n \"Context window exceeded\",\n \"Internal server error\",\n \"\",\n \"Some other error message\",\n ]\n\n for error_str in non_rate_limit_strings:\n assert not ExceptionCheckers.is_error_str_rate_limit(\n error_str\n ), f\"Should NOT detect rate limit in: {error_str}\"\n\n # Test edge cases\n assert not ExceptionCheckers.is_error_str_rate_limit(None) # type: ignore\n assert not ExceptionCheckers.is_error_str_rate_limit(42) # type: ignore\n\n\ndef test_anthropic_tool_calling_exception():\n \"\"\"\n Related - https://github.com/BerriAI/litellm/issues/4348\n \"\"\"\n tools = [\n {\n \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather in a given location\",\n \"parameters\": {},\n },\n }\n ]\n try:\n litellm.completion(\n model=\"claude-haiku-4-5-20251001\",\n messages=[{\"role\": \"user\", \"content\": \"Hey, how's it going?\"}],\n tools=tools,\n )\n except litellm.BadRequestError:\n pass\n\n\nfrom typing import Optional, Union\n\nfrom openai import AsyncOpenAI, OpenAI\n\n\ndef _pre_call_utils(\n call_type: str,\n data: dict,\n client: Union[OpenAI, AsyncOpenAI],\n sync_mode: bool,\n streaming: Optional[bool],\n):\n if call_type == \"embedding\":\n data[\"input\"] = \"Hello world!\"\n mapped_target: Any = client.embeddings.with_raw_response\n if sync_mode:\n original_function = litellm.embedding\n else:\n original_function = litellm.aembedding\n elif call_type == \"chat_completion\":\n data[\"messages\"] = [{\"role\": \"user\", \"content\": \"Hello world\"}]\n if streaming is True:\n data[\"stream\"] = True\n mapped_target = client.chat.completions.with_raw_response # type: ignore\n if sync_mode:\n original_function = litellm.completion\n else:\n original_function = litellm.acompletion\n elif call_type == \"completion\":\n data[\"prompt\"] = \"Hello world\"\n if streaming is True:\n data[\"stream\"] = True\n mapped_target = client.completions.with_raw_response # type: ignore\n if sync_mode:\n original_function = litellm.text_completion\n else:\n original_function = litellm.atext_completion\n\n return data, original_function, mapped_target\n\n\ndef _pre_call_utils_httpx(\n call_type: str,\n data: dict,\n client: Union[HTTPHandler, AsyncHTTPHandler],\n sync_mode: bool,\n streaming: Optional[bool],\n):\n mapped_target: Any = client.client\n if call_type == \"embedding\":\n data[\"input\"] = \"Hello world!\"\n\n if sync_mode:\n original_function = litellm.embedding\n else:\n original_function = litellm.aembedding\n elif call_type == \"chat_completion\":\n data[\"messages\"] = [{\"role\": \"user\", \"content\": \"Hello world\"}]\n if streaming is True:\n data[\"stream\"] = True\n\n if sync_mode:\n original_function = litellm.completion\n else:\n original_function = litellm.acompletion\n elif call_type == \"completion\":\n data[\"prompt\"] = \"Hello world\"\n if streaming is True:\n data[\"stream\"] = True\n if sync_mode:\n original_function = litellm.text_completion\n else:\n original_function = litellm.atext_completion\n\n return data, original_function, mapped_target\n\n\n@pytest.mark.parametrize(\n \"sync_mode\",\n [True, False],\n)\n@pytest.mark.parametrize(\n \"provider, model, call_type, streaming\",\n [\n (\"openai\", \"text-embedding-ada-002\", \"embedding\", None),\n (\"openai\", \"gpt-3.5-turbo\", \"chat_completion\", False),\n (\"openai\", \"gpt-3.5-turbo\", \"chat_completion\", True),\n (\"openai\", \"gpt-3.5-turbo-instruct\", \"completion\", True),\n (\"azure\", \"azure/gpt-4.1-mini\", \"chat_completion\", True),\n (\"azure\", \"azure/text-embedding-ada-002\", \"embedding\", True),\n (\"azure\", \"azure_text/gpt-3.5-turbo-instruct\", \"completion\", True),\n ],\n)\n@pytest.mark.asyncio\nasync def test_exception_with_headers(sync_mode, provider, model, call_type, streaming):\n \"\"\"\n User feedback: litellm says \"No deployments available for selected model, Try again in 60 seconds\"\n but Azure says to retry in at most 9s\n\n ```\n {\"message\": \"litellm.proxy.proxy_server.embeddings(): Exception occured - No deployments available for selected model, Try again in 60 seconds. Passed model=text-embedding-ada-002. pre-call-checks=False, allowed_model_region=n/a, cooldown_list=[('b49cbc9314273db7181fe69b1b19993f04efb88f2c1819947c538bac08097e4c', {'Exception Received': 'litellm.RateLimitError: AzureException RateLimitError - Requests to the Embeddings_Create Operation under Azure OpenAI API version 2023-09-01-preview have exceeded call rate limit of your current OpenAI S0 pricing tier. Please retry after 9 seconds. Please go here: https://aka.ms/oai/quotaincrease if you would like to further increase the default rate limit.', 'Status Code': '429'})]\", \"level\": \"ERROR\", \"timestamp\": \"2024-08-22T03:25:36.900476\"}\n ```\n \"\"\"\n print(f\"Received args: {locals()}\")\n import openai\n\n if sync_mode:\n if provider == \"openai\":\n openai_client = openai.OpenAI(api_key=\"\")\n elif provider == \"azure\":\n openai_client = openai.AzureOpenAI(\n api_key=\"\", base_url=\"\", api_version=litellm.AZURE_DEFAULT_API_VERSION\n )\n else:\n if provider == \"openai\":\n openai_client = openai.AsyncOpenAI(api_key=\"\")\n elif provider == \"azure\":\n openai_client = openai.AsyncAzureOpenAI(\n api_key=\"\", base_url=\"\", api_version=litellm.AZURE_DEFAULT_API_VERSION\n )\n\n data = {\"model\": model}\n data, original_function, mapped_target = _pre_call_utils(\n call_type=call_type,\n data=data,\n client=openai_client,\n sync_mode=sync_mode,\n streaming=streaming,\n )\n\n cooldown_time = 30.0\n\n def _return_exception(*args, **kwargs):\n import datetime\n\n from httpx import Headers, Request, Response\n\n kwargs = {\n \"request\": Request(\"POST\", \"https://www.google.com\"),\n \"message\": \"Error code: 429 - Rate Limit Error!\",\n \"body\": {\"detail\": \"Rate Limit Error!\"},\n \"code\": None,\n \"param\": None,\n \"type\": None,\n \"response\": Response(\n status_code=429,\n headers=Headers(\n {\n \"date\": \"Sat, 21 Sep 2024 22:56:53 GMT\",\n \"server\": \"uvicorn\",\n \"retry-after\": \"30\",\n \"content-length\": \"30\",\n \"content-type\": \"application/json\",\n }\n ),\n request=Request(\"POST\", \"http://0.0.0.0:9000/chat/completions\"),\n ),\n \"status_code\": 429,\n \"request_id\": None,\n }\n\n exception = Exception()\n for k, v in kwargs.items():\n setattr(exception, k, v)\n raise exception\n\n with patch.object(\n mapped_target,\n \"create\",\n side_effect=_return_exception,\n ):\n new_retry_after_mock_client = MagicMock(return_value=-1)\n\n litellm.utils._get_retry_after_from_exception_header = (\n new_retry_after_mock_client\n )\n\n exception_raised = False\n try:\n if sync_mode:\n resp = original_function(**data, client=openai_client)\n if streaming:\n for chunk in resp:\n continue\n else:\n resp = await original_function(**data, client=openai_client)\n\n if streaming:\n async for chunk in resp:\n continue\n\n except litellm.RateLimitError as e:\n exception_raised = True\n assert e.litellm_response_headers is not None\n assert int(e.litellm_response_headers[\"retry-after\"]) == cooldown_time\n\n if exception_raised is False:\n print(resp)\n assert exception_raised\n\n\ndef test_openai_gateway_timeout_error():\n \"\"\"\n Test that the OpenAI gateway timeout error is raised\n \"\"\"\n openai_client = OpenAI()\n mapped_target = openai_client.chat.completions.with_raw_response # type: ignore\n\n def _return_exception(*args, **kwargs):\n import datetime\n\n from httpx import Headers, Request, Response\n\n kwargs = {\n \"request\": Request(\"POST\", \"https://www.google.com\"),\n \"message\": \"Error code: 504 - Gateway Timeout Error!\",\n \"body\": {\"detail\": \"Gateway Timeout Error!\"},\n \"code\": None,\n \"param\": None,\n \"type\": None,\n \"response\": Response(\n status_code=504,\n headers=Headers(\n {\n \"date\": \"Sat, 21 Sep 2024 22:56:53 GMT\",\n \"server\": \"uvicorn\",\n \"content-length\": \"30\",\n \"content-type\": \"application/json\",\n }\n ),\n request=Request(\"POST\", \"http://0.0.0.0:9000/chat/completions\"),\n ),\n \"status_code\": 504,\n \"request_id\": None,\n }\n\n exception = Exception()\n for k, v in kwargs.items():\n setattr(exception, k, v)\n raise exception\n\n try:\n with patch.object(\n mapped_target,\n \"create\",\n side_effect=_return_exception,\n ):\n litellm.completion(\n model=\"openai/gpt-3.5-turbo\",\n messages=[{\"role\": \"user\", \"content\": \"Hello world\"}],\n client=openai_client,\n )\n pytest.fail(\"Expected to raise Timeout\")\n except litellm.Timeout as e:\n assert e.status_code == 504\n\n\n@pytest.mark.parametrize(\n \"sync_mode\",\n [True, False],\n)\n@pytest.mark.parametrize(\"streaming\", [True, False])\n@pytest.mark.parametrize(\n \"provider, model, call_type\",\n [\n (\"anthropic\", \"claude-3-haiku-20240307\", \"chat_completion\"),\n ],\n)\n@pytest.mark.asyncio\nasync def test_exception_with_headers_httpx(\n sync_mode, provider, model, call_type, streaming\n):\n \"\"\"\n User feedback: litellm says \"No deployments available for selected model, Try again in 60 seconds\"\n but Azure says to retry in at most 9s\n\n ```\n {\"message\": \"litellm.proxy.proxy_server.embeddings(): Exception occured - No deployments available for selected model, Try again in 60 seconds. Passed model=text-embedding-ada-002. pre-call-checks=False, allowed_model_region=n/a, cooldown_list=[('b49cbc9314273db7181fe69b1b19993f04efb88f2c1819947c538bac08097e4c', {'Exception Received': 'litellm.RateLimitError: AzureException RateLimitError - Requests to the Embeddings_Create Operation under Azure OpenAI API version 2023-09-01-preview have exceeded call rate limit of your current OpenAI S0 pricing tier. Please retry after 9 seconds. Please go here: https://aka.ms/oai/quotaincrease if you would like to further increase the default rate limit.', 'Status Code': '429'})]\", \"level\": \"ERROR\", \"timestamp\": \"2024-08-22T03:25:36.900476\"}\n ```\n \"\"\"\n print(f\"Received args: {locals()}\")\n import openai\n\n if sync_mode:\n client = HTTPHandler()\n else:\n client = AsyncHTTPHandler()\n\n data = {\"model\": model}\n data, original_function, mapped_target = _pre_call_utils_httpx(\n call_type=call_type,\n data=data,\n client=client,\n sync_mode=sync_mode,\n streaming=streaming,\n )\n\n cooldown_time = 30.0\n\n def _return_exception(*args, **kwargs):\n import datetime\n\n from httpx import Headers, HTTPStatusError, Request, Response\n\n # Create the Request object\n request = Request(\"POST\", \"http://0.0.0.0:9000/chat/completions\")\n\n # Create the Response object with the necessary headers and status code\n response = Response(\n status_code=429,\n headers=Headers(\n {\n \"date\": \"Sat, 21 Sep 2024 22:56:53 GMT\",\n \"server\": \"uvicorn\",\n \"retry-after\": \"30\",\n \"content-length\": \"30\",\n \"content-type\": \"application/json\",\n }\n ),\n request=request,\n )\n\n # Create and raise the HTTPStatusError exception\n raise HTTPStatusError(\n message=\"Error code: 429 - Rate Limit Error!\",\n request=request,\n response=response,\n )\n\n with patch.object(\n mapped_target,\n \"send\",\n side_effect=_return_exception,\n ):\n new_retry_after_mock_client = MagicMock(return_value=-1)\n\n litellm.utils._get_retry_after_from_exception_header = (\n new_retry_after_mock_client\n )\n\n exception_raised = False\n try:\n if sync_mode:\n resp = original_function(**data, client=client)\n if streaming:\n for chunk in resp:\n continue\n else:\n resp = await original_function(**data, client=client)\n\n if streaming:\n async for chunk in resp:\n continue\n\n except litellm.RateLimitError as e:\n exception_raised = True\n assert (\n e.litellm_response_headers is not None\n ), \"litellm_response_headers is None\"\n print(\"e.litellm_response_headers\", e.litellm_response_headers)\n assert int(e.litellm_response_headers[\"retry-after\"]) == cooldown_time\n\n if exception_raised is False:\n print(resp)\n assert exception_raised\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"model\", [\"azure/gpt-4.1-mini\", \"openai/gpt-3.5-turbo\"])\nasync def test_bad_request_error_contains_httpx_response(model):\n \"\"\"\n Test that the BadRequestError contains the httpx response\n\n Relevant issue: https://github.com/BerriAI/litellm/issues/6732\n \"\"\"\n try:\n await litellm.acompletion(\n model=model,\n messages=[{\"role\": \"user\", \"content\": \"Hello world\"}],\n bad_arg=\"bad_arg\",\n )\n pytest.fail(\"Expected to raise BadRequestError\")\n except litellm.BadRequestError as e:\n print(\"e.response\", e.response)\n print(\"vars(e.response)\", vars(e.response))\n assert e.response is not None\n\n\ndef test_exceptions_base_class():\n try:\n raise litellm.RateLimitError(\n message=\"BedrockException: Rate Limit Error\",\n model=\"model\",\n llm_provider=\"bedrock\",\n )\n except litellm.RateLimitError as e:\n assert isinstance(e, litellm.RateLimitError)\n assert e.code == \"429\"\n assert e.type == \"throttling_error\"\n\n\ndef test_context_window_exceeded_error_from_litellm_proxy():\n from httpx import Response\n from litellm.litellm_core_utils.exception_mapping_utils import (\n extract_and_raise_litellm_exception,\n )\n\n args = {\n \"response\": Response(status_code=400, text=\"Bad Request\"),\n \"error_str\": \"Error code: 400 - {'error': {'message': \\\"litellm.ContextWindowExceededError: litellm.BadRequestError: this is a mock context window exceeded error\\\\nmodel=gpt-3.5-turbo. context_window_fallbacks=None. fallbacks=None.\\\\n\\\\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks\\\\nReceived Model Group=gpt-3.5-turbo\\\\nAvailable Model Group Fallbacks=None\\\", 'type': None, 'param': None, 'code': '400'}}\",\n \"model\": \"gpt-3.5-turbo\",\n \"custom_llm_provider\": \"litellm_proxy\",\n }\n with pytest.raises(litellm.ContextWindowExceededError):\n extract_and_raise_litellm_exception(**args)\n\n\ndef test_bad_request_error_with_response_without_request():\n \"\"\"\n Test that BadRequestError handles Response objects without a request attribute.\n\n This simulates a real scenario where a Response is created without a request\n (e.g., in tests or when manually creating error responses), and we need to\n ensure it doesn't raise RuntimeError when the exception is created.\n \"\"\"\n from httpx import Response\n from litellm.litellm_core_utils.exception_mapping_utils import (\n extract_and_raise_litellm_exception,\n )\n\n # Create a Response without a request (simulates the scenario that was failing)\n response_without_request = Response(status_code=400, text=\"Bad Request\")\n\n # Test that extract_and_raise_litellm_exception can handle this\n args = {\n \"response\": response_without_request,\n \"error_str\": \"Error code: 400 - {'error': {'message': 'litellm.BadRequestError: Invalid request parameters', 'type': None, 'param': None, 'code': '400'}}\",\n \"model\": \"gpt-3.5-turbo\",\n \"custom_llm_provider\": \"openai\",\n }\n\n # This should raise BadRequestError without RuntimeError\n with pytest.raises(litellm.BadRequestError) as exc_info:\n extract_and_raise_litellm_exception(**args)\n\n # Verify the exception was created successfully\n error = exc_info.value\n assert error is not None\n assert error.model == \"gpt-3.5-turbo\"\n assert error.llm_provider == \"openai\"\n\n # Verify the exception has a response (should be minimal error response)\n assert error.response is not None\n # The response should have a request (minimal error response has one)\n assert getattr(error.response, \"_request\", None) is not None\n # Should be able to access request property without RuntimeError\n assert error.response.request is not None\n\n\n@pytest.mark.parametrize(\"sync_mode\", [True, False])\n@pytest.mark.parametrize(\"stream_mode\", [True, False])\n@pytest.mark.parametrize(\"model\", [\"gpt-4.1-nano\"]) # \"gpt-4o-mini\",\n@pytest.mark.asyncio\nasync def test_exception_bubbling_up(sync_mode, stream_mode, model):\n \"\"\"\n make sure code, param, and type are bubbled up\n \"\"\"\n import litellm\n\n litellm.set_verbose = True\n with pytest.raises(Exception) as exc_info:\n if sync_mode:\n litellm.completion(\n model=model,\n messages=[{\"role\": \"usera\", \"content\": \"hi\"}],\n stream=stream_mode,\n sync_stream=sync_mode,\n )\n else:\n await litellm.acompletion(\n model=model,\n messages=[{\"role\": \"usera\", \"content\": \"hi\"}],\n stream=stream_mode,\n sync_stream=sync_mode,\n )\n\n assert exc_info.value.code == \"invalid_value\"\n assert exc_info.value.param is not None\n assert exc_info.value.type == \"invalid_request_error\"\n"}}}} |