hack/tools/scripts/llm-key-hunter/results/_cache/content/aa-turner_top-pypi-sdists-2000.json

{"repo": "aa-turner/top-pypi-sdists-2000", "files": {"extracted/br/browser-use/browser_use/llm/vercel/chat.py": {"0c60adfb96c74a3afedf3eaadc0a6b52b7f55e92": {"t": 1785596051.079521, "n": 21593, "h": "3eb81402b7e8bd9d", "c": "import json\nimport os\nfrom collections.abc import Mapping\nfrom dataclasses import dataclass, field\nfrom typing import Any, Literal, TypeAlias, TypeVar, overload\n\nimport httpx\nfrom openai import APIConnectionError, APIStatusError, AsyncOpenAI, RateLimitError\nfrom openai.types.chat.chat_completion import ChatCompletion\nfrom openai.types.shared_params.response_format_json_schema import (\n\tJSONSchema,\n\tResponseFormatJSONSchema,\n)\nfrom pydantic import BaseModel\n\nfrom browser_use.llm.base import BaseChatModel\nfrom browser_use.llm.exceptions import ModelProviderError, ModelRateLimitError\nfrom browser_use.llm.messages import BaseMessage, ContentPartTextParam, SystemMessage\nfrom browser_use.llm.schema import SchemaOptimizer\nfrom browser_use.llm.vercel.serializer import VercelMessageSerializer\nfrom browser_use.llm.views import ChatInvokeCompletion, ChatInvokeUsage\n\nT = TypeVar('T', bound=BaseModel)\n\nChatVercelModel: TypeAlias = Literal[\n\t'alibaba/qwen-3-14b',\n\t'alibaba/qwen-3-235b',\n\t'alibaba/qwen-3-30b',\n\t'alibaba/qwen-3-32b',\n\t'alibaba/qwen3-235b-a22b-thinking',\n\t'alibaba/qwen3-coder',\n\t'alibaba/qwen3-coder-30b-a3b',\n\t'alibaba/qwen3-coder-next',\n\t'alibaba/qwen3-coder-plus',\n\t'alibaba/qwen3-embedding-0.6b',\n\t'alibaba/qwen3-embedding-4b',\n\t'alibaba/qwen3-embedding-8b',\n\t'alibaba/qwen3-max',\n\t'alibaba/qwen3-max-preview',\n\t'alibaba/qwen3-max-thinking',\n\t'alibaba/qwen3-next-80b-a3b-instruct',\n\t'alibaba/qwen3-next-80b-a3b-thinking',\n\t'alibaba/qwen3-vl-instruct',\n\t'alibaba/qwen3-vl-thinking',\n\t'alibaba/qwen3.5-flash',\n\t'alibaba/qwen3.5-plus',\n\t'alibaba/wan-v2.5-t2v-preview',\n\t'alibaba/wan-v2.6-i2v',\n\t'alibaba/wan-v2.6-i2v-flash',\n\t'alibaba/wan-v2.6-r2v',\n\t'alibaba/wan-v2.6-r2v-flash',\n\t'alibaba/wan-v2.6-t2v',\n\t'amazon/nova-2-lite',\n\t'amazon/nova-lite',\n\t'amazon/nova-micro',\n\t'amazon/nova-pro',\n\t'amazon/titan-embed-text-v2',\n\t'anthropic/claude-3-haiku',\n\t'anthropic/claude-3-opus',\n\t'anthropic/claude-3.5-haiku',\n\t'anthropic/claude-3.5-sonnet',\n\t'anthropic/claude-3.5-sonnet-20240620',\n\t'anthropic/claude-3.7-sonnet',\n\t'anthropic/claude-fable-5',\n\t'anthropic/claude-haiku-4.5',\n\t'anthropic/claude-opus-4',\n\t'anthropic/claude-opus-4.1',\n\t'anthropic/claude-opus-4.5',\n\t'anthropic/claude-opus-4.6',\n\t'anthropic/claude-sonnet-4',\n\t'anthropic/claude-sonnet-4.5',\n\t'anthropic/claude-sonnet-4.6',\n\t'arcee-ai/trinity-large-preview',\n\t'arcee-ai/trinity-mini',\n\t'bfl/flux-kontext-max',\n\t'bfl/flux-kontext-pro',\n\t'bfl/flux-pro-1.0-fill',\n\t'bfl/flux-pro-1.1',\n\t'bfl/flux-pro-1.1-ultra',\n\t'bytedance/seed-1.6',\n\t'bytedance/seed-1.8',\n\t'bytedance/seedance-v1.0-lite-i2v',\n\t'bytedance/seedance-v1.0-lite-t2v',\n\t'bytedance/seedance-v1.0-pro',\n\t'bytedance/seedance-v1.0-pro-fast',\n\t'bytedance/seedance-v1.5-pro',\n\t'cohere/command-a',\n\t'cohere/embed-v4.0',\n\t'deepseek/deepseek-r1',\n\t'deepseek/deepseek-v3',\n\t'deepseek/deepseek-v3.1',\n\t'deepseek/deepseek-v3.1-terminus',\n\t'deepseek/deepseek-v3.2',\n\t'deepseek/deepseek-v3.2-thinking',\n\t'google/gemini-2.0-flash',\n\t'google/gemini-2.0-flash-lite',\n\t'google/gemini-2.5-flash',\n\t'google/gemini-2.5-flash-image',\n\t'google/gemini-2.5-flash-lite',\n\t'google/gemini-2.5-flash-lite-preview-09-2025',\n\t'google/gemini-2.5-flash-preview-09-2025',\n\t'google/gemini-2.5-pro',\n\t'google/gemini-3-flash',\n\t'google/gemini-3-pro-image',\n\t'google/gemini-3-pro-preview',\n\t'google/gemini-3.1-flash-image-preview',\n\t'google/gemini-3.1-flash-lite-preview',\n\t'google/gemini-3.1-pro-preview',\n\t'google/gemini-embedding-001',\n\t'google/imagen-4.0-fast-generate-001',\n\t'google/imagen-4.0-generate-001',\n\t'google/imagen-4.0-ultra-generate-001',\n\t'google/text-embedding-005',\n\t'google/text-multilingual-embedding-002',\n\t'google/veo-3.0-fast-generate-001',\n\t'google/veo-3.0-generate-001',\n\t'google/veo-3.1-fast-generate-001',\n\t'google/veo-3.1-generate-001',\n\t'inception/mercury-2',\n\t'inception/mercury-coder-small',\n\t'klingai/kling-v2.5-turbo-i2v',\n\t'klingai/kling-v2.5-turbo-t2v',\n\t'klingai/kling-v2.6-i2v',\n\t'klingai/kling-v2.6-motion-control',\n\t'klingai/kling-v2.6-t2v',\n\t'klingai/kling-v3.0-i2v',\n\t'klingai/kling-v3.0-t2v',\n\t'kwaipilot/kat-coder-pro-v1',\n\t'meituan/longcat-flash-chat',\n\t'meituan/longcat-flash-thinking',\n\t'meta/llama-3.1-70b',\n\t'meta/llama-3.1-8b',\n\t'meta/llama-3.2-11b',\n\t'meta/llama-3.2-1b',\n\t'meta/llama-3.2-3b',\n\t'meta/llama-3.2-90b',\n\t'meta/llama-3.3-70b',\n\t'meta/llama-4-maverick',\n\t'meta/llama-4-scout',\n\t'minimax/minimax-m2',\n\t'minimax/minimax-m2.1',\n\t'minimax/minimax-m2.1-lightning',\n\t'minimax/minimax-m2.5',\n\t'minimax/minimax-m2.5-highspeed',\n\t'mistral/codestral',\n\t'mistral/codestral-embed',\n\t'mistral/devstral-2',\n\t'mistral/devstral-small',\n\t'mistral/devstral-small-2',\n\t'mistral/magistral-medium',\n\t'mistral/magistral-small',\n\t'mistral/ministral-14b',\n\t'mistral/ministral-3b',\n\t'mistral/ministral-8b',\n\t'mistral/mistral-embed',\n\t'mistral/mistral-large-3',\n\t'mistral/mistral-medium',\n\t'mistral/mistral-nemo',\n\t'mistral/mistral-small',\n\t'mistral/mixtral-8x22b-instruct',\n\t'mistral/pixtral-12b',\n\t'mistral/pixtral-large',\n\t'moonshotai/kimi-k2',\n\t'moonshotai/kimi-k2-0905',\n\t'moonshotai/kimi-k2-thinking',\n\t'moonshotai/kimi-k2-thinking-turbo',\n\t'moonshotai/kimi-k2-turbo',\n\t'moonshotai/kimi-k2.5',\n\t'morph/morph-v3-fast',\n\t'morph/morph-v3-large',\n\t'nvidia/nemotron-3-nano-30b-a3b',\n\t'nvidia/nemotron-nano-12b-v2-vl',\n\t'nvidia/nemotron-nano-9b-v2',\n\t'openai/gpt-3.5-turbo',\n\t'openai/gpt-3.5-turbo-instruct',\n\t'openai/gpt-4-turbo',\n\t'openai/gpt-4.1',\n\t'openai/gpt-4.1-mini',\n\t'openai/gpt-4.1-nano',\n\t'openai/gpt-4o',\n\t'openai/gpt-4o-mini',\n\t'openai/gpt-4o-mini-search-preview',\n\t'openai/gpt-5',\n\t'openai/gpt-5-chat',\n\t'openai/gpt-5-codex',\n\t'openai/gpt-5-mini',\n\t'openai/gpt-5-nano',\n\t'openai/gpt-5-pro',\n\t'openai/gpt-5.1-codex',\n\t'openai/gpt-5.1-codex-max',\n\t'openai/gpt-5.1-codex-mini',\n\t'openai/gpt-5.1-instant',\n\t'openai/gpt-5.1-thinking',\n\t'openai/gpt-5.2',\n\t'openai/gpt-5.2-chat',\n\t'openai/gpt-5.2-codex',\n\t'openai/gpt-5.2-pro',\n\t'openai/gpt-5.3-chat',\n\t'openai/gpt-5.3-codex',\n\t'openai/gpt-5.4',\n\t'openai/gpt-5.4-pro',\n\t'openai/gpt-image-1',\n\t'openai/gpt-image-1-mini',\n\t'openai/gpt-image-1.5',\n\t'openai/gpt-oss-120b',\n\t'openai/gpt-oss-20b',\n\t'openai/gpt-oss-safeguard-20b',\n\t'openai/o1',\n\t'openai/o3',\n\t'openai/o3-deep-research',\n\t'openai/o3-mini',\n\t'openai/o3-pro',\n\t'openai/o4-mini',\n\t'openai/text-embedding-3-large',\n\t'openai/text-embedding-3-small',\n\t'openai/text-embedding-ada-002',\n\t'perplexity/sonar',\n\t'perplexity/sonar-pro',\n\t'perplexity/sonar-reasoning',\n\t'perplexity/sonar-reasoning-pro',\n\t'prime-intellect/intellect-3',\n\t'recraft/recraft-v2',\n\t'recraft/recraft-v3',\n\t'recraft/recraft-v4',\n\t'recraft/recraft-v4-pro',\n\t'stealth/sonoma-dusk-alpha',\n\t'stealth/sonoma-sky-alpha',\n\t'vercel/v0-1.0-md',\n\t'vercel/v0-1.5-md',\n\t'voyage/voyage-3-large',\n\t'voyage/voyage-3.5',\n\t'voyage/voyage-3.5-lite',\n\t'voyage/voyage-4',\n\t'voyage/voyage-4-large',\n\t'voyage/voyage-4-lite',\n\t'voyage/voyage-code-2',\n\t'voyage/voyage-code-3',\n\t'voyage/voyage-finance-2',\n\t'voyage/voyage-law-2',\n\t'xai/grok-2-vision',\n\t'xai/grok-3',\n\t'xai/grok-3-fast',\n\t'xai/grok-3-mini',\n\t'xai/grok-3-mini-fast',\n\t'xai/grok-4',\n\t'xai/grok-4-fast-non-reasoning',\n\t'xai/grok-4-fast-reasoning',\n\t'xai/grok-4.1-fast-non-reasoning',\n\t'xai/grok-4.1-fast-reasoning',\n\t'xai/grok-4.20-multi-agent-beta',\n\t'xai/grok-4.20-non-reasoning-beta',\n\t'xai/grok-4.20-reasoning-beta',\n\t'xai/grok-code-fast-1',\n\t'xai/grok-imagine-image',\n\t'xai/grok-imagine-image-pro',\n\t'xai/grok-imagine-video',\n\t'xiaomi/mimo-v2-flash',\n\t'zai/glm-4.5',\n\t'zai/glm-4.5-air',\n\t'zai/glm-4.5v',\n\t'zai/glm-4.6',\n\t'zai/glm-4.6v',\n\t'zai/glm-4.6v-flash',\n\t'zai/glm-4.7',\n\t'zai/glm-4.7-flashx',\n\t'zai/glm-5',\n]\n\n\n@dataclass\nclass ChatVercel(BaseChatModel):\n\t\"\"\"\n\tA wrapper around Vercel AI Gateway's API, which provides OpenAI-compatible access\n\tto various LLM models with features like rate limiting, caching, and monitoring.\n\n\tExamples:\n\t\t```python\n\t        from browser_use import Agent, ChatVercel\n\n\t        llm = ChatVercel(model='openai/gpt-4o', api_key='your_vercel_api_key')\n\n\t        agent = Agent(task='Your task here', llm=llm)\n\t\t```\n\n\tArgs:\n\t    model: The model identifier\n\t    api_key: Your Vercel AI Gateway API key. If not provided, falls back to\n\t        AI_GATEWAY_API_KEY or VERCEL_OIDC_TOKEN environment variables.\n\t    base_url: The Vercel AI Gateway endpoint (defaults to https://ai-gateway.vercel.sh/v1)\n\t    temperature: Sampling temperature (0-2)\n\t    max_tokens: Maximum tokens to generate\n\t    reasoning_models: List of reasoning model patterns (e.g., 'o1', 'gpt-oss') that need\n\t        prompt-based JSON extraction. Auto-detects common reasoning models by default.\n\t    timeout: Request timeout in seconds\n\t    max_retries: Maximum number of retries for failed requests\n\t    provider_options: Provider routing options for the gateway. Use this to control which\n\t        providers are used and in what order. Example: {'gateway': {'order': ['vertex', 'anthropic']}}\n\t    reasoning: Optional provider-specific reasoning configuration. Merged into\n\t        providerOptions under the appropriate provider key. Example for Anthropic:\n\t        {'anthropic': {'thinking': {'type': 'adaptive'}}}. Example for OpenAI:\n\t        {'openai': {'reasoningEffort': 'high', 'reasoningSummary': 'detailed'}}.\n\t    model_fallbacks: Optional list of fallback model IDs tried in order if the primary\n\t        model fails. Passed as providerOptions.gateway.models.\n\t    caching: Optional caching mode for the gateway. Currently supports 'auto', which\n\t        enables provider-specific prompt caching via providerOptions.gateway.caching.\n\t\"\"\"\n\n\t# Model configuration\n\tmodel: ChatVercelModel | str\n\n\t# Model params\n\ttemperature: float | None = None\n\tmax_tokens: int | None = None\n\ttop_p: float | None = None\n\treasoning_models: list[str] | None = field(\n\t\tdefault_factory=lambda: [\n\t\t\t'o1',\n\t\t\t'o3',\n\t\t\t'o4',\n\t\t\t'gpt-oss',\n\t\t\t'gpt-5.2-pro',\n\t\t\t'gpt-5.4-pro',\n\t\t\t'deepseek-r1',\n\t\t\t'-thinking',\n\t\t\t'perplexity/sonar-reasoning',\n\t\t]\n\t)\n\n\t# Client initialization parameters\n\tapi_key: str | None = None\n\tbase_url: str | httpx.URL = 'https://ai-gateway.vercel.sh/v1'\n\ttimeout: float | httpx.Timeout | None = None\n\tmax_retries: int = 5\n\tdefault_headers: Mapping[str, str] | None = None\n\tdefault_query: Mapping[str, object] | None = None\n\thttp_client: httpx.AsyncClient | None = None\n\t_strict_response_validation: bool = False\n\tprovider_options: dict[str, Any] | None = None\n\treasoning: dict[str, dict[str, Any]] | None = None\n\tmodel_fallbacks: list[str] | None = None\n\tcaching: Literal['auto'] | None = None\n\n\t# Static\n\t@property\n\tdef provider(self) -> str:\n\t\treturn 'vercel'\n\n\tdef _get_client_params(self) -> dict[str, Any]:\n\t\t\"\"\"Prepare client parameters dictionary.\"\"\"\n\t\tapi_key = self.api_key or os.getenv('AI_GATEWAY_API_KEY') or os.getenv('VERCEL_OIDC_TOKEN')\n\n\t\tbase_params = {\n\t\t\t'api_key': api_key,\n\t\t\t'base_url': self.base_url,\n\t\t\t'timeout': self.timeout,\n\t\t\t'max_retries': self.max_retries,\n\t\t\t'default_headers': self.default_headers,\n\t\t\t'default_query': self.default_query,\n\t\t\t'_strict_response_validation': self._strict_response_validation,\n\t\t}\n\n\t\tclient_params = {k: v for k, v in base_params.items() if v is not None}\n\n\t\tif self.http_client is not None:\n\t\t\tclient_params['http_client'] = self.http_client\n\n\t\treturn client_params\n\n\tdef get_client(self) -> AsyncOpenAI:\n\t\t\"\"\"\n\t\tReturns an AsyncOpenAI client configured for Vercel AI Gateway.\n\n\t\tReturns:\n\t\t    AsyncOpenAI: An instance of the AsyncOpenAI client with Vercel base URL.\n\t\t\"\"\"\n\t\tif not hasattr(self, '_client'):\n\t\t\tclient_params = self._get_client_params()\n\t\t\tself._client = AsyncOpenAI(**client_params)\n\t\treturn self._client\n\n\t@property\n\tdef name(self) -> str:\n\t\treturn str(self.model)\n\n\tdef _get_usage(self, response: ChatCompletion) -> ChatInvokeUsage | None:\n\t\t\"\"\"Extract usage information from the Vercel response.\"\"\"\n\t\tif response.usage is None:\n\t\t\treturn None\n\n\t\tprompt_details = getattr(response.usage, 'prompt_tokens_details', None)\n\t\tcached_tokens = prompt_details.cached_tokens if prompt_details else None\n\n\t\treturn ChatInvokeUsage(\n\t\t\tprompt_tokens=response.usage.prompt_tokens,\n\t\t\tprompt_cached_tokens=cached_tokens,\n\t\t\tprompt_cache_creation_tokens=None,\n\t\t\tprompt_image_tokens=None,\n\t\t\tcompletion_tokens=response.usage.completion_tokens,\n\t\t\ttotal_tokens=response.usage.total_tokens,\n\t\t)\n\n\tdef _fix_gemini_schema(self, schema: dict[str, Any]) -> dict[str, Any]:\n\t\t\"\"\"\n\t\tConvert a Pydantic model to a Gemini-compatible schema.\n\n\t\tThis function removes unsupported properties like 'additionalProperties' and resolves\n\t\t$ref references that Gemini doesn't support.\n\t\t\"\"\"\n\n\t\t# Handle $defs and $ref resolution\n\t\tif '$defs' in schema:\n\t\t\tdefs = schema.pop('$defs')\n\n\t\t\tdef resolve_refs(obj: Any) -> Any:\n\t\t\t\tif isinstance(obj, dict):\n\t\t\t\t\tif '$ref' in obj:\n\t\t\t\t\t\tref = obj.pop('$ref')\n\t\t\t\t\t\tref_name = ref.split('/')[-1]\n\t\t\t\t\t\tif ref_name in defs:\n\t\t\t\t\t\t\t# Replace the reference with the actual definition\n\t\t\t\t\t\t\tresolved = defs[ref_name].copy()\n\t\t\t\t\t\t\t# Merge any additional properties from the reference\n\t\t\t\t\t\t\tfor key, value in obj.items():\n\t\t\t\t\t\t\t\tif key != '$ref':\n\t\t\t\t\t\t\t\t\tresolved[key] = value\n\t\t\t\t\t\t\treturn resolve_refs(resolved)\n\t\t\t\t\t\treturn obj\n\t\t\t\t\telse:\n\t\t\t\t\t\t# Recursively process all dictionary values\n\t\t\t\t\t\treturn {k: resolve_refs(v) for k, v in obj.items()}\n\t\t\t\telif isinstance(obj, list):\n\t\t\t\t\treturn [resolve_refs(item) for item in obj]\n\t\t\t\treturn obj\n\n\t\t\tschema = resolve_refs(schema)\n\n\t\t# Remove unsupported properties\n\t\tdef clean_schema(obj: Any) -> Any:\n\t\t\tif isinstance(obj, dict):\n\t\t\t\t# Remove unsupported properties\n\t\t\t\tcleaned = {}\n\t\t\t\tfor key, value in obj.items():\n\t\t\t\t\tif key not in ['additionalProperties', 'title', 'default']:\n\t\t\t\t\t\tcleaned_value = clean_schema(value)\n\t\t\t\t\t\t# Handle empty object properties - Gemini doesn't allow empty OBJECT types\n\t\t\t\t\t\tif (\n\t\t\t\t\t\t\tkey == 'properties'\n\t\t\t\t\t\t\tand isinstance(cleaned_value, dict)\n\t\t\t\t\t\t\tand len(cleaned_value) == 0\n\t\t\t\t\t\t\tand isinstance(obj.get('type', ''), str)\n\t\t\t\t\t\t\tand obj.get('type', '').upper() == 'OBJECT'\n\t\t\t\t\t\t):\n\t\t\t\t\t\t\t# Convert empty object to have at least one property\n\t\t\t\t\t\t\tcleaned['properties'] = {'_placeholder': {'type': 'string'}}\n\t\t\t\t\t\telse:\n\t\t\t\t\t\t\tcleaned[key] = cleaned_value\n\n\t\t\t\t# If this is an object type with empty properties, add a placeholder\n\t\t\t\tif (\n\t\t\t\t\tisinstance(cleaned.get('type', ''), str)\n\t\t\t\t\tand cleaned.get('type', '').upper() == 'OBJECT'\n\t\t\t\t\tand 'properties' in cleaned\n\t\t\t\t\tand isinstance(cleaned['properties'], dict)\n\t\t\t\t\tand len(cleaned['properties']) == 0\n\t\t\t\t):\n\t\t\t\t\tcleaned['properties'] = {'_placeholder': {'type': 'string'}}\n\n\t\t\t\t# Also remove 'title' from the required list if it exists\n\t\t\t\tif 'required' in cleaned and isinstance(cleaned.get('required'), list):\n\t\t\t\t\tcleaned['required'] = [p for p in cleaned['required'] if p != 'title']\n\n\t\t\t\treturn cleaned\n\t\t\telif isinstance(obj, list):\n\t\t\t\treturn [clean_schema(item) for item in obj]\n\t\t\treturn obj\n\n\t\treturn clean_schema(schema)\n\n\t@overload\n\tasync def ainvoke(\n\t\tself, messages: list[BaseMessage], output_format: None = None, **kwargs: Any\n\t) -> ChatInvokeCompletion[str]: ...\n\n\t@overload\n\tasync def ainvoke(self, messages: list[BaseMessage], output_format: type[T], **kwargs: Any) -> ChatInvokeCompletion[T]: ...\n\n\tasync def ainvoke(\n\t\tself, messages: list[BaseMessage], output_format: type[T] | None = None, **kwargs: Any\n\t) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:\n\t\t\"\"\"\n\t\tInvoke the model with the given messages through Vercel AI Gateway.\n\n\t\tArgs:\n\t\t    messages: List of chat messages\n\t\t    output_format: Optional Pydantic model class for structured output\n\n\t\tReturns:\n\t\t    Either a string response or an instance of output_format\n\t\t\"\"\"\n\t\tvercel_messages = VercelMessageSerializer.serialize_messages(messages)\n\n\t\ttry:\n\t\t\tmodel_params: dict[str, Any] = {}\n\t\t\tif self.temperature is not None:\n\t\t\t\tmodel_params['temperature'] = self.temperature\n\t\t\tif self.max_tokens is not None:\n\t\t\t\tmodel_params['max_tokens'] = self.max_tokens\n\t\t\tif self.top_p is not None:\n\t\t\t\tmodel_params['top_p'] = self.top_p\n\n\t\t\textra_body: dict[str, Any] = {}\n\n\t\t\tprovider_opts: dict[str, Any] = {}\n\t\t\tif self.provider_options:\n\t\t\t\tprovider_opts.update(self.provider_options)\n\n\t\t\tif self.reasoning:\n\t\t\t\t# Merge provider-specific reasoning options (ex: {'anthropic': {'thinking': ...}})\n\t\t\t\tfor provider_name, opts in self.reasoning.items():\n\t\t\t\t\texisting = provider_opts.get(provider_name, {})\n\t\t\t\t\texisting.update(opts)\n\t\t\t\t\tprovider_opts[provider_name] = existing\n\n\t\t\tgateway_opts: dict[str, Any] = provider_opts.get('gateway', {})\n\n\t\t\tif self.model_fallbacks:\n\t\t\t\tgateway_opts['models'] = self.model_fallbacks\n\n\t\t\tif self.caching:\n\t\t\t\tgateway_opts['caching'] = self.caching\n\n\t\t\tif gateway_opts:\n\t\t\t\tprovider_opts['gateway'] = gateway_opts\n\n\t\t\tif provider_opts:\n\t\t\t\textra_body['providerOptions'] = provider_opts\n\n\t\t\tif extra_body:\n\t\t\t\tmodel_params['extra_body'] = extra_body\n\n\t\t\tif output_format is None:\n\t\t\t\t# Return string response\n\t\t\t\tresponse = await self.get_client().chat.completions.create(\n\t\t\t\t\tmodel=self.model,\n\t\t\t\t\tmessages=vercel_messages,\n\t\t\t\t\t**model_params,\n\t\t\t\t)\n\n\t\t\t\tusage = self._get_usage(response)\n\t\t\t\treturn ChatInvokeCompletion(\n\t\t\t\t\tcompletion=response.choices[0].message.content or '',\n\t\t\t\t\tusage=usage,\n\t\t\t\t\tstop_reason=response.choices[0].finish_reason if response.choices else None,\n\t\t\t\t)\n\n\t\t\telse:\n\t\t\t\tis_google_model = self.model.startswith('google/')\n\t\t\t\tis_anthropic_model = self.model.startswith('anthropic/')\n\t\t\t\tis_reasoning_model = self.reasoning_models and any(\n\t\t\t\t\tstr(pattern).lower() in str(self.model).lower() for pattern in self.reasoning_models\n\t\t\t\t)\n\n\t\t\t\tif is_google_model or is_anthropic_model or is_reasoning_model:\n\t\t\t\t\tmodified_messages = [m.model_copy(deep=True) for m in messages]\n\n\t\t\t\t\tschema = SchemaOptimizer.create_gemini_optimized_schema(output_format)\n\t\t\t\t\tjson_instruction = f'\\n\\nIMPORTANT: You must respond with ONLY a valid JSON object (no markdown, no code blocks, no explanations) that exactly matches this schema:\\n{json.dumps(schema, indent=2)}'\n\n\t\t\t\t\tinstruction_added = False\n\t\t\t\t\tif modified_messages and modified_messages[0].role == 'system':\n\t\t\t\t\t\tif isinstance(modified_messages[0].content, str):\n\t\t\t\t\t\t\tmodified_messages[0].content += json_instruction\n\t\t\t\t\t\t\tinstruction_added = True\n\t\t\t\t\t\telif isinstance(modified_messages[0].content, list):\n\t\t\t\t\t\t\tmodified_messages[0].content.append(ContentPartTextParam(text=json_instruction))\n\t\t\t\t\t\t\tinstruction_added = True\n\t\t\t\t\telif modified_messages and modified_messages[-1].role == 'user':\n\t\t\t\t\t\tif isinstance(modified_messages[-1].content, str):\n\t\t\t\t\t\t\tmodified_messages[-1].content += json_instruction\n\t\t\t\t\t\t\tinstruction_added = True\n\t\t\t\t\t\telif isinstance(modified_messages[-1].content, list):\n\t\t\t\t\t\t\tmodified_messages[-1].content.append(ContentPartTextParam(text=json_instruction))\n\t\t\t\t\t\t\tinstruction_added = True\n\n\t\t\t\t\tif not instruction_added:\n\t\t\t\t\t\tmodified_messages.insert(0, SystemMessage(content=json_instruction))\n\n\t\t\t\t\tvercel_messages = VercelMessageSerializer.serialize_messages(modified_messages)\n\n\t\t\t\t\tresponse = await self.get_client().chat.completions.create(\n\t\t\t\t\t\tmodel=self.model,\n\t\t\t\t\t\tmessages=vercel_messages,\n\t\t\t\t\t\t**model_params,\n\t\t\t\t\t)\n\n\t\t\t\t\tcontent = response.choices[0].message.content if response.choices else None\n\n\t\t\t\t\tif not content:\n\t\t\t\t\t\traise ModelProviderError(\n\t\t\t\t\t\t\tmessage='No response from model',\n\t\t\t\t\t\t\tstatus_code=500,\n\t\t\t\t\t\t\tmodel=self.name,\n\t\t\t\t\t\t)\n\n\t\t\t\t\ttry:\n\t\t\t\t\t\ttext = content.strip()\n\t\t\t\t\t\tif text.startswith('```json') and text.endswith('```'):\n\t\t\t\t\t\t\ttext = text[7:-3].strip()\n\t\t\t\t\t\telif text.startswith('```') and text.endswith('```'):\n\t\t\t\t\t\t\ttext = text[3:-3].strip()\n\n\t\t\t\t\t\tparsed_data = json.loads(text)\n\t\t\t\t\t\tparsed = output_format.model_validate(parsed_data)\n\n\t\t\t\t\t\tusage = self._get_usage(response)\n\t\t\t\t\t\treturn ChatInvokeCompletion(\n\t\t\t\t\t\t\tcompletion=parsed,\n\t\t\t\t\t\t\tusage=usage,\n\t\t\t\t\t\t\tstop_reason=response.choices[0].finish_reason if response.choices else None,\n\t\t\t\t\t\t)\n\n\t\t\t\t\texcept (json.JSONDecodeError, ValueError) as e:\n\t\t\t\t\t\traise ModelProviderError(\n\t\t\t\t\t\t\tmessage=f'Failed to parse JSON response: {str(e)}. Raw response: {content[:200]}',\n\t\t\t\t\t\t\tstatus_code=500,\n\t\t\t\t\t\t\tmodel=self.name,\n\t\t\t\t\t\t) from e\n\n\t\t\t\telse:\n\t\t\t\t\tschema = SchemaOptimizer.create_optimized_json_schema(output_format)\n\n\t\t\t\t\tresponse_format_schema: JSONSchema = {\n\t\t\t\t\t\t'name': 'agent_output',\n\t\t\t\t\t\t'strict': True,\n\t\t\t\t\t\t'schema': schema,\n\t\t\t\t\t}\n\n\t\t\t\t\tresponse = await self.get_client().chat.completions.create(\n\t\t\t\t\t\tmodel=self.model,\n\t\t\t\t\t\tmessages=vercel_messages,\n\t\t\t\t\t\tresponse_format=ResponseFormatJSONSchema(\n\t\t\t\t\t\t\tjson_schema=response_format_schema,\n\t\t\t\t\t\t\ttype='json_schema',\n\t\t\t\t\t\t),\n\t\t\t\t\t\t**model_params,\n\t\t\t\t\t)\n\n\t\t\t\t\tcontent = response.choices[0].message.content if response.choices else None\n\n\t\t\t\t\tif not content:\n\t\t\t\t\t\traise ModelProviderError(\n\t\t\t\t\t\t\tmessage='Failed to parse structured output from model response - empty or null content',\n\t\t\t\t\t\t\tstatus_code=500,\n\t\t\t\t\t\t\tmodel=self.name,\n\t\t\t\t\t\t)\n\n\t\t\t\t\tusage = self._get_usage(response)\n\t\t\t\t\tparsed = output_format.model_validate_json(content)\n\n\t\t\t\t\treturn ChatInvokeCompletion(\n\t\t\t\t\t\tcompletion=parsed,\n\t\t\t\t\t\tusage=usage,\n\t\t\t\t\t\tstop_reason=response.choices[0].finish_reason if response.choices else None,\n\t\t\t\t\t)\n\n\t\texcept RateLimitError as e:\n\t\t\traise ModelRateLimitError(message=e.message, model=self.name) from e\n\n\t\texcept APIConnectionError as e:\n\t\t\traise ModelProviderError(message=str(e), model=self.name) from e\n\n\t\texcept APIStatusError as e:\n\t\t\traise ModelProviderError(message=e.message, status_code=e.status_code, model=self.name) from e\n\n\t\texcept Exception as e:\n\t\t\traise ModelProviderError(message=str(e), model=self.name) from e\n"}}, "extracted/de/deepeval/deepeval/config/settings.py": {"b9b038458f2cc819fd6dfb6d96a4a8625adc63df": {"t": 1785746918.9652, "n": 58681, "h": "66c2bf9bb4d96186", "c": "\"\"\"\nCentral config for DeepEval.\n\n- Autoloads dotenv files into os.environ without overwriting existing vars\n  (order: .env -> .env.{APP_ENV} -> .env.local).\n- Defines the Pydantic `Settings` model and `get_settings()` singleton.\n- Exposes an `edit()` context manager that diffs changes and persists them to\n  dotenv and the legacy JSON keystore (non-secret keys only), with validators and\n  type coercion.\n\"\"\"\n\nimport hashlib\nimport json\nimport logging\nimport math\nimport os\nimport re\nimport threading\n\nfrom contextvars import ContextVar\nfrom pathlib import Path\nfrom pydantic import (\n    AnyUrl,\n    computed_field,\n    confloat,\n    conint,\n    Field,\n    field_validator,\n    model_validator,\n    SecretStr,\n    PositiveFloat,\n)\nfrom pydantic_settings import BaseSettings, SettingsConfigDict\nfrom typing import (\n    Any,\n    Dict,\n    List,\n    Optional,\n    Union,\n    NamedTuple,\n    get_args,\n    get_origin,\n)\n\nfrom deepeval.config.utils import (\n    coerce_to_list,\n    constrain_between,\n    dedupe_preserve_order,\n    parse_bool,\n    read_dotenv_file,\n)\nfrom deepeval.constants import SUPPORTED_PROVIDER_SLUGS, slugify\n\nlogger = logging.getLogger(__name__)\n_SAVE_RE = re.compile(r\"^(?P<scheme>dotenv)(?::(?P<path>.+))?$\")\n\n_ACTIVE_SETTINGS_EDIT_CTX: ContextVar[Optional[\"Settings._SettingsEditCtx\"]] = (\n    ContextVar(\"_ACTIVE_SETTINGS_EDIT_CTX\", default=None)\n)\n\n# settings that were converted to computed fields with override counterparts\n_DEPRECATED_TO_OVERRIDE = {\n    \"DEEPEVAL_PER_TASK_TIMEOUT_SECONDS\": \"DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE\",\n    \"DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS\": \"DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE\",\n    \"DEEPEVAL_TASK_GATHER_BUFFER_SECONDS\": \"DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE\",\n}\n# Track which secrets we've warned about when loading from the legacy keyfile\n_LEGACY_KEYFILE_SECRET_WARNED: set[str] = set()\n\n\ndef _find_legacy_enum(env_key: str):\n    from deepeval.key_handler import (\n        ModelKeyValues,\n        EmbeddingKeyValues,\n        KeyValues,\n    )\n\n    enums = (ModelKeyValues, EmbeddingKeyValues, KeyValues)\n\n    for enum in enums:\n        try:\n            return getattr(enum, env_key)\n        except AttributeError:\n            pass\n\n    for enum in enums:\n        for member in enum:\n            if member.value == env_key:\n                return member\n    return None\n\n\ndef _is_secret_key(env_key: str) -> bool:\n    field = Settings.model_fields.get(env_key)\n    if not field:\n        return False\n    if field.annotation is SecretStr:\n        return True\n\n    origin = get_origin(field.annotation)\n    if origin is Union:\n        return any(arg is SecretStr for arg in get_args(field.annotation))\n    return False\n\n\ndef _merge_legacy_keyfile_into_env() -> None:\n    \"\"\"\n    Backwards compatibility: merge values from the legacy .deepeval/.deepeval\n    JSON keystore into os.environ for known Settings fields, without\n    overwriting existing process env vars.\n\n    This runs before we compute the Settings env fingerprint so that Pydantic\n    can see these values on first construction.\n\n    Precedence: process env -> dotenv -> legacy json\n    \"\"\"\n    # if somebody really wants to skip this behavior\n    if parse_bool(os.getenv(\"DEEPEVAL_DISABLE_LEGACY_KEYFILE\"), default=False):\n        return\n\n    from deepeval.constants import HIDDEN_DIR, KEY_FILE\n    from deepeval.key_handler import (\n        KeyValues,\n        ModelKeyValues,\n        EmbeddingKeyValues,\n    )\n\n    key_path = Path(HIDDEN_DIR) / KEY_FILE\n\n    try:\n        with key_path.open(\"r\", encoding=\"utf-8\") as f:\n            try:\n                data = json.load(f)\n            except json.JSONDecodeError:\n                # Corrupted file -> ignore, same as KeyFileHandler\n                return\n    except FileNotFoundError:\n        # No legacy store -> nothing to merge\n        return\n\n    if not isinstance(data, dict):\n        return\n\n    # Map JSON keys (enum .value) -> env keys (enum .name)\n    mapping: Dict[str, str] = {}\n    for enum in (KeyValues, ModelKeyValues, EmbeddingKeyValues):\n        for member in enum:\n            mapping[member.value] = member.name\n\n    for json_key, raw in data.items():\n        env_key = mapping.get(json_key)\n        if not env_key:\n            continue\n\n        # Process env always wins\n        if env_key in os.environ:\n            continue\n        if raw is None:\n            continue\n\n        # Mirror the legacy warning semantics for secrets, but only once per key\n        if env_key not in _LEGACY_KEYFILE_SECRET_WARNED and _is_secret_key(\n            env_key\n        ):\n            logger.warning(\n                \"Reading secret '%s' (legacy key '%s') from legacy %s/%s. \"\n                \"Persisting API keys in plaintext is deprecated. \"\n                \"Move this to your environment (.env / .env.local). \"\n                \"This fallback will be removed in a future release.\",\n                env_key,\n                json_key,\n                HIDDEN_DIR,\n                KEY_FILE,\n            )\n            _LEGACY_KEYFILE_SECRET_WARNED.add(env_key)\n        # Let Settings validators coerce types; we just inject the raw string\n        os.environ[env_key] = str(raw)\n\n\ndef _discover_app_env_from_files(env_dir: Path) -> Optional[str]:\n    # prefer base .env.local, then .env for APP_ENV discovery\n    for name in (\".env.local\", \".env\"):\n        v = read_dotenv_file(env_dir / name).get(\"APP_ENV\")\n        if v:\n            v = str(v).strip()\n            if v:\n                return v\n    return None\n\n\ndef resolve_env_dir() -> Path:\n    \"\"\"Directory containing .env files: ENV_DIR_PATH env var, else CWD.\"\"\"\n    raw_dir = os.getenv(\"ENV_DIR_PATH\")\n    if raw_dir:\n        return Path(os.path.expanduser(os.path.expandvars(raw_dir)))\n    return Path(os.getcwd())\n\n\ndef _resolve_app_env(env_dir: Path) -> Optional[str]:\n    # Pick APP_ENV (process -> .env.local -> .env -> none)\n    app_env = os.getenv(\"APP_ENV\") or _discover_app_env_from_files(env_dir)\n    if app_env is None:\n        return None\n    app_env = str(app_env).strip()\n    return app_env or None\n\n\ndef dotenv_search_paths(\n    env_dir: Optional[Path] = None, app_env: Optional[str] = None\n) -> List[Path]:\n    \"\"\"Dotenv files in load order (lowest -> highest precedence):\n    .env -> .env.{APP_ENV} -> .env.local.\n\n    Single source of truth for which files `autoload_dotenv()` reads;\n    `deepeval diagnose` uses it too so the display cannot drift from the\n    loader.\n    \"\"\"\n    env_dir = env_dir if env_dir is not None else resolve_env_dir()\n    app_env = app_env if app_env is not None else _resolve_app_env(env_dir)\n\n    paths = [env_dir / \".env\"]\n    if app_env:\n        paths.append(env_dir / f\".env.{app_env}\")\n    paths.append(env_dir / \".env.local\")\n    return paths\n\n\ndef autoload_dotenv() -> None:\n    \"\"\"\n    Load env vars from .env files without overriding existing process env.\n\n    Precedence (lowest -> highest): .env -> .env.{APP_ENV} -> .env.local\n    Process env always wins over file values.\n\n    Controls:\n      - DEEPEVAL_DISABLE_DOTENV=1 -> skip\n      - ENV_DIR_PATH -> directory containing .env files (default: CWD)\n    \"\"\"\n    if parse_bool(os.getenv(\"DEEPEVAL_DISABLE_DOTENV\"), default=False):\n        return\n\n    env_dir = resolve_env_dir()\n    app_env = _resolve_app_env(env_dir)\n\n    merged: Dict[str, str] = {}\n    if app_env:\n        # Seed the discovered APP_ENV; file values below may override it.\n        merged[\"APP_ENV\"] = app_env\n\n    # merge files in precedence order (later wins)\n    for path in dotenv_search_paths(env_dir, app_env):\n        merged.update(read_dotenv_file(path))\n\n    # Write only keys that aren’t already in process env\n    for k, v in merged.items():\n        if k not in os.environ:\n            os.environ[k] = v\n\n\nclass PersistResult(NamedTuple):\n    handled: bool\n    path: Optional[Path]\n    updated: Dict[str, Any]  # typed, validated and changed\n\n\nclass Settings(BaseSettings):\n    # def __init__(self):\n    #     super().__init__()\n    def __setattr__(self, name: str, value):\n        ctx = _ACTIVE_SETTINGS_EDIT_CTX.get()\n        if ctx is not None and name in type(self).model_fields:\n            ctx._touched.add(name)\n        return super().__setattr__(name, value)\n\n    model_config = SettingsConfigDict(\n        extra=\"ignore\",\n        case_sensitive=True,\n        validate_assignment=True,\n    )\n\n    #\n    # General\n    #\n\n    APP_ENV: str = Field(\n        \"dev\",\n        description=\"Application environment name used for dotenv selection (loads .env.<APP_ENV> if present).\",\n    )\n    LOG_LEVEL: Optional[int] = Field(\n        None,\n        description=\"Global logging level (e.g. DEBUG/INFO/WARNING/ERROR/CRITICAL or numeric).\",\n    )\n    PYTHONPATH: str = Field(\n        \".\",\n        description=\"Extra PYTHONPATH used by the CLI runner (default: current project '.').\",\n    )\n    CONFIDENT_REGION: Optional[str] = Field(\n        None,\n        description=\"Optional Confident AI region hint (uppercased).\",\n    )\n    CONFIDENT_OPEN_BROWSER: Optional[bool] = Field(\n        True,\n        description=\"Open a browser automatically for Confident AI links/flows when available.\",\n    )\n\n    #\n    # CLI\n    #\n    DEEPEVAL_DEFAULT_SAVE: Optional[str] = Field(\n        None,\n        description=\"Default persistence target for settings changes (e.g. 'dotenv' or 'dotenv:/path/to/.env.local').\",\n    )\n    DEEPEVAL_DISABLE_DOTENV: Optional[bool] = Field(\n        None,\n        description=\"Disable dotenv autoloading (.env → .env.<APP_ENV> → .env.local). Tip: set to 1 in pytest/CI to prevent loading env files on import.\",\n    )\n    ENV_DIR_PATH: Optional[Path] = Field(\n        None,\n        description=\"Directory containing .env files (default: current working directory).\",\n    )\n    DEEPEVAL_FILE_SYSTEM: Optional[str] = Field(\n        None,\n        description=\"Filesystem mode for runtime/CLI (currently supports READ_ONLY).\",\n    )\n    DEEPEVAL_IDENTIFIER: Optional[str] = Field(\n        None,\n        description=\"Identifier/tag to help identify your test run on Confident AI.\",\n    )\n\n    #\n    # Storage & Output\n    #\n\n    # When set, DeepEval will export a timestamped JSON of the latest test run\n    # into this directory. The directory will be created on demand.\n    DEEPEVAL_RESULTS_FOLDER: Optional[Path] = Field(\n        None,\n        description=\"If set, export a timestamped JSON of the latest test run into this folder (created if missing).\",\n    )\n\n    # When set, overrides the default DeepEval cache directory\n    DEEPEVAL_CACHE_FOLDER: Optional[Path] = Field(\n        \".deepeval\",\n        description=\"Path to the directory used by DeepEval to store cache files. If set, this overrides the default cache location. The directory will be created if it does not exist.\",\n    )\n\n    # Display / Truncation\n    DEEPEVAL_MAXLEN_TINY: Optional[int] = Field(\n        40,\n        description=\"Default truncation length for 'tiny' displays in logs/UI.\",\n    )\n    DEEPEVAL_MAXLEN_SHORT: Optional[int] = Field(\n        60,\n        description=\"Default truncation length for 'short' displays in logs/UI.\",\n    )\n    DEEPEVAL_MAXLEN_MEDIUM: Optional[int] = Field(\n        120,\n        description=\"Default truncation length for 'medium' displays in logs/UI.\",\n    )\n    DEEPEVAL_MAXLEN_LONG: Optional[int] = Field(\n        240,\n        description=\"Default truncation length for 'long' displays in logs/UI.\",\n    )\n\n    # If set, this overrides the default max_len used by deepeval/utils shorten\n    # falls back to DEEPEVAL_MAXLEN_LONG when None.\n    DEEPEVAL_SHORTEN_DEFAULT_MAXLEN: Optional[int] = Field(\n        None,\n        description=\"Override default max_len for deepeval.utils.shorten (falls back to DEEPEVAL_MAXLEN_LONG when unset).\",\n    )\n\n    # Optional global suffix (keeps your \"...\" default).\n    DEEPEVAL_SHORTEN_SUFFIX: Optional[str] = Field(\n        \"...\",\n        description=\"Suffix appended by deepeval.utils.shorten when truncating (default: '...').\",\n    )\n\n    #\n    # GPU and perf toggles\n    #\n\n    CUDA_LAUNCH_BLOCKING: Optional[bool] = Field(\n        None,\n        description=\"CUDA debug toggle (forces synchronous CUDA ops). Useful for debugging GPU errors.\",\n    )\n    CUDA_VISIBLE_DEVICES: Optional[str] = Field(\n        None,\n        description=\"CUDA device visibility mask (e.g. '0' or '0,1').\",\n    )\n    TOKENIZERS_PARALLELISM: Optional[bool] = Field(\n        None,\n        description=\"HuggingFace tokenizers parallelism toggle (set to false to reduce warnings/noise).\",\n    )\n    TRANSFORMERS_NO_ADVISORY_WARNINGS: Optional[bool] = Field(\n        None,\n        description=\"Disable advisory warnings from transformers (reduces console noise).\",\n    )\n\n    #\n    # Model Keys\n    #\n\n    CONFIDENT_API_KEY: Optional[SecretStr] = Field(\n        None,\n        description=\"Confident AI API key (used for uploading results/telemetry to Confident).\",\n    )\n\n    # ======\n    # Base URL for Confident AI API server\n    # ======\n    CONFIDENT_BASE_URL: Optional[str] = Field(\n        None,\n        description=\"Base URL for Confident AI API server (set only if using a custom/hosted endpoint).\",\n    )\n\n    CONFIDENT_DISABLE_SSL: Optional[bool] = Field(\n        None,\n        description=\"Disable SSL/TLS certificate verification for requests to the Confident AI API server (only use for self-hosted endpoints with self-signed certificates).\",\n    )\n\n    # General\n    TEMPERATURE: Optional[confloat(ge=0, le=2)] = Field(\n        None,\n        description=\"Global default model temperature (0–2). Model-specific constructors may override.\",\n    )\n\n    # Anthropic\n    USE_ANTHROPIC_MODEL: Optional[bool] = Field(\n        None,\n        description=\"Select Anthropic as the active LLM provider (USE_* flags are mutually exclusive in CLI helpers).\",\n    )\n    ANTHROPIC_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"Anthropic API key.\"\n    )\n    ANTHROPIC_MODEL_NAME: Optional[str] = Field(\n        None, description=\"Anthropic model name (e.g. 'claude-3-...').\"\n    )\n    ANTHROPIC_COST_PER_INPUT_TOKEN: Optional[PositiveFloat] = Field(\n        None,\n        description=\"Anthropic input token cost (used for cost reporting).\",\n    )\n    ANTHROPIC_COST_PER_OUTPUT_TOKEN: Optional[PositiveFloat] = Field(\n        None,\n        description=\"Anthropic output token cost (used for cost reporting).\",\n    )\n\n    # AWS\n    AWS_ACCESS_KEY_ID: Optional[SecretStr] = Field(\n        None,\n        description=\"AWS access key ID (for Bedrock or other AWS-backed integrations).\",\n    )\n    AWS_SECRET_ACCESS_KEY: Optional[SecretStr] = Field(\n        None,\n        description=\"AWS secret access key (for Bedrock or other AWS-backed integrations).\",\n    )\n    AWS_SESSION_TOKEN: Optional[SecretStr] = Field(\n        None,\n        description=\"AWS session token (for temporary credentials with Bedrock or other AWS-backed integrations).\",\n    )\n    # AWS Bedrock\n    USE_AWS_BEDROCK_MODEL: Optional[bool] = Field(\n        None, description=\"Select AWS Bedrock as the active LLM provider.\"\n    )\n    AWS_BEDROCK_MODEL_NAME: Optional[str] = Field(\n        None, description=\"AWS Bedrock model identifier.\"\n    )\n    AWS_BEDROCK_REGION: Optional[str] = Field(\n        None, description=\"AWS region for Bedrock (normalized to lowercase).\"\n    )\n    AWS_BEDROCK_COST_PER_INPUT_TOKEN: Optional[PositiveFloat] = Field(\n        None, description=\"Bedrock input token cost (used for cost reporting).\"\n    )\n    AWS_BEDROCK_COST_PER_OUTPUT_TOKEN: Optional[PositiveFloat] = Field(\n        None, description=\"Bedrock output token cost (used for cost reporting).\"\n    )\n    # Azure Open AI\n    USE_AZURE_OPENAI: Optional[bool] = Field(\n        None, description=\"Select Azure OpenAI as the active LLM provider.\"\n    )\n    AZURE_OPENAI_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"Azure OpenAI API key.\"\n    )\n    AZURE_OPENAI_AD_TOKEN: Optional[SecretStr] = Field(\n        None, description=\"Azure OpenAI Ad Token.\"\n    )\n    AZURE_OPENAI_ENDPOINT: Optional[AnyUrl] = Field(\n        None, description=\"Azure OpenAI endpoint URL.\"\n    )\n    OPENAI_API_VERSION: Optional[str] = Field(\n        None,\n        description=\"Azure OpenAI API version (if required by your deployment).\",\n    )\n    AZURE_DEPLOYMENT_NAME: Optional[str] = Field(\n        None,\n        description=\"Azure OpenAI deployment name (required for most Azure configs).\",\n    )\n    AZURE_MODEL_NAME: Optional[str] = Field(\n        None,\n        description=\"Azure model name label (informational; may be used in reporting).\",\n    )\n    AZURE_MODEL_VERSION: Optional[str] = Field(\n        None,\n        description=\"Azure model version label (informational; may be used in reporting).\",\n    )\n    # DeepSeek\n    USE_DEEPSEEK_MODEL: Optional[bool] = Field(\n        None, description=\"Select DeepSeek as the active LLM provider.\"\n    )\n    DEEPSEEK_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"DeepSeek API key.\"\n    )\n    DEEPSEEK_MODEL_NAME: Optional[str] = Field(\n        None, description=\"DeepSeek model name.\"\n    )\n    DEEPSEEK_COST_PER_INPUT_TOKEN: Optional[float] = Field(\n        None, description=\"DeepSeek input token cost (used for cost reporting).\"\n    )\n    DEEPSEEK_COST_PER_OUTPUT_TOKEN: Optional[float] = Field(\n        None,\n        description=\"DeepSeek output token cost (used for cost reporting).\",\n    )\n    # Gemini\n    USE_GEMINI_MODEL: Optional[bool] = Field(\n        None, description=\"Select Google Gemini as the active LLM provider.\"\n    )\n    GOOGLE_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"Google API key for Gemini (non-Vertex usage).\"\n    )\n    GEMINI_MODEL_NAME: Optional[str] = Field(\n        None, description=\"Gemini model name (e.g. 'gemini-...').\"\n    )\n    GOOGLE_GENAI_USE_VERTEXAI: Optional[bool] = Field(\n        None,\n        description=\"Use Vertex AI for Gemini requests instead of direct API key mode.\",\n    )\n    GOOGLE_CLOUD_PROJECT: Optional[str] = Field(\n        None,\n        description=\"GCP project ID for Vertex AI (required if GOOGLE_GENAI_USE_VERTEXAI=true).\",\n    )\n    GOOGLE_CLOUD_LOCATION: Optional[str] = Field(\n        None,\n        description=\"GCP region/location for Vertex AI (e.g. 'us-central1').\",\n    )\n    GOOGLE_SERVICE_ACCOUNT_KEY: Optional[SecretStr] = Field(\n        None,\n        description=\"Service account JSON key for Vertex AI auth (if not using ADC).\",\n    )\n    GEMINI_COST_PER_INPUT_TOKEN: Optional[float] = Field(\n        None, description=\"Gemini input token cost (used for cost reporting).\"\n    )\n    GEMINI_COST_PER_OUTPUT_TOKEN: Optional[float] = Field(\n        None, description=\"Gemini output token cost (used for cost reporting).\"\n    )\n    # Grok\n    USE_GROK_MODEL: Optional[bool] = Field(\n        None, description=\"Select Grok as the active LLM provider.\"\n    )\n    GROK_API_KEY: Optional[SecretStr] = Field(None, description=\"Grok API key.\")\n    GROK_MODEL_NAME: Optional[str] = Field(None, description=\"Grok model name.\")\n    GROK_COST_PER_INPUT_TOKEN: Optional[float] = Field(\n        None, description=\"Grok input token cost (used for cost reporting).\"\n    )\n    GROK_COST_PER_OUTPUT_TOKEN: Optional[float] = Field(\n        None, description=\"Grok output token cost (used for cost reporting).\"\n    )\n    # LiteLLM\n    USE_LITELLM: Optional[bool] = Field(\n        None, description=\"Select LiteLLM as the active LLM provider.\"\n    )\n    LITELLM_API_KEY: Optional[SecretStr] = Field(\n        None,\n        description=\"LiteLLM API key (if required by your LiteLLM deployment).\",\n    )\n    LITELLM_MODEL_NAME: Optional[str] = Field(\n        None,\n        description=\"LiteLLM model name (as exposed by your LiteLLM endpoint).\",\n    )\n    LITELLM_API_BASE: Optional[AnyUrl] = Field(\n        None, description=\"LiteLLM API base URL (direct).\"\n    )\n    LITELLM_PROXY_API_BASE: Optional[AnyUrl] = Field(\n        None, description=\"LiteLLM proxy base URL (if using proxy mode).\"\n    )\n    LITELLM_PROXY_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"LiteLLM proxy API key (if required).\"\n    )\n    # LM Studio\n    LM_STUDIO_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"LM Studio API key (if configured).\"\n    )\n    LM_STUDIO_MODEL_NAME: Optional[str] = Field(\n        None, description=\"LM Studio model name.\"\n    )\n    # Local Model\n    USE_LOCAL_MODEL: Optional[bool] = Field(\n        None,\n        description=\"Select a local/self-hosted model as the active LLM provider.\",\n    )\n    LOCAL_MODEL_API_KEY: Optional[SecretStr] = Field(\n        None,\n        description=\"API key for a local/self-hosted LLM endpoint (if required).\",\n    )\n    LOCAL_EMBEDDING_API_KEY: Optional[SecretStr] = Field(\n        None,\n        description=\"API key for a local/self-hosted embedding endpoint (if required).\",\n    )\n    LOCAL_MODEL_NAME: Optional[str] = Field(\n        None,\n        description=\"Local/self-hosted model name (informational / routing).\",\n    )\n    LOCAL_MODEL_BASE_URL: Optional[AnyUrl] = Field(\n        None, description=\"Base URL for a local/self-hosted LLM endpoint.\"\n    )\n    LOCAL_MODEL_FORMAT: Optional[str] = Field(\n        None,\n        description=\"Local model API format identifier (implementation-specific).\",\n    )\n    # Moonshot\n    USE_MOONSHOT_MODEL: Optional[bool] = Field(\n        None, description=\"Select Moonshot as the active LLM provider.\"\n    )\n    MOONSHOT_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"Moonshot API key.\"\n    )\n    MOONSHOT_MODEL_NAME: Optional[str] = Field(\n        None, description=\"Moonshot model name.\"\n    )\n    MOONSHOT_COST_PER_INPUT_TOKEN: Optional[float] = Field(\n        None, description=\"Moonshot input token cost (used for cost reporting).\"\n    )\n    MOONSHOT_COST_PER_OUTPUT_TOKEN: Optional[float] = Field(\n        None,\n        description=\"Moonshot output token cost (used for cost reporting).\",\n    )\n    # Ollama\n    OLLAMA_MODEL_NAME: Optional[str] = Field(\n        None,\n        description=\"Ollama model name (used when running via Ollama integration).\",\n    )\n    # OpenAI\n    USE_OPENAI_MODEL: Optional[bool] = Field(\n        None, description=\"Select OpenAI as the active LLM provider.\"\n    )\n    OPENAI_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"OpenAI API key.\"\n    )\n    OPENAI_MODEL_NAME: Optional[str] = Field(\n        None, description=\"OpenAI model name (e.g. 'gpt-4.1').\"\n    )\n    OPENAI_COST_PER_INPUT_TOKEN: Optional[float] = Field(\n        None, description=\"OpenAI input token cost (used for cost reporting).\"\n    )\n    OPENAI_COST_PER_OUTPUT_TOKEN: Optional[float] = Field(\n        None, description=\"OpenAI output token cost (used for cost reporting).\"\n    )\n    # PortKey\n    USE_PORTKEY_MODEL: Optional[bool] = Field(\n        None, description=\"Select Portkey as the active LLM provider.\"\n    )\n    PORTKEY_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"Portkey API key.\"\n    )\n    PORTKEY_MODEL_NAME: Optional[str] = Field(\n        None, description=\"Portkey model name (as configured in Portkey).\"\n    )\n    PORTKEY_BASE_URL: Optional[AnyUrl] = Field(\n        None, description=\"Portkey base URL (if using a custom endpoint).\"\n    )\n    PORTKEY_PROVIDER_NAME: Optional[str] = Field(\n        None, description=\"Provider name/routing hint for Portkey.\"\n    )\n    # OpenRouter\n    USE_OPENROUTER_MODEL: Optional[bool] = None\n    OPENROUTER_API_KEY: Optional[SecretStr] = None\n    OPENROUTER_MODEL_NAME: Optional[str] = None\n    OPENROUTER_COST_PER_INPUT_TOKEN: Optional[float] = None\n    OPENROUTER_COST_PER_OUTPUT_TOKEN: Optional[float] = None\n    OPENROUTER_BASE_URL: Optional[AnyUrl] = Field(\n        None, description=\"OpenRouter base URL (if using a custom endpoint).\"\n    )\n\n    # Vertex AI\n    VERTEX_AI_MODEL_NAME: Optional[str] = Field(\n        None,\n        description=\"Vertex AI model name (used by some Google integrations).\",\n    )\n    # VLLM\n    VLLM_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"vLLM API key (if required by your vLLM gateway).\"\n    )\n    VLLM_MODEL_NAME: Optional[str] = Field(None, description=\"vLLM model name.\")\n\n    #\n    # Embedding Keys\n    #\n\n    # Azure OpenAI\n    USE_AZURE_OPENAI_EMBEDDING: Optional[bool] = Field(\n        None, description=\"Use Azure OpenAI for embeddings.\"\n    )\n    AZURE_EMBEDDING_MODEL_NAME: Optional[str] = Field(\n        None, description=\"Azure embedding model name label.\"\n    )\n    AZURE_EMBEDDING_DEPLOYMENT_NAME: Optional[str] = Field(\n        None, description=\"Azure embedding deployment name.\"\n    )\n\n    # Local\n    USE_LOCAL_EMBEDDINGS: Optional[bool] = Field(\n        None, description=\"Use a local/self-hosted embeddings endpoint.\"\n    )\n    LOCAL_EMBEDDING_MODEL_NAME: Optional[str] = Field(\n        None,\n        description=\"Local embedding model name (informational / routing).\",\n    )\n    LOCAL_EMBEDDING_BASE_URL: Optional[AnyUrl] = Field(\n        None,\n        description=\"Base URL for a local/self-hosted embeddings endpoint.\",\n    )\n\n    #\n    # Retry Policy\n    #\n    # Controls how Tenacity retries provider calls when the SDK isn't doing its own retries.\n    # Key concepts:\n    # - attempts count includes the first call. e.g. 1 = no retries, 2 = one retry.\n    # - backoff sleeps follow exponential growth with a cap, plus jitter. Expected jitter\n    #   contribution is ~ JITTER/2 per sleep.\n    # - logging levels are looked up dynamically each attempt, so if you change LOG_LEVEL at runtime,\n    #   the retry loggers will honor it without restart.\n    DEEPEVAL_SDK_RETRY_PROVIDERS: Optional[List[str]] = Field(\n        None,\n        description=\"Providers for which retries should be delegated to the provider SDK (use ['*'] for all).\",\n    )\n    DEEPEVAL_RETRY_BEFORE_LOG_LEVEL: Optional[int] = Field(\n        None,\n        description=\"Log level for 'before retry' logs (defaults to LOG_LEVEL if set, else INFO).\",\n    )\n    DEEPEVAL_RETRY_AFTER_LOG_LEVEL: Optional[int] = Field(\n        None,\n        description=\"Log level for 'after retry' logs (defaults to ERROR).\",\n    )\n    DEEPEVAL_RETRY_MAX_ATTEMPTS: conint(ge=1) = Field(\n        2,\n        description=\"Max attempts per provider call (includes the first call; 1 = no retries).\",\n    )\n    DEEPEVAL_RETRY_INITIAL_SECONDS: confloat(ge=0) = Field(\n        1.0,\n        description=\"Initial backoff sleep (seconds) before the first retry.\",\n    )\n    DEEPEVAL_RETRY_EXP_BASE: confloat(ge=1) = Field(\n        2.0, description=\"Exponential backoff growth factor.\"\n    )\n    DEEPEVAL_RETRY_JITTER: confloat(ge=0) = Field(\n        2.0, description=\"Uniform jitter added to each retry sleep (seconds).\"\n    )\n    DEEPEVAL_RETRY_CAP_SECONDS: confloat(ge=0) = Field(\n        5.0, description=\"Maximum backoff sleep per retry (seconds).\"\n    )\n\n    #\n    # Telemetry and Debug\n    #\n    DEEPEVAL_DEBUG_ASYNC: Optional[bool] = Field(\n        None, description=\"Enable extra async debugging logs/behavior.\"\n    )\n    DEEPEVAL_TELEMETRY_OPT_OUT: Optional[bool] = Field(\n        None,\n        description=\"Opt out of DeepEval telemetry (OFF wins if conflicting legacy flags are set).\",\n    )\n    DEEPEVAL_UPDATE_WARNING_OPT_IN: Optional[bool] = Field(\n        None,\n        description=\"Opt in to update warnings in the CLI/runtime when new versions are available.\",\n    )\n    DEEPEVAL_GRPC_LOGGING: Optional[bool] = Field(\n        None,\n        description=\"Enable extra gRPC logging for Confident transport/debugging.\",\n    )\n    GRPC_VERBOSITY: Optional[str] = Field(\n        None, description=\"gRPC verbosity (grpc env var passthrough).\"\n    )\n    GRPC_TRACE: Optional[str] = Field(\n        None, description=\"gRPC trace categories (grpc env var passthrough).\"\n    )\n    ERROR_REPORTING: Optional[bool] = Field(\n        None,\n        description=\"Enable/disable error reporting (implementation/integration dependent).\",\n    )\n    IGNORE_DEEPEVAL_ERRORS: Optional[bool] = Field(\n        None,\n        description=\"Continue execution when DeepEval encounters certain recoverable errors.\",\n    )\n    SKIP_DEEPEVAL_MISSING_PARAMS: Optional[bool] = Field(\n        None,\n        description=\"Skip metrics/test cases with missing required params instead of raising.\",\n    )\n    DEEPEVAL_VERBOSE_MODE: Optional[bool] = Field(\n        None, description=\"Enable verbose logging and additional warnings.\"\n    )\n    DEEPEVAL_LOG_STACK_TRACES: Optional[bool] = Field(\n        None, description=\"Include stack traces in certain DeepEval error logs.\"\n    )\n    ENABLE_DEEPEVAL_CACHE: Optional[bool] = Field(\n        None,\n        description=\"Enable DeepEval caching where supported (may improve performance).\",\n    )\n\n    CONFIDENT_TRACE_FLUSH: Optional[bool] = Field(\n        None,\n        description=\"Flush traces eagerly (useful for debugging; may add overhead).\",\n    )\n    CONFIDENT_TRACE_ENVIRONMENT: Optional[str] = Field(\n        \"development\",\n        description=\"Trace environment label (e.g. development/staging/production).\",\n    )\n    CONFIDENT_TRACE_VERBOSE: Optional[bool] = Field(\n        True, description=\"Enable verbose trace logging for Confident tracing.\"\n    )\n    CONFIDENT_TRACE_SAMPLE_RATE: Optional[float] = Field(\n        1.0, description=\"Trace sampling rate (0–1). Lower to reduce overhead.\"\n    )\n    CONFIDENT_TRACE_INTERNAL: Optional[bool] = Field(\n        None,\n        description=\"Enable detailed internal tracing of metric and model methods inside @observe spans.\",\n    )\n\n    CONFIDENT_OTEL_URL: Optional[AnyUrl] = Field(\n        \"https://otel.confident-ai.com\",\n        description=\"OpenTelemetry OTLP exporter endpoint (if using OTEL export).\",\n    )\n\n    #\n    # Network\n    #\n    MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS: float = Field(\n        3.05,\n        description=\"Connect timeout (seconds) when fetching remote images for multimodal inputs.\",\n    )\n    MEDIA_IMAGE_READ_TIMEOUT_SECONDS: float = Field(\n        10.0,\n        description=\"Read timeout (seconds) when fetching remote images for multimodal inputs.\",\n    )\n    DEEPEVAL_DISABLE_TIMEOUTS: Optional[bool] = Field(\n        None,\n        description=\"Disable DeepEval-enforced timeouts (per-attempt, per-task, gather). Provider SDK timeouts may still apply.\",\n    )\n    # DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE\n    # Per-attempt timeout (seconds) for provider calls used by the retry policy.\n    # This is an OVERRIDE setting. The effective value you should rely on at runtime is\n    # the computed property: DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS.\n    #\n    # If this is None or 0 the DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS is computed from either:\n    #   - DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE: slice the outer budget\n    #     across attempts after subtracting expected backoff and a small safety buffer\n    #   - the default outer budget (180s) if no outer override is set.\n    #\n    # Tip: Set this OR the outer override, but generally not both\n    DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE: Optional[confloat(gt=0)] = (\n        Field(\n            None,\n            description=\"Override per-attempt provider call timeout (seconds). Leave unset to derive from task timeout.\",\n        )\n    )\n\n    #\n    # Async Document Pipelines\n    #\n\n    DEEPEVAL_MAX_CONCURRENT_DOC_PROCESSING: conint(ge=1) = Field(\n        2, description=\"Max concurrent async document processing tasks.\"\n    )\n\n    #\n    # Async Task Configuration\n    #\n    DEEPEVAL_TIMEOUT_THREAD_LIMIT: conint(ge=1) = Field(\n        128,\n        description=\"Max worker threads used for timeout enforcement in async execution.\",\n    )\n    DEEPEVAL_TIMEOUT_SEMAPHORE_WARN_AFTER_SECONDS: confloat(ge=0) = Field(\n        5.0,\n        description=\"Warn if waiting on the timeout semaphore longer than this many seconds.\",\n    )\n    # DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE\n    # Outer time budget (seconds) for a single metric/test-case, including retries and backoff.\n    # This is an OVERRIDE setting. If None or 0 the DEEPEVAL_PER_TASK_TIMEOUT_SECONDS field is computed:\n    #     attempts * per_attempt_timeout + expected_backoff + 1s safety\n    # (When neither override is set 180s is used.)\n    #\n    # If > 0, we use the value exactly and log a warning if it is likely too small\n    # to accommodate the configured attempts/backoff.\n    #\n    # usage:\n    #   - set DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE along with DEEPEVAL_RETRY_MAX_ATTEMPTS, or\n    #   - set DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE alone.\n    DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE: Optional[confloat(ge=0)] = (\n        Field(\n            None,\n            description=\"Override outer per-test-case timeout budget (seconds), including retries/backoff. Leave unset to auto-derive.\",\n        )\n    )\n\n    # Buffer time for gathering results from all tasks, added to the longest task duration\n    # Increase if many tasks are running concurrently\n    # DEEPEVAL_TASK_GATHER_BUFFER_SECONDS: confloat(ge=0) = (\n    #     30  # 15s seemed like not enough. we may make this computed later.\n    # )\n    DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE: Optional[confloat(ge=0)] = (\n        Field(\n            None,\n            description=\"Override buffer added to the longest task duration when gathering async results (seconds).\",\n        )\n    )\n\n    ###################\n    # Computed Fields #\n    ###################\n\n    def _calc_auto_outer_timeout(self) -> float:\n        \"\"\"Compute outer budget from per-attempt timeout + retries/backoff.\n        Never reference the computed property itself here.\n        \"\"\"\n        attempts = self.DEEPEVAL_RETRY_MAX_ATTEMPTS or 1\n        timeout_seconds = float(\n            self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE or 0\n        )\n        if timeout_seconds <= 0:\n            # No per-attempt timeout set -> default outer budget\n            return 180\n\n        backoff = self._expected_backoff(attempts)\n        safety_overhead = 1.0\n        return float(\n            math.ceil(attempts * timeout_seconds + backoff + safety_overhead)\n        )\n\n    @computed_field\n    @property\n    def DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS(self) -> float:\n        over = self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE\n        if over is not None and float(over) > 0:\n            return float(over)\n\n        attempts = int(self.DEEPEVAL_RETRY_MAX_ATTEMPTS or 1)\n        outer_over = self.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE\n\n        # If the user set an outer override, slice it up\n        if outer_over and float(outer_over) > 0 and attempts > 0:\n            backoff = self._expected_backoff(attempts)\n            safety = 1.0\n            usable = max(0.0, float(outer_over) - backoff - safety)\n            return 0.0 if usable <= 0 else (usable / attempts)\n\n        # NEW: when neither override is set, derive from the default outer (180s)\n        default_outer = 180.0\n        backoff = self._expected_backoff(attempts)\n        safety = 1.0\n        usable = max(0.0, default_outer - backoff - safety)\n        # Keep per-attempt sensible (cap to at least 1s)\n        return 0.0 if usable <= 0 else max(1.0, usable / attempts)\n\n    @computed_field\n    @property\n    def DEEPEVAL_PER_TASK_TIMEOUT_SECONDS(self) -> float:\n        \"\"\"If OVERRIDE is set (nonzero), return it; else return the derived budget.\"\"\"\n        outer = self.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE\n        if outer not in (None, 0):\n            # Warn if user-provided outer is likely to truncate retries\n            if (self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0) > 0:\n                min_needed = self._calc_auto_outer_timeout()\n                if float(outer) < min_needed:\n                    if self.DEEPEVAL_VERBOSE_MODE:\n                        logger.warning(\n                            \"Metric timeout (outer=%ss) is less than attempts × per-attempt \"\n                            \"timeout + backoff (≈%ss). Retries may be cut short.\",\n                            float(outer),\n                            min_needed,\n                        )\n            return float(outer)\n\n        # Auto mode\n        return self._calc_auto_outer_timeout()\n\n    @computed_field\n    @property\n    def DEEPEVAL_TASK_GATHER_BUFFER_SECONDS(self) -> float:\n        \"\"\"\n        Buffer time we add to the longest task’s duration to allow gather/drain\n        to complete. If an override is provided, use it; otherwise derive a\n        sensible default from the task-level budget:\n            buffer = constrain_between(0.15 * DEEPEVAL_PER_TASK_TIMEOUT_SECONDS, 10, 60)\n        \"\"\"\n        over = self.DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE\n        if over is not None and float(over) >= 0:\n            return float(over)\n\n        outer = float(self.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS or 0.0)\n        base = 0.15 * outer\n        return constrain_between(base, 10.0, 60.0)\n\n    ##############\n    # Validators #\n    ##############\n\n    @field_validator(\n        \"CONFIDENT_OPEN_BROWSER\",\n        \"CONFIDENT_DISABLE_SSL\",\n        \"CONFIDENT_TRACE_FLUSH\",\n        \"CONFIDENT_TRACE_INTERNAL\",\n        \"CONFIDENT_TRACE_VERBOSE\",\n        \"CUDA_LAUNCH_BLOCKING\",\n        \"DEEPEVAL_DEBUG_ASYNC\",\n        \"DEEPEVAL_LOG_STACK_TRACES\",\n        \"DEEPEVAL_DISABLE_TIMEOUTS\",\n        \"DEEPEVAL_VERBOSE_MODE\",\n        \"DEEPEVAL_GRPC_LOGGING\",\n        \"DEEPEVAL_DISABLE_DOTENV\",\n        \"DEEPEVAL_TELEMETRY_OPT_OUT\",\n        \"DEEPEVAL_UPDATE_WARNING_OPT_IN\",\n        \"ENABLE_DEEPEVAL_CACHE\",\n        \"ERROR_REPORTING\",\n        \"GOOGLE_GENAI_USE_VERTEXAI\",\n        \"IGNORE_DEEPEVAL_ERRORS\",\n        \"SKIP_DEEPEVAL_MISSING_PARAMS\",\n        \"TOKENIZERS_PARALLELISM\",\n        \"TRANSFORMERS_NO_ADVISORY_WARNINGS\",\n        \"USE_AWS_BEDROCK_MODEL\",\n        \"USE_OPENAI_MODEL\",\n        \"USE_AZURE_OPENAI\",\n        \"USE_LOCAL_MODEL\",\n        \"USE_GEMINI_MODEL\",\n        \"USE_MOONSHOT_MODEL\",\n        \"USE_GROK_MODEL\",\n        \"USE_DEEPSEEK_MODEL\",\n        \"USE_LITELLM\",\n        \"USE_AZURE_OPENAI_EMBEDDING\",\n        \"USE_LOCAL_EMBEDDINGS\",\n        \"USE_PORTKEY_MODEL\",\n        mode=\"before\",\n    )\n    @classmethod\n    def _coerce_yes_no(cls, v):\n        return None if v is None else parse_bool(v, default=False)\n\n    @field_validator(\n        \"DEEPEVAL_RESULTS_FOLDER\",\n        \"ENV_DIR_PATH\",\n        \"DEEPEVAL_CACHE_FOLDER\",\n        mode=\"before\",\n    )\n    @classmethod\n    def _coerce_path(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip()\n        if not s:\n            return None\n        # expand ~ and env vars;\n        # but don't resolve to avoid failing on non-existent paths\n        return Path(os.path.expandvars(os.path.expanduser(s)))\n\n    # Treat \"\", \"none\", \"null\" as None for numeric overrides\n    @field_validator(\n        \"OPENAI_COST_PER_INPUT_TOKEN\",\n        \"OPENAI_COST_PER_OUTPUT_TOKEN\",\n        \"GEMINI_COST_PER_INPUT_TOKEN\",\n        \"GEMINI_COST_PER_OUTPUT_TOKEN\",\n        \"AWS_BEDROCK_COST_PER_INPUT_TOKEN\",\n        \"AWS_BEDROCK_COST_PER_OUTPUT_TOKEN\",\n        \"TEMPERATURE\",\n        \"CONFIDENT_TRACE_SAMPLE_RATE\",\n        mode=\"before\",\n    )\n    @classmethod\n    def _none_or_float(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip().lower()\n        if s in {\"\", \"none\", \"null\"}:\n            return None\n        return float(v)\n\n    @field_validator(\"CONFIDENT_TRACE_SAMPLE_RATE\")\n    @classmethod\n    def _validate_sample_rate(cls, v):\n        if v is None:\n            return None\n        if not (0.0 <= float(v) <= 1.0):\n            raise ValueError(\n                \"CONFIDENT_TRACE_SAMPLE_RATE must be between 0 and 1\"\n            )\n        return float(v)\n\n    @field_validator(\"DEEPEVAL_DEFAULT_SAVE\", mode=\"before\")\n    @classmethod\n    def _validate_default_save(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip()\n        if not s:\n            return None\n        m = _SAVE_RE.match(s)\n        if not m:\n            raise ValueError(\n                \"DEEPEVAL_DEFAULT_SAVE must be 'dotenv' or 'dotenv:<path>'\"\n            )\n        path = m.group(\"path\")\n        if path is None:\n            return \"dotenv\"\n        path = os.path.expanduser(os.path.expandvars(path))\n        return f\"dotenv:{path}\"\n\n    @field_validator(\"DEEPEVAL_FILE_SYSTEM\", mode=\"before\")\n    @classmethod\n    def _normalize_fs(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip().upper()\n\n        # adds friendly aliases\n        if s in {\"READ_ONLY\", \"READ-ONLY\", \"READONLY\", \"RO\"}:\n            return \"READ_ONLY\"\n        raise ValueError(\n            \"DEEPEVAL_FILE_SYSTEM must be READ_ONLY (case-insensitive).\"\n        )\n\n    @field_validator(\"CONFIDENT_REGION\", mode=\"before\")\n    @classmethod\n    def _normalize_upper(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip()\n        if not s:\n            return None\n        return s.upper()\n\n    @field_validator(\"AWS_BEDROCK_REGION\", mode=\"before\")\n    @classmethod\n    def _normalize_lower(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip()\n        if not s:\n            return None\n        return s.lower()\n\n    @field_validator(\"DEEPEVAL_SDK_RETRY_PROVIDERS\", mode=\"before\")\n    @classmethod\n    def _coerce_to_list(cls, v):\n        # works with JSON list, comma/space/semicolon separated, or real lists\n        return coerce_to_list(v, lower=True)\n\n    @field_validator(\"DEEPEVAL_SDK_RETRY_PROVIDERS\", mode=\"after\")\n    @classmethod\n    def _validate_sdk_provider_list(cls, v):\n        if v is None:\n            return None\n\n        normalized: list[str] = []\n        star = False\n\n        for item in v:\n            s = str(item).strip()\n            if not s:\n                continue\n            if s == \"*\":\n                star = True\n                continue\n            s = slugify(s)\n            if s in SUPPORTED_PROVIDER_SLUGS:\n                normalized.append(s)\n            else:\n                if parse_bool(\n                    os.getenv(\"DEEPEVAL_VERBOSE_MODE\"), default=False\n                ):\n                    logger.warning(\"Unknown provider slug %r dropped\", item)\n\n        if star:\n            return [\"*\"]\n\n        # It is important to dedup after normalization to catch variants\n        normalized = dedupe_preserve_order(normalized)\n        return normalized or None\n\n    @field_validator(\n        \"DEEPEVAL_RETRY_BEFORE_LOG_LEVEL\",\n        \"DEEPEVAL_RETRY_AFTER_LOG_LEVEL\",\n        \"LOG_LEVEL\",\n        mode=\"before\",\n    )\n    @classmethod\n    def _coerce_log_level(cls, v):\n        if v is None:\n            return None\n        if isinstance(v, (int, float)):\n            return int(v)\n\n        s = str(v).strip().upper()\n        if not s:\n            return None\n\n        import logging\n\n        # Accept standard names or numeric strings\n        name_to_level = {\n            \"CRITICAL\": logging.CRITICAL,\n            \"ERROR\": logging.ERROR,\n            \"WARNING\": logging.WARNING,\n            \"INFO\": logging.INFO,\n            \"DEBUG\": logging.DEBUG,\n            \"NOTSET\": logging.NOTSET,\n        }\n        if s.isdigit() or (s.startswith(\"-\") and s[1:].isdigit()):\n            return int(s)\n        if s in name_to_level:\n            return name_to_level[s]\n        raise ValueError(\n            \"Retry log level must be one of DEBUG, INFO, WARNING, ERROR, \"\n            \"CRITICAL, NOTSET, or a numeric logging level.\"\n        )\n\n    @field_validator(\"DEEPEVAL_TELEMETRY_OPT_OUT\", mode=\"before\")\n    @classmethod\n    def _apply_telemetry_enabled_alias(cls, v):\n        \"\"\"\n        Precedence (most secure):\n        - Any OFF signal wins if both are set:\n          - DEEPEVAL_TELEMETRY_OPT_OUT = truthy  -> OFF\n          - DEEPEVAL_TELEMETRY_ENABLED = falsy   -> OFF\n        - Else, ON signal:\n          - DEEPEVAL_TELEMETRY_OPT_OUT = falsy   -> ON\n          - DEEPEVAL_TELEMETRY_ENABLED = truthy  -> ON\n        - Else None (unset) -> ON\n        \"\"\"\n\n        def normalize(x):\n            if x is None:\n                return None\n            s = str(x).strip()\n            return None if s == \"\" else parse_bool(s, default=False)\n\n        new_opt_out = normalize(v)  # True means OFF, False means ON\n        legacy_enabled = normalize(\n            os.getenv(\"DEEPEVAL_TELEMETRY_ENABLED\")\n        )  # True means ON, False means OFF\n\n        off_signal = (new_opt_out is True) or (legacy_enabled is False)\n        on_signal = (new_opt_out is False) or (legacy_enabled is True)\n\n        # Conflict: simultaneous OFF and ON signals\n        if off_signal and on_signal:\n            # Only warn if verbose or debug\n            if parse_bool(\n                os.getenv(\"DEEPEVAL_VERBOSE_MODE\"), default=False\n            ) or logger.isEnabledFor(logging.DEBUG):\n                logger.warning(\n                    \"Conflicting telemetry flags detected: DEEPEVAL_TELEMETRY_OPT_OUT=%r, \"\n                    \"DEEPEVAL_TELEMETRY_ENABLED=%r. Defaulting to OFF.\",\n                    new_opt_out,\n                    legacy_enabled,\n                )\n            return True  # OFF wins\n\n        # Clear winner\n        if off_signal:\n            return True  # OFF\n        if on_signal:\n            return False  # ON\n\n        # Unset means ON\n        return False\n\n    @model_validator(mode=\"after\")\n    def _apply_deprecated_computed_env_aliases(self):\n        \"\"\"\n        Backwards compatibility courtesy:\n        - If users still set a deprecated computed field in the environment,\n          emit a deprecation warning and mirror its value into the matching\n          *_OVERRIDE field (unless the override is already set).\n        - Override always wins if both are present.\n        \"\"\"\n        for old_key, override_key in _DEPRECATED_TO_OVERRIDE.items():\n            raw = os.getenv(old_key)\n            if raw is None or str(raw).strip() == \"\":\n                continue\n\n            # if override already set, ignore the deprecated one but log a warning\n            if getattr(self, override_key) is not None:\n                logger.warning(\n                    \"Config deprecation: %s is deprecated and was ignored because %s \"\n                    \"is already set. Please remove %s and use %s going forward.\",\n                    old_key,\n                    override_key,\n                    old_key,\n                    override_key,\n                )\n                continue\n\n            # apply the deprecated value into the override field.\n            try:\n                # let pydantic coerce the string to the target type on assignment\n                setattr(self, override_key, raw)\n                logger.warning(\n                    \"Config deprecation: %s is deprecated. Its value (%r) was applied to %s. \"\n                    \"Please migrate to %s and remove %s from your environment.\",\n                    old_key,\n                    raw,\n                    override_key,\n                    override_key,\n                    old_key,\n                )\n            except Exception as e:\n                # do not let exception bubble up, just warn\n                logger.warning(\n                    \"Config deprecation: %s is deprecated and could not be applied to %s \"\n                    \"(value=%r): %s\",\n                    old_key,\n                    override_key,\n                    raw,\n                    e,\n                )\n        return self\n\n    #######################\n    # Persistence support #\n    #######################\n    class _SettingsEditCtx:\n        # TODO: will generate this list in future PR\n        COMPUTED_FIELDS: frozenset[str] = frozenset(\n            {\n                \"DEEPEVAL_PER_TASK_TIMEOUT_SECONDS\",\n                \"DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS\",\n                \"DEEPEVAL_TASK_GATHER_BUFFER_SECONDS\",\n            }\n        )\n\n        def __init__(\n            self,\n            settings: \"Settings\",\n            save: Optional[str],\n            persist: Optional[bool],\n        ):\n            self._s = settings\n            self._save = save\n            self._persist = persist\n            self._before: Dict[str, Any] = {}\n            self._touched: set[str] = set()\n            self.result: Optional[PersistResult] = None\n\n        @property\n        def s(self) -> \"Settings\":\n            return self._s\n\n        def __enter__(self) -> \"Settings._SettingsEditCtx\":\n            # snapshot current state\n            self._token = _ACTIVE_SETTINGS_EDIT_CTX.set(self)\n            self._before = {\n                k: getattr(self._s, k) for k in type(self._s).model_fields\n            }\n            return self\n\n        def __exit__(self, exc_type, exc, tb):\n            try:\n                if exc_type is not None:\n                    return False  # don’t persist on error\n\n                from deepeval.config.settings_manager import (\n                    update_settings_and_persist,\n                    _normalize_for_env,\n                    _resolve_save_path,\n                )\n\n                # lazy import legacy JSON store deps\n                from deepeval.key_handler import KEY_FILE_HANDLER\n\n                model_fields = type(self._s).model_fields\n                # Exclude computed fields from persistence\n\n                # compute diff of changed fields\n                after = {k: getattr(self._s, k) for k in model_fields}\n\n                before_norm = {\n                    k: _normalize_for_env(v) for k, v in self._before.items()\n                }\n                after_norm = {\n                    k: _normalize_for_env(v) for k, v in after.items()\n                }\n\n                changed_keys = {\n                    k for k in after_norm if after_norm[k] != before_norm.get(k)\n                }\n                changed_keys -= self.COMPUTED_FIELDS\n                touched_keys = set(self._touched) - self.COMPUTED_FIELDS\n\n                # dotenv should persist union(changed, touched)\n                persist_dotenv = self._persist is not False\n                ok, resolved_path = _resolve_save_path(self._save)\n\n                existing_dotenv = {}\n                if persist_dotenv and ok and resolved_path is not None:\n                    existing_dotenv = read_dotenv_file(resolved_path)\n\n                candidate_keys_for_dotenv = (\n                    changed_keys | touched_keys\n                ) - self.COMPUTED_FIELDS\n\n                keys_for_dotenv: set[str] = set()\n                for key in candidate_keys_for_dotenv:\n                    desired = after_norm.get(key)  # normalized string or None\n                    if desired is None:\n                        # only need to unset if it's actually present in dotenv\n                        # if key in existing_dotenv:\n                        #     keys_for_dotenv.add(key)\n                        keys_for_dotenv.add(key)\n                    else:\n                        if existing_dotenv.get(key) != desired:\n                            keys_for_dotenv.add(key)\n\n                updates_for_dotenv = {\n                    key: after[key] for key in keys_for_dotenv\n                }\n\n                if not changed_keys and not updates_for_dotenv:\n                    if self._persist is False:\n                        # we report handled so that the cli does not mistakenly report invalid save option\n                        self.result = PersistResult(True, None, {})\n                        return False\n\n                    ok, resolved_path = _resolve_save_path(self._save)\n                    self.result = PersistResult(ok, resolved_path, {})\n                    return False\n\n                updates = {k: after[k] for k in changed_keys}\n\n                if \"LOG_LEVEL\" in updates:\n                    from deepeval.config.logging import (\n                        apply_deepeval_log_level,\n                    )\n\n                    apply_deepeval_log_level()\n\n                #\n                # .deepeval JSON support\n                #\n\n                if self._persist is not False:\n                    for k in changed_keys:\n                        legacy_member = _find_legacy_enum(k)\n                        if legacy_member is None:\n                            continue  # skip if not a defined as legacy field\n\n                        val = updates[k]\n                        # Remove from JSON if unset\n                        if val is None:\n                            KEY_FILE_HANDLER.remove_key(legacy_member)\n                            continue\n\n                        # Never store secrets in the JSON keystore\n                        if _is_secret_key(k):\n                            continue\n\n                        # For booleans, the legacy store expects \"YES\"/\"NO\"\n                        if isinstance(val, bool):\n                            KEY_FILE_HANDLER.write_key(\n                                legacy_member, \"YES\" if val else \"NO\"\n                            )\n                        else:\n                            # store as string\n                            KEY_FILE_HANDLER.write_key(legacy_member, str(val))\n\n                #\n                # dotenv store\n                #\n\n                # defer import to avoid cyclics\n                handled, path = update_settings_and_persist(\n                    updates_for_dotenv,\n                    save=self._save,\n                    persist_dotenv=persist_dotenv,\n                )\n                self.result = PersistResult(handled, path, updates_for_dotenv)\n                return False\n            finally:\n                if self._token is not None:\n                    _ACTIVE_SETTINGS_EDIT_CTX.reset(self._token)\n\n        def switch_model_provider(self, target) -> None:\n            \"\"\"\n            Flip USE_* settings within the target's provider family (LLM vs embeddings).\n            \"\"\"\n            from deepeval.key_handler import KEY_FILE_HANDLER\n\n            target_key = getattr(target, \"value\", str(target))\n\n            def _is_embedding_flag(k: str) -> bool:\n                return \"EMBEDDING\" in k\n\n            target_is_embedding = _is_embedding_flag(target_key)\n\n            use_fields = [\n                field\n                for field in type(self._s).model_fields\n                if field.startswith(\"USE_\")\n                and _is_embedding_flag(field) == target_is_embedding\n            ]\n\n            if target_key not in use_fields:\n                raise ValueError(\n                    f\"{target_key} is not a recognized USE_* field\"\n                )\n\n            for field in use_fields:\n                on = field == target_key\n                # Non-targets are *unset* (None) rather than written as an\n                # explicit False/\"NO\": keeps the keystore and dotenv clean\n                # and preserves Optional[bool] unset-vs-false semantics.\n                # `should_use_*()` treats a missing flag as off.\n                setattr(self._s, field, True if on else None)\n\n                if self._persist is not False:\n                    legacy_member = _find_legacy_enum(field)\n                    if legacy_member is not None:\n                        if on:\n                            KEY_FILE_HANDLER.write_key(legacy_member, \"YES\")\n                        else:\n                            KEY_FILE_HANDLER.remove_key(legacy_member)\n\n    def edit(\n        self, *, save: Optional[str] = None, persist: Optional[bool] = None\n    ):\n        \"\"\"Context manager for atomic, persisted updates.\n\n        Args:\n            save: 'dotenv[:path]' to explicitly write to a dotenv file.\n                  None (default) respects DEEPEVAL_DEFAULT_SAVE if set.\n            persist: If False, do not write (dotenv, JSON), update runtime only.\n                     If True or None, normal persistence rules apply.\n        \"\"\"\n        return self._SettingsEditCtx(self, save, persist)\n\n    def set_model_provider(self, target, *, save: Optional[str] = None):\n        \"\"\"\n        Convenience wrapper to switch providers outside of an existing edit() block.\n        Returns the PersistResult.\n        \"\"\"\n        with self.edit(save=save) as ctx:\n            ctx.switch_model_provider(target)\n        return ctx.result\n\n    def _expected_backoff(self, attempts: int) -> float:\n        \"\"\"Sum of expected sleeps for (attempts-1) retries, including jitter expectation.\"\"\"\n        sleeps = max(0, attempts - 1)\n        cur = float(self.DEEPEVAL_RETRY_INITIAL_SECONDS)\n        cap = float(self.DEEPEVAL_RETRY_CAP_SECONDS)\n        base = float(self.DEEPEVAL_RETRY_EXP_BASE)\n        jitter = float(self.DEEPEVAL_RETRY_JITTER)\n\n        backoff = 0.0\n        for _ in range(sleeps):\n            backoff += min(cap, cur)\n            cur *= base\n        backoff += sleeps * (jitter / 2.0)  # expected jitter\n        return backoff\n\n    def _constrain_between(self, value: float, lo: float, hi: float) -> float:\n        \"\"\"Return value constrained to the inclusive range [lo, hi].\"\"\"\n        return min(max(value, lo), hi)\n\n\n_settings_singleton: Optional[Settings] = None\n_settings_env_fingerprint: Optional[str] = None\n_settings_lock = threading.RLock()\n\n\ndef _calc_env_fingerprint() -> str:\n    # Pull legacy .deepeval JSON-based settings into the process env before hashing\n    _merge_legacy_keyfile_into_env()\n\n    env = os.environ.copy()\n    # must hash in a stable order.\n    keys = sorted(\n        key\n        for key in Settings.model_fields.keys()\n        if key != \"_DEPRECATED_TELEMETRY_ENABLED\"  # exclude deprecated\n    )\n    # encode as triples: (key, present?, value)\n    items = [(k, k in env, env.get(k)) for k in keys]\n    payload = json.dumps(items, ensure_ascii=False, separators=(\",\", \":\"))\n    return hashlib.sha256(payload.encode(\"utf-8\")).hexdigest()\n\n\ndef get_settings() -> Settings:\n    global _settings_singleton, _settings_env_fingerprint\n    fingerprint = _calc_env_fingerprint()\n\n    with _settings_lock:\n        if (\n            _settings_singleton is None\n            or _settings_env_fingerprint != fingerprint\n        ):\n            _settings_singleton = Settings()\n            _settings_env_fingerprint = fingerprint\n            from deepeval.config.logging import apply_deepeval_log_level\n\n            apply_deepeval_log_level()\n        return _settings_singleton\n\n\ndef reset_settings(*, reload_dotenv: bool = False) -> Settings:\n    \"\"\"\n    Drop the cached Settings singleton and rebuild it from the current process\n    environment.\n\n    Args:\n        reload_dotenv: When True, call `autoload_dotenv()` before re-instantiating,\n                       which merges .env values into os.environ (never overwriting\n                       existing process env vars).\n\n    Returns:\n        The fresh Settings instance.\n    \"\"\"\n    global _settings_singleton, _settings_env_fingerprint\n    with _settings_lock:\n        if reload_dotenv:\n            autoload_dotenv()\n        _settings_singleton = None\n        _settings_env_fingerprint = None\n    return get_settings()\n"}, "80bc06ee47898e8e78d3cc38fdfed63036dae975": {"t": 1785768985.225666, "n": 58681, "h": "66c2bf9bb4d96186", "c": "\"\"\"\nCentral config for DeepEval.\n\n- Autoloads dotenv files into os.environ without overwriting existing vars\n  (order: .env -> .env.{APP_ENV} -> .env.local).\n- Defines the Pydantic `Settings` model and `get_settings()` singleton.\n- Exposes an `edit()` context manager that diffs changes and persists them to\n  dotenv and the legacy JSON keystore (non-secret keys only), with validators and\n  type coercion.\n\"\"\"\n\nimport hashlib\nimport json\nimport logging\nimport math\nimport os\nimport re\nimport threading\n\nfrom contextvars import ContextVar\nfrom pathlib import Path\nfrom pydantic import (\n    AnyUrl,\n    computed_field,\n    confloat,\n    conint,\n    Field,\n    field_validator,\n    model_validator,\n    SecretStr,\n    PositiveFloat,\n)\nfrom pydantic_settings import BaseSettings, SettingsConfigDict\nfrom typing import (\n    Any,\n    Dict,\n    List,\n    Optional,\n    Union,\n    NamedTuple,\n    get_args,\n    get_origin,\n)\n\nfrom deepeval.config.utils import (\n    coerce_to_list,\n    constrain_between,\n    dedupe_preserve_order,\n    parse_bool,\n    read_dotenv_file,\n)\nfrom deepeval.constants import SUPPORTED_PROVIDER_SLUGS, slugify\n\nlogger = logging.getLogger(__name__)\n_SAVE_RE = re.compile(r\"^(?P<scheme>dotenv)(?::(?P<path>.+))?$\")\n\n_ACTIVE_SETTINGS_EDIT_CTX: ContextVar[Optional[\"Settings._SettingsEditCtx\"]] = (\n    ContextVar(\"_ACTIVE_SETTINGS_EDIT_CTX\", default=None)\n)\n\n# settings that were converted to computed fields with override counterparts\n_DEPRECATED_TO_OVERRIDE = {\n    \"DEEPEVAL_PER_TASK_TIMEOUT_SECONDS\": \"DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE\",\n    \"DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS\": \"DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE\",\n    \"DEEPEVAL_TASK_GATHER_BUFFER_SECONDS\": \"DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE\",\n}\n# Track which secrets we've warned about when loading from the legacy keyfile\n_LEGACY_KEYFILE_SECRET_WARNED: set[str] = set()\n\n\ndef _find_legacy_enum(env_key: str):\n    from deepeval.key_handler import (\n        ModelKeyValues,\n        EmbeddingKeyValues,\n        KeyValues,\n    )\n\n    enums = (ModelKeyValues, EmbeddingKeyValues, KeyValues)\n\n    for enum in enums:\n        try:\n            return getattr(enum, env_key)\n        except AttributeError:\n            pass\n\n    for enum in enums:\n        for member in enum:\n            if member.value == env_key:\n                return member\n    return None\n\n\ndef _is_secret_key(env_key: str) -> bool:\n    field = Settings.model_fields.get(env_key)\n    if not field:\n        return False\n    if field.annotation is SecretStr:\n        return True\n\n    origin = get_origin(field.annotation)\n    if origin is Union:\n        return any(arg is SecretStr for arg in get_args(field.annotation))\n    return False\n\n\ndef _merge_legacy_keyfile_into_env() -> None:\n    \"\"\"\n    Backwards compatibility: merge values from the legacy .deepeval/.deepeval\n    JSON keystore into os.environ for known Settings fields, without\n    overwriting existing process env vars.\n\n    This runs before we compute the Settings env fingerprint so that Pydantic\n    can see these values on first construction.\n\n    Precedence: process env -> dotenv -> legacy json\n    \"\"\"\n    # if somebody really wants to skip this behavior\n    if parse_bool(os.getenv(\"DEEPEVAL_DISABLE_LEGACY_KEYFILE\"), default=False):\n        return\n\n    from deepeval.constants import HIDDEN_DIR, KEY_FILE\n    from deepeval.key_handler import (\n        KeyValues,\n        ModelKeyValues,\n        EmbeddingKeyValues,\n    )\n\n    key_path = Path(HIDDEN_DIR) / KEY_FILE\n\n    try:\n        with key_path.open(\"r\", encoding=\"utf-8\") as f:\n            try:\n                data = json.load(f)\n            except json.JSONDecodeError:\n                # Corrupted file -> ignore, same as KeyFileHandler\n                return\n    except FileNotFoundError:\n        # No legacy store -> nothing to merge\n        return\n\n    if not isinstance(data, dict):\n        return\n\n    # Map JSON keys (enum .value) -> env keys (enum .name)\n    mapping: Dict[str, str] = {}\n    for enum in (KeyValues, ModelKeyValues, EmbeddingKeyValues):\n        for member in enum:\n            mapping[member.value] = member.name\n\n    for json_key, raw in data.items():\n        env_key = mapping.get(json_key)\n        if not env_key:\n            continue\n\n        # Process env always wins\n        if env_key in os.environ:\n            continue\n        if raw is None:\n            continue\n\n        # Mirror the legacy warning semantics for secrets, but only once per key\n        if env_key not in _LEGACY_KEYFILE_SECRET_WARNED and _is_secret_key(\n            env_key\n        ):\n            logger.warning(\n                \"Reading secret '%s' (legacy key '%s') from legacy %s/%s. \"\n                \"Persisting API keys in plaintext is deprecated. \"\n                \"Move this to your environment (.env / .env.local). \"\n                \"This fallback will be removed in a future release.\",\n                env_key,\n                json_key,\n                HIDDEN_DIR,\n                KEY_FILE,\n            )\n            _LEGACY_KEYFILE_SECRET_WARNED.add(env_key)\n        # Let Settings validators coerce types; we just inject the raw string\n        os.environ[env_key] = str(raw)\n\n\ndef _discover_app_env_from_files(env_dir: Path) -> Optional[str]:\n    # prefer base .env.local, then .env for APP_ENV discovery\n    for name in (\".env.local\", \".env\"):\n        v = read_dotenv_file(env_dir / name).get(\"APP_ENV\")\n        if v:\n            v = str(v).strip()\n            if v:\n                return v\n    return None\n\n\ndef resolve_env_dir() -> Path:\n    \"\"\"Directory containing .env files: ENV_DIR_PATH env var, else CWD.\"\"\"\n    raw_dir = os.getenv(\"ENV_DIR_PATH\")\n    if raw_dir:\n        return Path(os.path.expanduser(os.path.expandvars(raw_dir)))\n    return Path(os.getcwd())\n\n\ndef _resolve_app_env(env_dir: Path) -> Optional[str]:\n    # Pick APP_ENV (process -> .env.local -> .env -> none)\n    app_env = os.getenv(\"APP_ENV\") or _discover_app_env_from_files(env_dir)\n    if app_env is None:\n        return None\n    app_env = str(app_env).strip()\n    return app_env or None\n\n\ndef dotenv_search_paths(\n    env_dir: Optional[Path] = None, app_env: Optional[str] = None\n) -> List[Path]:\n    \"\"\"Dotenv files in load order (lowest -> highest precedence):\n    .env -> .env.{APP_ENV} -> .env.local.\n\n    Single source of truth for which files `autoload_dotenv()` reads;\n    `deepeval diagnose` uses it too so the display cannot drift from the\n    loader.\n    \"\"\"\n    env_dir = env_dir if env_dir is not None else resolve_env_dir()\n    app_env = app_env if app_env is not None else _resolve_app_env(env_dir)\n\n    paths = [env_dir / \".env\"]\n    if app_env:\n        paths.append(env_dir / f\".env.{app_env}\")\n    paths.append(env_dir / \".env.local\")\n    return paths\n\n\ndef autoload_dotenv() -> None:\n    \"\"\"\n    Load env vars from .env files without overriding existing process env.\n\n    Precedence (lowest -> highest): .env -> .env.{APP_ENV} -> .env.local\n    Process env always wins over file values.\n\n    Controls:\n      - DEEPEVAL_DISABLE_DOTENV=1 -> skip\n      - ENV_DIR_PATH -> directory containing .env files (default: CWD)\n    \"\"\"\n    if parse_bool(os.getenv(\"DEEPEVAL_DISABLE_DOTENV\"), default=False):\n        return\n\n    env_dir = resolve_env_dir()\n    app_env = _resolve_app_env(env_dir)\n\n    merged: Dict[str, str] = {}\n    if app_env:\n        # Seed the discovered APP_ENV; file values below may override it.\n        merged[\"APP_ENV\"] = app_env\n\n    # merge files in precedence order (later wins)\n    for path in dotenv_search_paths(env_dir, app_env):\n        merged.update(read_dotenv_file(path))\n\n    # Write only keys that aren’t already in process env\n    for k, v in merged.items():\n        if k not in os.environ:\n            os.environ[k] = v\n\n\nclass PersistResult(NamedTuple):\n    handled: bool\n    path: Optional[Path]\n    updated: Dict[str, Any]  # typed, validated and changed\n\n\nclass Settings(BaseSettings):\n    # def __init__(self):\n    #     super().__init__()\n    def __setattr__(self, name: str, value):\n        ctx = _ACTIVE_SETTINGS_EDIT_CTX.get()\n        if ctx is not None and name in type(self).model_fields:\n            ctx._touched.add(name)\n        return super().__setattr__(name, value)\n\n    model_config = SettingsConfigDict(\n        extra=\"ignore\",\n        case_sensitive=True,\n        validate_assignment=True,\n    )\n\n    #\n    # General\n    #\n\n    APP_ENV: str = Field(\n        \"dev\",\n        description=\"Application environment name used for dotenv selection (loads .env.<APP_ENV> if present).\",\n    )\n    LOG_LEVEL: Optional[int] = Field(\n        None,\n        description=\"Global logging level (e.g. DEBUG/INFO/WARNING/ERROR/CRITICAL or numeric).\",\n    )\n    PYTHONPATH: str = Field(\n        \".\",\n        description=\"Extra PYTHONPATH used by the CLI runner (default: current project '.').\",\n    )\n    CONFIDENT_REGION: Optional[str] = Field(\n        None,\n        description=\"Optional Confident AI region hint (uppercased).\",\n    )\n    CONFIDENT_OPEN_BROWSER: Optional[bool] = Field(\n        True,\n        description=\"Open a browser automatically for Confident AI links/flows when available.\",\n    )\n\n    #\n    # CLI\n    #\n    DEEPEVAL_DEFAULT_SAVE: Optional[str] = Field(\n        None,\n        description=\"Default persistence target for settings changes (e.g. 'dotenv' or 'dotenv:/path/to/.env.local').\",\n    )\n    DEEPEVAL_DISABLE_DOTENV: Optional[bool] = Field(\n        None,\n        description=\"Disable dotenv autoloading (.env → .env.<APP_ENV> → .env.local). Tip: set to 1 in pytest/CI to prevent loading env files on import.\",\n    )\n    ENV_DIR_PATH: Optional[Path] = Field(\n        None,\n        description=\"Directory containing .env files (default: current working directory).\",\n    )\n    DEEPEVAL_FILE_SYSTEM: Optional[str] = Field(\n        None,\n        description=\"Filesystem mode for runtime/CLI (currently supports READ_ONLY).\",\n    )\n    DEEPEVAL_IDENTIFIER: Optional[str] = Field(\n        None,\n        description=\"Identifier/tag to help identify your test run on Confident AI.\",\n    )\n\n    #\n    # Storage & Output\n    #\n\n    # When set, DeepEval will export a timestamped JSON of the latest test run\n    # into this directory. The directory will be created on demand.\n    DEEPEVAL_RESULTS_FOLDER: Optional[Path] = Field(\n        None,\n        description=\"If set, export a timestamped JSON of the latest test run into this folder (created if missing).\",\n    )\n\n    # When set, overrides the default DeepEval cache directory\n    DEEPEVAL_CACHE_FOLDER: Optional[Path] = Field(\n        \".deepeval\",\n        description=\"Path to the directory used by DeepEval to store cache files. If set, this overrides the default cache location. The directory will be created if it does not exist.\",\n    )\n\n    # Display / Truncation\n    DEEPEVAL_MAXLEN_TINY: Optional[int] = Field(\n        40,\n        description=\"Default truncation length for 'tiny' displays in logs/UI.\",\n    )\n    DEEPEVAL_MAXLEN_SHORT: Optional[int] = Field(\n        60,\n        description=\"Default truncation length for 'short' displays in logs/UI.\",\n    )\n    DEEPEVAL_MAXLEN_MEDIUM: Optional[int] = Field(\n        120,\n        description=\"Default truncation length for 'medium' displays in logs/UI.\",\n    )\n    DEEPEVAL_MAXLEN_LONG: Optional[int] = Field(\n        240,\n        description=\"Default truncation length for 'long' displays in logs/UI.\",\n    )\n\n    # If set, this overrides the default max_len used by deepeval/utils shorten\n    # falls back to DEEPEVAL_MAXLEN_LONG when None.\n    DEEPEVAL_SHORTEN_DEFAULT_MAXLEN: Optional[int] = Field(\n        None,\n        description=\"Override default max_len for deepeval.utils.shorten (falls back to DEEPEVAL_MAXLEN_LONG when unset).\",\n    )\n\n    # Optional global suffix (keeps your \"...\" default).\n    DEEPEVAL_SHORTEN_SUFFIX: Optional[str] = Field(\n        \"...\",\n        description=\"Suffix appended by deepeval.utils.shorten when truncating (default: '...').\",\n    )\n\n    #\n    # GPU and perf toggles\n    #\n\n    CUDA_LAUNCH_BLOCKING: Optional[bool] = Field(\n        None,\n        description=\"CUDA debug toggle (forces synchronous CUDA ops). Useful for debugging GPU errors.\",\n    )\n    CUDA_VISIBLE_DEVICES: Optional[str] = Field(\n        None,\n        description=\"CUDA device visibility mask (e.g. '0' or '0,1').\",\n    )\n    TOKENIZERS_PARALLELISM: Optional[bool] = Field(\n        None,\n        description=\"HuggingFace tokenizers parallelism toggle (set to false to reduce warnings/noise).\",\n    )\n    TRANSFORMERS_NO_ADVISORY_WARNINGS: Optional[bool] = Field(\n        None,\n        description=\"Disable advisory warnings from transformers (reduces console noise).\",\n    )\n\n    #\n    # Model Keys\n    #\n\n    CONFIDENT_API_KEY: Optional[SecretStr] = Field(\n        None,\n        description=\"Confident AI API key (used for uploading results/telemetry to Confident).\",\n    )\n\n    # ======\n    # Base URL for Confident AI API server\n    # ======\n    CONFIDENT_BASE_URL: Optional[str] = Field(\n        None,\n        description=\"Base URL for Confident AI API server (set only if using a custom/hosted endpoint).\",\n    )\n\n    CONFIDENT_DISABLE_SSL: Optional[bool] = Field(\n        None,\n        description=\"Disable SSL/TLS certificate verification for requests to the Confident AI API server (only use for self-hosted endpoints with self-signed certificates).\",\n    )\n\n    # General\n    TEMPERATURE: Optional[confloat(ge=0, le=2)] = Field(\n        None,\n        description=\"Global default model temperature (0–2). Model-specific constructors may override.\",\n    )\n\n    # Anthropic\n    USE_ANTHROPIC_MODEL: Optional[bool] = Field(\n        None,\n        description=\"Select Anthropic as the active LLM provider (USE_* flags are mutually exclusive in CLI helpers).\",\n    )\n    ANTHROPIC_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"Anthropic API key.\"\n    )\n    ANTHROPIC_MODEL_NAME: Optional[str] = Field(\n        None, description=\"Anthropic model name (e.g. 'claude-3-...').\"\n    )\n    ANTHROPIC_COST_PER_INPUT_TOKEN: Optional[PositiveFloat] = Field(\n        None,\n        description=\"Anthropic input token cost (used for cost reporting).\",\n    )\n    ANTHROPIC_COST_PER_OUTPUT_TOKEN: Optional[PositiveFloat] = Field(\n        None,\n        description=\"Anthropic output token cost (used for cost reporting).\",\n    )\n\n    # AWS\n    AWS_ACCESS_KEY_ID: Optional[SecretStr] = Field(\n        None,\n        description=\"AWS access key ID (for Bedrock or other AWS-backed integrations).\",\n    )\n    AWS_SECRET_ACCESS_KEY: Optional[SecretStr] = Field(\n        None,\n        description=\"AWS secret access key (for Bedrock or other AWS-backed integrations).\",\n    )\n    AWS_SESSION_TOKEN: Optional[SecretStr] = Field(\n        None,\n        description=\"AWS session token (for temporary credentials with Bedrock or other AWS-backed integrations).\",\n    )\n    # AWS Bedrock\n    USE_AWS_BEDROCK_MODEL: Optional[bool] = Field(\n        None, description=\"Select AWS Bedrock as the active LLM provider.\"\n    )\n    AWS_BEDROCK_MODEL_NAME: Optional[str] = Field(\n        None, description=\"AWS Bedrock model identifier.\"\n    )\n    AWS_BEDROCK_REGION: Optional[str] = Field(\n        None, description=\"AWS region for Bedrock (normalized to lowercase).\"\n    )\n    AWS_BEDROCK_COST_PER_INPUT_TOKEN: Optional[PositiveFloat] = Field(\n        None, description=\"Bedrock input token cost (used for cost reporting).\"\n    )\n    AWS_BEDROCK_COST_PER_OUTPUT_TOKEN: Optional[PositiveFloat] = Field(\n        None, description=\"Bedrock output token cost (used for cost reporting).\"\n    )\n    # Azure Open AI\n    USE_AZURE_OPENAI: Optional[bool] = Field(\n        None, description=\"Select Azure OpenAI as the active LLM provider.\"\n    )\n    AZURE_OPENAI_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"Azure OpenAI API key.\"\n    )\n    AZURE_OPENAI_AD_TOKEN: Optional[SecretStr] = Field(\n        None, description=\"Azure OpenAI Ad Token.\"\n    )\n    AZURE_OPENAI_ENDPOINT: Optional[AnyUrl] = Field(\n        None, description=\"Azure OpenAI endpoint URL.\"\n    )\n    OPENAI_API_VERSION: Optional[str] = Field(\n        None,\n        description=\"Azure OpenAI API version (if required by your deployment).\",\n    )\n    AZURE_DEPLOYMENT_NAME: Optional[str] = Field(\n        None,\n        description=\"Azure OpenAI deployment name (required for most Azure configs).\",\n    )\n    AZURE_MODEL_NAME: Optional[str] = Field(\n        None,\n        description=\"Azure model name label (informational; may be used in reporting).\",\n    )\n    AZURE_MODEL_VERSION: Optional[str] = Field(\n        None,\n        description=\"Azure model version label (informational; may be used in reporting).\",\n    )\n    # DeepSeek\n    USE_DEEPSEEK_MODEL: Optional[bool] = Field(\n        None, description=\"Select DeepSeek as the active LLM provider.\"\n    )\n    DEEPSEEK_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"DeepSeek API key.\"\n    )\n    DEEPSEEK_MODEL_NAME: Optional[str] = Field(\n        None, description=\"DeepSeek model name.\"\n    )\n    DEEPSEEK_COST_PER_INPUT_TOKEN: Optional[float] = Field(\n        None, description=\"DeepSeek input token cost (used for cost reporting).\"\n    )\n    DEEPSEEK_COST_PER_OUTPUT_TOKEN: Optional[float] = Field(\n        None,\n        description=\"DeepSeek output token cost (used for cost reporting).\",\n    )\n    # Gemini\n    USE_GEMINI_MODEL: Optional[bool] = Field(\n        None, description=\"Select Google Gemini as the active LLM provider.\"\n    )\n    GOOGLE_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"Google API key for Gemini (non-Vertex usage).\"\n    )\n    GEMINI_MODEL_NAME: Optional[str] = Field(\n        None, description=\"Gemini model name (e.g. 'gemini-...').\"\n    )\n    GOOGLE_GENAI_USE_VERTEXAI: Optional[bool] = Field(\n        None,\n        description=\"Use Vertex AI for Gemini requests instead of direct API key mode.\",\n    )\n    GOOGLE_CLOUD_PROJECT: Optional[str] = Field(\n        None,\n        description=\"GCP project ID for Vertex AI (required if GOOGLE_GENAI_USE_VERTEXAI=true).\",\n    )\n    GOOGLE_CLOUD_LOCATION: Optional[str] = Field(\n        None,\n        description=\"GCP region/location for Vertex AI (e.g. 'us-central1').\",\n    )\n    GOOGLE_SERVICE_ACCOUNT_KEY: Optional[SecretStr] = Field(\n        None,\n        description=\"Service account JSON key for Vertex AI auth (if not using ADC).\",\n    )\n    GEMINI_COST_PER_INPUT_TOKEN: Optional[float] = Field(\n        None, description=\"Gemini input token cost (used for cost reporting).\"\n    )\n    GEMINI_COST_PER_OUTPUT_TOKEN: Optional[float] = Field(\n        None, description=\"Gemini output token cost (used for cost reporting).\"\n    )\n    # Grok\n    USE_GROK_MODEL: Optional[bool] = Field(\n        None, description=\"Select Grok as the active LLM provider.\"\n    )\n    GROK_API_KEY: Optional[SecretStr] = Field(None, description=\"Grok API key.\")\n    GROK_MODEL_NAME: Optional[str] = Field(None, description=\"Grok model name.\")\n    GROK_COST_PER_INPUT_TOKEN: Optional[float] = Field(\n        None, description=\"Grok input token cost (used for cost reporting).\"\n    )\n    GROK_COST_PER_OUTPUT_TOKEN: Optional[float] = Field(\n        None, description=\"Grok output token cost (used for cost reporting).\"\n    )\n    # LiteLLM\n    USE_LITELLM: Optional[bool] = Field(\n        None, description=\"Select LiteLLM as the active LLM provider.\"\n    )\n    LITELLM_API_KEY: Optional[SecretStr] = Field(\n        None,\n        description=\"LiteLLM API key (if required by your LiteLLM deployment).\",\n    )\n    LITELLM_MODEL_NAME: Optional[str] = Field(\n        None,\n        description=\"LiteLLM model name (as exposed by your LiteLLM endpoint).\",\n    )\n    LITELLM_API_BASE: Optional[AnyUrl] = Field(\n        None, description=\"LiteLLM API base URL (direct).\"\n    )\n    LITELLM_PROXY_API_BASE: Optional[AnyUrl] = Field(\n        None, description=\"LiteLLM proxy base URL (if using proxy mode).\"\n    )\n    LITELLM_PROXY_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"LiteLLM proxy API key (if required).\"\n    )\n    # LM Studio\n    LM_STUDIO_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"LM Studio API key (if configured).\"\n    )\n    LM_STUDIO_MODEL_NAME: Optional[str] = Field(\n        None, description=\"LM Studio model name.\"\n    )\n    # Local Model\n    USE_LOCAL_MODEL: Optional[bool] = Field(\n        None,\n        description=\"Select a local/self-hosted model as the active LLM provider.\",\n    )\n    LOCAL_MODEL_API_KEY: Optional[SecretStr] = Field(\n        None,\n        description=\"API key for a local/self-hosted LLM endpoint (if required).\",\n    )\n    LOCAL_EMBEDDING_API_KEY: Optional[SecretStr] = Field(\n        None,\n        description=\"API key for a local/self-hosted embedding endpoint (if required).\",\n    )\n    LOCAL_MODEL_NAME: Optional[str] = Field(\n        None,\n        description=\"Local/self-hosted model name (informational / routing).\",\n    )\n    LOCAL_MODEL_BASE_URL: Optional[AnyUrl] = Field(\n        None, description=\"Base URL for a local/self-hosted LLM endpoint.\"\n    )\n    LOCAL_MODEL_FORMAT: Optional[str] = Field(\n        None,\n        description=\"Local model API format identifier (implementation-specific).\",\n    )\n    # Moonshot\n    USE_MOONSHOT_MODEL: Optional[bool] = Field(\n        None, description=\"Select Moonshot as the active LLM provider.\"\n    )\n    MOONSHOT_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"Moonshot API key.\"\n    )\n    MOONSHOT_MODEL_NAME: Optional[str] = Field(\n        None, description=\"Moonshot model name.\"\n    )\n    MOONSHOT_COST_PER_INPUT_TOKEN: Optional[float] = Field(\n        None, description=\"Moonshot input token cost (used for cost reporting).\"\n    )\n    MOONSHOT_COST_PER_OUTPUT_TOKEN: Optional[float] = Field(\n        None,\n        description=\"Moonshot output token cost (used for cost reporting).\",\n    )\n    # Ollama\n    OLLAMA_MODEL_NAME: Optional[str] = Field(\n        None,\n        description=\"Ollama model name (used when running via Ollama integration).\",\n    )\n    # OpenAI\n    USE_OPENAI_MODEL: Optional[bool] = Field(\n        None, description=\"Select OpenAI as the active LLM provider.\"\n    )\n    OPENAI_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"OpenAI API key.\"\n    )\n    OPENAI_MODEL_NAME: Optional[str] = Field(\n        None, description=\"OpenAI model name (e.g. 'gpt-4.1').\"\n    )\n    OPENAI_COST_PER_INPUT_TOKEN: Optional[float] = Field(\n        None, description=\"OpenAI input token cost (used for cost reporting).\"\n    )\n    OPENAI_COST_PER_OUTPUT_TOKEN: Optional[float] = Field(\n        None, description=\"OpenAI output token cost (used for cost reporting).\"\n    )\n    # PortKey\n    USE_PORTKEY_MODEL: Optional[bool] = Field(\n        None, description=\"Select Portkey as the active LLM provider.\"\n    )\n    PORTKEY_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"Portkey API key.\"\n    )\n    PORTKEY_MODEL_NAME: Optional[str] = Field(\n        None, description=\"Portkey model name (as configured in Portkey).\"\n    )\n    PORTKEY_BASE_URL: Optional[AnyUrl] = Field(\n        None, description=\"Portkey base URL (if using a custom endpoint).\"\n    )\n    PORTKEY_PROVIDER_NAME: Optional[str] = Field(\n        None, description=\"Provider name/routing hint for Portkey.\"\n    )\n    # OpenRouter\n    USE_OPENROUTER_MODEL: Optional[bool] = None\n    OPENROUTER_API_KEY: Optional[SecretStr] = None\n    OPENROUTER_MODEL_NAME: Optional[str] = None\n    OPENROUTER_COST_PER_INPUT_TOKEN: Optional[float] = None\n    OPENROUTER_COST_PER_OUTPUT_TOKEN: Optional[float] = None\n    OPENROUTER_BASE_URL: Optional[AnyUrl] = Field(\n        None, description=\"OpenRouter base URL (if using a custom endpoint).\"\n    )\n\n    # Vertex AI\n    VERTEX_AI_MODEL_NAME: Optional[str] = Field(\n        None,\n        description=\"Vertex AI model name (used by some Google integrations).\",\n    )\n    # VLLM\n    VLLM_API_KEY: Optional[SecretStr] = Field(\n        None, description=\"vLLM API key (if required by your vLLM gateway).\"\n    )\n    VLLM_MODEL_NAME: Optional[str] = Field(None, description=\"vLLM model name.\")\n\n    #\n    # Embedding Keys\n    #\n\n    # Azure OpenAI\n    USE_AZURE_OPENAI_EMBEDDING: Optional[bool] = Field(\n        None, description=\"Use Azure OpenAI for embeddings.\"\n    )\n    AZURE_EMBEDDING_MODEL_NAME: Optional[str] = Field(\n        None, description=\"Azure embedding model name label.\"\n    )\n    AZURE_EMBEDDING_DEPLOYMENT_NAME: Optional[str] = Field(\n        None, description=\"Azure embedding deployment name.\"\n    )\n\n    # Local\n    USE_LOCAL_EMBEDDINGS: Optional[bool] = Field(\n        None, description=\"Use a local/self-hosted embeddings endpoint.\"\n    )\n    LOCAL_EMBEDDING_MODEL_NAME: Optional[str] = Field(\n        None,\n        description=\"Local embedding model name (informational / routing).\",\n    )\n    LOCAL_EMBEDDING_BASE_URL: Optional[AnyUrl] = Field(\n        None,\n        description=\"Base URL for a local/self-hosted embeddings endpoint.\",\n    )\n\n    #\n    # Retry Policy\n    #\n    # Controls how Tenacity retries provider calls when the SDK isn't doing its own retries.\n    # Key concepts:\n    # - attempts count includes the first call. e.g. 1 = no retries, 2 = one retry.\n    # - backoff sleeps follow exponential growth with a cap, plus jitter. Expected jitter\n    #   contribution is ~ JITTER/2 per sleep.\n    # - logging levels are looked up dynamically each attempt, so if you change LOG_LEVEL at runtime,\n    #   the retry loggers will honor it without restart.\n    DEEPEVAL_SDK_RETRY_PROVIDERS: Optional[List[str]] = Field(\n        None,\n        description=\"Providers for which retries should be delegated to the provider SDK (use ['*'] for all).\",\n    )\n    DEEPEVAL_RETRY_BEFORE_LOG_LEVEL: Optional[int] = Field(\n        None,\n        description=\"Log level for 'before retry' logs (defaults to LOG_LEVEL if set, else INFO).\",\n    )\n    DEEPEVAL_RETRY_AFTER_LOG_LEVEL: Optional[int] = Field(\n        None,\n        description=\"Log level for 'after retry' logs (defaults to ERROR).\",\n    )\n    DEEPEVAL_RETRY_MAX_ATTEMPTS: conint(ge=1) = Field(\n        2,\n        description=\"Max attempts per provider call (includes the first call; 1 = no retries).\",\n    )\n    DEEPEVAL_RETRY_INITIAL_SECONDS: confloat(ge=0) = Field(\n        1.0,\n        description=\"Initial backoff sleep (seconds) before the first retry.\",\n    )\n    DEEPEVAL_RETRY_EXP_BASE: confloat(ge=1) = Field(\n        2.0, description=\"Exponential backoff growth factor.\"\n    )\n    DEEPEVAL_RETRY_JITTER: confloat(ge=0) = Field(\n        2.0, description=\"Uniform jitter added to each retry sleep (seconds).\"\n    )\n    DEEPEVAL_RETRY_CAP_SECONDS: confloat(ge=0) = Field(\n        5.0, description=\"Maximum backoff sleep per retry (seconds).\"\n    )\n\n    #\n    # Telemetry and Debug\n    #\n    DEEPEVAL_DEBUG_ASYNC: Optional[bool] = Field(\n        None, description=\"Enable extra async debugging logs/behavior.\"\n    )\n    DEEPEVAL_TELEMETRY_OPT_OUT: Optional[bool] = Field(\n        None,\n        description=\"Opt out of DeepEval telemetry (OFF wins if conflicting legacy flags are set).\",\n    )\n    DEEPEVAL_UPDATE_WARNING_OPT_IN: Optional[bool] = Field(\n        None,\n        description=\"Opt in to update warnings in the CLI/runtime when new versions are available.\",\n    )\n    DEEPEVAL_GRPC_LOGGING: Optional[bool] = Field(\n        None,\n        description=\"Enable extra gRPC logging for Confident transport/debugging.\",\n    )\n    GRPC_VERBOSITY: Optional[str] = Field(\n        None, description=\"gRPC verbosity (grpc env var passthrough).\"\n    )\n    GRPC_TRACE: Optional[str] = Field(\n        None, description=\"gRPC trace categories (grpc env var passthrough).\"\n    )\n    ERROR_REPORTING: Optional[bool] = Field(\n        None,\n        description=\"Enable/disable error reporting (implementation/integration dependent).\",\n    )\n    IGNORE_DEEPEVAL_ERRORS: Optional[bool] = Field(\n        None,\n        description=\"Continue execution when DeepEval encounters certain recoverable errors.\",\n    )\n    SKIP_DEEPEVAL_MISSING_PARAMS: Optional[bool] = Field(\n        None,\n        description=\"Skip metrics/test cases with missing required params instead of raising.\",\n    )\n    DEEPEVAL_VERBOSE_MODE: Optional[bool] = Field(\n        None, description=\"Enable verbose logging and additional warnings.\"\n    )\n    DEEPEVAL_LOG_STACK_TRACES: Optional[bool] = Field(\n        None, description=\"Include stack traces in certain DeepEval error logs.\"\n    )\n    ENABLE_DEEPEVAL_CACHE: Optional[bool] = Field(\n        None,\n        description=\"Enable DeepEval caching where supported (may improve performance).\",\n    )\n\n    CONFIDENT_TRACE_FLUSH: Optional[bool] = Field(\n        None,\n        description=\"Flush traces eagerly (useful for debugging; may add overhead).\",\n    )\n    CONFIDENT_TRACE_ENVIRONMENT: Optional[str] = Field(\n        \"development\",\n        description=\"Trace environment label (e.g. development/staging/production).\",\n    )\n    CONFIDENT_TRACE_VERBOSE: Optional[bool] = Field(\n        True, description=\"Enable verbose trace logging for Confident tracing.\"\n    )\n    CONFIDENT_TRACE_SAMPLE_RATE: Optional[float] = Field(\n        1.0, description=\"Trace sampling rate (0–1). Lower to reduce overhead.\"\n    )\n    CONFIDENT_TRACE_INTERNAL: Optional[bool] = Field(\n        None,\n        description=\"Enable detailed internal tracing of metric and model methods inside @observe spans.\",\n    )\n\n    CONFIDENT_OTEL_URL: Optional[AnyUrl] = Field(\n        \"https://otel.confident-ai.com\",\n        description=\"OpenTelemetry OTLP exporter endpoint (if using OTEL export).\",\n    )\n\n    #\n    # Network\n    #\n    MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS: float = Field(\n        3.05,\n        description=\"Connect timeout (seconds) when fetching remote images for multimodal inputs.\",\n    )\n    MEDIA_IMAGE_READ_TIMEOUT_SECONDS: float = Field(\n        10.0,\n        description=\"Read timeout (seconds) when fetching remote images for multimodal inputs.\",\n    )\n    DEEPEVAL_DISABLE_TIMEOUTS: Optional[bool] = Field(\n        None,\n        description=\"Disable DeepEval-enforced timeouts (per-attempt, per-task, gather). Provider SDK timeouts may still apply.\",\n    )\n    # DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE\n    # Per-attempt timeout (seconds) for provider calls used by the retry policy.\n    # This is an OVERRIDE setting. The effective value you should rely on at runtime is\n    # the computed property: DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS.\n    #\n    # If this is None or 0 the DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS is computed from either:\n    #   - DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE: slice the outer budget\n    #     across attempts after subtracting expected backoff and a small safety buffer\n    #   - the default outer budget (180s) if no outer override is set.\n    #\n    # Tip: Set this OR the outer override, but generally not both\n    DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE: Optional[confloat(gt=0)] = (\n        Field(\n            None,\n            description=\"Override per-attempt provider call timeout (seconds). Leave unset to derive from task timeout.\",\n        )\n    )\n\n    #\n    # Async Document Pipelines\n    #\n\n    DEEPEVAL_MAX_CONCURRENT_DOC_PROCESSING: conint(ge=1) = Field(\n        2, description=\"Max concurrent async document processing tasks.\"\n    )\n\n    #\n    # Async Task Configuration\n    #\n    DEEPEVAL_TIMEOUT_THREAD_LIMIT: conint(ge=1) = Field(\n        128,\n        description=\"Max worker threads used for timeout enforcement in async execution.\",\n    )\n    DEEPEVAL_TIMEOUT_SEMAPHORE_WARN_AFTER_SECONDS: confloat(ge=0) = Field(\n        5.0,\n        description=\"Warn if waiting on the timeout semaphore longer than this many seconds.\",\n    )\n    # DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE\n    # Outer time budget (seconds) for a single metric/test-case, including retries and backoff.\n    # This is an OVERRIDE setting. If None or 0 the DEEPEVAL_PER_TASK_TIMEOUT_SECONDS field is computed:\n    #     attempts * per_attempt_timeout + expected_backoff + 1s safety\n    # (When neither override is set 180s is used.)\n    #\n    # If > 0, we use the value exactly and log a warning if it is likely too small\n    # to accommodate the configured attempts/backoff.\n    #\n    # usage:\n    #   - set DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE along with DEEPEVAL_RETRY_MAX_ATTEMPTS, or\n    #   - set DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE alone.\n    DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE: Optional[confloat(ge=0)] = (\n        Field(\n            None,\n            description=\"Override outer per-test-case timeout budget (seconds), including retries/backoff. Leave unset to auto-derive.\",\n        )\n    )\n\n    # Buffer time for gathering results from all tasks, added to the longest task duration\n    # Increase if many tasks are running concurrently\n    # DEEPEVAL_TASK_GATHER_BUFFER_SECONDS: confloat(ge=0) = (\n    #     30  # 15s seemed like not enough. we may make this computed later.\n    # )\n    DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE: Optional[confloat(ge=0)] = (\n        Field(\n            None,\n            description=\"Override buffer added to the longest task duration when gathering async results (seconds).\",\n        )\n    )\n\n    ###################\n    # Computed Fields #\n    ###################\n\n    def _calc_auto_outer_timeout(self) -> float:\n        \"\"\"Compute outer budget from per-attempt timeout + retries/backoff.\n        Never reference the computed property itself here.\n        \"\"\"\n        attempts = self.DEEPEVAL_RETRY_MAX_ATTEMPTS or 1\n        timeout_seconds = float(\n            self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE or 0\n        )\n        if timeout_seconds <= 0:\n            # No per-attempt timeout set -> default outer budget\n            return 180\n\n        backoff = self._expected_backoff(attempts)\n        safety_overhead = 1.0\n        return float(\n            math.ceil(attempts * timeout_seconds + backoff + safety_overhead)\n        )\n\n    @computed_field\n    @property\n    def DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS(self) -> float:\n        over = self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE\n        if over is not None and float(over) > 0:\n            return float(over)\n\n        attempts = int(self.DEEPEVAL_RETRY_MAX_ATTEMPTS or 1)\n        outer_over = self.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE\n\n        # If the user set an outer override, slice it up\n        if outer_over and float(outer_over) > 0 and attempts > 0:\n            backoff = self._expected_backoff(attempts)\n            safety = 1.0\n            usable = max(0.0, float(outer_over) - backoff - safety)\n            return 0.0 if usable <= 0 else (usable / attempts)\n\n        # NEW: when neither override is set, derive from the default outer (180s)\n        default_outer = 180.0\n        backoff = self._expected_backoff(attempts)\n        safety = 1.0\n        usable = max(0.0, default_outer - backoff - safety)\n        # Keep per-attempt sensible (cap to at least 1s)\n        return 0.0 if usable <= 0 else max(1.0, usable / attempts)\n\n    @computed_field\n    @property\n    def DEEPEVAL_PER_TASK_TIMEOUT_SECONDS(self) -> float:\n        \"\"\"If OVERRIDE is set (nonzero), return it; else return the derived budget.\"\"\"\n        outer = self.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE\n        if outer not in (None, 0):\n            # Warn if user-provided outer is likely to truncate retries\n            if (self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0) > 0:\n                min_needed = self._calc_auto_outer_timeout()\n                if float(outer) < min_needed:\n                    if self.DEEPEVAL_VERBOSE_MODE:\n                        logger.warning(\n                            \"Metric timeout (outer=%ss) is less than attempts × per-attempt \"\n                            \"timeout + backoff (≈%ss). Retries may be cut short.\",\n                            float(outer),\n                            min_needed,\n                        )\n            return float(outer)\n\n        # Auto mode\n        return self._calc_auto_outer_timeout()\n\n    @computed_field\n    @property\n    def DEEPEVAL_TASK_GATHER_BUFFER_SECONDS(self) -> float:\n        \"\"\"\n        Buffer time we add to the longest task’s duration to allow gather/drain\n        to complete. If an override is provided, use it; otherwise derive a\n        sensible default from the task-level budget:\n            buffer = constrain_between(0.15 * DEEPEVAL_PER_TASK_TIMEOUT_SECONDS, 10, 60)\n        \"\"\"\n        over = self.DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE\n        if over is not None and float(over) >= 0:\n            return float(over)\n\n        outer = float(self.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS or 0.0)\n        base = 0.15 * outer\n        return constrain_between(base, 10.0, 60.0)\n\n    ##############\n    # Validators #\n    ##############\n\n    @field_validator(\n        \"CONFIDENT_OPEN_BROWSER\",\n        \"CONFIDENT_DISABLE_SSL\",\n        \"CONFIDENT_TRACE_FLUSH\",\n        \"CONFIDENT_TRACE_INTERNAL\",\n        \"CONFIDENT_TRACE_VERBOSE\",\n        \"CUDA_LAUNCH_BLOCKING\",\n        \"DEEPEVAL_DEBUG_ASYNC\",\n        \"DEEPEVAL_LOG_STACK_TRACES\",\n        \"DEEPEVAL_DISABLE_TIMEOUTS\",\n        \"DEEPEVAL_VERBOSE_MODE\",\n        \"DEEPEVAL_GRPC_LOGGING\",\n        \"DEEPEVAL_DISABLE_DOTENV\",\n        \"DEEPEVAL_TELEMETRY_OPT_OUT\",\n        \"DEEPEVAL_UPDATE_WARNING_OPT_IN\",\n        \"ENABLE_DEEPEVAL_CACHE\",\n        \"ERROR_REPORTING\",\n        \"GOOGLE_GENAI_USE_VERTEXAI\",\n        \"IGNORE_DEEPEVAL_ERRORS\",\n        \"SKIP_DEEPEVAL_MISSING_PARAMS\",\n        \"TOKENIZERS_PARALLELISM\",\n        \"TRANSFORMERS_NO_ADVISORY_WARNINGS\",\n        \"USE_AWS_BEDROCK_MODEL\",\n        \"USE_OPENAI_MODEL\",\n        \"USE_AZURE_OPENAI\",\n        \"USE_LOCAL_MODEL\",\n        \"USE_GEMINI_MODEL\",\n        \"USE_MOONSHOT_MODEL\",\n        \"USE_GROK_MODEL\",\n        \"USE_DEEPSEEK_MODEL\",\n        \"USE_LITELLM\",\n        \"USE_AZURE_OPENAI_EMBEDDING\",\n        \"USE_LOCAL_EMBEDDINGS\",\n        \"USE_PORTKEY_MODEL\",\n        mode=\"before\",\n    )\n    @classmethod\n    def _coerce_yes_no(cls, v):\n        return None if v is None else parse_bool(v, default=False)\n\n    @field_validator(\n        \"DEEPEVAL_RESULTS_FOLDER\",\n        \"ENV_DIR_PATH\",\n        \"DEEPEVAL_CACHE_FOLDER\",\n        mode=\"before\",\n    )\n    @classmethod\n    def _coerce_path(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip()\n        if not s:\n            return None\n        # expand ~ and env vars;\n        # but don't resolve to avoid failing on non-existent paths\n        return Path(os.path.expandvars(os.path.expanduser(s)))\n\n    # Treat \"\", \"none\", \"null\" as None for numeric overrides\n    @field_validator(\n        \"OPENAI_COST_PER_INPUT_TOKEN\",\n        \"OPENAI_COST_PER_OUTPUT_TOKEN\",\n        \"GEMINI_COST_PER_INPUT_TOKEN\",\n        \"GEMINI_COST_PER_OUTPUT_TOKEN\",\n        \"AWS_BEDROCK_COST_PER_INPUT_TOKEN\",\n        \"AWS_BEDROCK_COST_PER_OUTPUT_TOKEN\",\n        \"TEMPERATURE\",\n        \"CONFIDENT_TRACE_SAMPLE_RATE\",\n        mode=\"before\",\n    )\n    @classmethod\n    def _none_or_float(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip().lower()\n        if s in {\"\", \"none\", \"null\"}:\n            return None\n        return float(v)\n\n    @field_validator(\"CONFIDENT_TRACE_SAMPLE_RATE\")\n    @classmethod\n    def _validate_sample_rate(cls, v):\n        if v is None:\n            return None\n        if not (0.0 <= float(v) <= 1.0):\n            raise ValueError(\n                \"CONFIDENT_TRACE_SAMPLE_RATE must be between 0 and 1\"\n            )\n        return float(v)\n\n    @field_validator(\"DEEPEVAL_DEFAULT_SAVE\", mode=\"before\")\n    @classmethod\n    def _validate_default_save(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip()\n        if not s:\n            return None\n        m = _SAVE_RE.match(s)\n        if not m:\n            raise ValueError(\n                \"DEEPEVAL_DEFAULT_SAVE must be 'dotenv' or 'dotenv:<path>'\"\n            )\n        path = m.group(\"path\")\n        if path is None:\n            return \"dotenv\"\n        path = os.path.expanduser(os.path.expandvars(path))\n        return f\"dotenv:{path}\"\n\n    @field_validator(\"DEEPEVAL_FILE_SYSTEM\", mode=\"before\")\n    @classmethod\n    def _normalize_fs(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip().upper()\n\n        # adds friendly aliases\n        if s in {\"READ_ONLY\", \"READ-ONLY\", \"READONLY\", \"RO\"}:\n            return \"READ_ONLY\"\n        raise ValueError(\n            \"DEEPEVAL_FILE_SYSTEM must be READ_ONLY (case-insensitive).\"\n        )\n\n    @field_validator(\"CONFIDENT_REGION\", mode=\"before\")\n    @classmethod\n    def _normalize_upper(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip()\n        if not s:\n            return None\n        return s.upper()\n\n    @field_validator(\"AWS_BEDROCK_REGION\", mode=\"before\")\n    @classmethod\n    def _normalize_lower(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip()\n        if not s:\n            return None\n        return s.lower()\n\n    @field_validator(\"DEEPEVAL_SDK_RETRY_PROVIDERS\", mode=\"before\")\n    @classmethod\n    def _coerce_to_list(cls, v):\n        # works with JSON list, comma/space/semicolon separated, or real lists\n        return coerce_to_list(v, lower=True)\n\n    @field_validator(\"DEEPEVAL_SDK_RETRY_PROVIDERS\", mode=\"after\")\n    @classmethod\n    def _validate_sdk_provider_list(cls, v):\n        if v is None:\n            return None\n\n        normalized: list[str] = []\n        star = False\n\n        for item in v:\n            s = str(item).strip()\n            if not s:\n                continue\n            if s == \"*\":\n                star = True\n                continue\n            s = slugify(s)\n            if s in SUPPORTED_PROVIDER_SLUGS:\n                normalized.append(s)\n            else:\n                if parse_bool(\n                    os.getenv(\"DEEPEVAL_VERBOSE_MODE\"), default=False\n                ):\n                    logger.warning(\"Unknown provider slug %r dropped\", item)\n\n        if star:\n            return [\"*\"]\n\n        # It is important to dedup after normalization to catch variants\n        normalized = dedupe_preserve_order(normalized)\n        return normalized or None\n\n    @field_validator(\n        \"DEEPEVAL_RETRY_BEFORE_LOG_LEVEL\",\n        \"DEEPEVAL_RETRY_AFTER_LOG_LEVEL\",\n        \"LOG_LEVEL\",\n        mode=\"before\",\n    )\n    @classmethod\n    def _coerce_log_level(cls, v):\n        if v is None:\n            return None\n        if isinstance(v, (int, float)):\n            return int(v)\n\n        s = str(v).strip().upper()\n        if not s:\n            return None\n\n        import logging\n\n        # Accept standard names or numeric strings\n        name_to_level = {\n            \"CRITICAL\": logging.CRITICAL,\n            \"ERROR\": logging.ERROR,\n            \"WARNING\": logging.WARNING,\n            \"INFO\": logging.INFO,\n            \"DEBUG\": logging.DEBUG,\n            \"NOTSET\": logging.NOTSET,\n        }\n        if s.isdigit() or (s.startswith(\"-\") and s[1:].isdigit()):\n            return int(s)\n        if s in name_to_level:\n            return name_to_level[s]\n        raise ValueError(\n            \"Retry log level must be one of DEBUG, INFO, WARNING, ERROR, \"\n            \"CRITICAL, NOTSET, or a numeric logging level.\"\n        )\n\n    @field_validator(\"DEEPEVAL_TELEMETRY_OPT_OUT\", mode=\"before\")\n    @classmethod\n    def _apply_telemetry_enabled_alias(cls, v):\n        \"\"\"\n        Precedence (most secure):\n        - Any OFF signal wins if both are set:\n          - DEEPEVAL_TELEMETRY_OPT_OUT = truthy  -> OFF\n          - DEEPEVAL_TELEMETRY_ENABLED = falsy   -> OFF\n        - Else, ON signal:\n          - DEEPEVAL_TELEMETRY_OPT_OUT = falsy   -> ON\n          - DEEPEVAL_TELEMETRY_ENABLED = truthy  -> ON\n        - Else None (unset) -> ON\n        \"\"\"\n\n        def normalize(x):\n            if x is None:\n                return None\n            s = str(x).strip()\n            return None if s == \"\" else parse_bool(s, default=False)\n\n        new_opt_out = normalize(v)  # True means OFF, False means ON\n        legacy_enabled = normalize(\n            os.getenv(\"DEEPEVAL_TELEMETRY_ENABLED\")\n        )  # True means ON, False means OFF\n\n        off_signal = (new_opt_out is True) or (legacy_enabled is False)\n        on_signal = (new_opt_out is False) or (legacy_enabled is True)\n\n        # Conflict: simultaneous OFF and ON signals\n        if off_signal and on_signal:\n            # Only warn if verbose or debug\n            if parse_bool(\n                os.getenv(\"DEEPEVAL_VERBOSE_MODE\"), default=False\n            ) or logger.isEnabledFor(logging.DEBUG):\n                logger.warning(\n                    \"Conflicting telemetry flags detected: DEEPEVAL_TELEMETRY_OPT_OUT=%r, \"\n                    \"DEEPEVAL_TELEMETRY_ENABLED=%r. Defaulting to OFF.\",\n                    new_opt_out,\n                    legacy_enabled,\n                )\n            return True  # OFF wins\n\n        # Clear winner\n        if off_signal:\n            return True  # OFF\n        if on_signal:\n            return False  # ON\n\n        # Unset means ON\n        return False\n\n    @model_validator(mode=\"after\")\n    def _apply_deprecated_computed_env_aliases(self):\n        \"\"\"\n        Backwards compatibility courtesy:\n        - If users still set a deprecated computed field in the environment,\n          emit a deprecation warning and mirror its value into the matching\n          *_OVERRIDE field (unless the override is already set).\n        - Override always wins if both are present.\n        \"\"\"\n        for old_key, override_key in _DEPRECATED_TO_OVERRIDE.items():\n            raw = os.getenv(old_key)\n            if raw is None or str(raw).strip() == \"\":\n                continue\n\n            # if override already set, ignore the deprecated one but log a warning\n            if getattr(self, override_key) is not None:\n                logger.warning(\n                    \"Config deprecation: %s is deprecated and was ignored because %s \"\n                    \"is already set. Please remove %s and use %s going forward.\",\n                    old_key,\n                    override_key,\n                    old_key,\n                    override_key,\n                )\n                continue\n\n            # apply the deprecated value into the override field.\n            try:\n                # let pydantic coerce the string to the target type on assignment\n                setattr(self, override_key, raw)\n                logger.warning(\n                    \"Config deprecation: %s is deprecated. Its value (%r) was applied to %s. \"\n                    \"Please migrate to %s and remove %s from your environment.\",\n                    old_key,\n                    raw,\n                    override_key,\n                    override_key,\n                    old_key,\n                )\n            except Exception as e:\n                # do not let exception bubble up, just warn\n                logger.warning(\n                    \"Config deprecation: %s is deprecated and could not be applied to %s \"\n                    \"(value=%r): %s\",\n                    old_key,\n                    override_key,\n                    raw,\n                    e,\n                )\n        return self\n\n    #######################\n    # Persistence support #\n    #######################\n    class _SettingsEditCtx:\n        # TODO: will generate this list in future PR\n        COMPUTED_FIELDS: frozenset[str] = frozenset(\n            {\n                \"DEEPEVAL_PER_TASK_TIMEOUT_SECONDS\",\n                \"DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS\",\n                \"DEEPEVAL_TASK_GATHER_BUFFER_SECONDS\",\n            }\n        )\n\n        def __init__(\n            self,\n            settings: \"Settings\",\n            save: Optional[str],\n            persist: Optional[bool],\n        ):\n            self._s = settings\n            self._save = save\n            self._persist = persist\n            self._before: Dict[str, Any] = {}\n            self._touched: set[str] = set()\n            self.result: Optional[PersistResult] = None\n\n        @property\n        def s(self) -> \"Settings\":\n            return self._s\n\n        def __enter__(self) -> \"Settings._SettingsEditCtx\":\n            # snapshot current state\n            self._token = _ACTIVE_SETTINGS_EDIT_CTX.set(self)\n            self._before = {\n                k: getattr(self._s, k) for k in type(self._s).model_fields\n            }\n            return self\n\n        def __exit__(self, exc_type, exc, tb):\n            try:\n                if exc_type is not None:\n                    return False  # don’t persist on error\n\n                from deepeval.config.settings_manager import (\n                    update_settings_and_persist,\n                    _normalize_for_env,\n                    _resolve_save_path,\n                )\n\n                # lazy import legacy JSON store deps\n                from deepeval.key_handler import KEY_FILE_HANDLER\n\n                model_fields = type(self._s).model_fields\n                # Exclude computed fields from persistence\n\n                # compute diff of changed fields\n                after = {k: getattr(self._s, k) for k in model_fields}\n\n                before_norm = {\n                    k: _normalize_for_env(v) for k, v in self._before.items()\n                }\n                after_norm = {\n                    k: _normalize_for_env(v) for k, v in after.items()\n                }\n\n                changed_keys = {\n                    k for k in after_norm if after_norm[k] != before_norm.get(k)\n                }\n                changed_keys -= self.COMPUTED_FIELDS\n                touched_keys = set(self._touched) - self.COMPUTED_FIELDS\n\n                # dotenv should persist union(changed, touched)\n                persist_dotenv = self._persist is not False\n                ok, resolved_path = _resolve_save_path(self._save)\n\n                existing_dotenv = {}\n                if persist_dotenv and ok and resolved_path is not None:\n                    existing_dotenv = read_dotenv_file(resolved_path)\n\n                candidate_keys_for_dotenv = (\n                    changed_keys | touched_keys\n                ) - self.COMPUTED_FIELDS\n\n                keys_for_dotenv: set[str] = set()\n                for key in candidate_keys_for_dotenv:\n                    desired = after_norm.get(key)  # normalized string or None\n                    if desired is None:\n                        # only need to unset if it's actually present in dotenv\n                        # if key in existing_dotenv:\n                        #     keys_for_dotenv.add(key)\n                        keys_for_dotenv.add(key)\n                    else:\n                        if existing_dotenv.get(key) != desired:\n                            keys_for_dotenv.add(key)\n\n                updates_for_dotenv = {\n                    key: after[key] for key in keys_for_dotenv\n                }\n\n                if not changed_keys and not updates_for_dotenv:\n                    if self._persist is False:\n                        # we report handled so that the cli does not mistakenly report invalid save option\n                        self.result = PersistResult(True, None, {})\n                        return False\n\n                    ok, resolved_path = _resolve_save_path(self._save)\n                    self.result = PersistResult(ok, resolved_path, {})\n                    return False\n\n                updates = {k: after[k] for k in changed_keys}\n\n                if \"LOG_LEVEL\" in updates:\n                    from deepeval.config.logging import (\n                        apply_deepeval_log_level,\n                    )\n\n                    apply_deepeval_log_level()\n\n                #\n                # .deepeval JSON support\n                #\n\n                if self._persist is not False:\n                    for k in changed_keys:\n                        legacy_member = _find_legacy_enum(k)\n                        if legacy_member is None:\n                            continue  # skip if not a defined as legacy field\n\n                        val = updates[k]\n                        # Remove from JSON if unset\n                        if val is None:\n                            KEY_FILE_HANDLER.remove_key(legacy_member)\n                            continue\n\n                        # Never store secrets in the JSON keystore\n                        if _is_secret_key(k):\n                            continue\n\n                        # For booleans, the legacy store expects \"YES\"/\"NO\"\n                        if isinstance(val, bool):\n                            KEY_FILE_HANDLER.write_key(\n                                legacy_member, \"YES\" if val else \"NO\"\n                            )\n                        else:\n                            # store as string\n                            KEY_FILE_HANDLER.write_key(legacy_member, str(val))\n\n                #\n                # dotenv store\n                #\n\n                # defer import to avoid cyclics\n                handled, path = update_settings_and_persist(\n                    updates_for_dotenv,\n                    save=self._save,\n                    persist_dotenv=persist_dotenv,\n                )\n                self.result = PersistResult(handled, path, updates_for_dotenv)\n                return False\n            finally:\n                if self._token is not None:\n                    _ACTIVE_SETTINGS_EDIT_CTX.reset(self._token)\n\n        def switch_model_provider(self, target) -> None:\n            \"\"\"\n            Flip USE_* settings within the target's provider family (LLM vs embeddings).\n            \"\"\"\n            from deepeval.key_handler import KEY_FILE_HANDLER\n\n            target_key = getattr(target, \"value\", str(target))\n\n            def _is_embedding_flag(k: str) -> bool:\n                return \"EMBEDDING\" in k\n\n            target_is_embedding = _is_embedding_flag(target_key)\n\n            use_fields = [\n                field\n                for field in type(self._s).model_fields\n                if field.startswith(\"USE_\")\n                and _is_embedding_flag(field) == target_is_embedding\n            ]\n\n            if target_key not in use_fields:\n                raise ValueError(\n                    f\"{target_key} is not a recognized USE_* field\"\n                )\n\n            for field in use_fields:\n                on = field == target_key\n                # Non-targets are *unset* (None) rather than written as an\n                # explicit False/\"NO\": keeps the keystore and dotenv clean\n                # and preserves Optional[bool] unset-vs-false semantics.\n                # `should_use_*()` treats a missing flag as off.\n                setattr(self._s, field, True if on else None)\n\n                if self._persist is not False:\n                    legacy_member = _find_legacy_enum(field)\n                    if legacy_member is not None:\n                        if on:\n                            KEY_FILE_HANDLER.write_key(legacy_member, \"YES\")\n                        else:\n                            KEY_FILE_HANDLER.remove_key(legacy_member)\n\n    def edit(\n        self, *, save: Optional[str] = None, persist: Optional[bool] = None\n    ):\n        \"\"\"Context manager for atomic, persisted updates.\n\n        Args:\n            save: 'dotenv[:path]' to explicitly write to a dotenv file.\n                  None (default) respects DEEPEVAL_DEFAULT_SAVE if set.\n            persist: If False, do not write (dotenv, JSON), update runtime only.\n                     If True or None, normal persistence rules apply.\n        \"\"\"\n        return self._SettingsEditCtx(self, save, persist)\n\n    def set_model_provider(self, target, *, save: Optional[str] = None):\n        \"\"\"\n        Convenience wrapper to switch providers outside of an existing edit() block.\n        Returns the PersistResult.\n        \"\"\"\n        with self.edit(save=save) as ctx:\n            ctx.switch_model_provider(target)\n        return ctx.result\n\n    def _expected_backoff(self, attempts: int) -> float:\n        \"\"\"Sum of expected sleeps for (attempts-1) retries, including jitter expectation.\"\"\"\n        sleeps = max(0, attempts - 1)\n        cur = float(self.DEEPEVAL_RETRY_INITIAL_SECONDS)\n        cap = float(self.DEEPEVAL_RETRY_CAP_SECONDS)\n        base = float(self.DEEPEVAL_RETRY_EXP_BASE)\n        jitter = float(self.DEEPEVAL_RETRY_JITTER)\n\n        backoff = 0.0\n        for _ in range(sleeps):\n            backoff += min(cap, cur)\n            cur *= base\n        backoff += sleeps * (jitter / 2.0)  # expected jitter\n        return backoff\n\n    def _constrain_between(self, value: float, lo: float, hi: float) -> float:\n        \"\"\"Return value constrained to the inclusive range [lo, hi].\"\"\"\n        return min(max(value, lo), hi)\n\n\n_settings_singleton: Optional[Settings] = None\n_settings_env_fingerprint: Optional[str] = None\n_settings_lock = threading.RLock()\n\n\ndef _calc_env_fingerprint() -> str:\n    # Pull legacy .deepeval JSON-based settings into the process env before hashing\n    _merge_legacy_keyfile_into_env()\n\n    env = os.environ.copy()\n    # must hash in a stable order.\n    keys = sorted(\n        key\n        for key in Settings.model_fields.keys()\n        if key != \"_DEPRECATED_TELEMETRY_ENABLED\"  # exclude deprecated\n    )\n    # encode as triples: (key, present?, value)\n    items = [(k, k in env, env.get(k)) for k in keys]\n    payload = json.dumps(items, ensure_ascii=False, separators=(\",\", \":\"))\n    return hashlib.sha256(payload.encode(\"utf-8\")).hexdigest()\n\n\ndef get_settings() -> Settings:\n    global _settings_singleton, _settings_env_fingerprint\n    fingerprint = _calc_env_fingerprint()\n\n    with _settings_lock:\n        if (\n            _settings_singleton is None\n            or _settings_env_fingerprint != fingerprint\n        ):\n            _settings_singleton = Settings()\n            _settings_env_fingerprint = fingerprint\n            from deepeval.config.logging import apply_deepeval_log_level\n\n            apply_deepeval_log_level()\n        return _settings_singleton\n\n\ndef reset_settings(*, reload_dotenv: bool = False) -> Settings:\n    \"\"\"\n    Drop the cached Settings singleton and rebuild it from the current process\n    environment.\n\n    Args:\n        reload_dotenv: When True, call `autoload_dotenv()` before re-instantiating,\n                       which merges .env values into os.environ (never overwriting\n                       existing process env vars).\n\n    Returns:\n        The fresh Settings instance.\n    \"\"\"\n    global _settings_singleton, _settings_env_fingerprint\n    with _settings_lock:\n        if reload_dotenv:\n            autoload_dotenv()\n        _settings_singleton = None\n        _settings_env_fingerprint = None\n    return get_settings()\n"}, "7b474c6a0b77e985d8265bb58c8a0a472db0bedf": {"t": 1785769007.9269292, "n": 36541, "h": "781f9254d421f170", "c": "\"\"\"\nCentral config for DeepEval.\n\n- Autoloads dotenv files into os.environ without overwriting existing vars\n  (order: .env -> .env.{APP_ENV} -> .env.local).\n- Defines the Pydantic `Settings` model and `get_settings()` singleton.\n- Exposes an `edit()` context manager that diffs changes and persists them to\n  dotenv and the legacy JSON keystore (non-secret keys only), with validators and\n  type coercion.\n\"\"\"\n\nimport hashlib\nimport json\nimport logging\nimport math\nimport os\nimport re\nimport threading\n\nfrom dotenv import dotenv_values\nfrom pathlib import Path\nfrom pydantic import (\n    AnyUrl,\n    computed_field,\n    confloat,\n    conint,\n    field_validator,\n    model_validator,\n    SecretStr,\n)\nfrom pydantic_settings import BaseSettings, SettingsConfigDict\nfrom typing import Any, Dict, List, Optional, NamedTuple\n\nfrom deepeval.config.utils import (\n    parse_bool,\n    coerce_to_list,\n    constrain_between,\n    dedupe_preserve_order,\n)\nfrom deepeval.constants import SUPPORTED_PROVIDER_SLUGS, slugify\n\n\nlogger = logging.getLogger(__name__)\n_SAVE_RE = re.compile(r\"^(?P<scheme>dotenv)(?::(?P<path>.+))?$\")\n\n# settings that were converted to computed fields with override counterparts\n_DEPRECATED_TO_OVERRIDE = {\n    \"DEEPEVAL_PER_TASK_TIMEOUT_SECONDS\": \"DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE\",\n    \"DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS\": \"DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE\",\n    \"DEEPEVAL_TASK_GATHER_BUFFER_SECONDS\": \"DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE\",\n}\n\n\ndef _find_legacy_enum(env_key: str):\n    from deepeval.key_handler import (\n        ModelKeyValues,\n        EmbeddingKeyValues,\n        KeyValues,\n    )\n\n    enums = (ModelKeyValues, EmbeddingKeyValues, KeyValues)\n\n    for enum in enums:\n        try:\n            return getattr(enum, env_key)\n        except AttributeError:\n            pass\n\n    for enum in enums:\n        for member in enum:\n            if member.value == env_key:\n                return member\n    return None\n\n\ndef _is_secret_key(settings: \"Settings\", env_key: str) -> bool:\n    field = type(settings).model_fields.get(env_key)\n    if not field:\n        return False\n    if field.annotation is SecretStr:\n        return True\n    # Optional[SecretStr] etc.\n    from typing import get_origin, get_args, Union\n\n    origin = get_origin(field.annotation)\n    if origin is Union:\n        return any(arg is SecretStr for arg in get_args(field.annotation))\n    return False\n\n\ndef _read_env_file(path: Path) -> Dict[str, str]:\n    if not path.exists():\n        return {}\n    try:\n        # filter out None to avoid writing \"None\" later\n        return {\n            k: v for k, v in dotenv_values(str(path)).items() if v is not None\n        }\n    except Exception:\n        return {}\n\n\ndef _discover_app_env_from_files(env_dir: Path) -> Optional[str]:\n    # prefer base .env.local, then .env for APP_ENV discovery\n    for name in (\".env.local\", \".env\"):\n        v = _read_env_file(env_dir / name).get(\"APP_ENV\")\n        if v:\n            v = str(v).strip()\n            if v:\n                return v\n    return None\n\n\ndef autoload_dotenv() -> None:\n    \"\"\"\n    Load env vars from .env files without overriding existing process env.\n\n    Precedence (lowest -> highest): .env -> .env.{APP_ENV} -> .env.local\n    Process env always wins over file values.\n\n    Controls:\n      - DEEPEVAL_DISABLE_DOTENV=1 -> skip\n      - ENV_DIR_PATH -> directory containing .env files (default: CWD)\n    \"\"\"\n    if parse_bool(os.getenv(\"DEEPEVAL_DISABLE_DOTENV\"), default=False):\n        return\n\n    raw_dir = os.getenv(\"ENV_DIR_PATH\")\n    if raw_dir:\n        env_dir = Path(os.path.expanduser(os.path.expandvars(raw_dir)))\n    else:\n        env_dir = Path(os.getcwd())\n\n    # merge files in precedence order\n    base = _read_env_file(env_dir / \".env\")\n    local = _read_env_file(env_dir / \".env.local\")\n\n    # Pick APP_ENV (process -> .env.local -> .env -> default)\n    app_env = (\n        os.getenv(\"APP_ENV\") or _discover_app_env_from_files(env_dir) or None\n    )\n    merged: Dict[str, str] = {}\n    env_specific: Dict[str, str] = {}\n    if app_env is not None:\n        app_env = app_env.strip()\n        if app_env:\n            env_specific = _read_env_file(env_dir / f\".env.{app_env}\")\n            merged.setdefault(\"APP_ENV\", app_env)\n\n    merged.update(base)\n    merged.update(env_specific)\n    merged.update(local)\n\n    # Write only keys that aren’t already in process env\n    for k, v in merged.items():\n        if k not in os.environ:\n            os.environ[k] = v\n\n\nclass PersistResult(NamedTuple):\n    handled: bool\n    path: Optional[Path]\n    updated: Dict[str, Any]  # typed, validated and changed\n\n\nclass Settings(BaseSettings):\n    model_config = SettingsConfigDict(\n        extra=\"ignore\",\n        case_sensitive=True,\n        validate_assignment=True,\n    )\n\n    #\n    # General\n    #\n\n    APP_ENV: str = \"dev\"\n    LOG_LEVEL: Optional[int] = None\n    PYTHONPATH: str = \".\"\n    CONFIDENT_REGION: Optional[str] = None\n    CONFIDENT_OPEN_BROWSER: Optional[bool] = True\n\n    #\n    # CLI\n    #\n\n    DEEPEVAL_DEFAULT_SAVE: Optional[str] = None\n    DEEPEVAL_DISABLE_DOTENV: Optional[bool] = None\n    ENV_DIR_PATH: Optional[Path] = (\n        None  # where .env files live (CWD if not set)\n    )\n    DEEPEVAL_FILE_SYSTEM: Optional[str] = None\n    DEEPEVAL_IDENTIFIER: Optional[str] = None\n\n    #\n    # Storage & Output\n    #\n\n    # When set, DeepEval will export a timestamped JSON of the latest test run\n    # into this directory. The directory will be created on demand.\n    DEEPEVAL_RESULTS_FOLDER: Optional[Path] = None\n\n    # Display / Truncation\n    DEEPEVAL_MAXLEN_TINY: Optional[int] = 40\n    DEEPEVAL_MAXLEN_SHORT: Optional[int] = 60\n    DEEPEVAL_MAXLEN_MEDIUM: Optional[int] = 120\n    DEEPEVAL_MAXLEN_LONG: Optional[int] = 240\n\n    # If set, this overrides the default max_len used by deepeval/utils shorten\n    # falls back to DEEPEVAL_MAXLEN_LONG when None.\n    DEEPEVAL_SHORTEN_DEFAULT_MAXLEN: Optional[int] = None\n\n    # Optional global suffix (keeps your \"...\" default).\n    DEEPEVAL_SHORTEN_SUFFIX: Optional[str] = \"...\"\n\n    #\n    # GPU and perf toggles\n    #\n\n    CUDA_LAUNCH_BLOCKING: Optional[bool] = None\n    CUDA_VISIBLE_DEVICES: Optional[str] = None\n    TOKENIZERS_PARALLELISM: Optional[bool] = None\n    TRANSFORMERS_NO_ADVISORY_WARNINGS: Optional[bool] = None\n\n    #\n    # Model Keys\n    #\n\n    API_KEY: Optional[SecretStr] = None\n    CONFIDENT_API_KEY: Optional[SecretStr] = None\n\n    # ======\n    # Base URL for Confident AI API server\n    # ======\n    CONFIDENT_BASE_URL: Optional[str] = None\n\n    # General\n    TEMPERATURE: Optional[confloat(ge=0, le=2)] = None\n\n    # Anthropic\n    ANTHROPIC_API_KEY: Optional[SecretStr] = None\n    # Azure Open AI\n    AZURE_OPENAI_API_KEY: Optional[SecretStr] = None\n    AZURE_OPENAI_ENDPOINT: Optional[AnyUrl] = None\n    OPENAI_API_VERSION: Optional[str] = None\n    AZURE_DEPLOYMENT_NAME: Optional[str] = None\n    AZURE_MODEL_NAME: Optional[str] = None\n    AZURE_MODEL_VERSION: Optional[str] = None\n    USE_AZURE_OPENAI: Optional[bool] = None\n    # DeepSeek\n    USE_DEEPSEEK_MODEL: Optional[bool] = None\n    DEEPSEEK_API_KEY: Optional[SecretStr] = None\n    DEEPSEEK_MODEL_NAME: Optional[str] = None\n    # Gemini\n    USE_GEMINI_MODEL: Optional[bool] = None\n    GOOGLE_API_KEY: Optional[SecretStr] = None\n    GEMINI_MODEL_NAME: Optional[str] = None\n    GOOGLE_GENAI_USE_VERTEXAI: Optional[bool] = None\n    GOOGLE_CLOUD_PROJECT: Optional[str] = None\n    GOOGLE_CLOUD_LOCATION: Optional[str] = None\n    # Grok\n    USE_GROK_MODEL: Optional[bool] = None\n    GROK_API_KEY: Optional[SecretStr] = None\n    GROK_MODEL_NAME: Optional[str] = None\n    # LiteLLM\n    USE_LITELLM: Optional[bool] = None\n    LITELLM_API_KEY: Optional[SecretStr] = None\n    LITELLM_MODEL_NAME: Optional[str] = None\n    LITELLM_API_BASE: Optional[AnyUrl] = None\n    LITELLM_PROXY_API_BASE: Optional[AnyUrl] = None\n    LITELLM_PROXY_API_KEY: Optional[SecretStr] = None\n    # LM Studio\n    LM_STUDIO_API_KEY: Optional[SecretStr] = None\n    LM_STUDIO_MODEL_NAME: Optional[str] = None\n    # Local Model\n    USE_LOCAL_MODEL: Optional[bool] = None\n    LOCAL_MODEL_API_KEY: Optional[SecretStr] = None\n    LOCAL_EMBEDDING_API_KEY: Optional[SecretStr] = None\n    LOCAL_MODEL_NAME: Optional[str] = None\n    LOCAL_MODEL_BASE_URL: Optional[AnyUrl] = None\n    LOCAL_MODEL_FORMAT: Optional[str] = None\n    # Moonshot\n    USE_MOONSHOT_MODEL: Optional[bool] = None\n    MOONSHOT_API_KEY: Optional[SecretStr] = None\n    MOONSHOT_MODEL_NAME: Optional[str] = None\n    # Ollama\n    OLLAMA_MODEL_NAME: Optional[str] = None\n    # OpenAI\n    USE_OPENAI_MODEL: Optional[bool] = None\n    OPENAI_API_KEY: Optional[SecretStr] = None\n    OPENAI_MODEL_NAME: Optional[str] = None\n    OPENAI_COST_PER_INPUT_TOKEN: Optional[float] = None\n    OPENAI_COST_PER_OUTPUT_TOKEN: Optional[float] = None\n    # Vertex AI\n    VERTEX_AI_MODEL_NAME: Optional[str] = None\n    # VLLM\n    VLLM_API_KEY: Optional[SecretStr] = None\n    VLLM_MODEL_NAME: Optional[str] = None\n\n    #\n    # Embedding Keys\n    #\n\n    # Azure OpenAI\n    USE_AZURE_OPENAI_EMBEDDING: Optional[bool] = None\n    AZURE_EMBEDDING_DEPLOYMENT_NAME: Optional[str] = None\n    # Local\n    USE_LOCAL_EMBEDDINGS: Optional[bool] = None\n    LOCAL_EMBEDDING_MODEL_NAME: Optional[str] = None\n    LOCAL_EMBEDDING_BASE_URL: Optional[AnyUrl] = None\n\n    #\n    # Retry Policy\n    #\n    # Controls how Tenacity retries provider calls when the SDK isn't doing its own retries.\n    # Key concepts:\n    # - attempts count includes the first call. e.g. 1 = no retries, 2 = one retry.\n    # - backoff sleeps follow exponential growth with a cap, plus jitter. Expected jitter\n    #   contribution is ~ JITTER/2 per sleep.\n    # - logging levels are looked up dynamically each attempt, so if you change LOG_LEVEL at runtime,\n    #   the retry loggers will honor it without restart.\n    DEEPEVAL_SDK_RETRY_PROVIDERS: Optional[List[str]] = (\n        None  # [\"*\"] to delegate all retries to SDKs\n    )\n    DEEPEVAL_RETRY_BEFORE_LOG_LEVEL: Optional[int] = (\n        None  # default is LOG_LEVEL if set, else INFO\n    )\n    DEEPEVAL_RETRY_AFTER_LOG_LEVEL: Optional[int] = None  # default -> ERROR\n    DEEPEVAL_RETRY_MAX_ATTEMPTS: conint(ge=1) = (\n        2  # attempts = first try + retries\n    )\n    DEEPEVAL_RETRY_INITIAL_SECONDS: confloat(ge=0) = (\n        1.0  # first sleep before retry, if any\n    )\n    DEEPEVAL_RETRY_EXP_BASE: confloat(ge=1) = (\n        2.0  # exponential growth factor for sleeps\n    )\n    DEEPEVAL_RETRY_JITTER: confloat(ge=0) = 2.0  # uniform jitter\n    DEEPEVAL_RETRY_CAP_SECONDS: confloat(ge=0) = (\n        5.0  # cap for each backoff sleep\n    )\n\n    #\n    # Telemetry and Debug\n    #\n    DEEPEVAL_DEBUG_ASYNC: Optional[bool] = None\n    DEEPEVAL_TELEMETRY_OPT_OUT: Optional[bool] = None\n    DEEPEVAL_UPDATE_WARNING_OPT_IN: Optional[bool] = None\n    DEEPEVAL_GRPC_LOGGING: Optional[bool] = None\n    GRPC_VERBOSITY: Optional[str] = None\n    GRPC_TRACE: Optional[str] = None\n    ERROR_REPORTING: Optional[bool] = None\n    IGNORE_DEEPEVAL_ERRORS: Optional[bool] = None\n    SKIP_DEEPEVAL_MISSING_PARAMS: Optional[bool] = None\n    DEEPEVAL_VERBOSE_MODE: Optional[bool] = None\n    DEEPEVAL_LOG_STACK_TRACES: Optional[bool] = None\n    ENABLE_DEEPEVAL_CACHE: Optional[bool] = None\n\n    CONFIDENT_TRACE_FLUSH: Optional[bool] = None\n    CONFIDENT_TRACE_ENVIRONMENT: Optional[str] = \"development\"\n    CONFIDENT_TRACE_VERBOSE: Optional[bool] = True\n    CONFIDENT_TRACE_SAMPLE_RATE: Optional[float] = 1.0\n\n    CONFIDENT_METRIC_LOGGING_FLUSH: Optional[bool] = None\n    CONFIDENT_METRIC_LOGGING_VERBOSE: Optional[bool] = True\n    CONFIDENT_METRIC_LOGGING_SAMPLE_RATE: Optional[float] = 1.0\n    CONFIDENT_METRIC_LOGGING_ENABLED: Optional[bool] = True\n\n    OTEL_EXPORTER_OTLP_ENDPOINT: Optional[AnyUrl] = None\n\n    #\n    # Network\n    #\n    MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS: float = 3.05\n    MEDIA_IMAGE_READ_TIMEOUT_SECONDS: float = 10.0\n    # DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE\n    # Per-attempt timeout (seconds) for provider calls used by the retry policy.\n    # This is an OVERRIDE setting. The effective value you should rely on at runtime is\n    # the computed property: DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS.\n    #\n    # If this is None or 0 the DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS is computed from either:\n    #   - DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE: slice the outer budget\n    #     across attempts after subtracting expected backoff and a small safety buffer\n    #   - the default outer budget (180s) if no outer override is set.\n    #\n    # Tip: Set this OR the outer override, but generally not both\n    DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE: Optional[confloat(gt=0)] = (\n        None\n    )\n\n    #\n    # Async Document Pipelines\n    #\n\n    DEEPEVAL_MAX_CONCURRENT_DOC_PROCESSING: conint(ge=1) = 2\n\n    #\n    # Async Task Configuration\n    #\n    DEEPEVAL_TIMEOUT_THREAD_LIMIT: conint(ge=1) = 128\n    DEEPEVAL_TIMEOUT_SEMAPHORE_WARN_AFTER_SECONDS: confloat(ge=0) = 5.0\n    # DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE\n    # Outer time budget (seconds) for a single metric/test-case, including retries and backoff.\n    # This is an OVERRIDE setting. If None or 0 the DEEPEVAL_PER_TASK_TIMEOUT_SECONDS field is computed:\n    #     attempts * per_attempt_timeout + expected_backoff + 1s safety\n    # (When neither override is set 180s is used.)\n    #\n    # If > 0, we use the value exactly and log a warning if it is likely too small\n    # to accommodate the configured attempts/backoff.\n    #\n    # usage:\n    #   - set DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE along with DEEPEVAL_RETRY_MAX_ATTEMPTS, or\n    #   - set DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE alone.\n    DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE: Optional[confloat(ge=0)] = None\n\n    # Buffer time for gathering results from all tasks, added to the longest task duration\n    # Increase if many tasks are running concurrently\n    # DEEPEVAL_TASK_GATHER_BUFFER_SECONDS: confloat(ge=0) = (\n    #     30  # 15s seemed like not enough. we may make this computed later.\n    # )\n    DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE: Optional[confloat(ge=0)] = (\n        None\n    )\n\n    ###################\n    # Computed Fields #\n    ###################\n\n    def _calc_auto_outer_timeout(self) -> float:\n        \"\"\"Compute outer budget from per-attempt timeout + retries/backoff.\n        Never reference the computed property itself here.\n        \"\"\"\n        attempts = self.DEEPEVAL_RETRY_MAX_ATTEMPTS or 1\n        timeout_seconds = float(\n            self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE or 0\n        )\n        if timeout_seconds <= 0:\n            # No per-attempt timeout set -> default outer budget\n            return 180\n\n        backoff = self._expected_backoff(attempts)\n        safety_overhead = 1.0\n        return float(\n            math.ceil(attempts * timeout_seconds + backoff + safety_overhead)\n        )\n\n    @computed_field\n    @property\n    def DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS(self) -> float:\n        over = self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE\n        if over is not None and float(over) > 0:\n            return float(over)\n\n        attempts = int(self.DEEPEVAL_RETRY_MAX_ATTEMPTS or 1)\n        outer_over = self.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE\n\n        # If the user set an outer override, slice it up\n        if outer_over and float(outer_over) > 0 and attempts > 0:\n            backoff = self._expected_backoff(attempts)\n            safety = 1.0\n            usable = max(0.0, float(outer_over) - backoff - safety)\n            return 0.0 if usable <= 0 else (usable / attempts)\n\n        # NEW: when neither override is set, derive from the default outer (180s)\n        default_outer = 180.0\n        backoff = self._expected_backoff(attempts)\n        safety = 1.0\n        usable = max(0.0, default_outer - backoff - safety)\n        # Keep per-attempt sensible (cap to at least 1s)\n        return 0.0 if usable <= 0 else max(1.0, usable / attempts)\n\n    @computed_field\n    @property\n    def DEEPEVAL_PER_TASK_TIMEOUT_SECONDS(self) -> float:\n        \"\"\"If OVERRIDE is set (nonzero), return it; else return the derived budget.\"\"\"\n        outer = self.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE\n        if outer not in (None, 0):\n            # Warn if user-provided outer is likely to truncate retries\n            if (self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0) > 0:\n                min_needed = self._calc_auto_outer_timeout()\n                if float(outer) < min_needed:\n                    if self.DEEPEVAL_VERBOSE_MODE:\n                        logger.warning(\n                            \"Metric timeout (outer=%ss) is less than attempts × per-attempt \"\n                            \"timeout + backoff (≈%ss). Retries may be cut short.\",\n                            float(outer),\n                            min_needed,\n                        )\n            return float(outer)\n\n        # Auto mode\n        return self._calc_auto_outer_timeout()\n\n    @computed_field\n    @property\n    def DEEPEVAL_TASK_GATHER_BUFFER_SECONDS(self) -> float:\n        \"\"\"\n        Buffer time we add to the longest task’s duration to allow gather/drain\n        to complete. If an override is provided, use it; otherwise derive a\n        sensible default from the task-level budget:\n            buffer = constrain_between(0.15 * DEEPEVAL_PER_TASK_TIMEOUT_SECONDS, 10, 60)\n        \"\"\"\n        over = self.DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE\n        if over is not None and float(over) >= 0:\n            return float(over)\n\n        outer = float(self.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS or 0.0)\n        base = 0.15 * outer\n        return constrain_between(base, 10.0, 60.0)\n\n    ##############\n    # Validators #\n    ##############\n\n    @field_validator(\n        \"CONFIDENT_OPEN_BROWSER\",\n        \"CONFIDENT_TRACE_FLUSH\",\n        \"CONFIDENT_TRACE_VERBOSE\",\n        \"USE_OPENAI_MODEL\",\n        \"USE_AZURE_OPENAI\",\n        \"USE_LOCAL_MODEL\",\n        \"USE_GEMINI_MODEL\",\n        \"GOOGLE_GENAI_USE_VERTEXAI\",\n        \"USE_MOONSHOT_MODEL\",\n        \"USE_GROK_MODEL\",\n        \"USE_DEEPSEEK_MODEL\",\n        \"USE_LITELLM\",\n        \"USE_AZURE_OPENAI_EMBEDDING\",\n        \"USE_LOCAL_EMBEDDINGS\",\n        \"DEEPEVAL_GRPC_LOGGING\",\n        \"DEEPEVAL_DISABLE_DOTENV\",\n        \"DEEPEVAL_TELEMETRY_OPT_OUT\",\n        \"DEEPEVAL_UPDATE_WARNING_OPT_IN\",\n        \"TOKENIZERS_PARALLELISM\",\n        \"TRANSFORMERS_NO_ADVISORY_WARNINGS\",\n        \"CUDA_LAUNCH_BLOCKING\",\n        \"ERROR_REPORTING\",\n        \"IGNORE_DEEPEVAL_ERRORS\",\n        \"SKIP_DEEPEVAL_MISSING_PARAMS\",\n        \"DEEPEVAL_VERBOSE_MODE\",\n        \"ENABLE_DEEPEVAL_CACHE\",\n        mode=\"before\",\n    )\n    @classmethod\n    def _coerce_yes_no(cls, v):\n        return None if v is None else parse_bool(v, default=False)\n\n    @field_validator(\"DEEPEVAL_RESULTS_FOLDER\", \"ENV_DIR_PATH\", mode=\"before\")\n    @classmethod\n    def _coerce_path(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip()\n        if not s:\n            return None\n        # expand ~ and env vars;\n        # but don't resolve to avoid failing on non-existent paths\n        return Path(os.path.expandvars(os.path.expanduser(s)))\n\n    # Treat \"\", \"none\", \"null\" as None for numeric overrides\n    @field_validator(\n        \"OPENAI_COST_PER_INPUT_TOKEN\",\n        \"OPENAI_COST_PER_OUTPUT_TOKEN\",\n        \"TEMPERATURE\",\n        \"CONFIDENT_TRACE_SAMPLE_RATE\",\n        \"CONFIDENT_METRIC_LOGGING_SAMPLE_RATE\",\n        mode=\"before\",\n    )\n    @classmethod\n    def _none_or_float(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip().lower()\n        if s in {\"\", \"none\", \"null\"}:\n            return None\n        return float(v)\n\n    @field_validator(\n        \"CONFIDENT_TRACE_SAMPLE_RATE\", \"CONFIDENT_METRIC_LOGGING_SAMPLE_RATE\"\n    )\n    @classmethod\n    def _validate_sample_rate(cls, v):\n        if v is None:\n            return None\n        if not (0.0 <= float(v) <= 1.0):\n            raise ValueError(\n                \"CONFIDENT_TRACE_SAMPLE_RATE or CONFIDENT_METRIC_LOGGING_SAMPLE_RATE must be between 0 and 1\"\n            )\n        return float(v)\n\n    @field_validator(\"DEEPEVAL_DEFAULT_SAVE\", mode=\"before\")\n    @classmethod\n    def _validate_default_save(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip()\n        if not s:\n            return None\n        m = _SAVE_RE.match(s)\n        if not m:\n            raise ValueError(\n                \"DEEPEVAL_DEFAULT_SAVE must be 'dotenv' or 'dotenv:<path>'\"\n            )\n        path = m.group(\"path\")\n        if path is None:\n            return \"dotenv\"\n        path = os.path.expanduser(os.path.expandvars(path))\n        return f\"dotenv:{path}\"\n\n    @field_validator(\"DEEPEVAL_FILE_SYSTEM\", mode=\"before\")\n    @classmethod\n    def _normalize_fs(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip().upper()\n\n        # adds friendly aliases\n        if s in {\"READ_ONLY\", \"READ-ONLY\", \"READONLY\", \"RO\"}:\n            return \"READ_ONLY\"\n        raise ValueError(\n            \"DEEPEVAL_FILE_SYSTEM must be READ_ONLY (case-insensitive).\"\n        )\n\n    @field_validator(\"CONFIDENT_REGION\", mode=\"before\")\n    @classmethod\n    def _normalize_upper(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip()\n        if not s:\n            return None\n        return s.upper()\n\n    @field_validator(\"DEEPEVAL_SDK_RETRY_PROVIDERS\", mode=\"before\")\n    @classmethod\n    def _coerce_to_list(cls, v):\n        # works with JSON list, comma/space/semicolon separated, or real lists\n        return coerce_to_list(v, lower=True)\n\n    @field_validator(\"DEEPEVAL_SDK_RETRY_PROVIDERS\", mode=\"after\")\n    @classmethod\n    def _validate_sdk_provider_list(cls, v):\n        if v is None:\n            return None\n\n        normalized: list[str] = []\n        star = False\n\n        for item in v:\n            s = str(item).strip()\n            if not s:\n                continue\n            if s == \"*\":\n                star = True\n                continue\n            s = slugify(s)\n            if s in SUPPORTED_PROVIDER_SLUGS:\n                normalized.append(s)\n            else:\n                if parse_bool(\n                    os.getenv(\"DEEPEVAL_VERBOSE_MODE\"), default=False\n                ):\n                    logger.warning(\"Unknown provider slug %r dropped\", item)\n\n        if star:\n            return [\"*\"]\n\n        # It is important to dedup after normalization to catch variants\n        normalized = dedupe_preserve_order(normalized)\n        return normalized or None\n\n    @field_validator(\n        \"DEEPEVAL_RETRY_BEFORE_LOG_LEVEL\",\n        \"DEEPEVAL_RETRY_AFTER_LOG_LEVEL\",\n        \"LOG_LEVEL\",\n        mode=\"before\",\n    )\n    @classmethod\n    def _coerce_log_level(cls, v):\n        if v is None:\n            return None\n        if isinstance(v, (int, float)):\n            return int(v)\n\n        s = str(v).strip().upper()\n        if not s:\n            return None\n\n        import logging\n\n        # Accept standard names or numeric strings\n        name_to_level = {\n            \"CRITICAL\": logging.CRITICAL,\n            \"ERROR\": logging.ERROR,\n            \"WARNING\": logging.WARNING,\n            \"INFO\": logging.INFO,\n            \"DEBUG\": logging.DEBUG,\n            \"NOTSET\": logging.NOTSET,\n        }\n        if s.isdigit() or (s.startswith(\"-\") and s[1:].isdigit()):\n            return int(s)\n        if s in name_to_level:\n            return name_to_level[s]\n        raise ValueError(\n            \"Retry log level must be one of DEBUG, INFO, WARNING, ERROR, \"\n            \"CRITICAL, NOTSET, or a numeric logging level.\"\n        )\n\n    @field_validator(\"DEEPEVAL_TELEMETRY_OPT_OUT\", mode=\"before\")\n    @classmethod\n    def _apply_telemetry_enabled_alias(cls, v):\n        \"\"\"\n        Precedence (most secure):\n        - Any OFF signal wins if both are set:\n          - DEEPEVAL_TELEMETRY_OPT_OUT = truthy  -> OFF\n          - DEEPEVAL_TELEMETRY_ENABLED = falsy   -> OFF\n        - Else, ON signal:\n          - DEEPEVAL_TELEMETRY_OPT_OUT = falsy   -> ON\n          - DEEPEVAL_TELEMETRY_ENABLED = truthy  -> ON\n        - Else None (unset) -> ON\n        \"\"\"\n\n        def normalize(x):\n            if x is None:\n                return None\n            s = str(x).strip()\n            return None if s == \"\" else parse_bool(s, default=False)\n\n        new_opt_out = normalize(v)  # True means OFF, False means ON\n        legacy_enabled = normalize(\n            os.getenv(\"DEEPEVAL_TELEMETRY_ENABLED\")\n        )  # True means ON, False means OFF\n\n        off_signal = (new_opt_out is True) or (legacy_enabled is False)\n        on_signal = (new_opt_out is False) or (legacy_enabled is True)\n\n        # Conflict: simultaneous OFF and ON signals\n        if off_signal and on_signal:\n            # Only warn if verbose or debug\n            if parse_bool(\n                os.getenv(\"DEEPEVAL_VERBOSE_MODE\"), default=False\n            ) or logger.isEnabledFor(logging.DEBUG):\n                logger.warning(\n                    \"Conflicting telemetry flags detected: DEEPEVAL_TELEMETRY_OPT_OUT=%r, \"\n                    \"DEEPEVAL_TELEMETRY_ENABLED=%r. Defaulting to OFF.\",\n                    new_opt_out,\n                    legacy_enabled,\n                )\n            return True  # OFF wins\n\n        # Clear winner\n        if off_signal:\n            return True  # OFF\n        if on_signal:\n            return False  # ON\n\n        # Unset means ON\n        return False\n\n    @model_validator(mode=\"after\")\n    def _apply_deprecated_computed_env_aliases(self):\n        \"\"\"\n        Backwards compatibility courtesy:\n        - If users still set a deprecated computed field in the environment,\n          emit a deprecation warning and mirror its value into the matching\n          *_OVERRIDE field (unless the override is already set).\n        - Override always wins if both are present.\n        \"\"\"\n        for old_key, override_key in _DEPRECATED_TO_OVERRIDE.items():\n            raw = os.getenv(old_key)\n            if raw is None or str(raw).strip() == \"\":\n                continue\n\n            # if override already set, ignore the deprecated one but log a warning\n            if getattr(self, override_key) is not None:\n                logger.warning(\n                    \"Config deprecation: %s is deprecated and was ignored because %s \"\n                    \"is already set. Please remove %s and use %s going forward.\",\n                    old_key,\n                    override_key,\n                    old_key,\n                    override_key,\n                )\n                continue\n\n            # apply the deprecated value into the override field.\n            try:\n                # let pydantic coerce the string to the target type on assignment\n                setattr(self, override_key, raw)\n                logger.warning(\n                    \"Config deprecation: %s is deprecated. Its value (%r) was applied to %s. \"\n                    \"Please migrate to %s and remove %s from your environment.\",\n                    old_key,\n                    raw,\n                    override_key,\n                    override_key,\n                    old_key,\n                )\n            except Exception as e:\n                # do not let exception bubble up, just warn\n                logger.warning(\n                    \"Config deprecation: %s is deprecated and could not be applied to %s \"\n                    \"(value=%r): %s\",\n                    old_key,\n                    override_key,\n                    raw,\n                    e,\n                )\n        return self\n\n    #######################\n    # Persistence support #\n    #######################\n    class _SettingsEditCtx:\n        # TODO: will generate this list in future PR\n        COMPUTED_FIELDS: frozenset[str] = frozenset(\n            {\n                \"DEEPEVAL_PER_TASK_TIMEOUT_SECONDS\",\n                \"DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS\",\n                \"DEEPEVAL_TASK_GATHER_BUFFER_SECONDS\",\n            }\n        )\n\n        def __init__(\n            self,\n            settings: \"Settings\",\n            save: Optional[str],\n            persist: Optional[bool],\n        ):\n            self._s = settings\n            self._save = save\n            self._persist = persist\n            self._before: Dict[str, Any] = {}\n            self.result: Optional[PersistResult] = None\n\n        @property\n        def s(self) -> \"Settings\":\n            return self._s\n\n        def __enter__(self) -> \"Settings._SettingsEditCtx\":\n            # snapshot current state\n            self._before = {\n                k: getattr(self._s, k) for k in type(self._s).model_fields\n            }\n            return self\n\n        def __exit__(self, exc_type, exc, tb):\n            if exc_type is not None:\n                return False  # don’t persist on error\n\n            from deepeval.config.settings_manager import (\n                update_settings_and_persist,\n                _normalize_for_env,\n            )\n\n            # lazy import legacy JSON store deps\n            from deepeval.key_handler import KEY_FILE_HANDLER\n\n            model_fields = type(self._s).model_fields\n            # Exclude computed fields from persistence\n\n            # compute diff of changed fields\n            after = {k: getattr(self._s, k) for k in model_fields}\n\n            before_norm = {\n                k: _normalize_for_env(v) for k, v in self._before.items()\n            }\n            after_norm = {k: _normalize_for_env(v) for k, v in after.items()}\n\n            changed_keys = {\n                k for k in after_norm if after_norm[k] != before_norm.get(k)\n            }\n            changed_keys -= self.COMPUTED_FIELDS\n\n            if not changed_keys:\n                self.result = PersistResult(False, None, {})\n                return False\n\n            updates = {k: after[k] for k in changed_keys}\n\n            if \"LOG_LEVEL\" in updates:\n                from deepeval.config.logging import (\n                    apply_deepeval_log_level,\n                )\n\n                apply_deepeval_log_level()\n\n            #\n            # .deepeval JSON support\n            #\n\n            if self._persist is not False:\n                for k in changed_keys:\n                    legacy_member = _find_legacy_enum(k)\n                    if legacy_member is None:\n                        continue  # skip if not a defined as legacy field\n\n                    val = updates[k]\n                    # Remove from JSON if unset\n                    if val is None:\n                        KEY_FILE_HANDLER.remove_key(legacy_member)\n                        continue\n\n                    # Never store secrets in the JSON keystore\n                    if _is_secret_key(self._s, k):\n                        continue\n\n                    # For booleans, the legacy store expects \"YES\"/\"NO\"\n                    if isinstance(val, bool):\n                        KEY_FILE_HANDLER.write_key(\n                            legacy_member, \"YES\" if val else \"NO\"\n                        )\n                    else:\n                        # store as string\n                        KEY_FILE_HANDLER.write_key(legacy_member, str(val))\n\n            #\n            # dotenv store\n            #\n\n            # defer import to avoid cyclics\n            handled, path = update_settings_and_persist(\n                updates,\n                save=self._save,\n                persist_dotenv=(False if self._persist is False else True),\n            )\n            self.result = PersistResult(handled, path, updates)\n            return False\n\n        def switch_model_provider(self, target) -> None:\n            \"\"\"\n            Flip all USE_* toggles so that the one matching the target is True and the rest are False.\n            Also,  mirror this change into the legacy JSON keystore as \"YES\"/\"NO\".\n\n            `target` may be an Enum with `.value`, such as ModelKeyValues.USE_OPENAI_MODEL\n            or a plain string like \"USE_OPENAI_MODEL\".\n            \"\"\"\n            from deepeval.key_handler import KEY_FILE_HANDLER\n\n            # Target key is the env style string, such as \"USE_OPENAI_MODEL\"\n            target_key = getattr(target, \"value\", str(target))\n\n            use_fields = [\n                k for k in type(self._s).model_fields if k.startswith(\"USE_\")\n            ]\n            if target_key not in use_fields:\n                raise ValueError(\n                    f\"{target_key} is not a recognized USE_* field\"\n                )\n\n            for k in use_fields:\n                on = k == target_key\n                # dotenv persistence will serialize to \"1\"/\"0\"\n                setattr(self._s, k, on)\n                if self._persist is not False:\n                    # legacy json persistence will serialize to \"YES\"/\"NO\"\n                    legacy_member = _find_legacy_enum(k)\n                    if legacy_member is not None:\n                        KEY_FILE_HANDLER.write_key(\n                            legacy_member, \"YES\" if on else \"NO\"\n                        )\n\n    def edit(\n        self, *, save: Optional[str] = None, persist: Optional[bool] = None\n    ):\n        \"\"\"Context manager for atomic, persisted updates.\n\n        Args:\n            save: 'dotenv[:path]' to explicitly write to a dotenv file.\n                  None (default) respects DEEPEVAL_DEFAULT_SAVE if set.\n            persist: If False, do not write (dotenv, JSON), update runtime only.\n                     If True or None, normal persistence rules apply.\n        \"\"\"\n        return self._SettingsEditCtx(self, save, persist)\n\n    def set_model_provider(self, target, *, save: Optional[str] = None):\n        \"\"\"\n        Convenience wrapper to switch providers outside of an existing edit() block.\n        Returns the PersistResult.\n        \"\"\"\n        with self.edit(save=save) as ctx:\n            ctx.switch_model_provider(target)\n        return ctx.result\n\n    def _expected_backoff(self, attempts: int) -> float:\n        \"\"\"Sum of expected sleeps for (attempts-1) retries, including jitter expectation.\"\"\"\n        sleeps = max(0, attempts - 1)\n        cur = float(self.DEEPEVAL_RETRY_INITIAL_SECONDS)\n        cap = float(self.DEEPEVAL_RETRY_CAP_SECONDS)\n        base = float(self.DEEPEVAL_RETRY_EXP_BASE)\n        jitter = float(self.DEEPEVAL_RETRY_JITTER)\n\n        backoff = 0.0\n        for _ in range(sleeps):\n            backoff += min(cap, cur)\n            cur *= base\n        backoff += sleeps * (jitter / 2.0)  # expected jitter\n        return backoff\n\n    def _constrain_between(self, value: float, lo: float, hi: float) -> float:\n        \"\"\"Return value constrained to the inclusive range [lo, hi].\"\"\"\n        return min(max(value, lo), hi)\n\n\n_settings_singleton: Optional[Settings] = None\n_settings_env_fingerprint: \"str | None\" = None\n_settings_lock = threading.RLock()\n\n\ndef _calc_env_fingerprint() -> str:\n    env = os.environ.copy()\n    # must hash in a stable order.\n    keys = sorted(\n        key\n        for key in Settings.model_fields.keys()\n        if key != \"_DEPRECATED_TELEMETRY_ENABLED\"  # exclude deprecated\n    )\n    # encode as triples: (key, present?, value)\n    items = [(k, k in env, env.get(k)) for k in keys]\n    payload = json.dumps(items, ensure_ascii=False, separators=(\",\", \":\"))\n    return hashlib.sha256(payload.encode(\"utf-8\")).hexdigest()\n\n\ndef get_settings() -> Settings:\n    global _settings_singleton, _settings_env_fingerprint\n    fingerprint = _calc_env_fingerprint()\n\n    with _settings_lock:\n        if (\n            _settings_singleton is None\n            or _settings_env_fingerprint != fingerprint\n        ):\n            _settings_singleton = Settings()\n            _settings_env_fingerprint = fingerprint\n            from deepeval.config.logging import apply_deepeval_log_level\n\n            apply_deepeval_log_level()\n        return _settings_singleton\n\n\ndef reset_settings(*, reload_dotenv: bool = False) -> Settings:\n    \"\"\"\n    Drop the cached Settings singleton and rebuild it from the current process\n    environment.\n\n    Args:\n        reload_dotenv: When True, call `autoload_dotenv()` before re-instantiating,\n                       which merges .env values into os.environ (never overwriting\n                       existing process env vars).\n\n    Returns:\n        The fresh Settings instance.\n    \"\"\"\n    global _settings_singleton, _settings_env_fingerprint\n    with _settings_lock:\n        if reload_dotenv:\n            autoload_dotenv()\n        _settings_singleton = None\n        _settings_env_fingerprint = None\n    return get_settings()\n"}, "ecd3575eb654554c1f991912b4024290d144d30e": {"t": 1785769009.5284846, "n": 36426, "h": "a0ca5731759ace54", "c": "\"\"\"\nCentral config for DeepEval.\n\n- Autoloads dotenv files into os.environ without overwriting existing vars\n  (order: .env -> .env.{APP_ENV} -> .env.local).\n- Defines the Pydantic `Settings` model and `get_settings()` singleton.\n- Exposes an `edit()` context manager that diffs changes and persists them to\n  dotenv and the legacy JSON keystore (non-secret keys only), with validators and\n  type coercion.\n\"\"\"\n\nimport hashlib\nimport json\nimport logging\nimport math\nimport os\nimport re\nimport threading\n\nfrom dotenv import dotenv_values\nfrom pathlib import Path\nfrom pydantic import (\n    AnyUrl,\n    computed_field,\n    confloat,\n    conint,\n    field_validator,\n    model_validator,\n    SecretStr,\n)\nfrom pydantic_settings import BaseSettings, SettingsConfigDict\nfrom typing import Any, Dict, List, Optional, NamedTuple\n\nfrom deepeval.config.utils import (\n    parse_bool,\n    coerce_to_list,\n    constrain_between,\n    dedupe_preserve_order,\n)\nfrom deepeval.constants import SUPPORTED_PROVIDER_SLUGS, slugify\n\n\nlogger = logging.getLogger(__name__)\n_SAVE_RE = re.compile(r\"^(?P<scheme>dotenv)(?::(?P<path>.+))?$\")\n\n# settings that were converted to computed fields with override counterparts\n_DEPRECATED_TO_OVERRIDE = {\n    \"DEEPEVAL_PER_TASK_TIMEOUT_SECONDS\": \"DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE\",\n    \"DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS\": \"DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE\",\n    \"DEEPEVAL_TASK_GATHER_BUFFER_SECONDS\": \"DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE\",\n}\n\n\ndef _find_legacy_enum(env_key: str):\n    from deepeval.key_handler import (\n        ModelKeyValues,\n        EmbeddingKeyValues,\n        KeyValues,\n    )\n\n    enums = (ModelKeyValues, EmbeddingKeyValues, KeyValues)\n\n    for enum in enums:\n        try:\n            return getattr(enum, env_key)\n        except AttributeError:\n            pass\n\n    for enum in enums:\n        for member in enum:\n            if member.value == env_key:\n                return member\n    return None\n\n\ndef _is_secret_key(settings: \"Settings\", env_key: str) -> bool:\n    field = type(settings).model_fields.get(env_key)\n    if not field:\n        return False\n    if field.annotation is SecretStr:\n        return True\n    # Optional[SecretStr] etc.\n    from typing import get_origin, get_args, Union\n\n    origin = get_origin(field.annotation)\n    if origin is Union:\n        return any(arg is SecretStr for arg in get_args(field.annotation))\n    return False\n\n\ndef _read_env_file(path: Path) -> Dict[str, str]:\n    if not path.exists():\n        return {}\n    try:\n        # filter out None to avoid writing \"None\" later\n        return {\n            k: v for k, v in dotenv_values(str(path)).items() if v is not None\n        }\n    except Exception:\n        return {}\n\n\ndef _discover_app_env_from_files(env_dir: Path) -> Optional[str]:\n    # prefer base .env.local, then .env for APP_ENV discovery\n    for name in (\".env.local\", \".env\"):\n        v = _read_env_file(env_dir / name).get(\"APP_ENV\")\n        if v:\n            v = str(v).strip()\n            if v:\n                return v\n    return None\n\n\ndef autoload_dotenv() -> None:\n    \"\"\"\n    Load env vars from .env files without overriding existing process env.\n\n    Precedence (lowest -> highest): .env -> .env.{APP_ENV} -> .env.local\n    Process env always wins over file values.\n\n    Controls:\n      - DEEPEVAL_DISABLE_DOTENV=1 -> skip\n      - ENV_DIR_PATH -> directory containing .env files (default: CWD)\n    \"\"\"\n    if parse_bool(os.getenv(\"DEEPEVAL_DISABLE_DOTENV\"), default=False):\n        return\n\n    raw_dir = os.getenv(\"ENV_DIR_PATH\")\n    if raw_dir:\n        env_dir = Path(os.path.expanduser(os.path.expandvars(raw_dir)))\n    else:\n        env_dir = Path(os.getcwd())\n\n    # merge files in precedence order\n    base = _read_env_file(env_dir / \".env\")\n    local = _read_env_file(env_dir / \".env.local\")\n\n    # Pick APP_ENV (process -> .env.local -> .env -> default)\n    app_env = (\n        os.getenv(\"APP_ENV\") or _discover_app_env_from_files(env_dir) or None\n    )\n    merged: Dict[str, str] = {}\n    env_specific: Dict[str, str] = {}\n    if app_env is not None:\n        app_env = app_env.strip()\n        if app_env:\n            env_specific = _read_env_file(env_dir / f\".env.{app_env}\")\n            merged.setdefault(\"APP_ENV\", app_env)\n\n    merged.update(base)\n    merged.update(env_specific)\n    merged.update(local)\n\n    # Write only keys that aren’t already in process env\n    for k, v in merged.items():\n        if k not in os.environ:\n            os.environ[k] = v\n\n\nclass PersistResult(NamedTuple):\n    handled: bool\n    path: Optional[Path]\n    updated: Dict[str, Any]  # typed, validated and changed\n\n\nclass Settings(BaseSettings):\n    model_config = SettingsConfigDict(\n        extra=\"ignore\",\n        case_sensitive=True,\n        validate_assignment=True,\n    )\n\n    #\n    # General\n    #\n\n    APP_ENV: str = \"dev\"\n    LOG_LEVEL: Optional[int] = None\n    PYTHONPATH: str = \".\"\n    CONFIDENT_REGION: Optional[str] = None\n    CONFIDENT_OPEN_BROWSER: Optional[bool] = True\n\n    #\n    # CLI\n    #\n\n    DEEPEVAL_DEFAULT_SAVE: Optional[str] = None\n    DEEPEVAL_DISABLE_DOTENV: Optional[bool] = None\n    ENV_DIR_PATH: Optional[Path] = (\n        None  # where .env files live (CWD if not set)\n    )\n    DEEPEVAL_FILE_SYSTEM: Optional[str] = None\n    DEEPEVAL_IDENTIFIER: Optional[str] = None\n\n    #\n    # Storage & Output\n    #\n\n    # When set, DeepEval will export a timestamped JSON of the latest test run\n    # into this directory. The directory will be created on demand.\n    DEEPEVAL_RESULTS_FOLDER: Optional[Path] = None\n\n    # Display / Truncation\n    DEEPEVAL_MAXLEN_TINY: Optional[int] = 40\n    DEEPEVAL_MAXLEN_SHORT: Optional[int] = 60\n    DEEPEVAL_MAXLEN_MEDIUM: Optional[int] = 120\n    DEEPEVAL_MAXLEN_LONG: Optional[int] = 240\n\n    # If set, this overrides the default max_len used by deepeval/utils shorten\n    # falls back to DEEPEVAL_MAXLEN_LONG when None.\n    DEEPEVAL_SHORTEN_DEFAULT_MAXLEN: Optional[int] = None\n\n    # Optional global suffix (keeps your \"...\" default).\n    DEEPEVAL_SHORTEN_SUFFIX: Optional[str] = \"...\"\n\n    #\n    # GPU and perf toggles\n    #\n\n    CUDA_LAUNCH_BLOCKING: Optional[bool] = None\n    CUDA_VISIBLE_DEVICES: Optional[str] = None\n    TOKENIZERS_PARALLELISM: Optional[bool] = None\n    TRANSFORMERS_NO_ADVISORY_WARNINGS: Optional[bool] = None\n\n    #\n    # Model Keys\n    #\n\n    API_KEY: Optional[SecretStr] = None\n    CONFIDENT_API_KEY: Optional[SecretStr] = None\n\n    # General\n    TEMPERATURE: Optional[confloat(ge=0, le=2)] = None\n\n    # Anthropic\n    ANTHROPIC_API_KEY: Optional[SecretStr] = None\n    # Azure Open AI\n    AZURE_OPENAI_API_KEY: Optional[SecretStr] = None\n    AZURE_OPENAI_ENDPOINT: Optional[AnyUrl] = None\n    OPENAI_API_VERSION: Optional[str] = None\n    AZURE_DEPLOYMENT_NAME: Optional[str] = None\n    AZURE_MODEL_NAME: Optional[str] = None\n    AZURE_MODEL_VERSION: Optional[str] = None\n    USE_AZURE_OPENAI: Optional[bool] = None\n    # DeepSeek\n    USE_DEEPSEEK_MODEL: Optional[bool] = None\n    DEEPSEEK_API_KEY: Optional[SecretStr] = None\n    DEEPSEEK_MODEL_NAME: Optional[str] = None\n    # Gemini\n    USE_GEMINI_MODEL: Optional[bool] = None\n    GOOGLE_API_KEY: Optional[SecretStr] = None\n    GEMINI_MODEL_NAME: Optional[str] = None\n    GOOGLE_GENAI_USE_VERTEXAI: Optional[bool] = None\n    GOOGLE_CLOUD_PROJECT: Optional[str] = None\n    GOOGLE_CLOUD_LOCATION: Optional[str] = None\n    # Grok\n    USE_GROK_MODEL: Optional[bool] = None\n    GROK_API_KEY: Optional[SecretStr] = None\n    GROK_MODEL_NAME: Optional[str] = None\n    # LiteLLM\n    USE_LITELLM: Optional[bool] = None\n    LITELLM_API_KEY: Optional[SecretStr] = None\n    LITELLM_MODEL_NAME: Optional[str] = None\n    LITELLM_API_BASE: Optional[AnyUrl] = None\n    LITELLM_PROXY_API_BASE: Optional[AnyUrl] = None\n    LITELLM_PROXY_API_KEY: Optional[SecretStr] = None\n    # LM Studio\n    LM_STUDIO_API_KEY: Optional[SecretStr] = None\n    LM_STUDIO_MODEL_NAME: Optional[str] = None\n    # Local Model\n    USE_LOCAL_MODEL: Optional[bool] = None\n    LOCAL_MODEL_API_KEY: Optional[SecretStr] = None\n    LOCAL_EMBEDDING_API_KEY: Optional[SecretStr] = None\n    LOCAL_MODEL_NAME: Optional[str] = None\n    LOCAL_MODEL_BASE_URL: Optional[AnyUrl] = None\n    LOCAL_MODEL_FORMAT: Optional[str] = None\n    # Moonshot\n    USE_MOONSHOT_MODEL: Optional[bool] = None\n    MOONSHOT_API_KEY: Optional[SecretStr] = None\n    MOONSHOT_MODEL_NAME: Optional[str] = None\n    # Ollama\n    OLLAMA_MODEL_NAME: Optional[str] = None\n    # OpenAI\n    USE_OPENAI_MODEL: Optional[bool] = None\n    OPENAI_API_KEY: Optional[SecretStr] = None\n    OPENAI_MODEL_NAME: Optional[str] = None\n    OPENAI_COST_PER_INPUT_TOKEN: Optional[float] = None\n    OPENAI_COST_PER_OUTPUT_TOKEN: Optional[float] = None\n    # Vertex AI\n    VERTEX_AI_MODEL_NAME: Optional[str] = None\n    # VLLM\n    VLLM_API_KEY: Optional[SecretStr] = None\n    VLLM_MODEL_NAME: Optional[str] = None\n\n    #\n    # Embedding Keys\n    #\n\n    # Azure OpenAI\n    USE_AZURE_OPENAI_EMBEDDING: Optional[bool] = None\n    AZURE_EMBEDDING_DEPLOYMENT_NAME: Optional[str] = None\n    # Local\n    USE_LOCAL_EMBEDDINGS: Optional[bool] = None\n    LOCAL_EMBEDDING_MODEL_NAME: Optional[str] = None\n    LOCAL_EMBEDDING_BASE_URL: Optional[AnyUrl] = None\n\n    #\n    # Retry Policy\n    #\n    # Controls how Tenacity retries provider calls when the SDK isn't doing its own retries.\n    # Key concepts:\n    # - attempts count includes the first call. e.g. 1 = no retries, 2 = one retry.\n    # - backoff sleeps follow exponential growth with a cap, plus jitter. Expected jitter\n    #   contribution is ~ JITTER/2 per sleep.\n    # - logging levels are looked up dynamically each attempt, so if you change LOG_LEVEL at runtime,\n    #   the retry loggers will honor it without restart.\n    DEEPEVAL_SDK_RETRY_PROVIDERS: Optional[List[str]] = (\n        None  # [\"*\"] to delegate all retries to SDKs\n    )\n    DEEPEVAL_RETRY_BEFORE_LOG_LEVEL: Optional[int] = (\n        None  # default is LOG_LEVEL if set, else INFO\n    )\n    DEEPEVAL_RETRY_AFTER_LOG_LEVEL: Optional[int] = None  # default -> ERROR\n    DEEPEVAL_RETRY_MAX_ATTEMPTS: conint(ge=1) = (\n        2  # attempts = first try + retries\n    )\n    DEEPEVAL_RETRY_INITIAL_SECONDS: confloat(ge=0) = (\n        1.0  # first sleep before retry, if any\n    )\n    DEEPEVAL_RETRY_EXP_BASE: confloat(ge=1) = (\n        2.0  # exponential growth factor for sleeps\n    )\n    DEEPEVAL_RETRY_JITTER: confloat(ge=0) = 2.0  # uniform jitter\n    DEEPEVAL_RETRY_CAP_SECONDS: confloat(ge=0) = (\n        5.0  # cap for each backoff sleep\n    )\n\n    #\n    # Telemetry and Debug\n    #\n    DEEPEVAL_DEBUG_ASYNC: Optional[bool] = None\n    DEEPEVAL_TELEMETRY_OPT_OUT: Optional[bool] = None\n    DEEPEVAL_UPDATE_WARNING_OPT_IN: Optional[bool] = None\n    DEEPEVAL_GRPC_LOGGING: Optional[bool] = None\n    GRPC_VERBOSITY: Optional[str] = None\n    GRPC_TRACE: Optional[str] = None\n    ERROR_REPORTING: Optional[bool] = None\n    IGNORE_DEEPEVAL_ERRORS: Optional[bool] = None\n    SKIP_DEEPEVAL_MISSING_PARAMS: Optional[bool] = None\n    DEEPEVAL_VERBOSE_MODE: Optional[bool] = None\n    DEEPEVAL_LOG_STACK_TRACES: Optional[bool] = None\n    ENABLE_DEEPEVAL_CACHE: Optional[bool] = None\n\n    CONFIDENT_TRACE_FLUSH: Optional[bool] = None\n    CONFIDENT_TRACE_ENVIRONMENT: Optional[str] = \"development\"\n    CONFIDENT_TRACE_VERBOSE: Optional[bool] = True\n    CONFIDENT_TRACE_SAMPLE_RATE: Optional[float] = 1.0\n\n    CONFIDENT_METRIC_LOGGING_FLUSH: Optional[bool] = None\n    CONFIDENT_METRIC_LOGGING_VERBOSE: Optional[bool] = True\n    CONFIDENT_METRIC_LOGGING_SAMPLE_RATE: Optional[float] = 1.0\n    CONFIDENT_METRIC_LOGGING_ENABLED: Optional[bool] = True\n\n    OTEL_EXPORTER_OTLP_ENDPOINT: Optional[AnyUrl] = None\n\n    #\n    # Network\n    #\n    MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS: float = 3.05\n    MEDIA_IMAGE_READ_TIMEOUT_SECONDS: float = 10.0\n    # DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE\n    # Per-attempt timeout (seconds) for provider calls used by the retry policy.\n    # This is an OVERRIDE setting. The effective value you should rely on at runtime is\n    # the computed property: DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS.\n    #\n    # If this is None or 0 the DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS is computed from either:\n    #   - DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE: slice the outer budget\n    #     across attempts after subtracting expected backoff and a small safety buffer\n    #   - the default outer budget (180s) if no outer override is set.\n    #\n    # Tip: Set this OR the outer override, but generally not both\n    DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE: Optional[confloat(gt=0)] = (\n        None\n    )\n\n    #\n    # Async Document Pipelines\n    #\n\n    DEEPEVAL_MAX_CONCURRENT_DOC_PROCESSING: conint(ge=1) = 2\n\n    #\n    # Async Task Configuration\n    #\n    DEEPEVAL_TIMEOUT_THREAD_LIMIT: conint(ge=1) = 128\n    DEEPEVAL_TIMEOUT_SEMAPHORE_WARN_AFTER_SECONDS: confloat(ge=0) = 5.0\n    # DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE\n    # Outer time budget (seconds) for a single metric/test-case, including retries and backoff.\n    # This is an OVERRIDE setting. If None or 0 the DEEPEVAL_PER_TASK_TIMEOUT_SECONDS field is computed:\n    #     attempts * per_attempt_timeout + expected_backoff + 1s safety\n    # (When neither override is set 180s is used.)\n    #\n    # If > 0, we use the value exactly and log a warning if it is likely too small\n    # to accommodate the configured attempts/backoff.\n    #\n    # usage:\n    #   - set DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE along with DEEPEVAL_RETRY_MAX_ATTEMPTS, or\n    #   - set DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE alone.\n    DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE: Optional[confloat(ge=0)] = None\n\n    # Buffer time for gathering results from all tasks, added to the longest task duration\n    # Increase if many tasks are running concurrently\n    # DEEPEVAL_TASK_GATHER_BUFFER_SECONDS: confloat(ge=0) = (\n    #     30  # 15s seemed like not enough. we may make this computed later.\n    # )\n    DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE: Optional[confloat(ge=0)] = (\n        None\n    )\n\n    ###################\n    # Computed Fields #\n    ###################\n\n    def _calc_auto_outer_timeout(self) -> float:\n        \"\"\"Compute outer budget from per-attempt timeout + retries/backoff.\n        Never reference the computed property itself here.\n        \"\"\"\n        attempts = self.DEEPEVAL_RETRY_MAX_ATTEMPTS or 1\n        timeout_seconds = float(\n            self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE or 0\n        )\n        if timeout_seconds <= 0:\n            # No per-attempt timeout set -> default outer budget\n            return 180\n\n        backoff = self._expected_backoff(attempts)\n        safety_overhead = 1.0\n        return float(\n            math.ceil(attempts * timeout_seconds + backoff + safety_overhead)\n        )\n\n    @computed_field\n    @property\n    def DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS(self) -> float:\n        over = self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE\n        if over is not None and float(over) > 0:\n            return float(over)\n\n        attempts = int(self.DEEPEVAL_RETRY_MAX_ATTEMPTS or 1)\n        outer_over = self.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE\n\n        # If the user set an outer override, slice it up\n        if outer_over and float(outer_over) > 0 and attempts > 0:\n            backoff = self._expected_backoff(attempts)\n            safety = 1.0\n            usable = max(0.0, float(outer_over) - backoff - safety)\n            return 0.0 if usable <= 0 else (usable / attempts)\n\n        # NEW: when neither override is set, derive from the default outer (180s)\n        default_outer = 180.0\n        backoff = self._expected_backoff(attempts)\n        safety = 1.0\n        usable = max(0.0, default_outer - backoff - safety)\n        # Keep per-attempt sensible (cap to at least 1s)\n        return 0.0 if usable <= 0 else max(1.0, usable / attempts)\n\n    @computed_field\n    @property\n    def DEEPEVAL_PER_TASK_TIMEOUT_SECONDS(self) -> float:\n        \"\"\"If OVERRIDE is set (nonzero), return it; else return the derived budget.\"\"\"\n        outer = self.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE\n        if outer not in (None, 0):\n            # Warn if user-provided outer is likely to truncate retries\n            if (self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0) > 0:\n                min_needed = self._calc_auto_outer_timeout()\n                if float(outer) < min_needed:\n                    if self.DEEPEVAL_VERBOSE_MODE:\n                        logger.warning(\n                            \"Metric timeout (outer=%ss) is less than attempts × per-attempt \"\n                            \"timeout + backoff (≈%ss). Retries may be cut short.\",\n                            float(outer),\n                            min_needed,\n                        )\n            return float(outer)\n\n        # Auto mode\n        return self._calc_auto_outer_timeout()\n\n    @computed_field\n    @property\n    def DEEPEVAL_TASK_GATHER_BUFFER_SECONDS(self) -> float:\n        \"\"\"\n        Buffer time we add to the longest task’s duration to allow gather/drain\n        to complete. If an override is provided, use it; otherwise derive a\n        sensible default from the task-level budget:\n            buffer = constrain_between(0.15 * DEEPEVAL_PER_TASK_TIMEOUT_SECONDS, 10, 60)\n        \"\"\"\n        over = self.DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE\n        if over is not None and float(over) >= 0:\n            return float(over)\n\n        outer = float(self.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS or 0.0)\n        base = 0.15 * outer\n        return constrain_between(base, 10.0, 60.0)\n\n    ##############\n    # Validators #\n    ##############\n\n    @field_validator(\n        \"CONFIDENT_OPEN_BROWSER\",\n        \"CONFIDENT_TRACE_FLUSH\",\n        \"CONFIDENT_TRACE_VERBOSE\",\n        \"USE_OPENAI_MODEL\",\n        \"USE_AZURE_OPENAI\",\n        \"USE_LOCAL_MODEL\",\n        \"USE_GEMINI_MODEL\",\n        \"GOOGLE_GENAI_USE_VERTEXAI\",\n        \"USE_MOONSHOT_MODEL\",\n        \"USE_GROK_MODEL\",\n        \"USE_DEEPSEEK_MODEL\",\n        \"USE_LITELLM\",\n        \"USE_AZURE_OPENAI_EMBEDDING\",\n        \"USE_LOCAL_EMBEDDINGS\",\n        \"DEEPEVAL_GRPC_LOGGING\",\n        \"DEEPEVAL_DISABLE_DOTENV\",\n        \"DEEPEVAL_TELEMETRY_OPT_OUT\",\n        \"DEEPEVAL_UPDATE_WARNING_OPT_IN\",\n        \"TOKENIZERS_PARALLELISM\",\n        \"TRANSFORMERS_NO_ADVISORY_WARNINGS\",\n        \"CUDA_LAUNCH_BLOCKING\",\n        \"ERROR_REPORTING\",\n        \"IGNORE_DEEPEVAL_ERRORS\",\n        \"SKIP_DEEPEVAL_MISSING_PARAMS\",\n        \"DEEPEVAL_VERBOSE_MODE\",\n        \"ENABLE_DEEPEVAL_CACHE\",\n        mode=\"before\",\n    )\n    @classmethod\n    def _coerce_yes_no(cls, v):\n        return None if v is None else parse_bool(v, default=False)\n\n    @field_validator(\"DEEPEVAL_RESULTS_FOLDER\", \"ENV_DIR_PATH\", mode=\"before\")\n    @classmethod\n    def _coerce_path(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip()\n        if not s:\n            return None\n        # expand ~ and env vars;\n        # but don't resolve to avoid failing on non-existent paths\n        return Path(os.path.expandvars(os.path.expanduser(s)))\n\n    # Treat \"\", \"none\", \"null\" as None for numeric overrides\n    @field_validator(\n        \"OPENAI_COST_PER_INPUT_TOKEN\",\n        \"OPENAI_COST_PER_OUTPUT_TOKEN\",\n        \"TEMPERATURE\",\n        \"CONFIDENT_TRACE_SAMPLE_RATE\",\n        \"CONFIDENT_METRIC_LOGGING_SAMPLE_RATE\",\n        mode=\"before\",\n    )\n    @classmethod\n    def _none_or_float(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip().lower()\n        if s in {\"\", \"none\", \"null\"}:\n            return None\n        return float(v)\n\n    @field_validator(\n        \"CONFIDENT_TRACE_SAMPLE_RATE\", \"CONFIDENT_METRIC_LOGGING_SAMPLE_RATE\"\n    )\n    @classmethod\n    def _validate_sample_rate(cls, v):\n        if v is None:\n            return None\n        if not (0.0 <= float(v) <= 1.0):\n            raise ValueError(\n                \"CONFIDENT_TRACE_SAMPLE_RATE or CONFIDENT_METRIC_LOGGING_SAMPLE_RATE must be between 0 and 1\"\n            )\n        return float(v)\n\n    @field_validator(\"DEEPEVAL_DEFAULT_SAVE\", mode=\"before\")\n    @classmethod\n    def _validate_default_save(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip()\n        if not s:\n            return None\n        m = _SAVE_RE.match(s)\n        if not m:\n            raise ValueError(\n                \"DEEPEVAL_DEFAULT_SAVE must be 'dotenv' or 'dotenv:<path>'\"\n            )\n        path = m.group(\"path\")\n        if path is None:\n            return \"dotenv\"\n        path = os.path.expanduser(os.path.expandvars(path))\n        return f\"dotenv:{path}\"\n\n    @field_validator(\"DEEPEVAL_FILE_SYSTEM\", mode=\"before\")\n    @classmethod\n    def _normalize_fs(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip().upper()\n\n        # adds friendly aliases\n        if s in {\"READ_ONLY\", \"READ-ONLY\", \"READONLY\", \"RO\"}:\n            return \"READ_ONLY\"\n        raise ValueError(\n            \"DEEPEVAL_FILE_SYSTEM must be READ_ONLY (case-insensitive).\"\n        )\n\n    @field_validator(\"CONFIDENT_REGION\", mode=\"before\")\n    @classmethod\n    def _normalize_upper(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip()\n        if not s:\n            return None\n        return s.upper()\n\n    @field_validator(\"DEEPEVAL_SDK_RETRY_PROVIDERS\", mode=\"before\")\n    @classmethod\n    def _coerce_to_list(cls, v):\n        # works with JSON list, comma/space/semicolon separated, or real lists\n        return coerce_to_list(v, lower=True)\n\n    @field_validator(\"DEEPEVAL_SDK_RETRY_PROVIDERS\", mode=\"after\")\n    @classmethod\n    def _validate_sdk_provider_list(cls, v):\n        if v is None:\n            return None\n\n        normalized: list[str] = []\n        star = False\n\n        for item in v:\n            s = str(item).strip()\n            if not s:\n                continue\n            if s == \"*\":\n                star = True\n                continue\n            s = slugify(s)\n            if s in SUPPORTED_PROVIDER_SLUGS:\n                normalized.append(s)\n            else:\n                if parse_bool(\n                    os.getenv(\"DEEPEVAL_VERBOSE_MODE\"), default=False\n                ):\n                    logger.warning(\"Unknown provider slug %r dropped\", item)\n\n        if star:\n            return [\"*\"]\n\n        # It is important to dedup after normalization to catch variants\n        normalized = dedupe_preserve_order(normalized)\n        return normalized or None\n\n    @field_validator(\n        \"DEEPEVAL_RETRY_BEFORE_LOG_LEVEL\",\n        \"DEEPEVAL_RETRY_AFTER_LOG_LEVEL\",\n        \"LOG_LEVEL\",\n        mode=\"before\",\n    )\n    @classmethod\n    def _coerce_log_level(cls, v):\n        if v is None:\n            return None\n        if isinstance(v, (int, float)):\n            return int(v)\n\n        s = str(v).strip().upper()\n        if not s:\n            return None\n\n        import logging\n\n        # Accept standard names or numeric strings\n        name_to_level = {\n            \"CRITICAL\": logging.CRITICAL,\n            \"ERROR\": logging.ERROR,\n            \"WARNING\": logging.WARNING,\n            \"INFO\": logging.INFO,\n            \"DEBUG\": logging.DEBUG,\n            \"NOTSET\": logging.NOTSET,\n        }\n        if s.isdigit() or (s.startswith(\"-\") and s[1:].isdigit()):\n            return int(s)\n        if s in name_to_level:\n            return name_to_level[s]\n        raise ValueError(\n            \"Retry log level must be one of DEBUG, INFO, WARNING, ERROR, \"\n            \"CRITICAL, NOTSET, or a numeric logging level.\"\n        )\n\n    @field_validator(\"DEEPEVAL_TELEMETRY_OPT_OUT\", mode=\"before\")\n    @classmethod\n    def _apply_telemetry_enabled_alias(cls, v):\n        \"\"\"\n        Precedence (most secure):\n        - Any OFF signal wins if both are set:\n          - DEEPEVAL_TELEMETRY_OPT_OUT = truthy  -> OFF\n          - DEEPEVAL_TELEMETRY_ENABLED = falsy   -> OFF\n        - Else, ON signal:\n          - DEEPEVAL_TELEMETRY_OPT_OUT = falsy   -> ON\n          - DEEPEVAL_TELEMETRY_ENABLED = truthy  -> ON\n        - Else None (unset) -> ON\n        \"\"\"\n\n        def normalize(x):\n            if x is None:\n                return None\n            s = str(x).strip()\n            return None if s == \"\" else parse_bool(s, default=False)\n\n        new_opt_out = normalize(v)  # True means OFF, False means ON\n        legacy_enabled = normalize(\n            os.getenv(\"DEEPEVAL_TELEMETRY_ENABLED\")\n        )  # True means ON, False means OFF\n\n        off_signal = (new_opt_out is True) or (legacy_enabled is False)\n        on_signal = (new_opt_out is False) or (legacy_enabled is True)\n\n        # Conflict: simultaneous OFF and ON signals\n        if off_signal and on_signal:\n            # Only warn if verbose or debug\n            if parse_bool(\n                os.getenv(\"DEEPEVAL_VERBOSE_MODE\"), default=False\n            ) or logger.isEnabledFor(logging.DEBUG):\n                logger.warning(\n                    \"Conflicting telemetry flags detected: DEEPEVAL_TELEMETRY_OPT_OUT=%r, \"\n                    \"DEEPEVAL_TELEMETRY_ENABLED=%r. Defaulting to OFF.\",\n                    new_opt_out,\n                    legacy_enabled,\n                )\n            return True  # OFF wins\n\n        # Clear winner\n        if off_signal:\n            return True  # OFF\n        if on_signal:\n            return False  # ON\n\n        # Unset means ON\n        return False\n\n    @model_validator(mode=\"after\")\n    def _apply_deprecated_computed_env_aliases(self):\n        \"\"\"\n        Backwards compatibility courtesy:\n        - If users still set a deprecated computed field in the environment,\n          emit a deprecation warning and mirror its value into the matching\n          *_OVERRIDE field (unless the override is already set).\n        - Override always wins if both are present.\n        \"\"\"\n        for old_key, override_key in _DEPRECATED_TO_OVERRIDE.items():\n            raw = os.getenv(old_key)\n            if raw is None or str(raw).strip() == \"\":\n                continue\n\n            # if override already set, ignore the deprecated one but log a warning\n            if getattr(self, override_key) is not None:\n                logger.warning(\n                    \"Config deprecation: %s is deprecated and was ignored because %s \"\n                    \"is already set. Please remove %s and use %s going forward.\",\n                    old_key,\n                    override_key,\n                    old_key,\n                    override_key,\n                )\n                continue\n\n            # apply the deprecated value into the override field.\n            try:\n                # let pydantic coerce the string to the target type on assignment\n                setattr(self, override_key, raw)\n                logger.warning(\n                    \"Config deprecation: %s is deprecated. Its value (%r) was applied to %s. \"\n                    \"Please migrate to %s and remove %s from your environment.\",\n                    old_key,\n                    raw,\n                    override_key,\n                    override_key,\n                    old_key,\n                )\n            except Exception as e:\n                # do not let exception bubble up, just warn\n                logger.warning(\n                    \"Config deprecation: %s is deprecated and could not be applied to %s \"\n                    \"(value=%r): %s\",\n                    old_key,\n                    override_key,\n                    raw,\n                    e,\n                )\n        return self\n\n    #######################\n    # Persistence support #\n    #######################\n    class _SettingsEditCtx:\n        # TODO: will generate this list in future PR\n        COMPUTED_FIELDS: frozenset[str] = frozenset(\n            {\n                \"DEEPEVAL_PER_TASK_TIMEOUT_SECONDS\",\n                \"DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS\",\n                \"DEEPEVAL_TASK_GATHER_BUFFER_SECONDS\",\n            }\n        )\n\n        def __init__(\n            self,\n            settings: \"Settings\",\n            save: Optional[str],\n            persist: Optional[bool],\n        ):\n            self._s = settings\n            self._save = save\n            self._persist = persist\n            self._before: Dict[str, Any] = {}\n            self.result: Optional[PersistResult] = None\n\n        @property\n        def s(self) -> \"Settings\":\n            return self._s\n\n        def __enter__(self) -> \"Settings._SettingsEditCtx\":\n            # snapshot current state\n            self._before = {\n                k: getattr(self._s, k) for k in type(self._s).model_fields\n            }\n            return self\n\n        def __exit__(self, exc_type, exc, tb):\n            if exc_type is not None:\n                return False  # don’t persist on error\n\n            from deepeval.config.settings_manager import (\n                update_settings_and_persist,\n                _normalize_for_env,\n            )\n\n            # lazy import legacy JSON store deps\n            from deepeval.key_handler import KEY_FILE_HANDLER\n\n            model_fields = type(self._s).model_fields\n            # Exclude computed fields from persistence\n\n            # compute diff of changed fields\n            after = {k: getattr(self._s, k) for k in model_fields}\n\n            before_norm = {\n                k: _normalize_for_env(v) for k, v in self._before.items()\n            }\n            after_norm = {k: _normalize_for_env(v) for k, v in after.items()}\n\n            changed_keys = {\n                k for k in after_norm if after_norm[k] != before_norm.get(k)\n            }\n            changed_keys -= self.COMPUTED_FIELDS\n\n            if not changed_keys:\n                self.result = PersistResult(False, None, {})\n                return False\n\n            updates = {k: after[k] for k in changed_keys}\n\n            if \"LOG_LEVEL\" in updates:\n                from deepeval.config.logging import (\n                    apply_deepeval_log_level,\n                )\n\n                apply_deepeval_log_level()\n\n            #\n            # .deepeval JSON support\n            #\n\n            if self._persist is not False:\n                for k in changed_keys:\n                    legacy_member = _find_legacy_enum(k)\n                    if legacy_member is None:\n                        continue  # skip if not a defined as legacy field\n\n                    val = updates[k]\n                    # Remove from JSON if unset\n                    if val is None:\n                        KEY_FILE_HANDLER.remove_key(legacy_member)\n                        continue\n\n                    # Never store secrets in the JSON keystore\n                    if _is_secret_key(self._s, k):\n                        continue\n\n                    # For booleans, the legacy store expects \"YES\"/\"NO\"\n                    if isinstance(val, bool):\n                        KEY_FILE_HANDLER.write_key(\n                            legacy_member, \"YES\" if val else \"NO\"\n                        )\n                    else:\n                        # store as string\n                        KEY_FILE_HANDLER.write_key(legacy_member, str(val))\n\n            #\n            # dotenv store\n            #\n\n            # defer import to avoid cyclics\n            handled, path = update_settings_and_persist(\n                updates,\n                save=self._save,\n                persist_dotenv=(False if self._persist is False else True),\n            )\n            self.result = PersistResult(handled, path, updates)\n            return False\n\n        def switch_model_provider(self, target) -> None:\n            \"\"\"\n            Flip all USE_* toggles so that the one matching the target is True and the rest are False.\n            Also,  mirror this change into the legacy JSON keystore as \"YES\"/\"NO\".\n\n            `target` may be an Enum with `.value`, such as ModelKeyValues.USE_OPENAI_MODEL\n            or a plain string like \"USE_OPENAI_MODEL\".\n            \"\"\"\n            from deepeval.key_handler import KEY_FILE_HANDLER\n\n            # Target key is the env style string, such as \"USE_OPENAI_MODEL\"\n            target_key = getattr(target, \"value\", str(target))\n\n            use_fields = [\n                k for k in type(self._s).model_fields if k.startswith(\"USE_\")\n            ]\n            if target_key not in use_fields:\n                raise ValueError(\n                    f\"{target_key} is not a recognized USE_* field\"\n                )\n\n            for k in use_fields:\n                on = k == target_key\n                # dotenv persistence will serialize to \"1\"/\"0\"\n                setattr(self._s, k, on)\n                if self._persist is not False:\n                    # legacy json persistence will serialize to \"YES\"/\"NO\"\n                    legacy_member = _find_legacy_enum(k)\n                    if legacy_member is not None:\n                        KEY_FILE_HANDLER.write_key(\n                            legacy_member, \"YES\" if on else \"NO\"\n                        )\n\n    def edit(\n        self, *, save: Optional[str] = None, persist: Optional[bool] = None\n    ):\n        \"\"\"Context manager for atomic, persisted updates.\n\n        Args:\n            save: 'dotenv[:path]' to explicitly write to a dotenv file.\n                  None (default) respects DEEPEVAL_DEFAULT_SAVE if set.\n            persist: If False, do not write (dotenv, JSON), update runtime only.\n                     If True or None, normal persistence rules apply.\n        \"\"\"\n        return self._SettingsEditCtx(self, save, persist)\n\n    def set_model_provider(self, target, *, save: Optional[str] = None):\n        \"\"\"\n        Convenience wrapper to switch providers outside of an existing edit() block.\n        Returns the PersistResult.\n        \"\"\"\n        with self.edit(save=save) as ctx:\n            ctx.switch_model_provider(target)\n        return ctx.result\n\n    def _expected_backoff(self, attempts: int) -> float:\n        \"\"\"Sum of expected sleeps for (attempts-1) retries, including jitter expectation.\"\"\"\n        sleeps = max(0, attempts - 1)\n        cur = float(self.DEEPEVAL_RETRY_INITIAL_SECONDS)\n        cap = float(self.DEEPEVAL_RETRY_CAP_SECONDS)\n        base = float(self.DEEPEVAL_RETRY_EXP_BASE)\n        jitter = float(self.DEEPEVAL_RETRY_JITTER)\n\n        backoff = 0.0\n        for _ in range(sleeps):\n            backoff += min(cap, cur)\n            cur *= base\n        backoff += sleeps * (jitter / 2.0)  # expected jitter\n        return backoff\n\n    def _constrain_between(self, value: float, lo: float, hi: float) -> float:\n        \"\"\"Return value constrained to the inclusive range [lo, hi].\"\"\"\n        return min(max(value, lo), hi)\n\n\n_settings_singleton: Optional[Settings] = None\n_settings_env_fingerprint: \"str | None\" = None\n_settings_lock = threading.RLock()\n\n\ndef _calc_env_fingerprint() -> str:\n    env = os.environ.copy()\n    # must hash in a stable order.\n    keys = sorted(\n        key\n        for key in Settings.model_fields.keys()\n        if key != \"_DEPRECATED_TELEMETRY_ENABLED\"  # exclude deprecated\n    )\n    # encode as triples: (key, present?, value)\n    items = [(k, k in env, env.get(k)) for k in keys]\n    payload = json.dumps(items, ensure_ascii=False, separators=(\",\", \":\"))\n    return hashlib.sha256(payload.encode(\"utf-8\")).hexdigest()\n\n\ndef get_settings() -> Settings:\n    global _settings_singleton, _settings_env_fingerprint\n    fingerprint = _calc_env_fingerprint()\n\n    with _settings_lock:\n        if (\n            _settings_singleton is None\n            or _settings_env_fingerprint != fingerprint\n        ):\n            _settings_singleton = Settings()\n            _settings_env_fingerprint = fingerprint\n            from deepeval.config.logging import apply_deepeval_log_level\n\n            apply_deepeval_log_level()\n        return _settings_singleton\n\n\ndef reset_settings(*, reload_dotenv: bool = False) -> Settings:\n    \"\"\"\n    Drop the cached Settings singleton and rebuild it from the current process\n    environment.\n\n    Args:\n        reload_dotenv: When True, call `autoload_dotenv()` before re-instantiating,\n                       which merges .env values into os.environ (never overwriting\n                       existing process env vars).\n\n    Returns:\n        The fresh Settings instance.\n    \"\"\"\n    global _settings_singleton, _settings_env_fingerprint\n    with _settings_lock:\n        if reload_dotenv:\n            autoload_dotenv()\n        _settings_singleton = None\n        _settings_env_fingerprint = None\n    return get_settings()\n"}, "c5188d892b565179960264230a4d0d83b8c7e2f3": {"t": 1785769011.1035252, "n": 30911, "h": "534c7023554f3f86", "c": "\"\"\"\nCentral config for DeepEval.\n\n- Autoloads dotenv files into os.environ without overwriting existing vars\n  (order: .env -> .env.{APP_ENV} -> .env.local).\n- Defines the Pydantic `Settings` model and `get_settings()` singleton.\n- Exposes an `edit()` context manager that diffs changes and persists them to\n  dotenv and the legacy JSON keystore (non-secret keys only), with validators and\n  type coercion.\n\"\"\"\n\nimport logging\nimport math\nimport os\nimport re\n\nfrom dotenv import dotenv_values\nfrom pathlib import Path\nfrom pydantic import (\n    AnyUrl,\n    computed_field,\n    confloat,\n    conint,\n    field_validator,\n    SecretStr,\n)\nfrom pydantic_settings import BaseSettings, SettingsConfigDict\nfrom typing import Any, Dict, List, Optional, NamedTuple\n\nfrom deepeval.config.utils import (\n    parse_bool,\n    coerce_to_list,\n    constrain_between,\n    dedupe_preserve_order,\n)\nfrom deepeval.constants import SUPPORTED_PROVIDER_SLUGS, slugify\n\n\nlogger = logging.getLogger(__name__)\n_SAVE_RE = re.compile(r\"^(?P<scheme>dotenv)(?::(?P<path>.+))?$\")\n\n\ndef _find_legacy_enum(env_key: str):\n    from deepeval.key_handler import (\n        ModelKeyValues,\n        EmbeddingKeyValues,\n        KeyValues,\n    )\n\n    enums = (ModelKeyValues, EmbeddingKeyValues, KeyValues)\n\n    for enum in enums:\n        try:\n            return getattr(enum, env_key)\n        except AttributeError:\n            pass\n\n    for enum in enums:\n        for member in enum:\n            if member.value == env_key:\n                return member\n    return None\n\n\ndef _is_secret_key(settings: \"Settings\", env_key: str) -> bool:\n    field = type(settings).model_fields.get(env_key)\n    if not field:\n        return False\n    if field.annotation is SecretStr:\n        return True\n    # Optional[SecretStr] etc.\n    from typing import get_origin, get_args, Union\n\n    origin = get_origin(field.annotation)\n    if origin is Union:\n        return any(arg is SecretStr for arg in get_args(field.annotation))\n    return False\n\n\ndef _read_env_file(path: Path) -> Dict[str, str]:\n    if not path.exists():\n        return {}\n    try:\n        # filter out None to avoid writing \"None\" later\n        return {\n            k: v for k, v in dotenv_values(str(path)).items() if v is not None\n        }\n    except Exception:\n        return {}\n\n\ndef _discover_app_env_from_files(env_dir: Path) -> Optional[str]:\n    # prefer base .env.local, then .env for APP_ENV discovery\n    for name in (\".env.local\", \".env\"):\n        v = _read_env_file(env_dir / name).get(\"APP_ENV\")\n        if v:\n            v = str(v).strip()\n            if v:\n                return v\n    return None\n\n\ndef autoload_dotenv() -> None:\n    \"\"\"\n    Load env vars from .env files without overriding existing process env.\n\n    Precedence (lowest -> highest): .env -> .env.{APP_ENV} -> .env.local\n    Process env always wins over file values.\n\n    Controls:\n      - DEEPEVAL_DISABLE_DOTENV=1 -> skip\n      - ENV_DIR_PATH -> directory containing .env files (default: CWD)\n    \"\"\"\n    if parse_bool(os.getenv(\"DEEPEVAL_DISABLE_DOTENV\"), default=False):\n        return\n\n    raw_dir = os.getenv(\"ENV_DIR_PATH\")\n    if raw_dir:\n        env_dir = Path(os.path.expanduser(os.path.expandvars(raw_dir)))\n    else:\n        env_dir = Path(os.getcwd())\n\n    # merge files in precedence order\n    base = _read_env_file(env_dir / \".env\")\n    local = _read_env_file(env_dir / \".env.local\")\n\n    # Pick APP_ENV (process -> .env.local -> .env -> default)\n    app_env = (\n        os.getenv(\"APP_ENV\") or _discover_app_env_from_files(env_dir) or None\n    )\n    merged: Dict[str, str] = {}\n    env_specific: Dict[str, str] = {}\n    if app_env is not None:\n        app_env = app_env.strip()\n        if app_env:\n            env_specific = _read_env_file(env_dir / f\".env.{app_env}\")\n            merged.setdefault(\"APP_ENV\", app_env)\n\n    merged.update(base)\n    merged.update(env_specific)\n    merged.update(local)\n\n    # Write only keys that aren’t already in process env\n    for k, v in merged.items():\n        if k not in os.environ:\n            os.environ[k] = v\n\n\nclass PersistResult(NamedTuple):\n    handled: bool\n    path: Optional[Path]\n    updated: Dict[str, Any]  # typed, validated and changed\n\n\nclass Settings(BaseSettings):\n    model_config = SettingsConfigDict(\n        extra=\"ignore\",\n        case_sensitive=True,\n        validate_assignment=True,\n    )\n\n    #\n    # General\n    #\n\n    APP_ENV: str = \"dev\"\n    LOG_LEVEL: Optional[int] = None\n    PYTHONPATH: str = \".\"\n    CONFIDENT_REGION: Optional[str] = None\n    CONFIDENT_OPEN_BROWSER: Optional[bool] = True\n\n    #\n    # CLI\n    #\n\n    DEEPEVAL_DEFAULT_SAVE: Optional[str] = None\n    DEEPEVAL_DISABLE_DOTENV: Optional[bool] = None\n    ENV_DIR_PATH: Optional[Path] = (\n        None  # where .env files live (CWD if not set)\n    )\n    DEEPEVAL_FILE_SYSTEM: Optional[str] = None\n    DEEPEVAL_IDENTIFIER: Optional[str] = None\n\n    #\n    # Storage & Output\n    #\n\n    # When set, DeepEval will export a timestamped JSON of the latest test run\n    # into this directory. The directory will be created on demand.\n    DEEPEVAL_RESULTS_FOLDER: Optional[Path] = None\n\n    # Display / Truncation\n    DEEPEVAL_MAXLEN_TINY: Optional[int] = 40\n    DEEPEVAL_MAXLEN_SHORT: Optional[int] = 60\n    DEEPEVAL_MAXLEN_MEDIUM: Optional[int] = 120\n    DEEPEVAL_MAXLEN_LONG: Optional[int] = 240\n\n    # If set, this overrides the default max_len used by deepeval/utils shorten\n    # falls back to DEEPEVAL_MAXLEN_LONG when None.\n    DEEPEVAL_SHORTEN_DEFAULT_MAXLEN: Optional[int] = None\n\n    # Optional global suffix (keeps your \"...\" default).\n    DEEPEVAL_SHORTEN_SUFFIX: Optional[str] = \"...\"\n\n    #\n    # GPU and perf toggles\n    #\n\n    CUDA_LAUNCH_BLOCKING: Optional[bool] = None\n    CUDA_VISIBLE_DEVICES: Optional[str] = None\n    TOKENIZERS_PARALLELISM: Optional[bool] = None\n    TRANSFORMERS_NO_ADVISORY_WARNINGS: Optional[bool] = None\n\n    #\n    # Model Keys\n    #\n\n    API_KEY: Optional[SecretStr] = None\n    CONFIDENT_API_KEY: Optional[SecretStr] = None\n\n    # General\n    TEMPERATURE: Optional[confloat(ge=0, le=2)] = None\n\n    # Anthropic\n    ANTHROPIC_API_KEY: Optional[SecretStr] = None\n    # Azure Open AI\n    AZURE_OPENAI_API_KEY: Optional[SecretStr] = None\n    AZURE_OPENAI_ENDPOINT: Optional[AnyUrl] = None\n    OPENAI_API_VERSION: Optional[str] = None\n    AZURE_DEPLOYMENT_NAME: Optional[str] = None\n    AZURE_MODEL_NAME: Optional[str] = None\n    AZURE_MODEL_VERSION: Optional[str] = None\n    USE_AZURE_OPENAI: Optional[bool] = None\n    # DeepSeek\n    USE_DEEPSEEK_MODEL: Optional[bool] = None\n    DEEPSEEK_API_KEY: Optional[SecretStr] = None\n    DEEPSEEK_MODEL_NAME: Optional[str] = None\n    # Gemini\n    USE_GEMINI_MODEL: Optional[bool] = None\n    GOOGLE_API_KEY: Optional[SecretStr] = None\n    GEMINI_MODEL_NAME: Optional[str] = None\n    GOOGLE_GENAI_USE_VERTEXAI: Optional[bool] = None\n    GOOGLE_CLOUD_PROJECT: Optional[str] = None\n    GOOGLE_CLOUD_LOCATION: Optional[str] = None\n    # Grok\n    USE_GROK_MODEL: Optional[bool] = None\n    GROK_API_KEY: Optional[SecretStr] = None\n    GROK_MODEL_NAME: Optional[str] = None\n    # LiteLLM\n    USE_LITELLM: Optional[bool] = None\n    LITELLM_API_KEY: Optional[SecretStr] = None\n    LITELLM_MODEL_NAME: Optional[str] = None\n    LITELLM_API_BASE: Optional[AnyUrl] = None\n    LITELLM_PROXY_API_BASE: Optional[AnyUrl] = None\n    LITELLM_PROXY_API_KEY: Optional[SecretStr] = None\n    # LM Studio\n    LM_STUDIO_API_KEY: Optional[SecretStr] = None\n    LM_STUDIO_MODEL_NAME: Optional[str] = None\n    # Local Model\n    USE_LOCAL_MODEL: Optional[bool] = None\n    LOCAL_MODEL_API_KEY: Optional[SecretStr] = None\n    LOCAL_EMBEDDING_API_KEY: Optional[SecretStr] = None\n    LOCAL_MODEL_NAME: Optional[str] = None\n    LOCAL_MODEL_BASE_URL: Optional[AnyUrl] = None\n    LOCAL_MODEL_FORMAT: Optional[str] = None\n    # Moonshot\n    USE_MOONSHOT_MODEL: Optional[bool] = None\n    MOONSHOT_API_KEY: Optional[SecretStr] = None\n    MOONSHOT_MODEL_NAME: Optional[str] = None\n    # Ollama\n    OLLAMA_MODEL_NAME: Optional[str] = None\n    # OpenAI\n    USE_OPENAI_MODEL: Optional[bool] = None\n    OPENAI_API_KEY: Optional[SecretStr] = None\n    OPENAI_MODEL_NAME: Optional[str] = None\n    OPENAI_COST_PER_INPUT_TOKEN: Optional[float] = None\n    OPENAI_COST_PER_OUTPUT_TOKEN: Optional[float] = None\n    # Vertex AI\n    VERTEX_AI_MODEL_NAME: Optional[str] = None\n    # VLLM\n    VLLM_API_KEY: Optional[SecretStr] = None\n    VLLM_MODEL_NAME: Optional[str] = None\n\n    #\n    # Embedding Keys\n    #\n\n    # Azure OpenAI\n    USE_AZURE_OPENAI_EMBEDDING: Optional[bool] = None\n    AZURE_EMBEDDING_DEPLOYMENT_NAME: Optional[str] = None\n    # Local\n    USE_LOCAL_EMBEDDINGS: Optional[bool] = None\n    LOCAL_EMBEDDING_MODEL_NAME: Optional[str] = None\n    LOCAL_EMBEDDING_BASE_URL: Optional[AnyUrl] = None\n\n    #\n    # Retry Policy\n    #\n    # Controls how Tenacity retries provider calls when the SDK isn't doing its own retries.\n    # Key concepts:\n    # - attempts count includes the first call. e.g. 1 = no retries, 2 = one retry.\n    # - backoff sleeps follow exponential growth with a cap, plus jitter. Expected jitter\n    #   contribution is ~ JITTER/2 per sleep.\n    # - logging levels are looked up dynamically each attempt, so if you change LOG_LEVEL at runtime,\n    #   the retry loggers will honor it without restart.\n    DEEPEVAL_SDK_RETRY_PROVIDERS: Optional[List[str]] = (\n        None  # [\"*\"] to delegate all retries to SDKs\n    )\n    DEEPEVAL_RETRY_BEFORE_LOG_LEVEL: Optional[int] = (\n        None  # default is LOG_LEVEL if set, else INFO\n    )\n    DEEPEVAL_RETRY_AFTER_LOG_LEVEL: Optional[int] = None  # default -> ERROR\n    DEEPEVAL_RETRY_MAX_ATTEMPTS: conint(ge=1) = (\n        2  # attempts = first try + retries\n    )\n    DEEPEVAL_RETRY_INITIAL_SECONDS: confloat(ge=0) = (\n        1.0  # first sleep before retry, if any\n    )\n    DEEPEVAL_RETRY_EXP_BASE: confloat(ge=1) = (\n        2.0  # exponential growth factor for sleeps\n    )\n    DEEPEVAL_RETRY_JITTER: confloat(ge=0) = 2.0  # uniform jitter\n    DEEPEVAL_RETRY_CAP_SECONDS: confloat(ge=0) = (\n        5.0  # cap for each backoff sleep\n    )\n\n    #\n    # Telemetry and Debug\n    #\n    DEEPEVAL_DEBUG_ASYNC: Optional[bool] = None\n    DEEPEVAL_TELEMETRY_OPT_OUT: Optional[bool] = None\n    DEEPEVAL_UPDATE_WARNING_OPT_IN: Optional[bool] = None\n    DEEPEVAL_GRPC_LOGGING: Optional[bool] = None\n    GRPC_VERBOSITY: Optional[str] = None\n    GRPC_TRACE: Optional[str] = None\n    ERROR_REPORTING: Optional[bool] = None\n    IGNORE_DEEPEVAL_ERRORS: Optional[bool] = None\n    SKIP_DEEPEVAL_MISSING_PARAMS: Optional[bool] = None\n    DEEPEVAL_VERBOSE_MODE: Optional[bool] = None\n    DEEPEVAL_LOG_STACK_TRACES: Optional[bool] = None\n    ENABLE_DEEPEVAL_CACHE: Optional[bool] = None\n\n    CONFIDENT_TRACE_FLUSH: Optional[bool] = None\n    CONFIDENT_TRACE_ENVIRONMENT: Optional[str] = \"development\"\n    CONFIDENT_TRACE_VERBOSE: Optional[bool] = True\n    CONFIDENT_TRACE_SAMPLE_RATE: Optional[float] = 1.0\n\n    CONFIDENT_METRIC_LOGGING_FLUSH: Optional[bool] = None\n    CONFIDENT_METRIC_LOGGING_VERBOSE: Optional[bool] = True\n    CONFIDENT_METRIC_LOGGING_SAMPLE_RATE: Optional[float] = 1.0\n    CONFIDENT_METRIC_LOGGING_ENABLED: Optional[bool] = True\n\n    OTEL_EXPORTER_OTLP_ENDPOINT: Optional[AnyUrl] = None\n\n    #\n    # Network\n    #\n    MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS: float = 3.05\n    MEDIA_IMAGE_READ_TIMEOUT_SECONDS: float = 10.0\n    # DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE\n    # Per-attempt timeout (seconds) for provider calls used by the retry policy.\n    # This is an OVERRIDE setting. The effective value you should rely on at runtime is\n    # the computed property: DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS.\n    #\n    # If this is None or 0 the DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS is computed from either:\n    #   - DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE: slice the outer budget\n    #     across attempts after subtracting expected backoff and a small safety buffer\n    #   - the default outer budget (180s) if no outer override is set.\n    #\n    # Tip: Set this OR the outer override, but generally not both\n    DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE: Optional[confloat(gt=0)] = (\n        None\n    )\n\n    #\n    # Async Document Pipelines\n    #\n\n    DEEPEVAL_MAX_CONCURRENT_DOC_PROCESSING: conint(ge=1) = 2\n\n    #\n    # Async Task Configuration\n    #\n    DEEPEVAL_TIMEOUT_THREAD_LIMIT: conint(ge=1) = 128\n    DEEPEVAL_TIMEOUT_SEMAPHORE_WARN_AFTER_SECONDS: confloat(ge=0) = 5.0\n    # DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE\n    # Outer time budget (seconds) for a single metric/test-case, including retries and backoff.\n    # This is an OVERRIDE setting. If None or 0 the DEEPEVAL_PER_TASK_TIMEOUT_SECONDS field is computed:\n    #     attempts * per_attempt_timeout + expected_backoff + 1s safety\n    # (When neither override is set 180s is used.)\n    #\n    # If > 0, we use the value exactly and log a warning if it is likely too small\n    # to accommodate the configured attempts/backoff.\n    #\n    # usage:\n    #   - set DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE along with DEEPEVAL_RETRY_MAX_ATTEMPTS, or\n    #   - set DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE alone.\n    DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE: Optional[confloat(ge=0)] = None\n\n    # Buffer time for gathering results from all tasks, added to the longest task duration\n    # Increase if many tasks are running concurrently\n    # DEEPEVAL_TASK_GATHER_BUFFER_SECONDS: confloat(ge=0) = (\n    #     30  # 15s seemed like not enough. we may make this computed later.\n    # )\n    DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE: Optional[confloat(ge=0)] = (\n        None\n    )\n\n    ###################\n    # Computed Fields #\n    ###################\n\n    def _calc_auto_outer_timeout(self) -> float:\n        \"\"\"Compute outer budget from per-attempt timeout + retries/backoff.\n        Never reference the computed property itself here.\n        \"\"\"\n        attempts = self.DEEPEVAL_RETRY_MAX_ATTEMPTS or 1\n        timeout_seconds = float(\n            self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE or 0\n        )\n        if timeout_seconds <= 0:\n            # No per-attempt timeout set -> default outer budget\n            return 180\n\n        backoff = self._expected_backoff(attempts)\n        safety_overhead = 1.0\n        return float(\n            math.ceil(attempts * timeout_seconds + backoff + safety_overhead)\n        )\n\n    @computed_field\n    @property\n    def DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS(self) -> float:\n        over = self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE\n        if over is not None and float(over) > 0:\n            return float(over)\n\n        attempts = int(self.DEEPEVAL_RETRY_MAX_ATTEMPTS or 1)\n        outer_over = self.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE\n\n        # If the user set an outer override, slice it up\n        if outer_over and float(outer_over) > 0 and attempts > 0:\n            backoff = self._expected_backoff(attempts)\n            safety = 1.0\n            usable = max(0.0, float(outer_over) - backoff - safety)\n            return 0.0 if usable <= 0 else (usable / attempts)\n\n        # NEW: when neither override is set, derive from the default outer (180s)\n        default_outer = 180.0\n        backoff = self._expected_backoff(attempts)\n        safety = 1.0\n        usable = max(0.0, default_outer - backoff - safety)\n        # Keep per-attempt sensible (cap to at least 1s)\n        return 0.0 if usable <= 0 else max(1.0, usable / attempts)\n\n    @computed_field\n    @property\n    def DEEPEVAL_PER_TASK_TIMEOUT_SECONDS(self) -> float:\n        \"\"\"If OVERRIDE is set (nonzero), return it; else return the derived budget.\"\"\"\n        outer = self.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE\n        if outer not in (None, 0):\n            # Warn if user-provided outer is likely to truncate retries\n            if (self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0) > 0:\n                min_needed = self._calc_auto_outer_timeout()\n                if float(outer) < min_needed:\n                    if self.DEEPEVAL_VERBOSE_MODE:\n                        logger.warning(\n                            \"Metric timeout (outer=%ss) is less than attempts × per-attempt \"\n                            \"timeout + backoff (≈%ss). Retries may be cut short.\",\n                            float(outer),\n                            min_needed,\n                        )\n            return float(outer)\n\n        # Auto mode\n        return self._calc_auto_outer_timeout()\n\n    @computed_field\n    @property\n    def DEEPEVAL_TASK_GATHER_BUFFER_SECONDS(self) -> float:\n        \"\"\"\n        Buffer time we add to the longest task’s duration to allow gather/drain\n        to complete. If an override is provided, use it; otherwise derive a\n        sensible default from the task-level budget:\n            buffer = constrain_between(0.15 * DEEPEVAL_PER_TASK_TIMEOUT_SECONDS, 10, 60)\n        \"\"\"\n        over = self.DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE\n        if over is not None and float(over) >= 0:\n            return float(over)\n\n        outer = float(self.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS or 0.0)\n        base = 0.15 * outer\n        return constrain_between(base, 10.0, 60.0)\n\n    ##############\n    # Validators #\n    ##############\n\n    @field_validator(\n        \"CONFIDENT_OPEN_BROWSER\",\n        \"CONFIDENT_TRACE_FLUSH\",\n        \"CONFIDENT_TRACE_VERBOSE\",\n        \"USE_OPENAI_MODEL\",\n        \"USE_AZURE_OPENAI\",\n        \"USE_LOCAL_MODEL\",\n        \"USE_GEMINI_MODEL\",\n        \"GOOGLE_GENAI_USE_VERTEXAI\",\n        \"USE_MOONSHOT_MODEL\",\n        \"USE_GROK_MODEL\",\n        \"USE_DEEPSEEK_MODEL\",\n        \"USE_LITELLM\",\n        \"USE_AZURE_OPENAI_EMBEDDING\",\n        \"USE_LOCAL_EMBEDDINGS\",\n        \"DEEPEVAL_GRPC_LOGGING\",\n        \"DEEPEVAL_DISABLE_DOTENV\",\n        \"DEEPEVAL_TELEMETRY_OPT_OUT\",\n        \"DEEPEVAL_UPDATE_WARNING_OPT_IN\",\n        \"TOKENIZERS_PARALLELISM\",\n        \"TRANSFORMERS_NO_ADVISORY_WARNINGS\",\n        \"CUDA_LAUNCH_BLOCKING\",\n        \"ERROR_REPORTING\",\n        \"IGNORE_DEEPEVAL_ERRORS\",\n        \"SKIP_DEEPEVAL_MISSING_PARAMS\",\n        \"DEEPEVAL_VERBOSE_MODE\",\n        \"ENABLE_DEEPEVAL_CACHE\",\n        mode=\"before\",\n    )\n    @classmethod\n    def _coerce_yes_no(cls, v):\n        return None if v is None else parse_bool(v, default=False)\n\n    @field_validator(\"DEEPEVAL_RESULTS_FOLDER\", \"ENV_DIR_PATH\", mode=\"before\")\n    @classmethod\n    def _coerce_path(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip()\n        if not s:\n            return None\n        # expand ~ and env vars;\n        # but don't resolve to avoid failing on non-existent paths\n        return Path(os.path.expandvars(os.path.expanduser(s)))\n\n    # Treat \"\", \"none\", \"null\" as None for numeric overrides\n    @field_validator(\n        \"OPENAI_COST_PER_INPUT_TOKEN\",\n        \"OPENAI_COST_PER_OUTPUT_TOKEN\",\n        \"TEMPERATURE\",\n        \"CONFIDENT_TRACE_SAMPLE_RATE\",\n        \"CONFIDENT_METRIC_LOGGING_SAMPLE_RATE\",\n        mode=\"before\",\n    )\n    @classmethod\n    def _none_or_float(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip().lower()\n        if s in {\"\", \"none\", \"null\"}:\n            return None\n        return float(v)\n\n    @field_validator(\n        \"CONFIDENT_TRACE_SAMPLE_RATE\", \"CONFIDENT_METRIC_LOGGING_SAMPLE_RATE\"\n    )\n    @classmethod\n    def _validate_sample_rate(cls, v):\n        if v is None:\n            return None\n        if not (0.0 <= float(v) <= 1.0):\n            raise ValueError(\n                \"CONFIDENT_TRACE_SAMPLE_RATE or CONFIDENT_METRIC_LOGGING_SAMPLE_RATE must be between 0 and 1\"\n            )\n        return float(v)\n\n    @field_validator(\"DEEPEVAL_DEFAULT_SAVE\", mode=\"before\")\n    @classmethod\n    def _validate_default_save(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip()\n        if not s:\n            return None\n        m = _SAVE_RE.match(s)\n        if not m:\n            raise ValueError(\n                \"DEEPEVAL_DEFAULT_SAVE must be 'dotenv' or 'dotenv:<path>'\"\n            )\n        path = m.group(\"path\")\n        if path is None:\n            return \"dotenv\"\n        path = os.path.expanduser(os.path.expandvars(path))\n        return f\"dotenv:{path}\"\n\n    @field_validator(\"DEEPEVAL_FILE_SYSTEM\", mode=\"before\")\n    @classmethod\n    def _normalize_fs(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip().upper()\n\n        # adds friendly aliases\n        if s in {\"READ_ONLY\", \"READ-ONLY\", \"READONLY\", \"RO\"}:\n            return \"READ_ONLY\"\n        raise ValueError(\n            \"DEEPEVAL_FILE_SYSTEM must be READ_ONLY (case-insensitive).\"\n        )\n\n    @field_validator(\"CONFIDENT_REGION\", mode=\"before\")\n    @classmethod\n    def _normalize_upper(cls, v):\n        if v is None:\n            return None\n        s = str(v).strip()\n        if not s:\n            return None\n        return s.upper()\n\n    @field_validator(\"DEEPEVAL_SDK_RETRY_PROVIDERS\", mode=\"before\")\n    @classmethod\n    def _coerce_to_list(cls, v):\n        # works with JSON list, comma/space/semicolon separated, or real lists\n        return coerce_to_list(v, lower=True)\n\n    @field_validator(\"DEEPEVAL_SDK_RETRY_PROVIDERS\", mode=\"after\")\n    @classmethod\n    def _validate_sdk_provider_list(cls, v):\n        if v is None:\n            return None\n\n        normalized: list[str] = []\n        star = False\n\n        for item in v:\n            s = str(item).strip()\n            if not s:\n                continue\n            if s == \"*\":\n                star = True\n                continue\n            s = slugify(s)\n            if s in SUPPORTED_PROVIDER_SLUGS:\n                normalized.append(s)\n            else:\n                if parse_bool(\n                    os.getenv(\"DEEPEVAL_VERBOSE_MODE\"), default=False\n                ):\n                    logger.warning(\"Unknown provider slug %r dropped\", item)\n\n        if star:\n            return [\"*\"]\n\n        # It is important to dedup after normalization to catch variants\n        normalized = dedupe_preserve_order(normalized)\n        return normalized or None\n\n    @field_validator(\n        \"DEEPEVAL_RETRY_BEFORE_LOG_LEVEL\",\n        \"DEEPEVAL_RETRY_AFTER_LOG_LEVEL\",\n        \"LOG_LEVEL\",\n        mode=\"before\",\n    )\n    @classmethod\n    def _coerce_log_level(cls, v):\n        if v is None:\n            return None\n        if isinstance(v, (int, float)):\n            return int(v)\n\n        s = str(v).strip().upper()\n        if not s:\n            return None\n\n        import logging\n\n        # Accept standard names or numeric strings\n        name_to_level = {\n            \"CRITICAL\": logging.CRITICAL,\n            \"ERROR\": logging.ERROR,\n            \"WARNING\": logging.WARNING,\n            \"INFO\": logging.INFO,\n            \"DEBUG\": logging.DEBUG,\n            \"NOTSET\": logging.NOTSET,\n        }\n        if s.isdigit() or (s.startswith(\"-\") and s[1:].isdigit()):\n            return int(s)\n        if s in name_to_level:\n            return name_to_level[s]\n        raise ValueError(\n            \"Retry log level must be one of DEBUG, INFO, WARNING, ERROR, \"\n            \"CRITICAL, NOTSET, or a numeric logging level.\"\n        )\n\n    #######################\n    # Persistence support #\n    #######################\n    class _SettingsEditCtx:\n        COMPUTED_FIELDS: frozenset[str] = frozenset(\n            {\"DEEPEVAL_PER_TASK_TIMEOUT_SECONDS\"}\n        )\n\n        def __init__(\n            self,\n            settings: \"Settings\",\n            save: Optional[str],\n            persist: Optional[bool],\n        ):\n            self._s = settings\n            self._save = save\n            self._persist = persist\n            self._before: Dict[str, Any] = {}\n            self.result: Optional[PersistResult] = None\n\n        @property\n        def s(self) -> \"Settings\":\n            return self._s\n\n        def __enter__(self) -> \"Settings._SettingsEditCtx\":\n            # snapshot current state\n            self._before = {\n                k: getattr(self._s, k) for k in type(self._s).model_fields\n            }\n            return self\n\n        def __exit__(self, exc_type, exc, tb):\n            if exc_type is not None:\n                return False  # don’t persist on error\n\n            from deepeval.config.settings_manager import (\n                update_settings_and_persist,\n                _normalize_for_env,\n            )\n\n            # lazy import legacy JSON store deps\n            from deepeval.key_handler import KEY_FILE_HANDLER\n\n            model_fields = type(self._s).model_fields\n            # Exclude computed fields from persistence\n\n            # compute diff of changed fields\n            after = {k: getattr(self._s, k) for k in model_fields}\n\n            before_norm = {\n                k: _normalize_for_env(v) for k, v in self._before.items()\n            }\n            after_norm = {k: _normalize_for_env(v) for k, v in after.items()}\n\n            changed_keys = {\n                k for k in after_norm if after_norm[k] != before_norm.get(k)\n            }\n            changed_keys -= self.COMPUTED_FIELDS\n\n            if not changed_keys:\n                self.result = PersistResult(False, None, {})\n                return False\n\n            updates = {k: after[k] for k in changed_keys}\n\n            if \"LOG_LEVEL\" in updates:\n                from deepeval.config.logging import (\n                    apply_deepeval_log_level,\n                )\n\n                apply_deepeval_log_level()\n\n            #\n            # .deepeval JSON support\n            #\n\n            if self._persist is not False:\n                for k in changed_keys:\n                    legacy_member = _find_legacy_enum(k)\n                    if legacy_member is None:\n                        continue  # skip if not a defined as legacy field\n\n                    val = updates[k]\n                    # Remove from JSON if unset\n                    if val is None:\n                        KEY_FILE_HANDLER.remove_key(legacy_member)\n                        continue\n\n                    # Never store secrets in the JSON keystore\n                    if _is_secret_key(self._s, k):\n                        continue\n\n                    # For booleans, the legacy store expects \"YES\"/\"NO\"\n                    if isinstance(val, bool):\n                        KEY_FILE_HANDLER.write_key(\n                            legacy_member, \"YES\" if val else \"NO\"\n                        )\n                    else:\n                        # store as string\n                        KEY_FILE_HANDLER.write_key(legacy_member, str(val))\n\n            #\n            # dotenv store\n            #\n\n            # defer import to avoid cyclics\n            handled, path = update_settings_and_persist(\n                updates,\n                save=self._save,\n                persist_dotenv=(False if self._persist is False else True),\n            )\n            self.result = PersistResult(handled, path, updates)\n            return False\n\n        def switch_model_provider(self, target) -> None:\n            \"\"\"\n            Flip all USE_* toggles so that the one matching the target is True and the rest are False.\n            Also,  mirror this change into the legacy JSON keystore as \"YES\"/\"NO\".\n\n            `target` may be an Enum with `.value`, such as ModelKeyValues.USE_OPENAI_MODEL\n            or a plain string like \"USE_OPENAI_MODEL\".\n            \"\"\"\n            from deepeval.key_handler import KEY_FILE_HANDLER\n\n            # Target key is the env style string, such as \"USE_OPENAI_MODEL\"\n            target_key = getattr(target, \"value\", str(target))\n\n            use_fields = [\n                k for k in type(self._s).model_fields if k.startswith(\"USE_\")\n            ]\n            if target_key not in use_fields:\n                raise ValueError(\n                    f\"{target_key} is not a recognized USE_* field\"\n                )\n\n            for k in use_fields:\n                on = k == target_key\n                # dotenv persistence will serialize to \"1\"/\"0\"\n                setattr(self._s, k, on)\n                if self._persist is not False:\n                    # legacy json persistence will serialize to \"YES\"/\"NO\"\n                    legacy_member = _find_legacy_enum(k)\n                    if legacy_member is not None:\n                        KEY_FILE_HANDLER.write_key(\n                            legacy_member, \"YES\" if on else \"NO\"\n                        )\n\n    def edit(\n        self, *, save: Optional[str] = None, persist: Optional[bool] = None\n    ):\n        \"\"\"Context manager for atomic, persisted updates.\n\n        Args:\n            save: 'dotenv[:path]' to explicitly write to a dotenv file.\n                  None (default) respects DEEPEVAL_DEFAULT_SAVE if set.\n            persist: If False, do not write (dotenv, JSON), update runtime only.\n                     If True or None, normal persistence rules apply.\n        \"\"\"\n        return self._SettingsEditCtx(self, save, persist)\n\n    def set_model_provider(self, target, *, save: Optional[str] = None):\n        \"\"\"\n        Convenience wrapper to switch providers outside of an existing edit() block.\n        Returns the PersistResult.\n        \"\"\"\n        with self.edit(save=save) as ctx:\n            ctx.switch_model_provider(target)\n        return ctx.result\n\n    def _expected_backoff(self, attempts: int) -> float:\n        \"\"\"Sum of expected sleeps for (attempts-1) retries, including jitter expectation.\"\"\"\n        sleeps = max(0, attempts - 1)\n        cur = float(self.DEEPEVAL_RETRY_INITIAL_SECONDS)\n        cap = float(self.DEEPEVAL_RETRY_CAP_SECONDS)\n        base = float(self.DEEPEVAL_RETRY_EXP_BASE)\n        jitter = float(self.DEEPEVAL_RETRY_JITTER)\n\n        backoff = 0.0\n        for _ in range(sleeps):\n            backoff += min(cap, cur)\n            cur *= base\n        backoff += sleeps * (jitter / 2.0)  # expected jitter\n        return backoff\n\n    def _constrain_between(self, value: float, lo: float, hi: float) -> float:\n        \"\"\"Return value constrained to the inclusive range [lo, hi].\"\"\"\n        return min(max(value, lo), hi)\n\n\n_settings_singleton: Optional[Settings] = None\n\n\ndef get_settings() -> Settings:\n    global _settings_singleton\n    if _settings_singleton is None:\n        _settings_singleton = Settings()\n        from deepeval.config.logging import apply_deepeval_log_level\n\n        apply_deepeval_log_level()\n    return _settings_singleton\n\n\ndef reset_settings(*, reload_dotenv: bool = False) -> Settings:\n    \"\"\"\n    Drop the cached Settings singleton and rebuild it from the current process\n    environment.\n\n    Args:\n        reload_dotenv: When True, call `autoload_dotenv()` before re-instantiating,\n                       which merges .env values into os.environ (never overwriting\n                       existing process env vars).\n\n    Returns:\n        The fresh Settings instance.\n    \"\"\"\n    global _settings_singleton\n    if reload_dotenv:\n        autoload_dotenv()\n    _settings_singleton = None\n    return get_settings()\n"}}}}