{"repo": "anandaditya2002/rag", "files": {"langflow/Lib/site-packages/qianfan/trainer/actions.py": {"29a056bc26e963b0318bf4b8b9ff579e2a8e313c": {"t": 1785674666.6193182, "n": 38292, "h": "e0c4b0f819e77593", "c": "# Copyright (c) 2023 Baidu, Inc. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nimport time\nfrom pathlib import Path\nfrom typing import Any, Dict, List, Optional, Union, cast\n\nfrom qianfan import resources as api\nfrom qianfan.config import get_config\nfrom qianfan.dataset import BosDataSource, Dataset, QianfanDataSource\nfrom qianfan.errors import InternalError, InvalidArgumentError\nfrom qianfan.evaluation import EvaluationManager\nfrom qianfan.evaluation.evaluator import Evaluator, LocalEvaluator, QianfanEvaluator\nfrom qianfan.model import Model, Service\nfrom qianfan.model.configs import DeployConfig\nfrom qianfan.resources.console import consts as console_consts\nfrom qianfan.trainer.base import (\n    ActionState,\n    BaseAction,\n    with_event,\n)\nfrom qianfan.trainer.configs import (\n    DefaultPostPretrainTrainConfigMapping,\n    DefaultTrainConfigMapping,\n    PeftType,\n    TrainConfig,\n    TrainLimit,\n    get_model_info,\n)\nfrom qianfan.trainer.consts import ServiceStatus, TrainStatus\nfrom qianfan.utils import (\n    bos_uploader,\n    log_debug,\n    log_error,\n    log_info,\n    log_warn,\n    utils,\n)\nfrom qianfan.utils.bos_uploader import is_valid_bos_path\nfrom qianfan.utils.utils import first_lower_case, snake_to_camel\n\n\nclass LoadDataSetAction(BaseAction[Dict[str, Any], Dict[str, Any]]):\n    \"\"\"LoadDataSetAction\n    Action for dataset's loading, invokes the dataset's save method\n    to guarantee the dataset is loaded in Qianfan platform.\n    Sample:\n        ```\n        load_action = LoadDataSetAction(dataset=Dataset(id=1))\n        load_action.exec()\n        ```\n\n    input:\n        none\n    output:\n        ```\n            {\n                \"datasets\": {\n                \"sourceType\": (\n                   2\n                ),\n                \"versions\": [\n                        {\n                            \"versionBosUri\": \"bos:/bbb/\"\n                        }\n                    ],\n                }\n            }\n        ```\n    \"\"\"\n\n    from qianfan.dataset.dataset import Dataset\n\n    dataset: Optional[Dataset] = None\n    bos_path: Optional[str] = None\n\n    def __init__(\n        self,\n        dataset: Optional[Union[Dataset, str]] = None,\n        dataset_template: Optional[console_consts.DataTemplateType] = None,\n        **kwargs: Any,\n    ) -> None:\n        super().__init__(**kwargs)\n        if dataset is None:\n            raise InvalidArgumentError(\"dataset must be set\")\n        if isinstance(dataset, str):\n            if not is_valid_bos_path(dataset):\n                raise InvalidArgumentError(f\"invalid bos_path {dataset}\")\n            self.bos_path = dataset\n        elif isinstance(dataset.inner_data_source_cache, QianfanDataSource):\n            qf_data_src = cast(QianfanDataSource, dataset.inner_data_source_cache)\n            if (\n                dataset_template is not None\n                and qf_data_src.template_type != dataset_template\n            ):\n                raise InvalidArgumentError(\n                    f\"dataset must be `{dataset_template}` template.\"\n                )\n            self.dataset = dataset\n        elif isinstance(dataset.inner_data_source_cache, BosDataSource):\n            self.dataset = dataset\n        else:\n            raise InvalidArgumentError(\n                \"dataset must be either implemented with QianfanDataSource or\"\n                \" BosDataSource or a bos path\"\n            )\n\n    @with_event\n    def exec(self, input: Dict[str, Any] = {}, **kwargs: Dict) -> Dict[str, Any]:\n        return self._exec(input, **kwargs)\n\n    def _exec(self, input: Dict[str, Any] = {}, **kwargs: Dict) -> Dict[str, Any]:\n        \"\"\"\n        Load dataset implementation, may called by exec and resume.\n        \"\"\"\n        if self.bos_path is not None:\n            if not self.bos_path.endswith(\"/\"):\n                bos_path = f'{Path(f\"/{self.bos_path}\").parent}'\n                log_warn(\n                    f\"input bos_path {self.bos_path} is a file, auto_convert to dir:\"\n                    f\" {bos_path}\"\n                )\n            else:\n                bos_path = self.bos_path\n            return {\n                \"datasets\": {\n                    \"sourceType\": (\n                        console_consts.TrainDatasetSourceType.PrivateBos.value\n                    ),\n                    \"versions\": [{\"versionBosUri\": bos_path}],\n                }\n            }\n        from qianfan.dataset.data_source import BosDataSource, QianfanDataSource\n\n        if self.dataset is None:\n            raise InvalidArgumentError(\"dataset or bos_path must be set\")\n        if self.dataset.inner_data_source_cache is None:\n            raise InvalidArgumentError(\"invalid dataset\")\n        if isinstance(self.dataset.inner_data_source_cache, QianfanDataSource):\n            log_debug(\"[load_dataset_action] prepare train-set\")\n            qf_data_src = cast(QianfanDataSource, self.dataset.inner_data_source_cache)\n            is_released = qf_data_src.release_dataset(**kwargs)\n            if not is_released:\n                log_error(\"[load_dataset_action] dataset not released\")\n                raise InvalidArgumentError(\"dataset must be released\")\n            log_debug(\"[load_dataset_action] dataset loaded successfully\")\n            self.qf_dataset_id = qf_data_src.id\n            return {\n                \"datasets\": {\n                    \"sourceType\": console_consts.TrainDatasetSourceType.Platform.value,\n                    \"versions\": [\n                        {\n                            \"versionId\": qf_data_src.id,\n                        }\n                    ],\n                }\n            }\n        elif isinstance(self.dataset.inner_data_source_cache, BosDataSource):\n            log_debug(\"[load_dataset_action] prepare train-set in BOS\")\n            bos_data_src = cast(BosDataSource, self.dataset.inner_data_source_cache)\n            return {\n                \"datasets\": {\n                    \"sourceType\": (\n                        console_consts.TrainDatasetSourceType.PrivateBos.value\n                    ),\n                    \"versions\": [\n                        {\n                            \"versionBosUri\": bos_uploader.generate_bos_file_parent_path(\n                                bos_data_src.bucket, bos_data_src.bos_file_path\n                            )\n                        }\n                    ],\n                }\n            }\n        else:\n            raise InvalidArgumentError(\"dataset must be set\")\n\n    @with_event\n    def resume(self, **kwargs: Dict) -> Dict[str, Any]:\n        \"\"\"\n        resume method for load dataset action.\n\n        Returns:\n            Dict[str, Any]: datasets metainfo including\n            dataset_id and dataset_type.\n        \"\"\"\n        if self.qf_dataset_id:\n            log_debug(\"[load_dataset_action] dataset loading already done\")\n            return {\n                \"datasets\": [\n                    {\n                        \"id\": self.qf_dataset_id,\n                        \"type\": console_consts.TrainDatasetType.Platform.value,\n                    }\n                ]\n            }\n        log_debug(\"[load_dataset_action] dataset loading resumed\")\n        return self._exec(**kwargs)\n\n\nclass TrainAction(\n    BaseAction[Dict[str, Any], Dict[str, Any]],\n):\n    \"\"\"\n    Class for Train Action, Synchronous invocation of the training API,\n    taking a dataset metadata dict as input and producing a model metadata\n    as output. Concretely, `exec` is called for running.\n\n    Note: this action is not involved with model publishing, please use use\n    `ModelPublishAction` for publishing model.\n\n    Sample:\n\n    Input:\n    ```\n        {\n            \"datasets\": {\n            \"sourceType\": (\n                2\n            ),\n            \"versions\": [\n                    {\n                        \"versionBosUri\": \"bos:/bbb/\"\n                    }\n                ],\n            }\n        }\n    ```\n\n    Output:\n    ```\n        {'task_id': \"task-ddd\", 'job_id': \"job-xxxx\"}\n    ```\n    \"\"\"\n\n    task_id: Optional[str] = None\n    \"\"\"train task id\"\"\"\n    job_id: Optional[str] = None\n    \"\"\"train job id\"\"\"\n    train_type: Optional[str] = \"\"\n    \"\"\"train_type\"\"\"\n    is_incr: bool = False\n    \"\"\"if it's incremental train or not\"\"\"\n    train_config: Optional[TrainConfig] = None\n    \"\"\"train config\"\"\"\n    train_mode: console_consts.TrainMode\n    \"\"\"train mode\"\"\"\n    job_name: str = \"\"\n    \"\"\"train task name\"\"\"\n    task_description: Optional[str] = None\n    \"\"\"train task description\"\"\"\n    job_description: Optional[str] = None\n    \"\"\"train job description\"\"\"\n    _input: Optional[Dict[str, Any]] = None\n    \"\"\"train input\"\"\"\n    result: Optional[Dict[str, Any]] = None\n    \"\"\"\"train result\"\"\"\n    train_model_name: Optional[str] = None\n    \"\"\"real name to start training\"\"\"\n\n    def __init__(\n        self,\n        train_mode: console_consts.TrainMode,\n        train_type: Optional[str] = None,\n        train_config: Optional[TrainConfig] = None,\n        task_id: Optional[str] = None,\n        job_id: Optional[str] = None,\n        peft_type: PeftType = PeftType.ALL,\n        job_name: Optional[str] = None,\n        task_description: Optional[str] = None,\n        job_description: Optional[str] = None,\n        **kwargs: Any,\n    ) -> None:\n        \"\"\"\n\n        Parameters:\n            train_mode (Optional[console_consts.TrainMode], optional):\n                train mode, e.g. `SFT`, `PostPretrain`. Defaults to None.\n            train_type (Optional[str], optional):\n                train_type, must be specified when it's not increment training\n                like 'ERNIE-Bot-turbo-0725'\n            train_config (Optional[TrainConfig], optional):\n                train_config, e.g. `epoch=10, batch_size=32`.\n            base_model (Optional[str], optional):\n                base_mode, like 'ERNIE-Bot-turbo'. Defaults to None.\n            task_id (Optional[int], optional):\n                used in incr train, model train task_id. Defaults to None.\n            job_id (Optional[int], optional):\n                used in incr train, mod train job_id. Defaults to None.\n            job_name (Optional[str], optional):\n                train task name. Defaults to None.\n            task_description (Optional[str], optional):\n                train task description. Defaults to None.\n            job_description (Optional[str], optional):\n                train job description. Defaults to None.\n        \"\"\"\n        super().__init__(**kwargs)\n        self.task_id = task_id\n        self.job_id = job_id\n        self.train_mode = train_mode\n        self.train_model_name = train_type\n        if self.task_id is not None:\n            # if incremental train\n            pre_task_detail = api.FineTune.V2.task_detail(task_id=self.task_id)\n            # 获取增量任务的训练model\n            if pre_task_detail.get(\"result\") is not None:\n                self.train_type = pre_task_detail[\"result\"][\"model\"]\n                self.train_mode = train_mode\n            self.is_incr = True\n        else:\n            if train_type is None:\n                raise InvalidArgumentError(\"train_type must be specified\")\n            # 从基础模型开始训练\n            self.train_type = train_type\n            model_info = get_model_info(train_mode, self.train_type)\n            if model_info is None:\n                log_warn(f\"unknown train model type: {self.train_type} is not found\")\n            elif model_info.model != \"\":\n                # 兼容改名模型\n                self.train_model_name = model_info.model\n        assert self.train_type is not None\n        if train_config is None:\n            train_config = self.get_default_train_config(\n                self.train_type, self.train_mode, peft_type\n            )\n        self.train_config = train_config\n        self.validateTrainConfig(strict=kwargs.get(\"validate_strict\", True))\n        self.job_name = self._generate_job_name(job_name, self.train_type)\n        self.task_description = task_description\n        self.job_description = job_description\n\n    def _generate_job_name(\n        self, job_name: Optional[str], train_type: Optional[str]\n    ) -> str:\n        if job_name is not None:\n            return job_name\n        model_info = (\n            get_model_info(self.train_mode, train_type)\n            if train_type is not None\n            else None\n        )\n        return (\n            f\"job_{utils.generate_letter_num_random_id()}\"\n            if model_info is None\n            else f\"{model_info.short_name}_{utils.generate_letter_num_random_id(5)}\"\n        )\n\n    def validateTrainConfig(self, strict: bool = True) -> None:\n        \"\"\"\n        validate train_config with ModelInfo Limits\n\n        Raises:\n            InvalidArgumentError: _description_\n        \"\"\"\n        if self.train_config is None:\n            raise InvalidArgumentError(\"none train_config\")\n        else:\n            assert self.train_type\n            train_type_model_info = get_model_info(self.train_mode, self.train_type)\n            if train_type_model_info is None:\n                return\n            if (\n                self.train_config.peft_type\n                not in train_type_model_info.support_peft_types\n            ):\n                log_warn(\n                    f\"[train_action] train_type {self.train_type}, peft_type\"\n                    f\" {self.train_config.peft_type} not found, it may be not supported\"\n                )\n                if strict:\n                    raise InvalidArgumentError(\n                        f\"[train_action] train_type {self.train_type}, peft_type\"\n                        f\" {self.train_config.peft_type} not found, it may be not\"\n                        \" supported\"\n                    )\n\n            else:\n                assert train_type_model_info\n                res = False\n                if (\n                    train_type_model_info.specific_peft_types_params_limit is not None\n                    and self.train_config.peft_type\n                    in train_type_model_info.specific_peft_types_params_limit\n                ):\n                    res = self._validate_train_config(\n                        train_type_model_info.specific_peft_types_params_limit[\n                            self.train_config.peft_type\n                        ]\n                        | train_type_model_info.common_params_limit,\n                    )\n                else:\n                    res = self._validate_train_config(\n                        train_type_model_info.common_params_limit\n                    )\n                if not res and strict:\n                    raise InvalidArgumentError(\n                        \"invalid train_config, please check the config\"\n                    )\n\n    def _validate_train_config(self, train_limit: TrainLimit) -> bool:\n        \"\"\"\n        validate train_config with a specific train_limit\n\n        Args:\n            train_limit (TrainLimit): _description_\n\n        Raises:\n            InvalidArgumentError: _description_\n        \"\"\"\n        if self.train_config is None:\n            raise InvalidArgumentError(\"validate train_config is none\")\n        return self.train_config.validate_config(train_limit)\n\n    @with_event\n    def exec(self, input: Dict[str, Any] = {}, **kwargs: Dict) -> Dict[str, Any]:\n        \"\"\"\n        exec method for train action\n\n        Args:\n            input (Dict[str, Any], optional):\n                input with dataset meta:\n                for example:\n                    Input:\n                    ```\n                    {'datasets':[{'type': 1, 'id': 111}]}\n                    ```\n        Raises:\n            InvalidArgumentError: invalid dataset input\n\n        Returns:\n            Dict[str, Any]:\n                train task output with task_id and job_id\n                for example:\n                    Output:\n                    ```\n                    {'task_id': 47923, 'job_id': 33512}\n                    ```\n        \"\"\"\n        # for resume\n        if self._input is None:\n            self._input = input\n        return self._exec(self._input, **kwargs)\n\n    def _exec(self, input: Dict[str, Any] = {}, **kwargs: Any) -> Dict[str, Any]:\n        # 校验数据集\n        ds_config = input.get(\"datasets\")\n        if ds_config is None:\n            raise InvalidArgumentError(\"train set must be set\")\n        assert isinstance(ds_config, dict)\n        assert self.train_config\n        ds_config[\"splitRatio\"] = self.train_config.trainset_rate\n\n        if self.job_id is None:\n            # request for create model train task\n            assert self.train_type is not None\n            resp = api.FineTune.V2.create_job(\n                name=self.job_name,\n                description=self.task_description,\n                model=(\n                    self.train_model_name if self.train_model_name else self.train_type\n                ),\n                train_mode=self.train_mode,\n                **kwargs,\n            )\n\n            self.job_id = str(resp[\"result\"][\"jobId\"])\n            log_debug(\n                f\"[train_action] create {self.train_mode} train job: {self.job_id}\"\n            )\n\n        assert self.train_config is not None\n        hyper_params_dict = {\n            **self.train_config.dict(exclude={\"peft_type\", \"trainset_rate\", \"extras\"}),\n            **self.train_config.extras,\n        }\n        hyper_params_dict = {\n            first_lower_case(snake_to_camel(key)): value\n            for key, value in hyper_params_dict.items()\n            if value is not None\n        }\n        # custom fix\n        if \"packing\" in hyper_params_dict:\n            hyper_params_dict[\"Packing\"] = hyper_params_dict.pop(\"packing\")\n        ds_config = input[\"datasets\"]\n        log_debug(f\"train with ds_config: { ds_config}\")\n        log_debug(f\"train with hyper_params: { hyper_params_dict}\")\n        if self.is_incr:\n            # 增量训练\n            kwargs[\"incrementTaskId\"] = self.task_id\n            log_info(f\"train with incrementTaskId: { self.task_id}\")\n        assert self.train_config.peft_type is not None\n        create_task_resp = api.FineTune.V2.create_task(\n            job_id=self.job_id,\n            params_scale=self.train_config.peft_type,\n            hyper_params=hyper_params_dict,\n            dataset_config=ds_config,\n            **kwargs,\n        )\n        self.task_id = str(create_task_resp[\"result\"][\"taskId\"])\n        log_debug(f\"[train_action] create {self.train_mode} train task: {self.task_id}\")\n\n        # 获取job状态，是否训练完成\n        train_metrics = self._wait_model_trained(**kwargs)\n        self.result = {\n            **input,\n            \"task_id\": self.task_id,\n            \"job_id\": self.job_id,\n            **train_metrics,\n        }\n        assert self.result is not None\n        return self.result\n\n    def _wait_model_trained(self, **kwargs: Dict) -> Dict[str, Any]:\n        if self.task_id is None:\n            raise InvalidArgumentError(\"task_id must not be None\")\n        output = {}\n        while True:\n            task_status_resp = api.FineTune.V2.task_detail(\n                task_id=self.task_id,\n                **kwargs,\n            )\n            task_status_result = task_status_resp.get(\"result\", {})\n            task_status = task_status_result.get(\"runStatus\")\n\n            self.action_event(ActionState.Running, \"train running\", task_status_resp)\n            if task_status == console_consts.TrainStatus.Finish:\n                output[\"metrics\"] = task_status_result.get(\"metrics\", {})\n                log_info(f\"[train_action] training task metrics: {output['metrics']}\")\n                break\n            elif task_status in [\n                console_consts.TrainStatus.Fail,\n                console_consts.TrainStatus.Stop,\n            ]:\n                log_error(\n                    \"[train_action] training job\"\n                    f\" {self.job_id}/{self.task_id} has ended,\"\n                    f\" {task_status_resp}\"\n                )\n                raise InternalError(\n                    f\"[train_action]training job {self.job_id}/{self.task_id} has ended\"\n                    f\" with status: {task_status}\"\n                )\n            elif task_status == console_consts.TrainStatus.Running:\n                job_progress_str = task_status_result.get(\"runProgress\")\n                job_progress = int(job_progress_str[:-1])\n                log_prefix = (\n                    \"sft\"\n                    if self.train_mode == console_consts.TrainMode.SFT\n                    else \"postPretrain\"\n                )\n                log_info(\n                    \"[train_action] training ...\"\n                    f\" job_name:{self.job_name} current status: {task_status},\"\n                    f\" {job_progress}% check train task log in\"\n                    f\" https://console.bce.baidu.com/qianfan/train/{log_prefix}/{self.job_id}/{self.task_id}/detail/traininglog\"\n                )\n                if job_progress >= 50:\n                    log_info(\n                        f\" check vdl report in {task_status_result.get('vdlLink')}\"\n                    )\n                time.sleep(get_config().TRAIN_STATUS_POLLING_INTERVAL)\n            else:\n                raise InternalError(\n                    f\"[train_action] job: {self.job_name} task:\"\n                    f\" {self.job_id}/{self.task_id} get unknown status: {task_status}\"\n                )\n        log_info(\n            \"[train_action] training job has ended:\"\n            f\" {self.job_id}/{self.task_id} with status: {task_status}\"\n        )\n        return output\n\n    @with_event\n    def resume(self, **kwargs: Dict) -> Dict[str, Any]:\n        \"\"\"\n        resume method for train action\n\n        Parameters:\n            **kwargs (Dict):\n                input args for action resume\n\n        \"\"\"\n        if self.result is not None:\n            log_warn(\"[train_action] already done\")\n            return self.result\n        self.action_event(ActionState.Running, \"train resume\")\n        if self.task_id is not None and self.job_id is not None:\n            log_info(\n                f\"[train_action] resume from created job {self.task_id}/{self.job_id}\"\n            )\n            train_metrics = self._wait_model_trained(**kwargs)\n            self.result = {\n                \"task_id\": self.task_id,\n                \"job_id\": self.job_id,\n                **train_metrics,\n            }\n            return self.result\n        else:\n            if self._input is None:\n                self._input = {}\n            return self._exec(self._input, **kwargs)\n\n    def stop(self, **kwargs: Dict) -> None:\n        \"\"\"\n        stop method for train action\n\n        Parameters:\n            **kwargs (Dict):\n                input args for action stop\n        \"\"\"\n        if self.task_id is None or self.job_id is None:\n            log_warn(\"[train_action] task_id or job_id not set, training not started\")\n            return\n        resp = api.FineTune.V2.stop_task(self.task_id)\n        if resp.get(\"result\"):\n            log_debug(f\"train task {self.task_id}/{self.job_id} stopped successfully\")\n        else:\n            log_debug(f\"train task {self.task_id}/{self.job_id} stopped failed\")\n\n    def get_default_train_config(\n        self, model_type: str, train_mode: console_consts.TrainMode, peft_type: PeftType\n    ) -> TrainConfig:\n        if train_mode == console_consts.TrainMode.PostPretrain:\n            model_info = DefaultPostPretrainTrainConfigMapping.get(\n                model_type,\n                # DefaultTrainConfigMapping[get_config().DEFAULT_FINE_TUNE_TRAIN_TYPE],\n            )\n        else:\n            model_info = DefaultTrainConfigMapping.get(\n                model_type,\n                # DefaultTrainConfigMapping[get_config().DEFAULT_FINE_TUNE_TRAIN_TYPE],\n            )\n        if model_info is None:\n            raise InvalidArgumentError(\n                f\"can not find default config for {model_type} in {peft_type}\"\n            )\n        train_config = model_info[peft_type]\n        train_config.peft_type = peft_type\n        return train_config\n\n\nclass ModelPublishAction(BaseAction[Dict[str, Any], Dict[str, Any]]):\n    \"\"\"\n    Class for Model publish action, Commonly used after `TrainAction`.\n\n    Sample:\n\n    Input:\n    ```\n    {'task_id': \"task-xxx\", 'job_id': \"job-xxx\"}\n    ```\n\n    Output:\n    ```\n    {'task_id': \"task-xxx\", 'job_id': \"job-xxx\", 'model_id': \"xxx\",\n    'model_version_id': \"aaa\", \"model\": <Model>}\n    ```\n    \"\"\"\n\n    task_id: Optional[str] = None\n    \"\"\"task id\"\"\"\n    job_id: Optional[str] = None\n    \"\"\"job id\"\"\"\n    result: Optional[Dict[str, Any]] = None\n    \"\"\"result of model publish action\"\"\"\n    model: Optional[Model] = None\n    \"\"\"model object\"\"\"\n\n    @with_event\n    def exec(self, input: Dict[str, Any] = {}, **kwargs: Dict) -> Dict[str, Any]:\n        if self.task_id == \"\" or self.job_id == \"\":\n            raise InvalidArgumentError(\"task_id or job_id must be set\")\n        self.task_id = input.get(\"task_id\", \"\")\n        self.job_id = input.get(\"job_id\", \"\")\n        self.model = Model(task_id=self.task_id, job_id=self.job_id)\n        return self._exec(input, **kwargs)\n\n    def _exec(self, input: Dict[str, Any] = {}, **kwargs: Dict) -> Dict[str, Any]:\n        if self.model is None:\n            raise InvalidArgumentError(\"model must be set when in model publish._exec\")\n        log_debug(\n            f\"[model_publish_action] start model publish task:, {self.task_id},\"\n            f\" {self.job_id}\"\n        )\n        try:\n            self.action_event(\n                ActionState.Running,\n                \"model publish\",\n                {\n                    \"task_id\": self.task_id,\n                    \"job_id\": self.job_id,\n                },\n            )\n            self.model.publish(name=input.get(\"name\", \"\"), **kwargs)\n            log_debug(\n                f\"[model publish] model: {self.task_id}_{self.job_id} has been\"\n                \" published.\"\n            )\n\n            self.result = {\n                **input,\n                \"task_id\": self.task_id,\n                \"job_id\": self.job_id,\n                \"model_id\": self.model.id,\n                \"model_version_id\": self.model.version_id,\n                \"model\": self.model,\n            }\n            return self.result\n        except Exception as e:\n            log_error(f\"[model_publish_action] model publish error: {e}\")\n            raise e\n\n    @with_event\n    def resume(self, **kwargs: Dict) -> Dict[str, Any]:\n        # raise NotImplementedError(\"ModelPublishAction.resume() is not implemented\")\n        if self.result is not None:\n            return self.result\n        if self.model is not None:\n            return self._exec()\n        if self.task_id is None or self.job_id is None:\n            raise InvalidArgumentError(\"task_id or job_id not set, resume failed\")\n        self.model = Model(task_id=self.task_id, job_id=self.job_id)\n        return self._exec(**kwargs)\n\n\nclass DeployAction(BaseAction[Dict[str, Any], Dict[str, Any]]):\n    \"\"\"DeployAction\n    Action for model service deployment. A TrainConfig must be supplied\n    when instance initialized.\n    Sample:\n        ```\n        deploy_config = DeployConfig(replicas=1, pool_type=1)\n        deploy_action = DeployAction(deploy_config=deploy_config)\n\n        output = deploy_action.exec(input)\n        ```\n\n    input:\n        {'task_id': \"task-xxxx\", 'job_id': \"job-xxxx\", 'model_id': \"xx\",\n        'model_version_id': \"xxx\"}\n    output:\n        ```\n        {'task_id': \"task-xxx\", 'job_id': \"job-xxxx\", 'model_id': \"xx\",\n        'model_version_id': \"xxx\",\n        'service_id': 164, 'service_endpoint': 'xbiimimv_xxx'}\n        ```\n    \"\"\"\n\n    deploy_config: Optional[DeployConfig]\n    \"\"\"deploy config include replicas and so on\"\"\"\n    model_id: Optional[int]\n    \"\"\"model id\"\"\"\n    model_id_str: Optional[str]\n    \"\"\"model str id\"\"\"\n    model_version_id: Optional[int]\n    \"\"\"model version id\"\"\"\n    model_version_id_str: Optional[str]\n    \"\"\"model version str id \"\"\"\n    _input: Optional[Dict[str, Any]] = None\n    \"\"\"input of action\"\"\"\n    result: Optional[Dict[str, Any]] = None\n    \"\"\"result of action\"\"\"\n\n    def __init__(self, deploy_config: Optional[DeployConfig] = None, **kwargs: Any):\n        \"\"\"\n\n        Parameters:\n            deploy_config (Optional[DeployConfig], optional):\n                deploy config include replicas and so on. Defaults to None.\n        \"\"\"\n        super().__init__(**kwargs)\n        self.deploy_config = deploy_config\n\n    @with_event\n    def exec(self, input: Dict[str, Any] = {}, **kwargs: Any) -> Dict[str, Any]:\n        # for resume\n        if self._input is None:\n            self._input = input\n        if self.deploy_config is None:\n            raise InvalidArgumentError(\"deploy_config must be set\")\n        if input.get(\"model\") is None:\n            self.model_id = input.get(\"model_id\")\n            self.model_version_id = input.get(\"model_version_id\")\n            # TODO 迁移成str id\n            if self.model_id is None or self.model_version_id is None:\n                raise InvalidArgumentError(\"model_id or model_version_id must be set\")\n\n            self.model = Model(self.model_id, self.model_version_id, auto_complete=True)\n            self.model.auto_complete_info()\n        else:\n            self.model = cast(Model, input.get(\"model\"))\n            if self.model is None:\n                raise InvalidArgumentError(\n                    \"must input with model or model id and version id\"\n                )\n            self.model.auto_complete_info()\n            self.model_id = self.model.old_id\n            self.model_version_id = self.model.old_version_id\n        # 自动补全\n\n        return self._exec(**kwargs)\n\n    def _exec(self, input: Dict[str, Any] = {}, **kwargs: Dict) -> Dict[str, Any]:\n        if self.deploy_config is None:\n            raise InvalidArgumentError(\"deploy_config must be set in deploy._exec\")\n        log_debug(\n            f\"[deploy_action] try deploy model {self.model.id}_{self.model.version_id}\"\n        )\n        self.action_event(\n            ActionState.Running,\n            \"ready to deploy\",\n            {\n                \"model\": self.model,\n            },\n        )\n        # deploy model\n        self.model.deploy(self.deploy_config, **kwargs)\n        if self.model.service is not None:\n            log_debug(\n                \"[deploy_action] model\"\n                f\" {self.model_id}_{self.model_version_id} deployed successfully with\"\n                \" service:\"\n                f\" {self.model.service.id} endpoint:{self.model.service.endpoint}\"\n            )\n            return {\n                **input,\n                \"service_id\": self.model.service.id,\n                \"service_endpoint\": self.model.service.endpoint,\n                \"service\": self.model.service,\n                \"model\": self.model,\n            }\n        else:\n            raise InternalError(\"model.service is not available\")\n\n    @with_event\n    def resume(self, **kwargs: Dict) -> Dict[str, Any]:\n        \"\"\"\n        resume method for deploy action\n\n        Parameters:\n            **kwargs (Dict):\n                input args for action resume\n\n        \"\"\"\n        if self.model_id is not None and self.model_version_id is not None:\n            self.model = Model(self.model_id_str, self.model_version_id_str)\n            self.model.auto_complete_info()\n        elif self.model is None:\n            raise InvalidArgumentError(\n                \"either (model_id and version_id) or model must be set\"\n            )\n        return self._exec()\n\n\nclass EvaluateAction(BaseAction[Dict[str, Any], Dict[str, Any]]):\n    \"\"\"EvaluateAction\n    Action for evaluate models or services.\n    Sample:\n    input:\n        ```\n        {'model_id': \"am-xxxx\", 'model_version_id': \"amv-xxxx\"}\n        ```\n    output:\n        ```\n        {'eval_res': EvaluationResult ...}\n        ```\n    \"\"\"\n\n    eval_manager: Optional[EvaluationManager] = None\n    \"\"\"evaluation manager for evaluate models or services.\"\"\"\n    eval_dataset: Optional[Dataset] = None\n    _input: Optional[Dict[str, Any]] = None\n    \"\"\"input of action\"\"\"\n    result: Optional[Dict[str, Any]] = None\n    \"\"\"result of action\"\"\"\n\n    def __init__(\n        self, eval_dataset: Dataset, evaluators: List[Evaluator], **kwargs: Any\n    ):\n        \"\"\"\n        init method for evaluate action\n\n        Parameters:\n            eval_dataset Dataset:\n                dataset for evaluation, use Dataset.load() to create.\n            evaluators List[Evaluator]:\n                evaluators for evaluation, include local and qianfan remote evaluators.\n                Specifically, qianfan_evaluators are only available for Model.\n        \"\"\"\n        super().__init__(**kwargs)\n        self.eval_dataset = eval_dataset\n        local_evaluators = [\n            eval for eval in evaluators if isinstance(eval, LocalEvaluator)\n        ]\n        qianfan_evaluators = [\n            eval for eval in evaluators if isinstance(eval, QianfanEvaluator)\n        ]\n        self.eval_manager = EvaluationManager(\n            local_evaluators=local_evaluators if len(local_evaluators) > 0 else None,\n            qianfan_evaluators=(\n                qianfan_evaluators if len(qianfan_evaluators) > 0 else None\n            ),\n        )\n\n    @with_event\n    def exec(self, input: Dict[str, Any] = {}, **kwargs: Any) -> Dict[str, Any]:\n        \"\"\"\n        exec evaluation\n\n        Parameters:\n            input (Dict[str, Any], optional): input dict with model/service info.\n            Defaults to {}.\n\n        Returns:\n            Dict[str, Any]: output the result with the original input\n        \"\"\"\n        self._input = input\n        log_info(f\"[evaluation_action] begin to do evaluation, input: {self._input}\")\n        llm = self._parse_from_input(self._input)\n        res = self._exec(llm, **kwargs)\n        self.result = {\"eval_res\": res, **input}\n        return self.result\n\n    def _parse_from_input(self, input: Dict[str, Any] = {}) -> Union[Model, Service]:\n        \"\"\"\n        Parses and returns the model or service object based on the input parameters.\n\n        Parameters:\n            input (Dict[str, Any], optional): . Defaults to {}.\n\n        Returns:\n            Union[Model, Service]: parsed model or service object\n        \"\"\"\n\n        if input.get(\"service\"):\n            llm = input.get(\"service\")\n        elif input.get(\"model\"):\n            llm = input.get(\"model\")\n        elif input.get(\"model_id\") and input.get(\"model_version_id\"):\n            llm = Model(input[\"model_id\"], input[\"model_version_id\"])\n        else:\n            log_error(f\"[evaluation_action] invalid llm input error {self._input}\")\n            raise InvalidArgumentError(\n                \"model or service must be set in evaluation action\"\n            )\n        assert isinstance(llm, (Model, Service))\n        return llm\n\n    def _exec(self, llm: Union[Model, Service], **kwargs: Dict) -> Any:\n        \"\"\"\n        accept a llm model/service to do evaluation\n\n        Parameters:\n            llm (Union[Model, Service]): model to do evaluation\n\n        Returns:\n            Any: evaluation result object\n        \"\"\"\n        assert self.eval_manager is not None\n        if self.eval_dataset is None:\n            raise InvalidArgumentError(\"eval_dataset must be set\")\n        self.action_event(\n            ActionState.Running,\n            \"ready to evaluate\",\n            {\n                \"llm\": llm,\n                \"dataset\": self.eval_dataset,\n            },\n        )\n        log_info(\"[evaluation_action] running evaluation...\")\n        return self.eval_manager.eval([llm], self.eval_dataset, **kwargs)\n\n    @with_event\n    def resume(self, **kwargs: Dict) -> Dict[str, Any]:\n        \"\"\"\n        resume method for eval action\n\n        Parameters:\n            **kwargs (Dict):\n                input args for action resume\n\n        \"\"\"\n        if self._input is None:\n            log_error(\n                \"[evaluation_action] previous input not found, call run() instead.\"\n            )\n            raise ValueError(\"input not found\")\n        llm = self._parse_from_input(self._input)\n        res = self._exec(llm, **kwargs)\n        self.result = {\"eval_res\": res, **self._input}\n        return self.result\n\n\naction_mapping: Dict[str, Dict[str, Any]] = {\n    LoadDataSetAction.__class__.__name__: {\n        ActionState.Preceding: TrainStatus.DatasetLoading,\n        ActionState.Running: TrainStatus.DatasetLoading,\n        ActionState.Done: TrainStatus.DatasetLoaded,\n        ActionState.Error: TrainStatus.DatasetLoadFailed,\n        ActionState.Stopped: TrainStatus.DatasetLoadStopped,\n    },\n    TrainAction.__class__.__name__: {\n        ActionState.Preceding: TrainStatus.TrainCreated,\n        ActionState.Running: TrainStatus.Training,\n        ActionState.Done: TrainStatus.TrainFinished,\n        ActionState.Error: TrainStatus.TrainFailed,\n        ActionState.Stopped: TrainStatus.TrainStopped,\n    },\n    ModelPublishAction.__class__.__name__: {\n        ActionState.Preceding: TrainStatus.ModelPublishing,\n        ActionState.Running: TrainStatus.ModelPublishing,\n        ActionState.Done: TrainStatus.ModelPublished,\n        ActionState.Error: TrainStatus.ModelPublishFailed,\n        ActionState.Stopped: TrainStatus.ModelPublishFailed,\n    },\n    DeployAction.__class__.__name__: {\n        ActionState.Preceding: ServiceStatus.Created,\n        ActionState.Running: ServiceStatus.Deploying,\n        ActionState.Done: ServiceStatus.Deployed,\n        ActionState.Error: ServiceStatus.DeployFailed,\n        ActionState.Stopped: ServiceStatus.DeployStopped,\n    },\n    EvaluateAction.__class__.__name__: {\n        ActionState.Preceding: TrainStatus.EvaluationCreated,\n        ActionState.Running: TrainStatus.EvaluationRunning,\n        ActionState.Done: TrainStatus.EvaluationFinished,\n        ActionState.Error: TrainStatus.EvaluationFailed,\n        ActionState.Stopped: TrainStatus.EvaluationStopped,\n    },\n}\n"}}, "langflow/Lib/site-packages/litellm/__init__.py": {"29a056bc26e963b0318bf4b8b9ff579e2a8e313c": {"t": 1785746649.4992573, "n": 27228, "h": "cbd580d63181a294", "c": "### Hide pydantic namespace conflict warnings globally ###\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", message=\".*conflict with protected namespace.*\")\n### INIT VARIABLES ###\nimport threading, requests, os\nfrom typing import Callable, List, Optional, Dict, Union, Any, Literal\nfrom litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler\nfrom litellm.caching import Cache\nfrom litellm._logging import (\n    set_verbose,\n    _turn_on_debug,\n    verbose_logger,\n    json_logs,\n    _turn_on_json,\n)\nfrom litellm.proxy._types import (\n    KeyManagementSystem,\n    KeyManagementSettings,\n    LiteLLM_UpperboundKeyGenerateParams,\n)\nimport httpx\nimport dotenv\n\nlitellm_mode = os.getenv(\"LITELLM_MODE\", \"DEV\")  # \"PRODUCTION\", \"DEV\"\nif litellm_mode == \"DEV\":\n    dotenv.load_dotenv()\n#############################################\nif set_verbose == True:\n    _turn_on_debug()\n#############################################\n### Callbacks /Logging / Success / Failure Handlers ###\ninput_callback: List[Union[str, Callable]] = []\nsuccess_callback: List[Union[str, Callable]] = []\nfailure_callback: List[Union[str, Callable]] = []\nservice_callback: List[Union[str, Callable]] = []\n_custom_logger_compatible_callbacks_literal = Literal[\"lago\", \"openmeter\"]\ncallbacks: List[Union[Callable, _custom_logger_compatible_callbacks_literal]] = []\n_langfuse_default_tags: Optional[\n    List[\n        Literal[\n            \"user_api_key_alias\",\n            \"user_api_key_user_id\",\n            \"user_api_key_user_email\",\n            \"user_api_key_team_alias\",\n            \"semantic-similarity\",\n            \"proxy_base_url\",\n        ]\n    ]\n] = None\n_async_input_callback: List[Callable] = (\n    []\n)  # internal variable - async custom callbacks are routed here.\n_async_success_callback: List[Union[str, Callable]] = (\n    []\n)  # internal variable - async custom callbacks are routed here.\n_async_failure_callback: List[Callable] = (\n    []\n)  # internal variable - async custom callbacks are routed here.\npre_call_rules: List[Callable] = []\npost_call_rules: List[Callable] = []\nturn_off_message_logging: Optional[bool] = False\nredact_messages_in_exceptions: Optional[bool] = False\nstore_audit_logs = False  # Enterprise feature, allow users to see audit logs\n## end of callbacks #############\n\nemail: Optional[str] = (\n    None  # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648\n)\ntoken: Optional[str] = (\n    None  # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648\n)\ntelemetry = True\nmax_tokens = 256  # OpenAI Defaults\ndrop_params = False\nmodify_params = False\nretry = True\n### AUTH ###\napi_key: Optional[str] = None\nopenai_key: Optional[str] = None\ndatabricks_key: Optional[str] = None\nazure_key: Optional[str] = None\nanthropic_key: Optional[str] = None\nreplicate_key: Optional[str] = None\ncohere_key: Optional[str] = None\nclarifai_key: Optional[str] = None\nmaritalk_key: Optional[str] = None\nai21_key: Optional[str] = None\nollama_key: Optional[str] = None\nopenrouter_key: Optional[str] = None\npredibase_key: Optional[str] = None\nhuggingface_key: Optional[str] = None\nvertex_project: Optional[str] = None\nvertex_location: Optional[str] = None\npredibase_tenant_id: Optional[str] = None\ntogetherai_api_key: Optional[str] = None\ncloudflare_api_key: Optional[str] = None\nbaseten_key: Optional[str] = None\naleph_alpha_key: Optional[str] = None\nnlp_cloud_key: Optional[str] = None\ncommon_cloud_provider_auth_params: dict = {\n    \"params\": [\"project\", \"region_name\", \"token\"],\n    \"providers\": [\"vertex_ai\", \"bedrock\", \"watsonx\", \"azure\"],\n}\nuse_client: bool = False\nssl_verify: bool = True\nssl_certificate: Optional[str] = None\ndisable_streaming_logging: bool = False\nin_memory_llm_clients_cache: dict = {}\n### GUARDRAILS ###\nllamaguard_model_name: Optional[str] = None\nopenai_moderations_model_name: Optional[str] = None\npresidio_ad_hoc_recognizers: Optional[str] = None\ngoogle_moderation_confidence_threshold: Optional[float] = None\nllamaguard_unsafe_content_categories: Optional[str] = None\nblocked_user_list: Optional[Union[str, List]] = None\nbanned_keywords_list: Optional[Union[str, List]] = None\nllm_guard_mode: Literal[\"all\", \"key-specific\", \"request-specific\"] = \"all\"\n##################\n### PREVIEW FEATURES ###\nenable_preview_features: bool = False\n##################\nlogging: bool = True\ncaching: bool = (\n    False  # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648\n)\ncaching_with_models: bool = (\n    False  # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648\n)\ncache: Optional[Cache] = (\n    None  # cache object <- use this - https://docs.litellm.ai/docs/caching\n)\ndefault_in_memory_ttl: Optional[float] = None\ndefault_redis_ttl: Optional[float] = None\nmodel_alias_map: Dict[str, str] = {}\nmodel_group_alias_map: Dict[str, str] = {}\nmax_budget: float = 0.0  # set the max budget across all providers\nbudget_duration: Optional[str] = (\n    None  # proxy only - resets budget after fixed duration. You can set duration as seconds (\"30s\"), minutes (\"30m\"), hours (\"30h\"), days (\"30d\").\n)\ndefault_soft_budget: float = (\n    50.0  # by default all litellm proxy keys have a soft budget of 50.0\n)\n_openai_finish_reasons = [\"stop\", \"length\", \"function_call\", \"content_filter\", \"null\"]\n_openai_completion_params = [\n    \"functions\",\n    \"function_call\",\n    \"temperature\",\n    \"temperature\",\n    \"top_p\",\n    \"n\",\n    \"stream\",\n    \"stop\",\n    \"max_tokens\",\n    \"presence_penalty\",\n    \"frequency_penalty\",\n    \"logit_bias\",\n    \"user\",\n    \"request_timeout\",\n    \"api_base\",\n    \"api_version\",\n    \"api_key\",\n    \"deployment_id\",\n    \"organization\",\n    \"base_url\",\n    \"default_headers\",\n    \"timeout\",\n    \"response_format\",\n    \"seed\",\n    \"tools\",\n    \"tool_choice\",\n    \"max_retries\",\n]\n_litellm_completion_params = [\n    \"metadata\",\n    \"acompletion\",\n    \"caching\",\n    \"mock_response\",\n    \"api_key\",\n    \"api_version\",\n    \"api_base\",\n    \"force_timeout\",\n    \"logger_fn\",\n    \"verbose\",\n    \"custom_llm_provider\",\n    \"litellm_logging_obj\",\n    \"litellm_call_id\",\n    \"use_client\",\n    \"id\",\n    \"fallbacks\",\n    \"azure\",\n    \"headers\",\n    \"model_list\",\n    \"num_retries\",\n    \"context_window_fallback_dict\",\n    \"roles\",\n    \"final_prompt_value\",\n    \"bos_token\",\n    \"eos_token\",\n    \"request_timeout\",\n    \"complete_response\",\n    \"self\",\n    \"client\",\n    \"rpm\",\n    \"tpm\",\n    \"input_cost_per_token\",\n    \"output_cost_per_token\",\n    \"hf_model_name\",\n    \"model_info\",\n    \"proxy_server_request\",\n    \"preset_cache_key\",\n]\n_current_cost = 0  # private variable, used if max budget is set\nerror_logs: Dict = {}\nadd_function_to_prompt: bool = (\n    False  # if function calling not supported by api, append function call details to system prompt\n)\nclient_session: Optional[httpx.Client] = None\naclient_session: Optional[httpx.AsyncClient] = None\nmodel_fallbacks: Optional[List] = None  # Deprecated for 'litellm.fallbacks'\nmodel_cost_map_url: str = (\n    \"https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json\"\n)\nsuppress_debug_info = False\ndynamodb_table_name: Optional[str] = None\ns3_callback_params: Optional[Dict] = None\ngeneric_logger_headers: Optional[Dict] = None\ndefault_key_generate_params: Optional[Dict] = None\nupperbound_key_generate_params: Optional[LiteLLM_UpperboundKeyGenerateParams] = None\ndefault_user_params: Optional[Dict] = None\ndefault_team_settings: Optional[List] = None\nmax_user_budget: Optional[float] = None\nmax_end_user_budget: Optional[float] = None\n#### RELIABILITY ####\nrequest_timeout: float = 6000\nmodule_level_aclient = AsyncHTTPHandler(timeout=request_timeout)\nmodule_level_client = HTTPHandler(timeout=request_timeout)\nnum_retries: Optional[int] = None  # per model endpoint\ndefault_fallbacks: Optional[List] = None\nfallbacks: Optional[List] = None\ncontext_window_fallbacks: Optional[List] = None\nallowed_fails: int = 0\nnum_retries_per_request: Optional[int] = (\n    None  # for the request overall (incl. fallbacks + model retries)\n)\n####### SECRET MANAGERS #####################\nsecret_manager_client: Optional[Any] = (\n    None  # list of instantiated key management clients - e.g. azure kv, infisical, etc.\n)\n_google_kms_resource_name: Optional[str] = None\n_key_management_system: Optional[KeyManagementSystem] = None\n_key_management_settings: Optional[KeyManagementSettings] = None\n#### PII MASKING ####\noutput_parse_pii: bool = False\n#############################################\n\n\ndef get_model_cost_map(url: str):\n    if (\n        os.getenv(\"LITELLM_LOCAL_MODEL_COST_MAP\", False) == True\n        or os.getenv(\"LITELLM_LOCAL_MODEL_COST_MAP\", False) == \"True\"\n    ):\n        import importlib.resources\n        import json\n\n        with importlib.resources.open_text(\n            \"litellm\", \"model_prices_and_context_window_backup.json\"\n        ) as f:\n            content = json.load(f)\n            return content\n\n    try:\n        with requests.get(\n            url, timeout=5\n        ) as response:  # set a 5 second timeout for the get request\n            response.raise_for_status()  # Raise an exception if the request is unsuccessful\n            content = response.json()\n            return content\n    except Exception as e:\n        import importlib.resources\n        import json\n\n        with importlib.resources.open_text(\n            \"litellm\", \"model_prices_and_context_window_backup.json\"\n        ) as f:\n            content = json.load(f)\n            return content\n\n\nmodel_cost = get_model_cost_map(url=model_cost_map_url)\ncustom_prompt_dict: Dict[str, dict] = {}\n\n\n####### THREAD-SPECIFIC DATA ###################\nclass MyLocal(threading.local):\n    def __init__(self):\n        self.user = \"Hello World\"\n\n\n_thread_context = MyLocal()\n\n\ndef identify(event_details):\n    # Store user in thread local data\n    if \"user\" in event_details:\n        _thread_context.user = event_details[\"user\"]\n\n\n####### ADDITIONAL PARAMS ################### configurable params if you use proxy models like Helicone, map spend to org id, etc.\napi_base = None\nheaders = None\napi_version = None\norganization = None\nproject = None\nconfig_path = None\n####### COMPLETION MODELS ###################\nopen_ai_chat_completion_models: List = []\nopen_ai_text_completion_models: List = []\ncohere_models: List = []\ncohere_chat_models: List = []\nmistral_chat_models: List = []\nanthropic_models: List = []\nopenrouter_models: List = []\nvertex_language_models: List = []\nvertex_vision_models: List = []\nvertex_chat_models: List = []\nvertex_code_chat_models: List = []\nvertex_text_models: List = []\nvertex_code_text_models: List = []\nvertex_embedding_models: List = []\nvertex_anthropic_models: List = []\nai21_models: List = []\nnlp_cloud_models: List = []\naleph_alpha_models: List = []\nbedrock_models: List = []\ndeepinfra_models: List = []\nperplexity_models: List = []\nwatsonx_models: List = []\nfor key, value in model_cost.items():\n    if value.get(\"litellm_provider\") == \"openai\":\n        open_ai_chat_completion_models.append(key)\n    elif value.get(\"litellm_provider\") == \"text-completion-openai\":\n        open_ai_text_completion_models.append(key)\n    elif value.get(\"litellm_provider\") == \"cohere\":\n        cohere_models.append(key)\n    elif value.get(\"litellm_provider\") == \"cohere_chat\":\n        cohere_chat_models.append(key)\n    elif value.get(\"litellm_provider\") == \"mistral\":\n        mistral_chat_models.append(key)\n    elif value.get(\"litellm_provider\") == \"anthropic\":\n        anthropic_models.append(key)\n    elif value.get(\"litellm_provider\") == \"openrouter\":\n        openrouter_models.append(key)\n    elif value.get(\"litellm_provider\") == \"vertex_ai-text-models\":\n        vertex_text_models.append(key)\n    elif value.get(\"litellm_provider\") == \"vertex_ai-code-text-models\":\n        vertex_code_text_models.append(key)\n    elif value.get(\"litellm_provider\") == \"vertex_ai-language-models\":\n        vertex_language_models.append(key)\n    elif value.get(\"litellm_provider\") == \"vertex_ai-vision-models\":\n        vertex_vision_models.append(key)\n    elif value.get(\"litellm_provider\") == \"vertex_ai-chat-models\":\n        vertex_chat_models.append(key)\n    elif value.get(\"litellm_provider\") == \"vertex_ai-code-chat-models\":\n        vertex_code_chat_models.append(key)\n    elif value.get(\"litellm_provider\") == \"vertex_ai-embedding-models\":\n        vertex_embedding_models.append(key)\n    elif value.get(\"litellm_provider\") == \"vertex_ai-anthropic_models\":\n        key = key.replace(\"vertex_ai/\", \"\")\n        vertex_anthropic_models.append(key)\n    elif value.get(\"litellm_provider\") == \"ai21\":\n        ai21_models.append(key)\n    elif value.get(\"litellm_provider\") == \"nlp_cloud\":\n        nlp_cloud_models.append(key)\n    elif value.get(\"litellm_provider\") == \"aleph_alpha\":\n        aleph_alpha_models.append(key)\n    elif value.get(\"litellm_provider\") == \"bedrock\":\n        bedrock_models.append(key)\n    elif value.get(\"litellm_provider\") == \"deepinfra\":\n        deepinfra_models.append(key)\n    elif value.get(\"litellm_provider\") == \"perplexity\":\n        perplexity_models.append(key)\n    elif value.get(\"litellm_provider\") == \"watsonx\":\n        watsonx_models.append(key)\n\n# known openai compatible endpoints - we'll eventually move this list to the model_prices_and_context_window.json dictionary\nopenai_compatible_endpoints: List = [\n    \"api.perplexity.ai\",\n    \"api.endpoints.anyscale.com/v1\",\n    \"api.deepinfra.com/v1/openai\",\n    \"api.mistral.ai/v1\",\n    \"api.groq.com/openai/v1\",\n    \"api.deepseek.com/v1\",\n    \"api.together.xyz/v1\",\n]\n\n# this is maintained for Exception Mapping\nopenai_compatible_providers: List = [\n    \"anyscale\",\n    \"mistral\",\n    \"groq\",\n    \"deepseek\",\n    \"deepinfra\",\n    \"perplexity\",\n    \"xinference\",\n    \"together_ai\",\n    \"fireworks_ai\",\n]\n\n\n# well supported replicate llms\nreplicate_models: List = [\n    # llama replicate supported LLMs\n    \"replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf\",\n    \"a16z-infra/llama-2-13b-chat:2a7f981751ec7fdf87b5b91ad4db53683a98082e9ff7bfd12c8cd5ea85980a52\",\n    \"meta/codellama-13b:1c914d844307b0588599b8393480a3ba917b660c7e9dfae681542b5325f228db\",\n    # Vicuna\n    \"replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b\",\n    \"joehoover/instructblip-vicuna13b:c4c54e3c8c97cd50c2d2fec9be3b6065563ccf7d43787fb99f84151b867178fe\",\n    # Flan T-5\n    \"daanelson/flan-t5-large:ce962b3f6792a57074a601d3979db5839697add2e4e02696b3ced4c022d4767f\",\n    # Others\n    \"replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5\",\n    \"replit/replit-code-v1-3b:b84f4c074b807211cd75e3e8b1589b6399052125b4c27106e43d47189e8415ad\",\n]\n\nclarifai_models: List = [\n    \"clarifai/meta.Llama-3.Llama-3-8B-Instruct\",\n    \"clarifai/gcp.generate.gemma-1_1-7b-it\",\n    \"clarifai/mistralai.completion.mixtral-8x22B\",\n    \"clarifai/cohere.generate.command-r-plus\",\n    \"clarifai/databricks.drbx.dbrx-instruct\",\n    \"clarifai/mistralai.completion.mistral-large\",\n    \"clarifai/mistralai.completion.mistral-medium\",\n    \"clarifai/mistralai.completion.mistral-small\",\n    \"clarifai/mistralai.completion.mixtral-8x7B-Instruct-v0_1\",\n    \"clarifai/gcp.generate.gemma-2b-it\",\n    \"clarifai/gcp.generate.gemma-7b-it\",\n    \"clarifai/deci.decilm.deciLM-7B-instruct\",\n    \"clarifai/mistralai.completion.mistral-7B-Instruct\",\n    \"clarifai/gcp.generate.gemini-pro\",\n    \"clarifai/anthropic.completion.claude-v1\",\n    \"clarifai/anthropic.completion.claude-instant-1_2\",\n    \"clarifai/anthropic.completion.claude-instant\",\n    \"clarifai/anthropic.completion.claude-v2\",\n    \"clarifai/anthropic.completion.claude-2_1\",\n    \"clarifai/meta.Llama-2.codeLlama-70b-Python\",\n    \"clarifai/meta.Llama-2.codeLlama-70b-Instruct\",\n    \"clarifai/openai.completion.gpt-3_5-turbo-instruct\",\n    \"clarifai/meta.Llama-2.llama2-7b-chat\",\n    \"clarifai/meta.Llama-2.llama2-13b-chat\",\n    \"clarifai/meta.Llama-2.llama2-70b-chat\",\n    \"clarifai/openai.chat-completion.gpt-4-turbo\",\n    \"clarifai/microsoft.text-generation.phi-2\",\n    \"clarifai/meta.Llama-2.llama2-7b-chat-vllm\",\n    \"clarifai/upstage.solar.solar-10_7b-instruct\",\n    \"clarifai/openchat.openchat.openchat-3_5-1210\",\n    \"clarifai/togethercomputer.stripedHyena.stripedHyena-Nous-7B\",\n    \"clarifai/gcp.generate.text-bison\",\n    \"clarifai/meta.Llama-2.llamaGuard-7b\",\n    \"clarifai/fblgit.una-cybertron.una-cybertron-7b-v2\",\n    \"clarifai/openai.chat-completion.GPT-4\",\n    \"clarifai/openai.chat-completion.GPT-3_5-turbo\",\n    \"clarifai/ai21.complete.Jurassic2-Grande\",\n    \"clarifai/ai21.complete.Jurassic2-Grande-Instruct\",\n    \"clarifai/ai21.complete.Jurassic2-Jumbo-Instruct\",\n    \"clarifai/ai21.complete.Jurassic2-Jumbo\",\n    \"clarifai/ai21.complete.Jurassic2-Large\",\n    \"clarifai/cohere.generate.cohere-generate-command\",\n    \"clarifai/wizardlm.generate.wizardCoder-Python-34B\",\n    \"clarifai/wizardlm.generate.wizardLM-70B\",\n    \"clarifai/tiiuae.falcon.falcon-40b-instruct\",\n    \"clarifai/togethercomputer.RedPajama.RedPajama-INCITE-7B-Chat\",\n    \"clarifai/gcp.generate.code-gecko\",\n    \"clarifai/gcp.generate.code-bison\",\n    \"clarifai/mistralai.completion.mistral-7B-OpenOrca\",\n    \"clarifai/mistralai.completion.openHermes-2-mistral-7B\",\n    \"clarifai/wizardlm.generate.wizardLM-13B\",\n    \"clarifai/huggingface-research.zephyr.zephyr-7B-alpha\",\n    \"clarifai/wizardlm.generate.wizardCoder-15B\",\n    \"clarifai/microsoft.text-generation.phi-1_5\",\n    \"clarifai/databricks.Dolly-v2.dolly-v2-12b\",\n    \"clarifai/bigcode.code.StarCoder\",\n    \"clarifai/salesforce.xgen.xgen-7b-8k-instruct\",\n    \"clarifai/mosaicml.mpt.mpt-7b-instruct\",\n    \"clarifai/anthropic.completion.claude-3-opus\",\n    \"clarifai/anthropic.completion.claude-3-sonnet\",\n    \"clarifai/gcp.generate.gemini-1_5-pro\",\n    \"clarifai/gcp.generate.imagen-2\",\n    \"clarifai/salesforce.blip.general-english-image-caption-blip-2\",\n]\n\n\nhuggingface_models: List = [\n    \"meta-llama/Llama-2-7b-hf\",\n    \"meta-llama/Llama-2-7b-chat-hf\",\n    \"meta-llama/Llama-2-13b-hf\",\n    \"meta-llama/Llama-2-13b-chat-hf\",\n    \"meta-llama/Llama-2-70b-hf\",\n    \"meta-llama/Llama-2-70b-chat-hf\",\n    \"meta-llama/Llama-2-7b\",\n    \"meta-llama/Llama-2-7b-chat\",\n    \"meta-llama/Llama-2-13b\",\n    \"meta-llama/Llama-2-13b-chat\",\n    \"meta-llama/Llama-2-70b\",\n    \"meta-llama/Llama-2-70b-chat\",\n]  # these have been tested on extensively. But by default all text2text-generation and text-generation models are supported by liteLLM. - https://docs.litellm.ai/docs/providers\n\ntogether_ai_models: List = [\n    # llama llms - chat\n    \"togethercomputer/llama-2-70b-chat\",\n    # llama llms - language / instruct\n    \"togethercomputer/llama-2-70b\",\n    \"togethercomputer/LLaMA-2-7B-32K\",\n    \"togethercomputer/Llama-2-7B-32K-Instruct\",\n    \"togethercomputer/llama-2-7b\",\n    # falcon llms\n    \"togethercomputer/falcon-40b-instruct\",\n    \"togethercomputer/falcon-7b-instruct\",\n    # alpaca\n    \"togethercomputer/alpaca-7b\",\n    # chat llms\n    \"HuggingFaceH4/starchat-alpha\",\n    # code llms\n    \"togethercomputer/CodeLlama-34b\",\n    \"togethercomputer/CodeLlama-34b-Instruct\",\n    \"togethercomputer/CodeLlama-34b-Python\",\n    \"defog/sqlcoder\",\n    \"NumbersStation/nsql-llama-2-7B\",\n    \"WizardLM/WizardCoder-15B-V1.0\",\n    \"WizardLM/WizardCoder-Python-34B-V1.0\",\n    # language llms\n    \"NousResearch/Nous-Hermes-Llama2-13b\",\n    \"Austism/chronos-hermes-13b\",\n    \"upstage/SOLAR-0-70b-16bit\",\n    \"WizardLM/WizardLM-70B-V1.0\",\n]  # supports all together ai models, just pass in the model id e.g. completion(model=\"together_computer/replit_code_3b\",...)\n\n\nbaseten_models: List = [\n    \"qvv0xeq\",\n    \"q841o8w\",\n    \"31dxrj3\",\n]  # FALCON 7B  # WizardLM  # Mosaic ML\n\n\n# used for Cost Tracking & Token counting\n# https://azure.microsoft.com/en-in/pricing/details/cognitive-services/openai-service/\n# Azure returns gpt-35-turbo in their responses, we need to map this to azure/gpt-3.5-turbo for token counting\nazure_llms = {\n    \"gpt-35-turbo\": \"azure/gpt-35-turbo\",\n    \"gpt-35-turbo-16k\": \"azure/gpt-35-turbo-16k\",\n    \"gpt-35-turbo-instruct\": \"azure/gpt-35-turbo-instruct\",\n}\n\nazure_embedding_models = {\n    \"ada\": \"azure/ada\",\n}\n\npetals_models = [\n    \"petals-team/StableBeluga2\",\n]\n\nollama_models = [\"llama2\"]\n\nmaritalk_models = [\"maritalk\"]\n\nmodel_list = (\n    open_ai_chat_completion_models\n    + open_ai_text_completion_models\n    + cohere_models\n    + cohere_chat_models\n    + anthropic_models\n    + replicate_models\n    + openrouter_models\n    + huggingface_models\n    + vertex_chat_models\n    + vertex_text_models\n    + ai21_models\n    + together_ai_models\n    + baseten_models\n    + aleph_alpha_models\n    + nlp_cloud_models\n    + ollama_models\n    + bedrock_models\n    + deepinfra_models\n    + perplexity_models\n    + maritalk_models\n    + vertex_language_models\n    + watsonx_models\n)\n\nprovider_list: List = [\n    \"openai\",\n    \"custom_openai\",\n    \"text-completion-openai\",\n    \"cohere\",\n    \"cohere_chat\",\n    \"clarifai\",\n    \"anthropic\",\n    \"replicate\",\n    \"huggingface\",\n    \"together_ai\",\n    \"openrouter\",\n    \"vertex_ai\",\n    \"palm\",\n    \"gemini\",\n    \"ai21\",\n    \"baseten\",\n    \"azure\",\n    \"azure_text\",\n    \"sagemaker\",\n    \"bedrock\",\n    \"vllm\",\n    \"nlp_cloud\",\n    \"petals\",\n    \"oobabooga\",\n    \"ollama\",\n    \"ollama_chat\",\n    \"deepinfra\",\n    \"perplexity\",\n    \"anyscale\",\n    \"mistral\",\n    \"groq\",\n    \"deepseek\",\n    \"maritalk\",\n    \"voyage\",\n    \"cloudflare\",\n    \"xinference\",\n    \"fireworks_ai\",\n    \"watsonx\",\n    \"triton\",\n    \"predibase\",\n    \"databricks\",\n    \"custom\",  # custom apis\n]\n\nmodels_by_provider: dict = {\n    \"openai\": open_ai_chat_completion_models + open_ai_text_completion_models,\n    \"cohere\": cohere_models,\n    \"cohere_chat\": cohere_chat_models,\n    \"anthropic\": anthropic_models,\n    \"replicate\": replicate_models,\n    \"huggingface\": huggingface_models,\n    \"together_ai\": together_ai_models,\n    \"baseten\": baseten_models,\n    \"openrouter\": openrouter_models,\n    \"vertex_ai\": vertex_chat_models\n    + vertex_text_models\n    + vertex_anthropic_models\n    + vertex_vision_models\n    + vertex_language_models,\n    \"ai21\": ai21_models,\n    \"bedrock\": bedrock_models,\n    \"petals\": petals_models,\n    \"ollama\": ollama_models,\n    \"deepinfra\": deepinfra_models,\n    \"perplexity\": perplexity_models,\n    \"maritalk\": maritalk_models,\n    \"watsonx\": watsonx_models,\n}\n\n# mapping for those models which have larger equivalents\nlonger_context_model_fallback_dict: dict = {\n    # openai chat completion models\n    \"gpt-3.5-turbo\": \"gpt-3.5-turbo-16k\",\n    \"gpt-3.5-turbo-0301\": \"gpt-3.5-turbo-16k-0301\",\n    \"gpt-3.5-turbo-0613\": \"gpt-3.5-turbo-16k-0613\",\n    \"gpt-4\": \"gpt-4-32k\",\n    \"gpt-4-0314\": \"gpt-4-32k-0314\",\n    \"gpt-4-0613\": \"gpt-4-32k-0613\",\n    # anthropic\n    \"claude-instant-1\": \"claude-2\",\n    \"claude-instant-1.2\": \"claude-2\",\n    # vertexai\n    \"chat-bison\": \"chat-bison-32k\",\n    \"chat-bison@001\": \"chat-bison-32k\",\n    \"codechat-bison\": \"codechat-bison-32k\",\n    \"codechat-bison@001\": \"codechat-bison-32k\",\n    # openrouter\n    \"openrouter/openai/gpt-3.5-turbo\": \"openrouter/openai/gpt-3.5-turbo-16k\",\n    \"openrouter/anthropic/claude-instant-v1\": \"openrouter/anthropic/claude-2\",\n}\n\n####### EMBEDDING MODELS ###################\nopen_ai_embedding_models: List = [\"text-embedding-ada-002\"]\ncohere_embedding_models: List = [\n    \"embed-english-v3.0\",\n    \"embed-english-light-v3.0\",\n    \"embed-multilingual-v3.0\",\n    \"embed-english-v2.0\",\n    \"embed-english-light-v2.0\",\n    \"embed-multilingual-v2.0\",\n]\nbedrock_embedding_models: List = [\n    \"amazon.titan-embed-text-v1\",\n    \"cohere.embed-english-v3\",\n    \"cohere.embed-multilingual-v3\",\n]\n\nall_embedding_models = (\n    open_ai_embedding_models\n    + cohere_embedding_models\n    + bedrock_embedding_models\n    + vertex_embedding_models\n)\n\n####### IMAGE GENERATION MODELS ###################\nopenai_image_generation_models = [\"dall-e-2\", \"dall-e-3\"]\n\nfrom .timeout import timeout\nfrom .cost_calculator import completion_cost\nfrom .utils import (\n    client,\n    exception_type,\n    get_optional_params,\n    modify_integration,\n    token_counter,\n    create_pretrained_tokenizer,\n    create_tokenizer,\n    cost_per_token,\n    supports_function_calling,\n    supports_parallel_function_calling,\n    supports_vision,\n    get_litellm_params,\n    Logging,\n    acreate,\n    get_model_list,\n    get_max_tokens,\n    get_model_info,\n    register_prompt_template,\n    validate_environment,\n    check_valid_key,\n    get_llm_provider,\n    register_model,\n    encode,\n    decode,\n    _calculate_retry_after,\n    _should_retry,\n    get_secret,\n    get_supported_openai_params,\n    get_api_base,\n    get_first_chars_messages,\n    ModelResponse,\n    ImageResponse,\n    ImageObject,\n    get_provider_fields,\n)\nfrom .llms.huggingface_restapi import HuggingfaceConfig\nfrom .llms.anthropic import AnthropicConfig\nfrom .llms.databricks import DatabricksConfig, DatabricksEmbeddingConfig\nfrom .llms.predibase import PredibaseConfig\nfrom .llms.anthropic_text import AnthropicTextConfig\nfrom .llms.replicate import ReplicateConfig\nfrom .llms.cohere import CohereConfig\nfrom .llms.clarifai import ClarifaiConfig\nfrom .llms.ai21 import AI21Config\nfrom .llms.together_ai import TogetherAIConfig\nfrom .llms.cloudflare import CloudflareConfig\nfrom .llms.palm import PalmConfig\nfrom .llms.gemini import GeminiConfig\nfrom .llms.nlp_cloud import NLPCloudConfig\nfrom .llms.aleph_alpha import AlephAlphaConfig\nfrom .llms.petals import PetalsConfig\nfrom .llms.vertex_ai import VertexAIConfig\nfrom .llms.vertex_ai_anthropic import VertexAIAnthropicConfig\nfrom .llms.sagemaker import SagemakerConfig\nfrom .llms.ollama import OllamaConfig\nfrom .llms.ollama_chat import OllamaChatConfig\nfrom .llms.maritalk import MaritTalkConfig\nfrom .llms.bedrock_httpx import AmazonCohereChatConfig, AmazonConverseConfig\nfrom .llms.bedrock import (\n    AmazonTitanConfig,\n    AmazonAI21Config,\n    AmazonAnthropicConfig,\n    AmazonAnthropicClaude3Config,\n    AmazonCohereConfig,\n    AmazonLlamaConfig,\n    AmazonStabilityConfig,\n    AmazonMistralConfig,\n    AmazonBedrockGlobalConfig,\n)\nfrom .llms.openai import (\n    OpenAIConfig,\n    OpenAITextCompletionConfig,\n    MistralConfig,\n    DeepInfraConfig,\n)\nfrom .llms.azure import (\n    AzureOpenAIConfig,\n    AzureOpenAIError,\n    AzureOpenAIAssistantsAPIConfig,\n)\nfrom .llms.watsonx import IBMWatsonXAIConfig\nfrom .main import *  # type: ignore\nfrom .integrations import *\nfrom .exceptions import (\n    AuthenticationError,\n    InvalidRequestError,\n    BadRequestError,\n    NotFoundError,\n    RateLimitError,\n    ServiceUnavailableError,\n    OpenAIError,\n    ContextWindowExceededError,\n    ContentPolicyViolationError,\n    BudgetExceededError,\n    APIError,\n    Timeout,\n    APIConnectionError,\n    APIResponseValidationError,\n    UnprocessableEntityError,\n    InternalServerError,\n    LITELLM_EXCEPTION_TYPES,\n)\nfrom .budget_manager import BudgetManager\nfrom .proxy.proxy_cli import run_server\nfrom .router import Router\nfrom .assistants.main import *\nfrom .batches.main import *\nfrom .scheduler import *\nfrom .cost_calculator import response_cost_calculator\n"}, "2450556b5a090e47498b661ec10fb61756e152bd": {"t": 1785766559.6951542, "n": 27228, "h": "cbd580d63181a294", "c": "### Hide pydantic namespace conflict warnings globally ###\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", message=\".*conflict with protected namespace.*\")\n### INIT VARIABLES ###\nimport threading, requests, os\nfrom typing import Callable, List, Optional, Dict, Union, Any, Literal\nfrom litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler\nfrom litellm.caching import Cache\nfrom litellm._logging import (\n    set_verbose,\n    _turn_on_debug,\n    verbose_logger,\n    json_logs,\n    _turn_on_json,\n)\nfrom litellm.proxy._types import (\n    KeyManagementSystem,\n    KeyManagementSettings,\n    LiteLLM_UpperboundKeyGenerateParams,\n)\nimport httpx\nimport dotenv\n\nlitellm_mode = os.getenv(\"LITELLM_MODE\", \"DEV\")  # \"PRODUCTION\", \"DEV\"\nif litellm_mode == \"DEV\":\n    dotenv.load_dotenv()\n#############################################\nif set_verbose == True:\n    _turn_on_debug()\n#############################################\n### Callbacks /Logging / Success / Failure Handlers ###\ninput_callback: List[Union[str, Callable]] = []\nsuccess_callback: List[Union[str, Callable]] = []\nfailure_callback: List[Union[str, Callable]] = []\nservice_callback: List[Union[str, Callable]] = []\n_custom_logger_compatible_callbacks_literal = Literal[\"lago\", \"openmeter\"]\ncallbacks: List[Union[Callable, _custom_logger_compatible_callbacks_literal]] = []\n_langfuse_default_tags: Optional[\n    List[\n        Literal[\n            \"user_api_key_alias\",\n            \"user_api_key_user_id\",\n            \"user_api_key_user_email\",\n            \"user_api_key_team_alias\",\n            \"semantic-similarity\",\n            \"proxy_base_url\",\n        ]\n    ]\n] = None\n_async_input_callback: List[Callable] = (\n    []\n)  # internal variable - async custom callbacks are routed here.\n_async_success_callback: List[Union[str, Callable]] = (\n    []\n)  # internal variable - async custom callbacks are routed here.\n_async_failure_callback: List[Callable] = (\n    []\n)  # internal variable - async custom callbacks are routed here.\npre_call_rules: List[Callable] = []\npost_call_rules: List[Callable] = []\nturn_off_message_logging: Optional[bool] = False\nredact_messages_in_exceptions: Optional[bool] = False\nstore_audit_logs = False  # Enterprise feature, allow users to see audit logs\n## end of callbacks #############\n\nemail: Optional[str] = (\n    None  # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648\n)\ntoken: Optional[str] = (\n    None  # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648\n)\ntelemetry = True\nmax_tokens = 256  # OpenAI Defaults\ndrop_params = False\nmodify_params = False\nretry = True\n### AUTH ###\napi_key: Optional[str] = None\nopenai_key: Optional[str] = None\ndatabricks_key: Optional[str] = None\nazure_key: Optional[str] = None\nanthropic_key: Optional[str] = None\nreplicate_key: Optional[str] = None\ncohere_key: Optional[str] = None\nclarifai_key: Optional[str] = None\nmaritalk_key: Optional[str] = None\nai21_key: Optional[str] = None\nollama_key: Optional[str] = None\nopenrouter_key: Optional[str] = None\npredibase_key: Optional[str] = None\nhuggingface_key: Optional[str] = None\nvertex_project: Optional[str] = None\nvertex_location: Optional[str] = None\npredibase_tenant_id: Optional[str] = None\ntogetherai_api_key: Optional[str] = None\ncloudflare_api_key: Optional[str] = None\nbaseten_key: Optional[str] = None\naleph_alpha_key: Optional[str] = None\nnlp_cloud_key: Optional[str] = None\ncommon_cloud_provider_auth_params: dict = {\n    \"params\": [\"project\", \"region_name\", \"token\"],\n    \"providers\": [\"vertex_ai\", \"bedrock\", \"watsonx\", \"azure\"],\n}\nuse_client: bool = False\nssl_verify: bool = True\nssl_certificate: Optional[str] = None\ndisable_streaming_logging: bool = False\nin_memory_llm_clients_cache: dict = {}\n### GUARDRAILS ###\nllamaguard_model_name: Optional[str] = None\nopenai_moderations_model_name: Optional[str] = None\npresidio_ad_hoc_recognizers: Optional[str] = None\ngoogle_moderation_confidence_threshold: Optional[float] = None\nllamaguard_unsafe_content_categories: Optional[str] = None\nblocked_user_list: Optional[Union[str, List]] = None\nbanned_keywords_list: Optional[Union[str, List]] = None\nllm_guard_mode: Literal[\"all\", \"key-specific\", \"request-specific\"] = \"all\"\n##################\n### PREVIEW FEATURES ###\nenable_preview_features: bool = False\n##################\nlogging: bool = True\ncaching: bool = (\n    False  # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648\n)\ncaching_with_models: bool = (\n    False  # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648\n)\ncache: Optional[Cache] = (\n    None  # cache object <- use this - https://docs.litellm.ai/docs/caching\n)\ndefault_in_memory_ttl: Optional[float] = None\ndefault_redis_ttl: Optional[float] = None\nmodel_alias_map: Dict[str, str] = {}\nmodel_group_alias_map: Dict[str, str] = {}\nmax_budget: float = 0.0  # set the max budget across all providers\nbudget_duration: Optional[str] = (\n    None  # proxy only - resets budget after fixed duration. You can set duration as seconds (\"30s\"), minutes (\"30m\"), hours (\"30h\"), days (\"30d\").\n)\ndefault_soft_budget: float = (\n    50.0  # by default all litellm proxy keys have a soft budget of 50.0\n)\n_openai_finish_reasons = [\"stop\", \"length\", \"function_call\", \"content_filter\", \"null\"]\n_openai_completion_params = [\n    \"functions\",\n    \"function_call\",\n    \"temperature\",\n    \"temperature\",\n    \"top_p\",\n    \"n\",\n    \"stream\",\n    \"stop\",\n    \"max_tokens\",\n    \"presence_penalty\",\n    \"frequency_penalty\",\n    \"logit_bias\",\n    \"user\",\n    \"request_timeout\",\n    \"api_base\",\n    \"api_version\",\n    \"api_key\",\n    \"deployment_id\",\n    \"organization\",\n    \"base_url\",\n    \"default_headers\",\n    \"timeout\",\n    \"response_format\",\n    \"seed\",\n    \"tools\",\n    \"tool_choice\",\n    \"max_retries\",\n]\n_litellm_completion_params = [\n    \"metadata\",\n    \"acompletion\",\n    \"caching\",\n    \"mock_response\",\n    \"api_key\",\n    \"api_version\",\n    \"api_base\",\n    \"force_timeout\",\n    \"logger_fn\",\n    \"verbose\",\n    \"custom_llm_provider\",\n    \"litellm_logging_obj\",\n    \"litellm_call_id\",\n    \"use_client\",\n    \"id\",\n    \"fallbacks\",\n    \"azure\",\n    \"headers\",\n    \"model_list\",\n    \"num_retries\",\n    \"context_window_fallback_dict\",\n    \"roles\",\n    \"final_prompt_value\",\n    \"bos_token\",\n    \"eos_token\",\n    \"request_timeout\",\n    \"complete_response\",\n    \"self\",\n    \"client\",\n    \"rpm\",\n    \"tpm\",\n    \"input_cost_per_token\",\n    \"output_cost_per_token\",\n    \"hf_model_name\",\n    \"model_info\",\n    \"proxy_server_request\",\n    \"preset_cache_key\",\n]\n_current_cost = 0  # private variable, used if max budget is set\nerror_logs: Dict = {}\nadd_function_to_prompt: bool = (\n    False  # if function calling not supported by api, append function call details to system prompt\n)\nclient_session: Optional[httpx.Client] = None\naclient_session: Optional[httpx.AsyncClient] = None\nmodel_fallbacks: Optional[List] = None  # Deprecated for 'litellm.fallbacks'\nmodel_cost_map_url: str = (\n    \"https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json\"\n)\nsuppress_debug_info = False\ndynamodb_table_name: Optional[str] = None\ns3_callback_params: Optional[Dict] = None\ngeneric_logger_headers: Optional[Dict] = None\ndefault_key_generate_params: Optional[Dict] = None\nupperbound_key_generate_params: Optional[LiteLLM_UpperboundKeyGenerateParams] = None\ndefault_user_params: Optional[Dict] = None\ndefault_team_settings: Optional[List] = None\nmax_user_budget: Optional[float] = None\nmax_end_user_budget: Optional[float] = None\n#### RELIABILITY ####\nrequest_timeout: float = 6000\nmodule_level_aclient = AsyncHTTPHandler(timeout=request_timeout)\nmodule_level_client = HTTPHandler(timeout=request_timeout)\nnum_retries: Optional[int] = None  # per model endpoint\ndefault_fallbacks: Optional[List] = None\nfallbacks: Optional[List] = None\ncontext_window_fallbacks: Optional[List] = None\nallowed_fails: int = 0\nnum_retries_per_request: Optional[int] = (\n    None  # for the request overall (incl. fallbacks + model retries)\n)\n####### SECRET MANAGERS #####################\nsecret_manager_client: Optional[Any] = (\n    None  # list of instantiated key management clients - e.g. azure kv, infisical, etc.\n)\n_google_kms_resource_name: Optional[str] = None\n_key_management_system: Optional[KeyManagementSystem] = None\n_key_management_settings: Optional[KeyManagementSettings] = None\n#### PII MASKING ####\noutput_parse_pii: bool = False\n#############################################\n\n\ndef get_model_cost_map(url: str):\n    if (\n        os.getenv(\"LITELLM_LOCAL_MODEL_COST_MAP\", False) == True\n        or os.getenv(\"LITELLM_LOCAL_MODEL_COST_MAP\", False) == \"True\"\n    ):\n        import importlib.resources\n        import json\n\n        with importlib.resources.open_text(\n            \"litellm\", \"model_prices_and_context_window_backup.json\"\n        ) as f:\n            content = json.load(f)\n            return content\n\n    try:\n        with requests.get(\n            url, timeout=5\n        ) as response:  # set a 5 second timeout for the get request\n            response.raise_for_status()  # Raise an exception if the request is unsuccessful\n            content = response.json()\n            return content\n    except Exception as e:\n        import importlib.resources\n        import json\n\n        with importlib.resources.open_text(\n            \"litellm\", \"model_prices_and_context_window_backup.json\"\n        ) as f:\n            content = json.load(f)\n            return content\n\n\nmodel_cost = get_model_cost_map(url=model_cost_map_url)\ncustom_prompt_dict: Dict[str, dict] = {}\n\n\n####### THREAD-SPECIFIC DATA ###################\nclass MyLocal(threading.local):\n    def __init__(self):\n        self.user = \"Hello World\"\n\n\n_thread_context = MyLocal()\n\n\ndef identify(event_details):\n    # Store user in thread local data\n    if \"user\" in event_details:\n        _thread_context.user = event_details[\"user\"]\n\n\n####### ADDITIONAL PARAMS ################### configurable params if you use proxy models like Helicone, map spend to org id, etc.\napi_base = None\nheaders = None\napi_version = None\norganization = None\nproject = None\nconfig_path = None\n####### COMPLETION MODELS ###################\nopen_ai_chat_completion_models: List = []\nopen_ai_text_completion_models: List = []\ncohere_models: List = []\ncohere_chat_models: List = []\nmistral_chat_models: List = []\nanthropic_models: List = []\nopenrouter_models: List = []\nvertex_language_models: List = []\nvertex_vision_models: List = []\nvertex_chat_models: List = []\nvertex_code_chat_models: List = []\nvertex_text_models: List = []\nvertex_code_text_models: List = []\nvertex_embedding_models: List = []\nvertex_anthropic_models: List = []\nai21_models: List = []\nnlp_cloud_models: List = []\naleph_alpha_models: List = []\nbedrock_models: List = []\ndeepinfra_models: List = []\nperplexity_models: List = []\nwatsonx_models: List = []\nfor key, value in model_cost.items():\n    if value.get(\"litellm_provider\") == \"openai\":\n        open_ai_chat_completion_models.append(key)\n    elif value.get(\"litellm_provider\") == \"text-completion-openai\":\n        open_ai_text_completion_models.append(key)\n    elif value.get(\"litellm_provider\") == \"cohere\":\n        cohere_models.append(key)\n    elif value.get(\"litellm_provider\") == \"cohere_chat\":\n        cohere_chat_models.append(key)\n    elif value.get(\"litellm_provider\") == \"mistral\":\n        mistral_chat_models.append(key)\n    elif value.get(\"litellm_provider\") == \"anthropic\":\n        anthropic_models.append(key)\n    elif value.get(\"litellm_provider\") == \"openrouter\":\n        openrouter_models.append(key)\n    elif value.get(\"litellm_provider\") == \"vertex_ai-text-models\":\n        vertex_text_models.append(key)\n    elif value.get(\"litellm_provider\") == \"vertex_ai-code-text-models\":\n        vertex_code_text_models.append(key)\n    elif value.get(\"litellm_provider\") == \"vertex_ai-language-models\":\n        vertex_language_models.append(key)\n    elif value.get(\"litellm_provider\") == \"vertex_ai-vision-models\":\n        vertex_vision_models.append(key)\n    elif value.get(\"litellm_provider\") == \"vertex_ai-chat-models\":\n        vertex_chat_models.append(key)\n    elif value.get(\"litellm_provider\") == \"vertex_ai-code-chat-models\":\n        vertex_code_chat_models.append(key)\n    elif value.get(\"litellm_provider\") == \"vertex_ai-embedding-models\":\n        vertex_embedding_models.append(key)\n    elif value.get(\"litellm_provider\") == \"vertex_ai-anthropic_models\":\n        key = key.replace(\"vertex_ai/\", \"\")\n        vertex_anthropic_models.append(key)\n    elif value.get(\"litellm_provider\") == \"ai21\":\n        ai21_models.append(key)\n    elif value.get(\"litellm_provider\") == \"nlp_cloud\":\n        nlp_cloud_models.append(key)\n    elif value.get(\"litellm_provider\") == \"aleph_alpha\":\n        aleph_alpha_models.append(key)\n    elif value.get(\"litellm_provider\") == \"bedrock\":\n        bedrock_models.append(key)\n    elif value.get(\"litellm_provider\") == \"deepinfra\":\n        deepinfra_models.append(key)\n    elif value.get(\"litellm_provider\") == \"perplexity\":\n        perplexity_models.append(key)\n    elif value.get(\"litellm_provider\") == \"watsonx\":\n        watsonx_models.append(key)\n\n# known openai compatible endpoints - we'll eventually move this list to the model_prices_and_context_window.json dictionary\nopenai_compatible_endpoints: List = [\n    \"api.perplexity.ai\",\n    \"api.endpoints.anyscale.com/v1\",\n    \"api.deepinfra.com/v1/openai\",\n    \"api.mistral.ai/v1\",\n    \"api.groq.com/openai/v1\",\n    \"api.deepseek.com/v1\",\n    \"api.together.xyz/v1\",\n]\n\n# this is maintained for Exception Mapping\nopenai_compatible_providers: List = [\n    \"anyscale\",\n    \"mistral\",\n    \"groq\",\n    \"deepseek\",\n    \"deepinfra\",\n    \"perplexity\",\n    \"xinference\",\n    \"together_ai\",\n    \"fireworks_ai\",\n]\n\n\n# well supported replicate llms\nreplicate_models: List = [\n    # llama replicate supported LLMs\n    \"replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf\",\n    \"a16z-infra/llama-2-13b-chat:2a7f981751ec7fdf87b5b91ad4db53683a98082e9ff7bfd12c8cd5ea85980a52\",\n    \"meta/codellama-13b:1c914d844307b0588599b8393480a3ba917b660c7e9dfae681542b5325f228db\",\n    # Vicuna\n    \"replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b\",\n    \"joehoover/instructblip-vicuna13b:c4c54e3c8c97cd50c2d2fec9be3b6065563ccf7d43787fb99f84151b867178fe\",\n    # Flan T-5\n    \"daanelson/flan-t5-large:ce962b3f6792a57074a601d3979db5839697add2e4e02696b3ced4c022d4767f\",\n    # Others\n    \"replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5\",\n    \"replit/replit-code-v1-3b:b84f4c074b807211cd75e3e8b1589b6399052125b4c27106e43d47189e8415ad\",\n]\n\nclarifai_models: List = [\n    \"clarifai/meta.Llama-3.Llama-3-8B-Instruct\",\n    \"clarifai/gcp.generate.gemma-1_1-7b-it\",\n    \"clarifai/mistralai.completion.mixtral-8x22B\",\n    \"clarifai/cohere.generate.command-r-plus\",\n    \"clarifai/databricks.drbx.dbrx-instruct\",\n    \"clarifai/mistralai.completion.mistral-large\",\n    \"clarifai/mistralai.completion.mistral-medium\",\n    \"clarifai/mistralai.completion.mistral-small\",\n    \"clarifai/mistralai.completion.mixtral-8x7B-Instruct-v0_1\",\n    \"clarifai/gcp.generate.gemma-2b-it\",\n    \"clarifai/gcp.generate.gemma-7b-it\",\n    \"clarifai/deci.decilm.deciLM-7B-instruct\",\n    \"clarifai/mistralai.completion.mistral-7B-Instruct\",\n    \"clarifai/gcp.generate.gemini-pro\",\n    \"clarifai/anthropic.completion.claude-v1\",\n    \"clarifai/anthropic.completion.claude-instant-1_2\",\n    \"clarifai/anthropic.completion.claude-instant\",\n    \"clarifai/anthropic.completion.claude-v2\",\n    \"clarifai/anthropic.completion.claude-2_1\",\n    \"clarifai/meta.Llama-2.codeLlama-70b-Python\",\n    \"clarifai/meta.Llama-2.codeLlama-70b-Instruct\",\n    \"clarifai/openai.completion.gpt-3_5-turbo-instruct\",\n    \"clarifai/meta.Llama-2.llama2-7b-chat\",\n    \"clarifai/meta.Llama-2.llama2-13b-chat\",\n    \"clarifai/meta.Llama-2.llama2-70b-chat\",\n    \"clarifai/openai.chat-completion.gpt-4-turbo\",\n    \"clarifai/microsoft.text-generation.phi-2\",\n    \"clarifai/meta.Llama-2.llama2-7b-chat-vllm\",\n    \"clarifai/upstage.solar.solar-10_7b-instruct\",\n    \"clarifai/openchat.openchat.openchat-3_5-1210\",\n    \"clarifai/togethercomputer.stripedHyena.stripedHyena-Nous-7B\",\n    \"clarifai/gcp.generate.text-bison\",\n    \"clarifai/meta.Llama-2.llamaGuard-7b\",\n    \"clarifai/fblgit.una-cybertron.una-cybertron-7b-v2\",\n    \"clarifai/openai.chat-completion.GPT-4\",\n    \"clarifai/openai.chat-completion.GPT-3_5-turbo\",\n    \"clarifai/ai21.complete.Jurassic2-Grande\",\n    \"clarifai/ai21.complete.Jurassic2-Grande-Instruct\",\n    \"clarifai/ai21.complete.Jurassic2-Jumbo-Instruct\",\n    \"clarifai/ai21.complete.Jurassic2-Jumbo\",\n    \"clarifai/ai21.complete.Jurassic2-Large\",\n    \"clarifai/cohere.generate.cohere-generate-command\",\n    \"clarifai/wizardlm.generate.wizardCoder-Python-34B\",\n    \"clarifai/wizardlm.generate.wizardLM-70B\",\n    \"clarifai/tiiuae.falcon.falcon-40b-instruct\",\n    \"clarifai/togethercomputer.RedPajama.RedPajama-INCITE-7B-Chat\",\n    \"clarifai/gcp.generate.code-gecko\",\n    \"clarifai/gcp.generate.code-bison\",\n    \"clarifai/mistralai.completion.mistral-7B-OpenOrca\",\n    \"clarifai/mistralai.completion.openHermes-2-mistral-7B\",\n    \"clarifai/wizardlm.generate.wizardLM-13B\",\n    \"clarifai/huggingface-research.zephyr.zephyr-7B-alpha\",\n    \"clarifai/wizardlm.generate.wizardCoder-15B\",\n    \"clarifai/microsoft.text-generation.phi-1_5\",\n    \"clarifai/databricks.Dolly-v2.dolly-v2-12b\",\n    \"clarifai/bigcode.code.StarCoder\",\n    \"clarifai/salesforce.xgen.xgen-7b-8k-instruct\",\n    \"clarifai/mosaicml.mpt.mpt-7b-instruct\",\n    \"clarifai/anthropic.completion.claude-3-opus\",\n    \"clarifai/anthropic.completion.claude-3-sonnet\",\n    \"clarifai/gcp.generate.gemini-1_5-pro\",\n    \"clarifai/gcp.generate.imagen-2\",\n    \"clarifai/salesforce.blip.general-english-image-caption-blip-2\",\n]\n\n\nhuggingface_models: List = [\n    \"meta-llama/Llama-2-7b-hf\",\n    \"meta-llama/Llama-2-7b-chat-hf\",\n    \"meta-llama/Llama-2-13b-hf\",\n    \"meta-llama/Llama-2-13b-chat-hf\",\n    \"meta-llama/Llama-2-70b-hf\",\n    \"meta-llama/Llama-2-70b-chat-hf\",\n    \"meta-llama/Llama-2-7b\",\n    \"meta-llama/Llama-2-7b-chat\",\n    \"meta-llama/Llama-2-13b\",\n    \"meta-llama/Llama-2-13b-chat\",\n    \"meta-llama/Llama-2-70b\",\n    \"meta-llama/Llama-2-70b-chat\",\n]  # these have been tested on extensively. But by default all text2text-generation and text-generation models are supported by liteLLM. - https://docs.litellm.ai/docs/providers\n\ntogether_ai_models: List = [\n    # llama llms - chat\n    \"togethercomputer/llama-2-70b-chat\",\n    # llama llms - language / instruct\n    \"togethercomputer/llama-2-70b\",\n    \"togethercomputer/LLaMA-2-7B-32K\",\n    \"togethercomputer/Llama-2-7B-32K-Instruct\",\n    \"togethercomputer/llama-2-7b\",\n    # falcon llms\n    \"togethercomputer/falcon-40b-instruct\",\n    \"togethercomputer/falcon-7b-instruct\",\n    # alpaca\n    \"togethercomputer/alpaca-7b\",\n    # chat llms\n    \"HuggingFaceH4/starchat-alpha\",\n    # code llms\n    \"togethercomputer/CodeLlama-34b\",\n    \"togethercomputer/CodeLlama-34b-Instruct\",\n    \"togethercomputer/CodeLlama-34b-Python\",\n    \"defog/sqlcoder\",\n    \"NumbersStation/nsql-llama-2-7B\",\n    \"WizardLM/WizardCoder-15B-V1.0\",\n    \"WizardLM/WizardCoder-Python-34B-V1.0\",\n    # language llms\n    \"NousResearch/Nous-Hermes-Llama2-13b\",\n    \"Austism/chronos-hermes-13b\",\n    \"upstage/SOLAR-0-70b-16bit\",\n    \"WizardLM/WizardLM-70B-V1.0\",\n]  # supports all together ai models, just pass in the model id e.g. completion(model=\"together_computer/replit_code_3b\",...)\n\n\nbaseten_models: List = [\n    \"qvv0xeq\",\n    \"q841o8w\",\n    \"31dxrj3\",\n]  # FALCON 7B  # WizardLM  # Mosaic ML\n\n\n# used for Cost Tracking & Token counting\n# https://azure.microsoft.com/en-in/pricing/details/cognitive-services/openai-service/\n# Azure returns gpt-35-turbo in their responses, we need to map this to azure/gpt-3.5-turbo for token counting\nazure_llms = {\n    \"gpt-35-turbo\": \"azure/gpt-35-turbo\",\n    \"gpt-35-turbo-16k\": \"azure/gpt-35-turbo-16k\",\n    \"gpt-35-turbo-instruct\": \"azure/gpt-35-turbo-instruct\",\n}\n\nazure_embedding_models = {\n    \"ada\": \"azure/ada\",\n}\n\npetals_models = [\n    \"petals-team/StableBeluga2\",\n]\n\nollama_models = [\"llama2\"]\n\nmaritalk_models = [\"maritalk\"]\n\nmodel_list = (\n    open_ai_chat_completion_models\n    + open_ai_text_completion_models\n    + cohere_models\n    + cohere_chat_models\n    + anthropic_models\n    + replicate_models\n    + openrouter_models\n    + huggingface_models\n    + vertex_chat_models\n    + vertex_text_models\n    + ai21_models\n    + together_ai_models\n    + baseten_models\n    + aleph_alpha_models\n    + nlp_cloud_models\n    + ollama_models\n    + bedrock_models\n    + deepinfra_models\n    + perplexity_models\n    + maritalk_models\n    + vertex_language_models\n    + watsonx_models\n)\n\nprovider_list: List = [\n    \"openai\",\n    \"custom_openai\",\n    \"text-completion-openai\",\n    \"cohere\",\n    \"cohere_chat\",\n    \"clarifai\",\n    \"anthropic\",\n    \"replicate\",\n    \"huggingface\",\n    \"together_ai\",\n    \"openrouter\",\n    \"vertex_ai\",\n    \"palm\",\n    \"gemini\",\n    \"ai21\",\n    \"baseten\",\n    \"azure\",\n    \"azure_text\",\n    \"sagemaker\",\n    \"bedrock\",\n    \"vllm\",\n    \"nlp_cloud\",\n    \"petals\",\n    \"oobabooga\",\n    \"ollama\",\n    \"ollama_chat\",\n    \"deepinfra\",\n    \"perplexity\",\n    \"anyscale\",\n    \"mistral\",\n    \"groq\",\n    \"deepseek\",\n    \"maritalk\",\n    \"voyage\",\n    \"cloudflare\",\n    \"xinference\",\n    \"fireworks_ai\",\n    \"watsonx\",\n    \"triton\",\n    \"predibase\",\n    \"databricks\",\n    \"custom\",  # custom apis\n]\n\nmodels_by_provider: dict = {\n    \"openai\": open_ai_chat_completion_models + open_ai_text_completion_models,\n    \"cohere\": cohere_models,\n    \"cohere_chat\": cohere_chat_models,\n    \"anthropic\": anthropic_models,\n    \"replicate\": replicate_models,\n    \"huggingface\": huggingface_models,\n    \"together_ai\": together_ai_models,\n    \"baseten\": baseten_models,\n    \"openrouter\": openrouter_models,\n    \"vertex_ai\": vertex_chat_models\n    + vertex_text_models\n    + vertex_anthropic_models\n    + vertex_vision_models\n    + vertex_language_models,\n    \"ai21\": ai21_models,\n    \"bedrock\": bedrock_models,\n    \"petals\": petals_models,\n    \"ollama\": ollama_models,\n    \"deepinfra\": deepinfra_models,\n    \"perplexity\": perplexity_models,\n    \"maritalk\": maritalk_models,\n    \"watsonx\": watsonx_models,\n}\n\n# mapping for those models which have larger equivalents\nlonger_context_model_fallback_dict: dict = {\n    # openai chat completion models\n    \"gpt-3.5-turbo\": \"gpt-3.5-turbo-16k\",\n    \"gpt-3.5-turbo-0301\": \"gpt-3.5-turbo-16k-0301\",\n    \"gpt-3.5-turbo-0613\": \"gpt-3.5-turbo-16k-0613\",\n    \"gpt-4\": \"gpt-4-32k\",\n    \"gpt-4-0314\": \"gpt-4-32k-0314\",\n    \"gpt-4-0613\": \"gpt-4-32k-0613\",\n    # anthropic\n    \"claude-instant-1\": \"claude-2\",\n    \"claude-instant-1.2\": \"claude-2\",\n    # vertexai\n    \"chat-bison\": \"chat-bison-32k\",\n    \"chat-bison@001\": \"chat-bison-32k\",\n    \"codechat-bison\": \"codechat-bison-32k\",\n    \"codechat-bison@001\": \"codechat-bison-32k\",\n    # openrouter\n    \"openrouter/openai/gpt-3.5-turbo\": \"openrouter/openai/gpt-3.5-turbo-16k\",\n    \"openrouter/anthropic/claude-instant-v1\": \"openrouter/anthropic/claude-2\",\n}\n\n####### EMBEDDING MODELS ###################\nopen_ai_embedding_models: List = [\"text-embedding-ada-002\"]\ncohere_embedding_models: List = [\n    \"embed-english-v3.0\",\n    \"embed-english-light-v3.0\",\n    \"embed-multilingual-v3.0\",\n    \"embed-english-v2.0\",\n    \"embed-english-light-v2.0\",\n    \"embed-multilingual-v2.0\",\n]\nbedrock_embedding_models: List = [\n    \"amazon.titan-embed-text-v1\",\n    \"cohere.embed-english-v3\",\n    \"cohere.embed-multilingual-v3\",\n]\n\nall_embedding_models = (\n    open_ai_embedding_models\n    + cohere_embedding_models\n    + bedrock_embedding_models\n    + vertex_embedding_models\n)\n\n####### IMAGE GENERATION MODELS ###################\nopenai_image_generation_models = [\"dall-e-2\", \"dall-e-3\"]\n\nfrom .timeout import timeout\nfrom .cost_calculator import completion_cost\nfrom .utils import (\n    client,\n    exception_type,\n    get_optional_params,\n    modify_integration,\n    token_counter,\n    create_pretrained_tokenizer,\n    create_tokenizer,\n    cost_per_token,\n    supports_function_calling,\n    supports_parallel_function_calling,\n    supports_vision,\n    get_litellm_params,\n    Logging,\n    acreate,\n    get_model_list,\n    get_max_tokens,\n    get_model_info,\n    register_prompt_template,\n    validate_environment,\n    check_valid_key,\n    get_llm_provider,\n    register_model,\n    encode,\n    decode,\n    _calculate_retry_after,\n    _should_retry,\n    get_secret,\n    get_supported_openai_params,\n    get_api_base,\n    get_first_chars_messages,\n    ModelResponse,\n    ImageResponse,\n    ImageObject,\n    get_provider_fields,\n)\nfrom .llms.huggingface_restapi import HuggingfaceConfig\nfrom .llms.anthropic import AnthropicConfig\nfrom .llms.databricks import DatabricksConfig, DatabricksEmbeddingConfig\nfrom .llms.predibase import PredibaseConfig\nfrom .llms.anthropic_text import AnthropicTextConfig\nfrom .llms.replicate import ReplicateConfig\nfrom .llms.cohere import CohereConfig\nfrom .llms.clarifai import ClarifaiConfig\nfrom .llms.ai21 import AI21Config\nfrom .llms.together_ai import TogetherAIConfig\nfrom .llms.cloudflare import CloudflareConfig\nfrom .llms.palm import PalmConfig\nfrom .llms.gemini import GeminiConfig\nfrom .llms.nlp_cloud import NLPCloudConfig\nfrom .llms.aleph_alpha import AlephAlphaConfig\nfrom .llms.petals import PetalsConfig\nfrom .llms.vertex_ai import VertexAIConfig\nfrom .llms.vertex_ai_anthropic import VertexAIAnthropicConfig\nfrom .llms.sagemaker import SagemakerConfig\nfrom .llms.ollama import OllamaConfig\nfrom .llms.ollama_chat import OllamaChatConfig\nfrom .llms.maritalk import MaritTalkConfig\nfrom .llms.bedrock_httpx import AmazonCohereChatConfig, AmazonConverseConfig\nfrom .llms.bedrock import (\n    AmazonTitanConfig,\n    AmazonAI21Config,\n    AmazonAnthropicConfig,\n    AmazonAnthropicClaude3Config,\n    AmazonCohereConfig,\n    AmazonLlamaConfig,\n    AmazonStabilityConfig,\n    AmazonMistralConfig,\n    AmazonBedrockGlobalConfig,\n)\nfrom .llms.openai import (\n    OpenAIConfig,\n    OpenAITextCompletionConfig,\n    MistralConfig,\n    DeepInfraConfig,\n)\nfrom .llms.azure import (\n    AzureOpenAIConfig,\n    AzureOpenAIError,\n    AzureOpenAIAssistantsAPIConfig,\n)\nfrom .llms.watsonx import IBMWatsonXAIConfig\nfrom .main import *  # type: ignore\nfrom .integrations import *\nfrom .exceptions import (\n    AuthenticationError,\n    InvalidRequestError,\n    BadRequestError,\n    NotFoundError,\n    RateLimitError,\n    ServiceUnavailableError,\n    OpenAIError,\n    ContextWindowExceededError,\n    ContentPolicyViolationError,\n    BudgetExceededError,\n    APIError,\n    Timeout,\n    APIConnectionError,\n    APIResponseValidationError,\n    UnprocessableEntityError,\n    InternalServerError,\n    LITELLM_EXCEPTION_TYPES,\n)\nfrom .budget_manager import BudgetManager\nfrom .proxy.proxy_cli import run_server\nfrom .router import Router\nfrom .assistants.main import *\nfrom .batches.main import *\nfrom .scheduler import *\nfrom .cost_calculator import response_cost_calculator\n"}}}}