1 line
29 KiB
JSON
1 line
29 KiB
JSON
{"repo": "boshallen/psycopass", "files": {"opencompass/opencompass/models/ai360_api.py": {"aa830620f9bdb68cad059240766e95c98ff8beeb": {"t": 1785768164.7296627, "n": 6105, "h": "f0d3a1c18c6df379", "c": "from concurrent.futures import ThreadPoolExecutor\nfrom typing import Dict, List, Optional, Union\n\nimport requests\n\nfrom opencompass.utils.prompt import PromptList\n\nfrom .base_api import BaseAPIModel\n\nPromptType = Union[PromptList, str]\n\n\nclass AI360GPT(BaseAPIModel):\n \"\"\"Model wrapper around 360 GPT.\n\n Documentations: https://ai.360.com/platform/docs/overview\n\n Args:\n path (str): Model name\n key (str): Provide API Key\n url (str): Provided URL\n query_per_second (int): The maximum queries allowed per second\n between two consecutive calls of the API. Defaults to 2.\n max_seq_len (int): Unused here.\n meta_template (Dict, optional): The model's meta prompt\n template if needed, in case the requirement of injecting or\n wrapping of any meta instructions.\n retry (int): Number of retires if the API call fails. Defaults to 2.\n \"\"\"\n\n def __init__(\n self,\n path: str, # model name, e.g.: 360GPT_S2_V9\n key: str,\n url: str = 'https://api.360.cn/v1/chat/completions',\n query_per_second: int = 2,\n max_seq_len: int = 2048,\n meta_template: Optional[Dict] = None,\n retry: int = 2,\n generation_kwargs: Dict = {\n 'temperature': 0.9,\n 'max_tokens': 2048,\n 'top_p': 0.5,\n 'tok_k': 0,\n 'repetition_penalty': 1.05,\n }): # noqa E125\n super().__init__(path=path,\n max_seq_len=max_seq_len,\n query_per_second=query_per_second,\n meta_template=meta_template,\n retry=retry,\n generation_kwargs=generation_kwargs)\n self.headers = {\n 'Authorization': f'Bearer {key}',\n 'Content-Type': 'application/json',\n }\n self.model = path\n self.url = url\n\n def generate(\n self,\n inputs: List[PromptType],\n max_out_len: int = 512,\n ) -> List[str]:\n \"\"\"Generate results given a list of inputs.\n\n Args:\n inputs (List[PromptType]): A list of strings or PromptDicts.\n The PromptDict should be organized in OpenCompass'\n API format.\n max_out_len (int): The maximum length of the output.\n\n Returns:\n List[str]: A list of generated strings.\n \"\"\"\n with ThreadPoolExecutor() as executor:\n results = list(\n executor.map(self._generate, inputs,\n [max_out_len] * len(inputs)))\n self.flush()\n return results\n\n def _generate(\n self,\n input: PromptType,\n max_out_len: int = 512,\n ) -> str:\n \"\"\"Generate results given an input.\n\n Args:\n inputs (PromptType): A string or PromptDict.\n The PromptDict should be organized in OpenCompass'\n API format.\n max_out_len (int): The maximum length of the output.\n\n Returns:\n str: The generated string.\n \"\"\"\n assert isinstance(input, (str, PromptList))\n\n if isinstance(input, str):\n messages = [{'role': 'user', 'content': input}]\n else:\n messages = []\n for item in input:\n msg = {'content': item['prompt']}\n if item['role'] == 'HUMAN':\n msg['role'] = 'user'\n elif item['role'] == 'BOT':\n msg['role'] = 'assistant'\n elif item['role'] == 'SYSTEM':\n msg['role'] = 'system'\n messages.append(msg)\n\n data = {\n 'model': self.model,\n 'messages': messages,\n 'stream': False,\n # \"user\": \"OpenCompass\"\n }\n\n data.update(self.generation_kwargs)\n\n max_num_retries = 0\n while max_num_retries < self.retry:\n self.acquire()\n # payload = json.dumps(data)\n try:\n raw_response = requests.request('POST',\n url=self.url,\n headers=self.headers,\n json=data)\n except Exception as e:\n self.release()\n print(e)\n max_num_retries += 1\n continue\n response = raw_response.json()\n self.release()\n\n if response is None:\n print('Connection error, reconnect.')\n # if connect error, frequent requests will casuse\n # continuous unstable network, therefore wait here\n # to slow down the request\n self.wait()\n continue\n if raw_response.status_code == 200:\n msg = response['choices'][0]['message']['content'].strip()\n self.logger.debug(f'Generated: {msg}')\n return msg\n\n # sensitive content, prompt overlength, network error\n # or illegal prompt\n if raw_response.status_code in [400, 401, 402, 429, 500]:\n if 'error' not in response:\n print(raw_response.status_code)\n print(raw_response.text)\n continue\n print(response)\n # tpm(token per minitue) limit\n if response['error']['code'] == '1005':\n self.logger.debug('tpm limit, ignoring')\n continue\n elif response['error']['code'] == '1001':\n msg = '参数错误:messages参数过长或max_tokens参数值过大'\n self.logger.debug(f'Generated: {msg}')\n return msg\n else:\n print(response)\n\n self.logger.error('Find error message in response: ',\n str(response['error']))\n\n print(raw_response)\n max_num_retries += 1\n\n raise RuntimeError(raw_response.text)\n"}}, "opencompass/opencompass/models/sensetime_api.py": {"aa830620f9bdb68cad059240766e95c98ff8beeb": {"t": 1785768271.8086946, "n": 7426, "h": "f0f174ea18663ec4", "c": "import json\nimport os\nimport time\nfrom concurrent.futures import ThreadPoolExecutor\nfrom typing import Dict, List, Optional, Union\n\nimport requests\n\nfrom opencompass.utils.prompt import PromptList\n\nfrom .base_api import BaseAPIModel\n\nPromptType = Union[PromptList, str]\n\n\nclass SenseTime(BaseAPIModel):\n \"\"\"Model wrapper around SenseTime.\n\n Args:\n path (str): The name of SenseTime model.\n e.g. `nova-ptc-xl-v1`\n key (str): Authorization key.\n query_per_second (int): The maximum queries allowed per second\n between two consecutive calls of the API. Defaults to 1.\n max_seq_len (int): Unused here.\n meta_template (Dict, optional): The model's meta prompt\n template if needed, in case the requirement of injecting or\n wrapping of any meta instructions.\n retry (int): Number of retires if the API call fails. Defaults to 2.\n \"\"\"\n\n def __init__(\n self,\n path: str,\n url: str,\n key: str = 'ENV',\n query_per_second: int = 2,\n max_seq_len: int = 2048,\n meta_template: Optional[Dict] = None,\n retry: int = 2,\n parameters: Optional[Dict] = None,\n ):\n super().__init__(path=path,\n max_seq_len=max_seq_len,\n query_per_second=query_per_second,\n meta_template=meta_template,\n retry=retry)\n\n if isinstance(key, str):\n self.keys = os.getenv('SENSENOVA_API_KEY') if key == 'ENV' else key\n else:\n self.keys = key\n\n self.headers = {\n 'Content-Type': 'application/json',\n 'Authorization': f'Bearer {self.keys}'\n }\n self.url = url\n self.model = path\n self.params = parameters\n\n def generate(\n self,\n inputs: List[PromptType],\n max_out_len: int = 512,\n ) -> List[str]:\n \"\"\"Generate results given a list of inputs.\n\n Args:\n inputs (List[PromptType]): A list of strings or PromptDicts.\n The PromptDict should be organized in OpenCompass'\n API format.\n max_out_len (int): The maximum length of the output.\n\n Returns:\n List[str]: A list of generated strings.\n \"\"\"\n with ThreadPoolExecutor() as executor:\n results = list(\n executor.map(self._generate, inputs,\n [max_out_len] * len(inputs)))\n self.flush()\n return results\n\n def _generate(\n self,\n input: PromptType,\n max_out_len: int = 512,\n ) -> str:\n \"\"\"Generate results given an input.\n\n Args:\n inputs (PromptType): A string or PromptDict.\n The PromptDict should be organized in OpenCompass'\n API format.\n max_out_len (int): The maximum length of the output.\n\n Returns:\n str: The generated string.\n \"\"\"\n assert isinstance(input, (str, PromptList))\n\n if isinstance(input, str):\n messages = [{'role': 'user', 'content': input}]\n else:\n messages = []\n msg_buffer, last_role = [], None\n for item in input:\n if not item['prompt']:\n continue\n item['role'] = 'assistant' if item['role'] == 'BOT' else 'user'\n if item['role'] != last_role and last_role is not None:\n messages.append({\n 'content': '\\n'.join(msg_buffer),\n 'role': last_role\n })\n msg_buffer = []\n msg_buffer.append(item['prompt'])\n last_role = item['role']\n messages.append({\n 'content': '\\n'.join(msg_buffer),\n 'role': last_role\n })\n\n data = {'messages': messages, 'model': self.model}\n if self.params is not None:\n data.update(self.params)\n\n stream = data['stream']\n\n max_num_retries = 0\n while max_num_retries < self.retry:\n self.acquire()\n\n max_num_retries += 1\n try:\n raw_response = requests.request('POST',\n url=self.url,\n headers=self.headers,\n json=data)\n except Exception:\n time.sleep(1)\n continue\n requests_id = raw_response.headers['X-Request-Id'] # noqa\n self.release()\n\n if not stream:\n response = raw_response.json()\n\n if response is None:\n print('Connection error, reconnect.')\n # if connect error, frequent requests will casuse\n # continuous unstable network, therefore wait here\n # to slow down the request\n self.wait()\n continue\n if raw_response.status_code == 200:\n msg = response['data']['choices'][0]['message']\n return msg\n\n if (raw_response.status_code != 200):\n if response['error']['code'] == 18:\n # security issue\n return 'error:unsafe'\n if response['error']['code'] == 17:\n return 'error:too long'\n else:\n print(raw_response.text)\n from IPython import embed\n embed()\n exit()\n time.sleep(1)\n continue\n else:\n # stream data to msg\n raw_response.encoding = 'utf-8'\n\n if raw_response.status_code == 200:\n response_text = raw_response.text\n data_blocks = response_text.split('data:')\n data_blocks = data_blocks[1:]\n\n first_block = json.loads(data_blocks[0])\n if first_block['status']['code'] != 0:\n msg = f\"error:{first_block['status']['code']},\"\n f\" {first_block['status']['message']}\"\n self.logger.error(msg)\n return msg\n\n msg = ''\n for i, part in enumerate(data_blocks):\n # print(f'process {i}: {part}')\n try:\n if part.startswith('[DONE]'):\n break\n\n json_data = json.loads(part)\n choices = json_data['data']['choices']\n for c in choices:\n delta = c.get('delta')\n msg += delta\n except json.decoder.JSONDecodeError as err:\n print(err)\n self.logger.error(f'Error decoding JSON: {part}')\n return msg\n\n else:\n print(raw_response.text,\n raw_response.headers.get('X-Request-Id'))\n time.sleep(1)\n continue\n\n raise RuntimeError\n"}}, "lagent/lagent/llms/sensenova.py": {"aa830620f9bdb68cad059240766e95c98ff8beeb": {"t": 1785809900.8287513, "n": 14716, "h": "70405e01319d73a5", "c": "import json\nimport os\nimport time\nimport warnings\nfrom concurrent.futures import ThreadPoolExecutor\nfrom logging import getLogger\nfrom threading import Lock\nfrom typing import Dict, Generator, List, Optional, Tuple, Union\n\nimport requests\n\nfrom lagent.schema import ModelStatusCode\nfrom lagent.utils.util import filter_suffix\nfrom .base_api import BaseAPILLM\n\nwarnings.simplefilter('default')\n\nSENSENOVA_API_BASE = 'https://api.sensenova.cn/v1/llm/chat-completions'\n\nsensechat_models = {'SenseChat-5': 131072, 'SenseChat-5-Cantonese': 32768}\n\n\nclass SensenovaAPI(BaseAPILLM):\n \"\"\"Model wrapper around SenseTime's models.\n\n Args:\n model_type (str): The name of SenseTime's model.\n retry (int): Number of retires if the API call fails. Defaults to 2.\n key (str or List[str]): SenseTime key(s). In particular, when it\n is set to \"ENV\", the key will be fetched from the environment\n variable $SENSENOVA_API_KEY. If it's a list, the keys will be\n used in round-robin manner. Defaults to 'ENV'.\n meta_template (Dict, optional): The model's meta prompt\n template if needed, in case the requirement of injecting or\n wrapping of any meta instructions.\n sensenova_api_base (str): The base url of SenseTime's API. Defaults to\n 'https://api.sensenova.cn/v1/llm/chat-completions'.\n gen_params: Default generation configuration which could be overridden\n on the fly of generation.\n \"\"\"\n\n is_api: bool = True\n\n def __init__(\n self,\n model_type: str = 'SenseChat-5-Cantonese',\n retry: int = 2,\n json_mode: bool = False,\n key: Union[str, List[str]] = 'ENV',\n meta_template: Optional[Dict] = [\n dict(role='system', api_role='system'),\n dict(role='user', api_role='user'),\n dict(role='assistant', api_role='assistant'),\n dict(role='environment', api_role='system'),\n ],\n sensenova_api_base: str = SENSENOVA_API_BASE,\n proxies: Optional[Dict] = None,\n **gen_params,\n ):\n\n super().__init__(\n model_type=model_type,\n meta_template=meta_template,\n retry=retry,\n **gen_params,\n )\n self.logger = getLogger(__name__)\n\n if isinstance(key, str):\n # First, apply for SenseNova's ak and sk from SenseTime staff\n # Then, generated SENSENOVA_API_KEY using lagent.utils.gen_key.auto_gen_jwt_token(ak, sk)\n self.keys = [\n os.getenv('SENSENOVA_API_KEY') if key == 'ENV' else key\n ]\n else:\n self.keys = key\n\n # record invalid keys and skip them when requesting API\n # - keys have insufficient_quota\n self.invalid_keys = set()\n\n self.key_ctr = 0\n self.url = sensenova_api_base\n self.model_type = model_type\n self.proxies = proxies\n self.json_mode = json_mode\n\n def chat(\n self,\n inputs: Union[List[dict], List[List[dict]]],\n **gen_params,\n ) -> Union[str, List[str]]:\n \"\"\"Generate responses given the contexts.\n\n Args:\n inputs (Union[List[dict], List[List[dict]]]): a list of messages\n or list of lists of messages\n gen_params: additional generation configuration\n\n Returns:\n Union[str, List[str]]: generated string(s)\n \"\"\"\n assert isinstance(inputs, list)\n if 'max_tokens' in gen_params:\n raise NotImplementedError('unsupported parameter: max_tokens')\n gen_params = {**self.gen_params, **gen_params}\n with ThreadPoolExecutor(max_workers=20) as executor:\n tasks = [\n executor.submit(self._chat,\n self.template_parser._prompt2api(messages),\n **gen_params)\n for messages in (\n [inputs] if isinstance(inputs[0], dict) else inputs)\n ]\n ret = [task.result() for task in tasks]\n return ret[0] if isinstance(inputs[0], dict) else ret\n\n def stream_chat(\n self,\n inputs: List[dict],\n **gen_params,\n ) -> Generator[Tuple[ModelStatusCode, str, Optional[str]], None, None]:\n \"\"\"Generate responses given the contexts.\n\n Args:\n inputs (List[dict]): a list of messages\n gen_params: additional generation configuration\n\n Yields:\n Tuple[ModelStatusCode, str, Optional[str]]: Status code, generated string, and optional metadata\n \"\"\"\n assert isinstance(inputs, list)\n if 'max_tokens' in gen_params:\n raise NotImplementedError('unsupported parameter: max_tokens')\n gen_params = self.update_gen_params(**gen_params)\n gen_params['stream'] = True\n\n resp = ''\n finished = False\n stop_words = gen_params.get('stop_words') or []\n messages = self.template_parser._prompt2api(inputs)\n for text in self._stream_chat(messages, **gen_params):\n # TODO 测试 resp = text 还是 resp += text\n resp += text\n if not resp:\n continue\n # remove stop_words\n for sw in stop_words:\n if sw in resp:\n resp = filter_suffix(resp, stop_words)\n finished = True\n break\n yield ModelStatusCode.STREAM_ING, resp, None\n if finished:\n break\n yield ModelStatusCode.END, resp, None\n\n def _chat(self, messages: List[dict], **gen_params) -> str:\n \"\"\"Generate completion from a list of templates.\n\n Args:\n messages (List[dict]): a list of prompt dictionaries\n gen_params: additional generation configuration\n\n Returns:\n str: The generated string.\n \"\"\"\n assert isinstance(messages, list)\n\n header, data = self.generate_request_data(\n model_type=self.model_type,\n messages=messages,\n gen_params=gen_params,\n json_mode=self.json_mode,\n )\n\n max_num_retries = 0\n while max_num_retries < self.retry:\n self._wait()\n\n with Lock():\n if len(self.invalid_keys) == len(self.keys):\n raise RuntimeError('All keys have insufficient quota.')\n\n # find the next valid key\n while True:\n self.key_ctr += 1\n if self.key_ctr == len(self.keys):\n self.key_ctr = 0\n\n if self.keys[self.key_ctr] not in self.invalid_keys:\n break\n\n key = self.keys[self.key_ctr]\n header['Authorization'] = f'Bearer {key}'\n\n response = dict()\n try:\n raw_response = requests.post(\n self.url,\n headers=header,\n data=json.dumps(data),\n proxies=self.proxies,\n )\n response = raw_response.json()\n return response['choices'][0]['message']['content'].strip()\n except requests.ConnectionError:\n print('Got connection error, retrying...')\n continue\n except requests.JSONDecodeError:\n print('JsonDecode error, got', str(raw_response.content))\n continue\n except KeyError:\n if 'error' in response:\n if response['error']['code'] == 'rate_limit_exceeded':\n time.sleep(1)\n continue\n elif response['error']['code'] == 'insufficient_quota':\n self.invalid_keys.add(key)\n self.logger.warn(f'insufficient_quota key: {key}')\n continue\n\n print('Find error message in response: ',\n str(response['error']))\n except Exception as error:\n print(str(error))\n max_num_retries += 1\n\n raise RuntimeError('Calling SenseTime failed after retrying for '\n f'{max_num_retries} times. Check the logs for '\n 'details.')\n\n def _stream_chat(self, messages: List[dict], **gen_params) -> str:\n \"\"\"Generate completion from a list of templates.\n\n Args:\n messages (List[dict]): a list of prompt dictionaries\n gen_params: additional generation configuration\n\n Returns:\n str: The generated string.\n \"\"\"\n\n def streaming(raw_response):\n for chunk in raw_response.iter_lines():\n if chunk:\n try:\n decoded_chunk = chunk.decode('utf-8')\n # print(f\"Decoded chunk: {decoded_chunk}\")\n\n if decoded_chunk == 'data:[DONE]':\n # print(\"Stream ended\")\n break\n\n if decoded_chunk.startswith('data:'):\n json_str = decoded_chunk[5:]\n chunk_data = json.loads(json_str)\n\n if 'data' in chunk_data and 'choices' in chunk_data[\n 'data']:\n choice = chunk_data['data']['choices'][0]\n if 'delta' in choice:\n content = choice['delta']\n yield content\n else:\n print(f'Unexpected format: {decoded_chunk}')\n\n except json.JSONDecodeError as e:\n print(f'JSON parsing error: {e}')\n except Exception as e:\n print(\n f'An error occurred while processing the chunk: {e}'\n )\n\n assert isinstance(messages, list)\n\n header, data = self.generate_request_data(\n model_type=self.model_type,\n messages=messages,\n gen_params=gen_params,\n json_mode=self.json_mode,\n )\n\n max_num_retries = 0\n while max_num_retries < self.retry:\n if len(self.invalid_keys) == len(self.keys):\n raise RuntimeError('All keys have insufficient quota.')\n\n # find the next valid key\n while True:\n self.key_ctr += 1\n if self.key_ctr == len(self.keys):\n self.key_ctr = 0\n\n if self.keys[self.key_ctr] not in self.invalid_keys:\n break\n\n key = self.keys[self.key_ctr]\n header['Authorization'] = f'Bearer {key}'\n\n response = dict()\n try:\n raw_response = requests.post(\n self.url,\n headers=header,\n data=json.dumps(data),\n proxies=self.proxies,\n )\n return streaming(raw_response)\n except requests.ConnectionError:\n print('Got connection error, retrying...')\n continue\n except requests.JSONDecodeError:\n print('JsonDecode error, got', str(raw_response.content))\n continue\n except KeyError:\n if 'error' in response:\n if response['error']['code'] == 'rate_limit_exceeded':\n time.sleep(1)\n continue\n elif response['error']['code'] == 'insufficient_quota':\n self.invalid_keys.add(key)\n self.logger.warn(f'insufficient_quota key: {key}')\n continue\n\n print('Find error message in response: ',\n str(response['error']))\n except Exception as error:\n print(str(error))\n max_num_retries += 1\n\n raise RuntimeError('Calling SenseTime failed after retrying for '\n f'{max_num_retries} times. Check the logs for '\n 'details.')\n\n def generate_request_data(self,\n model_type,\n messages,\n gen_params,\n json_mode=False):\n \"\"\"\n Generates the request data for different model types.\n\n Args:\n model_type (str): The type of the model (e.g., 'sense').\n messages (list): The list of messages to be sent to the model.\n gen_params (dict): The generation parameters.\n json_mode (bool): Flag to determine if the response format should be JSON.\n\n Returns:\n tuple: A tuple containing the header and the request data.\n \"\"\"\n # Copy generation parameters to avoid modifying the original dictionary\n gen_params = gen_params.copy()\n\n # Hold out 100 tokens due to potential errors in token calculation\n max_tokens = min(gen_params.pop('max_new_tokens'), 4096)\n if max_tokens <= 0:\n return '', ''\n\n # Initialize the header\n header = {\n 'content-type': 'application/json',\n }\n\n # Common parameters processing\n gen_params['max_tokens'] = max_tokens\n if 'stop_words' in gen_params:\n gen_params['stop'] = gen_params.pop('stop_words')\n if 'repetition_penalty' in gen_params:\n gen_params['frequency_penalty'] = gen_params.pop(\n 'repetition_penalty')\n\n # Model-specific processing\n data = {}\n if model_type.lower().startswith('sense'):\n gen_params.pop('skip_special_tokens', None)\n gen_params.pop('session_id', None)\n data = {\n 'model': model_type,\n 'messages': messages,\n 'n': 1,\n **gen_params\n }\n if json_mode:\n data['response_format'] = {'type': 'json_object'}\n else:\n raise NotImplementedError(\n f'Model type {model_type} is not supported')\n\n return header, data\n\n def tokenize(self, prompt: str) -> list:\n \"\"\"Tokenize the input prompt.\n\n Args:\n prompt (str): Input string.\n\n Returns:\n list: token ids\n \"\"\"\n import tiktoken\n\n self.tiktoken = tiktoken\n enc = self.tiktoken.encoding_for_model('gpt-4o')\n return enc.encode(prompt)\n"}}}} |