1 line
93 KiB
JSON
1 line
93 KiB
JSON
{"repo": "char-lotte-xia/werewolfplus", "files": {"agent_manager/agents/civ_agent/civ_autogpt/GPTAgent.py": {"e3d3d0815d29ff5862423284fbe1676158a0f6be": {"t": 1785745989.2344434, "n": 16192, "h": "88111e04982f2009", "c": "import os\nimport openai\nimport random\nimport json\nimport pinecone\n\nfrom civrealm.freeciv.utils.freeciv_logging import fc_logger\nfrom .utils import num_tokens_from_messages, extract_json, TOKEN_LIMIT_TABLE\nfrom langchain_community.chat_models import ChatOpenAI, AzureChatOpenAI\nfrom langchain.memory import ConversationSummaryBufferMemory\n\nfrom langchain_community.embeddings.openai import OpenAIEmbeddings\nfrom langchain_community.vectorstores import Pinecone\nfrom langchain_community.llms import OpenAI, AzureOpenAI\nfrom langchain.chains.question_answering import load_qa_chain\nfrom agent_manager.agents.civ_agent.prompt_handlers.base_prompt_handler import BasePromptHandler\n\n\ncwd = os.getcwd()\nOPENAI_KEYS_FILE = os.path.join(cwd, \"agents/civ_autogpt/openai_keys.txt\")\nSAVED_DIALOGUE_FILE = os.path.join(\n cwd, \"agents/civ_autogpt/saved_dialogues/saved_dialogue.txt\")\ntask_prompt_file = os.path.join(cwd,\n \"agents/civ_autogpt/prompts/task_prompt.txt\")\nGPT_MODEL_NAME = \"gpt-3.5-turbo-16k\"\n\n\nclass GPTAgent:\n \"\"\"\n This agent uses GPT-3 to generate actions.\n \"\"\"\n def __init__(self, model):\n self.model = model\n self.dialogue = []\n self.taken_actions_list = []\n self.message = ''\n\n self.openai_api_keys = self.load_openai_keys()\n self.prompt_handler = BasePromptHandler()\n self.state_prompt = self._load_state_prompt()\n self.task_prompt = self._load_task_prompt()\n\n # For Azure OpenAI API only\n self.deployment_name = os.environ['DEPLOYMENT_NAME']\n\n if os.environ['OPENAI_API_TYPE'] == 'azure':\n self.change_api_base('azure')\n llm = AzureChatOpenAI(openai_api_base=openai.api_base,\n openai_api_version=openai.api_version,\n openai_api_key=openai.api_key,\n openai_api_type=openai.api_type,\n deployment_name=self.deployment_name,\n temperature=0.7)\n self.chain = load_qa_chain(AzureOpenAI(\n deployment_name=self.deployment_name, model_name=self.model),\n chain_type=\"stuff\")\n else:\n self.change_api_base('openai')\n llm = ChatOpenAI(temperature=0.7, openai_api_key=openai.api_key)\n self.chain = load_qa_chain(OpenAI(model_name=GPT_MODEL_NAME),\n chain_type=\"stuff\")\n\n self.memory = ConversationSummaryBufferMemory(llm=llm,\n max_token_limit=500)\n\n pinecone.init(api_key=os.environ[\"MY_PINECONE_API_KEY\"],\n environment=os.environ[\"MY_PINECONE_ENV\"])\n\n self.index = Pinecone.from_existing_index(\n index_name='langchain-demo',\n embedding=OpenAIEmbeddings(model=\"text-embedding-ada-002\"))\n\n def add_user_message_to_dialogue(self, message):\n self.dialogue.append({'role': 'user', 'content': message})\n\n def get_similiar_docs(self, query, k=2, score=False):\n index = self.index\n if score:\n similar_docs = index.similarity_search_with_score(query, k=k)\n else:\n similar_docs = index.similarity_search(query, k=k)\n return similar_docs\n\n def get_answer(self, query):\n similar_docs = self.get_similiar_docs(query)\n while True:\n try:\n answer = self.chain.run(input_documents=similar_docs,\n question=query)\n break\n except Exception as e:\n print('similiar_docs:', similar_docs)\n print('query:', query)\n print(e)\n self.update_openai_api_key()\n return answer\n\n def change_api_base(self, to_type):\n openai.api_type = to_type\n if to_type == 'azure':\n openai.api_version = os.environ[\"OPENAI_API_VERSION\"]\n openai.api_base = os.environ[\"OPENAI_API_BASE\"]\n openai.api_key = os.environ[\"OPENAI_API_KEY\"]\n else:\n assert openai.api_type == \"openai\"\n openai.api_version = \"\"\n openai.api_base = 'https://api.openai.com/v1'\n self.update_openai_api_key()\n\n @staticmethod\n def load_openai_keys():\n with open(OPENAI_KEYS_FILE, \"r\") as f:\n context = f.read()\n return context.split('\\n')\n\n def _load_state_prompt(self):\n state_prompt = self.prompt_handler.state_prompt()\n self.add_user_message_to_dialogue(state_prompt)\n return state_prompt\n\n def _load_task_prompt(self):\n task_prompt = self.prompt_handler.task_prompt()\n self.add_user_message_to_dialogue(task_prompt)\n return task_prompt\n\n def load_saved_dialogue(self, load_path=SAVED_DIALOGUE_FILE):\n # print(\"reading task prompt from {}\".format(task_prompt_file))\n with open(load_path, \"r\") as f:\n self.dialogue = eval(f.read())\n\n def save_dialogue_to_file(self, save_path=SAVED_DIALOGUE_FILE):\n with open(save_path, \"w\", encoding='utf-8') as f:\n for message in self.dialogue:\n f.write(str(message) + '\\n')\n\n def update_openai_api_key(self):\n if openai.api_type != 'openai':\n return\n\n curr_key = self.openai_api_keys[0]\n openai.api_key = os.environ[\"OPENAI_API_KEY\"] = curr_key\n self.openai_api_keys.pop(0)\n self.openai_api_keys.append(curr_key)\n\n def check_if_the_taken_actions_list_needed_update(self,\n check_content,\n check_num=3,\n top_k_charactors=0):\n if top_k_charactors == 0:\n if len(self.taken_actions_list) >= check_num:\n for i in range(check_num):\n if self.taken_actions_list[-1 - i] == check_content:\n if i == check_num - 1:\n return True\n else:\n continue\n else:\n return False\n\n return False\n else:\n if len(self.taken_actions_list) >= check_num:\n for i in range(check_num):\n if self.taken_actions_list[\n -1 - i][:top_k_charactors] == check_content:\n if i == check_num - 1:\n return True\n else:\n continue\n else:\n return False\n\n return False\n\n def process_command(self, command_json, obs_input_prompt,\n current_unit_name, current_avail_actions):\n '''\n manualAndHistorySearch\n askCurrentGameInformation\n finalDecision\n '''\n fc_logger.debug(f'Processing command: {command_json}')\n try:\n command_input = command_json['command']['input']\n command_name = command_json['command']['name']\n except Exception as e:\n print(e)\n print('Not in given json format, retrying...')\n if random.random() > 0.5:\n self.update_dialogue(obs_input_prompt +\n self.prompt_handler.insist_json(),\n pop_num=2)\n return None\n else:\n self.update_dialogue(obs_input_prompt, pop_num=2)\n return None\n if (command_name == 'finalDecision') and command_input['action']:\n # Here to implement controller\n print(command_input)\n exec_action = command_input['action']\n\n if command_input['action'] not in current_avail_actions:\n print('Not in the available action list, retrying...')\n # Here is the most time taking place.\n if random.random() > 0.5:\n self.update_dialogue(\n obs_input_prompt +\n self.prompt_handler.insist_avail_action(),\n pop_num=2)\n return None\n else:\n self.update_dialogue(obs_input_prompt, pop_num=2)\n return None\n\n else:\n self.taken_actions_list.append(command_input['action'])\n if self.check_if_the_taken_actions_list_needed_update(\n 'goto', 15, 4\n ) or self.check_if_the_taken_actions_list_needed_update(\n 'keep_activity', 15, 0):\n self.update_dialogue(\n obs_input_prompt +\n self.prompt_handler.insist_various_actions(\n action=\"goto\"),\n pop_num=2)\n # command_input = new_response['command']['input']\n self.taken_actions_list = []\n return None\n else:\n print('exec_action:', exec_action)\n return exec_action\n\n elif command_name == 'askCurrentGameInformation' and command_input[\n 'query']:\n print(command_input)\n self.taken_actions_list.append('askCurrentGameInformation')\n return None\n\n elif command_name == 'manualAndHistorySearch' and command_input[\n 'look_up']:\n print(command_input)\n\n if self.check_if_the_taken_actions_list_needed_update(\n 'look_up', 3, 0):\n answer = self.prompt_handler.generate(\"finish_look_for\")\n print('answer:', answer)\n self.add_user_message_to_dialogue(answer)\n self.taken_actions_list = []\n else:\n query = command_input['look_up']\n answer = self.get_answer(query)\n print('answer:', answer)\n if random.random() > 0.5:\n self.add_user_message_to_dialogue(\n answer + self.prompt_handler.finish_look_for())\n else:\n self.add_user_message_to_dialogue(answer)\n\n self.memory.save_context({'assistant': query},\n {'user': answer})\n self.taken_actions_list.append('look_up')\n\n return None\n else:\n print('error')\n print(command_json)\n\n if random.random() < 0.8:\n self.dialogue.pop(-1)\n else:\n self.add_user_message_to_dialogue(\n 'You should only use the given commands!')\n # self.update_dialogue(obs_input_prompt, pop_num = 1)\n\n return None\n\n def query(self, stop=None, temperature=0.7, top_p=0.95):\n self.restrict_dialogue()\n # TODO add retreat mech to cope with rate limit\n self.update_openai_api_key()\n\n fc_logger.debug(f'Querying with dialogue: {self.dialogue}')\n\n if self.model in ['gpt-3.5-turbo-0301', 'gpt-3.5-turbo']:\n assert openai.api_type == 'openai'\n response = openai.ChatCompletion.create(model=self.model,\n messages=self.dialogue,\n temperature=temperature,\n top_p=top_p)\n elif self.model in [\"gpt-35-turbo\", \"gpt-35-turbo-16k\"]:\n assert openai.api_type == 'azure'\n response = openai.ChatCompletion.create(\n deployment_id=self.deployment_name,\n model=self.model,\n messages=self.dialogue)\n\n else:\n chat = ChatOpenAI(model_name='deepseek-chat',\n openai_api_key='sk-c7743bb83dc54d078af3731f4d7885e1',\n temperature=0.2,\n max_tokens=1024,\n n=1,\n request_timeout=60,\n openai_api_base='https://api.deepseek.com'\n )\n generations = chat.generate([self.dialogue], stop=[stop] if stop is not None else None)\n response = [chat_gen.message.content for chat_gen in generations.generations[0]][0]\n\n return response\n\n def update_dialogue(self, chat_content, pop_num=0):\n if pop_num != 0:\n for i in range(pop_num):\n self.dialogue.pop(-1)\n\n return self.communicate(chat_content)\n\n def parse_response(self, response):\n fc_logger.debug(f'Parsing response: {response}')\n\n if self.model in [\n 'gpt-3.5-turbo-0301', 'gpt-3.5-turbo', 'gpt-4', 'gpt-4-0314'\n ]:\n return dict(response.choices[0].message)\n\n elif self.model in [\"gpt-35-turbo\", \"gpt-35-turbo-16k\"]:\n try:\n ans = extract_json(\n response.choices[0].message.content)\n except:\n return response.choices[0].message\n return {'role': 'assistant', 'content': ans}\n\n elif self.model in ['vicuna-33B', 'Llama2-70B-chat']:\n return {'role': 'assistant', 'content': extract_json(response)}\n\n else:\n # self.model in ['text-davinci-003', 'code-davinci-002']\n return {\n 'role': 'assistant',\n 'content': response[\"choices\"][0][\"text\"][2:]\n }\n\n def restrict_dialogue(self):\n limit = TOKEN_LIMIT_TABLE[self.model]\n \"\"\"\n The limit on token length for gpt-3.5-turbo-0301 is 4096.\n If token length exceeds the limit, we will remove the oldest messages.\n \"\"\"\n # TODO validate that the messages removed are obs and actions\n while num_tokens_from_messages(self.dialogue, self.model) >= limit:\n temp_message = {}\n user_tag = 0\n if self.dialogue[-1]['role'] == 'user':\n temp_message = self.dialogue[-1]\n user_tag = 1\n\n while len(self.dialogue) >= 3:\n self.dialogue.pop(-1)\n\n while True:\n try:\n self.add_user_message_to_dialogue(\n 'The former chat history can be summarized as: \\n' +\n self.memory.load_memory_variables({})['history'])\n break\n except Exception as e:\n print(e)\n self.update_openai_api_key()\n\n if user_tag == 1:\n self.dialogue.append(temp_message)\n user_tag = 0\n\n def communicate(self, content, parse_choice_tag=False):\n self.add_user_message_to_dialogue(content)\n while True:\n try:\n raw_response = self.query()\n self.message = self.parse_response(raw_response)\n self.dialogue.append(self.message)\n\n response = self.message[\"content\"]\n try:\n response = json.loads(response)\n except Exception as e:\n # self.dialogue.pop(-1)\n print(e)\n self.add_user_message_to_dialogue(\n 'You should only respond in JSON format as described')\n print('Not response json, retrying...')\n\n continue\n break\n\n except Exception as e:\n fc_logger.debug('Error in communicate: ' + str(e))\n fc_logger.debug('content: ' + content)\n print(e)\n print(\"retrying...\")\n continue\n return response\n\n def reset(self):\n\n self.dialogue = []\n self.message = ''\n self.taken_actions_list = []\n\n self.openai_api_keys = self.load_openai_keys()\n self.state_prompt = self._load_state_prompt()\n self.task_prompt = self._load_task_prompt()\n"}, "712d86460aa169e5b80d0f68affbbf75ea34a858": {"t": 1785758576.853193, "n": 16192, "h": "88111e04982f2009", "c": "import os\nimport openai\nimport random\nimport json\nimport pinecone\n\nfrom civrealm.freeciv.utils.freeciv_logging import fc_logger\nfrom .utils import num_tokens_from_messages, extract_json, TOKEN_LIMIT_TABLE\nfrom langchain_community.chat_models import ChatOpenAI, AzureChatOpenAI\nfrom langchain.memory import ConversationSummaryBufferMemory\n\nfrom langchain_community.embeddings.openai import OpenAIEmbeddings\nfrom langchain_community.vectorstores import Pinecone\nfrom langchain_community.llms import OpenAI, AzureOpenAI\nfrom langchain.chains.question_answering import load_qa_chain\nfrom agent_manager.agents.civ_agent.prompt_handlers.base_prompt_handler import BasePromptHandler\n\n\ncwd = os.getcwd()\nOPENAI_KEYS_FILE = os.path.join(cwd, \"agents/civ_autogpt/openai_keys.txt\")\nSAVED_DIALOGUE_FILE = os.path.join(\n cwd, \"agents/civ_autogpt/saved_dialogues/saved_dialogue.txt\")\ntask_prompt_file = os.path.join(cwd,\n \"agents/civ_autogpt/prompts/task_prompt.txt\")\nGPT_MODEL_NAME = \"gpt-3.5-turbo-16k\"\n\n\nclass GPTAgent:\n \"\"\"\n This agent uses GPT-3 to generate actions.\n \"\"\"\n def __init__(self, model):\n self.model = model\n self.dialogue = []\n self.taken_actions_list = []\n self.message = ''\n\n self.openai_api_keys = self.load_openai_keys()\n self.prompt_handler = BasePromptHandler()\n self.state_prompt = self._load_state_prompt()\n self.task_prompt = self._load_task_prompt()\n\n # For Azure OpenAI API only\n self.deployment_name = os.environ['DEPLOYMENT_NAME']\n\n if os.environ['OPENAI_API_TYPE'] == 'azure':\n self.change_api_base('azure')\n llm = AzureChatOpenAI(openai_api_base=openai.api_base,\n openai_api_version=openai.api_version,\n openai_api_key=openai.api_key,\n openai_api_type=openai.api_type,\n deployment_name=self.deployment_name,\n temperature=0.7)\n self.chain = load_qa_chain(AzureOpenAI(\n deployment_name=self.deployment_name, model_name=self.model),\n chain_type=\"stuff\")\n else:\n self.change_api_base('openai')\n llm = ChatOpenAI(temperature=0.7, openai_api_key=openai.api_key)\n self.chain = load_qa_chain(OpenAI(model_name=GPT_MODEL_NAME),\n chain_type=\"stuff\")\n\n self.memory = ConversationSummaryBufferMemory(llm=llm,\n max_token_limit=500)\n\n pinecone.init(api_key=os.environ[\"MY_PINECONE_API_KEY\"],\n environment=os.environ[\"MY_PINECONE_ENV\"])\n\n self.index = Pinecone.from_existing_index(\n index_name='langchain-demo',\n embedding=OpenAIEmbeddings(model=\"text-embedding-ada-002\"))\n\n def add_user_message_to_dialogue(self, message):\n self.dialogue.append({'role': 'user', 'content': message})\n\n def get_similiar_docs(self, query, k=2, score=False):\n index = self.index\n if score:\n similar_docs = index.similarity_search_with_score(query, k=k)\n else:\n similar_docs = index.similarity_search(query, k=k)\n return similar_docs\n\n def get_answer(self, query):\n similar_docs = self.get_similiar_docs(query)\n while True:\n try:\n answer = self.chain.run(input_documents=similar_docs,\n question=query)\n break\n except Exception as e:\n print('similiar_docs:', similar_docs)\n print('query:', query)\n print(e)\n self.update_openai_api_key()\n return answer\n\n def change_api_base(self, to_type):\n openai.api_type = to_type\n if to_type == 'azure':\n openai.api_version = os.environ[\"OPENAI_API_VERSION\"]\n openai.api_base = os.environ[\"OPENAI_API_BASE\"]\n openai.api_key = os.environ[\"OPENAI_API_KEY\"]\n else:\n assert openai.api_type == \"openai\"\n openai.api_version = \"\"\n openai.api_base = 'https://api.openai.com/v1'\n self.update_openai_api_key()\n\n @staticmethod\n def load_openai_keys():\n with open(OPENAI_KEYS_FILE, \"r\") as f:\n context = f.read()\n return context.split('\\n')\n\n def _load_state_prompt(self):\n state_prompt = self.prompt_handler.state_prompt()\n self.add_user_message_to_dialogue(state_prompt)\n return state_prompt\n\n def _load_task_prompt(self):\n task_prompt = self.prompt_handler.task_prompt()\n self.add_user_message_to_dialogue(task_prompt)\n return task_prompt\n\n def load_saved_dialogue(self, load_path=SAVED_DIALOGUE_FILE):\n # print(\"reading task prompt from {}\".format(task_prompt_file))\n with open(load_path, \"r\") as f:\n self.dialogue = eval(f.read())\n\n def save_dialogue_to_file(self, save_path=SAVED_DIALOGUE_FILE):\n with open(save_path, \"w\", encoding='utf-8') as f:\n for message in self.dialogue:\n f.write(str(message) + '\\n')\n\n def update_openai_api_key(self):\n if openai.api_type != 'openai':\n return\n\n curr_key = self.openai_api_keys[0]\n openai.api_key = os.environ[\"OPENAI_API_KEY\"] = curr_key\n self.openai_api_keys.pop(0)\n self.openai_api_keys.append(curr_key)\n\n def check_if_the_taken_actions_list_needed_update(self,\n check_content,\n check_num=3,\n top_k_charactors=0):\n if top_k_charactors == 0:\n if len(self.taken_actions_list) >= check_num:\n for i in range(check_num):\n if self.taken_actions_list[-1 - i] == check_content:\n if i == check_num - 1:\n return True\n else:\n continue\n else:\n return False\n\n return False\n else:\n if len(self.taken_actions_list) >= check_num:\n for i in range(check_num):\n if self.taken_actions_list[\n -1 - i][:top_k_charactors] == check_content:\n if i == check_num - 1:\n return True\n else:\n continue\n else:\n return False\n\n return False\n\n def process_command(self, command_json, obs_input_prompt,\n current_unit_name, current_avail_actions):\n '''\n manualAndHistorySearch\n askCurrentGameInformation\n finalDecision\n '''\n fc_logger.debug(f'Processing command: {command_json}')\n try:\n command_input = command_json['command']['input']\n command_name = command_json['command']['name']\n except Exception as e:\n print(e)\n print('Not in given json format, retrying...')\n if random.random() > 0.5:\n self.update_dialogue(obs_input_prompt +\n self.prompt_handler.insist_json(),\n pop_num=2)\n return None\n else:\n self.update_dialogue(obs_input_prompt, pop_num=2)\n return None\n if (command_name == 'finalDecision') and command_input['action']:\n # Here to implement controller\n print(command_input)\n exec_action = command_input['action']\n\n if command_input['action'] not in current_avail_actions:\n print('Not in the available action list, retrying...')\n # Here is the most time taking place.\n if random.random() > 0.5:\n self.update_dialogue(\n obs_input_prompt +\n self.prompt_handler.insist_avail_action(),\n pop_num=2)\n return None\n else:\n self.update_dialogue(obs_input_prompt, pop_num=2)\n return None\n\n else:\n self.taken_actions_list.append(command_input['action'])\n if self.check_if_the_taken_actions_list_needed_update(\n 'goto', 15, 4\n ) or self.check_if_the_taken_actions_list_needed_update(\n 'keep_activity', 15, 0):\n self.update_dialogue(\n obs_input_prompt +\n self.prompt_handler.insist_various_actions(\n action=\"goto\"),\n pop_num=2)\n # command_input = new_response['command']['input']\n self.taken_actions_list = []\n return None\n else:\n print('exec_action:', exec_action)\n return exec_action\n\n elif command_name == 'askCurrentGameInformation' and command_input[\n 'query']:\n print(command_input)\n self.taken_actions_list.append('askCurrentGameInformation')\n return None\n\n elif command_name == 'manualAndHistorySearch' and command_input[\n 'look_up']:\n print(command_input)\n\n if self.check_if_the_taken_actions_list_needed_update(\n 'look_up', 3, 0):\n answer = self.prompt_handler.generate(\"finish_look_for\")\n print('answer:', answer)\n self.add_user_message_to_dialogue(answer)\n self.taken_actions_list = []\n else:\n query = command_input['look_up']\n answer = self.get_answer(query)\n print('answer:', answer)\n if random.random() > 0.5:\n self.add_user_message_to_dialogue(\n answer + self.prompt_handler.finish_look_for())\n else:\n self.add_user_message_to_dialogue(answer)\n\n self.memory.save_context({'assistant': query},\n {'user': answer})\n self.taken_actions_list.append('look_up')\n\n return None\n else:\n print('error')\n print(command_json)\n\n if random.random() < 0.8:\n self.dialogue.pop(-1)\n else:\n self.add_user_message_to_dialogue(\n 'You should only use the given commands!')\n # self.update_dialogue(obs_input_prompt, pop_num = 1)\n\n return None\n\n def query(self, stop=None, temperature=0.7, top_p=0.95):\n self.restrict_dialogue()\n # TODO add retreat mech to cope with rate limit\n self.update_openai_api_key()\n\n fc_logger.debug(f'Querying with dialogue: {self.dialogue}')\n\n if self.model in ['gpt-3.5-turbo-0301', 'gpt-3.5-turbo']:\n assert openai.api_type == 'openai'\n response = openai.ChatCompletion.create(model=self.model,\n messages=self.dialogue,\n temperature=temperature,\n top_p=top_p)\n elif self.model in [\"gpt-35-turbo\", \"gpt-35-turbo-16k\"]:\n assert openai.api_type == 'azure'\n response = openai.ChatCompletion.create(\n deployment_id=self.deployment_name,\n model=self.model,\n messages=self.dialogue)\n\n else:\n chat = ChatOpenAI(model_name='deepseek-chat',\n openai_api_key='sk-c7743bb83dc54d078af3731f4d7885e1',\n temperature=0.2,\n max_tokens=1024,\n n=1,\n request_timeout=60,\n openai_api_base='https://api.deepseek.com'\n )\n generations = chat.generate([self.dialogue], stop=[stop] if stop is not None else None)\n response = [chat_gen.message.content for chat_gen in generations.generations[0]][0]\n\n return response\n\n def update_dialogue(self, chat_content, pop_num=0):\n if pop_num != 0:\n for i in range(pop_num):\n self.dialogue.pop(-1)\n\n return self.communicate(chat_content)\n\n def parse_response(self, response):\n fc_logger.debug(f'Parsing response: {response}')\n\n if self.model in [\n 'gpt-3.5-turbo-0301', 'gpt-3.5-turbo', 'gpt-4', 'gpt-4-0314'\n ]:\n return dict(response.choices[0].message)\n\n elif self.model in [\"gpt-35-turbo\", \"gpt-35-turbo-16k\"]:\n try:\n ans = extract_json(\n response.choices[0].message.content)\n except:\n return response.choices[0].message\n return {'role': 'assistant', 'content': ans}\n\n elif self.model in ['vicuna-33B', 'Llama2-70B-chat']:\n return {'role': 'assistant', 'content': extract_json(response)}\n\n else:\n # self.model in ['text-davinci-003', 'code-davinci-002']\n return {\n 'role': 'assistant',\n 'content': response[\"choices\"][0][\"text\"][2:]\n }\n\n def restrict_dialogue(self):\n limit = TOKEN_LIMIT_TABLE[self.model]\n \"\"\"\n The limit on token length for gpt-3.5-turbo-0301 is 4096.\n If token length exceeds the limit, we will remove the oldest messages.\n \"\"\"\n # TODO validate that the messages removed are obs and actions\n while num_tokens_from_messages(self.dialogue, self.model) >= limit:\n temp_message = {}\n user_tag = 0\n if self.dialogue[-1]['role'] == 'user':\n temp_message = self.dialogue[-1]\n user_tag = 1\n\n while len(self.dialogue) >= 3:\n self.dialogue.pop(-1)\n\n while True:\n try:\n self.add_user_message_to_dialogue(\n 'The former chat history can be summarized as: \\n' +\n self.memory.load_memory_variables({})['history'])\n break\n except Exception as e:\n print(e)\n self.update_openai_api_key()\n\n if user_tag == 1:\n self.dialogue.append(temp_message)\n user_tag = 0\n\n def communicate(self, content, parse_choice_tag=False):\n self.add_user_message_to_dialogue(content)\n while True:\n try:\n raw_response = self.query()\n self.message = self.parse_response(raw_response)\n self.dialogue.append(self.message)\n\n response = self.message[\"content\"]\n try:\n response = json.loads(response)\n except Exception as e:\n # self.dialogue.pop(-1)\n print(e)\n self.add_user_message_to_dialogue(\n 'You should only respond in JSON format as described')\n print('Not response json, retrying...')\n\n continue\n break\n\n except Exception as e:\n fc_logger.debug('Error in communicate: ' + str(e))\n fc_logger.debug('content: ' + content)\n print(e)\n print(\"retrying...\")\n continue\n return response\n\n def reset(self):\n\n self.dialogue = []\n self.message = ''\n self.taken_actions_list = []\n\n self.openai_api_keys = self.load_openai_keys()\n self.state_prompt = self._load_state_prompt()\n self.task_prompt = self._load_task_prompt()\n"}}, "agent_manager/agents/welfare_diplomacy/agents.py": {"e3d3d0815d29ff5862423284fbe1676158a0f6be": {"t": 1785747503.9265618, "n": 29485, "h": "4e696e65851d9f4b", "c": "\"\"\"\nRepresents different agent prompting systems (e.g. GPT, 🤗 Transformers, random).\n\nA given agent should take in information about the game and a power, prompt\nan underlying model for a response, and return back the extracted response. \n\"\"\"\n\nfrom abc import ABC, abstractmethod\nimport json\nimport random\nimport time\nimport yaml\nfrom box import Box\nfrom games.welfare_diplomacy.diplomacy import Game\nimport wandb\n\nfrom games.welfare_diplomacy.welfare_diplomacy_baselines.environment import mila_actions, diplomacy_state\nfrom games.welfare_diplomacy.welfare_diplomacy_baselines.baselines import no_press_policies\n\nfrom agent_manager.agents.welfare_diplomacy.backends import OpenAIChatBackend\nfrom agent_manager.agents.welfare_diplomacy.data_types import (\n AgentResponse,\n AgentParams,\n BackendResponse,\n)\n\nfrom agent_manager.prompts import welfare_diplomacy_prompt as prompts\n\nclass AgentCompletionError(ValueError):\n \"\"\"Raised when an agent fails to complete a prompt.\"\"\"\n\n\nclass Agent(ABC):\n \"\"\"Base agent class.\"\"\"\n\n def __init__(self, **_):\n \"\"\"Base init to ignore unused kwargs.\"\"\"\n\n def __repr__(self) -> str:\n \"\"\"Return a string representation of the agent.\"\"\"\n raise NotImplementedError\n\n @abstractmethod\n def respond(\n self,\n params: AgentParams,\n ) -> AgentResponse:\n \"\"\"Prompt the model for a response.\"\"\"\n\n\nclass RandomAgent(Agent):\n \"\"\"Takes random actions and sends 1 random message.\"\"\"\n\n def __repr__(self) -> str:\n return \"RandomAgent()\"\n\n def respond(self, params: AgentParams) -> AgentResponse:\n \"\"\"Randomly generate orders and messages.\"\"\"\n # For each power, randomly sampling a valid order\n power_orders = []\n for loc in params.game.get_orderable_locations(params.power.name):\n if params.possible_orders[loc]:\n # Sort for determinism\n params.possible_orders[loc].sort()\n\n # If this is a disbandable unit in an adjustment phase in welfare,\n # then randomly choose whether to disband or not\n if (\n \"ADJUSTMENTS\" in str(params.game.phase)\n and \" D\" in params.possible_orders[loc][0][-2:]\n and params.game.welfare\n ):\n power_orders.append(\n random.choice([\"WAIVE\", params.possible_orders[loc][0]])\n )\n else:\n power_orders.append(random.choice(params.possible_orders[loc]))\n # # Testing: Randomly add an invalid order\n # if random.random() < 0.1:\n # power_orders.append(\"Random invalid order\")\n\n # # Testing: More randomly raise a completion error\n # if random.random() < 0.05:\n # raise AgentCompletionError(\"Randomly raised completion error\")\n\n # For debugging prompting\n system_prompt = prompts.get_system_prompt(params)\n user_prompt = prompts.get_user_prompt(params)\n\n # Randomly sending a message to another power\n other_powers = [p for p in params.game.powers if p != params.power.name] + [\n \"GLOBAL\"\n ]\n recipient = random.choice(other_powers)\n message = f\"Hello {recipient}! I'm {params.power.name} contacting you on turn {params.game.get_current_phase()}. Here's a random number: {random.randint(0, 100)}.\"\n\n # Sleep to allow wandb to catch up\n sleep_time = (\n 0.0\n if isinstance(wandb.run.mode, wandb.sdk.lib.disabled.RunDisabled)\n else 0.3\n )\n time.sleep(sleep_time)\n\n return AgentResponse(\n reasoning=\"Randomly generated orders and messages.\",\n orders=power_orders,\n messages={recipient: message} if not params.game.no_press else {},\n system_prompt=system_prompt,\n user_prompt=user_prompt,\n prompt_tokens=0,\n completion_tokens=0,\n total_tokens=0,\n completion_time_sec=sleep_time,\n )\n\n\nclass ManualAgent(Agent):\n \"\"\"Manually specify orders with --manual_orders_path.\"\"\"\n\n def __init__(self, manual_orders_path: str, **_):\n \"\"\"Load the manual orders.\"\"\"\n assert manual_orders_path\n self.manual_orders_path = manual_orders_path\n with open(manual_orders_path, \"r\") as file:\n self.manual_orders: dict[str, list[str]] = yaml.safe_load(file)\n # Validate the file\n for phase, orders_list in self.manual_orders.items():\n assert isinstance(phase, str)\n assert len(phase) == 6 # e.g. \"F1905M\"\n assert phase[0] in \"SFW\"\n assert phase[1:5].isdigit()\n assert phase[-1] in \"MRA\"\n assert isinstance(orders_list, list)\n for order in orders_list:\n assert isinstance(order, str)\n assert len(order.split()) >= 3\n assert order.split()[0] in \"AF\"\n\n def __repr__(self) -> str:\n return f\"ManualAgent({self.manual_orders_path})\"\n\n def respond(self, params: AgentParams) -> AgentResponse:\n \"\"\"Submit the specified orders if available.\"\"\"\n power_orders = []\n\n current_phase = params.game.get_current_phase()\n if current_phase in self.manual_orders:\n orders_list = self.manual_orders[current_phase]\n for order in orders_list:\n unit_type = order.split()[0]\n loc = order.split()[1]\n if unit_type + \" \" + loc in params.power.units:\n assert (\n order in params.possible_orders[loc]\n ), f\"Manual order {order} not in possible orders for power {params.power.name}:\\n{params.possible_orders[loc]}\"\n power_orders.append(order)\n\n # For debugging prompting\n system_prompt = prompts.get_system_prompt(params)\n user_prompt = prompts.get_user_prompt(params)\n\n # Sleep to allow wandb to catch up\n sleep_time = (\n 0.0\n if isinstance(wandb.run.mode, wandb.sdk.lib.disabled.RunDisabled)\n else 0.3\n )\n time.sleep(sleep_time)\n\n return AgentResponse(\n reasoning=\"Manually specified orders and messages.\",\n orders=power_orders,\n messages={},\n system_prompt=system_prompt,\n user_prompt=user_prompt,\n prompt_tokens=0,\n completion_tokens=0,\n total_tokens=0,\n completion_time_sec=sleep_time,\n )\n\n\nclass LLMAgent2(Agent):\n \"\"\"Uses OpenAI/Claude Chat/Completion to generate orders and messages.\"\"\"\n\n def __init__(self, model_name: str, **kwargs):\n # Decide whether it's a chat or completion model\n disable_completion_preface = kwargs.pop(\"disable_completion_preface\", False)\n self.use_completion_preface = not disable_completion_preface\n args={\n \"model_name\":\"deepseek-chat\",\n \"api_key\":\"sk-e851421cc0224deb8fad0c8a30d5c862\",\n \"api_url\":\"https://api.deepseek.com\",\n \"max_tokens\":4096,\n \"timeout\":60,\n \"temperature\":0.2\n }\n from agent_manager.llm_models.llm_model import LLMModel\n from openai import OpenAI\n llm_model=LLMModel(args)\n self.backend=OpenAI(api_key=llm_model.api_key,base_url=llm_model.api_url)\n\n def __repr__(self) -> str:\n return f\"LLMAgent(Backend: {self.backend.model_name}, Temperature: {self.temperature}, Top P: {self.top_p})\"\n\n def respond(self, params: AgentParams) -> AgentResponse:\n \"\"\"Prompt the model for a response.\"\"\"\n system_prompt = prompts.get_system_prompt(params)\n user_prompt = prompts.get_user_prompt(params)\n response = None\n try:\n if self.use_completion_preface:\n preface_prompt = prompts.get_preface_prompt(params)\n response: BackendResponse = self.backend.complete(\n system_prompt,\n user_prompt,\n completion_preface=preface_prompt,\n temperature=self.temperature,\n top_p=self.top_p,\n )\n json_completion = preface_prompt + response.completion\n else:\n response: BackendResponse = self.backend.complete(\n system_prompt,\n user_prompt,\n temperature=self.temperature,\n top_p=self.top_p,\n )\n json_completion = response.completion\n # Remove repeated **system** from parroty completion models\n json_completion = json_completion.split(\"**\")[0].strip(\" `\\n\")\n\n # Claude likes to add junk around the actual JSON object, so find it manually\n start = json_completion.index(\"{\")\n end = json_completion.rindex(\"}\") + 1 # +1 to include the } in the slice\n json_completion = json_completion[start:end]\n\n # Load the JSON\n completion = json.loads(json_completion, strict=False)\n\n # Extract data from completion\n reasoning = (\n completion[\"reasoning\"]\n if \"reasoning\" in completion\n else \"*model outputted no reasoning*\"\n )\n orders = completion[\"orders\"]\n # Clean orders\n for order in orders:\n if not isinstance(order, str):\n raise AgentCompletionError(\n f\"Order is not a str\\n\\Response: {response}\"\n )\n # Enforce no messages in no_press\n if params.game.no_press:\n completion[\"messages\"] = {}\n # Turn recipients in messages into ALLCAPS for the engine\n messages = {}\n for recipient, message in completion[\"messages\"].items():\n if isinstance(message, list):\n # Handle weird model outputs\n message = \" \".join(message)\n if not isinstance(message, str):\n # Force each message into a string\n message = str(message)\n if not message:\n # Skip empty messages\n continue\n messages[recipient.upper()] = message\n except Exception as exc:\n raise AgentCompletionError(f\"Exception: {exc}\\n\\Response: {response}\")\n return AgentResponse(\n reasoning=reasoning,\n orders=orders,\n messages=messages,\n system_prompt=system_prompt,\n user_prompt=user_prompt,\n prompt_tokens=response.prompt_tokens,\n completion_tokens=response.completion_tokens,\n total_tokens=response.total_tokens,\n completion_time_sec=response.completion_time_sec,\n )\nclass LLMAgent(Agent):\n \"\"\"Uses OpenAI/Claude Chat/Completion to generate orders and messages.\"\"\"\n\n def __init__(self, model,logger=None, role=None,**kwargs):\n self.logger=logger\n self.llm_model=model\n # Chat models can't specify the start of the completion\n self.use_completion_preface = False\n self.backend = OpenAIChatBackend(model,logger,role)\n\n def __repr__(self) -> str:\n return f\"LLMAgent(Backend: {self.llm_model.model_name}, Temperature: {self.llm_model.temperature})\"\n\n def respond(self, params: AgentParams) -> AgentResponse:\n \"\"\"Prompt the model for a response.\"\"\"\n system_prompt = prompts.get_system_prompt(params)\n user_prompt = prompts.get_user_prompt(params)\n response = None\n try:\n if self.use_completion_preface:\n preface_prompt = prompts.get_preface_prompt(params)\n response: BackendResponse = self.backend.complete(\n system_prompt,\n user_prompt,\n completion_preface=preface_prompt\n )\n json_completion = preface_prompt + response.completion\n else:\n response: BackendResponse = self.backend.complete(\n system_prompt,\n user_prompt\n )\n json_completion = response.completion\n # Remove repeated **system** from parroty completion models\n json_completion = json_completion.split(\"**\")[0].strip(\" `\\n\")\n\n # Claude likes to add junk around the actual JSON object, so find it manually\n start = json_completion.index(\"{\")\n end = json_completion.rindex(\"}\") + 1 # +1 to include the } in the slice\n json_completion = json_completion[start:end]\n\n # Load the JSON\n completion = json.loads(json_completion, strict=False)\n\n # Extract data from completion\n reasoning = (\n completion[\"reasoning\"]\n if \"reasoning\" in completion\n else \"*model outputted no reasoning*\"\n )\n orders = completion[\"orders\"]\n self.update_traj_action(str(orders))\n # Clean orders\n for order in orders:\n if not isinstance(order, str):\n raise AgentCompletionError(\n f\"Order is not a str\\n\\Response: {response}\"\n )\n # Enforce no messages in no_press\n if params.game.no_press:\n completion[\"messages\"] = {}\n # Turn recipients in messages into ALLCAPS for the engine\n messages = {}\n for recipient, message in completion[\"messages\"].items():\n if isinstance(message, list):\n # Handle weird model outputs\n message = \" \".join(message)\n if not isinstance(message, str):\n # Force each message into a string\n message = str(message)\n if not message:\n # Skip empty messages\n continue\n messages[recipient.upper()] = message\n except Exception as exc:\n raise AgentCompletionError(f\"Exception: {exc}\\n\\Response: {response}\")\n return AgentResponse(\n reasoning=reasoning,\n orders=orders,\n messages=messages,\n system_prompt=system_prompt,\n user_prompt=user_prompt,\n prompt_tokens=response.prompt_tokens,\n completion_tokens=response.completion_tokens,\n total_tokens=response.total_tokens,\n completion_time_sec=response.completion_time_sec,\n )\n def update_traj_action(self,content):\n if len(self.backend.trajectory)>=1:\n self.backend.trajectory[-1]['content']=content\nclass LLMAgentNew(Agent):\n \"\"\"Uses OpenAI/Claude Chat/Completion to generate orders and messages.\"\"\"\n\n def __init__(self, model,logger=None,role=None, **kwargs):\n # Chat models can't specify the start of the completion\n self.use_completion_preface = False\n self.llm_mode=model\n self.backend = OpenAIChatBackend(model=model,logger=logger,role=role)\n\n def __repr__(self) -> str:\n return f\"LLMAgent(Backend: {self.llm_mode.model_name}, Temperature: {self.llm_mode.temperature})\"\n\n def respond(self, params: AgentParams) -> AgentResponse:\n \"\"\"Prompt the model for a response.\"\"\"\n system_prompt = prompts.get_system_prompt(params)\n user_prompt = prompts.get_user_prompt(params)\n response = None\n try:\n if self.use_completion_preface:\n preface_prompt = prompts.get_preface_prompt(params)\n response: BackendResponse = self.backend.complete(\n system_prompt,\n user_prompt,\n completion_preface=preface_prompt\n )\n json_completion = preface_prompt + response.completion\n else:\n response: BackendResponse = self.backend.complete(\n system_prompt,\n user_prompt\n )\n json_completion = response.completion\n # Remove repeated **system** from parroty completion models\n json_completion = json_completion.split(\"**\")[0].strip(\" `\\n\")\n\n # Claude likes to add junk around the actual JSON object, so find it manually\n start = json_completion.index(\"{\")\n end = json_completion.rindex(\"}\") + 1 # +1 to include the } in the slice\n json_completion = json_completion[start:end]\n\n # Load the JSON\n completion = json.loads(json_completion, strict=False)\n\n # Extract data from completion\n reasoning = (\n completion[\"reasoning\"]\n if \"reasoning\" in completion\n else \"*model outputted no reasoning*\"\n )\n orders = completion[\"orders\"]\n self.update_traj_action(str(orders))\n # Clean orders\n for order in orders:\n if not isinstance(order, str):\n raise AgentCompletionError(\n f\"Order is not a str\\n\\Response: {response}\"\n )\n # Enforce no messages in no_press\n if params.game.no_press:\n completion[\"messages\"] = {}\n # Turn recipients in messages into ALLCAPS for the engine\n messages = {}\n for recipient, message in completion[\"messages\"].items():\n if isinstance(message, list):\n # Handle weird model outputs\n message = \" \".join(message)\n if not isinstance(message, str):\n # Force each message into a string\n message = str(message)\n if not message:\n # Skip empty messages\n continue\n messages[recipient.upper()] = message\n except Exception as exc:\n raise AgentCompletionError(f\"Exception: {exc}\\n\\Response: {response}\")\n return AgentResponse(\n reasoning=reasoning,\n orders=orders,\n messages=messages,\n system_prompt=system_prompt,\n user_prompt=user_prompt,\n prompt_tokens=response.prompt_tokens,\n completion_tokens=response.completion_tokens,\n total_tokens=response.total_tokens,\n completion_time_sec=response.completion_time_sec,\n )\n def update_traj_action(self,content):\n if len(self.backend.trajectory)>=1:\n self.backend.trajectory[-1]['content']=content\n\n\n\nclass NoPressAgent(Agent):\n \"\"\"Follows a no-press policy from baselines.\n\n Args:\n policy_key: int to select a policy from no_press_policies.policy_map.\"\"\"\n\n def __init__(self, policy_key: int):\n self.policy_key = policy_key\n self.policy = no_press_policies.policy_map[policy_key]()\n\n def __repr__(self) -> str:\n return f\"NoPressAgent(key={self.policy_key})\"\n\n def respond(self, params: AgentParams) -> AgentResponse:\n power_slot = [sorted(params.game.map.powers).index(params.power.name)]\n state = diplomacy_state.WelfareDiplomacyState(params.game)\n observation = state.observation()\n legal_actions = state.legal_actions()\n\n self.policy.reset()\n actions = self.policy.actions(power_slot, observation, legal_actions)[0][\n 0\n ] # policy.actions returns a tuple: a list of lists of actions for each slot, and info about the step\n\n # Convert actions to MILA orders.\n orders = []\n for action in actions:\n candidate_orders = mila_actions.action_to_mila_actions(action)\n order = mila_actions.resolve_mila_orders(candidate_orders, params.game)\n orders.append(order)\n assert len(actions) == len(\n orders\n ), f\"Mapping from DM actions {actions} to MILA orders {orders} wasn't 1-1.\"\n\n return AgentResponse(\n reasoning=\"Orders from no-press baseline policy.\",\n orders=orders,\n messages={},\n system_prompt=\"\",\n user_prompt=\"\",\n prompt_tokens=0,\n completion_tokens=0,\n total_tokens=0,\n completion_time_sec=0.0,\n )\n\n\nclass ExploiterAgent(Agent):\n \"\"\"Initially uses OpenAI/Claude Chat/Completion to generate orders and messages.\n\n Once there are fewer than a certain number of enemy units, switches to zero-sum RL policy, and switches back to LLMAgent policy once enough supply centers are acquired.\n\n Kwargs:\n unit_threshold: int, number of enemy units on board below which the agent switches to RL policy.\n center_threshold: int, number of centers below which the agent switches back to LLMAgent policy.\n power: str, name of power to use as exploiter.\n\n and remaining kwargs for API model (temperature, top_p and manual_orders_path)\n \"\"\"\n\n def __init__(self, llm_model, **kwargs):\n \"\"\"Instantiate LLMAgent and RL policies.\"\"\"\n self.center_threshold = kwargs.pop(\"center_threshold\")\n self.unit_threshold = kwargs.pop(\"unit_threshold\")\n self.max_years = kwargs.pop(\"max_years\")\n self.final_year = 1901 + self.max_years - 1\n self.latest_year_to_switch = 1901 + int(self.max_years / 3)\n # Set exploiter policy\n self.rl_policy = no_press_policies.get_network_policy_instance()\n # Set intial \"cooperative\" policy\n self.llm_policy = LLMAgent(llm_model, **kwargs)\n self.exploiting = False\n self.triggered = False\n self.start_disbanding = False\n\n def __repr__(self) -> str:\n if self.exploiting:\n return f\"ExploiterAgent playing RL policy, {self.rl_policy}\"\n else:\n return f\"ExploiterAgent 'playing nice' with {self.llm_policy}\"\n\n def respond(self, params: AgentParams) -> AgentResponse:\n year = int(params.game.phase.split()[1])\n\n # Count number of enemy units\n enemy_units = sum(\n len(params.game.get_units(power))\n for power in params.game.map.powers\n if power != params.power.name\n )\n\n # Count number of centers\n centers = len(params.power.centers)\n\n # Switch to RL policy if few enough enemy units or end of 4th year:\n if not self.triggered and (\n enemy_units <= self.unit_threshold\n or (year == self.latest_year_to_switch and \"A\" in params.game.phase_type)\n ):\n self.triggered = True\n\n if year == self.final_year - 2 and \"A\" in params.game.phase_type:\n self.start_disbanding = True\n\n # Determine whether to play RL policy\n play_rl_policy = (\n self.triggered\n and centers <= self.center_threshold\n and not self.start_disbanding\n )\n\n if not play_rl_policy:\n self.exploiting = False\n return self.llm_policy.respond(params)\n else:\n start_time = time.time()\n state = diplomacy_state.WelfareDiplomacyState(params.game)\n self.exploiting = True\n self.rl_policy.reset()\n power_ix = Game().map.powers.index(params.power.name)\n actions, _ = self.rl_policy.actions(\n [power_ix], state.observation(), state.legal_actions()\n )\n # actions is a list of lists of actions for each slot\n actions = actions[0]\n\n # Convert actions to MILA orders.\n orders = []\n for action in actions:\n candidate_orders = mila_actions.action_to_mila_actions(action)\n order = mila_actions.resolve_mila_orders(candidate_orders, params.game)\n orders.append(order)\n assert len(actions) == len(\n orders\n ), f\"Mapping from DM actions {actions} to MILA orders {orders} wasn't 1-1.\"\n\n elapsed_time = time.time() - start_time\n return AgentResponse(\n reasoning=\"Orders from hybrid exploiter.\",\n orders=orders,\n messages={},\n system_prompt=\"\",\n user_prompt=\"\",\n prompt_tokens=0,\n completion_tokens=0,\n total_tokens=0,\n completion_time_sec=elapsed_time,\n )\n\nclass ExploiterAgentNew(Agent):\n \"\"\"Initially uses OpenAI/Claude Chat/Completion to generate orders and messages.\n\n Once there are fewer than a certain number of enemy units, switches to zero-sum RL policy, and switches back to LLMAgent policy once enough supply centers are acquired.\n\n Kwargs:\n unit_threshold: int, number of enemy units on board below which the agent switches to RL policy.\n center_threshold: int, number of centers below which the agent switches back to LLMAgent policy.\n power: str, name of power to use as exploiter.\n\n and remaining kwargs for API model (temperature, top_p and manual_orders_path)\n \"\"\"\n\n def __init__(self, llm_model,logger=None,role=None, **kwargs):\n self.logger=logger\n \"\"\"Instantiate LLMAgent and RL policies.\"\"\"\n self.center_threshold = kwargs.pop(\"center_threshold\")\n self.unit_threshold = kwargs.pop(\"unit_threshold\")\n self.max_years = kwargs.pop(\"max_years\")\n self.final_year = 1901 + self.max_years - 1\n self.latest_year_to_switch = 1901 + int(self.max_years / 3)\n # Set exploiter policy\n self.rl_policy = no_press_policies.get_network_policy_instance()\n # Set intial \"cooperative\" policy\n self.llm_policy = LLMAgentNew(llm_model,logger,role, **kwargs)\n self.exploiting = False\n self.triggered = False\n self.start_disbanding = False\n\n def __repr__(self) -> str:\n if self.exploiting:\n return f\"ExploiterAgent playing RL policy, {self.rl_policy}\"\n else:\n return f\"ExploiterAgent 'playing nice' with {self.llm_policy}\"\n\n def respond(self, params: AgentParams) -> AgentResponse:\n year = int(params.game.phase.split()[1])\n\n # Count number of enemy units\n enemy_units = sum(\n len(params.game.get_units(power))\n for power in params.game.map.powers\n if power != params.power.name\n )\n\n # Count number of centers\n centers = len(params.power.centers)\n\n # Switch to RL policy if few enough enemy units or end of 4th year:\n if not self.triggered and (\n enemy_units <= self.unit_threshold\n or (year == self.latest_year_to_switch and \"A\" in params.game.phase_type)\n ):\n self.triggered = True\n\n if year == self.final_year - 2 and \"A\" in params.game.phase_type:\n self.start_disbanding = True\n\n # Determine whether to play RL policy\n play_rl_policy = (\n self.triggered\n and centers <= self.center_threshold\n and not self.start_disbanding\n )\n\n if not play_rl_policy:\n self.exploiting = False\n return self.llm_policy.respond(params)\n else:\n start_time = time.time()\n state = diplomacy_state.WelfareDiplomacyState(params.game)\n self.exploiting = True\n self.rl_policy.reset()\n power_ix = Game().map.powers.index(params.power.name)\n actions, _ = self.rl_policy.actions(\n [power_ix], state.observation(), state.legal_actions()\n )\n # actions is a list of lists of actions for each slot\n actions = actions[0]\n\n # Convert actions to MILA orders.\n orders = []\n for action in actions:\n candidate_orders = mila_actions.action_to_mila_actions(action)\n order = mila_actions.resolve_mila_orders(candidate_orders, params.game)\n orders.append(order)\n assert len(actions) == len(\n orders\n ), f\"Mapping from DM actions {actions} to MILA orders {orders} wasn't 1-1.\"\n\n elapsed_time = time.time() - start_time\n return AgentResponse(\n reasoning=\"Orders from hybrid exploiter.\",\n orders=orders,\n messages={},\n system_prompt=\"\",\n user_prompt=\"\",\n prompt_tokens=0,\n completion_tokens=0,\n total_tokens=0,\n completion_time_sec=elapsed_time,\n )\n\ndef model_name_to_agent(model_name: str, **kwargs) -> Agent:\n \"\"\"Given a model name, return an instantiated corresponding agent.\"\"\"\n model_name = model_name.lower()\n if model_name == \"random\":\n return RandomAgent()\n elif model_name == \"manual\":\n return ManualAgent(**kwargs)\n elif model_name == \"nopress\":\n return NoPressAgent(**kwargs)\n elif model_name == \"exploiter\":\n return ExploiterAgent(**kwargs)\n elif (\n \"gpt-\" in model_name\n or \"davinci-\" in model_name\n or \"text-\" in model_name\n or \"claude\" in model_name\n or \"llama\" in model_name\n ):\n return LLMAgent(model_name, **kwargs)\n else:\n raise ValueError(f\"Unknown model name: {model_name}\")\n"}, "712d86460aa169e5b80d0f68affbbf75ea34a858": {"t": 1785758576.225356, "n": 29485, "h": "4e696e65851d9f4b", "c": "\"\"\"\nRepresents different agent prompting systems (e.g. GPT, 🤗 Transformers, random).\n\nA given agent should take in information about the game and a power, prompt\nan underlying model for a response, and return back the extracted response. \n\"\"\"\n\nfrom abc import ABC, abstractmethod\nimport json\nimport random\nimport time\nimport yaml\nfrom box import Box\nfrom games.welfare_diplomacy.diplomacy import Game\nimport wandb\n\nfrom games.welfare_diplomacy.welfare_diplomacy_baselines.environment import mila_actions, diplomacy_state\nfrom games.welfare_diplomacy.welfare_diplomacy_baselines.baselines import no_press_policies\n\nfrom agent_manager.agents.welfare_diplomacy.backends import OpenAIChatBackend\nfrom agent_manager.agents.welfare_diplomacy.data_types import (\n AgentResponse,\n AgentParams,\n BackendResponse,\n)\n\nfrom agent_manager.prompts import welfare_diplomacy_prompt as prompts\n\nclass AgentCompletionError(ValueError):\n \"\"\"Raised when an agent fails to complete a prompt.\"\"\"\n\n\nclass Agent(ABC):\n \"\"\"Base agent class.\"\"\"\n\n def __init__(self, **_):\n \"\"\"Base init to ignore unused kwargs.\"\"\"\n\n def __repr__(self) -> str:\n \"\"\"Return a string representation of the agent.\"\"\"\n raise NotImplementedError\n\n @abstractmethod\n def respond(\n self,\n params: AgentParams,\n ) -> AgentResponse:\n \"\"\"Prompt the model for a response.\"\"\"\n\n\nclass RandomAgent(Agent):\n \"\"\"Takes random actions and sends 1 random message.\"\"\"\n\n def __repr__(self) -> str:\n return \"RandomAgent()\"\n\n def respond(self, params: AgentParams) -> AgentResponse:\n \"\"\"Randomly generate orders and messages.\"\"\"\n # For each power, randomly sampling a valid order\n power_orders = []\n for loc in params.game.get_orderable_locations(params.power.name):\n if params.possible_orders[loc]:\n # Sort for determinism\n params.possible_orders[loc].sort()\n\n # If this is a disbandable unit in an adjustment phase in welfare,\n # then randomly choose whether to disband or not\n if (\n \"ADJUSTMENTS\" in str(params.game.phase)\n and \" D\" in params.possible_orders[loc][0][-2:]\n and params.game.welfare\n ):\n power_orders.append(\n random.choice([\"WAIVE\", params.possible_orders[loc][0]])\n )\n else:\n power_orders.append(random.choice(params.possible_orders[loc]))\n # # Testing: Randomly add an invalid order\n # if random.random() < 0.1:\n # power_orders.append(\"Random invalid order\")\n\n # # Testing: More randomly raise a completion error\n # if random.random() < 0.05:\n # raise AgentCompletionError(\"Randomly raised completion error\")\n\n # For debugging prompting\n system_prompt = prompts.get_system_prompt(params)\n user_prompt = prompts.get_user_prompt(params)\n\n # Randomly sending a message to another power\n other_powers = [p for p in params.game.powers if p != params.power.name] + [\n \"GLOBAL\"\n ]\n recipient = random.choice(other_powers)\n message = f\"Hello {recipient}! I'm {params.power.name} contacting you on turn {params.game.get_current_phase()}. Here's a random number: {random.randint(0, 100)}.\"\n\n # Sleep to allow wandb to catch up\n sleep_time = (\n 0.0\n if isinstance(wandb.run.mode, wandb.sdk.lib.disabled.RunDisabled)\n else 0.3\n )\n time.sleep(sleep_time)\n\n return AgentResponse(\n reasoning=\"Randomly generated orders and messages.\",\n orders=power_orders,\n messages={recipient: message} if not params.game.no_press else {},\n system_prompt=system_prompt,\n user_prompt=user_prompt,\n prompt_tokens=0,\n completion_tokens=0,\n total_tokens=0,\n completion_time_sec=sleep_time,\n )\n\n\nclass ManualAgent(Agent):\n \"\"\"Manually specify orders with --manual_orders_path.\"\"\"\n\n def __init__(self, manual_orders_path: str, **_):\n \"\"\"Load the manual orders.\"\"\"\n assert manual_orders_path\n self.manual_orders_path = manual_orders_path\n with open(manual_orders_path, \"r\") as file:\n self.manual_orders: dict[str, list[str]] = yaml.safe_load(file)\n # Validate the file\n for phase, orders_list in self.manual_orders.items():\n assert isinstance(phase, str)\n assert len(phase) == 6 # e.g. \"F1905M\"\n assert phase[0] in \"SFW\"\n assert phase[1:5].isdigit()\n assert phase[-1] in \"MRA\"\n assert isinstance(orders_list, list)\n for order in orders_list:\n assert isinstance(order, str)\n assert len(order.split()) >= 3\n assert order.split()[0] in \"AF\"\n\n def __repr__(self) -> str:\n return f\"ManualAgent({self.manual_orders_path})\"\n\n def respond(self, params: AgentParams) -> AgentResponse:\n \"\"\"Submit the specified orders if available.\"\"\"\n power_orders = []\n\n current_phase = params.game.get_current_phase()\n if current_phase in self.manual_orders:\n orders_list = self.manual_orders[current_phase]\n for order in orders_list:\n unit_type = order.split()[0]\n loc = order.split()[1]\n if unit_type + \" \" + loc in params.power.units:\n assert (\n order in params.possible_orders[loc]\n ), f\"Manual order {order} not in possible orders for power {params.power.name}:\\n{params.possible_orders[loc]}\"\n power_orders.append(order)\n\n # For debugging prompting\n system_prompt = prompts.get_system_prompt(params)\n user_prompt = prompts.get_user_prompt(params)\n\n # Sleep to allow wandb to catch up\n sleep_time = (\n 0.0\n if isinstance(wandb.run.mode, wandb.sdk.lib.disabled.RunDisabled)\n else 0.3\n )\n time.sleep(sleep_time)\n\n return AgentResponse(\n reasoning=\"Manually specified orders and messages.\",\n orders=power_orders,\n messages={},\n system_prompt=system_prompt,\n user_prompt=user_prompt,\n prompt_tokens=0,\n completion_tokens=0,\n total_tokens=0,\n completion_time_sec=sleep_time,\n )\n\n\nclass LLMAgent2(Agent):\n \"\"\"Uses OpenAI/Claude Chat/Completion to generate orders and messages.\"\"\"\n\n def __init__(self, model_name: str, **kwargs):\n # Decide whether it's a chat or completion model\n disable_completion_preface = kwargs.pop(\"disable_completion_preface\", False)\n self.use_completion_preface = not disable_completion_preface\n args={\n \"model_name\":\"deepseek-chat\",\n \"api_key\":\"sk-e851421cc0224deb8fad0c8a30d5c862\",\n \"api_url\":\"https://api.deepseek.com\",\n \"max_tokens\":4096,\n \"timeout\":60,\n \"temperature\":0.2\n }\n from agent_manager.llm_models.llm_model import LLMModel\n from openai import OpenAI\n llm_model=LLMModel(args)\n self.backend=OpenAI(api_key=llm_model.api_key,base_url=llm_model.api_url)\n\n def __repr__(self) -> str:\n return f\"LLMAgent(Backend: {self.backend.model_name}, Temperature: {self.temperature}, Top P: {self.top_p})\"\n\n def respond(self, params: AgentParams) -> AgentResponse:\n \"\"\"Prompt the model for a response.\"\"\"\n system_prompt = prompts.get_system_prompt(params)\n user_prompt = prompts.get_user_prompt(params)\n response = None\n try:\n if self.use_completion_preface:\n preface_prompt = prompts.get_preface_prompt(params)\n response: BackendResponse = self.backend.complete(\n system_prompt,\n user_prompt,\n completion_preface=preface_prompt,\n temperature=self.temperature,\n top_p=self.top_p,\n )\n json_completion = preface_prompt + response.completion\n else:\n response: BackendResponse = self.backend.complete(\n system_prompt,\n user_prompt,\n temperature=self.temperature,\n top_p=self.top_p,\n )\n json_completion = response.completion\n # Remove repeated **system** from parroty completion models\n json_completion = json_completion.split(\"**\")[0].strip(\" `\\n\")\n\n # Claude likes to add junk around the actual JSON object, so find it manually\n start = json_completion.index(\"{\")\n end = json_completion.rindex(\"}\") + 1 # +1 to include the } in the slice\n json_completion = json_completion[start:end]\n\n # Load the JSON\n completion = json.loads(json_completion, strict=False)\n\n # Extract data from completion\n reasoning = (\n completion[\"reasoning\"]\n if \"reasoning\" in completion\n else \"*model outputted no reasoning*\"\n )\n orders = completion[\"orders\"]\n # Clean orders\n for order in orders:\n if not isinstance(order, str):\n raise AgentCompletionError(\n f\"Order is not a str\\n\\Response: {response}\"\n )\n # Enforce no messages in no_press\n if params.game.no_press:\n completion[\"messages\"] = {}\n # Turn recipients in messages into ALLCAPS for the engine\n messages = {}\n for recipient, message in completion[\"messages\"].items():\n if isinstance(message, list):\n # Handle weird model outputs\n message = \" \".join(message)\n if not isinstance(message, str):\n # Force each message into a string\n message = str(message)\n if not message:\n # Skip empty messages\n continue\n messages[recipient.upper()] = message\n except Exception as exc:\n raise AgentCompletionError(f\"Exception: {exc}\\n\\Response: {response}\")\n return AgentResponse(\n reasoning=reasoning,\n orders=orders,\n messages=messages,\n system_prompt=system_prompt,\n user_prompt=user_prompt,\n prompt_tokens=response.prompt_tokens,\n completion_tokens=response.completion_tokens,\n total_tokens=response.total_tokens,\n completion_time_sec=response.completion_time_sec,\n )\nclass LLMAgent(Agent):\n \"\"\"Uses OpenAI/Claude Chat/Completion to generate orders and messages.\"\"\"\n\n def __init__(self, model,logger=None, role=None,**kwargs):\n self.logger=logger\n self.llm_model=model\n # Chat models can't specify the start of the completion\n self.use_completion_preface = False\n self.backend = OpenAIChatBackend(model,logger,role)\n\n def __repr__(self) -> str:\n return f\"LLMAgent(Backend: {self.llm_model.model_name}, Temperature: {self.llm_model.temperature})\"\n\n def respond(self, params: AgentParams) -> AgentResponse:\n \"\"\"Prompt the model for a response.\"\"\"\n system_prompt = prompts.get_system_prompt(params)\n user_prompt = prompts.get_user_prompt(params)\n response = None\n try:\n if self.use_completion_preface:\n preface_prompt = prompts.get_preface_prompt(params)\n response: BackendResponse = self.backend.complete(\n system_prompt,\n user_prompt,\n completion_preface=preface_prompt\n )\n json_completion = preface_prompt + response.completion\n else:\n response: BackendResponse = self.backend.complete(\n system_prompt,\n user_prompt\n )\n json_completion = response.completion\n # Remove repeated **system** from parroty completion models\n json_completion = json_completion.split(\"**\")[0].strip(\" `\\n\")\n\n # Claude likes to add junk around the actual JSON object, so find it manually\n start = json_completion.index(\"{\")\n end = json_completion.rindex(\"}\") + 1 # +1 to include the } in the slice\n json_completion = json_completion[start:end]\n\n # Load the JSON\n completion = json.loads(json_completion, strict=False)\n\n # Extract data from completion\n reasoning = (\n completion[\"reasoning\"]\n if \"reasoning\" in completion\n else \"*model outputted no reasoning*\"\n )\n orders = completion[\"orders\"]\n self.update_traj_action(str(orders))\n # Clean orders\n for order in orders:\n if not isinstance(order, str):\n raise AgentCompletionError(\n f\"Order is not a str\\n\\Response: {response}\"\n )\n # Enforce no messages in no_press\n if params.game.no_press:\n completion[\"messages\"] = {}\n # Turn recipients in messages into ALLCAPS for the engine\n messages = {}\n for recipient, message in completion[\"messages\"].items():\n if isinstance(message, list):\n # Handle weird model outputs\n message = \" \".join(message)\n if not isinstance(message, str):\n # Force each message into a string\n message = str(message)\n if not message:\n # Skip empty messages\n continue\n messages[recipient.upper()] = message\n except Exception as exc:\n raise AgentCompletionError(f\"Exception: {exc}\\n\\Response: {response}\")\n return AgentResponse(\n reasoning=reasoning,\n orders=orders,\n messages=messages,\n system_prompt=system_prompt,\n user_prompt=user_prompt,\n prompt_tokens=response.prompt_tokens,\n completion_tokens=response.completion_tokens,\n total_tokens=response.total_tokens,\n completion_time_sec=response.completion_time_sec,\n )\n def update_traj_action(self,content):\n if len(self.backend.trajectory)>=1:\n self.backend.trajectory[-1]['content']=content\nclass LLMAgentNew(Agent):\n \"\"\"Uses OpenAI/Claude Chat/Completion to generate orders and messages.\"\"\"\n\n def __init__(self, model,logger=None,role=None, **kwargs):\n # Chat models can't specify the start of the completion\n self.use_completion_preface = False\n self.llm_mode=model\n self.backend = OpenAIChatBackend(model=model,logger=logger,role=role)\n\n def __repr__(self) -> str:\n return f\"LLMAgent(Backend: {self.llm_mode.model_name}, Temperature: {self.llm_mode.temperature})\"\n\n def respond(self, params: AgentParams) -> AgentResponse:\n \"\"\"Prompt the model for a response.\"\"\"\n system_prompt = prompts.get_system_prompt(params)\n user_prompt = prompts.get_user_prompt(params)\n response = None\n try:\n if self.use_completion_preface:\n preface_prompt = prompts.get_preface_prompt(params)\n response: BackendResponse = self.backend.complete(\n system_prompt,\n user_prompt,\n completion_preface=preface_prompt\n )\n json_completion = preface_prompt + response.completion\n else:\n response: BackendResponse = self.backend.complete(\n system_prompt,\n user_prompt\n )\n json_completion = response.completion\n # Remove repeated **system** from parroty completion models\n json_completion = json_completion.split(\"**\")[0].strip(\" `\\n\")\n\n # Claude likes to add junk around the actual JSON object, so find it manually\n start = json_completion.index(\"{\")\n end = json_completion.rindex(\"}\") + 1 # +1 to include the } in the slice\n json_completion = json_completion[start:end]\n\n # Load the JSON\n completion = json.loads(json_completion, strict=False)\n\n # Extract data from completion\n reasoning = (\n completion[\"reasoning\"]\n if \"reasoning\" in completion\n else \"*model outputted no reasoning*\"\n )\n orders = completion[\"orders\"]\n self.update_traj_action(str(orders))\n # Clean orders\n for order in orders:\n if not isinstance(order, str):\n raise AgentCompletionError(\n f\"Order is not a str\\n\\Response: {response}\"\n )\n # Enforce no messages in no_press\n if params.game.no_press:\n completion[\"messages\"] = {}\n # Turn recipients in messages into ALLCAPS for the engine\n messages = {}\n for recipient, message in completion[\"messages\"].items():\n if isinstance(message, list):\n # Handle weird model outputs\n message = \" \".join(message)\n if not isinstance(message, str):\n # Force each message into a string\n message = str(message)\n if not message:\n # Skip empty messages\n continue\n messages[recipient.upper()] = message\n except Exception as exc:\n raise AgentCompletionError(f\"Exception: {exc}\\n\\Response: {response}\")\n return AgentResponse(\n reasoning=reasoning,\n orders=orders,\n messages=messages,\n system_prompt=system_prompt,\n user_prompt=user_prompt,\n prompt_tokens=response.prompt_tokens,\n completion_tokens=response.completion_tokens,\n total_tokens=response.total_tokens,\n completion_time_sec=response.completion_time_sec,\n )\n def update_traj_action(self,content):\n if len(self.backend.trajectory)>=1:\n self.backend.trajectory[-1]['content']=content\n\n\n\nclass NoPressAgent(Agent):\n \"\"\"Follows a no-press policy from baselines.\n\n Args:\n policy_key: int to select a policy from no_press_policies.policy_map.\"\"\"\n\n def __init__(self, policy_key: int):\n self.policy_key = policy_key\n self.policy = no_press_policies.policy_map[policy_key]()\n\n def __repr__(self) -> str:\n return f\"NoPressAgent(key={self.policy_key})\"\n\n def respond(self, params: AgentParams) -> AgentResponse:\n power_slot = [sorted(params.game.map.powers).index(params.power.name)]\n state = diplomacy_state.WelfareDiplomacyState(params.game)\n observation = state.observation()\n legal_actions = state.legal_actions()\n\n self.policy.reset()\n actions = self.policy.actions(power_slot, observation, legal_actions)[0][\n 0\n ] # policy.actions returns a tuple: a list of lists of actions for each slot, and info about the step\n\n # Convert actions to MILA orders.\n orders = []\n for action in actions:\n candidate_orders = mila_actions.action_to_mila_actions(action)\n order = mila_actions.resolve_mila_orders(candidate_orders, params.game)\n orders.append(order)\n assert len(actions) == len(\n orders\n ), f\"Mapping from DM actions {actions} to MILA orders {orders} wasn't 1-1.\"\n\n return AgentResponse(\n reasoning=\"Orders from no-press baseline policy.\",\n orders=orders,\n messages={},\n system_prompt=\"\",\n user_prompt=\"\",\n prompt_tokens=0,\n completion_tokens=0,\n total_tokens=0,\n completion_time_sec=0.0,\n )\n\n\nclass ExploiterAgent(Agent):\n \"\"\"Initially uses OpenAI/Claude Chat/Completion to generate orders and messages.\n\n Once there are fewer than a certain number of enemy units, switches to zero-sum RL policy, and switches back to LLMAgent policy once enough supply centers are acquired.\n\n Kwargs:\n unit_threshold: int, number of enemy units on board below which the agent switches to RL policy.\n center_threshold: int, number of centers below which the agent switches back to LLMAgent policy.\n power: str, name of power to use as exploiter.\n\n and remaining kwargs for API model (temperature, top_p and manual_orders_path)\n \"\"\"\n\n def __init__(self, llm_model, **kwargs):\n \"\"\"Instantiate LLMAgent and RL policies.\"\"\"\n self.center_threshold = kwargs.pop(\"center_threshold\")\n self.unit_threshold = kwargs.pop(\"unit_threshold\")\n self.max_years = kwargs.pop(\"max_years\")\n self.final_year = 1901 + self.max_years - 1\n self.latest_year_to_switch = 1901 + int(self.max_years / 3)\n # Set exploiter policy\n self.rl_policy = no_press_policies.get_network_policy_instance()\n # Set intial \"cooperative\" policy\n self.llm_policy = LLMAgent(llm_model, **kwargs)\n self.exploiting = False\n self.triggered = False\n self.start_disbanding = False\n\n def __repr__(self) -> str:\n if self.exploiting:\n return f\"ExploiterAgent playing RL policy, {self.rl_policy}\"\n else:\n return f\"ExploiterAgent 'playing nice' with {self.llm_policy}\"\n\n def respond(self, params: AgentParams) -> AgentResponse:\n year = int(params.game.phase.split()[1])\n\n # Count number of enemy units\n enemy_units = sum(\n len(params.game.get_units(power))\n for power in params.game.map.powers\n if power != params.power.name\n )\n\n # Count number of centers\n centers = len(params.power.centers)\n\n # Switch to RL policy if few enough enemy units or end of 4th year:\n if not self.triggered and (\n enemy_units <= self.unit_threshold\n or (year == self.latest_year_to_switch and \"A\" in params.game.phase_type)\n ):\n self.triggered = True\n\n if year == self.final_year - 2 and \"A\" in params.game.phase_type:\n self.start_disbanding = True\n\n # Determine whether to play RL policy\n play_rl_policy = (\n self.triggered\n and centers <= self.center_threshold\n and not self.start_disbanding\n )\n\n if not play_rl_policy:\n self.exploiting = False\n return self.llm_policy.respond(params)\n else:\n start_time = time.time()\n state = diplomacy_state.WelfareDiplomacyState(params.game)\n self.exploiting = True\n self.rl_policy.reset()\n power_ix = Game().map.powers.index(params.power.name)\n actions, _ = self.rl_policy.actions(\n [power_ix], state.observation(), state.legal_actions()\n )\n # actions is a list of lists of actions for each slot\n actions = actions[0]\n\n # Convert actions to MILA orders.\n orders = []\n for action in actions:\n candidate_orders = mila_actions.action_to_mila_actions(action)\n order = mila_actions.resolve_mila_orders(candidate_orders, params.game)\n orders.append(order)\n assert len(actions) == len(\n orders\n ), f\"Mapping from DM actions {actions} to MILA orders {orders} wasn't 1-1.\"\n\n elapsed_time = time.time() - start_time\n return AgentResponse(\n reasoning=\"Orders from hybrid exploiter.\",\n orders=orders,\n messages={},\n system_prompt=\"\",\n user_prompt=\"\",\n prompt_tokens=0,\n completion_tokens=0,\n total_tokens=0,\n completion_time_sec=elapsed_time,\n )\n\nclass ExploiterAgentNew(Agent):\n \"\"\"Initially uses OpenAI/Claude Chat/Completion to generate orders and messages.\n\n Once there are fewer than a certain number of enemy units, switches to zero-sum RL policy, and switches back to LLMAgent policy once enough supply centers are acquired.\n\n Kwargs:\n unit_threshold: int, number of enemy units on board below which the agent switches to RL policy.\n center_threshold: int, number of centers below which the agent switches back to LLMAgent policy.\n power: str, name of power to use as exploiter.\n\n and remaining kwargs for API model (temperature, top_p and manual_orders_path)\n \"\"\"\n\n def __init__(self, llm_model,logger=None,role=None, **kwargs):\n self.logger=logger\n \"\"\"Instantiate LLMAgent and RL policies.\"\"\"\n self.center_threshold = kwargs.pop(\"center_threshold\")\n self.unit_threshold = kwargs.pop(\"unit_threshold\")\n self.max_years = kwargs.pop(\"max_years\")\n self.final_year = 1901 + self.max_years - 1\n self.latest_year_to_switch = 1901 + int(self.max_years / 3)\n # Set exploiter policy\n self.rl_policy = no_press_policies.get_network_policy_instance()\n # Set intial \"cooperative\" policy\n self.llm_policy = LLMAgentNew(llm_model,logger,role, **kwargs)\n self.exploiting = False\n self.triggered = False\n self.start_disbanding = False\n\n def __repr__(self) -> str:\n if self.exploiting:\n return f\"ExploiterAgent playing RL policy, {self.rl_policy}\"\n else:\n return f\"ExploiterAgent 'playing nice' with {self.llm_policy}\"\n\n def respond(self, params: AgentParams) -> AgentResponse:\n year = int(params.game.phase.split()[1])\n\n # Count number of enemy units\n enemy_units = sum(\n len(params.game.get_units(power))\n for power in params.game.map.powers\n if power != params.power.name\n )\n\n # Count number of centers\n centers = len(params.power.centers)\n\n # Switch to RL policy if few enough enemy units or end of 4th year:\n if not self.triggered and (\n enemy_units <= self.unit_threshold\n or (year == self.latest_year_to_switch and \"A\" in params.game.phase_type)\n ):\n self.triggered = True\n\n if year == self.final_year - 2 and \"A\" in params.game.phase_type:\n self.start_disbanding = True\n\n # Determine whether to play RL policy\n play_rl_policy = (\n self.triggered\n and centers <= self.center_threshold\n and not self.start_disbanding\n )\n\n if not play_rl_policy:\n self.exploiting = False\n return self.llm_policy.respond(params)\n else:\n start_time = time.time()\n state = diplomacy_state.WelfareDiplomacyState(params.game)\n self.exploiting = True\n self.rl_policy.reset()\n power_ix = Game().map.powers.index(params.power.name)\n actions, _ = self.rl_policy.actions(\n [power_ix], state.observation(), state.legal_actions()\n )\n # actions is a list of lists of actions for each slot\n actions = actions[0]\n\n # Convert actions to MILA orders.\n orders = []\n for action in actions:\n candidate_orders = mila_actions.action_to_mila_actions(action)\n order = mila_actions.resolve_mila_orders(candidate_orders, params.game)\n orders.append(order)\n assert len(actions) == len(\n orders\n ), f\"Mapping from DM actions {actions} to MILA orders {orders} wasn't 1-1.\"\n\n elapsed_time = time.time() - start_time\n return AgentResponse(\n reasoning=\"Orders from hybrid exploiter.\",\n orders=orders,\n messages={},\n system_prompt=\"\",\n user_prompt=\"\",\n prompt_tokens=0,\n completion_tokens=0,\n total_tokens=0,\n completion_time_sec=elapsed_time,\n )\n\ndef model_name_to_agent(model_name: str, **kwargs) -> Agent:\n \"\"\"Given a model name, return an instantiated corresponding agent.\"\"\"\n model_name = model_name.lower()\n if model_name == \"random\":\n return RandomAgent()\n elif model_name == \"manual\":\n return ManualAgent(**kwargs)\n elif model_name == \"nopress\":\n return NoPressAgent(**kwargs)\n elif model_name == \"exploiter\":\n return ExploiterAgent(**kwargs)\n elif (\n \"gpt-\" in model_name\n or \"davinci-\" in model_name\n or \"text-\" in model_name\n or \"claude\" in model_name\n or \"llama\" in model_name\n ):\n return LLMAgent(model_name, **kwargs)\n else:\n raise ValueError(f\"Unknown model name: {model_name}\")\n"}}}} |