1 line
31 KiB
JSON
1 line
31 KiB
JSON
{"repo": "bambuuai/affine-rl", "files": {"scripts/PRINT_API_evaluation.ipynb": {"8cd51018c1a3e73a0b4bb126a427628f1f0318fb": {"t": 1785745497.762252, "n": 4203, "h": "b15210d016ecf869", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import sys\\n\",\n \"from pathlib import Path\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"\\n\",\n \"PROJECT_ROOT = Path(\\\"../\\\")\\n\",\n \"os.chdir(PROJECT_ROOT)\\n\",\n \"\\n\",\n \"load_dotenv() \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"sys.path.insert(0, 'AffineFoundation/affinetes/environments/trace')\\n\",\n \"sys.path.insert(0, 'AffineFoundation/affinetes/affinetes/templates')\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from env import Actor\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"actor = Actor()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"openai_api_key = os.getenv(\\\"OPENAI_API_KEY\\\")\\n\",\n \"deepseek_api_key = os.getenv(\\\"DEEPSEEK_API_KEY\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"api_url=\\\"https://api.deepseek.com/chat/completions\\\"\\n\",\n \"model=\\\"deepseek-reasoner\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"task_ids = [105, 177, 241, 418, 525, 569, 614, 785, 826, 1213, 1328, 1359, 1816, 1946, 2066, 2354, 2446, 2637, 2659, 2738, 2754, 2826, 2868, 2968, 3037, 3042, 3148, 3174, 3239, 3327, 3336, 3363, 3402, 3499, 3579, 3791, 4049, 4063, 4212, 4235, 4260, 4426, 4441, 4505, 4538, 4561, 4633, 5281, 5456, 5548, 5674, 5708, 5885, 6072, 6115, 6124, 6230, 6546, 6683, 6761, 6784, 6884, 7083, 7205, 7376, 7402, 7428, 7489, 7547, 7596, 7658, 7680, 7762, 7827, 7890, 8030, 8155, 8315, 8856, 8923, 9004, 9060, 9061, 9117, 9234, 9514, 9588, 9637, 9685, 9880, 9973, 10092, 10138, 10176, 10278, 10344, 10560, 10604, 10802, 11340, 11397, 11404, 11654, 11674, 11736, 11996, 12232, 12313, 12338, 12339, 12457, 12514, 12536, 12657, 12768, 13042, 13067, 13259, 13278, 13342, 13920, 13944, 14336, 14378, 14384, 14524, 14580, 14642, 14651, 14675, 14712, 14801, 15109, 15132, 15223, 15336, 15683, 15750, 15762, 15857, 15898, 15969, 15987, 16098, 16232, 16312, 16325, 16764, 16874, 17003, 17237, 17336, 17450, 17763, 17987, 18284, 18320, 18393, 18410, 18416, 18552, 18776, 19364, 19412, 19500, 19721, 19893, 20035, 20259, 20292, 20405, 20468, 20553, 20631, 20698, 20738, 20752, 20762, 20867, 20904, 20995, 21003, 21160, 21281, 21380, 21498, 21640, 21713, 21771, 21821, 21844, 22074, 22425, 22618, 22659, 22716, 22857, 22944, 22963, 23132]\\n\",\n \"\\n\",\n \"results = []\\n\",\n \"total_score = 0\\n\",\n \"for task_id in task_ids[:1]:\\n\",\n \" result = await actor.evaluate(\\n\",\n \" model=model,\\n\",\n \" base_url=api_url,\\n\",\n \" timeout=600,\\n\",\n \" temperature=None,\\n\",\n \" api_key=deepseek_api_key,\\n\",\n \" seed = None,\\n\",\n \" task_id = None\\n\",\n \" )\\n\",\n \"\\n\",\n \" results.append({\\\"Score\\\": result['score'], \\\"conversation\\\": result[\\\"extra\\\"]['conversation'],\\\"test_result\\\": result[\\\"extra\\\"]['test_result'],\\\"dataset_index\\\": result[\\\"extra\\\"]['dataset_index'], \\\"answer\\\": result['answer']})\\n\",\n \" total_score += result['score']\\n\",\n \" print(result['extra']['conversation'][1])\\n\",\n \" print(\\\"------------\\\")\\n\",\n \" print(result[\\\"answer\\\"])\\n\",\n \" print(f\\\"Task_id: {task_id}\\\")\\n\",\n \" print(f\\\"Score: {result['score']}\\\")\\n\",\n \" print(f\\\"test_result: {total_score} / {len(results)}\\\")\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"affine\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.14\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 4\n}\n"}, "6c98da9ccdc0ae7bb695517136617c0e05223380": {"t": 1785753716.2911432, "n": 4203, "h": "b15210d016ecf869", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import sys\\n\",\n \"from pathlib import Path\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"\\n\",\n \"PROJECT_ROOT = Path(\\\"../\\\")\\n\",\n \"os.chdir(PROJECT_ROOT)\\n\",\n \"\\n\",\n \"load_dotenv() \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"sys.path.insert(0, 'AffineFoundation/affinetes/environments/trace')\\n\",\n \"sys.path.insert(0, 'AffineFoundation/affinetes/affinetes/templates')\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from env import Actor\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"actor = Actor()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"openai_api_key = os.getenv(\\\"OPENAI_API_KEY\\\")\\n\",\n \"deepseek_api_key = os.getenv(\\\"DEEPSEEK_API_KEY\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"api_url=\\\"https://api.deepseek.com/chat/completions\\\"\\n\",\n \"model=\\\"deepseek-reasoner\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"task_ids = [105, 177, 241, 418, 525, 569, 614, 785, 826, 1213, 1328, 1359, 1816, 1946, 2066, 2354, 2446, 2637, 2659, 2738, 2754, 2826, 2868, 2968, 3037, 3042, 3148, 3174, 3239, 3327, 3336, 3363, 3402, 3499, 3579, 3791, 4049, 4063, 4212, 4235, 4260, 4426, 4441, 4505, 4538, 4561, 4633, 5281, 5456, 5548, 5674, 5708, 5885, 6072, 6115, 6124, 6230, 6546, 6683, 6761, 6784, 6884, 7083, 7205, 7376, 7402, 7428, 7489, 7547, 7596, 7658, 7680, 7762, 7827, 7890, 8030, 8155, 8315, 8856, 8923, 9004, 9060, 9061, 9117, 9234, 9514, 9588, 9637, 9685, 9880, 9973, 10092, 10138, 10176, 10278, 10344, 10560, 10604, 10802, 11340, 11397, 11404, 11654, 11674, 11736, 11996, 12232, 12313, 12338, 12339, 12457, 12514, 12536, 12657, 12768, 13042, 13067, 13259, 13278, 13342, 13920, 13944, 14336, 14378, 14384, 14524, 14580, 14642, 14651, 14675, 14712, 14801, 15109, 15132, 15223, 15336, 15683, 15750, 15762, 15857, 15898, 15969, 15987, 16098, 16232, 16312, 16325, 16764, 16874, 17003, 17237, 17336, 17450, 17763, 17987, 18284, 18320, 18393, 18410, 18416, 18552, 18776, 19364, 19412, 19500, 19721, 19893, 20035, 20259, 20292, 20405, 20468, 20553, 20631, 20698, 20738, 20752, 20762, 20867, 20904, 20995, 21003, 21160, 21281, 21380, 21498, 21640, 21713, 21771, 21821, 21844, 22074, 22425, 22618, 22659, 22716, 22857, 22944, 22963, 23132]\\n\",\n \"\\n\",\n \"results = []\\n\",\n \"total_score = 0\\n\",\n \"for task_id in task_ids[:1]:\\n\",\n \" result = await actor.evaluate(\\n\",\n \" model=model,\\n\",\n \" base_url=api_url,\\n\",\n \" timeout=600,\\n\",\n \" temperature=None,\\n\",\n \" api_key=deepseek_api_key,\\n\",\n \" seed = None,\\n\",\n \" task_id = None\\n\",\n \" )\\n\",\n \"\\n\",\n \" results.append({\\\"Score\\\": result['score'], \\\"conversation\\\": result[\\\"extra\\\"]['conversation'],\\\"test_result\\\": result[\\\"extra\\\"]['test_result'],\\\"dataset_index\\\": result[\\\"extra\\\"]['dataset_index'], \\\"answer\\\": result['answer']})\\n\",\n \" total_score += result['score']\\n\",\n \" print(result['extra']['conversation'][1])\\n\",\n \" print(\\\"------------\\\")\\n\",\n \" print(result[\\\"answer\\\"])\\n\",\n \" print(f\\\"Task_id: {task_id}\\\")\\n\",\n \" print(f\\\"Score: {result['score']}\\\")\\n\",\n \" print(f\\\"test_result: {total_score} / {len(results)}\\\")\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"affine\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.14\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 4\n}\n"}, "ab962fbc5d4eaa1d9484fabce599c2e36d7c8542": {"t": 1785753717.4729986, "n": 5356, "h": "28df55181eb2fa64", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# OpenSpiel Bot Testing\\n\",\n \"\\n\",\n \"Test notebook for:\\n\",\n \"- `HumanBot` - CLI interaction\\n\",\n \"- `AlgorithmBot` - Trajectory generation for LLM training\\n\",\n \"- `play_game()` - Unified game function\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": []\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import sys\\n\",\n \"from pathlib import Path\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"\\n\",\n \"PROJECT_ROOT = Path(\\\"../\\\")\\n\",\n \"os.chdir(PROJECT_ROOT)\\n\",\n \"\\n\",\n \"load_dotenv() \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"sys.path.insert(0, 'AffineFoundation/affinetes/environments/trace')\\n\",\n \"sys.path.insert(0, 'AffineFoundation/affinetes/affinetes/templates')\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from env import Actor\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"actor = Actor()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"openai_api_key = os.getenv(\\\"OPENAI_API_KEY\\\")\\n\",\n \"deepseek_api_key = os.getenv(\\\"DEEPSEEK_API_KEY\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"api_url=\\\"https://api.deepseek.com/chat/completions\\\"\\n\",\n \"model=\\\"deepseek-reasoner\\\"\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 1. Test AlgorithmBot - Trajectory Generation\\n\",\n \"\\n\",\n \"Generate training data using MCTS algorithm\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"print(openai_api_key)\\n\",\n \"print(deepseek_api_key)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"print(model)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"task_ids = [105, 177, 241, 418, 525, 569, 614, 785, 826, 1213, 1328, 1359, 1816, 1946, 2066, 2354, 2446, 2637, 2659, 2738, 2754, 2826, 2868, 2968, 3037, 3042, 3148, 3174, 3239, 3327, 3336, 3363, 3402, 3499, 3579, 3791, 4049, 4063, 4212, 4235, 4260, 4426, 4441, 4505, 4538, 4561, 4633, 5281, 5456, 5548, 5674, 5708, 5885, 6072, 6115, 6124, 6230, 6546, 6683, 6761, 6784, 6884, 7083, 7205, 7376, 7402, 7428, 7489, 7547, 7596, 7658, 7680, 7762, 7827, 7890, 8030, 8155, 8315, 8856, 8923, 9004, 9060, 9061, 9117, 9234, 9514, 9588, 9637, 9685, 9880, 9973, 10092, 10138, 10176, 10278, 10344, 10560, 10604, 10802, 11340, 11397, 11404, 11654, 11674, 11736, 11996, 12232, 12313, 12338, 12339, 12457, 12514, 12536, 12657, 12768, 13042, 13067, 13259, 13278, 13342, 13920, 13944, 14336, 14378, 14384, 14524, 14580, 14642, 14651, 14675, 14712, 14801, 15109, 15132, 15223, 15336, 15683, 15750, 15762, 15857, 15898, 15969, 15987, 16098, 16232, 16312, 16325, 16764, 16874, 17003, 17237, 17336, 17450, 17763, 17987, 18284, 18320, 18393, 18410, 18416, 18552, 18776, 19364, 19412, 19500, 19721, 19893, 20035, 20259, 20292, 20405, 20468, 20553, 20631, 20698, 20738, 20752, 20762, 20867, 20904, 20995, 21003, 21160, 21281, 21380, 21498, 21640, 21713, 21771, 21821, 21844, 22074, 22425, 22618, 22659, 22716, 22857, 22944, 22963, 23132]\\n\",\n \"\\n\",\n \"results = []\\n\",\n \"total_score = 0\\n\",\n \"for task_id in task_ids[:5]:\\n\",\n \" result = await actor.evaluate(\\n\",\n \" model=model,\\n\",\n \" base_url=api_url,\\n\",\n \" timeout=600,\\n\",\n \" temperature=None,\\n\",\n \" api_key=deepseek_api_key,\\n\",\n \" seed = None,\\n\",\n \" task_id = None\\n\",\n \" )\\n\",\n \"\\n\",\n \" results.append({\\\"Score\\\": result['score'], \\\"conversation\\\": result[\\\"extra\\\"]['conversation'],\\\"test_result\\\": result[\\\"extra\\\"]['test_result'],\\\"dataset_index\\\": result[\\\"extra\\\"]['dataset_index'], \\\"answer\\\": result['answer']})\\n\",\n \" total_score += result['score']\\n\",\n \" print(result['extra']['conversation'][1])\\n\",\n \" print(\\\"------------\\\")\\n\",\n \" print(result[\\\"answer\\\"])\\n\",\n \" print(f\\\"Task_id: {task_id}\\\")\\n\",\n \" print(f\\\"Score: {result['score']}\\\")\\n\",\n \" print(f\\\"test_result: {total_score} / {len(results)}\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"results[0][\\\"conversation\\\"][1]\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"affine-h\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.14\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 4\n}\n"}}, "scripts/LGC_API_evaluation.ipynb": {"8cd51018c1a3e73a0b4bb126a427628f1f0318fb": {"t": 1785745509.1842139, "n": 4229, "h": "36784f88fdf8c7f5", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import sys\\n\",\n \"from pathlib import Path\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"\\n\",\n \"PROJECT_ROOT = Path(\\\"../\\\")\\n\",\n \"os.chdir(PROJECT_ROOT)\\n\",\n \"\\n\",\n \"load_dotenv() \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"sys.path.insert(0, 'AffineFoundation/affinetes/environments/primeintellect/lgc-v2')\\n\",\n \"sys.path.insert(0, 'AffineFoundation/affinetes/affinetes/templates')\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from env import Actor\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"actor = Actor()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"api_key = os.getenv(\\\"OPENAI_API_KEY\\\")\\n\",\n \"# deepseek_api_key = os.getenv(\\\"DEEPSEEK_API_KEY\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"api_url=\\\"https://api.openai.com/v1\\\"\\n\",\n \"model=\\\"o3\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"api_url=\\\"https://api.deepseek.com/chat/completions\\\"\\n\",\n \"model=\\\"deepseek-reasoner\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"task_ids = [105, 177, 241, 418, 525, 569, 614, 785, 826, 1213, 1328, 1359, 1816, 1946, 2066, 2354, 2446, 2637, 2659, 2738, 2754, 2826, 2868, 2968, 3037, 3042, 3148, 3174, 3239, 3327, 3336, 3363, 3402, 3499, 3579, 3791, 4049, 4063, 4212, 4235, 4260, 4426, 4441, 4505, 4538, 4561, 4633, 5281, 5456, 5548, 5674, 5708, 5885, 6072, 6115, 6124, 6230, 6546, 6683, 6761, 6784, 6884, 7083, 7205, 7376, 7402, 7428, 7489, 7547, 7596, 7658, 7680, 7762, 7827, 7890, 8030, 8155, 8315, 8856, 8923, 9004, 9060, 9061, 9117, 9234, 9514, 9588, 9637, 9685, 9880, 9973, 10092, 10138, 10176, 10278, 10344, 10560, 10604, 10802, 11340, 11397, 11404, 11654, 11674, 11736, 11996, 12232, 12313, 12338, 12339, 12457, 12514, 12536, 12657, 12768, 13042, 13067, 13259, 13278, 13342, 13920, 13944, 14336, 14378, 14384, 14524, 14580, 14642, 14651, 14675, 14712, 14801, 15109, 15132, 15223, 15336, 15683, 15750, 15762, 15857, 15898, 15969, 15987, 16098, 16232, 16312, 16325, 16764, 16874, 17003, 17237, 17336, 17450, 17763, 17987, 18284, 18320, 18393, 18410, 18416, 18552, 18776, 19364, 19412, 19500, 19721, 19893, 20035, 20259, 20292, 20405, 20468, 20553, 20631, 20698, 20738, 20752, 20762, 20867, 20904, 20995, 21003, 21160, 21281, 21380, 21498, 21640, 21713, 21771, 21821, 21844, 22074, 22425, 22618, 22659, 22716, 22857, 22944, 22963, 23132]\\n\",\n \"\\n\",\n \"results = []\\n\",\n \"total_score = 0\\n\",\n \"for task_id in task_ids:\\n\",\n \" result = await actor.evaluate_openai(\\n\",\n \" model=model,\\n\",\n \" base_url=api_url,\\n\",\n \" temperature=None,\\n\",\n \" api_key=api_key,\\n\",\n \" seed = None,\\n\",\n \" task_id = None\\n\",\n \" )\\n\",\n \"\\n\",\n \" # print(result)\\n\",\n \" results.append(result)\\n\",\n \" total_score += result['score']\\n\",\n \" print(result['extra']['conversation'][1])\\n\",\n \" print(\\\"------------\\\")\\n\",\n \" print(result['extra'].get('task_metadata', {}).get('full_sequence', ''))\\n\",\n \" print(f\\\"Task_id: {task_id}\\\")\\n\",\n \" print(f\\\"Score: {result['score']}\\\")\\n\",\n \" print(f\\\"test_result: {total_score} / {len(results)}\\\")\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"affine\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.14\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 4\n}\n"}, "6c98da9ccdc0ae7bb695517136617c0e05223380": {"t": 1785753715.0423577, "n": 4229, "h": "36784f88fdf8c7f5", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import sys\\n\",\n \"from pathlib import Path\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"\\n\",\n \"PROJECT_ROOT = Path(\\\"../\\\")\\n\",\n \"os.chdir(PROJECT_ROOT)\\n\",\n \"\\n\",\n \"load_dotenv() \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"sys.path.insert(0, 'AffineFoundation/affinetes/environments/primeintellect/lgc-v2')\\n\",\n \"sys.path.insert(0, 'AffineFoundation/affinetes/affinetes/templates')\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from env import Actor\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"actor = Actor()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"api_key = os.getenv(\\\"OPENAI_API_KEY\\\")\\n\",\n \"# deepseek_api_key = os.getenv(\\\"DEEPSEEK_API_KEY\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"api_url=\\\"https://api.openai.com/v1\\\"\\n\",\n \"model=\\\"o3\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"api_url=\\\"https://api.deepseek.com/chat/completions\\\"\\n\",\n \"model=\\\"deepseek-reasoner\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"task_ids = [105, 177, 241, 418, 525, 569, 614, 785, 826, 1213, 1328, 1359, 1816, 1946, 2066, 2354, 2446, 2637, 2659, 2738, 2754, 2826, 2868, 2968, 3037, 3042, 3148, 3174, 3239, 3327, 3336, 3363, 3402, 3499, 3579, 3791, 4049, 4063, 4212, 4235, 4260, 4426, 4441, 4505, 4538, 4561, 4633, 5281, 5456, 5548, 5674, 5708, 5885, 6072, 6115, 6124, 6230, 6546, 6683, 6761, 6784, 6884, 7083, 7205, 7376, 7402, 7428, 7489, 7547, 7596, 7658, 7680, 7762, 7827, 7890, 8030, 8155, 8315, 8856, 8923, 9004, 9060, 9061, 9117, 9234, 9514, 9588, 9637, 9685, 9880, 9973, 10092, 10138, 10176, 10278, 10344, 10560, 10604, 10802, 11340, 11397, 11404, 11654, 11674, 11736, 11996, 12232, 12313, 12338, 12339, 12457, 12514, 12536, 12657, 12768, 13042, 13067, 13259, 13278, 13342, 13920, 13944, 14336, 14378, 14384, 14524, 14580, 14642, 14651, 14675, 14712, 14801, 15109, 15132, 15223, 15336, 15683, 15750, 15762, 15857, 15898, 15969, 15987, 16098, 16232, 16312, 16325, 16764, 16874, 17003, 17237, 17336, 17450, 17763, 17987, 18284, 18320, 18393, 18410, 18416, 18552, 18776, 19364, 19412, 19500, 19721, 19893, 20035, 20259, 20292, 20405, 20468, 20553, 20631, 20698, 20738, 20752, 20762, 20867, 20904, 20995, 21003, 21160, 21281, 21380, 21498, 21640, 21713, 21771, 21821, 21844, 22074, 22425, 22618, 22659, 22716, 22857, 22944, 22963, 23132]\\n\",\n \"\\n\",\n \"results = []\\n\",\n \"total_score = 0\\n\",\n \"for task_id in task_ids:\\n\",\n \" result = await actor.evaluate_openai(\\n\",\n \" model=model,\\n\",\n \" base_url=api_url,\\n\",\n \" temperature=None,\\n\",\n \" api_key=api_key,\\n\",\n \" seed = None,\\n\",\n \" task_id = None\\n\",\n \" )\\n\",\n \"\\n\",\n \" # print(result)\\n\",\n \" results.append(result)\\n\",\n \" total_score += result['score']\\n\",\n \" print(result['extra']['conversation'][1])\\n\",\n \" print(\\\"------------\\\")\\n\",\n \" print(result['extra'].get('task_metadata', {}).get('full_sequence', ''))\\n\",\n \" print(f\\\"Task_id: {task_id}\\\")\\n\",\n \" print(f\\\"Score: {result['score']}\\\")\\n\",\n \" print(f\\\"test_result: {total_score} / {len(results)}\\\")\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"affine\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.14\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 4\n}\n"}, "ab962fbc5d4eaa1d9484fabce599c2e36d7c8542": {"t": 1785753716.2411635, "n": 5372, "h": "398f9ae26b62f7c4", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# OpenSpiel Bot Testing\\n\",\n \"\\n\",\n \"Test notebook for:\\n\",\n \"- `HumanBot` - CLI interaction\\n\",\n \"- `AlgorithmBot` - Trajectory generation for LLM training\\n\",\n \"- `play_game()` - Unified game function\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": []\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import sys\\n\",\n \"from pathlib import Path\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"\\n\",\n \"PROJECT_ROOT = Path(\\\"../\\\")\\n\",\n \"os.chdir(PROJECT_ROOT)\\n\",\n \"\\n\",\n \"load_dotenv() \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"sys.path.insert(0, 'AffineFoundation/affinetes/environments/primeintellect/lgc-v2')\\n\",\n \"sys.path.insert(0, 'AffineFoundation/affinetes/affinetes/templates')\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from env import Actor\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"actor = Actor()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"openai_api_key = os.getenv(\\\"OPENAI_API_KEY\\\")\\n\",\n \"deepseek_api_key = os.getenv(\\\"DEEPSEEK_API_KEY\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"api_url=\\\"https://api.deepseek.com/chat/completions\\\"\\n\",\n \"model=\\\"deepseek-reasoner\\\"\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 1. Test AlgorithmBot - Trajectory Generation\\n\",\n \"\\n\",\n \"Generate training data using MCTS algorithm\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"print(openai_api_key)\\n\",\n \"print(deepseek_api_key)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"print(model)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"task_ids = [105, 177, 241, 418, 525, 569, 614, 785, 826, 1213, 1328, 1359, 1816, 1946, 2066, 2354, 2446, 2637, 2659, 2738, 2754, 2826, 2868, 2968, 3037, 3042, 3148, 3174, 3239, 3327, 3336, 3363, 3402, 3499, 3579, 3791, 4049, 4063, 4212, 4235, 4260, 4426, 4441, 4505, 4538, 4561, 4633, 5281, 5456, 5548, 5674, 5708, 5885, 6072, 6115, 6124, 6230, 6546, 6683, 6761, 6784, 6884, 7083, 7205, 7376, 7402, 7428, 7489, 7547, 7596, 7658, 7680, 7762, 7827, 7890, 8030, 8155, 8315, 8856, 8923, 9004, 9060, 9061, 9117, 9234, 9514, 9588, 9637, 9685, 9880, 9973, 10092, 10138, 10176, 10278, 10344, 10560, 10604, 10802, 11340, 11397, 11404, 11654, 11674, 11736, 11996, 12232, 12313, 12338, 12339, 12457, 12514, 12536, 12657, 12768, 13042, 13067, 13259, 13278, 13342, 13920, 13944, 14336, 14378, 14384, 14524, 14580, 14642, 14651, 14675, 14712, 14801, 15109, 15132, 15223, 15336, 15683, 15750, 15762, 15857, 15898, 15969, 15987, 16098, 16232, 16312, 16325, 16764, 16874, 17003, 17237, 17336, 17450, 17763, 17987, 18284, 18320, 18393, 18410, 18416, 18552, 18776, 19364, 19412, 19500, 19721, 19893, 20035, 20259, 20292, 20405, 20468, 20553, 20631, 20698, 20738, 20752, 20762, 20867, 20904, 20995, 21003, 21160, 21281, 21380, 21498, 21640, 21713, 21771, 21821, 21844, 22074, 22425, 22618, 22659, 22716, 22857, 22944, 22963, 23132]\\n\",\n \"\\n\",\n \"results = []\\n\",\n \"total_score = 0\\n\",\n \"for task_id in task_ids[:5]:\\n\",\n \" result = await actor.evaluate(\\n\",\n \" model=model,\\n\",\n \" base_url=api_url,\\n\",\n \" timeout=600,\\n\",\n \" temperature=None,\\n\",\n \" api_key=deepseek_api_key,\\n\",\n \" seed = None,\\n\",\n \" task_id = None\\n\",\n \" )\\n\",\n \"\\n\",\n \" results.append({\\\"Score\\\": result['score'], \\\"conversation\\\": result[\\\"extra\\\"]['conversation'],\\\"test_result\\\": result[\\\"extra\\\"]['test_result'],\\\"dataset_index\\\": result[\\\"extra\\\"]['dataset_index'], \\\"answer\\\": result['answer']})\\n\",\n \" total_score += result['score']\\n\",\n \" print(result['extra']['conversation'][1])\\n\",\n \" print(\\\"------------\\\")\\n\",\n \" print(result[\\\"answer\\\"])\\n\",\n \" print(f\\\"Task_id: {task_id}\\\")\\n\",\n \" print(f\\\"Score: {result['score']}\\\")\\n\",\n \" print(f\\\"test_result: {total_score} / {len(results)}\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"results[0][\\\"conversation\\\"][1]\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"affine-h\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.14\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 4\n}\n"}}}} |