1 line
76 KiB
JSON
1 line
76 KiB
JSON
{"repo": "6hris/pokermind", "files": {"pokerbench.ipynb": {"9c40b31179c07230e2d95982b84feba7e2164f95": {"t": 1785745472.7216434, "n": 12003, "h": "8c9037c19497ab54", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"id\": \"173e2309-a47e-4d16-819b-7e33f67e4e52\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from datasets import load_dataset\\n\",\n \"\\n\",\n \"ds = load_dataset(\\\"RZ412/PokerBench\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"id\": \"397779d8-3c77-420e-9371-9f8d621fe817\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"{'train': (563200, 2), 'test': (11000, 2)}\"\n ]\n },\n \"execution_count\": 2,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"ds.shape\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"id\": \"67208356-9f19-4d26-85d3-32ee2033ca88\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"\\n\",\n \"load_dotenv()\\n\",\n \"api_key = os.getenv(\\\"OPENROUTER_API_KEY\\\")\\n\",\n \"ds_key = os.getenv(\\\"DEEPSEEK_API_KEY\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 4,\n \"id\": \"223fa0f2-5abc-4370-8189-1e3e18f951b7\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"test_data = ds['test']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"id\": \"5d7f4349-6b56-4764-af13-af0a6129a9ee\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import requests\\n\",\n \"def query_llm(prompt, model):\\n\",\n \" headers = {\\n\",\n \" \\\"Authorization\\\": f\\\"Bearer {api_key}\\\",\\n\",\n \" \\\"Content-Type\\\": \\\"application/json\\\"\\n\",\n \" }\\n\",\n \" payload = {\\n\",\n \" \\\"model\\\": model,\\n\",\n \" \\\"prompt\\\": prompt,\\n\",\n \" # \\\"temperature\\\": 0.1,\\n\",\n \" #\\\"top_p\\\": 0.95\\n\",\n \" }\\n\",\n \" response = requests.post(\\\"https://openrouter.ai/api/v1/completions\\\", headers=headers, json=payload)\\n\",\n \" response_data = response.json()\\n\",\n \" if \\\"choices\\\" not in response_data:\\n\",\n \" print(response_data)\\n\",\n \" return\\n\",\n \" txt = response.json()[\\\"choices\\\"][0][\\\"text\\\"]\\n\",\n \" return txt\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 68,\n \"id\": \"8c119b41-291d-4615-8271-2d9fe752ba78\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from openai import OpenAI\\n\",\n \"\\n\",\n \"client = OpenAI(api_key=api_key, base_url=\\\"https://openrouter.ai/api/v1\\\")\\n\",\n \"\\n\",\n \"def query_deepseek(prompt):\\n\",\n \" try:\\n\",\n \" response = client.chat.completions.create(\\n\",\n \" model=\\\"deepseek/deepseek-r1-distill-qwen-32b\\\",\\n\",\n \" messages=[{\\\"role\\\": \\\"user\\\", \\\"content\\\": prompt}],\\n\",\n \" #temperature=0.1, \\n\",\n \" #top_p=0.95\\n\",\n \" )\\n\",\n \" if response.choices:\\n\",\n \" return response.choices[0].message.content\\n\",\n \" else:\\n\",\n \" print(\\\"No response choices available.\\\")\\n\",\n \" return None\\n\",\n \"\\n\",\n \" except Exception as e:\\n\",\n \" print(f\\\"Error querying DeepSeek API: {e}\\\")\\n\",\n \" return None\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 7,\n \"id\": \"41760969-10df-4b14-b6fc-2aec64db7c8c\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"model_deepseek = \\\"deepseek/deepseek-r1\\\"\\n\",\n \"model_llama = \\\"meta-llama/llama-3.3-70b-instruct\\\"\\n\",\n \"model_gemini = \\\"google/gemini-2.0-flash-lite-preview-02-05:free\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 8,\n \"id\": \"95e540b2-9b24-45b1-9b3b-3b19f1dce786\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"instructions = test_data['instruction']\\n\",\n \"outputs = test_data['output']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 69,\n \"id\": \"4c778822-47f3-49d4-a5ed-4f270811c93a\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"import time\\n\",\n \"\\n\",\n \"def batch_query_deepseek(test_subset, output_file=\\\"results.json\\\", save_every=50):\\n\",\n \" \\\"\\\"\\\"Batch queries DeepSeek and saves responses to a JSON file.\\\"\\\"\\\"\\n\",\n \" \\n\",\n \" results = []\\n\",\n \" \\n\",\n \" for i, entry in enumerate(test_subset):\\n\",\n \" response = query_deepseek(entry[\\\"instruction\\\"])\\n\",\n \" results.append({\\\"Index\\\": i, \\\"Response\\\": response})\\n\",\n \"\\n\",\n \" if (i + 1) % save_every == 0:\\n\",\n \" with open(output_file, \\\"w\\\") as f:\\n\",\n \" json.dump(results, f, indent=4)\\n\",\n \" print(f\\\"✅ Saved {i+1}/{len(test_subset)} results to {output_file}\\\")\\n\",\n \"\\n\",\n \" with open(output_file, \\\"w\\\") as f:\\n\",\n \" json.dump(results, f, indent=4)\\n\",\n \" \\n\",\n \" print(\\\"✅ Done! Results saved to:\\\", output_file)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 72,\n \"id\": \"8695824c-0778-4a30-a344-7e7569895a97\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"np.random.seed(42)\\n\",\n \"test_data_array = np.array(test_data)\\n\",\n \"random_indices = np.random.choice(len(test_data_array), size=400, replace=False)\\n\",\n \"test_subset = test_data_array[random_indices] \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 73,\n \"id\": \"86fe6187-7b3d-4b56-a37b-408fba01c4fe\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"✅ Saved 20/400 results to deepseek_results.json\\n\",\n \"✅ Saved 40/400 results to deepseek_results.json\\n\",\n \"✅ Saved 60/400 results to deepseek_results.json\\n\",\n \"✅ Saved 80/400 results to deepseek_results.json\\n\",\n \"No response choices available.\\n\",\n \"✅ Saved 100/400 results to deepseek_results.json\\n\",\n \"No response choices available.\\n\",\n \"✅ Saved 120/400 results to deepseek_results.json\\n\",\n \"No response choices available.\\n\",\n \"No response choices available.\\n\",\n \"✅ Saved 140/400 results to deepseek_results.json\\n\",\n \"✅ Saved 160/400 results to deepseek_results.json\\n\",\n \"No response choices available.\\n\",\n \"✅ Saved 180/400 results to deepseek_results.json\\n\",\n \"No response choices available.\\n\",\n \"No response choices available.\\n\",\n \"No response choices available.\\n\",\n \"✅ Saved 200/400 results to deepseek_results.json\\n\",\n \"✅ Saved 220/400 results to deepseek_results.json\\n\",\n \"✅ Saved 240/400 results to deepseek_results.json\\n\",\n \"✅ Saved 260/400 results to deepseek_results.json\\n\",\n \"✅ Saved 280/400 results to deepseek_results.json\\n\",\n \"✅ Saved 300/400 results to deepseek_results.json\\n\",\n \"✅ Saved 320/400 results to deepseek_results.json\\n\",\n \"✅ Saved 340/400 results to deepseek_results.json\\n\",\n \"✅ Saved 360/400 results to deepseek_results.json\\n\",\n \"✅ Saved 380/400 results to deepseek_results.json\\n\",\n \"✅ Saved 400/400 results to deepseek_results.json\\n\",\n \"✅ Done! Results saved to: deepseek_results.json\\n\"\n ]\n }\n ],\n \"source\": [\n \"batch_query_deepseek(test_subset, output_file=\\\"deepseek_results.json\\\", save_every=20)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 97,\n \"id\": \"d017ff8c-7c1c-464e-a5d0-4180f0757f0e\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import re\\n\",\n \"\\n\",\n \"def extract_action(response):\\n\",\n \" if not response:\\n\",\n \" return \\\"fail\\\"\\n\",\n \" # Define the regex pattern to match the action and optional value\\n\",\n \" pattern = r\\\"(bet|raise|call|fold|check)\\\\s*(\\\\d*)\\\"\\n\",\n \" \\n\",\n \" # Search for the pattern in the response\\n\",\n \" match = re.search(pattern, response, re.IGNORECASE)\\n\",\n \" \\n\",\n \" if match:\\n\",\n \" action = match.group(1).lower() # Extract the action (e.g., \\\"bet\\\")\\n\",\n \" value = match.group(2) if match.group(2) else None # Extract the value (e.g., \\\"4\\\")\\n\",\n \" \\n\",\n \" # Return the action and value (if applicable)\\n\",\n \" if value:\\n\",\n \" return f\\\"{action} {value}\\\"\\n\",\n \" else:\\n\",\n \" return action\\n\",\n \" else:\\n\",\n \" return \\\"fail\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 101,\n \"id\": \"838f3a40-0dab-4f86-ac85-32f2862742ed\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Check\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Bet 10 chips\\n\",\n \"Check\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Check\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Bet 20 chips\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Fold\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Action Accuracy (AA): 83.33%\\n\",\n \"Exact Match Accuracy (EM): 77.78%\\n\"\n ]\n }\n ],\n \"source\": [\n \"instructions = test_data['instruction']\\n\",\n \"outputs = test_data['output']\\n\",\n \"\\n\",\n \"predictions = [query_llm(instruction, model_gemini) for instruction in instructions[:18]]\\n\",\n \"\\n\",\n \"aa, em = evaluate_model(predictions, outputs[:18])\\n\",\n \"\\n\",\n \"print(f\\\"Action Accuracy (AA): {aa:.2f}%\\\")\\n\",\n \"print(f\\\"Exact Match Accuracy (EM): {em:.2f}%\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 93,\n \"id\": \"f2e67ddc-d47f-43bb-ad3f-8b42da9739ad\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with open(\\\"deepseek_results.json\\\", \\\"r\\\") as f:\\n\",\n \" deepseek_results_data = json.load(f)\\n\",\n \" \\n\",\n \"deepseek_results = [entry[\\\"Response\\\"] for entry in deepseek_results_data]\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 103,\n \"id\": \"8cc8637e-9060-455d-b332-1f9b16e69d20\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Action Accuracy (AA): 43.00%\\n\",\n \"Exact Match Accuracy (EM): 37.75%\\n\"\n ]\n }\n ],\n \"source\": [\n \"subset_outputs = [entry['output'] for entry in test_subset]\\n\",\n \"\\n\",\n \"deepseek_processed_results = [extract_action(res) for res in deepseek_results]\\n\",\n \"aa, em = evaluate_model(deepseek_processed_results, subset_outputs)\\n\",\n \"print(f\\\"Action Accuracy (AA): {aa:.2f}%\\\")\\n\",\n \"print(f\\\"Exact Match Accuracy (EM): {em:.2f}%\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 91,\n \"id\": \"870db2d6-78db-4c7d-a396-62d1594c0aff\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"def evaluate_model(predictions, ground_truths):\\n\",\n \" action_correct = 0\\n\",\n \" exact_match_correct = 0\\n\",\n \" total = len(ground_truths)\\n\",\n \"\\n\",\n \" for pred, true in zip(predictions, ground_truths):\\n\",\n \" pred_action = pred.lower().split()[0]\\n\",\n \" true_action = true.lower().split()[0]\\n\",\n \" \\n\",\n \" # Action Accuracy: Does the action match?\\n\",\n \" if pred_action == true_action:\\n\",\n \" action_correct += 1\\n\",\n \" \\n\",\n \" # Exact Match Accuracy: Does the entire response match?\\n\",\n \" if pred.lower() == true.lower():\\n\",\n \" exact_match_correct += 1\\n\",\n \"\\n\",\n \" action_accuracy = (action_correct / total) * 100\\n\",\n \" exact_match_accuracy = (exact_match_correct / total) * 100\\n\",\n \"\\n\",\n \" return action_accuracy, exact_match_accuracy\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"8799c604-5517-4f54-88d2-f166a52a41d7\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Python 3 (ipykernel)\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.10.16\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 5\n}\n"}, "b597e50bc3d382d4b332dbb8ae4495d0bff0a40a": {"t": 1785753591.1617444, "n": 12003, "h": "8c9037c19497ab54", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"id\": \"173e2309-a47e-4d16-819b-7e33f67e4e52\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from datasets import load_dataset\\n\",\n \"\\n\",\n \"ds = load_dataset(\\\"RZ412/PokerBench\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"id\": \"397779d8-3c77-420e-9371-9f8d621fe817\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"{'train': (563200, 2), 'test': (11000, 2)}\"\n ]\n },\n \"execution_count\": 2,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"ds.shape\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"id\": \"67208356-9f19-4d26-85d3-32ee2033ca88\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"\\n\",\n \"load_dotenv()\\n\",\n \"api_key = os.getenv(\\\"OPENROUTER_API_KEY\\\")\\n\",\n \"ds_key = os.getenv(\\\"DEEPSEEK_API_KEY\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 4,\n \"id\": \"223fa0f2-5abc-4370-8189-1e3e18f951b7\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"test_data = ds['test']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"id\": \"5d7f4349-6b56-4764-af13-af0a6129a9ee\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import requests\\n\",\n \"def query_llm(prompt, model):\\n\",\n \" headers = {\\n\",\n \" \\\"Authorization\\\": f\\\"Bearer {api_key}\\\",\\n\",\n \" \\\"Content-Type\\\": \\\"application/json\\\"\\n\",\n \" }\\n\",\n \" payload = {\\n\",\n \" \\\"model\\\": model,\\n\",\n \" \\\"prompt\\\": prompt,\\n\",\n \" # \\\"temperature\\\": 0.1,\\n\",\n \" #\\\"top_p\\\": 0.95\\n\",\n \" }\\n\",\n \" response = requests.post(\\\"https://openrouter.ai/api/v1/completions\\\", headers=headers, json=payload)\\n\",\n \" response_data = response.json()\\n\",\n \" if \\\"choices\\\" not in response_data:\\n\",\n \" print(response_data)\\n\",\n \" return\\n\",\n \" txt = response.json()[\\\"choices\\\"][0][\\\"text\\\"]\\n\",\n \" return txt\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 68,\n \"id\": \"8c119b41-291d-4615-8271-2d9fe752ba78\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from openai import OpenAI\\n\",\n \"\\n\",\n \"client = OpenAI(api_key=api_key, base_url=\\\"https://openrouter.ai/api/v1\\\")\\n\",\n \"\\n\",\n \"def query_deepseek(prompt):\\n\",\n \" try:\\n\",\n \" response = client.chat.completions.create(\\n\",\n \" model=\\\"deepseek/deepseek-r1-distill-qwen-32b\\\",\\n\",\n \" messages=[{\\\"role\\\": \\\"user\\\", \\\"content\\\": prompt}],\\n\",\n \" #temperature=0.1, \\n\",\n \" #top_p=0.95\\n\",\n \" )\\n\",\n \" if response.choices:\\n\",\n \" return response.choices[0].message.content\\n\",\n \" else:\\n\",\n \" print(\\\"No response choices available.\\\")\\n\",\n \" return None\\n\",\n \"\\n\",\n \" except Exception as e:\\n\",\n \" print(f\\\"Error querying DeepSeek API: {e}\\\")\\n\",\n \" return None\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 7,\n \"id\": \"41760969-10df-4b14-b6fc-2aec64db7c8c\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"model_deepseek = \\\"deepseek/deepseek-r1\\\"\\n\",\n \"model_llama = \\\"meta-llama/llama-3.3-70b-instruct\\\"\\n\",\n \"model_gemini = \\\"google/gemini-2.0-flash-lite-preview-02-05:free\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 8,\n \"id\": \"95e540b2-9b24-45b1-9b3b-3b19f1dce786\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"instructions = test_data['instruction']\\n\",\n \"outputs = test_data['output']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 69,\n \"id\": \"4c778822-47f3-49d4-a5ed-4f270811c93a\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"import time\\n\",\n \"\\n\",\n \"def batch_query_deepseek(test_subset, output_file=\\\"results.json\\\", save_every=50):\\n\",\n \" \\\"\\\"\\\"Batch queries DeepSeek and saves responses to a JSON file.\\\"\\\"\\\"\\n\",\n \" \\n\",\n \" results = []\\n\",\n \" \\n\",\n \" for i, entry in enumerate(test_subset):\\n\",\n \" response = query_deepseek(entry[\\\"instruction\\\"])\\n\",\n \" results.append({\\\"Index\\\": i, \\\"Response\\\": response})\\n\",\n \"\\n\",\n \" if (i + 1) % save_every == 0:\\n\",\n \" with open(output_file, \\\"w\\\") as f:\\n\",\n \" json.dump(results, f, indent=4)\\n\",\n \" print(f\\\"✅ Saved {i+1}/{len(test_subset)} results to {output_file}\\\")\\n\",\n \"\\n\",\n \" with open(output_file, \\\"w\\\") as f:\\n\",\n \" json.dump(results, f, indent=4)\\n\",\n \" \\n\",\n \" print(\\\"✅ Done! Results saved to:\\\", output_file)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 72,\n \"id\": \"8695824c-0778-4a30-a344-7e7569895a97\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"np.random.seed(42)\\n\",\n \"test_data_array = np.array(test_data)\\n\",\n \"random_indices = np.random.choice(len(test_data_array), size=400, replace=False)\\n\",\n \"test_subset = test_data_array[random_indices] \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 73,\n \"id\": \"86fe6187-7b3d-4b56-a37b-408fba01c4fe\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"✅ Saved 20/400 results to deepseek_results.json\\n\",\n \"✅ Saved 40/400 results to deepseek_results.json\\n\",\n \"✅ Saved 60/400 results to deepseek_results.json\\n\",\n \"✅ Saved 80/400 results to deepseek_results.json\\n\",\n \"No response choices available.\\n\",\n \"✅ Saved 100/400 results to deepseek_results.json\\n\",\n \"No response choices available.\\n\",\n \"✅ Saved 120/400 results to deepseek_results.json\\n\",\n \"No response choices available.\\n\",\n \"No response choices available.\\n\",\n \"✅ Saved 140/400 results to deepseek_results.json\\n\",\n \"✅ Saved 160/400 results to deepseek_results.json\\n\",\n \"No response choices available.\\n\",\n \"✅ Saved 180/400 results to deepseek_results.json\\n\",\n \"No response choices available.\\n\",\n \"No response choices available.\\n\",\n \"No response choices available.\\n\",\n \"✅ Saved 200/400 results to deepseek_results.json\\n\",\n \"✅ Saved 220/400 results to deepseek_results.json\\n\",\n \"✅ Saved 240/400 results to deepseek_results.json\\n\",\n \"✅ Saved 260/400 results to deepseek_results.json\\n\",\n \"✅ Saved 280/400 results to deepseek_results.json\\n\",\n \"✅ Saved 300/400 results to deepseek_results.json\\n\",\n \"✅ Saved 320/400 results to deepseek_results.json\\n\",\n \"✅ Saved 340/400 results to deepseek_results.json\\n\",\n \"✅ Saved 360/400 results to deepseek_results.json\\n\",\n \"✅ Saved 380/400 results to deepseek_results.json\\n\",\n \"✅ Saved 400/400 results to deepseek_results.json\\n\",\n \"✅ Done! Results saved to: deepseek_results.json\\n\"\n ]\n }\n ],\n \"source\": [\n \"batch_query_deepseek(test_subset, output_file=\\\"deepseek_results.json\\\", save_every=20)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 97,\n \"id\": \"d017ff8c-7c1c-464e-a5d0-4180f0757f0e\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import re\\n\",\n \"\\n\",\n \"def extract_action(response):\\n\",\n \" if not response:\\n\",\n \" return \\\"fail\\\"\\n\",\n \" # Define the regex pattern to match the action and optional value\\n\",\n \" pattern = r\\\"(bet|raise|call|fold|check)\\\\s*(\\\\d*)\\\"\\n\",\n \" \\n\",\n \" # Search for the pattern in the response\\n\",\n \" match = re.search(pattern, response, re.IGNORECASE)\\n\",\n \" \\n\",\n \" if match:\\n\",\n \" action = match.group(1).lower() # Extract the action (e.g., \\\"bet\\\")\\n\",\n \" value = match.group(2) if match.group(2) else None # Extract the value (e.g., \\\"4\\\")\\n\",\n \" \\n\",\n \" # Return the action and value (if applicable)\\n\",\n \" if value:\\n\",\n \" return f\\\"{action} {value}\\\"\\n\",\n \" else:\\n\",\n \" return action\\n\",\n \" else:\\n\",\n \" return \\\"fail\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 101,\n \"id\": \"838f3a40-0dab-4f86-ac85-32f2862742ed\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Check\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Bet 10 chips\\n\",\n \"Check\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Check\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Bet 20 chips\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Fold\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Action Accuracy (AA): 83.33%\\n\",\n \"Exact Match Accuracy (EM): 77.78%\\n\"\n ]\n }\n ],\n \"source\": [\n \"instructions = test_data['instruction']\\n\",\n \"outputs = test_data['output']\\n\",\n \"\\n\",\n \"predictions = [query_llm(instruction, model_gemini) for instruction in instructions[:18]]\\n\",\n \"\\n\",\n \"aa, em = evaluate_model(predictions, outputs[:18])\\n\",\n \"\\n\",\n \"print(f\\\"Action Accuracy (AA): {aa:.2f}%\\\")\\n\",\n \"print(f\\\"Exact Match Accuracy (EM): {em:.2f}%\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 93,\n \"id\": \"f2e67ddc-d47f-43bb-ad3f-8b42da9739ad\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"with open(\\\"deepseek_results.json\\\", \\\"r\\\") as f:\\n\",\n \" deepseek_results_data = json.load(f)\\n\",\n \" \\n\",\n \"deepseek_results = [entry[\\\"Response\\\"] for entry in deepseek_results_data]\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 103,\n \"id\": \"8cc8637e-9060-455d-b332-1f9b16e69d20\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Action Accuracy (AA): 43.00%\\n\",\n \"Exact Match Accuracy (EM): 37.75%\\n\"\n ]\n }\n ],\n \"source\": [\n \"subset_outputs = [entry['output'] for entry in test_subset]\\n\",\n \"\\n\",\n \"deepseek_processed_results = [extract_action(res) for res in deepseek_results]\\n\",\n \"aa, em = evaluate_model(deepseek_processed_results, subset_outputs)\\n\",\n \"print(f\\\"Action Accuracy (AA): {aa:.2f}%\\\")\\n\",\n \"print(f\\\"Exact Match Accuracy (EM): {em:.2f}%\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 91,\n \"id\": \"870db2d6-78db-4c7d-a396-62d1594c0aff\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"def evaluate_model(predictions, ground_truths):\\n\",\n \" action_correct = 0\\n\",\n \" exact_match_correct = 0\\n\",\n \" total = len(ground_truths)\\n\",\n \"\\n\",\n \" for pred, true in zip(predictions, ground_truths):\\n\",\n \" pred_action = pred.lower().split()[0]\\n\",\n \" true_action = true.lower().split()[0]\\n\",\n \" \\n\",\n \" # Action Accuracy: Does the action match?\\n\",\n \" if pred_action == true_action:\\n\",\n \" action_correct += 1\\n\",\n \" \\n\",\n \" # Exact Match Accuracy: Does the entire response match?\\n\",\n \" if pred.lower() == true.lower():\\n\",\n \" exact_match_correct += 1\\n\",\n \"\\n\",\n \" action_accuracy = (action_correct / total) * 100\\n\",\n \" exact_match_accuracy = (exact_match_correct / total) * 100\\n\",\n \"\\n\",\n \" return action_accuracy, exact_match_accuracy\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"8799c604-5517-4f54-88d2-f166a52a41d7\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Python 3 (ipykernel)\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.10.16\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 5\n}\n"}, "007f571e9a35bffcc55face68c4e4bb5702204f5": {"t": 1785753592.434147, "n": 7956, "h": "8ce7e634f681cf40", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"id\": \"173e2309-a47e-4d16-819b-7e33f67e4e52\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from datasets import load_dataset\\n\",\n \"\\n\",\n \"ds = load_dataset(\\\"RZ412/PokerBench\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"id\": \"397779d8-3c77-420e-9371-9f8d621fe817\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"{'train': (563200, 2), 'test': (11000, 2)}\"\n ]\n },\n \"execution_count\": 2,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"ds.shape\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"id\": \"67208356-9f19-4d26-85d3-32ee2033ca88\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"\\n\",\n \"load_dotenv()\\n\",\n \"api_key = os.getenv(\\\"OPENROUTER_API_KEY\\\")\\n\",\n \"ds_key = os.getenv(\\\"DEEPSEEK_API_KEY\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 4,\n \"id\": \"223fa0f2-5abc-4370-8189-1e3e18f951b7\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"test_data = ds['test']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"id\": \"5d7f4349-6b56-4764-af13-af0a6129a9ee\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import requests\\n\",\n \"def query_llm(prompt, model):\\n\",\n \" headers = {\\n\",\n \" \\\"Authorization\\\": f\\\"Bearer {api_key}\\\",\\n\",\n \" \\\"Content-Type\\\": \\\"application/json\\\"\\n\",\n \" }\\n\",\n \" payload = {\\n\",\n \" \\\"model\\\": model,\\n\",\n \" \\\"prompt\\\": prompt,\\n\",\n \" # \\\"temperature\\\": 0.1,\\n\",\n \" #\\\"top_p\\\": 0.95\\n\",\n \" }\\n\",\n \" response = requests.post(\\\"https://openrouter.ai/api/v1/completions\\\", headers=headers, json=payload)\\n\",\n \" response_data = response.json()\\n\",\n \" if \\\"choices\\\" not in response_data:\\n\",\n \" print(response_data)\\n\",\n \" return\\n\",\n \" txt = response.json()[\\\"choices\\\"][0][\\\"text\\\"]\\n\",\n \" return txt\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 24,\n \"id\": \"8c119b41-291d-4615-8271-2d9fe752ba78\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from openai import OpenAI\\n\",\n \"\\n\",\n \"client = OpenAI(api_key=api_key, base_url=\\\"https://openrouter.ai/api/v1\\\")\\n\",\n \"\\n\",\n \"def query_deepseek(prompt):\\n\",\n \" try:\\n\",\n \" response = client.chat.completions.create(\\n\",\n \" model=\\\"deepseek/deepseek-r1\\\",\\n\",\n \" messages=[{\\\"role\\\": \\\"user\\\", \\\"content\\\": prompt}],\\n\",\n \" #temperature=0.1, \\n\",\n \" #top_p=0.95\\n\",\n \" )\\n\",\n \" if response.choices:\\n\",\n \" return response.choices[0].message.content\\n\",\n \" else:\\n\",\n \" print(\\\"No response choices available.\\\")\\n\",\n \" return None\\n\",\n \"\\n\",\n \" except Exception as e:\\n\",\n \" print(f\\\"Error querying DeepSeek API: {e}\\\")\\n\",\n \" return None\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 7,\n \"id\": \"41760969-10df-4b14-b6fc-2aec64db7c8c\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"model_deepseek = \\\"deepseek/deepseek-r1\\\"\\n\",\n \"model_llama = \\\"meta-llama/llama-3.3-70b-instruct\\\"\\n\",\n \"model_gemini = \\\"google/gemini-2.0-flash-lite-preview-02-05:free\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 8,\n \"id\": \"95e540b2-9b24-45b1-9b3b-3b19f1dce786\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"instructions = test_data['instruction']\\n\",\n \"outputs = test_data['output']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 48,\n \"id\": \"4c778822-47f3-49d4-a5ed-4f270811c93a\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import json\\n\",\n \"import time\\n\",\n \"\\n\",\n \"def batch_query_deepseek(test_subset, output_file=\\\"results.json\\\", save_every=50):\\n\",\n \" \\\"\\\"\\\"Batch queries DeepSeek and saves responses to a JSON file.\\\"\\\"\\\"\\n\",\n \" \\n\",\n \" results = []\\n\",\n \" \\n\",\n \" for i, entry in enumerate(test_subset):\\n\",\n \" response = query_deepseek(entry[\\\"instruction\\\"])\\n\",\n \" results.append({\\\"Index\\\": i, \\\"Response\\\": response})\\n\",\n \"\\n\",\n \" if (i + 1) % save_every == 0:\\n\",\n \" with open(output_file, \\\"w\\\") as f:\\n\",\n \" json.dump(results, f, indent=4)\\n\",\n \" print(f\\\"✅ Saved {i+1}/{len(test_subset)} results to {output_file}\\\")\\n\",\n \"\\n\",\n \" with open(output_file, \\\"w\\\") as f:\\n\",\n \" json.dump(results, f, indent=4)\\n\",\n \" \\n\",\n \" print(\\\"✅ Done! Results saved to:\\\", output_file)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"8695824c-0778-4a30-a344-7e7569895a97\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"np.random.seed(42)\\n\",\n \"test_data_array = np.array(test_data)\\n\",\n \"random_indices = np.random.choice(len(test_data_array), size=200, replace=False)\\n\",\n \"test_subset = test_data_array[random_indices] \\n\",\n \"\\n\",\n \"batch_query_deepseek(test_subset, output_file=\\\"deepseek_results.json\\\", save_every=20)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"d017ff8c-7c1c-464e-a5d0-4180f0757f0e\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import re\\n\",\n \"\\n\",\n \"def extract_action(response):\\n\",\n \" # Define the regex pattern to match the action and optional value\\n\",\n \" pattern = r\\\"(bet|raise|call|fold|check)\\\\s*(\\\\d*)\\\"\\n\",\n \" \\n\",\n \" # Search for the pattern in the response\\n\",\n \" match = re.search(pattern, response, re.IGNORECASE)\\n\",\n \" \\n\",\n \" if match:\\n\",\n \" action = match.group(1).lower() # Extract the action (e.g., \\\"bet\\\")\\n\",\n \" value = match.group(2) if match.group(2) else None # Extract the value (e.g., \\\"4\\\")\\n\",\n \" \\n\",\n \" # Return the action and value (if applicable)\\n\",\n \" if value:\\n\",\n \" return f\\\"{action} {value}\\\"\\n\",\n \" else:\\n\",\n \" return action\\n\",\n \" else:\\n\",\n \" return \\\"fail\\\"\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 101,\n \"id\": \"838f3a40-0dab-4f86-ac85-32f2862742ed\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Check\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Bet 10 chips\\n\",\n \"Check\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Check\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Bet 20 chips\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Fold\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Action Accuracy (AA): 83.33%\\n\",\n \"Exact Match Accuracy (EM): 77.78%\\n\"\n ]\n }\n ],\n \"source\": [\n \"instructions = test_data['instruction']\\n\",\n \"outputs = test_data['output']\\n\",\n \"\\n\",\n \"predictions = [query_llm(instruction, model_gemini) for instruction in instructions[:18]]\\n\",\n \"\\n\",\n \"aa, em = evaluate_model(predictions, outputs[:18])\\n\",\n \"\\n\",\n \"print(f\\\"Action Accuracy (AA): {aa:.2f}%\\\")\\n\",\n \"print(f\\\"Exact Match Accuracy (EM): {em:.2f}%\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"8cc8637e-9060-455d-b332-1f9b16e69d20\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Python 3 (ipykernel)\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.10.16\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 5\n}\n"}, "f9a921b8d2d637ae2114631e095396e90e1cd75b": {"t": 1785753595.0782952, "n": 10474, "h": "5d5adcbfc0c855f8", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"id\": \"173e2309-a47e-4d16-819b-7e33f67e4e52\",\n \"metadata\": {\n \"collapsed\": true,\n \"jupyter\": {\n \"outputs_hidden\": true\n },\n \"scrolled\": true\n },\n \"outputs\": [\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"8ac5f8f85f6e4172aabe96a5e9768d76\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"README.md: 0%| | 0.00/5.16k [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"1ce33fb10cea4b1cbbc3d8ed1eee74ee\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"(…)lop_500k_train_set_prompt_and_label.json: 0%| | 0.00/607M [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"4116b049bfe24df2b9b7b4e477186a80\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"(…)flop_60k_train_set_prompt_and_label.json: 0%| | 0.00/62.0M [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"a5b453b8537f4b3e97bb255fb8a5d266\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"(…)tflop_10k_test_set_prompt_and_label.json: 0%| | 0.00/12.2M [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"b2454187d77047b28b3e18d17a4eb117\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"(…)reflop_1k_test_set_prompt_and_label.json: 0%| | 0.00/965k [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"16c975f1bb4d484489e0efd02b46bea5\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"Generating train split: 0%| | 0/563200 [00:00<?, ? examples/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"0e8fe666df1c48cf99bf2ebe4c5b0bbf\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"Generating test split: 0%| | 0/11000 [00:00<?, ? examples/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n }\n ],\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from datasets import load_dataset\\n\",\n \"\\n\",\n \"ds = load_dataset(\\\"RZ412/PokerBench\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 38,\n \"id\": \"397779d8-3c77-420e-9371-9f8d621fe817\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"{'train': (563200, 2), 'test': (11000, 2)}\"\n ]\n },\n \"execution_count\": 38,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"ds.shape\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 39,\n \"id\": \"67208356-9f19-4d26-85d3-32ee2033ca88\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"\\n\",\n \"load_dotenv()\\n\",\n \"api_key = os.getenv(\\\"OPENROUTER_API_KEY\\\")\\n\",\n \"ds_key = os.getenv(\\\"DEEPSEEK_API_KEY\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 40,\n \"id\": \"223fa0f2-5abc-4370-8189-1e3e18f951b7\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"test_data = ds['test']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 41,\n \"id\": \"5d7f4349-6b56-4764-af13-af0a6129a9ee\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import requests\\n\",\n \"def query_llm(prompt, model):\\n\",\n \" headers = {\\n\",\n \" \\\"Authorization\\\": f\\\"Bearer {api_key}\\\",\\n\",\n \" \\\"Content-Type\\\": \\\"application/json\\\"\\n\",\n \" }\\n\",\n \" payload = {\\n\",\n \" \\\"model\\\": model,\\n\",\n \" \\\"prompt\\\": prompt,\\n\",\n \" # \\\"temperature\\\": 0.1,\\n\",\n \" #\\\"top_p\\\": 0.95\\n\",\n \" }\\n\",\n \" response = requests.post(\\\"https://openrouter.ai/api/v1/completions\\\", headers=headers, json=payload)\\n\",\n \" response_data = response.json()\\n\",\n \" if \\\"choices\\\" not in response_data:\\n\",\n \" print(response_data)\\n\",\n \" return\\n\",\n \" txt = response.json()[\\\"choices\\\"][0][\\\"text\\\"]\\n\",\n \" return txt\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 51,\n \"id\": \"8c119b41-291d-4615-8271-2d9fe752ba78\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from openai import OpenAI\\n\",\n \"\\n\",\n \"client = OpenAI(api_key=api_key, base_url=\\\"https://openrouter.ai/api/v1\\\")\\n\",\n \"\\n\",\n \"def query_deepseek(prompt):\\n\",\n \" try:\\n\",\n \" response = client.chat.completions.create(\\n\",\n \" model=\\\"deepseek/deepseek-r1\\\",\\n\",\n \" messages=[{\\\"role\\\": \\\"user\\\", \\\"content\\\": prompt}],\\n\",\n \" #temperature=0.1, \\n\",\n \" #top_p=0.95\\n\",\n \" )\\n\",\n \" print(\\\"deekseek response: \\\", response)\\n\",\n \" if response.choices:\\n\",\n \" return response.choices[0].message.content\\n\",\n \" else:\\n\",\n \" print(\\\"No response choices available.\\\")\\n\",\n \" return None\\n\",\n \"\\n\",\n \" except Exception as e:\\n\",\n \" print(f\\\"Error querying DeepSeek API: {e}\\\")\\n\",\n \" return None\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 43,\n \"id\": \"41760969-10df-4b14-b6fc-2aec64db7c8c\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"model_deepseek = \\\"deepseek/deepseek-r1\\\"\\n\",\n \"model_llama = \\\"meta-llama/llama-3.3-70b-instruct\\\"\\n\",\n \"model_gemini = \\\"google/gemini-2.0-flash-lite-preview-02-05:free\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 33,\n \"id\": \"95e540b2-9b24-45b1-9b3b-3b19f1dce786\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"instructions = test_data['instruction']\\n\",\n \"outputs = test_data['output']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 53,\n \"id\": \"aa44518a-0f53-4613-b16f-b5368abdf9b3\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Your optimal action is: bet 4 chips.\\n\"\n ]\n }\n ],\n \"source\": [\n \"ans = query_llm2(instructions[0])\\n\",\n \"print(ans)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"4c778822-47f3-49d4-a5ed-4f270811c93a\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import numpy as np\\n\",\n \"\\n\",\n \"np.random.seed(42)\\n\",\n \"\\n\",\n \"test_data_array = np.array(test_data)\\n\",\n \"\\n\",\n \"random_indices = np.random.choice(len(test_data_array), size=1000, replace=False)\\n\",\n \"\\n\",\n \"random_test_sample = test_data_array[random_indices]\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"id\": \"d017ff8c-7c1c-464e-a5d0-4180f0757f0e\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Extracted Action: bet 4\\n\"\n ]\n }\n ],\n \"source\": [\n \"import re\\n\",\n \"\\n\",\n \"def extract_action(response):\\n\",\n \" # Define the regex pattern to match the action and optional value\\n\",\n \" pattern = r\\\"(bet|raise|call|fold|check)\\\\s*(\\\\d*)\\\"\\n\",\n \" \\n\",\n \" # Search for the pattern in the response\\n\",\n \" match = re.search(pattern, response, re.IGNORECASE)\\n\",\n \" \\n\",\n \" if match:\\n\",\n \" action = match.group(1).lower() # Extract the action (e.g., \\\"bet\\\")\\n\",\n \" value = match.group(2) if match.group(2) else None # Extract the value (e.g., \\\"4\\\")\\n\",\n \" \\n\",\n \" # Return the action and value (if applicable)\\n\",\n \" if value:\\n\",\n \" return f\\\"{action} {value}\\\"\\n\",\n \" else:\\n\",\n \" return action\\n\",\n \" else:\\n\",\n \" # If no action is found, return a fallback action (e.g., \\\"fold\\\")\\n\",\n \" return \\\"fold\\\"\\n\",\n \"\\n\",\n \"# Example usage\\n\",\n \"response = \\\"Your optimal action is: bet 4 chips.\\\"\\n\",\n \"extracted_action = extract_action(response)\\n\",\n \"print(f\\\"Extracted Action: {extracted_action}\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 101,\n \"id\": \"838f3a40-0dab-4f86-ac85-32f2862742ed\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Check\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Bet 10 chips\\n\",\n \"Check\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Check\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Bet 20 chips\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Fold\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Action Accuracy (AA): 83.33%\\n\",\n \"Exact Match Accuracy (EM): 77.78%\\n\"\n ]\n }\n ],\n \"source\": [\n \"instructions = test_data['instruction']\\n\",\n \"outputs = test_data['output']\\n\",\n \"\\n\",\n \"predictions = [query_llm(instruction, model_gemini) for instruction in instructions[:18]]\\n\",\n \"\\n\",\n \"aa, em = evaluate_model(predictions, outputs[:18])\\n\",\n \"\\n\",\n \"print(f\\\"Action Accuracy (AA): {aa:.2f}%\\\")\\n\",\n \"print(f\\\"Exact Match Accuracy (EM): {em:.2f}%\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"8cc8637e-9060-455d-b332-1f9b16e69d20\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Python 3 (ipykernel)\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.10.16\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 5\n}\n"}, "1e7647260285622964055fffbe58697614702ef0": {"t": 1785753596.374476, "n": 6261, "h": "458db0191a094d3f", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"id\": \"173e2309-a47e-4d16-819b-7e33f67e4e52\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from datasets import load_dataset\\n\",\n \"\\n\",\n \"ds = load_dataset(\\\"RZ412/PokerBench\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 38,\n \"id\": \"397779d8-3c77-420e-9371-9f8d621fe817\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"{'train': (563200, 2), 'test': (11000, 2)}\"\n ]\n },\n \"execution_count\": 38,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"ds.shape\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 39,\n \"id\": \"67208356-9f19-4d26-85d3-32ee2033ca88\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"\\n\",\n \"load_dotenv()\\n\",\n \"api_key = os.getenv(\\\"OPENROUTER_API_KEY\\\")\\n\",\n \"ds_key = os.getenv(\\\"DEEPSEEK_API_KEY\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 40,\n \"id\": \"223fa0f2-5abc-4370-8189-1e3e18f951b7\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"test_data = ds['test']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 41,\n \"id\": \"5d7f4349-6b56-4764-af13-af0a6129a9ee\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import requests\\n\",\n \"def query_llm(prompt, model):\\n\",\n \" headers = {\\n\",\n \" \\\"Authorization\\\": f\\\"Bearer {api_key}\\\",\\n\",\n \" \\\"Content-Type\\\": \\\"application/json\\\"\\n\",\n \" }\\n\",\n \" payload = {\\n\",\n \" \\\"model\\\": model,\\n\",\n \" \\\"prompt\\\": prompt,\\n\",\n \" # \\\"temperature\\\": 0.1,\\n\",\n \" #\\\"top_p\\\": 0.95\\n\",\n \" }\\n\",\n \" response = requests.post(\\\"https://openrouter.ai/api/v1/completions\\\", headers=headers, json=payload)\\n\",\n \" response_data = response.json()\\n\",\n \" if \\\"choices\\\" not in response_data:\\n\",\n \" print(response_data)\\n\",\n \" return\\n\",\n \" txt = response.json()[\\\"choices\\\"][0][\\\"text\\\"]\\n\",\n \" return txt\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 51,\n \"id\": \"8c119b41-291d-4615-8271-2d9fe752ba78\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from openai import OpenAI\\n\",\n \"\\n\",\n \"client = OpenAI(api_key=api_key, base_url=\\\"https://openrouter.ai/api/v1\\\")\\n\",\n \"\\n\",\n \"def query_deepseek(prompt):\\n\",\n \" try:\\n\",\n \" response = client.chat.completions.create(\\n\",\n \" model=\\\"deepseek/deepseek-r1\\\",\\n\",\n \" messages=[{\\\"role\\\": \\\"user\\\", \\\"content\\\": prompt}],\\n\",\n \" #temperature=0.1, \\n\",\n \" #top_p=0.95\\n\",\n \" )\\n\",\n \" print(\\\"deekseek response: \\\", response)\\n\",\n \" if response.choices:\\n\",\n \" return response.choices[0].message.content\\n\",\n \" else:\\n\",\n \" print(\\\"No response choices available.\\\")\\n\",\n \" return None\\n\",\n \"\\n\",\n \" except Exception as e:\\n\",\n \" print(f\\\"Error querying DeepSeek API: {e}\\\")\\n\",\n \" return None\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 43,\n \"id\": \"41760969-10df-4b14-b6fc-2aec64db7c8c\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"model_deepseek = \\\"deepseek/deepseek-r1\\\"\\n\",\n \"model_llama = \\\"meta-llama/llama-3.3-70b-instruct\\\"\\n\",\n \"model_gemini = \\\"google/gemini-2.0-flash-lite-preview-02-05:free\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 33,\n \"id\": \"95e540b2-9b24-45b1-9b3b-3b19f1dce786\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"instructions = test_data['instruction']\\n\",\n \"outputs = test_data['output']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 53,\n \"id\": \"aa44518a-0f53-4613-b16f-b5368abdf9b3\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Your optimal action is: bet 4 chips.\\n\"\n ]\n }\n ],\n \"source\": [\n \"ans = query_llm2(instructions[0])\\n\",\n \"print(ans)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"4c778822-47f3-49d4-a5ed-4f270811c93a\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import numpy as np\\n\",\n \"\\n\",\n \"np.random.seed(42)\\n\",\n \"\\n\",\n \"test_data_array = np.array(test_data)\\n\",\n \"\\n\",\n \"random_indices = np.random.choice(len(test_data_array), size=1000, replace=False)\\n\",\n \"\\n\",\n \"random_test_sample = test_data_array[random_indices]\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 101,\n \"id\": \"838f3a40-0dab-4f86-ac85-32f2862742ed\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Check\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Bet 10 chips\\n\",\n \"Check\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Check\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Bet 20 chips\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Fold\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Action Accuracy (AA): 83.33%\\n\",\n \"Exact Match Accuracy (EM): 77.78%\\n\"\n ]\n }\n ],\n \"source\": [\n \"instructions = test_data['instruction']\\n\",\n \"outputs = test_data['output']\\n\",\n \"\\n\",\n \"predictions = [query_llm(instruction, model_gemini) for instruction in instructions[:18]]\\n\",\n \"\\n\",\n \"aa, em = evaluate_model(predictions, outputs[:18])\\n\",\n \"\\n\",\n \"print(f\\\"Action Accuracy (AA): {aa:.2f}%\\\")\\n\",\n \"print(f\\\"Exact Match Accuracy (EM): {em:.2f}%\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"8cc8637e-9060-455d-b332-1f9b16e69d20\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Python 3 (ipykernel)\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.10.16\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 5\n}\n"}, "81eb17b3b74a70d7721e68a6db5045b77aa4787b": {"t": 1785753597.9361057, "n": 10107, "h": "ca9ef5ba57b3082d", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"id\": \"173e2309-a47e-4d16-819b-7e33f67e4e52\",\n \"metadata\": {\n \"collapsed\": true,\n \"jupyter\": {\n \"outputs_hidden\": true\n },\n \"scrolled\": true\n },\n \"outputs\": [\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"8ac5f8f85f6e4172aabe96a5e9768d76\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"README.md: 0%| | 0.00/5.16k [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"1ce33fb10cea4b1cbbc3d8ed1eee74ee\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"(…)lop_500k_train_set_prompt_and_label.json: 0%| | 0.00/607M [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"4116b049bfe24df2b9b7b4e477186a80\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"(…)flop_60k_train_set_prompt_and_label.json: 0%| | 0.00/62.0M [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"a5b453b8537f4b3e97bb255fb8a5d266\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"(…)tflop_10k_test_set_prompt_and_label.json: 0%| | 0.00/12.2M [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"b2454187d77047b28b3e18d17a4eb117\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"(…)reflop_1k_test_set_prompt_and_label.json: 0%| | 0.00/965k [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"16c975f1bb4d484489e0efd02b46bea5\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"Generating train split: 0%| | 0/563200 [00:00<?, ? examples/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"0e8fe666df1c48cf99bf2ebe4c5b0bbf\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"Generating test split: 0%| | 0/11000 [00:00<?, ? examples/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n }\n ],\n \"source\": [\n \"from datasets import load_dataset\\n\",\n \"\\n\",\n \"ds = load_dataset(\\\"RZ412/PokerBench\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"id\": \"397779d8-3c77-420e-9371-9f8d621fe817\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"{'train': (563200, 2), 'test': (11000, 2)}\"\n ]\n },\n \"execution_count\": 3,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"ds.shape\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 12,\n \"id\": \"67208356-9f19-4d26-85d3-32ee2033ca88\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"\\n\",\n \"load_dotenv()\\n\",\n \"api_key = os.getenv(\\\"OPENROUTER_API_KEY\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 32,\n \"id\": \"223fa0f2-5abc-4370-8189-1e3e18f951b7\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"test_data = ds['test']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 90,\n \"id\": \"5d7f4349-6b56-4764-af13-af0a6129a9ee\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import requests\\n\",\n \"def query_llm(prompt, model):\\n\",\n \" headers = {\\n\",\n \" \\\"Authorization\\\": f\\\"Bearer {api_key}\\\",\\n\",\n \" \\\"Content-Type\\\": \\\"application/json\\\"\\n\",\n \" }\\n\",\n \" payload = {\\n\",\n \" \\\"model\\\": model,\\n\",\n \" \\\"prompt\\\": prompt,\\n\",\n \" \\\"temperature\\\": 0.1,\\n\",\n \" \\\"top_p\\\": 0.95\\n\",\n \" }\\n\",\n \" response = requests.post(\\\"https://openrouter.ai/api/v1/completions\\\", headers=headers, json=payload)\\n\",\n \" response_data = response.json()\\n\",\n \" if \\\"choices\\\" not in response_data:\\n\",\n \" print(response_data)\\n\",\n \" return\\n\",\n \" txt = response.json()[\\\"choices\\\"][0][\\\"text\\\"].strip()\\n\",\n \" print(txt)\\n\",\n \" return txt\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 96,\n \"id\": \"41760969-10df-4b14-b6fc-2aec64db7c8c\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"model_deepseek = \\\"deepseek/deepseek-r1:free\\\"\\n\",\n \"model_llama = \\\"meta-llama/llama-3.3-70b-instruct\\\"\\n\",\n \"model_gemini = \\\"google/gemini-2.0-flash-lite-preview-02-05:free\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 97,\n \"id\": \"0f656850-2d22-4924-9ca7-c101ef568c09\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"def evaluate_model(predictions, ground_truths):\\n\",\n \" action_correct = 0\\n\",\n \" exact_match_correct = 0\\n\",\n \" total = len(ground_truths)\\n\",\n \"\\n\",\n \" for pred, true in zip(predictions, ground_truths):\\n\",\n \" pred_action = pred.lower().split()[0]\\n\",\n \" true_action = true.lower().split()[0]\\n\",\n \" \\n\",\n \" # Action Accuracy: Does the action match?\\n\",\n \" if pred_action == true_action:\\n\",\n \" action_correct += 1\\n\",\n \" \\n\",\n \" # Exact Match Accuracy: Does the entire response match?\\n\",\n \" if pred.lower() == true.lower():\\n\",\n \" exact_match_correct += 1\\n\",\n \"\\n\",\n \" action_accuracy = (action_correct / total) * 100\\n\",\n \" exact_match_accuracy = (exact_match_correct / total) * 100\\n\",\n \"\\n\",\n \" return action_accuracy, exact_match_accuracy\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 98,\n \"id\": \"95e540b2-9b24-45b1-9b3b-3b19f1dce786\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"instructions = test_data['instruction']\\n\",\n \"outputs = test_data['output']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 99,\n \"id\": \"9df7972d-aacb-432d-a3a7-40d3b36b8802\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Check\\n\"\n ]\n },\n {\n \"data\": {\n \"text/plain\": [\n \"'Check'\"\n ]\n },\n \"execution_count\": 99,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"query_llm(instructions[0], model_gemini)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"id\": \"d017ff8c-7c1c-464e-a5d0-4180f0757f0e\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Extracted Action: bet 4\\n\"\n ]\n }\n ],\n \"source\": [\n \"import re\\n\",\n \"\\n\",\n \"def extract_action(response):\\n\",\n \" # Define the regex pattern to match the action and optional value\\n\",\n \" pattern = r\\\"(bet|raise|call|fold|check)\\\\s*(\\\\d*)\\\"\\n\",\n \" \\n\",\n \" # Search for the pattern in the response\\n\",\n \" match = re.search(pattern, response, re.IGNORECASE)\\n\",\n \" \\n\",\n \" if match:\\n\",\n \" action = match.group(1).lower() # Extract the action (e.g., \\\"bet\\\")\\n\",\n \" value = match.group(2) if match.group(2) else None # Extract the value (e.g., \\\"4\\\")\\n\",\n \" \\n\",\n \" # Return the action and value (if applicable)\\n\",\n \" if value:\\n\",\n \" return f\\\"{action} {value}\\\"\\n\",\n \" else:\\n\",\n \" return action\\n\",\n \" else:\\n\",\n \" # If no action is found, return a fallback action (e.g., \\\"fold\\\")\\n\",\n \" return \\\"fold\\\"\\n\",\n \"\\n\",\n \"# Example usage\\n\",\n \"response = \\\"Your optimal action is: bet 4 chips.\\\"\\n\",\n \"extracted_action = extract_action(response)\\n\",\n \"print(f\\\"Extracted Action: {extracted_action}\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 101,\n \"id\": \"838f3a40-0dab-4f86-ac85-32f2862742ed\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Check\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Bet 10 chips\\n\",\n \"Check\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Check\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Bet 20 chips\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Fold\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Action Accuracy (AA): 83.33%\\n\",\n \"Exact Match Accuracy (EM): 77.78%\\n\"\n ]\n }\n ],\n \"source\": [\n \"instructions = test_data['instruction']\\n\",\n \"outputs = test_data['output']\\n\",\n \"\\n\",\n \"predictions = [query_llm(instruction, model_gemini) for instruction in instructions[:18]]\\n\",\n \"\\n\",\n \"aa, em = evaluate_model(predictions, outputs[:18])\\n\",\n \"\\n\",\n \"print(f\\\"Action Accuracy (AA): {aa:.2f}%\\\")\\n\",\n \"print(f\\\"Exact Match Accuracy (EM): {em:.2f}%\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"8cc8637e-9060-455d-b332-1f9b16e69d20\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Python 3 (ipykernel)\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.10.16\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 5\n}\n"}, "261d3a2d56371c2f050285d401185e3e07e9d6d5": {"t": 1785753600.326852, "n": 8589, "h": "14d6f7770902df4d", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"id\": \"173e2309-a47e-4d16-819b-7e33f67e4e52\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"8ac5f8f85f6e4172aabe96a5e9768d76\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"README.md: 0%| | 0.00/5.16k [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"1ce33fb10cea4b1cbbc3d8ed1eee74ee\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"(…)lop_500k_train_set_prompt_and_label.json: 0%| | 0.00/607M [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"4116b049bfe24df2b9b7b4e477186a80\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"(…)flop_60k_train_set_prompt_and_label.json: 0%| | 0.00/62.0M [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"a5b453b8537f4b3e97bb255fb8a5d266\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"(…)tflop_10k_test_set_prompt_and_label.json: 0%| | 0.00/12.2M [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"b2454187d77047b28b3e18d17a4eb117\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"(…)reflop_1k_test_set_prompt_and_label.json: 0%| | 0.00/965k [00:00<?, ?B/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"16c975f1bb4d484489e0efd02b46bea5\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"Generating train split: 0%| | 0/563200 [00:00<?, ? examples/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"0e8fe666df1c48cf99bf2ebe4c5b0bbf\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"Generating test split: 0%| | 0/11000 [00:00<?, ? examples/s]\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n }\n ],\n \"source\": [\n \"from datasets import load_dataset\\n\",\n \"\\n\",\n \"ds = load_dataset(\\\"RZ412/PokerBench\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"id\": \"397779d8-3c77-420e-9371-9f8d621fe817\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"{'train': (563200, 2), 'test': (11000, 2)}\"\n ]\n },\n \"execution_count\": 3,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"ds.shape\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 12,\n \"id\": \"67208356-9f19-4d26-85d3-32ee2033ca88\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"\\n\",\n \"load_dotenv()\\n\",\n \"api_key = os.getenv(\\\"OPENROUTER_API_KEY\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 32,\n \"id\": \"223fa0f2-5abc-4370-8189-1e3e18f951b7\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"test_data = ds['test']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 90,\n \"id\": \"5d7f4349-6b56-4764-af13-af0a6129a9ee\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import requests\\n\",\n \"def query_llm(prompt, model):\\n\",\n \" headers = {\\n\",\n \" \\\"Authorization\\\": f\\\"Bearer {api_key}\\\",\\n\",\n \" \\\"Content-Type\\\": \\\"application/json\\\"\\n\",\n \" }\\n\",\n \" payload = {\\n\",\n \" \\\"model\\\": model,\\n\",\n \" \\\"prompt\\\": prompt,\\n\",\n \" \\\"temperature\\\": 0.1,\\n\",\n \" \\\"top_p\\\": 0.95\\n\",\n \" }\\n\",\n \" response = requests.post(\\\"https://openrouter.ai/api/v1/completions\\\", headers=headers, json=payload)\\n\",\n \" response_data = response.json()\\n\",\n \" if \\\"choices\\\" not in response_data:\\n\",\n \" print(response_data)\\n\",\n \" return\\n\",\n \" txt = response.json()[\\\"choices\\\"][0][\\\"text\\\"].strip()\\n\",\n \" print(txt)\\n\",\n \" return txt\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 96,\n \"id\": \"41760969-10df-4b14-b6fc-2aec64db7c8c\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"model_deepseek = \\\"deepseek/deepseek-r1:free\\\"\\n\",\n \"model_llama = \\\"meta-llama/llama-3.3-70b-instruct\\\"\\n\",\n \"model_gemini = \\\"google/gemini-2.0-flash-lite-preview-02-05:free\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 97,\n \"id\": \"0f656850-2d22-4924-9ca7-c101ef568c09\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"def evaluate_model(predictions, ground_truths):\\n\",\n \" action_correct = 0\\n\",\n \" exact_match_correct = 0\\n\",\n \" total = len(ground_truths)\\n\",\n \"\\n\",\n \" for pred, true in zip(predictions, ground_truths):\\n\",\n \" pred_action = pred.lower().split()[0]\\n\",\n \" true_action = true.lower().split()[0]\\n\",\n \" \\n\",\n \" # Action Accuracy: Does the action match?\\n\",\n \" if pred_action == true_action:\\n\",\n \" action_correct += 1\\n\",\n \" \\n\",\n \" # Exact Match Accuracy: Does the entire response match?\\n\",\n \" if pred.lower() == true.lower():\\n\",\n \" exact_match_correct += 1\\n\",\n \"\\n\",\n \" action_accuracy = (action_correct / total) * 100\\n\",\n \" exact_match_accuracy = (exact_match_correct / total) * 100\\n\",\n \"\\n\",\n \" return action_accuracy, exact_match_accuracy\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 98,\n \"id\": \"95e540b2-9b24-45b1-9b3b-3b19f1dce786\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"instructions = test_data['instruction']\\n\",\n \"outputs = test_data['output']\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 99,\n \"id\": \"9df7972d-aacb-432d-a3a7-40d3b36b8802\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Check\\n\"\n ]\n },\n {\n \"data\": {\n \"text/plain\": [\n \"'Check'\"\n ]\n },\n \"execution_count\": 99,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"query_llm(instructions[0], model_gemini)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 101,\n \"id\": \"838f3a40-0dab-4f86-ac85-32f2862742ed\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Check\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Bet 10 chips\\n\",\n \"Check\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Check\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Bet 20 chips\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Check\\n\",\n \"Fold\\n\",\n \"Call\\n\",\n \"Call\\n\",\n \"Action Accuracy (AA): 83.33%\\n\",\n \"Exact Match Accuracy (EM): 77.78%\\n\"\n ]\n }\n ],\n \"source\": [\n \"instructions = test_data['instruction']\\n\",\n \"outputs = test_data['output']\\n\",\n \"\\n\",\n \"predictions = [query_llm(instruction, model_gemini) for instruction in instructions[:18]]\\n\",\n \"\\n\",\n \"aa, em = evaluate_model(predictions, outputs[:18])\\n\",\n \"\\n\",\n \"print(f\\\"Action Accuracy (AA): {aa:.2f}%\\\")\\n\",\n \"print(f\\\"Exact Match Accuracy (EM): {em:.2f}%\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"8cc8637e-9060-455d-b332-1f9b16e69d20\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Python 3 (ipykernel)\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.10.16\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 5\n}\n"}}}} |