1 line
17 KiB
JSON
1 line
17 KiB
JSON
{"repo": "atingupta2005/agentic-ai", "files": {"day-3/2-Frameworks-for-Building-AI-Agents/labs/3_Multi-Model_Evaluation_with_an_AI_Judge.ipynb": {"c0e8b44b091d2a1811778a4f0997a09d81d4b171": {"t": 1785745496.390615, "n": 15261, "h": "f2a10453f61d64b1", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Lab: Comparing and Evaluating Multiple LLMs\\n\",\n \"\\n\",\n \"This lab explores how to interact with various Large Language Models (LLMs) from different providers through their APIs. We will then implement an agentic pattern where one LLM evaluates the responses of others.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# === Imports ===\\n\",\n \"# A good practice is to group all imports at the top of the script.\\n\",\n \"\\n\",\n \"import os\\n\",\n \"import json\\n\",\n \"from dotenv import load_dotenv\\n\",\n \"from openai import OpenAI # For OpenAI, Groq, DeepSeek, and Google (using compatible endpoint)\\n\",\n \"from anthropic import Anthropic # For Claude models\\n\",\n \"from IPython.display import Markdown, display\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Load environment variables from the .env file.\\n\",\n \"# This is a crucial step to securely manage your API keys.\\n\",\n \"load_dotenv(override=True)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# === API Key Verification ===\\n\",\n \"# This cell verifies that all necessary API keys are available in the environment.\\n\",\n \"# It's a useful debugging step to ensure your setup is correct.\\n\",\n \"\\n\",\n \"openai_api_key = os.getenv('OPENAI_API_KEY')\\n\",\n \"anthropic_api_key = os.getenv('ANTHROPIC_API_KEY')\\n\",\n \"google_api_key = os.getenv('GOOGLE_API_KEY')\\n\",\n \"deepseek_api_key = os.getenv('DEEPSEEK_API_KEY')\\n\",\n \"groq_api_key = os.getenv('GROQ_API_KEY')\\n\",\n \"\\n\",\n \"def check_key(name, key):\\n\",\n \" if key:\\n\",\n \" # Print only a small, non-sensitive prefix of the key.\\n\",\n \" print(f\\\"{name} Key is set (starts with: {key[:4]}...)\\\")\\n\",\n \" else:\\n\",\n \" # Some keys are optional for this lab.\\n\",\n \" print(f\\\"{name} Key is not set (optional)\\\")\\n\",\n \"\\n\",\n \"check_key(\\\"OpenAI\\\", openai_api_key)\\n\",\n \"check_key(\\\"Anthropic\\\", anthropic_api_key)\\n\",\n \"check_key(\\\"Google\\\", google_api_key)\\n\",\n \"check_key(\\\"DeepSeek\\\", deepseek_api_key)\\n\",\n \"check_key(\\\"Groq\\\", groq_api_key)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### Step 1: Generate a Challenge Question\\n\",\n \"\\n\",\n \"First, we'll use an LLM to generate a single, high-quality question that we can then pose to all the other models. This ensures a fair and consistent comparison.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"request = \\\"Please come up with a challenging, nuanced question that I can ask a number of LLMs to evaluate their intelligence. \\\"\\n\",\n \"request += \\\"Answer only with the question, no explanation or preamble.\\\"\\n\",\n \"messages = [{\\\"role\\\": \\\"user\\\", \\\"content\\\": request}]\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"openai_client = OpenAI()\\n\",\n \"response = openai_client.chat.completions.create(\\n\",\n \" model=\\\"gpt-4o\\\", # Using a powerful model to generate a high-quality question\\n\",\n \" messages=messages,\\n\",\n \")\\n\",\n \"question = response.choices[0].message.content\\n\",\n \"print(\\\"The generated question for all models is:\\\")\\n\",\n \"print(question)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### Step 2: Query Each Model with the Same Question\\n\",\n \"\\n\",\n \"Now we'll iterate through a list of different models from various providers. We will store each model's name and its answer for later evaluation.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Initialize lists to store the names of the models and their answers.\\n\",\n \"competitors = []\\n\",\n \"answers = []\\n\",\n \"\\n\",\n \"# The message list now contains the single question we generated.\\n\",\n \"messages = [{\\\"role\\\": \\\"user\\\", \\\"content\\\": question}]\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# === Competitor 1: OpenAI (GPT-4o Mini) ===\\n\",\n \"model_name = \\\"gpt-4o-mini\\\"\\n\",\n \"print(f\\\"--- Querying {model_name} ---\\\")\\n\",\n \"\\n\",\n \"response = openai_client.chat.completions.create(model=model_name, messages=messages)\\n\",\n \"answer = response.choices[0].message.content\\n\",\n \"\\n\",\n \"display(Markdown(answer))\\n\",\n \"competitors.append(model_name)\\n\",\n \"answers.append(answer)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# === Competitor 2: Anthropic (Claude Sonnet) ===\\n\",\n \"# Note: Anthropic uses a slightly different client and response structure.\\n\",\n \"if anthropic_api_key:\\n\",\n \" model_name = \\\"claude-3-5-sonnet-20240620\\\"\\n\",\n \" print(f\\\"--- Querying {model_name} ---\\\")\\n\",\n \"\\n\",\n \" claude_client = Anthropic()\\n\",\n \" # `max_tokens` is a required parameter for Anthropic's API.\\n\",\n \" response = claude_client.messages.create(model=model_name, messages=messages, max_tokens=2048)\\n\",\n \" # The answer is located in `response.content[0].text`.\\n\",\n \" answer = response.content[0].text\\n\",\n \"\\n\",\n \" display(Markdown(answer))\\n\",\n \" competitors.append(model_name)\\n\",\n \" answers.append(answer)\\n\",\n \"else:\\n\",\n \" print(\\\"Skipping Anthropic model, no API key found.\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# === Competitor 3: Google (Gemini Flash) ===\\n\",\n \"# We can use the OpenAI client by pointing the `base_url` to Google's OpenAI-compatible endpoint.\\n\",\n \"if google_api_key:\\n\",\n \" model_name = \\\"gemini-1.5-flash-latest\\\"\\n\",\n \" print(f\\\"--- Querying {model_name} ---\\\")\\n\",\n \" \\n\",\n \" # Note the custom base_url to route requests to Google.\\n\",\n \" gemini_client = OpenAI(api_key=google_api_key, base_url=\\\"https://generativelanguage.googleapis.com/v1beta/models\\\")\\n\",\n \" \\n\",\n \" # We need to specify the model as part of the endpoint in this case\\n\",\n \" response = gemini_client.chat.completions.create(model=f\\\"{model_name}:generateContent\\\", messages=messages)\\n\",\n \" answer = response.choices[0].message.content\\n\",\n \"\\n\",\n \" display(Markdown(answer))\\n\",\n \" competitors.append(model_name)\\n\",\n \" answers.append(answer)\\n\",\n \"else:\\n\",\n \" print(\\\"Skipping Google model, no API key found.\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# === Competitor 4: DeepSeek ===\\n\",\n \"# DeepSeek also provides an OpenAI-compatible endpoint.\\n\",\n \"if deepseek_api_key:\\n\",\n \" model_name = \\\"deepseek-chat\\\"\\n\",\n \" print(f\\\"--- Querying {model_name} ---\\\")\\n\",\n \"\\n\",\n \" deepseek_client = OpenAI(api_key=deepseek_api_key, base_url=\\\"https://api.deepseek.com/v1\\\")\\n\",\n \" response = deepseek_client.chat.completions.create(model=model_name, messages=messages)\\n\",\n \" answer = response.choices[0].message.content\\n\",\n \"\\n\",\n \" display(Markdown(answer))\\n\",\n \" competitors.append(model_name)\\n\",\n \" answers.append(answer)\\n\",\n \"else:\\n\",\n \" print(\\\"Skipping DeepSeek model, no API key found.\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# === Competitor 5: Groq (Llama 3) ===\\n\",\n \"# Groq offers very fast inference on models like Llama via their compatible endpoint.\\n\",\n \"if groq_api_key:\\n\",\n \" model_name = \\\"llama3-70b-8192\\\"\\n\",\n \" print(f\\\"--- Querying {model_name} ---\\\")\\n\",\n \"\\n\",\n \" groq_client = OpenAI(api_key=groq_api_key, base_url=\\\"https://api.groq.com/openai/v1\\\")\\n\",\n \" response = groq_client.chat.completions.create(model=model_name, messages=messages)\\n\",\n \" answer = response.choices[0].message.content\\n\",\n \"\\n\",\n \" display(Markdown(answer))\\n\",\n \" competitors.append(model_name)\\n\",\n \" answers.append(answer)\\n\",\n \"else:\\n\",\n \" print(\\\"Skipping Groq model, no API key found.\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### Using Ollama for Local Models\\n\",\n \"\\n\",\n \"Ollama allows you to run open-source models directly on your own machine. It exposes a local server that is compatible with the OpenAI API, making it easy to integrate.\\n\",\n \"\\n\",\n \"1. **Installation**: If you haven't already, [download Ollama](https://ollama.com).\\n\",\n \"2. **Run the Server**: Open a terminal and run `ollama serve`. You should see \\\"Ollama is running\\\" at [http://localhost:11434](http://localhost:11434).\\n\",\n \"3. **Pull a Model**: In your terminal, pull a model to use. We'll use `llama3`, a powerful and reasonably sized model.\\n\",\n \" ```bash\\n\",\n \" ollama pull llama3\\n\",\n \" ```\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# === Competitor 6: Ollama (Local Llama 3) ===\\n\",\n \"# To use Ollama, we point the OpenAI client to the local server address.\\n\",\n \"# An API key is required but its value doesn't matter for local Ollama.\\n\",\n \"\\n\",\n \"try:\\n\",\n \" model_name = \\\"llama3\\\"\\n\",\n \" print(f\\\"--- Querying {model_name} (local) ---\\\")\\n\",\n \" ollama_client = OpenAI(base_url='http://localhost:11434/v1', api_key='ollama')\\n\",\n \"\\n\",\n \" # We can check if the model exists locally before calling it.\\n\",\n \" local_models = [m['name'] for m in ollama_client.models.list().data]\\n\",\n \" if f\\\"{model_name}:latest\\\" in local_models:\\n\",\n \" response = ollama_client.chat.completions.create(model=model_name, messages=messages)\\n\",\n \" answer = response.choices[0].message.content\\n\",\n \"\\n\",\n \" display(Markdown(answer))\\n\",\n \" competitors.append(model_name)\\n\",\n \" answers.append(answer)\\n\",\n \" else:\\n\",\n \" print(f\\\"Ollama model '{model_name}' not found. Please run 'ollama pull {model_name}' in your terminal.\\\")\\n\",\n \"\\n\",\n \"except Exception as e:\\n\",\n \" print(f\\\"Could not connect to Ollama. Is the server running? Error: {e}\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### Step 3: Evaluate the Results with a Judge LLM\\n\",\n \"\\n\",\n \"Now that we have all the responses, we can implement an **Evaluator** pattern. We will assemble all the answers into a single prompt and ask a powerful LLM to act as a judge, ranking the responses from best to worst.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# First, let's combine all the answers into a single string for the judge's prompt.\\n\",\n \"# The `enumerate` function is used to get both the index and the value.\\n\",\n \"\\n\",\n \"all_answers_text = \\\"\\\"\\n\",\n \"for index, answer in enumerate(answers):\\n\",\n \" all_answers_text += f\\\"# Response from competitor {index+1} ({competitors[index]})\\\\n\\\\n\\\"\\n\",\n \" all_answers_text += answer + \\\"\\\\n\\\\n---\\\\n\\\\n\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# This is the master prompt for our Judge agent.\\n\",\n \"# It includes the original question, all the answers, and instructions for the output format (JSON).\\n\",\n \"# Using f-strings makes it easy to inject our variables into the prompt text.\\n\",\n \"\\n\",\n \"judge_prompt = f\\\"\\\"\\\"You are an impartial judge in a competition between {len(competitors)} Large Language Models.\\n\",\n \"Your task is to evaluate each model's response for clarity, depth, accuracy, and strength of argument.\\n\",\n \"\\n\",\n \"The models were all asked this question:\\n\",\n \"--- QUESTION ---\\n\",\n \"{question}\\n\",\n \"--- END QUESTION ---\\n\",\n \"\\n\",\n \"Here are the responses from each competitor:\\n\",\n \"--- COMPETITOR RESPONSES ---\\n\",\n \"{all_answers_text}\\n\",\n \"--- END RESPONSES ---\\n\",\n \"\\n\",\n \"Please rank the competitors from best to worst based on their answers.\\n\",\n \"Respond with JSON, and only JSON. The JSON object should have a single key, \\\"results\\\", which is a list of the competitor numbers (as integers) in ranked order.\\n\",\n \"Example format: {{\\\"results\\\": [3, 1, 2, 4]}}\\n\",\n \"Do not include any other text, explanations, or markdown formatting.\\n\",\n \"\\\"\\\"\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Let's see the final prompt before sending it.\\n\",\n \"print(judge_prompt)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# It's time for the final judgement!\\n\",\n \"# We will use GPT-4o for this task as it has strong reasoning and instruction-following capabilities.\\n\",\n \"# We also enable JSON mode to ensure the output is valid JSON.\\n\",\n \"\\n\",\n \"judge_messages = [{\\\"role\\\": \\\"user\\\", \\\"content\\\": judge_prompt}]\\n\",\n \"\\n\",\n \"judgement_response = openai_client.chat.completions.create(\\n\",\n \" model=\\\"gpt-4o\\\",\\n\",\n \" messages=judge_messages,\\n\",\n \" response_format={\\\"type\\\": \\\"json_object\\\"} # Enforce JSON output\\n\",\n \")\\n\",\n \"results_json = judgement_response.choices[0].message.content\\n\",\n \"print(\\\"Raw JSON from judge:\\\")\\n\",\n \"print(results_json)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# === Final Results ===\\n\",\n \"# Now we parse the JSON response from the judge and print the final rankings.\\n\",\n \"\\n\",\n \"try:\\n\",\n \" results_dict = json.loads(results_json)\\n\",\n \" ranks = results_dict[\\\"results\\\"]\\n\",\n \" print(\\\"\\\\n--- FINAL RANKINGS ---\\\")\\n\",\n \" for rank_index, competitor_number in enumerate(ranks):\\n\",\n \" # The competitor number is 1-based, so we subtract 1 for the list index.\\n\",\n \" competitor_name = competitors[int(competitor_number)-1]\\n\",\n \" print(f\\\"Rank {rank_index+1}: {competitor_name}\\\")\\n\",\n \"except (json.JSONDecodeError, KeyError, IndexError) as e:\\n\",\n \" print(f\\\"\\\\nError parsing the judge's response. Please check the raw JSON output. Error: {e}\\\")\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \".venv\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.12.9\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n"}}}} |