hack/tools/scripts/llm-key-hunter/retry_unknowns.py

#!/usr/bin/env python3
"""
Re-verify unknown/timeout keys through a working proxy (OpenAI) or directly (others).
Re-classifies every key in results/medium/<Provider>/unknown.txt as if fresh.
"""
import json, urllib.request, urllib.error, re, sys, time, itertools
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor, as_completed

UA = "curl/8.5.0"
ROOT = Path(__file__).parent / "results/medium"
PROXY_POOL = [l.strip() for l in (Path(__file__).parent / "results/proxies/openai_capable.txt").read_text().splitlines() if l.strip()]

PROXY_CYCLE = itertools.cycle(PROXY_POOL)
_proxy_lock = __import__("threading").Lock()

def next_proxy():
    with _proxy_lock:
        return next(PROXY_CYCLE)

def http(method, url, headers, body=None, timeout=25, use_proxy=False):
    h = {"User-Agent": UA, "Accept": "*/*", **headers}
    data = json.dumps(body).encode() if body is not None else None
    req = urllib.request.Request(url, data=data, headers=h, method=method)
    handlers = []
    px = None
    if use_proxy:
        px = next_proxy()
        handlers.append(urllib.request.ProxyHandler({"http": px, "https": px}))
    opener = urllib.request.build_opener(*handlers) if handlers else urllib.request.build_opener()
    try:
        with opener.open(req, timeout=timeout) as resp:
            return resp.getcode(), resp.read().decode("utf-8", "replace"), px
    except urllib.error.HTTPError as e:
        try:
            return e.code, e.read().decode("utf-8", "replace"), px
        except Exception:
            return e.code, "", px
    except Exception as e:
        return 0, f"network: {type(e).__name__}: {e}", px

def verify_openai(key):
    # two layers: models (cheap) then a tiny gpt-4o-mini chat
    c1, b1, _ = http("GET", "https://api.openai.com/v1/models",
                     {"Authorization": f"Bearer {key}"}, use_proxy=True)
    if c1 == 401:
        return "DEAD", "401 models"
    if c1 == 200:
        # key recognized — actually try a chat to confirm quota/balance
        c2, b2, px = http("POST", "https://api.openai.com/v1/chat/completions",
            {"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
            {"model": "gpt-4o-mini", "messages":[{"role":"user","content":"hi"}],
             "max_tokens": 1, "temperature": 0}, use_proxy=True)
        if c2 == 200 and '"choices"' in b2:
            return "USABLE", f"chat OK via {px}"
        if c2 in (402, 429):
            return "NO_BALANCE", f"chat {c2}: {b2[:120]}"
        if c2 == 401:
            return "DEAD", "401 chat"
        if c2 == 0:
            return "UNKNOWN", f"chat net: {b2[:120]}"
        return "NO_ACCESS", f"chat {c2}: {b2[:120]}"
    if c1 == 0:
        return "UNKNOWN", f"models net: {b1[:120]}"
    return "NO_ACCESS", f"models {c1}: {b1[:120]}"

def verify_groq(key):
    c, b, _ = http("POST", "https://api.groq.com/openai/v1/chat/completions",
        {"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
        {"model": "llama-3.1-8b-instant", "messages":[{"role":"user","content":"hi"}],
         "max_tokens": 1, "temperature": 0})
    if c == 200 and '"choices"' in b: return "USABLE", "OK"
    if c == 401: return "DEAD", "401"
    if c in (402, 429): return "NO_BALANCE", f"{c}: {b[:120]}"
    if c == 0: return "UNKNOWN", b[:120]
    return "NO_ACCESS", f"{c}: {b[:120]}"

def verify_anthropic(key):
    c, b, _ = http("POST", "https://api.anthropic.com/v1/messages",
        {"x-api-key": key, "anthropic-version": "2023-06-01",
         "Content-Type": "application/json"},
        {"model": "claude-3-5-haiku-20241022",
         "max_tokens": 4, "messages":[{"role":"user","content":"hi"}]})
    if c == 200 and '"content"' in b: return "USABLE", "OK"
    if c == 401: return "DEAD", "401"
    if c in (402, 429): return "NO_BALANCE", f"{c}: {b[:120]}"
    if c == 0: return "UNKNOWN", b[:120]
    return "NO_ACCESS", f"{c}: {b[:120]}"

def verify_generic_openai_like(base, model, key):
    c, b, _ = http("POST", f"{base}/v1/chat/completions",
        {"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
        {"model": model, "messages":[{"role":"user","content":"hi"}],
         "max_tokens": 1, "temperature": 0})
    if c == 200 and '"choices"' in b: return "USABLE", "OK"
    if c == 401: return "DEAD", "401"
    if c in (402, 429): return "NO_BALANCE", f"{c}: {b[:120]}"
    if c == 0: return "UNKNOWN", b[:120]
    return "NO_ACCESS", f"{c}: {b[:120]}"

VERIFIERS = {
    "OpenAI":       verify_openai,
    "Groq":         verify_groq,
    "Anthropic":    verify_anthropic,
    "SiliconFlow":  lambda k: verify_generic_openai_like("https://api.siliconflow.cn", "Qwen/Qwen2.5-7B-Instruct", k),
    "LingyiWanwu":  lambda k: verify_generic_openai_like("https://api.lingyiwanwu.com", "yi-lightning", k),
    "StepFun":      lambda k: verify_generic_openai_like("https://api.stepfun.com", "step-1-flash", k),
}

def parse(line):
    parts = line.split("|", 2)
    if len(parts) < 2: return None
    return parts[0], parts[1], parts[2] if len(parts) > 2 else ""

def run_provider(name, workers):
    f = ROOT / name / "unknown.txt"
    if not f.exists(): return
    rows = [r for r in (parse(l) for l in f.read_text().splitlines() if l.strip()) if r]
    if not rows: return
    print(f"\n=== {name}: retrying {len(rows)} unknown keys ===", flush=True)
    ver = VERIFIERS[name]
    out = {k: [] for k in ("USABLE","NO_BALANCE","NO_ACCESS","UNKNOWN","DEAD")}
    done = 0
    with ThreadPoolExecutor(max_workers=workers) as pool:
        futs = {pool.submit(ver, k): (k, u, d) for k, u, d in rows}
        for f2 in as_completed(futs):
            k, u, d = futs[f2]
            try:
                v, det = f2.result()
            except Exception as e:
                v, det = "UNKNOWN", f"exc:{e}"
            out[v].append((k, u, det))
            done += 1
            if done % 25 == 0:
                print(f"  {name} {done}/{len(rows)}", flush=True)
    # append into existing bucket files (these were previously all unknown.txt)
    for label, items in out.items():
        if not items: continue
        p = ROOT / name / f"{label.lower()}.txt"
        existing = set()
        if p.exists():
            for l in p.read_text().splitlines():
                if "|" in l: existing.add(l.split("|",1)[0])
        with p.open("a") as fh:
            for k, u, det in items:
                if k in existing: continue
                fh.write(f"{k}|{u}|[retry] {det}\n")
    # clear unknown file since all re-classified
    f.write_text("")
    print(f"{name}: " + "  ".join(f"{k}={len(v)}" for k,v in out.items() if v), flush=True)

if __name__ == "__main__":
    providers = sys.argv[1:] if len(sys.argv) > 1 else list(VERIFIERS.keys())
    for name in providers:
        if name not in VERIFIERS:
            print(f"unknown provider: {name}"); continue
        run_provider(name, workers=8 if name == "OpenAI" else 20)