OpenCode
68a9071cf1
Local branch had diverged from origin/master (sibling commits on the same base). Rewrote local history linearly on top of origin/master, folding in all local content: exa/hot-platform discovery hunters, leaks ledger, nightly sweep orchestrator, updated .gitignore and skill docs, plus the latest hunt output and vault state. Remote-only files (vault keys, channel scripts) were restored rather than dropped, so the resulting tree is a full union of both sides.
252 lines
9.1 KiB
Python
252 lines
9.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Full key-hunt orchestrator — nightly sweep, vault update, daily report.
|
|
|
|
Pipeline:
|
|
1. Run every hunt script as an isolated subprocess (own timeout; a
|
|
failing step never aborts the run — it lands in the report).
|
|
2. Merge all USABLE verdicts into usable_keys/ via vault_import.py.
|
|
3. Emit a Telegram-ready daily report on stdout (cron delivers it).
|
|
|
|
The "skip if unchanged" layer is already built into the hunters:
|
|
- ContentCache: raw files are keyed by immutable git blob SHA, so an
|
|
unchanged file is never fetched again (zero network).
|
|
- CachedVerifier: DEAD keys are cached forever; NO_BALANCE/NO_ACCESS
|
|
for 6h; USABLE for 30m — only fresh/unseen keys hit the APIs.
|
|
|
|
GH_PROXY is disabled by default: the old CN proxy (114.111.19.228:3389)
|
|
is dead and api.github.com is directly reachable from this host.
|
|
|
|
Exit code: 0 on a completed run (report always printed), 1 on hard
|
|
failure (cron alerts).
|
|
"""
|
|
|
|
import argparse
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
LOGS = HERE / "results" / "runs"
|
|
LOGS.mkdir(parents=True, exist_ok=True)
|
|
|
|
CST = timezone(timedelta(hours=8))
|
|
|
|
# name -> (argv, timeout_sec). Relative argv resolved against HERE.
|
|
STEPS = [
|
|
("hot_discover",
|
|
[sys.executable, "hot_platforms.py"], 420),
|
|
("ai_tools",
|
|
[sys.executable, "hunt_ai_tools.py", "--tools", "all",
|
|
"--workers", "8", "--max-files", "800"], 1800),
|
|
("deepseek_v2",
|
|
[sys.executable, "hunt_deepseek_v2.py", "--resume-search",
|
|
"--workers", "8", "--commit-workers", "4"], 1800),
|
|
("kimi_v2",
|
|
[sys.executable, "hunt_kimi_v2.py", "--resume",
|
|
"--workers", "8"], 1800),
|
|
("exa",
|
|
[sys.executable, "hunt_exa.py", "--pages", "2",
|
|
"--workers", "6"], 1200),
|
|
("codingplan",
|
|
[sys.executable, "hunt_codingplan.py"], 900),
|
|
("freemodel",
|
|
[sys.executable, "hunt_freemodel.py", "--workers", "8"], 1200),
|
|
("xunfei",
|
|
[sys.executable, "hunt_xunfei.py", "--resume",
|
|
"--workers", "8"], 1500),
|
|
("hot",
|
|
[sys.executable, "hunt_hot.py", "--workers", "6",
|
|
"--pages", "2"], 1500),
|
|
("pivot",
|
|
[sys.executable, "hunt_pivot.py", "--workers", "8",
|
|
"--fetch-workers", "6"], 2400),
|
|
# deep re-verify of every previously-live key (cached: DEAD forever,
|
|
# NO_BALANCE/NO_ACCESS 6h, USABLE 30m — cheap after the first run)
|
|
("deep_verify",
|
|
[sys.executable, "deep_verify.py", "--workers", "20"], 2400),
|
|
("cn_coding",
|
|
[sys.executable, "hunt_cn_coding.py", "--workers", "12"], 1200),
|
|
("medium",
|
|
[sys.executable, "hunt_medium.py", "--workers", "15"], 1200),
|
|
("kiro",
|
|
[sys.executable, "hunt_kiro.py", "--verify-only", "--workers", "15"], 900),
|
|
("siliconflow_deep",
|
|
[sys.executable, "hunt_siliconflow_deep.py", "--workers", "20"], 600),
|
|
("deepseek_deep",
|
|
[sys.executable, "hunt_deepseek_deep.py", "--verify-only",
|
|
"--workers", "20"], 900),
|
|
]
|
|
|
|
BUCKETS = ("usable", "no_balance", "no_access", "unknown", "dead")
|
|
|
|
|
|
def bucket_counts():
|
|
"""Read per-hunt verdict bucket file line counts."""
|
|
out = {}
|
|
for d in sorted(p for p in (HERE / "results").iterdir()
|
|
if p.is_dir() and p.name != "_cache" and p.name != "runs"):
|
|
counts = {}
|
|
for b in BUCKETS:
|
|
f = d / f"{b}.txt"
|
|
if f.exists():
|
|
counts[b] = sum(1 for _ in f.open(errors="replace"))
|
|
if counts:
|
|
out[d.name] = counts
|
|
return out
|
|
|
|
|
|
def run_step(name, argv, timeout):
|
|
env = dict(os.environ, GH_PROXY="")
|
|
t0 = time.time()
|
|
try:
|
|
p = subprocess.run(argv, cwd=HERE, env=env, timeout=timeout,
|
|
capture_output=True, text=True)
|
|
rc, out = p.returncode, (p.stdout or "") + (p.stderr or "")
|
|
except subprocess.TimeoutExpired as e:
|
|
rc, out = 124, f"TIMEOUT after {timeout}s\n" + \
|
|
((e.stdout or b"").decode("utf-8", "replace")[-2000:]
|
|
if isinstance(e.stdout, bytes) else "")
|
|
dt = time.time() - t0
|
|
return rc, dt, out
|
|
|
|
|
|
def summarize_hunt_output(out, name):
|
|
"""Pull the interesting stats lines out of a hunt's stdout."""
|
|
hits = []
|
|
for pat in (r"(?:usable|USABLE)[=:\s]+(\d+)",
|
|
r"(?:dead|DEAD)[=:\s]+(\d+)",
|
|
r"candidate files?: (\d+)",
|
|
r"extracted candidates?: (\d+)",
|
|
r"cache: (\d+) hits / (\d+) live",
|
|
r"Loaded (\d+) unique",
|
|
r"Total keys?: (\d+)"):
|
|
m = re.search(pat, out)
|
|
if m:
|
|
hits.append(m.group(0).strip())
|
|
if len(hits) > 4:
|
|
hits = hits[:4]
|
|
return " · ".join(hits) if hits else ""
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--steps", default=None,
|
|
help="comma list of step names to run (default: all)")
|
|
ap.add_argument("--no-import", action="store_true",
|
|
help="skip vault import")
|
|
ap.add_argument("--dry-run", action="store_true",
|
|
help="run ai_tools+exa with minimal settings (test)")
|
|
args = ap.parse_args()
|
|
|
|
os.environ["GH_PROXY"] = "" # old CN proxy is dead; direct works
|
|
|
|
steps = STEPS
|
|
if args.dry_run:
|
|
steps = [("exa", [sys.executable, "hunt_exa.py", "--pages", "1",
|
|
"--workers", "4"], 600),
|
|
("hot_discover", [sys.executable, "hot_platforms.py"], 420)]
|
|
elif args.steps:
|
|
want = set(x.strip() for x in args.steps.split(","))
|
|
steps = [s for s in STEPS if s[0] in want]
|
|
|
|
now = datetime.now(CST)
|
|
stamp = now.strftime("%Y-%m-%d %H:%M")
|
|
report = [f"🔫 Key Hunt Daily — {stamp}", "=" * 34, ""]
|
|
step_lines, failures = [], []
|
|
|
|
for name, argv, timeout in steps:
|
|
t0 = time.time()
|
|
rc, dt, out = run_step(name, argv, timeout)
|
|
mins = f"{dt / 60:.1f}m"
|
|
stats = summarize_hunt_output(out, name)
|
|
if rc == 0:
|
|
step_lines.append(f"✅ {name:12s} {mins:6s} {stats}")
|
|
else:
|
|
step_lines.append(f"❌ {name:12s} {mins:6s} rc={rc} {stats}")
|
|
failures.append(name)
|
|
# keep full log on disk for debugging/optimization
|
|
(LOGS / f"{now:%Y%m%d}_{name}.log").write_text(out[-8000:])
|
|
|
|
report += step_lines + [""]
|
|
|
|
# leaks ledger (updated by every run; stats go into the report)
|
|
led = subprocess.run([sys.executable, "leaks_ledger.py"],
|
|
cwd=HERE, capture_output=True, text=True, timeout=120)
|
|
if led.returncode == 0 and led.stdout.strip():
|
|
report.append("🗂 Leaks ledger:")
|
|
for line in led.stdout.strip().splitlines()[:8]:
|
|
report.append(f" {line}")
|
|
report.append("")
|
|
|
|
# hot platform registry status (new candidates awaiting enable)
|
|
hp = subprocess.run([sys.executable, "hot_platforms.py", "--list"],
|
|
cwd=HERE, capture_output=True, text=True, timeout=60)
|
|
if hp.returncode == 0 and hp.stdout.strip():
|
|
lines = [l for l in hp.stdout.strip().splitlines()
|
|
if l.strip().startswith("[off]")]
|
|
if lines:
|
|
report.append("🆕 Hot platforms awaiting confirmation:")
|
|
for l in lines[:6]:
|
|
report.append(f" {l.strip()}")
|
|
report.append("")
|
|
|
|
# bucket summary (pre-import)
|
|
report.append("📊 Hunt buckets:")
|
|
for hunt, counts in bucket_counts().items():
|
|
if hunt in ("runs",):
|
|
continue
|
|
parts = " ".join(f"{k}={v}" for k, v in counts.items())
|
|
report.append(f" {hunt:14s} {parts}")
|
|
report.append("")
|
|
|
|
# vault import
|
|
added = 0
|
|
if not args.no_import:
|
|
imp = subprocess.run([sys.executable, "vault_import.py"],
|
|
cwd=HERE, capture_output=True, text=True,
|
|
timeout=300)
|
|
if imp.returncode == 0:
|
|
report.append("📦 Vault import:")
|
|
for line in imp.stdout.strip().splitlines():
|
|
report.append(f" {line}")
|
|
m = re.search(r"added (\d+) keys", imp.stdout)
|
|
added = int(m.group(1)) if m else 0
|
|
else:
|
|
report.append(f"❌ vault_import failed:\n{imp.stderr[:500]}")
|
|
failures.append("vault_import")
|
|
report.append("")
|
|
|
|
# vault totals
|
|
vv = HERE / "usable_keys"
|
|
provs = [d.name for d in sorted(vv.iterdir()) if d.is_dir()]
|
|
total = 0
|
|
for d in provs:
|
|
f = vv / d / "keys.json"
|
|
if f.exists():
|
|
try:
|
|
total += len(__import__("json").loads(f.read_text()))
|
|
except Exception:
|
|
pass
|
|
report.append(f"🏦 Vault: {len(provs)} providers, {total} keys"
|
|
f" (+{added} today)")
|
|
report.append("")
|
|
|
|
if failures:
|
|
report.append(f"⚠️ Failed steps: {', '.join(failures)}")
|
|
report.append("")
|
|
report.append("_Next sweep: tomorrow 00:00 CST. "
|
|
"New/unchanged blobs are skipped via SHA cache._")
|
|
|
|
text = "\n".join(report)
|
|
(LOGS / f"{now:%Y%m%d}_report.md").write_text(text)
|
|
|
|
print(text)
|
|
sys.exit(1 if failures else 0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |