#!/usr/bin/env python3
"""Persistent self-healing proxy pool — favors DURABLE proxies.

Free public proxies die within minutes. Instead of trusting whatever works
right now, this daemon:

  1. scrapes fresh candidates from many public lists every REFRESH_CYCLES,
  2. re-tests EVERY known proxy against the REAL target (api.github.com) every
     TEST_INTERVAL seconds,
  3. tracks per-proxy uptime, consecutive successes, and survival time,
  4. exposes two files for hunters:
       github_live.txt    — all proxies that passed the last round
       github_stable.txt  — durable proxies: survived MIN_ROUNDS rounds with
                            >= UPTIME_MIN uptime and a recent success

State persists in pool_state.json so durability accrues across restarts.
Run it once and leave it running (or schedule it). Logs to proxy_pool.log.

Usage:
  python3 proxy_pool.py --once          # one scrape + test round, then exit
  python3 proxy_pool.py                 # run forever
  python3 proxy_pool.py --workers 200
"""
import argparse, json, os, re, socket, subprocess, time, urllib.request, urllib.error
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from datetime import datetime, timezone

HERE = Path(__file__).resolve().parent
OUT = HERE / "results" / "proxies"
OUT.mkdir(parents=True, exist_ok=True)
STATE = OUT / "pool_state.json"
LIVE = OUT / "github_live.txt"
STABLE = OUT / "github_stable.txt"
VETERAN = OUT / "github_veteran.txt"
LOG = OUT / "proxy_pool.log"

UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/124.0 Safari/537.36"

# The actual things we need the proxy for: reaching GitHub API AND raw content.
# A proxy must reach BOTH to be considered "stable" (some proxies MITM or only
# allow certain hosts). 200/401/403/429/404 all mean "the proxy reached GitHub".
TARGETS = [
    "https://api.github.com/zen",
    "https://raw.githubusercontent.com/torvalds/linux/master/README",
]
PASS_CODES = {200, 401, 403, 429, 404}
# A proxy must pass at least this many of the TARGETS to count as a live round.
MIN_TARGETS_PASS = 2

SOURCES = [
    "https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt",
    "https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks5.txt",
    "https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt",
    "https://raw.githubusercontent.com/monosans/proxy-list/main/proxies_anonymous/http.txt",
    "https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt",
    "https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt",
    "https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/http.txt",
    "https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/https.txt",
    "https://raw.githubusercontent.com/roosterkid/openproxylist/main/HTTPS_RAW.txt",
    "https://raw.githubusercontent.com/mmpx12/proxy-list/master/http.txt",
    "https://raw.githubusercontent.com/mmpx12/proxy-list/master/https.txt",
    "https://raw.githubusercontent.com/proxifly/free-proxy-list/main/proxies/protocols/http/data.txt",
    "https://raw.githubusercontent.com/proxifly/free-proxy-list/main/proxies/countries/CN/data.txt",
    "https://raw.githubusercontent.com/zloi-user/hideip.me/main/http.txt",
    "https://raw.githubusercontent.com/zloi-user/hideip.me/main/https.txt",
    "https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt",
    "https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt",
    "https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt",
    "https://raw.githubusercontent.com/Zaeem20/FREE_PROXIES_LIST/master/http.txt",
    "https://raw.githubusercontent.com/Zaeem20/FREE_PROXIES_LIST/master/https.txt",
    "https://raw.githubusercontent.com/officialputuid/KangProxy/KangProxy/http/http.txt",
    "https://raw.githubusercontent.com/officialputuid/KangProxy/KangProxy/https/https.txt",
    "https://raw.githubusercontent.com/yemixzy/proxy-list/main/proxies/http.txt",
    "https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt",
    "https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/https.txt",
    "https://raw.githubusercontent.com/proxy4parsing/proxy-list/main/http.txt",
    "https://raw.githubusercontent.com/Anonym0usWork1221/Free-Proxies/main/proxy_files/http_proxies.txt",
]

# Durability thresholds
MIN_ROUNDS = 3        # must have been tested at least this many rounds
UPTIME_MIN = 0.50     # and succeed at least this fraction of the time
STREAK_MIN = 2        # and be alive on the last 2 consecutive rounds
# Veteran tier: the truly durable proxies that survive over time.
VET_MIN_ROUNDS = 12      # at least 12 rounds of history
VET_MIN_AGE_H = 1.0      # survived at least 1 wall-clock hour
VET_UPTIME_MIN = 0.70    # 70%+ uptime
VET_STREAK_MIN = 3       # alive the last 3 rounds
TEST_TIMEOUT = 5
MAX_AGE_FAILS = 8     # forget a NEW proxy after this many consecutive failures
VET_MAX_FAILS = 20    # give veterans a longer leash (they sometimes blip)
RECHECK_INTERVAL = 60   # re-test all live/known proxies every N seconds
SCRAPE_INTERVAL = 1800   # re-scrape sources every N seconds

IPPORT_RE = re.compile(rb"\b(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d{2,5})\b")
DIRECT = urllib.request.build_opener()


def log(msg):
    line = f"[{datetime.now().strftime('%H:%M:%S')}] {msg}"
    print(line, flush=True)
    with LOG.open("a") as f:
        f.write(line + "\n")


def scrape():
    found = set()
    for url in SOURCES:
        try:
            req = urllib.request.Request(url, headers={"User-Agent": UA})
            with DIRECT.open(req, timeout=20) as r:
                data = r.read()
            for m in IPPORT_RE.finditer(data):
                ip = m.group(1).decode()
                port = int(m.group(2))
                octets = [int(x) for x in ip.split(".")]
                if all(0 <= o <= 255 for o in octets) and 1 <= port <= 65535:
                    found.add(f"{ip}:{port}")
        except Exception:
            continue
    return found


def test_proxy(pp, timeout=TEST_TIMEOUT):
    """Return (pp, ok, latency_ms, code_or_err). Uses curl because Python's
    ProxyHandler falsely reports success when the tunnel hangs mid-transfer.
    A proxy is OK only if it reaches MIN_TARGETS_PASS of TARGETS."""
    scheme = "socks5" if pp.endswith("|s5") else "http"
    addr = pp.split("|")[0]
    proxy_url = f"{scheme}://{addr}"
    t0 = time.time()
    codes = []
    passed = 0
    for target in TARGETS:
        try:
            out = subprocess.run(
                ["curl", "-s", "-m", str(timeout), "-x", proxy_url,
                 "-o", "/dev/null", "-w", "%{http_code}",
                 "-A", UA, target],
                capture_output=True, text=True, timeout=timeout + 3)
            code = out.stdout.strip()
            codes.append(code)
            if code.isdigit() and int(code) in PASS_CODES:
                passed += 1
            else:
                # one failed target is enough to fail fast
                break
        except subprocess.TimeoutExpired:
            codes.append("timeout")
            break
        except Exception as e:
            codes.append(type(e).__name__)
            break
    lat = int((time.time() - t0) * 1000)
    ok = passed >= MIN_TARGETS_PASS
    return pp, ok, lat, ",".join(codes)


def load_state():
    if STATE.exists():
        try:
            return json.loads(STATE.read_text())
        except Exception:
            return {}
    return {}


def save_state(state):
    tmp = STATE.with_suffix(".tmp")
    tmp.write_text(json.dumps(state, ensure_ascii=False))
    os.replace(tmp, STATE)


def update(state, results, round_id):
    now = time.time()
    live = []
    for pp, ok, lat, code in results:
        e = state.setdefault(pp, {"first_seen": now, "successes": 0,
                                 "fails": 0, "streak": 0, "rounds": 0,
                                 "last_ok": 0, "last_code": "",
                                 "latencies": []})
        e["rounds"] += 1
        e["last_code"] = code
        if ok:
            e["successes"] += 1
            e["streak"] += 1
            e["fails"] = 0
            e["last_ok"] = now
            e["latencies"].append(lat)
            e["latencies"] = e["latencies"][-20:]
            live.append(pp)
        else:
            e["fails"] += 1
            e["streak"] = 0
    # evict dead weight; veterans get a longer leash (they blip sometimes)
    evicted = 0
    for pp in list(state.keys()):
        e = state[pp]
        is_vet = (e["rounds"] >= VET_MIN_ROUNDS
                  and (e["successes"] / max(1, e["successes"] + e["fails"])) >= VET_UPTIME_MIN)
        limit = VET_MAX_FAILS if is_vet else MAX_AGE_FAILS
        if e["fails"] >= limit:
            del state[pp]
            evicted += 1
    return live, evicted


def _avg_lat(e):
    return sum(e["latencies"]) / len(e["latencies"]) if e["latencies"] else 9999


def write_lists(state, live):
    live_set = set(live)
    stable = []
    veterans = []
    now = time.time()
    for pp, e in state.items():
        if pp not in live_set:
            continue
        total = e["successes"] + e["fails"]
        uptime = e["successes"] / total if total else 0
        age_h = (now - e["first_seen"]) / 3600
        avg_lat = _avg_lat(e)
        if (e["rounds"] >= MIN_ROUNDS and uptime >= UPTIME_MIN
                and e["streak"] >= STREAK_MIN):
            stable.append((pp, avg_lat, uptime, age_h, e["rounds"]))
        if (e["rounds"] >= VET_MIN_ROUNDS and age_h >= VET_MIN_AGE_H
                and uptime >= VET_UPTIME_MIN and e["streak"] >= VET_STREAK_MIN):
            veterans.append((pp, avg_lat, uptime, age_h, e["rounds"]))
    # write live (sorted by latency if known)
    live_sorted = sorted(live, key=lambda p: _avg_lat(state.get(p, {})))
    LIVE.write_text("\n".join(f"http://{pp.split('|')[0]}" for pp in live_sorted) + "\n")
    stable_sorted = sorted(stable, key=lambda x: x[1])
    STABLE.write_text("\n".join(
        f"http://{pp.split('|')[0]}\t{lat:.0f}ms\tup={up:.0%}\tage={age:.1f}h\trounds={r}"
        for pp, lat, up, age, r in stable_sorted) + "\n")
    VETERAN.write_text("\n".join(
        f"http://{pp.split('|')[0]}\t{lat:.0f}ms\tup={up:.0%}\tage={age:.1f}h\trounds={r}"
        for pp, lat, up, age, r in sorted(veterans, key=lambda x: x[1])
    ) + ("\n" if veterans else ""))
    return len(stable_sorted), len(veterans)


def run_round(state, workers, round_id, do_scrape, max_new=5000, max_test=10000):
    new_now = []
    if do_scrape:
        fresh = scrape()
        for p in fresh:
            if p not in state:
                state[p] = {"first_seen": time.time(), "successes": 0,
                            "fails": 0, "streak": 0, "rounds": 0,
                            "last_ok": 0, "last_code": "", "latencies": []}
                new_now.append(p)
        log(f"scraped {len(fresh)} candidates, {len(new_now)} new, {len(state)} total")
    # Always re-test known proxies first; add a bounded batch of untested new ones.
    known = [p for p, e in state.items() if e["rounds"] > 0]
    untested = [p for p, e in state.items() if e["rounds"] == 0]
    targets = known + untested[:max_new]
    if len(targets) > max_test:
        # keep all known, trim newest if still over
        targets = targets[:max_test]
    log(f"round {round_id}: testing {len(targets)} proxies "
        f"({len(known)} known, {len(targets)-len(known)} new; workers={workers})")
    results = []
    t0 = time.time()
    with ThreadPoolExecutor(max_workers=workers) as ex:
        futs = [ex.submit(test_proxy, p) for p in targets]
        for f in as_completed(futs):
            results.append(f.result())
    live, evicted = update(state, results, round_id)
    n_stable, n_vet = write_lists(state, live)
    save_state(state)
    log(f"round {round_id} done in {time.time()-t0:.0f}s: {len(live)} live, "
        f"{n_stable} stable, {n_vet} veterans, {evicted} evicted, "
        f"{max(0,len(untested)-max_new)} untested queued")
    return n_stable, n_vet


def show_status():
    state = load_state()
    now = time.time()
    rows = []
    for pp, e in state.items():
        total = e["successes"] + e["fails"]
        up = e["successes"] / total if total else 0
        age_h = (now - e["first_seen"]) / 3600
        rows.append((pp, _avg_lat(e), up, age_h, e["rounds"], e["streak"],
                     e.get("last_code", "")))
    live = [r for r in rows if r[5] > 0]
    stable = [r for r in live if r[4] >= MIN_ROUNDS and r[2] >= UPTIME_MIN and r[5] >= STREAK_MIN]
    vets = [r for r in live if r[4] >= VET_MIN_ROUNDS and r[3] >= VET_MIN_AGE_H
            and r[2] >= VET_UPTIME_MIN and r[5] >= VET_STREAK_MIN]
    print(f"tracked: {len(rows)} | live now: {len(live)} | stable: {len(stable)} | veterans: {len(vets)}")
    if vets:
        print("\n=== VETERANS (durable) ===")
        for pp, lat, up, age, rnd, st, code in sorted(vets, key=lambda x: x[1]):
            print(f"  http://{pp.split('|')[0]}\t{lat:.0f}ms\tup={up:.0%}\tage={age:.1f}h\trounds={rnd}\tstreak={st}")
    if stable:
        print("\n=== STABLE ===")
        for pp, lat, up, age, rnd, st, code in sorted(stable, key=lambda x: x[1])[:15]:
            print(f"  http://{pp.split('|')[0]}\t{lat:.0f}ms\tup={up:.0%}\tage={age:.1f}h\trounds={rnd}\tstreak={st}")


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--once", action="store_true")
    ap.add_argument("--status", action="store_true", help="print current pool status and exit")
    ap.add_argument("--workers", type=int, default=300)
    args = ap.parse_args()

    if args.status:
        show_status()
        return

    state = load_state()
    log(f"loaded {len(state)} known proxies from state")
    round_id = 0
    last_scrape = 0
    while True:
        round_id += 1
        do_scrape = (time.time() - last_scrape) > SCRAPE_INTERVAL or last_scrape == 0
        if do_scrape:
            last_scrape = time.time()
        try:
            run_round(state, args.workers, round_id, do_scrape)
        except Exception as e:
            log(f"round error: {type(e).__name__}: {e}")
        if args.once:
            break
        time.sleep(RECHECK_INTERVAL)


if __name__ == "__main__":
    main()