336 lines
14 KiB
Python
336 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Persistent self-healing proxy pool — favors DURABLE proxies.
|
|
|
|
Free public proxies die within minutes. Instead of trusting whatever works
|
|
right now, this daemon:
|
|
|
|
1. scrapes fresh candidates from many public lists every REFRESH_CYCLES,
|
|
2. re-tests EVERY known proxy against the REAL target (api.github.com) every
|
|
TEST_INTERVAL seconds,
|
|
3. tracks per-proxy uptime, consecutive successes, and survival time,
|
|
4. exposes two files for hunters:
|
|
github_live.txt — all proxies that passed the last round
|
|
github_stable.txt — durable proxies: survived MIN_ROUNDS rounds with
|
|
>= UPTIME_MIN uptime and a recent success
|
|
|
|
State persists in pool_state.json so durability accrues across restarts.
|
|
Run it once and leave it running (or schedule it). Logs to proxy_pool.log.
|
|
|
|
Usage:
|
|
python3 proxy_pool.py --once # one scrape + test round, then exit
|
|
python3 proxy_pool.py # run forever
|
|
python3 proxy_pool.py --workers 200
|
|
"""
|
|
import argparse, json, os, re, socket, subprocess, time, urllib.request, urllib.error
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from pathlib import Path
|
|
from datetime import datetime, timezone
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
OUT = HERE / "results" / "proxies"
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
STATE = OUT / "pool_state.json"
|
|
LIVE = OUT / "github_live.txt"
|
|
STABLE = OUT / "github_stable.txt"
|
|
VETERAN = OUT / "github_veteran.txt"
|
|
LOG = OUT / "proxy_pool.log"
|
|
|
|
UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/124.0 Safari/537.36"
|
|
|
|
# The actual things we need the proxy for: reaching GitHub API AND raw content.
|
|
# A proxy must reach BOTH to be considered "stable" (some proxies MITM or only
|
|
# allow certain hosts). 200/401/403/429/404 all mean "the proxy reached GitHub".
|
|
TARGETS = [
|
|
"https://api.github.com/zen",
|
|
"https://raw.githubusercontent.com/torvalds/linux/master/README",
|
|
]
|
|
PASS_CODES = {200, 401, 403, 429, 404}
|
|
# A proxy must pass at least this many of the TARGETS to count as a live round.
|
|
MIN_TARGETS_PASS = 2
|
|
|
|
SOURCES = [
|
|
"https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt",
|
|
"https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks5.txt",
|
|
"https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt",
|
|
"https://raw.githubusercontent.com/monosans/proxy-list/main/proxies_anonymous/http.txt",
|
|
"https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt",
|
|
"https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt",
|
|
"https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/http.txt",
|
|
"https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/https.txt",
|
|
"https://raw.githubusercontent.com/roosterkid/openproxylist/main/HTTPS_RAW.txt",
|
|
"https://raw.githubusercontent.com/mmpx12/proxy-list/master/http.txt",
|
|
"https://raw.githubusercontent.com/mmpx12/proxy-list/master/https.txt",
|
|
"https://raw.githubusercontent.com/proxifly/free-proxy-list/main/proxies/protocols/http/data.txt",
|
|
"https://raw.githubusercontent.com/proxifly/free-proxy-list/main/proxies/countries/CN/data.txt",
|
|
"https://raw.githubusercontent.com/zloi-user/hideip.me/main/http.txt",
|
|
"https://raw.githubusercontent.com/zloi-user/hideip.me/main/https.txt",
|
|
"https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt",
|
|
"https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt",
|
|
"https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt",
|
|
"https://raw.githubusercontent.com/Zaeem20/FREE_PROXIES_LIST/master/http.txt",
|
|
"https://raw.githubusercontent.com/Zaeem20/FREE_PROXIES_LIST/master/https.txt",
|
|
"https://raw.githubusercontent.com/officialputuid/KangProxy/KangProxy/http/http.txt",
|
|
"https://raw.githubusercontent.com/officialputuid/KangProxy/KangProxy/https/https.txt",
|
|
"https://raw.githubusercontent.com/yemixzy/proxy-list/main/proxies/http.txt",
|
|
"https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt",
|
|
"https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/https.txt",
|
|
"https://raw.githubusercontent.com/proxy4parsing/proxy-list/main/http.txt",
|
|
"https://raw.githubusercontent.com/Anonym0usWork1221/Free-Proxies/main/proxy_files/http_proxies.txt",
|
|
]
|
|
|
|
# Durability thresholds
|
|
MIN_ROUNDS = 3 # must have been tested at least this many rounds
|
|
UPTIME_MIN = 0.50 # and succeed at least this fraction of the time
|
|
STREAK_MIN = 2 # and be alive on the last 2 consecutive rounds
|
|
# Veteran tier: the truly durable proxies that survive over time.
|
|
VET_MIN_ROUNDS = 12 # at least 12 rounds of history
|
|
VET_MIN_AGE_H = 1.0 # survived at least 1 wall-clock hour
|
|
VET_UPTIME_MIN = 0.70 # 70%+ uptime
|
|
VET_STREAK_MIN = 3 # alive the last 3 rounds
|
|
TEST_TIMEOUT = 5
|
|
MAX_AGE_FAILS = 8 # forget a NEW proxy after this many consecutive failures
|
|
VET_MAX_FAILS = 20 # give veterans a longer leash (they sometimes blip)
|
|
RECHECK_INTERVAL = 60 # re-test all live/known proxies every N seconds
|
|
SCRAPE_INTERVAL = 1800 # re-scrape sources every N seconds
|
|
|
|
IPPORT_RE = re.compile(rb"\b(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d{2,5})\b")
|
|
DIRECT = urllib.request.build_opener()
|
|
|
|
|
|
def log(msg):
|
|
line = f"[{datetime.now().strftime('%H:%M:%S')}] {msg}"
|
|
print(line, flush=True)
|
|
with LOG.open("a") as f:
|
|
f.write(line + "\n")
|
|
|
|
|
|
def scrape():
|
|
found = set()
|
|
for url in SOURCES:
|
|
try:
|
|
req = urllib.request.Request(url, headers={"User-Agent": UA})
|
|
with DIRECT.open(req, timeout=20) as r:
|
|
data = r.read()
|
|
for m in IPPORT_RE.finditer(data):
|
|
ip = m.group(1).decode()
|
|
port = int(m.group(2))
|
|
octets = [int(x) for x in ip.split(".")]
|
|
if all(0 <= o <= 255 for o in octets) and 1 <= port <= 65535:
|
|
found.add(f"{ip}:{port}")
|
|
except Exception:
|
|
continue
|
|
return found
|
|
|
|
|
|
def test_proxy(pp, timeout=TEST_TIMEOUT):
|
|
"""Return (pp, ok, latency_ms, code_or_err). Uses curl because Python's
|
|
ProxyHandler falsely reports success when the tunnel hangs mid-transfer.
|
|
A proxy is OK only if it reaches MIN_TARGETS_PASS of TARGETS."""
|
|
scheme = "socks5" if pp.endswith("|s5") else "http"
|
|
addr = pp.split("|")[0]
|
|
proxy_url = f"{scheme}://{addr}"
|
|
t0 = time.time()
|
|
codes = []
|
|
passed = 0
|
|
for target in TARGETS:
|
|
try:
|
|
out = subprocess.run(
|
|
["curl", "-s", "-m", str(timeout), "-x", proxy_url,
|
|
"-o", "/dev/null", "-w", "%{http_code}",
|
|
"-A", UA, target],
|
|
capture_output=True, text=True, timeout=timeout + 3)
|
|
code = out.stdout.strip()
|
|
codes.append(code)
|
|
if code.isdigit() and int(code) in PASS_CODES:
|
|
passed += 1
|
|
else:
|
|
# one failed target is enough to fail fast
|
|
break
|
|
except subprocess.TimeoutExpired:
|
|
codes.append("timeout")
|
|
break
|
|
except Exception as e:
|
|
codes.append(type(e).__name__)
|
|
break
|
|
lat = int((time.time() - t0) * 1000)
|
|
ok = passed >= MIN_TARGETS_PASS
|
|
return pp, ok, lat, ",".join(codes)
|
|
|
|
|
|
def load_state():
|
|
if STATE.exists():
|
|
try:
|
|
return json.loads(STATE.read_text())
|
|
except Exception:
|
|
return {}
|
|
return {}
|
|
|
|
|
|
def save_state(state):
|
|
tmp = STATE.with_suffix(".tmp")
|
|
tmp.write_text(json.dumps(state, ensure_ascii=False))
|
|
os.replace(tmp, STATE)
|
|
|
|
|
|
def update(state, results, round_id):
|
|
now = time.time()
|
|
live = []
|
|
for pp, ok, lat, code in results:
|
|
e = state.setdefault(pp, {"first_seen": now, "successes": 0,
|
|
"fails": 0, "streak": 0, "rounds": 0,
|
|
"last_ok": 0, "last_code": "",
|
|
"latencies": []})
|
|
e["rounds"] += 1
|
|
e["last_code"] = code
|
|
if ok:
|
|
e["successes"] += 1
|
|
e["streak"] += 1
|
|
e["fails"] = 0
|
|
e["last_ok"] = now
|
|
e["latencies"].append(lat)
|
|
e["latencies"] = e["latencies"][-20:]
|
|
live.append(pp)
|
|
else:
|
|
e["fails"] += 1
|
|
e["streak"] = 0
|
|
# evict dead weight; veterans get a longer leash (they blip sometimes)
|
|
evicted = 0
|
|
for pp in list(state.keys()):
|
|
e = state[pp]
|
|
is_vet = (e["rounds"] >= VET_MIN_ROUNDS
|
|
and (e["successes"] / max(1, e["successes"] + e["fails"])) >= VET_UPTIME_MIN)
|
|
limit = VET_MAX_FAILS if is_vet else MAX_AGE_FAILS
|
|
if e["fails"] >= limit:
|
|
del state[pp]
|
|
evicted += 1
|
|
return live, evicted
|
|
|
|
|
|
def _avg_lat(e):
|
|
return sum(e["latencies"]) / len(e["latencies"]) if e["latencies"] else 9999
|
|
|
|
|
|
def write_lists(state, live):
|
|
live_set = set(live)
|
|
stable = []
|
|
veterans = []
|
|
now = time.time()
|
|
for pp, e in state.items():
|
|
if pp not in live_set:
|
|
continue
|
|
total = e["successes"] + e["fails"]
|
|
uptime = e["successes"] / total if total else 0
|
|
age_h = (now - e["first_seen"]) / 3600
|
|
avg_lat = _avg_lat(e)
|
|
if (e["rounds"] >= MIN_ROUNDS and uptime >= UPTIME_MIN
|
|
and e["streak"] >= STREAK_MIN):
|
|
stable.append((pp, avg_lat, uptime, age_h, e["rounds"]))
|
|
if (e["rounds"] >= VET_MIN_ROUNDS and age_h >= VET_MIN_AGE_H
|
|
and uptime >= VET_UPTIME_MIN and e["streak"] >= VET_STREAK_MIN):
|
|
veterans.append((pp, avg_lat, uptime, age_h, e["rounds"]))
|
|
# write live (sorted by latency if known)
|
|
live_sorted = sorted(live, key=lambda p: _avg_lat(state.get(p, {})))
|
|
LIVE.write_text("\n".join(f"http://{pp.split('|')[0]}" for pp in live_sorted) + "\n")
|
|
stable_sorted = sorted(stable, key=lambda x: x[1])
|
|
STABLE.write_text("\n".join(
|
|
f"http://{pp.split('|')[0]}\t{lat:.0f}ms\tup={up:.0%}\tage={age:.1f}h\trounds={r}"
|
|
for pp, lat, up, age, r in stable_sorted) + "\n")
|
|
VETERAN.write_text("\n".join(
|
|
f"http://{pp.split('|')[0]}\t{lat:.0f}ms\tup={up:.0%}\tage={age:.1f}h\trounds={r}"
|
|
for pp, lat, up, age, r in sorted(veterans, key=lambda x: x[1])
|
|
) + ("\n" if veterans else ""))
|
|
return len(stable_sorted), len(veterans)
|
|
|
|
|
|
def run_round(state, workers, round_id, do_scrape, max_new=5000, max_test=10000):
|
|
new_now = []
|
|
if do_scrape:
|
|
fresh = scrape()
|
|
for p in fresh:
|
|
if p not in state:
|
|
state[p] = {"first_seen": time.time(), "successes": 0,
|
|
"fails": 0, "streak": 0, "rounds": 0,
|
|
"last_ok": 0, "last_code": "", "latencies": []}
|
|
new_now.append(p)
|
|
log(f"scraped {len(fresh)} candidates, {len(new_now)} new, {len(state)} total")
|
|
# Always re-test known proxies first; add a bounded batch of untested new ones.
|
|
known = [p for p, e in state.items() if e["rounds"] > 0]
|
|
untested = [p for p, e in state.items() if e["rounds"] == 0]
|
|
targets = known + untested[:max_new]
|
|
if len(targets) > max_test:
|
|
# keep all known, trim newest if still over
|
|
targets = targets[:max_test]
|
|
log(f"round {round_id}: testing {len(targets)} proxies "
|
|
f"({len(known)} known, {len(targets)-len(known)} new; workers={workers})")
|
|
results = []
|
|
t0 = time.time()
|
|
with ThreadPoolExecutor(max_workers=workers) as ex:
|
|
futs = [ex.submit(test_proxy, p) for p in targets]
|
|
for f in as_completed(futs):
|
|
results.append(f.result())
|
|
live, evicted = update(state, results, round_id)
|
|
n_stable, n_vet = write_lists(state, live)
|
|
save_state(state)
|
|
log(f"round {round_id} done in {time.time()-t0:.0f}s: {len(live)} live, "
|
|
f"{n_stable} stable, {n_vet} veterans, {evicted} evicted, "
|
|
f"{max(0,len(untested)-max_new)} untested queued")
|
|
return n_stable, n_vet
|
|
|
|
|
|
def show_status():
|
|
state = load_state()
|
|
now = time.time()
|
|
rows = []
|
|
for pp, e in state.items():
|
|
total = e["successes"] + e["fails"]
|
|
up = e["successes"] / total if total else 0
|
|
age_h = (now - e["first_seen"]) / 3600
|
|
rows.append((pp, _avg_lat(e), up, age_h, e["rounds"], e["streak"],
|
|
e.get("last_code", "")))
|
|
live = [r for r in rows if r[5] > 0]
|
|
stable = [r for r in live if r[4] >= MIN_ROUNDS and r[2] >= UPTIME_MIN and r[5] >= STREAK_MIN]
|
|
vets = [r for r in live if r[4] >= VET_MIN_ROUNDS and r[3] >= VET_MIN_AGE_H
|
|
and r[2] >= VET_UPTIME_MIN and r[5] >= VET_STREAK_MIN]
|
|
print(f"tracked: {len(rows)} | live now: {len(live)} | stable: {len(stable)} | veterans: {len(vets)}")
|
|
if vets:
|
|
print("\n=== VETERANS (durable) ===")
|
|
for pp, lat, up, age, rnd, st, code in sorted(vets, key=lambda x: x[1]):
|
|
print(f" http://{pp.split('|')[0]}\t{lat:.0f}ms\tup={up:.0%}\tage={age:.1f}h\trounds={rnd}\tstreak={st}")
|
|
if stable:
|
|
print("\n=== STABLE ===")
|
|
for pp, lat, up, age, rnd, st, code in sorted(stable, key=lambda x: x[1])[:15]:
|
|
print(f" http://{pp.split('|')[0]}\t{lat:.0f}ms\tup={up:.0%}\tage={age:.1f}h\trounds={rnd}\tstreak={st}")
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--once", action="store_true")
|
|
ap.add_argument("--status", action="store_true", help="print current pool status and exit")
|
|
ap.add_argument("--workers", type=int, default=300)
|
|
args = ap.parse_args()
|
|
|
|
if args.status:
|
|
show_status()
|
|
return
|
|
|
|
state = load_state()
|
|
log(f"loaded {len(state)} known proxies from state")
|
|
round_id = 0
|
|
last_scrape = 0
|
|
while True:
|
|
round_id += 1
|
|
do_scrape = (time.time() - last_scrape) > SCRAPE_INTERVAL or last_scrape == 0
|
|
if do_scrape:
|
|
last_scrape = time.time()
|
|
try:
|
|
run_round(state, args.workers, round_id, do_scrape)
|
|
except Exception as e:
|
|
log(f"round error: {type(e).__name__}: {e}")
|
|
if args.once:
|
|
break
|
|
time.sleep(RECHECK_INTERVAL)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|