Initial commit: grok-free-register-oss

Open-source Grok free registration CLI, xai_enroller auth pipeline,
local auth service, tests and docs.

grok_register/core/admission.py

@@ -0,0 +1,129 @@
+"""Local admission gates for CSP workers.
+
+AdmissionGate is not a scheduler: it never chooses a role and never moves
+resources. It only lets producers reserve local production intent against
+inventory watermarks, so fixed S/P workers can self-throttle without a central
+dispatcher.
+"""
+import asyncio
+
+
+class _TProductionLease:
+    __slots__ = ("_gate", "_released")
+
+    def __init__(self, gate):
+        self._gate = gate
+        self._released = False
+
+    async def release(self):
+        if self._released:
+            return
+        self._released = True
+        async with self._gate._cond:
+            self._gate.t_in_progress -= 1
+            self._gate._cond.notify_all()
+
+
+class _QBatchLease:
+    __slots__ = ("_gate", "count", "_remaining")
+
+    def __init__(self, gate, count):
+        self._gate = gate
+        self.count = count
+        self._remaining = count
+
+    async def release_one(self):
+        if self._remaining <= 0:
+            return
+        async with self._gate._cond:
+            if self._remaining <= 0:
+                return
+            self._remaining -= 1
+            self._gate.q_inflight -= 1
+            self._gate._cond.notify_all()
+
+    async def release_all(self):
+        if self._remaining <= 0:
+            return
+        async with self._gate._cond:
+            if self._remaining <= 0:
+                return
+            self._gate.q_inflight -= self._remaining
+            self._remaining = 0
+            self._gate._cond.notify_all()
+
+
+class AdmissionGate:
+    """Watermark-based producer admission for T and Q.
+
+    T production uses high/low hysteresis because local T generation can be
+    heavy and should not keep filling an already sufficient inventory.
+
+    Q production reserves a batch by current demand:
+        q_high - q_depth - q_inflight
+    capped by the worker's max batch. Each reservation is released when the
+    corresponding pending Q requests have reached terminal state.
+    """
+
+    def __init__(self, inventory, *, t_low, t_high, q_low, q_high):
+        if t_low < 0 or q_low < 0:
+            raise ValueError("low watermarks must be non-negative")
+        if t_high <= 0 or q_high <= 0:
+            raise ValueError("high watermarks must be positive")
+        if t_low > t_high:
+            raise ValueError("t_low must be <= t_high")
+        if q_low > q_high:
+            raise ValueError("q_low must be <= q_high")
+
+        self.inventory = inventory
+        self.t_low = t_low
+        self.t_high = t_high
+        self.q_low = q_low
+        self.q_high = q_high
+        self.t_in_progress = 0
+        self.q_inflight = 0
+        self._t_paused = False
+        self._q_paused = False
+        self._cond = asyncio.Condition()
+
+    async def acquire_t_production(self):
+        """Reserve one T production attempt."""
+        async with self._cond:
+            while True:
+                total = self.inventory.t_depth + self.t_in_progress
+                if total >= self.t_high:
+                    self._t_paused = True
+                if self._t_paused and total <= self.t_low:
+                    self._t_paused = False
+                if not self._t_paused and total < self.t_high:
+                    self.t_in_progress += 1
+                    self._cond.notify_all()
+                    return _TProductionLease(self)
+                await self._cond.wait()
+
+    async def acquire_q_batch(self, *, max_batch):
+        """Reserve up to max_batch pending Q requests based on current demand."""
+        if max_batch <= 0:
+            raise ValueError("max_batch must be positive")
+
+        async with self._cond:
+            while True:
+                total = self.inventory.q_depth + self.q_inflight
+                if total >= self.q_high:
+                    self._q_paused = True
+                if self._q_paused and total <= self.q_low:
+                    self._q_paused = False
+                demand = self.q_high - total
+                if not self._q_paused and demand > 0:
+                    count = min(max_batch, demand)
+                    self.q_inflight += count
+                    if self.inventory.q_depth + self.q_inflight >= self.q_high:
+                        self._q_paused = True
+                    self._cond.notify_all()
+                    return _QBatchLease(self, count)
+                await self._cond.wait()
+
+    async def notify_changed(self):
+        """Wake waiters after inventory depth changes."""
+        async with self._cond:
+            self._cond.notify_all()

grok_register/core/envelope.py

@@ -0,0 +1,61 @@
+"""
+ResourceEnvelope — T/Q 资源实体与库存 slot 的生命周期绑定
+
+slot 获取与 envelope 创建必须通过 create_with_slot() 完成。
+release_slot_once() 幂等:重复调用只释放一次。
+"""
+import time
+import asyncio
+
+
+class ResourceEnvelope:
+    """绑定一个 T/Q 资源值与其对应的库存 slot。"""
+
+    __slots__ = ('kind', 'value', 'slot_sem', '_released', 'created_at', 'expires_at', 'meta')
+
+    def __init__(self, kind, value, slot_sem, *, created_at, expires_at=None, meta=None):
+        self.kind = kind          # 'T' or 'Q'
+        self.value = value        # T: turnstile token str; Q: {'email','password','code'}
+        self.slot_sem = slot_sem  # T_Slot_Sem or Q_Slot_Sem
+        self._released = False
+        self.created_at = created_at
+        self.expires_at = expires_at
+        self.meta = meta or {}
+
+    @classmethod
+    async def create_with_slot(cls, kind, value, slot_sem, *, expires_at=None, meta=None):
+        """先获取库存 slot,再构造 envelope。slot 获取失败则抛异常,不创建 envelope。"""
+        await slot_sem.acquire()
+        try:
+            return cls(
+                kind, value, slot_sem,
+                created_at=time.time(),
+                expires_at=expires_at,
+                meta=meta,
+            )
+        except BaseException:
+            slot_sem.release()
+            raise
+
+    def release_slot_once(self, reason=''):
+        """幂等释放 slot。重复调用安全。"""
+        if not self._released:
+            self._released = True
+            self.slot_sem.release()
+
+    def discard(self, reason=''):
+        """资源未被消费,释放库存占用。"""
+        self.release_slot_once(reason=f'discard:{reason}')
+
+    def is_expired(self, now=None):
+        """检查资源是否过期。"""
+        if self.expires_at is None:
+            return False
+        return (now or time.time()) > self.expires_at
+
+    @property
+    def released(self):
+        return self._released
+
+    def __repr__(self):
+        return f'Envelope({self.kind}, released={self._released})'

grok_register/core/inventory.py

@@ -0,0 +1,169 @@
+"""
+Inventory — 唯一库存门面 + PairLease
+
+put_t / put_q: 所有权从 Worker 转移到 Inventory
+claim_pair(): 等待完整 T/Q pair,通过 PairLease 原子转移所有权
+内部单锁 + Condition,不做分片,不做无锁。
+"""
+import asyncio
+import time
+from collections import deque
+
+from .envelope import ResourceEnvelope
+
+
+class PairLease:
+    """claim_pair 成功后的 pair 所有权对象。
+
+    用法:
+        async with inventory.claim_pair() as pair:
+            t_val, q_val = pair.t.value, pair.q.value
+            ...
+    离开上下文时无论成功/失败/取消,都核销 T/Q 并释放 slot。
+    """
+
+    __slots__ = ('t', 'q', '_inventory')
+
+    def __init__(self, t_env, q_env, inventory):
+        self.t = t_env
+        self.q = q_env
+        self._inventory = inventory
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb):
+        # 第一版策略: 无论结果如何,均核销 T/Q
+        self.t.release_slot_once(reason='pair_settle')
+        self.q.release_slot_once(reason='pair_settle')
+        # 通知可能在等待的 claim_pair
+        async with self._inventory._lock:
+            self._inventory._cond.notify_all()
+        return False
+
+
+class Inventory:
+    """T/Q 库存门面。
+
+    内部维护两个 deque 和一个 Lock+Condition。
+    put_t/put_q 在锁内 O(1) 操作;claim_pair 在锁外等待 Condition。
+    """
+
+    def __init__(self, metrics=None):
+        self._t_buf = deque()
+        self._q_buf = deque()
+        self._lock = asyncio.Lock()
+        self._cond = asyncio.Condition(self._lock)
+        self._metrics = metrics
+
+    # ── 入库 ──
+
+    async def put_t(self, env: ResourceEnvelope):
+        """将 T envelope 入库。调用前 env 属于 Worker,成功后属于 Inventory。"""
+        async with self._lock:
+            now = time.time()
+            if env.is_expired(now):
+                env.discard(reason='expired_on_put')
+                if self._metrics:
+                    self._metrics.t_expired += 1
+                return
+            self._cleanup_expired(now)
+            self._t_buf.append(env)
+            if self._metrics:
+                self._metrics.t_admitted += 1
+            self._cond.notify_all()
+
+    async def put_q(self, env: ResourceEnvelope):
+        """将 Q envelope 入库。调用前 env 属于 Worker,成功后属于 Inventory。"""
+        async with self._lock:
+            now = time.time()
+            if env.is_expired(now):
+                env.discard(reason='expired_on_put')
+                if self._metrics:
+                    self._metrics.q_expired += 1
+                return
+            self._cleanup_expired(now)
+            self._q_buf.append(env)
+            if self._metrics:
+                self._metrics.q_admitted += 1
+            self._cond.notify_all()
+
+    # ── Claim ──
+
+    def claim_pair(self):
+        """返回 PairLease 的 async context manager。
+
+        等待期间可取消,取消不弹出资源。成功后 T/Q 原子弹出并转移给 PairLease。
+        """
+        return _PairClaim(self, self._metrics)
+
+    # ── 内部 ──
+
+    def _try_pop_pair(self):
+        """在锁内尝试弹出一对未过期的 T/Q。返回 (t_env, q_env) 或 None。"""
+        now = time.time()
+        # 清理过期
+        self._cleanup_expired(now)
+        if self._t_buf and self._q_buf:
+            t_env = self._t_buf.popleft()
+            q_env = self._q_buf.popleft()
+            return t_env, q_env
+        return None
+
+    def _cleanup_expired(self, now):
+        """lazy cleanup: 清理已过期的 T/Q 并释放 slot。锁内调用。"""
+        self._cleanup_buf(self._t_buf, now, 't_expired')
+        self._cleanup_buf(self._q_buf, now, 'q_expired')
+
+    def _cleanup_buf(self, buf, now, metric_name):
+        """扫描完整库存,保留未过期实体的原始 FIFO 顺序。"""
+        kept = deque()
+        while buf:
+            env = buf.popleft()
+            if env.is_expired(now):
+                env.discard(reason='expired_in_inventory')
+                if self._metrics:
+                    setattr(self._metrics, metric_name, getattr(self._metrics, metric_name) + 1)
+            else:
+                kept.append(env)
+        buf.extend(kept)
+
+    @property
+    def t_depth(self):
+        return len(self._t_buf)
+
+    @property
+    def q_depth(self):
+        return len(self._q_buf)
+
+
+class _PairClaim:
+    """claim_pair() 返回的 async context manager。"""
+
+    __slots__ = ('_inventory', '_pair', '_metrics')
+
+    def __init__(self, inventory, metrics):
+        self._inventory = inventory
+        self._pair = None
+        self._metrics = metrics
+
+    async def __aenter__(self):
+        inv = self._inventory
+        async with inv._lock:
+            # 循环等待直到有 pair 或被取消
+            while True:
+                pair = inv._try_pop_pair()
+                if pair is not None:
+                    t_env, q_env = pair
+                    self._pair = PairLease(t_env, q_env, inv)
+                    if self._metrics:
+                        self._metrics.pair_claimed += 1
+                    return self._pair
+                # 没有 pair,等待 notify
+                await inv._cond.wait()
+
+    async def __aexit__(self, exc_type, exc, tb):
+        if self._pair is not None:
+            # PairLease.__aexit__ 处理核销
+            return await self._pair.__aexit__(exc_type, exc, tb)
+        return False

grok_register/core/observer.py

@@ -0,0 +1,215 @@
+"""
+Observer — 只读观测层
+
+只记录指标、生成日志。不修改 Semaphore,不调度 Worker,不释放资源。
+"""
+import time
+from collections import deque
+
+
+class Metrics:
+    """全局指标收集器。所有字段只在 asyncio 单线程中写入,无需加锁。"""
+
+    __slots__ = (
+        'start_time',
+        't_produced', 't_admitted', 't_claimed', 't_expired', 't_discarded',
+        't_solve_count', 't_solve_seconds', 't_solve_failed',
+        'solver_goto_seconds', 'solver_inject_seconds', 'solver_initial_seconds',
+        'solver_click_seconds', 'solver_wait_seconds', 'solver_reused_count',
+        'solver_visible_frame_count',
+        's_physical_count', 's_physical_wait_seconds', 's_physical_hold_seconds',
+        'p_physical_count', 'p_physical_wait_seconds', 'p_physical_hold_seconds',
+        'c_physical_count', 'c_physical_wait_seconds', 'c_physical_hold_seconds',
+        'p_email_create_count', 'p_email_create_seconds',
+        'p_page_prepare_count', 'p_page_prepare_seconds',
+        'p_send_count', 'p_send_seconds',
+        'c_page_acquire_count', 'c_page_acquire_seconds',
+        'c_verify_count', 'c_verify_seconds',
+        'c_register_count', 'c_register_seconds',
+        'c_hot_page_hits', 'c_hot_page_misses',
+        'q_sent', 'q_returned', 'q_admitted', 'q_claimed', 'q_expired', 'q_discarded',
+        'q_send_batches', 'q_send_batch_items',
+        'pair_claimed', 'pair_consumed_ok', 'pair_consumed_fail',
+        'success_count',
+        'registration_starts',
+        '_clock', 'started_monotonic', 'recent_success_times',
+    )
+
+    def __init__(self, clock=time.monotonic):
+        self._clock = clock
+        self.started_monotonic = clock()
+        self.recent_success_times = deque()
+        self.start_time = time.time()
+        # T 生命周期
+        self.t_produced = 0
+        self.t_admitted = 0
+        self.t_claimed = 0
+        self.t_expired = 0
+        self.t_discarded = 0
+        self.t_solve_count = 0
+        self.t_solve_seconds = 0.0
+        self.t_solve_failed = 0
+        self.solver_goto_seconds = 0.0
+        self.solver_inject_seconds = 0.0
+        self.solver_initial_seconds = 0.0
+        self.solver_click_seconds = 0.0
+        self.solver_wait_seconds = 0.0
+        self.solver_reused_count = 0
+        self.solver_visible_frame_count = 0
+        self.s_physical_count = 0
+        self.s_physical_wait_seconds = 0.0
+        self.s_physical_hold_seconds = 0.0
+        self.p_physical_count = 0
+        self.p_physical_wait_seconds = 0.0
+        self.p_physical_hold_seconds = 0.0
+        self.c_physical_count = 0
+        self.c_physical_wait_seconds = 0.0
+        self.c_physical_hold_seconds = 0.0
+        self.p_email_create_count = 0
+        self.p_email_create_seconds = 0.0
+        self.p_page_prepare_count = 0
+        self.p_page_prepare_seconds = 0.0
+        self.p_send_count = 0
+        self.p_send_seconds = 0.0
+        self.c_page_acquire_count = 0
+        self.c_page_acquire_seconds = 0.0
+        self.c_verify_count = 0
+        self.c_verify_seconds = 0.0
+        self.c_register_count = 0
+        self.c_register_seconds = 0.0
+        self.c_hot_page_hits = 0
+        self.c_hot_page_misses = 0
+        # Q 生命周期
+        self.q_sent = 0
+        self.q_returned = 0
+        self.q_admitted = 0
+        self.q_claimed = 0
+        self.q_expired = 0
+        self.q_discarded = 0
+        self.q_send_batches = 0
+        self.q_send_batch_items = 0
+        # Pair
+        self.pair_claimed = 0
+        self.pair_consumed_ok = 0
+        self.pair_consumed_fail = 0
+        # 成功数
+        self.success_count = 0
+        self.registration_starts = 0
+
+    def next_registration_task(self):
+        self.registration_starts += 1
+        return self.registration_starts
+
+    def record_success(self):
+        self.success_count += 1
+        self.recent_success_times.append(self._clock())
+
+    def five_minute_success_rate(self):
+        now = self._clock()
+        cutoff = now - 300.0
+        while self.recent_success_times and self.recent_success_times[0] < cutoff:
+            self.recent_success_times.popleft()
+        if not self.recent_success_times:
+            return None if self.success_count == 0 else 0.0
+        elapsed = max(1.0, min(300.0, now - self.started_monotonic))
+        return len(self.recent_success_times) * 60.0 / elapsed
+
+    def runtime_average_success_rate(self):
+        if self.success_count == 0:
+            return None
+        elapsed = max(1.0, self._clock() - self.started_monotonic)
+        return self.success_count * 60.0 / elapsed
+
+    def snapshot(self, inventory, sems):
+        """生成一行监控日志。"""
+        elapsed = time.time() - self.start_time
+        rate = self.success_count / (elapsed / 60) if elapsed > 60 else 0
+        p_batch_avg = (
+            self.q_send_batch_items / self.q_send_batches
+            if self.q_send_batches else 0
+        )
+        t_solve_avg = (
+            self.t_solve_seconds / self.t_solve_count
+            if self.t_solve_count else 0
+        )
+        solver_goto_avg = (
+            self.solver_goto_seconds / self.t_solve_count
+            if self.t_solve_count else 0
+        )
+        solver_inject_avg = (
+            self.solver_inject_seconds / self.t_solve_count
+            if self.t_solve_count else 0
+        )
+        solver_initial_avg = (
+            self.solver_initial_seconds / self.t_solve_count
+            if self.t_solve_count else 0
+        )
+        solver_click_avg = (
+            self.solver_click_seconds / self.t_solve_count
+            if self.t_solve_count else 0
+        )
+        solver_wait_avg = (
+            self.solver_wait_seconds / self.t_solve_count
+            if self.t_solve_count else 0
+        )
+        solver_reuse_ratio = (
+            self.solver_reused_count / self.t_solve_count
+            if self.t_solve_count else 0
+        )
+        solver_visible_ratio = (
+            self.solver_visible_frame_count / self.t_solve_count
+            if self.t_solve_count else 0
+        )
+        s_phys_wait, s_phys_hold = self._avg_pair(
+            self.s_physical_wait_seconds, self.s_physical_hold_seconds, self.s_physical_count
+        )
+        p_phys_wait, p_phys_hold = self._avg_pair(
+            self.p_physical_wait_seconds, self.p_physical_hold_seconds, self.p_physical_count
+        )
+        c_phys_wait, c_phys_hold = self._avg_pair(
+            self.c_physical_wait_seconds, self.c_physical_hold_seconds, self.c_physical_count
+        )
+        p_email_create = self._avg(self.p_email_create_seconds, self.p_email_create_count)
+        p_page_prepare = self._avg(self.p_page_prepare_seconds, self.p_page_prepare_count)
+        p_send_stage = self._avg(self.p_send_seconds, self.p_send_count)
+        c_page_acquire = self._avg(self.c_page_acquire_seconds, self.c_page_acquire_count)
+        c_verify = self._avg(self.c_verify_seconds, self.c_verify_count)
+        c_register = self._avg(self.c_register_seconds, self.c_register_count)
+        p_send_sem = sems.get("p_send")
+        admission = sems.get("admission")
+        p_send_part = f' p_send:{p_send_sem._value}' if p_send_sem is not None else ''
+        admission_part = (
+            f' t_prog:{admission.t_in_progress} q_inflight:{admission.q_inflight}'
+            if admission is not None else ''
+        )
+        return (
+            f'[*] T:{inventory.t_depth} Q:{inventory.q_depth} '
+            f'phys:{sems["physical"]._value}{p_send_part} t_slot:{sems["t_slot"]._value} '
+            f'q_slot:{sems["q_slot"]._value} q_pend:{sems["q_pending"]._value} '
+            f'p_batch:{p_batch_avg:.1f}{admission_part} '
+            f's_phys:{s_phys_wait:.2f}/{s_phys_hold:.2f} '
+            f'p_phys:{p_phys_wait:.2f}/{p_phys_hold:.2f} '
+            f'c_phys:{c_phys_wait:.2f}/{c_phys_hold:.2f} '
+            f'p_stage:{p_email_create:.2f}/{p_page_prepare:.2f}/{p_send_stage:.2f} '
+            f'c_stage:{c_page_acquire:.2f}/{c_verify:.2f}/{c_register:.2f} '
+            f'c_hot:{self.c_hot_page_hits}/{self.c_hot_page_misses} '
+            f't_solve_avg:{t_solve_avg:.1f} t_solve_fail:{self.t_solve_failed} '
+            f'solver_goto:{solver_goto_avg:.2f} solver_inject:{solver_inject_avg:.2f} '
+            f'solver_initial:{solver_initial_avg:.2f} solver_click:{solver_click_avg:.2f} '
+            f'solver_wait:{solver_wait_avg:.2f} solver_reuse:{solver_reuse_ratio:.2f} '
+            f'solver_visible:{solver_visible_ratio:.2f} '
+            f't_prod:{self.t_produced} t_adm:{self.t_admitted} t_exp:{self.t_expired} '
+            f'q_sent:{self.q_sent} q_ret:{self.q_returned} q_adm:{self.q_admitted} q_exp:{self.q_expired} '
+            f'pair:{self.pair_claimed} ok:{self.pair_consumed_ok} fail:{self.pair_consumed_fail} '
+            f'rate:{rate:.1f}/min #{self.success_count}'
+        )
+
+    @staticmethod
+    def _avg(total, count):
+        return total / count if count else 0
+
+    @staticmethod
+    def _avg_pair(wait_total, hold_total, count):
+        if not count:
+            return 0, 0
+        return wait_total / count, hold_total / count