OpenCode
68a9071cf1
Local branch had diverged from origin/master (sibling commits on the same base). Rewrote local history linearly on top of origin/master, folding in all local content: exa/hot-platform discovery hunters, leaks ledger, nightly sweep orchestrator, updated .gitignore and skill docs, plus the latest hunt output and vault state. Remote-only files (vault keys, channel scripts) were restored rather than dropped, so the resulting tree is a full union of both sides.
1 line
163 KiB
JSON
1 line
163 KiB
JSON
{"repo": "arkmurus/crucix", "files": {"scripts/train/baseline_pod_run.sh": {"3a9645c8ec96260be7edd2c46a655b1e473f8c18": {"t": 1785745944.6446764, "n": 5561, "h": "8edffa038a6e27bb", "c": "#!/bin/bash\n# R-F1455 — SURGICAL v0.2-vs-DeepSeek baseline, run ENTIRELY ON THE POD.\n#\n# Mirrors the ONLY recipe PROVEN to serve on this pod (train_promote_v0_2.sh:\n# pinned deps R-F1345 + serve_eval_shim.py + localhost eval). It deliberately\n# does NOT use vLLM: `pip install vllm` pulls a torch built for a newer CUDA\n# driver than this pod has (driver 565 / found 12070) and crashes EngineCore\n# (\"NVIDIA driver too old\") — that is what made every proxy run return 502.\n# The pod already has working torch 2.4.1+cu124; the shim uses it.\n#\n# Eval runs against http://localhost:8888 (NO RunPod proxy) — the proven path.\n#\n# Env in: DEEPSEEK_API_KEY (for the DeepSeek baseline leg)\n# Out: /workspace/eval/aria_llm_v02_eval.json + deepseek_baseline_eval.json\nset -uo pipefail\n\nBASE_MODEL=\"mistralai/Mistral-7B-Instruct-v0.3\" # v0.2 actual base (R-F1454)\nADAPTER=\"/workspace/checkpoints/aria_llm_v0_2_dpo\" # the DPO LoRA\nNAME=\"aria-llm-v0.2\"\nPORT=8888\nSCRIPTS=\"/workspace/crucix/scripts/train\"\nEVAL_SET=\"${EVAL_SET:-/workspace/datasets/aria_eval_500q.jsonl}\" # R-F1455: overridable (100-Q subset)\nEVAL_DIR=\"/workspace/eval\"\nLOGS=\"/workspace/logs\"\n# Base weights live in the PERSISTENT volume cache (container ~/.cache is wiped on\n# restart). Gated on HF + no token, so resolve strictly from the cache.\nexport HF_HOME=/workspace/.cache/huggingface\nexport HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1\n\nmkdir -p \"$EVAL_DIR\" \"$LOGS\"\nlog(){ echo \"[$(date -u +%H:%M:%S)] $*\"; }\nfail(){ echo \"[FATAL] $*\" >&2; exit 1; }\n\n# ── Preflight (no destructive action before everything is present) ────────────\n[ -d \"$ADAPTER\" ] || fail \"adapter missing: $ADAPTER\"\n[ -f \"$SCRIPTS/serve_eval_shim.py\" ] || fail \"shim missing: $SCRIPTS/serve_eval_shim.py\"\n[ -f \"$SCRIPTS/eval_aria_llm.py\" ] || fail \"eval missing: $SCRIPTS/eval_aria_llm.py\"\n[ -f \"$EVAL_SET\" ] || fail \"eval set missing: $EVAL_SET\"\n[ -d \"$HF_HOME/hub/models--mistralai--Mistral-7B-Instruct-v0.3\" ] \\\n || fail \"Mistral base not in volume cache: $HF_HOME\"\nADAPTER_BASE=$(python3 -c \"import json;print(json.load(open('$ADAPTER/adapter_config.json')).get('base_model_name_or_path',''))\" 2>/dev/null || echo \"\")\n[ \"$ADAPTER_BASE\" = \"$BASE_MODEL\" ] || fail \"adapter base ('$ADAPTER_BASE') != '$BASE_MODEL'\"\nlog \"preflight ok — eval set $(wc -l < \"$EVAL_SET\") lines; adapter base $ADAPTER_BASE\"\n\n# ── Deps: the EXACT pinned coherent set from train_promote_v0_2.sh (R-F1345),\n# serve+eval subset (drop trl/datasets — those are train-only). Mistral's\n# tokenizer NEEDS sentencepiece+protobuf; the shim NEEDS fastapi+uvicorn; the\n# eval NEEDS httpx. Do NOT pipe pip through tail (masks the exit code). ──────────\nlog \"installing pinned serve+eval deps…\"\npip install -q \"transformers==4.46.3\" \"peft==0.13.2\" \"accelerate>=0.34\" \\\n bitsandbytes sentencepiece protobuf fastapi uvicorn httpx \\\n || fail \"dep install failed\"\npython - <<'PY' || fail \"deps import check failed\"\nimport transformers, peft, accelerate, bitsandbytes # noqa\nimport sentencepiece, google.protobuf # noqa — Mistral tokenizer\nimport fastapi, uvicorn, httpx # noqa — shim + eval client\nprint(f\"deps ok: transformers {transformers.__version__} peft {peft.__version__}\")\nPY\n\n# ── Serve v0.2 via the shim (EXACT invocation from train_promote serve()) ─────\npkill -f serve_eval_shim 2>/dev/null || true\npkill -f vllm.entrypoints 2>/dev/null || true\nsleep 3\nADAPTER=\"$ADAPTER\" MODEL_NAME=\"$NAME\" PORT=$PORT BASE_MODEL=\"$BASE_MODEL\" \\\n setsid nohup python \"$SCRIPTS/serve_eval_shim.py\" > \"$LOGS/shim_$NAME.log\" 2>&1 < /dev/null &\nlog \"waiting for shim to serve $NAME (bf16 7B load ~3-5 min)…\"\nready=0\nfor i in $(seq 1 60); do # 60*10s = 10 min cap\n if curl -s --max-time 5 \"http://localhost:$PORT/v1/models\" | grep -q \"$NAME\"; then\n ready=1; log \"shim serving $NAME (attempt $i)\"; break\n fi\n sleep 10\ndone\nif [ \"$ready\" != 1 ]; then\n echo \"=== shim log tail ===\"; tail -40 \"$LOGS/shim_$NAME.log\"\n fail \"shim did not come up in 10 min\"\nfi\n\n# ── Eval v0.2 against localhost (PROVEN path — no RunPod proxy) ───────────────\nlog \"eval v0.2 (500-Q) against localhost…\"\npython \"$SCRIPTS/eval_aria_llm.py\" \\\n --target \"http://localhost:$PORT/v1\" --model \"$NAME\" \\\n --eval-set \"$EVAL_SET\" --out \"$EVAL_DIR/aria_llm_v02_eval.json\" 2>&1 | tee \"$LOGS/eval_v02.log\" \\\n || fail \"v0.2 eval failed\"\n\n# ── Eval DeepSeek baseline — SAME eval set, SAME instrument (apples-to-apples) ─\nif [ -n \"${DEEPSEEK_API_KEY:-}\" ]; then\n log \"eval DeepSeek baseline (deepseek-chat)…\"\n python \"$SCRIPTS/eval_aria_llm.py\" \\\n --target \"https://api.deepseek.com/v1\" --model \"deepseek-chat\" \\\n --api-key \"$DEEPSEEK_API_KEY\" \\\n --eval-set \"$EVAL_SET\" --out \"$EVAL_DIR/deepseek_baseline_eval.json\" 2>&1 | tee \"$LOGS/eval_deepseek.log\" \\\n || log \"WARN: DeepSeek eval failed (see log) — v0.2 result still valid\"\nelse\n log \"WARN: DEEPSEEK_API_KEY not set on pod — skipping DeepSeek baseline\"\nfi\n\n# ── Summary ──────────────────────────────────────────────────────────────────\nlog \"=== BASELINE RESULTS ===\"\npython - <<'PY'\nimport json, os\nfor f in [\"/workspace/eval/aria_llm_v02_eval.json\",\n \"/workspace/eval/deepseek_baseline_eval.json\"]:\n if not os.path.exists(f):\n print(f, \"MISSING\"); continue\n d = json.load(open(f))\n dd = d.get(\"defence_dd\") or {}\n pi = d.get(\"prompt_injection\") or {}\n print(f\"{d.get('model')}: dd_accuracy={dd.get('accuracy')} (n={dd.get('total')}) \"\n f\"| injection pass_rate={pi.get('pass_rate')} leak_rate={pi.get('leak_rate')}\")\nPY\npkill -f serve_eval_shim 2>/dev/null || true\nlog \"done.\"\n"}, "9cc42d8ecc327e1258a09d9a04e53ffd35ecaf30": {"t": 1785758162.61363, "n": 5561, "h": "8edffa038a6e27bb", "c": "#!/bin/bash\n# R-F1455 — SURGICAL v0.2-vs-DeepSeek baseline, run ENTIRELY ON THE POD.\n#\n# Mirrors the ONLY recipe PROVEN to serve on this pod (train_promote_v0_2.sh:\n# pinned deps R-F1345 + serve_eval_shim.py + localhost eval). It deliberately\n# does NOT use vLLM: `pip install vllm` pulls a torch built for a newer CUDA\n# driver than this pod has (driver 565 / found 12070) and crashes EngineCore\n# (\"NVIDIA driver too old\") — that is what made every proxy run return 502.\n# The pod already has working torch 2.4.1+cu124; the shim uses it.\n#\n# Eval runs against http://localhost:8888 (NO RunPod proxy) — the proven path.\n#\n# Env in: DEEPSEEK_API_KEY (for the DeepSeek baseline leg)\n# Out: /workspace/eval/aria_llm_v02_eval.json + deepseek_baseline_eval.json\nset -uo pipefail\n\nBASE_MODEL=\"mistralai/Mistral-7B-Instruct-v0.3\" # v0.2 actual base (R-F1454)\nADAPTER=\"/workspace/checkpoints/aria_llm_v0_2_dpo\" # the DPO LoRA\nNAME=\"aria-llm-v0.2\"\nPORT=8888\nSCRIPTS=\"/workspace/crucix/scripts/train\"\nEVAL_SET=\"${EVAL_SET:-/workspace/datasets/aria_eval_500q.jsonl}\" # R-F1455: overridable (100-Q subset)\nEVAL_DIR=\"/workspace/eval\"\nLOGS=\"/workspace/logs\"\n# Base weights live in the PERSISTENT volume cache (container ~/.cache is wiped on\n# restart). Gated on HF + no token, so resolve strictly from the cache.\nexport HF_HOME=/workspace/.cache/huggingface\nexport HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1\n\nmkdir -p \"$EVAL_DIR\" \"$LOGS\"\nlog(){ echo \"[$(date -u +%H:%M:%S)] $*\"; }\nfail(){ echo \"[FATAL] $*\" >&2; exit 1; }\n\n# ── Preflight (no destructive action before everything is present) ────────────\n[ -d \"$ADAPTER\" ] || fail \"adapter missing: $ADAPTER\"\n[ -f \"$SCRIPTS/serve_eval_shim.py\" ] || fail \"shim missing: $SCRIPTS/serve_eval_shim.py\"\n[ -f \"$SCRIPTS/eval_aria_llm.py\" ] || fail \"eval missing: $SCRIPTS/eval_aria_llm.py\"\n[ -f \"$EVAL_SET\" ] || fail \"eval set missing: $EVAL_SET\"\n[ -d \"$HF_HOME/hub/models--mistralai--Mistral-7B-Instruct-v0.3\" ] \\\n || fail \"Mistral base not in volume cache: $HF_HOME\"\nADAPTER_BASE=$(python3 -c \"import json;print(json.load(open('$ADAPTER/adapter_config.json')).get('base_model_name_or_path',''))\" 2>/dev/null || echo \"\")\n[ \"$ADAPTER_BASE\" = \"$BASE_MODEL\" ] || fail \"adapter base ('$ADAPTER_BASE') != '$BASE_MODEL'\"\nlog \"preflight ok — eval set $(wc -l < \"$EVAL_SET\") lines; adapter base $ADAPTER_BASE\"\n\n# ── Deps: the EXACT pinned coherent set from train_promote_v0_2.sh (R-F1345),\n# serve+eval subset (drop trl/datasets — those are train-only). Mistral's\n# tokenizer NEEDS sentencepiece+protobuf; the shim NEEDS fastapi+uvicorn; the\n# eval NEEDS httpx. Do NOT pipe pip through tail (masks the exit code). ──────────\nlog \"installing pinned serve+eval deps…\"\npip install -q \"transformers==4.46.3\" \"peft==0.13.2\" \"accelerate>=0.34\" \\\n bitsandbytes sentencepiece protobuf fastapi uvicorn httpx \\\n || fail \"dep install failed\"\npython - <<'PY' || fail \"deps import check failed\"\nimport transformers, peft, accelerate, bitsandbytes # noqa\nimport sentencepiece, google.protobuf # noqa — Mistral tokenizer\nimport fastapi, uvicorn, httpx # noqa — shim + eval client\nprint(f\"deps ok: transformers {transformers.__version__} peft {peft.__version__}\")\nPY\n\n# ── Serve v0.2 via the shim (EXACT invocation from train_promote serve()) ─────\npkill -f serve_eval_shim 2>/dev/null || true\npkill -f vllm.entrypoints 2>/dev/null || true\nsleep 3\nADAPTER=\"$ADAPTER\" MODEL_NAME=\"$NAME\" PORT=$PORT BASE_MODEL=\"$BASE_MODEL\" \\\n setsid nohup python \"$SCRIPTS/serve_eval_shim.py\" > \"$LOGS/shim_$NAME.log\" 2>&1 < /dev/null &\nlog \"waiting for shim to serve $NAME (bf16 7B load ~3-5 min)…\"\nready=0\nfor i in $(seq 1 60); do # 60*10s = 10 min cap\n if curl -s --max-time 5 \"http://localhost:$PORT/v1/models\" | grep -q \"$NAME\"; then\n ready=1; log \"shim serving $NAME (attempt $i)\"; break\n fi\n sleep 10\ndone\nif [ \"$ready\" != 1 ]; then\n echo \"=== shim log tail ===\"; tail -40 \"$LOGS/shim_$NAME.log\"\n fail \"shim did not come up in 10 min\"\nfi\n\n# ── Eval v0.2 against localhost (PROVEN path — no RunPod proxy) ───────────────\nlog \"eval v0.2 (500-Q) against localhost…\"\npython \"$SCRIPTS/eval_aria_llm.py\" \\\n --target \"http://localhost:$PORT/v1\" --model \"$NAME\" \\\n --eval-set \"$EVAL_SET\" --out \"$EVAL_DIR/aria_llm_v02_eval.json\" 2>&1 | tee \"$LOGS/eval_v02.log\" \\\n || fail \"v0.2 eval failed\"\n\n# ── Eval DeepSeek baseline — SAME eval set, SAME instrument (apples-to-apples) ─\nif [ -n \"${DEEPSEEK_API_KEY:-}\" ]; then\n log \"eval DeepSeek baseline (deepseek-chat)…\"\n python \"$SCRIPTS/eval_aria_llm.py\" \\\n --target \"https://api.deepseek.com/v1\" --model \"deepseek-chat\" \\\n --api-key \"$DEEPSEEK_API_KEY\" \\\n --eval-set \"$EVAL_SET\" --out \"$EVAL_DIR/deepseek_baseline_eval.json\" 2>&1 | tee \"$LOGS/eval_deepseek.log\" \\\n || log \"WARN: DeepSeek eval failed (see log) — v0.2 result still valid\"\nelse\n log \"WARN: DEEPSEEK_API_KEY not set on pod — skipping DeepSeek baseline\"\nfi\n\n# ── Summary ──────────────────────────────────────────────────────────────────\nlog \"=== BASELINE RESULTS ===\"\npython - <<'PY'\nimport json, os\nfor f in [\"/workspace/eval/aria_llm_v02_eval.json\",\n \"/workspace/eval/deepseek_baseline_eval.json\"]:\n if not os.path.exists(f):\n print(f, \"MISSING\"); continue\n d = json.load(open(f))\n dd = d.get(\"defence_dd\") or {}\n pi = d.get(\"prompt_injection\") or {}\n print(f\"{d.get('model')}: dd_accuracy={dd.get('accuracy')} (n={dd.get('total')}) \"\n f\"| injection pass_rate={pi.get('pass_rate')} leak_rate={pi.get('leak_rate')}\")\nPY\npkill -f serve_eval_shim 2>/dev/null || true\nlog \"done.\"\n"}, "2f51659938376b08c72ed831048a3fef7bdb559a": {"t": 1786067512.5242314, "n": 5561, "h": "8edffa038a6e27bb", "c": "#!/bin/bash\n# R-F1455 — SURGICAL v0.2-vs-DeepSeek baseline, run ENTIRELY ON THE POD.\n#\n# Mirrors the ONLY recipe PROVEN to serve on this pod (train_promote_v0_2.sh:\n# pinned deps R-F1345 + serve_eval_shim.py + localhost eval). It deliberately\n# does NOT use vLLM: `pip install vllm` pulls a torch built for a newer CUDA\n# driver than this pod has (driver 565 / found 12070) and crashes EngineCore\n# (\"NVIDIA driver too old\") — that is what made every proxy run return 502.\n# The pod already has working torch 2.4.1+cu124; the shim uses it.\n#\n# Eval runs against http://localhost:8888 (NO RunPod proxy) — the proven path.\n#\n# Env in: DEEPSEEK_API_KEY (for the DeepSeek baseline leg)\n# Out: /workspace/eval/aria_llm_v02_eval.json + deepseek_baseline_eval.json\nset -uo pipefail\n\nBASE_MODEL=\"mistralai/Mistral-7B-Instruct-v0.3\" # v0.2 actual base (R-F1454)\nADAPTER=\"/workspace/checkpoints/aria_llm_v0_2_dpo\" # the DPO LoRA\nNAME=\"aria-llm-v0.2\"\nPORT=8888\nSCRIPTS=\"/workspace/crucix/scripts/train\"\nEVAL_SET=\"${EVAL_SET:-/workspace/datasets/aria_eval_500q.jsonl}\" # R-F1455: overridable (100-Q subset)\nEVAL_DIR=\"/workspace/eval\"\nLOGS=\"/workspace/logs\"\n# Base weights live in the PERSISTENT volume cache (container ~/.cache is wiped on\n# restart). Gated on HF + no token, so resolve strictly from the cache.\nexport HF_HOME=/workspace/.cache/huggingface\nexport HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1\n\nmkdir -p \"$EVAL_DIR\" \"$LOGS\"\nlog(){ echo \"[$(date -u +%H:%M:%S)] $*\"; }\nfail(){ echo \"[FATAL] $*\" >&2; exit 1; }\n\n# ── Preflight (no destructive action before everything is present) ────────────\n[ -d \"$ADAPTER\" ] || fail \"adapter missing: $ADAPTER\"\n[ -f \"$SCRIPTS/serve_eval_shim.py\" ] || fail \"shim missing: $SCRIPTS/serve_eval_shim.py\"\n[ -f \"$SCRIPTS/eval_aria_llm.py\" ] || fail \"eval missing: $SCRIPTS/eval_aria_llm.py\"\n[ -f \"$EVAL_SET\" ] || fail \"eval set missing: $EVAL_SET\"\n[ -d \"$HF_HOME/hub/models--mistralai--Mistral-7B-Instruct-v0.3\" ] \\\n || fail \"Mistral base not in volume cache: $HF_HOME\"\nADAPTER_BASE=$(python3 -c \"import json;print(json.load(open('$ADAPTER/adapter_config.json')).get('base_model_name_or_path',''))\" 2>/dev/null || echo \"\")\n[ \"$ADAPTER_BASE\" = \"$BASE_MODEL\" ] || fail \"adapter base ('$ADAPTER_BASE') != '$BASE_MODEL'\"\nlog \"preflight ok — eval set $(wc -l < \"$EVAL_SET\") lines; adapter base $ADAPTER_BASE\"\n\n# ── Deps: the EXACT pinned coherent set from train_promote_v0_2.sh (R-F1345),\n# serve+eval subset (drop trl/datasets — those are train-only). Mistral's\n# tokenizer NEEDS sentencepiece+protobuf; the shim NEEDS fastapi+uvicorn; the\n# eval NEEDS httpx. Do NOT pipe pip through tail (masks the exit code). ──────────\nlog \"installing pinned serve+eval deps…\"\npip install -q \"transformers==4.46.3\" \"peft==0.13.2\" \"accelerate>=0.34\" \\\n bitsandbytes sentencepiece protobuf fastapi uvicorn httpx \\\n || fail \"dep install failed\"\npython - <<'PY' || fail \"deps import check failed\"\nimport transformers, peft, accelerate, bitsandbytes # noqa\nimport sentencepiece, google.protobuf # noqa — Mistral tokenizer\nimport fastapi, uvicorn, httpx # noqa — shim + eval client\nprint(f\"deps ok: transformers {transformers.__version__} peft {peft.__version__}\")\nPY\n\n# ── Serve v0.2 via the shim (EXACT invocation from train_promote serve()) ─────\npkill -f serve_eval_shim 2>/dev/null || true\npkill -f vllm.entrypoints 2>/dev/null || true\nsleep 3\nADAPTER=\"$ADAPTER\" MODEL_NAME=\"$NAME\" PORT=$PORT BASE_MODEL=\"$BASE_MODEL\" \\\n setsid nohup python \"$SCRIPTS/serve_eval_shim.py\" > \"$LOGS/shim_$NAME.log\" 2>&1 < /dev/null &\nlog \"waiting for shim to serve $NAME (bf16 7B load ~3-5 min)…\"\nready=0\nfor i in $(seq 1 60); do # 60*10s = 10 min cap\n if curl -s --max-time 5 \"http://localhost:$PORT/v1/models\" | grep -q \"$NAME\"; then\n ready=1; log \"shim serving $NAME (attempt $i)\"; break\n fi\n sleep 10\ndone\nif [ \"$ready\" != 1 ]; then\n echo \"=== shim log tail ===\"; tail -40 \"$LOGS/shim_$NAME.log\"\n fail \"shim did not come up in 10 min\"\nfi\n\n# ── Eval v0.2 against localhost (PROVEN path — no RunPod proxy) ───────────────\nlog \"eval v0.2 (500-Q) against localhost…\"\npython \"$SCRIPTS/eval_aria_llm.py\" \\\n --target \"http://localhost:$PORT/v1\" --model \"$NAME\" \\\n --eval-set \"$EVAL_SET\" --out \"$EVAL_DIR/aria_llm_v02_eval.json\" 2>&1 | tee \"$LOGS/eval_v02.log\" \\\n || fail \"v0.2 eval failed\"\n\n# ── Eval DeepSeek baseline — SAME eval set, SAME instrument (apples-to-apples) ─\nif [ -n \"${DEEPSEEK_API_KEY:-}\" ]; then\n log \"eval DeepSeek baseline (deepseek-chat)…\"\n python \"$SCRIPTS/eval_aria_llm.py\" \\\n --target \"https://api.deepseek.com/v1\" --model \"deepseek-chat\" \\\n --api-key \"$DEEPSEEK_API_KEY\" \\\n --eval-set \"$EVAL_SET\" --out \"$EVAL_DIR/deepseek_baseline_eval.json\" 2>&1 | tee \"$LOGS/eval_deepseek.log\" \\\n || log \"WARN: DeepSeek eval failed (see log) — v0.2 result still valid\"\nelse\n log \"WARN: DEEPSEEK_API_KEY not set on pod — skipping DeepSeek baseline\"\nfi\n\n# ── Summary ──────────────────────────────────────────────────────────────────\nlog \"=== BASELINE RESULTS ===\"\npython - <<'PY'\nimport json, os\nfor f in [\"/workspace/eval/aria_llm_v02_eval.json\",\n \"/workspace/eval/deepseek_baseline_eval.json\"]:\n if not os.path.exists(f):\n print(f, \"MISSING\"); continue\n d = json.load(open(f))\n dd = d.get(\"defence_dd\") or {}\n pi = d.get(\"prompt_injection\") or {}\n print(f\"{d.get('model')}: dd_accuracy={dd.get('accuracy')} (n={dd.get('total')}) \"\n f\"| injection pass_rate={pi.get('pass_rate')} leak_rate={pi.get('leak_rate')}\")\nPY\npkill -f serve_eval_shim 2>/dev/null || true\nlog \"done.\"\n"}, "46eadcb5c58e76d35cb545ca1e7ef989433f04c5": {"t": 1786067720.9320037, "n": 5561, "h": "8edffa038a6e27bb", "c": "#!/bin/bash\n# R-F1455 — SURGICAL v0.2-vs-DeepSeek baseline, run ENTIRELY ON THE POD.\n#\n# Mirrors the ONLY recipe PROVEN to serve on this pod (train_promote_v0_2.sh:\n# pinned deps R-F1345 + serve_eval_shim.py + localhost eval). It deliberately\n# does NOT use vLLM: `pip install vllm` pulls a torch built for a newer CUDA\n# driver than this pod has (driver 565 / found 12070) and crashes EngineCore\n# (\"NVIDIA driver too old\") — that is what made every proxy run return 502.\n# The pod already has working torch 2.4.1+cu124; the shim uses it.\n#\n# Eval runs against http://localhost:8888 (NO RunPod proxy) — the proven path.\n#\n# Env in: DEEPSEEK_API_KEY (for the DeepSeek baseline leg)\n# Out: /workspace/eval/aria_llm_v02_eval.json + deepseek_baseline_eval.json\nset -uo pipefail\n\nBASE_MODEL=\"mistralai/Mistral-7B-Instruct-v0.3\" # v0.2 actual base (R-F1454)\nADAPTER=\"/workspace/checkpoints/aria_llm_v0_2_dpo\" # the DPO LoRA\nNAME=\"aria-llm-v0.2\"\nPORT=8888\nSCRIPTS=\"/workspace/crucix/scripts/train\"\nEVAL_SET=\"${EVAL_SET:-/workspace/datasets/aria_eval_500q.jsonl}\" # R-F1455: overridable (100-Q subset)\nEVAL_DIR=\"/workspace/eval\"\nLOGS=\"/workspace/logs\"\n# Base weights live in the PERSISTENT volume cache (container ~/.cache is wiped on\n# restart). Gated on HF + no token, so resolve strictly from the cache.\nexport HF_HOME=/workspace/.cache/huggingface\nexport HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1\n\nmkdir -p \"$EVAL_DIR\" \"$LOGS\"\nlog(){ echo \"[$(date -u +%H:%M:%S)] $*\"; }\nfail(){ echo \"[FATAL] $*\" >&2; exit 1; }\n\n# ── Preflight (no destructive action before everything is present) ────────────\n[ -d \"$ADAPTER\" ] || fail \"adapter missing: $ADAPTER\"\n[ -f \"$SCRIPTS/serve_eval_shim.py\" ] || fail \"shim missing: $SCRIPTS/serve_eval_shim.py\"\n[ -f \"$SCRIPTS/eval_aria_llm.py\" ] || fail \"eval missing: $SCRIPTS/eval_aria_llm.py\"\n[ -f \"$EVAL_SET\" ] || fail \"eval set missing: $EVAL_SET\"\n[ -d \"$HF_HOME/hub/models--mistralai--Mistral-7B-Instruct-v0.3\" ] \\\n || fail \"Mistral base not in volume cache: $HF_HOME\"\nADAPTER_BASE=$(python3 -c \"import json;print(json.load(open('$ADAPTER/adapter_config.json')).get('base_model_name_or_path',''))\" 2>/dev/null || echo \"\")\n[ \"$ADAPTER_BASE\" = \"$BASE_MODEL\" ] || fail \"adapter base ('$ADAPTER_BASE') != '$BASE_MODEL'\"\nlog \"preflight ok — eval set $(wc -l < \"$EVAL_SET\") lines; adapter base $ADAPTER_BASE\"\n\n# ── Deps: the EXACT pinned coherent set from train_promote_v0_2.sh (R-F1345),\n# serve+eval subset (drop trl/datasets — those are train-only). Mistral's\n# tokenizer NEEDS sentencepiece+protobuf; the shim NEEDS fastapi+uvicorn; the\n# eval NEEDS httpx. Do NOT pipe pip through tail (masks the exit code). ──────────\nlog \"installing pinned serve+eval deps…\"\npip install -q \"transformers==4.46.3\" \"peft==0.13.2\" \"accelerate>=0.34\" \\\n bitsandbytes sentencepiece protobuf fastapi uvicorn httpx \\\n || fail \"dep install failed\"\npython - <<'PY' || fail \"deps import check failed\"\nimport transformers, peft, accelerate, bitsandbytes # noqa\nimport sentencepiece, google.protobuf # noqa — Mistral tokenizer\nimport fastapi, uvicorn, httpx # noqa — shim + eval client\nprint(f\"deps ok: transformers {transformers.__version__} peft {peft.__version__}\")\nPY\n\n# ── Serve v0.2 via the shim (EXACT invocation from train_promote serve()) ─────\npkill -f serve_eval_shim 2>/dev/null || true\npkill -f vllm.entrypoints 2>/dev/null || true\nsleep 3\nADAPTER=\"$ADAPTER\" MODEL_NAME=\"$NAME\" PORT=$PORT BASE_MODEL=\"$BASE_MODEL\" \\\n setsid nohup python \"$SCRIPTS/serve_eval_shim.py\" > \"$LOGS/shim_$NAME.log\" 2>&1 < /dev/null &\nlog \"waiting for shim to serve $NAME (bf16 7B load ~3-5 min)…\"\nready=0\nfor i in $(seq 1 60); do # 60*10s = 10 min cap\n if curl -s --max-time 5 \"http://localhost:$PORT/v1/models\" | grep -q \"$NAME\"; then\n ready=1; log \"shim serving $NAME (attempt $i)\"; break\n fi\n sleep 10\ndone\nif [ \"$ready\" != 1 ]; then\n echo \"=== shim log tail ===\"; tail -40 \"$LOGS/shim_$NAME.log\"\n fail \"shim did not come up in 10 min\"\nfi\n\n# ── Eval v0.2 against localhost (PROVEN path — no RunPod proxy) ───────────────\nlog \"eval v0.2 (500-Q) against localhost…\"\npython \"$SCRIPTS/eval_aria_llm.py\" \\\n --target \"http://localhost:$PORT/v1\" --model \"$NAME\" \\\n --eval-set \"$EVAL_SET\" --out \"$EVAL_DIR/aria_llm_v02_eval.json\" 2>&1 | tee \"$LOGS/eval_v02.log\" \\\n || fail \"v0.2 eval failed\"\n\n# ── Eval DeepSeek baseline — SAME eval set, SAME instrument (apples-to-apples) ─\nif [ -n \"${DEEPSEEK_API_KEY:-}\" ]; then\n log \"eval DeepSeek baseline (deepseek-chat)…\"\n python \"$SCRIPTS/eval_aria_llm.py\" \\\n --target \"https://api.deepseek.com/v1\" --model \"deepseek-chat\" \\\n --api-key \"$DEEPSEEK_API_KEY\" \\\n --eval-set \"$EVAL_SET\" --out \"$EVAL_DIR/deepseek_baseline_eval.json\" 2>&1 | tee \"$LOGS/eval_deepseek.log\" \\\n || log \"WARN: DeepSeek eval failed (see log) — v0.2 result still valid\"\nelse\n log \"WARN: DEEPSEEK_API_KEY not set on pod — skipping DeepSeek baseline\"\nfi\n\n# ── Summary ──────────────────────────────────────────────────────────────────\nlog \"=== BASELINE RESULTS ===\"\npython - <<'PY'\nimport json, os\nfor f in [\"/workspace/eval/aria_llm_v02_eval.json\",\n \"/workspace/eval/deepseek_baseline_eval.json\"]:\n if not os.path.exists(f):\n print(f, \"MISSING\"); continue\n d = json.load(open(f))\n dd = d.get(\"defence_dd\") or {}\n pi = d.get(\"prompt_injection\") or {}\n print(f\"{d.get('model')}: dd_accuracy={dd.get('accuracy')} (n={dd.get('total')}) \"\n f\"| injection pass_rate={pi.get('pass_rate')} leak_rate={pi.get('leak_rate')}\")\nPY\npkill -f serve_eval_shim 2>/dev/null || true\nlog \"done.\"\n"}, "4046b84d1710f56e6baf6752d32289ad60da382d": {"t": 1786068345.597922, "n": 5561, "h": "8edffa038a6e27bb", "c": "#!/bin/bash\n# R-F1455 — SURGICAL v0.2-vs-DeepSeek baseline, run ENTIRELY ON THE POD.\n#\n# Mirrors the ONLY recipe PROVEN to serve on this pod (train_promote_v0_2.sh:\n# pinned deps R-F1345 + serve_eval_shim.py + localhost eval). It deliberately\n# does NOT use vLLM: `pip install vllm` pulls a torch built for a newer CUDA\n# driver than this pod has (driver 565 / found 12070) and crashes EngineCore\n# (\"NVIDIA driver too old\") — that is what made every proxy run return 502.\n# The pod already has working torch 2.4.1+cu124; the shim uses it.\n#\n# Eval runs against http://localhost:8888 (NO RunPod proxy) — the proven path.\n#\n# Env in: DEEPSEEK_API_KEY (for the DeepSeek baseline leg)\n# Out: /workspace/eval/aria_llm_v02_eval.json + deepseek_baseline_eval.json\nset -uo pipefail\n\nBASE_MODEL=\"mistralai/Mistral-7B-Instruct-v0.3\" # v0.2 actual base (R-F1454)\nADAPTER=\"/workspace/checkpoints/aria_llm_v0_2_dpo\" # the DPO LoRA\nNAME=\"aria-llm-v0.2\"\nPORT=8888\nSCRIPTS=\"/workspace/crucix/scripts/train\"\nEVAL_SET=\"${EVAL_SET:-/workspace/datasets/aria_eval_500q.jsonl}\" # R-F1455: overridable (100-Q subset)\nEVAL_DIR=\"/workspace/eval\"\nLOGS=\"/workspace/logs\"\n# Base weights live in the PERSISTENT volume cache (container ~/.cache is wiped on\n# restart). Gated on HF + no token, so resolve strictly from the cache.\nexport HF_HOME=/workspace/.cache/huggingface\nexport HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1\n\nmkdir -p \"$EVAL_DIR\" \"$LOGS\"\nlog(){ echo \"[$(date -u +%H:%M:%S)] $*\"; }\nfail(){ echo \"[FATAL] $*\" >&2; exit 1; }\n\n# ── Preflight (no destructive action before everything is present) ────────────\n[ -d \"$ADAPTER\" ] || fail \"adapter missing: $ADAPTER\"\n[ -f \"$SCRIPTS/serve_eval_shim.py\" ] || fail \"shim missing: $SCRIPTS/serve_eval_shim.py\"\n[ -f \"$SCRIPTS/eval_aria_llm.py\" ] || fail \"eval missing: $SCRIPTS/eval_aria_llm.py\"\n[ -f \"$EVAL_SET\" ] || fail \"eval set missing: $EVAL_SET\"\n[ -d \"$HF_HOME/hub/models--mistralai--Mistral-7B-Instruct-v0.3\" ] \\\n || fail \"Mistral base not in volume cache: $HF_HOME\"\nADAPTER_BASE=$(python3 -c \"import json;print(json.load(open('$ADAPTER/adapter_config.json')).get('base_model_name_or_path',''))\" 2>/dev/null || echo \"\")\n[ \"$ADAPTER_BASE\" = \"$BASE_MODEL\" ] || fail \"adapter base ('$ADAPTER_BASE') != '$BASE_MODEL'\"\nlog \"preflight ok — eval set $(wc -l < \"$EVAL_SET\") lines; adapter base $ADAPTER_BASE\"\n\n# ── Deps: the EXACT pinned coherent set from train_promote_v0_2.sh (R-F1345),\n# serve+eval subset (drop trl/datasets — those are train-only). Mistral's\n# tokenizer NEEDS sentencepiece+protobuf; the shim NEEDS fastapi+uvicorn; the\n# eval NEEDS httpx. Do NOT pipe pip through tail (masks the exit code). ──────────\nlog \"installing pinned serve+eval deps…\"\npip install -q \"transformers==4.46.3\" \"peft==0.13.2\" \"accelerate>=0.34\" \\\n bitsandbytes sentencepiece protobuf fastapi uvicorn httpx \\\n || fail \"dep install failed\"\npython - <<'PY' || fail \"deps import check failed\"\nimport transformers, peft, accelerate, bitsandbytes # noqa\nimport sentencepiece, google.protobuf # noqa — Mistral tokenizer\nimport fastapi, uvicorn, httpx # noqa — shim + eval client\nprint(f\"deps ok: transformers {transformers.__version__} peft {peft.__version__}\")\nPY\n\n# ── Serve v0.2 via the shim (EXACT invocation from train_promote serve()) ─────\npkill -f serve_eval_shim 2>/dev/null || true\npkill -f vllm.entrypoints 2>/dev/null || true\nsleep 3\nADAPTER=\"$ADAPTER\" MODEL_NAME=\"$NAME\" PORT=$PORT BASE_MODEL=\"$BASE_MODEL\" \\\n setsid nohup python \"$SCRIPTS/serve_eval_shim.py\" > \"$LOGS/shim_$NAME.log\" 2>&1 < /dev/null &\nlog \"waiting for shim to serve $NAME (bf16 7B load ~3-5 min)…\"\nready=0\nfor i in $(seq 1 60); do # 60*10s = 10 min cap\n if curl -s --max-time 5 \"http://localhost:$PORT/v1/models\" | grep -q \"$NAME\"; then\n ready=1; log \"shim serving $NAME (attempt $i)\"; break\n fi\n sleep 10\ndone\nif [ \"$ready\" != 1 ]; then\n echo \"=== shim log tail ===\"; tail -40 \"$LOGS/shim_$NAME.log\"\n fail \"shim did not come up in 10 min\"\nfi\n\n# ── Eval v0.2 against localhost (PROVEN path — no RunPod proxy) ───────────────\nlog \"eval v0.2 (500-Q) against localhost…\"\npython \"$SCRIPTS/eval_aria_llm.py\" \\\n --target \"http://localhost:$PORT/v1\" --model \"$NAME\" \\\n --eval-set \"$EVAL_SET\" --out \"$EVAL_DIR/aria_llm_v02_eval.json\" 2>&1 | tee \"$LOGS/eval_v02.log\" \\\n || fail \"v0.2 eval failed\"\n\n# ── Eval DeepSeek baseline — SAME eval set, SAME instrument (apples-to-apples) ─\nif [ -n \"${DEEPSEEK_API_KEY:-}\" ]; then\n log \"eval DeepSeek baseline (deepseek-chat)…\"\n python \"$SCRIPTS/eval_aria_llm.py\" \\\n --target \"https://api.deepseek.com/v1\" --model \"deepseek-chat\" \\\n --api-key \"$DEEPSEEK_API_KEY\" \\\n --eval-set \"$EVAL_SET\" --out \"$EVAL_DIR/deepseek_baseline_eval.json\" 2>&1 | tee \"$LOGS/eval_deepseek.log\" \\\n || log \"WARN: DeepSeek eval failed (see log) — v0.2 result still valid\"\nelse\n log \"WARN: DEEPSEEK_API_KEY not set on pod — skipping DeepSeek baseline\"\nfi\n\n# ── Summary ──────────────────────────────────────────────────────────────────\nlog \"=== BASELINE RESULTS ===\"\npython - <<'PY'\nimport json, os\nfor f in [\"/workspace/eval/aria_llm_v02_eval.json\",\n \"/workspace/eval/deepseek_baseline_eval.json\"]:\n if not os.path.exists(f):\n print(f, \"MISSING\"); continue\n d = json.load(open(f))\n dd = d.get(\"defence_dd\") or {}\n pi = d.get(\"prompt_injection\") or {}\n print(f\"{d.get('model')}: dd_accuracy={dd.get('accuracy')} (n={dd.get('total')}) \"\n f\"| injection pass_rate={pi.get('pass_rate')} leak_rate={pi.get('leak_rate')}\")\nPY\npkill -f serve_eval_shim 2>/dev/null || true\nlog \"done.\"\n"}}, "scripts/train/v0_2_eval_pod_run.sh": {"3a9645c8ec96260be7edd2c46a655b1e473f8c18": {"t": 1785745956.6558871, "n": 5042, "h": "129dc5870e16ef37", "c": "#!/bin/bash\n# R-F1474 — v0.2 500-Q eval, run ON THE POD. Closes the v0.3-vs-v0.2 head-to-head.\n#\n# The 2026-06-09 cycle trained v0.3 (judge-DD 0.22 / 500) and evaluated it, but the\n# v0.2 comparison leg never finished before the pod went away — so we have v0.3's\n# absolute score but no apples-to-apples verdict. This re-serves the PERSISTED v0.2\n# DPO adapter and evals it on the SAME frozen 500-Q with the SAME validated judge.\n# NO training — adapter already exists on the volume. Mirrors v0_3_pod_run.sh's\n# serve()/eval_model() exactly (shim, NOT vLLM; localhost:8888; judge auto-on).\n#\n# Env in: DEEPSEEK_API_KEY (judge), EVAL_SET (default below).\n# Out: /workspace/eval/aria_llm_v0_2_eval.json\nset -uo pipefail\n\nBASE_MODEL=\"mistralai/Mistral-7B-Instruct-v0.3\"\nV02_ADAPTER=\"/workspace/checkpoints/aria_llm_v0_2_dpo\"\nEVAL_SET=\"${EVAL_SET:-/workspace/datasets/aria_eval_500q.jsonl}\"\nEVAL_DIR=\"/workspace/eval\"; LOGS=\"/workspace/logs\"; SCRIPTS=\"/workspace/crucix/scripts/train\"\nPORT=8888\nV02_REPORT=\"${EVAL_DIR}/aria_llm_v0_2_eval.json\"\nexport HF_HOME=/workspace/.cache/huggingface\nexport HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1\n\nmkdir -p \"$EVAL_DIR\" \"$LOGS\"\nrm -f \"$EVAL_DIR/_v0_2_status\"\ntrap 'rc=$?; echo \"$rc\" > \"$EVAL_DIR/_v0_2_status\" 2>/dev/null || true' EXIT\nlog(){ echo \"[$(date -u +%H:%M:%S)] $*\"; }\nfail(){ echo \"[FATAL] $*\" >&2; exit 1; }\n\nserve(){ # serve <lora_path> <name> — shim only\n local lora=\"$1\" name=\"$2\"\n pkill -f serve_eval_shim 2>/dev/null || true; sleep 3\n HF_HOME=\"$HF_HOME\" ADAPTER=\"$lora\" MODEL_NAME=\"$name\" PORT=$PORT BASE_MODEL=\"$BASE_MODEL\" \\\n setsid nohup python \"$SCRIPTS/serve_eval_shim.py\" > \"$LOGS/shim_${name}.log\" 2>&1 < /dev/null &\n for i in $(seq 1 60); do\n if curl -s --max-time 5 \"http://localhost:$PORT/v1/models\" | grep -q \"$name\"; then\n log \"shim serving $name (attempt $i)\"; return 0\n fi; sleep 10\n done\n echo \"=== shim log tail ($name) ===\"; tail -40 \"$LOGS/shim_${name}.log\"; return 1\n}\n\ncoherence_ok(){\n python - \"$1\" <<'PY'\nimport json, sys, urllib.request\nname = sys.argv[1]\ndef ask(q):\n body = json.dumps({\"model\": name, \"messages\": [{\"role\":\"user\",\"content\":q}],\n \"max_tokens\": 150, \"temperature\": 0.3}).encode()\n req = urllib.request.Request(\"http://localhost:8888/v1/chat/completions\", body, {\"Content-Type\":\"application/json\"})\n return json.load(urllib.request.urlopen(req, timeout=120))[\"choices\"][0][\"message\"][\"content\"]\nQS = [\"What is the main due-diligence risk with a sanctions-adjacent counterparty?\",\n \"In two sentences, why does ultimate beneficial ownership matter in KYC?\",\n \"What should a compliance team verify before onboarding a new supplier?\"]\nbad = 0\nfor q in QS:\n try: t = ask(q)\n except Exception as e: print(f\" ask failed: {e}\", file=sys.stderr); bad += 1; continue\n deg = any(m in t for m in (\"tier=\",\"self_hosted\",\"[Layers\",\"comp_web\",\"=10000\")) or len(t.strip()) < 25\n if deg: bad += 1; print(f\" DEGENERATE: {t[:120]!r}\", file=sys.stderr)\nprint(\"COHERENT\" if bad == 0 else \"DEGENERATE\")\nPY\n}\n\nlog \"=== ARIA-LLM v0.2 500-Q eval (comparison leg) ===\"\n[ -d \"$V02_ADAPTER\" ] || fail \"v0.2 adapter missing on volume: $V02_ADAPTER\"\n[ -f \"$V02_ADAPTER/adapter_config.json\" ] || fail \"no adapter_config.json in $V02_ADAPTER\"\n[ -f \"$EVAL_SET\" ] || fail \"eval set missing: $EVAL_SET\"\n[ -f \"$SCRIPTS/serve_eval_shim.py\" ] || fail \"serve_eval_shim.py missing: $SCRIPTS\"\n[ -f \"$SCRIPTS/eval_aria_llm.py\" ] || fail \"eval_aria_llm.py missing: $SCRIPTS\"\n[ -d \"$HF_HOME/hub/models--mistralai--Mistral-7B-Instruct-v0.3\" ] || fail \"Mistral base not in volume cache\"\nlog \"preflight ok — eval $(wc -l < \"$EVAL_SET\") Q against v0.2 adapter\"\n\n# Container site-packages are wiped on restart; only /workspace persists -> reinstall.\nlog \"installing pinned serve+eval deps…\"\npip install -q \"transformers==4.46.3\" \"peft==0.13.2\" \"accelerate>=0.34\" \\\n bitsandbytes sentencepiece protobuf fastapi uvicorn httpx || fail \"dep install failed\"\n\nserve \"$V02_ADAPTER\" \"aria-llm-v0.2\" || fail \"could not serve v0.2\"\nCOH=$(coherence_ok \"aria-llm-v0.2\"); log \"coherence(v0.2): $COH\"\n[ \"$COH\" = \"COHERENT\" ] || fail \"v0.2 DEGENERATE on coherence gate — inspect $LOGS/shim_aria-llm-v0.2.log\"\n\nlog \"eval v0.2 (500-Q, judge) against localhost…\"\nDEEPSEEK_API_KEY=\"${DEEPSEEK_API_KEY:-}\" \\\npython \"$SCRIPTS/eval_aria_llm.py\" \\\n --target \"http://localhost:$PORT/v1\" --model \"aria-llm-v0.2\" \\\n --eval-set \"$EVAL_SET\" --out \"$V02_REPORT\" 2>&1 | tee \"$LOGS/eval_v0_2.log\" || fail \"v0.2 eval failed\"\n\n[ -s \"$V02_REPORT\" ] || fail \"v0.2 report not written: $V02_REPORT\"\nlog \"=== v0.2 eval complete ===\"\npython - \"$V02_REPORT\" <<'PY'\nimport json, sys\nd = json.load(open(sys.argv[1], encoding=\"utf-8\"))\ndd = d.get(\"defence_dd\") or d.get(\"dd_eval\") or {}; pi = d.get(\"prompt_injection\") or {}\nprint(f\"v0.2 judge-DD: {dd.get('accuracy')} (n={dd.get('total')}) | injection leak_rate={pi.get('leak_rate')}\")\nPY\npkill -f serve_eval_shim 2>/dev/null || true\nlog \"done — shim stopped. (Pod stop is the orchestrator's EXIT trap.)\"\n"}, "355d148bd828b00439f475e4842616ce1de45c73": {"t": 1785758161.4943724, "n": 5042, "h": "129dc5870e16ef37", "c": "#!/bin/bash\n# R-F1474 — v0.2 500-Q eval, run ON THE POD. Closes the v0.3-vs-v0.2 head-to-head.\n#\n# The 2026-06-09 cycle trained v0.3 (judge-DD 0.22 / 500) and evaluated it, but the\n# v0.2 comparison leg never finished before the pod went away — so we have v0.3's\n# absolute score but no apples-to-apples verdict. This re-serves the PERSISTED v0.2\n# DPO adapter and evals it on the SAME frozen 500-Q with the SAME validated judge.\n# NO training — adapter already exists on the volume. Mirrors v0_3_pod_run.sh's\n# serve()/eval_model() exactly (shim, NOT vLLM; localhost:8888; judge auto-on).\n#\n# Env in: DEEPSEEK_API_KEY (judge), EVAL_SET (default below).\n# Out: /workspace/eval/aria_llm_v0_2_eval.json\nset -uo pipefail\n\nBASE_MODEL=\"mistralai/Mistral-7B-Instruct-v0.3\"\nV02_ADAPTER=\"/workspace/checkpoints/aria_llm_v0_2_dpo\"\nEVAL_SET=\"${EVAL_SET:-/workspace/datasets/aria_eval_500q.jsonl}\"\nEVAL_DIR=\"/workspace/eval\"; LOGS=\"/workspace/logs\"; SCRIPTS=\"/workspace/crucix/scripts/train\"\nPORT=8888\nV02_REPORT=\"${EVAL_DIR}/aria_llm_v0_2_eval.json\"\nexport HF_HOME=/workspace/.cache/huggingface\nexport HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1\n\nmkdir -p \"$EVAL_DIR\" \"$LOGS\"\nrm -f \"$EVAL_DIR/_v0_2_status\"\ntrap 'rc=$?; echo \"$rc\" > \"$EVAL_DIR/_v0_2_status\" 2>/dev/null || true' EXIT\nlog(){ echo \"[$(date -u +%H:%M:%S)] $*\"; }\nfail(){ echo \"[FATAL] $*\" >&2; exit 1; }\n\nserve(){ # serve <lora_path> <name> — shim only\n local lora=\"$1\" name=\"$2\"\n pkill -f serve_eval_shim 2>/dev/null || true; sleep 3\n HF_HOME=\"$HF_HOME\" ADAPTER=\"$lora\" MODEL_NAME=\"$name\" PORT=$PORT BASE_MODEL=\"$BASE_MODEL\" \\\n setsid nohup python \"$SCRIPTS/serve_eval_shim.py\" > \"$LOGS/shim_${name}.log\" 2>&1 < /dev/null &\n for i in $(seq 1 60); do\n if curl -s --max-time 5 \"http://localhost:$PORT/v1/models\" | grep -q \"$name\"; then\n log \"shim serving $name (attempt $i)\"; return 0\n fi; sleep 10\n done\n echo \"=== shim log tail ($name) ===\"; tail -40 \"$LOGS/shim_${name}.log\"; return 1\n}\n\ncoherence_ok(){\n python - \"$1\" <<'PY'\nimport json, sys, urllib.request\nname = sys.argv[1]\ndef ask(q):\n body = json.dumps({\"model\": name, \"messages\": [{\"role\":\"user\",\"content\":q}],\n \"max_tokens\": 150, \"temperature\": 0.3}).encode()\n req = urllib.request.Request(\"http://localhost:8888/v1/chat/completions\", body, {\"Content-Type\":\"application/json\"})\n return json.load(urllib.request.urlopen(req, timeout=120))[\"choices\"][0][\"message\"][\"content\"]\nQS = [\"What is the main due-diligence risk with a sanctions-adjacent counterparty?\",\n \"In two sentences, why does ultimate beneficial ownership matter in KYC?\",\n \"What should a compliance team verify before onboarding a new supplier?\"]\nbad = 0\nfor q in QS:\n try: t = ask(q)\n except Exception as e: print(f\" ask failed: {e}\", file=sys.stderr); bad += 1; continue\n deg = any(m in t for m in (\"tier=\",\"self_hosted\",\"[Layers\",\"comp_web\",\"=10000\")) or len(t.strip()) < 25\n if deg: bad += 1; print(f\" DEGENERATE: {t[:120]!r}\", file=sys.stderr)\nprint(\"COHERENT\" if bad == 0 else \"DEGENERATE\")\nPY\n}\n\nlog \"=== ARIA-LLM v0.2 500-Q eval (comparison leg) ===\"\n[ -d \"$V02_ADAPTER\" ] || fail \"v0.2 adapter missing on volume: $V02_ADAPTER\"\n[ -f \"$V02_ADAPTER/adapter_config.json\" ] || fail \"no adapter_config.json in $V02_ADAPTER\"\n[ -f \"$EVAL_SET\" ] || fail \"eval set missing: $EVAL_SET\"\n[ -f \"$SCRIPTS/serve_eval_shim.py\" ] || fail \"serve_eval_shim.py missing: $SCRIPTS\"\n[ -f \"$SCRIPTS/eval_aria_llm.py\" ] || fail \"eval_aria_llm.py missing: $SCRIPTS\"\n[ -d \"$HF_HOME/hub/models--mistralai--Mistral-7B-Instruct-v0.3\" ] || fail \"Mistral base not in volume cache\"\nlog \"preflight ok — eval $(wc -l < \"$EVAL_SET\") Q against v0.2 adapter\"\n\n# Container site-packages are wiped on restart; only /workspace persists -> reinstall.\nlog \"installing pinned serve+eval deps…\"\npip install -q \"transformers==4.46.3\" \"peft==0.13.2\" \"accelerate>=0.34\" \\\n bitsandbytes sentencepiece protobuf fastapi uvicorn httpx || fail \"dep install failed\"\n\nserve \"$V02_ADAPTER\" \"aria-llm-v0.2\" || fail \"could not serve v0.2\"\nCOH=$(coherence_ok \"aria-llm-v0.2\"); log \"coherence(v0.2): $COH\"\n[ \"$COH\" = \"COHERENT\" ] || fail \"v0.2 DEGENERATE on coherence gate — inspect $LOGS/shim_aria-llm-v0.2.log\"\n\nlog \"eval v0.2 (500-Q, judge) against localhost…\"\nDEEPSEEK_API_KEY=\"${DEEPSEEK_API_KEY:-}\" \\\npython \"$SCRIPTS/eval_aria_llm.py\" \\\n --target \"http://localhost:$PORT/v1\" --model \"aria-llm-v0.2\" \\\n --eval-set \"$EVAL_SET\" --out \"$V02_REPORT\" 2>&1 | tee \"$LOGS/eval_v0_2.log\" || fail \"v0.2 eval failed\"\n\n[ -s \"$V02_REPORT\" ] || fail \"v0.2 report not written: $V02_REPORT\"\nlog \"=== v0.2 eval complete ===\"\npython - \"$V02_REPORT\" <<'PY'\nimport json, sys\nd = json.load(open(sys.argv[1], encoding=\"utf-8\"))\ndd = d.get(\"defence_dd\") or d.get(\"dd_eval\") or {}; pi = d.get(\"prompt_injection\") or {}\nprint(f\"v0.2 judge-DD: {dd.get('accuracy')} (n={dd.get('total')}) | injection leak_rate={pi.get('leak_rate')}\")\nPY\npkill -f serve_eval_shim 2>/dev/null || true\nlog \"done — shim stopped. (Pod stop is the orchestrator's EXIT trap.)\"\n"}, "2f51659938376b08c72ed831048a3fef7bdb559a": {"t": 1786067516.0182097, "n": 5042, "h": "129dc5870e16ef37", "c": "#!/bin/bash\n# R-F1474 — v0.2 500-Q eval, run ON THE POD. Closes the v0.3-vs-v0.2 head-to-head.\n#\n# The 2026-06-09 cycle trained v0.3 (judge-DD 0.22 / 500) and evaluated it, but the\n# v0.2 comparison leg never finished before the pod went away — so we have v0.3's\n# absolute score but no apples-to-apples verdict. This re-serves the PERSISTED v0.2\n# DPO adapter and evals it on the SAME frozen 500-Q with the SAME validated judge.\n# NO training — adapter already exists on the volume. Mirrors v0_3_pod_run.sh's\n# serve()/eval_model() exactly (shim, NOT vLLM; localhost:8888; judge auto-on).\n#\n# Env in: DEEPSEEK_API_KEY (judge), EVAL_SET (default below).\n# Out: /workspace/eval/aria_llm_v0_2_eval.json\nset -uo pipefail\n\nBASE_MODEL=\"mistralai/Mistral-7B-Instruct-v0.3\"\nV02_ADAPTER=\"/workspace/checkpoints/aria_llm_v0_2_dpo\"\nEVAL_SET=\"${EVAL_SET:-/workspace/datasets/aria_eval_500q.jsonl}\"\nEVAL_DIR=\"/workspace/eval\"; LOGS=\"/workspace/logs\"; SCRIPTS=\"/workspace/crucix/scripts/train\"\nPORT=8888\nV02_REPORT=\"${EVAL_DIR}/aria_llm_v0_2_eval.json\"\nexport HF_HOME=/workspace/.cache/huggingface\nexport HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1\n\nmkdir -p \"$EVAL_DIR\" \"$LOGS\"\nrm -f \"$EVAL_DIR/_v0_2_status\"\ntrap 'rc=$?; echo \"$rc\" > \"$EVAL_DIR/_v0_2_status\" 2>/dev/null || true' EXIT\nlog(){ echo \"[$(date -u +%H:%M:%S)] $*\"; }\nfail(){ echo \"[FATAL] $*\" >&2; exit 1; }\n\nserve(){ # serve <lora_path> <name> — shim only\n local lora=\"$1\" name=\"$2\"\n pkill -f serve_eval_shim 2>/dev/null || true; sleep 3\n HF_HOME=\"$HF_HOME\" ADAPTER=\"$lora\" MODEL_NAME=\"$name\" PORT=$PORT BASE_MODEL=\"$BASE_MODEL\" \\\n setsid nohup python \"$SCRIPTS/serve_eval_shim.py\" > \"$LOGS/shim_${name}.log\" 2>&1 < /dev/null &\n for i in $(seq 1 60); do\n if curl -s --max-time 5 \"http://localhost:$PORT/v1/models\" | grep -q \"$name\"; then\n log \"shim serving $name (attempt $i)\"; return 0\n fi; sleep 10\n done\n echo \"=== shim log tail ($name) ===\"; tail -40 \"$LOGS/shim_${name}.log\"; return 1\n}\n\ncoherence_ok(){\n python - \"$1\" <<'PY'\nimport json, sys, urllib.request\nname = sys.argv[1]\ndef ask(q):\n body = json.dumps({\"model\": name, \"messages\": [{\"role\":\"user\",\"content\":q}],\n \"max_tokens\": 150, \"temperature\": 0.3}).encode()\n req = urllib.request.Request(\"http://localhost:8888/v1/chat/completions\", body, {\"Content-Type\":\"application/json\"})\n return json.load(urllib.request.urlopen(req, timeout=120))[\"choices\"][0][\"message\"][\"content\"]\nQS = [\"What is the main due-diligence risk with a sanctions-adjacent counterparty?\",\n \"In two sentences, why does ultimate beneficial ownership matter in KYC?\",\n \"What should a compliance team verify before onboarding a new supplier?\"]\nbad = 0\nfor q in QS:\n try: t = ask(q)\n except Exception as e: print(f\" ask failed: {e}\", file=sys.stderr); bad += 1; continue\n deg = any(m in t for m in (\"tier=\",\"self_hosted\",\"[Layers\",\"comp_web\",\"=10000\")) or len(t.strip()) < 25\n if deg: bad += 1; print(f\" DEGENERATE: {t[:120]!r}\", file=sys.stderr)\nprint(\"COHERENT\" if bad == 0 else \"DEGENERATE\")\nPY\n}\n\nlog \"=== ARIA-LLM v0.2 500-Q eval (comparison leg) ===\"\n[ -d \"$V02_ADAPTER\" ] || fail \"v0.2 adapter missing on volume: $V02_ADAPTER\"\n[ -f \"$V02_ADAPTER/adapter_config.json\" ] || fail \"no adapter_config.json in $V02_ADAPTER\"\n[ -f \"$EVAL_SET\" ] || fail \"eval set missing: $EVAL_SET\"\n[ -f \"$SCRIPTS/serve_eval_shim.py\" ] || fail \"serve_eval_shim.py missing: $SCRIPTS\"\n[ -f \"$SCRIPTS/eval_aria_llm.py\" ] || fail \"eval_aria_llm.py missing: $SCRIPTS\"\n[ -d \"$HF_HOME/hub/models--mistralai--Mistral-7B-Instruct-v0.3\" ] || fail \"Mistral base not in volume cache\"\nlog \"preflight ok — eval $(wc -l < \"$EVAL_SET\") Q against v0.2 adapter\"\n\n# Container site-packages are wiped on restart; only /workspace persists -> reinstall.\nlog \"installing pinned serve+eval deps…\"\npip install -q \"transformers==4.46.3\" \"peft==0.13.2\" \"accelerate>=0.34\" \\\n bitsandbytes sentencepiece protobuf fastapi uvicorn httpx || fail \"dep install failed\"\n\nserve \"$V02_ADAPTER\" \"aria-llm-v0.2\" || fail \"could not serve v0.2\"\nCOH=$(coherence_ok \"aria-llm-v0.2\"); log \"coherence(v0.2): $COH\"\n[ \"$COH\" = \"COHERENT\" ] || fail \"v0.2 DEGENERATE on coherence gate — inspect $LOGS/shim_aria-llm-v0.2.log\"\n\nlog \"eval v0.2 (500-Q, judge) against localhost…\"\nDEEPSEEK_API_KEY=\"${DEEPSEEK_API_KEY:-}\" \\\npython \"$SCRIPTS/eval_aria_llm.py\" \\\n --target \"http://localhost:$PORT/v1\" --model \"aria-llm-v0.2\" \\\n --eval-set \"$EVAL_SET\" --out \"$V02_REPORT\" 2>&1 | tee \"$LOGS/eval_v0_2.log\" || fail \"v0.2 eval failed\"\n\n[ -s \"$V02_REPORT\" ] || fail \"v0.2 report not written: $V02_REPORT\"\nlog \"=== v0.2 eval complete ===\"\npython - \"$V02_REPORT\" <<'PY'\nimport json, sys\nd = json.load(open(sys.argv[1], encoding=\"utf-8\"))\ndd = d.get(\"defence_dd\") or d.get(\"dd_eval\") or {}; pi = d.get(\"prompt_injection\") or {}\nprint(f\"v0.2 judge-DD: {dd.get('accuracy')} (n={dd.get('total')}) | injection leak_rate={pi.get('leak_rate')}\")\nPY\npkill -f serve_eval_shim 2>/dev/null || true\nlog \"done — shim stopped. (Pod stop is the orchestrator's EXIT trap.)\"\n"}, "4046b84d1710f56e6baf6752d32289ad60da382d": {"t": 1786068346.4015398, "n": 5042, "h": "129dc5870e16ef37", "c": "#!/bin/bash\n# R-F1474 — v0.2 500-Q eval, run ON THE POD. Closes the v0.3-vs-v0.2 head-to-head.\n#\n# The 2026-06-09 cycle trained v0.3 (judge-DD 0.22 / 500) and evaluated it, but the\n# v0.2 comparison leg never finished before the pod went away — so we have v0.3's\n# absolute score but no apples-to-apples verdict. This re-serves the PERSISTED v0.2\n# DPO adapter and evals it on the SAME frozen 500-Q with the SAME validated judge.\n# NO training — adapter already exists on the volume. Mirrors v0_3_pod_run.sh's\n# serve()/eval_model() exactly (shim, NOT vLLM; localhost:8888; judge auto-on).\n#\n# Env in: DEEPSEEK_API_KEY (judge), EVAL_SET (default below).\n# Out: /workspace/eval/aria_llm_v0_2_eval.json\nset -uo pipefail\n\nBASE_MODEL=\"mistralai/Mistral-7B-Instruct-v0.3\"\nV02_ADAPTER=\"/workspace/checkpoints/aria_llm_v0_2_dpo\"\nEVAL_SET=\"${EVAL_SET:-/workspace/datasets/aria_eval_500q.jsonl}\"\nEVAL_DIR=\"/workspace/eval\"; LOGS=\"/workspace/logs\"; SCRIPTS=\"/workspace/crucix/scripts/train\"\nPORT=8888\nV02_REPORT=\"${EVAL_DIR}/aria_llm_v0_2_eval.json\"\nexport HF_HOME=/workspace/.cache/huggingface\nexport HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1\n\nmkdir -p \"$EVAL_DIR\" \"$LOGS\"\nrm -f \"$EVAL_DIR/_v0_2_status\"\ntrap 'rc=$?; echo \"$rc\" > \"$EVAL_DIR/_v0_2_status\" 2>/dev/null || true' EXIT\nlog(){ echo \"[$(date -u +%H:%M:%S)] $*\"; }\nfail(){ echo \"[FATAL] $*\" >&2; exit 1; }\n\nserve(){ # serve <lora_path> <name> — shim only\n local lora=\"$1\" name=\"$2\"\n pkill -f serve_eval_shim 2>/dev/null || true; sleep 3\n HF_HOME=\"$HF_HOME\" ADAPTER=\"$lora\" MODEL_NAME=\"$name\" PORT=$PORT BASE_MODEL=\"$BASE_MODEL\" \\\n setsid nohup python \"$SCRIPTS/serve_eval_shim.py\" > \"$LOGS/shim_${name}.log\" 2>&1 < /dev/null &\n for i in $(seq 1 60); do\n if curl -s --max-time 5 \"http://localhost:$PORT/v1/models\" | grep -q \"$name\"; then\n log \"shim serving $name (attempt $i)\"; return 0\n fi; sleep 10\n done\n echo \"=== shim log tail ($name) ===\"; tail -40 \"$LOGS/shim_${name}.log\"; return 1\n}\n\ncoherence_ok(){\n python - \"$1\" <<'PY'\nimport json, sys, urllib.request\nname = sys.argv[1]\ndef ask(q):\n body = json.dumps({\"model\": name, \"messages\": [{\"role\":\"user\",\"content\":q}],\n \"max_tokens\": 150, \"temperature\": 0.3}).encode()\n req = urllib.request.Request(\"http://localhost:8888/v1/chat/completions\", body, {\"Content-Type\":\"application/json\"})\n return json.load(urllib.request.urlopen(req, timeout=120))[\"choices\"][0][\"message\"][\"content\"]\nQS = [\"What is the main due-diligence risk with a sanctions-adjacent counterparty?\",\n \"In two sentences, why does ultimate beneficial ownership matter in KYC?\",\n \"What should a compliance team verify before onboarding a new supplier?\"]\nbad = 0\nfor q in QS:\n try: t = ask(q)\n except Exception as e: print(f\" ask failed: {e}\", file=sys.stderr); bad += 1; continue\n deg = any(m in t for m in (\"tier=\",\"self_hosted\",\"[Layers\",\"comp_web\",\"=10000\")) or len(t.strip()) < 25\n if deg: bad += 1; print(f\" DEGENERATE: {t[:120]!r}\", file=sys.stderr)\nprint(\"COHERENT\" if bad == 0 else \"DEGENERATE\")\nPY\n}\n\nlog \"=== ARIA-LLM v0.2 500-Q eval (comparison leg) ===\"\n[ -d \"$V02_ADAPTER\" ] || fail \"v0.2 adapter missing on volume: $V02_ADAPTER\"\n[ -f \"$V02_ADAPTER/adapter_config.json\" ] || fail \"no adapter_config.json in $V02_ADAPTER\"\n[ -f \"$EVAL_SET\" ] || fail \"eval set missing: $EVAL_SET\"\n[ -f \"$SCRIPTS/serve_eval_shim.py\" ] || fail \"serve_eval_shim.py missing: $SCRIPTS\"\n[ -f \"$SCRIPTS/eval_aria_llm.py\" ] || fail \"eval_aria_llm.py missing: $SCRIPTS\"\n[ -d \"$HF_HOME/hub/models--mistralai--Mistral-7B-Instruct-v0.3\" ] || fail \"Mistral base not in volume cache\"\nlog \"preflight ok — eval $(wc -l < \"$EVAL_SET\") Q against v0.2 adapter\"\n\n# Container site-packages are wiped on restart; only /workspace persists -> reinstall.\nlog \"installing pinned serve+eval deps…\"\npip install -q \"transformers==4.46.3\" \"peft==0.13.2\" \"accelerate>=0.34\" \\\n bitsandbytes sentencepiece protobuf fastapi uvicorn httpx || fail \"dep install failed\"\n\nserve \"$V02_ADAPTER\" \"aria-llm-v0.2\" || fail \"could not serve v0.2\"\nCOH=$(coherence_ok \"aria-llm-v0.2\"); log \"coherence(v0.2): $COH\"\n[ \"$COH\" = \"COHERENT\" ] || fail \"v0.2 DEGENERATE on coherence gate — inspect $LOGS/shim_aria-llm-v0.2.log\"\n\nlog \"eval v0.2 (500-Q, judge) against localhost…\"\nDEEPSEEK_API_KEY=\"${DEEPSEEK_API_KEY:-}\" \\\npython \"$SCRIPTS/eval_aria_llm.py\" \\\n --target \"http://localhost:$PORT/v1\" --model \"aria-llm-v0.2\" \\\n --eval-set \"$EVAL_SET\" --out \"$V02_REPORT\" 2>&1 | tee \"$LOGS/eval_v0_2.log\" || fail \"v0.2 eval failed\"\n\n[ -s \"$V02_REPORT\" ] || fail \"v0.2 report not written: $V02_REPORT\"\nlog \"=== v0.2 eval complete ===\"\npython - \"$V02_REPORT\" <<'PY'\nimport json, sys\nd = json.load(open(sys.argv[1], encoding=\"utf-8\"))\ndd = d.get(\"defence_dd\") or d.get(\"dd_eval\") or {}; pi = d.get(\"prompt_injection\") or {}\nprint(f\"v0.2 judge-DD: {dd.get('accuracy')} (n={dd.get('total')}) | injection leak_rate={pi.get('leak_rate')}\")\nPY\npkill -f serve_eval_shim 2>/dev/null || true\nlog \"done — shim stopped. (Pod stop is the orchestrator's EXIT trap.)\"\n"}}, "scripts/train/run_baseline.sh": {"3a9645c8ec96260be7edd2c46a655b1e473f8c18": {"t": 1785745962.9184797, "n": 4285, "h": "4d31ebfbb2f875d5", "c": "#!/bin/bash\n# R-F1455 — LOCAL driver for the v0.2-vs-DeepSeek baseline. Runs from the operator\n# Windows box (has repo + .env + ~/.ssh/runpod_aria). Retries the pod start until a\n# GPU frees on the pinned host, then pushes + runs baseline_pod_run.sh ON THE POD\n# (proven shim recipe), pulls the reports, and ALWAYS stops the pod (EXIT trap).\n#\n# Cost-safe: start attempts on a capacity-blocked pod do not bill; the GPU only\n# bills once RUNNING, and the trap stops it on any exit.\nset -uo pipefail\n\nREPO=\"/c/code/crucix\"; cd \"$REPO\"\nPOD=\"7ei3hldcpz4j2v\"\nAPI=\"https://rest.runpod.io/v1\"\nAPI_KEY=$(grep -E \"^RUNPOD_API_KEY=\" .env | head -1 | cut -d= -f2- | tr -d '\"' | tr -d \"'\" | tr -d '\\r')\nDSK=$(grep -E \"^DEEPSEEK_API_KEY=\" .env | head -1 | cut -d= -f2- | tr -d '\"' | tr -d \"'\" | tr -d '\\r')\nKEY=\"/tmp/rpkey\"; cp ~/.ssh/runpod_aria \"$KEY\"; chmod 600 \"$KEY\"\nSSH=\"ssh -i $KEY -o StrictHostKeyChecking=no -o ConnectTimeout=10\"\n\nstop_pod(){ echo \"[driver] stopping pod $POD\"; curl -s -X POST \"$API/pods/$POD/stop\" -H \"Authorization: Bearer $API_KEY\" >/dev/null 2>&1 || true; }\ntrap stop_pod EXIT\n\njget(){ python -c \"import sys,json;d=json.load(sys.stdin);print(d.get('$1','') or '')\" 2>/dev/null; }\npmget(){ python -c \"import sys,json;d=json.load(sys.stdin);pm=d.get('portMappings') or {};print(pm.get('22') or '')\" 2>/dev/null; }\n\n# ── 1. Retry start until a GPU frees (cap ~60 min); $0 while blocked ──────────\necho \"[driver] retrying pod start until a GPU frees on the host…\"\nHOST=\"\"; PORT=\"\"\nfor i in $(seq 1 60); do\n R=$(curl -s -X POST \"$API/pods/$POD/start\" -H \"Authorization: Bearer $API_KEY\")\n ERR=$(echo \"$R\" | python -c \"import sys,json\ntry: d=json.load(sys.stdin)\nexcept: print('parse'); raise SystemExit\nprint(d.get('error','') if isinstance(d,dict) else '')\" 2>/dev/null || echo \"\")\n if [ -z \"$ERR\" ]; then\n for j in $(seq 1 18); do\n PD=$(curl -s \"$API/pods/$POD\" -H \"Authorization: Bearer $API_KEY\")\n ST=$(echo \"$PD\" | jget desiredStatus); HOST=$(echo \"$PD\" | jget publicIp); PORT=$(echo \"$PD\" | pmget)\n [ \"$ST\" = \"RUNNING\" ] && [ -n \"$HOST\" ] && [ -n \"$PORT\" ] && break\n sleep 8\n done\n [ -n \"$HOST\" ] && [ -n \"$PORT\" ] && { echo \"[driver] pod RUNNING $HOST:$PORT (attempt $i)\"; break; }\n fi\n echo \"[driver] start blocked (${ERR:-not-ready}) — retry in 60s ($i/60)\"\n HOST=\"\"; PORT=\"\"; sleep 60\ndone\n[ -n \"$HOST\" ] && [ -n \"$PORT\" ] || { echo \"[driver] FATAL: no GPU freed in ~60 min — recreate the pod on a host with availability (operator).\"; exit 1; }\n\n# ── 2. Wait for SSH ──────────────────────────────────────────────────────────\necho \"[driver] waiting for SSH…\"\nok=0; for i in $(seq 1 24); do $SSH -p \"$PORT\" root@\"$HOST\" \"echo ok\" 2>/dev/null | grep -q ok && { ok=1; echo \"[driver] SSH ready ($i)\"; break; }; sleep 5; done\n[ \"$ok\" = 1 ] || { echo \"[driver] FATAL: SSH never came up\"; exit 1; }\n\n# ── 3. Push runner + the frozen 500-Q eval set ───────────────────────────────\necho \"[driver] pushing runner + eval set…\"\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" scripts/train/baseline_pod_run.sh root@\"$HOST\":/workspace/ || { echo \"[driver] FATAL scp runner\"; exit 1; }\n$SSH -p \"$PORT\" root@\"$HOST\" \"mkdir -p /workspace/datasets\"\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" data/eval_reports/aria_eval_500q.jsonl root@\"$HOST\":/workspace/datasets/aria_eval_500q.jsonl || { echo \"[driver] FATAL scp eval set\"; exit 1; }\n\n# ── 4. Run the proven baseline ON THE POD ────────────────────────────────────\necho \"[driver] running baseline_pod_run.sh on the pod…\"\n$SSH -p \"$PORT\" root@\"$HOST\" \"DEEPSEEK_API_KEY='$DSK' bash /workspace/baseline_pod_run.sh\"\n\n# ── 5. Pull reports ──────────────────────────────────────────────────────────\necho \"[driver] pulling reports…\"\nmkdir -p data/eval_reports\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" root@\"$HOST\":/workspace/eval/aria_llm_v02_eval.json data/eval_reports/ 2>/dev/null || echo \"[driver] (v02 report not pulled)\"\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" root@\"$HOST\":/workspace/eval/deepseek_baseline_eval.json data/eval_reports/ 2>/dev/null || echo \"[driver] (deepseek report not pulled)\"\n\n# ── 6. Stop pod (trap also covers this) ──────────────────────────────────────\nstop_pod\necho \"[driver] DONE — reports in data/eval_reports/, pod stopped.\"\n"}, "9cc42d8ecc327e1258a09d9a04e53ffd35ecaf30": {"t": 1785758159.9877267, "n": 4285, "h": "4d31ebfbb2f875d5", "c": "#!/bin/bash\n# R-F1455 — LOCAL driver for the v0.2-vs-DeepSeek baseline. Runs from the operator\n# Windows box (has repo + .env + ~/.ssh/runpod_aria). Retries the pod start until a\n# GPU frees on the pinned host, then pushes + runs baseline_pod_run.sh ON THE POD\n# (proven shim recipe), pulls the reports, and ALWAYS stops the pod (EXIT trap).\n#\n# Cost-safe: start attempts on a capacity-blocked pod do not bill; the GPU only\n# bills once RUNNING, and the trap stops it on any exit.\nset -uo pipefail\n\nREPO=\"/c/code/crucix\"; cd \"$REPO\"\nPOD=\"7ei3hldcpz4j2v\"\nAPI=\"https://rest.runpod.io/v1\"\nAPI_KEY=$(grep -E \"^RUNPOD_API_KEY=\" .env | head -1 | cut -d= -f2- | tr -d '\"' | tr -d \"'\" | tr -d '\\r')\nDSK=$(grep -E \"^DEEPSEEK_API_KEY=\" .env | head -1 | cut -d= -f2- | tr -d '\"' | tr -d \"'\" | tr -d '\\r')\nKEY=\"/tmp/rpkey\"; cp ~/.ssh/runpod_aria \"$KEY\"; chmod 600 \"$KEY\"\nSSH=\"ssh -i $KEY -o StrictHostKeyChecking=no -o ConnectTimeout=10\"\n\nstop_pod(){ echo \"[driver] stopping pod $POD\"; curl -s -X POST \"$API/pods/$POD/stop\" -H \"Authorization: Bearer $API_KEY\" >/dev/null 2>&1 || true; }\ntrap stop_pod EXIT\n\njget(){ python -c \"import sys,json;d=json.load(sys.stdin);print(d.get('$1','') or '')\" 2>/dev/null; }\npmget(){ python -c \"import sys,json;d=json.load(sys.stdin);pm=d.get('portMappings') or {};print(pm.get('22') or '')\" 2>/dev/null; }\n\n# ── 1. Retry start until a GPU frees (cap ~60 min); $0 while blocked ──────────\necho \"[driver] retrying pod start until a GPU frees on the host…\"\nHOST=\"\"; PORT=\"\"\nfor i in $(seq 1 60); do\n R=$(curl -s -X POST \"$API/pods/$POD/start\" -H \"Authorization: Bearer $API_KEY\")\n ERR=$(echo \"$R\" | python -c \"import sys,json\ntry: d=json.load(sys.stdin)\nexcept: print('parse'); raise SystemExit\nprint(d.get('error','') if isinstance(d,dict) else '')\" 2>/dev/null || echo \"\")\n if [ -z \"$ERR\" ]; then\n for j in $(seq 1 18); do\n PD=$(curl -s \"$API/pods/$POD\" -H \"Authorization: Bearer $API_KEY\")\n ST=$(echo \"$PD\" | jget desiredStatus); HOST=$(echo \"$PD\" | jget publicIp); PORT=$(echo \"$PD\" | pmget)\n [ \"$ST\" = \"RUNNING\" ] && [ -n \"$HOST\" ] && [ -n \"$PORT\" ] && break\n sleep 8\n done\n [ -n \"$HOST\" ] && [ -n \"$PORT\" ] && { echo \"[driver] pod RUNNING $HOST:$PORT (attempt $i)\"; break; }\n fi\n echo \"[driver] start blocked (${ERR:-not-ready}) — retry in 60s ($i/60)\"\n HOST=\"\"; PORT=\"\"; sleep 60\ndone\n[ -n \"$HOST\" ] && [ -n \"$PORT\" ] || { echo \"[driver] FATAL: no GPU freed in ~60 min — recreate the pod on a host with availability (operator).\"; exit 1; }\n\n# ── 2. Wait for SSH ──────────────────────────────────────────────────────────\necho \"[driver] waiting for SSH…\"\nok=0; for i in $(seq 1 24); do $SSH -p \"$PORT\" root@\"$HOST\" \"echo ok\" 2>/dev/null | grep -q ok && { ok=1; echo \"[driver] SSH ready ($i)\"; break; }; sleep 5; done\n[ \"$ok\" = 1 ] || { echo \"[driver] FATAL: SSH never came up\"; exit 1; }\n\n# ── 3. Push runner + the frozen 500-Q eval set ───────────────────────────────\necho \"[driver] pushing runner + eval set…\"\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" scripts/train/baseline_pod_run.sh root@\"$HOST\":/workspace/ || { echo \"[driver] FATAL scp runner\"; exit 1; }\n$SSH -p \"$PORT\" root@\"$HOST\" \"mkdir -p /workspace/datasets\"\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" data/eval_reports/aria_eval_500q.jsonl root@\"$HOST\":/workspace/datasets/aria_eval_500q.jsonl || { echo \"[driver] FATAL scp eval set\"; exit 1; }\n\n# ── 4. Run the proven baseline ON THE POD ────────────────────────────────────\necho \"[driver] running baseline_pod_run.sh on the pod…\"\n$SSH -p \"$PORT\" root@\"$HOST\" \"DEEPSEEK_API_KEY='$DSK' bash /workspace/baseline_pod_run.sh\"\n\n# ── 5. Pull reports ──────────────────────────────────────────────────────────\necho \"[driver] pulling reports…\"\nmkdir -p data/eval_reports\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" root@\"$HOST\":/workspace/eval/aria_llm_v02_eval.json data/eval_reports/ 2>/dev/null || echo \"[driver] (v02 report not pulled)\"\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" root@\"$HOST\":/workspace/eval/deepseek_baseline_eval.json data/eval_reports/ 2>/dev/null || echo \"[driver] (deepseek report not pulled)\"\n\n# ── 6. Stop pod (trap also covers this) ──────────────────────────────────────\nstop_pod\necho \"[driver] DONE — reports in data/eval_reports/, pod stopped.\"\n"}, "2f51659938376b08c72ed831048a3fef7bdb559a": {"t": 1786067518.3738685, "n": 4285, "h": "4d31ebfbb2f875d5", "c": "#!/bin/bash\n# R-F1455 — LOCAL driver for the v0.2-vs-DeepSeek baseline. Runs from the operator\n# Windows box (has repo + .env + ~/.ssh/runpod_aria). Retries the pod start until a\n# GPU frees on the pinned host, then pushes + runs baseline_pod_run.sh ON THE POD\n# (proven shim recipe), pulls the reports, and ALWAYS stops the pod (EXIT trap).\n#\n# Cost-safe: start attempts on a capacity-blocked pod do not bill; the GPU only\n# bills once RUNNING, and the trap stops it on any exit.\nset -uo pipefail\n\nREPO=\"/c/code/crucix\"; cd \"$REPO\"\nPOD=\"7ei3hldcpz4j2v\"\nAPI=\"https://rest.runpod.io/v1\"\nAPI_KEY=$(grep -E \"^RUNPOD_API_KEY=\" .env | head -1 | cut -d= -f2- | tr -d '\"' | tr -d \"'\" | tr -d '\\r')\nDSK=$(grep -E \"^DEEPSEEK_API_KEY=\" .env | head -1 | cut -d= -f2- | tr -d '\"' | tr -d \"'\" | tr -d '\\r')\nKEY=\"/tmp/rpkey\"; cp ~/.ssh/runpod_aria \"$KEY\"; chmod 600 \"$KEY\"\nSSH=\"ssh -i $KEY -o StrictHostKeyChecking=no -o ConnectTimeout=10\"\n\nstop_pod(){ echo \"[driver] stopping pod $POD\"; curl -s -X POST \"$API/pods/$POD/stop\" -H \"Authorization: Bearer $API_KEY\" >/dev/null 2>&1 || true; }\ntrap stop_pod EXIT\n\njget(){ python -c \"import sys,json;d=json.load(sys.stdin);print(d.get('$1','') or '')\" 2>/dev/null; }\npmget(){ python -c \"import sys,json;d=json.load(sys.stdin);pm=d.get('portMappings') or {};print(pm.get('22') or '')\" 2>/dev/null; }\n\n# ── 1. Retry start until a GPU frees (cap ~60 min); $0 while blocked ──────────\necho \"[driver] retrying pod start until a GPU frees on the host…\"\nHOST=\"\"; PORT=\"\"\nfor i in $(seq 1 60); do\n R=$(curl -s -X POST \"$API/pods/$POD/start\" -H \"Authorization: Bearer $API_KEY\")\n ERR=$(echo \"$R\" | python -c \"import sys,json\ntry: d=json.load(sys.stdin)\nexcept: print('parse'); raise SystemExit\nprint(d.get('error','') if isinstance(d,dict) else '')\" 2>/dev/null || echo \"\")\n if [ -z \"$ERR\" ]; then\n for j in $(seq 1 18); do\n PD=$(curl -s \"$API/pods/$POD\" -H \"Authorization: Bearer $API_KEY\")\n ST=$(echo \"$PD\" | jget desiredStatus); HOST=$(echo \"$PD\" | jget publicIp); PORT=$(echo \"$PD\" | pmget)\n [ \"$ST\" = \"RUNNING\" ] && [ -n \"$HOST\" ] && [ -n \"$PORT\" ] && break\n sleep 8\n done\n [ -n \"$HOST\" ] && [ -n \"$PORT\" ] && { echo \"[driver] pod RUNNING $HOST:$PORT (attempt $i)\"; break; }\n fi\n echo \"[driver] start blocked (${ERR:-not-ready}) — retry in 60s ($i/60)\"\n HOST=\"\"; PORT=\"\"; sleep 60\ndone\n[ -n \"$HOST\" ] && [ -n \"$PORT\" ] || { echo \"[driver] FATAL: no GPU freed in ~60 min — recreate the pod on a host with availability (operator).\"; exit 1; }\n\n# ── 2. Wait for SSH ──────────────────────────────────────────────────────────\necho \"[driver] waiting for SSH…\"\nok=0; for i in $(seq 1 24); do $SSH -p \"$PORT\" root@\"$HOST\" \"echo ok\" 2>/dev/null | grep -q ok && { ok=1; echo \"[driver] SSH ready ($i)\"; break; }; sleep 5; done\n[ \"$ok\" = 1 ] || { echo \"[driver] FATAL: SSH never came up\"; exit 1; }\n\n# ── 3. Push runner + the frozen 500-Q eval set ───────────────────────────────\necho \"[driver] pushing runner + eval set…\"\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" scripts/train/baseline_pod_run.sh root@\"$HOST\":/workspace/ || { echo \"[driver] FATAL scp runner\"; exit 1; }\n$SSH -p \"$PORT\" root@\"$HOST\" \"mkdir -p /workspace/datasets\"\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" data/eval_reports/aria_eval_500q.jsonl root@\"$HOST\":/workspace/datasets/aria_eval_500q.jsonl || { echo \"[driver] FATAL scp eval set\"; exit 1; }\n\n# ── 4. Run the proven baseline ON THE POD ────────────────────────────────────\necho \"[driver] running baseline_pod_run.sh on the pod…\"\n$SSH -p \"$PORT\" root@\"$HOST\" \"DEEPSEEK_API_KEY='$DSK' bash /workspace/baseline_pod_run.sh\"\n\n# ── 5. Pull reports ──────────────────────────────────────────────────────────\necho \"[driver] pulling reports…\"\nmkdir -p data/eval_reports\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" root@\"$HOST\":/workspace/eval/aria_llm_v02_eval.json data/eval_reports/ 2>/dev/null || echo \"[driver] (v02 report not pulled)\"\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" root@\"$HOST\":/workspace/eval/deepseek_baseline_eval.json data/eval_reports/ 2>/dev/null || echo \"[driver] (deepseek report not pulled)\"\n\n# ── 6. Stop pod (trap also covers this) ──────────────────────────────────────\nstop_pod\necho \"[driver] DONE — reports in data/eval_reports/, pod stopped.\"\n"}, "4046b84d1710f56e6baf6752d32289ad60da382d": {"t": 1786068347.8895092, "n": 4285, "h": "4d31ebfbb2f875d5", "c": "#!/bin/bash\n# R-F1455 — LOCAL driver for the v0.2-vs-DeepSeek baseline. Runs from the operator\n# Windows box (has repo + .env + ~/.ssh/runpod_aria). Retries the pod start until a\n# GPU frees on the pinned host, then pushes + runs baseline_pod_run.sh ON THE POD\n# (proven shim recipe), pulls the reports, and ALWAYS stops the pod (EXIT trap).\n#\n# Cost-safe: start attempts on a capacity-blocked pod do not bill; the GPU only\n# bills once RUNNING, and the trap stops it on any exit.\nset -uo pipefail\n\nREPO=\"/c/code/crucix\"; cd \"$REPO\"\nPOD=\"7ei3hldcpz4j2v\"\nAPI=\"https://rest.runpod.io/v1\"\nAPI_KEY=$(grep -E \"^RUNPOD_API_KEY=\" .env | head -1 | cut -d= -f2- | tr -d '\"' | tr -d \"'\" | tr -d '\\r')\nDSK=$(grep -E \"^DEEPSEEK_API_KEY=\" .env | head -1 | cut -d= -f2- | tr -d '\"' | tr -d \"'\" | tr -d '\\r')\nKEY=\"/tmp/rpkey\"; cp ~/.ssh/runpod_aria \"$KEY\"; chmod 600 \"$KEY\"\nSSH=\"ssh -i $KEY -o StrictHostKeyChecking=no -o ConnectTimeout=10\"\n\nstop_pod(){ echo \"[driver] stopping pod $POD\"; curl -s -X POST \"$API/pods/$POD/stop\" -H \"Authorization: Bearer $API_KEY\" >/dev/null 2>&1 || true; }\ntrap stop_pod EXIT\n\njget(){ python -c \"import sys,json;d=json.load(sys.stdin);print(d.get('$1','') or '')\" 2>/dev/null; }\npmget(){ python -c \"import sys,json;d=json.load(sys.stdin);pm=d.get('portMappings') or {};print(pm.get('22') or '')\" 2>/dev/null; }\n\n# ── 1. Retry start until a GPU frees (cap ~60 min); $0 while blocked ──────────\necho \"[driver] retrying pod start until a GPU frees on the host…\"\nHOST=\"\"; PORT=\"\"\nfor i in $(seq 1 60); do\n R=$(curl -s -X POST \"$API/pods/$POD/start\" -H \"Authorization: Bearer $API_KEY\")\n ERR=$(echo \"$R\" | python -c \"import sys,json\ntry: d=json.load(sys.stdin)\nexcept: print('parse'); raise SystemExit\nprint(d.get('error','') if isinstance(d,dict) else '')\" 2>/dev/null || echo \"\")\n if [ -z \"$ERR\" ]; then\n for j in $(seq 1 18); do\n PD=$(curl -s \"$API/pods/$POD\" -H \"Authorization: Bearer $API_KEY\")\n ST=$(echo \"$PD\" | jget desiredStatus); HOST=$(echo \"$PD\" | jget publicIp); PORT=$(echo \"$PD\" | pmget)\n [ \"$ST\" = \"RUNNING\" ] && [ -n \"$HOST\" ] && [ -n \"$PORT\" ] && break\n sleep 8\n done\n [ -n \"$HOST\" ] && [ -n \"$PORT\" ] && { echo \"[driver] pod RUNNING $HOST:$PORT (attempt $i)\"; break; }\n fi\n echo \"[driver] start blocked (${ERR:-not-ready}) — retry in 60s ($i/60)\"\n HOST=\"\"; PORT=\"\"; sleep 60\ndone\n[ -n \"$HOST\" ] && [ -n \"$PORT\" ] || { echo \"[driver] FATAL: no GPU freed in ~60 min — recreate the pod on a host with availability (operator).\"; exit 1; }\n\n# ── 2. Wait for SSH ──────────────────────────────────────────────────────────\necho \"[driver] waiting for SSH…\"\nok=0; for i in $(seq 1 24); do $SSH -p \"$PORT\" root@\"$HOST\" \"echo ok\" 2>/dev/null | grep -q ok && { ok=1; echo \"[driver] SSH ready ($i)\"; break; }; sleep 5; done\n[ \"$ok\" = 1 ] || { echo \"[driver] FATAL: SSH never came up\"; exit 1; }\n\n# ── 3. Push runner + the frozen 500-Q eval set ───────────────────────────────\necho \"[driver] pushing runner + eval set…\"\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" scripts/train/baseline_pod_run.sh root@\"$HOST\":/workspace/ || { echo \"[driver] FATAL scp runner\"; exit 1; }\n$SSH -p \"$PORT\" root@\"$HOST\" \"mkdir -p /workspace/datasets\"\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" data/eval_reports/aria_eval_500q.jsonl root@\"$HOST\":/workspace/datasets/aria_eval_500q.jsonl || { echo \"[driver] FATAL scp eval set\"; exit 1; }\n\n# ── 4. Run the proven baseline ON THE POD ────────────────────────────────────\necho \"[driver] running baseline_pod_run.sh on the pod…\"\n$SSH -p \"$PORT\" root@\"$HOST\" \"DEEPSEEK_API_KEY='$DSK' bash /workspace/baseline_pod_run.sh\"\n\n# ── 5. Pull reports ──────────────────────────────────────────────────────────\necho \"[driver] pulling reports…\"\nmkdir -p data/eval_reports\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" root@\"$HOST\":/workspace/eval/aria_llm_v02_eval.json data/eval_reports/ 2>/dev/null || echo \"[driver] (v02 report not pulled)\"\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" root@\"$HOST\":/workspace/eval/deepseek_baseline_eval.json data/eval_reports/ 2>/dev/null || echo \"[driver] (deepseek report not pulled)\"\n\n# ── 6. Stop pod (trap also covers this) ──────────────────────────────────────\nstop_pod\necho \"[driver] DONE — reports in data/eval_reports/, pod stopped.\"\n"}}, "scripts/train/eval_only_v07.sh": {"3a9645c8ec96260be7edd2c46a655b1e473f8c18": {"t": 1785745975.0724564, "n": 7029, "h": "4e0b64870af0d813", "c": "#!/bin/bash\n# EVAL-ONLY v0.7 driver (R-F17xx) — operator approved an OFFLINE eval of the best\n# own-model. Serves the EXISTING on-volume adapter (NO retrain) and runs the 500-Q\n# open-book eval, then STOPS the pod. Hardened like run_v0_7_grounded_cycle.sh:\n# auto-arms the on-pod self-stop watcher + timeout-wraps every poll SSH, so a hung\n# Windows SSH can't freeze the loop and the pod self-stops even if this driver dies.\n# Does NOT set ARIA_LLM_URL. Does NOT retrain.\nset -uo pipefail\nREPO=\"/c/code/crucix\"; cd \"$REPO\"\nPOD=\"${RUNPOD_POD_ID:-2adkzeri6fa2zi}\"\nAPI=\"https://rest.runpod.io/v1\"\nAPI_KEY=$(grep -E \"^RUNPOD_API_KEY=\" .env | head -1 | cut -d= -f2- | tr -d '\"' | tr -d \"'\" | tr -d '\\r')\nDSK=$(grep -E \"^DEEPSEEK_API_KEY=\" .env | head -1 | cut -d= -f2- | tr -d '\"' | tr -d \"'\" | tr -d '\\r')\n[ -n \"$API_KEY\" ] || { echo \"[eval] FATAL: RUNPOD_API_KEY not in .env\"; exit 1; }\nEVAL_LOCAL=\"${EVAL_LOCAL:-data/eval_reports/aria_eval_500q_openbook.jsonl}\"\n[ -s \"$EVAL_LOCAL\" ] || EVAL_LOCAL=\"data/training/aria_eval_500q_openbook.jsonl\"\n[ -s \"$EVAL_LOCAL\" ] || { echo \"[eval] FATAL: open-book eval set not found\"; exit 1; }\necho \"[eval] open-book eval set: $EVAL_LOCAL ($(wc -l < \"$EVAL_LOCAL\") Q)\"\n\nKEY=\"/tmp/rpkey_eval\"; cp ~/.ssh/runpod_aria \"$KEY\"; chmod 600 \"$KEY\"\nSSH=\"ssh -i $KEY -o StrictHostKeyChecking=no -o ConnectTimeout=15 -o ServerAliveInterval=30 -o ServerAliveCountMax=6\"\nTSSH(){ timeout 75 $SSH \"$@\"; }\njget(){ python -c \"import sys,json;d=json.load(sys.stdin);print(d.get('$1','') or '')\" 2>/dev/null; }\npmget(){ python -c \"import sys,json;d=json.load(sys.stdin);pm=d.get('portMappings') or {};print(pm.get('22') or '')\" 2>/dev/null; }\nstop_pod(){ echo \"[eval] stopping pod $POD\"; curl -s -X POST \"$API/pods/$POD/stop\" -H \"Authorization: Bearer $API_KEY\" >/dev/null 2>&1 || true; }\ntrap stop_pod EXIT\n\n# 1. Ensure RUNNING (US-KS-2 is capacity-blocked intermittently → retry up to ~40 min)\nHOST=\"\"; PORT=\"\"\nfor i in $(seq 1 40); do\n PD=$(curl -s \"$API/pods/$POD\" -H \"Authorization: Bearer $API_KEY\")\n ST=$(echo \"$PD\" | jget desiredStatus)\n if [ \"$ST\" != \"RUNNING\" ]; then\n R=$(curl -s -X POST \"$API/pods/$POD/start\" -H \"Authorization: Bearer $API_KEY\")\n ERR=$(echo \"$R\" | python -c \"import sys,json\ntry: d=json.load(sys.stdin); print(d.get('error','') if isinstance(d,dict) else '')\nexcept: print('')\" 2>/dev/null || echo \"\")\n [ -n \"$ERR\" ] && { echo \"[eval] start blocked ($ERR) — retry 60s ($i/20)\"; sleep 60; continue; }\n fi\n for j in $(seq 1 18); do\n PD=$(curl -s \"$API/pods/$POD\" -H \"Authorization: Bearer $API_KEY\")\n ST=$(echo \"$PD\" | jget desiredStatus); HOST=$(echo \"$PD\" | jget publicIp); PORT=$(echo \"$PD\" | pmget)\n [ \"$ST\" = \"RUNNING\" ] && [ -n \"$HOST\" ] && [ -n \"$PORT\" ] && break\n sleep 8\n done\n [ -n \"$HOST\" ] && [ -n \"$PORT\" ] && { echo \"[eval] pod RUNNING $HOST:$PORT\"; break; }\n sleep 30\ndone\n[ -n \"$HOST\" ] && [ -n \"$PORT\" ] || { echo \"[eval] FATAL: pod not RUNNING\"; exit 1; }\n\n# 2. Wait SSH, then settle (sshd is flaky in the first ~30s after boot → \"Connection\n# closed by remote host\" on scp). R-F17xx: wait for THREE consecutive good probes.\nok=0\nfor i in $(seq 1 36); do\n if TSSH -p \"$PORT\" root@\"$HOST\" \"echo ok\" 2>/dev/null | grep -q ok; then ok=$((ok+1)); else ok=0; fi\n [ \"$ok\" -ge 3 ] && { echo \"[eval] SSH stable ($i)\"; break; }\n sleep 5\ndone\nsleep 15 # extra settle before bulk transfers\n\n# scp with retry — flaky post-boot sshd drops the first connections.\nRSCP(){ local src=\"$1\" dst=\"$2\" t; for t in 1 2 3 4 5; do\n if timeout 90 scp -i \"$KEY\" -o StrictHostKeyChecking=no -o ConnectTimeout=15 -P \"$PORT\" \"$src\" root@\"$HOST\":\"$dst\" 2>/dev/null; then return 0; fi\n echo \"[eval] scp retry $t for $(basename \"$src\")…\"; sleep 10\ndone; return 1; }\n\n# 3. Push runner + scripts + open-book eval set + self-stop watcher + eval deps\necho \"[eval] pushing runner + scripts + eval set + watcher…\"\nfor i in 1 2 3; do TSSH -p \"$PORT\" root@\"$HOST\" \"mkdir -p /workspace/datasets /workspace/crucix/scripts/train /workspace/crucix/aria_service/intel /workspace/logs\" 2>/dev/null && break; sleep 8; done\nRSCP scripts/train/pod_eval_only.sh /workspace/ || { echo \"[eval] FATAL scp runner\"; exit 1; }\nRSCP scripts/train/pod_selfstop_watch_v04.sh /workspace/ || { echo \"[eval] FATAL scp watcher\"; exit 1; }\nfor f in serve_eval_shim.py eval_aria_llm.py; do\n RSCP \"scripts/train/$f\" \"/workspace/crucix/scripts/train/$f\" || { echo \"[eval] FATAL scp $f\"; exit 1; }\ndone\nRSCP \"$EVAL_LOCAL\" /workspace/datasets/aria_eval_openbook.jsonl || { echo \"[eval] FATAL scp eval set\"; exit 1; }\nfor f in aria_service/__init__.py aria_service/intel/__init__.py aria_service/intel/engine_wiring.py aria_service/intel/prompt_injection_suite.py; do\n RSCP \"$f\" \"/workspace/crucix/$f\" || echo \"[eval] (optional dep $f not pushed)\"\ndone\n\n# 4. ARM the self-stop watcher FIRST (cost safety even if this driver dies), then launch eval-only DETACHED\necho \"[eval] ARMING on-pod self-stop watcher (GRACE=900)…\"\nTSSH -p \"$PORT\" root@\"$HOST\" \\\n \"POD_ID=$POD RP_KEY='$API_KEY' GRACE=900 setsid nohup bash /workspace/pod_selfstop_watch_v04.sh >/workspace/logs/_selfstop_launch.log 2>&1 < /dev/null & echo ARMED\" \\\n || echo \"[eval] WARN: watcher arm failed — relying on EXIT trap\"\necho \"[eval] launching EVAL-ONLY runner DETACHED…\"\nTSSH -p \"$PORT\" root@\"$HOST\" \\\n \"rm -f /workspace/eval/_cycle_status; DEEPSEEK_API_KEY='$DSK' \\\n setsid nohup bash /workspace/pod_eval_only.sh > /workspace/logs/eval_only.log 2>&1 < /dev/null & echo STARTED\" \\\n || { echo \"[eval] FATAL: could not launch eval runner\"; exit 1; }\n\n# 5. Poll for completion (cap ~1.5h; timeout-wrapped SSH)\necho \"[eval] polling (cap ~1.5h)…\"\nRC=\"\"\nfor i in $(seq 1 45); do\n sleep 120\n OUT=$(TSSH -p \"$PORT\" root@\"$HOST\" \\\n 'printf \"RC=%s\\n\" \"$(cat /workspace/eval/_cycle_status 2>/dev/null)\"; tail -1 /workspace/logs/eval_only.log 2>/dev/null' \\\n 2>/dev/null | tr -d '\\r')\n RC=$(printf '%s\\n' \"$OUT\" | sed -n 's/^RC=//p' | tr -d ' ')\n LINE=$(printf '%s\\n' \"$OUT\" | grep -v '^RC=' | tail -1)\n echo \"[eval] [$i/45] ${LINE:-(no log line yet)}\"\n [ -n \"$RC\" ] && { echo \"[eval] runner finished (exit $RC)\"; break; }\ndone\n\n# 6. Pull report\nmkdir -p data/eval_reports\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" root@\"$HOST\":/workspace/eval/aria_llm_v0_7_eval_rerun.json data/eval_reports/aria_llm_v0_7_eval_rerun.json 2>/dev/null || echo \"[eval] (report not pulled)\"\necho \"[eval] === RESULT ===\"\npython - <<'PY'\nimport json, os\nf=\"data/eval_reports/aria_llm_v0_7_eval_rerun.json\"\nif not os.path.exists(f): print(\"report MISSING — check pod logs\"); raise SystemExit\nd=json.load(open(f,encoding=\"utf-8\")); dd=d.get(\"defence_dd\") or {}; pi=d.get(\"prompt_injection\") or {}\na=dd.get(\"accuracy\")\nprint(f\"v0.7 (rerun, open-book): judge-DD={a} (n={dd.get('total')}) | leak_rate={pi.get('leak_rate')}\")\nprint(f\"DeepSeek parity=0.336 | prior v0.7=0.308 | G1: {'PASS' if (a or 0)>=0.336 else 'FAIL'}\")\nPY\nstop_pod\necho \"[eval] DONE — pod stopped.\"\n"}, "695edfe6da5a7f0b2bf510ef0bd15bfdfb68702f": {"t": 1785758160.226149, "n": 7029, "h": "4e0b64870af0d813", "c": "#!/bin/bash\n# EVAL-ONLY v0.7 driver (R-F17xx) — operator approved an OFFLINE eval of the best\n# own-model. Serves the EXISTING on-volume adapter (NO retrain) and runs the 500-Q\n# open-book eval, then STOPS the pod. Hardened like run_v0_7_grounded_cycle.sh:\n# auto-arms the on-pod self-stop watcher + timeout-wraps every poll SSH, so a hung\n# Windows SSH can't freeze the loop and the pod self-stops even if this driver dies.\n# Does NOT set ARIA_LLM_URL. Does NOT retrain.\nset -uo pipefail\nREPO=\"/c/code/crucix\"; cd \"$REPO\"\nPOD=\"${RUNPOD_POD_ID:-2adkzeri6fa2zi}\"\nAPI=\"https://rest.runpod.io/v1\"\nAPI_KEY=$(grep -E \"^RUNPOD_API_KEY=\" .env | head -1 | cut -d= -f2- | tr -d '\"' | tr -d \"'\" | tr -d '\\r')\nDSK=$(grep -E \"^DEEPSEEK_API_KEY=\" .env | head -1 | cut -d= -f2- | tr -d '\"' | tr -d \"'\" | tr -d '\\r')\n[ -n \"$API_KEY\" ] || { echo \"[eval] FATAL: RUNPOD_API_KEY not in .env\"; exit 1; }\nEVAL_LOCAL=\"${EVAL_LOCAL:-data/eval_reports/aria_eval_500q_openbook.jsonl}\"\n[ -s \"$EVAL_LOCAL\" ] || EVAL_LOCAL=\"data/training/aria_eval_500q_openbook.jsonl\"\n[ -s \"$EVAL_LOCAL\" ] || { echo \"[eval] FATAL: open-book eval set not found\"; exit 1; }\necho \"[eval] open-book eval set: $EVAL_LOCAL ($(wc -l < \"$EVAL_LOCAL\") Q)\"\n\nKEY=\"/tmp/rpkey_eval\"; cp ~/.ssh/runpod_aria \"$KEY\"; chmod 600 \"$KEY\"\nSSH=\"ssh -i $KEY -o StrictHostKeyChecking=no -o ConnectTimeout=15 -o ServerAliveInterval=30 -o ServerAliveCountMax=6\"\nTSSH(){ timeout 75 $SSH \"$@\"; }\njget(){ python -c \"import sys,json;d=json.load(sys.stdin);print(d.get('$1','') or '')\" 2>/dev/null; }\npmget(){ python -c \"import sys,json;d=json.load(sys.stdin);pm=d.get('portMappings') or {};print(pm.get('22') or '')\" 2>/dev/null; }\nstop_pod(){ echo \"[eval] stopping pod $POD\"; curl -s -X POST \"$API/pods/$POD/stop\" -H \"Authorization: Bearer $API_KEY\" >/dev/null 2>&1 || true; }\ntrap stop_pod EXIT\n\n# 1. Ensure RUNNING (US-KS-2 is capacity-blocked intermittently → retry up to ~40 min)\nHOST=\"\"; PORT=\"\"\nfor i in $(seq 1 40); do\n PD=$(curl -s \"$API/pods/$POD\" -H \"Authorization: Bearer $API_KEY\")\n ST=$(echo \"$PD\" | jget desiredStatus)\n if [ \"$ST\" != \"RUNNING\" ]; then\n R=$(curl -s -X POST \"$API/pods/$POD/start\" -H \"Authorization: Bearer $API_KEY\")\n ERR=$(echo \"$R\" | python -c \"import sys,json\ntry: d=json.load(sys.stdin); print(d.get('error','') if isinstance(d,dict) else '')\nexcept: print('')\" 2>/dev/null || echo \"\")\n [ -n \"$ERR\" ] && { echo \"[eval] start blocked ($ERR) — retry 60s ($i/20)\"; sleep 60; continue; }\n fi\n for j in $(seq 1 18); do\n PD=$(curl -s \"$API/pods/$POD\" -H \"Authorization: Bearer $API_KEY\")\n ST=$(echo \"$PD\" | jget desiredStatus); HOST=$(echo \"$PD\" | jget publicIp); PORT=$(echo \"$PD\" | pmget)\n [ \"$ST\" = \"RUNNING\" ] && [ -n \"$HOST\" ] && [ -n \"$PORT\" ] && break\n sleep 8\n done\n [ -n \"$HOST\" ] && [ -n \"$PORT\" ] && { echo \"[eval] pod RUNNING $HOST:$PORT\"; break; }\n sleep 30\ndone\n[ -n \"$HOST\" ] && [ -n \"$PORT\" ] || { echo \"[eval] FATAL: pod not RUNNING\"; exit 1; }\n\n# 2. Wait SSH, then settle (sshd is flaky in the first ~30s after boot → \"Connection\n# closed by remote host\" on scp). R-F17xx: wait for THREE consecutive good probes.\nok=0\nfor i in $(seq 1 36); do\n if TSSH -p \"$PORT\" root@\"$HOST\" \"echo ok\" 2>/dev/null | grep -q ok; then ok=$((ok+1)); else ok=0; fi\n [ \"$ok\" -ge 3 ] && { echo \"[eval] SSH stable ($i)\"; break; }\n sleep 5\ndone\nsleep 15 # extra settle before bulk transfers\n\n# scp with retry — flaky post-boot sshd drops the first connections.\nRSCP(){ local src=\"$1\" dst=\"$2\" t; for t in 1 2 3 4 5; do\n if timeout 90 scp -i \"$KEY\" -o StrictHostKeyChecking=no -o ConnectTimeout=15 -P \"$PORT\" \"$src\" root@\"$HOST\":\"$dst\" 2>/dev/null; then return 0; fi\n echo \"[eval] scp retry $t for $(basename \"$src\")…\"; sleep 10\ndone; return 1; }\n\n# 3. Push runner + scripts + open-book eval set + self-stop watcher + eval deps\necho \"[eval] pushing runner + scripts + eval set + watcher…\"\nfor i in 1 2 3; do TSSH -p \"$PORT\" root@\"$HOST\" \"mkdir -p /workspace/datasets /workspace/crucix/scripts/train /workspace/crucix/aria_service/intel /workspace/logs\" 2>/dev/null && break; sleep 8; done\nRSCP scripts/train/pod_eval_only.sh /workspace/ || { echo \"[eval] FATAL scp runner\"; exit 1; }\nRSCP scripts/train/pod_selfstop_watch_v04.sh /workspace/ || { echo \"[eval] FATAL scp watcher\"; exit 1; }\nfor f in serve_eval_shim.py eval_aria_llm.py; do\n RSCP \"scripts/train/$f\" \"/workspace/crucix/scripts/train/$f\" || { echo \"[eval] FATAL scp $f\"; exit 1; }\ndone\nRSCP \"$EVAL_LOCAL\" /workspace/datasets/aria_eval_openbook.jsonl || { echo \"[eval] FATAL scp eval set\"; exit 1; }\nfor f in aria_service/__init__.py aria_service/intel/__init__.py aria_service/intel/engine_wiring.py aria_service/intel/prompt_injection_suite.py; do\n RSCP \"$f\" \"/workspace/crucix/$f\" || echo \"[eval] (optional dep $f not pushed)\"\ndone\n\n# 4. ARM the self-stop watcher FIRST (cost safety even if this driver dies), then launch eval-only DETACHED\necho \"[eval] ARMING on-pod self-stop watcher (GRACE=900)…\"\nTSSH -p \"$PORT\" root@\"$HOST\" \\\n \"POD_ID=$POD RP_KEY='$API_KEY' GRACE=900 setsid nohup bash /workspace/pod_selfstop_watch_v04.sh >/workspace/logs/_selfstop_launch.log 2>&1 < /dev/null & echo ARMED\" \\\n || echo \"[eval] WARN: watcher arm failed — relying on EXIT trap\"\necho \"[eval] launching EVAL-ONLY runner DETACHED…\"\nTSSH -p \"$PORT\" root@\"$HOST\" \\\n \"rm -f /workspace/eval/_cycle_status; DEEPSEEK_API_KEY='$DSK' \\\n setsid nohup bash /workspace/pod_eval_only.sh > /workspace/logs/eval_only.log 2>&1 < /dev/null & echo STARTED\" \\\n || { echo \"[eval] FATAL: could not launch eval runner\"; exit 1; }\n\n# 5. Poll for completion (cap ~1.5h; timeout-wrapped SSH)\necho \"[eval] polling (cap ~1.5h)…\"\nRC=\"\"\nfor i in $(seq 1 45); do\n sleep 120\n OUT=$(TSSH -p \"$PORT\" root@\"$HOST\" \\\n 'printf \"RC=%s\\n\" \"$(cat /workspace/eval/_cycle_status 2>/dev/null)\"; tail -1 /workspace/logs/eval_only.log 2>/dev/null' \\\n 2>/dev/null | tr -d '\\r')\n RC=$(printf '%s\\n' \"$OUT\" | sed -n 's/^RC=//p' | tr -d ' ')\n LINE=$(printf '%s\\n' \"$OUT\" | grep -v '^RC=' | tail -1)\n echo \"[eval] [$i/45] ${LINE:-(no log line yet)}\"\n [ -n \"$RC\" ] && { echo \"[eval] runner finished (exit $RC)\"; break; }\ndone\n\n# 6. Pull report\nmkdir -p data/eval_reports\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" root@\"$HOST\":/workspace/eval/aria_llm_v0_7_eval_rerun.json data/eval_reports/aria_llm_v0_7_eval_rerun.json 2>/dev/null || echo \"[eval] (report not pulled)\"\necho \"[eval] === RESULT ===\"\npython - <<'PY'\nimport json, os\nf=\"data/eval_reports/aria_llm_v0_7_eval_rerun.json\"\nif not os.path.exists(f): print(\"report MISSING — check pod logs\"); raise SystemExit\nd=json.load(open(f,encoding=\"utf-8\")); dd=d.get(\"defence_dd\") or {}; pi=d.get(\"prompt_injection\") or {}\na=dd.get(\"accuracy\")\nprint(f\"v0.7 (rerun, open-book): judge-DD={a} (n={dd.get('total')}) | leak_rate={pi.get('leak_rate')}\")\nprint(f\"DeepSeek parity=0.336 | prior v0.7=0.308 | G1: {'PASS' if (a or 0)>=0.336 else 'FAIL'}\")\nPY\nstop_pod\necho \"[eval] DONE — pod stopped.\"\n"}, "2f51659938376b08c72ed831048a3fef7bdb559a": {"t": 1786067520.1810849, "n": 7029, "h": "4e0b64870af0d813", "c": "#!/bin/bash\n# EVAL-ONLY v0.7 driver (R-F17xx) — operator approved an OFFLINE eval of the best\n# own-model. Serves the EXISTING on-volume adapter (NO retrain) and runs the 500-Q\n# open-book eval, then STOPS the pod. Hardened like run_v0_7_grounded_cycle.sh:\n# auto-arms the on-pod self-stop watcher + timeout-wraps every poll SSH, so a hung\n# Windows SSH can't freeze the loop and the pod self-stops even if this driver dies.\n# Does NOT set ARIA_LLM_URL. Does NOT retrain.\nset -uo pipefail\nREPO=\"/c/code/crucix\"; cd \"$REPO\"\nPOD=\"${RUNPOD_POD_ID:-2adkzeri6fa2zi}\"\nAPI=\"https://rest.runpod.io/v1\"\nAPI_KEY=$(grep -E \"^RUNPOD_API_KEY=\" .env | head -1 | cut -d= -f2- | tr -d '\"' | tr -d \"'\" | tr -d '\\r')\nDSK=$(grep -E \"^DEEPSEEK_API_KEY=\" .env | head -1 | cut -d= -f2- | tr -d '\"' | tr -d \"'\" | tr -d '\\r')\n[ -n \"$API_KEY\" ] || { echo \"[eval] FATAL: RUNPOD_API_KEY not in .env\"; exit 1; }\nEVAL_LOCAL=\"${EVAL_LOCAL:-data/eval_reports/aria_eval_500q_openbook.jsonl}\"\n[ -s \"$EVAL_LOCAL\" ] || EVAL_LOCAL=\"data/training/aria_eval_500q_openbook.jsonl\"\n[ -s \"$EVAL_LOCAL\" ] || { echo \"[eval] FATAL: open-book eval set not found\"; exit 1; }\necho \"[eval] open-book eval set: $EVAL_LOCAL ($(wc -l < \"$EVAL_LOCAL\") Q)\"\n\nKEY=\"/tmp/rpkey_eval\"; cp ~/.ssh/runpod_aria \"$KEY\"; chmod 600 \"$KEY\"\nSSH=\"ssh -i $KEY -o StrictHostKeyChecking=no -o ConnectTimeout=15 -o ServerAliveInterval=30 -o ServerAliveCountMax=6\"\nTSSH(){ timeout 75 $SSH \"$@\"; }\njget(){ python -c \"import sys,json;d=json.load(sys.stdin);print(d.get('$1','') or '')\" 2>/dev/null; }\npmget(){ python -c \"import sys,json;d=json.load(sys.stdin);pm=d.get('portMappings') or {};print(pm.get('22') or '')\" 2>/dev/null; }\nstop_pod(){ echo \"[eval] stopping pod $POD\"; curl -s -X POST \"$API/pods/$POD/stop\" -H \"Authorization: Bearer $API_KEY\" >/dev/null 2>&1 || true; }\ntrap stop_pod EXIT\n\n# 1. Ensure RUNNING (US-KS-2 is capacity-blocked intermittently → retry up to ~40 min)\nHOST=\"\"; PORT=\"\"\nfor i in $(seq 1 40); do\n PD=$(curl -s \"$API/pods/$POD\" -H \"Authorization: Bearer $API_KEY\")\n ST=$(echo \"$PD\" | jget desiredStatus)\n if [ \"$ST\" != \"RUNNING\" ]; then\n R=$(curl -s -X POST \"$API/pods/$POD/start\" -H \"Authorization: Bearer $API_KEY\")\n ERR=$(echo \"$R\" | python -c \"import sys,json\ntry: d=json.load(sys.stdin); print(d.get('error','') if isinstance(d,dict) else '')\nexcept: print('')\" 2>/dev/null || echo \"\")\n [ -n \"$ERR\" ] && { echo \"[eval] start blocked ($ERR) — retry 60s ($i/20)\"; sleep 60; continue; }\n fi\n for j in $(seq 1 18); do\n PD=$(curl -s \"$API/pods/$POD\" -H \"Authorization: Bearer $API_KEY\")\n ST=$(echo \"$PD\" | jget desiredStatus); HOST=$(echo \"$PD\" | jget publicIp); PORT=$(echo \"$PD\" | pmget)\n [ \"$ST\" = \"RUNNING\" ] && [ -n \"$HOST\" ] && [ -n \"$PORT\" ] && break\n sleep 8\n done\n [ -n \"$HOST\" ] && [ -n \"$PORT\" ] && { echo \"[eval] pod RUNNING $HOST:$PORT\"; break; }\n sleep 30\ndone\n[ -n \"$HOST\" ] && [ -n \"$PORT\" ] || { echo \"[eval] FATAL: pod not RUNNING\"; exit 1; }\n\n# 2. Wait SSH, then settle (sshd is flaky in the first ~30s after boot → \"Connection\n# closed by remote host\" on scp). R-F17xx: wait for THREE consecutive good probes.\nok=0\nfor i in $(seq 1 36); do\n if TSSH -p \"$PORT\" root@\"$HOST\" \"echo ok\" 2>/dev/null | grep -q ok; then ok=$((ok+1)); else ok=0; fi\n [ \"$ok\" -ge 3 ] && { echo \"[eval] SSH stable ($i)\"; break; }\n sleep 5\ndone\nsleep 15 # extra settle before bulk transfers\n\n# scp with retry — flaky post-boot sshd drops the first connections.\nRSCP(){ local src=\"$1\" dst=\"$2\" t; for t in 1 2 3 4 5; do\n if timeout 90 scp -i \"$KEY\" -o StrictHostKeyChecking=no -o ConnectTimeout=15 -P \"$PORT\" \"$src\" root@\"$HOST\":\"$dst\" 2>/dev/null; then return 0; fi\n echo \"[eval] scp retry $t for $(basename \"$src\")…\"; sleep 10\ndone; return 1; }\n\n# 3. Push runner + scripts + open-book eval set + self-stop watcher + eval deps\necho \"[eval] pushing runner + scripts + eval set + watcher…\"\nfor i in 1 2 3; do TSSH -p \"$PORT\" root@\"$HOST\" \"mkdir -p /workspace/datasets /workspace/crucix/scripts/train /workspace/crucix/aria_service/intel /workspace/logs\" 2>/dev/null && break; sleep 8; done\nRSCP scripts/train/pod_eval_only.sh /workspace/ || { echo \"[eval] FATAL scp runner\"; exit 1; }\nRSCP scripts/train/pod_selfstop_watch_v04.sh /workspace/ || { echo \"[eval] FATAL scp watcher\"; exit 1; }\nfor f in serve_eval_shim.py eval_aria_llm.py; do\n RSCP \"scripts/train/$f\" \"/workspace/crucix/scripts/train/$f\" || { echo \"[eval] FATAL scp $f\"; exit 1; }\ndone\nRSCP \"$EVAL_LOCAL\" /workspace/datasets/aria_eval_openbook.jsonl || { echo \"[eval] FATAL scp eval set\"; exit 1; }\nfor f in aria_service/__init__.py aria_service/intel/__init__.py aria_service/intel/engine_wiring.py aria_service/intel/prompt_injection_suite.py; do\n RSCP \"$f\" \"/workspace/crucix/$f\" || echo \"[eval] (optional dep $f not pushed)\"\ndone\n\n# 4. ARM the self-stop watcher FIRST (cost safety even if this driver dies), then launch eval-only DETACHED\necho \"[eval] ARMING on-pod self-stop watcher (GRACE=900)…\"\nTSSH -p \"$PORT\" root@\"$HOST\" \\\n \"POD_ID=$POD RP_KEY='$API_KEY' GRACE=900 setsid nohup bash /workspace/pod_selfstop_watch_v04.sh >/workspace/logs/_selfstop_launch.log 2>&1 < /dev/null & echo ARMED\" \\\n || echo \"[eval] WARN: watcher arm failed — relying on EXIT trap\"\necho \"[eval] launching EVAL-ONLY runner DETACHED…\"\nTSSH -p \"$PORT\" root@\"$HOST\" \\\n \"rm -f /workspace/eval/_cycle_status; DEEPSEEK_API_KEY='$DSK' \\\n setsid nohup bash /workspace/pod_eval_only.sh > /workspace/logs/eval_only.log 2>&1 < /dev/null & echo STARTED\" \\\n || { echo \"[eval] FATAL: could not launch eval runner\"; exit 1; }\n\n# 5. Poll for completion (cap ~1.5h; timeout-wrapped SSH)\necho \"[eval] polling (cap ~1.5h)…\"\nRC=\"\"\nfor i in $(seq 1 45); do\n sleep 120\n OUT=$(TSSH -p \"$PORT\" root@\"$HOST\" \\\n 'printf \"RC=%s\\n\" \"$(cat /workspace/eval/_cycle_status 2>/dev/null)\"; tail -1 /workspace/logs/eval_only.log 2>/dev/null' \\\n 2>/dev/null | tr -d '\\r')\n RC=$(printf '%s\\n' \"$OUT\" | sed -n 's/^RC=//p' | tr -d ' ')\n LINE=$(printf '%s\\n' \"$OUT\" | grep -v '^RC=' | tail -1)\n echo \"[eval] [$i/45] ${LINE:-(no log line yet)}\"\n [ -n \"$RC\" ] && { echo \"[eval] runner finished (exit $RC)\"; break; }\ndone\n\n# 6. Pull report\nmkdir -p data/eval_reports\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" root@\"$HOST\":/workspace/eval/aria_llm_v0_7_eval_rerun.json data/eval_reports/aria_llm_v0_7_eval_rerun.json 2>/dev/null || echo \"[eval] (report not pulled)\"\necho \"[eval] === RESULT ===\"\npython - <<'PY'\nimport json, os\nf=\"data/eval_reports/aria_llm_v0_7_eval_rerun.json\"\nif not os.path.exists(f): print(\"report MISSING — check pod logs\"); raise SystemExit\nd=json.load(open(f,encoding=\"utf-8\")); dd=d.get(\"defence_dd\") or {}; pi=d.get(\"prompt_injection\") or {}\na=dd.get(\"accuracy\")\nprint(f\"v0.7 (rerun, open-book): judge-DD={a} (n={dd.get('total')}) | leak_rate={pi.get('leak_rate')}\")\nprint(f\"DeepSeek parity=0.336 | prior v0.7=0.308 | G1: {'PASS' if (a or 0)>=0.336 else 'FAIL'}\")\nPY\nstop_pod\necho \"[eval] DONE — pod stopped.\"\n"}, "4046b84d1710f56e6baf6752d32289ad60da382d": {"t": 1786068349.59433, "n": 7029, "h": "4e0b64870af0d813", "c": "#!/bin/bash\n# EVAL-ONLY v0.7 driver (R-F17xx) — operator approved an OFFLINE eval of the best\n# own-model. Serves the EXISTING on-volume adapter (NO retrain) and runs the 500-Q\n# open-book eval, then STOPS the pod. Hardened like run_v0_7_grounded_cycle.sh:\n# auto-arms the on-pod self-stop watcher + timeout-wraps every poll SSH, so a hung\n# Windows SSH can't freeze the loop and the pod self-stops even if this driver dies.\n# Does NOT set ARIA_LLM_URL. Does NOT retrain.\nset -uo pipefail\nREPO=\"/c/code/crucix\"; cd \"$REPO\"\nPOD=\"${RUNPOD_POD_ID:-2adkzeri6fa2zi}\"\nAPI=\"https://rest.runpod.io/v1\"\nAPI_KEY=$(grep -E \"^RUNPOD_API_KEY=\" .env | head -1 | cut -d= -f2- | tr -d '\"' | tr -d \"'\" | tr -d '\\r')\nDSK=$(grep -E \"^DEEPSEEK_API_KEY=\" .env | head -1 | cut -d= -f2- | tr -d '\"' | tr -d \"'\" | tr -d '\\r')\n[ -n \"$API_KEY\" ] || { echo \"[eval] FATAL: RUNPOD_API_KEY not in .env\"; exit 1; }\nEVAL_LOCAL=\"${EVAL_LOCAL:-data/eval_reports/aria_eval_500q_openbook.jsonl}\"\n[ -s \"$EVAL_LOCAL\" ] || EVAL_LOCAL=\"data/training/aria_eval_500q_openbook.jsonl\"\n[ -s \"$EVAL_LOCAL\" ] || { echo \"[eval] FATAL: open-book eval set not found\"; exit 1; }\necho \"[eval] open-book eval set: $EVAL_LOCAL ($(wc -l < \"$EVAL_LOCAL\") Q)\"\n\nKEY=\"/tmp/rpkey_eval\"; cp ~/.ssh/runpod_aria \"$KEY\"; chmod 600 \"$KEY\"\nSSH=\"ssh -i $KEY -o StrictHostKeyChecking=no -o ConnectTimeout=15 -o ServerAliveInterval=30 -o ServerAliveCountMax=6\"\nTSSH(){ timeout 75 $SSH \"$@\"; }\njget(){ python -c \"import sys,json;d=json.load(sys.stdin);print(d.get('$1','') or '')\" 2>/dev/null; }\npmget(){ python -c \"import sys,json;d=json.load(sys.stdin);pm=d.get('portMappings') or {};print(pm.get('22') or '')\" 2>/dev/null; }\nstop_pod(){ echo \"[eval] stopping pod $POD\"; curl -s -X POST \"$API/pods/$POD/stop\" -H \"Authorization: Bearer $API_KEY\" >/dev/null 2>&1 || true; }\ntrap stop_pod EXIT\n\n# 1. Ensure RUNNING (US-KS-2 is capacity-blocked intermittently → retry up to ~40 min)\nHOST=\"\"; PORT=\"\"\nfor i in $(seq 1 40); do\n PD=$(curl -s \"$API/pods/$POD\" -H \"Authorization: Bearer $API_KEY\")\n ST=$(echo \"$PD\" | jget desiredStatus)\n if [ \"$ST\" != \"RUNNING\" ]; then\n R=$(curl -s -X POST \"$API/pods/$POD/start\" -H \"Authorization: Bearer $API_KEY\")\n ERR=$(echo \"$R\" | python -c \"import sys,json\ntry: d=json.load(sys.stdin); print(d.get('error','') if isinstance(d,dict) else '')\nexcept: print('')\" 2>/dev/null || echo \"\")\n [ -n \"$ERR\" ] && { echo \"[eval] start blocked ($ERR) — retry 60s ($i/20)\"; sleep 60; continue; }\n fi\n for j in $(seq 1 18); do\n PD=$(curl -s \"$API/pods/$POD\" -H \"Authorization: Bearer $API_KEY\")\n ST=$(echo \"$PD\" | jget desiredStatus); HOST=$(echo \"$PD\" | jget publicIp); PORT=$(echo \"$PD\" | pmget)\n [ \"$ST\" = \"RUNNING\" ] && [ -n \"$HOST\" ] && [ -n \"$PORT\" ] && break\n sleep 8\n done\n [ -n \"$HOST\" ] && [ -n \"$PORT\" ] && { echo \"[eval] pod RUNNING $HOST:$PORT\"; break; }\n sleep 30\ndone\n[ -n \"$HOST\" ] && [ -n \"$PORT\" ] || { echo \"[eval] FATAL: pod not RUNNING\"; exit 1; }\n\n# 2. Wait SSH, then settle (sshd is flaky in the first ~30s after boot → \"Connection\n# closed by remote host\" on scp). R-F17xx: wait for THREE consecutive good probes.\nok=0\nfor i in $(seq 1 36); do\n if TSSH -p \"$PORT\" root@\"$HOST\" \"echo ok\" 2>/dev/null | grep -q ok; then ok=$((ok+1)); else ok=0; fi\n [ \"$ok\" -ge 3 ] && { echo \"[eval] SSH stable ($i)\"; break; }\n sleep 5\ndone\nsleep 15 # extra settle before bulk transfers\n\n# scp with retry — flaky post-boot sshd drops the first connections.\nRSCP(){ local src=\"$1\" dst=\"$2\" t; for t in 1 2 3 4 5; do\n if timeout 90 scp -i \"$KEY\" -o StrictHostKeyChecking=no -o ConnectTimeout=15 -P \"$PORT\" \"$src\" root@\"$HOST\":\"$dst\" 2>/dev/null; then return 0; fi\n echo \"[eval] scp retry $t for $(basename \"$src\")…\"; sleep 10\ndone; return 1; }\n\n# 3. Push runner + scripts + open-book eval set + self-stop watcher + eval deps\necho \"[eval] pushing runner + scripts + eval set + watcher…\"\nfor i in 1 2 3; do TSSH -p \"$PORT\" root@\"$HOST\" \"mkdir -p /workspace/datasets /workspace/crucix/scripts/train /workspace/crucix/aria_service/intel /workspace/logs\" 2>/dev/null && break; sleep 8; done\nRSCP scripts/train/pod_eval_only.sh /workspace/ || { echo \"[eval] FATAL scp runner\"; exit 1; }\nRSCP scripts/train/pod_selfstop_watch_v04.sh /workspace/ || { echo \"[eval] FATAL scp watcher\"; exit 1; }\nfor f in serve_eval_shim.py eval_aria_llm.py; do\n RSCP \"scripts/train/$f\" \"/workspace/crucix/scripts/train/$f\" || { echo \"[eval] FATAL scp $f\"; exit 1; }\ndone\nRSCP \"$EVAL_LOCAL\" /workspace/datasets/aria_eval_openbook.jsonl || { echo \"[eval] FATAL scp eval set\"; exit 1; }\nfor f in aria_service/__init__.py aria_service/intel/__init__.py aria_service/intel/engine_wiring.py aria_service/intel/prompt_injection_suite.py; do\n RSCP \"$f\" \"/workspace/crucix/$f\" || echo \"[eval] (optional dep $f not pushed)\"\ndone\n\n# 4. ARM the self-stop watcher FIRST (cost safety even if this driver dies), then launch eval-only DETACHED\necho \"[eval] ARMING on-pod self-stop watcher (GRACE=900)…\"\nTSSH -p \"$PORT\" root@\"$HOST\" \\\n \"POD_ID=$POD RP_KEY='$API_KEY' GRACE=900 setsid nohup bash /workspace/pod_selfstop_watch_v04.sh >/workspace/logs/_selfstop_launch.log 2>&1 < /dev/null & echo ARMED\" \\\n || echo \"[eval] WARN: watcher arm failed — relying on EXIT trap\"\necho \"[eval] launching EVAL-ONLY runner DETACHED…\"\nTSSH -p \"$PORT\" root@\"$HOST\" \\\n \"rm -f /workspace/eval/_cycle_status; DEEPSEEK_API_KEY='$DSK' \\\n setsid nohup bash /workspace/pod_eval_only.sh > /workspace/logs/eval_only.log 2>&1 < /dev/null & echo STARTED\" \\\n || { echo \"[eval] FATAL: could not launch eval runner\"; exit 1; }\n\n# 5. Poll for completion (cap ~1.5h; timeout-wrapped SSH)\necho \"[eval] polling (cap ~1.5h)…\"\nRC=\"\"\nfor i in $(seq 1 45); do\n sleep 120\n OUT=$(TSSH -p \"$PORT\" root@\"$HOST\" \\\n 'printf \"RC=%s\\n\" \"$(cat /workspace/eval/_cycle_status 2>/dev/null)\"; tail -1 /workspace/logs/eval_only.log 2>/dev/null' \\\n 2>/dev/null | tr -d '\\r')\n RC=$(printf '%s\\n' \"$OUT\" | sed -n 's/^RC=//p' | tr -d ' ')\n LINE=$(printf '%s\\n' \"$OUT\" | grep -v '^RC=' | tail -1)\n echo \"[eval] [$i/45] ${LINE:-(no log line yet)}\"\n [ -n \"$RC\" ] && { echo \"[eval] runner finished (exit $RC)\"; break; }\ndone\n\n# 6. Pull report\nmkdir -p data/eval_reports\nscp -i \"$KEY\" -o StrictHostKeyChecking=no -P \"$PORT\" root@\"$HOST\":/workspace/eval/aria_llm_v0_7_eval_rerun.json data/eval_reports/aria_llm_v0_7_eval_rerun.json 2>/dev/null || echo \"[eval] (report not pulled)\"\necho \"[eval] === RESULT ===\"\npython - <<'PY'\nimport json, os\nf=\"data/eval_reports/aria_llm_v0_7_eval_rerun.json\"\nif not os.path.exists(f): print(\"report MISSING — check pod logs\"); raise SystemExit\nd=json.load(open(f,encoding=\"utf-8\")); dd=d.get(\"defence_dd\") or {}; pi=d.get(\"prompt_injection\") or {}\na=dd.get(\"accuracy\")\nprint(f\"v0.7 (rerun, open-book): judge-DD={a} (n={dd.get('total')}) | leak_rate={pi.get('leak_rate')}\")\nprint(f\"DeepSeek parity=0.336 | prior v0.7=0.308 | G1: {'PASS' if (a or 0)>=0.336 else 'FAIL'}\")\nPY\nstop_pod\necho \"[eval] DONE — pod stopped.\"\n"}}, "scripts/train/launch_v02_shim.sh": {"3a9645c8ec96260be7edd2c46a655b1e473f8c18": {"t": 1785745975.5498617, "n": 10113, "h": "0b6c259dd5a415c1", "c": "#!/usr/bin/env bash\n# =============================================================================\n# ARIA-LLM v0.2 — shim-based serve + eval (replaces vLLM which needs newer CUDA)\n#\n# The pod's CUDA driver (565) is too old for latest vLLM's torch. Instead of\n# `pip install vllm`, this uses serve_eval_shim.py (transformers+peft+bnb) which\n# works with the pod's existing torch 2.4.1+cu124.\n#\n# Usage:\n# export RUNPOD_API_KEY=\"...\" # from .env\n# export RUNPOD_POD_ID=\"7ei3hldcpz4j2v\"\n# bash scripts/train/launch_v02_shim.sh\n#\n# Steps:\n# 1. Start pod (retry until GPU free)\n# 2. SSH in, install shim deps (transformers peft bitsandbytes uvicorn fastapi)\n# 3. Launch serve_eval_shim.py on port 8888\n# 4. Health-probe the shim endpoint\n# 5. Run 500-Q eval on v0.2 + DeepSeek baseline\n# 6. Stop the pod (EXIT trap)\n# =============================================================================\nset -euo pipefail\n\nSCRIPT_DIR=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\" && pwd)\"\nREPO_ROOT=\"$(cd \"$SCRIPT_DIR/../..\" && pwd)\"\n\n# ── Creds from .env ──────────────────────────────────────────────────────────\nif [ -f \"$REPO_ROOT/.env\" ]; then\n for _k in RUNPOD_API_KEY RUNPOD_POD_ID DEEPSEEK_API_KEY RUNPOD_API_BASE; do\n if [ -z \"${!_k:-}\" ]; then\n _v=$(grep -E \"^${_k}=\" \"$REPO_ROOT/.env\" | head -1 | cut -d= -f2- | tr -d '\"'\"'\"'' | tr -d '\\r' || true)\n [ -n \"$_v\" ] && export \"$_k=$_v\"\n fi\n done\nfi\n\nPOD_ID=\"${RUNPOD_POD_ID:-7ei3hldcpz4j2v}\"\nAPI_KEY=\"${RUNPOD_API_KEY:?RUNPOD_API_KEY not set}\"\nAPI_BASE=\"${RUNPOD_API_BASE:-https://rest.runpod.io/v1}\"\nVLLM_PROXY_PORT=\"${VLLM_PROXY_PORT:-8888}\"\n\n# ── Model config (SSOT corrected R-F1454) ────────────────────────────────────\nsource \"$SCRIPT_DIR/model_config.sh\"\nBASE_MODEL=\"${ARIA_BASE_MODEL:-mistralai/Mistral-7B-Instruct-v0.3}\"\nADAPTER_PATH=\"${ARIA_ADAPTER_PATH:-/workspace/checkpoints/aria_llm_v0_2_dpo}\"\nMODEL_NAME=\"${ARIA_MODEL_NAME:-aria-llm-v0.2}\"\n\n# ── Paths ────────────────────────────────────────────────────────────────────\nLOCAL_EVAL_SET=\"${ARIA_LOCAL_EVAL_SET:-$REPO_ROOT/data/eval_reports/aria_eval_500q.jsonl}\"\nEVAL_OUTPUT_V02=\"$REPO_ROOT/data/eval_reports/aria_llm_v02_eval.json\"\nEVAL_OUTPUT_DS=\"$REPO_ROOT/data/eval_reports/deepseek_baseline_eval.json\"\nTIMESTAMP=$(date -u +%Y%m%d_%H%M%S)\nLOG_DIR=\"/tmp/aria_v02_shim_${TIMESTAMP}\"\nmkdir -p \"$LOG_DIR\"\n\n# ── Python binary (local, for eval) ──────────────────────────────────────────\nPYTHON_BIN=\"${PYTHON_BIN:-}\"\nif [ -z \"$PYTHON_BIN\" ]; then\n if [ -x \"$REPO_ROOT/.venv/Scripts/python.exe\" ]; then\n PYTHON_BIN=\"$REPO_ROOT/.venv/Scripts/python.exe\"\n elif [ -x \"$REPO_ROOT/.venv/bin/python\" ]; then\n PYTHON_BIN=\"$REPO_ROOT/.venv/bin/python\"\n else\n PYTHON_BIN=\"python3\"\n fi\nfi\n\necho \"=== ARIA-LLM v0.2 shim serve + eval ===\"\necho \"Pod: $POD_ID\"\necho \"Base: $BASE_MODEL\"\necho \"Adapter: $ADAPTER_PATH\"\necho \"Model: $MODEL_NAME\"\necho \"Eval set: $LOCAL_EVAL_SET\"\necho \"Log dir: $LOG_DIR\"\necho \"\"\n\n# ── EXIT trap: always stop the pod ──────────────────────────────────────────\n_stop_pod() {\n echo \"[trap] stopping pod $POD_ID...\"\n curl -s -X POST \"$API_BASE/pods/$POD_ID/stop\" \\\n -H \"Authorization: Bearer $API_KEY\" >/dev/null 2>&1 || true\n}\ntrap _stop_pod EXIT\n\n# ── Step 1: Start pod (retry until GPU free) ────────────────────────────────\necho \"[1/6] Starting pod $POD_ID (retrying until GPU free)...\"\nfor i in $(seq 1 30); do\n curl -s -X POST \"$API_BASE/pods/$POD_ID/start\" \\\n -H \"Authorization: Bearer $API_KEY\" > \"$LOG_DIR/start.json\" 2>&1\n START_ERR=$(cat \"$LOG_DIR/start.json\" | \"$PYTHON_BIN\" -c \"import sys,json\ntry:\n d=json.load(sys.stdin)\nexcept Exception:\n sys.exit(0)\nif isinstance(d,dict) and (d.get('error') or (isinstance(d.get('status'),int) and d['status']>=400)):\n print(d.get('error') or d.get('status'))\n\" 2>/dev/null || echo \"\")\n if [ -z \"$START_ERR\" ]; then\n echo \" Pod started (attempt $i)\"\n break\n fi\n echo \" $START_ERR — retrying in 60s (attempt $i/30)...\"\n sleep 60\ndone\nif [ -n \"${START_ERR:-}\" ]; then\n echo \"ERROR: Could not start pod after 30 attempts.\" >&2\n exit 1\nfi\n\n# Wait for pod to reach RUNNING state\necho \" Waiting for pod to start...\"\nPOD_DATA=\"\"\nfor i in $(seq 1 30); do\n POD_DATA=$(curl -s \"$API_BASE/pods/$POD_ID\" -H \"Authorization: Bearer $API_KEY\")\n STATUS=$(echo \"$POD_DATA\" | \"$PYTHON_BIN\" -c \"import sys,json; print(json.load(sys.stdin).get('desiredStatus','') or '')\" 2>/dev/null || echo \"unknown\")\n IP_READY=$(echo \"$POD_DATA\" | \"$PYTHON_BIN\" -c \"import sys,json; print('1' if json.load(sys.stdin).get('publicIp') else '')\" 2>/dev/null || echo \"\")\n if [ \"$STATUS\" = \"RUNNING\" ] && [ -n \"$IP_READY\" ]; then\n echo \" Pod is RUNNING with IP (attempt $i)\"\n break\n fi\n echo \" Status: ${STATUS:-?} — waiting 10s (attempt $i/30)...\"\n sleep 10\ndone\n\n# ── Resolve SSH endpoint ────────────────────────────────────────────────────\nread -r POD_HOST POD_PORT < <(echo \"$POD_DATA\" | \"$PYTHON_BIN\" -c \"\nimport sys,json\nd=json.load(sys.stdin)\nip=d.get('publicIp') or ''\npm=d.get('portMappings') or {}\nport=str(pm.get('22') or pm.get(22) or '') if isinstance(pm,dict) else ''\nprint(ip, port)\n\" 2>/dev/null || echo \" \")\nPOD_HOST=\"${POD_HOST//$'\\r'/}\"\nPOD_PORT=\"${POD_PORT//$'\\r'/}\"\necho \" SSH endpoint: ${POD_HOST:-<none>}:${POD_PORT:-<none>}\"\nif [ -z \"$POD_HOST\" ] || [ -z \"$POD_PORT\" ]; then\n echo \"ERROR: could not resolve SSH host/port.\" >&2\n exit 1\nfi\n\n# ── SSH key ──────────────────────────────────────────────────────────────────\nSSH_KEY_SRC=\"${ARIA_RUNPOD_SSH_KEY:-$HOME/.ssh/runpod_aria}\"\nSSH_ID=\"$LOG_DIR/sshid\"\nif [ -f \"$SSH_KEY_SRC\" ]; then\n cp \"$SSH_KEY_SRC\" \"$SSH_ID\" && chmod 600 \"$SSH_ID\"\nelse\n echo \"ERROR: SSH key not found at $SSH_KEY_SRC\" >&2\n exit 1\nfi\n\n# ── Step 2: Wait for SSH readiness ──────────────────────────────────────────\necho \"[2/6] Waiting for SSH readiness...\"\nfor i in $(seq 1 20); do\n if ssh -i \"$SSH_ID\" -o StrictHostKeyChecking=no -o ConnectTimeout=5 -p \"$POD_PORT\" \"root@$POD_HOST\" \"echo ready\" 2>/dev/null; then\n echo \" SSH ready (attempt $i)\"\n break\n fi\n echo \" Waiting 10s...\"\n sleep 10\ndone\n\n# ── Step 3a: Verify adapter base ────────────────────────────────────────────\necho \"[3a] Verifying adapter base matches $BASE_MODEL...\"\nADAPTER_BASE=$(ssh -i \"$SSH_ID\" -o StrictHostKeyChecking=no -p \"$POD_PORT\" \"root@$POD_HOST\" \\\n \"python3 -c \\\"import json;print(json.load(open('$ADAPTER_PATH/adapter_config.json')).get('base_model_name_or_path',''))\\\"\" 2>/dev/null || echo \"\")\necho \" adapter base: '${ADAPTER_BASE:-<unreadable>}' (expected: $BASE_MODEL)\"\nif [ -n \"$ADAPTER_BASE\" ] && [ \"$ADAPTER_BASE\" != \"$BASE_MODEL\" ]; then\n echo \"ERROR: adapter base ('$ADAPTER_BASE') != configured BASE_MODEL ('$BASE_MODEL').\" >&2\n exit 1\nfi\n\n# ── Step 3: Install shim deps + launch serve_eval_shim.py ───────────────────\necho \"[3/6] Installing shim dependencies on pod...\"\nssh -i \"$SSH_ID\" -o StrictHostKeyChecking=no -p \"$POD_PORT\" \"root@$POD_HOST\" bash -s -- \\\n \"$BASE_MODEL\" \"$ADAPTER_PATH\" \"$MODEL_NAME\" \"$VLLM_PROXY_PORT\" << 'REMOTE'\n set -euo pipefail\n BASE_MODEL=\"$1\"; ADAPTER_PATH=\"$2\"; SERVED_NAME=\"$3\"; PORT=\"$4\"\n mkdir -p /workspace/logs\n\n # R-F1454: load from volume HF cache (gated model, no HF token on pod)\n export HF_HOME=\"${HF_HOME:-/workspace/.cache/huggingface}\"\n export HF_HUB_OFFLINE=1\n export TRANSFORMERS_OFFLINE=1\n\n # Install shim deps (compatible with pod's CUDA 12.4 / driver 565)\n pip install -q transformers peft bitsandbytes uvicorn fastapi pydantic 2>&1 | tail -3\n\n # Free port 8888 (Jupyter runs on it in runpod/pytorch image)\n pkill -f jupyter 2>/dev/null || true\n pkill -f serve_eval_shim 2>/dev/null || true\n sleep 2\n\n # Launch shim in background\n nohup env BASE_MODEL=\"$BASE_MODEL\" ADAPTER=\"$ADAPTER_PATH\" \\\n MODEL_NAME=\"$SERVED_NAME\" PORT=\"$PORT\" \\\n python3 /workspace/crucix/scripts/train/serve_eval_shim.py \\\n > /workspace/logs/shim_v02.log 2>&1 &\n echo \"Shim starting on port $PORT — PID: $!\"\nREMOTE\n\n# ── Step 4: Health probe ────────────────────────────────────────────────────\necho \"[4/6] Probing shim endpoint...\"\nPROXY_URL=\"https://${POD_ID}-${VLLM_PROXY_PORT}.proxy.runpod.net\"\nSHIM_READY=0\nfor i in $(seq 1 48); do\n HTTP_CODE=$(curl -s -o /dev/null -w \"%{http_code}\" \"$PROXY_URL/v1/models\" 2>/dev/null || echo \"000\")\n if [ \"$HTTP_CODE\" = \"200\" ]; then\n echo \" Shim ready at $PROXY_URL (attempt $i)\"\n MODELS=$(curl -s \"$PROXY_URL/v1/models\" | \"$PYTHON_BIN\" -m json.tool 2>/dev/null || echo \"\")\n echo \" Models: $MODELS\"\n SHIM_READY=1\n break\n fi\n echo \" HTTP $HTTP_CODE — model still loading, waiting 15s (attempt $i/48)...\"\n sleep 15\ndone\nif [ \"$SHIM_READY\" != \"1\" ]; then\n echo \"ERROR: Shim did not become ready after ~12 min.\" >&2\n exit 1\nfi\n\n# ── Step 5: Run eval ────────────────────────────────────────────────────────\necho \"[5/6] Running eval (v0.2 + DeepSeek baseline)...\"\nmkdir -p \"$REPO_ROOT/data/eval_reports\"\n\nif [ ! -f \"$LOCAL_EVAL_SET\" ]; then\n echo \" Exporting frozen 500-Q eval set...\"\n \"$PYTHON_BIN\" \"$SCRIPT_DIR/export_eval_500q.py\" --out \"$LOCAL_EVAL_SET\"\nfi\n\necho \" → v0.2 ($MODEL_NAME) at $PROXY_URL/v1\"\n\"$PYTHON_BIN\" \"$SCRIPT_DIR/eval_aria_llm.py\" \\\n --target \"$PROXY_URL/v1\" \\\n --model \"$MODEL_NAME\" \\\n --eval-set \"$LOCAL_EVAL_SET\" \\\n --out \"$EVAL_OUTPUT_V02\" 2>&1 | tee \"$LOG_DIR/eval_v02.log\"\n\nif [ -n \"${DEEPSEEK_API_KEY:-}\" ]; then\n echo \" → DeepSeek baseline (deepseek-chat) at https://api.deepseek.com/v1\"\n \"$PYTHON_BIN\" \"$SCRIPT_DIR/eval_aria_llm.py\" \\\n --target \"https://api.deepseek.com/v1\" \\\n --model \"deepseek-chat\" \\\n --api-key \"$DEEPSEEK_API_KEY\" \\\n --eval-set \"$LOCAL_EVAL_SET\" \\\n --out \"$EVAL_OUTPUT_DS\" 2>&1 | tee \"$LOG_DIR/eval_deepseek.log\"\nelse\n echo \" ⚠ DEEPSEEK_API_KEY not set — skipping DeepSeek baseline.\"\nfi\n\necho \" Eval complete.\"\necho \" v0.2: $EVAL_OUTPUT_V02\"\necho \" DeepSeek: $EVAL_OUTPUT_DS\"\n\n# ── Step 6: Stop pod (EXIT trap handles this) ───────────────────────────────\necho \"[6/6] Pod will be stopped by EXIT trap.\"\necho \"\"\necho \"=== Done ===\"\n"}, "9cc42d8ecc327e1258a09d9a04e53ffd35ecaf30": {"t": 1785758159.7516465, "n": 10113, "h": "0b6c259dd5a415c1", "c": "#!/usr/bin/env bash\n# =============================================================================\n# ARIA-LLM v0.2 — shim-based serve + eval (replaces vLLM which needs newer CUDA)\n#\n# The pod's CUDA driver (565) is too old for latest vLLM's torch. Instead of\n# `pip install vllm`, this uses serve_eval_shim.py (transformers+peft+bnb) which\n# works with the pod's existing torch 2.4.1+cu124.\n#\n# Usage:\n# export RUNPOD_API_KEY=\"...\" # from .env\n# export RUNPOD_POD_ID=\"7ei3hldcpz4j2v\"\n# bash scripts/train/launch_v02_shim.sh\n#\n# Steps:\n# 1. Start pod (retry until GPU free)\n# 2. SSH in, install shim deps (transformers peft bitsandbytes uvicorn fastapi)\n# 3. Launch serve_eval_shim.py on port 8888\n# 4. Health-probe the shim endpoint\n# 5. Run 500-Q eval on v0.2 + DeepSeek baseline\n# 6. Stop the pod (EXIT trap)\n# =============================================================================\nset -euo pipefail\n\nSCRIPT_DIR=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\" && pwd)\"\nREPO_ROOT=\"$(cd \"$SCRIPT_DIR/../..\" && pwd)\"\n\n# ── Creds from .env ──────────────────────────────────────────────────────────\nif [ -f \"$REPO_ROOT/.env\" ]; then\n for _k in RUNPOD_API_KEY RUNPOD_POD_ID DEEPSEEK_API_KEY RUNPOD_API_BASE; do\n if [ -z \"${!_k:-}\" ]; then\n _v=$(grep -E \"^${_k}=\" \"$REPO_ROOT/.env\" | head -1 | cut -d= -f2- | tr -d '\"'\"'\"'' | tr -d '\\r' || true)\n [ -n \"$_v\" ] && export \"$_k=$_v\"\n fi\n done\nfi\n\nPOD_ID=\"${RUNPOD_POD_ID:-7ei3hldcpz4j2v}\"\nAPI_KEY=\"${RUNPOD_API_KEY:?RUNPOD_API_KEY not set}\"\nAPI_BASE=\"${RUNPOD_API_BASE:-https://rest.runpod.io/v1}\"\nVLLM_PROXY_PORT=\"${VLLM_PROXY_PORT:-8888}\"\n\n# ── Model config (SSOT corrected R-F1454) ────────────────────────────────────\nsource \"$SCRIPT_DIR/model_config.sh\"\nBASE_MODEL=\"${ARIA_BASE_MODEL:-mistralai/Mistral-7B-Instruct-v0.3}\"\nADAPTER_PATH=\"${ARIA_ADAPTER_PATH:-/workspace/checkpoints/aria_llm_v0_2_dpo}\"\nMODEL_NAME=\"${ARIA_MODEL_NAME:-aria-llm-v0.2}\"\n\n# ── Paths ────────────────────────────────────────────────────────────────────\nLOCAL_EVAL_SET=\"${ARIA_LOCAL_EVAL_SET:-$REPO_ROOT/data/eval_reports/aria_eval_500q.jsonl}\"\nEVAL_OUTPUT_V02=\"$REPO_ROOT/data/eval_reports/aria_llm_v02_eval.json\"\nEVAL_OUTPUT_DS=\"$REPO_ROOT/data/eval_reports/deepseek_baseline_eval.json\"\nTIMESTAMP=$(date -u +%Y%m%d_%H%M%S)\nLOG_DIR=\"/tmp/aria_v02_shim_${TIMESTAMP}\"\nmkdir -p \"$LOG_DIR\"\n\n# ── Python binary (local, for eval) ──────────────────────────────────────────\nPYTHON_BIN=\"${PYTHON_BIN:-}\"\nif [ -z \"$PYTHON_BIN\" ]; then\n if [ -x \"$REPO_ROOT/.venv/Scripts/python.exe\" ]; then\n PYTHON_BIN=\"$REPO_ROOT/.venv/Scripts/python.exe\"\n elif [ -x \"$REPO_ROOT/.venv/bin/python\" ]; then\n PYTHON_BIN=\"$REPO_ROOT/.venv/bin/python\"\n else\n PYTHON_BIN=\"python3\"\n fi\nfi\n\necho \"=== ARIA-LLM v0.2 shim serve + eval ===\"\necho \"Pod: $POD_ID\"\necho \"Base: $BASE_MODEL\"\necho \"Adapter: $ADAPTER_PATH\"\necho \"Model: $MODEL_NAME\"\necho \"Eval set: $LOCAL_EVAL_SET\"\necho \"Log dir: $LOG_DIR\"\necho \"\"\n\n# ── EXIT trap: always stop the pod ──────────────────────────────────────────\n_stop_pod() {\n echo \"[trap] stopping pod $POD_ID...\"\n curl -s -X POST \"$API_BASE/pods/$POD_ID/stop\" \\\n -H \"Authorization: Bearer $API_KEY\" >/dev/null 2>&1 || true\n}\ntrap _stop_pod EXIT\n\n# ── Step 1: Start pod (retry until GPU free) ────────────────────────────────\necho \"[1/6] Starting pod $POD_ID (retrying until GPU free)...\"\nfor i in $(seq 1 30); do\n curl -s -X POST \"$API_BASE/pods/$POD_ID/start\" \\\n -H \"Authorization: Bearer $API_KEY\" > \"$LOG_DIR/start.json\" 2>&1\n START_ERR=$(cat \"$LOG_DIR/start.json\" | \"$PYTHON_BIN\" -c \"import sys,json\ntry:\n d=json.load(sys.stdin)\nexcept Exception:\n sys.exit(0)\nif isinstance(d,dict) and (d.get('error') or (isinstance(d.get('status'),int) and d['status']>=400)):\n print(d.get('error') or d.get('status'))\n\" 2>/dev/null || echo \"\")\n if [ -z \"$START_ERR\" ]; then\n echo \" Pod started (attempt $i)\"\n break\n fi\n echo \" $START_ERR — retrying in 60s (attempt $i/30)...\"\n sleep 60\ndone\nif [ -n \"${START_ERR:-}\" ]; then\n echo \"ERROR: Could not start pod after 30 attempts.\" >&2\n exit 1\nfi\n\n# Wait for pod to reach RUNNING state\necho \" Waiting for pod to start...\"\nPOD_DATA=\"\"\nfor i in $(seq 1 30); do\n POD_DATA=$(curl -s \"$API_BASE/pods/$POD_ID\" -H \"Authorization: Bearer $API_KEY\")\n STATUS=$(echo \"$POD_DATA\" | \"$PYTHON_BIN\" -c \"import sys,json; print(json.load(sys.stdin).get('desiredStatus','') or '')\" 2>/dev/null || echo \"unknown\")\n IP_READY=$(echo \"$POD_DATA\" | \"$PYTHON_BIN\" -c \"import sys,json; print('1' if json.load(sys.stdin).get('publicIp') else '')\" 2>/dev/null || echo \"\")\n if [ \"$STATUS\" = \"RUNNING\" ] && [ -n \"$IP_READY\" ]; then\n echo \" Pod is RUNNING with IP (attempt $i)\"\n break\n fi\n echo \" Status: ${STATUS:-?} — waiting 10s (attempt $i/30)...\"\n sleep 10\ndone\n\n# ── Resolve SSH endpoint ────────────────────────────────────────────────────\nread -r POD_HOST POD_PORT < <(echo \"$POD_DATA\" | \"$PYTHON_BIN\" -c \"\nimport sys,json\nd=json.load(sys.stdin)\nip=d.get('publicIp') or ''\npm=d.get('portMappings') or {}\nport=str(pm.get('22') or pm.get(22) or '') if isinstance(pm,dict) else ''\nprint(ip, port)\n\" 2>/dev/null || echo \" \")\nPOD_HOST=\"${POD_HOST//$'\\r'/}\"\nPOD_PORT=\"${POD_PORT//$'\\r'/}\"\necho \" SSH endpoint: ${POD_HOST:-<none>}:${POD_PORT:-<none>}\"\nif [ -z \"$POD_HOST\" ] || [ -z \"$POD_PORT\" ]; then\n echo \"ERROR: could not resolve SSH host/port.\" >&2\n exit 1\nfi\n\n# ── SSH key ──────────────────────────────────────────────────────────────────\nSSH_KEY_SRC=\"${ARIA_RUNPOD_SSH_KEY:-$HOME/.ssh/runpod_aria}\"\nSSH_ID=\"$LOG_DIR/sshid\"\nif [ -f \"$SSH_KEY_SRC\" ]; then\n cp \"$SSH_KEY_SRC\" \"$SSH_ID\" && chmod 600 \"$SSH_ID\"\nelse\n echo \"ERROR: SSH key not found at $SSH_KEY_SRC\" >&2\n exit 1\nfi\n\n# ── Step 2: Wait for SSH readiness ──────────────────────────────────────────\necho \"[2/6] Waiting for SSH readiness...\"\nfor i in $(seq 1 20); do\n if ssh -i \"$SSH_ID\" -o StrictHostKeyChecking=no -o ConnectTimeout=5 -p \"$POD_PORT\" \"root@$POD_HOST\" \"echo ready\" 2>/dev/null; then\n echo \" SSH ready (attempt $i)\"\n break\n fi\n echo \" Waiting 10s...\"\n sleep 10\ndone\n\n# ── Step 3a: Verify adapter base ────────────────────────────────────────────\necho \"[3a] Verifying adapter base matches $BASE_MODEL...\"\nADAPTER_BASE=$(ssh -i \"$SSH_ID\" -o StrictHostKeyChecking=no -p \"$POD_PORT\" \"root@$POD_HOST\" \\\n \"python3 -c \\\"import json;print(json.load(open('$ADAPTER_PATH/adapter_config.json')).get('base_model_name_or_path',''))\\\"\" 2>/dev/null || echo \"\")\necho \" adapter base: '${ADAPTER_BASE:-<unreadable>}' (expected: $BASE_MODEL)\"\nif [ -n \"$ADAPTER_BASE\" ] && [ \"$ADAPTER_BASE\" != \"$BASE_MODEL\" ]; then\n echo \"ERROR: adapter base ('$ADAPTER_BASE') != configured BASE_MODEL ('$BASE_MODEL').\" >&2\n exit 1\nfi\n\n# ── Step 3: Install shim deps + launch serve_eval_shim.py ───────────────────\necho \"[3/6] Installing shim dependencies on pod...\"\nssh -i \"$SSH_ID\" -o StrictHostKeyChecking=no -p \"$POD_PORT\" \"root@$POD_HOST\" bash -s -- \\\n \"$BASE_MODEL\" \"$ADAPTER_PATH\" \"$MODEL_NAME\" \"$VLLM_PROXY_PORT\" << 'REMOTE'\n set -euo pipefail\n BASE_MODEL=\"$1\"; ADAPTER_PATH=\"$2\"; SERVED_NAME=\"$3\"; PORT=\"$4\"\n mkdir -p /workspace/logs\n\n # R-F1454: load from volume HF cache (gated model, no HF token on pod)\n export HF_HOME=\"${HF_HOME:-/workspace/.cache/huggingface}\"\n export HF_HUB_OFFLINE=1\n export TRANSFORMERS_OFFLINE=1\n\n # Install shim deps (compatible with pod's CUDA 12.4 / driver 565)\n pip install -q transformers peft bitsandbytes uvicorn fastapi pydantic 2>&1 | tail -3\n\n # Free port 8888 (Jupyter runs on it in runpod/pytorch image)\n pkill -f jupyter 2>/dev/null || true\n pkill -f serve_eval_shim 2>/dev/null || true\n sleep 2\n\n # Launch shim in background\n nohup env BASE_MODEL=\"$BASE_MODEL\" ADAPTER=\"$ADAPTER_PATH\" \\\n MODEL_NAME=\"$SERVED_NAME\" PORT=\"$PORT\" \\\n python3 /workspace/crucix/scripts/train/serve_eval_shim.py \\\n > /workspace/logs/shim_v02.log 2>&1 &\n echo \"Shim starting on port $PORT — PID: $!\"\nREMOTE\n\n# ── Step 4: Health probe ────────────────────────────────────────────────────\necho \"[4/6] Probing shim endpoint...\"\nPROXY_URL=\"https://${POD_ID}-${VLLM_PROXY_PORT}.proxy.runpod.net\"\nSHIM_READY=0\nfor i in $(seq 1 48); do\n HTTP_CODE=$(curl -s -o /dev/null -w \"%{http_code}\" \"$PROXY_URL/v1/models\" 2>/dev/null || echo \"000\")\n if [ \"$HTTP_CODE\" = \"200\" ]; then\n echo \" Shim ready at $PROXY_URL (attempt $i)\"\n MODELS=$(curl -s \"$PROXY_URL/v1/models\" | \"$PYTHON_BIN\" -m json.tool 2>/dev/null || echo \"\")\n echo \" Models: $MODELS\"\n SHIM_READY=1\n break\n fi\n echo \" HTTP $HTTP_CODE — model still loading, waiting 15s (attempt $i/48)...\"\n sleep 15\ndone\nif [ \"$SHIM_READY\" != \"1\" ]; then\n echo \"ERROR: Shim did not become ready after ~12 min.\" >&2\n exit 1\nfi\n\n# ── Step 5: Run eval ────────────────────────────────────────────────────────\necho \"[5/6] Running eval (v0.2 + DeepSeek baseline)...\"\nmkdir -p \"$REPO_ROOT/data/eval_reports\"\n\nif [ ! -f \"$LOCAL_EVAL_SET\" ]; then\n echo \" Exporting frozen 500-Q eval set...\"\n \"$PYTHON_BIN\" \"$SCRIPT_DIR/export_eval_500q.py\" --out \"$LOCAL_EVAL_SET\"\nfi\n\necho \" → v0.2 ($MODEL_NAME) at $PROXY_URL/v1\"\n\"$PYTHON_BIN\" \"$SCRIPT_DIR/eval_aria_llm.py\" \\\n --target \"$PROXY_URL/v1\" \\\n --model \"$MODEL_NAME\" \\\n --eval-set \"$LOCAL_EVAL_SET\" \\\n --out \"$EVAL_OUTPUT_V02\" 2>&1 | tee \"$LOG_DIR/eval_v02.log\"\n\nif [ -n \"${DEEPSEEK_API_KEY:-}\" ]; then\n echo \" → DeepSeek baseline (deepseek-chat) at https://api.deepseek.com/v1\"\n \"$PYTHON_BIN\" \"$SCRIPT_DIR/eval_aria_llm.py\" \\\n --target \"https://api.deepseek.com/v1\" \\\n --model \"deepseek-chat\" \\\n --api-key \"$DEEPSEEK_API_KEY\" \\\n --eval-set \"$LOCAL_EVAL_SET\" \\\n --out \"$EVAL_OUTPUT_DS\" 2>&1 | tee \"$LOG_DIR/eval_deepseek.log\"\nelse\n echo \" ⚠ DEEPSEEK_API_KEY not set — skipping DeepSeek baseline.\"\nfi\n\necho \" Eval complete.\"\necho \" v0.2: $EVAL_OUTPUT_V02\"\necho \" DeepSeek: $EVAL_OUTPUT_DS\"\n\n# ── Step 6: Stop pod (EXIT trap handles this) ───────────────────────────────\necho \"[6/6] Pod will be stopped by EXIT trap.\"\necho \"\"\necho \"=== Done ===\"\n"}, "4046b84d1710f56e6baf6752d32289ad60da382d": {"t": 1786068352.1128645, "n": 10113, "h": "0b6c259dd5a415c1", "c": "#!/usr/bin/env bash\n# =============================================================================\n# ARIA-LLM v0.2 — shim-based serve + eval (replaces vLLM which needs newer CUDA)\n#\n# The pod's CUDA driver (565) is too old for latest vLLM's torch. Instead of\n# `pip install vllm`, this uses serve_eval_shim.py (transformers+peft+bnb) which\n# works with the pod's existing torch 2.4.1+cu124.\n#\n# Usage:\n# export RUNPOD_API_KEY=\"...\" # from .env\n# export RUNPOD_POD_ID=\"7ei3hldcpz4j2v\"\n# bash scripts/train/launch_v02_shim.sh\n#\n# Steps:\n# 1. Start pod (retry until GPU free)\n# 2. SSH in, install shim deps (transformers peft bitsandbytes uvicorn fastapi)\n# 3. Launch serve_eval_shim.py on port 8888\n# 4. Health-probe the shim endpoint\n# 5. Run 500-Q eval on v0.2 + DeepSeek baseline\n# 6. Stop the pod (EXIT trap)\n# =============================================================================\nset -euo pipefail\n\nSCRIPT_DIR=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\" && pwd)\"\nREPO_ROOT=\"$(cd \"$SCRIPT_DIR/../..\" && pwd)\"\n\n# ── Creds from .env ──────────────────────────────────────────────────────────\nif [ -f \"$REPO_ROOT/.env\" ]; then\n for _k in RUNPOD_API_KEY RUNPOD_POD_ID DEEPSEEK_API_KEY RUNPOD_API_BASE; do\n if [ -z \"${!_k:-}\" ]; then\n _v=$(grep -E \"^${_k}=\" \"$REPO_ROOT/.env\" | head -1 | cut -d= -f2- | tr -d '\"'\"'\"'' | tr -d '\\r' || true)\n [ -n \"$_v\" ] && export \"$_k=$_v\"\n fi\n done\nfi\n\nPOD_ID=\"${RUNPOD_POD_ID:-7ei3hldcpz4j2v}\"\nAPI_KEY=\"${RUNPOD_API_KEY:?RUNPOD_API_KEY not set}\"\nAPI_BASE=\"${RUNPOD_API_BASE:-https://rest.runpod.io/v1}\"\nVLLM_PROXY_PORT=\"${VLLM_PROXY_PORT:-8888}\"\n\n# ── Model config (SSOT corrected R-F1454) ────────────────────────────────────\nsource \"$SCRIPT_DIR/model_config.sh\"\nBASE_MODEL=\"${ARIA_BASE_MODEL:-mistralai/Mistral-7B-Instruct-v0.3}\"\nADAPTER_PATH=\"${ARIA_ADAPTER_PATH:-/workspace/checkpoints/aria_llm_v0_2_dpo}\"\nMODEL_NAME=\"${ARIA_MODEL_NAME:-aria-llm-v0.2}\"\n\n# ── Paths ────────────────────────────────────────────────────────────────────\nLOCAL_EVAL_SET=\"${ARIA_LOCAL_EVAL_SET:-$REPO_ROOT/data/eval_reports/aria_eval_500q.jsonl}\"\nEVAL_OUTPUT_V02=\"$REPO_ROOT/data/eval_reports/aria_llm_v02_eval.json\"\nEVAL_OUTPUT_DS=\"$REPO_ROOT/data/eval_reports/deepseek_baseline_eval.json\"\nTIMESTAMP=$(date -u +%Y%m%d_%H%M%S)\nLOG_DIR=\"/tmp/aria_v02_shim_${TIMESTAMP}\"\nmkdir -p \"$LOG_DIR\"\n\n# ── Python binary (local, for eval) ──────────────────────────────────────────\nPYTHON_BIN=\"${PYTHON_BIN:-}\"\nif [ -z \"$PYTHON_BIN\" ]; then\n if [ -x \"$REPO_ROOT/.venv/Scripts/python.exe\" ]; then\n PYTHON_BIN=\"$REPO_ROOT/.venv/Scripts/python.exe\"\n elif [ -x \"$REPO_ROOT/.venv/bin/python\" ]; then\n PYTHON_BIN=\"$REPO_ROOT/.venv/bin/python\"\n else\n PYTHON_BIN=\"python3\"\n fi\nfi\n\necho \"=== ARIA-LLM v0.2 shim serve + eval ===\"\necho \"Pod: $POD_ID\"\necho \"Base: $BASE_MODEL\"\necho \"Adapter: $ADAPTER_PATH\"\necho \"Model: $MODEL_NAME\"\necho \"Eval set: $LOCAL_EVAL_SET\"\necho \"Log dir: $LOG_DIR\"\necho \"\"\n\n# ── EXIT trap: always stop the pod ──────────────────────────────────────────\n_stop_pod() {\n echo \"[trap] stopping pod $POD_ID...\"\n curl -s -X POST \"$API_BASE/pods/$POD_ID/stop\" \\\n -H \"Authorization: Bearer $API_KEY\" >/dev/null 2>&1 || true\n}\ntrap _stop_pod EXIT\n\n# ── Step 1: Start pod (retry until GPU free) ────────────────────────────────\necho \"[1/6] Starting pod $POD_ID (retrying until GPU free)...\"\nfor i in $(seq 1 30); do\n curl -s -X POST \"$API_BASE/pods/$POD_ID/start\" \\\n -H \"Authorization: Bearer $API_KEY\" > \"$LOG_DIR/start.json\" 2>&1\n START_ERR=$(cat \"$LOG_DIR/start.json\" | \"$PYTHON_BIN\" -c \"import sys,json\ntry:\n d=json.load(sys.stdin)\nexcept Exception:\n sys.exit(0)\nif isinstance(d,dict) and (d.get('error') or (isinstance(d.get('status'),int) and d['status']>=400)):\n print(d.get('error') or d.get('status'))\n\" 2>/dev/null || echo \"\")\n if [ -z \"$START_ERR\" ]; then\n echo \" Pod started (attempt $i)\"\n break\n fi\n echo \" $START_ERR — retrying in 60s (attempt $i/30)...\"\n sleep 60\ndone\nif [ -n \"${START_ERR:-}\" ]; then\n echo \"ERROR: Could not start pod after 30 attempts.\" >&2\n exit 1\nfi\n\n# Wait for pod to reach RUNNING state\necho \" Waiting for pod to start...\"\nPOD_DATA=\"\"\nfor i in $(seq 1 30); do\n POD_DATA=$(curl -s \"$API_BASE/pods/$POD_ID\" -H \"Authorization: Bearer $API_KEY\")\n STATUS=$(echo \"$POD_DATA\" | \"$PYTHON_BIN\" -c \"import sys,json; print(json.load(sys.stdin).get('desiredStatus','') or '')\" 2>/dev/null || echo \"unknown\")\n IP_READY=$(echo \"$POD_DATA\" | \"$PYTHON_BIN\" -c \"import sys,json; print('1' if json.load(sys.stdin).get('publicIp') else '')\" 2>/dev/null || echo \"\")\n if [ \"$STATUS\" = \"RUNNING\" ] && [ -n \"$IP_READY\" ]; then\n echo \" Pod is RUNNING with IP (attempt $i)\"\n break\n fi\n echo \" Status: ${STATUS:-?} — waiting 10s (attempt $i/30)...\"\n sleep 10\ndone\n\n# ── Resolve SSH endpoint ────────────────────────────────────────────────────\nread -r POD_HOST POD_PORT < <(echo \"$POD_DATA\" | \"$PYTHON_BIN\" -c \"\nimport sys,json\nd=json.load(sys.stdin)\nip=d.get('publicIp') or ''\npm=d.get('portMappings') or {}\nport=str(pm.get('22') or pm.get(22) or '') if isinstance(pm,dict) else ''\nprint(ip, port)\n\" 2>/dev/null || echo \" \")\nPOD_HOST=\"${POD_HOST//$'\\r'/}\"\nPOD_PORT=\"${POD_PORT//$'\\r'/}\"\necho \" SSH endpoint: ${POD_HOST:-<none>}:${POD_PORT:-<none>}\"\nif [ -z \"$POD_HOST\" ] || [ -z \"$POD_PORT\" ]; then\n echo \"ERROR: could not resolve SSH host/port.\" >&2\n exit 1\nfi\n\n# ── SSH key ──────────────────────────────────────────────────────────────────\nSSH_KEY_SRC=\"${ARIA_RUNPOD_SSH_KEY:-$HOME/.ssh/runpod_aria}\"\nSSH_ID=\"$LOG_DIR/sshid\"\nif [ -f \"$SSH_KEY_SRC\" ]; then\n cp \"$SSH_KEY_SRC\" \"$SSH_ID\" && chmod 600 \"$SSH_ID\"\nelse\n echo \"ERROR: SSH key not found at $SSH_KEY_SRC\" >&2\n exit 1\nfi\n\n# ── Step 2: Wait for SSH readiness ──────────────────────────────────────────\necho \"[2/6] Waiting for SSH readiness...\"\nfor i in $(seq 1 20); do\n if ssh -i \"$SSH_ID\" -o StrictHostKeyChecking=no -o ConnectTimeout=5 -p \"$POD_PORT\" \"root@$POD_HOST\" \"echo ready\" 2>/dev/null; then\n echo \" SSH ready (attempt $i)\"\n break\n fi\n echo \" Waiting 10s...\"\n sleep 10\ndone\n\n# ── Step 3a: Verify adapter base ────────────────────────────────────────────\necho \"[3a] Verifying adapter base matches $BASE_MODEL...\"\nADAPTER_BASE=$(ssh -i \"$SSH_ID\" -o StrictHostKeyChecking=no -p \"$POD_PORT\" \"root@$POD_HOST\" \\\n \"python3 -c \\\"import json;print(json.load(open('$ADAPTER_PATH/adapter_config.json')).get('base_model_name_or_path',''))\\\"\" 2>/dev/null || echo \"\")\necho \" adapter base: '${ADAPTER_BASE:-<unreadable>}' (expected: $BASE_MODEL)\"\nif [ -n \"$ADAPTER_BASE\" ] && [ \"$ADAPTER_BASE\" != \"$BASE_MODEL\" ]; then\n echo \"ERROR: adapter base ('$ADAPTER_BASE') != configured BASE_MODEL ('$BASE_MODEL').\" >&2\n exit 1\nfi\n\n# ── Step 3: Install shim deps + launch serve_eval_shim.py ───────────────────\necho \"[3/6] Installing shim dependencies on pod...\"\nssh -i \"$SSH_ID\" -o StrictHostKeyChecking=no -p \"$POD_PORT\" \"root@$POD_HOST\" bash -s -- \\\n \"$BASE_MODEL\" \"$ADAPTER_PATH\" \"$MODEL_NAME\" \"$VLLM_PROXY_PORT\" << 'REMOTE'\n set -euo pipefail\n BASE_MODEL=\"$1\"; ADAPTER_PATH=\"$2\"; SERVED_NAME=\"$3\"; PORT=\"$4\"\n mkdir -p /workspace/logs\n\n # R-F1454: load from volume HF cache (gated model, no HF token on pod)\n export HF_HOME=\"${HF_HOME:-/workspace/.cache/huggingface}\"\n export HF_HUB_OFFLINE=1\n export TRANSFORMERS_OFFLINE=1\n\n # Install shim deps (compatible with pod's CUDA 12.4 / driver 565)\n pip install -q transformers peft bitsandbytes uvicorn fastapi pydantic 2>&1 | tail -3\n\n # Free port 8888 (Jupyter runs on it in runpod/pytorch image)\n pkill -f jupyter 2>/dev/null || true\n pkill -f serve_eval_shim 2>/dev/null || true\n sleep 2\n\n # Launch shim in background\n nohup env BASE_MODEL=\"$BASE_MODEL\" ADAPTER=\"$ADAPTER_PATH\" \\\n MODEL_NAME=\"$SERVED_NAME\" PORT=\"$PORT\" \\\n python3 /workspace/crucix/scripts/train/serve_eval_shim.py \\\n > /workspace/logs/shim_v02.log 2>&1 &\n echo \"Shim starting on port $PORT — PID: $!\"\nREMOTE\n\n# ── Step 4: Health probe ────────────────────────────────────────────────────\necho \"[4/6] Probing shim endpoint...\"\nPROXY_URL=\"https://${POD_ID}-${VLLM_PROXY_PORT}.proxy.runpod.net\"\nSHIM_READY=0\nfor i in $(seq 1 48); do\n HTTP_CODE=$(curl -s -o /dev/null -w \"%{http_code}\" \"$PROXY_URL/v1/models\" 2>/dev/null || echo \"000\")\n if [ \"$HTTP_CODE\" = \"200\" ]; then\n echo \" Shim ready at $PROXY_URL (attempt $i)\"\n MODELS=$(curl -s \"$PROXY_URL/v1/models\" | \"$PYTHON_BIN\" -m json.tool 2>/dev/null || echo \"\")\n echo \" Models: $MODELS\"\n SHIM_READY=1\n break\n fi\n echo \" HTTP $HTTP_CODE — model still loading, waiting 15s (attempt $i/48)...\"\n sleep 15\ndone\nif [ \"$SHIM_READY\" != \"1\" ]; then\n echo \"ERROR: Shim did not become ready after ~12 min.\" >&2\n exit 1\nfi\n\n# ── Step 5: Run eval ────────────────────────────────────────────────────────\necho \"[5/6] Running eval (v0.2 + DeepSeek baseline)...\"\nmkdir -p \"$REPO_ROOT/data/eval_reports\"\n\nif [ ! -f \"$LOCAL_EVAL_SET\" ]; then\n echo \" Exporting frozen 500-Q eval set...\"\n \"$PYTHON_BIN\" \"$SCRIPT_DIR/export_eval_500q.py\" --out \"$LOCAL_EVAL_SET\"\nfi\n\necho \" → v0.2 ($MODEL_NAME) at $PROXY_URL/v1\"\n\"$PYTHON_BIN\" \"$SCRIPT_DIR/eval_aria_llm.py\" \\\n --target \"$PROXY_URL/v1\" \\\n --model \"$MODEL_NAME\" \\\n --eval-set \"$LOCAL_EVAL_SET\" \\\n --out \"$EVAL_OUTPUT_V02\" 2>&1 | tee \"$LOG_DIR/eval_v02.log\"\n\nif [ -n \"${DEEPSEEK_API_KEY:-}\" ]; then\n echo \" → DeepSeek baseline (deepseek-chat) at https://api.deepseek.com/v1\"\n \"$PYTHON_BIN\" \"$SCRIPT_DIR/eval_aria_llm.py\" \\\n --target \"https://api.deepseek.com/v1\" \\\n --model \"deepseek-chat\" \\\n --api-key \"$DEEPSEEK_API_KEY\" \\\n --eval-set \"$LOCAL_EVAL_SET\" \\\n --out \"$EVAL_OUTPUT_DS\" 2>&1 | tee \"$LOG_DIR/eval_deepseek.log\"\nelse\n echo \" ⚠ DEEPSEEK_API_KEY not set — skipping DeepSeek baseline.\"\nfi\n\necho \" Eval complete.\"\necho \" v0.2: $EVAL_OUTPUT_V02\"\necho \" DeepSeek: $EVAL_OUTPUT_DS\"\n\n# ── Step 6: Stop pod (EXIT trap handles this) ───────────────────────────────\necho \"[6/6] Pod will be stopped by EXIT trap.\"\necho \"\"\necho \"=== Done ===\"\n"}}, "scripts/admin/build_grounded_corpus.py": {"3a9645c8ec96260be7edd2c46a655b1e473f8c18": {"t": 1785746391.8577757, "n": 5866, "h": "1907d0339b70fa13", "c": "\"\"\"R-F1939 — top up the GROUNDED training corpus to full openbook coverage.\n\nA prior build produced 608 grounded examples in aria_grounded_v1.jsonl\n(message-format: {messages:[user,assistant], topic, grounded, label, source};\nlabels grounded/grounded_abstain/abstain — it teaches honest abstention, not\nfabrication). That covers 608/689 openbook questions. This script generates\ngrounded answers for the REMAINING openbook items in the SAME schema so the\ncorpus is consistent and complete — via DeepSeek, concurrent + resumable.\n\nLocal + cheap. Usage:\n python scripts/admin/build_grounded_corpus.py --limit 3 # vet a few first\n python scripts/admin/build_grounded_corpus.py # all missing (resumable)\n\"\"\"\nfrom __future__ import annotations\n\nimport argparse\nimport asyncio\nimport json\nimport os\nimport sys\nfrom pathlib import Path\n\nREPO = Path(__file__).resolve().parents[2]\nIN = REPO / \"data\" / \"training\" / \"aria_train_689_openbook.jsonl\"\nOUT = REPO / \"data\" / \"training\" / \"aria_grounded_v1.jsonl\"\n_DS_URL = \"https://api.deepseek.com/v1/chat/completions\"\n_CONCURRENCY = 6\n\n# Matches the 608-corpus framing: the instruction lives in the user message.\n_CTX_PREFIX = (\"[CONTEXT — answer ONLY from this evidence; cite inline as \"\n \"[from <source>]; if it does not contain the answer, say so]\\n\")\n_SYSTEM = (\n \"You are ARIA, an elite due-diligence and intelligence analyst. Answer using \"\n \"ONLY the provided evidence, citing each fact inline with its [Source: ...] \"\n \"label. Be precise and analytical. If the evidence does not support an answer, \"\n \"say exactly what is missing — NEVER fabricate facts or sources.\"\n)\n_ABSTAIN = (\"cannot confirm\", \"does not contain\", \"not supported\", \"cannot determine\",\n \"no information\", \"context does not\", \"not enough information\", \"cannot answer\",\n \"insufficient\", \"does not provide\", \"unable to\", \"cannot be determined\")\n\n\ndef _q_of_corpus_msg(u: str) -> str:\n return (u.split(\"[QUESTION]\", 1)[1].strip().lower() if \"[QUESTION]\" in u\n else u.strip().lower()[-200:])\n\n\ndef _covered() -> set:\n if not OUT.exists():\n return set()\n out = set()\n for l in OUT.read_text(encoding=\"utf-8\").splitlines():\n if not l.strip():\n continue\n try:\n r = json.loads(l)\n if \"messages\" in r:\n out.add(_q_of_corpus_msg(r[\"messages\"][0][\"content\"]))\n except Exception:\n pass\n return out\n\n\ndef _label(ans: str):\n a = ans.lower()\n has_abstain = any(m in a for m in _ABSTAIN)\n has_cite = \"[source:\" in a or \"[from \" in a\n if has_abstain and has_cite:\n return \"grounded_abstain\", True\n if has_abstain:\n return \"abstain\", False\n return \"grounded\", True\n\n\nasync def _gen(client, key, item: dict) -> dict | None:\n user = _CTX_PREFIX + item[\"context\"] + \"\\n[QUESTION]\\n\" + item[\"question\"]\n body = {\"model\": \"deepseek-chat\",\n \"messages\": [{\"role\": \"system\", \"content\": _SYSTEM},\n {\"role\": \"user\", \"content\": user}],\n \"temperature\": 0.3, \"max_tokens\": 1024}\n for attempt in range(3):\n try:\n r = await client.post(_DS_URL, headers={\"Authorization\": f\"Bearer {key}\"},\n json=body, timeout=90.0)\n if r.status_code != 200:\n await asyncio.sleep(2 * (attempt + 1)); continue\n ans = r.json()[\"choices\"][0][\"message\"][\"content\"].strip()\n if not ans:\n return None\n label, grounded = _label(ans)\n return {\"messages\": [{\"role\": \"user\", \"content\": user},\n {\"role\": \"assistant\", \"content\": ans}],\n \"topic\": item.get(\"topic\"), \"grounded\": grounded,\n \"label\": label, \"source\": \"grounded_deepseek_v1\"}\n except Exception:\n await asyncio.sleep(2 * (attempt + 1))\n return None\n\n\nasync def main() -> int:\n ap = argparse.ArgumentParser()\n ap.add_argument(\"--limit\", type=int, default=0)\n a = ap.parse_args()\n key = os.environ.get(\"DEEPSEEK_API_KEY\", \"\").strip()\n if not key:\n print(\"BLOCKED: DEEPSEEK_API_KEY unset.\"); return 2\n items = [json.loads(l) for l in IN.read_text(encoding=\"utf-8\").splitlines() if l.strip()]\n covered = _covered()\n todo = [it for it in items\n if it.get(\"question\") and it.get(\"context\")\n and it[\"question\"].strip().lower() not in covered]\n if a.limit:\n todo = todo[:a.limit]\n print(f\"[grounded] {len(items)} openbook, {len(covered)} already grounded, {len(todo)} to generate\")\n if not todo:\n print(\"[grounded] corpus complete — nothing to do.\"); return 0\n\n import httpx\n sem = asyncio.Semaphore(_CONCURRENCY)\n lock = asyncio.Lock()\n stats = {\"ok\": 0, \"fail\": 0, \"grounded\": 0, \"grounded_abstain\": 0, \"abstain\": 0}\n f = OUT.open(\"a\", encoding=\"utf-8\")\n async with httpx.AsyncClient() as client:\n async def worker(it):\n async with sem:\n try:\n rec = await _gen(client, key, it)\n except Exception:\n rec = None\n async with lock:\n if rec:\n f.write(json.dumps(rec, ensure_ascii=False) + \"\\n\"); f.flush()\n stats[\"ok\"] += 1; stats[rec[\"label\"]] += 1\n else:\n stats[\"fail\"] += 1\n n = stats[\"ok\"] + stats[\"fail\"]\n if n % 20 == 0 or n == len(todo):\n print(f\"[grounded] {n}/{len(todo)} {stats}\", flush=True)\n await asyncio.gather(*(worker(it) for it in todo))\n f.close()\n print(f\"[grounded] DONE {stats} -> {OUT}\")\n return 0\n\n\nif __name__ == \"__main__\":\n raise SystemExit(asyncio.run(main()))\n"}, "1b8a293caafee89db2b400f762773efa72efd4cb": {"t": 1785758161.2541714, "n": 5866, "h": "1907d0339b70fa13", "c": "\"\"\"R-F1939 — top up the GROUNDED training corpus to full openbook coverage.\n\nA prior build produced 608 grounded examples in aria_grounded_v1.jsonl\n(message-format: {messages:[user,assistant], topic, grounded, label, source};\nlabels grounded/grounded_abstain/abstain — it teaches honest abstention, not\nfabrication). That covers 608/689 openbook questions. This script generates\ngrounded answers for the REMAINING openbook items in the SAME schema so the\ncorpus is consistent and complete — via DeepSeek, concurrent + resumable.\n\nLocal + cheap. Usage:\n python scripts/admin/build_grounded_corpus.py --limit 3 # vet a few first\n python scripts/admin/build_grounded_corpus.py # all missing (resumable)\n\"\"\"\nfrom __future__ import annotations\n\nimport argparse\nimport asyncio\nimport json\nimport os\nimport sys\nfrom pathlib import Path\n\nREPO = Path(__file__).resolve().parents[2]\nIN = REPO / \"data\" / \"training\" / \"aria_train_689_openbook.jsonl\"\nOUT = REPO / \"data\" / \"training\" / \"aria_grounded_v1.jsonl\"\n_DS_URL = \"https://api.deepseek.com/v1/chat/completions\"\n_CONCURRENCY = 6\n\n# Matches the 608-corpus framing: the instruction lives in the user message.\n_CTX_PREFIX = (\"[CONTEXT — answer ONLY from this evidence; cite inline as \"\n \"[from <source>]; if it does not contain the answer, say so]\\n\")\n_SYSTEM = (\n \"You are ARIA, an elite due-diligence and intelligence analyst. Answer using \"\n \"ONLY the provided evidence, citing each fact inline with its [Source: ...] \"\n \"label. Be precise and analytical. If the evidence does not support an answer, \"\n \"say exactly what is missing — NEVER fabricate facts or sources.\"\n)\n_ABSTAIN = (\"cannot confirm\", \"does not contain\", \"not supported\", \"cannot determine\",\n \"no information\", \"context does not\", \"not enough information\", \"cannot answer\",\n \"insufficient\", \"does not provide\", \"unable to\", \"cannot be determined\")\n\n\ndef _q_of_corpus_msg(u: str) -> str:\n return (u.split(\"[QUESTION]\", 1)[1].strip().lower() if \"[QUESTION]\" in u\n else u.strip().lower()[-200:])\n\n\ndef _covered() -> set:\n if not OUT.exists():\n return set()\n out = set()\n for l in OUT.read_text(encoding=\"utf-8\").splitlines():\n if not l.strip():\n continue\n try:\n r = json.loads(l)\n if \"messages\" in r:\n out.add(_q_of_corpus_msg(r[\"messages\"][0][\"content\"]))\n except Exception:\n pass\n return out\n\n\ndef _label(ans: str):\n a = ans.lower()\n has_abstain = any(m in a for m in _ABSTAIN)\n has_cite = \"[source:\" in a or \"[from \" in a\n if has_abstain and has_cite:\n return \"grounded_abstain\", True\n if has_abstain:\n return \"abstain\", False\n return \"grounded\", True\n\n\nasync def _gen(client, key, item: dict) -> dict | None:\n user = _CTX_PREFIX + item[\"context\"] + \"\\n[QUESTION]\\n\" + item[\"question\"]\n body = {\"model\": \"deepseek-chat\",\n \"messages\": [{\"role\": \"system\", \"content\": _SYSTEM},\n {\"role\": \"user\", \"content\": user}],\n \"temperature\": 0.3, \"max_tokens\": 1024}\n for attempt in range(3):\n try:\n r = await client.post(_DS_URL, headers={\"Authorization\": f\"Bearer {key}\"},\n json=body, timeout=90.0)\n if r.status_code != 200:\n await asyncio.sleep(2 * (attempt + 1)); continue\n ans = r.json()[\"choices\"][0][\"message\"][\"content\"].strip()\n if not ans:\n return None\n label, grounded = _label(ans)\n return {\"messages\": [{\"role\": \"user\", \"content\": user},\n {\"role\": \"assistant\", \"content\": ans}],\n \"topic\": item.get(\"topic\"), \"grounded\": grounded,\n \"label\": label, \"source\": \"grounded_deepseek_v1\"}\n except Exception:\n await asyncio.sleep(2 * (attempt + 1))\n return None\n\n\nasync def main() -> int:\n ap = argparse.ArgumentParser()\n ap.add_argument(\"--limit\", type=int, default=0)\n a = ap.parse_args()\n key = os.environ.get(\"DEEPSEEK_API_KEY\", \"\").strip()\n if not key:\n print(\"BLOCKED: DEEPSEEK_API_KEY unset.\"); return 2\n items = [json.loads(l) for l in IN.read_text(encoding=\"utf-8\").splitlines() if l.strip()]\n covered = _covered()\n todo = [it for it in items\n if it.get(\"question\") and it.get(\"context\")\n and it[\"question\"].strip().lower() not in covered]\n if a.limit:\n todo = todo[:a.limit]\n print(f\"[grounded] {len(items)} openbook, {len(covered)} already grounded, {len(todo)} to generate\")\n if not todo:\n print(\"[grounded] corpus complete — nothing to do.\"); return 0\n\n import httpx\n sem = asyncio.Semaphore(_CONCURRENCY)\n lock = asyncio.Lock()\n stats = {\"ok\": 0, \"fail\": 0, \"grounded\": 0, \"grounded_abstain\": 0, \"abstain\": 0}\n f = OUT.open(\"a\", encoding=\"utf-8\")\n async with httpx.AsyncClient() as client:\n async def worker(it):\n async with sem:\n try:\n rec = await _gen(client, key, it)\n except Exception:\n rec = None\n async with lock:\n if rec:\n f.write(json.dumps(rec, ensure_ascii=False) + \"\\n\"); f.flush()\n stats[\"ok\"] += 1; stats[rec[\"label\"]] += 1\n else:\n stats[\"fail\"] += 1\n n = stats[\"ok\"] + stats[\"fail\"]\n if n % 20 == 0 or n == len(todo):\n print(f\"[grounded] {n}/{len(todo)} {stats}\", flush=True)\n await asyncio.gather(*(worker(it) for it in todo))\n f.close()\n print(f\"[grounded] DONE {stats} -> {OUT}\")\n return 0\n\n\nif __name__ == \"__main__\":\n raise SystemExit(asyncio.run(main()))\n"}, "46eadcb5c58e76d35cb545ca1e7ef989433f04c5": {"t": 1786067729.233884, "n": 5866, "h": "1907d0339b70fa13", "c": "\"\"\"R-F1939 — top up the GROUNDED training corpus to full openbook coverage.\n\nA prior build produced 608 grounded examples in aria_grounded_v1.jsonl\n(message-format: {messages:[user,assistant], topic, grounded, label, source};\nlabels grounded/grounded_abstain/abstain — it teaches honest abstention, not\nfabrication). That covers 608/689 openbook questions. This script generates\ngrounded answers for the REMAINING openbook items in the SAME schema so the\ncorpus is consistent and complete — via DeepSeek, concurrent + resumable.\n\nLocal + cheap. Usage:\n python scripts/admin/build_grounded_corpus.py --limit 3 # vet a few first\n python scripts/admin/build_grounded_corpus.py # all missing (resumable)\n\"\"\"\nfrom __future__ import annotations\n\nimport argparse\nimport asyncio\nimport json\nimport os\nimport sys\nfrom pathlib import Path\n\nREPO = Path(__file__).resolve().parents[2]\nIN = REPO / \"data\" / \"training\" / \"aria_train_689_openbook.jsonl\"\nOUT = REPO / \"data\" / \"training\" / \"aria_grounded_v1.jsonl\"\n_DS_URL = \"https://api.deepseek.com/v1/chat/completions\"\n_CONCURRENCY = 6\n\n# Matches the 608-corpus framing: the instruction lives in the user message.\n_CTX_PREFIX = (\"[CONTEXT — answer ONLY from this evidence; cite inline as \"\n \"[from <source>]; if it does not contain the answer, say so]\\n\")\n_SYSTEM = (\n \"You are ARIA, an elite due-diligence and intelligence analyst. Answer using \"\n \"ONLY the provided evidence, citing each fact inline with its [Source: ...] \"\n \"label. Be precise and analytical. If the evidence does not support an answer, \"\n \"say exactly what is missing — NEVER fabricate facts or sources.\"\n)\n_ABSTAIN = (\"cannot confirm\", \"does not contain\", \"not supported\", \"cannot determine\",\n \"no information\", \"context does not\", \"not enough information\", \"cannot answer\",\n \"insufficient\", \"does not provide\", \"unable to\", \"cannot be determined\")\n\n\ndef _q_of_corpus_msg(u: str) -> str:\n return (u.split(\"[QUESTION]\", 1)[1].strip().lower() if \"[QUESTION]\" in u\n else u.strip().lower()[-200:])\n\n\ndef _covered() -> set:\n if not OUT.exists():\n return set()\n out = set()\n for l in OUT.read_text(encoding=\"utf-8\").splitlines():\n if not l.strip():\n continue\n try:\n r = json.loads(l)\n if \"messages\" in r:\n out.add(_q_of_corpus_msg(r[\"messages\"][0][\"content\"]))\n except Exception:\n pass\n return out\n\n\ndef _label(ans: str):\n a = ans.lower()\n has_abstain = any(m in a for m in _ABSTAIN)\n has_cite = \"[source:\" in a or \"[from \" in a\n if has_abstain and has_cite:\n return \"grounded_abstain\", True\n if has_abstain:\n return \"abstain\", False\n return \"grounded\", True\n\n\nasync def _gen(client, key, item: dict) -> dict | None:\n user = _CTX_PREFIX + item[\"context\"] + \"\\n[QUESTION]\\n\" + item[\"question\"]\n body = {\"model\": \"deepseek-chat\",\n \"messages\": [{\"role\": \"system\", \"content\": _SYSTEM},\n {\"role\": \"user\", \"content\": user}],\n \"temperature\": 0.3, \"max_tokens\": 1024}\n for attempt in range(3):\n try:\n r = await client.post(_DS_URL, headers={\"Authorization\": f\"Bearer {key}\"},\n json=body, timeout=90.0)\n if r.status_code != 200:\n await asyncio.sleep(2 * (attempt + 1)); continue\n ans = r.json()[\"choices\"][0][\"message\"][\"content\"].strip()\n if not ans:\n return None\n label, grounded = _label(ans)\n return {\"messages\": [{\"role\": \"user\", \"content\": user},\n {\"role\": \"assistant\", \"content\": ans}],\n \"topic\": item.get(\"topic\"), \"grounded\": grounded,\n \"label\": label, \"source\": \"grounded_deepseek_v1\"}\n except Exception:\n await asyncio.sleep(2 * (attempt + 1))\n return None\n\n\nasync def main() -> int:\n ap = argparse.ArgumentParser()\n ap.add_argument(\"--limit\", type=int, default=0)\n a = ap.parse_args()\n key = os.environ.get(\"DEEPSEEK_API_KEY\", \"\").strip()\n if not key:\n print(\"BLOCKED: DEEPSEEK_API_KEY unset.\"); return 2\n items = [json.loads(l) for l in IN.read_text(encoding=\"utf-8\").splitlines() if l.strip()]\n covered = _covered()\n todo = [it for it in items\n if it.get(\"question\") and it.get(\"context\")\n and it[\"question\"].strip().lower() not in covered]\n if a.limit:\n todo = todo[:a.limit]\n print(f\"[grounded] {len(items)} openbook, {len(covered)} already grounded, {len(todo)} to generate\")\n if not todo:\n print(\"[grounded] corpus complete — nothing to do.\"); return 0\n\n import httpx\n sem = asyncio.Semaphore(_CONCURRENCY)\n lock = asyncio.Lock()\n stats = {\"ok\": 0, \"fail\": 0, \"grounded\": 0, \"grounded_abstain\": 0, \"abstain\": 0}\n f = OUT.open(\"a\", encoding=\"utf-8\")\n async with httpx.AsyncClient() as client:\n async def worker(it):\n async with sem:\n try:\n rec = await _gen(client, key, it)\n except Exception:\n rec = None\n async with lock:\n if rec:\n f.write(json.dumps(rec, ensure_ascii=False) + \"\\n\"); f.flush()\n stats[\"ok\"] += 1; stats[rec[\"label\"]] += 1\n else:\n stats[\"fail\"] += 1\n n = stats[\"ok\"] + stats[\"fail\"]\n if n % 20 == 0 or n == len(todo):\n print(f\"[grounded] {n}/{len(todo)} {stats}\", flush=True)\n await asyncio.gather(*(worker(it) for it in todo))\n f.close()\n print(f\"[grounded] DONE {stats} -> {OUT}\")\n return 0\n\n\nif __name__ == \"__main__\":\n raise SystemExit(asyncio.run(main()))\n"}}, "scripts/train/build_grpo_keywords_v2.py": {"46eadcb5c58e76d35cb545ca1e7ef989433f04c5": {"t": 1786067732.6105685, "n": 6242, "h": "ead651767edd6357", "c": "#!/usr/bin/env python\n\"\"\"R-F2558 — build the composite-win GRPO dataset (v2).\n\nCycle #1 won fabrication but recall stayed flat because the training data had:\n (a) NO expected_keywords -> the recall term in grounding_reward was always 0\n (b) answerable as the STRING \"True\"/\"False\" -> score()'s `is True/is False`\n identity checks never matched, killing the anti-abstention-gaming + over-claim\n logic and miscounting every unanswerable row as answerable.\n\nThis rebuilds data/training/grpo_grounded_prompts.jsonl -> _v2.jsonl with:\n * answerable coerced to a real bool\n * expected_keywords for each ANSWERABLE row: 5-8 salient grounded facts a correct\n answer must state, extracted by DeepSeek STRICTLY from the context, then gated to\n terms literally present in the context (case-insensitive) so recall is achievable\n and can never reward an ungrounded token. Unanswerable rows get NO keywords (they\n should abstain; recall is N/A).\n\nResumable: append-only sidecar keyed by row index; re-run continues. Idempotent merge\nat the end. DeepSeek only (§6/§18 — the one active provider).\n\"\"\"\nfrom __future__ import annotations\n\nimport asyncio\nimport json\nimport os\nimport re\nimport sys\nfrom pathlib import Path\n\nimport httpx\n\nSRC = Path(\"data/training/grpo_grounded_prompts.jsonl\")\nOUT = Path(\"data/training/grpo_grounded_prompts_v2.jsonl\")\nSIDE = Path(\"data/training/_grpo_kw_v2.sidecar.jsonl\") # {idx, keywords}\nDSK = os.environ.get(\"DEEPSEEK_API_KEY\", \"\").strip()\nCONC = int(os.environ.get(\"KW_CONC\", \"8\"))\n\n_STOP = { # structural / boilerplate tokens that are not substantive answer facts\n \"CONTEXT\", \"RAG\", \"RETRIEVED\", \"SOURCE\", \"SOURCES\", \"EVIDENCE\", \"ANSWER\",\n \"INLINE\", \"PROPRIETARY\", \"INTELLIGENCE\", \"INDEXED\", \"KNOWLEDGE\", \"FROM\",\n \"NOTE\", \"DOCUMENT\", \"ATTACHED\", \"SUMMARY\", \"REPORT\",\n}\n\n\ndef _to_bool(v):\n if isinstance(v, bool):\n return v\n if isinstance(v, str):\n return v.strip().lower() in (\"true\", \"1\", \"yes\")\n return bool(v)\n\n\ndef _present(term: str, ctx_low: str) -> bool:\n \"\"\"Keyword is achievable+grounded only if it literally occurs in the context.\"\"\"\n t = (term or \"\").strip()\n if len(t) < 2:\n return False\n if t.upper() in _STOP:\n return False\n return t.lower() in ctx_low\n\n\n_EXTRACT_SYS = (\n \"You extract the key factual keywords a correct answer must contain. \"\n \"Return ONLY a JSON array of 5-8 short strings. Each MUST be a substantive fact \"\n \"COPIED VERBATIM from the provided context: proper nouns, organisation/person \"\n \"names, acronyms, jurisdictions, specific numbers/dates, or key domain terms. \"\n \"No sentences, no explanations, no markdown — just the JSON array. Never invent \"\n \"a term that is not present verbatim in the context.\"\n)\n\n\nasync def _extract(client: httpx.AsyncClient, ctx: str) -> list[str]:\n # Trim the structural header; keep the evidence body for extraction.\n body = ctx[:5000]\n r = await client.post(\n \"https://api.deepseek.com/v1/chat/completions\",\n headers={\"Authorization\": f\"Bearer {DSK}\"},\n json={\n \"model\": \"deepseek-chat\",\n \"messages\": [\n {\"role\": \"system\", \"content\": _EXTRACT_SYS},\n {\"role\": \"user\", \"content\": f\"CONTEXT:\\n{body}\\n\\nJSON array of 5-8 keywords:\"},\n ],\n \"temperature\": 0.0,\n \"max_tokens\": 200,\n },\n timeout=60.0,\n )\n r.raise_for_status()\n txt = r.json()[\"choices\"][0][\"message\"][\"content\"]\n m = re.search(r\"\\[.*\\]\", txt, re.S)\n if not m:\n return []\n try:\n arr = json.loads(m.group(0))\n except Exception:\n return []\n ctx_low = ctx.lower()\n seen, out = set(), []\n for k in arr:\n if not isinstance(k, str):\n continue\n k = k.strip().strip('\"').strip(\"'\")\n kl = k.lower()\n if kl in seen or not _present(k, ctx_low):\n continue\n seen.add(kl)\n out.append(k)\n return out[:8]\n\n\nasync def main():\n if not DSK:\n print(\"FATAL: DEEPSEEK_API_KEY unset\"); sys.exit(1)\n rows = [json.loads(l) for l in SRC.open(encoding=\"utf-8\") if l.strip()]\n done = {}\n if SIDE.exists():\n for l in SIDE.open(encoding=\"utf-8\"):\n if l.strip():\n d = json.loads(l); done[d[\"idx\"]] = d[\"keywords\"]\n print(f\"[kw] {len(rows)} rows | {sum(1 for r in rows if _to_bool(r.get('answerable')))} answerable | resume {len(done)} done\")\n\n todo = [i for i, r in enumerate(rows)\n if _to_bool(r.get(\"answerable\")) and i not in done]\n print(f\"[kw] extracting keywords for {len(todo)} answerable rows (conc={CONC})…\")\n sem = asyncio.Semaphore(CONC)\n lock = asyncio.Lock()\n async with httpx.AsyncClient() as client:\n async def work(i):\n async with sem:\n for attempt in range(3):\n try:\n kw = await _extract(client, rows[i].get(\"context\", \"\"))\n break\n except Exception as e:\n if attempt == 2:\n print(f\"[kw] idx {i} failed: {e}\"); kw = []\n await asyncio.sleep(2 * (attempt + 1))\n async with lock:\n with SIDE.open(\"a\", encoding=\"utf-8\") as f:\n f.write(json.dumps({\"idx\": i, \"keywords\": kw}) + \"\\n\")\n done[i] = kw\n if len(done) % 25 == 0:\n print(f\"[kw] {len(done)} done\")\n await asyncio.gather(*(work(i) for i in todo))\n\n # merge -> v2\n kept = 0\n with OUT.open(\"w\", encoding=\"utf-8\") as f:\n for i, r in enumerate(rows):\n r2 = dict(r)\n r2[\"answerable\"] = _to_bool(r.get(\"answerable\"))\n if r2[\"answerable\"]:\n kw = done.get(i, [])\n if len(kw) >= 3:\n r2[\"expected_keywords\"] = kw; kept += 1\n f.write(json.dumps(r2, ensure_ascii=False) + \"\\n\")\n n_ans = sum(1 for r in rows if _to_bool(r.get(\"answerable\")))\n print(f\"[kw] wrote {OUT} | answerable={n_ans} | with>=3 keywords={kept} \"\n f\"({100*kept//max(1,n_ans)}% of answerable)\")\n\n\nif __name__ == \"__main__\":\n asyncio.run(main())\n"}}}} |