OpenCode
68a9071cf1
Local branch had diverged from origin/master (sibling commits on the same base). Rewrote local history linearly on top of origin/master, folding in all local content: exa/hot-platform discovery hunters, leaks ledger, nightly sweep orchestrator, updated .gitignore and skill docs, plus the latest hunt output and vault state. Remote-only files (vault keys, channel scripts) were restored rather than dropped, so the resulting tree is a full union of both sides.
1 line
6.8 KiB
JSON
1 line
6.8 KiB
JSON
{"repo": "advatar/nextai", "files": {"self-improving-starter/rl/grpo_config.yaml": {"03385619fe8e11aca92c90c03e9ccc46c1e09ffe": {"t": 1785746859.626864, "n": 2123, "h": "4db75ea89f2312c4", "c": "# verl GRPO config skeleton for RLVR fine-tuning.\n#\n# This is a historical, deliberately disabled design skeleton, not a runnable\n# recipe. Before use it needs disjoint task-level datasets, an isolated reward\n# service, a verified trajectory schema, a pinned model, and a GPU cluster.\n# rl/reward.py refuses execution outside an explicitly isolated reward worker.\n#\n# Run (on the GPU host, after `pip install verl vllm torch datasets`):\n# python -m verl.trainer.main_ppo --config-path rl --config-name grpo_config\n#\n# Docs: https://github.com/volcengine/verl\n\nalgorithm:\n adv_estimator: grpo # Group Relative Policy Optimization (no value net)\n kl_ctrl:\n kl_coef: 0.001 # light KL to the reference policy; raise if it drifts\n\ndata:\n train_files: data/train.parquet # built by rl/dataset.py\n val_files: data/val.parquet # must contain disjoint HELD-OUT TASKS\n prompt_key: prompt\n max_prompt_length: 2048\n max_response_length: 2048\n train_batch_size: 64\n\nactor_rollout_ref:\n model:\n # Pick an open-weight model strong at code + reasoning. You cannot fine-tune\n # closed frontier models (Claude/GPT) this way — use an open checkpoint:\n # Qwen/Qwen3-Coder-* DeepSeek distills meta-llama/Llama-*\n path: Qwen/Qwen3-Coder-7B # <-- set to your chosen base checkpoint\n actor:\n optim:\n lr: 1.0e-6\n ppo_mini_batch_size: 16\n use_kl_loss: true\n rollout:\n name: vllm # vLLM backend for fast rollout generation\n n: 8 # samples per prompt (the \"group\" in GRPO)\n temperature: 1.0\n gpu_memory_utilization: 0.6\n\nreward_model:\n enable: false # RLVR: no learned reward model\n # verl loads the custom reward function below instead:\ncustom_reward_function:\n path: rl/reward.py\n name: compute_score\n\ntrainer:\n total_epochs: 3\n n_gpus_per_node: 8 # <-- set to your hardware\n nnodes: 1\n save_freq: 50\n test_freq: 10 # evaluate on the HELD-OUT set regularly; watch for reward-hacking\n project_name: self-improving-starter\n experiment_name: grpo-rlvr\n"}, "3e33f7d6d1f6f944a8df05b3908ed35da69e6fe3": {"t": 1785768104.4017725, "n": 2123, "h": "4db75ea89f2312c4", "c": "# verl GRPO config skeleton for RLVR fine-tuning.\n#\n# This is a historical, deliberately disabled design skeleton, not a runnable\n# recipe. Before use it needs disjoint task-level datasets, an isolated reward\n# service, a verified trajectory schema, a pinned model, and a GPU cluster.\n# rl/reward.py refuses execution outside an explicitly isolated reward worker.\n#\n# Run (on the GPU host, after `pip install verl vllm torch datasets`):\n# python -m verl.trainer.main_ppo --config-path rl --config-name grpo_config\n#\n# Docs: https://github.com/volcengine/verl\n\nalgorithm:\n adv_estimator: grpo # Group Relative Policy Optimization (no value net)\n kl_ctrl:\n kl_coef: 0.001 # light KL to the reference policy; raise if it drifts\n\ndata:\n train_files: data/train.parquet # built by rl/dataset.py\n val_files: data/val.parquet # must contain disjoint HELD-OUT TASKS\n prompt_key: prompt\n max_prompt_length: 2048\n max_response_length: 2048\n train_batch_size: 64\n\nactor_rollout_ref:\n model:\n # Pick an open-weight model strong at code + reasoning. You cannot fine-tune\n # closed frontier models (Claude/GPT) this way — use an open checkpoint:\n # Qwen/Qwen3-Coder-* DeepSeek distills meta-llama/Llama-*\n path: Qwen/Qwen3-Coder-7B # <-- set to your chosen base checkpoint\n actor:\n optim:\n lr: 1.0e-6\n ppo_mini_batch_size: 16\n use_kl_loss: true\n rollout:\n name: vllm # vLLM backend for fast rollout generation\n n: 8 # samples per prompt (the \"group\" in GRPO)\n temperature: 1.0\n gpu_memory_utilization: 0.6\n\nreward_model:\n enable: false # RLVR: no learned reward model\n # verl loads the custom reward function below instead:\ncustom_reward_function:\n path: rl/reward.py\n name: compute_score\n\ntrainer:\n total_epochs: 3\n n_gpus_per_node: 8 # <-- set to your hardware\n nnodes: 1\n save_freq: 50\n test_freq: 10 # evaluate on the HELD-OUT set regularly; watch for reward-hacking\n project_name: self-improving-starter\n experiment_name: grpo-rlvr\n"}, "9e93bf1a1183bac91e3e09156bbdebb6b2b99f00": {"t": 1786068508.5793436, "n": 2123, "h": "4db75ea89f2312c4", "c": "# verl GRPO config skeleton for RLVR fine-tuning.\n#\n# This is a historical, deliberately disabled design skeleton, not a runnable\n# recipe. Before use it needs disjoint task-level datasets, an isolated reward\n# service, a verified trajectory schema, a pinned model, and a GPU cluster.\n# rl/reward.py refuses execution outside an explicitly isolated reward worker.\n#\n# Run (on the GPU host, after `pip install verl vllm torch datasets`):\n# python -m verl.trainer.main_ppo --config-path rl --config-name grpo_config\n#\n# Docs: https://github.com/volcengine/verl\n\nalgorithm:\n adv_estimator: grpo # Group Relative Policy Optimization (no value net)\n kl_ctrl:\n kl_coef: 0.001 # light KL to the reference policy; raise if it drifts\n\ndata:\n train_files: data/train.parquet # built by rl/dataset.py\n val_files: data/val.parquet # must contain disjoint HELD-OUT TASKS\n prompt_key: prompt\n max_prompt_length: 2048\n max_response_length: 2048\n train_batch_size: 64\n\nactor_rollout_ref:\n model:\n # Pick an open-weight model strong at code + reasoning. You cannot fine-tune\n # closed frontier models (Claude/GPT) this way — use an open checkpoint:\n # Qwen/Qwen3-Coder-* DeepSeek distills meta-llama/Llama-*\n path: Qwen/Qwen3-Coder-7B # <-- set to your chosen base checkpoint\n actor:\n optim:\n lr: 1.0e-6\n ppo_mini_batch_size: 16\n use_kl_loss: true\n rollout:\n name: vllm # vLLM backend for fast rollout generation\n n: 8 # samples per prompt (the \"group\" in GRPO)\n temperature: 1.0\n gpu_memory_utilization: 0.6\n\nreward_model:\n enable: false # RLVR: no learned reward model\n # verl loads the custom reward function below instead:\ncustom_reward_function:\n path: rl/reward.py\n name: compute_score\n\ntrainer:\n total_epochs: 3\n n_gpus_per_node: 8 # <-- set to your hardware\n nnodes: 1\n save_freq: 50\n test_freq: 10 # evaluate on the HELD-OUT set regularly; watch for reward-hacking\n project_name: self-improving-starter\n experiment_name: grpo-rlvr\n"}}}} |