1 line
35 KiB
JSON
1 line
35 KiB
JSON
{"repo": "bgi-plant/onegenome-rice", "files": {"model/tests/functional_tests/test_cases/mixtral/deepseekv3_proxy_flex_tp1pp4emp16etp1cp1_release/model_config.yaml": {"f8a3bdaf5590a19a2917d09e44a4b2acb0b15d16": {"t": 1785746823.646297, "n": 4592, "h": "efe1d5d348aaeb39", "c": "# The proxy model is used for local code quality check.\n# The proxy model should contain all the necessary components and settings but fewer parameters.\nENV_VARS:\n TORCH_NCCL_AVOID_RECORD_STREAMS: 0\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 1\n PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True\n NCCL_NVLS_ENABLE: 0\n NVTE_FUSED_ATTN: 1\n NVTE_NORM_FWD_USE_CUDNN: 1\n NVTE_NORM_BWD_USE_CUDNN: 1\n PYTHONWARNINGS: ignore\n NCCL_DEBUG: VERSION\n NON_DETERMINSTIC_RESULTS: 1\n NVSHMEM_IB_ENABLE_IBGDA: 0\nTEST_TYPE: \"release\"\nMODEL_ARGS:\n # Distributed args\n --distributed-timeout-minutes: 60\n --tensor-model-parallel-size: 1\n --pipeline-model-parallel-size: 4\n --pipeline-model-parallel-layout: Et*2\\\\|\\\\(tt\\\\|\\\\)*5t\\\\|tmL # Et*2|(tt|)*5t|tmL\n --expert-model-parallel-size: 16\n --context-parallel-size: 1\n --expert-tensor-parallel-size: 1\n --use-distributed-optimizer: true\n --overlap-grad-reduce: true\n --overlap-param-gather: true\n\n # Training args\n --use-mcore-models: true\n --sequence-parallel: true\n --use-flash-attn: true\n --disable-bias-linear: true\n --micro-batch-size: 1\n --global-batch-size: 512\n --train-samples: 24414062\n --exit-duration-in-mins: 220\n --no-check-for-nan-in-loss-and-grad: true\n --cross-entropy-loss-fusion: true\n --cross-entropy-fusion-impl: te\n --manual-gc: true\n --manual-gc-interval: 10\n\n # Transformer Engine args\n --transformer-impl: transformer_engine\n\n # Data args\n --seq-length: 4096\n --data-cache-path: ${DATA_CACHE_PATH}\n --tokenizer-type: HuggingFaceTokenizer\n --tokenizer-model: ${TOKENIZER_PATH}\n --data-path: $DATA_BLEND\n --split: 99,1,0\n --no-mmap-bin-files: true\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 6\n\n # Add network size args\n --num-layers: 14 # original 61 layers\n --hidden-size: 7168\n --ffn-hidden-size: 18432\n --num-attention-heads: 128\n --kv-channels: 128\n --max-position-embeddings: 4096\n --position-embedding-type: rope\n --rotary-base: 10000\n --make-vocab-size-divisible-by: 3232\n --normalization: RMSNorm\n --norm-epsilon: 1e-6\n --swiglu: true\n --untie-embeddings-and-output-weights: true\n --multi-latent-attention: true\n --mtp-num-layers: 1\n --mtp-loss-scaling-factor: 0.1\n\n # Add regularization args\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --clip-grad: 1.0\n --weight-decay: 0.1\n --qk-layernorm: true\n\n # Add learning rate args\n --lr-decay-samples: 584765624\n --lr-warmup-samples: 1536000\n # Learning rate scaled down from 7.3e-6 (DeepSeek-V3 technical report, GBS=15360) to 3.9e-6 (GBS=8192)\n --lr-warmup-init: 3.9e-7\n --lr: 3.9e-6\n --min-lr: 3.9e-7\n --lr-decay-style: cosine\n --adam-beta1: 0.9\n --adam-beta2: 0.95\n\n # Add MoE args\n --num-experts: 64 # local 4 + 1 shared, EP16\n --moe-layer-freq: ([0]*3+[1]*11)\n --moe-ffn-hidden-size: 2048\n --moe-shared-expert-intermediate-size: 2048\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-router-topk: 8\n --moe-token-dispatcher-type: flex\n --moe-enable-deepep: true\n --moe-router-pre-softmax: true\n --moe-grouped-gemm: true\n --moe-aux-loss-coeff: 1e-4\n --moe-router-group-topk: 4\n --moe-router-num-groups: 8\n --moe-router-topk-scaling-factor: 2.5\n --moe-router-score-function: sigmoid\n --moe-router-enable-expert-bias: true\n --moe-router-bias-update-rate: 1e-3\n --moe-router-dtype: fp32\n --moe-permute-fusion: true\n\n # Add MLA args\n --q-lora-rank: 1536\n --kv-lora-rank: 512\n --qk-head-dim: 128\n --qk-pos-emb-head-dim: 64\n --v-head-dim: 128\n --rotary-scaling-factor: 40\n --mscale: 1.0\n --mscale-all-dim: 1.0\n\n # Add validation args\n --eval-iters: 32\n --eval-interval: 200\n\n # Add checkpointing args\n --no-load-optim: true\n --no-load-rng: true\n --auto-detect-ckpt-format:\n true\n # Add checkpointing args\n --save: ${CHECKPOINT_SAVE_PATH}\n --load: ${CHECKPOINT_LOAD_PATH}\n --save-interval: 500\n --save-retain-interval: 10000\n --dist-ckpt-strictness: log_all\n\n # Add initialization args\n --init-method-std: 0.02\n\n # Add logging args\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --log-num-zeros-in-grad: true\n --log-params-norm: true\n --log-validation-ppl-to-tensorboard: true\n --log-throughput: true\n --log-interval: 1\n --logging-level: 40\n --tensorboard-dir: ${TENSORBOARD_PATH}\n --wandb-project: megatron-core-release-runs\n --wandb-exp-name: ${WANDB_EXPERIMENT}\n --wandb-save-dir: ${WANDB_SAVE_PATH}\n\n # Add mixed precision args\n --bf16: true\n\n # enable experimental\n --enable-experimental: true\nMETRICS:\n - \"iteration-time\"\n - \"lm loss\"\n - \"mem-allocated-bytes\"\n - \"mem-max-allocated-bytes\"\n"}, "931b275029e26e093fa6b33ca83a9ad363b85d31": {"t": 1785767889.6607063, "n": 4592, "h": "efe1d5d348aaeb39", "c": "# The proxy model is used for local code quality check.\n# The proxy model should contain all the necessary components and settings but fewer parameters.\nENV_VARS:\n TORCH_NCCL_AVOID_RECORD_STREAMS: 0\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 1\n PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True\n NCCL_NVLS_ENABLE: 0\n NVTE_FUSED_ATTN: 1\n NVTE_NORM_FWD_USE_CUDNN: 1\n NVTE_NORM_BWD_USE_CUDNN: 1\n PYTHONWARNINGS: ignore\n NCCL_DEBUG: VERSION\n NON_DETERMINSTIC_RESULTS: 1\n NVSHMEM_IB_ENABLE_IBGDA: 0\nTEST_TYPE: \"release\"\nMODEL_ARGS:\n # Distributed args\n --distributed-timeout-minutes: 60\n --tensor-model-parallel-size: 1\n --pipeline-model-parallel-size: 4\n --pipeline-model-parallel-layout: Et*2\\\\|\\\\(tt\\\\|\\\\)*5t\\\\|tmL # Et*2|(tt|)*5t|tmL\n --expert-model-parallel-size: 16\n --context-parallel-size: 1\n --expert-tensor-parallel-size: 1\n --use-distributed-optimizer: true\n --overlap-grad-reduce: true\n --overlap-param-gather: true\n\n # Training args\n --use-mcore-models: true\n --sequence-parallel: true\n --use-flash-attn: true\n --disable-bias-linear: true\n --micro-batch-size: 1\n --global-batch-size: 512\n --train-samples: 24414062\n --exit-duration-in-mins: 220\n --no-check-for-nan-in-loss-and-grad: true\n --cross-entropy-loss-fusion: true\n --cross-entropy-fusion-impl: te\n --manual-gc: true\n --manual-gc-interval: 10\n\n # Transformer Engine args\n --transformer-impl: transformer_engine\n\n # Data args\n --seq-length: 4096\n --data-cache-path: ${DATA_CACHE_PATH}\n --tokenizer-type: HuggingFaceTokenizer\n --tokenizer-model: ${TOKENIZER_PATH}\n --data-path: $DATA_BLEND\n --split: 99,1,0\n --no-mmap-bin-files: true\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 6\n\n # Add network size args\n --num-layers: 14 # original 61 layers\n --hidden-size: 7168\n --ffn-hidden-size: 18432\n --num-attention-heads: 128\n --kv-channels: 128\n --max-position-embeddings: 4096\n --position-embedding-type: rope\n --rotary-base: 10000\n --make-vocab-size-divisible-by: 3232\n --normalization: RMSNorm\n --norm-epsilon: 1e-6\n --swiglu: true\n --untie-embeddings-and-output-weights: true\n --multi-latent-attention: true\n --mtp-num-layers: 1\n --mtp-loss-scaling-factor: 0.1\n\n # Add regularization args\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --clip-grad: 1.0\n --weight-decay: 0.1\n --qk-layernorm: true\n\n # Add learning rate args\n --lr-decay-samples: 584765624\n --lr-warmup-samples: 1536000\n # Learning rate scaled down from 7.3e-6 (DeepSeek-V3 technical report, GBS=15360) to 3.9e-6 (GBS=8192)\n --lr-warmup-init: 3.9e-7\n --lr: 3.9e-6\n --min-lr: 3.9e-7\n --lr-decay-style: cosine\n --adam-beta1: 0.9\n --adam-beta2: 0.95\n\n # Add MoE args\n --num-experts: 64 # local 4 + 1 shared, EP16\n --moe-layer-freq: ([0]*3+[1]*11)\n --moe-ffn-hidden-size: 2048\n --moe-shared-expert-intermediate-size: 2048\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-router-topk: 8\n --moe-token-dispatcher-type: flex\n --moe-enable-deepep: true\n --moe-router-pre-softmax: true\n --moe-grouped-gemm: true\n --moe-aux-loss-coeff: 1e-4\n --moe-router-group-topk: 4\n --moe-router-num-groups: 8\n --moe-router-topk-scaling-factor: 2.5\n --moe-router-score-function: sigmoid\n --moe-router-enable-expert-bias: true\n --moe-router-bias-update-rate: 1e-3\n --moe-router-dtype: fp32\n --moe-permute-fusion: true\n\n # Add MLA args\n --q-lora-rank: 1536\n --kv-lora-rank: 512\n --qk-head-dim: 128\n --qk-pos-emb-head-dim: 64\n --v-head-dim: 128\n --rotary-scaling-factor: 40\n --mscale: 1.0\n --mscale-all-dim: 1.0\n\n # Add validation args\n --eval-iters: 32\n --eval-interval: 200\n\n # Add checkpointing args\n --no-load-optim: true\n --no-load-rng: true\n --auto-detect-ckpt-format:\n true\n # Add checkpointing args\n --save: ${CHECKPOINT_SAVE_PATH}\n --load: ${CHECKPOINT_LOAD_PATH}\n --save-interval: 500\n --save-retain-interval: 10000\n --dist-ckpt-strictness: log_all\n\n # Add initialization args\n --init-method-std: 0.02\n\n # Add logging args\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --log-num-zeros-in-grad: true\n --log-params-norm: true\n --log-validation-ppl-to-tensorboard: true\n --log-throughput: true\n --log-interval: 1\n --logging-level: 40\n --tensorboard-dir: ${TENSORBOARD_PATH}\n --wandb-project: megatron-core-release-runs\n --wandb-exp-name: ${WANDB_EXPERIMENT}\n --wandb-save-dir: ${WANDB_SAVE_PATH}\n\n # Add mixed precision args\n --bf16: true\n\n # enable experimental\n --enable-experimental: true\nMETRICS:\n - \"iteration-time\"\n - \"lm loss\"\n - \"mem-allocated-bytes\"\n - \"mem-max-allocated-bytes\"\n"}}, "model/tests/functional_tests/test_cases/moe/gpt_static_inference_tp4_pp1_ep4_16B_logitsmatch/model_config.yaml": {"f8a3bdaf5590a19a2917d09e44a4b2acb0b15d16": {"t": 1785746825.636706, "n": 2876, "h": "155a8806e6916e72", "c": "ENV_VARS:\n CUDA_DEVICE_MAX_CONNECTIONS: 1\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n NCCL_ALGO: Ring\n CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n --log-num-zeros-in-grad: true\n --log-validation-ppl-to-tensorboard: true\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --timing-log-level: 0\n --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n --tokenizer-type: TikTokenizer\n --tiktoken-pattern: v2\n --distributed-backend: nccl\n --log-interval: 1\n --transformer-impl: transformer_engine\n --tensor-model-parallel-size: 4\n --pipeline-model-parallel-size: 1\n --expert-model-parallel-size: 4\n --expert-tensor-parallel-size: 1\n --sequence-parallel: true\n --use-mcore-models: true\n --moe-token-dispatcher-type: alltoall\n --moe-grouped-gemm: true\n --num-experts: 64\n --moe-router-topk: 6\n --moe-z-loss-coeff: 0\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-aux-loss-coeff: 1e-3\n --moe-router-score-function: sigmoid\n --untie-embeddings-and-output-weights: true\n --disable-bias-linear: true\n --init-method-std: 0.014\n --position-embedding-type: rope\n --rotary-base: 1000000\n --rotary-percent: 1.0\n --num-layers: 27\n --hidden-size: 2048\n --moe-ffn-hidden-size: 1408\n --moe-shared-expert-intermediate-size: 2816\n --ffn-hidden-size: 10944\n --num-attention-heads: 16\n --kv-channels: 128\n --normalization: RMSNorm\n --swiglu: true\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --seq-length: 4096\n --max-position-embeddings: 4096\n --micro-batch-size: 1\n --ckpt-format: torch_dist\n --ckpt-fully-parallel-save: true\n --ckpt-fully-parallel-load: true\n --ckpt-assume-constant-structure: true\n --dist-ckpt-strictness: log_unexpected\n --bf16: true\n --attention-backend: flash\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 8\n --flash-decode: true\n --use-checkpoint-args: true\n --no-use-tokenizer-model-from-checkpoint-args: true\n --no-load-optim: true\n --deterministic-mode: true\n --save-interval: 2000\n --temperature: 1.0\n --top_k: 1\n --return-log-probs: true\n --num-tokens-to-generate: 30\n --max-tokens-to-oom: 3600000\n --inference-max-seq-length: 4096\n --output-path: ${TENSORBOARD_PATH}\n --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n --incoming-requests-per-sec: -1 # all requests arrive up front.\nMETRICS:\n - \"generated_tokens\"\n - \"logprobs\"\n"}, "931b275029e26e093fa6b33ca83a9ad363b85d31": {"t": 1785767888.8417704, "n": 2876, "h": "155a8806e6916e72", "c": "ENV_VARS:\n CUDA_DEVICE_MAX_CONNECTIONS: 1\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n NCCL_ALGO: Ring\n CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n --log-num-zeros-in-grad: true\n --log-validation-ppl-to-tensorboard: true\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --timing-log-level: 0\n --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n --tokenizer-type: TikTokenizer\n --tiktoken-pattern: v2\n --distributed-backend: nccl\n --log-interval: 1\n --transformer-impl: transformer_engine\n --tensor-model-parallel-size: 4\n --pipeline-model-parallel-size: 1\n --expert-model-parallel-size: 4\n --expert-tensor-parallel-size: 1\n --sequence-parallel: true\n --use-mcore-models: true\n --moe-token-dispatcher-type: alltoall\n --moe-grouped-gemm: true\n --num-experts: 64\n --moe-router-topk: 6\n --moe-z-loss-coeff: 0\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-aux-loss-coeff: 1e-3\n --moe-router-score-function: sigmoid\n --untie-embeddings-and-output-weights: true\n --disable-bias-linear: true\n --init-method-std: 0.014\n --position-embedding-type: rope\n --rotary-base: 1000000\n --rotary-percent: 1.0\n --num-layers: 27\n --hidden-size: 2048\n --moe-ffn-hidden-size: 1408\n --moe-shared-expert-intermediate-size: 2816\n --ffn-hidden-size: 10944\n --num-attention-heads: 16\n --kv-channels: 128\n --normalization: RMSNorm\n --swiglu: true\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --seq-length: 4096\n --max-position-embeddings: 4096\n --micro-batch-size: 1\n --ckpt-format: torch_dist\n --ckpt-fully-parallel-save: true\n --ckpt-fully-parallel-load: true\n --ckpt-assume-constant-structure: true\n --dist-ckpt-strictness: log_unexpected\n --bf16: true\n --attention-backend: flash\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 8\n --flash-decode: true\n --use-checkpoint-args: true\n --no-use-tokenizer-model-from-checkpoint-args: true\n --no-load-optim: true\n --deterministic-mode: true\n --save-interval: 2000\n --temperature: 1.0\n --top_k: 1\n --return-log-probs: true\n --num-tokens-to-generate: 30\n --max-tokens-to-oom: 3600000\n --inference-max-seq-length: 4096\n --output-path: ${TENSORBOARD_PATH}\n --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n --incoming-requests-per-sec: -1 # all requests arrive up front.\nMETRICS:\n - \"generated_tokens\"\n - \"logprobs\"\n"}}, "model/tests/functional_tests/test_cases/moe/gpt_static_inference_cuda_graphs_pad_tp4_pp1_ep4_16B_logitsmatch/model_config.yaml": {"f8a3bdaf5590a19a2917d09e44a4b2acb0b15d16": {"t": 1785746836.1310086, "n": 3010, "h": "cae46104d1655d34", "c": "ENV_VARS:\n CUDA_DEVICE_MAX_CONNECTIONS: 1\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n NCCL_ALGO: Ring\n CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n --log-num-zeros-in-grad: true\n --log-validation-ppl-to-tensorboard: true\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --timing-log-level: 0\n --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n --tokenizer-type: TikTokenizer\n --tiktoken-pattern: v2\n --distributed-backend: nccl\n --log-interval: 1\n --transformer-impl: transformer_engine\n --tensor-model-parallel-size: 4\n --pipeline-model-parallel-size: 1\n --expert-model-parallel-size: 4\n --expert-tensor-parallel-size: 1\n --sequence-parallel: true\n --use-mcore-models: true\n --moe-token-dispatcher-type: alltoall\n --moe-grouped-gemm: true\n --num-experts: 64\n --moe-router-topk: 6\n --moe-z-loss-coeff: 0\n --moe-aux-loss-coeff: 1e-3\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-router-score-function: sigmoid\n --untie-embeddings-and-output-weights: true\n --disable-bias-linear: true\n --init-method-std: 0.014\n --position-embedding-type: rope\n --rotary-base: 1000000\n --rotary-percent: 1.0\n --num-layers: 27\n --hidden-size: 2048\n --moe-ffn-hidden-size: 1408\n --moe-shared-expert-intermediate-size: 2816\n --ffn-hidden-size: 10944\n --num-attention-heads: 16\n --kv-channels: 128\n --normalization: RMSNorm\n --swiglu: true\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --seq-length: 4096\n --max-position-embeddings: 4096\n --micro-batch-size: 1\n --flash-decode: true\n --enable-cuda-graph: true\n --te-rng-tracker: true\n --inference-rng-tracker: true\n --moe-pad-experts-for-cuda-graph-inference: true\n --ckpt-format: torch_dist\n --ckpt-fully-parallel-save: true\n --ckpt-fully-parallel-load: true\n --ckpt-assume-constant-structure: true\n --dist-ckpt-strictness: log_unexpected\n --bf16: true\n --attention-backend: flash\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 8\n --use-checkpoint-args: true\n --no-use-tokenizer-model-from-checkpoint-args: true\n --no-load-optim: true\n --deterministic-mode: true\n --save-interval: 2000\n --temperature: 1.0\n --top_k: 1\n --return-log-probs: true\n --num-tokens-to-generate: 30\n --max-tokens-to-oom: 3600000\n --inference-max-seq-length: 4096\n --inference-max-requests: 1\n --output-path: ${TENSORBOARD_PATH}\n --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n --incoming-requests-per-sec: -1\nMETRICS:\n - \"generated_tokens\"\n - \"logprobs\"\n"}, "931b275029e26e093fa6b33ca83a9ad363b85d31": {"t": 1785767889.7348137, "n": 3010, "h": "cae46104d1655d34", "c": "ENV_VARS:\n CUDA_DEVICE_MAX_CONNECTIONS: 1\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n NCCL_ALGO: Ring\n CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n --log-num-zeros-in-grad: true\n --log-validation-ppl-to-tensorboard: true\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --timing-log-level: 0\n --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n --tokenizer-type: TikTokenizer\n --tiktoken-pattern: v2\n --distributed-backend: nccl\n --log-interval: 1\n --transformer-impl: transformer_engine\n --tensor-model-parallel-size: 4\n --pipeline-model-parallel-size: 1\n --expert-model-parallel-size: 4\n --expert-tensor-parallel-size: 1\n --sequence-parallel: true\n --use-mcore-models: true\n --moe-token-dispatcher-type: alltoall\n --moe-grouped-gemm: true\n --num-experts: 64\n --moe-router-topk: 6\n --moe-z-loss-coeff: 0\n --moe-aux-loss-coeff: 1e-3\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-router-score-function: sigmoid\n --untie-embeddings-and-output-weights: true\n --disable-bias-linear: true\n --init-method-std: 0.014\n --position-embedding-type: rope\n --rotary-base: 1000000\n --rotary-percent: 1.0\n --num-layers: 27\n --hidden-size: 2048\n --moe-ffn-hidden-size: 1408\n --moe-shared-expert-intermediate-size: 2816\n --ffn-hidden-size: 10944\n --num-attention-heads: 16\n --kv-channels: 128\n --normalization: RMSNorm\n --swiglu: true\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --seq-length: 4096\n --max-position-embeddings: 4096\n --micro-batch-size: 1\n --flash-decode: true\n --enable-cuda-graph: true\n --te-rng-tracker: true\n --inference-rng-tracker: true\n --moe-pad-experts-for-cuda-graph-inference: true\n --ckpt-format: torch_dist\n --ckpt-fully-parallel-save: true\n --ckpt-fully-parallel-load: true\n --ckpt-assume-constant-structure: true\n --dist-ckpt-strictness: log_unexpected\n --bf16: true\n --attention-backend: flash\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 8\n --use-checkpoint-args: true\n --no-use-tokenizer-model-from-checkpoint-args: true\n --no-load-optim: true\n --deterministic-mode: true\n --save-interval: 2000\n --temperature: 1.0\n --top_k: 1\n --return-log-probs: true\n --num-tokens-to-generate: 30\n --max-tokens-to-oom: 3600000\n --inference-max-seq-length: 4096\n --inference-max-requests: 1\n --output-path: ${TENSORBOARD_PATH}\n --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n --incoming-requests-per-sec: -1\nMETRICS:\n - \"generated_tokens\"\n - \"logprobs\"\n"}}, "model/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_pp1_ep4_16B_logitsmatch/model_config.yaml": {"f8a3bdaf5590a19a2917d09e44a4b2acb0b15d16": {"t": 1785746856.4126694, "n": 2938, "h": "9cd5ee8d831043f9", "c": "ENV_VARS:\n CUDA_DEVICE_MAX_CONNECTIONS: 1\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n NCCL_ALGO: Ring\n CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n --log-num-zeros-in-grad: true\n --log-validation-ppl-to-tensorboard: true\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --timing-log-level: 0\n --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n --tokenizer-type: TikTokenizer\n --tiktoken-pattern: v2\n --distributed-backend: nccl\n --log-interval: 1\n --transformer-impl: transformer_engine\n --tensor-model-parallel-size: 4\n --pipeline-model-parallel-size: 1\n --expert-model-parallel-size: 4\n --expert-tensor-parallel-size: 1\n --sequence-parallel: true\n --use-mcore-models: true\n --moe-token-dispatcher-type: alltoall\n --moe-grouped-gemm: true\n --num-experts: 64\n --moe-router-topk: 6\n --moe-z-loss-coeff: 0\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-aux-loss-coeff: 1e-3\n --moe-router-score-function: sigmoid\n --untie-embeddings-and-output-weights: true\n --disable-bias-linear: true\n --init-method-std: 0.014\n --position-embedding-type: rope\n --rotary-base: 1000000\n --rotary-percent: 1.0\n --num-layers: 27\n --hidden-size: 2048\n --moe-ffn-hidden-size: 1408\n --moe-shared-expert-intermediate-size: 2816\n --ffn-hidden-size: 10944\n --num-attention-heads: 16\n --kv-channels: 128\n --normalization: RMSNorm\n --swiglu: true\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --seq-length: 4096\n --max-position-embeddings: 4096\n --micro-batch-size: 1\n --ckpt-format: torch_dist\n --ckpt-fully-parallel-save: true\n --ckpt-fully-parallel-load: true\n --ckpt-assume-constant-structure: true\n --dist-ckpt-strictness: log_unexpected\n --bf16: true\n --attention-backend: flash\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 8\n --use-checkpoint-args: true\n --no-use-tokenizer-model-from-checkpoint-args: true\n --no-load-optim: true\n --deterministic-mode: true # moe will use different ops for determinism for inference\n --save-interval: 2000\n --temperature: 1.0\n --top_k: 1\n --return-log-probs: true\n --num-tokens-to-generate: 30\n --max-tokens-to-oom: 3600000\n --inference-max-seq-length: 4096\n --output-path: ${TENSORBOARD_PATH}\n --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n --incoming-requests-per-sec: -1 # all requests arrive up front.\n --inference-repeat-n: 8\nMETRICS:\n - \"generated_tokens\"\n - \"logprobs\"\n"}, "931b275029e26e093fa6b33ca83a9ad363b85d31": {"t": 1785767889.2072768, "n": 2938, "h": "9cd5ee8d831043f9", "c": "ENV_VARS:\n CUDA_DEVICE_MAX_CONNECTIONS: 1\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n NCCL_ALGO: Ring\n CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n --log-num-zeros-in-grad: true\n --log-validation-ppl-to-tensorboard: true\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --timing-log-level: 0\n --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n --tokenizer-type: TikTokenizer\n --tiktoken-pattern: v2\n --distributed-backend: nccl\n --log-interval: 1\n --transformer-impl: transformer_engine\n --tensor-model-parallel-size: 4\n --pipeline-model-parallel-size: 1\n --expert-model-parallel-size: 4\n --expert-tensor-parallel-size: 1\n --sequence-parallel: true\n --use-mcore-models: true\n --moe-token-dispatcher-type: alltoall\n --moe-grouped-gemm: true\n --num-experts: 64\n --moe-router-topk: 6\n --moe-z-loss-coeff: 0\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-aux-loss-coeff: 1e-3\n --moe-router-score-function: sigmoid\n --untie-embeddings-and-output-weights: true\n --disable-bias-linear: true\n --init-method-std: 0.014\n --position-embedding-type: rope\n --rotary-base: 1000000\n --rotary-percent: 1.0\n --num-layers: 27\n --hidden-size: 2048\n --moe-ffn-hidden-size: 1408\n --moe-shared-expert-intermediate-size: 2816\n --ffn-hidden-size: 10944\n --num-attention-heads: 16\n --kv-channels: 128\n --normalization: RMSNorm\n --swiglu: true\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --seq-length: 4096\n --max-position-embeddings: 4096\n --micro-batch-size: 1\n --ckpt-format: torch_dist\n --ckpt-fully-parallel-save: true\n --ckpt-fully-parallel-load: true\n --ckpt-assume-constant-structure: true\n --dist-ckpt-strictness: log_unexpected\n --bf16: true\n --attention-backend: flash\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 8\n --use-checkpoint-args: true\n --no-use-tokenizer-model-from-checkpoint-args: true\n --no-load-optim: true\n --deterministic-mode: true # moe will use different ops for determinism for inference\n --save-interval: 2000\n --temperature: 1.0\n --top_k: 1\n --return-log-probs: true\n --num-tokens-to-generate: 30\n --max-tokens-to-oom: 3600000\n --inference-max-seq-length: 4096\n --output-path: ${TENSORBOARD_PATH}\n --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n --incoming-requests-per-sec: -1 # all requests arrive up front.\n --inference-repeat-n: 8\nMETRICS:\n - \"generated_tokens\"\n - \"logprobs\"\n"}}, "model/tests/functional_tests/test_cases/gpt/gpt_static_inference_tp1_pp1_16b_multiprompt_tokensmatch/model_config.yaml": {"f8a3bdaf5590a19a2917d09e44a4b2acb0b15d16": {"t": 1785746856.8219872, "n": 2835, "h": "692e4836a6f2a74f", "c": "ENV_VARS:\n CUDA_DEVICE_MAX_CONNECTIONS: 1\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n NCCL_ALGO: Ring\n CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n --log-num-zeros-in-grad: true\n --log-validation-ppl-to-tensorboard: true\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --timing-log-level: 0\n # See the mount paths defined in the top level tests/test_utils/recipes/gpt-static-inference.yaml\n --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n --tokenizer-type: TikTokenizer\n --tiktoken-pattern: v2\n --distributed-backend: nccl\n --log-interval: 1\n --transformer-impl: transformer_engine\n --tensor-model-parallel-size: 1\n --pipeline-model-parallel-size: 1\n --expert-model-parallel-size: 1\n --use-mcore-models: true\n --moe-token-dispatcher-type: alltoall\n --moe-grouped-gemm: true\n --num-experts: 64\n --moe-router-topk: 6\n --moe-z-loss-coeff: 0\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-aux-loss-coeff: 1e-3\n --moe-router-score-function: sigmoid\n --untie-embeddings-and-output-weights: true\n --disable-bias-linear: true\n --init-method-std: 0.014\n --position-embedding-type: rope\n --rotary-base: 1000000\n --rotary-percent: 1.0\n --num-layers: 27\n --hidden-size: 2048\n --moe-ffn-hidden-size: 1408\n --moe-shared-expert-intermediate-size: 2816\n --ffn-hidden-size: 10944\n --num-attention-heads: 16\n --kv-channels: 128\n --normalization: RMSNorm\n --swiglu: true\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --seq-length: 4096\n --max-position-embeddings: 4096\n # Make sure the batch size is 1 so we test consecutive calls to inference covering state cleanup:\n --micro-batch-size: 1\n --inference-max-requests: 1\n --ckpt-format: torch_dist\n --dist-ckpt-optim-fully-reshardable: true\n --ckpt-fully-parallel-save: true\n --ckpt-fully-parallel-load: true\n --ckpt-assume-constant-structure: true\n --dist-ckpt-strictness: log_unexpected\n --bf16: true\n --attention-backend: flash\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 8\n --flash-decode: true\n --use-checkpoint-args: true\n --no-use-tokenizer-model-from-checkpoint-args: true\n --no-load-optim: true\n --deterministic-mode: true\n --save-interval: 2000\n --temperature: 1.0\n --top_k: 1\n --return-log-probs: true\n --num-tokens-to-generate: 80\n --max-tokens-to-oom: 3600000\n --inference-max-seq-length: 4096\n --output-path: ${TENSORBOARD_PATH}\n --prompt-file: ./tests/functional_tests/test_cases/gpt/gpt_static_inference_tp1_pp1_16b_multiprompt_tokensmatch/test_prompts.jsonl\n --incoming-requests-per-sec: -1 # all requests arrive up front.\nMETRICS:\n - \"generated_text\"\n"}, "931b275029e26e093fa6b33ca83a9ad363b85d31": {"t": 1785767889.3362784, "n": 2835, "h": "692e4836a6f2a74f", "c": "ENV_VARS:\n CUDA_DEVICE_MAX_CONNECTIONS: 1\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n NCCL_ALGO: Ring\n CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n --log-num-zeros-in-grad: true\n --log-validation-ppl-to-tensorboard: true\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --timing-log-level: 0\n # See the mount paths defined in the top level tests/test_utils/recipes/gpt-static-inference.yaml\n --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n --tokenizer-type: TikTokenizer\n --tiktoken-pattern: v2\n --distributed-backend: nccl\n --log-interval: 1\n --transformer-impl: transformer_engine\n --tensor-model-parallel-size: 1\n --pipeline-model-parallel-size: 1\n --expert-model-parallel-size: 1\n --use-mcore-models: true\n --moe-token-dispatcher-type: alltoall\n --moe-grouped-gemm: true\n --num-experts: 64\n --moe-router-topk: 6\n --moe-z-loss-coeff: 0\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-aux-loss-coeff: 1e-3\n --moe-router-score-function: sigmoid\n --untie-embeddings-and-output-weights: true\n --disable-bias-linear: true\n --init-method-std: 0.014\n --position-embedding-type: rope\n --rotary-base: 1000000\n --rotary-percent: 1.0\n --num-layers: 27\n --hidden-size: 2048\n --moe-ffn-hidden-size: 1408\n --moe-shared-expert-intermediate-size: 2816\n --ffn-hidden-size: 10944\n --num-attention-heads: 16\n --kv-channels: 128\n --normalization: RMSNorm\n --swiglu: true\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --seq-length: 4096\n --max-position-embeddings: 4096\n # Make sure the batch size is 1 so we test consecutive calls to inference covering state cleanup:\n --micro-batch-size: 1\n --inference-max-requests: 1\n --ckpt-format: torch_dist\n --dist-ckpt-optim-fully-reshardable: true\n --ckpt-fully-parallel-save: true\n --ckpt-fully-parallel-load: true\n --ckpt-assume-constant-structure: true\n --dist-ckpt-strictness: log_unexpected\n --bf16: true\n --attention-backend: flash\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 8\n --flash-decode: true\n --use-checkpoint-args: true\n --no-use-tokenizer-model-from-checkpoint-args: true\n --no-load-optim: true\n --deterministic-mode: true\n --save-interval: 2000\n --temperature: 1.0\n --top_k: 1\n --return-log-probs: true\n --num-tokens-to-generate: 80\n --max-tokens-to-oom: 3600000\n --inference-max-seq-length: 4096\n --output-path: ${TENSORBOARD_PATH}\n --prompt-file: ./tests/functional_tests/test_cases/gpt/gpt_static_inference_tp1_pp1_16b_multiprompt_tokensmatch/test_prompts.jsonl\n --incoming-requests-per-sec: -1 # all requests arrive up front.\nMETRICS:\n - \"generated_text\"\n"}}}} |