{"repo": "anonymous452026/ngpt-nvfp4", "files": {"Megatron-LM/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_pp1_ep4_16B_logitsmatch/model_config.yaml": {"6454f40bb73ecdddf9b414a80cff2219ed702298": {"t": 1785746826.0881436, "n": 2988, "h": "0256650a3db65f5d", "c": "ENV_VARS:\n  CUDA_DEVICE_MAX_CONNECTIONS: 1\n  NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n  NCCL_ALGO: Ring\n  CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n  --log-num-zeros-in-grad: true\n  --log-validation-ppl-to-tensorboard: true\n  --log-timers-to-tensorboard: true\n  --log-memory-to-tensorboard: true\n  --timing-log-level: 0\n  --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n  --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n  --tokenizer-type: TikTokenizer\n  --tiktoken-pattern: v2\n  --distributed-backend: nccl\n  --log-interval: 1\n  --transformer-impl: transformer_engine\n  --tensor-model-parallel-size: 4\n  --pipeline-model-parallel-size: 1\n  --expert-model-parallel-size: 4\n  --expert-tensor-parallel-size: 1\n  --sequence-parallel: true\n  --use-mcore-models: true\n  --moe-token-dispatcher-type: alltoall\n  --moe-grouped-gemm: true\n  --num-experts: 64\n  --moe-router-topk: 6\n  --moe-z-loss-coeff: 0\n  --moe-router-load-balancing-type: seq_aux_loss\n  --moe-aux-loss-coeff: 1e-3\n  --moe-router-score-function: sigmoid\n  --untie-embeddings-and-output-weights: true\n  --disable-bias-linear: true\n  --init-method-std: 0.014\n  --position-embedding-type: rope\n  --rotary-base: 1000000\n  --rotary-percent: 1.0\n  --num-layers: 27\n  --hidden-size: 2048\n  --moe-ffn-hidden-size: 1408\n  --moe-shared-expert-intermediate-size: 2816\n  --ffn-hidden-size: 10944\n  --num-attention-heads: 16\n  --kv-channels: 128\n  --normalization: RMSNorm\n  --swiglu: true\n  --attention-dropout: 0.0\n  --hidden-dropout: 0.0\n  --seq-length: 4096\n  --max-position-embeddings: 4096\n  --micro-batch-size: 1\n  --ckpt-format: torch_dist\n  --ckpt-fully-parallel-save: true\n  --ckpt-fully-parallel-load: true\n  --ckpt-assume-constant-structure: true\n  --dist-ckpt-strictness: log_unexpected\n  --bf16: true\n  --attention-backend: flash\n  --no-create-attention-mask-in-dataloader: true\n  --num-workers: 8\n  --use-checkpoint-args: true\n  --no-use-tokenizer-model-from-checkpoint-args: true\n  --no-load-optim: true\n  --deterministic-mode: true # moe will use different ops for determinism for inference\n  --save-interval: 2000\n  --temperature: 1.0\n  --top_k: 1\n  --return-log-probs: true\n  --num-tokens-to-generate: 30\n  --max-tokens-to-oom: 3600000\n  --inference-max-seq-length: 4096\n  --output-path: ${TENSORBOARD_PATH}\n  --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n  --incoming-requests-per-sec: -1 # all requests arrive up front.\n  --inference-repeat-n: 8\n  --inference-dynamic-batching-buffer-size-gb: 20\nMETRICS:\n  - \"generated_tokens\"\n  - \"logprobs\"\n"}, "0bba0036e8375d6afea0a5067e698caf5b8a2549": {"t": 1785767913.1415098, "n": 2988, "h": "0256650a3db65f5d", "c": "ENV_VARS:\n  CUDA_DEVICE_MAX_CONNECTIONS: 1\n  NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n  NCCL_ALGO: Ring\n  CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n  --log-num-zeros-in-grad: true\n  --log-validation-ppl-to-tensorboard: true\n  --log-timers-to-tensorboard: true\n  --log-memory-to-tensorboard: true\n  --timing-log-level: 0\n  --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n  --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n  --tokenizer-type: TikTokenizer\n  --tiktoken-pattern: v2\n  --distributed-backend: nccl\n  --log-interval: 1\n  --transformer-impl: transformer_engine\n  --tensor-model-parallel-size: 4\n  --pipeline-model-parallel-size: 1\n  --expert-model-parallel-size: 4\n  --expert-tensor-parallel-size: 1\n  --sequence-parallel: true\n  --use-mcore-models: true\n  --moe-token-dispatcher-type: alltoall\n  --moe-grouped-gemm: true\n  --num-experts: 64\n  --moe-router-topk: 6\n  --moe-z-loss-coeff: 0\n  --moe-router-load-balancing-type: seq_aux_loss\n  --moe-aux-loss-coeff: 1e-3\n  --moe-router-score-function: sigmoid\n  --untie-embeddings-and-output-weights: true\n  --disable-bias-linear: true\n  --init-method-std: 0.014\n  --position-embedding-type: rope\n  --rotary-base: 1000000\n  --rotary-percent: 1.0\n  --num-layers: 27\n  --hidden-size: 2048\n  --moe-ffn-hidden-size: 1408\n  --moe-shared-expert-intermediate-size: 2816\n  --ffn-hidden-size: 10944\n  --num-attention-heads: 16\n  --kv-channels: 128\n  --normalization: RMSNorm\n  --swiglu: true\n  --attention-dropout: 0.0\n  --hidden-dropout: 0.0\n  --seq-length: 4096\n  --max-position-embeddings: 4096\n  --micro-batch-size: 1\n  --ckpt-format: torch_dist\n  --ckpt-fully-parallel-save: true\n  --ckpt-fully-parallel-load: true\n  --ckpt-assume-constant-structure: true\n  --dist-ckpt-strictness: log_unexpected\n  --bf16: true\n  --attention-backend: flash\n  --no-create-attention-mask-in-dataloader: true\n  --num-workers: 8\n  --use-checkpoint-args: true\n  --no-use-tokenizer-model-from-checkpoint-args: true\n  --no-load-optim: true\n  --deterministic-mode: true # moe will use different ops for determinism for inference\n  --save-interval: 2000\n  --temperature: 1.0\n  --top_k: 1\n  --return-log-probs: true\n  --num-tokens-to-generate: 30\n  --max-tokens-to-oom: 3600000\n  --inference-max-seq-length: 4096\n  --output-path: ${TENSORBOARD_PATH}\n  --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n  --incoming-requests-per-sec: -1 # all requests arrive up front.\n  --inference-repeat-n: 8\n  --inference-dynamic-batching-buffer-size-gb: 20\nMETRICS:\n  - \"generated_tokens\"\n  - \"logprobs\"\n"}}, "Megatron-LM/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_cuda_graphs_pad_tp4_pp1_ep4_16B_logitsmatch/model_config.yaml": {"6454f40bb73ecdddf9b414a80cff2219ed702298": {"t": 1785746837.99128, "n": 3131, "h": "ad783b463c41c568", "c": "ENV_VARS:\n  CUDA_DEVICE_MAX_CONNECTIONS: 1\n  NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n  NCCL_ALGO: Ring\n  CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n  --log-num-zeros-in-grad: true\n  --log-validation-ppl-to-tensorboard: true\n  --log-timers-to-tensorboard: true\n  --log-memory-to-tensorboard: true\n  --timing-log-level: 0\n  --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n  --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n  --tokenizer-type: TikTokenizer\n  --tiktoken-pattern: v2\n  --distributed-backend: nccl\n  --log-interval: 1\n  --transformer-impl: transformer_engine\n  --tensor-model-parallel-size: 4\n  --pipeline-model-parallel-size: 1\n  --expert-model-parallel-size: 4\n  --expert-tensor-parallel-size: 1\n  --sequence-parallel: true\n  --use-mcore-models: true\n  --moe-token-dispatcher-type: alltoall\n  --moe-grouped-gemm: true\n  --num-experts: 64\n  --moe-router-topk: 6\n  --moe-z-loss-coeff: 0\n  --moe-router-load-balancing-type: seq_aux_loss\n  --moe-aux-loss-coeff: 1e-3\n  --moe-router-score-function: sigmoid\n  --untie-embeddings-and-output-weights: true\n  --disable-bias-linear: true\n  --init-method-std: 0.014\n  --position-embedding-type: rope\n  --rotary-base: 1000000\n  --rotary-percent: 1.0\n  --num-layers: 27\n  --hidden-size: 2048\n  --moe-ffn-hidden-size: 1408\n  --moe-shared-expert-intermediate-size: 2816\n  --ffn-hidden-size: 10944\n  --num-attention-heads: 16\n  --kv-channels: 128\n  --normalization: RMSNorm\n  --swiglu: true\n  --attention-dropout: 0.0\n  --hidden-dropout: 0.0\n  --seq-length: 4096\n  --max-position-embeddings: 4096\n  --micro-batch-size: 1\n  --enable-cuda-graph: true\n  --te-rng-tracker: true\n  --inference-rng-tracker: true\n  --moe-pad-experts-for-cuda-graph-inference: true\n  --ckpt-format: torch_dist\n  --ckpt-fully-parallel-save: true\n  --ckpt-fully-parallel-load: true\n  --ckpt-assume-constant-structure: true\n  --dist-ckpt-strictness: log_unexpected\n  --bf16: true\n  --attention-backend: flash\n  --no-create-attention-mask-in-dataloader: true\n  --num-workers: 8\n  --use-checkpoint-args: true\n  --no-use-tokenizer-model-from-checkpoint-args: true\n  --no-load-optim: true\n  --deterministic-mode: true\n  --save-interval: 2000\n  --temperature: 1.0\n  --top_k: 1\n  --return-log-probs: true\n  --num-tokens-to-generate: 30\n  --max-tokens-to-oom: 3600000\n  --inference-max-seq-length: 4096\n  --output-path: ${TENSORBOARD_PATH}\n  --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n  --incoming-requests-per-sec: -1\n  --inference-repeat-n: 8\n  --inference-dynamic-batching-buffer-size-gb: 20\n  --inference-dynamic-batching-num-cuda-graphs: 1 \n  --inference-dynamic-batching-max-requests: 8\nMETRICS:\n  - \"generated_tokens\"\n  - \"logprobs\"\n"}, "0bba0036e8375d6afea0a5067e698caf5b8a2549": {"t": 1785767913.0826645, "n": 3131, "h": "ad783b463c41c568", "c": "ENV_VARS:\n  CUDA_DEVICE_MAX_CONNECTIONS: 1\n  NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n  NCCL_ALGO: Ring\n  CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n  --log-num-zeros-in-grad: true\n  --log-validation-ppl-to-tensorboard: true\n  --log-timers-to-tensorboard: true\n  --log-memory-to-tensorboard: true\n  --timing-log-level: 0\n  --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n  --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n  --tokenizer-type: TikTokenizer\n  --tiktoken-pattern: v2\n  --distributed-backend: nccl\n  --log-interval: 1\n  --transformer-impl: transformer_engine\n  --tensor-model-parallel-size: 4\n  --pipeline-model-parallel-size: 1\n  --expert-model-parallel-size: 4\n  --expert-tensor-parallel-size: 1\n  --sequence-parallel: true\n  --use-mcore-models: true\n  --moe-token-dispatcher-type: alltoall\n  --moe-grouped-gemm: true\n  --num-experts: 64\n  --moe-router-topk: 6\n  --moe-z-loss-coeff: 0\n  --moe-router-load-balancing-type: seq_aux_loss\n  --moe-aux-loss-coeff: 1e-3\n  --moe-router-score-function: sigmoid\n  --untie-embeddings-and-output-weights: true\n  --disable-bias-linear: true\n  --init-method-std: 0.014\n  --position-embedding-type: rope\n  --rotary-base: 1000000\n  --rotary-percent: 1.0\n  --num-layers: 27\n  --hidden-size: 2048\n  --moe-ffn-hidden-size: 1408\n  --moe-shared-expert-intermediate-size: 2816\n  --ffn-hidden-size: 10944\n  --num-attention-heads: 16\n  --kv-channels: 128\n  --normalization: RMSNorm\n  --swiglu: true\n  --attention-dropout: 0.0\n  --hidden-dropout: 0.0\n  --seq-length: 4096\n  --max-position-embeddings: 4096\n  --micro-batch-size: 1\n  --enable-cuda-graph: true\n  --te-rng-tracker: true\n  --inference-rng-tracker: true\n  --moe-pad-experts-for-cuda-graph-inference: true\n  --ckpt-format: torch_dist\n  --ckpt-fully-parallel-save: true\n  --ckpt-fully-parallel-load: true\n  --ckpt-assume-constant-structure: true\n  --dist-ckpt-strictness: log_unexpected\n  --bf16: true\n  --attention-backend: flash\n  --no-create-attention-mask-in-dataloader: true\n  --num-workers: 8\n  --use-checkpoint-args: true\n  --no-use-tokenizer-model-from-checkpoint-args: true\n  --no-load-optim: true\n  --deterministic-mode: true\n  --save-interval: 2000\n  --temperature: 1.0\n  --top_k: 1\n  --return-log-probs: true\n  --num-tokens-to-generate: 30\n  --max-tokens-to-oom: 3600000\n  --inference-max-seq-length: 4096\n  --output-path: ${TENSORBOARD_PATH}\n  --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n  --incoming-requests-per-sec: -1\n  --inference-repeat-n: 8\n  --inference-dynamic-batching-buffer-size-gb: 20\n  --inference-dynamic-batching-num-cuda-graphs: 1 \n  --inference-dynamic-batching-max-requests: 8\nMETRICS:\n  - \"generated_tokens\"\n  - \"logprobs\"\n"}}, "Megatron-LM/tests/functional_tests/test_cases/moe/gpt_static_inference_cuda_graphs_pad_tp4_pp1_ep4_16B_logitsmatch/model_config.yaml": {"6454f40bb73ecdddf9b414a80cff2219ed702298": {"t": 1785746848.8241973, "n": 3010, "h": "cae46104d1655d34", "c": "ENV_VARS:\n  CUDA_DEVICE_MAX_CONNECTIONS: 1\n  NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n  NCCL_ALGO: Ring\n  CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n  --log-num-zeros-in-grad: true\n  --log-validation-ppl-to-tensorboard: true\n  --log-timers-to-tensorboard: true\n  --log-memory-to-tensorboard: true\n  --timing-log-level: 0\n  --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n  --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n  --tokenizer-type: TikTokenizer\n  --tiktoken-pattern: v2\n  --distributed-backend: nccl\n  --log-interval: 1\n  --transformer-impl: transformer_engine\n  --tensor-model-parallel-size: 4\n  --pipeline-model-parallel-size: 1\n  --expert-model-parallel-size: 4\n  --expert-tensor-parallel-size: 1\n  --sequence-parallel: true\n  --use-mcore-models: true\n  --moe-token-dispatcher-type: alltoall\n  --moe-grouped-gemm: true\n  --num-experts: 64\n  --moe-router-topk: 6\n  --moe-z-loss-coeff: 0\n  --moe-aux-loss-coeff: 1e-3\n  --moe-router-load-balancing-type: seq_aux_loss\n  --moe-router-score-function: sigmoid\n  --untie-embeddings-and-output-weights: true\n  --disable-bias-linear: true\n  --init-method-std: 0.014\n  --position-embedding-type: rope\n  --rotary-base: 1000000\n  --rotary-percent: 1.0\n  --num-layers: 27\n  --hidden-size: 2048\n  --moe-ffn-hidden-size: 1408\n  --moe-shared-expert-intermediate-size: 2816\n  --ffn-hidden-size: 10944\n  --num-attention-heads: 16\n  --kv-channels: 128\n  --normalization: RMSNorm\n  --swiglu: true\n  --attention-dropout: 0.0\n  --hidden-dropout: 0.0\n  --seq-length: 4096\n  --max-position-embeddings: 4096\n  --micro-batch-size: 1\n  --flash-decode: true\n  --enable-cuda-graph: true\n  --te-rng-tracker: true\n  --inference-rng-tracker: true\n  --moe-pad-experts-for-cuda-graph-inference: true\n  --ckpt-format: torch_dist\n  --ckpt-fully-parallel-save: true\n  --ckpt-fully-parallel-load: true\n  --ckpt-assume-constant-structure: true\n  --dist-ckpt-strictness: log_unexpected\n  --bf16: true\n  --attention-backend: flash\n  --no-create-attention-mask-in-dataloader: true\n  --num-workers: 8\n  --use-checkpoint-args: true\n  --no-use-tokenizer-model-from-checkpoint-args: true\n  --no-load-optim: true\n  --deterministic-mode: true\n  --save-interval: 2000\n  --temperature: 1.0\n  --top_k: 1\n  --return-log-probs: true\n  --num-tokens-to-generate: 30\n  --max-tokens-to-oom: 3600000\n  --inference-max-seq-length: 4096\n  --inference-max-requests: 1\n  --output-path: ${TENSORBOARD_PATH}\n  --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n  --incoming-requests-per-sec: -1\nMETRICS:\n  - \"generated_tokens\"\n  - \"logprobs\"\n"}, "0bba0036e8375d6afea0a5067e698caf5b8a2549": {"t": 1785767915.6021495, "n": 3010, "h": "cae46104d1655d34", "c": "ENV_VARS:\n  CUDA_DEVICE_MAX_CONNECTIONS: 1\n  NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n  NCCL_ALGO: Ring\n  CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n  --log-num-zeros-in-grad: true\n  --log-validation-ppl-to-tensorboard: true\n  --log-timers-to-tensorboard: true\n  --log-memory-to-tensorboard: true\n  --timing-log-level: 0\n  --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n  --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n  --tokenizer-type: TikTokenizer\n  --tiktoken-pattern: v2\n  --distributed-backend: nccl\n  --log-interval: 1\n  --transformer-impl: transformer_engine\n  --tensor-model-parallel-size: 4\n  --pipeline-model-parallel-size: 1\n  --expert-model-parallel-size: 4\n  --expert-tensor-parallel-size: 1\n  --sequence-parallel: true\n  --use-mcore-models: true\n  --moe-token-dispatcher-type: alltoall\n  --moe-grouped-gemm: true\n  --num-experts: 64\n  --moe-router-topk: 6\n  --moe-z-loss-coeff: 0\n  --moe-aux-loss-coeff: 1e-3\n  --moe-router-load-balancing-type: seq_aux_loss\n  --moe-router-score-function: sigmoid\n  --untie-embeddings-and-output-weights: true\n  --disable-bias-linear: true\n  --init-method-std: 0.014\n  --position-embedding-type: rope\n  --rotary-base: 1000000\n  --rotary-percent: 1.0\n  --num-layers: 27\n  --hidden-size: 2048\n  --moe-ffn-hidden-size: 1408\n  --moe-shared-expert-intermediate-size: 2816\n  --ffn-hidden-size: 10944\n  --num-attention-heads: 16\n  --kv-channels: 128\n  --normalization: RMSNorm\n  --swiglu: true\n  --attention-dropout: 0.0\n  --hidden-dropout: 0.0\n  --seq-length: 4096\n  --max-position-embeddings: 4096\n  --micro-batch-size: 1\n  --flash-decode: true\n  --enable-cuda-graph: true\n  --te-rng-tracker: true\n  --inference-rng-tracker: true\n  --moe-pad-experts-for-cuda-graph-inference: true\n  --ckpt-format: torch_dist\n  --ckpt-fully-parallel-save: true\n  --ckpt-fully-parallel-load: true\n  --ckpt-assume-constant-structure: true\n  --dist-ckpt-strictness: log_unexpected\n  --bf16: true\n  --attention-backend: flash\n  --no-create-attention-mask-in-dataloader: true\n  --num-workers: 8\n  --use-checkpoint-args: true\n  --no-use-tokenizer-model-from-checkpoint-args: true\n  --no-load-optim: true\n  --deterministic-mode: true\n  --save-interval: 2000\n  --temperature: 1.0\n  --top_k: 1\n  --return-log-probs: true\n  --num-tokens-to-generate: 30\n  --max-tokens-to-oom: 3600000\n  --inference-max-seq-length: 4096\n  --inference-max-requests: 1\n  --output-path: ${TENSORBOARD_PATH}\n  --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n  --incoming-requests-per-sec: -1\nMETRICS:\n  - \"generated_tokens\"\n  - \"logprobs\"\n"}}, "Megatron-LM/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/model_config.yaml": {"6454f40bb73ecdddf9b414a80cff2219ed702298": {"t": 1785746853.1022198, "n": 3285, "h": "92400b8c12ae6175", "c": "ENV_VARS:\n  CUDA_DEVICE_MAX_CONNECTIONS: 1\n  NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n  NCCL_ALGO: Ring\n  CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n  --log-num-zeros-in-grad: true\n  --log-validation-ppl-to-tensorboard: true\n  --log-timers-to-tensorboard: true\n  --log-memory-to-tensorboard: true\n  --timing-log-level: 0\n  --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n  --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n  --tokenizer-type: TikTokenizer\n  --tiktoken-pattern: v2\n  --distributed-backend: nccl\n  --log-interval: 1\n  --transformer-impl: transformer_engine\n  --tensor-model-parallel-size: 4\n  --pipeline-model-parallel-size: 1\n  --expert-model-parallel-size: 8\n  --expert-tensor-parallel-size: 1\n  --sequence-parallel: true\n  --use-mcore-models: true\n  --moe-token-dispatcher-type: alltoall\n  --moe-grouped-gemm: true\n  --num-experts: 64\n  --moe-router-topk: 6\n  --moe-z-loss-coeff: 0\n  --moe-router-load-balancing-type: seq_aux_loss\n  --moe-aux-loss-coeff: 1e-3\n  --moe-router-score-function: sigmoid\n  --untie-embeddings-and-output-weights: true\n  --disable-bias-linear: true\n  --init-method-std: 0.014\n  --position-embedding-type: rope\n  --rotary-base: 1000000\n  --rotary-percent: 1.0\n  --num-layers: 27\n  --hidden-size: 2048\n  --moe-ffn-hidden-size: 1408\n  --moe-shared-expert-intermediate-size: 2816\n  --ffn-hidden-size: 10944\n  --num-attention-heads: 16\n  --kv-channels: 128\n  --normalization: RMSNorm\n  --swiglu: true\n  --attention-dropout: 0.0\n  --hidden-dropout: 0.0\n  --seq-length: 4096\n  --max-position-embeddings: 4096\n  --micro-batch-size: 1\n  --ckpt-format: torch_dist\n  --ckpt-fully-parallel-save: true\n  --ckpt-fully-parallel-load: true\n  --ckpt-assume-constant-structure: true\n  --dist-ckpt-strictness: log_unexpected\n  --bf16: true\n  --attention-backend: flash\n  --no-create-attention-mask-in-dataloader: true\n  --num-workers: 8\n  --use-checkpoint-args: true\n  --no-use-tokenizer-model-from-checkpoint-args: true\n  --no-load-optim: true\n  --deterministic-mode: true # moe will use different ops for determinism for inference\n  --save-interval: 2000\n  --temperature: 1.0\n  --top_k: 1\n  --return-log-probs: true\n  --num-tokens-to-generate: 30\n  --max-tokens-to-oom: 3600000\n  --inference-max-seq-length: 4096\n  --output-path: ${TENSORBOARD_PATH}\n  --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n  --incoming-requests-per-sec: -1 # all requests arrive up front.\n  --inference-repeat-n: 8\n  --inference-dynamic-batching-buffer-size-gb: 20\n  --cuda-graph-impl: local \n  --moe-pad-experts-for-cuda-graph-inference: true\n  --inference-dynamic-batching-buffer-size-gb: 20\n  --inference-dynamic-batching-num-cuda-graphs: 1 \n  --inference-dynamic-batching-max-requests: 512\n  --inference-logging-step-interval: 1\n  --sequence-parallel: true\n\nMETRICS:\n  - \"generated_tokens\"\n  - \"logprobs\"\n"}, "0bba0036e8375d6afea0a5067e698caf5b8a2549": {"t": 1785767915.1181734, "n": 3285, "h": "92400b8c12ae6175", "c": "ENV_VARS:\n  CUDA_DEVICE_MAX_CONNECTIONS: 1\n  NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n  NCCL_ALGO: Ring\n  CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n  --log-num-zeros-in-grad: true\n  --log-validation-ppl-to-tensorboard: true\n  --log-timers-to-tensorboard: true\n  --log-memory-to-tensorboard: true\n  --timing-log-level: 0\n  --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n  --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n  --tokenizer-type: TikTokenizer\n  --tiktoken-pattern: v2\n  --distributed-backend: nccl\n  --log-interval: 1\n  --transformer-impl: transformer_engine\n  --tensor-model-parallel-size: 4\n  --pipeline-model-parallel-size: 1\n  --expert-model-parallel-size: 8\n  --expert-tensor-parallel-size: 1\n  --sequence-parallel: true\n  --use-mcore-models: true\n  --moe-token-dispatcher-type: alltoall\n  --moe-grouped-gemm: true\n  --num-experts: 64\n  --moe-router-topk: 6\n  --moe-z-loss-coeff: 0\n  --moe-router-load-balancing-type: seq_aux_loss\n  --moe-aux-loss-coeff: 1e-3\n  --moe-router-score-function: sigmoid\n  --untie-embeddings-and-output-weights: true\n  --disable-bias-linear: true\n  --init-method-std: 0.014\n  --position-embedding-type: rope\n  --rotary-base: 1000000\n  --rotary-percent: 1.0\n  --num-layers: 27\n  --hidden-size: 2048\n  --moe-ffn-hidden-size: 1408\n  --moe-shared-expert-intermediate-size: 2816\n  --ffn-hidden-size: 10944\n  --num-attention-heads: 16\n  --kv-channels: 128\n  --normalization: RMSNorm\n  --swiglu: true\n  --attention-dropout: 0.0\n  --hidden-dropout: 0.0\n  --seq-length: 4096\n  --max-position-embeddings: 4096\n  --micro-batch-size: 1\n  --ckpt-format: torch_dist\n  --ckpt-fully-parallel-save: true\n  --ckpt-fully-parallel-load: true\n  --ckpt-assume-constant-structure: true\n  --dist-ckpt-strictness: log_unexpected\n  --bf16: true\n  --attention-backend: flash\n  --no-create-attention-mask-in-dataloader: true\n  --num-workers: 8\n  --use-checkpoint-args: true\n  --no-use-tokenizer-model-from-checkpoint-args: true\n  --no-load-optim: true\n  --deterministic-mode: true # moe will use different ops for determinism for inference\n  --save-interval: 2000\n  --temperature: 1.0\n  --top_k: 1\n  --return-log-probs: true\n  --num-tokens-to-generate: 30\n  --max-tokens-to-oom: 3600000\n  --inference-max-seq-length: 4096\n  --output-path: ${TENSORBOARD_PATH}\n  --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n  --incoming-requests-per-sec: -1 # all requests arrive up front.\n  --inference-repeat-n: 8\n  --inference-dynamic-batching-buffer-size-gb: 20\n  --cuda-graph-impl: local \n  --moe-pad-experts-for-cuda-graph-inference: true\n  --inference-dynamic-batching-buffer-size-gb: 20\n  --inference-dynamic-batching-num-cuda-graphs: 1 \n  --inference-dynamic-batching-max-requests: 512\n  --inference-logging-step-interval: 1\n  --sequence-parallel: true\n\nMETRICS:\n  - \"generated_tokens\"\n  - \"logprobs\"\n"}}, "Megatron-LM/tests/functional_tests/test_cases/moe/gpt_static_inference_tp1_pp1_ep1_16B_logitsmatch/model_config.yaml": {"6454f40bb73ecdddf9b414a80cff2219ed702298": {"t": 1785746856.5890167, "n": 2863, "h": "eb338b982701386c", "c": "ENV_VARS:\n  CUDA_DEVICE_MAX_CONNECTIONS: 1\n  NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n  NCCL_ALGO: Ring\n  CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n  --log-num-zeros-in-grad: true\n  --log-validation-ppl-to-tensorboard: true\n  --log-timers-to-tensorboard: true\n  --log-memory-to-tensorboard: true\n  --timing-log-level: 0\n  --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n  --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n  --tokenizer-type: TikTokenizer\n  --tiktoken-pattern: v2\n  --distributed-backend: nccl\n  --log-interval: 1\n  --transformer-impl: transformer_engine\n  --tensor-model-parallel-size: 1\n  --pipeline-model-parallel-size: 1\n  --expert-model-parallel-size: 1\n  --use-mcore-models: true\n  --moe-token-dispatcher-type: alltoall\n  --moe-grouped-gemm: true\n  --num-experts: 64\n  --moe-router-topk: 6\n  --moe-z-loss-coeff: 0\n  --moe-router-load-balancing-type: seq_aux_loss\n  --moe-aux-loss-coeff: 1e-3\n  --moe-router-score-function: sigmoid\n  --untie-embeddings-and-output-weights: true\n  --disable-bias-linear: true\n  --init-method-std: 0.014\n  --position-embedding-type: rope\n  --rotary-base: 1000000\n  --rotary-percent: 1.0\n  --num-layers: 27\n  --hidden-size: 2048\n  --moe-ffn-hidden-size: 1408\n  --moe-shared-expert-intermediate-size: 2816\n  --ffn-hidden-size: 10944\n  --num-attention-heads: 16\n  --kv-channels: 128\n  --normalization: RMSNorm\n  --swiglu: true\n  --attention-dropout: 0.0\n  --hidden-dropout: 0.0\n  --seq-length: 4096\n  --max-position-embeddings: 4096\n  --micro-batch-size: 1\n  --ckpt-format: torch_dist\n  --ckpt-fully-parallel-save: true\n  --ckpt-fully-parallel-load: true\n  --ckpt-assume-constant-structure: true\n  --dist-ckpt-strictness: log_unexpected\n  --bf16: true\n  --attention-backend: flash\n  --no-create-attention-mask-in-dataloader: true\n  --num-workers: 8\n  --flash-decode: true\n  --use-checkpoint-args: true\n  --no-use-tokenizer-model-from-checkpoint-args: true\n  --no-load-optim: true\n  --deterministic-mode: true\n  --save-interval: 2000\n  --temperature: 1.0\n  --top_k: 1\n  --return-log-probs: true\n  --num-tokens-to-generate: 30\n  --max-tokens-to-oom: 3600000\n  --inference-max-seq-length: 4096\n  --output-path: ${TENSORBOARD_PATH}\n  --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n  --incoming-requests-per-sec: -1 # all requests arrive up front.\n  --inference-dynamic-batching-buffer-size-gb: 20\nMETRICS:\n  - \"generated_tokens\"\n  - \"logprobs\"\n"}, "0bba0036e8375d6afea0a5067e698caf5b8a2549": {"t": 1785767914.1643357, "n": 2863, "h": "eb338b982701386c", "c": "ENV_VARS:\n  CUDA_DEVICE_MAX_CONNECTIONS: 1\n  NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n  NCCL_ALGO: Ring\n  CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n  --log-num-zeros-in-grad: true\n  --log-validation-ppl-to-tensorboard: true\n  --log-timers-to-tensorboard: true\n  --log-memory-to-tensorboard: true\n  --timing-log-level: 0\n  --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n  --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n  --tokenizer-type: TikTokenizer\n  --tiktoken-pattern: v2\n  --distributed-backend: nccl\n  --log-interval: 1\n  --transformer-impl: transformer_engine\n  --tensor-model-parallel-size: 1\n  --pipeline-model-parallel-size: 1\n  --expert-model-parallel-size: 1\n  --use-mcore-models: true\n  --moe-token-dispatcher-type: alltoall\n  --moe-grouped-gemm: true\n  --num-experts: 64\n  --moe-router-topk: 6\n  --moe-z-loss-coeff: 0\n  --moe-router-load-balancing-type: seq_aux_loss\n  --moe-aux-loss-coeff: 1e-3\n  --moe-router-score-function: sigmoid\n  --untie-embeddings-and-output-weights: true\n  --disable-bias-linear: true\n  --init-method-std: 0.014\n  --position-embedding-type: rope\n  --rotary-base: 1000000\n  --rotary-percent: 1.0\n  --num-layers: 27\n  --hidden-size: 2048\n  --moe-ffn-hidden-size: 1408\n  --moe-shared-expert-intermediate-size: 2816\n  --ffn-hidden-size: 10944\n  --num-attention-heads: 16\n  --kv-channels: 128\n  --normalization: RMSNorm\n  --swiglu: true\n  --attention-dropout: 0.0\n  --hidden-dropout: 0.0\n  --seq-length: 4096\n  --max-position-embeddings: 4096\n  --micro-batch-size: 1\n  --ckpt-format: torch_dist\n  --ckpt-fully-parallel-save: true\n  --ckpt-fully-parallel-load: true\n  --ckpt-assume-constant-structure: true\n  --dist-ckpt-strictness: log_unexpected\n  --bf16: true\n  --attention-backend: flash\n  --no-create-attention-mask-in-dataloader: true\n  --num-workers: 8\n  --flash-decode: true\n  --use-checkpoint-args: true\n  --no-use-tokenizer-model-from-checkpoint-args: true\n  --no-load-optim: true\n  --deterministic-mode: true\n  --save-interval: 2000\n  --temperature: 1.0\n  --top_k: 1\n  --return-log-probs: true\n  --num-tokens-to-generate: 30\n  --max-tokens-to-oom: 3600000\n  --inference-max-seq-length: 4096\n  --output-path: ${TENSORBOARD_PATH}\n  --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n  --incoming-requests-per-sec: -1 # all requests arrive up front.\n  --inference-dynamic-batching-buffer-size-gb: 20\nMETRICS:\n  - \"generated_tokens\"\n  - \"logprobs\"\n"}}, "Megatron-LM/tests/functional_tests/test_cases/moe/gpt_static_inference_tp4_pp1_ep4_16B_logitsmatch/model_config.yaml": {"6454f40bb73ecdddf9b414a80cff2219ed702298": {"t": 1785746856.7164679, "n": 2876, "h": "155a8806e6916e72", "c": "ENV_VARS:\n  CUDA_DEVICE_MAX_CONNECTIONS: 1\n  NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n  NCCL_ALGO: Ring\n  CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n  --log-num-zeros-in-grad: true\n  --log-validation-ppl-to-tensorboard: true\n  --log-timers-to-tensorboard: true\n  --log-memory-to-tensorboard: true\n  --timing-log-level: 0\n  --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n  --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n  --tokenizer-type: TikTokenizer\n  --tiktoken-pattern: v2\n  --distributed-backend: nccl\n  --log-interval: 1\n  --transformer-impl: transformer_engine\n  --tensor-model-parallel-size: 4\n  --pipeline-model-parallel-size: 1\n  --expert-model-parallel-size: 4\n  --expert-tensor-parallel-size: 1\n  --sequence-parallel: true\n  --use-mcore-models: true\n  --moe-token-dispatcher-type: alltoall\n  --moe-grouped-gemm: true\n  --num-experts: 64\n  --moe-router-topk: 6\n  --moe-z-loss-coeff: 0\n  --moe-router-load-balancing-type: seq_aux_loss\n  --moe-aux-loss-coeff: 1e-3\n  --moe-router-score-function: sigmoid\n  --untie-embeddings-and-output-weights: true\n  --disable-bias-linear: true\n  --init-method-std: 0.014\n  --position-embedding-type: rope\n  --rotary-base: 1000000\n  --rotary-percent: 1.0\n  --num-layers: 27\n  --hidden-size: 2048\n  --moe-ffn-hidden-size: 1408\n  --moe-shared-expert-intermediate-size: 2816\n  --ffn-hidden-size: 10944\n  --num-attention-heads: 16\n  --kv-channels: 128\n  --normalization: RMSNorm\n  --swiglu: true\n  --attention-dropout: 0.0\n  --hidden-dropout: 0.0\n  --seq-length: 4096\n  --max-position-embeddings: 4096\n  --micro-batch-size: 1\n  --ckpt-format: torch_dist\n  --ckpt-fully-parallel-save: true\n  --ckpt-fully-parallel-load: true\n  --ckpt-assume-constant-structure: true\n  --dist-ckpt-strictness: log_unexpected\n  --bf16: true\n  --attention-backend: flash\n  --no-create-attention-mask-in-dataloader: true\n  --num-workers: 8\n  --flash-decode: true\n  --use-checkpoint-args: true\n  --no-use-tokenizer-model-from-checkpoint-args: true\n  --no-load-optim: true\n  --deterministic-mode: true\n  --save-interval: 2000\n  --temperature: 1.0\n  --top_k: 1\n  --return-log-probs: true\n  --num-tokens-to-generate: 30\n  --max-tokens-to-oom: 3600000\n  --inference-max-seq-length: 4096\n  --output-path: ${TENSORBOARD_PATH}\n  --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n  --incoming-requests-per-sec: -1 # all requests arrive up front.\nMETRICS:\n  - \"generated_tokens\"\n  - \"logprobs\"\n"}, "0bba0036e8375d6afea0a5067e698caf5b8a2549": {"t": 1785767915.1260574, "n": 2876, "h": "155a8806e6916e72", "c": "ENV_VARS:\n  CUDA_DEVICE_MAX_CONNECTIONS: 1\n  NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n  NCCL_ALGO: Ring\n  CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n  --log-num-zeros-in-grad: true\n  --log-validation-ppl-to-tensorboard: true\n  --log-timers-to-tensorboard: true\n  --log-memory-to-tensorboard: true\n  --timing-log-level: 0\n  --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n  --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n  --tokenizer-type: TikTokenizer\n  --tiktoken-pattern: v2\n  --distributed-backend: nccl\n  --log-interval: 1\n  --transformer-impl: transformer_engine\n  --tensor-model-parallel-size: 4\n  --pipeline-model-parallel-size: 1\n  --expert-model-parallel-size: 4\n  --expert-tensor-parallel-size: 1\n  --sequence-parallel: true\n  --use-mcore-models: true\n  --moe-token-dispatcher-type: alltoall\n  --moe-grouped-gemm: true\n  --num-experts: 64\n  --moe-router-topk: 6\n  --moe-z-loss-coeff: 0\n  --moe-router-load-balancing-type: seq_aux_loss\n  --moe-aux-loss-coeff: 1e-3\n  --moe-router-score-function: sigmoid\n  --untie-embeddings-and-output-weights: true\n  --disable-bias-linear: true\n  --init-method-std: 0.014\n  --position-embedding-type: rope\n  --rotary-base: 1000000\n  --rotary-percent: 1.0\n  --num-layers: 27\n  --hidden-size: 2048\n  --moe-ffn-hidden-size: 1408\n  --moe-shared-expert-intermediate-size: 2816\n  --ffn-hidden-size: 10944\n  --num-attention-heads: 16\n  --kv-channels: 128\n  --normalization: RMSNorm\n  --swiglu: true\n  --attention-dropout: 0.0\n  --hidden-dropout: 0.0\n  --seq-length: 4096\n  --max-position-embeddings: 4096\n  --micro-batch-size: 1\n  --ckpt-format: torch_dist\n  --ckpt-fully-parallel-save: true\n  --ckpt-fully-parallel-load: true\n  --ckpt-assume-constant-structure: true\n  --dist-ckpt-strictness: log_unexpected\n  --bf16: true\n  --attention-backend: flash\n  --no-create-attention-mask-in-dataloader: true\n  --num-workers: 8\n  --flash-decode: true\n  --use-checkpoint-args: true\n  --no-use-tokenizer-model-from-checkpoint-args: true\n  --no-load-optim: true\n  --deterministic-mode: true\n  --save-interval: 2000\n  --temperature: 1.0\n  --top_k: 1\n  --return-log-probs: true\n  --num-tokens-to-generate: 30\n  --max-tokens-to-oom: 3600000\n  --inference-max-seq-length: 4096\n  --output-path: ${TENSORBOARD_PATH}\n  --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n  --incoming-requests-per-sec: -1 # all requests arrive up front.\nMETRICS:\n  - \"generated_tokens\"\n  - \"logprobs\"\n"}}, "Megatron-LM/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_zmq/model_config.yaml": {"6454f40bb73ecdddf9b414a80cff2219ed702298": {"t": 1785746860.5414164, "n": 3009, "h": "8392c90c686f2029", "c": "ENV_VARS:\n  CUDA_DEVICE_MAX_CONNECTIONS: 1\n  NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n  NCCL_ALGO: Ring\n  CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n  --log-num-zeros-in-grad: true\n  --log-validation-ppl-to-tensorboard: true\n  --log-timers-to-tensorboard: true\n  --log-memory-to-tensorboard: true\n  --timing-log-level: 0\n  --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n  --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n  --tokenizer-type: TikTokenizer\n  --tiktoken-pattern: v2\n  --distributed-backend: nccl\n  --log-interval: 1\n  --transformer-impl: transformer_engine\n  --tensor-model-parallel-size: 4\n  --pipeline-model-parallel-size: 1\n  --expert-model-parallel-size: 8\n  --expert-tensor-parallel-size: 1\n  --sequence-parallel: true\n  --use-mcore-models: true\n  --moe-token-dispatcher-type: alltoall\n  --moe-grouped-gemm: true\n  --num-experts: 64\n  --moe-router-topk: 6\n  --moe-z-loss-coeff: 0\n  --moe-router-load-balancing-type: seq_aux_loss\n  --moe-aux-loss-coeff: 1e-3\n  --moe-router-score-function: sigmoid\n  --untie-embeddings-and-output-weights: true\n  --disable-bias-linear: true\n  --init-method-std: 0.014\n  --position-embedding-type: rope\n  --rotary-base: 1000000\n  --rotary-percent: 1.0\n  --num-layers: 27\n  --hidden-size: 2048\n  --moe-ffn-hidden-size: 1408\n  --moe-shared-expert-intermediate-size: 2816\n  --ffn-hidden-size: 10944\n  --num-attention-heads: 16\n  --kv-channels: 128\n  --normalization: RMSNorm\n  --swiglu: true\n  --attention-dropout: 0.0\n  --hidden-dropout: 0.0\n  --seq-length: 4096\n  --max-position-embeddings: 4096\n  --micro-batch-size: 1\n  --ckpt-format: torch_dist\n  --ckpt-fully-parallel-save: true\n  --ckpt-fully-parallel-load: true\n  --ckpt-assume-constant-structure: true\n  --dist-ckpt-strictness: log_unexpected\n  --bf16: true\n  --attention-backend: flash\n  --no-create-attention-mask-in-dataloader: true\n  --num-workers: 8\n  --use-checkpoint-args: true\n  --no-use-tokenizer-model-from-checkpoint-args: true\n  --no-load-optim: true\n  --deterministic-mode: true # moe will use different ops for determinism for inference\n  --save-interval: 2000\n  --temperature: 1.0\n  --top_k: 1\n  --return-log-probs: true\n  --num-tokens-to-generate: 30\n  --max-tokens-to-oom: 3600000\n  --inference-max-seq-length: 4096\n  --output-path: ${TENSORBOARD_PATH}\n  --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n  --incoming-requests-per-sec: -1 # all requests arrive up front.\n  --inference-repeat-n: 8\n  --inference-logging-step-interval: 1 \n  --sequence-parallel: true\n  \nMETRICS:\n  - \"generated_tokens\"\n  - \"logprobs\"\n"}, "0bba0036e8375d6afea0a5067e698caf5b8a2549": {"t": 1785767913.6451888, "n": 3009, "h": "8392c90c686f2029", "c": "ENV_VARS:\n  CUDA_DEVICE_MAX_CONNECTIONS: 1\n  NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n  NCCL_ALGO: Ring\n  CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n  --log-num-zeros-in-grad: true\n  --log-validation-ppl-to-tensorboard: true\n  --log-timers-to-tensorboard: true\n  --log-memory-to-tensorboard: true\n  --timing-log-level: 0\n  --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n  --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n  --tokenizer-type: TikTokenizer\n  --tiktoken-pattern: v2\n  --distributed-backend: nccl\n  --log-interval: 1\n  --transformer-impl: transformer_engine\n  --tensor-model-parallel-size: 4\n  --pipeline-model-parallel-size: 1\n  --expert-model-parallel-size: 8\n  --expert-tensor-parallel-size: 1\n  --sequence-parallel: true\n  --use-mcore-models: true\n  --moe-token-dispatcher-type: alltoall\n  --moe-grouped-gemm: true\n  --num-experts: 64\n  --moe-router-topk: 6\n  --moe-z-loss-coeff: 0\n  --moe-router-load-balancing-type: seq_aux_loss\n  --moe-aux-loss-coeff: 1e-3\n  --moe-router-score-function: sigmoid\n  --untie-embeddings-and-output-weights: true\n  --disable-bias-linear: true\n  --init-method-std: 0.014\n  --position-embedding-type: rope\n  --rotary-base: 1000000\n  --rotary-percent: 1.0\n  --num-layers: 27\n  --hidden-size: 2048\n  --moe-ffn-hidden-size: 1408\n  --moe-shared-expert-intermediate-size: 2816\n  --ffn-hidden-size: 10944\n  --num-attention-heads: 16\n  --kv-channels: 128\n  --normalization: RMSNorm\n  --swiglu: true\n  --attention-dropout: 0.0\n  --hidden-dropout: 0.0\n  --seq-length: 4096\n  --max-position-embeddings: 4096\n  --micro-batch-size: 1\n  --ckpt-format: torch_dist\n  --ckpt-fully-parallel-save: true\n  --ckpt-fully-parallel-load: true\n  --ckpt-assume-constant-structure: true\n  --dist-ckpt-strictness: log_unexpected\n  --bf16: true\n  --attention-backend: flash\n  --no-create-attention-mask-in-dataloader: true\n  --num-workers: 8\n  --use-checkpoint-args: true\n  --no-use-tokenizer-model-from-checkpoint-args: true\n  --no-load-optim: true\n  --deterministic-mode: true # moe will use different ops for determinism for inference\n  --save-interval: 2000\n  --temperature: 1.0\n  --top_k: 1\n  --return-log-probs: true\n  --num-tokens-to-generate: 30\n  --max-tokens-to-oom: 3600000\n  --inference-max-seq-length: 4096\n  --output-path: ${TENSORBOARD_PATH}\n  --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n  --incoming-requests-per-sec: -1 # all requests arrive up front.\n  --inference-repeat-n: 8\n  --inference-logging-step-interval: 1 \n  --sequence-parallel: true\n  \nMETRICS:\n  - \"generated_tokens\"\n  - \"logprobs\"\n"}}}}