1 line
22 KiB
JSON
1 line
22 KiB
JSON
{"repo": "chen-hao-chao/mdm-prime-v2", "files": {"megatron/tests/functional_tests/test_cases/mixtral/deepseekv3_proxy_flex_tp1pp4emp16etp1cp1_release/model_config.yaml": {"ba62fdcc4dbf38abdf09388da0b56a1402528df5": {"t": 1785746836.5106611, "n": 4521, "h": "838c817131aed1b8", "c": "# The proxy model is used for local code quality check.\n# The proxy model should contain all the necessary components and settings but fewer parameters.\nENV_VARS:\n TORCH_NCCL_AVOID_RECORD_STREAMS: 0\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 1\n PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True\n NCCL_NVLS_ENABLE: 0\n NVTE_FUSED_ATTN: 1\n NVTE_NORM_FWD_USE_CUDNN: 1\n NVTE_NORM_BWD_USE_CUDNN: 1\n PYTHONWARNINGS: ignore\n NCCL_DEBUG: VERSION\n NON_DETERMINSTIC_RESULTS: 1\n NVSHMEM_IB_ENABLE_IBGDA: 0\nTEST_TYPE: \"release\"\nMODEL_ARGS:\n # Distributed args\n --distributed-timeout-minutes: 60\n --tensor-model-parallel-size: 1\n --pipeline-model-parallel-size: 4\n --pipeline-model-parallel-layout: Et*2\\\\|\\\\(tt\\\\|\\\\)*5t\\\\|tmL # Et*2|(tt|)*5t|tmL\n --expert-model-parallel-size: 16\n --context-parallel-size: 1\n --expert-tensor-parallel-size: 1\n --use-distributed-optimizer: true\n --overlap-grad-reduce: true\n --overlap-param-gather: true\n\n # Training args\n --use-mcore-models: true\n --sequence-parallel: true\n --use-flash-attn: true\n --disable-bias-linear: true\n --micro-batch-size: 1\n --global-batch-size: 512\n --train-samples: 24414062\n --exit-duration-in-mins: 220\n --no-check-for-nan-in-loss-and-grad: true\n --cross-entropy-loss-fusion: true\n --cross-entropy-fusion-impl: te\n --manual-gc: true\n --manual-gc-interval: 10\n\n # Transformer Engine args\n --transformer-impl: transformer_engine\n\n # Data args\n --seq-length: 4096\n --data-cache-path: ${DATA_CACHE_PATH}\n --tokenizer-type: HuggingFaceTokenizer\n --tokenizer-model: ${TOKENIZER_PATH}\n --data-path: $DATA_BLEND\n --split: 99,1,0\n --no-mmap-bin-files: true\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 6\n\n # Add network size args\n --num-layers: 14 # original 61 layers\n --hidden-size: 7168\n --ffn-hidden-size: 18432\n --num-attention-heads: 128\n --kv-channels: 128\n --max-position-embeddings: 4096\n --position-embedding-type: rope\n --rotary-base: 10000\n --make-vocab-size-divisible-by: 3232\n --normalization: RMSNorm\n --norm-epsilon: 1e-6\n --swiglu: true\n --untie-embeddings-and-output-weights: true\n --multi-latent-attention: true\n --mtp-num-layers: 1\n --mtp-loss-scaling-factor: 0.1\n\n # Add regularization args\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --clip-grad: 1.0\n --weight-decay: 0.1\n --qk-layernorm: true\n\n # Add learning rate args\n --lr-decay-samples: 584765624\n --lr-warmup-samples: 1536000\n # Learning rate scaled down from 7.3e-6 (DeepSeek-V3 technical report, GBS=15360) to 3.9e-6 (GBS=8192)\n --lr-warmup-init: 3.9e-7\n --lr: 3.9e-6\n --min-lr: 3.9e-7\n --lr-decay-style: cosine\n --adam-beta1: 0.9\n --adam-beta2: 0.95\n\n # Add MoE args\n --num-experts: 64 # local 4 + 1 shared, EP16\n --moe-layer-freq: ([0]*3+[1]*11)\n --moe-ffn-hidden-size: 2048\n --moe-shared-expert-intermediate-size: 2048\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-router-topk: 8\n --moe-token-dispatcher-type: flex\n --moe-enable-deepep: true\n --moe-router-pre-softmax: true\n --moe-grouped-gemm: true\n --moe-aux-loss-coeff: 1e-4\n --moe-router-group-topk: 4\n --moe-router-num-groups: 8\n --moe-router-topk-scaling-factor: 2.5\n --moe-router-score-function: sigmoid\n --moe-router-enable-expert-bias: true\n --moe-router-bias-update-rate: 1e-3\n --moe-router-dtype: fp32\n --moe-permute-fusion: true\n\n # Add MLA args\n --q-lora-rank: 1536\n --kv-lora-rank: 512\n --qk-head-dim: 128\n --qk-pos-emb-head-dim: 64\n --v-head-dim: 128\n --rotary-scaling-factor: 40\n --mscale: 1.0\n --mscale-all-dim: 1.0\n\n # Add validation args\n --eval-iters: 32\n --eval-interval: 200\n\n # Add checkpointing args\n --no-load-optim: true\n --no-load-rng: true\n --auto-detect-ckpt-format:\n true\n # Add checkpointing args\n --save: ${CHECKPOINT_SAVE_PATH}\n --load: ${CHECKPOINT_LOAD_PATH}\n --save-interval: 500\n --dist-ckpt-strictness: log_all\n\n # Add initialization args\n --init-method-std: 0.02\n\n # Add logging args\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --log-num-zeros-in-grad: true\n --log-params-norm: true\n --log-validation-ppl-to-tensorboard: true\n --log-throughput: true\n --log-interval: 1\n --logging-level: 40\n --tensorboard-dir: ${TENSORBOARD_PATH}\n --wandb-project: megatron-core-release-runs\n --wandb-exp-name: ${WANDB_EXPERIMENT}\n\n # Add mixed precision args\n --bf16: true\n\n # enable experimental\n --enable-experimental: true\nMETRICS:\n - \"iteration-time\"\n - \"lm loss\"\n - \"mem-allocated-bytes\"\n - \"mem-max-allocated-bytes\"\n"}, "06028347fb21584dda006f072809d6da50411c4d": {"t": 1785767974.216314, "n": 4521, "h": "838c817131aed1b8", "c": "# The proxy model is used for local code quality check.\n# The proxy model should contain all the necessary components and settings but fewer parameters.\nENV_VARS:\n TORCH_NCCL_AVOID_RECORD_STREAMS: 0\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 1\n PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True\n NCCL_NVLS_ENABLE: 0\n NVTE_FUSED_ATTN: 1\n NVTE_NORM_FWD_USE_CUDNN: 1\n NVTE_NORM_BWD_USE_CUDNN: 1\n PYTHONWARNINGS: ignore\n NCCL_DEBUG: VERSION\n NON_DETERMINSTIC_RESULTS: 1\n NVSHMEM_IB_ENABLE_IBGDA: 0\nTEST_TYPE: \"release\"\nMODEL_ARGS:\n # Distributed args\n --distributed-timeout-minutes: 60\n --tensor-model-parallel-size: 1\n --pipeline-model-parallel-size: 4\n --pipeline-model-parallel-layout: Et*2\\\\|\\\\(tt\\\\|\\\\)*5t\\\\|tmL # Et*2|(tt|)*5t|tmL\n --expert-model-parallel-size: 16\n --context-parallel-size: 1\n --expert-tensor-parallel-size: 1\n --use-distributed-optimizer: true\n --overlap-grad-reduce: true\n --overlap-param-gather: true\n\n # Training args\n --use-mcore-models: true\n --sequence-parallel: true\n --use-flash-attn: true\n --disable-bias-linear: true\n --micro-batch-size: 1\n --global-batch-size: 512\n --train-samples: 24414062\n --exit-duration-in-mins: 220\n --no-check-for-nan-in-loss-and-grad: true\n --cross-entropy-loss-fusion: true\n --cross-entropy-fusion-impl: te\n --manual-gc: true\n --manual-gc-interval: 10\n\n # Transformer Engine args\n --transformer-impl: transformer_engine\n\n # Data args\n --seq-length: 4096\n --data-cache-path: ${DATA_CACHE_PATH}\n --tokenizer-type: HuggingFaceTokenizer\n --tokenizer-model: ${TOKENIZER_PATH}\n --data-path: $DATA_BLEND\n --split: 99,1,0\n --no-mmap-bin-files: true\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 6\n\n # Add network size args\n --num-layers: 14 # original 61 layers\n --hidden-size: 7168\n --ffn-hidden-size: 18432\n --num-attention-heads: 128\n --kv-channels: 128\n --max-position-embeddings: 4096\n --position-embedding-type: rope\n --rotary-base: 10000\n --make-vocab-size-divisible-by: 3232\n --normalization: RMSNorm\n --norm-epsilon: 1e-6\n --swiglu: true\n --untie-embeddings-and-output-weights: true\n --multi-latent-attention: true\n --mtp-num-layers: 1\n --mtp-loss-scaling-factor: 0.1\n\n # Add regularization args\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --clip-grad: 1.0\n --weight-decay: 0.1\n --qk-layernorm: true\n\n # Add learning rate args\n --lr-decay-samples: 584765624\n --lr-warmup-samples: 1536000\n # Learning rate scaled down from 7.3e-6 (DeepSeek-V3 technical report, GBS=15360) to 3.9e-6 (GBS=8192)\n --lr-warmup-init: 3.9e-7\n --lr: 3.9e-6\n --min-lr: 3.9e-7\n --lr-decay-style: cosine\n --adam-beta1: 0.9\n --adam-beta2: 0.95\n\n # Add MoE args\n --num-experts: 64 # local 4 + 1 shared, EP16\n --moe-layer-freq: ([0]*3+[1]*11)\n --moe-ffn-hidden-size: 2048\n --moe-shared-expert-intermediate-size: 2048\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-router-topk: 8\n --moe-token-dispatcher-type: flex\n --moe-enable-deepep: true\n --moe-router-pre-softmax: true\n --moe-grouped-gemm: true\n --moe-aux-loss-coeff: 1e-4\n --moe-router-group-topk: 4\n --moe-router-num-groups: 8\n --moe-router-topk-scaling-factor: 2.5\n --moe-router-score-function: sigmoid\n --moe-router-enable-expert-bias: true\n --moe-router-bias-update-rate: 1e-3\n --moe-router-dtype: fp32\n --moe-permute-fusion: true\n\n # Add MLA args\n --q-lora-rank: 1536\n --kv-lora-rank: 512\n --qk-head-dim: 128\n --qk-pos-emb-head-dim: 64\n --v-head-dim: 128\n --rotary-scaling-factor: 40\n --mscale: 1.0\n --mscale-all-dim: 1.0\n\n # Add validation args\n --eval-iters: 32\n --eval-interval: 200\n\n # Add checkpointing args\n --no-load-optim: true\n --no-load-rng: true\n --auto-detect-ckpt-format:\n true\n # Add checkpointing args\n --save: ${CHECKPOINT_SAVE_PATH}\n --load: ${CHECKPOINT_LOAD_PATH}\n --save-interval: 500\n --dist-ckpt-strictness: log_all\n\n # Add initialization args\n --init-method-std: 0.02\n\n # Add logging args\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --log-num-zeros-in-grad: true\n --log-params-norm: true\n --log-validation-ppl-to-tensorboard: true\n --log-throughput: true\n --log-interval: 1\n --logging-level: 40\n --tensorboard-dir: ${TENSORBOARD_PATH}\n --wandb-project: megatron-core-release-runs\n --wandb-exp-name: ${WANDB_EXPERIMENT}\n\n # Add mixed precision args\n --bf16: true\n\n # enable experimental\n --enable-experimental: true\nMETRICS:\n - \"iteration-time\"\n - \"lm loss\"\n - \"mem-allocated-bytes\"\n - \"mem-max-allocated-bytes\"\n"}}, "megatron/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_pp1_ep4_16B_logitsmatch/model_config.yaml": {"ba62fdcc4dbf38abdf09388da0b56a1402528df5": {"t": 1785746837.952662, "n": 2810, "h": "a7506bdd6cacdd12", "c": "ENV_VARS:\n CUDA_DEVICE_MAX_CONNECTIONS: 1\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n NCCL_ALGO: Ring\n CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n --log-num-zeros-in-grad: true\n --log-validation-ppl-to-tensorboard: true\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --timing-log-level: 2\n --load: ${CHECKPOINT_LOAD_PATH}/deepseek_16b_pyt/model/checkpoints\n --tokenizer-model: ${DATA_PATH}/deepseek_16b_pyt/tokenizer/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n --tokenizer-type: TikTokenizer\n --tiktoken-pattern: v2\n --distributed-backend: nccl\n --log-interval: 1\n --transformer-impl: transformer_engine\n --tensor-model-parallel-size: 4\n --pipeline-model-parallel-size: 1\n --expert-model-parallel-size: 4\n --expert-tensor-parallel-size: 1\n --sequence-parallel: true\n --use-mcore-models: true\n --moe-token-dispatcher-type: alltoall\n --moe-grouped-gemm: true\n --num-experts: 64\n --moe-router-topk: 6\n --moe-z-loss-coeff: 0\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-aux-loss-coeff: 1e-3\n --moe-router-score-function: sigmoid\n --untie-embeddings-and-output-weights: true\n --disable-bias-linear: true\n --init-method-std: 0.014\n --position-embedding-type: rope\n --rotary-base: 1000000\n --rotary-percent: 1.0\n --num-layers: 27\n --hidden-size: 2048\n --moe-ffn-hidden-size: 1408\n --moe-shared-expert-intermediate-size: 2816\n --ffn-hidden-size: 10944\n --num-attention-heads: 16\n --kv-channels: 128\n --normalization: RMSNorm\n --swiglu: true\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --seq-length: 4096\n --max-position-embeddings: 4096\n --micro-batch-size: 1\n --ckpt-format: torch_dist\n --ckpt-fully-parallel-save: true\n --ckpt-fully-parallel-load: true\n --ckpt-assume-constant-structure: true\n --dist-ckpt-strictness: log_unexpected\n --bf16: true\n --attention-backend: flash\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 8\n --use-checkpoint-args: true\n --no-use-tokenizer-model-from-checkpoint-args: true\n --no-load-optim: true\n --deterministic-mode: true\n --save-interval: 2000\n --temperature: 1.0\n --top_k: 1\n --return-log-probs: true\n --num-tokens-to-generate: 30\n --max-tokens-to-oom: 3600000\n --inference-max-seq-length: 4096\n --output-path: ${TENSORBOARD_PATH}\n --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n --incoming-requests-per-sec: -1 # all requests arrive up front.\nMETRICS:\n - \"generated_tokens\"\n - \"logprobs\"\n"}, "06028347fb21584dda006f072809d6da50411c4d": {"t": 1785767973.5501876, "n": 2810, "h": "a7506bdd6cacdd12", "c": "ENV_VARS:\n CUDA_DEVICE_MAX_CONNECTIONS: 1\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n NCCL_ALGO: Ring\n CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n --log-num-zeros-in-grad: true\n --log-validation-ppl-to-tensorboard: true\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --timing-log-level: 2\n --load: ${CHECKPOINT_LOAD_PATH}/deepseek_16b_pyt/model/checkpoints\n --tokenizer-model: ${DATA_PATH}/deepseek_16b_pyt/tokenizer/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n --tokenizer-type: TikTokenizer\n --tiktoken-pattern: v2\n --distributed-backend: nccl\n --log-interval: 1\n --transformer-impl: transformer_engine\n --tensor-model-parallel-size: 4\n --pipeline-model-parallel-size: 1\n --expert-model-parallel-size: 4\n --expert-tensor-parallel-size: 1\n --sequence-parallel: true\n --use-mcore-models: true\n --moe-token-dispatcher-type: alltoall\n --moe-grouped-gemm: true\n --num-experts: 64\n --moe-router-topk: 6\n --moe-z-loss-coeff: 0\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-aux-loss-coeff: 1e-3\n --moe-router-score-function: sigmoid\n --untie-embeddings-and-output-weights: true\n --disable-bias-linear: true\n --init-method-std: 0.014\n --position-embedding-type: rope\n --rotary-base: 1000000\n --rotary-percent: 1.0\n --num-layers: 27\n --hidden-size: 2048\n --moe-ffn-hidden-size: 1408\n --moe-shared-expert-intermediate-size: 2816\n --ffn-hidden-size: 10944\n --num-attention-heads: 16\n --kv-channels: 128\n --normalization: RMSNorm\n --swiglu: true\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --seq-length: 4096\n --max-position-embeddings: 4096\n --micro-batch-size: 1\n --ckpt-format: torch_dist\n --ckpt-fully-parallel-save: true\n --ckpt-fully-parallel-load: true\n --ckpt-assume-constant-structure: true\n --dist-ckpt-strictness: log_unexpected\n --bf16: true\n --attention-backend: flash\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 8\n --use-checkpoint-args: true\n --no-use-tokenizer-model-from-checkpoint-args: true\n --no-load-optim: true\n --deterministic-mode: true\n --save-interval: 2000\n --temperature: 1.0\n --top_k: 1\n --return-log-probs: true\n --num-tokens-to-generate: 30\n --max-tokens-to-oom: 3600000\n --inference-max-seq-length: 4096\n --output-path: ${TENSORBOARD_PATH}\n --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n --incoming-requests-per-sec: -1 # all requests arrive up front.\nMETRICS:\n - \"generated_tokens\"\n - \"logprobs\"\n"}}, "megatron/tests/functional_tests/test_cases/moe/gpt_static_inference_tp1_pp1_ep1_16B_logitsmatch/model_config.yaml": {"ba62fdcc4dbf38abdf09388da0b56a1402528df5": {"t": 1785746843.8282568, "n": 2770, "h": "3154bbd065b3c8fb", "c": "ENV_VARS:\n CUDA_DEVICE_MAX_CONNECTIONS: 1\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n NCCL_ALGO: Ring\n CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n --log-num-zeros-in-grad: true\n --log-validation-ppl-to-tensorboard: true\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --timing-log-level: 2\n --load: ${CHECKPOINT_LOAD_PATH}/deepseek_16b_pyt/model/checkpoints\n --tokenizer-model: ${DATA_PATH}/deepseek_16b_pyt/tokenizer/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n --tokenizer-type: TikTokenizer\n --tiktoken-pattern: v2\n --distributed-backend: nccl\n --log-interval: 1\n --transformer-impl: transformer_engine\n --tensor-model-parallel-size: 1\n --pipeline-model-parallel-size: 1\n --expert-model-parallel-size: 1\n --use-mcore-models: true\n --moe-token-dispatcher-type: alltoall\n --moe-grouped-gemm: true\n --num-experts: 64\n --moe-router-topk: 6\n --moe-z-loss-coeff: 0\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-aux-loss-coeff: 1e-3\n --moe-router-score-function: sigmoid\n --untie-embeddings-and-output-weights: true\n --disable-bias-linear: true\n --init-method-std: 0.014\n --position-embedding-type: rope\n --rotary-base: 1000000\n --rotary-percent: 1.0\n --num-layers: 27\n --hidden-size: 2048\n --moe-ffn-hidden-size: 1408\n --moe-shared-expert-intermediate-size: 2816\n --ffn-hidden-size: 10944\n --num-attention-heads: 16\n --kv-channels: 128\n --normalization: RMSNorm\n --swiglu: true\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --seq-length: 4096\n --max-position-embeddings: 4096\n --micro-batch-size: 1\n --ckpt-format: torch_dist\n --ckpt-fully-parallel-save: true\n --ckpt-fully-parallel-load: true\n --ckpt-assume-constant-structure: true\n --dist-ckpt-strictness: log_unexpected\n --bf16: true\n --attention-backend: flash\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 8\n --flash-decode: true\n --use-checkpoint-args: true\n --no-use-tokenizer-model-from-checkpoint-args: true\n --no-load-optim: true\n --deterministic-mode: true\n --save-interval: 2000\n --temperature: 1.0\n --top_k: 1\n --return-log-probs: true\n --num-tokens-to-generate: 30\n --max-tokens-to-oom: 3600000\n --inference-max-seq-length: 4096\n --output-path: ${TENSORBOARD_PATH}\n --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n --incoming-requests-per-sec: -1 # all requests arrive up front.\nMETRICS:\n - \"generated_tokens\"\n - \"logprobs\"\n"}, "06028347fb21584dda006f072809d6da50411c4d": {"t": 1785767974.0241146, "n": 2770, "h": "3154bbd065b3c8fb", "c": "ENV_VARS:\n CUDA_DEVICE_MAX_CONNECTIONS: 1\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n NCCL_ALGO: Ring\n CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n --log-num-zeros-in-grad: true\n --log-validation-ppl-to-tensorboard: true\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --timing-log-level: 2\n --load: ${CHECKPOINT_LOAD_PATH}/deepseek_16b_pyt/model/checkpoints\n --tokenizer-model: ${DATA_PATH}/deepseek_16b_pyt/tokenizer/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n --tokenizer-type: TikTokenizer\n --tiktoken-pattern: v2\n --distributed-backend: nccl\n --log-interval: 1\n --transformer-impl: transformer_engine\n --tensor-model-parallel-size: 1\n --pipeline-model-parallel-size: 1\n --expert-model-parallel-size: 1\n --use-mcore-models: true\n --moe-token-dispatcher-type: alltoall\n --moe-grouped-gemm: true\n --num-experts: 64\n --moe-router-topk: 6\n --moe-z-loss-coeff: 0\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-aux-loss-coeff: 1e-3\n --moe-router-score-function: sigmoid\n --untie-embeddings-and-output-weights: true\n --disable-bias-linear: true\n --init-method-std: 0.014\n --position-embedding-type: rope\n --rotary-base: 1000000\n --rotary-percent: 1.0\n --num-layers: 27\n --hidden-size: 2048\n --moe-ffn-hidden-size: 1408\n --moe-shared-expert-intermediate-size: 2816\n --ffn-hidden-size: 10944\n --num-attention-heads: 16\n --kv-channels: 128\n --normalization: RMSNorm\n --swiglu: true\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --seq-length: 4096\n --max-position-embeddings: 4096\n --micro-batch-size: 1\n --ckpt-format: torch_dist\n --ckpt-fully-parallel-save: true\n --ckpt-fully-parallel-load: true\n --ckpt-assume-constant-structure: true\n --dist-ckpt-strictness: log_unexpected\n --bf16: true\n --attention-backend: flash\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 8\n --flash-decode: true\n --use-checkpoint-args: true\n --no-use-tokenizer-model-from-checkpoint-args: true\n --no-load-optim: true\n --deterministic-mode: true\n --save-interval: 2000\n --temperature: 1.0\n --top_k: 1\n --return-log-probs: true\n --num-tokens-to-generate: 30\n --max-tokens-to-oom: 3600000\n --inference-max-seq-length: 4096\n --output-path: ${TENSORBOARD_PATH}\n --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n --incoming-requests-per-sec: -1 # all requests arrive up front.\nMETRICS:\n - \"generated_tokens\"\n - \"logprobs\"\n"}}}} |