1 line
16 KiB
JSON
1 line
16 KiB
JSON
{"repo": "birgermoell/nvidia-megatron-lm-context-extension", "files": {"tests/functional_tests/test_cases/moe/gpt_dynamic_inference_cuda_graphs_pad_tp4_pp1_ep4_16B_logitsmatch/model_config.yaml": {"d3ee058092f66045286f0c0bc8d6ce4f21d5302c": {"t": 1785746845.0672448, "n": 3033, "h": "844b67608ee3fd6e", "c": "ENV_VARS:\n CUDA_DEVICE_MAX_CONNECTIONS: 1\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n NCCL_ALGO: Ring\n CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n --log-num-zeros-in-grad: true\n --log-validation-ppl-to-tensorboard: true\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --timing-log-level: 0\n --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n --tokenizer-type: TikTokenizer\n --tiktoken-pattern: v2\n --distributed-backend: nccl\n --log-interval: 1\n --transformer-impl: transformer_engine\n --tensor-model-parallel-size: 4\n --pipeline-model-parallel-size: 1\n --expert-model-parallel-size: 4\n --expert-tensor-parallel-size: 1\n --sequence-parallel: true\n --use-mcore-models: true\n --moe-token-dispatcher-type: alltoall\n --moe-grouped-gemm: true\n --num-experts: 64\n --moe-router-topk: 6\n --moe-z-loss-coeff: 0\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-aux-loss-coeff: 1e-3\n --moe-router-score-function: sigmoid\n --untie-embeddings-and-output-weights: true\n --disable-bias-linear: true\n --init-method-std: 0.014\n --position-embedding-type: rope\n --rotary-base: 1000000\n --rotary-percent: 1.0\n --num-layers: 27\n --hidden-size: 2048\n --moe-ffn-hidden-size: 1408\n --moe-shared-expert-intermediate-size: 2816\n --ffn-hidden-size: 10944\n --num-attention-heads: 16\n --kv-channels: 128\n --normalization: RMSNorm\n --swiglu: true\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --seq-length: 4096\n --max-position-embeddings: 4096\n --micro-batch-size: 1\n --enable-cuda-graph: true\n --te-rng-tracker: true\n --inference-rng-tracker: true\n --moe-pad-experts-for-cuda-graph-inference: true\n --ckpt-format: torch_dist\n --ckpt-fully-parallel-save: true\n --ckpt-fully-parallel-load: true\n --ckpt-assume-constant-structure: true\n --dist-ckpt-strictness: log_unexpected\n --bf16: true\n --attention-backend: flash\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 8\n --use-checkpoint-args: true\n --no-use-tokenizer-model-from-checkpoint-args: true\n --no-load-optim: true\n --deterministic-mode: true\n --save-interval: 2000\n --temperature: 1.0\n --top_k: 1\n --return-log-probs: true\n --num-tokens-to-generate: 30\n --max-tokens-to-oom: 3600000\n --inference-max-seq-length: 4096\n --output-path: ${TENSORBOARD_PATH}\n --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n --incoming-requests-per-sec: -1\n --inference-repeat-n: 8\n --inference-dynamic-batching-buffer-size-gb: 20\nMETRICS:\n - \"generated_tokens\"\n - \"logprobs\"\n"}, "9fd43fa9e2363939c25fa896e29eb181e5434d81": {"t": 1785768042.7822902, "n": 3040, "h": "cd3cb2f3219ba05f", "c": "ENV_VARS:\n CUDA_DEVICE_MAX_CONNECTIONS: 1\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n NCCL_ALGO: Ring\n CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n --log-num-zeros-in-grad: true\n --log-validation-ppl-to-tensorboard: true\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --timing-log-level: 0\n --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n --tokenizer-type: TikTokenizer\n --tiktoken-pattern: v2\n --distributed-backend: nccl\n --log-interval: 1\n --transformer-impl: transformer_engine\n --tensor-model-parallel-size: 4\n --pipeline-model-parallel-size: 1\n --expert-model-parallel-size: 4\n --expert-tensor-parallel-size: 1\n --sequence-parallel: true\n --use-mcore-models: true\n --moe-token-dispatcher-type: alltoall\n --moe-grouped-gemm: true\n --num-experts: 64\n --moe-router-topk: 6\n --moe-z-loss-coeff: 0\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-aux-loss-coeff: 1e-3\n --moe-router-score-function: sigmoid\n --untie-embeddings-and-output-weights: true\n --disable-bias-linear: true\n --init-method-std: 0.014\n --position-embedding-type: rope\n --rotary-base: 1000000\n --rotary-percent: 1.0\n --num-layers: 27\n --hidden-size: 2048\n --moe-ffn-hidden-size: 1408\n --moe-shared-expert-intermediate-size: 2816\n --ffn-hidden-size: 10944\n --num-attention-heads: 16\n --kv-channels: 128\n --normalization: RMSNorm\n --swiglu: true\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --seq-length: 4096\n --max-position-embeddings: 4096\n --micro-batch-size: 1\n --enable-cuda-graph: true\n --te-rng-tracker: true\n --inference-rng-tracker: true\n --moe-pad-experts-for-cuda-graph-inference: true\n --ckpt-format: torch_dist\n --ckpt-fully-parallel-save: true\n --ckpt-fully-parallel-load: true\n --ckpt-assume-constant-structure: true\n --dist-ckpt-strictness: log_unexpected\n --bf16: true\n --attention-backend: flash\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 8\n --use-checkpoint-args: true\n --no-use-tokenizer-model-from-checkpoint-args: true\n --no-load-optim: true\n --deterministic-mode: true\n --save-interval: 2000\n --temperature: 1.0\n --top_k: 1\n --return-log-probs: true\n --num-tokens-to-generate: 30\n --max-tokens-to-oom: 3600000\n --inference-max-seq-length: 4096\n --output-path: ${TENSORBOARD_PATH}\n --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n --incoming-requests-per-sec: -1\n --inference-repeat-n: 8\n --inference-dynamic-batching-active-buffer-size-gb: 20\nMETRICS:\n - \"generated_tokens\"\n - \"logprobs\"\n"}, "adc69dba2dec6c9014d51b6cad3edf1020731786": {"t": 1785768044.1207993, "n": 2983, "h": "85c9e744d688c7d5", "c": "ENV_VARS:\n CUDA_DEVICE_MAX_CONNECTIONS: 1\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n NCCL_ALGO: Ring\n CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n --log-num-zeros-in-grad: true\n --log-validation-ppl-to-tensorboard: true\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --timing-log-level: 0\n --load: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/checkpoints\n --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/deepseek_16b_pyt/dcp/mcore-v1_bf16/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n --tokenizer-type: TikTokenizer\n --tiktoken-pattern: v2\n --distributed-backend: nccl\n --log-interval: 1\n --transformer-impl: transformer_engine\n --tensor-model-parallel-size: 4\n --pipeline-model-parallel-size: 1\n --expert-model-parallel-size: 4\n --expert-tensor-parallel-size: 1\n --sequence-parallel: true\n --use-mcore-models: true\n --moe-token-dispatcher-type: alltoall\n --moe-grouped-gemm: true\n --num-experts: 64\n --moe-router-topk: 6\n --moe-z-loss-coeff: 0\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-aux-loss-coeff: 1e-3\n --moe-router-score-function: sigmoid\n --untie-embeddings-and-output-weights: true\n --disable-bias-linear: true\n --init-method-std: 0.014\n --position-embedding-type: rope\n --rotary-base: 1000000\n --rotary-percent: 1.0\n --num-layers: 27\n --hidden-size: 2048\n --moe-ffn-hidden-size: 1408\n --moe-shared-expert-intermediate-size: 2816\n --ffn-hidden-size: 10944\n --num-attention-heads: 16\n --kv-channels: 128\n --normalization: RMSNorm\n --swiglu: true\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --seq-length: 4096\n --max-position-embeddings: 4096\n --micro-batch-size: 1\n --enable-cuda-graph: true\n --te-rng-tracker: true\n --inference-rng-tracker: true\n --moe-pad-experts-for-cuda-graph-inference: true\n --ckpt-format: torch_dist\n --ckpt-fully-parallel-save: true\n --ckpt-fully-parallel-load: true\n --ckpt-assume-constant-structure: true\n --dist-ckpt-strictness: log_unexpected\n --bf16: true\n --attention-backend: flash\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 8\n --use-checkpoint-args: true\n --no-use-tokenizer-model-from-checkpoint-args: true\n --no-load-optim: true\n --deterministic-mode: true\n --save-interval: 2000\n --temperature: 1.0\n --top_k: 1\n --return-log-probs: true\n --num-tokens-to-generate: 30\n --max-tokens-to-oom: 3600000\n --inference-max-seq-length: 4096\n --output-path: ${TENSORBOARD_PATH}\n --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n --incoming-requests-per-sec: -1\n --inference-repeat-n: 8\nMETRICS:\n - \"generated_tokens\"\n - \"logprobs\"\n"}, "47d5049a266bc027c113811df0d74246932f50b4": {"t": 1785768045.3357606, "n": 2939, "h": "3bacb25fd36a4b8b", "c": "ENV_VARS:\n CUDA_DEVICE_MAX_CONNECTIONS: 1\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n NCCL_ALGO: Ring\n CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n --log-num-zeros-in-grad: true\n --log-validation-ppl-to-tensorboard: true\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --timing-log-level: 2\n --load: ${CHECKPOINT_LOAD_PATH}/deepseek_16b_pyt/model/checkpoints\n --tokenizer-model: ${DATA_PATH}/deepseek_16b_pyt/tokenizer/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n --tokenizer-type: TikTokenizer\n --tiktoken-pattern: v2\n --distributed-backend: nccl\n --log-interval: 1\n --transformer-impl: transformer_engine\n --tensor-model-parallel-size: 4\n --pipeline-model-parallel-size: 1\n --expert-model-parallel-size: 4\n --expert-tensor-parallel-size: 1\n --sequence-parallel: true\n --use-mcore-models: true\n --moe-token-dispatcher-type: alltoall\n --moe-grouped-gemm: true\n --num-experts: 64\n --moe-router-topk: 6\n --moe-z-loss-coeff: 0\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-aux-loss-coeff: 1e-3\n --moe-router-score-function: sigmoid\n --untie-embeddings-and-output-weights: true\n --disable-bias-linear: true\n --init-method-std: 0.014\n --position-embedding-type: rope\n --rotary-base: 1000000\n --rotary-percent: 1.0\n --num-layers: 27\n --hidden-size: 2048\n --moe-ffn-hidden-size: 1408\n --moe-shared-expert-intermediate-size: 2816\n --ffn-hidden-size: 10944\n --num-attention-heads: 16\n --kv-channels: 128\n --normalization: RMSNorm\n --swiglu: true\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --seq-length: 4096\n --max-position-embeddings: 4096\n --micro-batch-size: 1\n --enable-cuda-graph: true\n --te-rng-tracker: true\n --inference-rng-tracker: true\n --moe-pad-experts-for-cuda-graph-inference: true\n --ckpt-format: torch_dist\n --ckpt-fully-parallel-save: true\n --ckpt-fully-parallel-load: true\n --ckpt-assume-constant-structure: true\n --dist-ckpt-strictness: log_unexpected\n --bf16: true\n --attention-backend: flash\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 8\n --use-checkpoint-args: true\n --no-use-tokenizer-model-from-checkpoint-args: true\n --no-load-optim: true\n --deterministic-mode: true\n --save-interval: 2000\n --temperature: 1.0\n --top_k: 1\n --return-log-probs: true\n --num-tokens-to-generate: 30\n --max-tokens-to-oom: 3600000\n --inference-max-seq-length: 4096\n --output-path: ${TENSORBOARD_PATH}\n --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n --incoming-requests-per-sec: -1\n --inference-repeat-n: 8\nMETRICS:\n - \"generated_tokens\"\n - \"logprobs\""}, "56818f9e5090ff9eb0f13f10bfe408aae4031c5c": {"t": 1785768046.6497319, "n": 2913, "h": "37eee75dc7fde7f1", "c": "ENV_VARS:\n CUDA_DEVICE_MAX_CONNECTIONS: 1\n NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0\n NCCL_ALGO: Ring\n CUBLAS_WORKSPACE_CONFIG: :4096:8\nTEST_TYPE: frozen-start\nMODE: inference\nMODEL_ARGS:\n --log-num-zeros-in-grad: true\n --log-validation-ppl-to-tensorboard: true\n --log-timers-to-tensorboard: true\n --log-memory-to-tensorboard: true\n --timing-log-level: 2\n --load: ${CHECKPOINT_LOAD_PATH}/deepseek_16b_pyt/model/checkpoints\n --tokenizer-model: ${DATA_PATH}/deepseek_16b_pyt/tokenizer/multiMixV8.gpt4o_nc_sd.500000.128k.vocab.json\n --tokenizer-type: TikTokenizer\n --tiktoken-pattern: v2\n --distributed-backend: nccl\n --log-interval: 1\n --transformer-impl: transformer_engine\n --tensor-model-parallel-size: 4\n --pipeline-model-parallel-size: 1\n --expert-model-parallel-size: 4\n --expert-tensor-parallel-size: 1\n --sequence-parallel: true\n --use-mcore-models: true\n --moe-token-dispatcher-type: alltoall\n --moe-grouped-gemm: true\n --num-experts: 64\n --moe-router-topk: 6\n --moe-z-loss-coeff: 0\n --moe-router-load-balancing-type: seq_aux_loss\n --moe-aux-loss-coeff: 1e-3\n --moe-router-score-function: sigmoid\n --untie-embeddings-and-output-weights: true\n --disable-bias-linear: true\n --init-method-std: 0.014\n --position-embedding-type: rope\n --rotary-base: 1000000\n --rotary-percent: 1.0\n --num-layers: 27\n --hidden-size: 2048\n --moe-ffn-hidden-size: 1408\n --moe-shared-expert-intermediate-size: 2816\n --ffn-hidden-size: 10944\n --num-attention-heads: 16\n --kv-channels: 128\n --normalization: RMSNorm\n --swiglu: true\n --attention-dropout: 0.0\n --hidden-dropout: 0.0\n --seq-length: 4096\n --max-position-embeddings: 4096\n --micro-batch-size: 1\n --enable-cuda-graph: true\n --te-rng-tracker: true\n --inference-rng-tracker: true\n --moe-pad-experts-for-cuda-graph-inference: true\n --ckpt-format: torch_dist\n --ckpt-fully-parallel-save: true\n --ckpt-fully-parallel-load: true\n --ckpt-assume-constant-structure: true\n --dist-ckpt-strictness: log_unexpected\n --bf16: true\n --attention-backend: flash\n --no-create-attention-mask-in-dataloader: true\n --num-workers: 8\n --use-checkpoint-args: true\n --no-use-tokenizer-model-from-checkpoint-args: true\n --no-load-optim: true\n --deterministic-mode: true\n --save-interval: 2000\n --temperature: 1.0\n --top_k: 1\n --return-log-probs: true\n --num-tokens-to-generate: 30\n --max-tokens-to-oom: 3600000\n --inference-max-seq-length: 4096\n --output-path: ${TENSORBOARD_PATH}\n --prompts: \"Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.\"\n --incoming-requests-per-sec: -1\nMETRICS:\n - \"generated_tokens\"\n - \"logprobs\""}}}} |