1 line
59 KiB
JSON
1 line
59 KiB
JSON
{"repo": "adityak74/llm-serving-bench-lab", "files": {".vendor/sglang/python/sglang/srt/environ.py": {"b78a5a456858bb5f7dd7a4f6aa691ac81637f48e": {"t": 1785595776.5483916, "n": 50921, "h": "fbb6e0535aea66b4", "c": "import functools\nimport os\nimport subprocess\nimport warnings\nfrom contextlib import ExitStack, contextmanager\nfrom enum import IntEnum\nfrom typing import Any, Optional\n\n\n@functools.lru_cache(maxsize=1)\ndef _default_hip() -> bool:\n \"\"\"Lazy ROCm/HIP detection for platform-conditional env defaults.\n\n Avoids importing torch at environ import time (this module is intentionally\n stdlib-only and loaded very early). Resolved on first EnvField.get() that uses\n it as a default, by which point torch is already imported in any real run;\n falls back to False if torch is unavailable.\n \"\"\"\n try:\n import torch\n\n return torch.version.hip is not None\n except Exception:\n return False\n\n\nclass EnvField:\n _allow_set_name = True\n\n def __init__(self, default: Any):\n self.default = default\n # NOTE: environ can only accept str values, so we need a flag to indicate\n # whether the env var is explicitly set to None.\n self._set_to_none = False\n\n def __set_name__(self, owner, name):\n assert EnvField._allow_set_name, \"Usage like `a = envs.A` is not allowed\"\n self.name = name\n\n def parse(self, value: str) -> Any:\n raise NotImplementedError()\n\n def _resolve_default(self) -> Any:\n # Support a callable default for lazily/platform-computed defaults\n # (e.g. EnvBool(_default_hip)); evaluated only when the env is unset.\n return self.default() if callable(self.default) else self.default\n\n def get(self) -> Any:\n value = os.getenv(self.name)\n\n # Explicitly set to None\n if self._set_to_none:\n assert value == str(None)\n return None\n\n # Not set, return default\n if value is None:\n return self._resolve_default()\n\n try:\n return self.parse(value)\n except ValueError as e:\n default = self._resolve_default()\n warnings.warn(\n f'Invalid value for {self.name}: {e}, using default \"{default}\"'\n )\n return default\n\n def is_set(self):\n return self.name in os.environ\n\n def set(self, value: Any):\n self._set_to_none = value is None\n os.environ[self.name] = str(value)\n\n @contextmanager\n def override(self, value: Any):\n backup_present = self.name in os.environ\n backup_value = os.environ.get(self.name)\n backup_set_to_none = self._set_to_none\n self.set(value)\n yield\n if backup_present:\n os.environ[self.name] = backup_value\n else:\n os.environ.pop(self.name, None)\n self._set_to_none = backup_set_to_none\n\n def clear(self):\n os.environ.pop(self.name, None)\n self._set_to_none = False\n\n def __bool__(self):\n raise RuntimeError(\n \"Please use `envs.YOUR_FLAG.get()` instead of `envs.YOUR_FLAG`\"\n )\n\n def __len__(self):\n raise RuntimeError(\n \"Please use `envs.YOUR_FLAG.get()` instead of `envs.YOUR_FLAG`\"\n )\n\n\nclass EnvTuple(EnvField):\n def parse(self, value: str) -> tuple[str, ...]:\n return tuple(s.strip() for s in value.split(\",\") if s.strip())\n\n\nclass EnvStr(EnvField):\n def parse(self, value: str) -> str:\n return value\n\n\nclass EnvBool(EnvField):\n def parse(self, value: str) -> bool:\n value = value.lower()\n if value in [\"true\", \"1\", \"yes\", \"y\"]:\n return True\n if value in [\"false\", \"0\", \"no\", \"n\"]:\n return False\n raise ValueError(f'\"{value}\" is not a valid boolean value')\n\n\nclass EnvInt(EnvField):\n def parse(self, value: str) -> int:\n try:\n return int(value)\n except ValueError:\n raise ValueError(f'\"{value}\" is not a valid integer value')\n\n\nclass _DeprecatedEnvFallback:\n \"\"\"Mixin for EnvField subclasses: if the canonical env var is not set,\n check *deprecated_name* and emit DeprecationWarning before reading it.\n\n Usage:\n SGLANG_DSA_FUSE_TOPK = EnvBoolWithAlias(True, deprecated_name=\"SGLANG_NSA_FUSE_TOPK\")\n \"\"\"\n\n def __init__(self, default: Any, deprecated_name: str):\n super().__init__(default)\n self.deprecated_name = deprecated_name\n\n def get(self) -> Any:\n if os.getenv(self.name) is None:\n fallback = os.getenv(self.deprecated_name)\n if fallback is not None:\n warnings.warn(\n f\"Environment variable '{self.deprecated_name}' is deprecated; \"\n f\"use '{self.name}' instead. \"\n \"The alias will be removed in a future release.\",\n DeprecationWarning,\n stacklevel=2,\n )\n os.environ[self.name] = fallback\n return super().get()\n\n\nclass EnvBoolWithAlias(_DeprecatedEnvFallback, EnvBool):\n pass\n\n\nclass EnvIntWithAlias(_DeprecatedEnvFallback, EnvInt):\n pass\n\n\nclass EnvFloat(EnvField):\n def parse(self, value: str) -> float:\n try:\n return float(value)\n except ValueError:\n raise ValueError(f'\"{value}\" is not a valid float value')\n\n\nclass ToolStrictLevel(IntEnum):\n \"\"\"\n Defines the strictness levels for tool call parsing and validation.\n\n OFF: No strict validation\n FUNCTION: Enables structural tag constraints for all tools\n PARAMETER: Enforces strict parameter validation for all tools\n \"\"\"\n\n OFF = 0\n FUNCTION = 1\n PARAMETER = 2\n\n\nclass Envs:\n\n # Raise on bare server_args field assignments after resolution; mutation\n # must go through ServerArgs.override() (enabled by the test harness).\n SGLANG_STRICT_CONFIG_MUTATION = EnvBool(False)\n\n # Model & File Download\n SGLANG_USE_MODELSCOPE = EnvBool(False)\n # Controls weight-file ordering for load-time I/O optimization.\n # -1 : no sorting, no staggering; preserves original file order.\n # 0 : sort files only; maximizes ordering but may reduce cross-rank I/O concurrency.\n # k>0: sort files and stagger per-rank order with factor k.\n # Files are processed in groups of (tp_size * k), and rank r starts each\n # group at offset (r * k), improving multi-rank I/O concurrency while\n # keeping access relatively ordered.\n SGLANG_SORT_WEIGHT_FILES = EnvInt(0)\n SGLANG_DISABLED_MODEL_ARCHS = EnvTuple(tuple())\n SGLANG_PREFETCH_BLOCK_SIZE_MB = EnvInt(16)\n SGLANG_GEMMA_OUT_OF_PLACE_POSITION_MUTATION = EnvBool(False)\n\n # HTTP server\n # Decompress request bodies tagged with `x-body-compressed`.\n SGLANG_ENABLE_REQUEST_DECOMPRESSION = EnvBool(False)\n # Override parsed request fields from headers.\n SGLANG_ENABLE_REQUEST_HEADER_OVERRIDES = EnvBool(False)\n\n # Logging Options\n SGLANG_LOG_GC = EnvBool(False)\n SGLANG_LOG_FORWARD_ITERS = EnvBool(False)\n SGLANG_LOG_MS = EnvBool(False)\n SGLANG_LOG_REQUEST_EXCEEDED_MS = EnvInt(-1)\n SGLANG_LOG_REQUEST_HEADERS = EnvTuple(tuple())\n SGLANG_LOG_SCHEDULER_STATUS_TARGET = EnvStr(\"\")\n SGLANG_LOG_SCHEDULER_STATUS_INTERVAL = EnvFloat(60.0)\n\n # IPC\n SGLANG_USE_PICKLE_IPC = EnvBool(True)\n SGLANG_LOG_PICKLE_IPC_OBJECTS = EnvBool(False)\n\n # SGLang CI\n SGLANG_IS_IN_CI = EnvBool(False)\n SGLANG_IS_IN_CI_AMD = EnvBool(False)\n SGLANG_CUDA_COREDUMP = EnvBool(False)\n # None = unset, letting get_dump_dir() resolve the base (RUNNER_TEMP in CI,\n # else /tmp); see debug_utils/cuda_coredump.py.\n SGLANG_CUDA_COREDUMP_DIR = EnvStr(None)\n SGLANG_TEST_MAX_RETRY = EnvInt(None)\n\n # Constrained Decoding (Grammar)\n SGLANG_GRAMMAR_POLL_INTERVAL = EnvFloat(0.005)\n SGLANG_GRAMMAR_MAX_POLL_ITERATIONS = EnvInt(10000)\n SGLANG_DISABLE_OUTLINES_DISK_CACHE = EnvBool(False)\n\n # Test & Debug\n SGLANG_DETECT_SLOW_RANK = EnvBool(False)\n SGLANG_TEST_STUCK_DETOKENIZER = EnvFloat(0)\n SGLANG_TEST_STUCK_DP_CONTROLLER = EnvFloat(0)\n SGLANG_TEST_STUCK_SCHEDULER_INIT = EnvFloat(0)\n SGLANG_TEST_STUCK_TOKENIZER = EnvFloat(0)\n SGLANG_TEST_CRASH_AFTER_STREAM_OUTPUTS = EnvInt(0)\n IS_H200 = EnvBool(False)\n SGLANG_SET_CPU_AFFINITY = EnvBool(False)\n SGLANG_ENABLE_CP_V2 = EnvBool(False)\n SGLANG_PROFILE_WITH_STACK = EnvBool(True)\n SGLANG_PROFILE_RECORD_SHAPES = EnvBool(True)\n SGLANG_PROFILE_V2 = EnvBool(False)\n SGLANG_ENABLE_NVTX_SCHEDULER = EnvBoolWithAlias(\n False, deprecated_name=\"SGLANG_ENABLE_NVTX\"\n )\n SGLANG_ENABLE_NVTX_OPERATIONS = EnvBoolWithAlias(\n False, deprecated_name=\"SGLANG_OPERATIONS_ENABLE_PROFILE\"\n )\n SGLANG_RECORD_STEP_TIME = EnvBool(False)\n SGLANG_ENABLE_CUDA_GRAPH_CAPTURE_TRACE = EnvBool(False)\n SGLANG_FORCE_SHUTDOWN = EnvBool(False)\n SGLANG_DEBUG_MEMORY_POOL = EnvBool(False)\n SGLANG_DEBUG_REVERT_PR = EnvInt(0)\n SGLANG_PHASE_CHECKER_DEBUG = EnvBool(False)\n SGLANG_TEST_REQUEST_TIME_STATS = EnvBool(False)\n SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK = EnvBool(False)\n SGLANG_SIMULATE_ACC_LEN = EnvFloat(-1)\n SGLANG_SIMULATE_ACC_METHOD = EnvStr(\"match-expected\")\n SGLANG_SIMULATE_ACC_TOKEN_MODE = EnvStr(\"fixed\")\n SGLANG_SIMULATE_UNIFORM_EXPERTS = EnvBool(False)\n SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS = EnvBool(False)\n SGLANG_TORCH_PROFILER_DIR = EnvStr(\"/tmp\")\n SGLANG_OTLP_EXPORTER_SCHEDULE_DELAY_MILLIS = EnvInt(500)\n SGLANG_OTLP_EXPORTER_MAX_EXPORT_BATCH_SIZE = EnvInt(64)\n SGLANG_NATIVE_MOVE_KV_CACHE = EnvBool(False)\n # Disable lazy compaction in the unified memory pool allocator and\n # fall back to the per-free eager compaction. Used for production\n # A/B and quick rollback. Default False (lazy compaction on).\n SGLANG_DISABLE_LAZY_COMPACTION = EnvBool(False)\n # Sort the multi-ended allocator's free list after a merge (perf A/B knob).\n SGLANG_SORT_FREE_LIST_AFTER_MERGE = EnvBool(False)\n # Periodically log lazy-compaction stats per sub-pool (observability only).\n SGLANG_LOG_LAZY_COMPACTION_STATS = EnvBool(False)\n SGLANG_LOG_LAZY_COMPACTION_STATS_INTERVAL_SEC = EnvInt(30)\n SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK = EnvBool(True)\n SGLANG_TEST_DISAGG_FAILURE_PROB = EnvFloat(0.0)\n\n # HND KV layout folds (page, head) into one paged index for per-kv-head sparse\n # page tables (DP attn); paged backends like trtllm_mha consume it directly.\n SGLANG_USE_HND_KVCACHE = EnvBool(False)\n\n # size the KV pool after CUDA-graph capture\n SGLANG_ENABLE_POST_CAPTURE_KV_SIZING = EnvBool(False)\n\n # Scheduler: memory leak test\n SGLANG_TEST_RETRACT = EnvBool(False)\n SGLANG_TEST_RETRACT_INTERVAL = EnvInt(3)\n SGLANG_TEST_RETRACT_NO_PREFILL_BS = EnvInt(2**31)\n # Scheduler: force lazy extra_buffer prealloc to fail at decode boundaries\n SGLANG_TEST_MAMBA_LAZY_ALLOC_FAIL = EnvBool(False)\n # KL tests: skip the cache-hit count assertion (e.g. when alloc failure reduces hits)\n SGLANG_TEST_SKIP_CACHE_HIT_ASSERT = EnvBool(False)\n SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY = EnvInt(0)\n SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_IDLE = EnvBool(True)\n # Physical KV-page checks: committed<=allocated + no page alias.\n SGLANG_CHECK_KV_PAGE_INVARIANTS = EnvBool(False)\n\n # Load snapshot backend\n SGLANG_LOAD_SNAPSHOT_USE_ZMQ = EnvBool(False)\n\n # Scheduler: new token ratio hyperparameters\n SGLANG_INIT_NEW_TOKEN_RATIO = EnvFloat(0.7)\n SGLANG_MIN_NEW_TOKEN_RATIO_FACTOR = EnvFloat(0.14)\n SGLANG_NEW_TOKEN_RATIO_DECAY_STEPS = EnvInt(600)\n SGLANG_RETRACT_DECODE_STEPS = EnvInt(20)\n SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION = EnvInt(4096)\n\n # Scheduler: recv interval\n SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_DEFAULT = EnvInt(1000)\n SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_DECODE = EnvInt(1)\n SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_TARGET_VERIFY = EnvInt(1)\n SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_NONE = EnvInt(1)\n\n # PD Disaggregation (runtime)\n # NOTE: For SGLANG_DISAGGREGATION_THREAD_POOL_SIZE, the effective default is\n # computed dynamically at runtime based on cpu_count; see disaggregation backends.\n SGLANG_DISAGGREGATION_THREAD_POOL_SIZE = EnvInt(None)\n SGLANG_DISAGGREGATION_QUEUE_SIZE = EnvInt(4)\n SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT = EnvInt(300)\n SGLANG_DISAGGREGATION_HEARTBEAT_INTERVAL = EnvFloat(5.0)\n SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE = EnvInt(2)\n SGLANG_DISAGGREGATION_WAITING_TIMEOUT = EnvInt(300)\n SGLANG_DISAGGREGATION_NIXL_BACKEND = EnvStr(\"UCX\")\n SGLANG_DISAGGREGATION_NIXL_BACKEND_PARAMS = EnvStr(\"{}\")\n SGLANG_DISAGG_PREFILL_EARLY_SEND_CACHED_PREFIX = EnvBool(True)\n SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER = EnvBool(False)\n SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK = EnvBool(False)\n\n # Scheduler: others:\n # in seconds. Set if you observe high memory accumulation over a long serving period.\n SGLANG_EMPTY_CACHE_INTERVAL = EnvFloat(-1)\n SGLANG_DISABLE_CONSECUTIVE_PREFILL_OVERLAP = EnvBool(False)\n # Force-enable the WAR (write-after-read) barrier for the overlap scheduler\n # even when is_cuda() is False (e.g. AMD/ROCm). On CUDA the barrier is\n # already enabled regardless of this flag (see start_event_loop).\n SGLANG_ENABLE_WAR_BARRIER = EnvBool(False)\n # PP: skip output send/recv when the entire batch consists of non-final chunked prefill requests,\n # since process_batch_result_prefill discards next_token_ids for those anyway.\n SGLANG_PP_SKIP_PURE_CHUNKED_OUTPUT_COMM = EnvBool(False)\n SGLANG_SCHEDULER_MAX_RECV_PER_POLL = EnvInt(-1)\n SGLANG_EXPERIMENTAL_CPP_RADIX_TREE = EnvBool(False)\n SGLANG_RADIX_FORCE_MISS = EnvBool(False)\n SGLANG_DYNAMIC_CHUNKING_SMOOTH_FACTOR = EnvFloat(0.75)\n SGLANG_SCHEDULER_SKIP_ALL_GATHER = EnvBool(False)\n SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE = EnvBool(False)\n SGLANG_KILLPG_ON_SCHEDULER_EXCEPTION = EnvBool(False)\n SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES = EnvInt(None)\n SGLANG_PREFILL_DELAYER_TOKEN_USAGE_LOW_WATERMARK = EnvFloat(None)\n SGLANG_DATA_PARALLEL_BUDGET_INTERVAL = EnvInt(1)\n SGLANG_REQ_WAITING_TIMEOUT = EnvFloat(-1) # in seconds\n SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH = EnvBool(False)\n SGLANG_REQ_RUNNING_TIMEOUT = EnvFloat(-1) # in seconds\n SGLANG_DISAGGREGATION_BOOTSTRAP_ENTRY_CLEANUP_INTERVAL = EnvInt(120)\n # Decode batches between SWA out-of-window evictions.\n SGLANG_SWA_EVICTION_INTERVAL = EnvInt(128)\n # For non-streaming requests, the scheduler still flushes intermediate\n # output batches to the tokenizer manager every N decoded tokens so that\n # `first_token_time`/TTFT can be recorded. Lower this (e.g. to 1) to get\n # an accurate TTFT for benchmarking; the upstream default of 50 trades\n # off some TTFT-metric accuracy for less IPC overhead.\n SGLANG_FORCE_STREAM_INTERVAL = EnvInt(50)\n\n # Test: pd-disaggregation\n SGLANG_TEST_PD_DISAGG_BACKEND = EnvStr(\"mooncake\")\n SGLANG_TEST_PD_DISAGG_DEVICES = EnvStr(None)\n SGLANG_TEST_FORCE_OPTIMISTIC_PREFILL_RETRY_PROB = EnvFloat(0.0)\n\n SGLANG_TEST_SCRIPTED_RUNTIME = EnvBool(False)\n SGLANG_TEST_SCRIPTED_RUNTIME_IPC_ADDR = EnvStr(None)\n SGLANG_TEST_SCRIPTED_RUNTIME_OUT_OF_BAND_ERROR_PATH = EnvStr(None)\n SGLANG_TEST_SCRIPTED_RUNTIME_SYS_PATH_ENTRY = EnvStr(None)\n\n # Model Parallel\n SGLANG_USE_MESSAGE_QUEUE_BROADCASTER = EnvBool(True)\n SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS = EnvBool(False)\n # Comma-separated bundle indices for Ray Custom PG mode (e.g., \"0,1,2,7\").\n SGLANG_RAY_BUNDLE_INDICES = EnvStr(\"\")\n # Override the distributed init method used by torch.distributed.init_process_group.\n # Set to \"env://\" to use an externally-created TCPStore via MASTER_ADDR/MASTER_PORT.\n SGLANG_DISTRIBUTED_INIT_METHOD_OVERRIDE = EnvStr(None)\n SGLANG_TCP_STORE_PORT = EnvInt(29600)\n\n # Base port hint for ephemeral sockets (ZMQ, SHM broadcaster, etc.).\n # When set, get_open_port() and shm_broadcast search upwards from this\n # value instead of asking the OS for a random port. Useful to keep all\n # SGLang ports in a predictable range behind a firewall.\n SGLANG_PORT = EnvInt(None)\n\n # Tool Calling\n SGLANG_FORWARD_UNKNOWN_TOOLS = EnvBool(False)\n\n # Native web search (Exa). EXA_API_KEY is the vendor BYOK credential\n # (kept as-is, not renamed to SGLANG_*); the SGLANG_EXA_* knobs tune the\n # request defaults for the built-in GPT-OSS web_search tool.\n EXA_API_KEY = EnvStr(None)\n SGLANG_EXA_NUM_RESULTS = EnvInt(10)\n SGLANG_EXA_SEARCH_TYPE = EnvStr(\"auto\")\n SGLANG_EXA_INCLUDE_HIGHLIGHTS = EnvBool(True)\n\n # Hi-Cache\n SGLANG_HICACHE_HF3FS_CONFIG_PATH = EnvStr(None)\n SGLANG_HICACHE_DECODE_OFFLOAD_STRIDE = EnvInt(None)\n SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None)\n # File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, \"0\" disables).\n SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None)\n SGLANG_HICACHE_FILE_BACKEND_EVICTION_RATIO = EnvFloat(0.9)\n SGLANG_HICACHE_FILE_BACKEND_MIN_FREE_SPACE = EnvStr(\"0\")\n # Enable client-side metadata caching to optimize filesystem checks (e.g. for Lustre/NFS/FUSE)\n SGLANG_HICACHE_FILE_BACKEND_ENABLE_METADATA_CACHE = EnvBool(False)\n # Positive cache TTL for filesystem metadata lookups (-1 disables positive expiration)\n SGLANG_HICACHE_FILE_BACKEND_METADATA_TTL = EnvFloat(5.0)\n SGLANG_HICACHE_NIXL_BACKEND_STORAGE_DIR = EnvStr(None)\n # Enable O_DIRECT when opening NIXL POSIX backend files (bypasses OS page cache).\n # Disable with SGLANG_HICACHE_NIXL_USE_DIRECT_IO=0 or via the\n # \"use_direct_io\": false key in --hicache-storage-backend-extra-config.\n SGLANG_HICACHE_NIXL_USE_DIRECT_IO = EnvBool(True)\n SGLANG_HUGEPAGE_SIZE = EnvStr(\"\")\n # Staging buffer for heterogeneous TP KV transfer\n SGLANG_DISAGG_STAGING_BUFFER = EnvBool(False)\n SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB = EnvInt(64)\n SGLANG_DISAGG_STAGING_POOL_SIZE_MB = EnvInt(4096)\n # TODO(yangminl): remove SGLANG_STAGING_USE_TORCH and the torch fallback in\n # staging_buffer.py once Triton kernels are fully validated in production.\n SGLANG_STAGING_USE_TORCH = EnvBool(False)\n # Mooncake KV Transfer\n SGLANG_MOONCAKE_CUSTOM_MEM_POOL = EnvStr(None)\n ENABLE_ASCEND_TRANSFER_WITH_MOONCAKE = EnvBool(False)\n ASCEND_NPU_PHY_ID = EnvInt(-1)\n SGLANG_MOONCAKE_SEND_AUX_TCP = EnvBool(False)\n SGLANG_ENABLE_FAILED_SESSION_PROBE = EnvBool(False)\n SGLANG_FAILED_SESSION_PROBE_INTERVAL_S = EnvFloat(30.0)\n\n # Mooncake Store\n SGLANG_HICACHE_MOONCAKE_CONFIG_PATH = EnvStr(None)\n SGLANG_HICACHE_MOONCAKE_REUSE_TE = EnvBool(True)\n MOONCAKE_MASTER = EnvStr(None)\n MOONCAKE_CLIENT = EnvStr(None)\n MOONCAKE_LOCAL_HOSTNAME = EnvStr(\"localhost\")\n MOONCAKE_TE_META_DATA_SERVER = EnvStr(\"P2PHANDSHAKE\")\n MOONCAKE_GLOBAL_SEGMENT_SIZE = EnvStr(\"4gb\")\n MOONCAKE_PROTOCOL = EnvStr(\"rdma\")\n MOONCAKE_DEVICE = EnvStr(\"\")\n MOONCAKE_MASTER_METRICS_PORT = EnvInt(9003)\n MOONCAKE_CHECK_SERVER = EnvBool(False)\n MOONCAKE_STANDALONE_STORAGE = EnvBool(False)\n MOONCAKE_ENABLE_SSD_OFFLOAD = EnvBool(False)\n MOONCAKE_OFFLOAD_FILE_STORAGE_PATH = EnvStr(None)\n\n # MoRI KV Transfer\n # Send CPU-resident AUX data via RDMA instead of ZMQ TCP (default: TCP).\n SGLANG_MORI_SEND_AUX_RDMA = EnvBool(False)\n # Number of RDMA Queue Pairs (QPs) used per transfer operation. Higher\n # values can increase parallelism and bandwidth utilization.\n SGLANG_MORI_QP_PER_TRANSFER = EnvInt(4)\n # Number of RDMA work requests posted in a single batch to each QP. Larger\n # batch sizes reduce per-operation overhead and improve throughput at the\n # cost of higher latency. -1 selects automatic sizing based on the number\n # of merged work requests and available endpoints.\n SGLANG_MORI_POST_BATCH_SIZE = EnvInt(-1)\n # Number of worker threads in the RDMA executor thread pool. More workers\n # can improve parallelism for large batch transfers across multiple QPs,\n # but excessive threads may cause contention.\n SGLANG_MORI_NUM_WORKERS = EnvInt(4)\n # Number of sharded synchronous worker threads that drain KV transfers.\n # Also the bound on outstanding (posted-but-not-completed) transfers, so it\n # is the primary throttle keeping the RDMA send queue from overflowing.\n SGLANG_MORI_TRANSFER_SHARDS = EnvInt(8)\n # Poll cadence (ms) at which a transfer worker wakes to check the SLA while\n # waiting for completion; real completion still wakes it immediately.\n SGLANG_MORI_WAIT_POLL_MS = EnvInt(1000)\n # Per-transfer SLA (ms) before a KV transfer is failed; 0 disables the SLA\n # and relies on the RDMA retry-exceeded timeout only.\n SGLANG_MORI_TRANSFER_TIMEOUT_MS = EnvInt(0)\n\n # AMD & ROCm\n SGLANG_USE_AITER = EnvBool(False)\n SGLANG_USE_AITER_AG = EnvBool(True)\n # Use reduce_scatter (instead of all_reduce + dp_scatter) for the equal-chunk\n # MAX_LEN DP-MoE combine. Default ON for ROCm/HIP (uses the aiter custom\n # symmetric-memory kernel), OFF elsewhere (would fall back to RCCL); override\n # explicitly to force on/off on any platform.\n SGLANG_DP_USE_REDUCE_SCATTER = EnvBool(_default_hip)\n SGLANG_USE_AITER_UNIFIED_ATTN = EnvBool(False)\n # Select the gate/up tile layout for AITER MoE: True -> interleave\n # (matches FlyDSL `gate_mode=\"interleave\"` kernels), False -> separated\n # (matches `gate_mode=\"separated\"`, the layout used by gptoss_fp4 tuned\n # configs and by Mxfp4MoEMethod's post-fix weight shuffle).\n SGLANG_USE_AITER_MOE_GU_ITLV = EnvBool(True)\n # Fuse the `residual_add + RMSNorm + zero-pad` triplet that appears\n # before the MoE block for models whose MoE input hidden_size must be\n # padded up to a stride (e.g. GPT-OSS MXFP4 needs pad to multiple of\n # 256). When False (default) the pad runs as a separate\n # torch.nn.functional.pad call inside the MoE method. When True, the\n # aiter Triton kernel `fused_add_rmsnorm_pad` produces a padded\n # post-attention layernorm output in one launch and the MoE method\n # skips the explicit pad. Currently only takes effect on the\n # post_attention_layernorm path with aiter backend and TP=1.\n SGLANG_AITER_FUSE_RMSNORM_PAD = EnvBool(False)\n # Physical layout for MHA KV cache. \"nhd\" (default) keeps the existing\n # (size, head_num, head_dim) per-token storage that\n # `aiter.mha.mha_batch_prefill_func`/`unified_attention` consume directly.\n # \"vectorized_5d\" allocates K as (num_blocks, H_kv, head_dim/x, page_size, x)\n # and V as (num_blocks, H_kv, page_size/x, head_dim, x) (x = 16 / dtype_size),\n # matching the SHUFFLE layout that aiter's CK FmhaBatchPrefill kernel and\n # `aiter.ops.triton.gluon.pa_decode_gluon` both consume natively. This is\n # the SHUFFLE KV layout that enables pa_decode_gluon for full-attn\n # decode without runtime permutes.\n SGLANG_AITER_KV_CACHE_LAYOUT = EnvStr(\"nhd\")\n SGLANG_ROCM_FUSED_DECODE_MLA = EnvBool(False)\n SGLANG_ROCM_DISABLE_LINEARQUANT = EnvBool(False)\n SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(4096)\n # Enable dual-stream MoE (shared experts vs routed experts) on the\n # ROCm/AITER path. Requires GPU_MAX_HW_QUEUES>=5 to avoid HW-queue serialization.\n SGLANG_ROCM_USE_MULTI_STREAM = EnvBool(False)\n SGLANG_HACK_FLASHMLA_BACKEND = EnvStr(\"tilelang\")\n\n # MPS (Apple Silicon)\n SGLANG_USE_MLX = EnvBool(False)\n SGLANG_MLX_USE_CUSTOM_ROPE = EnvBool(False)\n SGLANG_MLX_FUSE_SWIGLU = EnvBool(False)\n # Number of decode steps between periodic mx.clear_cache() calls.\n # Set to 0 to disable cache clearing entirely.\n SGLANG_MLX_CLEAR_CACHE_STEPS = EnvInt(256)\n\n # NPU\n SGLANG_NPU_DISABLE_ACL_FORMAT_WEIGHT = EnvBool(False)\n SGLANG_NPU_USE_MULTI_STREAM = EnvBool(False)\n SGLANG_NPU_USE_MLAPO = EnvBool(False)\n # Forward native implementation for activation gelu tanh for model Skywork-Reward-Gemma-2-27B-v0.2\n SGLANG_NPU_FORWARD_NATIVE_GELUTANH = EnvBool(False)\n # Forward native implementation for gemma rms norm for model Skywork-Reward-Gemma-2-27B-v0.2\n SGLANG_NPU_FORWARD_NATIVE_GEMMA_RMS_NORM = EnvBool(False)\n # Delay all-gather after qlora for better performance for Deepseek v3.2\n SGLANG_USE_AG_AFTER_QLORA = EnvBool(False)\n # Master switch for the experimental TRT-LLM LoRA fast path; when OFF (default) every\n # fine-grained opt switch reads False, keeping non-experimental paths byte-identical.\n SGLANG_EXPERIMENTAL_LORA_OPTI = EnvBool(False)\n # Quantize x to int8 in the dispatch operator\n DEEP_NORMAL_MODE_USE_INT8_QUANT = EnvBool(False) # This argument is deprecated\n SGLANG_NPU_FUSED_MOE_MODE = EnvInt(1)\n\n # MTHREADS & MUSA\n SGLANG_MUSA_FA3_FORCE_UPDATE_METADATA = EnvBool(False)\n\n # Quantization\n SGLANG_INT4_WEIGHT = EnvBool(False)\n SGLANG_CPU_QUANTIZATION = EnvBool(False)\n SGLANG_USE_DYNAMIC_MXFP4_LINEAR = EnvBool(False)\n SGLANG_FORCE_FP8_MARLIN = EnvBool(False)\n SGLANG_MOE_NVFP4_DISPATCH = EnvBool(False)\n SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN = EnvBool(False)\n SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE = EnvBool(False)\n SGLANG_QUANT_ALLOW_DOWNCASTING = EnvBool(False)\n SGLANG_FP8_IGNORED_LAYERS = EnvStr(\"\")\n SGLANG_FP4_IGNORED_LAYERS = EnvStr(\"\")\n\n # Flashinfer\n SGLANG_IS_FLASHINFER_AVAILABLE = EnvBool(True)\n SGLANG_FLASHINFER_USE_PAGED = EnvBool(False)\n # Default to the pick from flashinfer\n SGLANG_FLASHINFER_WORKSPACE_SIZE = EnvInt(384 * 1024 * 1024)\n # Enable NVFP4 per-token activation scaling path for FlashInfer TRT-LLM MoE.\n SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION = EnvBool(False)\n # SGLang needs to know FlashInfer NVFP4 4over6 config to compute the global scale factor.\n FLASHINFER_NVFP4_4OVER6 = EnvBool(False)\n FLASHINFER_NVFP4_4OVER6_E4M3_USE_256 = EnvBool(False)\n # Skip-softmax threshold scale factor for TRT-LLM attention (prefill and decode separately).\n # None = standard attention. See https://arxiv.org/abs/2512.12087\n SGLANG_SKIP_SOFTMAX_PREFILL_THRESHOLD_SCALE_FACTOR = EnvFloat(None)\n SGLANG_SKIP_SOFTMAX_DECODE_THRESHOLD_SCALE_FACTOR = EnvFloat(None)\n # SM120 FlashMLA decode backend: \"flashinfer\" (default), \"triton\", or \"torch\".\n SGLANG_SM120_FLASHMLA_BACKEND = EnvStr(\"flashinfer\")\n\n # Triton\n SGLANG_TRITON_DECODE_ATTN_STATIC_KV_SPLITS = EnvBool(False)\n SGLANG_USE_CUSTOM_TRITON_KERNEL_CACHE = EnvBool(False)\n\n # Torch Compile\n SGLANG_ENABLE_TORCH_COMPILE = EnvBool(False)\n\n # EPLB\n SGLANG_EXPERT_LOCATION_UPDATER_LOG_INPUT = EnvBool(False)\n SGLANG_EXPERT_LOCATION_UPDATER_CANARY = EnvBool(False)\n SGLANG_EXPERT_LOCATION_UPDATER_LOG_METRICS = EnvBool(False)\n SGLANG_LOG_EXPERT_LOCATION_METADATA = EnvBool(False)\n SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR = EnvStr(\"/tmp\")\n SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL = EnvInt(0)\n SGLANG_ENABLE_EPLB_BALANCEDNESS_METRIC = EnvBool(False)\n\n # TBO\n SGLANG_TBO_DEBUG = EnvBool(False)\n\n # DeepGemm\n SGLANG_ENABLE_JIT_DEEPGEMM = EnvBool(True)\n SGLANG_JIT_DEEPGEMM_PRECOMPILE = EnvBool(True)\n SGLANG_JIT_DEEPGEMM_FAST_WARMUP = EnvBool(False)\n SGLANG_JIT_DEEPGEMM_COMPILE_WORKERS = EnvInt(4)\n SGLANG_IN_DEEPGEMM_PRECOMPILE_STAGE = EnvBool(False)\n SGLANG_DG_CACHE_DIR = EnvStr(os.path.expanduser(\"~/.cache/deep_gemm\"))\n SGLANG_DG_USE_NVRTC = EnvBool(False)\n SGLANG_USE_DEEPGEMM_BMM = EnvBool(False)\n SGLANG_DEEPGEMM_SANITY_CHECK = EnvBool(False)\n SGLANG_DEEPGEMM_PDL = EnvBool(True)\n SGLANG_PP_PARALLEL_DEEPGEMM_WARMUP = EnvBool(False)\n\n # DeepSeek MHA Optimization\n SGLANG_CHUNKED_PREFIX_CACHE_THRESHOLD = EnvInt(8192)\n SGLANG_MAX_KV_CHUNK_CAPACITY = EnvInt(128 * 1024)\n\n # DeepEP\n SGLANG_DEEPEP_BF16_DISPATCH = EnvBool(False) # This argument is deprecated\n SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(128)\n SGLANG_DEEPEP_LL_COMBINE_SEND_NUM_SMS = EnvInt(32)\n SGLANG_BLACKWELL_OVERLAP_SHARED_EXPERTS_OUTSIDE_SBO = EnvBool(False)\n # Force dynamic DeepEP Waterfill with runtime EP all-reduce instead of the\n # default static local-batch path.\n SGLANG_DISABLE_STATIC_WATERFILL = EnvBool(False)\n\n # NIXL-EP\n SGLANG_NIXL_EP_BF16_DISPATCH = EnvBool(False)\n SGLANG_NIXL_EP_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(128)\n\n # DSA Backend (canonical names; fall back to SGLANG_NSA_* with deprecation warning)\n SGLANG_DSA_FUSE_TOPK = EnvBoolWithAlias(\n True, deprecated_name=\"SGLANG_NSA_FUSE_TOPK\"\n )\n SGLANG_DSA_TOPK_FLASHINFER_DETERMINISTIC = EnvBool(False)\n SGLANG_DSA_TOPK_FLASHINFER_TIE_BREAK = EnvStr(None)\n SGLANG_DSA_PREFILL_DENSE_ATTN_KV_LEN_THRESHOLD = EnvIntWithAlias(\n 2048, deprecated_name=\"SGLANG_NSA_PREFILL_DENSE_ATTN_KV_LEN_THRESHOLD\"\n )\n SGLANG_DSA_HIP_DISABLE_PRESHUFFLE = EnvBoolWithAlias(\n False, deprecated_name=\"SGLANG_NSA_HIP_DISABLE_PRESHUFFLE\"\n )\n SGLANG_DSA_MQA_LOGITS_FREE_MEM_FRACTION = EnvFloat(0.2)\n SGLANG_ENABLE_PCG_DSV2_DUAL_STREAM = EnvBool(False)\n SGLANG_DSA_TOPK_BROADCAST = EnvBool(False)\n SGLANG_DISABLE_DSA_INDEXER_FUSION = EnvBool(False)\n\n # sgl-kernel\n SGLANG_SKIP_SGL_KERNEL_VERSION_CHECK = EnvBool(False)\n\n # Flash Attention\n SGLANG_USE_SGL_FA3_KERNEL = EnvBool(True)\n\n # Kernels\n USE_TRITON_W8A8_FP8_KERNEL = EnvBool(False)\n SGLANG_RETURN_ORIGINAL_LOGPROB = EnvBool(False)\n SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN = EnvBool(False)\n SGLANG_MOE_PADDING = EnvBool(False)\n SGLANG_CUTLASS_MOE = EnvBool(False)\n HF_HUB_DISABLE_XET = EnvBool(False)\n DISABLE_OPENAPI_DOC = EnvBool(False)\n SGLANG_ENABLE_TORCH_INFERENCE_MODE = EnvBool(False)\n SGLANG_IS_FIRST_RANK_ON_NODE = EnvBool(True)\n SGLANG_SYNC_TOKEN_IDS_ACROSS_TP = EnvBool(False)\n SGLANG_ENABLE_COLOCATED_BATCH_GEN = EnvBool(False)\n\n # Deterministic inference\n SGLANG_ENABLE_DETERMINISTIC_INFERENCE = EnvBool(False)\n # Use 1-stage all-reduce kernel on AMD (deterministic, fixed accumulation order)\n # If not set: auto (enabled when --enable-deterministic-inference is on)\n # Set to 1: force enable (even without --enable-deterministic-inference)\n # Set to 0: force disable (use default Aiter AR even with --enable-deterministic-inference)\n SGLANG_USE_1STAGE_ALLREDUCE = EnvBool(False)\n SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 = EnvBool(True)\n SGLANG_FLASHINFER_PREFILL_SPLIT_TILE_SIZE = EnvInt(4096)\n SGLANG_FLASHINFER_DECODE_SPLIT_TILE_SIZE = EnvInt(2048)\n SGLANG_TRITON_PREFILL_TRUNCATION_ALIGN_SIZE = EnvInt(4096)\n SGLANG_TRITON_DECODE_SPLIT_TILE_SIZE = EnvInt(256)\n\n # RoPE cache configuration\n SGLANG_SPEC_EXPANSION_SAFETY_FACTOR = EnvInt(2)\n SGLANG_ROPE_CACHE_SAFETY_MARGIN = EnvInt(256)\n SGLANG_ROPE_CACHE_ALIGN = EnvInt(128)\n\n # Overlap Spec V2\n SGLANG_ENABLE_OVERLAP_PLAN_STREAM = EnvBool(False)\n\n # Spec Config\n SGLANG_SPEC_ENABLE_STRICT_FILTER_CHECK = EnvBool(True)\n # Skip draft_extend while adaptive spec is at steps=0 (drafting disabled).\n # Saves the per-step draft forward, but the draft KV goes stale: an upshift\n # back to steps>0 starts from a cold draft state (low accept until it recovers).\n SGLANG_SPEC_SKIP_ZERO_STEP_DRAFT_EXTEND = EnvBool(False)\n # Use the split-KV (flash-decode) kernel for EAGLE target-verify on the\n # Triton backend (ROCm). Only active at speculative topk == 1; falls back to\n # extend_attention_fwd for unsupported cases or when set false (e.g. for\n # debugging). Correctness is unaffected; this only changes performance.\n SGLANG_ENABLE_SPLITKV_VERIFY = EnvBool(True)\n # Master switch for all async-asserted invariant probes (NaN, Inf, OOB,\n # page alignment). Off in prod; tests turn it on to fail-fast on\n # numerical / index violations instead of getting silent NaN cascades.\n SGLANG_ENABLE_ASYNC_ASSERT = EnvBool(False)\n # Sanitize NaN logits before sampling kernels and log a throttled warning\n # (see sanitize_nan_logits).\n SGLANG_SANITIZE_NAN_LOGITS = EnvBool(False)\n\n # VLM\n SGLANG_VLM_CACHE_SIZE_MB = EnvInt(100)\n SGLANG_IMAGE_MAX_PIXELS = EnvInt(16384 * 28 * 28)\n SGLANG_RESIZE_RESAMPLE = EnvStr(\"\")\n SGLANG_MM_BUFFER_SIZE_MB = EnvInt(0)\n SGLANG_MM_PRECOMPUTE_HASH = EnvBool(False)\n SGLANG_VIT_ENABLE_CUDA_GRAPH = EnvBool(False)\n # Use the fully-vectorized ViT position-embedding interpolation (no per-image\n # Python loop / CPU<->GPU sync). Bit-exact with the legacy implementation;\n # set False to fall back to the per-image loop.\n SGLANG_VIT_ENABLE_VECTORIZED_POS_EMBED = EnvBool(True)\n SGLANG_MM_SKIP_COMPUTE_HASH = EnvBool(False)\n # For pre-tokenized (list[int]) multimodal prompts,\n # preserve the user's original tokens to avoid retokenization drift.\n SGLANG_MM_AVOID_RETOKENIZE = EnvBool(True)\n\n # VLM Item CUDA IPC Transport\n SGLANG_USE_CUDA_IPC_TRANSPORT = EnvBool(False)\n SGLANG_USE_IPC_POOL_HANDLE_CACHE = EnvBool(False)\n SGLANG_MM_FEATURE_CACHE_MB = EnvInt(1 * 1024)\n SGLANG_MM_ITEM_MEM_POOL_RECYCLE_INTERVAL_SEC = EnvFloat(0.05)\n\n # Mamba\n SGLANG_MAMBA_CONV_DTYPE = EnvStr(\"bfloat16\")\n SGLANG_MAMBA_SSM_DTYPE = EnvStr(None)\n\n # Unified Radix Tree\n SGLANG_ENABLE_UNIFIED_RADIX_TREE = EnvBool(False)\n\n # CUDA Graph\n SGLANG_USE_BREAKABLE_CUDA_GRAPH = EnvBool(False)\n # Guards CUDA graph executable dedup via cudaGraphExecUpdate.\n SGLANG_ENABLE_CUDA_GRAPH_DEDUP = EnvBool(False)\n\n # Release & Resume Memory\n SGLANG_MEMORY_SAVER_CUDA_GRAPH = EnvBool(False)\n\n # Sparse Embeddings\n SGLANG_EMBEDDINGS_SPARSE_HEAD = EnvStr(None)\n\n # Logits processor\n SGLANG_ENABLE_LOGITS_PROCESSER_CHUNK = EnvBool(False)\n SGLANG_LOGITS_PROCESSER_CHUNK_SIZE = EnvInt(2048)\n\n # Tool-Call behavior\n SGLANG_TOOL_STRICT_LEVEL = EnvInt(ToolStrictLevel.OFF)\n\n # Think tokens budget: negative means unlimited, >= 0 caps thinking tokens\n SGLANG_MAX_THINK_TOKENS = EnvInt(-1)\n\n # Ngram\n SGLANG_NGRAM_FORCE_GREEDY_VERIFY = EnvBool(False)\n\n # Warmup\n # in seconds. If a warmup forward batch takes longer than this, the server will crash to prevent hanging.\n # Recommend to increase warmup timeout to 1800 to accommodate some kernel JIT precache e.g. deep gemm\n SGLANG_WARMUP_TIMEOUT = EnvFloat(-1)\n\n # HTTP Server\n SGLANG_TIMEOUT_KEEP_ALIVE = EnvInt(5)\n # Uvicorn multiprocess supervisor pings each worker on this interval; default 5s is\n # too short when many workers cold-start and load tokenizers in parallel.\n SGLANG_UVICORN_WORKER_HEALTHCHECK_TIMEOUT = EnvInt(10)\n\n # Health Check\n SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION = EnvBool(True)\n\n # Crash diagnostics\n SGLANG_PYSPY_DUMP_BEFORE_CRASH = EnvBool(True)\n SGLANG_CUDA_COREDUMP_BEFORE_CRASH = EnvBool(True)\n SGLANG_CUDA_COREDUMP_BEFORE_CRASH_WAIT_SECS = EnvFloat(60.0)\n\n # Encoder gRPC\n SGLANG_ENCODER_GRPC_TIMEOUT_SECS = EnvInt(60)\n # Encoder receiver selection: http|grpc (used by EPD paths).\n SGLANG_ENCODER_MM_RECEIVER_MODE = EnvStr(\"http\")\n\n # Native gRPC server. SGLANG_GRPC_PORT is the env fallback for the\n # --grpc-port CLI flag; setting either enables the native server alongside\n # HTTP. The worker-threads knob stays env-only (internal tuning, no CLI\n # surface).\n SGLANG_GRPC_PORT = EnvInt(None)\n SGLANG_GRPC_WORKER_THREADS = EnvInt(4)\n\n # External models\n SGLANG_EXTERNAL_MODEL_PACKAGE = EnvStr(\"\")\n SGLANG_EXTERNAL_MM_MODEL_ARCH = EnvStr(\"\")\n SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE = EnvStr(\"\")\n\n # Numa\n SGLANG_NUMA_BIND_V2 = EnvBool(True)\n SGLANG_AUTO_NUMA_BIND = EnvBool(False)\n SGLANG_CRASH_ON_NUMA_BIND_FAILURE = EnvBool(False)\n\n # Metrics\n SGLANG_ENABLE_METRICS_DEVICE_TIMER = EnvBool(False)\n SGLANG_ENABLE_METRICS_DP_ATTENTION = EnvBool(False)\n\n # Tokenizer (Kimi tiktoken: cache all_special_tokens / all_special_ids; the ITL can differ by +10x under high batch size).\n SGLANG_PATCH_TOKENIZER = EnvBool(True)\n\n # TokenizerManager\n SGLANG_REQUEST_STATE_WAIT_TIMEOUT = EnvInt(4)\n\n # ZBAL, zero buffer accelerate library, currently worked only in npu\n SGLANG_ZBAL_LOCAL_MEM_SIZE = EnvInt(0)\n SGLANG_ZBAL_BOOTSTRAP_URL = EnvStr(\"\")\n\n SGLANG_DEFAULT_THINKING = EnvBool(False)\n\n # ====================================================================\n # DeepSeek V4\n SGLANG_OPT_DPSK_V4_RADIX = EnvBool(True)\n SGLANG_OPT_USE_OLD_COMPRESSOR = EnvBool(False)\n SGLANG_OPT_USE_TRITON_SWA_PREPARE = EnvBool(True)\n SGLANG_OPT_USE_AITER_MHC_PRE = EnvBool(True)\n SGLANG_OPT_USE_AITER_MHC_POST = EnvBool(True)\n SGLANG_OPT_USE_AITER_SILU_MUL = EnvBool(False)\n SGLANG_OPT_USE_FUSED_COMPRESS = EnvBool(False)\n SGLANG_OPT_USE_FUSED_COMPRESS_TRITON = EnvBool(False)\n SGLANG_OPT_USE_FUSED_QK_NORM_ROPE = EnvBool(True)\n SGLANG_OPT_USE_FUSED_CLAMP_ACT_MUL = EnvBool(True)\n SGLANG_ENABLE_NVFP4_GEMM_SWIGLU_FUSION = EnvBool(True)\n SGLANG_FIX_MTP_HC_HIDDEN = EnvBool(False)\n # ====================================================================\n\n # Set False when using FP4-to-FP8 converted DeepSeek V4 checkpoint.\n SGLANG_DSV4_FP4_EXPERTS = EnvBool(True)\n SGLANG_DSV4_FP4_DEQUANT = EnvBool(False)\n # Default reasoning_effort for dsv4 chat encoder when request doesn't set it.\n # Accepts \"\", \"max\", \"high\" (empty string means unset); other values filtered to None.\n SGLANG_DSV4_REASONING_EFFORT = EnvStr(\"\")\n\n # CUDA kernels\n SGLANG_OPT_DEEPGEMM_HC_PRENORM = EnvBool(True)\n SGLANG_OPT_USE_TILELANG_MHC_PRE = EnvBool(True)\n SGLANG_OPT_USE_TILELANG_MHC_POST = EnvBool(True)\n SGLANG_DSV4_MHC_PREWARM = EnvBool(True)\n SGLANG_OPT_USE_TRITON_FUSED_MHC = EnvBool(True)\n SGLANG_OPT_FUSE_MHC_POST_PRE = EnvBool(False)\n SGLANG_OPT_USE_TILELANG_INDEXER = EnvBool(False)\n SGLANG_OPT_USE_AITER_INDEXER = EnvBool(False)\n SGLANG_OPT_DSV4_NONPAGED_INDEXER = EnvBool(True)\n # Per-rank local query rows (after DP-attention sharding when enabled),\n # not request ISL.\n SGLANG_OPT_DSV4_NONPAGED_INDEXER_MIN_QUERY_TOKENS = EnvInt(8192)\n SGLANG_OPT_USE_JIT_INDEXER_METADATA = EnvBool(True)\n SGLANG_OPT_USE_ONLINE_COMPRESS = EnvBool(False)\n SGLANG_EXPERIMENTAL_ONLINE_C128_MTP = EnvBool(False)\n SGLANG_DSV4_COMPRESS_STATE_DTYPE = EnvStr(\"float32\")\n # Deprecated: DSV4 compressor V2 is always used.\n SGLANG_OPT_USE_COMPRESSOR_V2 = EnvBool(True)\n SGLANG_FP8_PAGED_MQA_LOGITS_TORCH = EnvBool(False)\n SGLANG_TOPK_TRANSFORM_512_TORCH = EnvBool(False)\n SGLANG_OPT_FLASHMLA_SPARSE_PREFILL = EnvBool(True)\n\n # SWA radix cache\n # TODO(DSV4): @ispobock this has bug on main branch when retract\n SGLANG_OPT_SWA_RADIX_CACHE_COMPACT = EnvBool(False)\n SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT = EnvBool(False)\n SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW = EnvBool(False)\n\n # Unified radix cache\n SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS = EnvBool(False)\n\n # DeepGemm Mega MoE\n SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE = EnvBool(False)\n SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK = EnvInt(1024)\n\n # When set, the mega-MoE x slot is packed E2M1 (FP4) instead of FP8 E4M3.\n # Halves symm-buffer footprint and unlocks the MXF4 mainloop downstream.\n # Setting this also exports DG_USE_FP4_ACTS=1 so DeepGEMM's symm-buffer\n # sizing + fp8_fp4_mega_moe pick up the FP4 layout.\n SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS = EnvBool(False)\n # Switches the L1+L2 mainloops from kind::mxf8f6f4 (K=32 with-padding) to\n # kind::mxf4 (K=64 dense) inside fp8_fp4_mega_moe. No effect unless\n # SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS is also set; DeepGEMM asserts\n # this combination on the host side.\n SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND = EnvBool(False)\n SGLANG_OPT_FIX_MEGA_MOE_MEMORY = EnvBool(False)\n\n # TopK\n SGLANG_OPT_USE_FUSED_HASH_TOPK = EnvBool(True)\n SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK = EnvBool(True)\n # Opt-in: route DeepSeek-V3 grouped topk through the unified Triton router\n # instead of the flashinfer/AOT grouped kernels. Off by default (flashinfer is\n # the tuned production path); the Triton path is bit-exact on DeepSeek-V3.2 e2e\n # and benchmarks at parity, so this is a consolidation escape hatch, not a perf flip.\n SGLANG_OPT_USE_JIT_KERNEL_GROUPED_TOPK = EnvBool(False)\n SGLANG_OPT_USE_TOPK_V2 = EnvBool(True)\n\n # MiniMax-M3 sparse decode indexer: single JIT radix-select kernel replaces the 2-stage split-K Triton topk.\n SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX = EnvBool(True)\n\n # Fused JIT store (minimax_store_kv_index) of main+index K/V instead of separate\n # set_*_buffer copies; falls back when main/index dtypes differ or non-CUDA.\n SGLANG_OPT_USE_MINIMAX_FUSED_KV_INDEX_STORE = EnvBool(True)\n\n # MiniMax-M3 MXFP8 MoE experimental fusion toggles (default off; A/B only).\n SGLANG_MINIMAX_M3_FUSED_SWIGLU_MXFP8 = EnvBool(False)\n SGLANG_MINIMAX_M3_FUSED_MOE_COMBINE = EnvBool(False)\n\n # GEMM / kernel fusion\n SGLANG_OPT_FP8_WO_A_GEMM = EnvBool(True)\n SGLANG_OPT_BF16_FP32_GEMM_ALGO = EnvStr(\"cublas\")\n SGLANG_OPT_USE_JIT_EP_ACTIVATION = EnvBool(True)\n SGLANG_OPT_FUSE_WQA_WKV = EnvBool(True)\n SGLANG_OPT_SWIGLU_CLAMP_FUSION = EnvBool(True)\n\n # Cache / overlap\n SGLANG_OPT_USE_FUSED_STORE_CACHE = EnvBool(True)\n SGLANG_OPT_USE_JIT_NORM = EnvBool(True)\n SGLANG_OPT_USE_MULTI_STREAM_OVERLAP = EnvBool(True)\n\n # CUDA graph\n SGLANG_PREP_IN_CUDA_GRAPH = EnvBool(True)\n\n # Eager forward wraps the ForwardBatch's own tensors instead of copying them\n # into the CUDA graph buffer registry (no per-iter device-to-device copy).\n SGLANG_EAGER_INPUT_NO_COPY = EnvBool(False)\n\n # Distributed\n SGLANG_DSV4_FIX_TP_ATTN_A2A_SCATTER = EnvBool(True)\n SGLANG_SHARED_EXPERT_TP1 = EnvBool(False)\n # Replicate the input embedding across TP ranks instead of sharding it\n # along the vocab dimension (saves an all-reduce/all-gather in the embed\n # lookup at the cost of replicated embedding weights). Drives both the\n # target and every draft that shares its embedding (see\n # get_embedding_tp_kwargs); they must stay in lock-step. Currently only\n # applies to the Deepseek-V2 family (Deepseek V3.1, Kimi K2.5) + drafts.\n SGLANG_ENABLE_EMBED_REPLICATION = EnvBool(False)\n # Symmetric Memory\n SGLANG_SYMM_MEM_PREALLOC_GB_SIZE = EnvInt(-1)\n SGLANG_DEBUG_SYMM_MEM = EnvBool(False)\n\n # Aiter\n SGLANG_USE_AITER_FP8_PER_TOKEN = EnvBool(False)\n\n # EPD\n SGLANG_ENCODER_RECV_TIMEOUT = EnvFloat(180.0)\n SGLANG_ENCODER_SEND_TIMEOUT = EnvFloat(180.0)\n SGLANG_ENCODER_HTTP_TIMEOUT = EnvFloat(1800.0)\n SGLANG_ENCODER_REQ_TIMEOUT = EnvFloat(180.0)\n SGLANG_ENCODER_DISPATCH_MIN_ITEMS = EnvInt(2)\n SGLANG_ENCODER_IMAGE_PROCESSOR_USE_GPU = EnvBool(False)\n SGLANG_ENCODER_MAX_BATCH_SIZE = EnvInt(8)\n SGLANG_ENCODER_PREPROC_WORKERS = EnvInt(8)\n # EncoderBootstrapServer health-check tuning. Interval == 0 disables it.\n SGLANG_ENCODER_BOOTSTRAP_HEALTH_CHECK_INTERVAL = EnvFloat(10.0)\n SGLANG_ENCODER_BOOTSTRAP_HEALTH_CHECK_TIMEOUT = EnvFloat(2.0)\n # Persistent receiver-side GPU embedding pool size for mooncake EPD transport.\n # 0 disables (per-request register/deregister). 4096 = 4GB default per TP\n SGLANG_EMBEDDING_POOL_SIZE_MB = EnvInt(4096)\n SGLANG_ENCODER_DP_WORKER_MAX_INFLIGHT = EnvInt(64)\n\n # Elastic EP Backup Port\n SGLANG_BACKUP_PORT_BASE = EnvInt(10000)\n\n # Sglang Cache Dir\n SGLANG_CACHE_DIR = EnvStr(os.path.expanduser(\"~/.cache/sglang\"))\n SGLANG_FLASHINFER_AUTOTUNE_CACHE = EnvBool(True)\n SGLANG_ENABLE_MOE_DEFERRED_FINALIZE = EnvBool(False)\n\n # Plugin system\n SGLANG_PLATFORM = EnvStr(\"\")\n SGLANG_PLUGINS = EnvStr(\"\")\n\n # ===================================================================\n # KV-Canary / Token-Oracle (testing-only)\n # ===================================================================\n SGLANG_KV_CANARY_RING_CAPACITY = EnvInt(1024)\n SGLANG_KV_CANARY_STATS_PRINT_EVERY_N_STEPS = EnvInt(100)\n SGLANG_KV_CANARY_ENABLE_WRITE_INPUT_ASSERT = EnvBool(False)\n SGLANG_KV_CANARY_PERTURB_REQ_TO_TOKEN_PROB = EnvFloat(0.0)\n SGLANG_KV_CANARY_PERTURB_WARMUP_STEPS = EnvInt(50)\n SGLANG_KV_CANARY_PERTURB_REAL_KV_USED_PROB = EnvFloat(0.0)\n SGLANG_KV_CANARY_PERTURB_REAL_KV_UNUSED_CACHE_PROB = EnvFloat(0.0)\n SGLANG_KV_CANARY_PERTURB_REAL_KV_POST_FORWARD_PROB = EnvFloat(0.0)\n SGLANG_KV_CANARY_PERTURB_TARGET_GROUP = EnvStr(None)\n SGLANG_KV_CANARY_PERTURB_NEXT_TOKEN_SWAP_PROB = EnvFloat(0.0)\n SGLANG_KV_CANARY_ENABLE_TOKEN_ORACLE = EnvBool(False)\n SGLANG_KV_CANARY_ENABLE_VERIFY_TOKEN_ASSERT = EnvBool(False)\n SGLANG_KV_CANARY_SWA_DIVERGENCE_STATS_INTERVAL = EnvInt(0)\n SGLANG_KV_CANARY_ENABLE_MHA_V = EnvBool(False)\n\n\nenvs = Envs()\nEnvField._allow_set_name = False\n\n\ndef _print_deprecated_env(old_name: str, new_name: Optional[str] = None):\n if old_name in os.environ:\n if new_name is None:\n warnings.warn(f\"Environment variable {old_name} has been deprecated.\")\n else:\n warnings.warn(\n f\"Environment variable {old_name} will be deprecated, please use {new_name} instead\"\n )\n os.environ[new_name] = os.environ[old_name]\n\n\ndef _warn_deprecated_env_to_cli_flag(env_name: str, suggestion: str):\n \"\"\"Warn when a deprecated environment variable is used.\n\n This is for env vars that are deprecated in favor of CLI flags.\n \"\"\"\n if env_name in os.environ:\n warnings.warn(f\"Environment variable {env_name} is deprecated. {suggestion}\")\n\n\ndef _convert_SGL_to_SGLANG():\n _print_deprecated_env(\"SGLANG_GC_LOG\", \"SGLANG_LOG_GC\")\n _print_deprecated_env(\n \"SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH\", \"SGLANG_MOE_NVFP4_DISPATCH\"\n )\n _print_deprecated_env(\n \"SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK\",\n \"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK\",\n )\n _print_deprecated_env(\"SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2\")\n _print_deprecated_env(\"SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN\")\n _print_deprecated_env(\"SGLANG_ENABLE_THINKING\", \"SGLANG_DEFAULT_THINKING\")\n _print_deprecated_env(\"SGLANG_REASONING_EFFORT\", \"SGLANG_DSV4_REASONING_EFFORT\")\n _print_deprecated_env(\n \"SGLANG_USE_JIT_ALL_REDUCE\", \"SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2\"\n )\n _deprecated_ms_to_s = {\n \"SGLANG_QUEUED_TIMEOUT_MS\": \"SGLANG_REQ_WAITING_TIMEOUT\",\n \"SGLANG_FORWARD_TIMEOUT_MS\": \"SGLANG_REQ_RUNNING_TIMEOUT\",\n }\n for old_name, new_name in _deprecated_ms_to_s.items():\n if old_name in os.environ:\n ms_val = os.environ[old_name]\n warnings.warn(\n f\"Environment variable {old_name} (in ms) is deprecated, \"\n f\"please use {new_name} (in seconds) instead\"\n )\n os.environ[new_name] = str(float(ms_val) / 1000.0)\n\n for key, value in os.environ.items():\n if key.startswith(\"SGL_\"):\n new_key = key.replace(\"SGL_\", \"SGLANG_\", 1)\n warnings.warn(\n f\"Environment variable {key} is deprecated, please use {new_key}\"\n )\n os.environ[new_key] = value\n\n\n_convert_SGL_to_SGLANG()\n_warn_deprecated_env_to_cli_flag(\n \"SGLANG_ENABLE_GRPC\",\n \"Please use '--grpc-port' to enable the native gRPC server.\",\n)\n_warn_deprecated_env_to_cli_flag(\n \"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE\",\n \"Please use '--enable-prefill-delayer' instead.\",\n)\n_warn_deprecated_env_to_cli_flag(\n \"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES\",\n \"Please use '--prefill-delayer-max-delay-passes' instead.\",\n)\n_warn_deprecated_env_to_cli_flag(\n \"SGLANG_PREFILL_DELAYER_TOKEN_USAGE_LOW_WATERMARK\",\n \"Please use '--prefill-delayer-token-usage-low-watermark' instead.\",\n)\n_warn_deprecated_env_to_cli_flag(\n \"SGLANG_DFLASH_PREFILL_REFILL_TARGET\",\n \"DFlash now auto-enables the min-free-slots delay; unset this env. To \"\n \"override the threshold, use '--min-free-slots-delay'.\",\n)\n\n# Import cuda_coredump to trigger auto-injection of CUDA env vars\n# when SGLANG_CUDA_COREDUMP=1. Best-effort; for strict guarantees,\n# set CUDA_* env vars in the shell before launching Python.\nimport sglang.srt.debug_utils.cuda_coredump # noqa: F401, E402 # isort: skip\n\n\ndef example_with_exit_stack():\n # Use this style of context manager in unit test\n exit_stack = ExitStack()\n exit_stack.enter_context(envs.SGLANG_TEST_RETRACT.override(False))\n assert envs.SGLANG_TEST_RETRACT.get() is False\n exit_stack.close()\n assert envs.SGLANG_TEST_RETRACT.get() is None\n\n\ndef example_with_subprocess():\n command = [\"python\", \"-c\", \"import os; print(os.getenv('SGLANG_TEST_RETRACT'))\"]\n with envs.SGLANG_TEST_RETRACT.override(True):\n process = subprocess.Popen(\n command, stdout=subprocess.PIPE, stderr=subprocess.PIPE\n )\n process.wait()\n output = process.stdout.read().decode(\"utf-8\").strip()\n assert output == \"True\"\n\n process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n output = process.stdout.read().decode(\"utf-8\").strip()\n assert output == \"None\"\n\n\ndef example_with_implicit_bool_avoidance():\n @contextmanager\n def assert_throws(message_matcher: str):\n try:\n yield\n except Exception as e:\n assert message_matcher in str(e), f\"{e=}\"\n print(f\"assert_throws find expected error: {e}\")\n return\n raise AssertionError(\"assert_throws do not see exceptions\")\n\n with assert_throws(\"Please use `envs.YOUR_FLAG.get()` instead of `envs.YOUR_FLAG`\"):\n if envs.SGLANG_TEST_RETRACT:\n pass\n\n with assert_throws(\"Please use `envs.YOUR_FLAG.get()` instead of `envs.YOUR_FLAG`\"):\n if (1 != 1) or envs.SGLANG_TEST_RETRACT:\n pass\n\n with assert_throws(\"Please use `envs.YOUR_FLAG.get()` instead of `envs.YOUR_FLAG`\"):\n if envs.SGLANG_TEST_RETRACT or (1 == 1):\n pass\n\n\ndef examples():\n # Example usage for envs\n envs.SGLANG_TEST_RETRACT.clear()\n assert envs.SGLANG_TEST_RETRACT.get() is False\n\n envs.SGLANG_TEST_RETRACT.set(None)\n assert envs.SGLANG_TEST_RETRACT.is_set() and envs.SGLANG_TEST_RETRACT.get() is None\n\n envs.SGLANG_TEST_RETRACT.clear()\n assert not envs.SGLANG_TEST_RETRACT.is_set()\n\n envs.SGLANG_TEST_RETRACT.set(True)\n assert envs.SGLANG_TEST_RETRACT.get() is True\n\n with envs.SGLANG_TEST_RETRACT.override(None):\n assert (\n envs.SGLANG_TEST_RETRACT.is_set() and envs.SGLANG_TEST_RETRACT.get() is None\n )\n\n assert envs.SGLANG_TEST_RETRACT.get() is True\n\n envs.SGLANG_TEST_RETRACT.set(None)\n with envs.SGLANG_TEST_RETRACT.override(True):\n assert envs.SGLANG_TEST_RETRACT.get() is True\n\n assert envs.SGLANG_TEST_RETRACT.is_set() and envs.SGLANG_TEST_RETRACT.get() is None\n\n example_with_exit_stack()\n example_with_subprocess()\n example_with_implicit_bool_avoidance()\n\n\nif __name__ == \"__main__\":\n examples()\n"}}, ".vendor/sglang/docs_new/cookbook/autoregressive/Meituan/LongCat-2.0.mdx": {"b78a5a456858bb5f7dd7a4f6aa691ac81637f48e": {"t": 1785674294.5951118, "n": 7563, "h": "083250ac404c1091", "c": "---\ntitle: LongCat-2.0\ndescription: \"Deploy LongCat-2.0-FP8 with SGLang - config-driven recipes for Meituan's 1.6T sparse MoE model on B300, B200, H200, and H20 GPUs.\"\ntag: NEW\n---\n\n## Deployment\n\n<a id=\"install\" />\n\n<Accordion title=\"Install SGLang\">\n\nFor all methods and hardware platforms, see the [official SGLang installation guide](../../../docs/get-started/install). LongCat-2.0 support is on SGLang `main`; use a nightly wheel or rolling nightly Docker image until the next tagged release includes it. The two paths below match the **Python / Docker** toggle in the command panel.\n\n<Tabs>\n\n<Tab title=\"Python (pip / uv)\">\n\n```bash Command\npip install --upgrade pip\npip install uv\n\n# Choose the nightly wheel index for your CUDA runtime.\nSGLANG_WHL_INDEX=https://docs.sglang.ai/whl/cu130 # B300 / CUDA 13\n# SGLANG_WHL_INDEX=https://docs.sglang.ai/whl/cu129 # CUDA 12.9\n\nuv pip install --prerelease=allow --extra-index-url \"${SGLANG_WHL_INDEX}\" \"sglang[all]\"\n```\n\nThen run the **Python** output of the command panel below in that environment.\n\n</Tab>\n\n<Tab title=\"Docker\">\n\n```bash Command\n# Choose the rolling nightly image for your hardware.\nSGLANG_DOCKER_IMAGE=lmsysorg/sglang:dev-cu13 # B300 / CUDA 13\n# SGLANG_DOCKER_IMAGE=lmsysorg/sglang:dev # Other supported hardware\n\ndocker pull \"${SGLANG_DOCKER_IMAGE}\"\n```\n\nFor how to launch the image, see [Install -> Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker). Substitute the inner `sglang serve ...` with what the command generator below produces.\n\n</Tab>\n\n</Tabs>\n\n</Accordion>\n\nPick your hardware + recipe to generate the launch command. LongCat-2.0 currently exposes one model-card-aligned serving strategy:\n\n- **Balanced** - the validated B300 recipe and the 2-node H200/B200/H20 topology use TP/EP parallelism with LongCat sparse attention prefill.\n\nimport { Deployment } from \"/src/snippets/_deployment.jsx\";\nimport { config } from \"/src/snippets/configs/meituan-longcat/longcat-2.0.jsx\";\nimport { benchmarks } from \"/src/snippets/configs/meituan-longcat/longcat-2.0-benchmarks.jsx\";\n\n<Deployment config={config} benchmarks={benchmarks} />\n\n<Note>\n The B300 single-node recipe was validated end-to-end with CUDA graph capture enabled. H200, B200, and H20 are shown as 2-node recipes because LongCat-2.0-FP8 needs 16 ranks for those GPU memory profiles.\n</Note>\n\n## Playground\n\nThe Playground is where you experiment with **SGLang features beyond the verified matrix**. The Deploy panel above only emits combinations the SGLang team has signed off on; the Playground lets you turn on additional knobs on top of whichever cell the Deploy panel is currently showing.\n\nimport { Playground } from \"/src/snippets/_playground.jsx\";\n\n<Playground config={config} />\n\n## 1. Model Introduction\n\n[LongCat-2.0-FP8](https://huggingface.co/meituan-longcat/LongCat-2.0-FP8) is the FP8 checkpoint of Meituan LongCat-2.0, a large sparse Mixture-of-Experts language model with 1.6T total parameters and about 48B activated parameters per token. It combines LongCat Sparse Attention (LSA), expert parallel MoE layers, and an n-gram/token-table embedding path for serving long-context workloads efficiently.\n\n<table style={{width: \"100%\", borderCollapse: \"collapse\", tableLayout: \"fixed\"}}>\n <thead>\n <tr style={{borderBottom: \"2px solid #d55816\"}}>\n <th style={{textAlign: \"left\", padding: \"10px 12px\", fontWeight: 700}}>Model</th>\n <th style={{textAlign: \"left\", padding: \"10px 12px\", fontWeight: 700}}>Architecture</th>\n <th style={{textAlign: \"left\", padding: \"10px 12px\", fontWeight: 700}}>Serving precision</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <td style={{padding: \"9px 12px\"}}><strong><a href=\"https://huggingface.co/meituan-longcat/LongCat-2.0-FP8\">LongCat-2.0-FP8</a></strong></td>\n <td style={{padding: \"9px 12px\"}}>Sparse MoE · LongCat Sparse Attention · n-gram embedding</td>\n <td style={{padding: \"9px 12px\"}}>FP8 weights, BF16 KV cache</td>\n </tr>\n </tbody>\n</table>\n\n**Resources:** [LongCat-2.0-FP8](https://huggingface.co/meituan-longcat/LongCat-2.0-FP8).\n\n## 2. Configuration Tips\n\n- **Remote code.** Use `--trust-remote-code` for the Hugging Face checkpoint.\n- **Topology.** The 8x B300 recipe uses TP=8 and EP=8. H200, B200, and H20 use a 2-node 16 GPU layout with TP=16 and EP=16; the command panel injects the multi-node rank flags for you.\n- **LongCat sparse attention.** Keep `--nsa-prefill-backend fa3` with `--chunked-prefill-size 2048` for the model-card-aligned prefill path.\n- **Memory.** The recipe uses `--kv-cache-dtype bfloat16` and starts at `--mem-fraction-static 0.92`. Tune memory only after the generated command launches cleanly on your cluster.\n- **Weight loading.** `--model-loader-extra-config '{\"enable_multithread_load\":true,\"num_threads\":12}'` loads checkpoint shards in parallel and reduces startup time.\n- **FP8 backend selection.** Do not pass `--fp8-gemm-runner-backend` manually. SGLang selects the correct backend for the LongCat FP8 scale layout.\n- **Host, port, and ranks.** Use the command panel environment fields for `HOST_IP`, `PORT`, `NODE0_IP`, and `NODE_RANK` instead of hardcoding them in the recipe.\n\n## 3. Advanced Usage\n\n### 3.1 Test the deployment\n\n<Accordion title=\"Chat completion example (cURL)\">\n\n```bash Command\ncurl http://localhost:30000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"meituan-longcat/LongCat-2.0-FP8\",\n \"messages\": [\n {\"role\": \"user\", \"content\": \"A shop has 17 apples and sells 8. Then it buys 6 more. How many apples are there? Answer with only the final number.\"}\n ],\n \"max_tokens\": 32,\n \"chat_template_kwargs\": {\"enable_thinking\": false}\n }'\n```\n\n</Accordion>\n\n<Accordion title=\"Expected output\">\n\n```text Output\n15\n```\n\n</Accordion>\n\n<Accordion title=\"OpenAI-compatible client (Python)\">\n\n```python Example\nfrom openai import OpenAI\n\nclient = OpenAI(\n base_url=\"http://localhost:30000/v1\",\n api_key=\"EMPTY\",\n)\n\nresponse = client.chat.completions.create(\n model=\"meituan-longcat/LongCat-2.0-FP8\",\n messages=[\n {\n \"role\": \"user\",\n \"content\": \"Solve: A shop has 17 apples and sells 8, then buys 6 more. Answer with only the final number.\",\n }\n ],\n max_tokens=32,\n extra_body={\"chat_template_kwargs\": {\"enable_thinking\": False}},\n)\n\nprint(response.choices[0].message.content)\n```\n\n</Accordion>\n\n<Accordion title=\"Example output\">\n\n```text Output\n15\n```\n\n</Accordion>\n\n## 4. Validation\n\nThe B300 recipe was validated with `meituan-longcat/LongCat-2.0-FP8` on 8x B300 using the command generated above.\n\n<table style={{width: \"100%\", borderCollapse: \"collapse\", tableLayout: \"fixed\"}}>\n <thead>\n <tr style={{borderBottom: \"2px solid #d55816\"}}>\n <th style={{textAlign: \"left\", padding: \"10px 12px\", fontWeight: 700}}>Evaluation</th>\n <th style={{textAlign: \"left\", padding: \"10px 12px\", fontWeight: 700}}>Examples</th>\n <th style={{textAlign: \"left\", padding: \"10px 12px\", fontWeight: 700}}>Accuracy</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <td style={{padding: \"9px 12px\"}}>GSM8K</td>\n <td style={{padding: \"9px 12px\"}}>200</td>\n <td style={{padding: \"9px 12px\"}}>98.0%</td>\n </tr>\n <tr>\n <td style={{padding: \"9px 12px\"}}>GSM8K</td>\n <td style={{padding: \"9px 12px\"}}>1314</td>\n <td style={{padding: \"9px 12px\"}}>95.8904109589041%</td>\n </tr>\n </tbody>\n</table>\n\nCUDA graph was enabled, and decode CUDA graph capture completed successfully during serving validation.\n"}}}} |