Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,7 @@ base:
model: {}
container: {}
dynamo:
install: true
source:
wheel: "1.6.0.dev20261016"
install: false
environment:
HF_HUB_OFFLINE: "1"
HF_HUB_CACHE: /hf_hub_cache
Expand Down Expand Up @@ -59,6 +57,8 @@ base:
PYTHONUNBUFFERED: "1"
SGLANG_TIMEOUT_KEEP_ALIVE: "900"
SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1"
SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION: "0"
SGLANG_TRTLLM_MHA_FULL_ATTN_KV_SPLITS: "16"
args:
served-model-name: "meta-models/Muse-Spark-Open-816B-A42B-NVFP4"
trust-remote-code: true
Expand All @@ -83,92 +83,82 @@ base:
enable-metrics: true
weight-loader-drop-cache-after-load: false
model-loader-extra-config: '{"enable_multithread_load":true}'
stream-interval: 20
sbatch_directives:
mem: "0"
srun_options:
mem: "0"
benchmark:
env:
HF_HUB_OFFLINE: "0"
TOKENIZER: /model
AIPERF_HTTP_TCP_USER_TIMEOUT: "900000"
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:"
AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true"
AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
AGENTIC_WARMUP_GRACE_PERIOD: "3600"

override_tp4_c40:
override_tp4_c56:
roles:
agg:
env:
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096"
SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: "4"
args:
mem-fraction-static: 0.95
swa-full-tokens-ratio: 0.3
max-running-requests: 24
cuda-graph-max-bs-decode: 24
swa-full-tokens-ratio: 0.2
max-running-requests: 32
cuda-graph-max-bs-decode: 32
chunked-prefill-size: 16384
max-prefill-tokens: 16384
cuda-graph-backend-prefill: breakable
schedule-policy: shortest-prefill-first
prefill-decode-interval: 32
schedule-policy: fcfs
prefill-decode-interval: 8
enable-hierarchical-cache: true
hicache-size: 175
hicache-size: 140
hicache-write-policy: write_back
speculative-draft-attention-backend: trtllm_mha

override_tp4_c32:
override_tp4_c50:
roles:
agg:
env:
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096"
args:
mem-fraction-static: 0.95
swa-full-tokens-ratio: 0.3
swa-full-tokens-ratio: 0.2
max-running-requests: 32
cuda-graph-max-bs-decode: 32
chunked-prefill-size: 8192
schedule-policy: shortest-prefill-first
prefill-decode-interval: 56
chunked-prefill-size: 16384
max-prefill-tokens: 16384
cuda-graph-backend-prefill: breakable
schedule-policy: fcfs
prefill-decode-interval: 20
enable-hierarchical-cache: true
hicache-size: 170
hicache-write-policy: write_through
hicache-size: 140
hicache-write-policy: write_back
speculative-draft-attention-backend: trtllm_mha

override_tp4_c20:
override_tp4_c46:
roles:
agg:
env:
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096"
args:
mem-fraction-static: 0.95
swa-full-tokens-ratio: 0.3
max-running-requests: 10
cuda-graph-max-bs-decode: 10
chunked-prefill-size: 8192
schedule-policy: shortest-prefill-first
prefill-decode-interval: 48
swa-full-tokens-ratio: 0.2
max-running-requests: 32
cuda-graph-max-bs-decode: 32
chunked-prefill-size: 16384
max-prefill-tokens: 16384
cuda-graph-backend-prefill: breakable
schedule-policy: fcfs
prefill-decode-interval: 24
enable-hierarchical-cache: true
hicache-size: 120
hicache-write-policy: write_through

override_tp4_c16:
roles:
agg:
args:
mem-fraction-static: 0.95
swa-full-tokens-ratio: 0.1
max-running-requests: 12
cuda-graph-max-bs-decode: 12
chunked-prefill-size: 8192
schedule-policy: shortest-prefill-first
prefill-decode-interval: 128

override_tp4_c8:
roles:
agg:
args:
mem-fraction-static: 0.88
swa-full-tokens-ratio: 0.3
max-running-requests: 8
cuda-graph-max-bs-decode: 8
chunked-prefill-size: 8192
prefill-decode-interval: 64
hicache-size: 140
hicache-write-policy: write_back
speculative-draft-attention-backend: trtllm_mha

override_tp8_c14:
override_tp8_c36:
roles:
agg:
nodes: 2
Expand All @@ -181,47 +171,13 @@ override_tp8_c14:
expert-parallel-size: 8
mem-fraction-static: 0.88
swa-full-tokens-ratio: 0.3
max-running-requests: 14
cuda-graph-max-bs-decode: 14
prefill-decode-interval: 64
max-running-requests: 36
cuda-graph-max-bs-decode: 36
schedule-policy: shortest-prefill-first
prefill-decode-interval: 128
speculative-draft-attention-backend: fa4

override_tp8_c10:
roles:
agg:
nodes: 2
env:
NCCL_CUMEM_ENABLE: "1"
NCCL_MNNVL_ENABLE: "1"
PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True"
args:
tensor-parallel-size: 8
expert-parallel-size: 8
mem-fraction-static: 0.88
swa-full-tokens-ratio: 0.3
max-running-requests: 2
cuda-graph-max-bs-decode: 2
prefill-decode-interval: 64

override_tp8_c4:
roles:
agg:
nodes: 2
env:
NCCL_CUMEM_ENABLE: "1"
NCCL_MNNVL_ENABLE: "1"
PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True"
args:
tensor-parallel-size: 8
expert-parallel-size: 8
mem-fraction-static: 0.88
swa-full-tokens-ratio: 0.3
max-running-requests: 2
cuda-graph-max-bs-decode: 2
prefill-decode-interval: 256
page-size: 128

override_tp8_c3:
override_tp8_c1:
roles:
agg:
nodes: 2
Expand All @@ -237,4 +193,4 @@ override_tp8_c3:
max-running-requests: 1
cuda-graph-max-bs-decode: 1
prefill-decode-interval: 64
page-size: 128
speculative-draft-attention-backend: trtllm_mha
Loading
Loading