diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/agg-variants.yaml index 41fb29a8a9..fb1a63b943 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/agg-variants.yaml @@ -6,9 +6,7 @@ base: model: {} container: {} dynamo: - install: true - source: - wheel: "1.6.0.dev20261016" + install: false environment: HF_HUB_OFFLINE: "1" HF_HUB_CACHE: /hf_hub_cache @@ -59,6 +57,8 @@ base: PYTHONUNBUFFERED: "1" SGLANG_TIMEOUT_KEEP_ALIVE: "900" SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION: "0" + SGLANG_TRTLLM_MHA_FULL_ATTN_KV_SPLITS: "16" args: served-model-name: "meta-models/Muse-Spark-Open-816B-A42B-NVFP4" trust-remote-code: true @@ -83,12 +83,14 @@ base: enable-metrics: true weight-loader-drop-cache-after-load: false model-loader-extra-config: '{"enable_multithread_load":true}' + stream-interval: 20 sbatch_directives: mem: "0" srun_options: mem: "0" benchmark: env: + HF_HUB_OFFLINE: "0" TOKENIZER: /model AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" @@ -96,79 +98,67 @@ base: AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AGENTIC_WARMUP_GRACE_PERIOD: "3600" -override_tp4_c40: +override_tp4_c56: roles: agg: env: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: "4" args: mem-fraction-static: 0.95 - swa-full-tokens-ratio: 0.3 - max-running-requests: 24 - cuda-graph-max-bs-decode: 24 + swa-full-tokens-ratio: 0.2 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 chunked-prefill-size: 16384 max-prefill-tokens: 16384 cuda-graph-backend-prefill: breakable - schedule-policy: shortest-prefill-first - prefill-decode-interval: 32 + schedule-policy: fcfs + prefill-decode-interval: 8 enable-hierarchical-cache: true - hicache-size: 175 + hicache-size: 140 + hicache-write-policy: write_back + speculative-draft-attention-backend: trtllm_mha -override_tp4_c32: +override_tp4_c50: roles: agg: + env: + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" args: mem-fraction-static: 0.95 - swa-full-tokens-ratio: 0.3 + swa-full-tokens-ratio: 0.2 max-running-requests: 32 cuda-graph-max-bs-decode: 32 - chunked-prefill-size: 8192 - schedule-policy: shortest-prefill-first - prefill-decode-interval: 56 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + cuda-graph-backend-prefill: breakable + schedule-policy: fcfs + prefill-decode-interval: 20 enable-hierarchical-cache: true - hicache-size: 170 - hicache-write-policy: write_through + hicache-size: 140 + hicache-write-policy: write_back + speculative-draft-attention-backend: trtllm_mha -override_tp4_c20: +override_tp4_c46: roles: agg: + env: + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" args: mem-fraction-static: 0.95 - swa-full-tokens-ratio: 0.3 - max-running-requests: 10 - cuda-graph-max-bs-decode: 10 - chunked-prefill-size: 8192 - schedule-policy: shortest-prefill-first - prefill-decode-interval: 48 + swa-full-tokens-ratio: 0.2 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + cuda-graph-backend-prefill: breakable + schedule-policy: fcfs + prefill-decode-interval: 24 enable-hierarchical-cache: true - hicache-size: 120 - hicache-write-policy: write_through - -override_tp4_c16: - roles: - agg: - args: - mem-fraction-static: 0.95 - swa-full-tokens-ratio: 0.1 - max-running-requests: 12 - cuda-graph-max-bs-decode: 12 - chunked-prefill-size: 8192 - schedule-policy: shortest-prefill-first - prefill-decode-interval: 128 - -override_tp4_c8: - roles: - agg: - args: - mem-fraction-static: 0.88 - swa-full-tokens-ratio: 0.3 - max-running-requests: 8 - cuda-graph-max-bs-decode: 8 - chunked-prefill-size: 8192 - prefill-decode-interval: 64 + hicache-size: 140 + hicache-write-policy: write_back + speculative-draft-attention-backend: trtllm_mha -override_tp8_c14: +override_tp8_c36: roles: agg: nodes: 2 @@ -181,47 +171,13 @@ override_tp8_c14: expert-parallel-size: 8 mem-fraction-static: 0.88 swa-full-tokens-ratio: 0.3 - max-running-requests: 14 - cuda-graph-max-bs-decode: 14 - prefill-decode-interval: 64 + max-running-requests: 36 + cuda-graph-max-bs-decode: 36 + schedule-policy: shortest-prefill-first + prefill-decode-interval: 128 speculative-draft-attention-backend: fa4 -override_tp8_c10: - roles: - agg: - nodes: 2 - env: - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" - args: - tensor-parallel-size: 8 - expert-parallel-size: 8 - mem-fraction-static: 0.88 - swa-full-tokens-ratio: 0.3 - max-running-requests: 2 - cuda-graph-max-bs-decode: 2 - prefill-decode-interval: 64 - -override_tp8_c4: - roles: - agg: - nodes: 2 - env: - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" - args: - tensor-parallel-size: 8 - expert-parallel-size: 8 - mem-fraction-static: 0.88 - swa-full-tokens-ratio: 0.3 - max-running-requests: 2 - cuda-graph-max-bs-decode: 2 - prefill-decode-interval: 256 - page-size: 128 - -override_tp8_c3: +override_tp8_c1: roles: agg: nodes: 2 @@ -237,4 +193,4 @@ override_tp8_c3: max-running-requests: 1 cuda-graph-max-bs-decode: 1 prefill-decode-interval: 64 - page-size: 128 + speculative-draft-attention-backend: trtllm_mha diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..1fdf2655f3 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,319 @@ +schema: 2 + +base: + name: "musespark-fp4-gb300-dynamo-sglang-agentx-disagg" + identity: + model: {} + container: {} + dynamo: + install: false + environment: + HF_HUB_OFFLINE: "1" + HF_HUB_CACHE: /hf_hub_cache + slurm: + time_limit: "4:00:00" + health_check: + max_attempts: 1440 + interval_seconds: 5 + resources: + gpu_type: "gb300" + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + DYN_ROUTER_USE_KV_EVENTS: "false" + DYN_ROUTER_TTL_SECS: "14400" + args: + router-mode: "kv" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + placement: + node: head + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + prefill: + env: + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + HF_HUB_CACHE: /hf_hub_cache + HF_HUB_OFFLINE: "1" + MC_FORCE_MNNVL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + PYTHONNOUSERSITE: "1" + PYTHONUNBUFFERED: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "14400" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "14400" + SGLANG_DISAGG_PREFILL_EARLY_SEND_CACHED_PREFIX: "0" + SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION: "0" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32" + SGLANG_DISAGGREGATION_QUEUE_SIZE: "8" + args: + chunked-prefill-size: 32768 + context-length: 1048576 + cuda-graph-max-bs-decode: 64 + disaggregation-transfer-backend: mooncake + dyn-reasoning-parser: muse + dyn-tool-call-parser: muse + enable-cache-report: true + enable-hierarchical-cache: true + enable-layernorm-sp: true + enable-megamoe-for-layernorm-sp: true + enable-metrics: true + expert-parallel-size: 8 + hicache-size: 150 + hicache-write-policy: write_back + kv-cache-dtype: fp8_e4m3 + max-prefill-tokens: 32768 + max-running-requests: 256 + mem-fraction-static: 0.85 + mm-feature-transport: cpu + moe-runner-backend: flashinfer_trtllm + optimistic-prefill-attempts: 0 + preferred-sampling-params: '{"temperature":1.0,"top_p":1.0,"top_k":64}' + quantization: modelopt_fp4 + reasoning-parser: muse + schedule-policy: shortest-prefill-first + served-model-name: "meta-models/Muse-Spark-Open-816B-A42B-NVFP4" + speculative-algorithm: DFLASH + speculative-draft-model-path: "RadixArk/Muse-Spark-Open-DFlash2" + speculative-draft-model-revision: "46c90f412549c472941a68382d91a99e5e6372a0" + speculative-num-draft-tokens: 8 + stream-interval: 20 + swa-full-tokens-ratio: 0.16 + tensor-parallel-size: 8 + trust-remote-code: true + watchdog-timeout: 3600 + weight-loader-drop-cache-after-load: false + model-loader-extra-config: '{"enable_multithread_load":true}' + speculative-draft-attention-backend: trtllm_mha + decode: + env: + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + HF_HUB_CACHE: /hf_hub_cache + HF_HUB_OFFLINE: "1" + MC_FORCE_MNNVL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + PYTHONNOUSERSITE: "1" + PYTHONUNBUFFERED: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "14400" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "14400" + SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION: "0" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_TRTLLM_MHA_FULL_ATTN_KV_SPLITS: "16" + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32" + SGLANG_DISAGGREGATION_QUEUE_SIZE: "8" + SGLANG_DISAGG_PREFILL_EARLY_SEND_CACHED_PREFIX: "0" + args: + context-length: 1048576 + disaggregation-transfer-backend: mooncake + dyn-reasoning-parser: muse + dyn-tool-call-parser: muse + enable-cache-report: true + enable-layernorm-sp: false + enable-megamoe-for-layernorm-sp: false + enable-metrics: true + expert-parallel-size: 8 + kv-cache-dtype: fp8_e4m3 + mem-fraction-static: 0.92 + mm-feature-transport: cpu + moe-runner-backend: flashinfer_trtllm + preferred-sampling-params: '{"temperature":1.0,"top_p":1.0,"top_k":64}' + quantization: modelopt_fp4 + reasoning-parser: muse + served-model-name: "meta-models/Muse-Spark-Open-816B-A42B-NVFP4" + speculative-algorithm: DFLASH + speculative-draft-model-path: "RadixArk/Muse-Spark-Open-DFlash2" + speculative-draft-model-revision: "46c90f412549c472941a68382d91a99e5e6372a0" + speculative-num-draft-tokens: 8 + stream-interval: 20 + swa-full-tokens-ratio: 0.1 + tensor-parallel-size: 8 + trust-remote-code: true + watchdog-timeout: 3600 + weight-loader-drop-cache-after-load: false + model-loader-extra-config: '{"enable_multithread_load":true}' + speculative-draft-attention-backend: trtllm_mha + sbatch_directives: + mem: "0" + srun_options: + mem: "0" + benchmark: + placement: + node: head + env: + HF_HUB_OFFLINE: "0" + TOKENIZER: /model + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AGENTIC_WARMUP_GRACE_PERIOD: "3600" + +override_3p4d_tp8_tp8_c672: + roles: + prefill: + nodes: 6 + workers: 3 + gpus: 8 + decode: + nodes: 8 + workers: 4 + gpus: 8 + args: + cuda-graph-max-bs-decode: 56 + max-running-requests: 56 + disaggregation-decode-extra-slots: 56 + frontend: + args: + router-session-affinity-mode: soft + router-session-affinity-ttl-secs: 14400 + +override_2p3d_tp8_tp8_c352: + roles: + prefill: + nodes: 4 + workers: 2 + gpus: 8 + decode: + nodes: 6 + workers: 3 + gpus: 8 + args: + cuda-graph-max-bs-decode: 40 + max-running-requests: 40 + disaggregation-decode-extra-slots: 40 + frontend: + args: + router-session-affinity-mode: soft + router-session-affinity-ttl-secs: 14400 + +override_1p2d_tp8_tp8_c224: + roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + decode: + nodes: 4 + workers: 2 + gpus: 8 + args: + cuda-graph-max-bs-decode: 36 + max-running-requests: 36 + disaggregation-decode-extra-slots: 36 + +override_1p2d_tp8_tp8_c160: + roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + decode: + nodes: 4 + workers: 2 + gpus: 8 + args: + cuda-graph-max-bs-decode: 24 + max-running-requests: 24 + disaggregation-decode-extra-slots: 24 + +override_1p3d_tp8_tp8_c224: + roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + decode: + nodes: 6 + workers: 3 + gpus: 8 + args: + cuda-graph-max-bs-decode: 20 + max-running-requests: 20 + disaggregation-decode-extra-slots: 20 + +override_1p4d_tp8_tp8_c112: + roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + decode: + nodes: 8 + workers: 4 + gpus: 8 + args: + cuda-graph-max-bs-decode: 6 + max-running-requests: 6 + disaggregation-decode-extra-slots: 6 + +override_1p5d_tp8_tp8_c120: + roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + decode: + nodes: 10 + workers: 5 + gpus: 8 + args: + cuda-graph-max-bs-decode: 5 + max-running-requests: 5 + disaggregation-decode-extra-slots: 5 + +override_1p4d_tp8_tp8_c56: + roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + decode: + nodes: 8 + workers: 4 + gpus: 8 + args: + cuda-graph-max-bs-decode: 3 + max-running-requests: 3 + disaggregation-decode-extra-slots: 3 + +override_1p4d_tp8_tp8_c32: + roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + decode: + nodes: 8 + workers: 4 + gpus: 8 + args: + cuda-graph-max-bs-decode: 3 + max-running-requests: 3 + disaggregation-decode-extra-slots: 3 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index a095e1f681..e5004a0760 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -6949,14 +6949,14 @@ musespark-fp4-gb300-sglang-agentic-dflash: search-space: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [24, 30, 40, 44, 48], srt-recipe: agentic.yaml } -musespark-fp4-gb300-dynamo-sglang-agentic-dflash: - image: lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a +musespark-fp4-gb300-dynamo-sglang-agentic-dflash-agg: + image: lmsysorg/sglang:nightly-dev-cu13-20261103-797af717@sha256:b4323a896f349d43a267c419c0267538b06c64a73110a0d0fd87496234df5dcd model: meta-models/Muse-Spark-Open-816B-A42B-NVFP4 model-prefix: musespark runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.6.0.dev20261016" } + router: { name: dynamo-router, version: "1.7.0.dev20261102" } multinode: true disagg: false srt-recipe-dir: musespark/sglang/gb300-fp4/agentx @@ -6964,99 +6964,211 @@ musespark-fp4-gb300-dynamo-sglang-agentic-dflash: agentic-coding: - search-space: - spec-decoding: draft_model - conc-list: [16] + conc-list: [36] + kv-offloading: none + num-nodes: 2 + srt-recipe: agg-variants.yaml:override_tp8_c36 + worker: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + - spec-decoding: draft_model + conc-list: [1] kv-offloading: none + num-nodes: 2 + srt-recipe: agg-variants.yaml:override_tp8_c1 + worker: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + - dram-utilization: 0.80 + search-space: + - spec-decoding: draft_model + conc-list: [56] + kv-offloading: dram + kv-offload-backend: { name: hicache } num-nodes: 1 - srt-recipe: agg-variants.yaml:override_tp4_c16 + srt-recipe: agg-variants.yaml:override_tp4_c56 worker: num-worker: 1 tp: 4 ep: 4 dp-attn: false - spec-decoding: draft_model - conc-list: [8] - kv-offloading: none + conc-list: [50] + kv-offloading: dram + kv-offload-backend: { name: hicache } num-nodes: 1 - srt-recipe: agg-variants.yaml:override_tp4_c8 + srt-recipe: agg-variants.yaml:override_tp4_c50 worker: num-worker: 1 tp: 4 ep: 4 dp-attn: false - spec-decoding: draft_model - conc-list: [14] - kv-offloading: none - num-nodes: 2 - srt-recipe: agg-variants.yaml:override_tp8_c14 + conc-list: [46] + kv-offloading: dram + kv-offload-backend: { name: hicache } + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c46 worker: num-worker: 1 + tp: 4 + ep: 4 + dp-attn: false + +musespark-fp4-gb300-dynamo-sglang-agentic-dflash-disagg: + image: lmsysorg/sglang:nightly-dev-cu13-20261103-797af717@sha256:b4323a896f349d43a267c419c0267538b06c64a73110a0d0fd87496234df5dcd + model: meta-models/Muse-Spark-Open-816B-A42B-NVFP4 + model-prefix: musespark + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.7.0.dev20261102" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + srt-recipe-dir: musespark/sglang/gb300-fp4/agentx + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: draft_model + conc-list: [672] + kv-offloading: dram + kv-offload-backend: { name: hicache } + srt-recipe: disagg-variants.yaml:override_3p4d_tp8_tp8_c672 + prefill: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: false + decode: + num-worker: 4 tp: 8 ep: 8 dp-attn: false - spec-decoding: draft_model - conc-list: [10] - kv-offloading: none - num-nodes: 2 - srt-recipe: agg-variants.yaml:override_tp8_c10 - worker: + conc-list: [352] + kv-offloading: dram + kv-offload-backend: { name: hicache } + srt-recipe: disagg-variants.yaml:override_2p3d_tp8_tp8_c352 + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: false + decode: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: false + - spec-decoding: draft_model + conc-list: [224] + kv-offloading: dram + kv-offload-backend: { name: hicache } + srt-recipe: disagg-variants.yaml:override_1p2d_tp8_tp8_c224 + prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: false + decode: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: false - spec-decoding: draft_model - conc-list: [4] - kv-offloading: none - num-nodes: 2 - srt-recipe: agg-variants.yaml:override_tp8_c4 - worker: + conc-list: [160] + kv-offloading: dram + kv-offload-backend: { name: hicache } + srt-recipe: disagg-variants.yaml:override_1p2d_tp8_tp8_c160 + prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: false + decode: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: false - spec-decoding: draft_model - conc-list: [3] - kv-offloading: none - num-nodes: 2 - srt-recipe: agg-variants.yaml:override_tp8_c3 - worker: + conc-list: [224] + kv-offloading: dram + kv-offload-backend: { name: hicache } + srt-recipe: disagg-variants.yaml:override_1p3d_tp8_tp8_c224 + prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: false - - dram-utilization: 0.80 - search-space: + decode: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: false - spec-decoding: draft_model - conc-list: [40] + conc-list: [112] kv-offloading: dram kv-offload-backend: { name: hicache } - num-nodes: 1 - srt-recipe: agg-variants.yaml:override_tp4_c40 - worker: + srt-recipe: disagg-variants.yaml:override_1p4d_tp8_tp8_c112 + prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 8 + ep: 8 + dp-attn: false + decode: + num-worker: 4 + tp: 8 + ep: 8 dp-attn: false - spec-decoding: draft_model - conc-list: [32] + conc-list: [120] kv-offloading: dram kv-offload-backend: { name: hicache } - num-nodes: 1 - srt-recipe: agg-variants.yaml:override_tp4_c32 - worker: + srt-recipe: disagg-variants.yaml:override_1p5d_tp8_tp8_c120 + prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 8 + ep: 8 + dp-attn: false + decode: + num-worker: 5 + tp: 8 + ep: 8 dp-attn: false - spec-decoding: draft_model - conc-list: [20] + conc-list: [56] kv-offloading: dram kv-offload-backend: { name: hicache } - num-nodes: 1 - srt-recipe: agg-variants.yaml:override_tp4_c20 - worker: + srt-recipe: disagg-variants.yaml:override_1p4d_tp8_tp8_c56 + prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 8 + ep: 8 + dp-attn: false + decode: + num-worker: 4 + tp: 8 + ep: 8 + dp-attn: false + - spec-decoding: draft_model + conc-list: [32] + kv-offloading: dram + kv-offload-backend: { name: hicache } + srt-recipe: disagg-variants.yaml:override_1p4d_tp8_tp8_c32 + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + decode: + num-worker: 4 + tp: 8 + ep: 8 dp-attn: false musespark-fp4-gb300-trt-agentic-dflash: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index f2c2307c97..bf7286cc7b 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12594,3 +12594,15 @@ - "Aggregated megakernel rows: c2 with max-running-requests 1; c4 with max-running-requests 2 and K31MK_* megakernel schedule options." - "Remove 3P1D c300, 1P1D c76, 1P3D c60 and c36, 1P4D c32 and the c1 megakernel row." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4893 + +- config-keys: + - musespark-fp4-gb300-dynamo-sglang-agentic-dflash-agg + - musespark-fp4-gb300-dynamo-sglang-agentic-dflash-disagg + scenario-type: + - agentic-coding + description: + - "Run the GB300 Muse Spark Open Dynamo + SGLang points on lmsysorg/sglang:nightly-dev-cu13-20261103-797af717 (digest sha256:b4323a896f349d43a267c419c0267538b06c64a73110a0d0fd87496234df5dcd) with the ai-dynamo 1.7.0.dev20261102 it ships (dynamo.install: false)." + - "Aggregated: add musespark-fp4-gb300-dynamo-sglang-agentic-dflash-agg with TP4 at c46, c50 and c56 (140 GB per-rank HiCache, 0.80 DRAM utilization) and two-node TP8 at c1 and c36." + - "Disaggregated: add musespark-fp4-gb300-dynamo-sglang-agentic-dflash-disagg with TP8 prefill and TP8 decode workers: 1P2D at c160 and c224, 1P3D at c224, 1P4D at c32, c56 and c112, 1P5D at c120, 2P3D at c352 and 3P4D at c672. Mooncake KV transfer and prefill HiCache (150 GB per rank, 0.80 DRAM utilization)." + - "Remove musespark-fp4-gb300-dynamo-sglang-agentic-dflash (TP4 c8, c16, c20, c32 and c40 and TP8 c3, c4, c10 and c14 on the dev-muse-spark-open image)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4757