Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
schema: 2
name: wan22-sglang-serving-client-g1

resources:
gpu_type: h200

services:
- name: video-server
type: generic
nodes: 1
critical: true
start: before_workers
command: [python3, -m, infx.bench, video, serve]
args:
- --model-view
- snapshot
- --
- /opt/sglang/bin/python3
- -m
- evaluator.srt_selected
- serve
- --model-type
- diffusion
- --model-path
- "{model_path}"
- --model-id
- "{model_id}"
- --revision
- "{model_revision}"
- --num-gpus
- "{gpus_per_node}"
# Deployment only: one GPU runs both CFG branches with no sequence parallelism.
- --cfg-parallel-size
- "1"
- --ulysses-degree
- "1"
- --tp-size
- "1"
- --encoder-parallel
- auto
# SGLang takes the UniPC shift from this flag and ignores the request field.
- --flow-shift
- "12.0"
- --warmup-resolutions
- 832x480
# The release set reuses each prompt across five seeds; cached text embeddings would skip work.
- --disable-conditioning-cache
- --performance-mode
- speed
- --host
- 0.0.0.0
- --port
- "30283"
- --enable-torch-compile
- "false"
- --dit-cpu-offload
- "false"
env:
PYTHONPATH: /video-package:/infmax-workspace
PYTHONUNBUFFERED: "1"
VIDEO_SERVER_PORT: "30283"
VIDEO_RUNTIME_REVISION: 297e54fd105e974cdf401e5b4d20253a22c16f90
HF_HUB_OFFLINE: "1"
TRANSFORMERS_OFFLINE: "1"
readiness:
http:
port: 30283
path: /health
timeout_seconds: 3600
interval_seconds: 5
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
schema: 2
name: wan22-sglang-serving-client-g2

resources:
gpu_type: h200

services:
- name: video-server
type: generic
nodes: 1
critical: true
start: before_workers
command: [python3, -m, infx.bench, video, serve]
args:
- --model-view
- snapshot
- --
- /opt/sglang/bin/python3
- -m
- evaluator.srt_selected
- serve
- --model-type
- diffusion
- --model-path
- "{model_path}"
- --model-id
- "{model_id}"
- --revision
- "{model_revision}"
- --num-gpus
- "{gpus_per_node}"
# Deployment only: CFG branches on two GPUs, no sequence parallelism.
- --cfg-parallel-size
- "2"
- --ulysses-degree
- "1"
- --tp-size
- "1"
- --encoder-parallel
- auto
# SGLang takes the UniPC shift from this flag and ignores the request field.
- --flow-shift
- "12.0"
- --warmup-resolutions
- 832x480
# The release set reuses each prompt across five seeds; cached text embeddings would skip work.
- --disable-conditioning-cache
- --performance-mode
- speed
- --host
- 0.0.0.0
- --port
- "30283"
- --enable-torch-compile
- "false"
- --dit-cpu-offload
- "false"
env:
PYTHONPATH: /video-package:/infmax-workspace
PYTHONUNBUFFERED: "1"
VIDEO_SERVER_PORT: "30283"
VIDEO_RUNTIME_REVISION: 297e54fd105e974cdf401e5b4d20253a22c16f90
HF_HUB_OFFLINE: "1"
TRANSFORMERS_OFFLINE: "1"
readiness:
http:
port: 30283
path: /health
timeout_seconds: 3600
interval_seconds: 5
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
schema: 2
name: wan22-sglang-serving-client-g4

resources:
gpu_type: h200

services:
- name: video-server
type: generic
nodes: 1
critical: true
start: before_workers
command: [python3, -m, infx.bench, video, serve]
args:
- --model-view
- snapshot
- --
- /opt/sglang/bin/python3
- -m
- evaluator.srt_selected
- serve
- --model-type
- diffusion
- --model-path
- "{model_path}"
- --model-id
- "{model_id}"
- --revision
- "{model_revision}"
- --num-gpus
- "{gpus_per_node}"
# Deployment only: CFG branches on two GPU groups, Ulysses sequence parallelism within each.
- --cfg-parallel-size
- "2"
- --ulysses-degree
- "2"
- --tp-size
- "1"
- --encoder-parallel
- auto
# SGLang takes the UniPC shift from this flag and ignores the request field.
- --flow-shift
- "12.0"
- --warmup-resolutions
- 832x480
# The release set reuses each prompt across five seeds; cached text embeddings would skip work.
- --disable-conditioning-cache
- --performance-mode
- speed
- --host
- 0.0.0.0
- --port
- "30283"
- --enable-torch-compile
- "false"
- --dit-cpu-offload
- "false"
env:
PYTHONPATH: /video-package:/infmax-workspace
PYTHONUNBUFFERED: "1"
VIDEO_SERVER_PORT: "30283"
VIDEO_RUNTIME_REVISION: 297e54fd105e974cdf401e5b4d20253a22c16f90
HF_HUB_OFFLINE: "1"
TRANSFORMERS_OFFLINE: "1"
readiness:
http:
port: 30283
path: /health
timeout_seconds: 3600
interval_seconds: 5
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
schema: 2
name: wan22-sglang-serving-client

resources:
gpu_type: h200

services:
- name: video-server
type: generic
nodes: 1
critical: true
start: before_workers
command: [python3, -m, infx.bench, video, serve]
args:
- --model-view
- snapshot
- --
- /opt/sglang/bin/python3
- -m
- evaluator.srt_selected
- serve
- --model-type
- diffusion
- --model-path
- "{model_path}"
- --model-id
- "{model_id}"
- --revision
- "{model_revision}"
- --num-gpus
- "{gpus_per_node}"
# Deployment only: CFG branches on two GPU groups, Ulysses sequence parallelism within each.
- --cfg-parallel-size
- "2"
- --ulysses-degree
- "4"
- --tp-size
- "1"
- --encoder-parallel
- auto
# SGLang takes the UniPC shift from this flag and ignores the request field.
- --flow-shift
- "12.0"
- --warmup-resolutions
- 832x480
# The release set reuses each prompt across five seeds; cached text embeddings would skip work.
- --disable-conditioning-cache
- --performance-mode
- speed
- --host
- 0.0.0.0
- --port
- "30283"
- --enable-torch-compile
- "false"
- --dit-cpu-offload
- "false"
env:
PYTHONPATH: /video-package:/infmax-workspace
PYTHONUNBUFFERED: "1"
VIDEO_SERVER_PORT: "30283"
VIDEO_RUNTIME_REVISION: 297e54fd105e974cdf401e5b4d20253a22c16f90
HF_HUB_OFFLINE: "1"
TRANSFORMERS_OFFLINE: "1"
readiness:
http:
port: 30283
path: /health
timeout_seconds: 3600
interval_seconds: 5
27 changes: 27 additions & 0 deletions inferencex-e2e/configs/nvidia-master.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8086,6 +8086,33 @@ wan22-bf16-b300-vllm-omni-video:
srt-recipe: serving-client.yaml
conc-list: [1]

wan22-bf16-h200-sglang-video:
image: lmsysorg/sglang:v0.5.23@sha256:f7876bef696ccf9cf75a2bf7ccb82970f5fac577f082fb8ca95c9a59d2160eb5
model: Wan-AI/Wan2.2-T2V-A14B-Diffusers
model-prefix: wan22
runner: cluster:h200-dgxc
precision: bf16
framework: sglang
multinode: false
srt-recipe-dir: wan22/sglang/h200-bf16
scenarios:
video-generation:
- spec: wan22.yaml
set: release-v1
search-space:
- gpus: 1
srt-recipe: serving-client-g1.yaml
conc-list: [1]
- gpus: 2
srt-recipe: serving-client-g2.yaml
conc-list: [1]
- gpus: 4
srt-recipe: serving-client-g4.yaml
conc-list: [1]
- gpus: 8
srt-recipe: serving-client.yaml
conc-list: [1]

glm5.3-fp4-b300-sglang-agentic-mtp:
image: lmsysorg/sglang:v0.5.20-cu130@sha256:b27fce60bc5494c118c4910702812bcfa8cee67abcdd1ff8b0902f21647552f4
model: nvidia/GLM-5.3-NVFP4
Expand Down
2 changes: 1 addition & 1 deletion inferencex-e2e/docs/MODELS.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ For non-grandfathered models, `ROCM_QUICK_REDUCE_QUANTIZATION`, `VLLM_ROCM_QUICK
|---|---|---|
| Agentic coding | Long Context, Multi Turn Realistic traffic trace replay with sub agents | Active. This is the trace-replay agentic-coding benchmark (see the srt-slurm recipes under [`benchmarks/single_node/srt-slurm-recipes/`](../benchmarks/single_node/srt-slurm-recipes) and the shared client [`benchmarks/srt_agentic.sh`](../benchmarks/srt_agentic.sh)). Going forward, new models will likely be onboarded with agentic coding only. Speculative decoding may be enabled or disabled to produce the best Pareto points; a separate non-spec-decode A/B baseline is not required (see [Deprecation policy](#deprecation-policy)). |
| Single-turn 8k1k | 8192 / 1024 | Active. This is the primary fixed-sequence-length scenario. |
| Video generation | H3: 124 frames at 24 fps, 1344x768, C1 | MiniMax-H3 uses native srt-slurm: SGLang selects 20-clip `h3-v2.yaml` C1 points on 1, 2, 4 and 8 B200, H200 or MI355X GPUs and 2, 4 and 8 H100 GPUs, and vLLM-Omni the same B200 points. Wan2.2-T2V-A14B (`wan22.yaml`: 81 frames at 16 fps, 832x480, 20 clips) runs on vLLM-Omni on 1, 2, 4 and 8 B200 or B300 GPUs. The client and server live in [`video-generation`](../../video-generation); energy is reported per valid clip. Config presence does not qualify execution, power, or quality. See [video delivery](video-generation.md). |
| Video generation | H3: 124 frames at 24 fps, 1344x768, C1 | MiniMax-H3 uses native srt-slurm: SGLang selects 20-clip `h3-v2.yaml` C1 points on 1, 2, 4 and 8 B200, H200 or MI355X GPUs and 2, 4 and 8 H100 GPUs, and vLLM-Omni the same B200 points. Wan2.2-T2V-A14B (`wan22.yaml`: 81 frames at 16 fps, 832x480, 20 clips) runs on vLLM-Omni on 1, 2, 4 and 8 B200 or B300 GPUs and on SGLang on 1, 2, 4 and 8 H200 GPUs. The client and server live in [`video-generation`](../../video-generation); energy is reported per valid clip. Config presence does not qualify execution, power, or quality. See [video delivery](video-generation.md). |
| Single-turn 1k1k | 1024 / 1024 | Deprecated since 2026-07-17 ([#2263](https://github.com/SemiAnalysisAI/InferenceX/pull/2263)), to save GPU cluster time for higher-priority real-world agentic-coding benchmarks and new frontier models. Archived configs lived in `configs/deprecated/` until it was deleted (#3463); see git history. The GLM-5.1 B200 TileRT point added later in [#2533](https://github.com/SemiAnalysisAI/InferenceX/pull/2533) was the last 1k1k config; it was retired on 2026-10-01 ([#3638](https://github.com/SemiAnalysisAI/InferenceX/pull/3638)). |
| Single-turn 1k8k | 1024 / 8192 | **Deprecated for all models** since 2026-03-27 ([#911](https://github.com/SemiAnalysisAI/InferenceX/pull/911)), to save GPU cluster time for higher-priority real-world agentic-coding benchmarks and new frontier models. Configs were removed, not archived. |

Expand Down
11 changes: 8 additions & 3 deletions inferencex-e2e/docs/video-generation.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@ one 80 GB GPU), and vLLM-Omni the same four B200 deployments, all using
`h3-v2.yaml`: 20 distinct measured prompts, a separate warmup, 49 denoising
updates, and 124 frames at 24 fps and 1344×768. A five-second request produces
124/24 seconds of video. Wan2.2-T2V-A14B runs vLLM-Omni on 1, 2, 4 and 8
B200 or B300 GPUs (CFG parallel 1 x Ulysses 1 on one GPU, then CFG parallel 2 x
Ulysses 1, 2 and 4) using `wan22.yaml` `release-v1`: four
B200 or B300 GPUs and SGLang on 1, 2, 4 and 8 H200 GPUs (CFG parallel 1 x
Ulysses 1 on one GPU, then CFG parallel 2 x Ulysses 1, 2 and 4) using
`wan22.yaml` `release-v1`: four
prompts by five seeds, one warmup, 40 UniPC steps, and 81 silent frames at 16 fps
and 832×480; its `smoke` set sends the same request controls for one case.
Config presence is not GPU or quality qualification.
Expand Down Expand Up @@ -49,7 +50,11 @@ includes the declared settings.

The Python entrypoint validates the workload and GPU identity, prepares an
explicitly selected model view, records the final command, then starts it.
It adds no engine flags or performance environment defaults. The SGLang child
It adds no engine flags or performance environment defaults. For Wan2.2 on
SGLang it also requires `--flow-shift` and the checkpoint's `boundary_ratio` to
equal the plan, because SGLang reads neither from a request. SGLang's Wan2.2
requests set `x264_preset: ultrafast` and `output_compression: 63` (CRF 18),
the libx264 encoding vLLM-Omni v0.32.0 delivers by default. The SGLang child
checks the assigned UUID order before importing its engine. The previous B200
`frozen-c124` four-clip JSON and YAML artifacts remain readable as historical evidence.
The App shows their four-sample cells in History, below its ten-valid-sample chart
Expand Down
9 changes: 9 additions & 0 deletions inferencex-e2e/perf-changelog.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12979,3 +12979,12 @@
- "The dcp8-disagg and agg frontends drop router-session-affinity-ttl-secs 1: with one-second session affinity a share of 1P5D decode picks stayed on an engine that was already busy."
- "Aggregated TP16/DCP16 at concurrency 1 moves to the new image and Dynamo with no other change."
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/5069

- config-keys:
- wan22-bf16-h200-sglang-video
scenario-type: [video-generation]
description:
- "Add an H200 Wan2.2-T2V-A14B baseline on SGLang v0.5.23, by the digest the H3 SGLang rows use: wan22.yaml release-v1 (four prompts by five seeds, one warmup; 832x480, 81 frames at 16 fps, 40 UniPC steps, guidance 4.0 high-noise / 3.0 low-noise, boundary 0.875, flow_shift 12.0, empty negative prompt, BF16), C1 on 1, 2, 4 and 8 GPUs."
- "SGLang takes flow_shift from the --flow-shift server flag and boundary_ratio from the checkpoint's model_index.json, so the server refuses to start when either differs from the spec; the client sends every other control per request. Conditioning cache disabled."
- "Deployment per count, the vLLM-Omni rows' layout: 1 GPU runs both CFG branches with Ulysses 1; 2 GPUs CFG parallel 2 x Ulysses 1; 4 GPUs CFG parallel 2 x Ulysses 2; 8 GPUs CFG parallel 2 x Ulysses 4. TP 1, performance mode speed, no torch.compile, no DiT offload."
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/5113
Loading
Loading