diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/h3/sglang/h100-bf16/serving-client-tp2-ulysses4.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/h3/sglang/h100-bf16/serving-client-tp2-ulysses4.yaml new file mode 100644 index 0000000000..c09ae2077a --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/h3/sglang/h100-bf16/serving-client-tp2-ulysses4.yaml @@ -0,0 +1,65 @@ +schema: 2 +name: h3-sglang-serving-client + +resources: + gpu_type: h100 + +services: + - name: video-server + type: generic + nodes: 1 + critical: true + start: before_workers + command: [python3, -m, infx.bench, video, serve] + args: + - --model-view + - snapshot + - -- + - /opt/sglang/bin/python3 + - -m + - evaluator.srt_selected + - serve + - --model-type + - diffusion + - --model-path + - "{model_path}" + - --model-id + - "{model_id}" + - --revision + - "{model_revision}" + - --model-variant + - fl2va + - --num-gpus + - "{gpus_per_node}" + - --ulysses-degree + - "4" + - --tp-size + - "2" + - --encoder-parallel + - auto + - --warmup-resolutions + - 1344x768 + - --disable-conditioning-cache + - --performance-mode + - speed + - --host + - 0.0.0.0 + - --port + - "30283" + - --enable-torch-compile + - "false" + - --dit-cpu-offload + - "false" + env: + PYTHONPATH: /video-package:/infmax-workspace + PYTHONUNBUFFERED: "1" + VIDEO_SERVER_PORT: "30283" + VIDEO_RUNTIME_REVISION: 297e54fd105e974cdf401e5b4d20253a22c16f90 + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + readiness: + http: + port: 30283 + path: /health + timeout_seconds: 1800 + interval_seconds: 5 diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/h3/sglang/h100-bf16/serving-client-tp2.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/h3/sglang/h100-bf16/serving-client-tp2.yaml new file mode 100644 index 0000000000..c37238dee1 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/h3/sglang/h100-bf16/serving-client-tp2.yaml @@ -0,0 +1,65 @@ +schema: 2 +name: h3-sglang-serving-client + +resources: + gpu_type: h100 + +services: + - name: video-server + type: generic + nodes: 1 + critical: true + start: before_workers + command: [python3, -m, infx.bench, video, serve] + args: + - --model-view + - snapshot + - -- + - /opt/sglang/bin/python3 + - -m + - evaluator.srt_selected + - serve + - --model-type + - diffusion + - --model-path + - "{model_path}" + - --model-id + - "{model_id}" + - --revision + - "{model_revision}" + - --model-variant + - fl2va + - --num-gpus + - "{gpus_per_node}" + - --ulysses-degree + - "1" + - --tp-size + - "2" + - --encoder-parallel + - auto + - --warmup-resolutions + - 1344x768 + - --disable-conditioning-cache + - --performance-mode + - speed + - --host + - 0.0.0.0 + - --port + - "30283" + - --enable-torch-compile + - "false" + - --dit-cpu-offload + - "false" + env: + PYTHONPATH: /video-package:/infmax-workspace + PYTHONUNBUFFERED: "1" + VIDEO_SERVER_PORT: "30283" + VIDEO_RUNTIME_REVISION: 297e54fd105e974cdf401e5b4d20253a22c16f90 + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + readiness: + http: + port: 30283 + path: /health + timeout_seconds: 1800 + interval_seconds: 5 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 40db965f44..69eaaa2b15 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -7479,9 +7479,15 @@ h3-bf16-h100-sglang-video: - spec: h3-v2.yaml set: release-v1 search-space: + - gpus: 2 + srt-recipe: serving-client-tp2.yaml + conc-list: [1] - gpus: 4 srt-recipe: serving-client.yaml conc-list: [1] + - gpus: 8 + srt-recipe: serving-client-tp2-ulysses4.yaml + conc-list: [1] h3-bf16-b200-sglang-video: image: lmsysorg/sglang:v0.5.23@sha256:f7876bef696ccf9cf75a2bf7ccb82970f5fac577f082fb8ca95c9a59d2160eb5 diff --git a/inferencex-e2e/docs/MODELS.md b/inferencex-e2e/docs/MODELS.md index 404327641b..2255d5898c 100644 --- a/inferencex-e2e/docs/MODELS.md +++ b/inferencex-e2e/docs/MODELS.md @@ -36,7 +36,7 @@ For non-grandfathered models, `ROCM_QUICK_REDUCE_QUANTIZATION`, `VLLM_ROCM_QUICK |---|---|---| | Agentic coding | Long Context, Multi Turn Realistic traffic trace replay with sub agents | Active. This is the trace-replay agentic-coding benchmark (see the srt-slurm recipes under [`benchmarks/single_node/srt-slurm-recipes/`](../benchmarks/single_node/srt-slurm-recipes) and the shared client [`benchmarks/srt_agentic.sh`](../benchmarks/srt_agentic.sh)). Going forward, new models will likely be onboarded with agentic coding only. Speculative decoding may be enabled or disabled to produce the best Pareto points; a separate non-spec-decode A/B baseline is not required (see [Deprecation policy](#deprecation-policy)). | | Single-turn 8k1k | 8192 / 1024 | Active. This is the primary fixed-sequence-length scenario. | -| Video generation | H3: 124 frames at 24 fps, 1344x768, C1 | MiniMax-H3 uses native srt-slurm: SGLang selects 20-clip `h3-v2.yaml` C1 points on 1, 2, 4 and 8 B200 GPUs and 4 H100 or MI355X GPUs. The client and server live in [`video-generation`](../../video-generation); energy is reported per valid clip. Config presence does not qualify execution, power, or quality. See [video delivery](video-generation.md). | +| Video generation | H3: 124 frames at 24 fps, 1344x768, C1 | MiniMax-H3 uses native srt-slurm: SGLang selects 20-clip `h3-v2.yaml` C1 points on 1, 2, 4 and 8 B200 GPUs, 2, 4 and 8 H100 GPUs and 4 MI355X GPUs. The client and server live in [`video-generation`](../../video-generation); energy is reported per valid clip. Config presence does not qualify execution, power, or quality. See [video delivery](video-generation.md). | | Single-turn 1k1k | 1024 / 1024 | Deprecated since 2026-07-17 ([#2263](https://github.com/SemiAnalysisAI/InferenceX/pull/2263)), to save GPU cluster time for higher-priority real-world agentic-coding benchmarks and new frontier models. Archived configs lived in `configs/deprecated/` until it was deleted (#3463); see git history. The GLM-5.1 B200 TileRT point added later in [#2533](https://github.com/SemiAnalysisAI/InferenceX/pull/2533) was the last 1k1k config; it was retired on 2026-10-01 ([#3638](https://github.com/SemiAnalysisAI/InferenceX/pull/3638)). | | Single-turn 1k8k | 1024 / 8192 | **Deprecated for all models** since 2026-03-27 ([#911](https://github.com/SemiAnalysisAI/InferenceX/pull/911)), to save GPU cluster time for higher-priority real-world agentic-coding benchmarks and new frontier models. Configs were removed, not archived. | diff --git a/inferencex-e2e/docs/video-generation.md b/inferencex-e2e/docs/video-generation.md index 090962dcb5..4b51acd3b8 100644 --- a/inferencex-e2e/docs/video-generation.md +++ b/inferencex-e2e/docs/video-generation.md @@ -1,8 +1,9 @@ # Native video delivery The `video-generation` scenario schedules one server job per deployment. SGLang -selects C1 deployments on 1, 2, 4 and 8 B200 GPUs (Ulysses across all GPUs) and -on 4 H100 or MI355X GPUs, all using +selects C1 deployments on 1, 2, 4 and 8 B200 GPUs (Ulysses across all GPUs), on +2, 4 and 8 H100 GPUs (TP2, since the model does not fit one 80 GB GPU) and on 4 +MI355X GPUs, all using `h3-v2.yaml`: 20 distinct measured prompts, a separate warmup, 49 denoising updates, and 124 frames at 24 fps and 1344×768. A five-second request produces 124/24 seconds of video. Config presence is not GPU or quality qualification. diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 9e141416db..fbbe353571 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12269,3 +12269,10 @@ description: - "Sweep H3 C1 latency on 1, 2, 4 and 8 B200 GPUs with SGLang Ulysses across all GPUs; the B200 recipe now sets the Ulysses degree from the GPU count, which keeps the 4-GPU point unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4852 + +- config-keys: + - h3-bf16-h100-sglang-video + scenario-type: [video-generation] + description: + - "Sweep H3 C1 latency on 2, 4 and 8 H100 GPUs with SGLang TP2 and Ulysses 1, 2 and 4; H100 starts at 2 GPUs because the model does not fit one 80 GB GPU, and the 4-GPU point keeps its recipe." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4857