Skip to content

Commit 55dd4f1

Browse files
committed
feat: enable Kimi-K3 GB200 aggregate power
为 GB200 Kimi-K3 聚合部署启用必需实测功耗。
1 parent 18f1be8 commit 55dd4f1

10 files changed

Lines changed: 224 additions & 31 deletions

‎benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-dspark4-maxseq2-mooncake-agentic.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,10 +130,27 @@ srun_options:
130130
mem: "0"
131131
container-remap-root: ""
132132

133+
telemetry:
134+
enabled: true
135+
provider: dcgm-power
136+
default_frequency: 1.0
137+
storage_subdir: power
138+
required: true
139+
startup_timeout_seconds: 120
140+
request_timeout_seconds: 2
141+
collector_join_timeout_seconds: 12
142+
dcgm_exporter:
143+
container_image: dcgm-exporter
144+
port: 9401
145+
133146
benchmark:
134147
type: custom
148+
client_placement: head
149+
concurrencies: [1, 2, 4, 8, 16]
135150
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
136151
env:
152+
ENABLE_AGENTX_POWER: "1"
153+
REQUIRE_POWER: "1"
137154
INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
138155
AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
139156
AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"

‎benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-nospec-mooncake-agentic.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,10 +126,27 @@ srun_options:
126126
mem: "0"
127127
container-remap-root: ""
128128

129+
telemetry:
130+
enabled: true
131+
provider: dcgm-power
132+
default_frequency: 1.0
133+
storage_subdir: power
134+
required: true
135+
startup_timeout_seconds: 120
136+
request_timeout_seconds: 2
137+
collector_join_timeout_seconds: 12
138+
dcgm_exporter:
139+
container_image: dcgm-exporter
140+
port: 9401
141+
129142
benchmark:
130143
type: custom
144+
client_placement: head
145+
concurrencies: [8, 40, 48]
131146
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
132147
env:
148+
ENABLE_AGENTX_POWER: "1"
149+
REQUIRE_POWER: "1"
133150
INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
134151
AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
135152
AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"

‎benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c16-agentic.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,10 +111,27 @@ sbatch_directives:
111111
srun_options:
112112
container-remap-root: ""
113113

114+
telemetry:
115+
enabled: true
116+
provider: dcgm-power
117+
default_frequency: 1.0
118+
storage_subdir: power
119+
required: true
120+
startup_timeout_seconds: 120
121+
request_timeout_seconds: 2
122+
collector_join_timeout_seconds: 12
123+
dcgm_exporter:
124+
container_image: dcgm-exporter
125+
port: 9401
126+
114127
benchmark:
115128
type: custom
129+
client_placement: head
130+
concurrencies: [16]
116131
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
117132
env:
133+
ENABLE_AGENTX_POWER: "1"
134+
REQUIRE_POWER: "1"
118135
INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
119136
AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
120137
AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"

‎benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c32-agentic.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,10 +111,27 @@ sbatch_directives:
111111
srun_options:
112112
container-remap-root: ""
113113

114+
telemetry:
115+
enabled: true
116+
provider: dcgm-power
117+
default_frequency: 1.0
118+
storage_subdir: power
119+
required: true
120+
startup_timeout_seconds: 120
121+
request_timeout_seconds: 2
122+
collector_join_timeout_seconds: 12
123+
dcgm_exporter:
124+
container_image: dcgm-exporter
125+
port: 9401
126+
114127
benchmark:
115128
type: custom
129+
client_placement: head
130+
concurrencies: [32]
116131
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
117132
env:
133+
ENABLE_AGENTX_POWER: "1"
134+
REQUIRE_POWER: "1"
118135
INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
119136
AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
120137
AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"

‎benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c48-agentic.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,10 +110,27 @@ sbatch_directives:
110110
srun_options:
111111
container-remap-root: ""
112112

113+
telemetry:
114+
enabled: true
115+
provider: dcgm-power
116+
default_frequency: 1.0
117+
storage_subdir: power
118+
required: true
119+
startup_timeout_seconds: 120
120+
request_timeout_seconds: 2
121+
collector_join_timeout_seconds: 12
122+
dcgm_exporter:
123+
container_image: dcgm-exporter
124+
port: 9401
125+
113126
benchmark:
114127
type: custom
128+
client_placement: head
129+
concurrencies: [48]
115130
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
116131
env:
132+
ENABLE_AGENTX_POWER: "1"
133+
REQUIRE_POWER: "1"
117134
INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
118135
AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
119136
AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"

‎benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c72-agentic.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,10 +110,27 @@ sbatch_directives:
110110
srun_options:
111111
container-remap-root: ""
112112

113+
telemetry:
114+
enabled: true
115+
provider: dcgm-power
116+
default_frequency: 1.0
117+
storage_subdir: power
118+
required: true
119+
startup_timeout_seconds: 120
120+
request_timeout_seconds: 2
121+
collector_join_timeout_seconds: 12
122+
dcgm_exporter:
123+
container_image: dcgm-exporter
124+
port: 9401
125+
113126
benchmark:
114127
type: custom
128+
client_placement: head
129+
concurrencies: [72]
115130
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
116131
env:
132+
ENABLE_AGENTX_POWER: "1"
133+
REQUIRE_POWER: "1"
117134
INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
118135
AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
119136
AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"

‎benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c96-agentic.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,10 +110,27 @@ sbatch_directives:
110110
srun_options:
111111
container-remap-root: ""
112112

113+
telemetry:
114+
enabled: true
115+
provider: dcgm-power
116+
default_frequency: 1.0
117+
storage_subdir: power
118+
required: true
119+
startup_timeout_seconds: 120
120+
request_timeout_seconds: 2
121+
collector_join_timeout_seconds: 12
122+
dcgm_exporter:
123+
container_image: dcgm-exporter
124+
port: 9401
125+
113126
benchmark:
114127
type: custom
128+
client_placement: head
129+
concurrencies: [96]
115130
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
116131
env:
132+
ENABLE_AGENTX_POWER: "1"
133+
REQUIRE_POWER: "1"
117134
INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
118135
AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
119136
AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"

‎perf-changelog.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7408,3 +7408,14 @@
74087408
- "Enable required power for the seven existing B200 Kimi-K3 recipes, use the pinned AgentX producer and shared collector, fix the exporter import URI, and record DCP8 with offload disabled to match the serving commands."
74097409
- "启用 Kimi-K3 B200 实测功耗并修正 DCP 与 Offload 元数据。"
74107410
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3043
7411+
7412+
- config-keys:
7413+
- kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-dcp16-agg
7414+
- kimik3-fp4-gb200-dynamo-vllm-agentic-mooncake-dcp16-agg
7415+
- kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2
7416+
scenario-type:
7417+
- agentic-coding
7418+
description:
7419+
- "Enable required telemetry on the seven existing GB200 aggregate recipes, pin the AgentX producer, and validate power after telemetry drain while preserving failed-job artifacts."
7420+
- "为 GB200 Kimi-K3 聚合部署启用必需实测功耗。"
7421+
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3045

‎runners/launch_gb200-nv.sh‎

Lines changed: 53 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ SQUASH_DIR="/mnt/lustre01/users-public/sa-shared"
1515
# srt-slurm merge lands.
1616
POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git"
1717
POWER_SRT_SLURM_PIN="6fc1bed01a0b82dae0088a105c03ce0cfb353443"
18+
AGENTX_POWER_SRT_SLURM_PIN="80d7203e424f903c9017de4608ee2044afce9574"
1819

1920
# Enroot 3.x does not parse Docker's tag@digest syntax. For digest-pinned
2021
# images, use its explicit registry syntax and pass the digest as the
@@ -332,12 +333,15 @@ if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk '
332333
USES_DCGM_POWER=1
333334
fi
334335

335-
# Note (wenyao): the producer pin follows the srt-slurm main lineage that the
336-
# dynamo-sglang lanes run on (fp8 validated end-to-end, fp4 recipes
337-
# parse-verified against the pin); other frameworks clone diverging refs
338-
# (aflowers branch, sa-submission), so fail fast for them instead.
339-
if [[ "$USES_DCGM_POWER" == "1" && "$FRAMEWORK" != "dynamo-sglang" ]]; then
340-
echo "Error: dcgm-power lanes are only validated for FRAMEWORK=dynamo-sglang, got: $FRAMEWORK" >&2
336+
USES_AGENTX_POWER=0
337+
if [[ "$USES_DCGM_POWER" == "1" && "$IS_AGENTIC" == "1" &&
338+
"$MODEL_PREFIX" == "kimik3" && "$PRECISION" == "fp4" &&
339+
"$FRAMEWORK" == "dynamo-vllm" &&
340+
"$_RECIPE_REL" == recipes/vllm/kimi-k3/agentic/* ]]; then
341+
USES_AGENTX_POWER=1
342+
fi
343+
if [[ "$USES_DCGM_POWER" == "1" && "$FRAMEWORK" != "dynamo-sglang" && "$USES_AGENTX_POWER" != "1" ]]; then
344+
echo "Error: dcgm-power requires dynamo-sglang or the supported Kimi-K3 AgentX route" >&2
341345
exit 1
342346
fi
343347

@@ -472,12 +476,20 @@ elif [[ "$IS_AGENTIC" == "1" && (( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION"
472476
fi
473477
# Kimi-K3 requires direct multi-node vLLM frontend support from srt-slurm.
474478
elif [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "kimik3" ]]; then
475-
git clone --branch v1.0.53 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1
476-
cd "$SRT_REPO_DIR" || exit 1
477-
test "$(git rev-parse HEAD)" = "217f94387abeddfed7149a71955dc523e07cd765" || {
478-
echo "Error: NVIDIA/srt-slurm v1.0.53 resolved to an unexpected commit" >&2
479-
exit 1
480-
}
479+
if [[ "$USES_AGENTX_POWER" == "1" ]]; then
480+
git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" || exit 1
481+
cd "$SRT_REPO_DIR" || exit 1
482+
git checkout "$AGENTX_POWER_SRT_SLURM_PIN" || exit 1
483+
test "$(git rev-parse HEAD)" = "$AGENTX_POWER_SRT_SLURM_PIN" || exit 1
484+
git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt"
485+
else
486+
git clone --branch v1.0.53 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1
487+
cd "$SRT_REPO_DIR" || exit 1
488+
test "$(git rev-parse HEAD)" = "217f94387abeddfed7149a71955dc523e07cd765" || {
489+
echo "Error: NVIDIA/srt-slurm v1.0.53 resolved to an unexpected commit" >&2
490+
exit 1
491+
}
492+
fi
481493
python3 "$GITHUB_WORKSPACE/runners/patch_srt_vllm_dp_ranks.py" "$(pwd)" || exit 1
482494
mkdir -p recipes/vllm/kimi-k3/agentic || exit 1
483495
cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \
@@ -759,6 +771,12 @@ sed -i "s/^name:.*/name: \"${SRT_SLURM_JOB_NAME}\"/" "$CONFIG_PATH"
759771
python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \
760772
"$CONFIG_PATH" "$FRAMEWORK" || exit 1
761773

774+
if [[ "$USES_AGENTX_POWER" == "1" ]]; then
775+
read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST"
776+
python3 "$GITHUB_WORKSPACE/runners/inject_srt_power_concurrencies.py" \
777+
"$CONFIG_PATH" "${POWER_CONCURRENCIES[@]}" || exit 1
778+
fi
779+
762780
# Don't leak the login-node venv to the compute-node orchestrator. sbatch's
763781
# default --export=ALL propagates VIRTUAL_ENV (set by `source
764782
# .venv/bin/activate` above) into job_script_minimal.j2, whose
@@ -822,13 +840,24 @@ trap 'exit 143' TERM HUP
822840
LOGS_DIR="outputs/$JOB_ID/logs"
823841
LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log"
824842

825-
stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1
843+
AGENTX_POWER_RC=0
844+
stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || AGENTX_POWER_RC=$?
845+
if [[ "$AGENTX_POWER_RC" != "0" && "$USES_AGENTX_POWER" != "1" ]]; then
846+
exit "$AGENTX_POWER_RC"
847+
fi
826848

827849
set -x
828850

829851
echo "Job $JOB_ID completed!"
830852
echo "Collecting results..."
831853

854+
if [[ "$USES_AGENTX_POWER" == "1" && "${EVAL_ONLY:-false}" != "true" ]]; then
855+
read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST"
856+
collect_agentic_power_results "$JOB_ID" "$LOGS_DIR" "$INFMAX_WORKSPACE" \
857+
"$GITHUB_WORKSPACE" "$RESULT_FILENAME" "$AGENTX_POWER_SRT_SLURM_PIN" \
858+
"${POWER_CONCURRENCIES[@]}" || AGENTX_POWER_RC=$?
859+
fi
860+
832861
if [ -d "$LOGS_DIR" ]; then
833862
echo "Found logs directory: $LOGS_DIR"
834863
# Provenance markers travel inside the server-logs bundle so the offline
@@ -844,6 +873,11 @@ else
844873
echo "Warning: Logs directory not found at $LOGS_DIR"
845874
fi
846875

876+
if [[ "$AGENTX_POWER_RC" != "0" ]]; then
877+
echo "ERROR: AgentX job or power validation failed after staging audit artifacts" >&2
878+
exit "$AGENTX_POWER_RC"
879+
fi
880+
847881
if [[ "${EVAL_ONLY:-false}" != "true" ]]; then
848882
if [ ! -d "$LOGS_DIR" ]; then
849883
exit 1
@@ -854,10 +888,12 @@ if [[ "${EVAL_ONLY:-false}" != "true" ]]; then
854888
# INFMAX_WORKSPACE mount. Its aggregation step writes one
855889
# ${RESULT_FILENAME}_conc<N>.json there per point; stage those files
856890
# back to GITHUB_WORKSPACE for the workflow guard and artifact upload.
857-
copy_agentic_results \
858-
"$INFMAX_WORKSPACE" \
859-
"$GITHUB_WORKSPACE" \
860-
"$RESULT_FILENAME" || exit 1
891+
if [[ "$USES_AGENTX_POWER" != "1" ]]; then
892+
copy_agentic_results \
893+
"$INFMAX_WORKSPACE" \
894+
"$GITHUB_WORKSPACE" \
895+
"$RESULT_FILENAME" || exit 1
896+
fi
861897
else
862898
# Find all fixed-sequence result subdirectories.
863899
RESULT_SUBDIRS=$(find "$LOGS_DIR" -maxdepth 1 -type d -name "*isl*osl*" 2>/dev/null)

0 commit comments

Comments
 (0)