Skip to content

Commit 57199f0

Browse files
committed
fix: preserve AMD power telemetry through benchmark shutdown
中文:等待 AMD 功耗采样覆盖基准窗口,保留取消清理与边界异常行审计。
1 parent 5d2a883 commit 57199f0

7 files changed

Lines changed: 713 additions & 28 deletions

File tree

‎benchmarks/benchmark_lib.sh‎

Lines changed: 128 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,8 @@ unset _benchmark_caller
155155
GPU_MONITOR_PID=""
156156
GPU_MONITOR_VENDOR=""
157157
GPU_MONITOR_INTERVAL=1
158+
# Bounded wait for AMD telemetry to cover a stop request; 0 skips the wait.
159+
AMD_MONITOR_STOP_TIMEOUT_S="${AMD_MONITOR_STOP_TIMEOUT_S:-30}"
158160
GPU_METRICS_CSV="${GPU_METRICS_CSV:-gpu_metrics.csv}"
159161
NVIDIA_GPU_MONITOR_QUERY="timestamp,index,power.draw,temperature.gpu,clocks.current.sm,clocks.current.memory,utilization.gpu,utilization.memory"
160162
export GPU_METRICS_CSV
@@ -215,17 +217,18 @@ start_gpu_monitor() {
215217
# Stop the background GPU monitor and report file size.
216218
stop_gpu_monitor() {
217219
if [[ -n "$GPU_MONITOR_PID" ]] && kill -0 "$GPU_MONITOR_PID" 2>/dev/null; then
218-
# benchmark_end_time_unix is recorded shortly before the benchmark
219-
# process exits, so the stream must cover one more sample past it for
220-
# deterministic boundary interpolation. NVIDIA appends a one-shot
221-
# post-exit sample below; amd-smi one-shot CSV has no timestamp column,
222-
# so the AMD path instead lets the watch stream emit final ticks before
223-
# the kill. Two extra intervals: amd-smi stamps integer seconds, so a
224-
# tick in the same second as the window end still fails bracketing —
225-
# the stream needs a tick at the NEXT whole second (measured on MI355X:
226-
# end=...153.325 vs last sample ...153.0).
220+
# The aggregator requires, for every GPU, a usable sample stamped at or
221+
# after the (fractional) benchmark window end, which is always <= the
222+
# wall clock when this stop runs. NVIDIA appends a one-shot post-exit
223+
# sample below; amd-smi one-shot CSV has no timestamp column, so the
224+
# AMD path polls the output file until every GPU's watch stream shows
225+
# a usable tick at the next whole second — amd-smi stamps integer
226+
# seconds, so that tick strictly covers any fractional window end
227+
# (measured on MI355X: end=...609.157 vs last sample ...605). Observing
228+
# the file rather than sleeping also defeats pipe-buffer loss when the
229+
# awk consumer is killed: covered rows are already on disk.
227230
if [[ "$GPU_MONITOR_VENDOR" == "amd" ]]; then
228-
sleep $(( ${GPU_MONITOR_INTERVAL:-1} + 2 ))
231+
_wait_for_amd_stop_coverage
229232
fi
230233
kill "$GPU_MONITOR_PID" 2>/dev/null
231234
wait "$GPU_MONITOR_PID" 2>/dev/null || true
@@ -271,6 +274,109 @@ _repair_truncated_gpu_metrics_tail() {
271274
return 0
272275
}
273276

277+
# Print the newest telemetry tick (whole epoch seconds) that EVERY observed
278+
# GPU has covered with a usable sample (numeric epoch timestamp, numeric
279+
# power > 0), or nothing when the stream holds no usable epoch-stamped row
280+
# (e.g. an amd-smi build emitting ISO timestamps). Column detection mirrors
281+
# _POWER_COL_RE/_POWER_EXCLUDE_RE/_GPU_INDEX_COL_RE in utils/aggregate_power.py.
282+
# POSIX awk only: the ROCm container images ship mawk/busybox awk.
283+
_amd_monitor_min_covered_tick() {
284+
[[ -f "$GPU_METRICS_CSV" ]] || return 0
285+
awk -F, '
286+
NR == 1 {
287+
for (i = 1; i <= NF; i++) {
288+
name = tolower($i)
289+
gsub(/^ +| +$/, "", name)
290+
sub(/\r$/, "", name)
291+
if (!power_col && name ~ /power/ && name !~ /limit|cap|max|min/)
292+
power_col = i
293+
if (!gpu_col && name ~ /^(index|gpu|gpu_id|gpu_index|card|device)$/)
294+
gpu_col = i
295+
}
296+
next
297+
}
298+
!power_col || !gpu_col { next }
299+
{
300+
# amd-smi quotes list-valued cells that embed commas; neutralize
301+
# them so the power cell keeps its header-relative position.
302+
line = $0
303+
sub(/\r$/, "", line)
304+
if (line ~ /"/) {
305+
n = split(line, seg, /"/)
306+
line = ""
307+
for (i = 1; i <= n; i++) {
308+
if (i % 2 == 0) gsub(/,/, ";", seg[i])
309+
line = line seg[i]
310+
}
311+
}
312+
count = split(line, cell, /,/)
313+
if (count < power_col || count < gpu_col) next
314+
if (cell[1] !~ /^[0-9]+(\.[0-9]+)?$/) next
315+
if (cell[power_col] !~ /^[0-9]+(\.[0-9]+)?$/) next
316+
if (cell[power_col] + 0 <= 0) next
317+
if (cell[gpu_col] == "") next
318+
ts = cell[1] + 0
319+
# Mirror _parse_timestamp in utils/aggregate_power.py: normalize
320+
# millisecond epochs so a ms-stamping amd-smi build cannot
321+
# trivially satisfy any second-scale stop target.
322+
if (ts > 1e12) ts /= 1000
323+
gpu = cell[gpu_col]
324+
if (!(gpu in newest) || ts > newest[gpu])
325+
newest[gpu] = ts
326+
}
327+
END {
328+
have = 0
329+
for (gpu in newest)
330+
if (!have || newest[gpu] < min) { min = newest[gpu]; have = 1 }
331+
if (have) printf "%d\n", min
332+
}
333+
' "$GPU_METRICS_CSV" 2>/dev/null
334+
return 0
335+
}
336+
337+
# Block until every observed GPU has a usable tick at/after the first whole
338+
# second past stop entry, so any window end preceding the stop request is
339+
# bracketed on file. Bounded by AMD_MONITOR_STOP_TIMEOUT_S; always returns 0 —
340+
# on timeout or early monitor death it warns and lets aggregation attribute
341+
# the missing coverage (fail-safe, never fail-silent).
342+
_wait_for_amd_stop_coverage() {
343+
local target deadline covered timeout_s
344+
# A non-integer timeout (e.g. "30s") would abort the whole stop_gpu_monitor
345+
# call under `set -e` at the arithmetic below, leaking the monitor process
346+
# and skipping tail repair + the energy sidecar; fall back to the default.
347+
timeout_s="${AMD_MONITOR_STOP_TIMEOUT_S:-30}"
348+
if [[ ! "$timeout_s" =~ ^-?[0-9]+$ ]]; then
349+
echo "[GPU Monitor] Warning: ignoring non-integer AMD_MONITOR_STOP_TIMEOUT_S='$timeout_s', using 30" >&2
350+
timeout_s=30
351+
fi
352+
if [[ "$timeout_s" -le 0 ]]; then
353+
return 0
354+
fi
355+
target=$(( $(date +%s) + 1 ))
356+
deadline=$(( target + timeout_s ))
357+
while :; do
358+
covered=$(_amd_monitor_min_covered_tick)
359+
if [[ -z "$covered" ]]; then
360+
# Non-epoch timestamps or an unusable stream: keep the legacy
361+
# fixed tail so older amd-smi builds behave exactly as before.
362+
sleep $(( ${GPU_MONITOR_INTERVAL:-1} + 2 ))
363+
return 0
364+
fi
365+
if [[ "$covered" -ge "$target" ]]; then
366+
return 0
367+
fi
368+
if ! _background_process_is_running "$GPU_MONITOR_PID"; then
369+
echo "[GPU Monitor] Warning: AMD monitor exited before covering the stop request (covered=$covered target=$target)" >&2
370+
return 0
371+
fi
372+
if [[ "$(date +%s)" -ge "$deadline" ]]; then
373+
echo "[GPU Monitor] Warning: AMD telemetry never covered the stop request within ${timeout_s}s (covered=$covered target=$target)" >&2
374+
return 0
375+
fi
376+
sleep 1
377+
done
378+
}
379+
274380
# Write one best-effort amd-smi snapshot; remove the file rather than keep a
275381
# partial one when the invocation fails.
276382
_write_amd_smi_sidecar() {
@@ -3283,9 +3389,15 @@ run_agentic_replay_and_write_outputs() (
32833389
esac
32843390

32853391
_stop_agentx_power_monitor() {
3392+
local mode="${1:-}"
32863393
if [ "$agentx_monitor_stopped" = "0" ]; then
3287-
agentx_monitor_stopped=1
3394+
if [ "$mode" = "abort" ]; then
3395+
# A cancelled run's power validity is moot; skip the AMD
3396+
# coverage wait so signal teardown stays fast.
3397+
AMD_MONITOR_STOP_TIMEOUT_S=0
3398+
fi
32883399
stop_gpu_monitor
3400+
agentx_monitor_stopped=1
32893401
fi
32903402
}
32913403

@@ -3329,10 +3441,11 @@ run_agentic_replay_and_write_outputs() (
33293441
agentx_monitor_stopped=0
33303442
# This function runs in a subshell, so these handlers cannot replace
33313443
# launcher-owned traps. The stopped flag keeps explicit and signal/EXIT
3332-
# cleanup idempotent.
3333-
trap '_stop_agentx_power_monitor' EXIT
3334-
trap '_stop_agentx_power_monitor; exit 130' INT
3335-
trap '_stop_agentx_power_monitor; exit 143' TERM
3444+
# cleanup idempotent after stopping completes. If a signal interrupts
3445+
# the normal coverage wait, abort cleanup must still kill the monitor.
3446+
trap '_stop_agentx_power_monitor abort' EXIT
3447+
trap '_stop_agentx_power_monitor abort; exit 130' INT
3448+
trap '_stop_agentx_power_monitor abort; exit 143' TERM
33363449
fi
33373450

33383451
echo "$REPLAY_CMD" > "$result_dir/benchmark_command.txt"

‎docs/results-and-ingestion.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,12 @@ Single-node GPU count is `tp * pp * pcp_size`. DCP does not multiply the physica
9696

9797
InferenceX-app treats routing fields as columns or config dimensions and stores numeric measurements in `benchmark_results.metrics` JSONB. The mapper supports v1 shared topology, v2 split prefill/decode topology, and nested v3 AgentX metrics. Unknown numeric metrics are retained and warned about, which permits schema growth without silently losing numeric data.
9898

99+
### Power telemetry stop and validation
100+
101+
The single-node AMD monitor in [`benchmark_lib.sh`](../benchmarks/benchmark_lib.sh) waits for every observed GPU to have a positive, numeric power sample at or beyond the first whole second after the stop request. `AMD_MONITOR_STOP_TIMEOUT_S` bounds the wait (default `30`; `0` skips it). Streams without usable epoch timestamps use the legacy fixed tail wait. A timeout does not certify coverage: the aggregator still rejects an unbracketed benchmark window. AgentX cancellation skips the coverage wait and stops the monitor, including when cancellation arrives during a normal stop.
102+
103+
[`single_node.py`](../infx/results/power/single_node.py) excludes unusable power rows outside the formal window from integration and bracketing. Within the boundary search band, these rows are counted per GPU in `power_validation.json` under `boundary_degenerate_rows`, even when no usable samples remain. In-window validation is unchanged; these counters are diagnostic evidence, not replacement measurements.
104+
99105
## Eval artifacts
100106

101107
### Per-config identity and collection

‎docs/results-and-ingestion_zh.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,12 @@ shape: array of benchmark row objects
9696

9797
InferenceX-app 将路由字段作为列或配置维度,并把数值测量存入 `benchmark_results.metrics` JSONB。映射器支持共享拓扑的 v1、拆分 prefill/decode 拓扑的 v2,以及嵌套 AgentX 指标的 v3。未知数值指标会被保留并产生警告,因此架构可以扩展,同时不会无提示地丢失数值数据。
9898

99+
### 功耗遥测停止与校验
100+
101+
[`benchmark_lib.sh`](../benchmarks/benchmark_lib.sh) 中的单节点 AMD 监控会等待每个已观测 GPU 都记录到有效的正数功耗样本,且时间戳不早于停止请求之后的第一个整秒。`AMD_MONITOR_STOP_TIMEOUT_S` 限制等待时长(默认 `30`;设为 `0` 可跳过等待)。没有可用 Unix 时间戳样本的数据流沿用原有的固定尾部等待。超时不代表覆盖有效:聚合器仍会拒绝未被样本完整包围的基准窗口。AgentX 取消运行时会跳过覆盖等待并停止监控,即使取消发生在正常停止的等待过程中。
102+
103+
[`single_node.py`](../infx/results/power/single_node.py) 不会将正式窗口之外的无效功耗行用于积分或边界覆盖。在边界搜索范围内,这些行会按 GPU 计入 `power_validation.json` 的 `boundary_degenerate_rows`,即使最终没有任何可用样本也会保留计数。窗口内的校验规则保持不变;这些计数仅供诊断,不能替代实测数据。
104+
99105
## 评测工件
100106

101107
### 单配置身份和收集

‎infx/results/power/single_node.py‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,11 @@
99
aggregate and a validation sidecar, but does not fail the benchmark. Power
1010
studies can set ``REQUIRE_POWER=1`` to fail after those audit artifacts exist.
1111
The aggregate carries numeric ``power_valid`` (1/0) for metric ingestion; the
12-
sidecar is the canonical source for boolean validity and reason codes.
12+
sidecar is the canonical source for boolean validity and reason codes. Rows in
13+
the ingest band but outside the formal window whose power is missing,
14+
non-finite, or <= 0 are teardown noise: they are skipped and counted in the
15+
sidecar's ``boundary_degenerate_rows`` instead of poisoning validity or faking
16+
window bracketing.
1317
"""
1418

1519
from __future__ import annotations
@@ -22,7 +26,7 @@
2226
import os
2327
import re
2428
import sys
25-
from dataclasses import dataclass
29+
from dataclasses import dataclass, field
2630
from datetime import datetime, timezone
2731
from pathlib import Path
2832
from statistics import mean
@@ -65,6 +69,10 @@ class PowerIntegration:
6569
per_gpu_max_sample_gap_s: dict[str, float]
6670
per_gpu_energy_j: dict[str, float]
6771
device_issues: dict[str, list[str]]
72+
# Rows in the ingest band but outside the formal window whose power was
73+
# missing/N-A/non-finite/<=0, skipped and counted per GPU; "unknown"
74+
# buckets rows without a GPU identity.
75+
boundary_degenerate_rows: dict[str, int] = field(default_factory=dict)
6876
avg_power_w: float | None = None
6977
p75_power_w: float | None = None
7078
p75_total_gpu_power_w: float | None = None
@@ -237,6 +245,7 @@ def _empty_integration(
237245
*,
238246
expected_num_gpus: int | None,
239247
reasons: list[str],
248+
boundary_degenerate_rows: dict[str, int] | None = None,
240249
) -> PowerIntegration:
241250
"""Build an invalid integration result when no device data is available."""
242251
return PowerIntegration(
@@ -248,6 +257,7 @@ def _empty_integration(
248257
per_gpu_max_sample_gap_s={},
249258
per_gpu_energy_j={},
250259
device_issues={},
260+
boundary_degenerate_rows=boundary_degenerate_rows or {},
251261
)
252262

253263

@@ -308,6 +318,7 @@ def integrate_power(
308318
# expose timestamps at lower resolution than their sampling cadence, so
309319
# duplicate-timestamp readings are averaged rather than treated as corrupt.
310320
raw_samples: dict[str, dict[float, list[float]]] = {}
321+
boundary_degenerate: dict[str, int] = {}
311322
saw_missing_gpu_identity = False
312323
try:
313324
with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f:
@@ -348,6 +359,15 @@ def integrate_power(
348359

349360
power = _parse_power((row.get(power_col) or "").strip())
350361
gpu_id = (row.get(gpu_col) or "").strip()
362+
if (power is None or not math.isfinite(power) or power <= 0.0) and (
363+
timestamp < start_unix or timestamp > end_unix
364+
):
365+
# SMI teardown rows can carry N/A or 0 W cells: outside the
366+
# formal window they are counted, never used to satisfy
367+
# bracketing or to poison in-window validity.
368+
key = gpu_id or "unknown"
369+
boundary_degenerate[key] = boundary_degenerate.get(key, 0) + 1
370+
continue
351371
if power is None:
352372
_append_reason(reasons, "invalid_power_sample")
353373
continue
@@ -364,6 +384,7 @@ def integrate_power(
364384
return _empty_integration(
365385
expected_num_gpus=expected_num_gpus,
366386
reasons=reasons,
387+
boundary_degenerate_rows=boundary_degenerate,
367388
)
368389

369390
if saw_missing_gpu_identity:
@@ -373,6 +394,7 @@ def integrate_power(
373394
return _empty_integration(
374395
expected_num_gpus=expected_num_gpus,
375396
reasons=reasons,
397+
boundary_degenerate_rows=boundary_degenerate,
376398
)
377399

378400
observed_gpu_ids = tuple(sorted(raw_samples, key=_gpu_sort_key))
@@ -450,6 +472,7 @@ def integrate_power(
450472
per_gpu_max_sample_gap_s=per_gpu_max_sample_gap_s,
451473
per_gpu_energy_j=per_gpu_energy_j,
452474
device_issues=device_issues,
475+
boundary_degenerate_rows=boundary_degenerate,
453476
avg_power_w=avg_power_w,
454477
p75_power_w=p75_total / len(observed_gpu_ids) if p75_total is not None else None,
455478
p75_total_gpu_power_w=p75_total,
@@ -679,6 +702,7 @@ def _validation_payload(
679702
"per_gpu_max_sample_gap_s": integration.per_gpu_max_sample_gap_s,
680703
"per_gpu_energy_j": integration.per_gpu_energy_j,
681704
"device_issues": integration.device_issues,
705+
"boundary_degenerate_rows": integration.boundary_degenerate_rows,
682706
"accumulator_check": accumulator_check,
683707
"metrics": audit_metrics(metrics),
684708
}

0 commit comments

Comments
 (0)