Skip to content

Commit ba0275c

Browse files
committed
feat: separate native multinode power collection contract
拆分原生多节点功耗采集契约、UTC 上下文和生命周期屏障,不启用新路径。
1 parent c304072 commit ba0275c

15 files changed

Lines changed: 1057 additions & 18 deletions

‎.github/workflows/benchmark-multinode-tmpl.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -499,6 +499,7 @@ jobs:
499499
agg_${{ env.RESULT_FILENAME }}_*.json
500500
power_validation_${{ env.RESULT_FILENAME }}_*.json
501501
LOGS/power/**
502+
LOGS/native_power/**
502503
result_processing_${{ env.RESULT_FILENAME }}.json
503504
LOGS/*/results_*.json
504505
LOGS/agentic/**/agentic_power_concurrency_*.json

‎.github/workflows/benchmark-tmpl.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -445,11 +445,13 @@ jobs:
445445
name: ${{ inputs.eval-only && 'eval_gpu_metrics_' || 'gpu_metrics_' }}${{ env.RESULT_FILENAME }}
446446
path: |
447447
gpu_metrics.csv
448+
gpu_metrics_context.json
448449
gpu_metrics_energy_start.csv
449450
gpu_metrics_energy_end.csv
450451
gpu_metrics_identity.json
451452
gpu_metrics_identity.csv
452453
results/gpu_metrics*.csv
454+
results/gpu_metrics_context.json
453455
results/gpu_metrics_identity.json
454456
if-no-files-found: ignore
455457

@@ -462,12 +464,14 @@ jobs:
462464
${{ env.RESULT_FILENAME }}.json
463465
agg_${{ env.RESULT_FILENAME }}.json
464466
gpu_metrics.csv
467+
gpu_metrics_context.json
465468
gpu_metrics_energy_start.csv
466469
gpu_metrics_energy_end.csv
467470
gpu_metrics_identity.json
468471
gpu_metrics_identity.csv
469472
power_validation_${{ env.RESULT_FILENAME }}.json
470473
results/gpu_metrics*.csv
474+
results/gpu_metrics_context.json
471475
results/gpu_metrics_identity.json
472476
results/agentic_power_window.json
473477
results/agentic_power_timezone_offset.txt

‎benchmarks/benchmark_lib.sh‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,7 @@ start_gpu_monitor() {
177177
GPU_METRICS_CSV="$output"
178178
GPU_MONITOR_INTERVAL="$interval"
179179
export GPU_METRICS_CSV
180+
printf '{"timestamp_timezone":"UTC"}\n' > "${output%.csv}_context.json"
180181

181182
if command -v nvidia-smi &>/dev/null; then
182183
GPU_MONITOR_VENDOR="nvidia"
@@ -185,7 +186,7 @@ start_gpu_monitor() {
185186
rm -f "${output%.csv}_identity.csv"
186187
echo "[GPU Monitor] Warning: NVIDIA identity sidecar failed" >&2
187188
fi
188-
nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \
189+
TZ=UTC nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \
189190
--format=csv -l "$interval" > "$output" 2>/dev/null &
190191
GPU_MONITOR_PID=$!
191192
echo "[GPU Monitor] Started NVIDIA (PID=$GPU_MONITOR_PID, interval=${interval}s, output=$output)"
@@ -196,7 +197,7 @@ start_gpu_monitor() {
196197
# Python; measured on MI355X: trailing ticks were lost at kill without it).
197198
# Pipe through awk to: skip preamble lines, keep first CSV header, skip repeated
198199
# headers, and flush every row so killing the pipe cannot discard buffered samples.
199-
PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \
200+
TZ=UTC PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \
200201
| awk '/^timestamp,/{if(!h){print;h=1};next} h{print;fflush()}' > "$output" &
201202
GPU_MONITOR_PID=$!
202203
# Hardware energy-accumulator + identity snapshots; the end-side twin in
@@ -232,7 +233,7 @@ stop_gpu_monitor() {
232233
case "$GPU_MONITOR_VENDOR" in
233234
nvidia)
234235
if _repair_truncated_gpu_metrics_tail; then
235-
nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \
236+
TZ=UTC nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \
236237
--format=csv,noheader >> "$GPU_METRICS_CSV" 2>/dev/null ||
237238
echo "[GPU Monitor] Warning: final NVIDIA sample failed" >&2
238239
fi

‎benchmarks/native_power_collect.sh‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
#!/usr/bin/env bash
2+
# One native SMI collector per serving node. Raw files stay on node-local scratch;
3+
# the launcher stages them as its host user after containers stop.
4+
set -uo pipefail
5+
power_dir=$1
6+
control_dir=$2
7+
vendor=$3
8+
rank=$4
9+
role=$5
10+
gpu_indices=$6
11+
num_nodes=$7
12+
repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
13+
export PYTHONPATH="$repo_root${PYTHONPATH:+:$PYTHONPATH}"
14+
source "$repo_root/benchmarks/benchmark_lib.sh"
15+
mkdir -p "$power_dir"
16+
collector_rc=0
17+
finished=0
18+
19+
write_control() {
20+
local path="$control_dir/$1"
21+
local pending="$path.tmp"
22+
printf '%s\n' "$2" > "$pending" || return
23+
# The control directory is created by the host user before Docker starts.
24+
# Containers must not strand root-owned files in the shared runner tree.
25+
if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then
26+
chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$pending" || return
27+
fi
28+
mv -f "$pending" "$path"
29+
}
30+
31+
finish() {
32+
local incoming_rc=$?
33+
[[ "$finished" == 0 ]] || return
34+
if [[ "$incoming_rc" != 0 ]]; then collector_rc=$incoming_rc; fi
35+
if ! _background_process_is_running "${GPU_MONITOR_PID:-}"; then collector_rc=1; fi
36+
stop_gpu_monitor
37+
if [[ "$vendor" == amd ]]; then
38+
_write_amd_smi_sidecar "$power_dir/gpu_metrics_devices_end.json" list --json
39+
else
40+
nvidia-smi --query-gpu=index,uuid,pci.bus_id,name,driver_version --format=csv \
41+
> "$power_dir/gpu_metrics_identity_end.csv" || collector_rc=1
42+
fi
43+
python3 -m infx.results.power.native_multinode end --directory "$power_dir" \
44+
--collector-exit-code "$collector_rc" || collector_rc=1
45+
if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then
46+
chown -R "$POWERX_HOST_UID:$POWERX_HOST_GID" "$power_dir" || collector_rc=1
47+
fi
48+
write_control "done-$rank" "$collector_rc"
49+
finished=1
50+
}
51+
trap finish EXIT
52+
trap 'collector_rc=130; AMD_MONITOR_STOP_TIMEOUT_S=0; exit 130' INT
53+
trap 'collector_rc=143; AMD_MONITOR_STOP_TIMEOUT_S=0; exit 143' TERM HUP
54+
55+
case "${POWERX_CLOCK_SYNCHRONIZED:-false}" in
56+
yes|true) clock_synchronized=true ;;
57+
*) clock_synchronized=false ;;
58+
esac
59+
60+
python3 -m infx.results.power.native_multinode begin --directory "$power_dir" \
61+
--vendor "$vendor" --rank "$rank" --role "$role" --gpu-indices "$gpu_indices" \
62+
--num-nodes "$num_nodes" --clock-synchronized "$clock_synchronized" || exit 1
63+
start_gpu_monitor --output "$power_dir/gpu_metrics.csv" || exit 1
64+
[[ "$GPU_MONITOR_VENDOR" == "$vendor" ]] || exit 1
65+
if [[ "$vendor" == amd ]]; then
66+
_write_amd_smi_sidecar "$power_dir/gpu_metrics_devices.json" list --json
67+
fi
68+
write_control "ready-$rank" ready
69+
while [[ ! -f "$control_dir/stop" ]]; do
70+
_background_process_is_running "$GPU_MONITOR_PID" || exit 1
71+
sleep 1 &
72+
wait $! || true
73+
done
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
#!/usr/bin/env bash
2+
# Shared barriers for native collectors. Launchers own host scratch and mounts.
3+
4+
powerx_start_collector() {
5+
local power_dir="$1" control_dir="$2" vendor="$3" rank="$4" role="$5" gpus="$6" nodes="$7"
6+
local indices
7+
indices=$(seq -s, 0 "$((gpus - 1))") || return 1
8+
POWERX_CONTROL_DIR="$control_dir"
9+
POWERX_NUM_NODES="$nodes"
10+
bash "$(dirname "${BASH_SOURCE[0]}")/native_power_collect.sh" \
11+
"$power_dir" "$control_dir" "$vendor" "$rank" "$role" "$indices" "$nodes" &
12+
POWERX_COLLECTOR_PID=$!
13+
}
14+
15+
powerx_write_control() {
16+
local path="$POWERX_CONTROL_DIR/$1"
17+
local pending="$path.tmp"
18+
printf '%s\n' "$2" > "$pending" || return
19+
if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then
20+
chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$pending" || return
21+
fi
22+
mv -f "$pending" "$path"
23+
}
24+
25+
powerx_wait_collectors() {
26+
local phase="$1" deadline=$((SECONDS + ${POWERX_BARRIER_TIMEOUT_S:-60})) rank pending failed
27+
while :; do
28+
pending=0
29+
failed=0
30+
for ((rank=0; rank<POWERX_NUM_NODES; rank++)); do
31+
if [[ "$phase" == ready && -f "$POWERX_CONTROL_DIR/done-$rank" ]]; then
32+
echo "PowerX collector $rank stopped before benchmark readiness" >&2
33+
return 1
34+
fi
35+
if [[ ! -f "$POWERX_CONTROL_DIR/$phase-$rank" ]]; then
36+
pending=1
37+
elif [[ "$phase" == done && "$(cat "$POWERX_CONTROL_DIR/done-$rank")" != 0 ]]; then
38+
failed=1
39+
fi
40+
done
41+
if [[ "$pending" == 0 ]]; then
42+
[[ "$failed" == 0 ]] || echo "One or more PowerX collectors failed" >&2
43+
return "$failed"
44+
fi
45+
if (( SECONDS >= deadline )); then
46+
echo "Timed out waiting for PowerX $phase receipts" >&2
47+
return 1
48+
fi
49+
sleep 1
50+
done
51+
}
52+
53+
powerx_stop_collectors() {
54+
local rc=0
55+
powerx_write_control stop stop || rc=$?
56+
powerx_wait_collectors done || rc=$?
57+
powerx_reap_collector || rc=$?
58+
return "$rc"
59+
}
60+
61+
powerx_reap_collector() {
62+
[[ -n "${POWERX_COLLECTOR_PID:-}" ]] || return 0
63+
local deadline=$((SECONDS + ${POWERX_BARRIER_TIMEOUT_S:-60})) rc=0
64+
while kill -0 "$POWERX_COLLECTOR_PID" 2>/dev/null; do
65+
if (( SECONDS >= deadline )); then
66+
kill -TERM "$POWERX_COLLECTOR_PID" 2>/dev/null || true
67+
sleep 2
68+
kill -KILL "$POWERX_COLLECTOR_PID" 2>/dev/null || true
69+
rc=1
70+
break
71+
fi
72+
sleep 1
73+
done
74+
wait "$POWERX_COLLECTOR_PID" || rc=$?
75+
POWERX_COLLECTOR_PID=""
76+
return "$rc"
77+
}

‎docs/results-and-ingestion.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,12 @@ For multinode fixed-sequence jobs, `utils/process_result.py --all` processes eve
106106

107107
Processing and diagnostic power-audit uploads run after launcher or validation failure, retaining raw and aggregate JSON. Normal `bmk_*` upload requires successful benchmark and processing steps, so an incomplete batch or failed Slurm job does not publish diagnostic rows. The main-branch ingest trigger can still publish other successful configurations from a partially failed sweep; it does not establish complete fleet coverage. Downstream importers can use the retained outcome to reject explicitly failed benchmarks.
108108

109+
### Native multinode telemetry
110+
111+
`native_power_collect.sh` and `native_power_lifecycle.sh` provide per-node collection and bounded ready/stop receipts. Launchers opt into the native package under `LOGS/native_power`; this prerequisite enables no new recipe. The adapter validates serving GPU identity, synchronized clocks, collector completion, and complete formal-window coverage. It preserves per-node failures, sample counts, and collector revision in the audit.
112+
113+
SMI collection records UTC context beside each CSV for portable replay. Unusable samples outside the formal window do not establish coverage; `boundary_degenerate_rows` retains their per-GPU counts.
114+
109115
## Eval artifacts
110116

111117
### Per-config identity and collection

‎docs/results-and-ingestion_zh.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,12 @@ InferenceX-app 将路由字段作为列或配置维度,并把数值测量存
106106

107107
启动器或验证失败后仍会运行处理和功耗诊断上传,并在审计工件中保留原始及聚合 JSON。正常 `bmk_*` 上传要求基准和处理步骤成功,因此不完整批次或 Slurm 失败不会发布诊断数据。主分支的入库触发器仍可发布部分失败 sweep 中其他成功配置的数据;这并不证明整个硬件范围已完成覆盖。下游导入器可利用保留的状态拒绝明确失败的基准结果。
108108

109+
### 原生多节点遥测
110+
111+
`native_power_collect.sh` 和 `native_power_lifecycle.sh` 提供每节点采集及有时限的就绪/停止状态文件。启动器可使用 `LOGS/native_power` 下的原生产物;此前置改动不会启用新 recipe。适配器验证服务 GPU 身份、时钟同步、采集完成及正式窗口完整覆盖,并在审计中保留节点故障、样本数和采集器版本。
112+
113+
SMI 采集在 CSV 旁记录 UTC 上下文以支持跨环境回放。正式窗口外的无效样本不能构成覆盖;`boundary_degenerate_rows` 保留其逐 GPU 计数。
114+
109115
## 评测工件
110116

111117
### 单配置身份和收集

‎infx/results/fixed_sequence.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,21 @@ def aggregate_power_result(
248248
expected_num_gpus = int(env['TP']) * int(env.get('PP_SIZE', '1')) * int(env.get('PCP_SIZE', '1'))
249249
try:
250250
if is_multinode:
251+
native_dir = Path(env.get('POWERX_NATIVE_DIR', 'LOGS/native_power'))
252+
if env.get('POWERX_NATIVE_DIR') or native_dir.is_dir():
253+
if source.is_dir() and source != native_dir:
254+
raise ValueError('Both native and SRT power packages are present')
255+
source = native_dir
256+
from .power.native_multinode import run
257+
258+
return run(
259+
native_dir, bench_path, agg_path,
260+
expected_prefill_gpus=prefill_gpus,
261+
expected_decode_gpus=decode_gpus,
262+
expected_aggregate_gpus=aggregate_gpus,
263+
validation_result=validation_path,
264+
require_power=require_power,
265+
)
251266
from .power.multinode import run
252267

253268
return run(

0 commit comments

Comments
 (0)