|
| 1 | +#!/usr/bin/env bash |
| 2 | +# One native SMI collector per serving node. Raw files stay on node-local scratch; |
| 3 | +# the launcher stages them as its host user after containers stop. |
| 4 | +set -uo pipefail |
| 5 | +power_dir=$1 |
| 6 | +control_dir=$2 |
| 7 | +vendor=$3 |
| 8 | +rank=$4 |
| 9 | +role=$5 |
| 10 | +gpu_indices=$6 |
| 11 | +num_nodes=$7 |
| 12 | +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) |
| 13 | +export PYTHONPATH="$repo_root${PYTHONPATH:+:$PYTHONPATH}" |
| 14 | +source "$repo_root/benchmarks/benchmark_lib.sh" |
| 15 | +mkdir -p "$power_dir" |
| 16 | +collector_rc=0 |
| 17 | +finished=0 |
| 18 | + |
| 19 | +write_control() { |
| 20 | + local path="$control_dir/$1" |
| 21 | + local pending="$path.tmp" |
| 22 | + printf '%s\n' "$2" > "$pending" || return |
| 23 | + # The control directory is created by the host user before Docker starts. |
| 24 | + # Containers must not strand root-owned files in the shared runner tree. |
| 25 | + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then |
| 26 | + chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$pending" || return |
| 27 | + fi |
| 28 | + mv -f "$pending" "$path" |
| 29 | +} |
| 30 | + |
| 31 | +finish() { |
| 32 | + local incoming_rc=$? |
| 33 | + [[ "$finished" == 0 ]] || return |
| 34 | + if [[ "$incoming_rc" != 0 ]]; then collector_rc=$incoming_rc; fi |
| 35 | + if ! _background_process_is_running "${GPU_MONITOR_PID:-}"; then collector_rc=1; fi |
| 36 | + stop_gpu_monitor |
| 37 | + if [[ "$vendor" == amd ]]; then |
| 38 | + _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices_end.json" list --json |
| 39 | + else |
| 40 | + nvidia-smi --query-gpu=index,uuid,pci.bus_id,name,driver_version --format=csv \ |
| 41 | + > "$power_dir/gpu_metrics_identity_end.csv" || collector_rc=1 |
| 42 | + fi |
| 43 | + python3 -m infx.results.power.native_multinode end --directory "$power_dir" \ |
| 44 | + --collector-exit-code "$collector_rc" || collector_rc=1 |
| 45 | + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then |
| 46 | + chown -R "$POWERX_HOST_UID:$POWERX_HOST_GID" "$power_dir" || collector_rc=1 |
| 47 | + fi |
| 48 | + write_control "done-$rank" "$collector_rc" |
| 49 | + finished=1 |
| 50 | +} |
| 51 | +trap finish EXIT |
| 52 | +trap 'collector_rc=130; AMD_MONITOR_STOP_TIMEOUT_S=0; exit 130' INT |
| 53 | +trap 'collector_rc=143; AMD_MONITOR_STOP_TIMEOUT_S=0; exit 143' TERM HUP |
| 54 | + |
| 55 | +case "${POWERX_CLOCK_SYNCHRONIZED:-false}" in |
| 56 | + yes|true) clock_synchronized=true ;; |
| 57 | + *) clock_synchronized=false ;; |
| 58 | +esac |
| 59 | + |
| 60 | +python3 -m infx.results.power.native_multinode begin --directory "$power_dir" \ |
| 61 | + --vendor "$vendor" --rank "$rank" --role "$role" --gpu-indices "$gpu_indices" \ |
| 62 | + --num-nodes "$num_nodes" --clock-synchronized "$clock_synchronized" || exit 1 |
| 63 | +start_gpu_monitor --output "$power_dir/gpu_metrics.csv" || exit 1 |
| 64 | +[[ "$GPU_MONITOR_VENDOR" == "$vendor" ]] || exit 1 |
| 65 | +if [[ "$vendor" == amd ]]; then |
| 66 | + _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices.json" list --json |
| 67 | +fi |
| 68 | +write_control "ready-$rank" ready |
| 69 | +while [[ ! -f "$control_dir/stop" ]]; do |
| 70 | + _background_process_is_running "$GPU_MONITOR_PID" || exit 1 |
| 71 | + sleep 1 & |
| 72 | + wait $! || true |
| 73 | +done |
0 commit comments