@@ -15,6 +15,7 @@ SQUASH_DIR="/mnt/lustre01/users-public/sa-shared"
1515# srt-slurm merge lands.
1616POWER_SRT_SLURM_URL=" https://github.com/edwingao28/srt-slurm.git"
1717POWER_SRT_SLURM_PIN=" 6fc1bed01a0b82dae0088a105c03ce0cfb353443"
18+ AGENTX_POWER_SRT_SLURM_PIN=" 80d7203e424f903c9017de4608ee2044afce9574"
1819
1920# Enroot 3.x does not parse Docker's tag@digest syntax. For digest-pinned
2021# images, use its explicit registry syntax and pass the digest as the
@@ -332,12 +333,15 @@ if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk '
332333 USES_DCGM_POWER=1
333334fi
334335
335- # Note (wenyao): the producer pin follows the srt-slurm main lineage that the
336- # dynamo-sglang lanes run on (fp8 validated end-to-end, fp4 recipes
337- # parse-verified against the pin); other frameworks clone diverging refs
338- # (aflowers branch, sa-submission), so fail fast for them instead.
339- if [[ " $USES_DCGM_POWER " == " 1" && " $FRAMEWORK " != " dynamo-sglang" ]]; then
340- echo " Error: dcgm-power lanes are only validated for FRAMEWORK=dynamo-sglang, got: $FRAMEWORK " >&2
336+ USES_AGENTX_POWER=0
337+ if [[ " $USES_DCGM_POWER " == " 1" && " $IS_AGENTIC " == " 1" &&
338+ " $MODEL_PREFIX " == " kimik3" && " $PRECISION " == " fp4" &&
339+ " $FRAMEWORK " == " dynamo-vllm" &&
340+ " $_RECIPE_REL " == recipes/vllm/kimi-k3/agentic/* ]]; then
341+ USES_AGENTX_POWER=1
342+ fi
343+ if [[ " $USES_DCGM_POWER " == " 1" && " $FRAMEWORK " != " dynamo-sglang" && " $USES_AGENTX_POWER " != " 1" ]]; then
344+ echo " Error: dcgm-power requires dynamo-sglang or the supported Kimi-K3 AgentX route" >&2
341345 exit 1
342346fi
343347
@@ -472,12 +476,20 @@ elif [[ "$IS_AGENTIC" == "1" && (( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION"
472476 fi
473477# Kimi-K3 requires direct multi-node vLLM frontend support from srt-slurm.
474478elif [[ " $IS_AGENTIC " == " 1" && " $MODEL_PREFIX " == " kimik3" ]]; then
475- git clone --branch v1.0.53 --single-branch https://github.com/NVIDIA/srt-slurm.git " $SRT_REPO_DIR " || exit 1
476- cd " $SRT_REPO_DIR " || exit 1
477- test " $( git rev-parse HEAD) " = " 217f94387abeddfed7149a71955dc523e07cd765" || {
478- echo " Error: NVIDIA/srt-slurm v1.0.53 resolved to an unexpected commit" >&2
479- exit 1
480- }
479+ if [[ " $USES_AGENTX_POWER " == " 1" ]]; then
480+ git clone " $POWER_SRT_SLURM_URL " " $SRT_REPO_DIR " || exit 1
481+ cd " $SRT_REPO_DIR " || exit 1
482+ git checkout " $AGENTX_POWER_SRT_SLURM_PIN " || exit 1
483+ test " $( git rev-parse HEAD) " = " $AGENTX_POWER_SRT_SLURM_PIN " || exit 1
484+ git rev-parse HEAD > " $GITHUB_WORKSPACE /power-producer-sha.txt"
485+ else
486+ git clone --branch v1.0.53 --single-branch https://github.com/NVIDIA/srt-slurm.git " $SRT_REPO_DIR " || exit 1
487+ cd " $SRT_REPO_DIR " || exit 1
488+ test " $( git rev-parse HEAD) " = " 217f94387abeddfed7149a71955dc523e07cd765" || {
489+ echo " Error: NVIDIA/srt-slurm v1.0.53 resolved to an unexpected commit" >&2
490+ exit 1
491+ }
492+ fi
481493 python3 " $GITHUB_WORKSPACE /runners/patch_srt_vllm_dp_ranks.py" " $( pwd) " || exit 1
482494 mkdir -p recipes/vllm/kimi-k3/agentic || exit 1
483495 cp -rT " $GITHUB_WORKSPACE /benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \
@@ -759,6 +771,12 @@ sed -i "s/^name:.*/name: \"${SRT_SLURM_JOB_NAME}\"/" "$CONFIG_PATH"
759771python3 " $GITHUB_WORKSPACE /runners/inject_synthetic_acceptance.py" \
760772 " $CONFIG_PATH " " $FRAMEWORK " || exit 1
761773
774+ if [[ " $USES_AGENTX_POWER " == " 1" ]]; then
775+ read -r -a POWER_CONCURRENCIES <<< " $CONC_LIST"
776+ python3 " $GITHUB_WORKSPACE /runners/inject_srt_power_concurrencies.py" \
777+ " $CONFIG_PATH " " ${POWER_CONCURRENCIES[@]} " || exit 1
778+ fi
779+
762780# Don't leak the login-node venv to the compute-node orchestrator. sbatch's
763781# default --export=ALL propagates VIRTUAL_ENV (set by `source
764782# .venv/bin/activate` above) into job_script_minimal.j2, whose
@@ -822,13 +840,24 @@ trap 'exit 143' TERM HUP
822840LOGS_DIR=" outputs/$JOB_ID /logs"
823841LOG_FILE=" $LOGS_DIR /sweep_${JOB_ID} .log"
824842
825- stream_slurm_job_log " $JOB_ID " " $LOG_FILE " || exit 1
843+ AGENTX_POWER_RC=0
844+ stream_slurm_job_log " $JOB_ID " " $LOG_FILE " || AGENTX_POWER_RC=$?
845+ if [[ " $AGENTX_POWER_RC " != " 0" && " $USES_AGENTX_POWER " != " 1" ]]; then
846+ exit " $AGENTX_POWER_RC "
847+ fi
826848
827849set -x
828850
829851echo " Job $JOB_ID completed!"
830852echo " Collecting results..."
831853
854+ if [[ " $USES_AGENTX_POWER " == " 1" && " ${EVAL_ONLY:- false} " != " true" ]]; then
855+ read -r -a POWER_CONCURRENCIES <<< " $CONC_LIST"
856+ collect_agentic_power_results " $JOB_ID " " $LOGS_DIR " " $INFMAX_WORKSPACE " \
857+ " $GITHUB_WORKSPACE " " $RESULT_FILENAME " " $AGENTX_POWER_SRT_SLURM_PIN " \
858+ " ${POWER_CONCURRENCIES[@]} " || AGENTX_POWER_RC=$?
859+ fi
860+
832861if [ -d " $LOGS_DIR " ]; then
833862 echo " Found logs directory: $LOGS_DIR "
834863 # Provenance markers travel inside the server-logs bundle so the offline
@@ -844,6 +873,11 @@ else
844873 echo " Warning: Logs directory not found at $LOGS_DIR "
845874fi
846875
876+ if [[ " $AGENTX_POWER_RC " != " 0" ]]; then
877+ echo " ERROR: AgentX job or power validation failed after staging audit artifacts" >&2
878+ exit " $AGENTX_POWER_RC "
879+ fi
880+
847881if [[ " ${EVAL_ONLY:- false} " != " true" ]]; then
848882 if [ ! -d " $LOGS_DIR " ]; then
849883 exit 1
@@ -854,10 +888,12 @@ if [[ "${EVAL_ONLY:-false}" != "true" ]]; then
854888 # INFMAX_WORKSPACE mount. Its aggregation step writes one
855889 # ${RESULT_FILENAME}_conc<N>.json there per point; stage those files
856890 # back to GITHUB_WORKSPACE for the workflow guard and artifact upload.
857- copy_agentic_results \
858- " $INFMAX_WORKSPACE " \
859- " $GITHUB_WORKSPACE " \
860- " $RESULT_FILENAME " || exit 1
891+ if [[ " $USES_AGENTX_POWER " != " 1" ]]; then
892+ copy_agentic_results \
893+ " $INFMAX_WORKSPACE " \
894+ " $GITHUB_WORKSPACE " \
895+ " $RESULT_FILENAME " || exit 1
896+ fi
861897 else
862898 # Find all fixed-sequence result subdirectories.
863899 RESULT_SUBDIRS=$( find " $LOGS_DIR " -maxdepth 1 -type d -name " *isl*osl*" 2> /dev/null)
0 commit comments