Fix #1865

.ci/docker/build.sh

@@ -114,6 +114,14 @@ case "$tag" in
     TRITON=yes
     INSTALL_MINGW=yes
     ;;
+  pytorch-linux-jammy-cuda13.0-cudnn9-py3.12-gcc11)
+    CUDA_VERSION=13.0.2
+    ANACONDA_PYTHON_VERSION=3.12
+    GCC_VERSION=11
+    KATEX=yes
+    TRITON=yes
+    INSTALL_MINGW=yes
+    ;;
   pytorch-linux-jammy-cuda13.0-cudnn9-py3-gcc11-inductor-benchmarks)
     CUDA_VERSION=13.0.2
     ANACONDA_PYTHON_VERSION=3.10
@@ -122,6 +130,14 @@ case "$tag" in
     TRITON=yes
     INDUCTOR_BENCHMARKS=yes
     ;;
+  pytorch-linux-jammy-cuda13.0-cudnn9-py3.12-gcc11-inductor-benchmarks)
+    CUDA_VERSION=13.0.2
+    ANACONDA_PYTHON_VERSION=3.12
+    GCC_VERSION=11
+    KATEX=yes
+    TRITON=yes
+    INDUCTOR_BENCHMARKS=yes
+    ;;
   pytorch-linux-jammy-cuda13.0-cudnn9-py3.12-gcc11-vllm)
     CUDA_VERSION=13.0.2
     ANACONDA_PYTHON_VERSION=3.12
@@ -248,20 +264,22 @@ case "$tag" in
     ;;
   pytorch-linux-jammy-linter)
     PYTHON_VERSION=3.10
+    CLANG_VERSION=18
     ;;
   pytorch-linux-jammy-cuda13.0-cudnn9-py3.10-linter)
     PYTHON_VERSION=3.10
     CUDA_VERSION=13.0.2
+    CLANG_VERSION=18
     ;;
-  pytorch-linux-jammy-aarch64-py3.10-gcc13)
+  pytorch-linux-jammy-aarch64-py3.10-gcc15)
     ANACONDA_PYTHON_VERSION=3.10
-    GCC_VERSION=13
+    GCC_VERSION=15
     ACL=yes
     OPENBLAS=yes
     ;;
-  pytorch-linux-jammy-aarch64-py3.10-gcc13-inductor-benchmarks)
+  pytorch-linux-jammy-aarch64-py3.10-gcc15-inductor-benchmarks)
     ANACONDA_PYTHON_VERSION=3.10
-    GCC_VERSION=13
+    GCC_VERSION=15
     ACL=yes
     OPENBLAS=yes
     INDUCTOR_BENCHMARKS=yes
@@ -309,26 +327,34 @@ esac
 
 tmp_tag=$(basename "$(mktemp -u)" | tr '[:upper:]' '[:lower:]')
 
-no_cache_flag=""
 progress_flag=""
-# Do not use cache and progress=plain when in CI
+# Plain (non-TTY) progress output in CI for complete, readable build logs.
 if [[ -n "${CI:-}" ]]; then
-  no_cache_flag="--no-cache"
   progress_flag="--progress=plain"
 fi
 
 # On a remote buildkit builder there is no local Docker daemon to receive
 # `--load`, so push directly to the registry instead. The caller is expected
 # to have logged in to the target registry already (e.g. via ecr-login).
 output_flag="--load -t ${tmp_tag}"
+cache_flag=""
 if [[ -n "${REMOTE_BUILDKIT:-}" ]]; then
   output_flag="--push"
+  # Remote BuildKit pods are ephemeral with pod-private caches, so share a cache
+  # via a per-image (hence per-arch) tag in the same registry. mode=max caches
+  # intermediate stages; image-manifest/oci-mediatypes keep it ECR-compatible.
+  if [[ -n "${DOCKER_IMAGE:-}" ]]; then
+    cache_ref="${DOCKER_IMAGE%:*}:${image}-buildcache"
+    cache_flag="--cache-from type=registry,ref=${cache_ref}"
+    cache_flag="${cache_flag} --cache-to type=registry,ref=${cache_ref},mode=max,image-manifest=true,oci-mediatypes=true"
+  fi
 fi
 
 # Build image
-docker buildx build \
-       ${no_cache_flag} \
+build_image() {
+  docker buildx build \
        ${progress_flag} \
+       ${cache_flag} \
        --build-arg "BUILD_ENVIRONMENT=${image}" \
        --build-arg "LLVMDEV=${LLVMDEV:-}" \
        --build-arg "UBUNTU_VERSION=${UBUNTU_VERSION}" \
@@ -364,6 +390,39 @@ docker buildx build \
        ${output_flag} \
        "$@" \
        .
+}
+
+if [[ -z "${REMOTE_BUILDKIT:-}" ]]; then
+  build_image "$@"
+else
+  # The autoscaled pool may be cold / at capacity at start, where buildx's ~20s
+  # connect (gRPC default) fails before scale-up. Retry connection failures (not
+  # build errors) for ~2h so a capacity-limited build waits for a free pod instead
+  # of hard-failing — still within the 240m job timeout.
+  attempts="${REMOTE_BUILDKIT_CONNECT_ATTEMPTS:-360}"
+  delay="${REMOTE_BUILDKIT_CONNECT_DELAY:-15}"
+  for attempt in $(seq 1 "${attempts}"); do
+    build_log="$(mktemp)"
+    set +e
+    build_image "$@" 2>&1 | tee "${build_log}"
+    rc="${PIPESTATUS[0]}"
+    set -e
+    if [[ "${rc}" -eq 0 ]]; then
+      rm -f "${build_log}"
+      break
+    fi
+    if [[ "${attempt}" -lt "${attempts}" ]] && grep -qiE \
+      "waiting for connection|context deadline exceeded|server preface|failed to (dial|list workers)|connection (refused|reset)|no such host|transport: Error|i/o timeout|use of closed network connection|EOF" \
+      "${build_log}"; then
+      echo "Remote BuildKit not ready yet (attempt ${attempt}/${attempts}); retrying in ${delay}s..." >&2
+      rm -f "${build_log}"
+      sleep "${delay}"
+      continue
+    fi
+    rm -f "${build_log}"
+    exit "${rc}"
+  done
+fi
 
 # NVIDIA dockers for RC releases use tag names like `11.0-cudnn9-devel-ubuntu18.04-rc`,
 # for this case we will set UBUNTU_VERSION to `18.04-rc` so that the Dockerfile could

.ci/docker/ci_commit_pins/huggingface-requirements.txt

@@ -1,2 +1,2 @@
-transformers==5.7.0
+transformers==5.10.1
 soxr==0.5.0

.ci/docker/ci_commit_pins/nccl.txt

@@ -1 +1 @@
-v2.29.7-1
+v2.30.7-1

.ci/docker/ci_commit_pins/triton-xpu.txt

@@ -1 +1 @@
-21033c4e2be9b42c9e6ce7a39a70ead2aba279b4
+5fcc14d918045eb2c866336b55229d1253d6d60c

.ci/docker/ci_commit_pins/triton.txt

@@ -1 +1 @@
-88b227e23f0445f3f695bad05bbf1a363b4f50e0
+5d6048aa0a324e090ada215b609ea76620133845

.ci/docker/common/install_conda.sh

@@ -92,6 +92,9 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
   # Install some other packages, including those needed for Python test reporting
   pip_install -r /opt/conda/requirements-ci.txt
 
+  # Installed spmd-types with --no-deps to avoid pulling torch dependency at this point
+  pip_install --no-deps spmd-types==0.2.1
+
   if [ -n "$DOCS" ]; then
     apt-get update
     apt-get -y install expect-dev

.ci/docker/common/install_cuda.sh

@@ -113,7 +113,7 @@ function install_126 {
 }
 
 function install_129 {
-  CUDNN_VERSION=9.20.0.48
+  CUDNN_VERSION=9.23.1.3
   CUSPARSELT_VERSION=0.8.1.1
   echo "Installing CUDA 12.9.1 and cuDNN ${CUDNN_VERSION} and NVSHMEM and NCCL and cuSparseLt-${CUSPARSELT_VERSION}"
   # install CUDA 12.9.1 in the same container
@@ -132,7 +132,7 @@ function install_129 {
 }
 
 function install_128 {
-  CUDNN_VERSION=9.20.0.48
+  CUDNN_VERSION=9.23.1.3
   CUSPARSELT_VERSION=0.7.1.0
   echo "Installing CUDA 12.8.1 and cuDNN ${CUDNN_VERSION} and NVSHMEM and NCCL and cuSparseLt-${CUSPARSELT_VERSION}"
   # install CUDA 12.8.1 in the same container
@@ -151,7 +151,7 @@ function install_128 {
 }
 
 function install_130 {
-  CUDNN_VERSION=9.20.0.48
+  CUDNN_VERSION=9.23.1.3
   CUSPARSELT_VERSION=0.8.1.1
   echo "Installing CUDA 13.0 and cuDNN ${CUDNN_VERSION} and NVSHMEM and NCCL and cuSparseLt-${CUSPARSELT_VERSION}"
   # install CUDA 13.0 in the same container
@@ -170,7 +170,7 @@ function install_130 {
 }
 
 function install_132 {
-  CUDNN_VERSION=9.20.0.48
+  CUDNN_VERSION=9.23.1.3
   CUSPARSELT_VERSION=0.8.1.1
   echo "Installing CUDA 13.2 and cuDNN ${CUDNN_VERSION} and NVSHMEM and NCCL and cuSparseLt-${CUSPARSELT_VERSION}"
   # install CUDA 13.2 in the same container

.ci/docker/common/install_inductor_benchmark_deps.sh

@@ -12,7 +12,7 @@ function install_timm() {
   local commit
   commit=$(get_pinned_commit timm)
 
-  pip_install "git+https://github.com/huggingface/pytorch-image-models@${commit}"
+  pip_install --no-deps "git+https://github.com/huggingface/pytorch-image-models@${commit}"
 }
 
 function install_torchbench() {

.ci/docker/common/install_linter.sh

@@ -4,7 +4,7 @@ set -ex
 
 if [ -n "${UBUNTU_VERSION}" ]; then
   apt update
-  apt-get install -y clang doxygen git graphviz nodejs npm libtinfo5
+  apt-get install -y doxygen git graphviz nodejs npm libtinfo5
 fi
 
 # Do shallow clone of PyTorch so that we can init lintrunner in Docker build context

.ci/docker/common/install_openblas.sh

@@ -11,7 +11,7 @@ git clone https://github.com/OpenMathLib/OpenBLAS.git -b "${OPENBLAS_VERSION}" -
 OPENBLAS_CHECKOUT_DIR="OpenBLAS"
 OPENBLAS_BUILD_FLAGS="
 CC=gcc
-NUM_THREADS=128
+NUM_THREADS=256
 USE_OPENMP=1
 NO_SHARED=0
 DYNAMIC_ARCH=1

.ci/docker/common/install_rocm.sh

@@ -166,7 +166,7 @@ EOF
 
     # we want the patch version of 7.2 instead
     if [[ $(ver $ROCM_VERSION) -eq $(ver 7.2) ]]; then
-        ROCM_VERSION="${ROCM_VERSION}.1"
+        ROCM_VERSION="${ROCM_VERSION}.3"
     fi
 
     # Default url values

.ci/docker/common/install_torch_tpu.sh

@@ -178,14 +178,22 @@ fi
 # Using the confirmed path: requirements/requirements.txt
 # Filter out torch pins to prevent downgrading the CI build
 grep -vE "torch|torchvision|torchaudio" requirements/requirements.txt > requirements_no_torch.txt
-pip_install -r requirements_no_torch.txt
+# The requirements file is fully hash-locked, so every dependency is already
+# pinned explicitly. Use --no-deps so pip does not try to re-resolve the
+# torch* packages we just stripped out (e.g. torchvision pulled in by timm),
+# which would fail in pip's --require-hashes mode.
+pip_install --no-deps -r requirements_no_torch.txt
 rm requirements_no_torch.txt
 
 # 10. Build
 echo "Building TorchTPU Wheel..."
 export TORCH_SOURCE=$(python -c "import torch; import os; print(os.path.dirname(os.path.dirname(torch.__file__)))")
 
-as_jenkins env TORCH_SOURCE="${TORCH_SOURCE}" bazel build //ci/wheel:torch_tpu_wheel --config=local --define WHEEL_VERSION=0.1.0 --define TORCH_SOURCE=local
+# --repo_env is required so the torch_tpu repository rule can see TORCH_SOURCE
+# and locate the local torch install's ATen headers. Bazel sanitizes the
+# ambient environment for hermeticity, so a plain `env TORCH_SOURCE=...` is not
+# visible to repository rules (only --define TORCH_SOURCE=local is not enough).
+as_jenkins env TORCH_SOURCE="${TORCH_SOURCE}" bazel build //ci/wheel:torch_tpu_wheel --config=local --define WHEEL_VERSION=0.1.0 --define TORCH_SOURCE=local --repo_env=TORCH_SOURCE="${TORCH_SOURCE}" --action_env=JAX_PLATFORMS=cpu
 
 # 11. Install
 pip_install bazel-bin/ci/wheel/*.whl
@@ -201,4 +209,4 @@ export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${TORCH_LIB_PATH}"
 echo "Updated LD_LIBRARY_PATH to include: ${TORCH_LIB_PATH}"
 
 # Verify installation
-python -c "import torch; from torch_tpu import api; print(f'Success! Device: {api.tpu_device()}')"
+python -c "import torch; print(f'Success! Device: {torch.device(\"tpu\")}')"

.ci/docker/linter-cuda/Dockerfile

@@ -17,6 +17,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends libomp-dev && a
 COPY ./common/install_user.sh install_user.sh
 RUN bash ./install_user.sh && rm install_user.sh
 
+# Install clang
+ARG CLANG_VERSION
+COPY ./common/install_clang.sh install_clang.sh
+RUN bash ./install_clang.sh && rm install_clang.sh
+
 # Install conda and other packages (e.g., numpy, pytest)
 ARG PYTHON_VERSION
 ARG PIP_CMAKE

.ci/docker/linter/Dockerfile

@@ -14,6 +14,11 @@ RUN bash ./install_base.sh && rm install_base.sh
 COPY ./common/install_user.sh install_user.sh
 RUN bash ./install_user.sh && rm install_user.sh
 
+# Install clang
+ARG CLANG_VERSION
+COPY ./common/install_clang.sh install_clang.sh
+RUN bash ./install_clang.sh && rm install_clang.sh
+
 # Install conda and other packages (e.g., numpy, pytest)
 ARG PYTHON_VERSION
 ENV PATH /var/lib/jenkins/ci_env/bin:$PATH

.ci/docker/manywheel/Dockerfile_2_28_aarch64

@@ -1,6 +1,6 @@
 FROM quay.io/pypa/manylinux_2_28_aarch64 as base
 
-ARG GCCTOOLSET_VERSION=13
+ARG DEVTOOLSET_VERSION=15
 
 # Language variables
 ENV LC_ALL=en_US.UTF-8
@@ -35,15 +35,15 @@ RUN yum install -y \
   yasm \
   zstd \
   sudo \
-  gcc-toolset-${GCCTOOLSET_VERSION}-gcc \
-  gcc-toolset-${GCCTOOLSET_VERSION}-gcc-c++ \
-  gcc-toolset-${GCCTOOLSET_VERSION}-gcc-gfortran \
-  gcc-toolset-${GCCTOOLSET_VERSION}-gdb
+  gcc-toolset-${DEVTOOLSET_VERSION}-gcc \
+  gcc-toolset-${DEVTOOLSET_VERSION}-gcc-c++ \
+  gcc-toolset-${DEVTOOLSET_VERSION}-gcc-gfortran \
+  gcc-toolset-${DEVTOOLSET_VERSION}-gdb
 RUN yum install -y --enablerepo=powertools ninja-build
 
 # Ensure the expected devtoolset is used
-ENV PATH=/opt/rh/gcc-toolset-${GCCTOOLSET_VERSION}/root/usr/bin:$PATH
-ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${GCCTOOLSET_VERSION}/root/usr/lib64:/opt/rh/gcc-toolset-${GCCTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH
+ENV PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH
+ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH
 
 # Build a newer version of libgomp than that supported in in Almalinux 8.
 COPY ./common/install_libgomp.sh install_libgomp.sh

.ci/docker/manywheel/Dockerfile_cuda_aarch64

@@ -1,6 +1,6 @@
 FROM quay.io/pypa/manylinux_2_28_aarch64 as base
 
-# Cuda ARM build needs gcc 11
+# Cuda ARM build needs gcc 13
 ARG DEVTOOLSET_VERSION=13
 
 # Language variables

.ci/docker/manywheel/build.sh

@@ -40,7 +40,7 @@ case ${image} in
     manylinux2_28_aarch64-builder:cpu-aarch64)
         TARGET=final
         GPU_IMAGE=arm64v8/almalinux:8
-        DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=13"
+        DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=15"
         MANY_LINUX_VERSION="2_28_aarch64"
         ;;
     manylinuxs390x-builder:cpu-s390x)

.ci/docker/requirements-ci.txt

@@ -403,10 +403,14 @@ tlparse==0.4.0
 filelock==3.20.3
 #Description: required for inductor testing
 
-cuda-bindings>=12.0,<13.0 ; platform_machine != "s390x" and platform_machine != "riscv64" and platform_system != "Darwin"
-#Description: required for testing CUDAGraph::raw_cuda_graph(). See https://nvidia.github.io/cuda-python/cuda-bindings/latest/support.html for how this version was chosen. Note "Any fix in the latest bindings would be backported to the prior major version" means that only the newest version of cuda-bindings will get fixes. Depending on the latest version of 12.x is okay because all 12.y versions will be supported via "CUDA minor version compatibility". Pytorch builds against 13.z versions of cuda toolkit work with 12.x versions of cuda-bindings as well because newer drivers work with old toolkits.
+cuda-bindings>=13.0,<14.0 ; platform_machine != "s390x" and platform_machine != "riscv64" and platform_system != "Darwin"
+#Description: required for testing CUDAGraph::raw_cuda_graph(). See https://nvidia.github.io/cuda-python/cuda-bindings/latest/support.html for how this version was chosen. PyTorch CI CUDA builds use CUDA 13, so depend on the matching cuda-bindings major while allowing minor updates within that major.
 #test that import: test_cuda.py
 
+cupti-python>13.0 ; platform_system == "Linux" and (platform_machine == "x86_64" or platform_machine == "aarch64") and python_version < "3.14"
+#Description: required for the experimental CUPTI monitor (torch.profiler._cupti_monitor). >13.0 to match CUDA 13.x toolkits. cupti-python only ships Linux x86_64/aarch64 CPython wheels (no 3.14/free-threaded yet), so guard to those.
+#test that import: test_profiler.py
+
 setuptools-git-versioning==2.1.0
 scikit-build==0.18.1
 pyre-extensions==0.0.32
@@ -417,5 +421,15 @@ tqdm>=4.66.0
 #Description: progress bar library required for dynamo benchmarks
 #test that import: benchmarks/dynamo/*
 
-aiohttp==3.13.4
+aiohttp==3.14.1
 #Description: required for torch.distributed.debug
+
+spin==0.17
+#Description: required for testing spin commands
+#Pinned versions: 0.17
+#test that import: spincli/test_spin.py
+
+uv
+#Description: required for running spin commands via uvx
+#Pinned version:
+#test that import:

.ci/docker/triton_version.txt

@@ -1 +1 @@
-3.7.0
+3.7.1

.ci/docker/triton_xpu_version.txt

@@ -1 +1 @@
-3.7.1
+3.7.2