zxd1997066 · Aug 31, 2026 · Sep 6, 2026 · Oct 12, 2026 · Oct 15, 2026
diff --git a/.github/actions/linux-uttest/action.yml b/.github/actions/linux-uttest/action.yml
@@ -168,19 +168,53 @@ runs:
 
     - name: xpu_distributed
       shell: timeout 36000 bash -xeu -o pipefail {0}
-      if: ${{ inputs.ut_name == 'xpu_distributed' }}
+      if: ${{ inputs.ut_name == 'xpu_distributed' || inputs.ut_name == 'xpu_distributed_weekly' }}
       run: |
+        test_scope="${{ inputs.ut_name }}"
         xpu-smi topology -m
-        mkdir -p ut_log/xpu_distributed
-        XCCL_ENABLE=$(python -c "import torch;print(torch.distributed.is_xccl_available())")
+        mkdir -p "ut_log/${test_scope}"
+        if [[ "${test_scope}" == "xpu_distributed_weekly" ]]; then
+          git -C pytorch/third_party/torch-xpu-ops fetch --depth=1 \
+            https://github.com/intel/torch-xpu-ops.git daisyden/distributed_2.16
+          git -C pytorch/third_party/torch-xpu-ops checkout \
+            -B daisyden/distributed_2.16 FETCH_HEAD
+        fi
+
+        XCCL_ENABLE=$(python -c "import torch; print(torch.distributed.is_xccl_available())")
         if [[ "${XCCL_ENABLE,,}" == 'false' ]] || [[ "${XCCL_ENABLE}" == '0' ]]; then
           echo -e "[ERROR] XCCL is not enabled"
           exit 1
         fi
         cd pytorch/third_party/torch-xpu-ops/test/xpu
-        python run_distributed.py \
-          2> ${{ github.workspace }}/ut_log/xpu_distributed/xpu_distributed_test_error.log | \
-          tee ${{ github.workspace }}/ut_log/xpu_distributed/xpu_distributed_test.log
+        if [[ "${test_scope}" == "xpu_distributed_weekly" ]]; then
+          export TEMP_DIR=/tmp
+          export BACKEND=xccl
+          export WORLD_SIZE=4
+          export USE_CCL_V2=1
+          uv pip install transformers pulp
+          export TORCHINDUCTOR_CACHE_DIR="${RUNNER_TEMP}/torchinductor-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          torch_distributed_test=$(python -c "import inspect, torch.testing._internal.distributed.distributed_test as m; print(inspect.getsourcefile(m) or m.__file__)")
+          torch_distributed_test_backup="${RUNNER_TEMP}/distributed_test-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.py"
+          cp "${torch_distributed_test}" "${torch_distributed_test_backup}"
+          restore_torch_distributed_test() {
+            local test_status=$?
+            trap - EXIT
+            if ! cp "${torch_distributed_test_backup}" "${torch_distributed_test}"; then
+              echo "::error::Failed to restore ${torch_distributed_test}"
+              exit 1
+            fi
+            exit "${test_status}"
+          }
+          trap restore_torch_distributed_test EXIT
+          cp distributed/distributed_test.py "${torch_distributed_test}"
+          python run_distributed_local.py \
+            2> ${{ github.workspace }}/ut_log/${test_scope}/${test_scope}_test_error.log | \
+            tee ${{ github.workspace }}/ut_log/${test_scope}/${test_scope}_test.log
+        else
+          python run_distributed.py \
+            2> ${{ github.workspace }}/ut_log/${test_scope}/${test_scope}_test_error.log | \
+            tee ${{ github.workspace }}/ut_log/${test_scope}/${test_scope}_test.log
+        fi
         find ../ -type f -name "*.xml" -exec cp {} ${{ github.workspace }}/ut_log/ \;
 
     - name: xpu_p0

diff --git a/.github/scripts/ut_result_check.sh b/.github/scripts/ut_result_check.sh
@@ -2,7 +2,7 @@
 # Test Suite Runner for Intel Torch-XPU-Ops
 # Usage: TEST_PLATFORM=linux ./script.sh <test_suite>
 
-# Available suites: op_regression, op_extended, op_ut, test_xpu, upstream_default, upstream_inductor, xpu_distributed, upstream_distributed, skipped_ut
+# Available suites: op_regression, op_extended, op_ut, test_xpu, upstream_default, upstream_inductor, xpu_distributed, xpu_distributed_weekly, upstream_distributed, skipped_ut
 readonly ut_suite="${1:-op_regression}"  # Default to op_regression if no suite specified
 readonly inputs_pytorch="${2:-nightly_wheel}"
 readonly REPO="intel/torch-xpu-ops"
@@ -486,7 +486,7 @@ case "$ut_suite" in
     op_regression|op_regression_dev1|op_extended|op_ut|test_xpu|upstream_default|upstream_inductor|upstream_distributed)
         run_main_tests "$ut_suite"
         ;;
-    xpu_distributed)
+    xpu_distributed|xpu_distributed_weekly)
         run_distributed_tests "$ut_suite"
         ;;
     skipped_ut)
@@ -498,6 +498,6 @@ case "$ut_suite" in
     *)
         echo "❌ Unknown test suite: ${ut_suite}" >&2
         printf "💡 Available: op_regression, op_regression_dev1, op_extended, op_ut, test_xpu, " >&2
-        printf "upstream_default, upstream_inductor, xpu_distributed, upstream_distributed, skipped_ut, xpu_profiling\n" >&2
+        printf "xpu_distributed, xpu_distributed_weekly, upstream_default, upstream_inductor, upstream_distributed, skipped_ut, xpu_profiling\n" >&2
         ;;
 esac
diff --git a/.github/workflows/_linux_ut.yml b/.github/workflows/_linux_ut.yml
@@ -22,7 +22,7 @@ on:
       ut:
         required: true
         type: string
-        description: UT scope. one of `op_regression,op_extended,op_ut,skipped_ut,torch_xpu,op_regression_dev1,xpu_distributed,upstream_distributed,upstream_inductor,upstream_default`
+        description: UT scope. one of `op_regression,op_extended,op_ut,skipped_ut,torch_xpu,op_regression_dev1,xpu_distributed,xpu_distributed_weekly,upstream_distributed,upstream_inductor,upstream_default`
       category:
         type: string
         default: ''
@@ -78,14 +78,14 @@ jobs:
         if: ${{ ! cancelled() }}
         uses: actions/upload-artifact@v7
         with:
-          name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ inputs.ut }}-${{ github.run_id }}-${{ github.run_attempt }}${{ inputs.category != '' && format('-{0}', inputs.category) || '' }}
+          name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ inputs.ut }}-${{ github.run_id }}-${{ github.run_attempt }}${{ inputs.category != '' && format('-{0}', inputs.category) || '' }}${{ inputs.ut == 'xpu_distributed' && format('-{0}', inputs.runner) || '' }}
           path: ${{ github.workspace }}/ut_log
           if-no-files-found: ignore
       - name: Upload XPU UT Failure list
         if: ${{ ! cancelled() }}
         uses: actions/upload-artifact@v7
         with:
-          name: XPU-UT-Failure-List-${{ github.event.pull_request.number || github.sha }}-${{ inputs.ut }}-${{ github.run_id }}-${{ github.run_attempt }}${{ inputs.category != '' && format('-{0}', inputs.category) || '' }}
+          name: XPU-UT-Failure-List-${{ github.event.pull_request.number || github.sha }}-${{ inputs.ut }}-${{ github.run_id }}-${{ github.run_attempt }}${{ inputs.category != '' && format('-{0}', inputs.category) || '' }}${{ inputs.ut == 'xpu_distributed' && format('-{0}', inputs.runner) || '' }}
           path: ${{ github.workspace }}/ut_log/ut_failure_list.csv
           if-no-files-found: ignore
       - name: Cleanup workspace
@@ -120,14 +120,14 @@ jobs:
         if: ${{ ! cancelled() }}
         uses: actions/upload-artifact@v7
         with:
-          name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ env.UT_LABEL }}-${{ github.run_id }}-${{ github.run_attempt }}${{ inputs.category != '' && format('-{0}', inputs.category) || '' }}
+          name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ env.UT_LABEL }}-${{ github.run_id }}-${{ github.run_attempt }}${{ inputs.category != '' && format('-{0}', inputs.category) || '' }}${{ inputs.ut == 'xpu_distributed' && format('-{0}', inputs.runner) || '' }}
           path: ${{ github.workspace }}/ut_log
           if-no-files-found: ignore
       - name: Upload XPU UT Failure list
         if: ${{ ! cancelled() }}
         uses: actions/upload-artifact@v7
         with:
-          name: XPU-UT-Failure-List-${{ github.event.pull_request.number || github.sha }}-${{ env.UT_LABEL }}-${{ github.run_id }}-${{ github.run_attempt }}${{ inputs.category != '' && format('-{0}', inputs.category) || '' }}
+          name: XPU-UT-Failure-List-${{ github.event.pull_request.number || github.sha }}-${{ env.UT_LABEL }}-${{ github.run_id }}-${{ github.run_attempt }}${{ inputs.category != '' && format('-{0}', inputs.category) || '' }}${{ inputs.ut == 'xpu_distributed' && format('-{0}', inputs.runner) || '' }}
           path: ${{ github.workspace }}/ut_log/ut_failure_list.csv
           if-no-files-found: ignore
 
@@ -146,7 +146,7 @@ jobs:
       - name: Download XPU UT Logs
         uses: actions/download-artifact@v8
         with:
-          pattern: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ inputs.ut }}-*${{ inputs.category != '' && format('-{0}', inputs.category) || '' }}
+          pattern: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ inputs.ut }}-*${{ inputs.category != '' && format('-{0}', inputs.category) || '' }}${{ inputs.ut == 'xpu_distributed' && format('-{0}', inputs.runner) || '' }}
           path: ${{ github.workspace }}/ut_log
       - name: Download known-failure issues snapshot
         continue-on-error: true
@@ -161,7 +161,7 @@ jobs:
           ls -al ${{ github.workspace }}/ut_log
           cd ${{ github.workspace }}/ut_log/
           # download-artifact v8 drops the per-artifact dir when only one artifact matches; fall back to ut_log root
-          latest_dir=$(find . -type d -name "Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ inputs.ut }}-*" |sort -V |tail -n 1)
+          latest_dir=$(find . -type d -name "Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ inputs.ut }}-*${{ inputs.category != '' && format('-{0}', inputs.category) || '' }}${{ inputs.ut == 'xpu_distributed' && format('-{0}', inputs.runner) || '' }}" |sort -V |tail -n 1)
           latest_dir="${latest_dir:-.}"
           cd "${latest_dir}/${{ inputs.ut }}"
           find "${{ github.workspace }}/ut_log" -type f \
@@ -203,7 +203,8 @@ jobs:
           for ut_name in ${ut_list}
           do
             cp Known_issue.log.tmp Known_issue.log
-            awk -v r="${ut_name}" 'BEGIN{ print_row = 0 }{
+            ut_filter_name="${ut_name/xpu_distributed_weekly/xpu_distributed}"
+            awk -v r="${ut_filter_name}" 'BEGIN{ print_row = 0 }{
               if ( ! ( $0 ~ /[a-zA-Z0-9]/ ) ) { print_row = 0 };
               if ( print_row == 1 && $1 ~ r ) { print $0 };
               if ( $0 ~ /Cases:/ ) { print_row = 1 };
@@ -218,7 +219,7 @@ jobs:
         run: |
           cd ${{ github.workspace }}/ut_log/
           # download-artifact v8 drops the per-artifact dir when only one artifact matches; fall back to ut_log root
-          latest_dir=$(find . -type d -name "Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ inputs.ut }}-*" |sort -V |tail -n 1)
+          latest_dir=$(find . -type d -name "Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ inputs.ut }}-*${{ inputs.category != '' && format('-{0}', inputs.category) || '' }}${{ inputs.ut == 'xpu_distributed' && format('-{0}', inputs.runner) || '' }}" |sort -V |tail -n 1)
           latest_dir="${latest_dir:-.}"
           ut_dir="${latest_dir}/${{ inputs.ut }}"
           if [ "${{ inputs.ut }}" == "basic" ]; then
@@ -264,13 +265,13 @@ jobs:
         id: upload-new-failures
         uses: actions/upload-artifact@v7
         with:
-          name: New-UT-Failures-${{ github.event.pull_request.number || github.sha }}-${{ inputs.ut }}-${{ github.run_id }}-${{ github.run_attempt }}
+          name: New-UT-Failures-${{ github.event.pull_request.number || github.sha }}-${{ inputs.ut }}-${{ github.run_id }}-${{ github.run_attempt }}${{ inputs.category != '' && format('-{0}', inputs.category) || '' }}${{ inputs.ut == 'xpu_distributed' && format('-{0}', inputs.runner) || '' }}
           path: ${{ github.workspace }}/ut_log/new_ut_failure_list.csv
       - name: Upload Inductor XPU UT Log
         if: ${{ ! cancelled() }}
         uses: actions/upload-artifact@v7
         with:
-          name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ inputs.ut }}-${{ github.run_id }}-${{ github.run_attempt }}${{ inputs.category != '' && format('-{0}', inputs.category) || '' }}
+          name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ inputs.ut }}-${{ github.run_id }}-${{ github.run_attempt }}${{ inputs.category != '' && format('-{0}', inputs.category) || '' }}${{ inputs.ut == 'xpu_distributed' && format('-{0}', inputs.runner) || '' }}
           path: ${{ github.workspace }}/ut_log
           overwrite: true
       - name: Comment on PR for new failures

diff --git a/.github/workflows/nightly_ondemand.yml b/.github/workflows/nightly_ondemand.yml
@@ -29,7 +29,7 @@ on:
       ut:
         type: string
         default: ''
-        description: UT scope. 'op_regression,op_extended,op_ut,skipped_ut,xpu_profiling,torch_xpu,op_regression_dev1,xpu_distributed,upstream_distributed,upstream_inductor,upstream_default,microbench,windows'. Delimiter is comma
+        description: UT scope. 'op_regression,op_extended,op_ut,skipped_ut,xpu_profiling,torch_xpu,op_regression_dev1,xpu_distributed,xpu_distributed_weekly,upstream_distributed,upstream_inductor,upstream_default,microbench,windows'. Delimiter is comma
       suite:
         type: string
         default: ''
@@ -100,12 +100,15 @@ jobs:
               echo "No such scheduler: ${{ github.event.schedule }}"
               exit 1
             fi
-            ut='["basic","op_ut","skipped_ut","xpu_profiling","xpu_distributed"]'
             suite='["huggingface","timm_models","torchbench"]'
+            ut='["basic","op_ut","skipped_ut","xpu_profiling","xpu_distributed"]'
             # Weekly also runs the upstream UT scopes
             if [[ "${test_type}" == *"-weekly" ]];then
               ut='["basic","op_ut","skipped_ut","xpu_profiling","xpu_distributed","upstream_distributed","upstream_inductor","upstream_default"]'
             fi
+            if [[ "${test_type}" == "wheel-weekly" ]];then
+              ut="${ut/xpu_distributed/xpu_distributed_weekly}"
+            fi
             scenario='["accuracy","performance"]'
             triton=''
             python=''

diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml
@@ -285,9 +285,10 @@ jobs:
       fail-fast: false
       matrix:
         ut_name: [xpu_distributed]
+        runner: [distributed, bmg-test]
     uses: ./.github/workflows/_linux_ut.yml
     with:
-      runner: distributed
+      runner: ${{ matrix.runner }}
       pytorch: ${{ contains(github.event.pull_request.labels.*.name, 'disable_build') && 'nightly_wheel' || github.base_ref }}
       torch_xpu_ops: ${{ contains(github.event.pull_request.labels.*.name, 'disable_build') && 'pinned' || 'main' }}
       ut: ${{ matrix.ut_name }}

diff --git a/test/xpu/run_distributed.py b/test/xpu/run_distributed.py
@@ -11,6 +11,7 @@
 import sys
 
 from skip_list_dist import skip_dict
+from torch.testing._internal.common_xpu import get_xpu_codename, XPUCodename
 from xpu_test_utils import launch_test
 
 res = 0
@@ -47,6 +48,24 @@
         if len(value) > len(max_affinity):
             max_affinity = value
 
+    # BMG has no XL-linked GPU groups (all-SYS topology), so use the first
+    # WORLD_SIZE GPUs only when PyTorch identifies the device as BMG.
+    if "," not in max_affinity:
+        if get_xpu_codename() != XPUCodename.BMG:
+            print(
+                "No multi-GPU XELINK group found; refusing to select GPUs by fallback."
+            )
+            sys.exit(255)
+        if not gpu_dict:
+            print("No GPUs found in xpu-smi topology output.")
+            sys.exit(255)
+
+        world_size = int(os.environ.get("WORLD_SIZE", "4"))
+        max_affinity = ",".join(str(i) for i in range(min(world_size, len(gpu_dict))))
+        print(
+            f"No multi-GPU XELINK group found on BMG; falling back to {max_affinity}."
+        )
+
     os.environ["ZE_AFFINITY_MASK"] = str(max_affinity)
     print(str("ZE_AFFINITY_MASK=" + os.environ.get("ZE_AFFINITY_MASK")))