Merge branch 'main' into revert-7218-xfail_vit

Author: weiwangmeta

.circleci/config.yml

@@ -61,12 +61,8 @@ def build_workflows(prefix="", filter_branch=None, upload=False, indentation=6,
                             fb = "/.*/"
 
                         # Disable all Linux Wheels Workflows from CircleCI
-                        # since those will now be done through Nova. We'll keep
-                        # around the py3.8 CPU Linux Wheels build since the docs
-                        # job depends on it.
                         if os_type == "linux" and btype == "wheel":
-                            if not (python_version == "3.8" and cu_version == "cpu"):
-                                continue
+                            continue
 
                         # Disable all Macos Wheels Workflows from CircleCI.
                         if os_type == "macos" and btype == "wheel":
@@ -76,21 +72,10 @@ def build_workflows(prefix="", filter_branch=None, upload=False, indentation=6,
                         if os_type != "win" and btype == "conda":
                             continue
 
-                        # Not supporting Python 3.11 conda packages at the
-                        # moment since the necessary dependencies are not
-                        # available. Windows 3.11 Wheels will be built from
-                        # CircleCI here, however.
-                        if python_version == "3.11" and btype == "conda":
-                            continue
-
                         w += workflow_pair(
                             btype, os_type, python_version, cu_version, unicode, prefix, upload, filter_branch=fb
                         )
 
-    if not filter_branch:
-        # Build on every pull request, but upload only on nightly and tags
-        w += build_doc_job("/.*/")
-        w += upload_doc_job("nightly")
     return indent(indentation, w)
 
 
@@ -121,35 +106,6 @@ def workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix=""
     return w
 
 
-def build_doc_job(filter_branch):
-    job = {
-        "name": "build_docs",
-        "python_version": "3.8",
-        "requires": [
-            "binary_linux_wheel_py3.8_cpu",
-        ],
-    }
-
-    if filter_branch:
-        job["filters"] = gen_filter_branch_tree(filter_branch, tags_list=RC_PATTERN)
-    return [{"build_docs": job}]
-
-
-def upload_doc_job(filter_branch):
-    job = {
-        "name": "upload_docs",
-        "context": "org-member",
-        "python_version": "3.8",
-        "requires": [
-            "build_docs",
-        ],
-    }
-
-    if filter_branch:
-        job["filters"] = gen_filter_branch_tree(filter_branch, tags_list=RC_PATTERN)
-    return [{"upload_docs": job}]
-
-
 manylinux_images = {
     "cu117": "pytorch/manylinux-cuda117",
     "cu118": "pytorch/manylinux-cuda118",
@@ -262,13 +218,17 @@ def indent(indentation, data_list):
 
 def unittest_workflows(indentation=6):
     jobs = []
-    for os_type in ["linux", "windows", "macos"]:
+    for os_type in ["windows"]:
         for device_type in ["cpu", "gpu"]:
             if os_type == "macos" and device_type == "gpu":
                 continue
-            if os_type == "linux" and device_type == "cpu":
-                continue
+
             for i, python_version in enumerate(PYTHON_VERSIONS):
+
+                # Turn off unit tests for 3.11, unit test are not setup properly
+                if python_version == "3.11":
+                    continue
+
                 job = {
                     "name": f"unittest_{os_type}_{device_type}_py{python_version}",
                     "python_version": python_version,
@@ -297,7 +257,7 @@ def cmake_workflows(indentation=6):
 
             job["cu_version"] = "cu117" if device == "gpu" else "cpu"
             if device == "gpu" and os_type == "linux":
-                job["wheel_docker_image"] = "pytorch/manylinux-cuda116"
+                job["wheel_docker_image"] = "pytorch/manylinux-cuda117"
             jobs.append({f"cmake_{os_type}_{device}": job})
     return indent(indentation, jobs)
 

.circleci/config.yml.in

@@ -5,7 +5,7 @@ unset PYTORCH_VERSION
 # so no need to set PYTORCH_VERSION.
 # In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config.
 
-set -e
+set -ex
 
 eval "$(./conda/bin/conda shell.bash hook)"
 conda activate ./env
@@ -22,6 +22,10 @@ else
     echo "Using CUDA $CUDA_VERSION as determined by CU_VERSION: ${CU_VERSION} "
     version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")"
     cudatoolkit="pytorch-cuda=${version}"
+
+     # make sure local cuda is set to required cuda version and not CUDA version by default
+    rm -f /usr/local/cuda
+    ln -s /usr/local/cuda-${version} /usr/local/cuda
 fi
 
 case "$(uname -s)" in
@@ -36,5 +40,6 @@ else
     conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c nvidia "pytorch-${UPLOAD_CHANNEL}"::pytorch[build="*${version}*"] "${cudatoolkit}"
 fi
 
+
 printf "* Installing torchvision\n"
 python setup.py develop

.circleci/regenerate.py

@@ -6,4 +6,17 @@ eval "$(./conda/bin/conda shell.bash hook)"
 conda activate ./env
 
 python -m torch.utils.collect_env
-pytest --junitxml=test-results/junit.xml -v --durations 20
+
+case "$(uname -s)" in
+  Darwin*)
+    # The largest macOS runner is not able to handle the regular test suite plus the transforms v2 tests at the same
+    # time due to insufficient resources. Thus, we ignore the transforms v2 tests at first and run them in a separate
+    # step afterwards.
+    GLOB='test/test_transforms_v2*'
+    pytest --junitxml=test-results/junit.xml -v --durations 20 --ignore-glob="${GLOB}"
+    eval "pytest --junitxml=test-results/junit-transforms-v2.xml -v --durations 20 ${GLOB}"
+    ;;
+  *)
+    pytest --junitxml=test-results/junit.xml -v --durations 20
+    ;;
+esac

.circleci/unittest/linux/scripts/install.sh

@@ -5,7 +5,7 @@
 #
 # Do not install PyTorch and torchvision here, otherwise they also get cached.
 
-set -e
+set -ex
 
 this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 # Avoid error: "fatal: unsafe repository"
@@ -39,7 +39,7 @@ conda activate "${env_dir}"
 # 3. Install Conda dependencies
 printf "* Installing dependencies (except PyTorch)\n"
 FFMPEG_PIN="=4.2"
-if [[ "${PYTHON_VERSION}" = "3.9" ]]; then
+if [[ "${PYTHON_VERSION}" == "3.9" ]]; then
     FFMPEG_PIN=">=4.2"
 fi
 

.circleci/unittest/linux/scripts/run_test.sh

@@ -5,7 +5,7 @@
 #
 # Do not install PyTorch and torchvision here, otherwise they also get cached.
 
-set -e
+set -ex
 
 this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 root_dir="$(git rev-parse --show-toplevel)"

.circleci/unittest/linux/scripts/setup_env.sh

@@ -1 +1,10 @@
 tracking_issue: 2447
+
+# List of workflows that will be re-run in case of failures
+# https://github.com/pytorch/test-infra/blob/main/torchci/lib/bot/retryBot.ts
+retryable_workflows:
+- Build Linux
+- Build Macos
+- Build M1
+- Tests on Linux
+- Tests on macOS

.circleci/unittest/windows/scripts/setup_env.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Prepare conda
+CONDA_PATH=$(which conda)
+eval "$(${CONDA_PATH} shell.bash hook)"
+
+# Setup the OS_TYPE environment variable that should be used for conditions involving the OS below.
+case $(uname) in
+  Linux)
+    OS_TYPE=linux
+    ;;
+  Darwin)
+    OS_TYPE=macos
+    ;;
+  MSYS*)
+    OS_TYPE=windows
+    ;;
+  *)
+    echo "Unknown OS type:" $(uname)
+    exit 1
+    ;;
+esac
+
+if [[ "${OS_TYPE}" == "macos" && $(uname -m) == x86_64 ]]; then
+  echo '::group::Uninstall system JPEG libraries on macOS'
+  # The x86 macOS runners, e.g. the GitHub Actions native "macos-12" runner, has some JPEG libraries installed by
+  # default that interfere with our build. We uninstall them here and use the one from conda below.
+  JPEG_LIBS=$(brew list | grep jpeg)
+  echo $JPEG_LIBS
+  for lib in $JPEG_LIBS; do
+    brew uninstall --ignore-dependencies --force $lib || true
+  done
+  echo '::endgroup::'
+fi
+
+echo '::group::Create build environment'
+# See https://github.com/pytorch/vision/issues/7296 for ffmpeg
+conda create \
+  --name ci \
+  --quiet --yes \
+  python="${PYTHON_VERSION}" pip \
+  ninja \
+  libpng jpeg \
+  'ffmpeg<4.3'
+conda activate ci
+pip install --progress-bar=off --upgrade setuptools
+
+# See https://github.com/pytorch/vision/issues/6790
+if [[ "${PYTHON_VERSION}" != "3.11" ]]; then
+  pip install --progress-bar=off av!=10.0.0
+fi
+
+echo '::endgroup::'
+
+echo '::group::Install PyTorch'
+# TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`.
+if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
+  CHANNEL=test
+else
+  CHANNEL=nightly
+fi
+
+pip install --progress-bar=off light-the-torch
+ltt install --progress-bar=off \
+  --pytorch-computation-backend="${GPU_ARCH_TYPE}${GPU_ARCH_VERSION}" \
+  --pytorch-channel="${CHANNEL}" \
+  torch
+
+if [[ $GPU_ARCH_TYPE == 'cuda' ]]; then
+  python -c "import torch; exit(not torch.cuda.is_available())"
+fi
+echo '::endgroup::'
+
+if [[ "${OS_TYPE}" == "windows" ]]; then
+  echo '::group::Install third party dependencies prior to TorchVision install on Windows'
+  # `easy_install`, i.e. `python setup.py` has problems downloading the dependencies due to SSL.
+  # Thus, we install them upfront with `pip` to avoid that.
+  # Instead of fixing the SSL error, we can probably maintain this special case until we switch away from the deprecated
+  # `easy_install` anyway.
+  python setup.py egg_info
+  # The requires.txt cannot be used with `pip install -r` directly. The requirements are listed at the top and the
+  # optional dependencies come in non-standard syntax after a blank line. Thus, we just extract the header.
+  sed -e '/^$/,$d' *.egg-info/requires.txt > requirements.txt
+  pip install --progress-bar=off -r requirements.txt
+  echo '::endgroup::'
+fi
+
+echo '::group::Install TorchVision'
+python setup.py develop
+echo '::endgroup::'
+
+echo '::group::Collect PyTorch environment information'
+python -m torch.utils.collect_env
+echo '::endgroup::'

.github/pytorch-probot.yml

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+./.github/scripts/setup-env.sh
+
+# Activate conda environment
+eval "$($(which conda) shell.bash hook)" && conda deactivate && conda activate ci
+
+echo '::group::Install testing utilities'
+pip install --progress-bar=off pytest pytest-mock pytest-cov
+echo '::endgroup::'
+
+pytest --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v --durations=25