From 502d5da9a5a000d2a7f37f07d8b8d60481f8d2fe Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 14 Mar 2023 16:36:45 +0100 Subject: [PATCH 01/59] migrate cmake workflows from CircleCI to GHA --- .github/scripts/setup-env.sh | 2 +- .github/workflows/build-cmake-linux.yml | 42 +++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build-cmake-linux.yml diff --git a/.github/scripts/setup-env.sh b/.github/scripts/setup-env.sh index 3a9217e9ef8..9955d6ed7b6 100755 --- a/.github/scripts/setup-env.sh +++ b/.github/scripts/setup-env.sh @@ -38,7 +38,7 @@ conda create \ --name ci \ --quiet --yes \ python="${PYTHON_VERSION}" pip \ - ninja \ + ninja cmake \ libpng jpeg \ 'ffmpeg<4.3' conda activate ci diff --git a/.github/workflows/build-cmake-linux.yml b/.github/workflows/build-cmake-linux.yml new file mode 100644 index 00000000000..0eb20f5fdac --- /dev/null +++ b/.github/workflows/build-cmake-linux.yml @@ -0,0 +1,42 @@ +name: Cmake on Linux + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + unittests: + strategy: + matrix: + include: + - runner: linux.12xlarge + gpu-arch-type: cpu + - runner: linux.g5.4xlarge.nvidia.gpu + gpu-arch-type: cuda + gpu-arch-version: "11.7" + fail-fast: false + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + gpu-arch-type: ${{ matrix.gpu-arch-type }} + gpu-arch-version: ${{ matrix.gpu-arch-version }} + script: | + set -euo pipefail + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + ./.github/scripts/setup-env.sh + + # Prepare conda + CONDA_PATH=$(which conda) + eval "$(${CONDA_PATH} shell.bash hook)" + conda activate ci + + packaging/build_cmake.sh From 29373c0e66a92bebd149573b5e9a18fda3650898 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 9 May 2023 11:05:38 +0200 Subject: [PATCH 02/59] update --- .github/workflows/build-cmake-linux.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-cmake-linux.yml b/.github/workflows/build-cmake-linux.yml index 0eb20f5fdac..d8372c550a4 100644 --- a/.github/workflows/build-cmake-linux.yml +++ b/.github/workflows/build-cmake-linux.yml @@ -33,10 +33,8 @@ jobs: export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} ./.github/scripts/setup-env.sh - - # Prepare conda - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - conda activate ci - + + # Activate conda environment + eval "$($(which conda) shell.bash hook)" && conda deactivate && conda activate ci + packaging/build_cmake.sh From f9c371c65e9c66859de807bba5d8628e60e1c7c4 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 9 May 2023 11:06:03 +0200 Subject: [PATCH 03/59] [REVERTME] disable unrelated CI --- .github/workflows/build-conda-linux.yml | 51 --------- .github/workflows/build-conda-m1.yml | 52 --------- .github/workflows/build-conda-macos.yml | 52 --------- .github/workflows/build-conda-windows.yml | 53 --------- .github/workflows/build-wheels-linux.yml | 50 --------- .github/workflows/build-wheels-m1.yml | 51 --------- .github/workflows/build-wheels-macos.yml | 51 --------- .github/workflows/build-wheels-windows.yml | 54 --------- .github/workflows/docs.yml | 68 ------------ .github/workflows/lint.yml | 98 ---------------- .github/workflows/pr-labels.yml | 35 ------ .../workflows/prototype-tests-linux-gpu.yml | 54 --------- .github/workflows/test-linux.yml | 105 ------------------ .github/workflows/test-macos.yml | 40 ------- .github/workflows/test-windows.yml | 43 ------- .github/workflows/tests-schedule.yml | 57 ---------- .github/workflows/update-viablestrict.yml | 23 ---- 17 files changed, 937 deletions(-) delete mode 100644 .github/workflows/build-conda-linux.yml delete mode 100644 .github/workflows/build-conda-m1.yml delete mode 100644 .github/workflows/build-conda-macos.yml delete mode 100644 .github/workflows/build-conda-windows.yml delete mode 100644 .github/workflows/build-wheels-linux.yml delete mode 100644 .github/workflows/build-wheels-m1.yml delete mode 100644 .github/workflows/build-wheels-macos.yml delete mode 100644 .github/workflows/build-wheels-windows.yml delete mode 100644 .github/workflows/docs.yml delete mode 100644 .github/workflows/lint.yml delete mode 100644 .github/workflows/pr-labels.yml delete mode 100644 .github/workflows/prototype-tests-linux-gpu.yml delete mode 100644 .github/workflows/test-linux.yml delete mode 100644 .github/workflows/test-macos.yml delete mode 100644 .github/workflows/test-windows.yml delete mode 100644 .github/workflows/tests-schedule.yml delete mode 100644 .github/workflows/update-viablestrict.yml diff --git a/.github/workflows/build-conda-linux.yml b/.github/workflows/build-conda-linux.yml deleted file mode 100644 index e44bb877ed6..00000000000 --- a/.github/workflows/build-conda-linux.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build Linux Conda - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: "" - post-script: "" - conda-package-directory: packaging/torchvision - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_linux.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-m1.yml b/.github/workflows/build-conda-m1.yml deleted file mode 100644 index 7006b6714aa..00000000000 --- a/.github/workflows/build-conda-m1.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Build M1 Conda - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: macos-arm64 - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: "" - post-script: "" - conda-package-directory: packaging/torchvision - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - runner-type: macos-m1-12 - trigger-event: ${{ github.event_name }} - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-macos.yml b/.github/workflows/build-conda-macos.yml deleted file mode 100644 index 39d00534b3a..00000000000 --- a/.github/workflows/build-conda-macos.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Build Macos Conda - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: macos - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: "" - post-script: "" - conda-package-directory: packaging/torchvision - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - runner-type: macos-12 - trigger-event: ${{ github.event_name }} - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-windows.yml b/.github/workflows/build-conda-windows.yml deleted file mode 100644 index 98c68377189..00000000000 --- a/.github/workflows/build-conda-windows.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Build Windows Conda - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: conda - os: windows - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - env-script: packaging/windows/internal/vc_env_helper.bat - post-script: "" - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_conda_windows.yml@main - with: - conda-package-directory: ${{ matrix.conda-package-directory }} - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - # Using "development" as trigger event so these binaries are not uploaded - # to official channels yet - trigger-event: development - secrets: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-wheels-linux.yml b/.github/workflows/build-wheels-linux.yml deleted file mode 100644 index e997d648ec2..00000000000 --- a/.github/workflows/build-wheels-linux.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Build Linux Wheels - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-m1.yml b/.github/workflows/build-wheels-m1.yml deleted file mode 100644 index 66c7687acc9..00000000000 --- a/.github/workflows/build-wheels-m1.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build M1 Wheels - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: macos-arm64 - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - runner-type: macos-m1-12 - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-macos.yml b/.github/workflows/build-wheels-macos.yml deleted file mode 100644 index 6c5ebc0fc37..00000000000 --- a/.github/workflows/build-wheels-macos.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build Macos Wheels - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: macos - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - runner-type: macos-12 - smoke-test-script: ${{ matrix.smoke-test-script }} - trigger-event: ${{ github.event_name }} - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-windows.yml b/.github/workflows/build-wheels-windows.yml deleted file mode 100644 index e238135979c..00000000000 --- a/.github/workflows/build-wheels-windows.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Build Windows Wheels - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: windows - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/vision - pre-script: packaging/pre_build_script.sh - env-script: packaging/windows/internal/vc_env_helper.bat - post-script: "python packaging/wheel/relocate.py" - smoke-test-script: test/smoke_test.py - package-name: torchvision - name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main - with: - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - env-script: ${{ matrix.env-script }} - post-script: ${{ matrix.post-script }} - package-name: ${{ matrix.package-name }} - smoke-test-script: ${{ matrix.smoke-test-script }} - # Using "development" as trigger event so these binaries are not uploaded - # to official channels yet - trigger-event: development - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 22ff4ab84b3..00000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Docs - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -jobs: - build: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - upload-artifact: docs - script: | - set -euo pipefail - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=cpu - export GPU_ARCH_VERSION='' - ./.github/scripts/setup-env.sh - - # Prepare conda - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - conda activate ci - # FIXME: not sure why we need this. `ldd torchvision/video_reader.so` shows that it - # already links against the one pulled from conda. However, at runtime it pulls from - # /lib64 - # Should we maybe always do this in `./.github/scripts/setup-env.sh` so that we don't - # have to pay attention in all other workflows? - export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}" - - cd docs - - echo '::group::Install doc requirements' - pip install --progress-bar=off -r requirements.txt - echo '::endgroup::' - - echo '::group::Build HTML docs' - # The runner does not have sufficient memory to run with as many processes as their are - # cores (`-j auto`). Thus, we limit to a single process (`-j 1`) here. - sed -i -e 's/-j auto/-j 1/' Makefile - make html - echo '::endgroup::' - - mv build/html "${RUNNER_ARTIFACT_DIR}" - - upload-preview: - if: github.event_name == 'pull_request' - needs: [build] - runs-on: [self-hosted, linux.2xlarge] - steps: - - uses: actions/download-artifact@v3 - with: - name: docs - - - name: Upload docs preview - uses: seemethere/upload-artifact-s3@v5 - with: - retention-days: 14 - s3-bucket: doc-previews - if-no-files-found: error - path: html - s3-prefix: pytorch/vision/${{ github.event.pull_request.number }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index ae5249a9219..00000000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,98 +0,0 @@ -name: Lint - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -jobs: - python-source-and-configs: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - script: | - set -euo pipefail - - echo '::group::Setup environment' - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - conda create --name ci --quiet --yes python=3.8 pip - conda activate ci - echo '::endgroup::' - - echo '::group::Install lint tools' - pip install --progress-bar=off pre-commit - echo '::endgroup::' - - echo '::group::Lint Python source and configs' - set +e - pre-commit run --all-files - - if [ $? -ne 0 ]; then - git --no-pager diff - exit 1 - fi - echo '::endgroup::' - - c-source: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - script: | - set -euo pipefail - - echo '::group::Setup environment' - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - # clang-format needs some shared libraries that conflict with the system ones. Thus, we install them from conda - # and prepend the libraries to linker path to prioritize them. `ncurses=5` is only available on the conda-forge - # channel. Since we are not building or testing here, this is fine. - conda create --name ci --quiet --yes -c conda-forge python=3.8 ncurses=5 libgcc - conda activate ci - export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}" - echo '::endgroup::' - - echo '::group::Install lint tools' - curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o ./clang-format - chmod +x ./clang-format - echo '::endgroup::' - - echo '::group::Lint C source' - set +e - ./.circleci/unittest/linux/scripts/run-clang-format.py -r torchvision/csrc --clang-format-executable ./clang-format - - if [ $? -ne 0 ]; then - git --no-pager diff - exit 1 - fi - echo '::endgroup::' - - - python-types: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - script: | - set -euo pipefail - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=cpu - export GPU_ARCH_VERSION='' - - ./.github/scripts/setup-env.sh - - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - conda activate ci - - echo '::group::Install lint tools' - pip install --progress-bar=off mypy - echo '::endgroup::' - - echo '::group::Lint Python types' - mypy --install-types --non-interactive --config-file mypy.ini - echo '::endgroup::' diff --git a/.github/workflows/pr-labels.yml b/.github/workflows/pr-labels.yml deleted file mode 100644 index 20c37e4fd88..00000000000 --- a/.github/workflows/pr-labels.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: pr-labels - -on: - push: - branches: - - main - -jobs: - is-properly-labeled: - runs-on: ubuntu-latest - - steps: - - name: Set up python - uses: actions/setup-python@v2 - - - name: Install requests - run: pip install requests - - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Process commit and find merger responsible for labeling - id: commit - run: echo "::set-output name=merger::$(python .github/process_commit.py ${{ github.sha }})" - - - name: Ping merger responsible for labeling if necessary - if: ${{ steps.commit.outputs.merger != '' }} - uses: mshick/add-pr-comment@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - message: | - Hey ${{ steps.commit.outputs.merger }}! - - You merged this PR, but no labels were added. The list of valid labels is available at https://github.com/pytorch/vision/blob/main/.github/process_commit.py diff --git a/.github/workflows/prototype-tests-linux-gpu.yml b/.github/workflows/prototype-tests-linux-gpu.yml deleted file mode 100644 index dee425054d5..00000000000 --- a/.github/workflows/prototype-tests-linux-gpu.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Prototype tests on Linux - -on: - pull_request: - -jobs: - unittests-prototype: - strategy: - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" - runner: ["linux.12xlarge"] - gpu-arch-type: ["cpu"] - include: - - python-version: "3.8" - runner: linux.g5.4xlarge.nvidia.gpu - gpu-arch-type: cuda - gpu-arch-version: "11.7" - fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - runner: ${{ matrix.runner }} - gpu-arch-type: ${{ matrix.gpu-arch-type }} - gpu-arch-version: ${{ matrix.gpu-arch-version }} - timeout: 120 - script: | - set -euo pipefail - - export PYTHON_VERSION=${{ matrix.python-version }} - export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} - export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} - ./.github/scripts/setup-env.sh - - # Prepare conda - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - conda activate ci - - echo '::group::Install testing utilities' - pip install --progress-bar=off pytest pytest-mock pytest-cov - echo '::endgroup::' - - # We don't want to run the prototype datasets tests. Since the positional glob into `pytest`, i.e. - # `test/test_prototype*.py` takes the highest priority, neither `--ignore` nor `--ignore-glob` can help us here. - rm test/test_prototype_datasets*.py - pytest \ - -v --durations=25 \ - --cov=torchvision/prototype --cov-report=term-missing \ - --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" \ - test/test_prototype_*.py diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml deleted file mode 100644 index b9b01b7a55e..00000000000 --- a/.github/workflows/test-linux.yml +++ /dev/null @@ -1,105 +0,0 @@ -name: Tests on Linux - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -jobs: - unittests: - strategy: - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" - runner: ["linux.12xlarge"] - gpu-arch-type: ["cpu"] - include: - - python-version: 3.8 - runner: linux.g5.4xlarge.nvidia.gpu - gpu-arch-type: cuda - gpu-arch-version: "11.7" - fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - runner: ${{ matrix.runner }} - gpu-arch-type: ${{ matrix.gpu-arch-type }} - gpu-arch-version: ${{ matrix.gpu-arch-version }} - timeout: 120 - script: | - set -euo pipefail - - export PYTHON_VERSION=${{ matrix.python-version }} - export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} - export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} - - ./.github/scripts/unittest.sh - - onnx: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - script: | - set -euo pipefail - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=cpu - export GPU_ARCH_VERSION='' - - ./.github/scripts/setup-env.sh - - # Prepare conda - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - conda activate ci - - echo '::group::Install ONNX' - pip install --progress-bar=off onnx onnxruntime - echo '::endgroup::' - - echo '::group::Install testing utilities' - pip install --progress-bar=off pytest - echo '::endgroup::' - - echo '::group::Run ONNX tests' - pytest --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v --durations=25 test/test_onnx.py - echo '::endgroup::' - - unittests-extended: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - script: | - set -euo pipefail - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=cpu - export GPU_ARCH_VERSION='' - - ./.github/scripts/setup-env.sh - - # Prepare conda - CONDA_PATH=$(which conda) - eval "$(${CONDA_PATH} shell.bash hook)" - conda activate ci - - echo '::group::Pre-download model weights' - pip install --progress-bar=off aiohttp aiofiles tqdm - python scripts/download_model_urls.py - echo '::endgroup::' - - echo '::group::Install testing utilities' - pip install --progress-bar=off pytest - echo '::endgroup::' - - echo '::group::Run extended unittests' - export PYTORCH_TEST_WITH_EXTENDED=1 - pytest --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v --durations=25 test/test_extended_*.py - echo '::endgroup::' diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml deleted file mode 100644 index 03e4b2db121..00000000000 --- a/.github/workflows/test-macos.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Tests on macOS - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -jobs: - unittests: - strategy: - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" - runner: ["macos-12"] - include: - - python-version: "3.8" - runner: macos-m1-12 - fail-fast: false - uses: pytorch/test-infra/.github/workflows/macos_job.yml@main - with: - repository: pytorch/vision - # We need an increased timeout here, since the macos-12 runner is the free one from GH - # and needs roughly 2 hours to just run the test suite - timeout: 240 - runner: ${{ matrix.runner }} - script: | - set -euo pipefail - - export PYTHON_VERSION=${{ matrix.python-version }} - export GPU_ARCH_TYPE=cpu - export GPU_ARCH_VERSION='' - - ./.github/scripts/unittest.sh diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml deleted file mode 100644 index b300a5d0e83..00000000000 --- a/.github/workflows/test-windows.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Tests on Windows - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -jobs: - unittests: - strategy: - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" - runner: ["windows.4xlarge"] - gpu-arch-type: ["cpu"] - include: - - python-version: "3.8" - runner: windows.g5.4xlarge.nvidia.gpu - gpu-arch-type: cuda - gpu-arch-version: "11.7" - fail-fast: false - uses: pytorch/test-infra/.github/workflows/windows_job.yml@main - with: - repository: pytorch/vision - runner: ${{ matrix.runner }} - gpu-arch-type: ${{ matrix.gpu-arch-type }} - gpu-arch-version: ${{ matrix.gpu-arch-version }} - timeout: 120 - script: | - set -euxo pipefail - - export PYTHON_VERSION=${{ matrix.python-version }} - export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} - export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} - - ./.github/scripts/unittest.sh diff --git a/.github/workflows/tests-schedule.yml b/.github/workflows/tests-schedule.yml deleted file mode 100644 index 5426fdc997a..00000000000 --- a/.github/workflows/tests-schedule.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: tests - -on: - pull_request: - paths: - - "test/test_datasets_download.py" - - ".github/failed_schedule_issue_template.md" - - ".github/workflows/tests-schedule.yml" - - schedule: - - cron: "0 9 * * *" - -jobs: - download: - runs-on: ubuntu-latest - - steps: - - name: Set up python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Upgrade system packages - run: python -m pip install --upgrade pip setuptools wheel - - - name: SSL - run: python -c 'import ssl; print(ssl.OPENSSL_VERSION)' - - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Install torch nightly build - run: pip install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - - - name: Install torchvision - run: pip install --no-build-isolation --editable . - - - name: Install all optional dataset requirements - run: pip install scipy pycocotools lmdb requests - - - name: Install tests requirements - run: pip install pytest - - - name: Run tests - run: pytest -ra -v test/test_datasets_download.py - - - uses: JasonEtco/create-an-issue@v2.4.0 - name: Create issue if download tests failed - if: failure() && github.event_name == 'schedule' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - WORKFLOW: ${{ github.workflow }} - JOB: ${{ github.job }} - ID: ${{ github.run_id }} - with: - filename: .github/failed_schedule_issue_template.md diff --git a/.github/workflows/update-viablestrict.yml b/.github/workflows/update-viablestrict.yml deleted file mode 100644 index 2d9c2265676..00000000000 --- a/.github/workflows/update-viablestrict.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Update viable/strict - -on: - pull_request: - paths: - - .github/workflows/update-viablestrict.yml - schedule: - - cron: 10,40 * * * * - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }} - cancel-in-progress: false - -jobs: - do_update_viablestrict: - uses: pytorch/test-infra/.github/workflows/update-viablestrict.yml@main - with: - repository: pytorch/vision - required_checks: "Build Linux,Build M1,Build Macos,Tests on Linux,Tests on macOS,Docs,Lint" - secrets: - ROCKSET_API_KEY: ${{ secrets.ROCKSET_API_KEY }} - GITHUB_DEPLOY_KEY : ${{ secrets.VISION_GITHUB_DEPLOY_KEY }} From e69c38cdbd12502383fa54a9e1bce20a5b34a1ff Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 9 May 2023 11:33:30 +0200 Subject: [PATCH 04/59] standalone cmake build --- .github/scripts/cmake.sh | 23 +++++++++++++++++++++++ .github/scripts/setup-env.sh | 4 ++-- .github/workflows/build-cmake-linux.yml | 6 +----- 3 files changed, 26 insertions(+), 7 deletions(-) create mode 100755 .github/scripts/cmake.sh diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh new file mode 100755 index 00000000000..767851bd626 --- /dev/null +++ b/.github/scripts/cmake.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +./.github/scripts/setup-env.sh + +# Activate conda environment +eval "$($(which conda) shell.bash hook)" && conda deactivate && conda activate ci + +Torch_DIR=$(python -c "import torch; print(torch.__path__[0])")/share/cmake/Torch +if [[ "${GPU_ARCH_TYPE}" == "cuda" ]]; then + WITH_CUDA=1 +else + WITH_CUDA=0 +fi + +mkdir cpp_build +cd cpp_build + +cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" + +make -j$PARALLELISM +make install diff --git a/.github/scripts/setup-env.sh b/.github/scripts/setup-env.sh index 2d895e35dcb..973e6ec4f42 100755 --- a/.github/scripts/setup-env.sh +++ b/.github/scripts/setup-env.sh @@ -3,8 +3,7 @@ set -euo pipefail # Prepare conda -CONDA_PATH=$(which conda) -eval "$(${CONDA_PATH} shell.bash hook)" +eval "$($(which conda) shell.bash hook)" # Setup the OS_TYPE environment variable that should be used for conditions involving the OS below. case $(uname) in @@ -88,6 +87,7 @@ if [[ "${OS_TYPE}" == "windows" ]]; then fi echo '::group::Install TorchVision' +# FIXME: can we run this upfront or does this needs to happen after the cmake build? python setup.py develop echo '::endgroup::' diff --git a/.github/workflows/build-cmake-linux.yml b/.github/workflows/build-cmake-linux.yml index d8372c550a4..15e250b3e93 100644 --- a/.github/workflows/build-cmake-linux.yml +++ b/.github/workflows/build-cmake-linux.yml @@ -32,9 +32,5 @@ jobs: export PYTHON_VERSION=3.8 export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} - ./.github/scripts/setup-env.sh - # Activate conda environment - eval "$($(which conda) shell.bash hook)" && conda deactivate && conda activate ci - - packaging/build_cmake.sh + ./.github/scripts/cmake.sh From 3abebff14520db11b8eae41891072bbb905e2d2c Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 9 May 2023 11:34:47 +0200 Subject: [PATCH 05/59] fix parallelism --- .github/scripts/cmake.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index 767851bd626..b4f1962887b 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -19,5 +19,10 @@ cd cpp_build cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" +PARALLELISM=8 +if [ -n "$MAX_JOBS" ]; then + PARALLELISM=$MAX_JOBS +fi + make -j$PARALLELISM make install From 5824c74003360a80b21a91553caefe57f013c9d4 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 9 May 2023 12:33:21 +0200 Subject: [PATCH 06/59] fix naming --- .github/scripts/setup-env.sh | 2 +- .github/workflows/build-cmake-linux.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/setup-env.sh b/.github/scripts/setup-env.sh index 973e6ec4f42..46958df1e36 100755 --- a/.github/scripts/setup-env.sh +++ b/.github/scripts/setup-env.sh @@ -88,7 +88,7 @@ fi echo '::group::Install TorchVision' # FIXME: can we run this upfront or does this needs to happen after the cmake build? -python setup.py develop +# python setup.py develop echo '::endgroup::' echo '::group::Collect PyTorch environment information' diff --git a/.github/workflows/build-cmake-linux.yml b/.github/workflows/build-cmake-linux.yml index 15e250b3e93..5e90997465e 100644 --- a/.github/workflows/build-cmake-linux.yml +++ b/.github/workflows/build-cmake-linux.yml @@ -10,7 +10,7 @@ on: workflow_dispatch: jobs: - unittests: + cmake: strategy: matrix: include: From 399f545906ac966206e0e63fa7c4ad937d4f05de Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 10 May 2023 13:02:13 +0200 Subject: [PATCH 07/59] trigger CI From fb4a55c4a38ebb8da6d247e58a5cc8e2410dcd31 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 10 May 2023 13:04:57 +0200 Subject: [PATCH 08/59] put back lint to debug --- .github/workflows/lint.yml | 109 +++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000000..c6198775fcd --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,109 @@ +name: Lint + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + python-source-and-configs: + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + script: | + set -euo pipefail + + echo '::group::Setup environment' + CONDA_PATH=$(which conda) + eval "$(${CONDA_PATH} shell.bash hook)" + conda create --name ci --quiet --yes python=3.8 pip + conda activate ci + echo '::endgroup::' + + echo '::group::Install lint tools' + pip install --progress-bar=off pre-commit + echo '::endgroup::' + + echo '::group::Lint Python source and configs' + set +e + pre-commit run --all-files + + if [ $? -ne 0 ]; then + git --no-pager diff + exit 1 + fi + echo '::endgroup::' + + c-source: + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + script: | + set -euo pipefail + + echo '::group::Setup environment' + CONDA_PATH=$(which conda) + eval "$(${CONDA_PATH} shell.bash hook)" + # clang-format needs some shared libraries that conflict with the system ones. Thus, we install them from conda + # and prepend the libraries to linker path to prioritize them. `ncurses=5` is only available on the conda-forge + # channel. Since we are not building or testing here, this is fine. + conda create --name ci --quiet --yes -c conda-forge python=3.8 ncurses=5 libgcc + conda activate ci + export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}" + echo '::endgroup::' + + echo '::group::Install lint tools' + curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o ./clang-format + chmod +x ./clang-format + echo '::endgroup::' + + echo '::group::Lint C source' + set +e + ./.circleci/unittest/linux/scripts/run-clang-format.py -r torchvision/csrc --clang-format-executable ./clang-format + + if [ $? -ne 0 ]; then + git --no-pager diff + exit 1 + fi + echo '::endgroup::' + + + python-types: + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + script: | + set -euo pipefail + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=cpu + export GPU_ARCH_VERSION='' + + ./.github/scripts/setup-env.sh + + CONDA_PATH=$(which conda) + eval "$(${CONDA_PATH} shell.bash hook)" + conda activate ci + + echo '::group::Install lint tools' + pip install --progress-bar=off mypy + echo '::endgroup::' + + echo '::group::Lint Python types' + mypy --install-types --non-interactive --config-file mypy.ini + echo '::endgroup::' + + bc: + if: github.event.pull_request + runs-on: ubuntu-latest + steps: + - name: Run BC Lint Action + uses: pytorch/test-infra/.github/actions/bc-lint@main + with: + repo: ${{ github.event.pull_request.head.repo.full_name }} + base_sha: ${{ github.event.pull_request.base.sha }} + head_sha: ${{ github.event.pull_request.head.sha }} From de5d2985c8fb5a1c0356bbbd24b6fa7e7f0a9f2c Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 10 May 2023 14:22:06 +0200 Subject: [PATCH 09/59] Revert "[REVERTME] disable unrelated CI" This reverts commit f9c371c65e9c66859de807bba5d8628e60e1c7c4. --- .github/workflows/build-conda-linux.yml | 51 +++++++++ .github/workflows/build-conda-m1.yml | 52 +++++++++ .github/workflows/build-conda-macos.yml | 52 +++++++++ .github/workflows/build-conda-windows.yml | 53 +++++++++ .github/workflows/build-wheels-linux.yml | 50 +++++++++ .github/workflows/build-wheels-m1.yml | 51 +++++++++ .github/workflows/build-wheels-macos.yml | 51 +++++++++ .github/workflows/build-wheels-windows.yml | 54 +++++++++ .github/workflows/docs.yml | 68 ++++++++++++ .github/workflows/pr-labels.yml | 35 ++++++ .../workflows/prototype-tests-linux-gpu.yml | 54 +++++++++ .github/workflows/test-linux.yml | 105 ++++++++++++++++++ .github/workflows/test-macos.yml | 40 +++++++ .github/workflows/test-windows.yml | 43 +++++++ .github/workflows/tests-schedule.yml | 57 ++++++++++ .github/workflows/update-viablestrict.yml | 23 ++++ 16 files changed, 839 insertions(+) create mode 100644 .github/workflows/build-conda-linux.yml create mode 100644 .github/workflows/build-conda-m1.yml create mode 100644 .github/workflows/build-conda-macos.yml create mode 100644 .github/workflows/build-conda-windows.yml create mode 100644 .github/workflows/build-wheels-linux.yml create mode 100644 .github/workflows/build-wheels-m1.yml create mode 100644 .github/workflows/build-wheels-macos.yml create mode 100644 .github/workflows/build-wheels-windows.yml create mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/pr-labels.yml create mode 100644 .github/workflows/prototype-tests-linux-gpu.yml create mode 100644 .github/workflows/test-linux.yml create mode 100644 .github/workflows/test-macos.yml create mode 100644 .github/workflows/test-windows.yml create mode 100644 .github/workflows/tests-schedule.yml create mode 100644 .github/workflows/update-viablestrict.yml diff --git a/.github/workflows/build-conda-linux.yml b/.github/workflows/build-conda-linux.yml new file mode 100644 index 00000000000..e44bb877ed6 --- /dev/null +++ b/.github/workflows/build-conda-linux.yml @@ -0,0 +1,51 @@ +name: Build Linux Conda + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: conda + os: linux + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: "" + post-script: "" + conda-package-directory: packaging/torchvision + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_conda_linux.yml@main + with: + conda-package-directory: ${{ matrix.conda-package-directory }} + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + secrets: + CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-m1.yml b/.github/workflows/build-conda-m1.yml new file mode 100644 index 00000000000..7006b6714aa --- /dev/null +++ b/.github/workflows/build-conda-m1.yml @@ -0,0 +1,52 @@ +name: Build M1 Conda + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: conda + os: macos-arm64 + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: "" + post-script: "" + conda-package-directory: packaging/torchvision + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main + with: + conda-package-directory: ${{ matrix.conda-package-directory }} + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + runner-type: macos-m1-12 + trigger-event: ${{ github.event_name }} + secrets: + CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-macos.yml b/.github/workflows/build-conda-macos.yml new file mode 100644 index 00000000000..39d00534b3a --- /dev/null +++ b/.github/workflows/build-conda-macos.yml @@ -0,0 +1,52 @@ +name: Build Macos Conda + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: conda + os: macos + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: "" + post-script: "" + conda-package-directory: packaging/torchvision + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main + with: + conda-package-directory: ${{ matrix.conda-package-directory }} + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + runner-type: macos-12 + trigger-event: ${{ github.event_name }} + secrets: + CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-conda-windows.yml b/.github/workflows/build-conda-windows.yml new file mode 100644 index 00000000000..98c68377189 --- /dev/null +++ b/.github/workflows/build-conda-windows.yml @@ -0,0 +1,53 @@ +name: Build Windows Conda + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: conda + os: windows + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: packaging/pre_build_script.sh + env-script: packaging/windows/internal/vc_env_helper.bat + post-script: "" + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_conda_windows.yml@main + with: + conda-package-directory: ${{ matrix.conda-package-directory }} + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + # Using "development" as trigger event so these binaries are not uploaded + # to official channels yet + trigger-event: development + secrets: + CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-wheels-linux.yml b/.github/workflows/build-wheels-linux.yml new file mode 100644 index 00000000000..e997d648ec2 --- /dev/null +++ b/.github/workflows/build-wheels-linux.yml @@ -0,0 +1,50 @@ +name: Build Linux Wheels + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: packaging/pre_build_script.sh + post-script: packaging/post_build_script.sh + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + secrets: + AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-m1.yml b/.github/workflows/build-wheels-m1.yml new file mode 100644 index 00000000000..66c7687acc9 --- /dev/null +++ b/.github/workflows/build-wheels-m1.yml @@ -0,0 +1,51 @@ +name: Build M1 Wheels + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: macos-arm64 + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: packaging/pre_build_script.sh + post-script: packaging/post_build_script.sh + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + runner-type: macos-m1-12 + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + secrets: + AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-macos.yml b/.github/workflows/build-wheels-macos.yml new file mode 100644 index 00000000000..6c5ebc0fc37 --- /dev/null +++ b/.github/workflows/build-wheels-macos.yml @@ -0,0 +1,51 @@ +name: Build Macos Wheels + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: macos + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: packaging/pre_build_script.sh + post-script: packaging/post_build_script.sh + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + runner-type: macos-12 + smoke-test-script: ${{ matrix.smoke-test-script }} + trigger-event: ${{ github.event_name }} + secrets: + AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-wheels-windows.yml b/.github/workflows/build-wheels-windows.yml new file mode 100644 index 00000000000..e238135979c --- /dev/null +++ b/.github/workflows/build-wheels-windows.yml @@ -0,0 +1,54 @@ +name: Build Windows Wheels + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: windows + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/vision + pre-script: packaging/pre_build_script.sh + env-script: packaging/windows/internal/vc_env_helper.bat + post-script: "python packaging/wheel/relocate.py" + smoke-test-script: test/smoke_test.py + package-name: torchvision + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + env-script: ${{ matrix.env-script }} + post-script: ${{ matrix.post-script }} + package-name: ${{ matrix.package-name }} + smoke-test-script: ${{ matrix.smoke-test-script }} + # Using "development" as trigger event so these binaries are not uploaded + # to official channels yet + trigger-event: development + secrets: + AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000000..22ff4ab84b3 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,68 @@ +name: Docs + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + build: + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + upload-artifact: docs + script: | + set -euo pipefail + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=cpu + export GPU_ARCH_VERSION='' + ./.github/scripts/setup-env.sh + + # Prepare conda + CONDA_PATH=$(which conda) + eval "$(${CONDA_PATH} shell.bash hook)" + conda activate ci + # FIXME: not sure why we need this. `ldd torchvision/video_reader.so` shows that it + # already links against the one pulled from conda. However, at runtime it pulls from + # /lib64 + # Should we maybe always do this in `./.github/scripts/setup-env.sh` so that we don't + # have to pay attention in all other workflows? + export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}" + + cd docs + + echo '::group::Install doc requirements' + pip install --progress-bar=off -r requirements.txt + echo '::endgroup::' + + echo '::group::Build HTML docs' + # The runner does not have sufficient memory to run with as many processes as their are + # cores (`-j auto`). Thus, we limit to a single process (`-j 1`) here. + sed -i -e 's/-j auto/-j 1/' Makefile + make html + echo '::endgroup::' + + mv build/html "${RUNNER_ARTIFACT_DIR}" + + upload-preview: + if: github.event_name == 'pull_request' + needs: [build] + runs-on: [self-hosted, linux.2xlarge] + steps: + - uses: actions/download-artifact@v3 + with: + name: docs + + - name: Upload docs preview + uses: seemethere/upload-artifact-s3@v5 + with: + retention-days: 14 + s3-bucket: doc-previews + if-no-files-found: error + path: html + s3-prefix: pytorch/vision/${{ github.event.pull_request.number }} diff --git a/.github/workflows/pr-labels.yml b/.github/workflows/pr-labels.yml new file mode 100644 index 00000000000..20c37e4fd88 --- /dev/null +++ b/.github/workflows/pr-labels.yml @@ -0,0 +1,35 @@ +name: pr-labels + +on: + push: + branches: + - main + +jobs: + is-properly-labeled: + runs-on: ubuntu-latest + + steps: + - name: Set up python + uses: actions/setup-python@v2 + + - name: Install requests + run: pip install requests + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Process commit and find merger responsible for labeling + id: commit + run: echo "::set-output name=merger::$(python .github/process_commit.py ${{ github.sha }})" + + - name: Ping merger responsible for labeling if necessary + if: ${{ steps.commit.outputs.merger != '' }} + uses: mshick/add-pr-comment@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + message: | + Hey ${{ steps.commit.outputs.merger }}! + + You merged this PR, but no labels were added. The list of valid labels is available at https://github.com/pytorch/vision/blob/main/.github/process_commit.py diff --git a/.github/workflows/prototype-tests-linux-gpu.yml b/.github/workflows/prototype-tests-linux-gpu.yml new file mode 100644 index 00000000000..dee425054d5 --- /dev/null +++ b/.github/workflows/prototype-tests-linux-gpu.yml @@ -0,0 +1,54 @@ +name: Prototype tests on Linux + +on: + pull_request: + +jobs: + unittests-prototype: + strategy: + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + runner: ["linux.12xlarge"] + gpu-arch-type: ["cpu"] + include: + - python-version: "3.8" + runner: linux.g5.4xlarge.nvidia.gpu + gpu-arch-type: cuda + gpu-arch-version: "11.7" + fail-fast: false + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + gpu-arch-type: ${{ matrix.gpu-arch-type }} + gpu-arch-version: ${{ matrix.gpu-arch-version }} + timeout: 120 + script: | + set -euo pipefail + + export PYTHON_VERSION=${{ matrix.python-version }} + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + ./.github/scripts/setup-env.sh + + # Prepare conda + CONDA_PATH=$(which conda) + eval "$(${CONDA_PATH} shell.bash hook)" + conda activate ci + + echo '::group::Install testing utilities' + pip install --progress-bar=off pytest pytest-mock pytest-cov + echo '::endgroup::' + + # We don't want to run the prototype datasets tests. Since the positional glob into `pytest`, i.e. + # `test/test_prototype*.py` takes the highest priority, neither `--ignore` nor `--ignore-glob` can help us here. + rm test/test_prototype_datasets*.py + pytest \ + -v --durations=25 \ + --cov=torchvision/prototype --cov-report=term-missing \ + --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" \ + test/test_prototype_*.py diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml new file mode 100644 index 00000000000..b9b01b7a55e --- /dev/null +++ b/.github/workflows/test-linux.yml @@ -0,0 +1,105 @@ +name: Tests on Linux + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + unittests: + strategy: + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + runner: ["linux.12xlarge"] + gpu-arch-type: ["cpu"] + include: + - python-version: 3.8 + runner: linux.g5.4xlarge.nvidia.gpu + gpu-arch-type: cuda + gpu-arch-version: "11.7" + fail-fast: false + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + gpu-arch-type: ${{ matrix.gpu-arch-type }} + gpu-arch-version: ${{ matrix.gpu-arch-version }} + timeout: 120 + script: | + set -euo pipefail + + export PYTHON_VERSION=${{ matrix.python-version }} + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + + ./.github/scripts/unittest.sh + + onnx: + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + script: | + set -euo pipefail + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=cpu + export GPU_ARCH_VERSION='' + + ./.github/scripts/setup-env.sh + + # Prepare conda + CONDA_PATH=$(which conda) + eval "$(${CONDA_PATH} shell.bash hook)" + conda activate ci + + echo '::group::Install ONNX' + pip install --progress-bar=off onnx onnxruntime + echo '::endgroup::' + + echo '::group::Install testing utilities' + pip install --progress-bar=off pytest + echo '::endgroup::' + + echo '::group::Run ONNX tests' + pytest --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v --durations=25 test/test_onnx.py + echo '::endgroup::' + + unittests-extended: + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + script: | + set -euo pipefail + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=cpu + export GPU_ARCH_VERSION='' + + ./.github/scripts/setup-env.sh + + # Prepare conda + CONDA_PATH=$(which conda) + eval "$(${CONDA_PATH} shell.bash hook)" + conda activate ci + + echo '::group::Pre-download model weights' + pip install --progress-bar=off aiohttp aiofiles tqdm + python scripts/download_model_urls.py + echo '::endgroup::' + + echo '::group::Install testing utilities' + pip install --progress-bar=off pytest + echo '::endgroup::' + + echo '::group::Run extended unittests' + export PYTORCH_TEST_WITH_EXTENDED=1 + pytest --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v --durations=25 test/test_extended_*.py + echo '::endgroup::' diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml new file mode 100644 index 00000000000..03e4b2db121 --- /dev/null +++ b/.github/workflows/test-macos.yml @@ -0,0 +1,40 @@ +name: Tests on macOS + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + unittests: + strategy: + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + runner: ["macos-12"] + include: + - python-version: "3.8" + runner: macos-m1-12 + fail-fast: false + uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + with: + repository: pytorch/vision + # We need an increased timeout here, since the macos-12 runner is the free one from GH + # and needs roughly 2 hours to just run the test suite + timeout: 240 + runner: ${{ matrix.runner }} + script: | + set -euo pipefail + + export PYTHON_VERSION=${{ matrix.python-version }} + export GPU_ARCH_TYPE=cpu + export GPU_ARCH_VERSION='' + + ./.github/scripts/unittest.sh diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml new file mode 100644 index 00000000000..b300a5d0e83 --- /dev/null +++ b/.github/workflows/test-windows.yml @@ -0,0 +1,43 @@ +name: Tests on Windows + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + unittests: + strategy: + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + runner: ["windows.4xlarge"] + gpu-arch-type: ["cpu"] + include: + - python-version: "3.8" + runner: windows.g5.4xlarge.nvidia.gpu + gpu-arch-type: cuda + gpu-arch-version: "11.7" + fail-fast: false + uses: pytorch/test-infra/.github/workflows/windows_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + gpu-arch-type: ${{ matrix.gpu-arch-type }} + gpu-arch-version: ${{ matrix.gpu-arch-version }} + timeout: 120 + script: | + set -euxo pipefail + + export PYTHON_VERSION=${{ matrix.python-version }} + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + + ./.github/scripts/unittest.sh diff --git a/.github/workflows/tests-schedule.yml b/.github/workflows/tests-schedule.yml new file mode 100644 index 00000000000..5426fdc997a --- /dev/null +++ b/.github/workflows/tests-schedule.yml @@ -0,0 +1,57 @@ +name: tests + +on: + pull_request: + paths: + - "test/test_datasets_download.py" + - ".github/failed_schedule_issue_template.md" + - ".github/workflows/tests-schedule.yml" + + schedule: + - cron: "0 9 * * *" + +jobs: + download: + runs-on: ubuntu-latest + + steps: + - name: Set up python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Upgrade system packages + run: python -m pip install --upgrade pip setuptools wheel + + - name: SSL + run: python -c 'import ssl; print(ssl.OPENSSL_VERSION)' + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install torch nightly build + run: pip install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + + - name: Install torchvision + run: pip install --no-build-isolation --editable . + + - name: Install all optional dataset requirements + run: pip install scipy pycocotools lmdb requests + + - name: Install tests requirements + run: pip install pytest + + - name: Run tests + run: pytest -ra -v test/test_datasets_download.py + + - uses: JasonEtco/create-an-issue@v2.4.0 + name: Create issue if download tests failed + if: failure() && github.event_name == 'schedule' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + WORKFLOW: ${{ github.workflow }} + JOB: ${{ github.job }} + ID: ${{ github.run_id }} + with: + filename: .github/failed_schedule_issue_template.md diff --git a/.github/workflows/update-viablestrict.yml b/.github/workflows/update-viablestrict.yml new file mode 100644 index 00000000000..2d9c2265676 --- /dev/null +++ b/.github/workflows/update-viablestrict.yml @@ -0,0 +1,23 @@ +name: Update viable/strict + +on: + pull_request: + paths: + - .github/workflows/update-viablestrict.yml + schedule: + - cron: 10,40 * * * * + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + do_update_viablestrict: + uses: pytorch/test-infra/.github/workflows/update-viablestrict.yml@main + with: + repository: pytorch/vision + required_checks: "Build Linux,Build M1,Build Macos,Tests on Linux,Tests on macOS,Docs,Lint" + secrets: + ROCKSET_API_KEY: ${{ secrets.ROCKSET_API_KEY }} + GITHUB_DEPLOY_KEY : ${{ secrets.VISION_GITHUB_DEPLOY_KEY }} From db44fef45ddaa47ebe75e5c2cd87cd33325b6e94 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 10 May 2023 14:29:39 +0200 Subject: [PATCH 10/59] allow unset MAX_JOBS --- .github/scripts/cmake.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index b4f1962887b..5844aea979b 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -20,9 +20,11 @@ cd cpp_build cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" PARALLELISM=8 +set +u if [ -n "$MAX_JOBS" ]; then PARALLELISM=$MAX_JOBS fi +set -u make -j$PARALLELISM make install From 405ea393695f24c231724f606728f9f39e1029a7 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 10 May 2023 14:40:06 +0200 Subject: [PATCH 11/59] use full cmake script --- .github/scripts/cmake.sh | 51 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index 5844aea979b..fb10ccb287a 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -7,6 +7,8 @@ set -euxo pipefail # Activate conda environment eval "$($(which conda) shell.bash hook)" && conda deactivate && conda activate ci +#### + Torch_DIR=$(python -c "import torch; print(torch.__path__[0])")/share/cmake/Torch if [[ "${GPU_ARCH_TYPE}" == "cuda" ]]; then WITH_CUDA=1 @@ -14,11 +16,6 @@ else WITH_CUDA=0 fi -mkdir cpp_build -cd cpp_build - -cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" - PARALLELISM=8 set +u if [ -n "$MAX_JOBS" ]; then @@ -26,5 +23,49 @@ if [ -n "$MAX_JOBS" ]; then fi set -u +#### + +mkdir cpp_build +pushd cpp_build + +cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" + make -j$PARALLELISM make install + +popd + +#### + +python setup.py develop + +#### + +pushd test/tracing/frcnn +mkdir build + +python trace_model.py +cp fasterrcnn_resnet50_fpn.pt build + +cd build +cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" +make -j$PARALLELISM + +./test_frcnn_tracing + +popd + +#### + +pushd examples/cpp/hello_world +mkdir build + +python trace_model.py +cp resnet18.pt build + +cd build +cmake .. -DTorch_DIR="${Torch_DIR}" + +make -j$PARALLELISM + +./hello-world From 58c539496dd392e29b0fba0f21da03f74349f721 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 10 May 2023 15:10:32 +0200 Subject: [PATCH 12/59] refactor --- .github/scripts/cmake.sh | 54 +++++++++++++++++++----------------- .github/scripts/setup-env.sh | 3 +- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index fb10ccb287a..02675a3f341 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -7,8 +7,6 @@ set -euxo pipefail # Activate conda environment eval "$($(which conda) shell.bash hook)" && conda deactivate && conda activate ci -#### - Torch_DIR=$(python -c "import torch; print(torch.__path__[0])")/share/cmake/Torch if [[ "${GPU_ARCH_TYPE}" == "cuda" ]]; then WITH_CUDA=1 @@ -23,49 +21,53 @@ if [ -n "$MAX_JOBS" ]; then fi set -u -#### +echo '::group::Prepare CMake builds' +mkdir -p cpp_build + +pushd test/tracing/frcnn +python trace_model.py +mkdir -p build +mv fasterrcnn_resnet50_fpn.pt build +popd + +pushd examples/cpp/hello_world +python trace_model.py +mkdir -p build +mv resnet18.pt build +popd -mkdir cpp_build +# This was only needed for the tracing above +pip uninstall -y torchvision +echo '::endgroup::' + +echo '::group::Build and install libtorchvision' pushd cpp_build cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" - make -j$PARALLELISM make install popd +echo '::endgroup::' -#### - -python setup.py develop +echo '::group::Build and run project that uses Faster-RCNN' +pushd test/tracing/frcnn/build -#### - -pushd test/tracing/frcnn -mkdir build - -python trace_model.py -cp fasterrcnn_resnet50_fpn.pt build - -cd build cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" make -j$PARALLELISM ./test_frcnn_tracing popd +echo '::endgroup::' -#### - -pushd examples/cpp/hello_world -mkdir build - -python trace_model.py -cp resnet18.pt build +echo '::group::Build and run C++ example' +pushd examples/cpp/hello_world/build -cd build cmake .. -DTorch_DIR="${Torch_DIR}" - make -j$PARALLELISM ./hello-world + +popd +echo '::endgroup::' diff --git a/.github/scripts/setup-env.sh b/.github/scripts/setup-env.sh index 46958df1e36..302a1247d6e 100755 --- a/.github/scripts/setup-env.sh +++ b/.github/scripts/setup-env.sh @@ -87,8 +87,7 @@ if [[ "${OS_TYPE}" == "windows" ]]; then fi echo '::group::Install TorchVision' -# FIXME: can we run this upfront or does this needs to happen after the cmake build? -# python setup.py develop +python setup.py develop echo '::endgroup::' echo '::group::Collect PyTorch environment information' From 6ed54005e0a521df5353938d4aa7b9db05ecc5df Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 10 May 2023 15:29:06 +0200 Subject: [PATCH 13/59] add support for macOS and Windows --- .github/workflows/build-cmake-macos.yml | 31 +++++++++++++++++++ .github/workflows/build-cmake-windows.yml | 36 +++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 .github/workflows/build-cmake-macos.yml create mode 100644 .github/workflows/build-cmake-windows.yml diff --git a/.github/workflows/build-cmake-macos.yml b/.github/workflows/build-cmake-macos.yml new file mode 100644 index 00000000000..a5e133349a6 --- /dev/null +++ b/.github/workflows/build-cmake-macos.yml @@ -0,0 +1,31 @@ +name: Tests on macOS + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + unittests: + strategy: + matrix: + include: + - runner: macos-12 + - runner: macos-m1-12 + fail-fast: false + uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + script: | + set -euo pipefail + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=cpu + export GPU_ARCH_VERSION='' + + ./.github/scripts/cmake.sh diff --git a/.github/workflows/build-cmake-windows.yml b/.github/workflows/build-cmake-windows.yml new file mode 100644 index 00000000000..33264efeb3f --- /dev/null +++ b/.github/workflows/build-cmake-windows.yml @@ -0,0 +1,36 @@ +name: Tests on Windows + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + unittests: + strategy: + matrix: + include: + - runner: windows.4xlarge + gpu-arch-type: cpu + - runner: windows.g5.4xlarge.nvidia.gpu + gpu-arch-type: cuda + gpu-arch-version: "11.7" + fail-fast: false + uses: pytorch/test-infra/.github/workflows/windows_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + gpu-arch-type: ${{ matrix.gpu-arch-type }} + gpu-arch-version: ${{ matrix.gpu-arch-version }} + script: | + set -euo pipefail + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + + ./.github/scripts/cmake.sh From 426887f13bb1a2feba2fff6f97a40e390eb9afbc Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 10 May 2023 15:35:06 +0200 Subject: [PATCH 14/59] remove parallelism flag --- .github/scripts/cmake.sh | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index 02675a3f341..17566744e3a 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -14,13 +14,6 @@ else WITH_CUDA=0 fi -PARALLELISM=8 -set +u -if [ -n "$MAX_JOBS" ]; then - PARALLELISM=$MAX_JOBS -fi -set -u - echo '::group::Prepare CMake builds' mkdir -p cpp_build @@ -44,7 +37,7 @@ echo '::group::Build and install libtorchvision' pushd cpp_build cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" -make -j$PARALLELISM +make make install popd @@ -54,7 +47,7 @@ echo '::group::Build and run project that uses Faster-RCNN' pushd test/tracing/frcnn/build cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" -make -j$PARALLELISM +make ./test_frcnn_tracing @@ -65,7 +58,7 @@ echo '::group::Build and run C++ example' pushd examples/cpp/hello_world/build cmake .. -DTorch_DIR="${Torch_DIR}" -make -j$PARALLELISM +make ./hello-world From 298a2366e9f291b06a616069279800bdfb6179d8 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 10 May 2023 16:24:59 +0200 Subject: [PATCH 15/59] trigger CI From 1af9dbb265c0ad20fba3543915fe08441067eaec Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 09:19:41 +0200 Subject: [PATCH 16/59] trigger CI From 1dc42f929600bca4950605bbf04737374b6be1b6 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 09:32:24 +0200 Subject: [PATCH 17/59] fix naming --- .github/workflows/build-cmake-linux.yml | 2 +- .github/workflows/build-cmake-macos.yml | 4 ++-- .github/workflows/build-cmake-windows.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-cmake-linux.yml b/.github/workflows/build-cmake-linux.yml index 5e90997465e..a41deb11a86 100644 --- a/.github/workflows/build-cmake-linux.yml +++ b/.github/workflows/build-cmake-linux.yml @@ -1,4 +1,4 @@ -name: Cmake on Linux +name: CMake on Linux on: pull_request: diff --git a/.github/workflows/build-cmake-macos.yml b/.github/workflows/build-cmake-macos.yml index a5e133349a6..b7ccb872f23 100644 --- a/.github/workflows/build-cmake-macos.yml +++ b/.github/workflows/build-cmake-macos.yml @@ -1,4 +1,4 @@ -name: Tests on macOS +name: CMake on macOS on: pull_request: @@ -10,7 +10,7 @@ on: workflow_dispatch: jobs: - unittests: + cmake: strategy: matrix: include: diff --git a/.github/workflows/build-cmake-windows.yml b/.github/workflows/build-cmake-windows.yml index 33264efeb3f..17c7eef8d53 100644 --- a/.github/workflows/build-cmake-windows.yml +++ b/.github/workflows/build-cmake-windows.yml @@ -1,4 +1,4 @@ -name: Tests on Windows +name: CMake on Windows on: pull_request: @@ -10,7 +10,7 @@ on: workflow_dispatch: jobs: - unittests: + cmake: strategy: matrix: include: From 16d0293d1e70b5c9685d34f47a11e9657fc4d9eb Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 09:35:02 +0200 Subject: [PATCH 18/59] consolidate workflows --- .github/workflows/build-cmake-linux.yml | 36 ---------- .github/workflows/build-cmake-macos.yml | 31 --------- .github/workflows/build-cmake-windows.yml | 36 ---------- .github/workflows/build-cmake.yml | 81 +++++++++++++++++++++++ 4 files changed, 81 insertions(+), 103 deletions(-) delete mode 100644 .github/workflows/build-cmake-linux.yml delete mode 100644 .github/workflows/build-cmake-macos.yml delete mode 100644 .github/workflows/build-cmake-windows.yml create mode 100644 .github/workflows/build-cmake.yml diff --git a/.github/workflows/build-cmake-linux.yml b/.github/workflows/build-cmake-linux.yml deleted file mode 100644 index a41deb11a86..00000000000 --- a/.github/workflows/build-cmake-linux.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: CMake on Linux - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -jobs: - cmake: - strategy: - matrix: - include: - - runner: linux.12xlarge - gpu-arch-type: cpu - - runner: linux.g5.4xlarge.nvidia.gpu - gpu-arch-type: cuda - gpu-arch-version: "11.7" - fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: pytorch/vision - runner: ${{ matrix.runner }} - gpu-arch-type: ${{ matrix.gpu-arch-type }} - gpu-arch-version: ${{ matrix.gpu-arch-version }} - script: | - set -euo pipefail - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} - export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} - - ./.github/scripts/cmake.sh diff --git a/.github/workflows/build-cmake-macos.yml b/.github/workflows/build-cmake-macos.yml deleted file mode 100644 index b7ccb872f23..00000000000 --- a/.github/workflows/build-cmake-macos.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: CMake on macOS - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -jobs: - cmake: - strategy: - matrix: - include: - - runner: macos-12 - - runner: macos-m1-12 - fail-fast: false - uses: pytorch/test-infra/.github/workflows/macos_job.yml@main - with: - repository: pytorch/vision - runner: ${{ matrix.runner }} - script: | - set -euo pipefail - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=cpu - export GPU_ARCH_VERSION='' - - ./.github/scripts/cmake.sh diff --git a/.github/workflows/build-cmake-windows.yml b/.github/workflows/build-cmake-windows.yml deleted file mode 100644 index 17c7eef8d53..00000000000 --- a/.github/workflows/build-cmake-windows.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: CMake on Windows - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: - -jobs: - cmake: - strategy: - matrix: - include: - - runner: windows.4xlarge - gpu-arch-type: cpu - - runner: windows.g5.4xlarge.nvidia.gpu - gpu-arch-type: cuda - gpu-arch-version: "11.7" - fail-fast: false - uses: pytorch/test-infra/.github/workflows/windows_job.yml@main - with: - repository: pytorch/vision - runner: ${{ matrix.runner }} - gpu-arch-type: ${{ matrix.gpu-arch-type }} - gpu-arch-version: ${{ matrix.gpu-arch-version }} - script: | - set -euo pipefail - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} - export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} - - ./.github/scripts/cmake.sh diff --git a/.github/workflows/build-cmake.yml b/.github/workflows/build-cmake.yml new file mode 100644 index 00000000000..d164b6200b9 --- /dev/null +++ b/.github/workflows/build-cmake.yml @@ -0,0 +1,81 @@ +name: CMake + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + linux: + strategy: + matrix: + include: + - runner: linux.12xlarge + gpu-arch-type: cpu + - runner: linux.g5.4xlarge.nvidia.gpu + gpu-arch-type: cuda + gpu-arch-version: "11.7" + fail-fast: false + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + gpu-arch-type: ${{ matrix.gpu-arch-type }} + gpu-arch-version: ${{ matrix.gpu-arch-version }} + script: | + set -euo pipefail + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + + ./.github/scripts/cmake.sh + + macos: + strategy: + matrix: + include: + - runner: macos-12 + - runner: macos-m1-12 + fail-fast: false + uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + script: | + set -euo pipefail + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=cpu + export GPU_ARCH_VERSION='' + + ./.github/scripts/cmake.sh + + windows: + strategy: + matrix: + include: + - runner: windows.4xlarge + gpu-arch-type: cpu + - runner: windows.g5.4xlarge.nvidia.gpu + gpu-arch-type: cuda + gpu-arch-version: "11.7" + fail-fast: false + uses: pytorch/test-infra/.github/workflows/windows_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + gpu-arch-type: ${{ matrix.gpu-arch-type }} + gpu-arch-version: ${{ matrix.gpu-arch-version }} + script: | + set -euo pipefail + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + + ./.github/scripts/cmake.sh From 40eecaef694f3444145093fab451b11ca8daeec3 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 10:04:58 +0200 Subject: [PATCH 19/59] fix cmake install prefix --- .github/scripts/cmake.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index 17566744e3a..9197270df00 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -36,7 +36,7 @@ echo '::endgroup::' echo '::group::Build and install libtorchvision' pushd cpp_build -cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" +cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" -DCMAKE_INSTALL_PREFIX="${CONDA_PREFIX}" make make install From 280868adb338483f7ae87b40b362f4b470d6281a Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 10:15:14 +0200 Subject: [PATCH 20/59] fix torchdir on Windows --- .github/scripts/cmake.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index 9197270df00..ef4b788bd43 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -7,7 +7,7 @@ set -euxo pipefail # Activate conda environment eval "$($(which conda) shell.bash hook)" && conda deactivate && conda activate ci -Torch_DIR=$(python -c "import torch; print(torch.__path__[0])")/share/cmake/Torch +Torch_DIR=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]).joinpath('share/cmake/Torch'))") if [[ "${GPU_ARCH_TYPE}" == "cuda" ]]; then WITH_CUDA=1 else From 82c038a3adc778a5d84bae4b41cdc116644b60e4 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 10:28:08 +0200 Subject: [PATCH 21/59] also uninstall system libpng on macos x86 --- .github/scripts/setup-env.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/scripts/setup-env.sh b/.github/scripts/setup-env.sh index 302a1247d6e..dce1cabcff1 100755 --- a/.github/scripts/setup-env.sh +++ b/.github/scripts/setup-env.sh @@ -24,12 +24,12 @@ esac if [[ "${OS_TYPE}" == "macos" && $(uname -m) == x86_64 ]]; then echo '::group::Uninstall system JPEG libraries on macOS' - # The x86 macOS runners, e.g. the GitHub Actions native "macos-12" runner, has some JPEG libraries installed by - # default that interfere with our build. We uninstall them here and use the one from conda below. - JPEG_LIBS=$(brew list | grep jpeg) - echo $JPEG_LIBS - for lib in $JPEG_LIBS; do - brew uninstall --ignore-dependencies --force $lib || true + # The x86 macOS runners, e.g. the GitHub Actions native "macos-12" runner, has some JPEG and PNG libraries + # installed by default that interfere with our build. We uninstall them here and use the one from conda below. + IMAGE_LIBS=$(brew list | grep -E "jpeg|png") + echo "${IMAGE_LIBS}" + for lib in "${IMAGE_LIBS}"; do + brew uninstall --ignore-dependencies --force "${lib}" || true done echo '::endgroup::' fi From 26c6d86977150772189b58154e852721e49db014 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 10:31:18 +0200 Subject: [PATCH 22/59] put back parallel builds --- .github/scripts/cmake.sh | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index ef4b788bd43..d3ec998dce9 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -7,6 +7,25 @@ set -euxo pipefail # Activate conda environment eval "$($(which conda) shell.bash hook)" && conda deactivate && conda activate ci +# Setup the OS_TYPE environment variable that should be used for conditions involving the OS below. +case $(uname) in + Linux) + OS_TYPE=linux + ;; + Darwin) + OS_TYPE=macos + ;; + MSYS*) + OS_TYPE=windows + ;; + *) + echo "Unknown OS type:" $(uname) + exit 1 + ;; +esac + +JOBS=$(nproc) + Torch_DIR=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]).joinpath('share/cmake/Torch'))") if [[ "${GPU_ARCH_TYPE}" == "cuda" ]]; then WITH_CUDA=1 @@ -37,7 +56,7 @@ echo '::group::Build and install libtorchvision' pushd cpp_build cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" -DCMAKE_INSTALL_PREFIX="${CONDA_PREFIX}" -make +make -j$JOBS make install popd @@ -47,7 +66,7 @@ echo '::group::Build and run project that uses Faster-RCNN' pushd test/tracing/frcnn/build cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" -make +make -jJOBS ./test_frcnn_tracing @@ -58,7 +77,7 @@ echo '::group::Build and run C++ example' pushd examples/cpp/hello_world/build cmake .. -DTorch_DIR="${Torch_DIR}" -make +make -jJOBS ./hello-world From 7917396341ee755b7963a1dd9df9f87d1c77f7a6 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 10:43:30 +0200 Subject: [PATCH 23/59] try windows helper --- .github/scripts/cmake.sh | 50 ++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index d3ec998dce9..6a733f774bd 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -25,6 +25,7 @@ case $(uname) in esac JOBS=$(nproc) +PACKAGING_DIR="${PWD}/packaging" Torch_DIR=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]).joinpath('share/cmake/Torch'))") if [[ "${GPU_ARCH_TYPE}" == "cuda" ]]; then @@ -56,30 +57,35 @@ echo '::group::Build and install libtorchvision' pushd cpp_build cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" -DCMAKE_INSTALL_PREFIX="${CONDA_PREFIX}" -make -j$JOBS -make install -popd -echo '::endgroup::' - -echo '::group::Build and run project that uses Faster-RCNN' -pushd test/tracing/frcnn/build - -cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" -make -jJOBS - -./test_frcnn_tracing +if [[ $OS_TYPE == windows ]]; then + "${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_cmake.bat" $JOBS +else + make -j$JOBS + make install +fi popd echo '::endgroup::' -echo '::group::Build and run C++ example' -pushd examples/cpp/hello_world/build - -cmake .. -DTorch_DIR="${Torch_DIR}" -make -jJOBS - -./hello-world - -popd -echo '::endgroup::' +#echo '::group::Build and run project that uses Faster-RCNN' +#pushd test/tracing/frcnn/build +# +#cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" +#make -jJOBS +# +#./test_frcnn_tracing +# +#popd +#echo '::endgroup::' +# +#echo '::group::Build and run C++ example' +#pushd examples/cpp/hello_world/build +# +#cmake .. -DTorch_DIR="${Torch_DIR}" +#make -jJOBS +# +#./hello-world +# +#popd +#echo '::endgroup::' From e28f1cbc53b51b947f29608bde7907327f7eb66c Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 11:48:20 +0200 Subject: [PATCH 24/59] fix number of jobs on macos --- .github/scripts/cmake.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index 6a733f774bd..b163c1b35e3 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -24,9 +24,15 @@ case $(uname) in ;; esac -JOBS=$(nproc) PACKAGING_DIR="${PWD}/packaging" +if [[ $OS_TYPE == macos ]]; then + JOBS=$(sysctl -n hw.logicalcpu) +else + JOBS=$(nproc) +fi + + Torch_DIR=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]).joinpath('share/cmake/Torch'))") if [[ "${GPU_ARCH_TYPE}" == "cuda" ]]; then WITH_CUDA=1 From 7cec87c0cd684f4267c3992b359376d3670f9be1 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 11:51:41 +0200 Subject: [PATCH 25/59] try adding more to the PATH on Windows --- .circleci/unittest/windows/scripts/set_cuda_envs.sh | 4 +++- .github/scripts/cmake.sh | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.circleci/unittest/windows/scripts/set_cuda_envs.sh b/.circleci/unittest/windows/scripts/set_cuda_envs.sh index 7db3137b594..93d1cca9406 100644 --- a/.circleci/unittest/windows/scripts/set_cuda_envs.sh +++ b/.circleci/unittest/windows/scripts/set_cuda_envs.sh @@ -20,7 +20,9 @@ fi # Unless there's an error, Don't exit. if [[ "$version" != "cpu" ]]; then # set cuda envs - export PATH="/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/bin:/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/libnvvp:$PATH" + export PATH=" + /c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/bin: + /c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/libnvvp:$PATH" export CUDA_PATH_V${version/./_}="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${version}" export CUDA_PATH="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${version}" diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index b163c1b35e3..f0347807725 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -32,6 +32,14 @@ else JOBS=$(nproc) fi +if [[ $OS_TYPE == windows ]]; then + echo $CUDA_PATH + echo $CUDA_HOME + echo $PATH + export PATH="${CUDA_PATH}/libnvvp:${PATH}" + JOBS=$(sysctl -n hw.logicalcpu) +fi + Torch_DIR=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]).joinpath('share/cmake/Torch'))") if [[ "${GPU_ARCH_TYPE}" == "cuda" ]]; then From 0dc5587a0ccc3dbd98051092728ce3a165085c6a Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 11:51:58 +0200 Subject: [PATCH 26/59] reenable the examples --- .github/scripts/cmake.sh | 42 ++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index f0347807725..2bbe40e5415 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -82,24 +82,24 @@ fi popd echo '::endgroup::' -#echo '::group::Build and run project that uses Faster-RCNN' -#pushd test/tracing/frcnn/build -# -#cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" -#make -jJOBS -# -#./test_frcnn_tracing -# -#popd -#echo '::endgroup::' -# -#echo '::group::Build and run C++ example' -#pushd examples/cpp/hello_world/build -# -#cmake .. -DTorch_DIR="${Torch_DIR}" -#make -jJOBS -# -#./hello-world -# -#popd -#echo '::endgroup::' +echo '::group::Build and run project that uses Faster-RCNN' +pushd test/tracing/frcnn/build + +cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" +make -jJOBS + +./test_frcnn_tracing + +popd +echo '::endgroup::' + +echo '::group::Build and run C++ example' +pushd examples/cpp/hello_world/build + +cmake .. -DTorch_DIR="${Torch_DIR}" +make -jJOBS + +./hello-world + +popd +echo '::endgroup::' From ca04d9c20f691d9a49d33c75d37623d8c30c0e45 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 11:58:20 +0200 Subject: [PATCH 27/59] fix parallel --- .github/scripts/cmake.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index 2bbe40e5415..7f5e0758f7f 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -86,7 +86,7 @@ echo '::group::Build and run project that uses Faster-RCNN' pushd test/tracing/frcnn/build cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" -make -jJOBS +make -j$JOBS ./test_frcnn_tracing @@ -97,7 +97,7 @@ echo '::group::Build and run C++ example' pushd examples/cpp/hello_world/build cmake .. -DTorch_DIR="${Torch_DIR}" -make -jJOBS +make -j$JOBS ./hello-world From 3d01320e9218d2aad27b2ef63f51eba90f6aebe7 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 13:05:20 +0200 Subject: [PATCH 28/59] fix windows examples --- .github/scripts/cmake.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index 7f5e0758f7f..9d03bbc8ffd 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -71,7 +71,6 @@ echo '::group::Build and install libtorchvision' pushd cpp_build cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" -DCMAKE_INSTALL_PREFIX="${CONDA_PREFIX}" - if [[ $OS_TYPE == windows ]]; then "${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_cmake.bat" $JOBS else @@ -86,7 +85,11 @@ echo '::group::Build and run project that uses Faster-RCNN' pushd test/tracing/frcnn/build cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" -make -j$JOBS +if [[ $OS_TYPE == windows ]]; then + "${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_frcnn.bat" $JOBS +else + make -j$JOBS +fi ./test_frcnn_tracing @@ -97,7 +100,11 @@ echo '::group::Build and run C++ example' pushd examples/cpp/hello_world/build cmake .. -DTorch_DIR="${Torch_DIR}" -make -j$JOBS +if [[ $OS_TYPE == windows ]]; then + "${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_cpp_example.bat" $JOBS +else + make -j$JOBS +fi ./hello-world From 3a465102e99fabe726d76179f8f9137dfa215af6 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 13:39:06 +0200 Subject: [PATCH 29/59] reduce verbosity during conda eval --- .github/scripts/cmake.sh | 2 +- .github/scripts/setup-env.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index 9d03bbc8ffd..2c37ab162bd 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -5,7 +5,7 @@ set -euxo pipefail ./.github/scripts/setup-env.sh # Activate conda environment -eval "$($(which conda) shell.bash hook)" && conda deactivate && conda activate ci +set +x && eval "$($(which conda) shell.bash hook)" && conda deactivate && conda activate ci && set -x # Setup the OS_TYPE environment variable that should be used for conditions involving the OS below. case $(uname) in diff --git a/.github/scripts/setup-env.sh b/.github/scripts/setup-env.sh index dce1cabcff1..a7ba4a6bd12 100755 --- a/.github/scripts/setup-env.sh +++ b/.github/scripts/setup-env.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -set -euo pipefail +set -euxo pipefail # Prepare conda -eval "$($(which conda) shell.bash hook)" +set +x && eval "$($(which conda) shell.bash hook)" && set -x # Setup the OS_TYPE environment variable that should be used for conditions involving the OS below. case $(uname) in From 2aea9bbefb178d76ee4058393945ced100c3ba1d Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 13:40:14 +0200 Subject: [PATCH 30/59] fix Windows CUDA --- .github/scripts/cmake.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index 2c37ab162bd..cde7a79d664 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -32,12 +32,11 @@ else JOBS=$(nproc) fi -if [[ $OS_TYPE == windows ]]; then +if [[ $OS_TYPE == windows && $GPU_ARCH_TYPE == cuda ]]; then echo $CUDA_PATH echo $CUDA_HOME echo $PATH export PATH="${CUDA_PATH}/libnvvp:${PATH}" - JOBS=$(sysctl -n hw.logicalcpu) fi From d88228d6cd5424763ce82379d02dbd359165bc39 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 13:57:04 +0200 Subject: [PATCH 31/59] add debug print --- .github/scripts/cmake.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index cde7a79d664..3f2ba1e8e78 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -69,6 +69,7 @@ echo '::endgroup::' echo '::group::Build and install libtorchvision' pushd cpp_build +ls "${CONDA_PREFIX}/lib" | grep -E "jpeg|png" cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" -DCMAKE_INSTALL_PREFIX="${CONDA_PREFIX}" if [[ $OS_TYPE == windows ]]; then "${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_cmake.bat" $JOBS From 33a4df19442b712eed6642fa0ac7c342ab3bedd6 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 14:00:14 +0200 Subject: [PATCH 32/59] fix prefix path --- .github/scripts/cmake.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index 3f2ba1e8e78..d9a5f297159 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -70,7 +70,7 @@ echo '::group::Build and install libtorchvision' pushd cpp_build ls "${CONDA_PREFIX}/lib" | grep -E "jpeg|png" -cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" -DCMAKE_INSTALL_PREFIX="${CONDA_PREFIX}" +cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" -DCMAKE_PREFIX_PATH="${CONDA_PREFIX}" -DCMAKE_INSTALL_PREFIX="${CONDA_PREFIX}" if [[ $OS_TYPE == windows ]]; then "${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_cmake.bat" $JOBS else From 08ae9e46bfcd127259049d9823fa949dbc270bfa Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 14:16:05 +0200 Subject: [PATCH 33/59] try fix windows --- .github/scripts/cmake.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index d9a5f297159..d50bfef62be 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -5,7 +5,7 @@ set -euxo pipefail ./.github/scripts/setup-env.sh # Activate conda environment -set +x && eval "$($(which conda) shell.bash hook)" && conda deactivate && conda activate ci && set -x +set +x && eval "$($(which conda) shell.bash hook)" && set -x && conda deactivate && conda activate ci # Setup the OS_TYPE environment variable that should be used for conditions involving the OS below. case $(uname) in @@ -33,13 +33,9 @@ else fi if [[ $OS_TYPE == windows && $GPU_ARCH_TYPE == cuda ]]; then - echo $CUDA_PATH - echo $CUDA_HOME - echo $PATH export PATH="${CUDA_PATH}/libnvvp:${PATH}" fi - Torch_DIR=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]).joinpath('share/cmake/Torch'))") if [[ "${GPU_ARCH_TYPE}" == "cuda" ]]; then WITH_CUDA=1 @@ -69,7 +65,10 @@ echo '::endgroup::' echo '::group::Build and install libtorchvision' pushd cpp_build -ls "${CONDA_PREFIX}/lib" | grep -E "jpeg|png" +if [[ $OS_TYPE == macos ]]; then + # DEBUG + ls "${CONDA_PREFIX}/lib" | grep -E "jpeg|png" +fi cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" -DCMAKE_PREFIX_PATH="${CONDA_PREFIX}" -DCMAKE_INSTALL_PREFIX="${CONDA_PREFIX}" if [[ $OS_TYPE == windows ]]; then "${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_cmake.bat" $JOBS From 64d2b0e3c433ba4c17df926cede10a8fda7797c4 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 14:16:48 +0200 Subject: [PATCH 34/59] revert unrelated --- .circleci/unittest/windows/scripts/set_cuda_envs.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.circleci/unittest/windows/scripts/set_cuda_envs.sh b/.circleci/unittest/windows/scripts/set_cuda_envs.sh index 93d1cca9406..7db3137b594 100644 --- a/.circleci/unittest/windows/scripts/set_cuda_envs.sh +++ b/.circleci/unittest/windows/scripts/set_cuda_envs.sh @@ -20,9 +20,7 @@ fi # Unless there's an error, Don't exit. if [[ "$version" != "cpu" ]]; then # set cuda envs - export PATH=" - /c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/bin: - /c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/libnvvp:$PATH" + export PATH="/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/bin:/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/libnvvp:$PATH" export CUDA_PATH_V${version/./_}="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${version}" export CUDA_PATH="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${version}" From f1e12010ac79b7676680f62101f31396c2f27bac Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 15:42:00 +0200 Subject: [PATCH 35/59] add find debug --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 405f947c233..ebceffec95c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,8 @@ option(WITH_PNG "Enable features requiring LibPNG." ON) option(WITH_JPEG "Enable features requiring LibJPEG." ON) option(USE_PYTHON "Link to Python when building" OFF) +set(CMAKE_FIND_DEBUG_MODE TRUE) + if(WITH_CUDA) enable_language(CUDA) add_definitions(-D__CUDA_NO_HALF_OPERATORS__) @@ -32,6 +34,8 @@ if (USE_PYTHON) find_package(Python3 REQUIRED COMPONENTS Development) endif() +set(CMAKE_FIND_DEBUG_MODE FALSE) + function(CUDA_CONVERT_FLAGS EXISTING_TARGET) get_property(old_flags TARGET ${EXISTING_TARGET} PROPERTY INTERFACE_COMPILE_OPTIONS) if(NOT "${old_flags}" STREQUAL "") From ce90113ca6164de6ac5d6d8af4de0d240df27d6e Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 11 May 2023 17:36:45 +0200 Subject: [PATCH 36/59] trigger CI From 10a6b802027671cc86d52550824f6c446e507b8f Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 12 May 2023 10:14:18 +0200 Subject: [PATCH 37/59] add more debug --- .github/scripts/cmake.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index d50bfef62be..cd32f5a39f7 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -68,6 +68,8 @@ pushd cpp_build if [[ $OS_TYPE == macos ]]; then # DEBUG ls "${CONDA_PREFIX}/lib" | grep -E "jpeg|png" + cat "${CONDA_PREFIX}/include/jpeglib.h" | grep VERSION + cat "${CONDA_PREFIX}/include/png.h" | grep VERSION_STRING fi cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" -DCMAKE_PREFIX_PATH="${CONDA_PREFIX}" -DCMAKE_INSTALL_PREFIX="${CONDA_PREFIX}" if [[ $OS_TYPE == windows ]]; then From 534147dd1f68ade30a8588f5831b621c4f322c30 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 12 May 2023 10:21:52 +0200 Subject: [PATCH 38/59] use otool --- .github/scripts/cmake.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index cd32f5a39f7..b718bd50fcc 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -67,7 +67,10 @@ pushd cpp_build if [[ $OS_TYPE == macos ]]; then # DEBUG - ls "${CONDA_PREFIX}/lib" | grep -E "jpeg|png" + IMAGE_LIBS=$(ls "${CONDA_PREFIX}/lib" | grep -E "(jpeg|png).*?\.dylib$") + for lib in "${IMAGE_LIBS}"; do + otool -L "${lib}" + done cat "${CONDA_PREFIX}/include/jpeglib.h" | grep VERSION cat "${CONDA_PREFIX}/include/png.h" | grep VERSION_STRING fi From 0f7f12656b97d4dac1f661e4227d3adfd5697482 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 12 May 2023 10:34:19 +0200 Subject: [PATCH 39/59] set CMAKE_FIND_FRAMEWORK --- .github/scripts/cmake.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index b718bd50fcc..90b1191934d 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -74,7 +74,10 @@ if [[ $OS_TYPE == macos ]]; then cat "${CONDA_PREFIX}/include/jpeglib.h" | grep VERSION cat "${CONDA_PREFIX}/include/png.h" | grep VERSION_STRING fi -cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" -DCMAKE_PREFIX_PATH="${CONDA_PREFIX}" -DCMAKE_INSTALL_PREFIX="${CONDA_PREFIX}" +cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" \ + -DCMAKE_PREFIX_PATH="${CONDA_PREFIX}" \ + -DCMAKE_FIND_FRAMEWORK=NEVER \ + -DCMAKE_INSTALL_PREFIX="${CONDA_PREFIX}" if [[ $OS_TYPE == windows ]]; then "${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_cmake.bat" $JOBS else From 8048e47a0909d629ddc723fdd1186dc7c4753168 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 12 May 2023 10:37:28 +0200 Subject: [PATCH 40/59] fix otool invocation --- .github/scripts/cmake.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index 90b1191934d..8517aa48d7e 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -69,7 +69,7 @@ if [[ $OS_TYPE == macos ]]; then # DEBUG IMAGE_LIBS=$(ls "${CONDA_PREFIX}/lib" | grep -E "(jpeg|png).*?\.dylib$") for lib in "${IMAGE_LIBS}"; do - otool -L "${lib}" + otool -L "${CONDA_PREFIX}/lib/${lib}" done cat "${CONDA_PREFIX}/include/jpeglib.h" | grep VERSION cat "${CONDA_PREFIX}/include/png.h" | grep VERSION_STRING From c6e77d293d9091c51f6feee5db4f80c574695bbc Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 12 May 2023 10:48:13 +0200 Subject: [PATCH 41/59] drop otool --- .github/scripts/cmake.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index 8517aa48d7e..12d34570687 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -68,9 +68,7 @@ pushd cpp_build if [[ $OS_TYPE == macos ]]; then # DEBUG IMAGE_LIBS=$(ls "${CONDA_PREFIX}/lib" | grep -E "(jpeg|png).*?\.dylib$") - for lib in "${IMAGE_LIBS}"; do - otool -L "${CONDA_PREFIX}/lib/${lib}" - done + echo $IMAGE_LIBS cat "${CONDA_PREFIX}/include/jpeglib.h" | grep VERSION cat "${CONDA_PREFIX}/include/png.h" | grep VERSION_STRING fi From 8a3f9d7c77bdda391137cec916e9b80f5d1f14ec Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 12 May 2023 11:09:04 +0200 Subject: [PATCH 42/59] remove debug and add comments --- .github/scripts/cmake.sh | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index 12d34570687..2dc733ea43f 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -65,15 +65,14 @@ echo '::endgroup::' echo '::group::Build and install libtorchvision' pushd cpp_build -if [[ $OS_TYPE == macos ]]; then - # DEBUG - IMAGE_LIBS=$(ls "${CONDA_PREFIX}/lib" | grep -E "(jpeg|png).*?\.dylib$") - echo $IMAGE_LIBS - cat "${CONDA_PREFIX}/include/jpeglib.h" | grep VERSION - cat "${CONDA_PREFIX}/include/png.h" | grep VERSION_STRING -fi cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" \ -DCMAKE_PREFIX_PATH="${CONDA_PREFIX}" \ + # On macOS, CMake is looking for the library (*.dylib) and the header (*.h) separately. By default, it prefers to load + # the header from other packages that install the library. This easily leads to a mismatch if the library installed + # from conda doesn't have the exact same version. Thus, we need to explicitly tell it to load anything from other + # installed frameworks. Resources: + # https://stackoverflow.com/questions/36523911/osx-homebrew-cmake-libpng-version-mismatch-issue + # https://cmake.org/cmake/help/latest/variable/CMAKE_FIND_FRAMEWORK.html -DCMAKE_FIND_FRAMEWORK=NEVER \ -DCMAKE_INSTALL_PREFIX="${CONDA_PREFIX}" if [[ $OS_TYPE == windows ]]; then @@ -89,7 +88,10 @@ echo '::endgroup::' echo '::group::Build and run project that uses Faster-RCNN' pushd test/tracing/frcnn/build -cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" +cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" \ + -DCMAKE_PREFIX_PATH="${CONDA_PREFIX}" \ + # See above for details + -DCMAKE_FIND_FRAMEWORK=NEVER \ if [[ $OS_TYPE == windows ]]; then "${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_frcnn.bat" $JOBS else @@ -104,7 +106,10 @@ echo '::endgroup::' echo '::group::Build and run C++ example' pushd examples/cpp/hello_world/build -cmake .. -DTorch_DIR="${Torch_DIR}" +cmake .. -DTorch_DIR="${Torch_DIR}" \ + -DCMAKE_PREFIX_PATH="${CONDA_PREFIX}" \ + # See above for details + -DCMAKE_FIND_FRAMEWORK=NEVER \ if [[ $OS_TYPE == windows ]]; then "${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_cpp_example.bat" $JOBS else From 27725ca1a97a7bc9bd28531739a762629069d426 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 12 May 2023 11:09:51 +0200 Subject: [PATCH 43/59] disable windows for now --- .github/workflows/build-cmake.yml | 48 +++++++++++++++---------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build-cmake.yml b/.github/workflows/build-cmake.yml index d164b6200b9..c6887353c89 100644 --- a/.github/workflows/build-cmake.yml +++ b/.github/workflows/build-cmake.yml @@ -55,27 +55,27 @@ jobs: ./.github/scripts/cmake.sh - windows: - strategy: - matrix: - include: - - runner: windows.4xlarge - gpu-arch-type: cpu - - runner: windows.g5.4xlarge.nvidia.gpu - gpu-arch-type: cuda - gpu-arch-version: "11.7" - fail-fast: false - uses: pytorch/test-infra/.github/workflows/windows_job.yml@main - with: - repository: pytorch/vision - runner: ${{ matrix.runner }} - gpu-arch-type: ${{ matrix.gpu-arch-type }} - gpu-arch-version: ${{ matrix.gpu-arch-version }} - script: | - set -euo pipefail - - export PYTHON_VERSION=3.8 - export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} - export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} - - ./.github/scripts/cmake.sh +# windows: +# strategy: +# matrix: +# include: +# - runner: windows.4xlarge +# gpu-arch-type: cpu +# - runner: windows.g5.4xlarge.nvidia.gpu +# gpu-arch-type: cuda +# gpu-arch-version: "11.7" +# fail-fast: false +# uses: pytorch/test-infra/.github/workflows/windows_job.yml@main +# with: +# repository: pytorch/vision +# runner: ${{ matrix.runner }} +# gpu-arch-type: ${{ matrix.gpu-arch-type }} +# gpu-arch-version: ${{ matrix.gpu-arch-version }} +# script: | +# set -euo pipefail +# +# export PYTHON_VERSION=3.8 +# export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} +# export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} +# +# ./.github/scripts/cmake.sh From 9db8e0c416238cc08d58908a838f846591814258 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 12 May 2023 11:18:36 +0200 Subject: [PATCH 44/59] remove CMake find debug --- CMakeLists.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ebceffec95c..405f947c233 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,8 +8,6 @@ option(WITH_PNG "Enable features requiring LibPNG." ON) option(WITH_JPEG "Enable features requiring LibJPEG." ON) option(USE_PYTHON "Link to Python when building" OFF) -set(CMAKE_FIND_DEBUG_MODE TRUE) - if(WITH_CUDA) enable_language(CUDA) add_definitions(-D__CUDA_NO_HALF_OPERATORS__) @@ -34,8 +32,6 @@ if (USE_PYTHON) find_package(Python3 REQUIRED COMPONENTS Development) endif() -set(CMAKE_FIND_DEBUG_MODE FALSE) - function(CUDA_CONVERT_FLAGS EXISTING_TARGET) get_property(old_flags TARGET ${EXISTING_TARGET} PROPERTY INTERFACE_COMPILE_OPTIONS) if(NOT "${old_flags}" STREQUAL "") From b056963768a37b08bd94f7f7293a37301c1c3aa7 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 12 May 2023 11:22:37 +0200 Subject: [PATCH 45/59] remove unnecessary path --- .github/scripts/cmake.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index 2dc733ea43f..89d5cc3fdc0 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -32,10 +32,6 @@ else JOBS=$(nproc) fi -if [[ $OS_TYPE == windows && $GPU_ARCH_TYPE == cuda ]]; then - export PATH="${CUDA_PATH}/libnvvp:${PATH}" -fi - Torch_DIR=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]).joinpath('share/cmake/Torch'))") if [[ "${GPU_ARCH_TYPE}" == "cuda" ]]; then WITH_CUDA=1 From 1d213c548f2d84106c84d56d4697822a4a8aa31a Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 12 May 2023 11:25:15 +0200 Subject: [PATCH 46/59] put windows CPU back --- .github/workflows/build-cmake.yml | 42 +++++++++++++++---------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-cmake.yml b/.github/workflows/build-cmake.yml index c6887353c89..5e28ce4698d 100644 --- a/.github/workflows/build-cmake.yml +++ b/.github/workflows/build-cmake.yml @@ -55,27 +55,27 @@ jobs: ./.github/scripts/cmake.sh -# windows: -# strategy: -# matrix: -# include: -# - runner: windows.4xlarge -# gpu-arch-type: cpu + windows: + strategy: + matrix: + include: + - runner: windows.4xlarge + gpu-arch-type: cpu # - runner: windows.g5.4xlarge.nvidia.gpu # gpu-arch-type: cuda # gpu-arch-version: "11.7" -# fail-fast: false -# uses: pytorch/test-infra/.github/workflows/windows_job.yml@main -# with: -# repository: pytorch/vision -# runner: ${{ matrix.runner }} -# gpu-arch-type: ${{ matrix.gpu-arch-type }} -# gpu-arch-version: ${{ matrix.gpu-arch-version }} -# script: | -# set -euo pipefail -# -# export PYTHON_VERSION=3.8 -# export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} -# export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} -# -# ./.github/scripts/cmake.sh + fail-fast: false + uses: pytorch/test-infra/.github/workflows/windows_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + gpu-arch-type: ${{ matrix.gpu-arch-type }} + gpu-arch-version: ${{ matrix.gpu-arch-version }} + script: | + set -euo pipefail + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + + ./.github/scripts/cmake.sh From 9c6f463aff873269f88650f7928e3a15d7e88de9 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 12 May 2023 11:28:40 +0200 Subject: [PATCH 47/59] fix inline comments --- .github/scripts/cmake.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index 89d5cc3fdc0..ba36db64206 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -61,14 +61,14 @@ echo '::endgroup::' echo '::group::Build and install libtorchvision' pushd cpp_build +# On macOS, CMake is looking for the library (*.dylib) and the header (*.h) separately. By default, it prefers to load +# the header from other packages that install the library. This easily leads to a mismatch if the library installed +# from conda doesn't have the exact same version. Thus, we need to explicitly set CMAKE_FIND_FRAMEWORK=NEVER to force +# it to not load anything from other installed frameworks. Resources: +# https://stackoverflow.com/questions/36523911/osx-homebrew-cmake-libpng-version-mismatch-issue +# https://cmake.org/cmake/help/latest/variable/CMAKE_FIND_FRAMEWORK.html cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" \ -DCMAKE_PREFIX_PATH="${CONDA_PREFIX}" \ - # On macOS, CMake is looking for the library (*.dylib) and the header (*.h) separately. By default, it prefers to load - # the header from other packages that install the library. This easily leads to a mismatch if the library installed - # from conda doesn't have the exact same version. Thus, we need to explicitly tell it to load anything from other - # installed frameworks. Resources: - # https://stackoverflow.com/questions/36523911/osx-homebrew-cmake-libpng-version-mismatch-issue - # https://cmake.org/cmake/help/latest/variable/CMAKE_FIND_FRAMEWORK.html -DCMAKE_FIND_FRAMEWORK=NEVER \ -DCMAKE_INSTALL_PREFIX="${CONDA_PREFIX}" if [[ $OS_TYPE == windows ]]; then @@ -86,8 +86,7 @@ pushd test/tracing/frcnn/build cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" \ -DCMAKE_PREFIX_PATH="${CONDA_PREFIX}" \ - # See above for details - -DCMAKE_FIND_FRAMEWORK=NEVER \ + -DCMAKE_FIND_FRAMEWORK=NEVER if [[ $OS_TYPE == windows ]]; then "${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_frcnn.bat" $JOBS else @@ -104,8 +103,7 @@ pushd examples/cpp/hello_world/build cmake .. -DTorch_DIR="${Torch_DIR}" \ -DCMAKE_PREFIX_PATH="${CONDA_PREFIX}" \ - # See above for details - -DCMAKE_FIND_FRAMEWORK=NEVER \ + -DCMAKE_FIND_FRAMEWORK=NEVER if [[ $OS_TYPE == windows ]]; then "${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_cpp_example.bat" $JOBS else From 1938a0d11c1acbacce96a9436d25f29784ecd1ee Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 12 May 2023 12:01:26 +0200 Subject: [PATCH 48/59] add torch to PATH on Windows --- .github/scripts/cmake.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index ba36db64206..3aa9edadf73 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -24,15 +24,19 @@ case $(uname) in ;; esac -PACKAGING_DIR="${PWD}/packaging" - if [[ $OS_TYPE == macos ]]; then JOBS=$(sysctl -n hw.logicalcpu) else JOBS=$(nproc) fi -Torch_DIR=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]).joinpath('share/cmake/Torch'))") +TORCH_PATH=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0])).as_posix()") +if [[ $OS_TYPE == windows ]]; then + PACKAGING_DIR="${PWD}/packaging" + export PATH="${TORCH_PATH}/lib:${PATH}" +fi + +Torch_DIR="${TORCH_PATH}/share/cmake/Torch" if [[ "${GPU_ARCH_TYPE}" == "cuda" ]]; then WITH_CUDA=1 else @@ -89,6 +93,8 @@ cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" \ -DCMAKE_FIND_FRAMEWORK=NEVER if [[ $OS_TYPE == windows ]]; then "${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_frcnn.bat" $JOBS + cd Release + cp ../fasterrcnn_resnet50_fpn.pt . else make -j$JOBS fi @@ -106,6 +112,8 @@ cmake .. -DTorch_DIR="${Torch_DIR}" \ -DCMAKE_FIND_FRAMEWORK=NEVER if [[ $OS_TYPE == windows ]]; then "${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_cpp_example.bat" $JOBS + cd Release + cp ../resnet18.pt . else make -j$JOBS fi From 1ec05142c7b9062d5b67bb4dc67cd304f2629b6f Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 12 May 2023 12:40:04 +0200 Subject: [PATCH 49/59] fix TORCH_PATH --- .github/scripts/cmake.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index 3aa9edadf73..ccc8d5e33b4 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -30,7 +30,7 @@ else JOBS=$(nproc) fi -TORCH_PATH=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0])).as_posix()") +TORCH_PATH=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]).as_posix())") if [[ $OS_TYPE == windows ]]; then PACKAGING_DIR="${PWD}/packaging" export PATH="${TORCH_PATH}/lib:${PATH}" From 1db13d6160ca7c17348e9b73fef476d23c79c356 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 12 May 2023 14:16:58 +0200 Subject: [PATCH 50/59] fix TORCH_PATH on Windows --- .github/scripts/cmake.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh index ccc8d5e33b4..ef3e5f61dad 100755 --- a/.github/scripts/cmake.sh +++ b/.github/scripts/cmake.sh @@ -5,7 +5,7 @@ set -euxo pipefail ./.github/scripts/setup-env.sh # Activate conda environment -set +x && eval "$($(which conda) shell.bash hook)" && set -x && conda deactivate && conda activate ci +set +x && eval "$($(which conda) shell.bash hook)" && conda deactivate && conda activate ci && set -x # Setup the OS_TYPE environment variable that should be used for conditions involving the OS below. case $(uname) in @@ -30,7 +30,7 @@ else JOBS=$(nproc) fi -TORCH_PATH=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]).as_posix())") +TORCH_PATH=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]))") if [[ $OS_TYPE == windows ]]; then PACKAGING_DIR="${PWD}/packaging" export PATH="${TORCH_PATH}/lib:${PATH}" From 9f8f48c711df3b351e6825b69f4648fb11fd6a5d Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 12 May 2023 14:45:40 +0200 Subject: [PATCH 51/59] put Windows GPU jobs back --- .github/workflows/build-cmake.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-cmake.yml b/.github/workflows/build-cmake.yml index 5e28ce4698d..d164b6200b9 100644 --- a/.github/workflows/build-cmake.yml +++ b/.github/workflows/build-cmake.yml @@ -61,9 +61,9 @@ jobs: include: - runner: windows.4xlarge gpu-arch-type: cpu -# - runner: windows.g5.4xlarge.nvidia.gpu -# gpu-arch-type: cuda -# gpu-arch-version: "11.7" + - runner: windows.g5.4xlarge.nvidia.gpu + gpu-arch-type: cuda + gpu-arch-version: "11.7" fail-fast: false uses: pytorch/test-infra/.github/workflows/windows_job.yml@main with: From 72835b9471f3668e7f6b6e268055eb5f3b905a65 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 16 May 2023 13:14:36 +0200 Subject: [PATCH 52/59] try nuclear CUDA reinstall on GPU runner --- .github/workflows/build-cmake.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build-cmake.yml b/.github/workflows/build-cmake.yml index d164b6200b9..24a387275a3 100644 --- a/.github/workflows/build-cmake.yml +++ b/.github/workflows/build-cmake.yml @@ -74,6 +74,14 @@ jobs: script: | set -euo pipefail + source packaging/windows/internal/vc_install_helper.sh + + if [[ ${{ matrix.gpu-arch-type }} == cuda ]]; then + export CU_VERSION=cu$(echo ${{ matrix.gpu-arch-version }} | sed 's/\.//') + echo CU_VERSION="${CU_VERSION}" + packaging/windows/internal/cuda_install.bat + fi + export PYTHON_VERSION=3.8 export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} From 427c82e2c2b71c7d1e05c5655b5fd4241aef276b Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 16 May 2023 13:46:30 +0200 Subject: [PATCH 53/59] add find debug for CUDA --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 405f947c233..9ba50be88b6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,6 +9,9 @@ option(WITH_JPEG "Enable features requiring LibJPEG." ON) option(USE_PYTHON "Link to Python when building" OFF) if(WITH_CUDA) + set(CMAKE_FIND_DEBUG_MODE TRUE) + find_package(CUDA) + set(CMAKE_FIND_DEBUG_MODE FALSE) enable_language(CUDA) add_definitions(-D__CUDA_NO_HALF_OPERATORS__) add_definitions(-DWITH_CUDA) From 862f268e895c19e9d36808f69e716ffcf6c78a94 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 16 May 2023 14:40:35 +0200 Subject: [PATCH 54/59] dont delete extracted file --- packaging/windows/internal/cuda_install.bat | 3 --- 1 file changed, 3 deletions(-) diff --git a/packaging/windows/internal/cuda_install.bat b/packaging/windows/internal/cuda_install.bat index 25aee6a9575..6820a84c156 100644 --- a/packaging/windows/internal/cuda_install.bat +++ b/packaging/windows/internal/cuda_install.bat @@ -165,6 +165,3 @@ echo Installing cuDNN... xcopy /Y "%SRC_DIR%\temp_build\cudnn\%CUDNN_FOLDER%\bin\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin" xcopy /Y "%SRC_DIR%\temp_build\cudnn\%CUDNN_FOLDER%\%CUDNN_LIB_FOLDER%\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\lib\x64" xcopy /Y "%SRC_DIR%\temp_build\cudnn\%CUDNN_FOLDER%\include\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\include" - -echo Cleaning temp files -rd /s /q "%SRC_DIR%\temp_build" || ver > nul From 9baaf701d890317f74ca529052bce840c6aa5238 Mon Sep 17 00:00:00 2001 From: Andrey Talman Date: Fri, 19 May 2023 12:50:34 -0400 Subject: [PATCH 55/59] Change integration path to reflect new AMI --- packaging/windows/internal/cuda_install.bat | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packaging/windows/internal/cuda_install.bat b/packaging/windows/internal/cuda_install.bat index 6820a84c156..2065973af95 100644 --- a/packaging/windows/internal/cuda_install.bat +++ b/packaging/windows/internal/cuda_install.bat @@ -127,14 +127,9 @@ echo %errorlevel% popd -echo Installing VS integration... -rem It's for VS 2019 -if "%CUDA_VER_MAJOR%" == "10" ( - xcopy /Y "%SRC_DIR%\temp_build\cuda\CUDAVisualStudioIntegration\extras\visual_studio_integration\MSBuildExtensions\*.*" "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Microsoft\VC\v160\BuildCustomizations" -) -if "%CUDA_VER_MAJOR%" == "11" ( - xcopy /Y "%SRC_DIR%\temp_build\cuda\visual_studio_integration\CUDAVisualStudioIntegration\extras\visual_studio_integration\MSBuildExtensions\*.*" "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Microsoft\VC\v160\BuildCustomizations" -) +echo Installing VS 2019 integration... +xcopy /YI "%SRC_DIR%\temp_build\cuda\visual_studio_integration\CUDAVisualStudioIntegration\extras\visual_studio_integration\MSBuildExtensions\*.*" "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Microsoft\VC\v160\BuildCustomizations" + echo Installing NvToolsExt... 7z x %SRC_DIR%\temp_build\NvToolsExt.7z -o"%SRC_DIR%\temp_build\NvToolsExt" From 9a86559339c940caabe406ded23b3072b8ba6461 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Sat, 20 May 2023 21:55:18 +0200 Subject: [PATCH 56/59] CXX_STANDARD 14 -> 17 --- examples/cpp/hello_world/CMakeLists.txt | 2 +- test/tracing/frcnn/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/cpp/hello_world/CMakeLists.txt b/examples/cpp/hello_world/CMakeLists.txt index 3ca59e4c199..7d49178b8b3 100644 --- a/examples/cpp/hello_world/CMakeLists.txt +++ b/examples/cpp/hello_world/CMakeLists.txt @@ -17,4 +17,4 @@ add_executable(hello-world main.cpp) # which also adds all the necessary torch dependencies. target_compile_features(hello-world PUBLIC cxx_range_for) target_link_libraries(hello-world TorchVision::TorchVision) -set_property(TARGET hello-world PROPERTY CXX_STANDARD 14) +set_property(TARGET hello-world PROPERTY CXX_STANDARD 17) diff --git a/test/tracing/frcnn/CMakeLists.txt b/test/tracing/frcnn/CMakeLists.txt index c79382470bd..8ede462e34b 100644 --- a/test/tracing/frcnn/CMakeLists.txt +++ b/test/tracing/frcnn/CMakeLists.txt @@ -10,4 +10,4 @@ find_package(Python3 COMPONENTS Development) add_executable(test_frcnn_tracing test_frcnn_tracing.cpp) target_compile_features(test_frcnn_tracing PUBLIC cxx_range_for) target_link_libraries(test_frcnn_tracing ${TORCH_LIBRARIES} TorchVision::TorchVision Python3::Python) -set_property(TARGET test_frcnn_tracing PROPERTY CXX_STANDARD 14) +set_property(TARGET test_frcnn_tracing PROPERTY CXX_STANDARD 17) From d0f41cea20944087042fc2cb35ab4b501e2d6845 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Sat, 20 May 2023 22:33:54 +0200 Subject: [PATCH 57/59] cleanup --- .github/workflows/build-cmake.yml | 2 ++ CMakeLists.txt | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-cmake.yml b/.github/workflows/build-cmake.yml index 24a387275a3..62ef463a70c 100644 --- a/.github/workflows/build-cmake.yml +++ b/.github/workflows/build-cmake.yml @@ -76,6 +76,8 @@ jobs: source packaging/windows/internal/vc_install_helper.sh + # FIXME: Basically, we are reinstalling CUDA here. We only need this, because we need to copy some files that + # can be extracted from the CUDA installer, but are not available on our Windows AMI. if [[ ${{ matrix.gpu-arch-type }} == cuda ]]; then export CU_VERSION=cu$(echo ${{ matrix.gpu-arch-version }} | sed 's/\.//') echo CU_VERSION="${CU_VERSION}" diff --git a/CMakeLists.txt b/CMakeLists.txt index 9ba50be88b6..405f947c233 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,9 +9,6 @@ option(WITH_JPEG "Enable features requiring LibJPEG." ON) option(USE_PYTHON "Link to Python when building" OFF) if(WITH_CUDA) - set(CMAKE_FIND_DEBUG_MODE TRUE) - find_package(CUDA) - set(CMAKE_FIND_DEBUG_MODE FALSE) enable_language(CUDA) add_definitions(-D__CUDA_NO_HALF_OPERATORS__) add_definitions(-DWITH_CUDA) From b98cf3e1d8832ebf63b57a2521ea97ee8f396aba Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Sat, 20 May 2023 23:16:11 +0200 Subject: [PATCH 58/59] add ref --- .github/workflows/build-cmake.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-cmake.yml b/.github/workflows/build-cmake.yml index 62ef463a70c..602fb16b9d9 100644 --- a/.github/workflows/build-cmake.yml +++ b/.github/workflows/build-cmake.yml @@ -78,6 +78,7 @@ jobs: # FIXME: Basically, we are reinstalling CUDA here. We only need this, because we need to copy some files that # can be extracted from the CUDA installer, but are not available on our Windows AMI. + # See https://github.com/pytorch/test-infra/pull/4189 if [[ ${{ matrix.gpu-arch-type }} == cuda ]]; then export CU_VERSION=cu$(echo ${{ matrix.gpu-arch-version }} | sed 's/\.//') echo CU_VERSION="${CU_VERSION}" From 79ae3e6e2851623a86acdb196d19fef18f4bff9a Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 23 May 2023 09:10:57 +0200 Subject: [PATCH 59/59] CUDA 11.7 -> 11.8 --- .github/workflows/build-cmake.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-cmake.yml b/.github/workflows/build-cmake.yml index 602fb16b9d9..8fdf99c84d1 100644 --- a/.github/workflows/build-cmake.yml +++ b/.github/workflows/build-cmake.yml @@ -18,7 +18,7 @@ jobs: gpu-arch-type: cpu - runner: linux.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "11.7" + gpu-arch-version: "11.8" fail-fast: false uses: pytorch/test-infra/.github/workflows/linux_job.yml@main with: @@ -63,7 +63,7 @@ jobs: gpu-arch-type: cpu - runner: windows.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "11.7" + gpu-arch-version: "11.8" fail-fast: false uses: pytorch/test-infra/.github/workflows/windows_job.yml@main with: