diff --git a/.github/scripts/cmake.sh b/.github/scripts/cmake.sh new file mode 100755 index 00000000000..ef3e5f61dad --- /dev/null +++ b/.github/scripts/cmake.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +./.github/scripts/setup-env.sh + +# Activate conda environment +set +x && eval "$($(which conda) shell.bash hook)" && conda deactivate && conda activate ci && set -x + +# Setup the OS_TYPE environment variable that should be used for conditions involving the OS below. +case $(uname) in + Linux) + OS_TYPE=linux + ;; + Darwin) + OS_TYPE=macos + ;; + MSYS*) + OS_TYPE=windows + ;; + *) + echo "Unknown OS type:" $(uname) + exit 1 + ;; +esac + +if [[ $OS_TYPE == macos ]]; then + JOBS=$(sysctl -n hw.logicalcpu) +else + JOBS=$(nproc) +fi + +TORCH_PATH=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]))") +if [[ $OS_TYPE == windows ]]; then + PACKAGING_DIR="${PWD}/packaging" + export PATH="${TORCH_PATH}/lib:${PATH}" +fi + +Torch_DIR="${TORCH_PATH}/share/cmake/Torch" +if [[ "${GPU_ARCH_TYPE}" == "cuda" ]]; then + WITH_CUDA=1 +else + WITH_CUDA=0 +fi + +echo '::group::Prepare CMake builds' +mkdir -p cpp_build + +pushd test/tracing/frcnn +python trace_model.py +mkdir -p build +mv fasterrcnn_resnet50_fpn.pt build +popd + +pushd examples/cpp/hello_world +python trace_model.py +mkdir -p build +mv resnet18.pt build +popd + +# This was only needed for the tracing above +pip uninstall -y torchvision +echo '::endgroup::' + +echo '::group::Build and install libtorchvision' +pushd cpp_build + +# On macOS, CMake is looking for the library (*.dylib) and the header (*.h) separately. By default, it prefers to load +# the header from other packages that install the library. This easily leads to a mismatch if the library installed +# from conda doesn't have the exact same version. Thus, we need to explicitly set CMAKE_FIND_FRAMEWORK=NEVER to force +# it to not load anything from other installed frameworks. Resources: +# https://stackoverflow.com/questions/36523911/osx-homebrew-cmake-libpng-version-mismatch-issue +# https://cmake.org/cmake/help/latest/variable/CMAKE_FIND_FRAMEWORK.html +cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" \ + -DCMAKE_PREFIX_PATH="${CONDA_PREFIX}" \ + -DCMAKE_FIND_FRAMEWORK=NEVER \ + -DCMAKE_INSTALL_PREFIX="${CONDA_PREFIX}" +if [[ $OS_TYPE == windows ]]; then + "${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_cmake.bat" $JOBS +else + make -j$JOBS + make install +fi + +popd +echo '::endgroup::' + +echo '::group::Build and run project that uses Faster-RCNN' +pushd test/tracing/frcnn/build + +cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" \ + -DCMAKE_PREFIX_PATH="${CONDA_PREFIX}" \ + -DCMAKE_FIND_FRAMEWORK=NEVER +if [[ $OS_TYPE == windows ]]; then + "${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_frcnn.bat" $JOBS + cd Release + cp ../fasterrcnn_resnet50_fpn.pt . +else + make -j$JOBS +fi + +./test_frcnn_tracing + +popd +echo '::endgroup::' + +echo '::group::Build and run C++ example' +pushd examples/cpp/hello_world/build + +cmake .. -DTorch_DIR="${Torch_DIR}" \ + -DCMAKE_PREFIX_PATH="${CONDA_PREFIX}" \ + -DCMAKE_FIND_FRAMEWORK=NEVER +if [[ $OS_TYPE == windows ]]; then + "${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_cpp_example.bat" $JOBS + cd Release + cp ../resnet18.pt . +else + make -j$JOBS +fi + +./hello-world + +popd +echo '::endgroup::' diff --git a/.github/scripts/setup-env.sh b/.github/scripts/setup-env.sh index 635fdb2654a..a7ba4a6bd12 100755 --- a/.github/scripts/setup-env.sh +++ b/.github/scripts/setup-env.sh @@ -1,10 +1,9 @@ #!/usr/bin/env bash -set -euo pipefail +set -euxo pipefail # Prepare conda -CONDA_PATH=$(which conda) -eval "$(${CONDA_PATH} shell.bash hook)" +set +x && eval "$($(which conda) shell.bash hook)" && set -x # Setup the OS_TYPE environment variable that should be used for conditions involving the OS below. case $(uname) in @@ -25,12 +24,12 @@ esac if [[ "${OS_TYPE}" == "macos" && $(uname -m) == x86_64 ]]; then echo '::group::Uninstall system JPEG libraries on macOS' - # The x86 macOS runners, e.g. the GitHub Actions native "macos-12" runner, has some JPEG libraries installed by - # default that interfere with our build. We uninstall them here and use the one from conda below. - JPEG_LIBS=$(brew list | grep jpeg) - echo $JPEG_LIBS - for lib in $JPEG_LIBS; do - brew uninstall --ignore-dependencies --force $lib || true + # The x86 macOS runners, e.g. the GitHub Actions native "macos-12" runner, has some JPEG and PNG libraries + # installed by default that interfere with our build. We uninstall them here and use the one from conda below. + IMAGE_LIBS=$(brew list | grep -E "jpeg|png") + echo "${IMAGE_LIBS}" + for lib in "${IMAGE_LIBS}"; do + brew uninstall --ignore-dependencies --force "${lib}" || true done echo '::endgroup::' fi @@ -41,7 +40,7 @@ conda create \ --name ci \ --quiet --yes \ python="${PYTHON_VERSION}" pip \ - ninja \ + ninja cmake \ libpng jpeg \ 'ffmpeg<4.3' conda activate ci diff --git a/.github/workflows/build-cmake.yml b/.github/workflows/build-cmake.yml new file mode 100644 index 00000000000..8fdf99c84d1 --- /dev/null +++ b/.github/workflows/build-cmake.yml @@ -0,0 +1,92 @@ +name: CMake + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + linux: + strategy: + matrix: + include: + - runner: linux.12xlarge + gpu-arch-type: cpu + - runner: linux.g5.4xlarge.nvidia.gpu + gpu-arch-type: cuda + gpu-arch-version: "11.8" + fail-fast: false + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + gpu-arch-type: ${{ matrix.gpu-arch-type }} + gpu-arch-version: ${{ matrix.gpu-arch-version }} + script: | + set -euo pipefail + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + + ./.github/scripts/cmake.sh + + macos: + strategy: + matrix: + include: + - runner: macos-12 + - runner: macos-m1-12 + fail-fast: false + uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + script: | + set -euo pipefail + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=cpu + export GPU_ARCH_VERSION='' + + ./.github/scripts/cmake.sh + + windows: + strategy: + matrix: + include: + - runner: windows.4xlarge + gpu-arch-type: cpu + - runner: windows.g5.4xlarge.nvidia.gpu + gpu-arch-type: cuda + gpu-arch-version: "11.8" + fail-fast: false + uses: pytorch/test-infra/.github/workflows/windows_job.yml@main + with: + repository: pytorch/vision + runner: ${{ matrix.runner }} + gpu-arch-type: ${{ matrix.gpu-arch-type }} + gpu-arch-version: ${{ matrix.gpu-arch-version }} + script: | + set -euo pipefail + + source packaging/windows/internal/vc_install_helper.sh + + # FIXME: Basically, we are reinstalling CUDA here. We only need this, because we need to copy some files that + # can be extracted from the CUDA installer, but are not available on our Windows AMI. + # See https://github.com/pytorch/test-infra/pull/4189 + if [[ ${{ matrix.gpu-arch-type }} == cuda ]]; then + export CU_VERSION=cu$(echo ${{ matrix.gpu-arch-version }} | sed 's/\.//') + echo CU_VERSION="${CU_VERSION}" + packaging/windows/internal/cuda_install.bat + fi + + export PYTHON_VERSION=3.8 + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + + ./.github/scripts/cmake.sh diff --git a/packaging/windows/internal/cuda_install.bat b/packaging/windows/internal/cuda_install.bat index 25aee6a9575..2065973af95 100644 --- a/packaging/windows/internal/cuda_install.bat +++ b/packaging/windows/internal/cuda_install.bat @@ -127,14 +127,9 @@ echo %errorlevel% popd -echo Installing VS integration... -rem It's for VS 2019 -if "%CUDA_VER_MAJOR%" == "10" ( - xcopy /Y "%SRC_DIR%\temp_build\cuda\CUDAVisualStudioIntegration\extras\visual_studio_integration\MSBuildExtensions\*.*" "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Microsoft\VC\v160\BuildCustomizations" -) -if "%CUDA_VER_MAJOR%" == "11" ( - xcopy /Y "%SRC_DIR%\temp_build\cuda\visual_studio_integration\CUDAVisualStudioIntegration\extras\visual_studio_integration\MSBuildExtensions\*.*" "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Microsoft\VC\v160\BuildCustomizations" -) +echo Installing VS 2019 integration... +xcopy /YI "%SRC_DIR%\temp_build\cuda\visual_studio_integration\CUDAVisualStudioIntegration\extras\visual_studio_integration\MSBuildExtensions\*.*" "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Microsoft\VC\v160\BuildCustomizations" + echo Installing NvToolsExt... 7z x %SRC_DIR%\temp_build\NvToolsExt.7z -o"%SRC_DIR%\temp_build\NvToolsExt" @@ -165,6 +160,3 @@ echo Installing cuDNN... xcopy /Y "%SRC_DIR%\temp_build\cudnn\%CUDNN_FOLDER%\bin\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin" xcopy /Y "%SRC_DIR%\temp_build\cudnn\%CUDNN_FOLDER%\%CUDNN_LIB_FOLDER%\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\lib\x64" xcopy /Y "%SRC_DIR%\temp_build\cudnn\%CUDNN_FOLDER%\include\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\include" - -echo Cleaning temp files -rd /s /q "%SRC_DIR%\temp_build" || ver > nul