Skip to content

Commit 4055bfb

Browse files
NicolasHugfacebook-github-bot
NicolasHug
authored and
facebook-github-bot
committed
[fbsync] migrate cmake workflows from CircleCI to GHA (#7417)
Summary: Co-authored-by: Andrey Talman <[email protected]> Reviewed By: vmoens Differential Revision: D46314040 fbshipit-source-id: 30d32462ee3e6615a23b6914b4d62e6e23070cec
1 parent b9ffac9 commit 4055bfb

File tree

4 files changed

+228
-21
lines changed

4 files changed

+228
-21
lines changed

‎.github/scripts/cmake.sh

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
#!/usr/bin/env bash
2+
3+
set -euxo pipefail
4+
5+
./.github/scripts/setup-env.sh
6+
7+
# Activate conda environment
8+
set +x && eval "$($(which conda) shell.bash hook)" && conda deactivate && conda activate ci && set -x
9+
10+
# Setup the OS_TYPE environment variable that should be used for conditions involving the OS below.
11+
case $(uname) in
12+
Linux)
13+
OS_TYPE=linux
14+
;;
15+
Darwin)
16+
OS_TYPE=macos
17+
;;
18+
MSYS*)
19+
OS_TYPE=windows
20+
;;
21+
*)
22+
echo "Unknown OS type:" $(uname)
23+
exit 1
24+
;;
25+
esac
26+
27+
if [[ $OS_TYPE == macos ]]; then
28+
JOBS=$(sysctl -n hw.logicalcpu)
29+
else
30+
JOBS=$(nproc)
31+
fi
32+
33+
TORCH_PATH=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]))")
34+
if [[ $OS_TYPE == windows ]]; then
35+
PACKAGING_DIR="${PWD}/packaging"
36+
export PATH="${TORCH_PATH}/lib:${PATH}"
37+
fi
38+
39+
Torch_DIR="${TORCH_PATH}/share/cmake/Torch"
40+
if [[ "${GPU_ARCH_TYPE}" == "cuda" ]]; then
41+
WITH_CUDA=1
42+
else
43+
WITH_CUDA=0
44+
fi
45+
46+
echo '::group::Prepare CMake builds'
47+
mkdir -p cpp_build
48+
49+
pushd test/tracing/frcnn
50+
python trace_model.py
51+
mkdir -p build
52+
mv fasterrcnn_resnet50_fpn.pt build
53+
popd
54+
55+
pushd examples/cpp/hello_world
56+
python trace_model.py
57+
mkdir -p build
58+
mv resnet18.pt build
59+
popd
60+
61+
# This was only needed for the tracing above
62+
pip uninstall -y torchvision
63+
echo '::endgroup::'
64+
65+
echo '::group::Build and install libtorchvision'
66+
pushd cpp_build
67+
68+
# On macOS, CMake is looking for the library (*.dylib) and the header (*.h) separately. By default, it prefers to load
69+
# the header from other packages that install the library. This easily leads to a mismatch if the library installed
70+
# from conda doesn't have the exact same version. Thus, we need to explicitly set CMAKE_FIND_FRAMEWORK=NEVER to force
71+
# it to not load anything from other installed frameworks. Resources:
72+
# https://stackoverflow.com/questions/36523911/osx-homebrew-cmake-libpng-version-mismatch-issue
73+
# https://cmake.org/cmake/help/latest/variable/CMAKE_FIND_FRAMEWORK.html
74+
cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" \
75+
-DCMAKE_PREFIX_PATH="${CONDA_PREFIX}" \
76+
-DCMAKE_FIND_FRAMEWORK=NEVER \
77+
-DCMAKE_INSTALL_PREFIX="${CONDA_PREFIX}"
78+
if [[ $OS_TYPE == windows ]]; then
79+
"${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_cmake.bat" $JOBS
80+
else
81+
make -j$JOBS
82+
make install
83+
fi
84+
85+
popd
86+
echo '::endgroup::'
87+
88+
echo '::group::Build and run project that uses Faster-RCNN'
89+
pushd test/tracing/frcnn/build
90+
91+
cmake .. -DTorch_DIR="${Torch_DIR}" -DWITH_CUDA="${WITH_CUDA}" \
92+
-DCMAKE_PREFIX_PATH="${CONDA_PREFIX}" \
93+
-DCMAKE_FIND_FRAMEWORK=NEVER
94+
if [[ $OS_TYPE == windows ]]; then
95+
"${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_frcnn.bat" $JOBS
96+
cd Release
97+
cp ../fasterrcnn_resnet50_fpn.pt .
98+
else
99+
make -j$JOBS
100+
fi
101+
102+
./test_frcnn_tracing
103+
104+
popd
105+
echo '::endgroup::'
106+
107+
echo '::group::Build and run C++ example'
108+
pushd examples/cpp/hello_world/build
109+
110+
cmake .. -DTorch_DIR="${Torch_DIR}" \
111+
-DCMAKE_PREFIX_PATH="${CONDA_PREFIX}" \
112+
-DCMAKE_FIND_FRAMEWORK=NEVER
113+
if [[ $OS_TYPE == windows ]]; then
114+
"${PACKAGING_DIR}/windows/internal/vc_env_helper.bat" "${PACKAGING_DIR}/windows/internal/build_cpp_example.bat" $JOBS
115+
cd Release
116+
cp ../resnet18.pt .
117+
else
118+
make -j$JOBS
119+
fi
120+
121+
./hello-world
122+
123+
popd
124+
echo '::endgroup::'

‎.github/scripts/setup-env.sh

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
#!/usr/bin/env bash
22

3-
set -euo pipefail
3+
set -euxo pipefail
44

55
# Prepare conda
6-
CONDA_PATH=$(which conda)
7-
eval "$(${CONDA_PATH} shell.bash hook)"
6+
set +x && eval "$($(which conda) shell.bash hook)" && set -x
87

98
# Setup the OS_TYPE environment variable that should be used for conditions involving the OS below.
109
case $(uname) in
@@ -25,12 +24,12 @@ esac
2524

2625
if [[ "${OS_TYPE}" == "macos" && $(uname -m) == x86_64 ]]; then
2726
echo '::group::Uninstall system JPEG libraries on macOS'
28-
# The x86 macOS runners, e.g. the GitHub Actions native "macos-12" runner, has some JPEG libraries installed by
29-
# default that interfere with our build. We uninstall them here and use the one from conda below.
30-
JPEG_LIBS=$(brew list | grep jpeg)
31-
echo $JPEG_LIBS
32-
for lib in $JPEG_LIBS; do
33-
brew uninstall --ignore-dependencies --force $lib || true
27+
# The x86 macOS runners, e.g. the GitHub Actions native "macos-12" runner, has some JPEG and PNG libraries
28+
# installed by default that interfere with our build. We uninstall them here and use the one from conda below.
29+
IMAGE_LIBS=$(brew list | grep -E "jpeg|png")
30+
echo "${IMAGE_LIBS}"
31+
for lib in "${IMAGE_LIBS}"; do
32+
brew uninstall --ignore-dependencies --force "${lib}" || true
3433
done
3534
echo '::endgroup::'
3635
fi
@@ -41,7 +40,7 @@ conda create \
4140
--name ci \
4241
--quiet --yes \
4342
python="${PYTHON_VERSION}" pip \
44-
ninja \
43+
ninja cmake \
4544
libpng jpeg \
4645
'ffmpeg<4.3'
4746
conda activate ci

‎.github/workflows/build-cmake.yml

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
name: CMake
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- nightly
8+
- main
9+
- release/*
10+
workflow_dispatch:
11+
12+
jobs:
13+
linux:
14+
strategy:
15+
matrix:
16+
include:
17+
- runner: linux.12xlarge
18+
gpu-arch-type: cpu
19+
- runner: linux.g5.4xlarge.nvidia.gpu
20+
gpu-arch-type: cuda
21+
gpu-arch-version: "11.8"
22+
fail-fast: false
23+
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
24+
with:
25+
repository: pytorch/vision
26+
runner: ${{ matrix.runner }}
27+
gpu-arch-type: ${{ matrix.gpu-arch-type }}
28+
gpu-arch-version: ${{ matrix.gpu-arch-version }}
29+
script: |
30+
set -euo pipefail
31+
32+
export PYTHON_VERSION=3.8
33+
export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }}
34+
export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }}
35+
36+
./.github/scripts/cmake.sh
37+
38+
macos:
39+
strategy:
40+
matrix:
41+
include:
42+
- runner: macos-12
43+
- runner: macos-m1-12
44+
fail-fast: false
45+
uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
46+
with:
47+
repository: pytorch/vision
48+
runner: ${{ matrix.runner }}
49+
script: |
50+
set -euo pipefail
51+
52+
export PYTHON_VERSION=3.8
53+
export GPU_ARCH_TYPE=cpu
54+
export GPU_ARCH_VERSION=''
55+
56+
./.github/scripts/cmake.sh
57+
58+
windows:
59+
strategy:
60+
matrix:
61+
include:
62+
- runner: windows.4xlarge
63+
gpu-arch-type: cpu
64+
- runner: windows.g5.4xlarge.nvidia.gpu
65+
gpu-arch-type: cuda
66+
gpu-arch-version: "11.8"
67+
fail-fast: false
68+
uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
69+
with:
70+
repository: pytorch/vision
71+
runner: ${{ matrix.runner }}
72+
gpu-arch-type: ${{ matrix.gpu-arch-type }}
73+
gpu-arch-version: ${{ matrix.gpu-arch-version }}
74+
script: |
75+
set -euo pipefail
76+
77+
source packaging/windows/internal/vc_install_helper.sh
78+
79+
# FIXME: Basically, we are reinstalling CUDA here. We only need this, because we need to copy some files that
80+
# can be extracted from the CUDA installer, but are not available on our Windows AMI.
81+
# See https://github.com/pytorch/test-infra/pull/4189
82+
if [[ ${{ matrix.gpu-arch-type }} == cuda ]]; then
83+
export CU_VERSION=cu$(echo ${{ matrix.gpu-arch-version }} | sed 's/\.//')
84+
echo CU_VERSION="${CU_VERSION}"
85+
packaging/windows/internal/cuda_install.bat
86+
fi
87+
88+
export PYTHON_VERSION=3.8
89+
export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }}
90+
export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }}
91+
92+
./.github/scripts/cmake.sh

‎packaging/windows/internal/cuda_install.bat

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -127,14 +127,9 @@ echo %errorlevel%
127127

128128
popd
129129

130-
echo Installing VS integration...
131-
rem It's for VS 2019
132-
if "%CUDA_VER_MAJOR%" == "10" (
133-
xcopy /Y "%SRC_DIR%\temp_build\cuda\CUDAVisualStudioIntegration\extras\visual_studio_integration\MSBuildExtensions\*.*" "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Microsoft\VC\v160\BuildCustomizations"
134-
)
135-
if "%CUDA_VER_MAJOR%" == "11" (
136-
xcopy /Y "%SRC_DIR%\temp_build\cuda\visual_studio_integration\CUDAVisualStudioIntegration\extras\visual_studio_integration\MSBuildExtensions\*.*" "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Microsoft\VC\v160\BuildCustomizations"
137-
)
130+
echo Installing VS 2019 integration...
131+
xcopy /YI "%SRC_DIR%\temp_build\cuda\visual_studio_integration\CUDAVisualStudioIntegration\extras\visual_studio_integration\MSBuildExtensions\*.*" "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Microsoft\VC\v160\BuildCustomizations"
132+
138133

139134
echo Installing NvToolsExt...
140135
7z x %SRC_DIR%\temp_build\NvToolsExt.7z -o"%SRC_DIR%\temp_build\NvToolsExt"
@@ -165,6 +160,3 @@ echo Installing cuDNN...
165160
xcopy /Y "%SRC_DIR%\temp_build\cudnn\%CUDNN_FOLDER%\bin\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin"
166161
xcopy /Y "%SRC_DIR%\temp_build\cudnn\%CUDNN_FOLDER%\%CUDNN_LIB_FOLDER%\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\lib\x64"
167162
xcopy /Y "%SRC_DIR%\temp_build\cudnn\%CUDNN_FOLDER%\include\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\include"
168-
169-
echo Cleaning temp files
170-
rd /s /q "%SRC_DIR%\temp_build" || ver > nul

0 commit comments

Comments
 (0)