Skip to content

Use ccache for wheel building #3513

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 9 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/wheels-nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ on:

jobs:
fetch-latest-commit:
name: 'Fetch the latest commit of NEURON nightly branch'
runs-on: ubuntu-latest
outputs:
commit: ${{ steps.save-commit.outputs.commit }}
Expand Down
41 changes: 41 additions & 0 deletions .github/workflows/wheels-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,35 @@ jobs:
brew uninstall --ignore-dependencies libomp || echo "libomp doesn't exist"
echo "$(brew --prefix)/opt/cmake/bin:$(brew --prefix)/opt/flex/bin:$(brew --prefix)/opt/bison/bin" >> $GITHUB_PATH

- name: Install ccache
run: |
ccache_version='4.11.3'
ccache_srcdir="ccache-${ccache_version}"
ccache_builddir="ccache-build"
wget "https://github.com/ccache/ccache/releases/download/v${ccache_version}/ccache-${ccache_version}.tar.gz"
tar xf "${ccache_srcdir}.tar.gz"
cmake -B "${ccache_builddir}" -S "${ccache_srcdir}"
cmake --build "${ccache_builddir}" --parallel
cmake --install "${ccache_builddir}"

- name: Set ccache dir
run: |
if [ $(uname -s) = Darwin ]; then
echo "CCACHE_DIR=${{ github.workspace }}/ccache" >> $GITHUB_ENV
elif [ $(uname -s) = Linux ]; then
# cibuildwheel mounts the host machine under `/host`
echo "CCACHE_DIR=/host/${{ github.workspace }}/ccache" >> $GITHUB_ENV
fi

- name: Restore compiler cache
id: restore-compiler-cache
uses: actions/cache/restore@v4
with:
path: ${{ github.workspace }}/ccache
key: ccache-wheels-${{inputs.platform}}-${{inputs.python_version}}-${{github.ref_name}}
restore-keys: |
ccache-wheels-${{inputs.platform}}-${{inputs.python_version}}-master

- name: Install readline
if: runner.os == 'macOS'
run: |
Expand All @@ -105,14 +134,26 @@ jobs:

- name: Build wheel
run: |
# for debugging ccache hits
ccache -z
ccache -svv
bash packaging/python/build_wheels.bash $(uname -s) ${{ inputs.python_version }}
ccache -svv

- name: Upload wheel files
uses: actions/upload-artifact@v4
with:
name: wheels-${{ inputs.python_version }}-${{ inputs.platform }}
path: wheelhouse/*.whl

- name: Save compiler cache
id: save-compiler-cache
if: always()
uses: actions/cache/save@v4
with:
key: ${{ steps.restore-compiler-cache.outputs.cache-primary-key }}
path: ${{ github.workspace }}/ccache

- name: Setup Python ${{ inputs.python_version }} for testing
uses: actions/setup-python@v5
with:
Expand Down
24 changes: 23 additions & 1 deletion azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,12 +67,23 @@ stages:
condition: and(in(variables['Build.Reason'], 'Manual'), eq(variables['NRN_RELEASE_UPLOAD'], 'true'))
displayName: 'Change name of package for release'

# Restore ccache
- task: Cache@2
displayName: 'Restore ccache'
inputs:
key: 'ccache | "$(Agent.OS)" | "$(python.nodot)" | "$(Build.SourceVersion)"'
restoreKeys: |
ccache | "$(Agent.OS)" | "$(python.nodot)"
ccache | "$(Agent.OS)"
path: '$(Pipeline.Workspace)/.ccache'

# Note that mpt headers must be mounted in the docker imager under `/nrnwheel/mpt`
# This path is checked by `packaging/python/build_wheels.bash` when run in the image.
- script: |
if [ "$BUILD_REASON" = "Manual" ] || [ "$BUILD_REASON" = "Schedule" ]; then
export NRN_RX3D_OPT_LEVEL=1
fi
export CCACHE_DIR=$(Pipeline.Workspace)/.ccache
sudo mkdir -p /opt/nrnwheel/mpt
sudo tar -zxf $(mpt_headersSF.secureFilePath) --directory /opt/nrnwheel/mpt
packaging/python/build_wheels.bash linux $(python.nodot)
Expand Down Expand Up @@ -130,7 +141,7 @@ stages:

- script: |
brew install --cask xquartz
brew install flex bison mpich
brew install flex bison mpich ccache
brew unlink mpich && brew install openmpi
cmake --version
# the version of bash used by Azure (which is the default one
Expand Down Expand Up @@ -164,12 +175,23 @@ stages:
condition: and(in(variables['Build.Reason'], 'Manual'), eq(variables['NRN_RELEASE_UPLOAD'], 'true'))
displayName: 'Change name of package for release'

# Restore ccache
- task: Cache@2
displayName: 'Restore ccache'
inputs:
key: 'ccache | "$(Agent.OS)" | "$(python.nodot)" | "$(Build.SourceVersion)"'
restoreKeys: |
ccache | "$(Agent.OS)" | "$(python.nodot)"
ccache | "$(Agent.OS)"
path: '$(Pipeline.Workspace)/.ccache'

- script: |
export PATH=/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH
export SDKROOT=$(xcrun --sdk macosx --show-sdk-path)
if [ "$BUILD_REASON" = "Manual" ] || [ "$BUILD_REASON" = "Schedule" ]; then
export NRN_RX3D_OPT_LEVEL=1
fi
export CCACHE_DIR=$(Pipeline.Workspace)/.ccache
sudo mkdir -p /opt/nrnwheel/$(uname -m)
sudo tar -zxf $(readlineSF.secureFilePath) --directory /opt/nrnwheel/$(uname -m)
packaging/python/build_wheels.bash osx $(python.nodot)
Expand Down
11 changes: 11 additions & 0 deletions packaging/python/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,17 @@ RUN curl -L -o Python-3.10.0.tar.gz https://www.python.org/ftp/python/3.10.0/Pyt
RUN yum -y install epel-release libX11-devel libXcomposite-devel vim-enhanced && yum -y clean all && rm -rf /var/cache
RUN yum -y remove ncurses-devel

# install ccache for speeding-up builds
RUN export ccache_version='4.11.3' \
&& ccache_srcdir="ccache-${ccache_version}" \
&& ccache_builddir="ccache-build" \
&& wget "https://github.com/ccache/ccache/releases/download/v${ccache_version}/ccache-${ccache_version}.tar.gz" \
&& tar xf "ccache-${ccache_version}.tar.gz" \
&& cmake -B "${ccache_builddir}" -S "${ccache_srcdir}" \
&& cmake --build "${ccache_builddir}" --parallel \
&& cmake --install "${ccache_builddir}" \
&& rm -fr "ccache-${ccache_version}.tar.gz" "${ccache_srcdir}" "${ccache_builddir}"

# build wheels from there
WORKDIR /root

Expand Down
30 changes: 29 additions & 1 deletion packaging/python/build_wheels.bash
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,9 @@ set_cibw_environment() {
# seems that 10.15 is actually needed for std::filesystem::path.
# 11.0 is required on ARM machines
[MACOSX_DEPLOYMENT_TARGET]="10.15"
[CMAKE_C_COMPILER_LAUNCHER]=""
[CMAKE_CXX_COMPILER_LAUNCHER]=""
[CCACHE_DIR]=""
)
elif [ "${platform}" = 'linux' ]; then
declare -A defaults=(
Expand All @@ -110,6 +113,9 @@ set_cibw_environment() {
[CORENRN_ENABLE_OPENMP]="ON"
[NRN_BINARY_DIST_BUILD]="ON"
[NRN_RX3D_OPT_LEVEL]="0"
[CMAKE_C_COMPILER_LAUNCHER]=""
[CMAKE_CXX_COMPILER_LAUNCHER]=""
[CCACHE_DIR]=""
)
fi

Expand Down Expand Up @@ -152,9 +158,31 @@ build_wheel_portable() {
fi
fi

# use ccache if it is available (it's always available for Linux builds)
if [ "${platform}" = 'linux' ] || ([ "${platform}" = 'macos' ] && command -v ccache >& /dev/null); then
export CCACHE_DEBUG=1
export CMAKE_C_COMPILER_LAUNCHER=ccache
export CMAKE_CXX_COMPILER_LAUNCHER=ccache
if [ "${platform}" = 'linux' ]; then
# the host filesystem is available in the container at `/host`
export CCACHE_DIR="${CCACHE_DIR:-/host/tmp/ccache}"
export CCACHE_NODIRECT=
elif [ "${platform}" = 'macos' ]; then
export CCACHE_DIR="${CCACHE_DIR:-/tmp/ccache}"
fi
CCACHE_STATS_COMMAND="ccache -svv"
CCACHE_ZERO_COMMAND="ccache -z"
else
CCACHE_STATS_COMMAND=":"
CCACHE_ZERO_COMMAND=":"
fi

set_cibw_environment "${platform}"

CIBW_BUILD_VERBOSITY=1 python -m cibuildwheel --debug-traceback --platform "${platform}" --output-dir wheelhouse
export CIBW_TEST_COMMAND="${CCACHE_STATS_COMMAND}"
export CIBW_BEFORE_BUILD="${CCACHE_ZERO_COMMAND} && ${CCACHE_STATS_COMMAND}"
export CIBW_BUILD_VERBOSITY=1
python -m cibuildwheel --debug-traceback --platform "${platform}" --output-dir wheelhouse

deactivate
}
Expand Down
18 changes: 15 additions & 3 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -122,10 +122,17 @@ NRN_ENABLE_MPI_DYNAMIC = { default = "ON", env = "NRN_ENABLE_MPI_DYNAMIC" }
# MPI headers are platform-specific, so we do not set defaults
NRN_MPI_DYNAMIC = { env = "NRN_MPI_DYNAMIC" }

# in case we want to speed-up builds
CMAKE_C_COMPILER_LAUNCHER = { env = "CMAKE_C_COMPILER_LAUNCHER" }
CMAKE_CXX_COMPILER_LAUNCHER = { env = "CMAKE_CXX_COMPILER_LAUNCHER" }

# we do not need NMODL line numbers
NMODL_ENABLE_FLEX_BISON_LINES = "OFF"

[tool.cibuildwheel]
# We can specify a custom Docker image when building NEURON
manylinux-aarch64-image = "docker.io/neuronsimulator/neuron_wheel:manylinux_2_28_aarch64"
manylinux-x86_64-image = "docker.io/neuronsimulator/neuron_wheel:manylinux_2_28_x86_64"
manylinux-aarch64-image = "docker.io/neuronsimulator/neuron_wheel:manylinux_2_28_aarch64_ccache"
manylinux-x86_64-image = "docker.io/neuronsimulator/neuron_wheel:manylinux_2_28_x86_64_ccache"

# certain platforms are not supported
skip = [ "pp*", "*-win32", "*-manylinux_i686", "*-musllinux_i686", "*-musllinux_x86_64", "*-musllinux_aarch64" ]
Expand All @@ -143,10 +150,15 @@ environment-pass = [
"CORENRN_ENABLE_OPENMP",
"NRN_BINARY_DIST_BUILD",
"NRN_RX3D_OPT_LEVEL",
"CCACHE_DEBUG",
"CCACHE_NODIRECT",
# when making a release, we need to override the version
"SETUPTOOLS_SCM_PRETEND_VERSION",
# in case the user wants to use less resources
"CMAKE_BUILD_PARALLEL_LEVEL",
"CMAKE_C_COMPILER_LAUNCHER",
"CMAKE_CXX_COMPILER_LAUNCHER",
"CCACHE_DIR",
]

# comment originally from `build_wheels.bash`:
Expand All @@ -157,7 +169,7 @@ environment-pass = [
# libnrniv: we ship precompiled version of neurondemo containing libnrnmech.so
# which is linked to libnrniv.so. auditwheel manipulate rpaths and
# ships an extra copy of libnrniv.so and hence exclude it here.
repair-wheel-command = "auditwheel -v repair -w {dest_dir} {wheel} --exclude 'libgomp.so.1' --exclude 'libnrniv.so'"
repair-wheel-command = "auditwheel -v repair -w {dest_dir} {wheel} --exclude 'libgomp.so.1' --exclude 'libnrniv.so' >& /dev/null"

[tool.cibuildwheel.macos]
repair-wheel-command = "delocate-listdeps {wheel} && delocate-wheel -v -w {dest_dir} {wheel}"
Loading