Use ccache for wheel building

.github/workflows/wheels-nightly.yml

@@ -25,6 +25,7 @@ on:
 
 jobs:
   fetch-latest-commit:
+    name: 'Fetch the latest commit of NEURON nightly branch'
     runs-on: ubuntu-latest
     outputs:
       commit: ${{ steps.save-commit.outputs.commit }}

.github/workflows/wheels-template.yml

@@ -81,6 +81,35 @@ jobs:
           brew uninstall --ignore-dependencies libomp || echo "libomp doesn't exist"
           echo "$(brew --prefix)/opt/cmake/bin:$(brew --prefix)/opt/flex/bin:$(brew --prefix)/opt/bison/bin" >> $GITHUB_PATH
 
+      - name: Install ccache
+        run: |
+          ccache_version='4.11.3'
+          ccache_srcdir="ccache-${ccache_version}"
+          ccache_builddir="ccache-build"
+          wget "https://github.com/ccache/ccache/releases/download/v${ccache_version}/ccache-${ccache_version}.tar.gz"
+          tar xf "${ccache_srcdir}.tar.gz"
+          cmake -B "${ccache_builddir}" -S "${ccache_srcdir}"
+          cmake --build "${ccache_builddir}" --parallel
+          cmake --install "${ccache_builddir}"
+
+      - name: Set ccache dir
+        run: |
+          if [ $(uname -s) = Darwin ]; then
+            echo "CCACHE_DIR=${{ github.workspace }}/ccache" >> $GITHUB_ENV
+          elif [ $(uname -s) = Linux ]; then
+            # cibuildwheel mounts the host machine under `/host`
+            echo "CCACHE_DIR=/host/${{ github.workspace }}/ccache" >> $GITHUB_ENV
+          fi
+
+      - name: Restore compiler cache
+        id: restore-compiler-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ github.workspace }}/ccache
+          key: ccache-wheels-${{inputs.platform}}-${{inputs.python_version}}-${{github.ref_name}}
+          restore-keys: |
+            ccache-wheels-${{inputs.platform}}-${{inputs.python_version}}-master
+
       - name: Install readline
         if: runner.os == 'macOS'
         run: |
@@ -105,14 +134,26 @@ jobs:
 
       - name: Build wheel
         run: |
+          # for debugging ccache hits
+          ccache -z
+          ccache -svv
           bash packaging/python/build_wheels.bash $(uname -s) ${{ inputs.python_version }}
+          ccache -svv
 
       - name: Upload wheel files
         uses: actions/upload-artifact@v4
         with:
           name: wheels-${{ inputs.python_version }}-${{ inputs.platform }}
           path: wheelhouse/*.whl
 
+      - name: Save compiler cache
+        id: save-compiler-cache
+        if: always()
+        uses: actions/cache/save@v4
+        with:
+          key: ${{ steps.restore-compiler-cache.outputs.cache-primary-key }}
+          path: ${{ github.workspace }}/ccache
+
       - name: Setup Python ${{ inputs.python_version }} for testing
         uses: actions/setup-python@v5
         with:

azure-pipelines.yml

@@ -67,12 +67,23 @@ stages:
         condition: and(in(variables['Build.Reason'], 'Manual'), eq(variables['NRN_RELEASE_UPLOAD'], 'true'))
         displayName: 'Change name of package for release'
 
+      # Restore ccache
+      - task: Cache@2
+        displayName: 'Restore ccache'
+        inputs:
+          key: 'ccache | "$(Agent.OS)" | "$(python.nodot)" | "$(Build.SourceVersion)"'
+          restoreKeys: |
+            ccache | "$(Agent.OS)" | "$(python.nodot)"
+            ccache | "$(Agent.OS)"
+          path: '$(Pipeline.Workspace)/.ccache'
+
       # Note that mpt headers must be mounted in the docker imager under `/nrnwheel/mpt`
       # This path is checked by `packaging/python/build_wheels.bash` when run in the image.
       - script: |
           if [ "$BUILD_REASON" = "Manual" ] || [ "$BUILD_REASON" = "Schedule" ]; then
             export NRN_RX3D_OPT_LEVEL=1
           fi
+          export CCACHE_DIR=$(Pipeline.Workspace)/.ccache
           sudo mkdir -p /opt/nrnwheel/mpt
           sudo tar -zxf $(mpt_headersSF.secureFilePath) --directory /opt/nrnwheel/mpt
           packaging/python/build_wheels.bash linux $(python.nodot)
@@ -130,7 +141,7 @@ stages:
 
       - script: |
           brew install --cask xquartz
-          brew install flex bison mpich
+          brew install flex bison mpich ccache
           brew unlink mpich && brew install openmpi
           cmake --version
           # the version of bash used by Azure (which is the default one
@@ -164,12 +175,23 @@ stages:
         condition: and(in(variables['Build.Reason'], 'Manual'), eq(variables['NRN_RELEASE_UPLOAD'], 'true'))
         displayName: 'Change name of package for release'
 
+      # Restore ccache
+      - task: Cache@2
+        displayName: 'Restore ccache'
+        inputs:
+          key: 'ccache | "$(Agent.OS)" | "$(python.nodot)" | "$(Build.SourceVersion)"'
+          restoreKeys: |
+            ccache | "$(Agent.OS)" | "$(python.nodot)"
+            ccache | "$(Agent.OS)"
+          path: '$(Pipeline.Workspace)/.ccache'
+
       - script: |
           export PATH=/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH
           export SDKROOT=$(xcrun --sdk macosx --show-sdk-path)
           if [ "$BUILD_REASON" = "Manual" ] || [ "$BUILD_REASON" = "Schedule" ]; then
             export NRN_RX3D_OPT_LEVEL=1
           fi
+          export CCACHE_DIR=$(Pipeline.Workspace)/.ccache
           sudo mkdir -p /opt/nrnwheel/$(uname -m)
           sudo tar -zxf $(readlineSF.secureFilePath) --directory /opt/nrnwheel/$(uname -m)
           packaging/python/build_wheels.bash osx $(python.nodot)

packaging/python/Dockerfile

@@ -83,6 +83,17 @@ RUN curl -L -o Python-3.10.0.tar.gz https://www.python.org/ftp/python/3.10.0/Pyt
 RUN yum -y install epel-release libX11-devel libXcomposite-devel vim-enhanced && yum -y clean all && rm -rf /var/cache
 RUN yum -y remove ncurses-devel
 
+# install ccache for speeding-up builds
+RUN export ccache_version='4.11.3' \
+    && ccache_srcdir="ccache-${ccache_version}" \
+    && ccache_builddir="ccache-build" \
+    && wget "https://github.com/ccache/ccache/releases/download/v${ccache_version}/ccache-${ccache_version}.tar.gz" \
+    && tar xf "ccache-${ccache_version}.tar.gz" \
+    && cmake -B "${ccache_builddir}" -S "${ccache_srcdir}" \
+    && cmake --build "${ccache_builddir}" --parallel \
+    && cmake --install "${ccache_builddir}" \
+    && rm -fr "ccache-${ccache_version}.tar.gz" "${ccache_srcdir}" "${ccache_builddir}"
+
 # build wheels from there
 WORKDIR /root
 

packaging/python/build_wheels.bash

@@ -99,6 +99,9 @@ set_cibw_environment() {
             # seems that 10.15 is actually needed for std::filesystem::path.
             # 11.0 is required on ARM machines
             [MACOSX_DEPLOYMENT_TARGET]="10.15"
+            [CMAKE_C_COMPILER_LAUNCHER]=""
+            [CMAKE_CXX_COMPILER_LAUNCHER]=""
+            [CCACHE_DIR]=""
         )
     elif [ "${platform}" = 'linux' ]; then
         declare -A defaults=(
@@ -110,6 +113,9 @@ set_cibw_environment() {
             [CORENRN_ENABLE_OPENMP]="ON"
             [NRN_BINARY_DIST_BUILD]="ON"
             [NRN_RX3D_OPT_LEVEL]="0"
+            [CMAKE_C_COMPILER_LAUNCHER]=""
+            [CMAKE_CXX_COMPILER_LAUNCHER]=""
+            [CCACHE_DIR]=""
         )
     fi
 
@@ -152,9 +158,31 @@ build_wheel_portable() {
         fi
     fi
 
+    # use ccache if it is available (it's always available for Linux builds)
+    if [ "${platform}" = 'linux' ] || ([ "${platform}" = 'macos' ] && command -v ccache >& /dev/null); then
+        export CCACHE_DEBUG=1
+        export CMAKE_C_COMPILER_LAUNCHER=ccache
+        export CMAKE_CXX_COMPILER_LAUNCHER=ccache
+        if [ "${platform}" = 'linux' ]; then
+            # the host filesystem is available in the container at `/host`
+            export CCACHE_DIR="${CCACHE_DIR:-/host/tmp/ccache}"
+            export CCACHE_NODIRECT=
+        elif [ "${platform}" = 'macos' ]; then
+            export CCACHE_DIR="${CCACHE_DIR:-/tmp/ccache}"
+        fi
+        CCACHE_STATS_COMMAND="ccache -svv"
+        CCACHE_ZERO_COMMAND="ccache -z"
+    else
+        CCACHE_STATS_COMMAND=":"
+        CCACHE_ZERO_COMMAND=":"
+    fi
+
     set_cibw_environment "${platform}"
 
-    CIBW_BUILD_VERBOSITY=1 python -m cibuildwheel --debug-traceback --platform "${platform}" --output-dir wheelhouse
+    export CIBW_TEST_COMMAND="${CCACHE_STATS_COMMAND}"
+    export CIBW_BEFORE_BUILD="${CCACHE_ZERO_COMMAND} && ${CCACHE_STATS_COMMAND}"
+    export CIBW_BUILD_VERBOSITY=1
+    python -m cibuildwheel --debug-traceback --platform "${platform}" --output-dir wheelhouse
 
     deactivate
 }

pyproject.toml

@@ -122,10 +122,17 @@ NRN_ENABLE_MPI_DYNAMIC = { default = "ON", env = "NRN_ENABLE_MPI_DYNAMIC" }
 # MPI headers are platform-specific, so we do not set defaults
 NRN_MPI_DYNAMIC = { env = "NRN_MPI_DYNAMIC" }
 
+# in case we want to speed-up builds
+CMAKE_C_COMPILER_LAUNCHER = { env = "CMAKE_C_COMPILER_LAUNCHER" }
+CMAKE_CXX_COMPILER_LAUNCHER = { env = "CMAKE_CXX_COMPILER_LAUNCHER" }
+
+# we do not need NMODL line numbers
+NMODL_ENABLE_FLEX_BISON_LINES = "OFF"
+
 [tool.cibuildwheel]
 # We can specify a custom Docker image when building NEURON
-manylinux-aarch64-image = "docker.io/neuronsimulator/neuron_wheel:manylinux_2_28_aarch64"
-manylinux-x86_64-image = "docker.io/neuronsimulator/neuron_wheel:manylinux_2_28_x86_64"
+manylinux-aarch64-image = "docker.io/neuronsimulator/neuron_wheel:manylinux_2_28_aarch64_ccache"
+manylinux-x86_64-image = "docker.io/neuronsimulator/neuron_wheel:manylinux_2_28_x86_64_ccache"
 
 # certain platforms are not supported
 skip = [ "pp*", "*-win32", "*-manylinux_i686", "*-musllinux_i686", "*-musllinux_x86_64", "*-musllinux_aarch64" ]
@@ -143,10 +150,15 @@ environment-pass = [
   "CORENRN_ENABLE_OPENMP",
   "NRN_BINARY_DIST_BUILD",
   "NRN_RX3D_OPT_LEVEL",
+  "CCACHE_DEBUG",
+  "CCACHE_NODIRECT",
   # when making a release, we need to override the version
   "SETUPTOOLS_SCM_PRETEND_VERSION",
   # in case the user wants to use less resources
   "CMAKE_BUILD_PARALLEL_LEVEL",
+  "CMAKE_C_COMPILER_LAUNCHER",
+  "CMAKE_CXX_COMPILER_LAUNCHER",
+  "CCACHE_DIR",
 ]
 
 # comment originally from `build_wheels.bash`:
@@ -157,7 +169,7 @@ environment-pass = [
 #   libnrniv: we ship precompiled version of neurondemo containing libnrnmech.so
 #             which is linked to libnrniv.so. auditwheel manipulate rpaths and
 #             ships an extra copy of libnrniv.so and hence exclude it here.
-repair-wheel-command = "auditwheel -v repair -w {dest_dir} {wheel} --exclude 'libgomp.so.1' --exclude 'libnrniv.so'"
+repair-wheel-command = "auditwheel -v repair -w {dest_dir} {wheel} --exclude 'libgomp.so.1' --exclude 'libnrniv.so' >& /dev/null"
 
 [tool.cibuildwheel.macos]
 repair-wheel-command = "delocate-listdeps {wheel} && delocate-wheel -v -w {dest_dir} {wheel}"