CI for benchmarks [WIP]

.github/workflows/benchmarks-cron.yml

@@ -0,0 +1,47 @@
+name: Benchmark cron
+
+on:
+  schedule:
+    - cron: '23 */6 * * * '
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+jobs:
+
+  benchmark_linux:
+    name: HEAD vs HEAD^
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
+      - name: Setup Miniconda
+        uses: conda-incubator/setup-miniconda@v2
+
+      - name: Setup some dependencies
+        shell: bash -l {0}
+        run:
+          pip install asv
+
+      - name: Run benchmarks
+        shell: bash -l {0}
+        run: |
+          set -x
+
+          # ID this runner
+          asv machine --yes
+
+          echo "Baseline:  HEAD^"
+          echo "Contender: HEAD"
+
+          # Run benchmarks for last commit against previous one
+          asv continuous --interleave-processes --split --show-stderr 'HEAD^' 'HEAD'
+
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: asv-benchmark-results
+          path: .asv/results

.github/workflows/benchmarks.yml

@@ -0,0 +1,51 @@
+name: Benchmark PR
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+jobs:
+
+  benchmark_linux:
+    name: Linux
+    runs-on: ubuntu-20.04
+
+    steps:
+      # We need the full repo to avoid this issue
+      # https://github.com/actions/checkout/issues/23
+      - name: Checkout full repo
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Setup Miniconda
+        uses: conda-incubator/setup-miniconda@v2
+
+      - name: Setup some dependencies
+        shell: bash -l {0}
+        run:
+          pip install asv
+
+      - name: Run benchmarks
+        shell: bash -l {0}
+        run: |
+          set -x
+
+          # ID this runner
+          asv machine --yes
+
+          echo "Baseline:  ${{ github.event.pull_request.base.sha }} (${{ github.event.pull_request.base.label }})"
+          echo "Contender: ${GITHUB_SHA} (${{ github.event.pull_request.head.label }})"
+
+          # Run benchmarks for current commit against base
+          # See https://github.com/airspeed-velocity/asv/issues/595 for discussion on interleaved processes
+          asv continuous --split --show-stderr --verbose --interleave-processes ${{ github.event.pull_request.base.sha }} ${GITHUB_SHA} | sed "/Traceback \|failed$\|PERFORMANCE DECREASED/ s/^/::error::/" | tee benchmarks.log
+          if grep "Traceback \|failed\|PERFORMANCE DECREASED" benchmarks.log > /dev/null ; then
+              exit 1
+          fi
+
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: asv-benchmark-results
+          path: .asv/results

.github/workflows/tests.yml → .github/workflows/tests.yml.disabled

@@ -118,7 +118,9 @@ jobs:
             fi
             source tools/github/script.sh
 
-
+      - name: Check benchmarks
+        run: |
+          asv check -E existing
 
   test_skimage_macos:
     name: macos-cp${{ matrix.python-version }}
@@ -190,4 +192,6 @@ jobs:
             fi
             source tools/github/script.sh
 
-
+      - name: Check benchmarks
+        run: |
+          asv check -E existing

asv.conf.json

@@ -18,8 +18,9 @@
     "pythons": ["3.7"],
     "matrix": {
          "cython": [],
-         "numpy": ["1.15", "1.16"],
-         "scipy": []
+         "numpy": ["1.17"],
+         "scipy": [],
+         "pooch": []
     },
 
     "env_dir": ".asv/env",

benchmarks/benchmark_registration.py

@@ -41,10 +41,10 @@ def time_ilk(self):
 
 class PhaseCrossCorrelationRegistration:
     """Benchmarks for registration.phase_cross_correlation in scikit-image"""
-    param_names = ["ndims", "image_size", "upscale_factor"]
+    param_names = ["ndims", "image_size", "upsample_factor"]
     params = [(2, 3), (32, 100), (1, 5, 10)]
 
-    def setup(self, ndims, image_size, upscale_factor, *args):
+    def setup(self, ndims, image_size, upsample_factor, *args):
         if phase_cross_correlation is None:
             raise NotImplementedError("phase_cross_correlation unavailable")
         shifts = (-2.3, 1.7, 5.4, -3.2)[:ndims]
@@ -53,10 +53,10 @@ def setup(self, ndims, image_size, upscale_factor, *args):
         self.reference_image = np.fft.fftn(phantom)
         self.shifted_image = ndi.fourier_shift(self.reference_image, shifts)
 
-    def time_phase_cross_correlation(self, ndims, image_size, upscale_factor):
+    def time_phase_cross_correlation(self, ndims, image_size, upsample_factor):
         result = phase_cross_correlation(self.reference_image,
                                          self.shifted_image,
-                                         upscale_factor,
+                                         upsample_factor=upsample_factor,
                                          space="fourier")
 
     def peakmem_reference(self, *args):
@@ -74,8 +74,8 @@ def peakmem_reference(self, *args):
         pass
 
     def peakmem_phase_cross_correlation(self, ndims, image_size,
-                                        upscale_factor):
+                                        upsample_factor):
         result = phase_cross_correlation(self.reference_image,
                                          self.shifted_image,
-                                         upscale_factor,
+                                         upsample_factor=upsample_factor,
                                          space="fourier")

requirements/test.txt

@@ -5,3 +5,4 @@ pytest-faulthandler
 flake8
 codecov
 pooch>=1.3.0
+asv