CI refactor

.github/workflows/ci.yaml

@@ -33,96 +33,47 @@ jobs:
       # https://stackoverflow.com/a/41673702
       CXXFLAGS: >-
         -Wall -Wextra -Wwrite-strings -Wunreachable-code -Wpointer-arith -Wredundant-decls -Wno-unknown-warning-option
-      CLANG_TIDY_ARGS: --fix --fix-errors --format-style=file
-      DOCKER_IMAGE: moveit/moveit2:${{ matrix.env.IMAGE }}
-      UPSTREAM_WORKSPACE: >
-        moveit2.repos
-        $(f="moveit2_$(sed 's/-.*$//' <<< "${{ matrix.env.IMAGE }}").repos"; test -r $f && echo $f)
-      # Pull any updates to the upstream workspace (after restoring it from cache)
-      AFTER_SETUP_UPSTREAM_WORKSPACE: vcs pull $BASEDIR/upstream_ws/src
-      AFTER_SETUP_DOWNSTREAM_WORKSPACE: vcs pull $BASEDIR/downstream_ws/src
-      # Clear the ccache stats before and log the stats after the build
-      AFTER_SETUP_CCACHE: ccache --zero-stats --max-size=10.0G
-      BEFORE_BUILD_UPSTREAM_WORKSPACE: ccache -z
-      AFTER_BUILD_TARGET_WORKSPACE: ccache -s
       # Changing linker to lld as ld has a behavior where it takes a long time to finish
+      CMAKE_EXE_LINKER_FLAGS: -fuse-ld=lld
+      CMAKE_SHARED_LINKER_FLAGS: -fuse-ld=lld
+      CMAKE_MODULE_LINKER_FLAGS: -fuse-ld=lld
       # Compile CCOV with Debug. Enable -Werror.
-      TARGET_CMAKE_ARGS: >
-        -DCMAKE_EXE_LINKER_FLAGS=-fuse-ld=lld
-        -DCMAKE_SHARED_LINKER_FLAGS=-fuse-ld=lld
-        -DCMAKE_MODULE_LINKER_FLAGS=-fuse-ld=lld
-        -DCMAKE_BUILD_TYPE=${{ matrix.env.CCOV && 'Debug' || 'Release'}}
-        -DCMAKE_CXX_FLAGS="-Werror $CXXFLAGS${{ matrix.env.CCOV && ' --coverage -O2 -fno-omit-frame-pointer' || ''}}"
-      UPSTREAM_CMAKE_ARGS: "-DCMAKE_CXX_FLAGS=''"
-      DOWNSTREAM_CMAKE_ARGS: -DCMAKE_CXX_FLAGS="-Wall -Wextra"
+      CMAKE_BUILD_TYPE: ${{ matrix.env.CCOV && 'Debug' || 'Release'}}
+      CMAKE_CXX_FLAGS: "-Werror $CXXFLAGS${{ matrix.env.CCOV && ' --coverage -O2 -fno-omit-frame-pointer' || ''}}"
+      CMAKE_CXX_COMPILER_LAUNCHER: ccache
       CCACHE_DIR: ${{ github.workspace }}/.ccache
       BASEDIR: ${{ github.workspace }}/.work
       CLANG_TIDY_BASE_REF: ${{ github.event_name != 'workflow_dispatch' && (github.base_ref || github.ref) || '' }}
-      BEFORE_CLANG_TIDY_CHECKS: |
-        # Show list of applied checks
-        (cd $TARGET_REPO_PATH; clang-tidy --list-checks)
-        # Disable clang-tidy for ikfast plugins as we cannot fix the generated code
-        find $BASEDIR/target_ws/build -iwholename "*_ikfast_plugin/compile_commands.json" -exec rm {} \;
-        find $BASEDIR/target_ws/build -iwholename "*_ikfast_manipulator_plugin/compile_commands.json" -exec rm {} \;
       CC: ${{ matrix.env.CLANG_TIDY && 'clang' }}
       CXX: ${{ matrix.env.CLANG_TIDY && 'clang++' }}
       ADDITIONAL_DEBS: lld
+      SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 
     name: ${{ matrix.env.IMAGE }}${{ matrix.env.CCOV && ' + ccov' || ''}}${{ matrix.env.IKFAST_TEST && ' + ikfast' || ''}}${{ matrix.env.CLANG_TIDY && (github.event_name != 'workflow_dispatch' && ' + clang-tidy (delta)' || ' + clang-tidy (all)') || '' }}
     runs-on: ubuntu-latest
+    container: moveit/moveit2:${{ matrix.env.IMAGE }}
+    defaults:
+      run:
+        shell: bash
     steps:
-      - name: "Free up disk space"
-        if: matrix.env.CCOV
-        run: |
-          sudo apt-get -qq purge "ghc*"
-          sudo apt-get clean
-          # cleanup docker images not used by us
-          docker system prune -af
-          # shift ccache folder to /mnt on a separate disk
-          sudo mkdir /mnt/ccache
-          mkdir ${{ env.CCACHE_DIR }}
-          sudo mount --bind ${{ env.CCACHE_DIR }} /mnt/ccache
-          # free up a lot of stuff from /usr/local
-          sudo rm -rf /usr/local
-          df -h
       - uses: actions/checkout@v6
-# NOTE: Testspace is temporarily disabled and needs to be installed for the MoveIt org
-# See: https://github.com/moveit/moveit2/issues/2852
-#       - uses: testspace-com/setup-testspace@v1
-#         if: github.repository == 'moveit/moveit2'
-#         with:
-#           domain: moveit
+        with:
+          # checkout into a "src" directory so we look like a ROS workspace
+          path: src
       - name: Get latest release date for rosdistro
         id: rosdistro_release_date
         uses: JafarAbdi/latest-rosdistro-release-date-action@main
         with:
           rosdistro: ${{ matrix.env.ROS_DISTRO }}
-      - name: Get latest timestamp repos file has been edited
-        id: repos_edit_timestamp
-        uses: vatanaksoytezer/latest-file-edit-timestamp-action@main
-        with:
-          file: moveit2.repos
-      - name: Cache upstream workspace
-        uses: rhaschke/cache@main
-        with:
-          path: ${{ env.BASEDIR }}/upstream_ws
-          key: ${{ env.CACHE_PREFIX }}-${{ github.run_id }}
-          restore-keys: ${{ env.CACHE_PREFIX }}
-        env:
-          GHA_CACHE_SAVE: always
-          CACHE_PREFIX: ${{ steps.rosdistro_release_date.outputs.date }}-upstream_ws-${{ steps.repos_edit_timestamp.outputs.timestamp }}-${{ matrix.env.IMAGE }}-${{ hashFiles('moveit2*.repos', '.github/workflows/ci.yaml') }}
-      # The target directory cache doesn't include the source directory because
-      # that comes from the checkout.  See "prepare target_ws for cache" task below
-      - name: Cache target workspace
-        if: "!matrix.env.CCOV"
+      - name: Cache build
         uses: rhaschke/cache@main
         with:
-          path: ${{ env.BASEDIR }}/target_ws
+          path: ${{ env.BASEDIR }}/build
           key: ${{ env.CACHE_PREFIX }}-${{ github.run_id }}
           restore-keys: ${{ env.CACHE_PREFIX }}
         env:
           GHA_CACHE_SAVE: always
-          CACHE_PREFIX: target_ws${{ matrix.env.CCOV && '-ccov' || '' }}-${{ matrix.env.IMAGE }}-${{ hashFiles('**/CMakeLists.txt', '**/package.xml', '.github/workflows/ci.yaml') }}
+          CACHE_PREFIX: build${{ matrix.env.CCOV && '-ccov' || '' }}-${{ matrix.env.IMAGE }}-${{ hashFiles('**/CMakeLists.txt', '**/package.xml', '.github/workflows/ci.yaml') }}
       - name: Cache ccache
         uses: rhaschke/cache@main
         with:
@@ -134,53 +85,55 @@ jobs:
         env:
           GHA_CACHE_SAVE: always
           CACHE_PREFIX: ccache-${{ matrix.env.IMAGE }}${{ matrix.env.CCOV && '-ccov' || '' }}
-      - name: Configure ccache
-        run: |
-          mkdir -p ${{ env.CCACHE_DIR }}
-          cp .github/ccache.conf ${{ env.CCACHE_DIR }}/ccache.conf
+      - name: Configure ccache and rezero its stats
+        run: ccache --zero-stats --max-size 10.0G
+      - name: vcs import
+        run: vcs import < moveit2.repos
+        working-directory: src
       - name: Generate ikfast packages
         if: matrix.env.IKFAST_TEST
         run: moveit_kinematics/test/test_ikfast_plugins.sh
-      - id: ici
-        name: Run industrial_ci
-        uses: ros-industrial/industrial_ci@master
-        env: ${{ matrix.env }}
-# NOTE: Testspace is temporarily disabled and needs to be installed for the MoveIt org
-# See: https://github.com/moveit/moveit2/issues/2852
-#      - name: Push result to Testspace
-#        if: always() && (github.repository == 'moveit/moveit2')
-#        run: |
-#          testspace "[ ${{ matrix.env.IMAGE }} ]${{ env.BASEDIR }}/target_ws/**/test_results/**/*.xml"
-      - name: Upload test artifacts (on failure)
+        working-directory: src
+      - name: Build workspace
+        run: source /opt/ros/${{ matrix.env.ROS_DISTRO }}/setup.bash && colcon build
+      - name: Test workspace
+        run: source /opt/ros/${{ matrix.env.ROS_DISTRO }}/setup.bash && colcon test --return-code-on-test-failure
+      - name: Show failed tests
+        if: failure()
+        run: colcon test-result
+      - name: clang-tidy
+        if: matrix.env.CLANG_TIDY
+        run: run-clang-tidy -p build/ src
+      - name: Upload test artifacts
         uses: actions/upload-artifact@v5
-        if: failure() && (steps.ici.outputs.run_target_test || steps.ici.outputs.target_test_results)
+        if: always()
         with:
           name: test-results-${{ matrix.env.IMAGE }}
           path: ${{ env.BASEDIR }}/target_ws/**/test_results/**/*.xml
           overwrite: true
       - name: Generate codecov report
         uses: rhaschke/lcov-action@main
-        if: always() && matrix.env.CCOV && steps.ici.outputs.target_test_results == '0'
+        if: always() && matrix.env.CCOV
         with:
-          docker: $DOCKER_IMAGE
           workdir: ${{ env.BASEDIR }}/target_ws
           lcov_capture_args: --ignore-errors=gcov,gcov,mismatch,mismatch,negative,negative
-          ignore: '"*/target_ws/build/*" "*/target_ws/install/*" "*/test/*"'
+          ignore: '"build/*" "install/*" "*/test/*"'
       - name: Upload codecov report
         uses: codecov/codecov-action@v5
-        if: always() && matrix.env.CCOV && steps.ici.outputs.target_test_results == '0'
+        if: always() && matrix.env.CCOV
         with:
-          files: ${{ env.BASEDIR }}/target_ws/coverage.info
-      - name: Upload clang-tidy changes
-        uses: rhaschke/upload-git-patch-action@main
-        if: always() && matrix.env.CLANG_TIDY
+          files: ${{ env.BASEDIR }}/coverage.info
+      - name: SonarQube Scan
+        # Don't run if SONAR_TOKEN is not set (i.e. PR job from a fork)
+        if: matrix.env.CCOV && env.SONAR_TOKEN != ''
+        uses: SonarSource/[email protected]
         with:
-          name: clang-tidy
-          path: ${{ env.BASEDIR }}/target_ws/src/$(basename $(pwd))
-      - name: Prepare target_ws for cache
-        if: always() && !matrix.env.CCOV
-        run: |
-          du -sh ${{ env.BASEDIR }}/target_ws
-          sudo find ${{ env.BASEDIR }}/target_ws -wholename '*/test_results/*' -delete
-          sudo rm -rf ${{ env.BASEDIR }}/target_ws/src
-          du -sh ${{ env.BASEDIR }}/target_ws
+          projectBaseDir: ${{ env.BASEDIR }}/target_ws
+  docker:
+    uses: ./.github/workflows/docker.yaml
+  docker_lint:
+    uses: ./.github/workflows/docker_lint.yaml
+  format:
+    uses: ./.github/workflows/format.yaml
+  tutorial_docker:
+    uses: ./.github/workflows/tutorial_docker.yaml

.github/workflows/docker.yaml

@@ -3,16 +3,9 @@ name: docker
 on:
   schedule:
     # 5 PM UTC every Sunday
-    - cron:  '0 17 * * 6'
+    - cron: "0 17 * * 6"
   workflow_dispatch:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths:
-      - .docker/**
-      - .github/workflows/docker.yaml
-      - moveit2.repos
+  workflow_call:
 
 jobs:
   release:
@@ -156,7 +149,7 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: "Remove .dockerignore"
-        run: rm .dockerignore  # enforce full source context
+        run: rm .dockerignore # enforce full source context
       - name: Build and Push
         uses: docker/build-push-action@v6
         with:

.github/workflows/docker_lint.yaml

@@ -2,14 +2,7 @@ name: Docker Lint
 
 on:
   workflow_dispatch:
-  push:
-    paths:
-      - .docker/**
-      - .github/workflows/docker_lint.yaml
-  pull_request:
-    paths:
-      - .docker/**
-      - .github/workflows/docker_lint.yaml
+  workflow_call:
 
 jobs:
   docker-lint:

.github/workflows/format.yaml

@@ -5,27 +5,23 @@ name: Formatting (pre-commit)
 
 on:
   workflow_dispatch:
-  pull_request:
-  merge_group:
-  push:
-    branches:
-      - main
+  workflow_call:
 
 jobs:
   pre-commit:
     name: Format
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v6
-    - uses: actions/setup-python@v6
-      with:
-        python-version: 3.x
-    - name: Install clang-format-14
-      run: sudo apt-get install clang-format-14
-    - uses: pre-commit/[email protected]
-      id: precommit
-    - name: Upload pre-commit changes
-      if: failure() && steps.precommit.outcome == 'failure'
-      uses: rhaschke/upload-git-patch-action@main
-      with:
-        name: pre-commit
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: 3.x
+      - name: Install clang-format-14
+        run: sudo apt-get install clang-format-14
+      - uses: pre-commit/[email protected]
+        id: precommit
+      - name: Upload pre-commit changes
+        if: failure() && steps.precommit.outcome == 'failure'
+        uses: rhaschke/upload-git-patch-action@main
+        with:
+          name: pre-commit