From d7c4d47f01fc88ac09de776d7162f673f64089db Mon Sep 17 00:00:00 2001 From: per1234 Date: Fri, 14 Oct 2022 06:03:12 -0700 Subject: [PATCH] Use workflow job matrix for builds Previously, a single GitHub Actions workflow job was used to make tester, nighly, and release builds of the application. These builds were done serially. A more efficient approach is to run the builds in parallel. This is accomplished by using a job matrix, where each target has its own generated job in the workflow. Previously, checksum generation code was present both in the taskfile and in the workflows (due to the need to update the macOS checksum after the notarization process done in the workflow). The checksum generation is now done exclusively in the build workflows. --- .github/workflows/publish-go-nightly-task.yml | 39 +++--- .github/workflows/publish-go-tester-task.yml | 124 +++++++++++------- .github/workflows/release-go-task.yml | 41 +++--- DistTasks.yml | 23 ---- 4 files changed, 117 insertions(+), 110 deletions(-) diff --git a/.github/workflows/publish-go-nightly-task.yml b/.github/workflows/publish-go-nightly-task.yml index 4c7808b2..db94b4a6 100644 --- a/.github/workflows/publish-go-nightly-task.yml +++ b/.github/workflows/publish-go-nightly-task.yml @@ -22,6 +22,19 @@ jobs: create-nightly-artifacts: runs-on: ubuntu-latest + strategy: + matrix: + os: + - Windows_32bit + - Windows_64bit + - Linux_32bit + - Linux_64bit + - Linux_ARMv6 + - Linux_ARMv7 + - Linux_ARM64 + - macOS_64bit + - macOS_ARM64 + steps: - name: Checkout repository uses: actions/checkout@v3 @@ -35,7 +48,7 @@ jobs: - name: Build env: NIGHTLY: true - run: task dist:all + run: task dist:${{ matrix.os }} - name: Upload artifacts uses: actions/upload-artifact@v3 @@ -128,14 +141,10 @@ jobs: run: | gon "${{ env.GON_CONFIG_PATH }}" - - name: Re-package binary and output checksum + - name: Re-package binary id: re-package working-directory: ${{ env.DIST_DIR }} - # This step performs the following: - # 1. Repackage the signed binary replaced in place by Gon (ignoring the output zip file) - # 2. Recalculate package checksum - # 3. Output the new checksum to include in the nnnnnn-checksums.txt file - # (it cannot be done there because of workflow job parallelization) + # Repackage the signed binary replaced in place by Gon (ignoring the output zip file) run: | # GitHub's upload/download-artifact actions don't preserve file permissions, # so we need to add execution permission back until the action is made to do this. @@ -145,11 +154,9 @@ jobs: tar -czvf "$PACKAGE_FILENAME" \ -C "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/" "${{ env.PROJECT_NAME }}" \ -C ../../ LICENSE.txt - CHECKSUM_LINE="$(shasum -a 256 $PACKAGE_FILENAME)" echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV - echo "::set-output name=checksum-${{ matrix.artifact.name }}::$CHECKSUM_LINE" - - name: Upload artifacts + - name: Upload artifact uses: actions/upload-artifact@v3 with: if-no-files-found: error @@ -167,15 +174,11 @@ jobs: name: ${{ env.ARTIFACT_NAME }} path: ${{ env.DIST_DIR }} - - name: Update checksum + - name: Create checksum file + working-directory: ${{ env.DIST_DIR}} run: | - declare -a checksum_lines=("${{ needs.notarize-macos.outputs.checksum-darwin_amd64 }}" "${{ needs.notarize-macos.outputs.checksum-darwin_arm64 }}") - for checksum_line in "${checksum_lines[@]}" - do - CHECKSUM=$(echo ${checksum_line} | cut -d " " -f 1) - PACKAGE_FILENAME=$(echo ${checksum_line} | cut -d " " -f 2) - perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM} ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt - done + TAG="nightly-$(date -u +"%Y%m%d")" + sha256sum ${{ env.PROJECT_NAME }}_${TAG}* > ${TAG}-checksums.txt - name: Upload release files on Arduino downloads servers uses: docker://plugins/s3 diff --git a/.github/workflows/publish-go-tester-task.yml b/.github/workflows/publish-go-tester-task.yml index 80e72b32..b97a385c 100644 --- a/.github/workflows/publish-go-tester-task.yml +++ b/.github/workflows/publish-go-tester-task.yml @@ -24,9 +24,10 @@ on: repository_dispatch: env: + # As defined by the Taskfile's PROJECT_NAME variable + PROJECT_NAME: arduino-lint # As defined by the Taskfile's DIST_DIR variable DIST_DIR: dist - BUILDS_ARTIFACT: build-artifacts jobs: run-determination: @@ -52,86 +53,107 @@ jobs: echo "::set-output name=result::$RESULT" - build: + package-name-prefix: needs: run-determination if: needs.run-determination.outputs.result == 'true' runs-on: ubuntu-latest - + outputs: + prefix: ${{ steps.calculation.outputs.prefix }} steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Install Task - uses: arduino/setup-task@v1 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - version: 3.x - - - name: Build + - name: package name prefix calculation + id: calculation run: | PACKAGE_NAME_PREFIX="test" if [ "${{ github.event_name }}" = "pull_request" ]; then PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.event.number }}" fi PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.sha }}-" - export PACKAGE_NAME_PREFIX - task dist:all - # Transfer builds to artifacts job - - name: Upload combined builds artifact - uses: actions/upload-artifact@v3 - with: - path: ${{ env.DIST_DIR }} - name: ${{ env.BUILDS_ARTIFACT }} + echo "::set-output name=prefix::$PACKAGE_NAME_PREFIX" - artifacts: - name: ${{ matrix.artifact.name }} artifact - needs: build + build: + needs: package-name-prefix + name: Build ${{ matrix.os.name }} runs-on: ubuntu-latest strategy: matrix: - artifact: - - path: "*checksums.txt" - name: checksums - - path: "*Linux_32bit.tar.gz" + os: + - task: Windows_32bit + path: "*Windows_32bit.zip" + name: Windows_X86-32 + - task: Windows_64bit + path: "*Windows_64bit.zip" + name: Windows_X86-64 + - task: Linux_32bit + path: "*Linux_32bit.tar.gz" name: Linux_X86-32 - - path: "*Linux_64bit.tar.gz" + - task: Linux_64bit + path: "*Linux_64bit.tar.gz" name: Linux_X86-64 - - path: "*Linux_ARM64.tar.gz" - name: Linux_ARM64 - - path: "*Linux_ARMv6.tar.gz" + - task: Linux_ARMv6 + path: "*Linux_ARMv6.tar.gz" name: Linux_ARMv6 - - path: "*Linux_ARMv7.tar.gz" + - task: Linux_ARMv7 + path: "*Linux_ARMv7.tar.gz" name: Linux_ARMv7 - - path: "*macOS_64bit.tar.gz" + - task: Linux_ARM64 + path: "*Linux_ARM64.tar.gz" + name: Linux_ARM64 + - task: macOS_64bit + path: "*macOS_64bit.tar.gz" name: macOS_64 - - path: "*macOS_ARM64.tar.gz" + - task: macOS_ARM64 + path: "*macOS_ARM64.tar.gz" name: macOS_ARM64 - - path: "*Windows_32bit.zip" - name: Windows_X86-32 - - path: "*Windows_64bit.zip" - name: Windows_X86-64 steps: - - name: Download combined builds artifact - uses: actions/download-artifact@v3 + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install Task + uses: arduino/setup-task@v1 with: - name: ${{ env.BUILDS_ARTIFACT }} - path: ${{ env.BUILDS_ARTIFACT }} + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: 3.x - - name: Upload individual build artifact + - name: Build + run: | + PACKAGE_NAME_PREFIX=${{ needs.package-name-prefix.outputs.prefix }} + export PACKAGE_NAME_PREFIX + task dist:${{ matrix.os.task }} + + # Transfer builds to artifacts job + - name: Upload build artifact uses: actions/upload-artifact@v3 with: - path: ${{ env.BUILDS_ARTIFACT }}/${{ matrix.artifact.path }} - name: ${{ matrix.artifact.name }} + path: ${{ env.DIST_DIR }}/${{ matrix.os.path }} + name: ${{ matrix.os.name }} - clean: - needs: artifacts + checksums: + needs: + - build + - package-name-prefix runs-on: ubuntu-latest steps: - - name: Remove unneeded combined builds artifact - uses: geekyeggo/delete-artifact@v2 + - name: Download build artifacts + uses: actions/download-artifact@v3 + + - name: Create checksum file + run: | + TAG="${{ needs.package-name-prefix.outputs.prefix }}git-snapshot" + declare -a artifacts=($(ls -d */)) + for artifact in ${artifacts[@]} + do + cd $artifact + checksum=$(sha256sum ${{ env.PROJECT_NAME }}_${TAG}*) + cd .. + echo $checksum >> ${TAG}-checksums.txt + done + + - name: Upload checksum artifact + uses: actions/upload-artifact@v3 with: - name: ${{ env.BUILDS_ARTIFACT }} + path: ./*checksums.txt + name: checksums diff --git a/.github/workflows/release-go-task.yml b/.github/workflows/release-go-task.yml index 5affbf47..9f01394c 100644 --- a/.github/workflows/release-go-task.yml +++ b/.github/workflows/release-go-task.yml @@ -19,6 +19,19 @@ jobs: create-release-artifacts: runs-on: ubuntu-latest + strategy: + matrix: + os: + - Windows_32bit + - Windows_64bit + - Linux_32bit + - Linux_64bit + - Linux_ARMv6 + - Linux_ARMv7 + - Linux_ARM64 + - macOS_64bit + - macOS_ARM64 + steps: - name: Checkout repository uses: actions/checkout@v3 @@ -26,6 +39,8 @@ jobs: fetch-depth: 0 - name: Create changelog + # Avoid creating the same changelog for each os + if: matrix.os == 'Windows_32bit' uses: arduino/create-changelog@v1 with: tag-regex: '^[0-9]+\.[0-9]+\.[0-9]+.*$' @@ -40,7 +55,7 @@ jobs: version: 3.x - name: Build - run: task dist:all + run: task dist:${{ matrix.os }} - name: Upload artifacts uses: actions/upload-artifact@v3 @@ -132,14 +147,10 @@ jobs: run: | gon "${{ env.GON_CONFIG_PATH }}" - - name: Re-package binary and output checksum + - name: Re-package binary id: re-package working-directory: ${{ env.DIST_DIR }} - # This step performs the following: - # 1. Repackage the signed binary replaced in place by Gon (ignoring the output zip file) - # 2. Recalculate package checksum - # 3. Output the new checksum to include in the nnnnnn-checksums.txt file - # (it cannot be done there because of workflow job parallelization) + # Repackage the signed binary replaced in place by Gon (ignoring the output zip file) run: | # GitHub's upload/download-artifact actions don't preserve file permissions, # so we need to add execution permission back until the action is made to do this. @@ -149,11 +160,9 @@ jobs: tar -czvf "$PACKAGE_FILENAME" \ -C "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/" "${{ env.PROJECT_NAME }}" \ -C ../../ LICENSE.txt - CHECKSUM_LINE="$(shasum -a 256 $PACKAGE_FILENAME)" echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV - echo "::set-output name=checksum-${{ matrix.artifact.name }}::$CHECKSUM_LINE" - - name: Upload artifacts + - name: Upload artifact uses: actions/upload-artifact@v3 with: if-no-files-found: error @@ -171,15 +180,11 @@ jobs: name: ${{ env.ARTIFACT_NAME }} path: ${{ env.DIST_DIR }} - - name: Update checksum + - name: Create checksum file + working-directory: ${{ env.DIST_DIR}} run: | - declare -a checksum_lines=("${{ needs.notarize-macos.outputs.checksum-darwin_amd64 }}" "${{ needs.notarize-macos.outputs.checksum-darwin_arm64 }}") - for checksum_line in "${checksum_lines[@]}" - do - CHECKSUM=$(echo ${checksum_line} | cut -d " " -f 1) - PACKAGE_FILENAME=$(echo ${checksum_line} | cut -d " " -f 2) - perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM} ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt - done + TAG="${GITHUB_REF/refs\/tags\//}" + sha256sum ${{ env.PROJECT_NAME }}_${TAG}* > ${TAG}-checksums.txt - name: Identify Prerelease # This is a workaround while waiting for create-release action diff --git a/DistTasks.yml b/DistTasks.yml index e81131ff..307cdca9 100644 --- a/DistTasks.yml +++ b/DistTasks.yml @@ -20,22 +20,8 @@ version: "3" vars: CONTAINER: "docker.elastic.co/beats-dev/golang-crossbuild" GO_VERSION: "1.17.8" - CHECKSUM_FILE: "{{.VERSION}}-checksums.txt" tasks: - all: - desc: Build for distribution for all platforms - cmds: - - task: Windows_32bit - - task: Windows_64bit - - task: Linux_32bit - - task: Linux_64bit - - task: Linux_ARMv6 - - task: Linux_ARMv7 - - task: Linux_ARM64 - - task: macOS_64bit - - task: macOS_ARM64 - Windows_32bit: desc: Builds Windows 32 bit binaries dir: "{{.DIST_DIR}}" @@ -48,7 +34,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" zip {{.PACKAGE_NAME}} {{.PLATFORM_DIR}}/{{.PROJECT_NAME}}.exe ../LICENSE.txt -j - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_windows_386" @@ -70,7 +55,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" zip {{.PACKAGE_NAME}} {{.PLATFORM_DIR}}/{{.PROJECT_NAME}}.exe ../LICENSE.txt -j - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_windows_amd64" @@ -92,7 +76,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_amd32" @@ -114,7 +97,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_amd64" @@ -136,7 +118,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_arm_7" @@ -158,7 +139,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_arm_6" @@ -208,7 +188,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_arm_64" @@ -230,7 +209,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_osx_darwin_amd64" @@ -265,7 +243,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_osx_darwin_arm64"