Merge pull request #334 from soburi/fix_builtin_led_macro #1097

Workflow file for this run

.github/workflows/package_core.yml at 87421b4

	# Copyright ARDUINO SRL (https://www.arduino.cc)
	# SPDX-License-Identifier: Apache-2.0

	# CI workflow to build, package, test and upload the ArduinoCore-zephyr core.

	name: Package, test and upload core

	on:
	push:
	pull_request:

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}-${{ github.actor }}
	cancel-in-progress: true

	env:
	# use the head SHA for PR hashes, instead of the merge commit
	HEAD_REF: ${{ github.event.pull_request.head.sha \|\| github.ref }}

	jobs:



	# Prepare the Zephyr build environment once for all boards, to speed up the
	# overall process and ensure all builds use the same environment and metadata.
	# Several outputs are made available for later jobs to use.

	build-env:
	name: Prepare build environment
	runs-on: ubuntu-latest
	outputs:
	# User-friendly SemVer-compatible version for the build. Matches the tag
	# name if building a tagged commit, otherwise it is "slightly more" than
	# the latest tag.
	CORE_VER: ${{ env.CORE_VER }}
	# Tag (if found) or hash identifying this core build, should be used for
	# user-friendly strings and file names
	CORE_TAG: ${{ env.CORE_TAG }}
	# Hash identifying this core build, should be used for exact version match
	CORE_HASH: ${{ env.CORE_HASH }}
	# JSON describing all boards to build, collected by get_board_details.sh
	ALL_BOARD_DATA: ${{ env.ALL_BOARD_DATA }}
	# Extended version of ALL_BOARD_DATA with link_mode alternatives added
	ALL_BOARD_FQBNS: ${{ env.ALL_BOARD_FQBNS }}
	# List of artifacts to produce, including the "zephyr" umbrella artifact
	ARTIFACTS: ${{ env.ARTIFACTS }}
	# List of architectures the artifacts will be built for
	SUB_ARCHES: ${{ env.SUB_ARCHES }}
	steps:

	- name: Install OS dependencies
	working-directory: /opt
	run: \|
	sudo apt-get remove --purge man-db -y # skips the mandb triggers
	sudo apt-get update
	sudo apt-get install -y --no-install-recommends git cmake wget python3-pip ninja-build

	- uses: actions/checkout@v4
	with:
	submodules: 'recursive'
	fetch-depth: 0
	persist-credentials: false
	fetch-tags: true # needed for get_core_version.sh

	- name: Initialize Zephyr environment
	run: \|
	# Install Zephyr as a core user would, but faster
	yes \| ./extra/bootstrap.sh -o=--filter=tree:0
	# Set up basic environment for later steps
	echo "CORE_HASH=$(git describe --always $HEAD_REF)" >> "$GITHUB_ENV"
	echo "ALL_BOARD_DATA=$(extra/get_board_details.sh \| jq -c 'sort_by(.variant)')" >> "$GITHUB_ENV"
	echo "CORE_VER=$(extra/get_core_version.sh)" >> "$GITHUB_ENV"

	- name: Map output packages
	# needs the above env vars
	run: \|
	echo "## Building \`$CORE_VER\`" >> "$GITHUB_STEP_SUMMARY"
	echo "CORE_TAG=$(git describe --tags --exact-match 2>/dev/null \|\| echo $CORE_HASH)" >> "$GITHUB_ENV"
	echo "ALL_BOARD_FQBNS=$(jq -c 'map((. + {link_mode: "static"}), (. + {link_mode: "dynamic"}))' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV"
	echo "ARTIFACTS=$(jq -c '["zephyr"] + (map(.artifact) \| unique)' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV"
	echo "SUB_ARCHES=$(jq -c 'map(.subarch) \| unique' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV"
	# Archive the build environment for later jobs
	(cd && tar cphf - .cmake work zephyr-sdk-* \| zstd > build-env.tar.zstd)
	# The API is only needed for packaging, not for building, so it's separate
	tar cphf - cores/arduino/api \| zstd > arduino-api.tar.zstd

	- name: Archive build environment
	uses: actions/upload-artifact@v4
	with:
	name: build-env
	path: ~/build-env.tar.zstd
	retention-days: 1

	- name: Archive API snapshot
	uses: actions/upload-artifact@v4
	with:
	name: arduino-api
	path: arduino-api.tar.zstd
	retention-days: 1



	# Extract the build environment and compile the loader project for each board
	# in parallel. The build logs and artifacts are intermediate results archived
	# for later jobs.

	build-board:
	# NOTE: this name is hardcoded in ci_inspect_logs.py
	name: Build for ${{ matrix.board }}
	runs-on: ubuntu-latest
	needs:
	- build-env
	env:
	# ccache, don't look for the spec file anywhere
	CCACHE_IGNOREOPTIONS: -specs=*
	# build-specific name of the generated artifacts
	ARTIFACT_TAG: ${{ needs.build-env.outputs.CORE_HASH }}-${{ matrix.board }}
	# report file stem for this board
	REPORT: reports/zephyr-${{ matrix.variant }}
	strategy:
	matrix:
	include:
	${{ fromJSON( needs.build-env.outputs.ALL_BOARD_DATA ) }}
	fail-fast: false
	steps:

	- uses: actions/download-artifact@v4
	with:
	path: /home/runner
	name: build-env

	- name: Restore build environment
	run: \|
	sudo apt-get remove --purge man-db -y # skips the mandb triggers
	sudo apt-get update
	sudo apt-get install -y --no-install-recommends git cmake wget python3-pip ninja-build ccache
	(cd ~ && tar --use-compress-program=unzstd -xpf build-env.tar.zstd && rm build-env.tar.zstd)

	- name: ccache
	uses: hendrikmuhs/ccache-action@v1.2
	with:
	verbose: 1
	key: ${{ github.job }}-${{ matrix.board }} # independent caches

	- name: Build loader
	shell: bash
	run: \|
	# create the report directory
	mkdir -p reports
	# run the build, capturing stdout and stderr
	if ! ./extra/build.sh ${{ matrix.board }} 1> >(tee $REPORT.stdout) 2> >(tee $REPORT.stderr) ; then
	# build failed, capture errors in summary and stop job without artifacts
	echo "### :x: ${{ matrix.board }} (\`${{ matrix.variant }}\`) build errors" > $GITHUB_STEP_SUMMARY
	echo >> $GITHUB_STEP_SUMMARY
	echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
	cat $REPORT.stderr >> $GITHUB_STEP_SUMMARY
	echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
	exit 1
	fi

	# look for warnings (errors are a happy path!)
	grep -i "warning:" $REPORT.stdout > $REPORT.warnings \|\| true

	# extract the memory usage table (from the header to the first non-% line)
	# override the size of the Flash with the size of the loader partition
	# and add the size of the sketch partition (not reported by Zephyr)
	LOADER_SIZE=$(( $(cat variants/${{ matrix.variant }}/syms-static.ld \| grep '_loader_max_size' \| cut -d '=' -f 2 \| tr -d ') ;') ))
	SKETCH_SIZE=$(( $(cat variants/${{ matrix.variant }}/syms-static.ld \| grep '_sketch_max_size' \| cut -d '=' -f 2 \| tr -d ') ;') ))
	cat $REPORT.stdout \| sed -n '/^Memory region/,/^[^%]*$/p' \| head -n -1 \
	\| awk 'BEGIN {split("B KB MB GB", u); for(i in u) m[u[i]]=1024^(i-1)} /:/ {print "[\"" $1 "\"," $2m[$3] "," $4m[$5] "]"}' \
	\| sort \
	\| jq -s "map((select(.[0] == \"FLASH:\") \| .[2]) \|= ${LOADER_SIZE}) + [ [ \"SKETCH:\", 0, ${SKETCH_SIZE} ] ]" > $REPORT.meminfo
	jq

	- name: Package board artifacts
	if: ${{ !cancelled() }}
	run: \|
	# add the config file to the report files, list them for later archiving
	cp firmwares/zephyr-${{ matrix.variant }}.config $REPORT.config
	echo "REPORT_FILES<<EOF" >> $GITHUB_ENV
	ls reports/* >> $GITHUB_ENV
	echo "EOF" >> $GITHUB_ENV

	# Preprare an archive of the built binaries (and build dir on failure)
	[ "${{ job.status }}" == "failure" ] && FULL_BUILD_DIR="build/${{ matrix.variant }}/"
	tar chf - \
	firmwares/${{ matrix.variant }} \
	variants/${{ matrix.variant }}/ \
	${FULL_BUILD_DIR} \
	\| zstd > binaries-${ARTIFACT_TAG}.tar.zstd

	- name: Archive board binaries
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@v4
	with:
	# prefix the name with 'failed-' if the build failed, so that it is not removed
	name: ${{ format('{0}binaries-{1}', (job.status == 'failure') && 'failed-' \|\| '', env.ARTIFACT_TAG) }}
	path: binaries-${{ env.ARTIFACT_TAG }}.tar.zstd
	retention-days: 7 # for failed build inspection

	- name: Archive build reports
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@v4
	with:
	name: build-report-${{ env.ARTIFACT_TAG }}
	path: ${{ env.REPORT_FILES }}
	retention-days: 1



	# Compose the core packages for each architecture in parallel, by retrieving
	# the API snapshot and the built binaries for all boards and packaging them.
	# The resulting core packages are final artifacts, ready for publishing.
	# The 'zephyr' umbrella package is also created here by including all others.

	package-core:
	name: Package ${{ matrix.artifact }}
	runs-on: ubuntu-latest
	needs:
	- build-env
	- build-board
	env:
	ALL_BOARD_DATA: ${{ needs.build-env.outputs.ALL_BOARD_DATA }}
	CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.build-env.outputs.CORE_HASH }}
	CORE_TAG: ${{ needs.build-env.outputs.CORE_TAG }}
	strategy:
	matrix:
	artifact: ${{ fromJSON( needs.build-env.outputs.ARTIFACTS ) }}
	fail-fast: false
	if: ${{ !cancelled() && needs.build-env.result == 'success' }}
	steps:

	- uses: actions/checkout@v4
	with:
	submodules: 'recursive'
	fetch-depth: 0
	persist-credentials: false
	fetch-tags: true

	# Get the API snapshot
	- uses: actions/download-artifact@v4
	with:
	path: .
	name: arduino-api

	# Get all built binaries
	- uses: actions/download-artifact@v4
	with:
	path: .
	pattern: binaries-*
	merge-multiple: true

	- name: Package core
	run: \|
	# Extract the API snapshot and board binaries in place
	rm -f cores/arduino/api # remove broken symlink
	tar --use-compress-program=unzstd -xpf arduino-api.tar.zstd
	for f in binaries-*.tar.zstd ; do
	tar --use-compress-program=unzstd -xpf $f
	done
	# Create the core artifact for this set of boards
	./extra/package_core.sh ${{ matrix.artifact }} ${CORE_TAG} distrib/${CORE_ARTIFACT}.tar.bz2

	- uses: actions/upload-artifact@v4
	if: ${{ success() \|\| failure() }}
	with:
	name: ${{ env.CORE_ARTIFACT }}
	path: distrib/${{ env.CORE_ARTIFACT }}.tar.bz2



	# After generating the final cores, the API and per-board binary archives are no longer needed.
	# The build environment archive can also be removed.

	cleanup-build:
	name: Clean up intermediates
	runs-on: ubuntu-latest
	needs:
	- package-core
	steps:

	- uses: geekyeggo/delete-artifact@v5.1.0
	with:
	name: \|
	arduino-api
	binaries-*
	build-env
	failOnError: false



	# For each board and link mode, compile all test sketches using the just
	# built core. The compilation reports are archived for later inspection.
	# This step never fails, so that expected errors do not block the CI run.

	test-core:
	# NOTE: this name is hardcoded in ci_inspect_logs.py
	name: Test ${{ matrix.board }}:${{ matrix.link_mode }}
	runs-on: ubuntu-latest
	needs:
	- build-env
	- package-core
	strategy:
	matrix:
	include:
	${{ fromJSON( needs.build-env.outputs.ALL_BOARD_FQBNS ) }}
	fail-fast: false
	env:
	PLAT: arduino:${{ matrix.subarch }}
	FQBN: arduino:${{ matrix.subarch }}:${{ matrix.board }}:link_mode=${{ matrix.link_mode }}
	CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.build-env.outputs.CORE_HASH }}
	ARTIFACT_TAG: ${{ needs.build-env.outputs.CORE_HASH }}-${{ matrix.board }}-${{ matrix.link_mode }}
	if: ${{ !cancelled() && needs.build-env.result == 'success' }}
	steps:

	# only needs the test list generator and artifact information
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0
	persist-credentials: false
	sparse-checkout: \|
	extra/ci_test_list.sh
	extra/artifacts/

	- uses: actions/download-artifact@v4
	with:
	name: ${{ env.CORE_ARTIFACT }}

	- name: Set up core
	run: \|
	tar xf ${CORE_ARTIFACT}.tar.bz2 # will create ArduinoCore-zephyr/
	echo "REPORT_FILE=$(echo ${FQBN} \| tr ':' '-').json" >> $GITHUB_ENV

	- name: Get test sketches
	run: \|
	# sets ALL_TESTS and ALL_LIBRARIES env vars in GITHUB_ENV
	extra/ci_test_list.sh ${{ matrix.artifact }} ${{ matrix.variant }}

	- name: Compile tests for ${{ matrix.board }}
	uses: pillo79/compile-sketches@4a1dead03155c17ddedc373699d7aa80a78a6c7d # next
	with:
	fqbn: ${{ env.FQBN }}
	platforms: \|
	# Use Board Manager version first to install the tools
	- name: ${{ env.PLAT }}
	# ... then replace that with the local version
	- name: ${{ env.PLAT }}
	source-path: "ArduinoCore-zephyr"
	sketch-paths: \|
	${{ env.ALL_TESTS }}
	libraries: \|
	${{ env.ALL_LIBRARIES }}
	cli-compile-flags: \|
	- '--build-property'
	- 'compiler.c.extra_flags=-Wno-type-limits -Wno-missing-field-initializers'
	- '--build-property'
	- 'compiler.cpp.extra_flags=-Wno-type-limits -Wno-missing-field-initializers'
	verbose: 'false'
	enable-deltas-report: 'false'
	enable-issues-report: 'true'
	always-succeed: 'true'

	- name: Prepare log
	if: ${{ success() \|\| failure() }}
	run: \|
	# make sure to create an empty report if none was created
	[ ! -f sketches-reports/${REPORT_FILE} ] && mkdir -p sketches-reports && echo "{}" > sketches-reports/${REPORT_FILE}
	# remove base path of the core install from all file references to make logs cleaner
	sed -i -e 's!/home/runner/.arduino15/packages/arduino/hardware/zephyr/[^/]*/!!g' sketches-reports/${REPORT_FILE}

	# archive the compile report for later
	- uses: actions/upload-artifact@v4
	if: ${{ success() \|\| failure() }}
	with:
	name: test-report-${{ env.ARTIFACT_TAG }}
	path: sketches-reports/*
	retention-days: 1



	# Collect all build and test reports, analyze them and produce a summary.
	# See ci_inspect_logs.py for details.

	verify-core:
	runs-on: ubuntu-latest
	needs:
	- build-env
	- package-core
	- test-core
	if: ${{ !cancelled() }}
	env:
	ALL_BOARD_DATA: ${{ needs.build-env.outputs.ALL_BOARD_DATA }}
	CORE_VER: ${{ needs.build-env.outputs.CORE_VER }}
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	GH_REPO: ${{ github.repository }}
	steps:

	- uses: actions/checkout@v4
	with:
	fetch-depth: 0
	persist-credentials: false
	fetch-tags: true

	- uses: actions/download-artifact@v4
	with:
	path: .
	pattern: "-report-"
	merge-multiple: true

	- run: \|
	# gather the array of job metadata (especially name and ID) for the current workflow run
	export WORKFLOW_JOBS=$(gh run view ${{ github.run_id }} --attempt ${{ github.run_attempt }} --json jobs --jq '.jobs')
	# Run the log inspection script
	extra/ci_inspect_logs.py result summary full_log

	# Display the summary and full log in the step summary
	cat summary >> $GITHUB_STEP_SUMMARY
	cat full_log >> $GITHUB_STEP_SUMMARY

	# Create a comment for PRs using the summary file
	if [ "${{ github.event_name }}" == "pull_request" ]; then
	mkdir -p comment-request
	echo "${{ github.event.pull_request.number }}" > comment-request/pr_number
	echo -e "## Built \`$CORE_VER\`\n" > comment-request/comment_body
	# strip LaTeX color commands and escape sequences for mail readability
	cat summary \| sed -e 's!\${\\color{\S}$.$}\$!\1!g' -e 's!\\%!%!g' >> comment-request/comment_body
	fi

	# upload comment request artifact (will be retrieved by leave_pr_comment.yml)
	- name: Archive comment information
	uses: actions/upload-artifact@v4
	if: ${{ github.event_name == 'pull_request' }}
	with:
	name: comment-request
	path: comment-request/
	retention-days: 1

	# teports are no longer needed now
	- name: Clean up intermediate artifacts
	uses: geekyeggo/delete-artifact@v5.1.0
	with:
	name: \|
	build-report-*
	test-report-*
	failOnError: false

	# The last line makes the job fail if the 'result' file does not contain "PASSED".
	# No further actions are allowed in case of error.
	- run: \|
	cat result
	grep "PASSED" result >& /dev/null # otherwise CI test failed



	# Upload the built core packages to the S3 bucket for public distribution.

	publish-core:
	name: Publish core
	runs-on: ubuntu-latest
	if: ${{ github.event_name == 'push' && github.repository == 'arduino/ArduinoCore-zephyr' }}
	needs:
	- build-env
	- package-core
	- verify-core
	environment: production
	permissions:
	id-token: write
	contents: read
	steps:

	- uses: actions/download-artifact@v4
	with:
	path: .
	pattern: ArduinoCore-*
	merge-multiple: true

	- name: Configure AWS credentials
	uses: aws-actions/configure-aws-credentials@v4
	with:
	role-to-assume: ${{ secrets.IAM_ROLE }}
	aws-region: ${{ secrets.AWS_REGION }}

	- name: Upload artifact
	run: \|
	for f in ArduinoCore-*.tar.bz2 ; do
	aws s3 cp $f s3://${{ secrets.S3_BUCKET }}/
	done

	# Prepare the package index JSON snippets for the newly built core packages.

	prepare-json:
	name: Prepare jsons
	runs-on: ubuntu-latest
	needs:
	- build-env
	- package-core
	- verify-core
	env:
	CORE_TAG: ${{ needs.build-env.outputs.CORE_TAG }}
	CORE_HASH: ${{ needs.build-env.outputs.CORE_HASH }}
	ARTIFACTS: ${{ needs.build-env.outputs.ARTIFACTS }}
	steps:

	- uses: actions/checkout@v4
	with:
	fetch-depth: 0
	persist-credentials: false
	fetch-tags: true

	- uses: actions/download-artifact@v4
	with:
	path: .
	pattern: ArduinoCore-*
	merge-multiple: true

	- name: Prepare package index snippets
	run: \|
	jq -cr '.[]' <<< ${ARTIFACTS} \| while read -r artifact; do
	# note: the archive names are always hash-based so they use a
	# predictable pattern. JSON files can use the tag for user-friendliness.
	ARTIFACT_FILE=ArduinoCore-${artifact}-${CORE_HASH}.tar.bz2
	PACKAGE_JSON=ArduinoCore-${artifact}-${CORE_TAG}.json
	./extra/gen_package_index_json.sh ${artifact} ${ARTIFACT_FILE} ${PACKAGE_JSON}
	done

	- name: Archive package index snippets
	uses: actions/upload-artifact@v4
	with:
	name: ArduinoCore-zephyr-${{ env.CORE_TAG }}-jsons
	path: ArduinoCore-*-${{ env.CORE_TAG }}.json

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Merge pull request #334 from soburi/fix_builtin_led_macro #1097

Workflow file

Merge pull request #334 from soburi/fix_builtin_led_macro #1097

Uh oh!

Workflow file for this run