chore(opt): Fetch set of Brillig entry points without reachability and recursive functions computation #13629
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Reports | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| jobs: | |
| rust_benchmarks: | |
| name: Rust Benchmarks | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 15 | |
| permissions: | |
| pull-requests: write | |
| # deployments permission to deploy GitHub pages website | |
| deployments: write | |
| # contents permission to update benchmark contents in gh-pages branch | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Setup toolchain | |
| uses: dtolnay/[email protected] | |
| with: | |
| targets: x86_64-unknown-linux-gnu | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: x86_64-unknown-linux-gnu-bench | |
| cache-on-failure: true | |
| save-if: ${{ github.event_name != 'merge_group' }} | |
| - name: Run ACVM benchmarks | |
| run: | | |
| cargo bench -p acvm -- --output-format bencher | tee acvm-bench.txt | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Store ACVM benchmarks result | |
| uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 | |
| with: | |
| name: "ACVM Benchmarks" | |
| tool: "cargo" | |
| output-file-path: ./acvm-bench.txt | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| # We want this to only run on master to avoid garbage data from PRs being added. | |
| auto-push: ${{ github.ref == 'refs/heads/master' }} | |
| alert-threshold: "120%" | |
| comment-on-alert: true | |
| comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }} | |
| fail-on-alert: false | |
| alert-comment-cc-users: "@TomAFrench" | |
| max-items-in-chart: 50 | |
| benchmark-projects-list: | |
| name: Load benchmark projects list | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 5 | |
| outputs: | |
| projects: ${{ steps.get_bench_projects.outputs.projects }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| - name: Build list of projects | |
| id: get_bench_projects | |
| run: | | |
| PROJECTS=$(yq ./.github/benchmark_projects.yml -o json | jq -c '.projects | to_entries | map(.value + { name: .key })') | |
| echo "projects=$PROJECTS" | |
| echo "projects=$PROJECTS" >> $GITHUB_OUTPUT | |
| build-nargo: | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Checkout Noir repo | |
| uses: actions/checkout@v5 | |
| - name: Setup toolchain | |
| uses: dtolnay/[email protected] | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: x86_64-unknown-linux-gnu | |
| cache-on-failure: true | |
| save-if: ${{ github.event_name != 'merge_group' }} | |
| - name: Build Nargo | |
| run: cargo build --package nargo_cli --release | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Package artifacts | |
| run: | | |
| mkdir dist | |
| cp ./target/release/nargo ./dist/nargo | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: nargo | |
| path: ./dist/* | |
| retention-days: 3 | |
| build-noir-inspector: | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Checkout Noir repo | |
| uses: actions/checkout@v5 | |
| - name: Setup toolchain | |
| uses: dtolnay/[email protected] | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: x86_64-unknown-linux-gnu-inspector | |
| cache-on-failure: true | |
| save-if: ${{ github.event_name != 'merge_group' }} | |
| - name: Build Nargo | |
| run: cargo build --package noir_inspector --release | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Package artifacts | |
| run: | | |
| mkdir dist | |
| cp ./target/release/noir-inspector ./dist/noir-inspector | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: noir-inspector | |
| path: ./dist/* | |
| retention-days: 3 | |
| compare_gates_reports: | |
| name: Circuit sizes | |
| needs: [build-nargo] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 20 | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Install `bb` | |
| run: | | |
| ./scripts/install_bb.sh | |
| echo "$HOME/.bb/" >> $GITHUB_PATH | |
| - name: Download nargo binary | |
| uses: ./.github/actions/download-nargo | |
| - name: Build Noir circuits | |
| working-directory: ./test_programs | |
| run: ./rebuild.sh | |
| - name: Generate gates report | |
| working-directory: ./test_programs | |
| run: | | |
| ./gates_report.sh | |
| mv gates_report.json ../gates_report.json | |
| - name: Compare gates reports | |
| id: gates_diff | |
| uses: noir-lang/noir-gates-diff@dbe920a8dcc3370af4be4f702ca9cef29317bec1 | |
| with: | |
| report: gates_report.json | |
| summaryQuantile: 0.9 # only display the 10% most significant circuit size diffs in the summary (defaults to 20%) | |
| - name: Add gates diff to sticky comment | |
| # We don't run this step on external PRs | |
| if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository | |
| uses: marocchino/sticky-pull-request-comment@v2 | |
| with: | |
| # delete the comment in case changes no longer impact circuit sizes | |
| delete: ${{ !steps.gates_diff.outputs.markdown }} | |
| message: ${{ steps.gates_diff.outputs.markdown }} | |
| compare_brillig_bytecode_size_reports: | |
| name: Brillig bytecode sizes | |
| needs: [build-nargo] | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 20 | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Download nargo binary | |
| uses: ./.github/actions/download-nargo | |
| - name: Generate Brillig bytecode size report | |
| working-directory: ./test_programs | |
| run: | | |
| set -e | |
| mkdir ./reports | |
| ./gates_report_brillig.sh 9223372036854775807 | |
| jq '.programs |= map(.package_name |= (. + "_inliner_max"))' gates_report_brillig.json > ./reports/gates_report_brillig_inliner_max.json | |
| ./gates_report_brillig.sh 0 | |
| jq '.programs |= map(.package_name |= (. + "_inliner_zero"))' gates_report_brillig.json > ./reports/gates_report_brillig_inliner_zero.json | |
| ./gates_report_brillig.sh -9223372036854775808 | |
| jq '.programs |= map(.package_name |= (. + "_inliner_min"))' gates_report_brillig.json > ./reports/gates_report_brillig_inliner_min.json | |
| # Merge all reports | |
| jq -s '{ programs: map(.programs) | add }' ./reports/* > ../gates_report_brillig.json | |
| - name: Compare Brillig bytecode size reports | |
| id: brillig_bytecode_diff | |
| uses: noir-lang/noir-gates-diff@dbe920a8dcc3370af4be4f702ca9cef29317bec1 | |
| with: | |
| report: gates_report_brillig.json | |
| header: | | |
| # Changes to Brillig bytecode sizes | |
| brillig_report: true | |
| summaryQuantile: 0.9 # only display the 10% most significant bytecode size diffs in the summary (defaults to 20%) | |
| - name: Add bytecode size diff to sticky comment | |
| if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target' | |
| uses: marocchino/sticky-pull-request-comment@v2 | |
| with: | |
| header: brillig | |
| # delete the comment in case changes no longer impact brillig bytecode sizes | |
| delete: ${{ !steps.brillig_bytecode_diff.outputs.markdown }} | |
| message: ${{ steps.brillig_bytecode_diff.outputs.markdown }} | |
| compare_brillig_execution_reports: | |
| name: Brillig execution trace sizes | |
| needs: [build-nargo] | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 20 | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Download nargo binary | |
| uses: ./.github/actions/download-nargo | |
| - name: Generate Brillig execution report | |
| working-directory: ./test_programs | |
| run: | | |
| set -e | |
| mkdir ./reports | |
| ./gates_report_brillig_execution.sh 9223372036854775807 | |
| jq '.programs |= map(.package_name |= (. + "_inliner_max"))' gates_report_brillig_execution.json > ./reports/gates_report_brillig_execution_inliner_max.json | |
| ./gates_report_brillig_execution.sh 0 | |
| jq '.programs |= map(.package_name |= (. + "_inliner_zero"))' gates_report_brillig_execution.json > ./reports/gates_report_brillig_execution_inliner_zero.json | |
| ./gates_report_brillig_execution.sh -9223372036854775808 | |
| jq '.programs |= map(.package_name |= (. + "_inliner_min"))' gates_report_brillig_execution.json > ./reports/gates_report_brillig_execution_inliner_min.json | |
| # Merge all reports | |
| jq -s '{ programs: map(.programs) | add }' ./reports/* > ../gates_report_brillig_execution.json | |
| - name: Compare Brillig execution reports | |
| id: brillig_execution_diff | |
| uses: noir-lang/noir-gates-diff@dbe920a8dcc3370af4be4f702ca9cef29317bec1 | |
| with: | |
| report: gates_report_brillig_execution.json | |
| header: | | |
| # Changes to number of Brillig opcodes executed | |
| brillig_report: true | |
| summaryQuantile: 0.9 # only display the 10% most significant bytecode size diffs in the summary (defaults to 20%) | |
| - name: Add bytecode size diff to sticky comment | |
| if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target' | |
| uses: marocchino/sticky-pull-request-comment@v2 | |
| with: | |
| header: brillig_execution | |
| # delete the comment in case changes no longer impact brillig bytecode sizes | |
| delete: ${{ !steps.brillig_execution_diff.outputs.markdown }} | |
| message: ${{ steps.brillig_execution_diff.outputs.markdown }} | |
| generate_memory_report: | |
| name: Peak memory usage | |
| needs: [build-nargo] | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 20 | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Download nargo binary | |
| uses: ./.github/actions/download-nargo | |
| - name: Generate Memory report | |
| working-directory: ./test_programs | |
| run: | | |
| ./memory_report.sh | |
| mv memory_report.json ../memory_report.json | |
| - name: Upload compilation memory report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: in_progress_compilation_mem_report | |
| path: memory_report.json | |
| retention-days: 3 | |
| overwrite: true | |
| - name: Generate execution memory report | |
| working-directory: ./test_programs | |
| run: | | |
| ./memory_report.sh 0 1 | |
| mv memory_report.json ../memory_report.json | |
| - name: Upload execution memory report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: in_progress_execution_mem_report | |
| path: memory_report.json | |
| retention-days: 3 | |
| overwrite: true | |
| external_repo_reports: | |
| needs: [build-nargo, build-noir-inspector, benchmark-projects-list] | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 15 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson( needs.benchmark-projects-list.outputs.projects )}} | |
| name: External repo compilation and execution reports - ${{ matrix.repo }}/${{ matrix.path }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/download-nargo/action.yml | |
| .github/actions/download-noir-inspector/action.yml | |
| scripts/gather_benchmark_data.sh | |
| scripts/process_benchmark_data.sh | |
| sparse-checkout-cone-mode: false | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| with: | |
| repository: ${{ matrix.repo }} | |
| path: test-repo | |
| ref: ${{ matrix.ref }} | |
| - name: Download nargo binary | |
| uses: ./.github/actions/download-nargo | |
| - name: Download noir-inspector binary | |
| uses: ./.github/actions/download-noir-inspector | |
| - name: Generate compilation report | |
| id: compilation_report | |
| run: | | |
| PACKAGE_NAME=$(basename ${{ matrix.path }}) | |
| REPORT_NAME=external_report_$PACKAGE_NAME.json | |
| REPORT_PATH=$(pwd)/$REPORT_NAME | |
| echo "report_name=$REPORT_NAME" >> $GITHUB_OUTPUT | |
| echo "report_path=$REPORT_PATH" >> $GITHUB_OUTPUT | |
| - name: Generate report data | |
| run: ./scripts/gather_benchmark_data.sh | |
| env: | |
| REPO_SLUG: ${{ matrix.repo }} | |
| PROJECT_TAG: ${{ matrix.ref }} | |
| REPO_DIR: test-repo | |
| PROJECT_DIR: ${{ matrix.path }} | |
| OUTPUT_DIR: ${{ github.workspace }}/output | |
| NUM_RUNS: ${{ matrix.num_runs }} | |
| - name: Process report data | |
| run: | | |
| ./scripts/process_benchmark_data.sh $INPUT_DIR > ${{ steps.compilation_report.outputs.report_path }} | |
| env: | |
| INPUT_DIR: ${{ github.workspace }}/output | |
| PROJECT_NAME: ${{ matrix.name }} | |
| - name: Upload compilation report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ steps.compilation_report.outputs.report_name }} | |
| path: ${{ steps.compilation_report.outputs.report_path }} | |
| retention-days: 3 | |
| overwrite: true | |
| - name: Check compilation time limit | |
| run: | | |
| TIME=$(jq '.compilation_time.value' ${{ steps.compilation_report.outputs.report_path }}) | |
| TIME_LIMIT=${{ matrix.compilation-timeout }} | |
| if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "$TIME_LIMIT"; then | |
| # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause. | |
| echo "Failing due to compilation exceeding timeout..." | |
| echo "Timeout: "$TIME_LIMIT"s" | |
| echo "Compilation took: "$TIME"s". | |
| exit 1 | |
| fi | |
| - name: Check execution time limit | |
| if: ${{ !matrix.cannot_execute }} | |
| run: | | |
| TIME=$(jq '.execution_time.value' ${{ steps.execution_report.outputs.report_path }}) | |
| TIME_LIMIT=${{ matrix.execution-timeout }} | |
| if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "$TIME_LIMIT"; then | |
| # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause. | |
| echo "Failing due to execution exceeding timeout..." | |
| echo "Timeout: "$TIME_LIMIT"s" | |
| echo "Execution took: "$TIME"s". | |
| exit 1 | |
| fi | |
| external_repo_memory_report: | |
| needs: [build-nargo, benchmark-projects-list] | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson( needs.benchmark-projects-list.outputs.projects )}} | |
| name: External repo memory report - ${{ matrix.repo }}/${{ matrix.path }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| path: scripts | |
| sparse-checkout: | | |
| .github/actions/download-nargo/action.yml | |
| test_programs/memory_report.sh | |
| test_programs/parse_memory.sh | |
| sparse-checkout-cone-mode: false | |
| - name: Download nargo binary | |
| uses: ./scripts/.github/actions/download-nargo | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ matrix.repo }} | |
| path: test-repo | |
| ref: ${{ matrix.ref }} | |
| - name: Generate compilation memory report | |
| id: compilation_report | |
| working-directory: ./test-repo/${{ matrix.path }} | |
| run: | | |
| mv /home/runner/work/noir/noir/scripts/test_programs/memory_report.sh ./memory_report.sh | |
| mv /home/runner/work/noir/noir/scripts/test_programs/parse_memory.sh ./parse_memory.sh | |
| ./memory_report.sh 1 | |
| # Rename the memory report as the execution report is about to write to the same file | |
| PACKAGE_NAME=${{ matrix.path }} | |
| PACKAGE_NAME=$(basename $PACKAGE_NAME) | |
| REPORT_NAME=compilation_mem_report_$PACKAGE_NAME.json | |
| REPORT_PATH=$(pwd)/$REPORT_NAME | |
| mv ./memory_report.json $REPORT_PATH | |
| echo "report_name=$REPORT_NAME" >> $GITHUB_OUTPUT | |
| echo "report_path=$REPORT_PATH" >> $GITHUB_OUTPUT | |
| env: | |
| FLAGS: ${{ matrix.flags }} | |
| - name: Upload compilation memory report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ steps.compilation_report.outputs.report_name }} | |
| path: ${{ steps.compilation_report.outputs.report_path }} | |
| retention-days: 3 | |
| overwrite: true | |
| - name: Generate execution memory report | |
| id: execution_report | |
| working-directory: ./test-repo/${{ matrix.path }} | |
| if: ${{ !matrix.cannot_execute }} | |
| run: | | |
| ./memory_report.sh 1 1 | |
| PACKAGE_NAME=${{ matrix.path }} | |
| PACKAGE_NAME=$(basename $PACKAGE_NAME) | |
| REPORT_NAME=execution_mem_report_$PACKAGE_NAME.json | |
| REPORT_PATH=$(pwd)/$REPORT_NAME | |
| mv ./memory_report.json $REPORT_PATH | |
| echo "report_name=$REPORT_NAME" >> $GITHUB_OUTPUT | |
| echo "report_path=$REPORT_PATH" >> $GITHUB_OUTPUT | |
| - name: Upload execution memory report | |
| if: ${{ !matrix.cannot_execute }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ steps.execution_report.outputs.report_name }} | |
| path: ${{ steps.execution_report.outputs.report_path }} | |
| retention-days: 3 | |
| overwrite: true | |
| - name: Check compilation memory limit | |
| run: | | |
| MEMORY=$(jq '.[0].value' ${{ steps.compilation_report.outputs.report_path }}) | |
| MEMORY_LIMIT=${{ matrix.compilation-memory-limit }} | |
| if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$MEMORY" "$MEMORY_LIMIT"; then | |
| # Don't bump this limit without understanding why this has happened and confirming that you're not the cause. | |
| echo "Failing due to compilation exceeding memory limit..." | |
| echo "Limit: "$MEMORY_LIMIT"MB" | |
| echo "Compilation took: "$MEMORY"MB". | |
| exit 1 | |
| fi | |
| - name: Check execution memory limit | |
| if: ${{ !matrix.cannot_execute }} | |
| run: | | |
| MEMORY=$(jq '.[0].value' ${{ steps.execution_report.outputs.report_path }}) | |
| MEMORY_LIMIT=${{ matrix.execution-memory-limit }} | |
| if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$MEMORY" "$MEMORY_LIMIT"; then | |
| # Don't bump this limit without understanding why this has happened and confirming that you're not the cause. | |
| echo "Failing due to execution exceeding memory limit..." | |
| echo "Limit: "$MEMORY_LIMIT"MB" | |
| echo "Execution took: "$MEMORY"MB". | |
| exit 1 | |
| fi | |
| upload_compilation_reports: | |
| name: Upload compilation reports | |
| needs: [external_repo_reports] | |
| timeout-minutes: 5 | |
| # We want this job to run even if one variation of the matrix in `external_repo_reports` fails | |
| if: always() | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| pull-requests: write | |
| # deployments permission to deploy GitHub pages website | |
| deployments: write | |
| # contents permission to update benchmark contents in gh-pages branch | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Download matrix compilation reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: external_report_* | |
| path: ./reports | |
| merge-multiple: true | |
| - name: Merge compilation reports using jq | |
| run: | | |
| # Github actions seems to not expand "**" in globs by default. | |
| shopt -s globstar | |
| jq --slurp 'map(.compilation_time | select(.))' ./reports/* | tee time_bench.json | |
| - name: Store benchmark result | |
| uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 | |
| with: | |
| name: "Compilation Time" | |
| tool: "customSmallerIsBetter" | |
| output-file-path: ./time_bench.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| # We want this to only run on master to avoid garbage data from PRs being added. | |
| auto-push: ${{ github.ref == 'refs/heads/master' }} | |
| alert-threshold: "120%" | |
| comment-on-alert: true | |
| comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }} | |
| fail-on-alert: false | |
| alert-comment-cc-users: "@TomAFrench" | |
| max-items-in-chart: 50 | |
| upload_execution_reports: | |
| name: Upload execution reports | |
| needs: [external_repo_reports] | |
| timeout-minutes: 5 | |
| # We want this job to run even if one variation of the matrix in `external_repo_reports` fails | |
| if: always() | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| pull-requests: write | |
| # deployments permission to deploy GitHub pages website | |
| deployments: write | |
| # contents permission to update benchmark contents in gh-pages branch | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Download matrix compilation reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: external_report_* | |
| path: ./reports | |
| merge-multiple: true | |
| - name: Merge execution reports using jq | |
| run: | | |
| # Github actions seems to not expand "**" in globs by default. | |
| shopt -s globstar | |
| jq --slurp 'map(.execution_time | select(.))' ./reports/* | tee time_bench.json | |
| - name: Store benchmark result | |
| uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 | |
| with: | |
| name: "Execution Time" | |
| tool: "customSmallerIsBetter" | |
| output-file-path: ./time_bench.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| # We want this to only run on master to avoid garbage data from PRs being added. | |
| auto-push: ${{ github.ref == 'refs/heads/master' }} | |
| alert-threshold: "120%" | |
| comment-on-alert: true | |
| comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }} | |
| fail-on-alert: false | |
| alert-comment-cc-users: "@TomAFrench" | |
| max-items-in-chart: 50 | |
| upload_artifact_reports: | |
| name: Upload artifact reports | |
| needs: [external_repo_reports] | |
| timeout-minutes: 5 | |
| # We want this job to run even if one variation of the matrix in `external_repo_reports` fails | |
| if: always() | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| pull-requests: write | |
| # deployments permission to deploy GitHub pages website | |
| deployments: write | |
| # contents permission to update benchmark contents in gh-pages branch | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Download matrix compilation reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: external_report_* | |
| path: ./reports | |
| merge-multiple: true | |
| - name: Merge artifact size reports using jq | |
| run: | | |
| # Github actions seems to not expand "**" in globs by default. | |
| shopt -s globstar | |
| jq --slurp 'map(.artifact_size | select(.))' ./reports/* | tee artifact_size.json | |
| - name: Store benchmark result | |
| uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 | |
| with: | |
| name: "Artifact Size" | |
| tool: "customSmallerIsBetter" | |
| output-file-path: ./artifact_size.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| # We want this to only run on master to avoid garbage data from PRs being added. | |
| auto-push: ${{ github.ref == 'refs/heads/master' }} | |
| alert-threshold: "120%" | |
| comment-on-alert: true | |
| comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }} | |
| fail-on-alert: false | |
| alert-comment-cc-users: "@TomAFrench" | |
| max-items-in-chart: 50 | |
| upload_opcode_reports: | |
| name: Upload opcode reports | |
| needs: [external_repo_reports] | |
| timeout-minutes: 5 | |
| # We want this job to run even if one variation of the matrix in `external_repo_reports` fails | |
| if: always() | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| pull-requests: write | |
| # deployments permission to deploy GitHub pages website | |
| deployments: write | |
| # contents permission to update benchmark contents in gh-pages branch | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Download matrix compilation reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: external_report_* | |
| path: ./reports | |
| merge-multiple: true | |
| - name: Merge opcode count reports using jq | |
| run: | | |
| # Github actions seems to not expand "**" in globs by default. | |
| shopt -s globstar | |
| jq --slurp 'map(.num_opcodes | select(.))' ./reports/* | tee num_opcodes.json | |
| - name: Store benchmark result | |
| uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 | |
| with: | |
| name: "Opcode count" | |
| tool: "customSmallerIsBetter" | |
| output-file-path: ./num_opcodes.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| # We want this to only run on master to avoid garbage data from PRs being added. | |
| auto-push: ${{ github.ref == 'refs/heads/master' }} | |
| alert-threshold: "110%" | |
| comment-on-alert: true | |
| comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }} | |
| fail-on-alert: false | |
| alert-comment-cc-users: "@TomAFrench" | |
| max-items-in-chart: 50 | |
| upload_compilation_memory_report: | |
| name: Upload compilation memory report | |
| needs: [generate_memory_report, external_repo_memory_report] | |
| timeout-minutes: 5 | |
| # We want this job to run even if one variation of the matrix in `external_repo_memory_report` fails | |
| if: always() | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| pull-requests: write | |
| # deployments permission to deploy GitHub pages website | |
| deployments: write | |
| # contents permission to update benchmark contents in gh-pages branch | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Download initial memory report | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: in_progress_compilation_mem_report | |
| - name: Download matrix memory reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: compilation_mem_report_* | |
| path: ./reports | |
| merge-multiple: true | |
| - name: Merge memory reports using jq | |
| run: | | |
| # Github actions seems to not expand "**" in globs by default. | |
| shopt -s globstar | |
| jq --slurp '. | flatten' ./reports/* | tee memory_bench.json | |
| - name: Store benchmark result | |
| uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 | |
| with: | |
| name: "Compilation Memory" | |
| tool: "customSmallerIsBetter" | |
| output-file-path: ./memory_bench.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| # We want this to only run on master to avoid garbage data from PRs being added. | |
| auto-push: ${{ github.ref == 'refs/heads/master' }} | |
| alert-threshold: "120%" | |
| comment-on-alert: true | |
| comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }} | |
| fail-on-alert: false | |
| alert-comment-cc-users: "@TomAFrench" | |
| max-items-in-chart: 50 | |
| upload_execution_memory_report: | |
| name: Upload execution memory report | |
| needs: [generate_memory_report, external_repo_memory_report] | |
| timeout-minutes: 5 | |
| # We want this job to run even if one variation of the matrix in `external_repo_memory_report` fails | |
| if: always() | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| pull-requests: write | |
| # deployments permission to deploy GitHub pages website | |
| deployments: write | |
| # contents permission to update benchmark contents in gh-pages branch | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Download initial memory report | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: in_progress_execution_mem_report | |
| - name: Download matrix memory reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: execution_mem_report_* | |
| path: ./reports | |
| merge-multiple: true | |
| - name: Merge memory reports using jq | |
| run: | | |
| # Github actions seems to not expand "**" in globs by default. | |
| shopt -s globstar | |
| jq --slurp '. | flatten' ./reports/* | tee memory_bench.json | |
| - name: Store benchmark result | |
| uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 | |
| with: | |
| name: "Execution Memory" | |
| tool: "customSmallerIsBetter" | |
| output-file-path: ./memory_bench.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| # We want this to only run on master to avoid garbage data from PRs being added. | |
| auto-push: ${{ github.ref == 'refs/heads/master' }} | |
| alert-threshold: "120%" | |
| comment-on-alert: true | |
| comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }} | |
| fail-on-alert: false | |
| alert-comment-cc-users: "@TomAFrench" | |
| max-items-in-chart: 50 | |
| # This is a job which depends on all test jobs and reports the overall status. | |
| # This allows us to add/remove test jobs without having to update the required workflows. | |
| reports-end: | |
| name: End | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 5 | |
| # We want this job to always run (even if the dependant jobs fail) as we want this job to fail rather than skipping. | |
| if: ${{ always() }} | |
| needs: | |
| - rust_benchmarks | |
| - upload_compilation_reports | |
| - upload_execution_reports | |
| - upload_artifact_reports | |
| - upload_opcode_reports | |
| - upload_compilation_memory_report | |
| - upload_execution_memory_report | |
| steps: | |
| - name: Report overall success | |
| run: | | |
| if [[ $FAIL == true ]]; then | |
| exit 1 | |
| else | |
| exit 0 | |
| fi | |
| env: | |
| # We treat any skipped or failing jobs as a failure for the workflow as a whole. | |
| FAIL: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') }} |