Partition coverage into multiple jobs for diskspace in CI

.github/workflows/coverage.yml

@@ -8,39 +8,43 @@ on:
 
 name: Code Coverage
 
+env:
+  # We partition coverage tests into multiple parts to avoid filling diskspace in a single runner
+  PARTITIONS_COUNT: 5
+
 jobs:
   coverage:
     runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        # This range spans from `0` to `PARTITIONS_COUNT - 1`, where `PARTITIONS_COUNT` is the number of partitions (defined in env var above)
+        partition: [0, 1, 2, 3, 4]
+
     steps:
-      - id: skip-check
-        uses: fkirc/[email protected]
-        with:
-          concurrent_skipping: "same_content_newer"
-          skip_after_successful_duplicate: "true"
       - name: Install dependencies
         run: sudo apt-get install -yqq --no-install-recommends build-essential libgtk-3-dev python3 python3-toml
       - uses: actions/checkout@v1
       - uses: actions-rs/toolchain@v1
         with:
-          # TODO: Inspect coverage, and change toolchain to stable
           toolchain: nightly-2023-08-01
           override: true
       - uses: actions-rs/cargo@v1
         with:
           command: clean
-      - uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --all-features --no-fail-fast
+      - name: Run coverage tests
+        run: python3 build-tools/workspace-partition.py ${{ env.PARTITIONS_COUNT }} ${{ matrix.partition }} -p | xargs cargo test
         env:
           RUST_LOG: debug
           RUST_BACKTRACE: full
           CARGO_INCREMENTAL: 0
-          RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests"
-          RUSTDOCFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests"
+          RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off"
+          RUSTDOCFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off"
       - id: coverage
         uses: actions-rs/[email protected]
+        with:
+          config: build-tools/coverage/grcov.toml
       - uses: actions/upload-artifact@v2
         with:
-          name: code-coverage-report
+          name: code-coverage-report-${{ matrix.partition }}
           path: ${{ steps.coverage.outputs.report }}

build-tools/coverage/README.md

@@ -0,0 +1,33 @@
+# Generating test coverage report
+
+Using the configuration in this directory, Github Actions CI will generate a test coverage report for the whole project. The report possibly will be split into multiple parts due to disk-space constraints related to github CI limits. This tutorial describes how to generate an html report out of this data.
+
+## Prerequisites
+- The source code of the project
+- The coverage report data generated by the CI
+- lcov installed, together with genhtml.
+
+## Steps
+1. Download the reports data from CI. The data is stored in the artifacts of the CI job. You can download it manually from the CI job page.
+2. Unzip all the downloaded files into a single directory. They all will have the prefix `grcov-report-`.
+3. Now we need to run the following command with all the files in the directory:
+
+```bash
+lcov -a grcov-report-1 -a grcov-report-2 -a grcov-report-3 -a grcov-report-4 -a grcov-report-5 -o grcov-report.info
+```
+
+where these suffixes can be different every time. The easiest way is to run this magic bash command which will concatenate all the files in the directory, prefix them with `-a`, and pass them to lcov:
+
+```bash
+ls -1a grcov-report-* | sed 's/^/-a /' | tr '\n' ' ' | xargs lcov -o grcov-report.info
+```
+
+4. Now we need to generate the html report. Go to the source code's root dir (and note where you put that grcov-report.info file)
+
+5. run the following command:
+
+```bash
+genhtml --exclude /rustc/* --exclude */.cargo/registry/* --exclude */.cargo/git/* --exclude /cargo/registry/* --exclude "*target/*" -o grcov-report --ignore-errors unmapped /path/to/grcov-report.info
+```
+
+6. Now you can open the `grcov-report/index.html` file in your browser and see the report.

build-tools/coverage/grcov.yml

@@ -0,0 +1,2 @@
+output-type: lcov
+output-path: ./lcov.info

build-tools/workspace-partition.py

@@ -0,0 +1,102 @@
+import sys
+import toml
+import os
+
+def read_cargo_toml(file_path):
+    with open(file_path, "r") as cargo_toml_file:
+        cargo_toml = toml.load(cargo_toml_file)
+    return cargo_toml
+
+def get_package_name(crate_dir):
+    cargo_toml_path = os.path.join(crate_dir, "Cargo.toml")
+    cargo_toml = read_cargo_toml(cargo_toml_path)
+
+    if cargo_toml is not None:
+        package = cargo_toml.get("package", {})
+        return package.get("name", None)
+    else:
+        raise Exception("Cargo.toml not found for crate {}.".format(crate_dir))
+
+def group_crates_by_first_directory(members):
+    '''
+    Here we ensure that all crates in the same directory are grouped together, to minimize fracturing of coverage.
+    We assume here that crates are tested within their directory. This is a fair assumption since we put test-suites
+    in the same directory as the crate they are testing.
+    '''
+    crate_groups = {}
+    for crate_dir in members:
+        # Split the path name with the directory separator
+        dir_parts = crate_dir.split(os.path.sep)
+        if dir_parts:
+            # Group crates by the first element and join them back using the separator
+            dir_name = os.path.sep.join(dir_parts[:1])
+            if dir_name not in crate_groups:
+                crate_groups[dir_name] = []
+            crate_groups[dir_name].append(crate_dir)
+    return crate_groups
+
+def partition_workspace(m, n):
+    cargo_toml = read_cargo_toml("Cargo.toml")
+
+    if cargo_toml is None:
+        raise Exception("Cargo.toml not found.")
+
+    workspace = cargo_toml.get("workspace", {})
+    members = workspace.get("members", [])
+
+    if not members:
+        raise Exception("No members found in the workspace.")
+
+    # Group crates based on directory structure using directory separators
+    crate_directory_groups = group_crates_by_first_directory(members)
+
+    # Calculate elements per partition and remainder based on the number of crate directories
+    total_directories = len(crate_directory_groups)
+    elements_per_partition = total_directories // m
+    remainder = total_directories % m
+
+    # Calculate the start and end indices for the current partition
+    start_idx = n * elements_per_partition + min(n, remainder)
+    end_idx = start_idx + elements_per_partition + (1 if n < remainder else 0)
+
+    # Get the crate directories for the current partition
+    partition_directories = []
+    for dir_path in list(crate_directory_groups.keys())[start_idx:end_idx]:
+        partition_directories.extend(crate_directory_groups[dir_path])
+
+    # Get the package names from the crate directories in this partition
+    package_names = [get_package_name(crate_dir) for crate_dir in partition_directories]
+
+    return package_names
+
+if __name__ == "__main__":
+    if len(sys.argv) == 2 and (sys.argv[1] == "--help" or sys.argv[1] == "-h"):
+        print("Usage: python partition_workspace.py <total_partitions> <partition_index> [prefix]")
+        print("")
+        print("Partitions the workspace into m partitions and returns the crates in the n-th partition.")
+        print("This can be used to split the workload of running tests across multiple CI jobs.")
+        print("")
+        print("To run the tests for the n-th partition, use the following command (for 3 partitions):")
+        print("---")
+        print("python3 {} 3 0 -p | xargs cargo test".format(sys.argv[0]))
+        print("python3 {} 3 1 -p | xargs cargo test".format(sys.argv[0]))
+        print("python3 {} 3 2 -p | xargs cargo test".format(sys.argv[0]))
+        print("---")
+        sys.exit(1)
+
+    if len(sys.argv) < 3 or len(sys.argv) > 4:
+        print("Usage: python3 {} <total_partitions> <partition_index> [prefix]".format(sys.argv[0]))
+        sys.exit(1)
+
+    total_splits = int(sys.argv[1])
+    partition_index = int(sys.argv[2])
+    prefix = sys.argv[3] + " " if len(sys.argv) == 4 else ""
+
+    if total_splits <= 0 or partition_index < 0 or partition_index >= total_splits:
+        print("Invalid input.")
+        sys.exit(1)
+
+    partition = partition_workspace(total_splits, partition_index)
+    partition_with_prefix = [prefix + member for member in partition]
+
+    print("{}".format(" ".join(partition_with_prefix)))