From b81391ec76f8f18b92b0aee7369c7bda2864e0d9 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Tue, 3 Dec 2024 08:17:33 +0100
Subject: [PATCH] CI: use free runners for i686-gnu jobs
---
.../host-x86_64/i686-gnu-nopt/Dockerfile | 4 +-
src/ci/docker/host-x86_64/i686-gnu/Dockerfile | 9 +--
src/ci/docker/run.sh | 35 +++++++-----
src/ci/github-actions/jobs.yml | 55 +++++++++++++++++--
4 files changed, 77 insertions(+), 26 deletions(-)
diff --git a/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile b/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile
index e2b66c2cff1d8..e273672060732 100644
--- a/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile
+++ b/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile
@@ -27,5 +27,5 @@ RUN echo "[rust]" > /config/nopt-std-config.toml
RUN echo "optimize = false" >> /config/nopt-std-config.toml
ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu --disable-optimize-tests
-ENV SCRIPT python3 ../x.py test --stage 0 --config /config/nopt-std-config.toml library/std \
- && python3 ../x.py --stage 2 test
+ARG SCRIPT_ARG
+ENV SCRIPT=${SCRIPT_ARG}
diff --git a/src/ci/docker/host-x86_64/i686-gnu/Dockerfile b/src/ci/docker/host-x86_64/i686-gnu/Dockerfile
index 61811c41904c4..dec25461bb4e8 100644
--- a/src/ci/docker/host-x86_64/i686-gnu/Dockerfile
+++ b/src/ci/docker/host-x86_64/i686-gnu/Dockerfile
@@ -24,10 +24,5 @@ COPY scripts/sccache.sh /scripts/
RUN sh /scripts/sccache.sh
ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu
-# Skip some tests that are unlikely to be platform specific, to speed up
-# this slow job.
-ENV SCRIPT python3 ../x.py --stage 2 test \
- --skip src/bootstrap \
- --skip tests/rustdoc-js \
- --skip src/tools/error_index_generator \
- --skip src/tools/linkchecker
+ARG SCRIPT_ARG
+ENV SCRIPT=${SCRIPT_ARG}
diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh
index d554186df4cfe..a0adf60b6b2c9 100755
--- a/src/ci/docker/run.sh
+++ b/src/ci/docker/run.sh
@@ -105,6 +105,23 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
# It seems that it cannot be the same as $IMAGE_TAG, otherwise it overwrites the cache
CACHE_IMAGE_TAG=${REGISTRY}/${REGISTRY_USERNAME}/rust-ci-cache:${cksum}
+ # Docker build arguments.
+ build_args=(
+ "build"
+ "--rm"
+ "-t" "rust-ci"
+ "-f" "$dockerfile"
+ "$context"
+ )
+
+ # If the environment variable DOCKER_SCRIPT is defined,
+ # set the build argument SCRIPT_ARG to DOCKER_SCRIPT.
+ # In this way, we run the script defined in CI,
+ # instead of the one defined in the Dockerfile.
+ if [ -n "${DOCKER_SCRIPT+x}" ]; then
+ build_args+=("--build-arg" "SCRIPT_ARG=${DOCKER_SCRIPT}")
+ fi
+
# On non-CI jobs, we try to download a pre-built image from the rust-lang-ci
# ghcr.io registry. If it is not possible, we fall back to building the image
# locally.
@@ -115,7 +132,7 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
docker tag "${IMAGE_TAG}" rust-ci
else
echo "Building local Docker image"
- retry docker build --rm -t rust-ci -f "$dockerfile" "$context"
+ retry docker "${build_args[@]}"
fi
# On PR CI jobs, we don't have permissions to write to the registry cache,
# but we can still read from it.
@@ -127,13 +144,9 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
# Build the image using registry caching backend
retry docker \
buildx \
- build \
- --rm \
- -t rust-ci \
- -f "$dockerfile" \
+ "${build_args[@]}" \
--cache-from type=registry,ref=${CACHE_IMAGE_TAG} \
- --output=type=docker \
- "$context"
+ --output=type=docker
# On auto/try builds, we can also write to the cache.
else
# Log into the Docker registry, so that we can read/write cache and the final image
@@ -147,14 +160,10 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
# Build the image using registry caching backend
retry docker \
buildx \
- build \
- --rm \
- -t rust-ci \
- -f "$dockerfile" \
+ "${build_args[@]}" \
--cache-from type=registry,ref=${CACHE_IMAGE_TAG} \
--cache-to type=registry,ref=${CACHE_IMAGE_TAG},compression=zstd \
- --output=type=docker \
- "$context"
+ --output=type=docker
# Print images for debugging purposes
docker images
diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml
index 454c855c75d25..e74e4875453e4 100644
--- a/src/ci/github-actions/jobs.yml
+++ b/src/ci/github-actions/jobs.yml
@@ -58,6 +58,22 @@ envs:
NO_DEBUG_ASSERTIONS: 1
NO_OVERFLOW_CHECKS: 1
+ # Different set of tests to run tests in parallel in multiple jobs.
+ stage_2_test_set1: &stage_2_test_set1
+ DOCKER_SCRIPT: >-
+ python3 ../x.py --stage 2 test
+ --skip compiler
+ --skip src
+
+ stage_2_test_set2: &stage_2_test_set2
+ DOCKER_SCRIPT: >-
+ python3 ../x.py --stage 2 test
+ --skip tests
+ --skip coverage-map
+ --skip coverage-run
+ --skip library
+ --skip tidyselftest
+
production:
&production
DEPLOY_BUCKET: rust-lang-ci2
@@ -212,11 +228,42 @@ auto:
- image: dist-x86_64-netbsd
<<: *job-linux-4c
- - image: i686-gnu
- <<: *job-linux-8c
+ # The i686-gnu job is split into multiple jobs to run tests in parallel.
+ # i686-gnu-1 skips tests that run in i686-gnu-2.
+ - image: i686-gnu-1
+ env:
+ IMAGE: i686-gnu
+ <<: *stage_2_test_set1
+ <<: *job-linux-4c
- - image: i686-gnu-nopt
- <<: *job-linux-8c
+ # Skip tests that run in i686-gnu-1
+ - image: i686-gnu-2
+ env:
+ IMAGE: i686-gnu
+ <<: *stage_2_test_set2
+ <<: *job-linux-4c
+
+ # The i686-gnu-nopt job is split into multiple jobs to run tests in parallel.
+ # i686-gnu-nopt-1 skips tests that run in i686-gnu-nopt-2
+ - image: i686-gnu-nopt-1
+ env:
+ IMAGE: i686-gnu-nopt
+ <<: *stage_2_test_set1
+ <<: *job-linux-4c
+
+ # Skip tests that run in i686-gnu-nopt-1
+ - image: i686-gnu-nopt-2
+ env:
+ IMAGE: i686-gnu-nopt
+ DOCKER_SCRIPT: >-
+ python3 ../x.py test --stage 0 --config /config/nopt-std-config.toml library/std &&
+ python3 ../x.py --stage 2 test
+ --skip tests
+ --skip coverage-map
+ --skip coverage-run
+ --skip library
+ --skip tidyselftest
+ <<: *job-linux-4c
- image: mingw-check
<<: *job-linux-4c