From 59536c4ff5b88d2080c51b082a3507b08065cf6e Mon Sep 17 00:00:00 2001 From: eric Date: Fri, 10 Nov 2023 10:58:35 -0500 Subject: [PATCH] Refactor dockerfile to support Buildkite AND Github Actions This change adds the image used by the self-hosted Github Actions builders. In an attempt to make the transition simple, all of the different images share as much of the same state as possible, including packages, users, etc... This results in bigger images, but that shouldn't be a problem. This change also renames all of the packages for consistency. Bots will have to be changed to use the new package names eventually. Again, docker-compose was used as the source of truth for defining argumentThis change also renames all of the packages for consistency. Bots will have to be changed to use the new package names eventually. Again, docker-compose was used as the source of truth for defining arguments I have already pushed example images to ghcr.io/libcxx/:testing --- libcxx/utils/ci/Dockerfile | 348 ++++++++++++++++------ libcxx/utils/ci/docker-compose.yml | 37 ++- libcxx/utils/ci/vendor/android/Dockerfile | 83 ------ libcxx/utils/data/ignore_format.txt | 1 - 4 files changed, 287 insertions(+), 182 deletions(-) delete mode 100644 libcxx/utils/ci/vendor/android/Dockerfile diff --git a/libcxx/utils/ci/Dockerfile b/libcxx/utils/ci/Dockerfile index c70a6ca8f1dcd..5510e4d6201b1 100644 --- a/libcxx/utils/ci/Dockerfile +++ b/libcxx/utils/ci/Dockerfile @@ -5,11 +5,29 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # #===----------------------------------------------------------------------===## - # -# This Dockerfile describes the base image used to run the various libc++ -# build bots. By default, the image runs the Buildkite Agent, however one -# can also just start the image with a shell to debug CI failures. +# This file defines the buildkite and github actions builder images. +# You can build & push both images using: +# +# docker compose build +# docker compose push +# +# Or you can select a single image to build & push using: +# +# docker compose build buildkite-builder +# docker compose push buildkite-builder +# +# The final images can be found at +# +# ghcr.io/libcxx/buildkite-builder +# ghcr.io/libcxx/actions-builder +# ghcr.io/libcxx/android-buildkite-builder +# +# Members of the github.com/libcxx/ organizations have permissions required to push new images. +# +# ===----------------------------------------------------------------------===## +# Running the buildkite image +# ===----------------------------------------------------------------------===## # # To start a Buildkite Agent, run it as: # $ docker run --env-file -it $(docker build -q libcxx/utils/ci) @@ -21,42 +39,82 @@ # # If you're only looking to run the Docker image locally for debugging a # build bot, see the `run-buildbot-container` script located in this directory. -# -# A pre-built version of this image is maintained on Github under the libc++ organization, as ghcr.io/libcxx/libcxx-builder. -# To update the image, rebuild it and push it to github (all members of the libc++ organization should be able to do this). -# -# $ docker compose build -# $ docker compose push -# -FROM ubuntu:jammy + +# HACK: We set the base image in the docker-compose file depending on the final target (buildkite vs github actions). +# This means we have a much slower container build, but we can use the same Dockerfile for both targets. +ARG BASE_IMAGE +FROM $BASE_IMAGE AS builder-base # Make sure apt-get doesn't try to prompt for stuff like our time zone, etc. ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y bash curl +# populated in the docker-compose file +ARG GCC_LATEST_VERSION +ENV GCC_LATEST_VERSION=${GCC_LATEST_VERSION} + +# populated in the docker-compose file +ARG LLVM_HEAD_VERSION +ENV LLVM_HEAD_VERSION=${LLVM_HEAD_VERSION} + +# HACK: The github actions runner image already has sudo and requires its use. The buildkite base image does not. +# Reconcile this. +RUN <> /etc/locale.gen -RUN mkdir /usr/local/share/i1en/ -RUN printf "fr_CA ISO-8859-1\ncs_CZ ISO-8859-2" >> /usr/local/share/i1en/SUPPORTED -RUN locale-gen +RUN <, and ToT, which are the ones we support. # We also install because we need to support the "latest-1" of the @@ -65,75 +123,179 @@ RUN locale-gen # LLVM 15, we still need to have Clang 12 in this Docker image because the LLVM # 14 release branch CI uses it. The tip-of-trunk CI will never use Clang 12, # though. -ARG LLVM_HEAD_VERSION # populated in the docker-compose file -ENV LLVM_HEAD_VERSION=${LLVM_HEAD_VERSION} -RUN apt-get update && apt-get install -y lsb-release wget software-properties-common -RUN wget https://apt.llvm.org/llvm.sh -O /tmp/llvm.sh -RUN bash /tmp/llvm.sh $(($LLVM_HEAD_VERSION - 3)) # for CI transitions -RUN bash /tmp/llvm.sh $(($LLVM_HEAD_VERSION - 2)) # previous release -RUN bash /tmp/llvm.sh $(($LLVM_HEAD_VERSION - 1)) # latest release -RUN bash /tmp/llvm.sh $LLVM_HEAD_VERSION # current ToT - -# Install clang-scan-deps, which is required to build modules; always all supported versions. -RUN apt-get update && apt-get install -y clang-tools-$(($LLVM_HEAD_VERSION - 3)) \ - clang-tools-$(($LLVM_HEAD_VERSION - 2)) \ - clang-tools-$(($LLVM_HEAD_VERSION - 1)) \ - clang-tools-$LLVM_HEAD_VERSION - -# Install clang-format; always use the lastest stable branch. -RUN apt-get update && apt-get install -y clang-format-$(($LLVM_HEAD_VERSION - 2)) clang-format-$(($LLVM_HEAD_VERSION - 1)) - -# Install clang-tidy -# TODO(LLVM-17) revert D148831 to only install $(($LLVM_HEAD_VERSION - 1)) and $LLVM_HEAD_VERSION -# The usage of the ToT version is needed due to module issues with Clang 16 -RUN apt-get update && apt-get install -y clang-tidy-$(($LLVM_HEAD_VERSION - 2)) clang-tidy-$(($LLVM_HEAD_VERSION - 1)) clang-tidy-$LLVM_HEAD_VERSION - -# Install llvm-dev and libclang-dev to compile custom clang-tidy checks -# TODO(LLVM-17) revert D148831 to only install $(($LLVM_HEAD_VERSION - 1)) and $LLVM_HEAD_VERSION -# The usage of the ToT version is needed due to module issues with Clang 16 -RUN apt-get update && apt-get install -y llvm-$(($LLVM_HEAD_VERSION - 2))-dev llvm-$(($LLVM_HEAD_VERSION - 1))-dev llvm-$LLVM_HEAD_VERSION-dev \ - libclang-$(($LLVM_HEAD_VERSION - 2))-dev libclang-$(($LLVM_HEAD_VERSION - 1))-dev libclang-$LLVM_HEAD_VERSION-dev \ - libomp5-$LLVM_HEAD_VERSION +RUN <> /etc/sudoers -RUN useradd --create-home libcxx-builder +RUN <> /home/libcxx-builder/.buildkite-agent/buildkite-agent.cfg +EOF + +USER libcxx-builder +WORKDIR /home/libcxx-builder + ENV PATH="${PATH}:/home/libcxx-builder/.buildkite-agent/bin" -RUN echo "tags=\"queue=libcxx-builders,arch=$(uname -m),os=linux\"" >> "/home/libcxx-builder/.buildkite-agent/buildkite-agent.cfg" -# By default, start the Buildkite agent (this requires a token). CMD ["buildkite-agent", "start"] + +# ===----------------------------------------------------------------------===## +# Android Buildkite Builder Image +# ===----------------------------------------------------------------------===## +# +# IMAGE: ghcr.io/libcxx/android-buildkite-builder. +# +FROM buildkite-builder AS android-buildkite-builder + +COPY --from=android-builder-base /opt/android /opt/android +COPY ./vendor/android/container-setup.sh /opt/android/container-setup.sh + +ENV PATH="/opt/android/sdk/platform-tools:${PATH}" + +USER libcxx-builder +WORKDIR /home/libcxx-builder + +# Reset the configuration, we pass the configuration via the environment. +RUN cp /home/libcxx-builder/.buildkite-agent/buildkite-agent.dist.cfg \ + /home/libcxx-builder/.buildkite-agent/buildkite-agent.cfg + +# Modify the Buildkite agent cmdline to do Android setup stuff first. +CMD /opt/android/container-setup.sh && buildkite-agent start + +# ===----------------------------------------------------------------------===## +# Github Actions Builder Image +# ===----------------------------------------------------------------------===## +# +# IMAGE: ghcr.io/libcxx/actions-builder. +# +FROM builder-base AS actions-builder + +WORKDIR /home/runner +USER runner + + + diff --git a/libcxx/utils/ci/docker-compose.yml b/libcxx/utils/ci/docker-compose.yml index 7f03d8f3af0f2..240e5b2828aa0 100644 --- a/libcxx/utils/ci/docker-compose.yml +++ b/libcxx/utils/ci/docker-compose.yml @@ -1,11 +1,38 @@ version: '3.7' + +x-versions: &compiler_versions + GCC_LATEST_VERSION: 13 + LLVM_HEAD_VERSION: 18 + services: - libcxx-builder: - image: ghcr.io/libcxx/libcxx-builder + buildkite-builder: + image: ghcr.io/libcxx/buildkite-builder:testing build: context: . dockerfile: Dockerfile + target: buildkite-builder args: - GCC_LATEST_VERSION: 13 - # LLVM POST-BRANCH bump version - LLVM_HEAD_VERSION: 18 + BASE_IMAGE: ubuntu:jammy + <<: *compiler_versions + actions-builder: + image: ghcr.io/libcxx/actions-builder:testing + build: + context: . + dockerfile: Dockerfile + target: actions-builder + args: + BASE_IMAGE: ghcr.io/actions/actions-runner:latest + <<: *compiler_versions + android-buildkite-builder: + image: ghcr.io/libcxx/android-buildkite-builder:testing + build: + context: . + dockerfile: Dockerfile + target: android-buildkite-builder + args: + BASE_IMAGE: ubuntu:jammy + ANDROID_CLANG_VERSION: r498229b + ANDROID_CLANG_PREBUILTS_COMMIT: 5186d132c99aa75dc25207c392e3ea5b93d0107e + ANDROID_SYSROOT_BID: 10957860 + <<: *compiler_versions + diff --git a/libcxx/utils/ci/vendor/android/Dockerfile b/libcxx/utils/ci/vendor/android/Dockerfile deleted file mode 100644 index 4d35334f89bb2..0000000000000 --- a/libcxx/utils/ci/vendor/android/Dockerfile +++ /dev/null @@ -1,83 +0,0 @@ -#===----------------------------------------------------------------------===## -# -# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# -#===----------------------------------------------------------------------===## - -# -# This Dockerfile extends ghcr.io/libcxx/libcxx-builder with Android support, including -# Android Clang and sysroot, Android platform-tools, and the Docker client. -# -# $ docker build -t libcxx-builder-android libcxx/utils/ci/vendor/android -# - -FROM ghcr.io/libcxx/libcxx-builder - -# Switch back to the root user to install things into /opt and /usr. -USER root -WORKDIR / - -# Install the Android platform tools (e.g. adb) into /opt/android/sdk. -RUN apt-get update && apt-get install -y unzip -RUN mkdir -p /opt/android/sdk && cd /opt/android/sdk && \ - curl -LO https://dl.google.com/android/repository/platform-tools-latest-linux.zip && \ - unzip platform-tools-latest-linux.zip && \ - rm platform-tools-latest-linux.zip - -# Install the current Android compiler. Specify the prebuilts commit to retrieve -# this compiler version even after it's removed from HEAD. -ENV ANDROID_CLANG_VERSION=r498229b -ENV ANDROID_CLANG_PREBUILTS_COMMIT=5186d132c99aa75dc25207c392e3ea5b93d0107e -RUN git clone --filter=blob:none --sparse \ - https://android.googlesource.com/platform/prebuilts/clang/host/linux-x86 \ - /opt/android/clang && \ - git -C /opt/android/clang checkout ${ANDROID_CLANG_PREBUILTS_COMMIT} && \ - git -C /opt/android/clang sparse-checkout add clang-${ANDROID_CLANG_VERSION} && \ - rm -fr /opt/android/clang/.git && \ - ln -sf /opt/android/clang/clang-${ANDROID_CLANG_VERSION} /opt/android/clang/clang-current && \ - # The "git sparse-checkout" and "ln" commands succeed even if nothing was - # checked out, so use this "ls" command to fix that. - ls /opt/android/clang/clang-current/bin/clang - -# Install an Android sysroot. New AOSP sysroots are available at -# https://ci.android.com/builds/branches/aosp-main/grid, the "ndk" target. The -# NDK also makes its sysroot prebuilt available at -# https://android.googlesource.com/platform/prebuilts/ndk/+/refs/heads/dev/platform/sysroot. -ENV ANDROID_SYSROOT_BID=10957860 -RUN cd /opt/android && \ - curl -L -o ndk_platform.tar.bz2 \ - https://androidbuildinternal.googleapis.com/android/internal/build/v3/builds/${ANDROID_SYSROOT_BID}/ndk/attempts/latest/artifacts/ndk_platform.tar.bz2/url && \ - tar xf ndk_platform.tar.bz2 && \ - rm ndk_platform.tar.bz2 - -# Install Docker. Mark the binary setuid so it can be run without prefixing it -# with sudo. Adding the container user to the docker group doesn't work because -# /var/run/docker.sock is owned by the host's docker GID, not the container's -# docker GID. -RUN apt-get update && apt-get install -y gpg && \ - install -m 0755 -d /etc/apt/keyrings && \ - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \ - chmod a+r /etc/apt/keyrings/docker.gpg && \ - echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ - "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ - tee /etc/apt/sources.list.d/docker.list > /dev/null && \ - apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin && \ - chmod u+s /usr/bin/docker - -COPY ./container-setup.sh /opt/android/container-setup.sh - -USER libcxx-builder -WORKDIR /home/libcxx-builder - -# Add Android platform-tools to the PATH. -ENV PATH="/opt/android/sdk/platform-tools:${PATH}" - -# Reset the buildkite-agent.cfg file. The tags are provided by an environment -# variable instead. -RUN cp /home/libcxx-builder/.buildkite-agent/buildkite-agent.dist.cfg \ - /home/libcxx-builder/.buildkite-agent/buildkite-agent.cfg - -# Modify the Buildkite agent cmdline to do Android setup stuff first. -CMD /opt/android/container-setup.sh && buildkite-agent start diff --git a/libcxx/utils/data/ignore_format.txt b/libcxx/utils/data/ignore_format.txt index 48b3c04adb542..8ea614e0286a3 100644 --- a/libcxx/utils/data/ignore_format.txt +++ b/libcxx/utils/data/ignore_format.txt @@ -7282,6 +7282,5 @@ libcxx/utils/ci/Dockerfile libcxx/utils/ci/macos-ci-setup libcxx/utils/ci/run-buildbot libcxx/utils/ci/run-buildbot-container -libcxx/utils/ci/vendor/android/Dockerfile libcxx/utils/ci/vendor/android/run-buildbot-container libcxx/utils/libcxx-lit