[manylinux2014] Rework scripts to be more docker cache friendly

This won't change travis-ci build time (at least for now) but allows maintainers/contributors to benefit from docker caching for local developments (faster builds at the expense of higher disk usage)

Round 1

Use docker multi-stage build to build the manylinux image in 3 steps:

1. runtime_base: this image has all the necessary bits that are common to the build_base image and the manylinux image. This includes system packages required for runtime, the gcc toolchain & some basic tools.

2. build_base: this image builds all the remaining tools required for the manylinux image. Any tool requiring specific system development package not required in the manylinux image shall be built from this image. For tools that used to be installed in /usr/local, they are staged in /manylinux-rootfs for an easy copy in the third stage.

3. manylinux: This image uses the runtime_base image as a base image. Tools built in the 2nd step are copied directly in the final filesystem. A finalization script is then run to install the remaining python tools/dependencies and run checks.

Author: mayeut

docker/Dockerfile

@@ -2,11 +2,11 @@
 ARG BASEIMAGE=amd64/centos:7
 ARG POLICY=manylinux2014
 ARG PLATFORM=x86_64
-ARG DEVTOOLSET_ROOTPATH=
-ARG LD_LIBRARY_PATH_ARG=
-ARG PREPEND_PATH=
+ARG DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root
+ARG LD_LIBRARY_PATH_ARG=${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
+ARG PREPEND_PATH=${DEVTOOLSET_ROOTPATH}/usr/bin:
 
-FROM $BASEIMAGE
+FROM $BASEIMAGE AS runtime_base
 ARG POLICY
 ARG PLATFORM
 ARG DEVTOOLSET_ROOTPATH
@@ -22,14 +22,36 @@ ENV PATH=${PREPEND_PATH}${PATH}
 ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
 
 # setup entrypoint, this will wrap commands with `linux32` with i686 images
-COPY build_scripts/install-entrypoint.sh /build_scripts/install-entrypoint.sh
+COPY build_scripts/install-entrypoint.sh /build_scripts/
 RUN bash /build_scripts/install-entrypoint.sh && rm -rf build_scripts
 COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint
 ENTRYPOINT ["manylinux-entrypoint"]
 
-COPY build_scripts /build_scripts
+
+COPY build_scripts/install-runtime-packages.sh /build_scripts/
+COPY build_scripts/build_utils.sh /build_scripts/
+COPY build_scripts/build_env_runtime.sh /build_scripts/
+RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts
+
+FROM runtime_base AS build_base
+COPY build_scripts/*pubkey*.txt /build_scripts/
+COPY build_scripts/build_utils.sh /build_scripts/
+COPY build_scripts/build.sh /build_scripts/
+COPY build_scripts/build_env.sh /build_scripts/
 RUN manylinux-entrypoint bash build_scripts/build.sh && rm -rf build_scripts
 
+FROM runtime_base
+COPY --from=build_base /manylinux-rootfs /
+COPY --from=build_base /opt/_internal /opt/_internal/
+COPY build_scripts/build_utils.sh /build_scripts/build_utils.sh
+COPY build_scripts/finalize.sh /build_scripts/finalize.sh
+COPY build_scripts/python-tag-abi-tag.py /build_scripts/python-tag-abi-tag.py
+COPY build_scripts/ssl-check.py /build_scripts/ssl-check.py
+COPY build_scripts/manylinux-check.py /build_scripts/manylinux-check.py
+COPY build_scripts/requirements.txt /build_scripts/requirements.txt
+COPY build_scripts/requirements-tools.txt /build_scripts/requirements-tools.txt
+RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts
+
 ENV SSL_CERT_FILE=/opt/_internal/certs.pem
 
 CMD ["/bin/bash"]

docker/build_scripts/build.sh

@@ -6,119 +6,40 @@ set -ex
 
 # Set build environment variables
 MY_DIR=$(dirname "${BASH_SOURCE[0]}")
-. $MY_DIR/build_env.sh
-
-# Dependencies for compiling Python that we want to remove from
-# the final image after compiling Python
-PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel expat-devel ncurses-devel readline-devel tk-devel gdbm-devel libdb-devel libpcap-devel xz-devel openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel perl-devel"
-
-# Libraries that are allowed as part of the manylinux2014 profile
-# Extract from PEP: https://www.python.org/dev/peps/pep-0599/#the-manylinux2014-policy
-# On RPM-based systems, they are provided by these packages:
-# Package:    Libraries
-# glib2:      libglib-2.0.so.0, libgthread-2.0.so.0, libgobject-2.0.so.0
-# glibc:      libresolv.so.2, libutil.so.1, libnsl.so.1, librt.so.1, libpthread.so.0, libdl.so.2, libm.so.6, libc.so.6
-# libICE:     libICE.so.6
-# libX11:     libX11.so.6
-# libXext:    libXext.so.6
-# libXrender: libXrender.so.1
-# libgcc:     libgcc_s.so.1
-# libstdc++:  libstdc++.so.6
-# mesa:       libGL.so.1
-#
-# PEP is missing the package for libSM.so.6 for RPM based system
-# Install development packages (except for libgcc which is provided by gcc install)
-MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel"
-
-CMAKE_DEPS="openssl-devel zlib-devel libcurl-devel"
+source $MY_DIR/build_env.sh
 
 # Get build utilities
 source $MY_DIR/build_utils.sh
 
-# See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages
-echo "multilib_policy=best" >> /etc/yum.conf
-# Error out if requested packages do not exist
-echo "skip_missing_names_on_install=False" >> /etc/yum.conf
-# Make sure that locale will not be removed
-sed -i '/^override_install_langs=/d' /etc/yum.conf
-
-# https://hub.docker.com/_/centos/
-# "Additionally, images with minor version tags that correspond to install
-# media are also offered. These images DO NOT recieve updates as they are
-# intended to match installation iso contents. If you choose to use these
-# images it is highly recommended that you include RUN yum -y update && yum
-# clean all in your Dockerfile, or otherwise address any potential security
-# concerns."
-# Decided not to clean at this point: https://github.com/pypa/manylinux/pull/129
-yum -y update
-yum -y install yum-utils curl
-yum-config-manager --enable extras
-
-if ! which localedef &> /dev/null; then
-    # somebody messed up glibc-common package to squeeze image size, reinstall the package
-    yum -y reinstall glibc-common
-fi
-
-# upgrading glibc-common can end with removal on en_US.UTF-8 locale
-localedef -i en_US -f UTF-8 en_US.UTF-8
-
-TOOLCHAIN_DEPS="devtoolset-9-binutils devtoolset-9-gcc devtoolset-9-gcc-c++ devtoolset-9-gcc-gfortran"
-if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then
-    # Software collection (for devtoolset-9)
-    yum -y install centos-release-scl-rh
-    # EPEL support (for yasm)
-    yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-    YASM=yasm
-elif [ "${AUDITWHEEL_ARCH}" == "aarch64" ] || [ "${AUDITWHEEL_ARCH}" == "ppc64le" ] || [ "${AUDITWHEEL_ARCH}" == "s390x" ]; then
-    # Software collection (for devtoolset-9)
-    yum -y install centos-release-scl-rh
-elif [ "${AUDITWHEEL_ARCH}" == "i686" ]; then
-    # No yasm on i686
-    # Install mayeut/devtoolset-9 repo to get devtoolset-9
-    curl -fsSLo /etc/yum.repos.d/mayeut-devtoolset-9.repo https://copr.fedorainfracloud.org/coprs/mayeut/devtoolset-9/repo/custom-1/mayeut-devtoolset-9-custom-1.repo
-fi
+# Dependencies for compiling Python that we want to remove from
+# the final image after compiling Python
+PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel expat-devel ncurses-devel readline-devel tk-devel gdbm-devel libdb-devel libpcap-devel xz-devel openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel perl-devel libffi-devel kernel-devel"
+CMAKE_DEPS="openssl-devel zlib-devel libcurl-devel"
 
 # Development tools and libraries
-yum -y install \
-    autoconf \
-    automake \
-    bison \
-    bzip2 \
-    ${TOOLCHAIN_DEPS} \
-    diffutils \
-    gettext \
-    file \
-    kernel-devel \
-    libffi-devel \
-    make \
-    patch \
-    unzip \
-    which \
-    ${YASM} \
-    ${PYTHON_COMPILE_DEPS} \
-    ${CMAKE_DEPS}
+yum -y install ${PYTHON_COMPILE_DEPS} ${CMAKE_DEPS}
 
 # Install git
 build_git $GIT_ROOT $GIT_HASH
-git version
-
-# Install newest automake
-build_automake $AUTOMAKE_ROOT $AUTOMAKE_HASH
-automake --version
-
-# Install newest libtool
-build_libtool $LIBTOOL_ROOT $LIBTOOL_HASH
-libtool --version
+/manylinux-rootfs/usr/local/bin/git version
 
 # Install a more recent SQLite3
 curl -fsSLO $SQLITE_AUTOCONF_DOWNLOAD_URL/$SQLITE_AUTOCONF_ROOT.tar.gz
 check_sha256sum $SQLITE_AUTOCONF_ROOT.tar.gz $SQLITE_AUTOCONF_HASH
 tar xfz $SQLITE_AUTOCONF_ROOT.tar.gz
 cd $SQLITE_AUTOCONF_ROOT
-do_standard_install
+DESTDIR=/sqlite3 do_standard_install
 cd ..
 rm -rf $SQLITE_AUTOCONF_ROOT*
-rm /usr/local/lib/libsqlite3.a
+rm /sqlite3/usr/local/lib/libsqlite3.a
+# Install for build
+cp -rf /sqlite3/* /
+# Clean-up for runtime
+rm -rf /sqlite3/usr/local/bin /sqlite3/usr/local/include /sqlite3/usr/local/lib/pkg-config /sqlite3/usr/local/share
+# Install for runtime
+cp -rf /sqlite3/* /manylinux-rootfs/
+# clean-up
+rm -rf /sqlite3
 
 # Install a recent version of cmake3
 curl -L -O $CMAKE_DOWNLOAD_URL/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz
@@ -127,101 +48,29 @@ tar -xzf cmake-${CMAKE_VERSION}.tar.gz
 cd cmake-${CMAKE_VERSION}
 ./bootstrap --system-curl --parallel=$(nproc)
 make -j$(nproc)
-make install
+make install DESTDIR=/manylinux-rootfs
 cd ..
-rm -rf cmake-${CMAKE_VERSION}
-
-# Install libcrypt.so.1 and libcrypt.so.2
-build_libxcrypt "$LIBXCRYPT_DOWNLOAD_URL" "$LIBXCRYPT_VERSION" "$LIBXCRYPT_HASH"
+rm -rf cmake-${CMAKE_VERSION}*
 
 # Compile the latest Python releases.
 # (In order to have a proper SSL module, Python is compiled
 # against a recent openssl [see env vars above], which is linked
 # statically.
-mkdir -p /opt/python
 build_cpythons $CPYTHON_VERSIONS
 
-# Create venv for auditwheel & certifi
-TOOLS_PATH=/opt/_internal/tools
-/opt/python/cp37-cp37m/bin/python -m venv $TOOLS_PATH
-source $TOOLS_PATH/bin/activate
-
-# Install default packages
-pip install -U --require-hashes -r $MY_DIR/requirements.txt
-# Install certifi and auditwheel
-pip install -U --require-hashes -r $MY_DIR/requirements-tools.txt
-
-# Make auditwheel available in PATH
-ln -s $TOOLS_PATH/bin/auditwheel /usr/local/bin/auditwheel
-
-# Our openssl doesn't know how to find the system CA trust store
-#   (https://github.com/pypa/manylinux/issues/53)
-# And it's not clear how up-to-date that is anyway
-# So let's just use the same one pip and everyone uses
-ln -s $(python -c 'import certifi; print(certifi.where())') /opt/_internal/certs.pem
-# If you modify this line you also have to modify the versions in the Dockerfiles:
-export SSL_CERT_FILE=/opt/_internal/certs.pem
-
-# Deactivate the tools virtual environment
-deactivate
-
-# Install patchelf (latest with unreleased bug fixes) and apply our patches
-build_patchelf $PATCHELF_VERSION $PATCHELF_HASH
-
-# Clean up development headers and other unnecessary stuff for
-# final image
-yum -y erase \
-    avahi \
-    bitstream-vera-fonts \
-    freetype \
-    gettext \
-    gtk2 \
-    hicolor-icon-theme \
-    libX11 \
-    wireless-tools \
-    ${PYTHON_COMPILE_DEPS} > /dev/null 2>&1
-yum -y install ${MANYLINUX_DEPS}
-yum -y clean all > /dev/null 2>&1
-yum list installed
-
 # we don't need libpython*.a, and they're many megabytes
 find /opt/_internal -name '*.a' -print0 | xargs -0 rm -f
 
 # Strip what we can -- and ignore errors, because this just attempts to strip
 # *everything*, including non-ELF files:
 find /opt/_internal -type f -print0 \
     | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true
-find /usr/local -type f -print0 \
+find /manylinux-rootfs -type f -print0 \
     | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true
 
-for PYTHON in /opt/python/*/bin/python; do
-    # Smoke test to make sure that our Pythons work, and do indeed detect as
-    # being manylinux compatible:
-    $PYTHON $MY_DIR/manylinux-check.py
-    # Make sure that SSL cert checking works
-    $PYTHON $MY_DIR/ssl-check.py
-done
-
 # We do not need the Python test suites, or indeed the precompiled .pyc and
 # .pyo files. Partially cribbed from:
 #    https://github.com/docker-library/python/blob/master/3.4/slim/Dockerfile
 find /opt/_internal -depth \
      \( -type d -a -name test -o -name tests \) \
   -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) | xargs rm -rf
-
-# Fix libc headers to remain compatible with C99 compilers.
-find /usr/include/ -type f -exec sed -i 's/\bextern _*inline_*\b/extern __inline __attribute__ ((__gnu_inline__))/g' {} +
-
-if [ "${DEVTOOLSET_ROOTPATH:-}" != "" ]; then
-    # remove useless things that have been installed by devtoolset
-    rm -rf $DEVTOOLSET_ROOTPATH/usr/share/man
-    find $DEVTOOLSET_ROOTPATH/usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf
-fi
-rm -rf /usr/share/backgrounds
-# if we updated glibc, we need to strip locales again...
-localedef --list-archive | grep -v -i ^en_US.utf8 | xargs localedef --delete-from-archive
-mv -f /usr/lib/locale/locale-archive /usr/lib/locale/locale-archive.tmpl
-build-locale-archive
-find /usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf
-find /usr/local/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf
-rm -rf /usr/local/share/man

docker/build_scripts/build_env.sh

@@ -4,18 +4,6 @@ PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python
 # of the form <maj>.<min>.<rev> or <maj>.<min>.<rev>rc<n>
 CPYTHON_VERSIONS="3.5.10 3.6.12 3.7.9 3.8.7 3.9.1"
 
-PATCHELF_VERSION=0.12
-PATCHELF_HASH=3dca33fb862213b3541350e1da262249959595903f559eae0fbc68966e9c3f56
-PATCHELF_DOWNLOAD_URL=https://github.com/NixOS/patchelf/archive
-
-AUTOMAKE_ROOT=automake-1.16.2
-AUTOMAKE_HASH=b2f361094b410b4acbf4efba7337bdb786335ca09eb2518635a09fb7319ca5c1
-AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake
-
-LIBTOOL_ROOT=libtool-2.4.6
-LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3
-LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool
-
 SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3340000
 SQLITE_AUTOCONF_HASH=bf6db7fae37d51754737747aaaf413b4d6b3b5fbacd52bdb2d0d6e5b2edd9aee
 SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2020
@@ -24,10 +12,6 @@ CMAKE_VERSION=3.18.3
 CMAKE_HASH=2c89f4e30af4914fd6fb5d00f863629812ada848eee4e2d29ec7e456d7fa32e5
 CMAKE_DOWNLOAD_URL=https://github.com/Kitware/CMake/releases/download
 
-LIBXCRYPT_VERSION=4.4.17
-LIBXCRYPT_HASH=7665168d0409574a03f7b484682e68334764c29c21ca5df438955a381384ca07
-LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive
-
 GIT_ROOT=git-2.30.0
 GIT_HASH=d24c4fa2a658318c2e66e25ab67cc30038a35696d2d39e6b12ceccf024de1e5e
 GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git

docker/build_scripts/build_env_runtime.sh

@@ -0,0 +1,17 @@
+# source me
+
+PATCHELF_VERSION=0.12
+PATCHELF_HASH=3dca33fb862213b3541350e1da262249959595903f559eae0fbc68966e9c3f56
+PATCHELF_DOWNLOAD_URL=https://github.com/NixOS/patchelf/archive
+
+AUTOMAKE_ROOT=automake-1.16.2
+AUTOMAKE_HASH=b2f361094b410b4acbf4efba7337bdb786335ca09eb2518635a09fb7319ca5c1
+AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake
+
+LIBTOOL_ROOT=libtool-2.4.6
+LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3
+LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool
+
+LIBXCRYPT_VERSION=4.4.17
+LIBXCRYPT_HASH=7665168d0409574a03f7b484682e68334764c29c21ca5df438955a381384ca07
+LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive

docker/build_scripts/build_utils.sh

@@ -35,25 +35,11 @@ function do_cpython_build {
     pushd Python-$py_ver
     local prefix="/opt/_internal/cpython-${py_ver}"
     mkdir -p ${prefix}/lib
-    ./configure --prefix=${prefix} --disable-shared > /dev/null
+    ./configure --prefix=${prefix} --disable-shared --with-ensurepip=no > /dev/null
     make -j$(nproc) > /dev/null
     make -j$(nproc) install > /dev/null
     popd
     rm -rf Python-$py_ver
-    # Some python's install as bin/python3. Make them available as
-    # bin/python.
-    if [ -e ${prefix}/bin/python3 ] && [ ! -e ${prefix}/bin/python ]; then
-        ln -s python3 ${prefix}/bin/python
-    fi
-    ${prefix}/bin/python -m ensurepip
-    if [ -e ${prefix}/bin/pip3 ] && [ ! -e ${prefix}/bin/pip ]; then
-        ln -s pip3 ${prefix}/bin/pip
-    fi
-    # Since we fall back on a canned copy of pip, we might not have
-    # the latest pip and friends. Upgrade them to make sure.
-    ${prefix}/bin/pip install -U --require-hashes -r ${MY_DIR}/requirements.txt
-    local abi_tag=$(${prefix}/bin/python ${MY_DIR}/python-tag-abi-tag.py)
-    ln -s ${prefix} /opt/python/${abi_tag}
 }
 
 
@@ -121,7 +107,7 @@ function build_git {
     fetch_source ${git_fname}.tar.gz ${GIT_DOWNLOAD_URL}
     check_sha256sum ${git_fname}.tar.gz ${git_sha256}
     tar -xzf ${git_fname}.tar.gz
-    (cd ${git_fname} && make -j$(nproc) install prefix=/usr/local NO_GETTEXT=1 NO_TCLTK=1 > /dev/null)
+    (cd ${git_fname} && make -j$(nproc) install prefix=/usr/local NO_GETTEXT=1 NO_TCLTK=1 DESTDIR=/manylinux-rootfs > /dev/null)
     rm -rf ${git_fname} ${git_fname}.tar.gz
 }