Split cpython build by minor version

Author: mayeut

docker/Dockerfile

@@ -85,17 +85,48 @@ RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3340000 && \
     export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2020 && \
     manylinux-entrypoint /build_scripts/build-sqlite3.sh
 
-COPY build_scripts/*pubkey*.txt /build_scripts/
-COPY build_scripts/build_env.sh /build_scripts/
 COPY build_scripts/build-cpython.sh /build_scripts/
-RUN manylinux-entrypoint /build_scripts/build-cpython.sh
+
+
+FROM build_cpython AS build_cpython35
+COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint gpg --import /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.5.10
+
+
+FROM build_cpython AS build_cpython36
+COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint gpg --import /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.6.12
+
+
+FROM build_cpython AS build_cpython37
+COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint gpg --import /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.7.9
+
+
+FROM build_cpython AS build_cpython38
+COPY build_scripts/ambv-pubkey.txt /build_scripts/ambv-pubkey.txt
+RUN manylinux-entrypoint gpg --import /build_scripts/ambv-pubkey.txt
+RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.7
+
+
+FROM build_cpython AS build_cpython39
+COPY build_scripts/ambv-pubkey.txt /build_scripts/ambv-pubkey.txt
+RUN manylinux-entrypoint gpg --import /build_scripts/ambv-pubkey.txt
+RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.1
 
 
 FROM runtime_base
 COPY --from=build_git /manylinux-rootfs /
 COPY --from=build_cmake /manylinux-rootfs /
 COPY --from=build_cpython /manylinux-rootfs /
-COPY --from=build_cpython /opt/_internal /opt/_internal/
+COPY --from=build_cpython35 /opt/_internal /opt/_internal/
+COPY --from=build_cpython36 /opt/_internal /opt/_internal/
+COPY --from=build_cpython37 /opt/_internal /opt/_internal/
+COPY --from=build_cpython38 /opt/_internal /opt/_internal/
+COPY --from=build_cpython39 /opt/_internal /opt/_internal/
 COPY build_scripts/finalize.sh /build_scripts/finalize.sh
 COPY build_scripts/python-tag-abi-tag.py /build_scripts/python-tag-abi-tag.py
 COPY build_scripts/ssl-check.py /build_scripts/ssl-check.py

docker/build_scripts/build-cpython.sh

@@ -2,34 +2,50 @@
 # Top-level build script called from Dockerfile
 
 # Stop at any error, show all commands
-set -ex
+set -exuo pipefail
 
-# Set build environment variables
+# Get script directory
 MY_DIR=$(dirname "${BASH_SOURCE[0]}")
-source $MY_DIR/build_env.sh
 
 # Get build utilities
 source $MY_DIR/build_utils.sh
 
-# Compile the latest Python releases.
-# (In order to have a proper SSL module, Python is compiled
-# against a recent openssl [see env vars above], which is linked
-# statically.
-build_cpythons $CPYTHON_VERSIONS
+
+CPYTHON_VERSION=$1
+CPYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python
+
+
+function pyver_dist_dir {
+	# Echoes the dist directory name of given pyver, removing alpha/beta prerelease
+	# Thus:
+	# 3.2.1   -> 3.2.1
+	# 3.7.0b4 -> 3.7.0
+	echo $1 | awk -F "." '{printf "%d.%d.%d", $1, $2, $3}'
+}
+
+
+CPYTHON_DIST_DIR=$(pyver_dist_dir ${CPYTHON_VERSION})
+fetch_source Python-${CPYTHON_VERSION}.tgz ${CPYTHON_DOWNLOAD_URL}/${CPYTHON_DIST_DIR}
+fetch_source Python-${CPYTHON_VERSION}.tgz.asc ${CPYTHON_DOWNLOAD_URL}/${CPYTHON_DIST_DIR}
+gpg --verify Python-${CPYTHON_VERSION}.tgz.asc
+tar -xzf Python-${CPYTHON_VERSION}.tgz
+pushd Python-${CPYTHON_VERSION}
+PREFIX="/opt/_internal/cpython-${CPYTHON_VERSION}"
+mkdir -p ${PREFIX}/lib
+./configure --prefix=${PREFIX} --disable-shared --with-ensurepip=no > /dev/null
+make -j$(nproc) > /dev/null
+make -j$(nproc) install > /dev/null
+popd
+rm -rf Python-${CPYTHON_VERSION} Python-${CPYTHON_VERSION}.tgz Python-${CPYTHON_VERSION}.tgz.asc
 
 # we don't need libpython*.a, and they're many megabytes
-find /opt/_internal -name '*.a' -print0 | xargs -0 rm -f
-
-# Strip what we can -- and ignore errors, because this just attempts to strip
-# *everything*, including non-ELF files:
-find /opt/_internal -type f -print0 \
-    | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true
-find /manylinux-rootfs -type f -print0 \
-    | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true
-
-# We do not need the Python test suites, or indeed the precompiled .pyc and
-# .pyo files. Partially cribbed from:
-#    https://github.com/docker-library/python/blob/master/3.4/slim/Dockerfile
-find /opt/_internal -depth \
-     \( -type d -a -name test -o -name tests \) \
-  -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) | xargs rm -rf
+find ${PREFIX} -name '*.a' -print0 | xargs -0 rm -f
+
+# We do not need the Python test suites
+find ${PREFIX} -depth \( -type d -a -name test -o -name tests \) | xargs rm -rf
+
+# We do not need precompiled .pyc and .pyo files.
+clean_pyc ${PREFIX}
+
+# Strip ELF files found in ${PREFIX}
+strip_ ${PREFIX}

docker/build_scripts/build_env.sh

@@ -10,67 +10,6 @@ function check_var {
 }
 
 
-function lex_pyver {
-    # Echoes Python version string padded with zeros
-    # Thus:
-    # 3.2.1 -> 003002001
-    # 3     -> 003000000
-    echo $1 | awk -F "." '{printf "%03d%03d%03d", $1, $2, $3}'
-}
-
-
-function pyver_dist_dir {
-    # Echoes the dist directory name of given pyver, removing alpha/beta prerelease
-    # Thus:
-    # 3.2.1   -> 3.2.1
-    # 3.7.0b4 -> 3.7.0
-    echo $1 | awk -F "." '{printf "%d.%d.%d", $1, $2, $3}'
-}
-
-
-function do_cpython_build {
-    local py_ver=$1
-    check_var $py_ver
-    tar -xzf Python-$py_ver.tgz
-    pushd Python-$py_ver
-    local prefix="/opt/_internal/cpython-${py_ver}"
-    mkdir -p ${prefix}/lib
-    ./configure --prefix=${prefix} --disable-shared --with-ensurepip=no > /dev/null
-    make -j$(nproc) > /dev/null
-    make -j$(nproc) install > /dev/null
-    popd
-    rm -rf Python-$py_ver
-}
-
-
-function build_cpython {
-    local py_ver=$1
-    check_var $py_ver
-    check_var $PYTHON_DOWNLOAD_URL
-    local py_dist_dir=$(pyver_dist_dir $py_ver)
-    curl -fsSLO $PYTHON_DOWNLOAD_URL/$py_dist_dir/Python-$py_ver.tgz
-    curl -fsSLO $PYTHON_DOWNLOAD_URL/$py_dist_dir/Python-$py_ver.tgz.asc
-    gpg --verify Python-$py_ver.tgz.asc
-    do_cpython_build $py_ver
-    rm -f Python-$py_ver.tgz
-    rm -f Python-$py_ver.tgz.asc
-}
-
-
-function build_cpythons {
-    # Import public keys used to verify downloaded Python source tarballs.
-    # https://www.python.org/static/files/pubkeys.txt
-    gpg --import ${MY_DIR}/cpython-pubkeys.txt
-    # Add version 3.8, 3.9 release manager's key
-    gpg --import ${MY_DIR}/ambv-pubkey.txt
-    for py_ver in $@; do
-        build_cpython $py_ver
-    done
-    # Remove GPG hidden directory.
-    rm -rf /root/.gnupg/
-}
-
-
 function fetch_source {
     # This is called both inside and outside the build context (e.g. in Travis) to prefetch
     # source tarballs, where curl exists (and works)
@@ -109,3 +48,7 @@ function strip_ {
 	# *everything*, including non-ELF files:
 	find $1 -type f -print0 | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true
 }
+
+function clean_pyc {
+	find $1 -type f -a \( -name '*.pyc' -o -name '*.pyo' \) -delete
+}

docker/build_scripts/build_utils.sh

@@ -61,9 +61,8 @@ for PYTHON in /opt/python/*/bin/python; do
 	$PYTHON $MY_DIR/ssl-check.py
 done
 
-# We do not need the precompiled .pyc and .pyo files. Partially cribbed from:
-#    https://github.com/docker-library/python/blob/master/3.4/slim/Dockerfile
-find /opt/_internal -type f -a \( -name '*.pyc' -o -name '*.pyo' \) | xargs rm -rf
+# We do not need the precompiled .pyc and .pyo files.
+clean_pyc /opt/_internal
 
 # remove cache
 rm -rf /root/.cache