Allow manylinux2010 build from manylinux2014 branch (#979)

Author: mayeut

build.sh

@@ -24,7 +24,23 @@ else
 fi
 
 # setup BASEIMAGE and its specific properties
-if [ "${POLICY}" == "manylinux2014" ]; then
+if [ "${POLICY}" == "manylinux2010" ]; then
+	if [ "${PLATFORM}" == "x86_64" ]; then
+		BASEIMAGE="quay.io/pypa/manylinux2010_x86_64_centos6_no_vsyscall"
+	elif [ "${PLATFORM}" == "i686" ]; then
+		BASEIMAGE="${MULTIARCH_PREFIX}centos:6"
+	else
+		echo "Policy '${POLICY}' does not support platform '${PLATFORM}'"
+		exit 1
+	fi
+	DEVTOOLSET_ROOTPATH="/opt/rh/devtoolset-8/root"
+	PREPEND_PATH="${DEVTOOLSET_ROOTPATH}/usr/bin:"
+	if [ "${PLATFORM}" == "i686" ]; then
+		LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib"
+	else
+		LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib"
+	fi
+elif [ "${POLICY}" == "manylinux2014" ]; then
 	if [ "${PLATFORM}" == "s390x" ]; then
 		BASEIMAGE="s390x/clefos:7"
 	else

docker/Dockerfile

@@ -21,6 +21,10 @@ ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG}
 ENV PATH=${PREPEND_PATH}${PATH}
 ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
 
+# first fixup mirrors, keep the script around
+COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors
+RUN fixup-mirrors
+
 # setup entrypoint, this will wrap commands with `linux32` with i686 images
 COPY build_scripts/install-entrypoint.sh /build_scripts/
 RUN bash /build_scripts/install-entrypoint.sh && rm -rf build_scripts
@@ -102,6 +106,12 @@ RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3340000 && \
     export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2020 && \
     manylinux-entrypoint /build_scripts/build-sqlite3.sh
 
+COPY build_scripts/build-openssl.sh /build_scripts/
+RUN export OPENSSL_ROOT=openssl-1.1.1i && \
+    export OPENSSL_HASH=e8be6a35fe41d10603c3cc635e93289ed00bf34b79671a3a4de64fcee00d5242 && \
+    export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source && \
+    manylinux-entrypoint /build_scripts/build-openssl.sh
+
 COPY build_scripts/build-cpython.sh /build_scripts/
 
 

docker/build_scripts/build-cmake.sh

@@ -32,7 +32,7 @@ rm -rf /manylinux-rootfs/usr/local/share/cmake-*/Help
 strip_ /manylinux-rootfs
 
 # Install
-cp -rf /manylinux-rootfs/* /
+cp -rlf /manylinux-rootfs/* /
 
 
 cmake --version

docker/build_scripts/build-git.sh

@@ -28,6 +28,6 @@ rm -rf ${GIT_ROOT} ${GIT_ROOT}.tar.gz
 strip_ /manylinux-rootfs
 
 # Install
-cp -rf /manylinux-rootfs/* /
+cp -rlf /manylinux-rootfs/* /
 
 git version

docker/build_scripts/build-openssl.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Top-level build script called from Dockerfile
+
+# Stop at any error, show all commands
+set -exuo pipefail
+
+# Get script directory
+MY_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+# Get build utilities
+source $MY_DIR/build_utils.sh
+
+# Install a more recent openssl
+check_var ${OPENSSL_ROOT}
+check_var ${OPENSSL_HASH}
+check_var ${OPENSSL_DOWNLOAD_URL}
+
+OPENSSL_VERSION=${OPENSSL_ROOT#*-}
+OPENSSL_MIN_VERSION=1.0.2
+
+INSTALLED=$(openssl version | head -1 | awk '{ print $2 }')
+SMALLEST=$(echo -e "${INSTALLED}\n${OPENSSL_MIN_VERSION}" | sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4n | head -1)
+if [ "${SMALLEST}" == "${OPENSSL_MIN_VERSION}" ]; then
+	echo "skipping installation of openssl ${OPENSSL_VERSION}, system provides openssl ${INSTALLED} which is newer than openssl ${OPENSSL_MIN_VERSION}"
+	exit 0
+fi
+
+if which yum; then
+	yum erase -y openssl-devel
+else
+	apt-get remove -y libssl-dev
+fi
+
+fetch_source ${OPENSSL_ROOT}.tar.gz ${OPENSSL_DOWNLOAD_URL}
+check_sha256sum ${OPENSSL_ROOT}.tar.gz ${OPENSSL_HASH}
+tar -xzf ${OPENSSL_ROOT}.tar.gz
+pushd ${OPENSSL_ROOT}
+./config no-shared -fPIC --prefix=/usr/local/ssl --openssldir=/usr/local/ssl > /dev/null
+make > /dev/null
+make install_sw > /dev/null
+popd
+rm -rf ${OPENSSL_ROOT} ${OPENSSL_ROOT}.tar.gz
+
+
+/usr/local/ssl/bin/openssl version

docker/build_scripts/build-sqlite3.sh

@@ -29,7 +29,7 @@ rm /manylinux-rootfs/usr/local/lib/libsqlite3.a
 strip_ /manylinux-rootfs
 
 # Install
-cp -rf /manylinux-rootfs/* /
+cp -rlf /manylinux-rootfs/* /
 
 # Clean-up for runtime
 rm -rf /manylinux-rootfs/usr/local/bin /manylinux-rootfs/usr/local/include /manylinux-rootfs/usr/local/lib/pkg-config /manylinux-rootfs/usr/local/share

docker/build_scripts/build-swig.sh

@@ -35,6 +35,6 @@ rm -rf ${SWIG_ROOT}*
 strip_ /manylinux-rootfs
 
 # Install
-cp -rf /manylinux-rootfs/* /
+cp -rlf /manylinux-rootfs/* /
 
 swig -version

docker/build_scripts/finalize.sh

@@ -56,7 +56,7 @@ deactivate
 for PYTHON in /opt/python/*/bin/python; do
 	# Smoke test to make sure that our Pythons work, and do indeed detect as
 	# being manylinux compatible:
-	$PYTHON $MY_DIR/manylinux-check.py
+	$PYTHON $MY_DIR/manylinux-check.py ${AUDITWHEEL_POLICY} ${AUDITWHEEL_ARCH}
 	# Make sure that SSL cert checking works
 	$PYTHON $MY_DIR/ssl-check.py
 done

docker/build_scripts/fixup-mirrors.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# Fix up mirrors once distro reaches EOL
+
+# Stop at any error, show all commands
+set -exuo pipefail
+
+if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then
+	# Centos 6 is EOL and is no longer available from the usual mirrors, so switch
+	# to https://vault.centos.org
+	sed -i 's/enabled=1/enabled=0/g' /etc/yum/pluginconf.d/fastestmirror.conf
+	sed -i 's/^mirrorlist/#mirrorlist/g' /etc/yum.repos.d/*.repo
+	sed -i 's;^.*baseurl=http://mirror;baseurl=https://vault;g' /etc/yum.repos.d/*.repo
+	sed -i 's;^.*baseurl=http://download.fedoraproject.org/pub;baseurl=https://archives.fedoraproject.org/pub/archive;g' /etc/yum.repos.d/*.repo
+fi

docker/build_scripts/install-autoconf.sh

@@ -39,7 +39,7 @@ rm -rf ${AUTOCONF_ROOT} ${AUTOCONF_ROOT}.tar.gz
 strip_ /manylinux-rootfs
 
 # Install
-cp -rf /manylinux-rootfs/* /
+cp -rlf /manylinux-rootfs/* /
 
 # Remove temporary rootfs
 rm -rf /manylinux-rootfs

docker/build_scripts/install-automake.sh

@@ -37,7 +37,7 @@ rm -rf ${AUTOMAKE_ROOT} ${AUTOMAKE_ROOT}.tar.gz
 strip_ /manylinux-rootfs
 
 # Install
-cp -rf /manylinux-rootfs/* /
+cp -rlf /manylinux-rootfs/* /
 
 # Remove temporary rootfs
 rm -rf /manylinux-rootfs

docker/build_scripts/install-build-packages.sh

@@ -1,16 +1,32 @@
 #!/bin/bash
-# Top-level build script called from Dockerfile
+# Install packages that will be needed at runtime
 
 # Stop at any error, show all commands
-set -ex
+set -exuo pipefail
 
-# Set build environment variables
-MY_DIR=$(dirname "${BASH_SOURCE[0]}")
 
-# Dependencies for compiling Python that we want to remove from
-# the final image after compiling Python
-PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel expat-devel ncurses-devel readline-devel tk-devel gdbm-devel libdb-devel libpcap-devel xz-devel openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel perl-devel libffi-devel kernel-devel"
-CMAKE_DEPS="openssl-devel zlib-devel libcurl-devel"
+# if a devel package is added to COMPILE_DEPS,
+# make sure the corresponding library is added to RUNTIME_DEPS if applicable
 
-# Development tools and libraries
-yum -y install ${PYTHON_COMPILE_DEPS} ${CMAKE_DEPS}
+if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
+	PACKAGE_MANAGER=yum
+	COMPILE_DEPS="zlib-devel bzip2-devel expat-devel ncurses-devel readline-devel tk-devel gdbm-devel libpcap-devel xz-devel openssl openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel uuid-devel libffi-devel kernel-headers"
+	if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then
+		COMPILE_DEPS="${COMPILE_DEPS} db4-devel"
+	else
+		COMPILE_DEPS="${COMPILE_DEPS} libdb-devel"
+	fi
+else
+	echo "Unsupported policy: '${AUDITWHEEL_POLICY}'"
+	exit 1
+fi
+
+
+if [ ${PACKAGE_MANAGER} == yum ]; then
+	yum -y install ${COMPILE_DEPS}
+	yum clean all
+	rm -rf /var/cache/yum
+else
+	echo "Not implemented"
+	exit 1
+fi

docker/build_scripts/install-entrypoint.sh

@@ -5,7 +5,7 @@
 # Stop at any error, show all commands
 set -exuo pipefail
 
-if [ "${AUDITWHEEL_PLAT}" == "manylinux2014_i686" ]; then
+if [ "${AUDITWHEEL_PLAT}" == "manylinux2010_i686" ] || [ "${AUDITWHEEL_PLAT}" == "manylinux2014_i686" ]; then
 	echo "i386" > /etc/yum/vars/basearch
 	yum -y update
 	yum install -y util-linux-ng

docker/build_scripts/install-libtool.sh

@@ -26,7 +26,7 @@ rm -rf ${LIBTOOL_ROOT} ${LIBTOOL_ROOT}.tar.gz
 strip_ /manylinux-rootfs
 
 # Install
-cp -rf /manylinux-rootfs/* /
+cp -rlf /manylinux-rootfs/* /
 
 # Remove temporary rootfs
 rm -rf /manylinux-rootfs

docker/build_scripts/install-libxcrypt.sh

@@ -41,7 +41,7 @@ rm -rf "v${LIBXCRYPT_VERSION}.tar.gz" "libxcrypt-${LIBXCRYPT_VERSION}"
 strip_ /manylinux-rootfs
 
 # Install
-cp -rf /manylinux-rootfs/* /
+cp -rlf /manylinux-rootfs/* /
 
 # Remove temporary rootfs
 rm -rf /manylinux-rootfs

docker/build_scripts/install-patchelf.sh

@@ -27,7 +27,7 @@ rm -rf ${PATCHELF_VERSION}.tar.gz patchelf-${PATCHELF_VERSION}
 strip_ /manylinux-rootfs
 
 # Install
-cp -rf /manylinux-rootfs/* /
+cp -rlf /manylinux-rootfs/* /
 
 # Remove temporary rootfs
 rm -rf /manylinux-rootfs

docker/build_scripts/install-runtime-packages.sh

@@ -24,17 +24,47 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}")
 # PEP is missing the package for libSM.so.6 for RPM based system
 
 # MANYLINUX_DEPS: Install development packages (except for libgcc which is provided by gcc install)
-# RUNTIME_DEPS: Runtime dependencies. c.f. build.sh
-if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
+if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
 	MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel"
-	RUNTIME_DEPS="zlib bzip2 expat ncurses readline tk gdbm libdb libpcap xz openssl keyutils-libs libkadm5 libcom_err libidn libcurl uuid libffi"
 else
 	echo "Unsupported policy: '${AUDITWHEEL_POLICY}'"
 	exit 1
 fi
 
-BASETOOLS="autoconf automake bison bzip2 diffutils file make patch unzip"
-if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
+# RUNTIME_DEPS: Runtime dependencies. c.f. install-build-packages.sh
+if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
+	RUNTIME_DEPS="zlib bzip2 expat ncurses readline tk gdbm libpcap xz openssl keyutils-libs libkadm5 libcom_err libidn libcurl uuid libffi"
+	if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then
+		RUNTIME_DEPS="${RUNTIME_DEPS} db4"
+	else
+		RUNTIME_DEPS="${RUNTIME_DEPS} libdb"
+	fi
+else
+	echo "Unsupported policy: '${AUDITWHEEL_POLICY}'"
+	exit 1
+fi
+
+BASETOOLS="autoconf automake bison bzip2 diffutils file hardlink make patch unzip"
+if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then
+	PACKAGE_MANAGER=yum
+	BASETOOLS="${BASETOOLS} which"
+	# See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages
+	echo "multilib_policy=best" >> /etc/yum.conf
+	yum -y update
+	yum -y install https://archives.fedoraproject.org/pub/archive/epel/6/x86_64/epel-release-6-8.noarch.rpm curl
+	fixup-mirrors
+	TOOLCHAIN_DEPS="devtoolset-8-binutils devtoolset-8-gcc devtoolset-8-gcc-c++ devtoolset-8-gcc-gfortran yasm"
+	if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then
+		# Software collection (for devtoolset-8)
+		yum -y install centos-release-scl
+		fixup-mirrors
+	elif [ "${AUDITWHEEL_ARCH}" == "i686" ]; then
+		# Add libgfortran4 for devtoolset-7 compat
+		TOOLCHAIN_DEPS="${TOOLCHAIN_DEPS} libgfortran4"
+		# Install mayeut/devtoolset-8 repo to get devtoolset-8
+		curl -fsSLo /etc/yum.repos.d/mayeut-devtoolset-8.repo https://copr.fedorainfracloud.org/coprs/mayeut/devtoolset-8-i386/repo/custom-1/mayeut-devtoolset-8-i386-custom-1.repo
+	fi
+elif [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
 	PACKAGE_MANAGER=yum
 	BASETOOLS="${BASETOOLS} which"
 	# See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages

docker/build_scripts/manylinux-check.py

@@ -1,8 +1,44 @@
-# Logic copied from PEP 599
+# Logic copied from PEP 513, PEP 599
 
 import sys
 
 
+def is_manylinux1_compatible():
+    # Only Linux, and only x86-64 / i686
+    from distutils.util import get_platform
+    if get_platform() not in ["linux-x86_64", "linux-i686"]:
+        return False
+
+    # Check for presence of _manylinux module
+    try:
+        import _manylinux
+        return bool(_manylinux.manylinux1_compatible)
+    except (ImportError, AttributeError):
+        # Fall through to heuristic check below
+        pass
+
+    # Check glibc version. CentOS 5 uses glibc 2.5.
+    return have_compatible_glibc(2, 5)
+
+
+def is_manylinux2010_compatible():
+    # Only Linux, and only x86-64 / i686
+    from distutils.util import get_platform
+    if get_platform() not in ["linux-x86_64", "linux-i686"]:
+        return False
+
+    # Check for presence of _manylinux module
+    try:
+        import _manylinux
+        return bool(_manylinux.manylinux2010_compatible)
+    except (ImportError, AttributeError):
+        # Fall through to heuristic check below
+        pass
+
+    # Check glibc version. CentOS 6 uses glibc 2.12.
+    return have_compatible_glibc(2, 12)
+
+
 def is_manylinux2014_compatible():
     # Only Linux, and only supported architectures
     from distutils.util import get_platform
@@ -58,9 +94,27 @@ def have_compatible_glibc(major, minimum_minor):
     return True
 
 
-if is_manylinux2014_compatible():
-    print("%s is manylinux2014 compatible" % (sys.executable,))
-    sys.exit(0)
-else:
-    print("%s is NOT manylinux2014 compatible" % (sys.executable,))
-    sys.exit(1)
+exit_code = 0
+
+if sys.argv[2] in {"x86_64", "i686"} and (sys.argv[1] in {"manylinux1", "manylinux2010", "manylinux2014"} or sys.argv[1].startswith("manylinux_")):
+    if is_manylinux1_compatible():
+        print("%s %s is manylinux1 compatible" % (sys.argv[1], sys.executable))
+    else:
+        print("%s %s is NOT manylinux1 compatible" % (sys.argv[1], sys.executable))
+        exit_code = 1
+
+if sys.argv[2] in {"x86_64", "i686"} and (sys.argv[1] in {"manylinux2010", "manylinux2014"} or sys.argv[1].startswith("manylinux_")):
+    if is_manylinux2010_compatible():
+        print("%s %s is manylinux2010 compatible" % (sys.argv[1], sys.executable))
+    else:
+        print("%s %s is NOT manylinux2010 compatible" % (sys.argv[1], sys.executable))
+        exit_code = 1
+
+if sys.argv[1] in {"manylinux2014"} or sys.argv[1].startswith("manylinux_"):
+    if is_manylinux2014_compatible():
+        print("%s %s is manylinux2014 compatible" % (sys.argv[1], sys.executable))
+    else:
+        print("%s %s is NOT manylinux2014 compatible" % (sys.argv[1], sys.executable))
+        exit_code = 1
+
+sys.exit(exit_code)