diff --git a/docker/build_scripts/build.sh b/docker/build_scripts/build.sh index 3a9fdfb8..7c67968c 100755 --- a/docker/build_scripts/build.sh +++ b/docker/build_scripts/build.sh @@ -11,7 +11,7 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}") # Dependencies for compiling Python that we want to remove from # the final image after compiling Python # GPG installed to verify signatures on Python source tarballs. -PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel gpg libffi-devel" +PYTHON_COMPILE_DEPS="expat-devel gettext tcl-devel libtermcap-devel zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel gpg libffi-devel" # Libraries that are allowed as part of the manylinux1 profile MANYLINUX1_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel ncurses-devel" @@ -68,8 +68,6 @@ yum -y install \ devtoolset-2-gcc-c++ \ devtoolset-2-gcc-gfortran \ diffutils \ - expat-devel \ - gettext \ kernel-devel-`uname -r` \ file \ make \ @@ -88,10 +86,6 @@ hash -r curl --version curl-config --features -# Install a git we link against OpenSSL so that we can use TLS 1.2 -build_git $GIT_ROOT $GIT_HASH -git version - # Install newest autoconf build_autoconf $AUTOCONF_ROOT $AUTOCONF_HASH autoconf --version @@ -104,6 +98,45 @@ automake --version build_libtool $LIBTOOL_ROOT $LIBTOOL_HASH libtool --version +# Install patchelf (latest with unreleased bug fixes) +curl -fsSL -o patchelf.tar.gz https://github.com/NixOS/patchelf/archive/$PATCHELF_VERSION.tar.gz +check_sha256sum patchelf.tar.gz $PATCHELF_HASH +tar -xzf patchelf.tar.gz +(cd patchelf-$PATCHELF_VERSION && ./bootstrap.sh && do_standard_install) +rm -rf patchelf.tar.gz patchelf-$PATCHELF_VERSION + +# We strip curl now because stripping after patchelf breaks it +strip --strip-unneeded /opt/_internal/_vendor/lib/libcurl.so.4 +strip --strip-unneeded /usr/local/bin/curl + +# Let's patch curl & openssl: +CHECKSUM=$(sha256sum /opt/_internal/_vendor/lib/libcrypto.so.1.0.0) +CHECKSUM=${CHECKSUM:0:8} +patchelf --set-soname /opt/_internal/_vendor/lib/libcrypto.so.1.0.0-${CHECKSUM} /opt/_internal/_vendor/lib/libcrypto.so.1.0.0 +mv /opt/_internal/_vendor/lib/libcrypto.so.1.0.0 /opt/_internal/_vendor/lib/libcrypto.so.1.0.0-${CHECKSUM} +ln -sf libcrypto.so.1.0.0-${CHECKSUM} /opt/_internal/_vendor/lib/libcrypto.so +patchelf --replace-needed libcrypto.so.1.0.0 /opt/_internal/_vendor/lib/libcrypto.so.1.0.0-${CHECKSUM} /opt/_internal/_vendor/lib/libssl.so.1.0.0 +patchelf --replace-needed libcrypto.so.1.0.0 /opt/_internal/_vendor/lib/libcrypto.so.1.0.0-${CHECKSUM} /opt/_internal/_vendor/lib/libcurl.so.4 +patchelf --replace-needed libcrypto.so.1.0.0 /opt/_internal/_vendor/lib/libcrypto.so.1.0.0-${CHECKSUM} /usr/local/bin/curl + +CHECKSUM=$(sha256sum /opt/_internal/_vendor/lib/libssl.so.1.0.0) +CHECKSUM=${CHECKSUM:0:8} +patchelf --set-soname /opt/_internal/_vendor/lib/libssl.so.1.0.0-${CHECKSUM} /opt/_internal/_vendor/lib/libssl.so.1.0.0 +mv /opt/_internal/_vendor/lib/libssl.so.1.0.0 /opt/_internal/_vendor/lib/libssl.so.1.0.0-${CHECKSUM} +ln -sf libssl.so.1.0.0-${CHECKSUM} /opt/_internal/_vendor/lib/libssl.so +patchelf --replace-needed libssl.so.1.0.0 /opt/_internal/_vendor/lib/libssl.so.1.0.0-${CHECKSUM} /opt/_internal/_vendor/lib/libcurl.so.4 +patchelf --replace-needed libssl.so.1.0.0 /opt/_internal/_vendor/lib/libssl.so.1.0.0-${CHECKSUM} /usr/local/bin/curl + +CHECKSUM=$(sha256sum /opt/_internal/_vendor/lib/libcurl.so.4) +CHECKSUM=${CHECKSUM:0:8} +patchelf --set-soname /opt/_internal/_vendor/lib/libcurl.so.4-${CHECKSUM} /opt/_internal/_vendor/lib/libcurl.so.4 +mv /opt/_internal/_vendor/lib/libcurl.so.4 /opt/_internal/_vendor/lib/libcurl.so.4-${CHECKSUM} +patchelf --replace-needed libcurl.so.4 /opt/_internal/_vendor/lib/libcurl.so.4-${CHECKSUM} /usr/local/bin/curl + +# Install a git we link against OpenSSL so that we can use TLS 1.2 +build_git $GIT_ROOT $GIT_HASH +git version + # Install a more recent SQLite3 curl -fsSLO $SQLITE_AUTOCONF_DOWNLOAD_URL/$SQLITE_AUTOCONF_VERSION.tar.gz check_sha256sum $SQLITE_AUTOCONF_VERSION.tar.gz $SQLITE_AUTOCONF_HASH @@ -112,11 +145,14 @@ cd $SQLITE_AUTOCONF_VERSION do_standard_install cd .. rm -rf $SQLITE_AUTOCONF_VERSION* +rm -f /usr/local/lib/libsqlite3.a + +# Create a symbolic link for python to find openssl +ln -s /opt/_internal/_vendor /usr/local/ssl # Compile the latest Python releases. # (In order to have a proper SSL module, Python is compiled -# against a recent openssl [see env vars above], which is linked -# statically. +# against a recent openssl [see env vars above]) mkdir -p /opt/python build_cpythons $CPYTHON_VERSIONS @@ -136,14 +172,8 @@ ln -s $($PY36_BIN/python -c 'import certifi; print(certifi.where())') \ export SSL_CERT_FILE=/opt/_internal/certs.pem # Now we can delete our built OpenSSL headers/static libs since we've linked everything we need -rm -rf /usr/local/ssl - -# Install patchelf (latest with unreleased bug fixes) -curl -fsSL -o patchelf.tar.gz https://github.com/NixOS/patchelf/archive/$PATCHELF_VERSION.tar.gz -check_sha256sum patchelf.tar.gz $PATCHELF_HASH -tar -xzf patchelf.tar.gz -(cd patchelf-$PATCHELF_VERSION && ./bootstrap.sh && do_standard_install) -rm -rf patchelf.tar.gz patchelf-$PATCHELF_VERSION +unlink /usr/local/ssl +find /opt/_internal/_vendor -mindepth 1 -maxdepth 1 -not -path '/opt/_internal/_vendor/lib*' | xargs rm -rf ln -s $PY36_BIN/auditwheel /usr/local/bin/auditwheel @@ -166,16 +196,17 @@ yum list installed find /opt/_internal -name '*.a' -print0 | xargs -0 rm -f # Strip what we can -- and ignore errors, because this just attempts to strip -# *everything*, including non-ELF files: -find /opt/_internal -type f -print0 \ +# *everything*, including non-ELF files. +# We ignore curl which was stripped before patching it with patchelf +find /opt/_internal -type f -not -name 'libcurl.so*' -print0 \ + | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true +find /usr/local -type f -not -name 'curl' -print0 \ | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true -# We do not need the Python test suites, or indeed the precompiled .pyc and -# .pyo files. Partially cribbed from: -# https://github.com/docker-library/python/blob/master/3.4/slim/Dockerfile -find /opt/_internal -depth \ - \( -type d -a -name test -o -name tests \) \ - -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) | xargs rm -rf +# Make sure curl is still working after stripping, patchelf can break things +# libcurl is in exceptions for stripping and this will help catch errors if openssl gets broken after an update +curl -fsSLO $GET_PIP_URL +rm -f get-pip.py for PYTHON in /opt/python/*/bin/python; do # Smoke test to make sure that our Pythons work, and do indeed detect as @@ -185,5 +216,12 @@ for PYTHON in /opt/python/*/bin/python; do $PYTHON $MY_DIR/ssl-check.py done +# We do not need the Python test suites, or indeed the precompiled .pyc and +# .pyo files. Partially cribbed from: +# https://github.com/docker-library/python/blob/master/3.4/slim/Dockerfile +find /opt/_internal -depth \ + \( -type d -a -name test -o -name tests \) \ + -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) | xargs rm -rf + # Fix libc headers to remain compatible with C99 compilers. find /usr/include/ -type f -exec sed -i 's/\bextern _*inline_*\b/extern __inline __attribute__ ((__gnu_inline__))/g' {} + diff --git a/docker/build_scripts/build_utils.sh b/docker/build_scripts/build_utils.sh index dd0780e2..e1ffabc0 100755 --- a/docker/build_scripts/build_utils.sh +++ b/docker/build_scripts/build_utils.sh @@ -104,9 +104,10 @@ function build_cpythons { function do_openssl_build { - ./config no-ssl2 no-shared -fPIC --prefix=/usr/local/ssl > /dev/null + ./config no-ssl2 shared --prefix=/opt/_internal/_vendor --openssldir=/opt/_internal/_vendor > /dev/null make > /dev/null make install_sw > /dev/null + rm -rf /opt/_internal/_vendor/bin /opt/_internal/_vendor/lib/*.a } @@ -170,15 +171,13 @@ function build_git { fetch_source v${git_fname}.tar.gz ${GIT_DOWNLOAD_URL} check_sha256sum v${git_fname}.tar.gz ${git_sha256} tar -xzf v${git_fname}.tar.gz - (cd git-${git_fname} && make install prefix=/usr/local LDFLAGS="-L/usr/local/ssl/lib -ldl" CFLAGS="-I/usr/local/ssl/include" > /dev/null) + (cd git-${git_fname} && make install prefix=/usr/local NO_GETTEXT=1 LDFLAGS="-L/opt/_internal/_vendor/lib -ldl" CFLAGS="-I/opt/_internal/_vendor/include" > /dev/null) rm -rf git-${git_fname} v${git_fname}.tar.gz } function do_curl_build { - # We do this shared to avoid obnoxious linker issues where git couldn't - # link properly. If anyone wants to make this build statically go for it. - LIBS=-ldl CFLAGS=-Wl,--exclude-libs,ALL ./configure --with-ssl --disable-static > /dev/null + LD_LIBRARY_PATH=/opt/_internal/_vendor/lib:$LD_LIBRARY_PATH LIBS=-ldl CFLAGS=-Wl,--exclude-libs,ALL ./configure --with-ssl=/opt/_internal/_vendor --disable-static --prefix=/opt/_internal/_vendor --bindir=/usr/local/bin > /dev/null make > /dev/null make install > /dev/null }