Skip to content

[WIP] Reduce image size #210

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

[WIP] Reduce image size #210

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5c75b16
Uninstall unecessary packages
mayeut Jun 1, 2018
d168c8f
Remove python cache after tests
mayeut May 26, 2018
6ff333c
Build shared OpenSSL
mayeut May 26, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 63 additions & 25 deletions docker/build_scripts/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}")
# Dependencies for compiling Python that we want to remove from
# the final image after compiling Python
# GPG installed to verify signatures on Python source tarballs.
PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel gpg libffi-devel"
PYTHON_COMPILE_DEPS="expat-devel gettext tcl-devel libtermcap-devel zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel gpg libffi-devel"

# Libraries that are allowed as part of the manylinux1 profile
MANYLINUX1_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel ncurses-devel"
Expand Down Expand Up @@ -68,8 +68,6 @@ yum -y install \
devtoolset-2-gcc-c++ \
devtoolset-2-gcc-gfortran \
diffutils \
expat-devel \
gettext \
kernel-devel-`uname -r` \
file \
make \
Expand All @@ -88,10 +86,6 @@ hash -r
curl --version
curl-config --features

# Install a git we link against OpenSSL so that we can use TLS 1.2
build_git $GIT_ROOT $GIT_HASH
git version

# Install newest autoconf
build_autoconf $AUTOCONF_ROOT $AUTOCONF_HASH
autoconf --version
Expand All @@ -104,6 +98,45 @@ automake --version
build_libtool $LIBTOOL_ROOT $LIBTOOL_HASH
libtool --version

# Install patchelf (latest with unreleased bug fixes)
curl -fsSL -o patchelf.tar.gz https://github.com/NixOS/patchelf/archive/$PATCHELF_VERSION.tar.gz
check_sha256sum patchelf.tar.gz $PATCHELF_HASH
tar -xzf patchelf.tar.gz
(cd patchelf-$PATCHELF_VERSION && ./bootstrap.sh && do_standard_install)
rm -rf patchelf.tar.gz patchelf-$PATCHELF_VERSION

# We strip curl now because stripping after patchelf breaks it
strip --strip-unneeded /opt/_internal/_vendor/lib/libcurl.so.4
strip --strip-unneeded /usr/local/bin/curl

# Let's patch curl & openssl:
CHECKSUM=$(sha256sum /opt/_internal/_vendor/lib/libcrypto.so.1.0.0)
CHECKSUM=${CHECKSUM:0:8}
patchelf --set-soname /opt/_internal/_vendor/lib/libcrypto.so.1.0.0-${CHECKSUM} /opt/_internal/_vendor/lib/libcrypto.so.1.0.0
mv /opt/_internal/_vendor/lib/libcrypto.so.1.0.0 /opt/_internal/_vendor/lib/libcrypto.so.1.0.0-${CHECKSUM}
ln -sf libcrypto.so.1.0.0-${CHECKSUM} /opt/_internal/_vendor/lib/libcrypto.so
patchelf --replace-needed libcrypto.so.1.0.0 /opt/_internal/_vendor/lib/libcrypto.so.1.0.0-${CHECKSUM} /opt/_internal/_vendor/lib/libssl.so.1.0.0
patchelf --replace-needed libcrypto.so.1.0.0 /opt/_internal/_vendor/lib/libcrypto.so.1.0.0-${CHECKSUM} /opt/_internal/_vendor/lib/libcurl.so.4
patchelf --replace-needed libcrypto.so.1.0.0 /opt/_internal/_vendor/lib/libcrypto.so.1.0.0-${CHECKSUM} /usr/local/bin/curl

CHECKSUM=$(sha256sum /opt/_internal/_vendor/lib/libssl.so.1.0.0)
CHECKSUM=${CHECKSUM:0:8}
patchelf --set-soname /opt/_internal/_vendor/lib/libssl.so.1.0.0-${CHECKSUM} /opt/_internal/_vendor/lib/libssl.so.1.0.0
mv /opt/_internal/_vendor/lib/libssl.so.1.0.0 /opt/_internal/_vendor/lib/libssl.so.1.0.0-${CHECKSUM}
ln -sf libssl.so.1.0.0-${CHECKSUM} /opt/_internal/_vendor/lib/libssl.so
patchelf --replace-needed libssl.so.1.0.0 /opt/_internal/_vendor/lib/libssl.so.1.0.0-${CHECKSUM} /opt/_internal/_vendor/lib/libcurl.so.4
patchelf --replace-needed libssl.so.1.0.0 /opt/_internal/_vendor/lib/libssl.so.1.0.0-${CHECKSUM} /usr/local/bin/curl

CHECKSUM=$(sha256sum /opt/_internal/_vendor/lib/libcurl.so.4)
CHECKSUM=${CHECKSUM:0:8}
patchelf --set-soname /opt/_internal/_vendor/lib/libcurl.so.4-${CHECKSUM} /opt/_internal/_vendor/lib/libcurl.so.4
mv /opt/_internal/_vendor/lib/libcurl.so.4 /opt/_internal/_vendor/lib/libcurl.so.4-${CHECKSUM}
patchelf --replace-needed libcurl.so.4 /opt/_internal/_vendor/lib/libcurl.so.4-${CHECKSUM} /usr/local/bin/curl

# Install a git we link against OpenSSL so that we can use TLS 1.2
build_git $GIT_ROOT $GIT_HASH
git version

# Install a more recent SQLite3
curl -fsSLO $SQLITE_AUTOCONF_DOWNLOAD_URL/$SQLITE_AUTOCONF_VERSION.tar.gz
check_sha256sum $SQLITE_AUTOCONF_VERSION.tar.gz $SQLITE_AUTOCONF_HASH
Expand All @@ -112,11 +145,14 @@ cd $SQLITE_AUTOCONF_VERSION
do_standard_install
cd ..
rm -rf $SQLITE_AUTOCONF_VERSION*
rm -f /usr/local/lib/libsqlite3.a

# Create a symbolic link for python to find openssl
ln -s /opt/_internal/_vendor /usr/local/ssl

# Compile the latest Python releases.
# (In order to have a proper SSL module, Python is compiled
# against a recent openssl [see env vars above], which is linked
# statically.
# against a recent openssl [see env vars above])
mkdir -p /opt/python
build_cpythons $CPYTHON_VERSIONS

Expand All @@ -136,14 +172,8 @@ ln -s $($PY36_BIN/python -c 'import certifi; print(certifi.where())') \
export SSL_CERT_FILE=/opt/_internal/certs.pem

# Now we can delete our built OpenSSL headers/static libs since we've linked everything we need
rm -rf /usr/local/ssl

# Install patchelf (latest with unreleased bug fixes)
curl -fsSL -o patchelf.tar.gz https://github.com/NixOS/patchelf/archive/$PATCHELF_VERSION.tar.gz
check_sha256sum patchelf.tar.gz $PATCHELF_HASH
tar -xzf patchelf.tar.gz
(cd patchelf-$PATCHELF_VERSION && ./bootstrap.sh && do_standard_install)
rm -rf patchelf.tar.gz patchelf-$PATCHELF_VERSION
unlink /usr/local/ssl
find /opt/_internal/_vendor -mindepth 1 -maxdepth 1 -not -path '/opt/_internal/_vendor/lib*' | xargs rm -rf

ln -s $PY36_BIN/auditwheel /usr/local/bin/auditwheel

Expand All @@ -166,16 +196,17 @@ yum list installed
find /opt/_internal -name '*.a' -print0 | xargs -0 rm -f

# Strip what we can -- and ignore errors, because this just attempts to strip
# *everything*, including non-ELF files:
find /opt/_internal -type f -print0 \
# *everything*, including non-ELF files.
# We ignore curl which was stripped before patching it with patchelf
find /opt/_internal -type f -not -name 'libcurl.so*' -print0 \
| xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true
find /usr/local -type f -not -name 'curl' -print0 \
| xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true

# We do not need the Python test suites, or indeed the precompiled .pyc and
# .pyo files. Partially cribbed from:
# https://github.com/docker-library/python/blob/master/3.4/slim/Dockerfile
find /opt/_internal -depth \
\( -type d -a -name test -o -name tests \) \
-o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) | xargs rm -rf
# Make sure curl is still working after stripping, patchelf can break things
# libcurl is in exceptions for stripping and this will help catch errors if openssl gets broken after an update
curl -fsSLO $GET_PIP_URL
rm -f get-pip.py

for PYTHON in /opt/python/*/bin/python; do
# Smoke test to make sure that our Pythons work, and do indeed detect as
Expand All @@ -185,5 +216,12 @@ for PYTHON in /opt/python/*/bin/python; do
$PYTHON $MY_DIR/ssl-check.py
done

# We do not need the Python test suites, or indeed the precompiled .pyc and
# .pyo files. Partially cribbed from:
# https://github.com/docker-library/python/blob/master/3.4/slim/Dockerfile
find /opt/_internal -depth \
\( -type d -a -name test -o -name tests \) \
-o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) | xargs rm -rf

# Fix libc headers to remain compatible with C99 compilers.
find /usr/include/ -type f -exec sed -i 's/\bextern _*inline_*\b/extern __inline __attribute__ ((__gnu_inline__))/g' {} +
9 changes: 4 additions & 5 deletions docker/build_scripts/build_utils.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,9 +104,10 @@ function build_cpythons {


function do_openssl_build {
./config no-ssl2 no-shared -fPIC --prefix=/usr/local/ssl > /dev/null
./config no-ssl2 shared --prefix=/opt/_internal/_vendor --openssldir=/opt/_internal/_vendor > /dev/null
make > /dev/null
make install_sw > /dev/null
rm -rf /opt/_internal/_vendor/bin /opt/_internal/_vendor/lib/*.a
}


Expand Down Expand Up @@ -170,15 +171,13 @@ function build_git {
fetch_source v${git_fname}.tar.gz ${GIT_DOWNLOAD_URL}
check_sha256sum v${git_fname}.tar.gz ${git_sha256}
tar -xzf v${git_fname}.tar.gz
(cd git-${git_fname} && make install prefix=/usr/local LDFLAGS="-L/usr/local/ssl/lib -ldl" CFLAGS="-I/usr/local/ssl/include" > /dev/null)
(cd git-${git_fname} && make install prefix=/usr/local NO_GETTEXT=1 LDFLAGS="-L/opt/_internal/_vendor/lib -ldl" CFLAGS="-I/opt/_internal/_vendor/include" > /dev/null)
rm -rf git-${git_fname} v${git_fname}.tar.gz
}


function do_curl_build {
# We do this shared to avoid obnoxious linker issues where git couldn't
# link properly. If anyone wants to make this build statically go for it.
LIBS=-ldl CFLAGS=-Wl,--exclude-libs,ALL ./configure --with-ssl --disable-static > /dev/null
LD_LIBRARY_PATH=/opt/_internal/_vendor/lib:$LD_LIBRARY_PATH LIBS=-ldl CFLAGS=-Wl,--exclude-libs,ALL ./configure --with-ssl=/opt/_internal/_vendor --disable-static --prefix=/opt/_internal/_vendor --bindir=/usr/local/bin > /dev/null
make > /dev/null
make install > /dev/null
}
Expand Down