Merge pull request #297 from rabbitmq/compile-openssl-otp

Compile OpenSSL & Erlang/OTP from source, use Ubuntu as the base image

Author: yosifkit

.travis.yml

@@ -2,9 +2,9 @@ language: bash
 services: docker
 
 env:
-  - VERSION=3.7-rc VARIANT=debian
+  - VERSION=3.7-rc VARIANT=ubuntu
   - VERSION=3.7-rc VARIANT=alpine
-  - VERSION=3.7 VARIANT=debian
+  - VERSION=3.7 VARIANT=ubuntu
   - VERSION=3.7 VARIANT=alpine
 
 install:

3.7-rc/alpine/Dockerfile

@@ -1,92 +1,230 @@
+# Alpine Linux is not officially supported by the RabbitMQ team -- use at your own risk!
 FROM alpine:3.8
 
-# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added
-RUN addgroup -S rabbitmq && adduser -S -h /var/lib/rabbitmq -G rabbitmq rabbitmq
-
 # grab su-exec for easy step-down from root
-RUN apk add --no-cache 'su-exec>=0.2'
+# bash for docker-entrypoint.sh
+RUN apk add --no-cache 'su-exec>=0.2' bash
+
+# Default to a PGP keyserver that pgp-happy-eyeballs recognizes, but allow for substitutions locally
+ARG PGP_KEYSERVER=ha.pool.sks-keyservers.net
+# If you are building this image locally and are getting `gpg: keyserver receive failed: No data` errors,
+# run the build with a different PGP_KEYSERVER, e.g. docker build --tag rabbitmq:3.7 --build-arg PGP_KEYSERVER=pgpkeys.eu 3.7/ubuntu
+# For context, see https://github.com/docker-library/official-images/issues/4252
 
-RUN apk add --no-cache \
-# Bash for docker-entrypoint
-		bash \
-# Procps for rabbitmqctl
-		procps \
-# Erlang for RabbitMQ
-		erlang-asn1 \
-		erlang-hipe \
-		erlang-crypto \
-		erlang-eldap \
-		erlang-inets \
-		erlang-mnesia \
-		erlang \
-		erlang-os-mon \
-		erlang-public-key \
-		erlang-sasl \
-		erlang-ssl \
-		erlang-syntax-tools \
-		erlang-xmerl
+# Using the latest OpenSSL LTS release, with support until September 2023 - https://www.openssl.org/source/
+ENV OPENSSL_VERSION="1.1.1a"
+ENV OPENSSL_SOURCE_SHA256="fc20130f8b7cbd2fb918b2f14e2f429e109c31ddd0fb38fc5d71d9ffed3f9f41"
+# https://www.openssl.org/community/omc.html
+ENV OPENSSL_PGP_KEY_ID="0x8657ABB260F056B1E5190839D9C4D26D0E604491"
 
-# get logs to stdout (thanks @dumbbell for pushing this upstream! :D)
-ENV RABBITMQ_LOGS=- RABBITMQ_SASL_LOGS=-
-# https://github.com/rabbitmq/rabbitmq-server/commit/53af45bf9a162dec849407d114041aad3d84feaf
+# Use the latest stable Erlang/OTP release (https://github.com/erlang/otp/tags)
+ENV OTP_VERSION="21.2.3"
+# TODO add PGP checking when the feature will be added to Erlang/OTP's build system
+# http://erlang.org/pipermail/erlang-questions/2019-January/097067.html
+ENV OTP_SOURCE_SHA256="109a5722e398bdcd3aeb4f4833cde90bf441a9c014006439643aab550a770923"
+
+# Install dependencies required to build Erlang/OTP from source
+# http://erlang.org/doc/installation_guide/INSTALL.html
+# autoconf: Required to configure Erlang/OTP before compiling
+# dpkg-dev: Required to set up host & build type when compiling Erlang/OTP
+# gnupg: Required to verify OpenSSL artefacts
+# libncurses5-dev: Required for Erlang/OTP new shell & observer_cli - https://github.com/zhongwencool/observer_cli
+# m4: Required for Erlang/OTP HiPE support
+RUN set -eux; \
+	\
+	apk add --no-cache --virtual .build-deps \
+		autoconf \
+		ca-certificates \
+		dpkg-dev dpkg \
+		gcc \
+		gnupg \
+		libc-dev \
+		linux-headers \
+		m4 \
+		make \
+		ncurses-dev \
+		wget \
+	; \
+	\
+	OPENSSL_SOURCE_URL="https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz"; \
+	OPENSSL_PATH="/usr/local/src/openssl-$OPENSSL_VERSION"; \
+	OPENSSL_CONFIG_DIR=/usr/local/etc/ssl; \
+	\
+# /usr/local/src doesn't exist in Alpine by default
+	mkdir /usr/local/src; \
+	\
+# Required by the crypto & ssl Erlang/OTP applications
+	wget --progress dot:giga --output-document "$OPENSSL_PATH.tar.gz.asc" "$OPENSSL_SOURCE_URL.asc"; \
+	wget --progress dot:giga --output-document "$OPENSSL_PATH.tar.gz" "$OPENSSL_SOURCE_URL"; \
+	export GNUPGHOME="$(mktemp -d)"; \
+	gpg --batch --keyserver "$PGP_KEYSERVER" --recv-keys "$OPENSSL_PGP_KEY_ID"; \
+	gpg --batch --verify "$OPENSSL_PATH.tar.gz.asc" "$OPENSSL_PATH.tar.gz"; \
+	gpgconf --kill all; \
+	rm -rf "$GNUPGHOME"; \
+	echo "$OPENSSL_SOURCE_SHA256 *$OPENSSL_PATH.tar.gz" | sha256sum -c -; \
+	mkdir -p "$OPENSSL_PATH"; \
+	tar --extract --file "$OPENSSL_PATH.tar.gz" --directory "$OPENSSL_PATH" --strip-components 1; \
+	\
+# Configure OpenSSL for compilation
+	cd "$OPENSSL_PATH"; \
+	./config --openssldir="$OPENSSL_CONFIG_DIR"; \
+# Compile, install OpenSSL, verify that the command-line works & development headers are present
+	make -j "$(getconf _NPROCESSORS_ONLN)"; \
+	make install_sw install_ssldirs; \
+	cd ..; \
+	rm -rf "$OPENSSL_PATH"*; \
+# use Alpine's CA certificates
+	rmdir "$OPENSSL_CONFIG_DIR/certs" "$OPENSSL_CONFIG_DIR/private"; \
+	ln -sf /etc/ssl/certs /etc/ssl/private "$OPENSSL_CONFIG_DIR"; \
+# smoke test
+	openssl version; \
+	\
+	OTP_SOURCE_URL="https://github.com/erlang/otp/archive/OTP-$OTP_VERSION.tar.gz"; \
+	OTP_PATH="/usr/local/src/otp-$OTP_VERSION"; \
+	\
+# Download, verify & extract OTP_SOURCE
+	mkdir -p "$OTP_PATH"; \
+	wget --progress dot:giga --output-document "$OTP_PATH.tar.gz" "$OTP_SOURCE_URL"; \
+	echo "$OTP_SOURCE_SHA256 *$OTP_PATH.tar.gz" | sha256sum -c -; \
+	tar --extract --file "$OTP_PATH.tar.gz" --directory "$OTP_PATH" --strip-components 1; \
+	\
+# Configure Erlang/OTP for compilation, disable unused features & applications
+# http://erlang.org/doc/applications.html
+# ERL_TOP is required for Erlang/OTP makefiles to find the absolute path for the installation
+	cd "$OTP_PATH"; \
+	export ERL_TOP="$OTP_PATH"; \
+	./otp_build autoconf; \
+	CFLAGS="$(dpkg-buildflags --get CFLAGS)"; export CFLAGS; \
+	hostArch="$(dpkg-architecture --query DEB_HOST_GNU_TYPE)"; \
+	buildArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \
+	./configure \
+		--host="$hostArch" \
+		--build="$buildArch" \
+		--disable-dynamic-ssl-lib \
+		--disable-sctp \
+		--disable-silent-rules \
+		--enable-clock-gettime \
+		--enable-hipe \
+		--enable-hybrid-heap \
+		--enable-kernel-poll \
+		--enable-shared-zlib \
+		--enable-smp-support \
+		--enable-threads \
+		--with-microstate-accounting=extra \
+		--without-common_test \
+		--without-debugger \
+		--without-dialyzer \
+		--without-diameter \
+		--without-edoc \
+		--without-erl_docgen \
+		--without-erl_interface \
+		--without-et \
+		--without-eunit \
+		--without-ftp \
+		--without-jinterface \
+		--without-megaco \
+		--without-observer \
+		--without-odbc \
+		--without-reltool \
+		--without-ssh \
+		--without-tftp \
+		--without-wx \
+	; \
+# Compile & install Erlang/OTP
+	make -j "$(getconf _NPROCESSORS_ONLN)" GEN_OPT_FLGS="-O2 -fno-strict-aliasing"; \
+	make install; \
+	cd ..; \
+	rm -rf "$OTP_PATH"* /usr/local/lib/erlang/lib/*/src; \
+	\
+	runDeps="$( \
+		scanelf --needed --nobanner --format '%n#p' --recursive /usr/local \
+			| tr ',' '\n' \
+			| sort -u \
+			| awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \
+	)"; \
+	apk add --no-cache --virtual .otp-run-deps $runDeps; \
+	apk del --no-network .build-deps; \
+	\
+# Check that OpenSSL still works after purging build dependencies
+	openssl version; \
+# Check that Erlang/OTP crypto & ssl were compiled against OpenSSL correctly
+	erl -noshell -eval 'io:format("~p~n~n~p~n~n", [crypto:supports(), ssl:versions()]), init:stop().'
 
-ENV RABBITMQ_HOME /opt/rabbitmq
-ENV PATH $RABBITMQ_HOME/sbin:$PATH
+ENV RABBITMQ_DATA_DIR=/var/lib/rabbitmq
+# Create rabbitmq system user & group, fix permissions & allow root user to connect to the RabbitMQ Erlang VM
+RUN set -eux; \
+	addgroup -g 101 -S rabbitmq; \
+	adduser -u 100 -S -h "$RABBITMQ_DATA_DIR" -G rabbitmq rabbitmq; \
+	mkdir -p "$RABBITMQ_DATA_DIR" /etc/rabbitmq /tmp/rabbitmq-ssl /var/log/rabbitmq; \
+	chown -fR rabbitmq:rabbitmq "$RABBITMQ_DATA_DIR" /etc/rabbitmq /tmp/rabbitmq-ssl /var/log/rabbitmq; \
+	chmod 777 "$RABBITMQ_DATA_DIR" /etc/rabbitmq /tmp/rabbitmq-ssl /var/log/rabbitmq; \
+	ln -sf "$RABBITMQ_DATA_DIR/.erlang.cookie" /root/.erlang.cookie
 
-# gpg: key 6026DFCA: public key "RabbitMQ Release Signing Key <[email protected]>" imported
-ENV RABBITMQ_GPG_KEY 0A9AF2115F4687BD29803A206B73A36E6026DFCA
+# Use the latest stable RabbitMQ release (https://www.rabbitmq.com/download.html)
+ENV RABBITMQ_VERSION="3.7.11-rc.1"
+# https://www.rabbitmq.com/signatures.html#importing-gpg
+ENV RABBITMQ_PGP_KEY_ID="0x0A9AF2115F4687BD29803A206B73A36E6026DFCA"
+ENV RABBITMQ_HOME=/opt/rabbitmq
 
-ENV RABBITMQ_VERSION 3.7.8-rc.4
-ENV RABBITMQ_GITHUB_TAG v3.7.8-rc.4
+# Add RabbitMQ to PATH, send all logs to TTY
+ENV PATH=$RABBITMQ_HOME/sbin:$PATH \
+	RABBITMQ_LOGS=- RABBITMQ_SASL_LOGS=-
 
-RUN set -ex; \
+# Install RabbitMQ
+RUN set -eux; \
 	\
 	apk add --no-cache --virtual .build-deps \
 		ca-certificates \
 		gnupg \
-		libressl \
 		wget \
 		xz \
 	; \
 	\
-	wget -O rabbitmq-server.tar.xz.asc "https://github.com/rabbitmq/rabbitmq-server/releases/download/$RABBITMQ_GITHUB_TAG/rabbitmq-server-generic-unix-${RABBITMQ_VERSION}.tar.xz.asc"; \
-	wget -O rabbitmq-server.tar.xz     "https://github.com/rabbitmq/rabbitmq-server/releases/download/$RABBITMQ_GITHUB_TAG/rabbitmq-server-generic-unix-${RABBITMQ_VERSION}.tar.xz"; \
+	RABBITMQ_SOURCE_URL="https://github.com/rabbitmq/rabbitmq-server/releases/download/v$RABBITMQ_VERSION/rabbitmq-server-generic-unix-$RABBITMQ_VERSION.tar.xz"; \
+	RABBITMQ_PATH="/usr/local/src/rabbitmq-$RABBITMQ_VERSION"; \
+	\
+	wget --progress dot:giga --output-document "$RABBITMQ_PATH.tar.xz.asc" "$RABBITMQ_SOURCE_URL.asc"; \
+	wget --progress dot:giga --output-document "$RABBITMQ_PATH.tar.xz" "$RABBITMQ_SOURCE_URL"; \
 	\
 	export GNUPGHOME="$(mktemp -d)"; \
-	gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys "$RABBITMQ_GPG_KEY"; \
-	gpg --batch --verify rabbitmq-server.tar.xz.asc rabbitmq-server.tar.xz; \
-	command -v gpgconf && gpgconf --kill all || :; \
+	gpg --batch --keyserver "$PGP_KEYSERVER" --recv-keys "$RABBITMQ_PGP_KEY_ID"; \
+	gpg --batch --verify "$RABBITMQ_PATH.tar.xz.asc" "$RABBITMQ_PATH.tar.xz"; \
+	gpgconf --kill all; \
 	rm -rf "$GNUPGHOME"; \
 	\
 	mkdir -p "$RABBITMQ_HOME"; \
-	tar \
-		--extract \
-		--verbose \
-		--file rabbitmq-server.tar.xz \
-		--directory "$RABBITMQ_HOME" \
-		--strip-components 1 \
-	; \
-	rm -f rabbitmq-server.tar.xz*; \
-	\
-# update SYS_PREFIX (first making sure it's set to what we expect it to be)
+	tar --extract --file "$RABBITMQ_PATH.tar.xz" --directory "$RABBITMQ_HOME" --strip-components 1; \
+	rm -rf "$RABBITMQ_PATH"*; \
+# Do not default SYS_PREFIX to RABBITMQ_HOME, leave it empty
 	grep -qE '^SYS_PREFIX=\$\{RABBITMQ_HOME\}$' "$RABBITMQ_HOME/sbin/rabbitmq-defaults"; \
-	sed -ri 's!^(SYS_PREFIX=).*$!\1!g' "$RABBITMQ_HOME/sbin/rabbitmq-defaults"; \
+	sed -i 's/^SYS_PREFIX=.*$/SYS_PREFIX=/' "$RABBITMQ_HOME/sbin/rabbitmq-defaults"; \
 	grep -qE '^SYS_PREFIX=$' "$RABBITMQ_HOME/sbin/rabbitmq-defaults"; \
+	chown -R rabbitmq:rabbitmq "$RABBITMQ_HOME"; \
 	\
-	apk del .build-deps
-
-# set home so that any `--user` knows where to put the erlang cookie
-ENV HOME /var/lib/rabbitmq
+	apk del .build-deps; \
+	\
+# verify assumption of no stale cookies
+	[ ! -e "$RABBITMQ_DATA_DIR/.erlang.cookie" ]; \
+# Ensure RabbitMQ was installed correctly by running a few commands that do not depend on a running server, as the rabbitmq user
+# If they all succeed, it's safe to assume that things have been set up correctly
+	su-exec rabbitmq rabbitmqctl help; \
+	su-exec rabbitmq rabbitmqctl list_ciphers; \
+	su-exec rabbitmq rabbitmq-plugins list; \
+# no stale cookies
+	rm "$RABBITMQ_DATA_DIR/.erlang.cookie"
 
-RUN mkdir -p /var/lib/rabbitmq /etc/rabbitmq /var/log/rabbitmq /tmp/rabbitmq-ssl \
-	&& chown -R rabbitmq:rabbitmq /var/lib/rabbitmq /etc/rabbitmq /var/log/rabbitmq /tmp/rabbitmq-ssl \
-	&& chmod -R 777 /var/lib/rabbitmq /etc/rabbitmq /var/log/rabbitmq /tmp/rabbitmq-ssl
-VOLUME /var/lib/rabbitmq
+# Added for backwards compatibility - users can simply COPY custom plugins to /plugins
+RUN ln -sf /opt/rabbitmq/plugins /plugins
 
-# add a symlink to the .erlang.cookie in /root so we can "docker exec rabbitmqctl ..." without gosu
-RUN ln -sf /var/lib/rabbitmq/.erlang.cookie /root/
+# set home so that any `--user` knows where to put the erlang cookie
+ENV HOME $RABBITMQ_DATA_DIR
+# Hint that the data (a.k.a. home dir) dir should be separate volume
+VOLUME $RABBITMQ_DATA_DIR
 
-RUN ln -sf "$RABBITMQ_HOME/plugins" /plugins
+# warning: the VM is running with native name encoding of latin1 which may cause Elixir to malfunction as it expects utf8. Please ensure your locale is set to UTF-8 (which can be verified by running "locale" in your shell)
+# Setting all environment variables that control language preferences, behaviour differs - https://www.gnu.org/software/gettext/manual/html_node/The-LANGUAGE-variable.html#The-LANGUAGE-variable
+# https://docs.docker.com/samples/library/ubuntu/#locales
+ENV LANG=C.UTF-8 LANGUAGE=C.UTF-8 LC_ALL=C.UTF-8
 
 COPY docker-entrypoint.sh /usr/local/bin/
 ENTRYPOINT ["docker-entrypoint.sh"]

3.7-rc/alpine/docker-entrypoint.sh

@@ -365,7 +365,7 @@ if [ "$1" = 'rabbitmq-server' ] && [ "$shouldWriteConfig" ]; then
 
 	# if management plugin is installed, generate config for it
 	# https://www.rabbitmq.com/management.html#configuration
-	if [ "$(rabbitmq-plugins list -m -e rabbitmq_management)" ]; then
+	if [ "$(rabbitmq-plugins list -q -m -e rabbitmq_management)" ]; then
 		if [ "$haveManagementSslConfig" ]; then
 			rabbit_set_config 'management.listener.port' 15671
 			rabbit_set_config 'management.listener.ssl' 'true'