diff --git a/.scripts/upload_new_async-profiler_version.sh b/.scripts/upload_new_async-profiler_version.sh index e70e27ee2..209625785 100755 --- a/.scripts/upload_new_async-profiler_version.sh +++ b/.scripts/upload_new_async-profiler_version.sh @@ -43,10 +43,10 @@ for arch in "${ARCHITECTURES[@]}"; do file=async-profiler-$VERSION-linux-$arch.tar.gz echo "Downloading $file from github.com" - curl --fail -LOs "https://github.com/async-profiler/async-profiler/releases/download/v$VERSION/$file" + curl --fail -LO --progress-bar "https://github.com/async-profiler/async-profiler/releases/download/v$VERSION/$file" echo "Uploading $file to Nexus" - curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" \ + curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" \ --upload-file "$file" \ 'https://repo.stackable.tech/repository/packages/async-profiler/' done diff --git a/.scripts/upload_new_inotify-tools_version.sh b/.scripts/upload_new_inotify-tools_version.sh index 8d5294a77..18f1683f6 100755 --- a/.scripts/upload_new_inotify-tools_version.sh +++ b/.scripts/upload_new_inotify-tools_version.sh @@ -49,10 +49,10 @@ for arch in "${ARCHITECTURES[@]}"; do file=inotify-tools-$VERSION.$arch.rpm echo "Downloading $file from dl.fedoraproject.org" - curl --fail -LOs "https://dl.fedoraproject.org/pub/epel/$EPEL_VERSION/Everything/$arch/Packages/i/$file" + curl --fail -LO --progress-bar "https://dl.fedoraproject.org/pub/epel/$EPEL_VERSION/Everything/$arch/Packages/i/$file" echo "Uploading $file to Nexus" - curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" \ + curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" \ --upload-file "$file" \ 'https://repo.stackable.tech/repository/packages/inotify-tools/' done diff --git a/.scripts/upload_new_jmx_exporter_version.sh b/.scripts/upload_new_jmx_exporter_version.sh index 33386270c..8ad909d2a 100755 --- a/.scripts/upload_new_jmx_exporter_version.sh +++ b/.scripts/upload_new_jmx_exporter_version.sh @@ -35,15 +35,15 @@ JAR_FILE="jmx_prometheus_javaagent-$VERSION.jar" SUM_FILE="$JAR_FILE.sha256" echo "Downloading JMX Exporter" -curl --fail -LOs "https://github.com/prometheus/jmx_exporter/releases/download/$VERSION/$JAR_FILE" -curl --fail -LOs "https://github.com/prometheus/jmx_exporter/releases/download/$VERSION/$SUM_FILE" +curl --fail -LO --progress-bar "https://github.com/prometheus/jmx_exporter/releases/download/$VERSION/$JAR_FILE" +curl --fail -LO --progress-bar "https://github.com/prometheus/jmx_exporter/releases/download/$VERSION/$SUM_FILE" # Check that sha256 sum matches before uploading sha256sum --check --status "$SUM_FILE" && echo "SHA256 Sum matches" echo "Uploading to Nexus" -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$JAR_FILE" 'https://repo.stackable.tech/repository/packages/jmx-exporter/' -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$SUM_FILE" 'https://repo.stackable.tech/repository/packages/jmx-exporter/' +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$JAR_FILE" 'https://repo.stackable.tech/repository/packages/jmx-exporter/' +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$SUM_FILE" 'https://repo.stackable.tech/repository/packages/jmx-exporter/' echo "Successfully uploaded new version of the JMX Exporter ($VERSION) Jar to Nexus" echo "https://repo.stackable.tech/service/rest/repository/browse/packages/jmx-exporter/" diff --git a/.scripts/upload_new_kcat_version.sh b/.scripts/upload_new_kcat_version.sh index a6b91b3b3..5915d8858 100755 --- a/.scripts/upload_new_kcat_version.sh +++ b/.scripts/upload_new_kcat_version.sh @@ -38,7 +38,7 @@ echo "Downloading kcat" curl --fail -Ls -o "kcat-$VERSION.tar.gz" "https://github.com/edenhill/kcat/archive/refs/tags/$VERSION.tar.gz" echo "Uploading to Nexus" -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "kcat-$VERSION.tar.gz" 'https://repo.stackable.tech/repository/packages/kcat/' +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "kcat-$VERSION.tar.gz" 'https://repo.stackable.tech/repository/packages/kcat/' echo "Successfully uploaded new version of kcat ($VERSION) to Nexus" diff --git a/.scripts/upload_new_protoc_version.sh b/.scripts/upload_new_protoc_version.sh index 29a72b6c2..889f2ccdd 100755 --- a/.scripts/upload_new_protoc_version.sh +++ b/.scripts/upload_new_protoc_version.sh @@ -50,7 +50,7 @@ for arch in "${ARCHITECTURES[@]}"; do FILE_NAME=$(basename "$DOWNLOAD_URL") echo "Uploading protoc to Nexus" - if ! curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$FILE_NAME" 'https://repo.stackable.tech/repository/packages/protoc/'; then + if ! curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$FILE_NAME" 'https://repo.stackable.tech/repository/packages/protoc/'; then echo "Failed to upload protoc to Nexus" exit 1 fi diff --git a/.scripts/upload_new_tini_version.sh b/.scripts/upload_new_tini_version.sh index 96744887a..e22df07be 100755 --- a/.scripts/upload_new_tini_version.sh +++ b/.scripts/upload_new_tini_version.sh @@ -41,7 +41,7 @@ for arch in "${ARCHITECTURES[@]}"; do curl --fail -Ls -o "tini-$VERSION-$arch" "https://github.com/krallin/tini/releases/download/v$VERSION/tini-$arch" echo "Uploading to Nexus" - curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "tini-$VERSION-$arch" 'https://repo.stackable.tech/repository/packages/tini/' + curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "tini-$VERSION-$arch" 'https://repo.stackable.tech/repository/packages/tini/' done echo "Successfully uploaded new version of TINI ($VERSION) to Nexus" diff --git a/druid/upload_new_druid_version.sh b/druid/upload_new_druid_version.sh index dd54b25dc..0a3a0929e 100755 --- a/druid/upload_new_druid_version.sh +++ b/druid/upload_new_druid_version.sh @@ -5,6 +5,11 @@ set -euo pipefail VERSION=${1:?"Missing version number argument (arg 1)"} NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"} +# We prefer fast downloads... +BASE_DOWNLOAD_URL="https://dlcdn.apache.org/druid" +# However, if the version is not available, use the slow archive instead: +# BASE_DOWNLOAD_URL="https://archive.apache.org/dist/druid" + read -r -s -p "Nexus Password: " NEXUS_PASSWORD echo "" @@ -33,10 +38,10 @@ cd "$WORK_DIR" || exit src_file="apache-druid-${VERSION}-src.tar.gz" -echo "Downloading Druid (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" -curl --fail -LOs "https://archive.apache.org/dist/druid/${VERSION}/${src_file}" -curl --fail -LOs "https://archive.apache.org/dist/druid/${VERSION}/${src_file}.asc" -curl --fail -LOs "https://archive.apache.org/dist/druid/${VERSION}/${src_file}.sha512" +echo "Downloading Druid source (if this fails, try switching the BASE_DOWNLOAD_URL to the archive)" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/${VERSION}/${src_file}" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/${VERSION}/${src_file}.asc" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/${VERSION}/${src_file}.sha512" # It is probably redundant to check both the checksum and the signature but it's cheap and why not echo "Validating SHA512 Checksum" @@ -46,18 +51,18 @@ if ! (sha512sum "${src_file}" | cut -d " " -f 1 | diff -Z - "${src_file}.sha512" fi echo "Validating signature" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://www.apache.org/dist/druid/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://archive.apache.org/dist/druid/KEYS | gpg --import")' - if ! (gpg --verify "${src_file}.asc" "${src_file}" 2> /dev/null); then echo "ERROR: The signature could not be verified" + echo "--> Make sure you have imported the KEYS file (${BASE_DOWNLOAD_URL}/KEYS) into GPG: https://www.apache.org/info/verification.html" + echo "--> e.g. \"curl ${BASE_DOWNLOAD_URL}/KEYS | gpg --import\"" exit 1 fi echo "Uploading everything to Nexus" EXIT_STATUS=0 -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}" 'https://repo.stackable.tech/repository/packages/druid/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.asc" 'https://repo.stackable.tech/repository/packages/druid/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.sha512" 'https://repo.stackable.tech/repository/packages/druid/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}" 'https://repo.stackable.tech/repository/packages/druid/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.asc" 'https://repo.stackable.tech/repository/packages/druid/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.sha512" 'https://repo.stackable.tech/repository/packages/druid/' || EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then echo "ERROR: Upload failed" diff --git a/hadoop/upload_new_hadoop_version.sh b/hadoop/upload_new_hadoop_version.sh index 15f58a9f9..9c390c967 100755 --- a/hadoop/upload_new_hadoop_version.sh +++ b/hadoop/upload_new_hadoop_version.sh @@ -39,16 +39,16 @@ cd "$WORK_DIR" || exit bin_file=hadoop-$VERSION.tar.gz src_file=hadoop-$VERSION-src.tar.gz -echo "Downloading Hadoop (if this fails, try switching the BASE_DOWNLOAD_URL to the archive)" +echo "Downloading Hadoop binary (if this fails, try switching the BASE_DOWNLOAD_URL to the archive)" curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hadoop-$VERSION/$bin_file" curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hadoop-$VERSION/$bin_file.asc" curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hadoop-$VERSION/$bin_file.sha512" +echo "Downloading Hadoop source (if this fails, try switching the BASE_DOWNLOAD_URL to the archive)" curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hadoop-$VERSION/$src_file" curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hadoop-$VERSION/$src_file.asc" curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hadoop-$VERSION/$src_file.sha512" - # It is probably redundant to check both the checksum and the signature but it's cheap and why not echo "Validating SHA512 Checksums" if ! (sha512sum --tag "$bin_file" | diff - "$bin_file.sha512" && sha512sum --tag "$src_file" | diff - "$src_file.sha512"); then @@ -58,22 +58,22 @@ if ! (sha512sum --tag "$bin_file" | diff - "$bin_file.sha512" && sha512sum --tag fi echo "Validating signatures" -echo "--> NOTE: Make sure you have downloaded and added the KEYS file (${BASE_DOWNLOAD_URL}/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using \"curl ${BASE_DOWNLOAD_URL}/KEYS | gpg --import\")" - if ! (gpg --verify "$bin_file.asc" "$bin_file" 2> /dev/null && gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then echo "ERROR: One of the signatures could not be verified" + echo "--> Make sure you have imported the KEYS file (${BASE_DOWNLOAD_URL}/KEYS) into GPG: https://www.apache.org/info/verification.html" + echo "--> e.g. \"curl ${BASE_DOWNLOAD_URL}/KEYS | gpg --import\"" exit 1 fi echo "Uploading everything to Nexus" EXIT_STATUS=0 -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file" 'https://repo.stackable.tech/repository/packages/hadoop/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file.asc" 'https://repo.stackable.tech/repository/packages/hadoop/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file.sha512" 'https://repo.stackable.tech/repository/packages/hadoop/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file" 'https://repo.stackable.tech/repository/packages/hadoop/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file.asc" 'https://repo.stackable.tech/repository/packages/hadoop/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file.sha512" 'https://repo.stackable.tech/repository/packages/hadoop/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file" 'https://repo.stackable.tech/repository/packages/hadoop/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.asc" 'https://repo.stackable.tech/repository/packages/hadoop/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.sha512" 'https://repo.stackable.tech/repository/packages/hadoop/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file" 'https://repo.stackable.tech/repository/packages/hadoop/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.asc" 'https://repo.stackable.tech/repository/packages/hadoop/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.sha512" 'https://repo.stackable.tech/repository/packages/hadoop/' || EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then echo "ERROR: Upload failed" diff --git a/hbase/upload_new_hbase-operator-tools_version.sh b/hbase/upload_new_hbase-operator-tools_version.sh index 0872e2ee3..aa33a3379 100755 --- a/hbase/upload_new_hbase-operator-tools_version.sh +++ b/hbase/upload_new_hbase-operator-tools_version.sh @@ -5,6 +5,11 @@ set -euo pipefail VERSION=${1:?"Missing version number argument (arg 1)"} NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"} +# We prefer fast downloads... +BASE_DOWNLOAD_URL="https://dlcdn.apache.org/hbase" +# However, if the version is not available, use the slow archive instead: +# BASE_DOWNLOAD_URL="https://archive.apache.org/dist/hbase" + read -r -s -p "Nexus Password: " NEXUS_PASSWORD echo "" @@ -33,11 +38,10 @@ cd "$WORK_DIR" || exit src_file=hbase-operator-tools-$VERSION-src.tar.gz -echo "Downloading hbase-operator-tools (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" -curl --fail -LOs "https://archive.apache.org/dist/hbase/hbase-operator-tools-$VERSION/$src_file" -curl --fail -LOs "https://archive.apache.org/dist/hbase/hbase-operator-tools-$VERSION/$src_file.asc" -curl --fail -LOs "https://archive.apache.org/dist/hbase/hbase-operator-tools-$VERSION/$src_file.sha512" - +echo "Downloading hbase-operator-tools source (if this fails, try switching the BASE_DOWNLOAD_URL to the archive)" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hbase-operator-tools-$VERSION/$src_file" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hbase-operator-tools-$VERSION/$src_file.asc" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hbase-operator-tools-$VERSION/$src_file.sha512" # It is probably redundant to check both the checksum and the signature but it's cheap and why not echo "Validating SHA512 Checksums" @@ -47,9 +51,10 @@ if ! (gpg --print-md SHA512 "$src_file" | diff - "$src_file.sha512"); then fi echo "Validating signatures" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/hbase/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://downloads.apache.org/hbase/KEYS | gpg --import")' if ! (gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then echo "ERROR: One of the signatures could not be verified" + echo "--> Make sure you have imported the KEYS file (${BASE_DOWNLOAD_URL}/KEYS) into GPG: https://www.apache.org/info/verification.html" + echo "--> e.g. \"curl ${BASE_DOWNLOAD_URL}/KEYS | gpg --import\"" exit 1 fi @@ -57,9 +62,9 @@ fi echo "Uploading everything to Nexus" EXIT_STATUS=0 -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file" 'https://repo.stackable.tech/repository/packages/hbase-operator-tools/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.asc" 'https://repo.stackable.tech/repository/packages/hbase-operator-tools/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.sha512" 'https://repo.stackable.tech/repository/packages/hbase-operator-tools/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file" 'https://repo.stackable.tech/repository/packages/hbase-operator-tools/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.asc" 'https://repo.stackable.tech/repository/packages/hbase-operator-tools/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.sha512" 'https://repo.stackable.tech/repository/packages/hbase-operator-tools/' || EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then diff --git a/hbase/upload_new_hbase_version.sh b/hbase/upload_new_hbase_version.sh index 80594d4d5..9b0c9ceef 100755 --- a/hbase/upload_new_hbase_version.sh +++ b/hbase/upload_new_hbase_version.sh @@ -5,6 +5,11 @@ set -euo pipefail VERSION=${1:?"Missing version number argument (arg 1)"} NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"} +# We prefer fast downloads... +BASE_DOWNLOAD_URL="https://dlcdn.apache.org/hbase" +# However, if the version is not available, use the slow archive instead: +# BASE_DOWNLOAD_URL="https://archive.apache.org/dist/hbase" + read -r -s -p "Nexus Password: " NEXUS_PASSWORD echo "" @@ -33,11 +38,10 @@ cd "$WORK_DIR" || exit src_file=hbase-$VERSION-src.tar.gz -echo "Downloading HBase source (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" -curl --fail -LOs "https://archive.apache.org/dist/hbase/$VERSION/$src_file" -curl --fail -LOs "https://archive.apache.org/dist/hbase/$VERSION/$src_file.asc" -curl --fail -LOs "https://archive.apache.org/dist/hbase/$VERSION/$src_file.sha512" - +echo "Downloading HBase source (if this fails, try switching the BASE_DOWNLOAD_URL to the archive)" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/$VERSION/$src_file" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/$VERSION/$src_file.asc" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/$VERSION/$src_file.sha512" # It is probably redundant to check both the checksum and the signature but it's cheap and why not echo "Validating SHA512 Checksums" @@ -47,17 +51,18 @@ if ! (gpg --print-md SHA512 "$src_file" | diff - "$src_file.sha512"); then fi echo "Validating signatures" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/hbase/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://downloads.apache.org/hbase/KEYS | gpg --import")' if ! (gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then echo "ERROR: One of the signatures could not be verified" + echo "--> Make sure you have imported the KEYS file (${BASE_DOWNLOAD_URL}/KEYS) into GPG: https://www.apache.org/info/verification.html" + echo "--> e.g. \"curl ${BASE_DOWNLOAD_URL}/KEYS | gpg --import\"" exit 1 fi echo "Uploading everything to Nexus" EXIT_STATUS=0 -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file" 'https://repo.stackable.tech/repository/packages/hbase/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.asc" 'https://repo.stackable.tech/repository/packages/hbase/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.sha512" 'https://repo.stackable.tech/repository/packages/hbase/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file" 'https://repo.stackable.tech/repository/packages/hbase/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.asc" 'https://repo.stackable.tech/repository/packages/hbase/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.sha512" 'https://repo.stackable.tech/repository/packages/hbase/' || EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then echo "ERROR: Upload failed" diff --git a/hbase/upload_new_phoenix_version.sh b/hbase/upload_new_phoenix_version.sh index 83c099df3..921af8625 100755 --- a/hbase/upload_new_phoenix_version.sh +++ b/hbase/upload_new_phoenix_version.sh @@ -5,6 +5,11 @@ set -euo pipefail VERSION=${1:?"Missing version number argument (arg 1)"} NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"} +# We prefer fast downloads... +BASE_DOWNLOAD_URL="https://dlcdn.apache.org/phoenix" +# However, if the version is not available, use the slow archive instead: +# BASE_DOWNLOAD_URL="https://archive.apache.org/dist/phoenix" + read -r -s -p "Nexus Password: " NEXUS_PASSWORD echo "" @@ -33,11 +38,10 @@ cd "$WORK_DIR" || exit src_file=phoenix-$VERSION-src.tar.gz -echo "Downloading phoenix (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" -curl --fail -LOs "https://archive.apache.org/dist/phoenix/phoenix-$VERSION/$src_file" -curl --fail -LOs "https://archive.apache.org/dist/phoenix/phoenix-$VERSION/$src_file.asc" -curl --fail -LOs "https://archive.apache.org/dist/phoenix/phoenix-$VERSION/$src_file.sha512" - +echo "Downloading phoenix source (if this fails, try switching the BASE_DOWNLOAD_URL to the archive)" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/phoenix-$VERSION/$src_file" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/phoenix-$VERSION/$src_file.asc" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/phoenix-$VERSION/$src_file.sha512" # It is probably redundant to check both the checksum and the signature but it's cheap and why not echo "Validating SHA512 Checksums" @@ -47,9 +51,10 @@ if ! (gpg --print-md SHA512 "$src_file" | diff - "$src_file.sha512"); then fi echo "Validating signatures" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/phoenix/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://downloads.apache.org/phoenix/KEYS | gpg --import")' if ! (gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then echo "ERROR: One of the signatures could not be verified" + echo "--> Make sure you have imported the KEYS file (${BASE_DOWNLOAD_URL}/KEYS) into GPG: https://www.apache.org/info/verification.html" + echo "--> e.g. \"curl ${BASE_DOWNLOAD_URL}/KEYS | gpg --import\"" exit 1 fi @@ -57,9 +62,9 @@ fi echo "Uploading everything to Nexus" EXIT_STATUS=0 -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file" 'https://repo.stackable.tech/repository/packages/phoenix/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.asc" 'https://repo.stackable.tech/repository/packages/phoenix/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.sha512" 'https://repo.stackable.tech/repository/packages/phoenix/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file" 'https://repo.stackable.tech/repository/packages/phoenix/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.asc" 'https://repo.stackable.tech/repository/packages/phoenix/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.sha512" 'https://repo.stackable.tech/repository/packages/phoenix/' || EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then diff --git a/hive/upload_new_hive_version.sh b/hive/upload_new_hive_version.sh index 1ed8bbb59..2c7e51804 100755 --- a/hive/upload_new_hive_version.sh +++ b/hive/upload_new_hive_version.sh @@ -5,6 +5,11 @@ set -euo pipefail VERSION=${1:?"Missing version number argument (arg 1)"} NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"} +# We prefer fast downloads... +BASE_DOWNLOAD_URL="https://dlcdn.apache.org/hive" +# However, if the version is not available, use the slow archive instead: +# BASE_DOWNLOAD_URL="https://archive.apache.org/dist/hive" + read -r -s -p "Nexus Password: " NEXUS_PASSWORD echo "" @@ -34,16 +39,15 @@ cd "$WORK_DIR" || exit bin_file="apache-hive-${VERSION}-bin.tar.gz" src_file="apache-hive-$VERSION-src.tar.gz" -echo "Downloading Hive (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" -curl --fail -LOs "https://dlcdn.apache.org/hive/hive-${VERSION}/${bin_file}" -curl --fail -LOs "https://dlcdn.apache.org/hive/hive-${VERSION}/${bin_file}.asc" -curl --fail -LOs "https://dlcdn.apache.org/hive/hive-${VERSION}/${bin_file}.sha256" - -echo "Downloading Hive (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" -curl --fail -LOs "https://dlcdn.apache.org/hive/hive-${VERSION}/${src_file}" -curl --fail -LOs "https://dlcdn.apache.org/hive/hive-${VERSION}/${src_file}.asc" -curl --fail -LOs "https://dlcdn.apache.org/hive/hive-${VERSION}/${src_file}.sha256" +echo "Downloading Hive binary (if this fails, try switching the BASE_DOWNLOAD_URL to the archive)" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hive-${VERSION}/${bin_file}" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hive-${VERSION}/${bin_file}.asc" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hive-${VERSION}/${bin_file}.sha256" +echo "Downloading Hive source (if this fails, try switching the BASE_DOWNLOAD_URL to the archive)" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hive-${VERSION}/${src_file}" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hive-${VERSION}/${src_file}.asc" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hive-${VERSION}/${src_file}.sha256" # It is probably redundant to check both the checksum and the signature but it's cheap and why not echo "Validating SHA256 Checksums" @@ -53,22 +57,22 @@ if ! (sha256sum "${bin_file}" | diff - "${bin_file}.sha256" && sha256sum "${src_ fi echo "Validating signatures" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://dlcdn.apache.org/hive/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://dlcdn.apache.org/hive/KEYS | gpg --import")' - if ! (gpg --verify "$bin_file.asc" "$bin_file" 2> /dev/null && gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then echo "ERROR: Signature could not be verified" + echo "--> Make sure you have imported the KEYS file (${BASE_DOWNLOAD_URL}/KEYS) into GPG: https://www.apache.org/info/verification.html" + echo "--> e.g. \"curl ${BASE_DOWNLOAD_URL}/KEYS | gpg --import\"" exit 1 fi echo "Uploading everything to Nexus" EXIT_STATUS=0 -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file" 'https://repo.stackable.tech/repository/packages/hive/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file.asc" 'https://repo.stackable.tech/repository/packages/hive/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file.sha256" 'https://repo.stackable.tech/repository/packages/hive/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file" 'https://repo.stackable.tech/repository/packages/hive/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file.asc" 'https://repo.stackable.tech/repository/packages/hive/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file.sha256" 'https://repo.stackable.tech/repository/packages/hive/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file" 'https://repo.stackable.tech/repository/packages/hive/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.asc" 'https://repo.stackable.tech/repository/packages/hive/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.sha256" 'https://repo.stackable.tech/repository/packages/hive/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file" 'https://repo.stackable.tech/repository/packages/hive/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.asc" 'https://repo.stackable.tech/repository/packages/hive/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.sha256" 'https://repo.stackable.tech/repository/packages/hive/' || EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then diff --git a/kafka/upload_new_kafka_version.sh b/kafka/upload_new_kafka_version.sh index 546880da7..8145f672e 100755 --- a/kafka/upload_new_kafka_version.sh +++ b/kafka/upload_new_kafka_version.sh @@ -5,6 +5,11 @@ set -euo pipefail VERSION=${1:?"Missing version number argument (arg 1)"} NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"} +# We prefer fast downloads... +BASE_DOWNLOAD_URL="https://dlcdn.apache.org/kafka" +# However, if the version is not available, use the slow archive instead: +# BASE_DOWNLOAD_URL="https://archive.apache.org/dist/kafka" + read -r -s -p "Nexus Password: " NEXUS_PASSWORD echo "" @@ -34,14 +39,15 @@ cd "$WORK_DIR" || exit bin_file=kafka_2.13-$VERSION.tgz src_file=kafka-$VERSION-src.tgz -echo "Downloading Kafka (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" -curl --fail -LOs "https://archive.apache.org/dist/kafka/$VERSION/$bin_file" -curl --fail -LOs "https://archive.apache.org/dist/kafka/$VERSION/$bin_file.asc" -curl --fail -LOs "https://archive.apache.org/dist/kafka/$VERSION/$bin_file.sha512" +echo "Downloading Kafka binary (if this fails, try switching the BASE_DOWNLOAD_URL to the archive)" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/$VERSION/$bin_file" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/$VERSION/$bin_file.asc" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/$VERSION/$bin_file.sha512" -curl --fail -LOs "https://archive.apache.org/dist/kafka/$VERSION/$src_file" -curl --fail -LOs "https://archive.apache.org/dist/kafka/$VERSION/$src_file.asc" -curl --fail -LOs "https://archive.apache.org/dist/kafka/$VERSION/$src_file.sha512" +echo "Downloading Kafka source (if this fails, try switching the BASE_DOWNLOAD_URL to the archive)" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/$VERSION/$src_file" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/$VERSION/$src_file.asc" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/$VERSION/$src_file.sha512" # It is probably redundant to check both the checksum and the signature but it's cheap and why not echo "Validating SHA512 Checksum" @@ -51,22 +57,22 @@ if ! (gpg --print-md SHA512 "$bin_file" | diff - "$bin_file.sha512" && gpg --pri fi echo "Validating signatures" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/kafka/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://downloads.apache.org/kafka/KEYS | gpg --import")' - if ! (gpg --verify "$bin_file.asc" "$bin_file" 2> /dev/null && gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then echo "ERROR: One of the signatures could not be verified" + echo "--> Make sure you have imported the KEYS file (${BASE_DOWNLOAD_URL}/KEYS) into GPG: https://www.apache.org/info/verification.html" + echo "--> e.g. \"curl ${BASE_DOWNLOAD_URL}/KEYS | gpg --import\"" exit 1 fi echo "Uploading everything to Nexus" EXIT_STATUS=0 -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file.asc" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file.sha512" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file.asc" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file.sha512" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.asc" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.sha512" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.asc" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.sha512" 'https://repo.stackable.tech/repository/packages/kafka/' || EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then echo "ERROR: Upload failed" diff --git a/nifi/upload_new_nifi_version.sh b/nifi/upload_new_nifi_version.sh index 1b053253e..a05445af4 100755 --- a/nifi/upload_new_nifi_version.sh +++ b/nifi/upload_new_nifi_version.sh @@ -5,6 +5,11 @@ set -euo pipefail VERSION=${1:?"Missing version number argument (arg 1)"} NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"} +# We prefer fast downloads... +BASE_DOWNLOAD_URL="https://dlcdn.apache.org/nifi" +# However, if the version is not available, use the slow archive instead: +# BASE_DOWNLOAD_URL="https://archive.apache.org/dist/nifi" + read -r -s -p "Nexus Password: " NEXUS_PASSWORD echo "" @@ -32,12 +37,11 @@ trap cleanup EXIT cd "$WORK_DIR" || exit src_file="nifi-$VERSION-source-release.zip" -download_url="https://archive.apache.org/dist/nifi/${VERSION}" -echo "Downloading NiFi source (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" -curl --fail -LOs "${download_url}/${src_file}" -curl --fail -LOs "${download_url}/${src_file}.asc" -curl --fail -LOs "${download_url}/${src_file}.sha512" +echo "Downloading NiFi source (if this fails, try switching the BASE_DOWNLOAD_URL to the archive)" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/$VERSION/${src_file}" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/$VERSION/${src_file}.asc" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/$VERSION/${src_file}.sha512" # It is probably redundant to check both the checksum and the signature but it's cheap and why not echo "Validating SHA512 Checksums" @@ -51,18 +55,18 @@ if ! (sha512sum "$src_file" | cut -d ' ' -f 1 | diff - <(echo -e "$(<"${src_file fi echo "Validating signatures" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/nifi/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://archive.apache.org/dist/nifi/KEYS | gpg --import")' - if ! (gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then echo "ERROR: One of the signatures could not be verified" + echo "--> Make sure you have imported the KEYS file (${BASE_DOWNLOAD_URL}/KEYS) into GPG: https://www.apache.org/info/verification.html" + echo "--> e.g. \"curl ${BASE_DOWNLOAD_URL}/KEYS | gpg --import\"" exit 1 fi echo "Uploading everything to Nexus" EXIT_STATUS=0 -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.asc" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.sha512" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.asc" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.sha512" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then echo "ERROR: Upload failed" diff --git a/omid/upload_new_omid_version.sh b/omid/upload_new_omid_version.sh index cbfc5db94..d3d9ea7b5 100755 --- a/omid/upload_new_omid_version.sh +++ b/omid/upload_new_omid_version.sh @@ -5,6 +5,11 @@ set -e VERSION=${1:?"Missing version number argument (arg 1)"} NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"} +# We prefer fast downloads... +BASE_DOWNLOAD_URL="https://dlcdn.apache.org/phoenix" +# However, if the version is not available, use the slow archive instead: +# BASE_DOWNLOAD_URL="https://archive.apache.org/dist/phoenix" + read -r -s -p "Nexus Password: " NEXUS_PASSWORD echo "" @@ -32,11 +37,10 @@ trap cleanup EXIT cd "$WORK_DIR" || exit src_file=phoenix-omid-$VERSION-src.tar.gz -echo "Downloading Omid (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" -curl --fail -LOs "https://downloads.apache.org/phoenix/phoenix-omid-${VERSION}/${src_file}" -curl --fail -LOs "https://downloads.apache.org/phoenix/phoenix-omid-$VERSION/$src_file.asc" -curl --fail -LOs "https://downloads.apache.org/phoenix/phoenix-omid-$VERSION/$src_file.sha512" - +echo "Downloading Omid source (if this fails, try switching the BASE_DOWNLOAD_URL to the archive)" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/phoenix-omid-${VERSION}/${src_file}" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/phoenix-omid-$VERSION/$src_file.asc" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/phoenix-omid-$VERSION/$src_file.sha512" # It is probably redundant to check both the checksum and the signature but it's cheap and why not echo "Validating SHA512 Checksums" @@ -46,17 +50,18 @@ if ! (gpg --print-md SHA512 "$src_file" | diff - "$src_file.sha512"); then fi echo "Validating signatures" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/phoenix/KEYS) to GPG: https://www.apache.org/info/verification.html' if ! (gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then echo "ERROR: One of the signatures could not be verified" + echo "--> Make sure you have imported the KEYS file (${BASE_DOWNLOAD_URL}/KEYS) into GPG: https://www.apache.org/info/verification.html" + echo "--> e.g. \"curl ${BASE_DOWNLOAD_URL}/KEYS | gpg --import\"" exit 1 fi echo "Uploading everything to Nexus" EXIT_STATUS=0 -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file" 'https://repo.stackable.tech/repository/packages/omid/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.asc" 'https://repo.stackable.tech/repository/packages/omid/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.sha512" 'https://repo.stackable.tech/repository/packages/omid/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file" 'https://repo.stackable.tech/repository/packages/omid/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.asc" 'https://repo.stackable.tech/repository/packages/omid/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.sha512" 'https://repo.stackable.tech/repository/packages/omid/' || EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then echo "ERROR: Upload failed" diff --git a/opa/upload_new_opa_version.sh b/opa/upload_new_opa_version.sh index d26ead135..0f2b797ef 100755 --- a/opa/upload_new_opa_version.sh +++ b/opa/upload_new_opa_version.sh @@ -40,7 +40,7 @@ curl --fail -L -o "${tar_gz_file}" "${download_url}" echo "Uploading OPA source to Nexus" EXIT_STATUS=0 -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${tar_gz_file}" 'https://repo.stackable.tech/repository/packages/opa/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${tar_gz_file}" 'https://repo.stackable.tech/repository/packages/opa/' || EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then echo "ERROR: Upload failed" diff --git a/spark-k8s/upload_new_hbase-connector_version.sh b/spark-k8s/upload_new_hbase-connector_version.sh index 6cc74533a..46a19af66 100755 --- a/spark-k8s/upload_new_hbase-connector_version.sh +++ b/spark-k8s/upload_new_hbase-connector_version.sh @@ -40,7 +40,7 @@ curl --fail -L -o "${tar_gz_file}" "${download_url}" echo "Uploading hbase-connectors source to Nexus" EXIT_STATUS=0 -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${tar_gz_file}" 'https://repo.stackable.tech/repository/packages/hbase-connectors/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${tar_gz_file}" 'https://repo.stackable.tech/repository/packages/hbase-connectors/' || EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then echo "ERROR: Upload failed" diff --git a/spark-k8s/upload_new_spark_version.sh b/spark-k8s/upload_new_spark_version.sh index 3fe0994c3..4ed6906c7 100755 --- a/spark-k8s/upload_new_spark_version.sh +++ b/spark-k8s/upload_new_spark_version.sh @@ -5,6 +5,11 @@ set -euo pipefail VERSION=${1:?"Missing version number argument (arg 1)"} NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"} +# We prefer fast downloads... +BASE_DOWNLOAD_URL="https://dlcdn.apache.org/spark" +# However, if the version is not available, use the slow archive instead: +# BASE_DOWNLOAD_URL="https://archive.apache.org/dist/spark" + read -r -s -p "Nexus Password: " NEXUS_PASSWORD echo "" @@ -33,10 +38,10 @@ cd "$WORK_DIR" || exit src_file="spark-${VERSION}.tgz" -echo "Downloading Spark (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" -curl --fail -LOs "https://archive.apache.org/dist/spark/spark-${VERSION}/${src_file}" -curl --fail -LOs "https://archive.apache.org/dist/spark/spark-${VERSION}/${src_file}.asc" -curl --fail -LOs "https://archive.apache.org/dist/spark/spark-${VERSION}/${src_file}.sha512" +echo "Downloading Spark source (if this fails, try switching the BASE_DOWNLOAD_URL to the archive)" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/spark-${VERSION}/${src_file}" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/spark-${VERSION}/${src_file}.asc" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/spark-${VERSION}/${src_file}.sha512" # It is probably redundant to check both the checksum and the signature but it's cheap and why not echo "Validating SHA512 Checksum" @@ -46,18 +51,18 @@ if ! (sha512sum "${src_file}" | diff - "${src_file}.sha512"); then fi echo "Validating signature" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/spark/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://archive.apache.org/dist/spark/KEYS | gpg --import")' - if ! (gpg --verify "${src_file}.asc" "${src_file}" 2>/dev/null); then echo "ERROR: The signature could not be verified" + echo "--> Make sure you have imported the KEYS file (${BASE_DOWNLOAD_URL}/KEYS) into GPG: https://www.apache.org/info/verification.html" + echo "--> e.g. \"curl ${BASE_DOWNLOAD_URL}/KEYS | gpg --import\"" exit 1 fi echo "Uploading everything to Nexus" EXIT_STATUS=0 -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}" 'https://repo.stackable.tech/repository/packages/spark/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.asc" 'https://repo.stackable.tech/repository/packages/spark/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.sha512" 'https://repo.stackable.tech/repository/packages/spark/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}" 'https://repo.stackable.tech/repository/packages/spark/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.asc" 'https://repo.stackable.tech/repository/packages/spark/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.sha512" 'https://repo.stackable.tech/repository/packages/spark/' || EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then echo "ERROR: Upload failed" diff --git a/statsd_exporter/upload_new_statsd_exporter_version.sh b/statsd_exporter/upload_new_statsd_exporter_version.sh index 5a23f4899..0a37fb23c 100755 --- a/statsd_exporter/upload_new_statsd_exporter_version.sh +++ b/statsd_exporter/upload_new_statsd_exporter_version.sh @@ -32,10 +32,10 @@ trap cleanup EXIT cd "$WORK_DIR" || exit echo "Downloading STATSD EXPORTER source code" -curl --fail -LOs "https://github.com/prometheus/statsd_exporter/archive/refs/tags/v$VERSION.tar.gz" && mv "v$VERSION.tar.gz" "statsd_exporter-$VERSION.src.tar.gz" +curl --fail -LO --progress-bar "https://github.com/prometheus/statsd_exporter/archive/refs/tags/v$VERSION.tar.gz" && mv "v$VERSION.tar.gz" "statsd_exporter-$VERSION.src.tar.gz" echo "Uploading to Nexus" -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "statsd_exporter-$VERSION.src.tar.gz" 'https://repo.stackable.tech/repository/packages/statsd_exporter/' +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "statsd_exporter-$VERSION.src.tar.gz" 'https://repo.stackable.tech/repository/packages/statsd_exporter/' echo "Successfully uploaded new version of STATSD-EXPORTER source code ($VERSION) to Nexus" echo "https://repo.stackable.tech/service/rest/repository/browse/packages/statsd_exporter/" diff --git a/trino-cli/upload_new_trino_version.sh b/trino-cli/upload_new_trino_version.sh index 3ed018aa6..fea3cf748 100755 --- a/trino-cli/upload_new_trino_version.sh +++ b/trino-cli/upload_new_trino_version.sh @@ -34,9 +34,9 @@ cd "$WORK_DIR" || exit bin_file=trino-cli-${VERSION}-executable.jar echo "Downloading Trino (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" -curl --fail -LOs "https://repo1.maven.org/maven2/io/trino/trino-cli/${VERSION}/${bin_file}" -curl --fail -LOs "https://repo1.maven.org/maven2/io/trino/trino-cli/${VERSION}/${bin_file}.asc" -curl --fail -LOs "https://repo1.maven.org/maven2/io/trino/trino-cli/${VERSION}/${bin_file}.sha1" +curl --fail -LO --progress-bar "https://repo1.maven.org/maven2/io/trino/trino-cli/${VERSION}/${bin_file}" +curl --fail -LO --progress-bar "https://repo1.maven.org/maven2/io/trino/trino-cli/${VERSION}/${bin_file}.asc" +curl --fail -LO --progress-bar "https://repo1.maven.org/maven2/io/trino/trino-cli/${VERSION}/${bin_file}.sha1" # It is probably redundant to check both the checksum and the signature but it's cheap and why not echo "Validating SHA1 Checksum" @@ -89,9 +89,9 @@ fi echo "Uploading everything to Nexus" EXIT_STATUS=0 -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}" 'https://repo.stackable.tech/repository/packages/trino-cli/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.asc" 'https://repo.stackable.tech/repository/packages/trino-cli/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.sha1" 'https://repo.stackable.tech/repository/packages/trino-cli/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}" 'https://repo.stackable.tech/repository/packages/trino-cli/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.asc" 'https://repo.stackable.tech/repository/packages/trino-cli/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.sha1" 'https://repo.stackable.tech/repository/packages/trino-cli/' || EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then echo "ERROR: Upload failed" diff --git a/trino-storage-connector/upload_new_trino_storage_connector_version.sh b/trino-storage-connector/upload_new_trino_storage_connector_version.sh index d260e823d..737b063d5 100755 --- a/trino-storage-connector/upload_new_trino_storage_connector_version.sh +++ b/trino-storage-connector/upload_new_trino_storage_connector_version.sh @@ -43,8 +43,8 @@ sha256sum "${src_file}" | cut --delimiter=' ' --field=1 > "${src_file}.sha256" echo "Uploading everything to Nexus" EXIT_STATUS=0 -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}" 'https://repo.stackable.tech/repository/packages/trino-storage/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.sha256" 'https://repo.stackable.tech/repository/packages/trino-storage/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}" 'https://repo.stackable.tech/repository/packages/trino-storage/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.sha256" 'https://repo.stackable.tech/repository/packages/trino-storage/' || EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then echo "ERROR: Upload failed" diff --git a/trino/upload_new_trino_version.sh b/trino/upload_new_trino_version.sh index 4386e50b6..2dfda18ea 100755 --- a/trino/upload_new_trino_version.sh +++ b/trino/upload_new_trino_version.sh @@ -43,8 +43,8 @@ sha256sum "${src_file}" | cut --delimiter=' ' --field=1 > "${src_file}.sha256" echo "Uploading everything to Nexus" EXIT_STATUS=0 -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}" 'https://repo.stackable.tech/repository/packages/trino-server/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.sha256" 'https://repo.stackable.tech/repository/packages/trino-server/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}" 'https://repo.stackable.tech/repository/packages/trino-server/' || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.sha256" 'https://repo.stackable.tech/repository/packages/trino-server/' || EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then echo "ERROR: Upload failed" diff --git a/vector/upload_new_vector_version.sh b/vector/upload_new_vector_version.sh index 4ec13cf40..2d71eb08c 100755 --- a/vector/upload_new_vector_version.sh +++ b/vector/upload_new_vector_version.sh @@ -32,11 +32,6 @@ for arch in "${ARCHITECTURES[@]}"; do "https://yum.vector.dev/stable/vector-$major_version/$arch/$file" echo "Validating signature" - echo "--> NOTE: Make sure you have downloaded and added Datadog's \ -public key (https://keys.datadoghq.com/DATADOG_RPM_KEY_B01082D3.public) \ -to the RPM package database: -rpmkeys --import --dbpath $RPM_PACKAGE_DB_PATH DATADOG_APT_KEY_CURRENT.public" - EXIT_STATUS=0 # `rpmkeys --checksig` also succeeds if the digests of an unsigned # package are okay. Therefore, test explicitly if the output @@ -50,11 +45,15 @@ rpmkeys --import --dbpath $RPM_PACKAGE_DB_PATH DATADOG_APT_KEY_CURRENT.public" EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then echo "ERROR: The signature could not be verified." + echo "--> NOTE: Make sure you have downloaded and added Datadog's \ +public key (https://keys.datadoghq.com/DATADOG_RPM_KEY_B01082D3.public) \ +to the RPM package database: +rpmkeys --import --dbpath $RPM_PACKAGE_DB_PATH DATADOG_APT_KEY_CURRENT.public" exit 1 fi echo "Uploading $file to Nexus" - curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" \ + curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" \ --upload-file "$file" \ 'https://repo.stackable.tech/repository/packages/vector/' diff --git a/zookeeper/upload_new_zookeeper_version.sh b/zookeeper/upload_new_zookeeper_version.sh index efaa6ccac..d3b74f65a 100755 --- a/zookeeper/upload_new_zookeeper_version.sh +++ b/zookeeper/upload_new_zookeeper_version.sh @@ -5,6 +5,11 @@ set -euo pipefail VERSION=${1:?"Missing version number argument (arg 1)"} NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"} +# We prefer fast downloads... +BASE_DOWNLOAD_URL="https://dlcdn.apache.org/zookeeper" +# However, if the version is not available, use the slow archive instead: +# BASE_DOWNLOAD_URL="https://archive.apache.org/dist/zookeeper" + read -r -s -p "Nexus Password: " NEXUS_PASSWORD echo "" @@ -33,18 +38,16 @@ cd "$WORK_DIR" || exit bin_file=apache-zookeeper-$VERSION-bin.tar.gz src_file=apache-zookeeper-$VERSION.tar.gz -download_url=https://archive.apache.org/dist/zookeeper - -echo "Downloading ZooKeeper (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" -curl --fail -LOs "$download_url/zookeeper-$VERSION/$bin_file" -curl --fail -LOs "$download_url/zookeeper-$VERSION/$bin_file.asc" -curl --fail -LOs "$download_url/zookeeper-$VERSION/$bin_file.sha512" -echo "Downloading ZooKeeper sources (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" -curl --fail -LOs "$download_url/zookeeper-$VERSION/$src_file" -curl --fail -LOs "$download_url/zookeeper-$VERSION/$src_file.asc" -curl --fail -LOs "$download_url/zookeeper-$VERSION/$src_file.sha512" +echo "Downloading ZooKeeper binary (if this fails, try switching the BASE_DOWNLOAD_URL to the archive)" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/zookeeper-$VERSION/$bin_file" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/zookeeper-$VERSION/$bin_file.asc" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/zookeeper-$VERSION/$bin_file.sha512" +echo "Downloading ZooKeeper source (if this fails, try switching the BASE_DOWNLOAD_URL to the archive)" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/zookeeper-$VERSION/$src_file" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/zookeeper-$VERSION/$src_file.asc" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/zookeeper-$VERSION/$src_file.sha512" # It is probably redundant to check both the checksum and the signature but it's cheap and why not echo "Validating SHA512 Checksums for binary releases" @@ -59,30 +62,32 @@ if ! (sha512sum "$src_file" | diff -Z - "$src_file.sha512"); then fi echo "Validating signatures for binary releases" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/zookeeper/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://archive.apache.org/dist/zookeeper/KEYS | gpg --import")' - if ! (gpg --verify "$bin_file.asc" "$bin_file" 2> /dev/null); then echo "ERROR: One of the signatures could not be verified for a binary release" + echo "--> Make sure you have imported the KEYS file (${BASE_DOWNLOAD_URL}/KEYS) into GPG: https://www.apache.org/info/verification.html" + echo "--> e.g. \"curl ${BASE_DOWNLOAD_URL}/KEYS | gpg --import\"" exit 1 fi echo "Validating signatures for source releases" if ! (gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then - echo "ERROR: One of the signatures could not be verified for a source release" - exit 1 + echo "ERROR: One of the signatures could not be verified for a source release" + echo "--> Make sure you have imported the KEYS file (${BASE_DOWNLOAD_URL}/KEYS) into GPG: https://www.apache.org/info/verification.html" + echo "--> e.g. \"curl ${BASE_DOWNLOAD_URL}/KEYS | gpg --import\"" + exit 1 fi echo "Uploading everything to Nexus" EXIT_STATUS=0 repo_url=https://repo.stackable.tech/repository/packages/zookeeper/ -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file" "$repo_url" || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file.asc" "$repo_url" || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file.sha512" "$repo_url" || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file" "$repo_url" || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file.asc" "$repo_url" || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$bin_file.sha512" "$repo_url" || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file" "$repo_url" || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.asc" "$repo_url" || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.sha512" "$repo_url" || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file" "$repo_url" || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.asc" "$repo_url" || EXIT_STATUS=$? +curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$src_file.sha512" "$repo_url" || EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then echo "ERROR: Upload failed"