From c482ffd1a3300989cca6b741e32b75f7c3059104 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 14 Dec 2020 14:52:25 +0100 Subject: [PATCH 1/4] bump rust-installer submodule --- src/tools/rust-installer | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/rust-installer b/src/tools/rust-installer index d66f476b4d5e7..5254dbfd25d52 160000 --- a/src/tools/rust-installer +++ b/src/tools/rust-installer @@ -1 +1 @@ -Subproject commit d66f476b4d5e7fdf1ec215c9ac16c923dc292324 +Subproject commit 5254dbfd25d5284728ab624dca1969d61427a0db From 0a2034dca46f72d24d94c04b47c48d30149693bd Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 14 Dec 2020 15:53:07 +0100 Subject: [PATCH 2/4] bootstrap: add the dist.compression-formats option The option allows to add or remove compression formats used while producing dist tarballs. --- config.toml.example | 4 ++++ src/bootstrap/config.rs | 3 +++ src/bootstrap/configure.py | 4 ++++ src/bootstrap/tarball.rs | 16 +++++++++++++++- 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/config.toml.example b/config.toml.example index b1fb8904ca988..9e08ce9b27e0c 100644 --- a/config.toml.example +++ b/config.toml.example @@ -669,3 +669,7 @@ changelog-seen = 2 # Whether to allow failures when building tools #missing-tools = false + +# List of compression formats to use when generating dist tarballs. The list of +# formats is provided to rust-installer, which must support all of them. +#compression-formats = ["gz", "xz"] diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index def8f21543626..8a3b936d80d5c 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -148,6 +148,7 @@ pub struct Config { pub dist_sign_folder: Option, pub dist_upload_addr: Option, pub dist_gpg_password_file: Option, + pub dist_compression_formats: Option>, // libstd features pub backtrace: bool, // support for RUST_BACKTRACE @@ -438,6 +439,7 @@ struct Dist { upload_addr: Option, src_tarball: Option, missing_tools: Option, + compression_formats: Option>, } #[derive(Deserialize)] @@ -936,6 +938,7 @@ impl Config { config.dist_sign_folder = t.sign_folder.map(PathBuf::from); config.dist_gpg_password_file = t.gpg_password_file.map(PathBuf::from); config.dist_upload_addr = t.upload_addr; + config.dist_compression_formats = t.compression_formats; set(&mut config.rust_dist_src, t.src_tarball); set(&mut config.missing_tools, t.missing_tools); } diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index 42f00ce962174..2cabaee68ea67 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -147,6 +147,8 @@ def v(*args): "experimental LLVM targets to build") v("release-channel", "rust.channel", "the name of the release channel to build") v("release-description", "rust.description", "optional descriptive string for version output") +v("dist-compression-formats", None, + "comma-separated list of compression formats to use") # Used on systems where "cc" is unavailable v("default-linker", "rust.default-linker", "the default linker") @@ -349,6 +351,8 @@ def set(key, value): elif option.name == 'option-checking': # this was handled above pass + elif option.name == 'dist-compression-formats': + set('dist.compression-formats', value.split(',')) else: raise RuntimeError("unhandled option {}".format(option.name)) diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index 5d73a655427b2..a918677f2e507 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -288,11 +288,25 @@ impl<'a> Tarball<'a> { build_cli(&self, &mut cmd); cmd.arg("--work-dir").arg(&self.temp_dir); + if let Some(formats) = &self.builder.config.dist_compression_formats { + assert!(!formats.is_empty(), "dist.compression-formats can't be empty"); + cmd.arg("--compression-formats").arg(formats.join(",")); + } self.builder.run(&mut cmd); if self.delete_temp_dir { t!(std::fs::remove_dir_all(&self.temp_dir)); } - crate::dist::distdir(self.builder).join(format!("{}.tar.gz", package_name)) + // Use either the first compression format defined, or "gz" as the default. + let ext = self + .builder + .config + .dist_compression_formats + .as_ref() + .and_then(|formats| formats.get(0)) + .map(|s| s.as_str()) + .unwrap_or("gz"); + + crate::dist::distdir(self.builder).join(format!("{}.tar.{}", package_name, ext)) } } From d0d0ee0b5d1fc459ee8d697dedf37d642430c6f5 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 28 Dec 2020 13:00:50 +0100 Subject: [PATCH 3/4] ci: stop producing gzip-compressed dist tarballs --- src/ci/run.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ci/run.sh b/src/ci/run.sh index 181a7fcb73266..1958b6ee41d7f 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -53,6 +53,11 @@ RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-locked-deps" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-cargo-native-static" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-units-std=1" +# Only produce xz tarballs on CI. gz tarballs will be generated by the release +# process by recompressing the existing xz ones. This decreases the storage +# space required for CI artifacts. +RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --dist-compression-formats=xz" + if [ "$DIST_SRC" = "" ]; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-dist-src" fi From 5526d902508178293b98b4d8b7b39802107dc395 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 30 Dec 2020 16:05:57 +0100 Subject: [PATCH 4/4] bootstrap: extract from any compression algorithm during distcheck --- src/bootstrap/test.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index b99692e8ba5b8..1b9523426e735 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -1968,7 +1968,7 @@ impl Step for Distcheck { builder.ensure(dist::Src); let mut cmd = Command::new("tar"); - cmd.arg("-xzf") + cmd.arg("-xf") .arg(builder.ensure(dist::PlainSourceTarball)) .arg("--strip-components=1") .current_dir(&dir); @@ -1992,10 +1992,7 @@ impl Step for Distcheck { t!(fs::create_dir_all(&dir)); let mut cmd = Command::new("tar"); - cmd.arg("-xzf") - .arg(builder.ensure(dist::Src)) - .arg("--strip-components=1") - .current_dir(&dir); + cmd.arg("-xf").arg(builder.ensure(dist::Src)).arg("--strip-components=1").current_dir(&dir); builder.run(&mut cmd); let toml = dir.join("rust-src/lib/rustlib/src/rust/library/std/Cargo.toml");