Stop recompressing to-be-pruned artifacts

docker-compose.yml

@@ -3,7 +3,7 @@ version: '3'
 
 services:
   minio:
-    image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
+    image: quay.io/minio/minio:RELEASE.2023-04-13T03-08-07Z
     command: minio server /data
     ports:
       - 9000:9000

local/Dockerfile

@@ -5,7 +5,7 @@
 # regularly takes 2 minutes to download 20MB of binary). The only other way
 # they distribute the CLI is from Docker, so we load their image as a stage and
 # then copy the binary from it later in the build.
-FROM quay.io/minio/mc:RELEASE.2022-10-22T03-39-29Z AS mc
+FROM quay.io/minio/mc:RELEASE.2023-04-12T02-21-51Z AS mc
 
 FROM ubuntu:22.04
 

prod/Dockerfile

@@ -56,8 +56,8 @@ RUN curl https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustu
     /root/.cargo/bin/rustup toolchain remove stable
 ENV PATH=/root/.cargo/bin:$PATH
 
-# Roughly 2x number of CPU cores we have in production.
-RUN aws configure set default.s3.max_concurrent_requests 150
+RUN aws configure set default.s3.max_concurrent_requests 500
+RUN aws configure set default.s3.max_queue_size 10000
 
 COPY --from=build /tmp/source/target/release/promote-release /usr/local/bin/
 COPY prod/load-gpg-keys.sh /usr/local/bin/load-gpg-keys

src/build_manifest.rs

@@ -37,6 +37,11 @@ impl<'a> BuildManifest<'a> {
         })
     }
 
+    pub(crate) fn clear_checksum_cache(&self) -> Result<(), Error> {
+        std::fs::remove_file(&self.checksum_cache_path)?;
+        Ok(())
+    }
+
     pub(crate) fn run(&self, upload_base: &str, dest: &Path) -> Result<Execution, Error> {
         let config = &self.builder.config;
 
@@ -102,33 +107,19 @@ impl<'a> BuildManifest<'a> {
 }
 
 pub(crate) struct Execution {
-    pub(crate) shipped_files: Option<HashSet<PathBuf>>,
+    pub(crate) shipped_files: HashSet<PathBuf>,
     pub(crate) checksum_cache: HashMap<PathBuf, String>,
 }
 
 impl Execution {
     fn new(shipped_files_path: &Path, checksum_cache_path: &Path) -> Result<Self, Error> {
-        // Once https://github.com/rust-lang/rust/pull/78196 reaches stable we can assume the
-        // "shipped files" file is always generated, and we can remove the Option<_>.
-        let shipped_files = if shipped_files_path.is_file() {
-            Some(
-                std::fs::read_to_string(shipped_files_path)?
-                    .lines()
-                    .filter(|line| !line.trim().is_empty())
-                    .map(PathBuf::from)
-                    .collect(),
-            )
-        } else {
-            None
-        };
+        let shipped_files = std::fs::read_to_string(shipped_files_path)?
+            .lines()
+            .filter(|line| !line.trim().is_empty())
+            .map(PathBuf::from)
+            .collect();
 
-        // Once https://github.com/rust-lang/rust/pull/78409 reaches stable we can assume the
-        // checksum cache will always be generated, and we can remove the if branch.
-        let checksum_cache = if checksum_cache_path.is_file() {
-            serde_json::from_slice(&std::fs::read(checksum_cache_path)?)?
-        } else {
-            HashMap::new()
-        };
+        let checksum_cache = serde_json::from_slice(&std::fs::read(checksum_cache_path)?)?;
 
         Ok(Execution {
             shipped_files,

src/main.rs

@@ -25,6 +25,7 @@ use chrono::Utc;
 use curl::easy::Easy;
 use fs2::FileExt;
 use github::{CreateTag, Github};
+use rayon::prelude::*;
 
 use crate::config::{Channel, Config};
 
@@ -172,17 +173,54 @@ impl Context {
 
         self.assert_all_components_present()?;
 
+        // Quickly produce gzip compressed artifacts that are needed for successful manifest
+        // building.
+        //
+        // Nightly (1.71+) supports this upstream without the extra recompression, see
+        // https://github.com/rust-lang/rust/pull/110436.
+        if self.config.channel != Channel::Nightly {
+            let version = match self.config.channel {
+                Channel::Stable => self.current_version.as_deref().unwrap(),
+                Channel::Beta => "beta",
+                Channel::Nightly => "nightly",
+            };
+            let recompress = [
+                self.dl_dir()
+                    .join(format!("rust-{}-x86_64-unknown-linux-gnu.tar.xz", version)),
+                self.dl_dir()
+                    .join(format!("cargo-{}-x86_64-unknown-linux-gnu.tar.xz", version)),
+            ];
+            recompress.par_iter().try_for_each(|tarball| {
+                recompress::recompress_file(tarball, false, flate2::Compression::fast(), false)
+            })?;
+        }
+
         // Ok we've now determined that a release needs to be done.
 
         let mut signer = Signer::new(&self.config)?;
 
         let build_manifest = BuildManifest::new(self)?;
         let smoke_test = SmokeTester::new(&[self.smoke_manifest_dir(), self.dl_dir()])?;
 
-        // First of all, the real manifests are generated, pointing to the public download
-        // endpoint. This will also collect the list of files shipped in the release (used
-        // later to prune the files we're not shipping) and a cache of all the checksums
-        // generated by build-manifest.
+        // This step is just a discovery of unused files so we can prune them prior to
+        // recompression...
+        let execution = build_manifest.run(
+            &format!("{}/{}", self.config.upload_addr, self.config.upload_dir),
+            &self.real_manifest_dir(),
+        )?;
+
+        // Removes files that we are not shipping from the files we're about to upload.
+        self.prune_unused_files(&execution.shipped_files)?;
+
+        // Generate recompressed artifacts from the input set. This invalidates signatures etc
+        // produced in the earlier step so we'll need to re-run the manifest building.
+        self.recompress(&self.dl_dir())?;
+
+        // Since we recompressed, need to clear out the checksum cache.
+        build_manifest.clear_checksum_cache()?;
+
+        // Now generate the real manifests, pointing to the public download endpoint. This will
+        // also generate a cache of all the checksums generated by build-manifest.
         let execution = build_manifest.run(
             &format!("{}/{}", self.config.upload_addr, self.config.upload_dir),
             &self.real_manifest_dir(),
@@ -195,11 +233,6 @@ impl Context {
             &self.smoke_manifest_dir(),
         )?;
 
-        // Removes files that we are not shipping from the files we're about to upload.
-        if let Some(shipped_files) = &execution.shipped_files {
-            self.prune_unused_files(shipped_files)?;
-        }
-
         // Sign both the downloaded artifacts and all the generated manifests. The signatures
         // of the downloaded files and the real manifests are permanent, while the signatures
         // for the smoke test manifests will be discarded later.
@@ -252,13 +285,13 @@ impl Context {
         for e in self.dl_dir().read_dir()? {
             let e = e?;
             let filename = e.file_name().into_string().unwrap();
-            if !filename.starts_with("rustc-") || !filename.ends_with(".tar.gz") {
+            if !filename.starts_with("rustc-") || !filename.ends_with(".tar.xz") {
                 continue;
             }
             println!("looking inside {} for a version", filename);
 
             let file = File::open(e.path())?;
-            let reader = flate2::read::GzDecoder::new(file);
+            let reader = xz2::read::XzDecoder::new(file);
             let mut archive = tar::Archive::new(reader);
 
             let mut version_file = None;
@@ -368,31 +401,18 @@ impl Context {
         // 2. We're making a stable release. The stable release is first signed
         //    with the dev key and then it's signed with the prod key later. We
         //    want the prod key to overwrite the dev key signatures.
-        //
-        // Also, collect paths that need to be recompressed
-        let mut to_recompress = Vec::new();
         for file in dl.read_dir()? {
             let file = file?;
             let path = file.path();
             match path.extension().and_then(|s| s.to_str()) {
-                // Store off the input files for potential recompression.
-                Some("xz") => {
-                    to_recompress.push(path.to_path_buf());
-                }
                 // Delete signature/hash files...
                 Some("asc") | Some("sha256") => {
                     fs::remove_file(&path)?;
                 }
-                Some("gz") if self.config.recompress_gz => {
-                    fs::remove_file(&path)?;
-                }
                 _ => {}
             }
         }
 
-        // Generate recompressed artifacts from the input set.
-        self.recompress(to_recompress)?;
-
         Ok(())
     }