rustwide_builder: allow using a try build

.env.sample

@@ -1,6 +1,10 @@
 export DOCSRS_PREFIX=ignored/cratesfyi-prefix
 export DOCSRS_DATABASE_URL=postgresql://cratesfyi:password@localhost:15432
 export DOCSRS_LOG=docs_rs=debug,rustwide=info
+# To build with a PR that hasn't landed in a rust dist toolchain yet,
+# you can set this to the git sha of a try build:
+# https://forge.rust-lang.org/infra/docs/rustc-ci.html#try-builds
+export DOCSRS_TOOLCHAIN=nightly
 export AWS_ACCESS_KEY_ID=cratesfyi
 export AWS_SECRET_ACCESS_KEY=secret_key
 export S3_ENDPOINT=http://localhost:9000

Cargo.toml

@@ -52,7 +52,7 @@ schemamama = "0.3"
 schemamama_postgres = "0.3"
 systemstat = "0.2.0"
 prometheus = { version = "0.13.0", default-features = false }
-rustwide = "0.15.0"
+rustwide = { version = "0.15.0", features = ["unstable-toolchain-ci"] }
 mime_guess = "2"
 zstd = "0.11.0"
 hostname = "0.3.1"

src/docbuilder/rustwide_builder.rs

@@ -18,6 +18,7 @@ use anyhow::{anyhow, bail, Error};
 use docsrs_metadata::{Metadata, DEFAULT_TARGETS, HOST_TARGET};
 use failure::Error as FailureError;
 use postgres::Client;
+use regex::Regex;
 use rustwide::cmd::{Command, CommandError, SandboxBuilder, SandboxImage};
 use rustwide::logging::{self, LogStorage};
 use rustwide::toolchain::ToolchainError;
@@ -73,7 +74,16 @@ impl RustwideBuilder {
             .purge_all_build_dirs()
             .map_err(FailureError::compat)?;
 
-        let toolchain = Toolchain::dist(&config.toolchain);
+        // If the toolchain is all hex, assume it references an artifact from
+        // CI, for instance an `@bors try` build.
+        let re = Regex::new(r"^[a-fA-F0-9]+$").unwrap();
+        let toolchain = if re.is_match(&config.toolchain) {
+            debug!("using CI build {}", &config.toolchain);
+            Toolchain::ci(&config.toolchain, false)
+        } else {
+            debug!("using toolchain {}", &config.toolchain);
+            Toolchain::dist(&config.toolchain)
+        };
 
         Ok(RustwideBuilder {
             workspace,
@@ -108,6 +118,23 @@ impl RustwideBuilder {
     }
 
     pub fn update_toolchain(&mut self) -> Result<bool> {
+        // For CI builds, a lot of the normal update_toolchain things don't apply.
+        // CI builds are only for one platform (https://forge.rust-lang.org/infra/docs/rustc-ci.html#try-builds)
+        // so we only try installing for the current platform. If that's not a match,
+        // for instance if we're running on macOS or Windows, this will error.
+        // Also, detecting the rustc version relies on calling rustc through rustup with the
+        // +channel argument, but the +channel argument doesn't work for CI builds. So
+        // we fake the rustc version and install from scratch every time since we can't detect
+        // the already-installed rustc version.
+        if let Some(ci) = self.toolchain.as_ci() {
+            self.toolchain
+                .install(&self.workspace)
+                .map_err(FailureError::compat)?;
+            self.rustc_version = format!("rustc 1.9999.0-nightly ({} 2999-12-29)", ci.sha());
+            self.add_essential_files()?;
+            return Ok(true);
+        }
+
         // Ignore errors if detection fails.
         let old_version = self.detect_rustc_version().ok();
 
@@ -174,6 +201,8 @@ impl RustwideBuilder {
         Ok(has_changed)
     }
 
+    /// Return a string containing the output of `rustc --version`. Only valid
+    /// for dist toolchains. Will error if run with a CI toolchain.
     fn detect_rustc_version(&self) -> Result<String> {
         info!("detecting rustc's version...");
         let res = Command::new(&self.workspace, self.toolchain.rustc())
@@ -190,7 +219,6 @@ impl RustwideBuilder {
     }
 
     pub fn add_essential_files(&mut self) -> Result<()> {
-        self.rustc_version = self.detect_rustc_version()?;
         let rustc_version = parse_rustc_version(&self.rustc_version)?;
 
         info!("building a dummy crate to get essential files");
@@ -883,16 +911,6 @@ mod tests {
                 .map(|v| v.as_str().unwrap().to_owned())
                 .collect();
             targets.sort();
-            assert_eq!(
-                targets,
-                vec![
-                    "i686-pc-windows-msvc",
-                    "i686-unknown-linux-gnu",
-                    "x86_64-apple-darwin",
-                    "x86_64-pc-windows-msvc",
-                    "x86_64-unknown-linux-gnu",
-                ]
-            );
 
             let web = env.frontend();
 
@@ -919,26 +937,51 @@ mod tests {
                 web,
             )?;
 
-            // other targets too
-            for target in DEFAULT_TARGETS {
-                let target_docs_present = storage.exists_in_archive(
-                    &doc_archive,
-                    &format!("{}/{}/index.html", target, crate_path),
-                )?;
+            assert!(!storage.exists_in_archive(
+                &doc_archive,
+                &format!("{}/{}/index.html", default_target, crate_path),
+            )?);
+
+            let default_target_url = format!(
+                "/{}/{}/{}/{}/index.html",
+                crate_, version, default_target, crate_path
+            );
+            assert_redirect(
+                &default_target_url,
+                &format!("/{}/{}/{}/index.html", crate_, version, crate_path),
+                web,
+            )?;
 
-                let target_url = format!(
-                    "/{}/{}/{}/{}/index.html",
-                    crate_, version, target, crate_path
+            // Non-dist toolchains only have a single target, and of course
+            // if include_default_targets is false we won't have this full list
+            // of targets.
+            if builder.toolchain.as_dist().is_some() && env.config().include_default_targets {
+                assert_eq!(
+                    targets,
+                    vec![
+                        "i686-pc-windows-msvc",
+                        "i686-unknown-linux-gnu",
+                        "x86_64-apple-darwin",
+                        "x86_64-pc-windows-msvc",
+                        "x86_64-unknown-linux-gnu",
+                    ]
                 );
 
-                if target == &default_target {
-                    assert!(!target_docs_present);
-                    assert_redirect(
-                        &target_url,
-                        &format!("/{}/{}/{}/index.html", crate_, version, crate_path),
-                        web,
+                // other targets too
+                for target in DEFAULT_TARGETS {
+                    if target == &default_target {
+                        continue;
+                    }
+                    let target_docs_present = storage.exists_in_archive(
+                        &doc_archive,
+                        &format!("{}/{}/index.html", target, crate_path),
                     )?;
-                } else {
+
+                    let target_url = format!(
+                        "/{}/{}/{}/{}/index.html",
+                        crate_, version, target, crate_path
+                    );
+
                     assert!(target_docs_present);
                     assert_success(&target_url, web)?;
                 }
@@ -1033,6 +1076,9 @@ mod tests {
             let crate_ = "windows-win";
             let version = "2.4.1";
             let mut builder = RustwideBuilder::init(env).unwrap();
+            if builder.toolchain.as_ci().is_some() {
+                return Ok(());
+            }
             assert!(builder.build_package(crate_, version, PackageKind::CratesIo)?);
 
             let storage = env.storage();

src/test/mod.rs

@@ -279,6 +279,8 @@ impl TestEnvironment {
         // are actually different.
         config.cache_control_stale_while_revalidate = Some(86400);
 
+        config.include_default_targets = true;
+
         config
     }
 

src/utils/rustc_version.rs

@@ -9,7 +9,7 @@ pub fn parse_rustc_version<S: AsRef<str>>(version: S) -> Result<String> {
     let version_regex = Regex::new(r" ([\w.-]+) \((\w+) (\d+)-(\d+)-(\d+)\)")?;
     let captures = version_regex
         .captures(version.as_ref())
-        .with_context(|| anyhow!("Failed to parse rustc version"))?;
+        .with_context(|| anyhow!("Failed to parse rustc version '{}'", version.as_ref()))?;
 
     Ok(format!(
         "{}{}{}-{}-{}",