diff --git a/.env.sample b/.env.sample index 6997e46e6..2610c5e2a 100644 --- a/.env.sample +++ b/.env.sample @@ -1,6 +1,10 @@ export DOCSRS_PREFIX=ignored/cratesfyi-prefix export DOCSRS_DATABASE_URL=postgresql://cratesfyi:password@localhost:15432 export DOCSRS_LOG=docs_rs=debug,rustwide=info +# To build with a PR that hasn't landed in a rust dist toolchain yet, +# you can set this to the git sha of a try build: +# https://forge.rust-lang.org/infra/docs/rustc-ci.html#try-builds +export DOCSRS_TOOLCHAIN=nightly export AWS_ACCESS_KEY_ID=cratesfyi export AWS_SECRET_ACCESS_KEY=secret_key export S3_ENDPOINT=http://localhost:9000 diff --git a/Cargo.toml b/Cargo.toml index da7250cdd..dc8149a28 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,7 +52,7 @@ schemamama = "0.3" schemamama_postgres = "0.3" systemstat = "0.2.0" prometheus = { version = "0.13.0", default-features = false } -rustwide = "0.15.0" +rustwide = { version = "0.15.0", features = ["unstable-toolchain-ci"] } mime_guess = "2" zstd = "0.11.0" hostname = "0.3.1" diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index ff63baa58..bca21bcc6 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -18,6 +18,7 @@ use anyhow::{anyhow, bail, Error}; use docsrs_metadata::{Metadata, DEFAULT_TARGETS, HOST_TARGET}; use failure::Error as FailureError; use postgres::Client; +use regex::Regex; use rustwide::cmd::{Command, CommandError, SandboxBuilder, SandboxImage}; use rustwide::logging::{self, LogStorage}; use rustwide::toolchain::ToolchainError; @@ -73,7 +74,16 @@ impl RustwideBuilder { .purge_all_build_dirs() .map_err(FailureError::compat)?; - let toolchain = Toolchain::dist(&config.toolchain); + // If the toolchain is all hex, assume it references an artifact from + // CI, for instance an `@bors try` build. + let re = Regex::new(r"^[a-fA-F0-9]+$").unwrap(); + let toolchain = if re.is_match(&config.toolchain) { + debug!("using CI build {}", &config.toolchain); + Toolchain::ci(&config.toolchain, false) + } else { + debug!("using toolchain {}", &config.toolchain); + Toolchain::dist(&config.toolchain) + }; Ok(RustwideBuilder { workspace, @@ -108,6 +118,23 @@ impl RustwideBuilder { } pub fn update_toolchain(&mut self) -> Result { + // For CI builds, a lot of the normal update_toolchain things don't apply. + // CI builds are only for one platform (https://forge.rust-lang.org/infra/docs/rustc-ci.html#try-builds) + // so we only try installing for the current platform. If that's not a match, + // for instance if we're running on macOS or Windows, this will error. + // Also, detecting the rustc version relies on calling rustc through rustup with the + // +channel argument, but the +channel argument doesn't work for CI builds. So + // we fake the rustc version and install from scratch every time since we can't detect + // the already-installed rustc version. + if let Some(ci) = self.toolchain.as_ci() { + self.toolchain + .install(&self.workspace) + .map_err(FailureError::compat)?; + self.rustc_version = format!("rustc 1.9999.0-nightly ({} 2999-12-29)", ci.sha()); + self.add_essential_files()?; + return Ok(true); + } + // Ignore errors if detection fails. let old_version = self.detect_rustc_version().ok(); @@ -174,6 +201,8 @@ impl RustwideBuilder { Ok(has_changed) } + /// Return a string containing the output of `rustc --version`. Only valid + /// for dist toolchains. Will error if run with a CI toolchain. fn detect_rustc_version(&self) -> Result { info!("detecting rustc's version..."); let res = Command::new(&self.workspace, self.toolchain.rustc()) @@ -190,7 +219,6 @@ impl RustwideBuilder { } pub fn add_essential_files(&mut self) -> Result<()> { - self.rustc_version = self.detect_rustc_version()?; let rustc_version = parse_rustc_version(&self.rustc_version)?; info!("building a dummy crate to get essential files"); @@ -883,16 +911,6 @@ mod tests { .map(|v| v.as_str().unwrap().to_owned()) .collect(); targets.sort(); - assert_eq!( - targets, - vec![ - "i686-pc-windows-msvc", - "i686-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-msvc", - "x86_64-unknown-linux-gnu", - ] - ); let web = env.frontend(); @@ -919,26 +937,51 @@ mod tests { web, )?; - // other targets too - for target in DEFAULT_TARGETS { - let target_docs_present = storage.exists_in_archive( - &doc_archive, - &format!("{}/{}/index.html", target, crate_path), - )?; + assert!(!storage.exists_in_archive( + &doc_archive, + &format!("{}/{}/index.html", default_target, crate_path), + )?); + + let default_target_url = format!( + "/{}/{}/{}/{}/index.html", + crate_, version, default_target, crate_path + ); + assert_redirect( + &default_target_url, + &format!("/{}/{}/{}/index.html", crate_, version, crate_path), + web, + )?; - let target_url = format!( - "/{}/{}/{}/{}/index.html", - crate_, version, target, crate_path + // Non-dist toolchains only have a single target, and of course + // if include_default_targets is false we won't have this full list + // of targets. + if builder.toolchain.as_dist().is_some() && env.config().include_default_targets { + assert_eq!( + targets, + vec![ + "i686-pc-windows-msvc", + "i686-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-unknown-linux-gnu", + ] ); - if target == &default_target { - assert!(!target_docs_present); - assert_redirect( - &target_url, - &format!("/{}/{}/{}/index.html", crate_, version, crate_path), - web, + // other targets too + for target in DEFAULT_TARGETS { + if target == &default_target { + continue; + } + let target_docs_present = storage.exists_in_archive( + &doc_archive, + &format!("{}/{}/index.html", target, crate_path), )?; - } else { + + let target_url = format!( + "/{}/{}/{}/{}/index.html", + crate_, version, target, crate_path + ); + assert!(target_docs_present); assert_success(&target_url, web)?; } @@ -1033,6 +1076,9 @@ mod tests { let crate_ = "windows-win"; let version = "2.4.1"; let mut builder = RustwideBuilder::init(env).unwrap(); + if builder.toolchain.as_ci().is_some() { + return Ok(()); + } assert!(builder.build_package(crate_, version, PackageKind::CratesIo)?); let storage = env.storage(); diff --git a/src/test/mod.rs b/src/test/mod.rs index af15c1a58..69e4a4667 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -279,6 +279,8 @@ impl TestEnvironment { // are actually different. config.cache_control_stale_while_revalidate = Some(86400); + config.include_default_targets = true; + config } diff --git a/src/utils/rustc_version.rs b/src/utils/rustc_version.rs index 4f128ae13..4f62ea7fc 100644 --- a/src/utils/rustc_version.rs +++ b/src/utils/rustc_version.rs @@ -9,7 +9,7 @@ pub fn parse_rustc_version>(version: S) -> Result { let version_regex = Regex::new(r" ([\w.-]+) \((\w+) (\d+)-(\d+)-(\d+)\)")?; let captures = version_regex .captures(version.as_ref()) - .with_context(|| anyhow!("Failed to parse rustc version"))?; + .with_context(|| anyhow!("Failed to parse rustc version '{}'", version.as_ref()))?; Ok(format!( "{}{}{}-{}-{}",