Auto merge of #10129 - jyn514:fetch-build-std, r=weihanglo

Add support for `-Zbuild-std` to `cargo fetch`

This allows downloading the dependencies for libstd in advance, which
can be useful in e.g. sandboxed build environments.

Fixes rust-lang/wg-cargo-std-aware#22.

r? `@ehuss`

Author: bors

src/cargo/core/compiler/build_config.rs

@@ -84,6 +84,11 @@ impl BuildConfig {
             anyhow::bail!("jobs may not be 0");
         }
 
+        if config.cli_unstable().build_std.is_some() && requested_kinds[0].is_host() {
+            // TODO: This should eventually be fixed.
+            anyhow::bail!("-Zbuild-std requires --target");
+        }
+
         Ok(BuildConfig {
             requested_kinds,
             jobs,

src/cargo/core/compiler/standard_lib.rs

@@ -9,10 +9,13 @@ use crate::core::resolver::HasDevUnits;
 use crate::core::{Dependency, PackageId, PackageSet, Resolve, SourceId, Workspace};
 use crate::ops::{self, Packages};
 use crate::util::errors::CargoResult;
+use crate::Config;
 use std::collections::{HashMap, HashSet};
 use std::env;
 use std::path::PathBuf;
 
+use super::BuildConfig;
+
 /// Parse the `-Zbuild-std` flag.
 pub fn parse_unstable_flag(value: Option<&str>) -> Vec<String> {
     // This is a temporary hack until there is a more principled way to
@@ -31,13 +34,45 @@ pub fn parse_unstable_flag(value: Option<&str>) -> Vec<String> {
     crates.into_iter().map(|s| s.to_string()).collect()
 }
 
+pub(crate) fn std_crates(config: &Config, units: Option<&[Unit]>) -> Option<Vec<String>> {
+    let crates = config.cli_unstable().build_std.as_ref()?.clone();
+
+    // Only build libtest if it looks like it is needed.
+    let mut crates = crates.clone();
+    // If we know what units we're building, we can filter for libtest depending on the jobs.
+    if let Some(units) = units {
+        if units
+            .iter()
+            .any(|unit| unit.mode.is_rustc_test() && unit.target.harness())
+        {
+            // Only build libtest when libstd is built (libtest depends on libstd)
+            if crates.iter().any(|c| c == "std") && !crates.iter().any(|c| c == "test") {
+                crates.push("test".to_string());
+            }
+        }
+    } else {
+        // We don't know what jobs are going to be run, so download libtest just in case.
+        if !crates.iter().any(|c| c == "test") {
+            crates.push("test".to_string())
+        }
+    }
+
+    Some(crates)
+}
+
 /// Resolve the standard library dependencies.
 pub fn resolve_std<'cfg>(
     ws: &Workspace<'cfg>,
     target_data: &RustcTargetData<'cfg>,
-    requested_targets: &[CompileKind],
+    build_config: &BuildConfig,
     crates: &[String],
 ) -> CargoResult<(PackageSet<'cfg>, Resolve, ResolvedFeatures)> {
+    if build_config.build_plan {
+        ws.config()
+            .shell()
+            .warn("-Zbuild-std does not currently fully support --build-plan")?;
+    }
+
     let src_path = detect_sysroot_src_path(target_data)?;
     let to_patch = [
         "rustc-std-workspace-core",
@@ -114,7 +149,7 @@ pub fn resolve_std<'cfg>(
     let resolve = ops::resolve_ws_with_opts(
         &std_ws,
         target_data,
-        requested_targets,
+        &build_config.requested_kinds,
         &cli_features,
         &specs,
         HasDevUnits::No,

src/cargo/ops/cargo_compile.rs

@@ -401,19 +401,8 @@ pub fn create_bcx<'a, 'cfg>(
     } = resolve;
 
     let std_resolve_features = if let Some(crates) = &config.cli_unstable().build_std {
-        if build_config.build_plan {
-            config
-                .shell()
-                .warn("-Zbuild-std does not currently fully support --build-plan")?;
-        }
-        if build_config.requested_kinds[0].is_host() {
-            // TODO: This should eventually be fixed. Unfortunately it is not
-            // easy to get the host triple in BuildConfig. Consider changing
-            // requested_target to an enum, or some other approach.
-            anyhow::bail!("-Zbuild-std requires --target");
-        }
         let (std_package_set, std_resolve, std_features) =
-            standard_lib::resolve_std(ws, &target_data, &build_config.requested_kinds, crates)?;
+            standard_lib::resolve_std(ws, &target_data, &build_config, crates)?;
         pkg_set.add_set(std_package_set);
         Some((std_resolve, std_features))
     } else {
@@ -554,19 +543,7 @@ pub fn create_bcx<'a, 'cfg>(
         None => Vec::new(),
     };
 
-    let std_roots = if let Some(crates) = &config.cli_unstable().build_std {
-        // Only build libtest if it looks like it is needed.
-        let mut crates = crates.clone();
-        if !crates.iter().any(|c| c == "test")
-            && units
-                .iter()
-                .any(|unit| unit.mode.is_rustc_test() && unit.target.harness())
-        {
-            // Only build libtest when libstd is built (libtest depends on libstd)
-            if crates.iter().any(|c| c == "std") {
-                crates.push("test".to_string());
-            }
-        }
+    let std_roots = if let Some(crates) = standard_lib::std_crates(config, Some(&units)) {
         let (std_resolve, std_features) = std_resolve_features.as_ref().unwrap();
         standard_lib::generate_std_roots(
             &crates,

src/cargo/ops/cargo_fetch.rs

@@ -1,3 +1,4 @@
+use crate::core::compiler::standard_lib;
 use crate::core::compiler::{BuildConfig, CompileMode, RustcTargetData};
 use crate::core::{PackageSet, Resolve, Workspace};
 use crate::ops;
@@ -17,7 +18,7 @@ pub fn fetch<'a>(
     options: &FetchOptions<'a>,
 ) -> CargoResult<(Resolve, PackageSet<'a>)> {
     ws.emit_warnings()?;
-    let (packages, resolve) = ops::resolve_ws(ws)?;
+    let (mut packages, resolve) = ops::resolve_ws(ws)?;
 
     let jobs = Some(1);
     let keep_going = false;
@@ -64,6 +65,14 @@ pub fn fetch<'a>(
             .map(|(id, _deps)| id);
         deps_to_fetch.extend(deps);
     }
+
+    // If -Zbuild-std was passed, download dependencies for the standard library.
+    // We don't know ahead of time what jobs we'll be running, so tell `std_crates` that.
+    if let Some(crates) = standard_lib::std_crates(config, None) {
+        let (std_package_set, _, _) = standard_lib::resolve_std(ws, &data, &build_config, &crates)?;
+        packages.add_set(std_package_set);
+    }
+
     packages.get_many(to_download)?;
 
     Ok((resolve, packages))

tests/testsuite/standard_lib.rs

@@ -690,3 +690,22 @@ fn proc_macro_only() {
         .with_stderr_contains("[FINISHED] [..]")
         .run();
 }
+
+#[cargo_test]
+fn fetch() {
+    let setup = match setup() {
+        Some(s) => s,
+        None => return,
+    };
+    let p = project().file("src/main.rs", "fn main() {}").build();
+    p.cargo("fetch")
+        .build_std(&setup)
+        .target_host()
+        .with_stderr_contains("[DOWNLOADED] [..]")
+        .run();
+    p.cargo("build")
+        .build_std(&setup)
+        .target_host()
+        .with_stderr_does_not_contain("[DOWNLOADED] [..]")
+        .run();
+}