Build-std: Add builtin dependencies

crates/cargo-util-schemas/src/core/package_id_spec.rs

@@ -130,14 +130,20 @@ impl PackageIdSpec {
                     // Leave `sparse` as part of URL, see `SourceId::new`
                     // url = strip_url_protocol(&url);
                 }
-                "path" => {
+                kind_str @ ("path" | "builtin") => {
                     if url.query().is_some() {
                         return Err(ErrorKind::UnexpectedQueryString(url).into());
                     }
                     if scheme != "file" {
                         return Err(ErrorKind::UnsupportedPathPlusScheme(scheme.into()).into());
                     }
-                    kind = Some(SourceKind::Path);
+                    //TODO: This affects Cargo's json output, and needs consideration for what we
+                    //want for different commands.
+                    kind = if kind_str == "path" {
+                        Some(SourceKind::Path)
+                    } else {
+                        Some(SourceKind::Builtin)
+                    };
                     url = strip_url_protocol(&url);
                 }
                 kind => return Err(ErrorKind::UnsupportedProtocol(kind.into()).into()),

crates/cargo-util-schemas/src/core/source_kind.rs

@@ -15,6 +15,8 @@ pub enum SourceKind {
     LocalRegistry,
     /// A directory-based registry.
     Directory,
+    /// Package sources distributed with the rust toolchain
+    Builtin,
 }
 
 // The hash here is important for what folder packages get downloaded into.
@@ -40,6 +42,7 @@ impl SourceKind {
             SourceKind::SparseRegistry => None,
             SourceKind::LocalRegistry => Some("local-registry"),
             SourceKind::Directory => Some("directory"),
+            SourceKind::Builtin => Some("builtin"),
         }
     }
 }
@@ -71,6 +74,10 @@ impl Ord for SourceKind {
             (_, SourceKind::Directory) => Ordering::Greater,
 
             (SourceKind::Git(a), SourceKind::Git(b)) => a.cmp(b),
+            (SourceKind::Git(_), _) => Ordering::Less,
+            (_, SourceKind::Git(_)) => Ordering::Greater,
+
+            (SourceKind::Builtin, SourceKind::Builtin) => Ordering::Equal,
         }
     }
 }

crates/resolver-tests/src/lib.rs

@@ -204,6 +204,7 @@ pub fn resolve_with_global_context_raw(
         &version_prefs,
         ResolveVersion::with_rust_version(None),
         Some(gctx),
+        &[],
     );
 
     // The largest test in our suite takes less then 30 secs.

src/cargo/core/compiler/standard_lib.rs

@@ -15,7 +15,11 @@ use std::path::PathBuf;
 
 use super::BuildConfig;
 
-fn std_crates<'a>(crates: &'a [String], default: &'static str, units: &[Unit]) -> HashSet<&'a str> {
+pub fn std_crates<'a>(
+    crates: &'a [String],
+    default: &'static str,
+    units: &[Unit],
+) -> HashSet<&'a str> {
     let mut crates = HashSet::from_iter(crates.iter().map(|s| s.as_str()));
     // This is a temporary hack until there is a more principled way to
     // declare dependencies in Cargo.toml.
@@ -59,6 +63,7 @@ pub fn resolve_std<'gctx>(
     // TODO: Consider doing something to enforce --locked? Or to prevent the
     // lock file from being written, such as setting ephemeral.
     let mut std_ws = Workspace::new(&std_ws_manifest_path, gctx)?;
+    std_ws.set_is_std(true);
     // Don't require optional dependencies in this workspace, aka std's own
     // `[dev-dependencies]`. No need for us to generate a `Resolve` which has
     // those included because we'll never use them anyway.
@@ -216,7 +221,7 @@ fn generate_roots(
     Ok(())
 }
 
-fn detect_sysroot_src_path(target_data: &RustcTargetData<'_>) -> CargoResult<PathBuf> {
+pub fn detect_sysroot_src_path(target_data: &RustcTargetData<'_>) -> CargoResult<PathBuf> {
     if let Some(s) = target_data.gctx.get_env_os("__CARGO_TESTS_ONLY_SRC_ROOT") {
         return Ok(s.into());
     }

src/cargo/core/compiler/unit_dependencies.rs

@@ -53,6 +53,8 @@ struct State<'a, 'gctx> {
     std_resolve: Option<&'a Resolve>,
     /// Like `usr_features` but for building standard library (`-Zbuild-std`).
     std_features: Option<&'a ResolvedFeatures>,
+    // The root units of any opaque dependencies present in the user resolve
+    opaque_roots: &'a HashMap<CompileKind, Vec<Unit>>,
     /// `true` while generating the dependencies for the standard library.
     is_std: bool,
     /// The high-level operation requested by the user.
@@ -93,7 +95,7 @@ pub fn build_unit_dependencies<'a, 'gctx>(
     resolve: &'a Resolve,
     features: &'a ResolvedFeatures,
     std_resolve: Option<&'a (Resolve, ResolvedFeatures)>,
-    roots: &[Unit],
+    roots: &[Unit], //TODO: builtins can be roots if requested on the command line
     scrape_units: &[Unit],
     std_roots: &HashMap<CompileKind, Vec<Unit>>,
     intent: UserIntent,
@@ -120,6 +122,7 @@ pub fn build_unit_dependencies<'a, 'gctx>(
         usr_features: features,
         std_resolve,
         std_features,
+        opaque_roots: std_roots,
         is_std: false,
         intent,
         target_data,
@@ -130,15 +133,14 @@ pub fn build_unit_dependencies<'a, 'gctx>(
     };
 
     let std_unit_deps = calc_deps_of_std(&mut state, std_roots)?;
+    if let Some(std_unit_deps) = std_unit_deps {
+        attach_std_deps(&mut state, std_unit_deps);
+    }
 
     deps_of_roots(roots, &mut state)?;
     super::links::validate_links(state.resolve(), &state.unit_dependencies)?;
     // Hopefully there aren't any links conflicts with the standard library?
 
-    if let Some(std_unit_deps) = std_unit_deps {
-        attach_std_deps(&mut state, std_roots, std_unit_deps);
-    }
-
     connect_run_custom_build_deps(&mut state);
 
     // Dependencies are used in tons of places throughout the backend, many of
@@ -189,38 +191,14 @@ fn calc_deps_of_std(
     Ok(Some(std::mem::take(&mut state.unit_dependencies)))
 }
 
-/// Add the standard library units to the `unit_dependencies`.
-fn attach_std_deps(
-    state: &mut State<'_, '_>,
-    std_roots: &HashMap<CompileKind, Vec<Unit>>,
-    std_unit_deps: UnitGraph,
-) {
-    // Attach the standard library as a dependency of every target unit.
-    let mut found = false;
-    for (unit, deps) in state.unit_dependencies.iter_mut() {
-        if !unit.kind.is_host() && !unit.mode.is_run_custom_build() {
-            deps.extend(std_roots[&unit.kind].iter().map(|unit| UnitDep {
-                unit: unit.clone(),
-                unit_for: UnitFor::new_normal(unit.kind),
-                extern_crate_name: unit.pkg.name(),
-                dep_name: None,
-                // TODO: Does this `public` make sense?
-                public: true,
-                noprelude: true,
-                nounused: true,
-                // Artificial dependency
-                manifest_deps: Unhashed(None),
-            }));
-            found = true;
+/// Add the dependencies of standard library units to the `unit_dependencies`.
+fn attach_std_deps(state: &mut State<'_, '_>, std_unit_deps: UnitGraph) {
+    for (unit, deps) in std_unit_deps.into_iter() {
+        if unit.pkg.package_id().name() == "sysroot" {
+            continue;
         }
-    }
-    // And also include the dependencies of the standard library itself. Don't
-    // include these if no units actually needed the standard library.
-    if found {
-        for (unit, deps) in std_unit_deps.into_iter() {
-            if let Some(other_unit) = state.unit_dependencies.insert(unit, deps) {
-                panic!("std unit collision with existing unit: {:?}", other_unit);
-            }
+        if let Some(other_unit) = state.unit_dependencies.insert(unit, deps) {
+            panic!("std unit collision with existing unit: {:?}", other_unit);
         }
     }
 }
@@ -341,17 +319,42 @@ fn compute_deps(
             )?;
             ret.push(unit_dep);
         } else {
-            let unit_dep = new_unit_dep(
-                state,
-                unit,
-                dep_pkg,
-                dep_lib,
-                Some(manifest_deps),
-                dep_unit_for,
-                unit.kind.for_target(dep_lib),
-                mode,
-                IS_NO_ARTIFACT_DEP,
-            )?;
+            // if builtin, return from state.opaque_roots
+            let unit_dep = if dep_pkg_id.source_id().is_builtin() {
+                if unit_for.is_for_host() {
+                    // Build scripts/proc_macros shouldn't use build-std
+                    continue;
+                }
+                let unit = state
+                    .opaque_roots
+                    .get(&unit.kind.for_target(dep_lib))
+                    .expect("Std was resolved for all requested targets")
+                    .iter()
+                    .find(|&u| u.pkg.name() == dep_pkg_id.name())
+                    .expect("libstd was resolved with all possible builtin deps as roots");
+                UnitDep {
+                    unit: unit.clone(),
+                    unit_for: UnitFor::new_normal(unit.kind),
+                    extern_crate_name: unit.pkg.name(),
+                    dep_name: None,
+                    public: true,
+                    noprelude: true,
+                    nounused: true,
+                    manifest_deps: Unhashed(None),
+                }
+            } else {
+                new_unit_dep(
+                    state,
+                    unit,
+                    dep_pkg,
+                    dep_lib,
+                    Some(manifest_deps),
+                    dep_unit_for,
+                    unit.kind.for_target(dep_lib),
+                    mode,
+                    IS_NO_ARTIFACT_DEP,
+                )?
+            };
             ret.push(unit_dep);
         }
 
@@ -367,6 +370,30 @@ fn compute_deps(
     }
     state.dev_dependency_edges.extend(dev_deps);
 
+    if state.gctx.cli_unstable().build_std.is_some()
+        && unit.mode.is_rustc_test()
+        && unit.target.harness()
+        && !unit_for.is_for_host()
+    {
+        // If test isn't found, we were probably compiling for no-std.
+        if let Some(test) = state.opaque_roots[&unit.kind]
+            .iter()
+            .find(|u| u.pkg.name() == "test")
+        {
+            let unitdep = UnitDep {
+                unit: test.clone(),
+                unit_for: UnitFor::new_normal(test.kind),
+                extern_crate_name: test.pkg.name(),
+                dep_name: None,
+                public: true,
+                noprelude: true,
+                nounused: true,
+                manifest_deps: Unhashed(None),
+            };
+            ret.push(unitdep);
+        }
+    }
+
     // If this target is a build script, then what we've collected so far is
     // all we need. If this isn't a build script, then it depends on the
     // build script if there is one.
@@ -651,6 +678,10 @@ fn compute_deps_doc(
     // the documentation of the library being built.
     let mut ret = Vec::new();
     for (id, deps) in state.deps(unit, unit_for) {
+        if id.source_id().is_builtin() {
+            // Build-std for cargo doc is not yet implemented
+            continue;
+        }
         let Some(dep_lib) = calc_artifact_deps(unit, unit_for, id, &deps, state, &mut ret)? else {
             continue;
         };

src/cargo/core/dependency.rs

@@ -1,4 +1,5 @@
 use cargo_platform::Platform;
+use cargo_util_schemas::core::SourceKind;
 use semver::VersionReq;
 use serde::Serialize;
 use serde::ser;
@@ -51,6 +52,10 @@ struct Inner {
     // This dependency should be used only for this platform.
     // `None` means *all platforms*.
     platform: Option<Platform>,
+
+    // Opaque dependencies should be resolved with a separate resolver run, and handled
+    // by unit generation.
+    opaque: bool,
 }
 
 #[derive(Serialize)]
@@ -162,10 +167,42 @@ impl Dependency {
                 platform: None,
                 explicit_name_in_toml: None,
                 artifact: None,
+                opaque: false,
             }),
         }
     }
 
+    pub fn new_for_builtin_summary(s: Summary) -> CargoResult<Self> {
+        if let SourceKind::Builtin = s.source_id().kind() {
+            Ok(Dependency {
+                // Most of these fields are ignored by the resolver/registry, which will just match this
+                // dependency up with the given Summary
+                inner: Arc::new(Inner {
+                    name: s.name(),
+                    source_id: s.source_id(),
+                    registry_id: None,
+                    req: OptVersionReq::Any,
+                    kind: DepKind::Normal,
+                    only_match_name: true,
+                    optional: false,
+                    public: true,
+                    features: Vec::new(),
+                    default_features: true,
+                    specified_req: false,
+                    platform: None,
+                    explicit_name_in_toml: None,
+                    artifact: None,
+                    opaque: true,
+                }),
+            })
+        } else {
+            Err(anyhow::format_err!(
+                "Can't create builtin dependency for a non-builtin \"{}\"",
+                s.source_id()
+            ))
+        }
+    }
+
     pub fn serialized(
         &self,
         unstable_flags: &CliUnstable,
@@ -467,6 +504,10 @@ impl Dependency {
     pub(crate) fn maybe_lib(&self) -> bool {
         self.artifact().map(|a| a.is_lib).unwrap_or(true)
     }
+
+    pub fn is_opaque(&self) -> bool {
+        self.inner.opaque
+    }
 }
 
 /// The presence of an artifact turns an ordinary dependency into an Artifact dependency.

src/cargo/core/resolver/dep_cache.rs

@@ -24,6 +24,7 @@ use crate::util::LocalPollAdapter;
 use crate::util::closest_msg;
 use crate::util::errors::CargoResult;
 use crate::util::interning::{INTERNED_DEFAULT, InternedString};
+use crate::util::network::PollExt;
 
 use anyhow::Context as _;
 use std::cell::RefCell;
@@ -191,13 +192,16 @@ pub struct RegistryQueryer<'a, T: Registry> {
         (Option<PackageId>, Summary, ResolveOpts),
         (Rc<(HashSet<InternedString>, Rc<Vec<DepInfo>>)>, bool),
     >,
+    /// The set of builtin dependencies to inject when appropriate
+    implicit_builtin_deps: &'a [Dependency],
 }
 
 impl<'a, T: Registry> RegistryQueryer<'a, T> {
     pub fn new(
         registry: &'a T,
         replacements: &'a [(PackageIdSpec, Dependency)],
         version_prefs: &'a VersionPreferences,
+        implicit_builtin_deps: &'a [Dependency],
     ) -> Self {
         let inner = Rc::new(RegistryQueryerAsync::new(
             registry,
@@ -208,6 +212,7 @@ impl<'a, T: Registry> RegistryQueryer<'a, T> {
             inner: inner.clone(),
             poller: LocalPollAdapter::new(inner),
             summary_cache: HashMap::new(),
+            implicit_builtin_deps,
         }
     }
 
@@ -301,6 +306,19 @@ impl<'a, T: Registry> RegistryQueryer<'a, T> {
             })
             .collect::<CargoResult<Vec<DepInfo>>>()?;
 
+        if opts.inject_builtins {
+            for dep in self.implicit_builtin_deps {
+                // TODO: This kicks off multiple queries per package searched. What's the
+                // performance impact?
+                let candidates = self
+                    .query(dep, first_version)
+                    .expect("Builtin packages should be immediately available")
+                    .expect("Builtin names should be valid by this point");
+
+                deps.push((dep.clone(), candidates, Rc::new(Default::default())));
+            }
+        }
+
         // Attempt to resolve dependencies with fewer candidates before trying
         // dependencies with more candidates. This way if the dependency with
         // only one candidate can't be resolved we don't have to do a bunch of