refactor(network): use async for registry network operations

Author: arlosi

Cargo.lock

@@ -45,8 +45,11 @@ curl = "0.4.49"
 # Do not upgrade curl-sys past 0.4.83
 # https://github.com/rust-lang/cargo/issues/16357
 curl-sys = "=0.4.83"
+dynosaur = "0.3"
 filetime = "0.2.27"
 flate2 = { version = "1.1.9", default-features = false, features = ["zlib-rs"] }
+futures = { version = "0.3", default-features = false, features = ["std", "executor"]}
+futures-timer = "3.0.3"
 git2 = "0.20.4"
 git2-curl = "0.21.0"
 # When updating this, also see if `gix-transport` further down needs updating or some auth-related tests will fail.
@@ -58,6 +61,7 @@ heck = "0.5.0"
 hex = "0.4.3"
 hmac = "0.12.1"
 home = "0.5.12"
+http = "1.4.0"
 http-auth = { version = "0.1.10", default-features = false }
 ignore = "0.4.25"
 im-rc = "15.1.0"
@@ -172,8 +176,10 @@ color-print.workspace = true
 crates-io.workspace = true
 curl = { workspace = true, features = ["http2"] }
 curl-sys.workspace = true
+dynosaur.workspace = true
 filetime.workspace = true
 flate2.workspace = true
+futures.workspace = true
 git2.workspace = true
 git2-curl.workspace = true
 gix.workspace = true
@@ -182,6 +188,7 @@ heck.workspace = true
 hex.workspace = true
 hmac.workspace = true
 home.workspace = true
+http.workspace = true
 http-auth.workspace = true
 ignore.workspace = true
 im-rc.workspace = true
@@ -225,6 +232,7 @@ unicode-ident.workspace = true
 url.workspace = true
 walkdir.workspace = true
 winnow.workspace = true
+futures-timer.workspace = true
 
 [target.'cfg(target_has_atomic = "64")'.dependencies]
 tracing-chrome.workspace = true

Cargo.toml

@@ -823,7 +823,6 @@ impl HttpServer {
                 url,
                 body,
             };
-            println!("req: {:#?}", req);
             let response = self.route(&req);
             let buf = buf.get_mut();
             write!(buf, "HTTP/1.1 {}\r\n", response.code).unwrap();
@@ -1029,7 +1028,7 @@ impl HttpServer {
         Response {
             code: 401,
             headers: vec![
-                r#"WWW-Authenticate: Cargo login_url="https://test-registry-login/me""#.to_string(),
+                r#"www-authenticate: Cargo login_url="https://test-registry-login/me""#.to_string(),
             ],
             body: b"Unauthorized message from server.".to_vec(),
         }

crates/cargo-test-support/src/registry.rs

@@ -7,16 +7,17 @@
 pub mod helpers;
 pub mod sat;
 
+use std::cell::RefCell;
 use std::cmp::{max, min};
 use std::collections::{BTreeMap, HashSet};
 use std::fmt;
-use std::task::Poll;
 use std::time::Instant;
 
 use cargo::core::Resolve;
 use cargo::core::ResolveVersion;
 use cargo::core::SourceId;
 use cargo::core::dependency::DepKind;
+use cargo::core::registry::DynRegistry;
 use cargo::core::resolver::{self, ResolveOpts, VersionOrdering, VersionPreferences};
 use cargo::core::{Dependency, PackageId, Registry, Summary};
 use cargo::sources::IndexSummary;
@@ -131,15 +132,16 @@ pub fn resolve_with_global_context_raw(
 ) -> CargoResult<Resolve> {
     struct MyRegistry<'a> {
         list: &'a [Summary],
-        used: HashSet<PackageId>,
+        used: RefCell<HashSet<PackageId>>,
     }
+
     impl<'a> Registry for MyRegistry<'a> {
-        fn query(
-            &mut self,
+        async fn query(
+            &self,
             dep: &Dependency,
             kind: QueryKind,
             f: &mut dyn FnMut(IndexSummary),
-        ) -> Poll<CargoResult<()>> {
+        ) -> CargoResult<()> {
             for summary in self.list.iter() {
                 let matched = match kind {
                     QueryKind::Exact => dep.matches(summary),
@@ -148,11 +150,11 @@ pub fn resolve_with_global_context_raw(
                     QueryKind::Normalized => true,
                 };
                 if matched {
-                    self.used.insert(summary.package_id());
+                    self.used.borrow_mut().insert(summary.package_id());
                     f(IndexSummary::Candidate(summary.clone()));
                 }
             }
-            Poll::Ready(Ok(()))
+            Ok(())
         }
 
         fn describe_source(&self, _src: SourceId) -> String {
@@ -162,22 +164,18 @@ pub fn resolve_with_global_context_raw(
         fn is_replaced(&self, _src: SourceId) -> bool {
             false
         }
-
-        fn block_until_ready(&mut self) -> CargoResult<()> {
-            Ok(())
-        }
     }
     impl<'a> Drop for MyRegistry<'a> {
         fn drop(&mut self) {
-            if std::thread::panicking() && self.list.len() != self.used.len() {
+            if std::thread::panicking() && self.list.len() != self.used.get_mut().len() {
                 // we found a case that causes a panic and did not use all of the input.
                 // lets print the part of the input that was used for minimization.
                 eprintln!(
                     "Part used before drop: {:?}",
                     PrettyPrintRegistry(
                         self.list
                             .iter()
-                            .filter(|s| { self.used.contains(&s.package_id()) })
+                            .filter(|s| { self.used.get_mut().contains(&s.package_id()) })
                             .cloned()
                             .collect()
                     )
@@ -187,7 +185,7 @@ pub fn resolve_with_global_context_raw(
     }
     let mut registry = MyRegistry {
         list: registry,
-        used: HashSet::new(),
+        used: RefCell::new(HashSet::new()),
     };
 
     let root_summary =
@@ -204,7 +202,7 @@ pub fn resolve_with_global_context_raw(
     let resolve = resolver::resolve(
         &[(root_summary, opts)],
         &[],
-        &mut registry,
+        DynRegistry::from_mut(&mut registry),
         &version_prefs,
         ResolveVersion::with_rust_version(None),
         Some(gctx),

crates/resolver-tests/src/lib.rs

@@ -13,6 +13,7 @@ git2.workspace = true
 semver.workspace = true
 tracing-subscriber.workspace = true
 tracing.workspace = true
+futures.workspace = true
 
 [lints]
 workspace = true

crates/xtask-bump-check/Cargo.toml

@@ -15,7 +15,6 @@
 use std::collections::HashMap;
 use std::fmt::Write;
 use std::fs;
-use std::task;
 
 use cargo::CargoResult;
 use cargo::core::Package;
@@ -433,15 +432,9 @@ fn check_crates_io<'a>(
         let current = member.version();
         let version_req = format!(">={current}");
         let query = Dependency::parse(*name, Some(&version_req), source_id)?;
-        let possibilities = loop {
-            // Exact to avoid returning all for path/git
-            match registry.query_vec(&query, QueryKind::Exact) {
-                task::Poll::Ready(res) => {
-                    break res?;
-                }
-                task::Poll::Pending => registry.block_until_ready()?,
-            }
-        };
+        // Exact to avoid returning all for path/git
+        let possibilities =
+            futures::executor::block_on(registry.query_vec(&query, QueryKind::Exact))?;
         if possibilities.is_empty() {
             tracing::trace!("dep `{name}` has no version greater than or equal to `{current}`");
         } else {

crates/xtask-bump-check/src/xtask.rs

@@ -397,6 +397,7 @@ use tracing::{debug, info};
 
 use crate::core::Package;
 use crate::core::compiler::unit_graph::UnitDep;
+use crate::sources::source::Source as _;
 use crate::util;
 use crate::util::errors::CargoResult;
 use crate::util::interning::InternedString;