From 4fc325d874b0a967603cd97e197c7980e619a90b Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 1 Feb 2023 13:51:20 +0000 Subject: [PATCH 1/3] Support sparse indexes And explicitly opt in to them for cargos which support it. This should very significantly speed up repinning, as only the metadata for crates needed for the build, rather than the whole index, will need fetching. --- crate_universe/private/srcs.bzl | 1 + crate_universe/src/cli/splice.rs | 4 +- crate_universe/src/cli/vendor.rs | 4 +- crate_universe/src/metadata.rs | 49 +++++-- crate_universe/src/splicing.rs | 121 +++++++++--------- crate_universe/src/splicing/cargo_config.rs | 8 +- .../src/splicing/crate_index_lookup.rs | 51 ++++++++ 7 files changed, 167 insertions(+), 71 deletions(-) create mode 100644 crate_universe/src/splicing/crate_index_lookup.rs diff --git a/crate_universe/private/srcs.bzl b/crate_universe/private/srcs.bzl index 030ff4c092..1f83e2dba7 100644 --- a/crate_universe/private/srcs.bzl +++ b/crate_universe/private/srcs.bzl @@ -29,6 +29,7 @@ CARGO_BAZEL_SRCS = [ Label("//crate_universe:src/rendering/templates/vendor_module.j2"), Label("//crate_universe:src/splicing.rs"), Label("//crate_universe:src/splicing/cargo_config.rs"), + Label("//crate_universe:src/splicing/crate_index_lookup.rs"), Label("//crate_universe:src/splicing/splicer.rs"), Label("//crate_universe:src/test.rs"), Label("//crate_universe:src/utils.rs"), diff --git a/crate_universe/src/cli/splice.rs b/crate_universe/src/cli/splice.rs index cb88df7d2c..207c33e752 100644 --- a/crate_universe/src/cli/splice.rs +++ b/crate_universe/src/cli/splice.rs @@ -98,9 +98,11 @@ pub fn splice(opt: SpliceOptions) -> Result<()> { )?; // Write the registry url info to the manifest now that a lockfile has been generated WorkspaceMetadata::write_registry_urls_and_feature_map( + &cargo, &cargo_lockfile, feature_map, - &manifest_path, + manifest_path.as_path_buf(), + manifest_path.as_path_buf(), )?; let output_dir = opt.output_dir.clone(); diff --git a/crate_universe/src/cli/vendor.rs b/crate_universe/src/cli/vendor.rs index 97da5d390c..21311773a6 100644 --- a/crate_universe/src/cli/vendor.rs +++ b/crate_universe/src/cli/vendor.rs @@ -151,9 +151,11 @@ pub fn vendor(opt: VendorOptions) -> Result<()> { // Write the registry url info to the manifest now that a lockfile has been generated WorkspaceMetadata::write_registry_urls_and_feature_map( + &cargo, &cargo_lockfile, feature_map, - &manifest_path, + manifest_path.as_path_buf(), + manifest_path.as_path_buf(), )?; // Write metadata to the workspace for future reuse diff --git a/crate_universe/src/metadata.rs b/crate_universe/src/metadata.rs index 392c2bff7c..6f44e01823 100644 --- a/crate_universe/src/metadata.rs +++ b/crate_universe/src/metadata.rs @@ -16,6 +16,7 @@ use crate::lockfile::Digest; use anyhow::{anyhow, bail, Context, Result}; use cargo_lock::Lockfile as CargoLockfile; use cargo_metadata::{Metadata as CargoMetadata, MetadataCommand}; +use semver::Version; use crate::config::CrateId; use crate::utils::starlark::SelectList; @@ -75,7 +76,7 @@ impl MetadataGenerator for Generator { let metadata = self .cargo_bin - .metadata_command() + .metadata_command()? .current_dir(manifest_dir) .manifest_path(manifest_path.as_ref()) .other_options(["--locked".to_owned()]) @@ -103,15 +104,20 @@ impl Cargo { } /// Returns a new `Command` for running this cargo. - pub fn command(&self) -> Command { - Command::new(&self.path) + pub fn command(&self) -> Result { + let mut command = Command::new(&self.path); + command.envs(self.env()?); + Ok(command) } /// Returns a new `MetadataCommand` using this cargo. - pub fn metadata_command(&self) -> MetadataCommand { + pub fn metadata_command(&self) -> Result { let mut command = MetadataCommand::new(); command.cargo_path(&self.path); - command + for (k, v) in self.env()? { + command.env(k, v); + } + Ok(command) } /// Returns the output of running `cargo version`, trimming any leading or trailing whitespace. @@ -124,6 +130,29 @@ impl Cargo { } Ok(full_version.clone().unwrap()) } + + pub fn use_sparse_registries_for_crates_io(&self) -> Result { + let full_version = self.full_version()?; + let version_str = full_version.split(' ').nth(1); + if let Some(version_str) = version_str { + let version = Version::parse(version_str).context("Failed to parse cargo version")?; + return Ok(version + >= Version::parse("1.68.0").expect("1.68.0 is a known-to-parse semver version")); + } + bail!("Couldn't parse cargo version"); + } + + fn env(&self) -> Result> { + let mut map = BTreeMap::new(); + + if self.use_sparse_registries_for_crates_io()? { + map.insert( + "CARGO_REGISTRIES_CRATES_IO_PROTOCOL".into(), + "sparse".into(), + ); + } + Ok(map) + } } /// A configuration desrcibing how to invoke [cargo update](https://doc.rust-lang.org/cargo/commands/cargo-update.html). @@ -192,7 +221,7 @@ impl CargoUpdateRequest { // Simply invoke `cargo update` let output = cargo_bin - .command() + .command()? // Cargo detects config files based on `pwd` when running so // to ensure user provided Cargo config files are used, it's // critical to set the working directory to the manifest dir. @@ -267,7 +296,7 @@ impl LockGenerator { // of having just generated a new one let output = self .cargo_bin - .command() + .command()? // Cargo detects config files based on `pwd` when running so // to ensure user provided Cargo config files are used, it's // critical to set the working directory to the manifest dir. @@ -295,7 +324,7 @@ impl LockGenerator { // Simply invoke `cargo generate-lockfile` let output = self .cargo_bin - .command() + .command()? // Cargo detects config files based on `pwd` when running so // to ensure user provided Cargo config files are used, it's // critical to set the working directory to the manifest dir. @@ -347,7 +376,7 @@ impl VendorGenerator { // Simply invoke `cargo generate-lockfile` let output = self .cargo_bin - .command() + .command()? // Cargo detects config files based on `pwd` when running so // to ensure user provided Cargo config files are used, it's // critical to set the working directory to the manifest dir. @@ -410,7 +439,7 @@ impl FeatureGenerator { // - https://github.com/bazelbuild/rules_rust/issues/1662 let output = self .cargo_bin - .command() + .command()? .current_dir(manifest_dir) .arg("tree") .arg("--locked") diff --git a/crate_universe/src/splicing.rs b/crate_universe/src/splicing.rs index 9833e63a5c..43085dec6b 100644 --- a/crate_universe/src/splicing.rs +++ b/crate_universe/src/splicing.rs @@ -1,6 +1,7 @@ //! This module is responsible for finding a Cargo workspace pub(crate) mod cargo_config; +mod crate_index_lookup; mod splicer; use std::collections::{BTreeMap, BTreeSet}; @@ -9,9 +10,8 @@ use std::fs; use std::path::{Path, PathBuf}; use std::str::FromStr; -use anyhow::{bail, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use cargo_toml::Manifest; -use hex::ToHex; use serde::{Deserialize, Serialize}; use crate::config::CrateId; @@ -19,6 +19,7 @@ use crate::metadata::{Cargo, CargoUpdateRequest, LockGenerator}; use crate::utils::starlark::{Label, SelectList}; use self::cargo_config::CargoConfig; +use self::crate_index_lookup::CrateIndexLookup; pub use self::splicer::*; type DirectPackageManifest = BTreeMap; @@ -242,11 +243,13 @@ impl WorkspaceMetadata { } pub fn write_registry_urls_and_feature_map( + cargo: &Cargo, lockfile: &cargo_lock::Lockfile, features: BTreeMap>, - manifest_path: &SplicedManifest, + input_manifest_path: &Path, + output_manifest_path: &Path, ) -> Result<()> { - let mut manifest = read_manifest(manifest_path.as_path_buf())?; + let mut manifest = read_manifest(input_manifest_path)?; let mut workspace_metaata = WorkspaceMetadata::try_from( manifest @@ -259,7 +262,7 @@ impl WorkspaceMetadata { .clone(), )?; - // Locate all packages soruced from a registry + // Locate all packages sourced from a registry let pkg_sources: Vec<&cargo_lock::Package> = lockfile .packages .iter() @@ -276,8 +279,7 @@ impl WorkspaceMetadata { // Load the cargo config let cargo_config = { // Note that this path must match the one defined in `splicing::setup_cargo_config` - let config_path = manifest_path - .as_path_buf() + let config_path = input_manifest_path .parent() .unwrap() .join(".cargo") @@ -294,80 +296,83 @@ impl WorkspaceMetadata { let crate_indexes = index_urls .into_iter() .map(|url| { - let index = { - // Ensure the correct registry is mapped based on the give Cargo config. - let index_url = if let Some(config) = &cargo_config { - if let Some(source) = config.get_source_from_url(&url) { - if let Some(replace_with) = &source.replace_with { - if let Some(replacement) = config.get_registry_index_url_by_name(replace_with) { - replacement - } else { - bail!("Tried to replace registry {} with registry named {} but didn't have metadata about the replacement", url, replace_with); - } + // Ensure the correct registry is mapped based on the give Cargo config. + let index_url = if let Some(config) = &cargo_config { + if let Some(source) = config.get_source_from_url(&url) { + if let Some(replace_with) = &source.replace_with { + if let Some(replacement) = config.get_registry_index_url_by_name(replace_with) { + replacement } else { - &url + bail!("Tried to replace registry {} with registry named {} but didn't have metadata about the replacement", url, replace_with); } } else { &url } } else { &url + } + } else { + &url + }; + let index = if cargo.use_sparse_registries_for_crates_io()? && index_url == "https://github.com/rust-lang/crates.io-index" { + CrateIndexLookup::Http(crates_index::SparseIndex::from_url("sparse+https://index.crates.io/")?) + } else if index_url.starts_with("sparse+https://") { + CrateIndexLookup::Http(crates_index::SparseIndex::from_url(index_url)?) + } else { + let index = { + // Load the index for the current url + let index = crates_index::Index::from_url(index_url) + .with_context(|| format!("Failed to load index for url: {index_url}"))?; + + // Ensure each index has a valid index config + index.index_config().with_context(|| { + format!("`config.json` not found in index: {index_url}") + })?; + + index }; - // Load the index for the current url - let index = crates_index::Index::from_url(index_url) - .with_context(|| format!("Failed to load index for url: {index_url}"))?; - - // Ensure each index has a valid index config - index.index_config().with_context(|| { - format!("`config.json` not found in index: {index_url}") - })?; - - index + CrateIndexLookup::Git(index) }; - Ok((url, index)) }) - .collect::>>() + .collect::>>() .context("Failed to locate crate indexes")?; // Get the download URL of each package based on it's registry url. let additional_sources = pkg_sources .iter() - .filter_map(|pkg| { + .map(|pkg| { let source_id = pkg.source.as_ref().unwrap(); - let index = &crate_indexes[&source_id.url().to_string()]; - let index_config = index.index_config().unwrap(); - - index.crate_(pkg.name.as_str()).map(|crate_idx| { - crate_idx - .versions() - .iter() - .find(|v| v.version() == pkg.version.to_string()) - .and_then(|v| { - v.download_url(&index_config).map(|url| { - let crate_id = - CrateId::new(v.name().to_owned(), v.version().to_owned()); - let sha256 = pkg - .checksum - .as_ref() - .and_then(|sum| { - sum.as_sha256().map(|sum| sum.encode_hex::()) - }) - .unwrap_or_else(|| v.checksum().encode_hex::()); - let source_info = SourceInfo { url, sha256 }; - (crate_id, source_info) - }) - }) + let source_url = source_id.url().to_string(); + let lookup = crate_indexes.get(&source_url).ok_or_else(|| { + anyhow!( + "Couldn't find crate_index data for SourceID {:?}", + source_id + ) + })?; + lookup.get_source_info(pkg).map(|source_info| { + ( + CrateId::new(pkg.name.as_str().to_owned(), pkg.version.to_string()), + source_info, + ) }) }) - .flatten(); - - workspace_metaata.sources.extend(additional_sources); + .collect::>>()?; + + workspace_metaata + .sources + .extend( + additional_sources + .into_iter() + .filter_map(|(crate_id, source_info)| { + source_info.map(|source_info| (crate_id, source_info)) + }), + ); workspace_metaata.features = features; workspace_metaata.inject_into(&mut manifest)?; - write_root_manifest(manifest_path.as_path_buf(), manifest)?; + write_root_manifest(output_manifest_path, manifest)?; Ok(()) } diff --git a/crate_universe/src/splicing/cargo_config.rs b/crate_universe/src/splicing/cargo_config.rs index 5f4a9aeeb8..ed1f289939 100644 --- a/crate_universe/src/splicing/cargo_config.rs +++ b/crate_universe/src/splicing/cargo_config.rs @@ -122,7 +122,13 @@ impl CargoConfig { /// Look up a registry [Source] by its url. pub fn get_source_from_url(&self, url: &str) -> Option<&Source> { - self.source.values().find(|v| v.registry == url) + if let Some(found) = self.source.values().find(|v| v.registry == url) { + Some(found) + } else if url == "https://github.com/rust-lang/crates.io-index" { + self.source.get("crates-io") + } else { + None + } } pub fn get_registry_index_url_by_name(&self, name: &str) -> Option<&str> { diff --git a/crate_universe/src/splicing/crate_index_lookup.rs b/crate_universe/src/splicing/crate_index_lookup.rs new file mode 100644 index 0000000000..735c529103 --- /dev/null +++ b/crate_universe/src/splicing/crate_index_lookup.rs @@ -0,0 +1,51 @@ +use crate::splicing::SourceInfo; +use anyhow::{Context, Result}; +use crates_index::IndexConfig; +use hex::ToHex; + +pub enum CrateIndexLookup { + Git(crates_index::Index), + Http(crates_index::SparseIndex), +} + +impl CrateIndexLookup { + pub fn get_source_info(&self, pkg: &cargo_lock::Package) -> Result> { + let index_config = self + .index_config() + .context("Failed to get crate index config")?; + let crate_ = match self { + // The crates we care about should all be in the cache already, + // because `cargo metadata` ran which should have fetched them. + Self::Http(index) => Some( + index + .crate_from_cache(pkg.name.as_str()) + .with_context(|| format!("Failed to get crate from cache for {pkg:?}"))?, + ), + Self::Git(index) => index.crate_(pkg.name.as_str()), + }; + let source_info = crate_.and_then(|crate_idx| { + crate_idx + .versions() + .iter() + .find(|v| v.version() == pkg.version.to_string()) + .and_then(|v| { + v.download_url(&index_config).map(|url| { + let sha256 = pkg + .checksum + .as_ref() + .and_then(|sum| sum.as_sha256().map(|sum| sum.encode_hex::())) + .unwrap_or_else(|| v.checksum().encode_hex::()); + SourceInfo { url, sha256 } + }) + }) + }); + Ok(source_info) + } + + fn index_config(&self) -> Result { + match self { + Self::Git(index) => index.index_config(), + Self::Http(index) => index.index_config(), + } + } +} From 18518e02768d6769a06f2897694525f335db5d37 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 14 Mar 2023 22:27:18 +0000 Subject: [PATCH 2/3] Add some tests --- crate_universe/src/splicing.rs | 29 ++-- crate_universe/src/splicing/cargo_config.rs | 153 +++++++++++++++++- .../src/splicing/crate_index_lookup.rs | 102 ++++++++++++ crate_universe/src/utils.rs | 2 + .../.cache/la/zy/lazy_static | Bin 0 -> 9840 bytes .../config.json | 4 + .../.cache/la/zy/lazy_static | Bin 0 -> 9840 bytes .../config.json | 4 + 8 files changed, 274 insertions(+), 20 deletions(-) create mode 100644 crate_universe/test_data/crate_indexes/lazy_static/cargo_home/registry/index/index.crates.io-6f17d22bba15001f/.cache/la/zy/lazy_static create mode 100644 crate_universe/test_data/crate_indexes/lazy_static/cargo_home/registry/index/index.crates.io-6f17d22bba15001f/config.json create mode 100644 crate_universe/test_data/crate_indexes/rewritten_lazy_static/cargo_home/registry/index/index.crates.io-6f17d22bba15001f/.cache/la/zy/lazy_static create mode 100644 crate_universe/test_data/crate_indexes/rewritten_lazy_static/cargo_home/registry/index/index.crates.io-6f17d22bba15001f/config.json diff --git a/crate_universe/src/splicing.rs b/crate_universe/src/splicing.rs index 43085dec6b..a4651233bd 100644 --- a/crate_universe/src/splicing.rs +++ b/crate_universe/src/splicing.rs @@ -16,6 +16,7 @@ use serde::{Deserialize, Serialize}; use crate::config::CrateId; use crate::metadata::{Cargo, CargoUpdateRequest, LockGenerator}; +use crate::utils; use crate::utils::starlark::{Label, SelectList}; use self::cargo_config::CargoConfig; @@ -298,31 +299,25 @@ impl WorkspaceMetadata { .map(|url| { // Ensure the correct registry is mapped based on the give Cargo config. let index_url = if let Some(config) = &cargo_config { - if let Some(source) = config.get_source_from_url(&url) { - if let Some(replace_with) = &source.replace_with { - if let Some(replacement) = config.get_registry_index_url_by_name(replace_with) { - replacement - } else { - bail!("Tried to replace registry {} with registry named {} but didn't have metadata about the replacement", url, replace_with); - } - } else { - &url - } - } else { - &url - } + config.resolve_replacement_url(&url)? } else { &url }; - let index = if cargo.use_sparse_registries_for_crates_io()? && index_url == "https://github.com/rust-lang/crates.io-index" { - CrateIndexLookup::Http(crates_index::SparseIndex::from_url("sparse+https://index.crates.io/")?) + let index = if cargo.use_sparse_registries_for_crates_io()? + && index_url == utils::CRATES_IO_INDEX_URL + { + CrateIndexLookup::Http(crates_index::SparseIndex::from_url( + "sparse+https://index.crates.io/", + )?) } else if index_url.starts_with("sparse+https://") { CrateIndexLookup::Http(crates_index::SparseIndex::from_url(index_url)?) } else { let index = { // Load the index for the current url - let index = crates_index::Index::from_url(index_url) - .with_context(|| format!("Failed to load index for url: {index_url}"))?; + let index = + crates_index::Index::from_url(index_url).with_context(|| { + format!("Failed to load index for url: {index_url}") + })?; // Ensure each index has a valid index config index.index_config().with_context(|| { diff --git a/crate_universe/src/splicing/cargo_config.rs b/crate_universe/src/splicing/cargo_config.rs index ed1f289939..09f715b596 100644 --- a/crate_universe/src/splicing/cargo_config.rs +++ b/crate_universe/src/splicing/cargo_config.rs @@ -5,7 +5,8 @@ use std::fs; use std::path::Path; use std::str::FromStr; -use anyhow::Result; +use crate::utils; +use anyhow::{bail, Result}; use serde::{Deserialize, Serialize}; /// The [`[registry]`](https://doc.rust-lang.org/cargo/reference/config.html#registry) @@ -34,7 +35,7 @@ pub struct Source { /// This is the default registry url per what's defined by Cargo. fn default_registry_url() -> String { - "https://github.com/rust-lang/crates.io-index".to_owned() + utils::CRATES_IO_INDEX_URL.to_owned() } #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] @@ -124,7 +125,7 @@ impl CargoConfig { pub fn get_source_from_url(&self, url: &str) -> Option<&Source> { if let Some(found) = self.source.values().find(|v| v.registry == url) { Some(found) - } else if url == "https://github.com/rust-lang/crates.io-index" { + } else if url == utils::CRATES_IO_INDEX_URL { self.source.get("crates-io") } else { None @@ -140,6 +141,22 @@ impl CargoConfig { None } } + + pub fn resolve_replacement_url<'a>(&'a self, url: &'a str) -> Result<&'a str> { + if let Some(source) = self.get_source_from_url(url) { + if let Some(replace_with) = &source.replace_with { + if let Some(replacement) = self.get_registry_index_url_by_name(replace_with) { + Ok(replacement) + } else { + bail!("Tried to replace registry {} with registry named {} but didn't have metadata about the replacement", url, replace_with); + } + } else { + Ok(url) + } + } else { + Ok(url) + } + } } #[cfg(test)] @@ -247,4 +264,134 @@ mod test { Some("https://artprod.mycompany/artifactory/git/cargo-remote.git"), ); } + + #[test] + fn registry_settings_get_source_from_url() { + let temp_dir = tempfile::tempdir().unwrap(); + let config = temp_dir.as_ref().join("config.toml"); + + fs::write( + &config, + textwrap::dedent( + r##" + [source.some-mirror] + registry = "https://artmirror.mycompany/artifactory/cargo-mirror.git" + "##, + ), + ) + .unwrap(); + + let config = CargoConfig::try_from_path(&config).unwrap(); + assert_eq!( + config + .get_source_from_url("https://artmirror.mycompany/artifactory/cargo-mirror.git") + .map(|s| s.registry.as_str()), + Some("https://artmirror.mycompany/artifactory/cargo-mirror.git"), + ); + } + + #[test] + fn resolve_replacement_url_no_replacement() { + let temp_dir = tempfile::tempdir().unwrap(); + let config = temp_dir.as_ref().join("config.toml"); + + fs::write(&config, "").unwrap(); + + let config = CargoConfig::try_from_path(&config).unwrap(); + + assert_eq!( + config + .resolve_replacement_url(utils::CRATES_IO_INDEX_URL) + .unwrap(), + utils::CRATES_IO_INDEX_URL + ); + assert_eq!( + config + .resolve_replacement_url("https://artmirror.mycompany/artifactory/cargo-mirror.git") + .unwrap(), + "https://artmirror.mycompany/artifactory/cargo-mirror.git" + ); + } + + #[test] + fn resolve_replacement_url_registry() { + let temp_dir = tempfile::tempdir().unwrap(); + let config = temp_dir.as_ref().join("config.toml"); + + fs::write(&config, textwrap::dedent( + r##" + [registries] + art-crates-remote = { index = "https://artprod.mycompany/artifactory/git/cargo-remote.git" } + + [source.crates-io] + replace-with = "some-mirror" + + [source.some-mirror] + registry = "https://artmirror.mycompany/artifactory/cargo-mirror.git" + "##, + )).unwrap(); + + let config = CargoConfig::try_from_path(&config).unwrap(); + assert_eq!( + config + .resolve_replacement_url(utils::CRATES_IO_INDEX_URL) + .unwrap(), + "https://artmirror.mycompany/artifactory/cargo-mirror.git" + ); + assert_eq!( + config + .resolve_replacement_url("https://artmirror.mycompany/artifactory/cargo-mirror.git") + .unwrap(), + "https://artmirror.mycompany/artifactory/cargo-mirror.git" + ); + assert_eq!( + config + .resolve_replacement_url( + "https://artprod.mycompany/artifactory/git/cargo-remote.git" + ) + .unwrap(), + "https://artprod.mycompany/artifactory/git/cargo-remote.git" + ); + } + + #[test] + fn resolve_replacement_url_source() { + let temp_dir = tempfile::tempdir().unwrap(); + let config = temp_dir.as_ref().join("config.toml"); + + fs::write(&config, textwrap::dedent( + r##" + [registries] + art-crates-remote = { index = "https://artprod.mycompany/artifactory/git/cargo-remote.git" } + + [source.crates-io] + replace-with = "art-crates-remote" + + [source.some-mirror] + registry = "https://artmirror.mycompany/artifactory/cargo-mirror.git" + "##, + )).unwrap(); + + let config = CargoConfig::try_from_path(&config).unwrap(); + assert_eq!( + config + .resolve_replacement_url(utils::CRATES_IO_INDEX_URL) + .unwrap(), + "https://artprod.mycompany/artifactory/git/cargo-remote.git" + ); + assert_eq!( + config + .resolve_replacement_url("https://artmirror.mycompany/artifactory/cargo-mirror.git") + .unwrap(), + "https://artmirror.mycompany/artifactory/cargo-mirror.git" + ); + assert_eq!( + config + .resolve_replacement_url( + "https://artprod.mycompany/artifactory/git/cargo-remote.git" + ) + .unwrap(), + "https://artprod.mycompany/artifactory/git/cargo-remote.git" + ); + } } diff --git a/crate_universe/src/splicing/crate_index_lookup.rs b/crate_universe/src/splicing/crate_index_lookup.rs index 735c529103..3ba4fdad45 100644 --- a/crate_universe/src/splicing/crate_index_lookup.rs +++ b/crate_universe/src/splicing/crate_index_lookup.rs @@ -49,3 +49,105 @@ impl CrateIndexLookup { } } } + +#[cfg(test)] +mod test { + use crate::splicing::crate_index_lookup::CrateIndexLookup; + use semver::Version; + use std::ffi::OsString; + + // TODO: Avoid global state (env vars) in these tests. + // TODO: These should be separate tests methods but they have conflicting state. + + #[test] + fn sparse_index() { + let runfiles = runfiles::Runfiles::create().unwrap(); + { + let _e = EnvVarResetter::set( + "CARGO_HOME", + runfiles.rlocation( + "rules_rust/crate_universe/test_data/crate_indexes/lazy_static/cargo_home", + ), + ); + + let index = CrateIndexLookup::Http( + crates_index::SparseIndex::from_url("sparse+https://index.crates.io/").unwrap(), + ); + let source_info = index + .get_source_info(&cargo_lock::Package { + name: "lazy_static".parse().unwrap(), + version: Version::parse("1.4.0").unwrap(), + source: None, + checksum: None, + dependencies: Vec::new(), + replace: None, + }) + .unwrap() + .unwrap(); + assert_eq!( + source_info.url, + "https://crates.io/api/v1/crates/lazy_static/1.4.0/download" + ); + assert_eq!( + source_info.sha256, + "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + ); + } + { + let _e = EnvVarResetter::set("CARGO_HOME", runfiles.rlocation("rules_rust/crate_universe/test_data/crate_indexes/rewritten_lazy_static/cargo_home")); + + let index = CrateIndexLookup::Http( + crates_index::SparseIndex::from_url("sparse+https://index.crates.io/").unwrap(), + ); + let source_info = index + .get_source_info(&cargo_lock::Package { + name: "lazy_static".parse().unwrap(), + version: Version::parse("1.4.0").unwrap(), + source: None, + checksum: None, + dependencies: Vec::new(), + replace: None, + }) + .unwrap() + .unwrap(); + assert_eq!( + source_info.url, + "https://some-mirror.com/api/v1/crates/lazy_static/1.4.0/download" + ); + assert_eq!( + source_info.sha256, + "fffffffffbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + ); + } + } + + struct EnvVarResetter { + key: OsString, + value: Option, + } + + impl EnvVarResetter { + fn set, V: Into>(key: K, value: V) -> EnvVarResetter { + let key = key.into(); + let value = value.into(); + let old_value = std::env::var_os(&key); + + std::env::set_var(&key, value); + + EnvVarResetter { + key, + value: old_value, + } + } + } + + impl Drop for EnvVarResetter { + fn drop(&mut self) { + if let Some(old_value) = self.value.as_ref() { + std::env::set_var(&self.key, old_value); + } else { + std::env::remove_var(&self.key); + } + } + } +} diff --git a/crate_universe/src/utils.rs b/crate_universe/src/utils.rs index 8953c819a1..c7f12c2258 100644 --- a/crate_universe/src/utils.rs +++ b/crate_universe/src/utils.rs @@ -2,6 +2,8 @@ pub mod starlark; +pub const CRATES_IO_INDEX_URL: &str = "https://github.com/rust-lang/crates.io-index"; + /// Convert a string into a valid crate module name by applying transforms to invalid characters pub fn sanitize_module_name(name: &str) -> String { name.replace('-', "_") diff --git a/crate_universe/test_data/crate_indexes/lazy_static/cargo_home/registry/index/index.crates.io-6f17d22bba15001f/.cache/la/zy/lazy_static b/crate_universe/test_data/crate_indexes/lazy_static/cargo_home/registry/index/index.crates.io-6f17d22bba15001f/.cache/la/zy/lazy_static new file mode 100644 index 0000000000000000000000000000000000000000..2fb8101cac6f263cae84d31b1aa6236ec5e1b6ca GIT binary patch literal 9840 zcmd5?U2_}B4OAt6B*kZiH<-bE>5t^$Dsf>37@x{AWZy_qe#b2S2KP!jC`y`*PJk+&_H8Z=zFU;S?!k>*RZI z7O5paS`RGV5r?XpYHXTaY1(@3R@tW>ea;#G=`h(o+!_AkFOQe2&$9Voy1E*2b8nBk z=x9N7bUHeZcdyxn!KwP}D0R`%iZk()2O{rDgKIf^p)fQWbt58fL6uru5}jWVoj)0! zZ+Ljn;d>d{JbR9#@F+qygjw--_8~==ogsza7UR~JNAPezxMxiS!26V5;vjy#-&o`dI>28#>MNfHT>MW()0DIR@m5`FU1 zE6?VRmXHOAKp9~=SEgOfqSVxwT?ckEX+`g9NIYs=R$E#` z7pM8ceCdRTcaRs3S3ry4q~Q{PV`N5?h&IIU)6~q>5h@RaX6r5bsEay-mJkM=W{iU4 zWlo%`Z=7^!2kg4&O0wK*D4H4?Ca@;2*~gZu6&y`uvU>G-5w6e@%AnJnkqR_G4+a=< zGP=vCu21F%2u>WATsfs)8?Fi>ZRP_~8j_dLM~k-T{4F63I?Wn*y1A^KtLx3ES{?#Y z$Rz}h1uUlMj^au%Tpq&-WQ!Y3Az%{6aD*+Z&Gq|Vwl|+AMf45jvV=nDG>1fBixM@* z4j~|2b!0)!p#c&Xq{~sOptKMiMXh+R!WwIoD5*0~0CCXE3u0YDD0G@p`k<1)yhS7q zXibnFGUi(c3bx7!&c|xhfa19-4F|>sKsg5~k(rKE;9RVJfnLQuClLrZVq_ek@X#I* zs_cEET5HkRCxiy}NOd9x-24uXEAa52pMLlC7ps%aziF^JMsY}w#L1#8>J9X==f00Wb5)Nf8W3zzn+J1-1C=6-N=?9$#ZuBjO9iM zfH=rfoHH-t66lOFGD#COYXSl`V775HNQsocgL1#~)XlSt-!(2D=bA%1V#xM^bYxKQl*r-*hY(_bI#4Wy&IwMyQNzCe)XkHfUmtMY8B@m)#670a zFoR)UbSb9TIdfK#3}uxuNVtOh5{_DqVT6i=RqNRx|LZYdy*_pG-0Rl|)O*I%VZtH| zJf3=n?E@haDkUO#69BxSV9}{vq4uH-YL+l$YUwfw)qu%%LY+NB;XPhW6Z4a%Umr}G zXH1=hU>VL6yo-*ZA*Y-jYtC?;n2N!fa|2gOQ21$1U_@au!cYWJ2dTpP=Bb+>gZ=to z3_fG(Lc=V^Q5q;!6>=Re0em09po{sHdc-v0U=y_Jd=-r);*SFM0=!00@0+J?en$7} zgBksdsRL|9I3_~?bnxXWVM0hG85+f#IkYC^3pk{Uj}9J6fkE7x%Z!W6dpUq zs|u$&K`5yObMUf3_UH*;2);lPm`#R5KT5~tV~#q=-6X*eewkT)&m=}7( zsvvTL@O^&RaYTQBL$SHpH_@qs@c3c{gZ0qtz`eT%{c)^{4<2^*j{Oq~i$xB+^PP-D z%?3t0Pab7UQAhB+C7?>ct7<@OI+cJ1N%MxQx$9h=V5urS2*3Nux_VJ{nDQzlniK;7UH!#sge7XqZP zqNL^<>`rhA43%Sr@TUDR@6pownMeC=rS(+{9x(FicGGQ=Eb|uVZ;4*&fl7cG1(k(O zXpE&|;s}B(WT9Y%hXtt_KA?c7f~5-vk4VWXj8J2&QPpGu%6**QC+1z-zi>WZbVFFs z4Y+4PH^kjt`|$n;`tjuuzRi+>Z6w(2P;!ssnjZL#FO5(`RF{pd(AEkPSs~esf&(QoQbDA5P44;T+7)Dg`v@?8xe5}s?_3===_4{{A_f- z;o(7t?`3H7>^Y9YqX^j$X2rkRhZJ2P>d8j(G4!Ii%jMLfTM}JZ5M7v!E)rYwE*BW>4b-OkQa_uK#Sm{;SzvjWJZ&SHpK7K)XdcpDi4Ha>n-}Ii#mgr5C+XNM#1qi zCr;HjPCB#$c3pHOS?)CyO$`kbSd-W6V@uTvjwUi$z52WeS7-@k&^%|P0u9iE0Y;pR z?lP+DllcLH6UQZ2PN~<1tAa?I`GAy$p3qaib{&OyU@huw}Kme*g3K=JTY8zM))}PzcR)NCdVhQFH7N z0@77S7StRXAaOyu9JLBc3&By;itkleV~r9eb>;~m4tjY(tV;-m<{70ADhbS6MB;$f z1nD7TzIC8rtBl}$tVRteo~zPuU~B-CbC435=|}~hiq$XBtGH(tfq)}M#sLZs?E#_6 z-Z!eX7L9#EXkd?2Ct}b&;2CC~z$j-legzo(SXTY|G#KHYeWXt^{QlGR2AA*bKM>jf zcI)Xp@5?zU@Y_$@>)Vaotgg0q4>qCgkcXS?yZv2XuW#k`|HCsaS>KJIGX941S z?0e&KHGP|Rn_J8kc=%6GpMCvcHQPLXeHtfm&z~Z7Nt!tbP0O)B4!2$)?KGwgo`}`q z^l%w%FnJ7NIBG3{D_VA=6aoqzAF31Cx_rvtH*m+VPeVBF`BS8BWXq7`xw`OHRG?s^^6UheWgaKHDijT zktYS`nnOEc$o7GBWKi&w$l?Wu5MqEjP%MSc2~NOK!@mC1ohCcKKH$1Drj8+qdrYBW z2E)AQQcSUP=By$a$|_@!a0U4#9JL(72o(vd*0VwW*JHkVed0Pu!_MW=R!+KV!%S;CO1rOPB#118%Eb@mK}@9}DyI6Z0l^}(ch z#?(m&mf<|XyXY7ia?07U<_y<~sTiC&H*lo{g`egGMieF^3`G!ikSeTip1RXxuwNgH z!DmceXqd$~N&}^;LaxIlfbSz1bTPkDkC-MLY=TyuucEO;{3u{Az-t8czIp0S&**-A zFr%L_b%3o1$7BeA4!&F^ObCf2L!)>zht`CA0f%()(ZNG0Fo=6|nUV1URs80uTVNnS z6CcR;r{wk9kDsn@Y-{)1ckrbT(4qi6M9zEd;!52sZv&nGiN9nkH%uxrqn|Dpbz{5Wvy9UD=vKPQh1(O$W{|uGJ7mnIt>P&rlzZ`u#1Jz82n^Ju@Vw7zP=14drmZn{m9<+KI*YoeEWpb}t4L1keR z8e^%LID+5`StwZHVL@t!7bxJVVCjOvBT}*oBh(mcR5h7^av$gSiFw!dFPzU8-4GUZ z1MXSS4RLqZKD^&U*fcnKz7C)9mH^)RfV>&3cyGZMD?VGn76rqtVhqN94Lpd0i1l;F z&JtiFyQR%Oz%R$P@xqkxf|TK&bEa$$AbQ*X4x0aHo9!WdgdOSbbMd@bzXd+jet8BS zdSE3bfY_g@Sztu)>;472D5IsSJ)V$&AY6P$2 zV_&MXVm;o1FvV^XZBqsBkZ|pTVHR0rNXy9%sXkSV3#iQyZ2Ix$5Z=v_fo&#w?|8-V KBO}8J#{M64egq2u literal 0 HcmV?d00001 diff --git a/crate_universe/test_data/crate_indexes/rewritten_lazy_static/cargo_home/registry/index/index.crates.io-6f17d22bba15001f/config.json b/crate_universe/test_data/crate_indexes/rewritten_lazy_static/cargo_home/registry/index/index.crates.io-6f17d22bba15001f/config.json new file mode 100644 index 0000000000..d623d80e62 --- /dev/null +++ b/crate_universe/test_data/crate_indexes/rewritten_lazy_static/cargo_home/registry/index/index.crates.io-6f17d22bba15001f/config.json @@ -0,0 +1,4 @@ +{ + "dl": "https://some-mirror.com/api/v1/crates", + "api": "https://crates.io" +} From dd33a7826274a9af73b1204472ca3159ca2c2a26 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 14 Mar 2023 22:32:24 +0000 Subject: [PATCH 3/3] Compare major+minor not against 1.68.0 Nightlies are technically not >= their released version --- crate_universe/src/metadata.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crate_universe/src/metadata.rs b/crate_universe/src/metadata.rs index 6f44e01823..c84bfbb8c1 100644 --- a/crate_universe/src/metadata.rs +++ b/crate_universe/src/metadata.rs @@ -136,8 +136,7 @@ impl Cargo { let version_str = full_version.split(' ').nth(1); if let Some(version_str) = version_str { let version = Version::parse(version_str).context("Failed to parse cargo version")?; - return Ok(version - >= Version::parse("1.68.0").expect("1.68.0 is a known-to-parse semver version")); + return Ok(version.major >= 1 && version.minor >= 68); } bail!("Couldn't parse cargo version"); }