From 5c160b544c44b538abdee1c932c1d338febd9a0c Mon Sep 17 00:00:00 2001 From: duriantaco Date: Mon, 6 Apr 2026 16:06:42 +0800 Subject: [PATCH 1/8] Add fyn torch doctor --- crates/fyn-cli/src/lib.rs | 31 ++ crates/fyn/src/commands/mod.rs | 2 + crates/fyn/src/commands/torch/doctor.rs | 462 ++++++++++++++++++++++++ crates/fyn/src/commands/torch/mod.rs | 1 + crates/fyn/src/lib.rs | 16 +- crates/fyn/tests/it/help.rs | 31 ++ crates/fyn/tests/it/main.rs | 3 + crates/fyn/tests/it/torch.rs | 96 +++++ docs/guides/integration/pytorch.md | 32 ++ 9 files changed, 673 insertions(+), 1 deletion(-) create mode 100644 crates/fyn/src/commands/torch/doctor.rs create mode 100644 crates/fyn/src/commands/torch/mod.rs create mode 100644 crates/fyn/tests/it/torch.rs diff --git a/crates/fyn-cli/src/lib.rs b/crates/fyn-cli/src/lib.rs index 955dd1c45..b130b24cd 100644 --- a/crates/fyn-cli/src/lib.rs +++ b/crates/fyn-cli/src/lib.rs @@ -484,6 +484,12 @@ pub enum Commands { after_long_help = "" )] Pip(PipNamespace), + /// Diagnose PyTorch backend selection for the current machine. + #[command( + after_help = "Use `fyn help torch` for more details.", + after_long_help = "" + )] + Torch(TorchNamespace), /// Create a virtual environment. /// /// By default, creates a virtual environment named `.venv` in the working @@ -617,6 +623,13 @@ pub struct StatusArgs { pub json: bool, } +#[derive(Args, Debug, Clone)] +pub struct TorchDoctorArgs { + /// Display the diagnosis in JSON format. + #[arg(long)] + pub json: bool, +} + #[derive(Args)] #[command(group = clap::ArgGroup::new("operation"))] pub struct VersionArgs { @@ -7291,6 +7304,24 @@ pub struct ToolUpgradeArgs { pub build: BuildOptionsArgs, } +#[derive(Args)] +pub struct TorchNamespace { + #[command(subcommand)] + pub command: TorchCommand, +} + +#[derive(Subcommand)] +pub enum TorchCommand { + /// Inspect the current machine and recommend a PyTorch backend. + /// + /// Detects the current accelerator, recommends the corresponding `--torch-backend` value, and + /// prints the exact `fyn pip install` command to run next. + /// + /// This command only reports information. It never installs packages or modifies project + /// files. + Doctor(TorchDoctorArgs), +} + #[derive(Args)] pub struct PythonNamespace { #[command(subcommand)] diff --git a/crates/fyn/src/commands/mod.rs b/crates/fyn/src/commands/mod.rs index 3b9c3b5cd..23e80f924 100644 --- a/crates/fyn/src/commands/mod.rs +++ b/crates/fyn/src/commands/mod.rs @@ -73,6 +73,7 @@ pub(crate) use tool::run::run as tool_run; pub(crate) use tool::uninstall::uninstall as tool_uninstall; pub(crate) use tool::update_shell::update_shell as tool_update_shell; pub(crate) use tool::upgrade::upgrade as tool_upgrade; +pub(crate) use torch::doctor::doctor as torch_doctor; pub(crate) use venv::venv; pub(crate) use workspace::dir::dir; pub(crate) use workspace::list::list; @@ -100,6 +101,7 @@ mod self_update; mod shell; mod status; mod tool; +mod torch; mod venv; mod workspace; diff --git a/crates/fyn/src/commands/torch/doctor.rs b/crates/fyn/src/commands/torch/doctor.rs new file mode 100644 index 000000000..5fdc7ee09 --- /dev/null +++ b/crates/fyn/src/commands/torch/doctor.rs @@ -0,0 +1,462 @@ +use std::fmt::Write; +use std::path::Path; +use std::process::Command; +use std::str::FromStr; + +use anyhow::Context; +use fyn_cache::Cache; +use fyn_fs::Simplified; +use fyn_platform::Libc; +use fyn_platform_tags::{Arch, Os, Platform}; +use fyn_preview::Preview; +use fyn_python::{EnvironmentPreference, PythonEnvironment, PythonPreference, PythonRequest}; +use fyn_static::EnvVars; +use fyn_torch::{Accelerator, TorchBackend, TorchSource, TorchStrategy}; +use owo_colors::OwoColorize; +use serde::{Deserialize, Serialize}; + +use crate::commands::ExitStatus; +use crate::printer::Printer; + +#[derive(Debug, Serialize)] +struct TorchDoctorEnvironment { + path: String, + python: String, + version: String, +} + +#[derive(Debug, Default, Serialize)] +struct TorchDoctorPackage { + #[serde(skip_serializing_if = "Option::is_none")] + version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + backend: Option, +} + +#[derive(Debug, Default, Serialize)] +struct TorchDoctorPackages { + torch: TorchDoctorPackage, + torchvision: TorchDoctorPackage, + torchaudio: TorchDoctorPackage, +} + +#[derive(Debug, Serialize)] +struct TorchDoctorAccelerator { + kind: &'static str, + display: String, + #[serde(skip_serializing_if = "Option::is_none")] + driver_version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + gpu_architecture: Option, +} + +#[derive(Debug, Serialize)] +struct TorchDoctorReport { + environment: Option, + platform: String, + accelerator: TorchDoctorAccelerator, + recommended_backend: String, + reason: String, + installed_packages: TorchDoctorPackages, + next_command: String, + notes: Vec, +} + +#[derive(Debug, Deserialize)] +struct PythonPackageVersions { + torch: Option, + torchvision: Option, + torchaudio: Option, +} + +pub(crate) async fn doctor( + json: bool, + python_preference: PythonPreference, + cache: &Cache, + preview: Preview, + printer: Printer, +) -> anyhow::Result { + let environment = PythonEnvironment::find( + &PythonRequest::default(), + EnvironmentPreference::from_system_flag(false, false), + python_preference, + cache, + preview, + ) + .ok(); + + let platform = environment + .as_ref() + .map(|environment| environment.interpreter().platform().clone()) + .unwrap_or_else(host_platform); + let accelerator = Accelerator::detect()?; + let strategy = strategy_from_accelerator(&platform.os().clone(), accelerator.clone()); + let recommended_backend = recommended_backend(&strategy)?; + let reason = recommendation_reason(&platform, accelerator.as_ref(), &recommended_backend); + let next_command = format!( + "fyn pip install torch torchvision torchaudio --torch-backend={recommended_backend}" + ); + + let mut notes = Vec::new(); + if environment.is_none() { + notes.push( + "No Python environment was discovered; the recommendation is based on the host machine." + .to_string(), + ); + } + if std::env::var_os(EnvVars::UV_CUDA_DRIVER_VERSION).is_some() { + notes.push("Accelerator detection was overridden by `UV_CUDA_DRIVER_VERSION`.".to_string()); + } + if std::env::var_os(EnvVars::UV_AMD_GPU_ARCHITECTURE).is_some() { + notes + .push("Accelerator detection was overridden by `UV_AMD_GPU_ARCHITECTURE`.".to_string()); + } + + let installed_packages = if let Some(environment) = environment.as_ref() { + match inspect_installed_packages(environment.python_executable()) { + Ok(packages) => { + add_backend_mismatch_notes(&packages, &recommended_backend, &mut notes); + packages + } + Err(err) => { + notes.push(format!( + "Could not inspect installed PyTorch packages: {}.", + err.to_string().trim() + )); + TorchDoctorPackages::default() + } + } + } else { + TorchDoctorPackages::default() + }; + + notes.push("`fyn torch doctor` only reports the recommended backend.".to_string()); + notes.push("Recreate the environment when switching GPU or backend families.".to_string()); + + let report = TorchDoctorReport { + environment: environment + .as_ref() + .map(|environment| TorchDoctorEnvironment { + path: environment.root().simplified_display().to_string(), + python: environment + .python_executable() + .simplified_display() + .to_string(), + version: environment.interpreter().python_full_version().to_string(), + }), + platform: platform.pretty(), + accelerator: accelerator_report(accelerator.as_ref()), + recommended_backend: recommended_backend.clone(), + reason, + installed_packages, + next_command, + notes, + }; + + if json { + writeln!( + printer.stdout(), + "{}", + serde_json::to_string_pretty(&report)? + )?; + return Ok(ExitStatus::Success); + } + + writeln!(printer.stdout(), "{}", "PyTorch doctor".bold())?; + if let Some(environment) = report.environment.as_ref() { + writeln!(printer.stdout(), "environment: {}", environment.path.cyan())?; + writeln!( + printer.stdout(), + "python: {} ({})", + environment.python.cyan(), + environment.version.cyan() + )?; + } else { + writeln!(printer.stdout(), "environment: {}", "not found".dimmed())?; + writeln!(printer.stdout(), "python: {}", "not found".dimmed())?; + } + writeln!(printer.stdout(), "platform: {}", report.platform.cyan())?; + writeln!( + printer.stdout(), + "accelerator: {}", + report.accelerator.display.cyan() + )?; + writeln!( + printer.stdout(), + "recommended backend: {}", + report.recommended_backend.cyan() + )?; + writeln!(printer.stdout(), "reason: {}", report.reason)?; + writeln!(printer.stdout())?; + writeln!(printer.stdout(), "installed packages:")?; + write_package_line(printer, "torch", &report.installed_packages.torch)?; + write_package_line( + printer, + "torchvision", + &report.installed_packages.torchvision, + )?; + write_package_line(printer, "torchaudio", &report.installed_packages.torchaudio)?; + writeln!(printer.stdout())?; + writeln!(printer.stdout(), "next command:")?; + writeln!(printer.stdout(), " {}", report.next_command.cyan())?; + if !report.notes.is_empty() { + writeln!(printer.stdout())?; + writeln!(printer.stdout(), "notes:")?; + for note in &report.notes { + writeln!(printer.stdout(), "- {note}")?; + } + } + + Ok(ExitStatus::Success) +} + +fn strategy_from_accelerator(os: &Os, accelerator: Option) -> TorchStrategy { + match accelerator { + Some(Accelerator::Cuda { driver_version }) => TorchStrategy::Cuda { + os: os.clone(), + driver_version, + source: TorchSource::PyTorch, + }, + Some(Accelerator::Amd { gpu_architecture }) => TorchStrategy::Amd { + os: os.clone(), + gpu_architecture, + source: TorchSource::PyTorch, + }, + Some(Accelerator::Xpu) => TorchStrategy::Xpu { + os: os.clone(), + source: TorchSource::PyTorch, + }, + None => TorchStrategy::Backend { + backend: TorchBackend::Cpu, + source: TorchSource::PyTorch, + }, + } +} + +fn recommended_backend(strategy: &TorchStrategy) -> anyhow::Result { + let index = strategy + .index_urls() + .next() + .context("No compatible PyTorch backend was found for this machine")?; + let backend = index + .url() + .path_segments() + .and_then(|segments| segments.filter(|segment| !segment.is_empty()).next_back()) + .context("Failed to extract the recommended PyTorch backend from the resolved index URL")?; + Ok(backend.to_string()) +} + +fn recommendation_reason( + platform: &Platform, + accelerator: Option<&Accelerator>, + recommended_backend: &str, +) -> String { + match accelerator { + Some(Accelerator::Cuda { driver_version }) if recommended_backend == "cpu" => format!( + "Detected NVIDIA driver {driver_version}, but no compatible CUDA backend is available for {}.", + platform.pretty() + ), + Some(Accelerator::Cuda { driver_version }) => format!( + "Detected NVIDIA driver {driver_version}; selected the highest compatible CUDA backend for {}.", + platform.pretty() + ), + Some(Accelerator::Amd { gpu_architecture }) if recommended_backend == "cpu" => format!( + "Detected AMD GPU architecture {gpu_architecture}, but no matching ROCm backend is available for {}.", + platform.pretty() + ), + Some(Accelerator::Amd { gpu_architecture }) => format!( + "Detected AMD GPU architecture {gpu_architecture}; selected the matching ROCm backend." + ), + Some(Accelerator::Xpu) if recommended_backend == "xpu" => { + "Detected an Intel GPU; selected the XPU backend.".to_string() + } + Some(Accelerator::Xpu) => format!( + "Detected an Intel GPU, but XPU wheels are not available for {}.", + platform.pretty() + ), + None => "No supported GPU accelerator was detected; using CPU wheels.".to_string(), + } +} + +fn accelerator_report(accelerator: Option<&Accelerator>) -> TorchDoctorAccelerator { + match accelerator { + Some(Accelerator::Cuda { driver_version }) => TorchDoctorAccelerator { + kind: "cuda", + display: format!("CUDA {driver_version}"), + driver_version: Some(driver_version.to_string()), + gpu_architecture: None, + }, + Some(Accelerator::Amd { gpu_architecture }) => TorchDoctorAccelerator { + kind: "amd", + display: format!("AMD {gpu_architecture}"), + driver_version: None, + gpu_architecture: Some(gpu_architecture.to_string()), + }, + Some(Accelerator::Xpu) => TorchDoctorAccelerator { + kind: "xpu", + display: "Intel GPU (XPU)".to_string(), + driver_version: None, + gpu_architecture: None, + }, + None => TorchDoctorAccelerator { + kind: "none", + display: "none detected".to_string(), + driver_version: None, + gpu_architecture: None, + }, + } +} + +fn inspect_installed_packages(python: &Path) -> anyhow::Result { + let script = r#" +import importlib.metadata as metadata +import json + +packages = {} +for name in ("torch", "torchvision", "torchaudio"): + try: + packages[name] = metadata.version(name) + except metadata.PackageNotFoundError: + packages[name] = None + +print(json.dumps(packages)) +"#; + + let output = Command::new(python) + .arg("-I") + .arg("-c") + .arg(script) + .output() + .with_context(|| { + format!( + "failed to run `{}` to inspect installed packages", + python.display() + ) + })?; + + if !output.status.success() { + anyhow::bail!(String::from_utf8_lossy(&output.stderr).trim().to_string()); + } + + let stdout = String::from_utf8(output.stdout)?; + let versions: PythonPackageVersions = serde_json::from_str(stdout.trim())?; + Ok(TorchDoctorPackages { + torch: package_from_version(versions.torch), + torchvision: package_from_version(versions.torchvision), + torchaudio: package_from_version(versions.torchaudio), + }) +} + +fn package_from_version(version: Option) -> TorchDoctorPackage { + let backend = version.as_deref().and_then(version_backend); + TorchDoctorPackage { version, backend } +} + +fn version_backend(version: &str) -> Option { + let (_, backend) = version.split_once('+')?; + TorchBackend::from_str(backend).ok()?; + Some(backend.to_string()) +} + +fn add_backend_mismatch_notes( + packages: &TorchDoctorPackages, + recommended_backend: &str, + notes: &mut Vec, +) { + let backends = [ + ("torch", packages.torch.backend.as_deref()), + ("torchvision", packages.torchvision.backend.as_deref()), + ("torchaudio", packages.torchaudio.backend.as_deref()), + ]; + + let mut detected = backends + .into_iter() + .filter_map(|(name, backend)| backend.map(|backend| (name, backend))) + .collect::>(); + + if detected.is_empty() { + return; + } + + detected.sort_unstable_by_key(|(name, _)| *name); + + let distinct = detected + .iter() + .map(|(_, backend)| *backend) + .collect::>(); + if distinct.len() > 1 { + notes.push(format!( + "Installed PyTorch packages report multiple backend suffixes: {}.", + detected + .iter() + .map(|(name, backend)| format!("{name}={backend}")) + .collect::>() + .join(", ") + )); + } else if let Some((name, backend)) = detected.first() + && *backend != recommended_backend + { + notes.push(format!( + "Installed {name} reports backend `{backend}`, but this machine recommends `{recommended_backend}`." + )); + } +} + +fn write_package_line( + printer: Printer, + name: &str, + package: &TorchDoctorPackage, +) -> Result<(), std::fmt::Error> { + match package.version.as_deref() { + Some(version) => writeln!(printer.stdout(), "{name}: {}", version.cyan()), + None => writeln!(printer.stdout(), "{name}: {}", "not installed".dimmed()), + } +} + +fn host_platform() -> Platform { + let arch = Arch::from_str(std::env::consts::ARCH).unwrap_or(Arch::X86_64); + let os = match std::env::consts::OS { + "linux" => match Libc::from_env() { + Ok(libc) if libc.is_musl() => Os::Musllinux { major: 1, minor: 2 }, + _ => Os::Manylinux { + major: 2, + minor: 28, + }, + }, + "macos" => Os::Macos { + major: 13, + minor: 0, + }, + "windows" => Os::Windows, + "freebsd" => Os::FreeBsd { + release: "unknown".to_string(), + }, + "netbsd" => Os::NetBsd { + release: "unknown".to_string(), + }, + "openbsd" => Os::OpenBsd { + release: "unknown".to_string(), + }, + "dragonfly" => Os::Dragonfly { + release: "unknown".to_string(), + }, + "illumos" => Os::Illumos { + release: "unknown".to_string(), + arch: arch.to_string(), + }, + "haiku" => Os::Haiku { + release: "unknown".to_string(), + }, + "android" => Os::Android { api_level: 0 }, + "ios" => Os::Ios { + major: 0, + minor: 0, + simulator: false, + }, + _ => Os::Manylinux { + major: 2, + minor: 28, + }, + }; + Platform::new(os, arch) +} diff --git a/crates/fyn/src/commands/torch/mod.rs b/crates/fyn/src/commands/torch/mod.rs new file mode 100644 index 000000000..d5d6513c9 --- /dev/null +++ b/crates/fyn/src/commands/torch/mod.rs @@ -0,0 +1 @@ +pub(crate) mod doctor; diff --git a/crates/fyn/src/lib.rs b/crates/fyn/src/lib.rs index 0e992d395..d91bb1a1d 100644 --- a/crates/fyn/src/lib.rs +++ b/crates/fyn/src/lib.rs @@ -31,7 +31,8 @@ use fyn_cli::{ AuthCommand, AuthHelperCommand, AuthNamespace, BuildBackendCommand, CacheCommand, CacheNamespace, Cli, Commands, PipCommand, PipNamespace, ProjectCommand, PythonCommand, PythonNamespace, SelfCommand, SelfNamespace, StatusArgs, ToolCommand, ToolNamespace, - TopLevelArgs, WorkspaceCommand, WorkspaceNamespace, compat::CompatArgs, + TopLevelArgs, TorchCommand, TorchDoctorArgs, TorchNamespace, WorkspaceCommand, + WorkspaceNamespace, compat::CompatArgs, }; use fyn_client::BaseClientBuilder; use fyn_configuration::min_stack_size; @@ -1528,6 +1529,19 @@ async fn run(mut cli: Cli) -> Result { ) .await } + Commands::Torch(TorchNamespace { + command: TorchCommand::Doctor(TorchDoctorArgs { json }), + }) => { + let cache = cache.init().await?; + commands::torch_doctor( + json, + globals.python_preference, + &cache, + globals.preview, + printer, + ) + .await + } Commands::Build(args) => { // Resolve the settings from the command-line arguments and workspace configuration. let args = settings::BuildSettings::resolve(args, filesystem, environment); diff --git a/crates/fyn/tests/it/help.rs b/crates/fyn/tests/it/help.rs index 0aac4d4fa..b22b8db9e 100644 --- a/crates/fyn/tests/it/help.rs +++ b/crates/fyn/tests/it/help.rs @@ -32,6 +32,7 @@ fn help() { tool Run and install commands provided by Python packages python Manage Python versions and installations pip Manage Python packages directly in an environment + torch Diagnose PyTorch backend selection for the current machine venv Create a virtual environment shell Activate the virtual environment in a new shell build Build Python packages into source distributions and wheels @@ -117,6 +118,7 @@ fn help_flag() { tool Run and install commands provided by Python packages python Manage Python versions and installations pip Manage Python packages directly in an environment + torch Diagnose PyTorch backend selection for the current machine venv Create a virtual environment shell Activate the virtual environment in a new shell build Build Python packages into source distributions and wheels @@ -200,6 +202,7 @@ fn help_short_flag() { tool Run and install commands provided by Python packages python Manage Python versions and installations pip Manage Python packages directly in an environment + torch Diagnose PyTorch backend selection for the current machine venv Create a virtual environment shell Activate the virtual environment in a new shell build Build Python packages into source distributions and wheels @@ -898,6 +901,30 @@ fn help_flag_subsubcommand() { "#); } +#[test] +fn help_torch_subcommand() { + let context = fyn_test::test_context_with_versions!(&[]); + let output = context.help().arg("torch").output().unwrap(); + assert!(output.status.success()); + + let stdout = String::from_utf8(output.stdout).unwrap(); + assert!(stdout.contains("Diagnose PyTorch backend selection for the current machine")); + assert!(stdout.contains("Usage: fyn torch [OPTIONS] ")); + assert!(stdout.contains("doctor")); +} + +#[test] +fn help_torch_doctor_subcommand() { + let context = fyn_test::test_context_with_versions!(&[]); + let output = context.help().arg("torch").arg("doctor").output().unwrap(); + assert!(output.status.success()); + + let stdout = String::from_utf8(output.stdout).unwrap(); + assert!(stdout.contains("Inspect the current machine and recommend a PyTorch backend.")); + assert!(stdout.contains("Usage: fyn torch doctor [OPTIONS]")); + assert!(stdout.contains("--json")); +} + #[test] fn help_unknown_subcommand() { let context = fyn_test::test_context_with_versions!(&[]); @@ -925,6 +952,7 @@ fn help_unknown_subcommand() { tool python pip + torch venv shell build @@ -958,6 +986,7 @@ fn help_unknown_subcommand() { tool python pip + torch venv shell build @@ -1020,6 +1049,7 @@ fn help_with_global_option() { tool Run and install commands provided by Python packages python Manage Python versions and installations pip Manage Python packages directly in an environment + torch Diagnose PyTorch backend selection for the current machine venv Create a virtual environment shell Activate the virtual environment in a new shell build Build Python packages into source distributions and wheels @@ -1146,6 +1176,7 @@ fn help_with_no_pager() { tool Run and install commands provided by Python packages python Manage Python versions and installations pip Manage Python packages directly in an environment + torch Diagnose PyTorch backend selection for the current machine venv Create a virtual environment shell Activate the virtual environment in a new shell build Build Python packages into source distributions and wheels diff --git a/crates/fyn/tests/it/main.rs b/crates/fyn/tests/it/main.rs index c22f40a24..9893b6bb8 100644 --- a/crates/fyn/tests/it/main.rs +++ b/crates/fyn/tests/it/main.rs @@ -130,6 +130,9 @@ mod show_settings; #[cfg(feature = "test-python")] mod status; +#[cfg(feature = "test-python")] +mod torch; + #[cfg(all(feature = "test-python", feature = "test-pypi"))] mod sync; diff --git a/crates/fyn/tests/it/torch.rs b/crates/fyn/tests/it/torch.rs new file mode 100644 index 000000000..c34237bc1 --- /dev/null +++ b/crates/fyn/tests/it/torch.rs @@ -0,0 +1,96 @@ +use anyhow::Result; +use serde_json::Value; + +use fyn_static::EnvVars; + +#[test] +fn torch_doctor_json_cuda_override() -> Result<()> { + let context = fyn_test::test_context!("3.12"); + let output = context + .command() + .arg("torch") + .arg("doctor") + .arg("--json") + .env(EnvVars::UV_CUDA_DRIVER_VERSION, "525.60.13") + .env_remove(EnvVars::UV_AMD_GPU_ARCHITECTURE) + .output()?; + + assert!(output.status.success()); + + let report: Value = serde_json::from_slice(&output.stdout)?; + let environment_path = context.venv.path().display().to_string(); + assert_eq!( + report["environment"]["path"].as_str(), + Some(environment_path.as_str()) + ); + assert_eq!(report["accelerator"]["kind"].as_str(), Some("cuda")); + assert_eq!( + report["accelerator"]["driver_version"].as_str(), + Some("525.60.13") + ); + + let expected_backend = if cfg!(target_os = "linux") { + "cu129" + } else if cfg!(windows) { + "cu118" + } else { + "cpu" + }; + assert_eq!( + report["recommended_backend"].as_str(), + Some(expected_backend) + ); + let expected_command = + format!("fyn pip install torch torchvision torchaudio --torch-backend={expected_backend}"); + assert_eq!( + report["next_command"].as_str(), + Some(expected_command.as_str()) + ); + + let notes = report["notes"].as_array().expect("notes array"); + assert!(notes.iter().any(|note| { + note.as_str() == Some("Accelerator detection was overridden by `UV_CUDA_DRIVER_VERSION`.") + })); + + Ok(()) +} + +#[test] +fn torch_doctor_json_amd_override() -> Result<()> { + let context = fyn_test::test_context!("3.12"); + let output = context + .command() + .arg("torch") + .arg("doctor") + .arg("--json") + .env(EnvVars::UV_AMD_GPU_ARCHITECTURE, "gfx1100") + .env_remove(EnvVars::UV_CUDA_DRIVER_VERSION) + .output()?; + + assert!(output.status.success()); + + let report: Value = serde_json::from_slice(&output.stdout)?; + assert_eq!(report["accelerator"]["kind"].as_str(), Some("amd")); + assert_eq!( + report["accelerator"]["gpu_architecture"].as_str(), + Some("gfx1100") + ); + + let expected_backend = if cfg!(target_os = "linux") { + "rocm7.1" + } else { + "cpu" + }; + assert_eq!( + report["recommended_backend"].as_str(), + Some(expected_backend) + ); + assert!( + report["reason"] + .as_str() + .expect("reason") + .contains("AMD GPU architecture gfx1100") + ); + + Ok(()) +} diff --git a/docs/guides/integration/pytorch.md b/docs/guides/integration/pytorch.md index 206e0bfb9..50ce03a22 100644 --- a/docs/guides/integration/pytorch.md +++ b/docs/guides/integration/pytorch.md @@ -449,6 +449,38 @@ To use the same workflow with fyn, replace `pip3` with `fyn pip`: $ fyn pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu ``` +## Diagnosing the recommended backend + +If you switch between machines or GPU families, use `fyn torch doctor` to inspect the current +machine and print the recommended `--torch-backend` value without modifying `pyproject.toml`: + +```shell +$ fyn torch doctor +PyTorch doctor +environment: /workspace/.venv +python: /workspace/.venv/bin/python3 (3.12.8) +platform: manylinux_2_28_x86_64 +accelerator: CUDA 570.86.15 +recommended backend: cu130 +reason: Detected NVIDIA driver 570.86.15; selected the highest compatible CUDA backend for manylinux_2_28_x86_64. + +installed packages: +torch: not installed +torchvision: not installed +torchaudio: not installed + +next command: + fyn pip install torch torchvision torchaudio --torch-backend=cu130 + +notes: +- `fyn torch doctor` only reports the recommended backend. +- Recreate the environment when switching GPU or backend families. +``` + +If PyTorch is already installed, `fyn torch doctor` also reports the installed `torch`, +`torchvision`, and `torchaudio` versions, including any backend suffixes such as `+cu128`. Use +`--json` to consume the same report in scripts or editor tooling. + ## Automatic backend selection fyn supports automatic selection of the appropriate PyTorch index via the `--torch-backend=auto` From f2c803a762e7df16945210387bbf5412143964d1 Mon Sep 17 00:00:00 2001 From: duriantaco Date: Mon, 6 Apr 2026 16:24:36 +0800 Subject: [PATCH 2/8] Fix torch doctor clippy lint --- crates/fyn/src/commands/torch/doctor.rs | 28 ++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/crates/fyn/src/commands/torch/doctor.rs b/crates/fyn/src/commands/torch/doctor.rs index 5fdc7ee09..33731be8b 100644 --- a/crates/fyn/src/commands/torch/doctor.rs +++ b/crates/fyn/src/commands/torch/doctor.rs @@ -241,7 +241,7 @@ fn recommended_backend(strategy: &TorchStrategy) -> anyhow::Result { let backend = index .url() .path_segments() - .and_then(|segments| segments.filter(|segment| !segment.is_empty()).next_back()) + .and_then(|mut segments| segments.rfind(|segment| !segment.is_empty())) .context("Failed to extract the recommended PyTorch backend from the resolved index URL")?; Ok(backend.to_string()) } @@ -460,3 +460,29 @@ fn host_platform() -> Platform { }; Platform::new(os, arch) } + +#[cfg(test)] +mod tests { + use super::recommended_backend; + use fyn_torch::{TorchBackend, TorchSource, TorchStrategy}; + + #[test] + fn recommended_backend_extracts_trailing_segment() { + let strategy = TorchStrategy::Backend { + backend: TorchBackend::Cu130, + source: TorchSource::PyTorch, + }; + + assert_eq!(recommended_backend(&strategy).unwrap(), "cu130"); + } + + #[test] + fn recommended_backend_handles_rocm_backend_names() { + let strategy = TorchStrategy::Backend { + backend: TorchBackend::Rocm71, + source: TorchSource::PyTorch, + }; + + assert_eq!(recommended_backend(&strategy).unwrap(), "rocm7.1"); + } +} From e14bc98a24dc2c84fb94408ebac4959c28e58b14 Mon Sep 17 00:00:00 2001 From: duriantaco Date: Mon, 6 Apr 2026 19:31:43 +0800 Subject: [PATCH 3/8] docs: refresh project and command guides --- docs/getting-started/features.md | 12 +++++ docs/guides/projects.md | 93 ++++++++++++++++++++++++++++++-- docs/index.md | 45 +++++++++++++++- 3 files changed, 145 insertions(+), 5 deletions(-) diff --git a/docs/getting-started/features.md b/docs/getting-started/features.md index 5f50607c5..5351fddec 100644 --- a/docs/getting-started/features.md +++ b/docs/getting-started/features.md @@ -36,7 +36,10 @@ Creating and working on Python projects, i.e., with a `pyproject.toml`. - `fyn remove`: Remove a dependency from the project. - `fyn sync`: Sync the project's dependencies with the environment. - `fyn lock`: Create a lockfile for the project's dependencies. +- `fyn upgrade`: Upgrade all or selected project dependencies. - `fyn run`: Run a command in the project environment. +- `fyn shell`: Open a shell with the project environment activated. +- `fyn status`: Inspect the current project and environment state. - `fyn tree`: View the dependency tree for the project. - `fyn build`: Build the project into distribution archives. - `fyn publish`: Publish the project to a package index. @@ -90,6 +93,15 @@ See the documentation on [locking environments](../pip/compile.md) for details. These commands do not exactly implement the interfaces and behavior of the tools they are based on. The further you stray from common workflows, the more likely you are to encounter differences. Consult the [pip-compatibility guide](../pip/compatibility.md) for details. +## PyTorch + +Inspect the current machine and recommend the right backend for PyTorch installs: + +- `fyn torch doctor`: Recommend a `--torch-backend` value, show the next install command, and + report the currently installed PyTorch package variants. + +See the [PyTorch integration guide](../guides/integration/pytorch.md) for details. + ## Utility Managing and inspecting fyn's state, such as the cache, storage directories, or performing a diff --git a/docs/guides/projects.md b/docs/guides/projects.md index 3a7e3d293..2a5e75735 100644 --- a/docs/guides/projects.md +++ b/docs/guides/projects.md @@ -148,7 +148,22 @@ To remove a package, you can use `fyn remove`: $ fyn remove requests ``` -To upgrade a package, run `fyn lock` with the `--upgrade-package` flag: +To upgrade a package, run `fyn upgrade` with the package name: + +```console +$ fyn upgrade requests +``` + +`fyn upgrade` is a convenience command that re-locks with upgrade semantics and then syncs the +environment. If you want to preview the change first, use `--dry-run`. If you want to update the +lockfile without syncing the environment yet, use `--no-sync`. + +```console +$ fyn upgrade --dry-run +$ fyn upgrade --no-sync requests +``` + +If you want the lower-level equivalent, run `fyn lock` with the `--upgrade-package` flag: ```console $ fyn lock --upgrade-package requests @@ -160,6 +175,67 @@ version, while keeping the rest of the lockfile intact. See the documentation on [managing dependencies](../concepts/projects/dependencies.md) for more details. +## Checking project status + +Use `fyn status` to inspect what fyn sees in the current directory: + +```console +$ fyn status +current directory: /path/to/project +project directory: /path/to/project +managed project: yes +workspace root: /path/to/project +pyproject.toml: yes +fyn.lock: yes +pip-in-project: warn +environment: /path/to/project/.venv +python: /path/to/project/.venv/bin/python3 (3.12.0) +``` + +This is useful when you want to confirm whether you're inside a managed project, whether the +lockfile is present, and which environment and Python interpreter fyn is currently using. + +For automation, use `--check` to fail when obvious project checks do not pass: + +```console +$ fyn status --check +``` + +Today, `--check` reports failure if you are not inside a managed project. When you are inside a +managed project, it also fails if `pyproject.toml` or `fyn.lock` is missing from the workspace +root. + +For editor integrations, scripts, or CI tooling, use JSON output: + +```console +$ fyn status --json +``` + +## Opening an activated shell + +If you want to work inside the project environment directly, use `fyn shell`: + +```console +$ fyn shell +success: Activated virtual environment at .venv +Type exit to deactivate. +``` + +`fyn shell` spawns a new shell process with the environment activated. When selecting the +environment, it prefers: + +1. An explicit path passed to `fyn shell` +2. `VIRTUAL_ENV`, if it is set +3. The discovered project or workspace environment +4. A local `.venv` fallback + +Use `--no-project` if you want to skip project discovery and only check for `.venv` in the current +directory: + +```console +$ fyn shell --no-project +``` + ## Viewing your version The `fyn version` command can be used to read your package's version. @@ -227,8 +303,17 @@ print("hello world") $ fyn run example.py ``` -Alternatively, you can use `fyn sync` to manually update the environment then activate it before -executing a command: +Alternatively, you can use `fyn sync` to manually update the environment and then either activate it +yourself or open an activated shell with `fyn shell` before executing commands directly: + +```console +$ fyn sync +$ fyn shell +success: Activated virtual environment at .venv +Type exit to deactivate. +``` + +If you prefer manual activation, the platform-specific commands are: === "macOS and Linux" @@ -250,7 +335,7 @@ executing a command: !!! note - The virtual environment must be active to run scripts and commands in the project without `fyn run`. Virtual environment activation differs per shell and platform. + The virtual environment must be active to run scripts and commands in the project without `fyn run`. `fyn shell` is the simplest cross-shell way to do that. Manual activation still differs per shell and platform. See the documentation on [running commands and scripts](../concepts/projects/run.md) in projects for more details. diff --git a/docs/index.md b/docs/index.md index e35a0cf2b..beb878c02 100644 --- a/docs/index.md +++ b/docs/index.md @@ -32,7 +32,8 @@ An extremely fast Python package and project manager, written in Rust. - Installable without Rust or Python via prebuilt distributions or `pip`. - Supports macOS, Linux, and Windows. -fyn is an independent community fork of [uv](https://github.com/astral-sh/uv). +fyn started as an independent community fork of [uv](https://github.com/astral-sh/uv) and now has +its own commands and configuration surface. ## Installation @@ -80,6 +81,48 @@ Resolved 2 packages in 0.70ms Checked 1 package in 0.02ms ``` +### Inspecting and entering the project environment + +Use `fyn status` to see whether fyn considers the current directory a managed project, whether +`fyn.lock` is present, and which environment and Python interpreter it is using: + +```console +$ fyn status +current directory: /home/user/example +project directory: /home/user/example +managed project: yes +workspace root: /home/user/example +pyproject.toml: yes +fyn.lock: yes +pip-in-project: warn +environment: /home/user/example/.venv +python: /home/user/example/.venv/bin/python3 (3.12.0) +``` + +Use `fyn upgrade` to refresh all dependencies or only the packages you name: + +```console +$ fyn upgrade +$ fyn upgrade requests flask +``` + +Use `fyn shell` to open a new shell with the project environment activated: + +```console +$ fyn shell +success: Activated virtual environment at .venv +Type exit to deactivate. +``` + +When you need to choose a PyTorch backend for the current machine without rewriting project +metadata, use `fyn torch doctor`: + +```console +$ fyn torch doctor +PyTorch doctor +recommended backend: cu130 +``` + See the [project guide](./guides/projects.md) to get started. fyn also supports building and publishing projects, even if they're not managed with fyn. See the From 252c9ade28da75da718849bad9ea0e3a5a751390 Mon Sep 17 00:00:00 2001 From: duriantaco Date: Mon, 6 Apr 2026 19:41:39 +0800 Subject: [PATCH 4/8] docs: format markdown --- docs/getting-started/features.md | 4 ++-- docs/guides/projects.md | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/getting-started/features.md b/docs/getting-started/features.md index 5351fddec..e0107acb8 100644 --- a/docs/getting-started/features.md +++ b/docs/getting-started/features.md @@ -97,8 +97,8 @@ See the documentation on [locking environments](../pip/compile.md) for details. Inspect the current machine and recommend the right backend for PyTorch installs: -- `fyn torch doctor`: Recommend a `--torch-backend` value, show the next install command, and - report the currently installed PyTorch package variants. +- `fyn torch doctor`: Recommend a `--torch-backend` value, show the next install command, and report + the currently installed PyTorch package variants. See the [PyTorch integration guide](../guides/integration/pytorch.md) for details. diff --git a/docs/guides/projects.md b/docs/guides/projects.md index 2a5e75735..b8b5acb1d 100644 --- a/docs/guides/projects.md +++ b/docs/guides/projects.md @@ -202,8 +202,7 @@ $ fyn status --check ``` Today, `--check` reports failure if you are not inside a managed project. When you are inside a -managed project, it also fails if `pyproject.toml` or `fyn.lock` is missing from the workspace -root. +managed project, it also fails if `pyproject.toml` or `fyn.lock` is missing from the workspace root. For editor integrations, scripts, or CI tooling, use JSON output: From 65fa9be9a7c4f42ab53046fc19583c7a2c47eafd Mon Sep 17 00:00:00 2001 From: duriantaco Date: Mon, 6 Apr 2026 20:59:31 +0800 Subject: [PATCH 5/8] fix: respect newly activated extras during sync --- crates/fyn-resolver/src/lock/installable.rs | 30 ++++++--- crates/fyn/tests/it/sync.rs | 74 +++++++++++++++++++++ 2 files changed, 95 insertions(+), 9 deletions(-) diff --git a/crates/fyn-resolver/src/lock/installable.rs b/crates/fyn-resolver/src/lock/installable.rs index 859ce2808..d5c161ef7 100644 --- a/crates/fyn-resolver/src/lock/installable.rs +++ b/crates/fyn-resolver/src/lock/installable.rs @@ -16,9 +16,22 @@ use fyn_normalize::{ExtraName, GroupName, PackageName}; use fyn_platform_tags::Tags; use fyn_pypi_types::ResolverMarkerEnvironment; -use crate::lock::{HashedDist, LockErrorKind, Package, TagPolicy}; +use crate::lock::{Dependency, HashedDist, LockErrorKind, Package, TagPolicy}; use crate::{Lock, LockError}; +fn newly_activated_extras<'lock>( + dep: &'lock Dependency, + activated_extras: &[(&'lock PackageName, &'lock ExtraName)], +) -> Vec<(&'lock PackageName, &'lock ExtraName)> { + dep.extra + .iter() + .filter_map(|extra| { + let key = (&dep.package_id.name, extra); + (!activated_extras.contains(&key)).then_some(key) + }) + .collect() +} + pub trait Installable<'lock> { /// Return the root install path. fn install_path(&self) -> &'lock Path; @@ -143,10 +156,14 @@ pub trait Installable<'lock> { }) .flatten() { + let additional_activated_extras = newly_activated_extras(dep, &activated_extras); if !dep.complexified_marker.evaluate( marker_env, activated_projects.iter().copied(), - activated_extras.iter().copied(), + activated_extras + .iter() + .chain(additional_activated_extras.iter()) + .copied(), activated_groups.iter().copied(), ) { continue; @@ -370,13 +387,8 @@ pub trait Installable<'lock> { Either::Right(package.dependencies.iter()) }; for dep in deps { - let mut additional_activated_extras = vec![]; - for extra in &dep.extra { - let key = (&dep.package_id.name, extra); - if !activated_extras_set.contains(&key) { - additional_activated_extras.push(key); - } - } + let additional_activated_extras = + newly_activated_extras(dep, &activated_extras); if !dep.complexified_marker.evaluate( marker_env, activated_projects.iter().copied(), diff --git a/crates/fyn/tests/it/sync.rs b/crates/fyn/tests/it/sync.rs index aaffac35f..1af3108d5 100644 --- a/crates/fyn/tests/it/sync.rs +++ b/crates/fyn/tests/it/sync.rs @@ -4964,6 +4964,80 @@ fn sync_group_self() -> Result<()> { Ok(()) } +/// Regression test for: +#[test] +fn sync_workspace_member_group_self_conflicting_extra() -> Result<()> { + let context = fyn_test::test_context!("3.12").with_filtered_counts(); + + context.temp_dir.child("pyproject.toml").write_str( + r#" + [tool.fyn.workspace] + members = ["member"] + "#, + )?; + + let member = context.temp_dir.child("member"); + member.child("pyproject.toml").write_str( + r#" + [project] + name = "member" + version = "0.0.0" + requires-python = ">=3.12" + + [project.optional-dependencies] + cpu = ["idna>=3"] + gpu = [] + + [dependency-groups] + ci = ["member[cpu]"] + + [build-system] + requires = ["fyn-build>=0.7,<10000"] + build-backend = "fyn_build" + + [tool.fyn] + package = true + conflicts = [ + [ + { extra = "cpu" }, + { extra = "gpu" }, + ], + ] + "#, + )?; + member + .child("src") + .child("member") + .child("__init__.py") + .touch()?; + + fyn_snapshot!( + context.filters(), + context + .sync() + .arg("--package") + .arg("member") + .arg("--group") + .arg("ci"), + @" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved [N] packages in [TIME] + Prepared [N] packages in [TIME] + Installed [N] packages in [TIME] + + idna==3.6 + + member==0.0.0 (from file://[TEMP_DIR]/member) + " + ); + + context.assert_command("import idna").success(); + + Ok(()) +} + #[test] fn sync_non_existent_extra() -> Result<()> { let context = fyn_test::test_context!("3.12"); From 20f28bab2424e3260c42baeee6e15fee3d02a5b7 Mon Sep 17 00:00:00 2001 From: duriantaco Date: Mon, 6 Apr 2026 22:03:45 +0800 Subject: [PATCH 6/8] docs: refresh README and manifesto --- MANIFESTO.md | 55 +++++++++++++++++++++------------- README.md | 84 +++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 102 insertions(+), 37 deletions(-) diff --git a/MANIFESTO.md b/MANIFESTO.md index 8ec90b4d4..70fc0a671 100644 --- a/MANIFESTO.md +++ b/MANIFESTO.md @@ -1,11 +1,14 @@ -# fyn — A uv fork with a smaller package-index `User-Agent` +# fyn — A community fork of uv with its own commands and config **fyn** is a Python package manager and project manager written in Rust. It's an independent fork of -[uv](https://github.com/astral-sh/uv), which is the fastest Python package installer around, with a -smaller package-index `User-Agent`, missing features added, and bugs fixed. +[uv](https://github.com/astral-sh/uv), which is the fastest Python package installer around. It +started as a relatively close fork, but fyn now ships fork-specific commands, config namespace, +defaults, policies, and behavior, along with a smaller package-index `User-Agent`, added features, +and bug fixes. -If you've used uv, fyn should feel familiar. Most day-to-day commands carry over, migration is -usually straightforward, and the main behavior differences are below. +If you've used uv, fyn should still feel familiar. Many day-to-day commands carry over and +migration is usually straightforward, but fyn is no longer just uv with a different binary name. +The main concrete differences are below. ## Why fork uv? @@ -19,8 +22,10 @@ So we forked it. We called it **fyn**. You can infer the first letter however yo ## What's actually different -We're not trying to rewrite the thing. uv is really solid and we're keeping most of it. We're just -fixing the stuff that bugged us and adding the things people kept asking for. +We're not trying to rewrite the thing from scratch. uv is really solid and the shared ancestry still +matters. But fyn has diverged enough that "uv with a smaller package-index `User-Agent`" is no +longer a very good description. The project now has its own command surface, project conventions, +policies, and bug fixes. ### Reduced package-index request metadata @@ -35,27 +40,37 @@ This reduces what is exposed in the header, but it does not make package install indexes still see normal network and request information, including your IP address and the packages you ask for. -### New features we added +### Some of the larger user-visible differences +- **Fork-specific project namespace** — fyn uses `[tool.fyn]` and `fyn.lock`, not `[tool.uv]` and + `uv.lock`. +- **Task runner** — define project tasks in `[tool.fyn.tasks]` and list them with + `fyn run --list-tasks`. - **`fyn shell`** - **`fyn upgrade`** +- **`fyn status`** +- **Managed-project pip guardrails** — configure `pip-in-project = "warn" | "error" | "allow"` to + control direct `fyn pip` mutations inside managed projects. - **Cache size limits** — set `UV_CACHE_MAX_SIZE=2G` and the cache cleans itself. uv's cache just grew forever. +- **Custom lockfile name** — set `UV_LOCKFILE` to use a non-default lockfile path. ## fyn vs uv — feature comparison -| Feature | uv | fyn | -| ------------------------------- | ------------------------------------- | ----------------------- | -| Speed (10-100x faster than pip) | Yes | Yes | -| Package index User-Agent | `uv/` plus LineHaul metadata | Minimal `fyn/` | -| `shell` command | Not available | `fyn shell` | -| `upgrade` command | Must chain two commands | `fyn upgrade` | -| Cache size limit | No limit | `UV_CACHE_MAX_SIZE` | -| Custom lockfile name | Not available | `UV_LOCKFILE` | -| `remove --group` sync behavior | Wipes other group packages | Fixed | -| Drop-in replacement for pip | Yes | Yes | -| Python version management | Yes | Yes | -| Lockfile support | Yes (`uv.lock`) | Yes (`fyn.lock`) | +| Area | uv | fyn | +| ------------------------------- | ------------------------------------- | ----------------------------------- | +| Speed (10-100x faster than pip) | Yes | Yes | +| Config namespace and lockfile | `[tool.uv]`, `uv.lock` | `[tool.fyn]`, `fyn.lock` | +| Package index User-Agent | `uv/` plus LineHaul metadata | Minimal `fyn/` | +| Task runner | No `[tool.uv.tasks]` | `[tool.fyn.tasks]` | +| `shell` command | No `uv shell` | `fyn shell` | +| `upgrade` command | No `uv upgrade` | `fyn upgrade` | +| `status` command | No `uv status` | `fyn status` | +| Managed-project `pip` policy | No `pip-in-project` setting | `pip-in-project = warn|error|allow` | +| Cache size limit | No `UV_CACHE_MAX_SIZE` | `UV_CACHE_MAX_SIZE` | +| Custom lockfile name | No `UV_LOCKFILE` | `UV_LOCKFILE` | +| Python version management | Yes | Yes | +| Lockfile support | Yes (`uv.lock`) | Yes (`fyn.lock`) | ## Roadmap diff --git a/README.md b/README.md index 8cf2f427c..54db75a0f 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,10 @@ An extremely fast Python package and project manager, written in Rust. -**fyn** is an independent Python package manager built on [uv](https://github.com/astral-sh/uv)'s -foundation, with reduced package-index request metadata, new features added, and long-standing bugs -fixed. See [MANIFESTO.md](MANIFESTO.md) for the full story. +**fyn** is an independent community fork of [uv](https://github.com/astral-sh/uv). It started on +uv's foundation, but it now has its own commands, settings, defaults, and behavior, alongside +reduced package-index request metadata, added features, and long-standing bug fixes. See +[MANIFESTO.md](MANIFESTO.md) for the full story. ## Highlights @@ -119,6 +120,10 @@ Type exit to deactivate. Works with bash, zsh, fish, nushell, powershell, and cmd. +If you pass a path, `fyn shell` activates that environment directly. Otherwise it uses `VIRTUAL_ENV` +when set, then the discovered project environment, then a local `.venv`. Use `--no-project` to +skip project discovery and only check the current directory. + ### Upgrade dependencies Upgrade all or specific dependencies in one command: @@ -135,6 +140,44 @@ success: Dependencies upgraded successfully. Supports `--dry-run` and `--no-sync`. +`fyn upgrade` is the convenience form of running `fyn lock --upgrade` and then `fyn sync`. + +### Project status + +Inspect the current project and environment state: + +```console +$ fyn status +current directory: /home/user/example +project directory: /home/user/example +managed project: yes +workspace root: /home/user/example +pyproject.toml: yes +fyn.lock: yes +pip-in-project: warn +environment: /home/user/example/.venv +python: /home/user/example/.venv/bin/python3 (3.12.0) +``` + +Use `--check` to fail when obvious project checks do not pass, or `--json` for scripting and editor +integrations. + +### PyTorch backend diagnosis + +Inspect the current machine and environment before installing or reinstalling PyTorch: + +```console +$ fyn torch doctor +PyTorch doctor +recommended backend: cu130 + +next command: + fyn pip install torch torchvision torchaudio --torch-backend=cu130 +``` + +Use `--json` for scripting. `fyn torch doctor` reports the recommendation and current package +state, but does not modify `pyproject.toml`. + ### Scripts fyn manages dependencies and environments for single-file scripts. @@ -314,22 +357,28 @@ The same ones as uv: macOS, Linux, and Windows, across x86_64 and aarch64. #### Is fyn compatible with uv? -Mostly at the workflow level, but not as a byte-for-byte drop-in replacement. Commands and many -`UV_*` environment variables carry over, but projects need `[tool.uv]` renamed to `[tool.fyn]` and -`uv.lock` renamed to `fyn.lock` unless you override the lockfile name. +At the workflow level, often yes, but not as a drop-in replacement. Many commands and `UV_*` +environment variables carry over, but fyn now has fork-specific commands, config, defaults, and +behavior. Projects still need `[tool.uv]` renamed to `[tool.fyn]` and `uv.lock` renamed to +`fyn.lock` unless you override the lockfile name. #### What's different from uv? -See [MANIFESTO.md](MANIFESTO.md) for the full comparison, or the table below for a quick summary: - -| Feature | uv | fyn | -| ------------------------ | ------------------------------------- | ----------------------- | -| Package index User-Agent | `uv/` plus LineHaul metadata | Minimal `fyn/` | -| Task runner | Not available | `[tool.fyn.tasks]` | -| `shell` command | Not available | `fyn shell` | -| `upgrade` command | Must chain two commands | `fyn upgrade` | -| Cache size limit | No limit | `UV_CACHE_MAX_SIZE` | -| Custom lockfile name | Not available | `UV_LOCKFILE` | +See [MANIFESTO.md](MANIFESTO.md) for the fuller comparison, or the table below for some of the +larger user-visible differences: + +| Area | uv | fyn | +| ----------------------------- | ------------------------------------- | ----------------------------------- | +| Config namespace and lockfile | `[tool.uv]`, `uv.lock` | `[tool.fyn]`, `fyn.lock` | +| Package index User-Agent | `uv/` plus LineHaul metadata | Minimal `fyn/` | +| Task runner | No `[tool.uv.tasks]` | `[tool.fyn.tasks]` | +| `shell` command | No `uv shell` | `fyn shell` | +| `upgrade` command | No `uv upgrade` | `fyn upgrade` | +| `status` command | No `uv status` | `fyn status` | +| `torch doctor` command | No `uv torch doctor` | `fyn torch doctor` | +| Managed-project `pip` policy | No `pip-in-project` setting | `pip-in-project = warn|error|allow` | +| Cache size limit | No `UV_CACHE_MAX_SIZE` | `UV_CACHE_MAX_SIZE` | +| Custom lockfile name | No `UV_LOCKFILE` | `UV_LOCKFILE` | ## Acknowledgements @@ -337,7 +386,8 @@ fyn's dependency resolver uses [PubGrub](https://github.com/pubgrub-rs/pubgrub) We're grateful to the PubGrub maintainers, especially [Jacob Finkelman](https://github.com/Eh2406), for their support. -fyn's core is derived from [uv](https://github.com/astral-sh/uv) by Astral. +fyn started as a fork of [uv](https://github.com/astral-sh/uv) by Astral and still shares +substantial ancestry with it. fyn's Git implementation is based on [Cargo](https://github.com/rust-lang/cargo). From f477f0238e2943696a21f9df5a63b1abd919c397 Mon Sep 17 00:00:00 2001 From: duriantaco Date: Mon, 6 Apr 2026 22:05:25 +0800 Subject: [PATCH 7/8] docs: refresh README and manifesto --- MANIFESTO.md | 34 +++++++++++++++++----------------- README.md | 32 ++++++++++++++++---------------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/MANIFESTO.md b/MANIFESTO.md index 70fc0a671..77642d6ce 100644 --- a/MANIFESTO.md +++ b/MANIFESTO.md @@ -6,9 +6,9 @@ started as a relatively close fork, but fyn now ships fork-specific commands, co defaults, policies, and behavior, along with a smaller package-index `User-Agent`, added features, and bug fixes. -If you've used uv, fyn should still feel familiar. Many day-to-day commands carry over and -migration is usually straightforward, but fyn is no longer just uv with a different binary name. -The main concrete differences are below. +If you've used uv, fyn should still feel familiar. Many day-to-day commands carry over and migration +is usually straightforward, but fyn is no longer just uv with a different binary name. The main +concrete differences are below. ## Why fork uv? @@ -57,20 +57,20 @@ you ask for. ## fyn vs uv — feature comparison -| Area | uv | fyn | -| ------------------------------- | ------------------------------------- | ----------------------------------- | -| Speed (10-100x faster than pip) | Yes | Yes | -| Config namespace and lockfile | `[tool.uv]`, `uv.lock` | `[tool.fyn]`, `fyn.lock` | -| Package index User-Agent | `uv/` plus LineHaul metadata | Minimal `fyn/` | -| Task runner | No `[tool.uv.tasks]` | `[tool.fyn.tasks]` | -| `shell` command | No `uv shell` | `fyn shell` | -| `upgrade` command | No `uv upgrade` | `fyn upgrade` | -| `status` command | No `uv status` | `fyn status` | -| Managed-project `pip` policy | No `pip-in-project` setting | `pip-in-project = warn|error|allow` | -| Cache size limit | No `UV_CACHE_MAX_SIZE` | `UV_CACHE_MAX_SIZE` | -| Custom lockfile name | No `UV_LOCKFILE` | `UV_LOCKFILE` | -| Python version management | Yes | Yes | -| Lockfile support | Yes (`uv.lock`) | Yes (`fyn.lock`) | +| Area | uv | fyn | +| ------------------------------- | ------------------------------------- | ------------------------ | ----- | ------ | +| Speed (10-100x faster than pip) | Yes | Yes | +| Config namespace and lockfile | `[tool.uv]`, `uv.lock` | `[tool.fyn]`, `fyn.lock` | +| Package index User-Agent | `uv/` plus LineHaul metadata | Minimal `fyn/` | +| Task runner | No `[tool.uv.tasks]` | `[tool.fyn.tasks]` | +| `shell` command | No `uv shell` | `fyn shell` | +| `upgrade` command | No `uv upgrade` | `fyn upgrade` | +| `status` command | No `uv status` | `fyn status` | +| Managed-project `pip` policy | No `pip-in-project` setting | `pip-in-project = warn | error | allow` | +| Cache size limit | No `UV_CACHE_MAX_SIZE` | `UV_CACHE_MAX_SIZE` | +| Custom lockfile name | No `UV_LOCKFILE` | `UV_LOCKFILE` | +| Python version management | Yes | Yes | +| Lockfile support | Yes (`uv.lock`) | Yes (`fyn.lock`) | ## Roadmap diff --git a/README.md b/README.md index 54db75a0f..8866e7558 100644 --- a/README.md +++ b/README.md @@ -121,8 +121,8 @@ Type exit to deactivate. Works with bash, zsh, fish, nushell, powershell, and cmd. If you pass a path, `fyn shell` activates that environment directly. Otherwise it uses `VIRTUAL_ENV` -when set, then the discovered project environment, then a local `.venv`. Use `--no-project` to -skip project discovery and only check the current directory. +when set, then the discovered project environment, then a local `.venv`. Use `--no-project` to skip +project discovery and only check the current directory. ### Upgrade dependencies @@ -175,8 +175,8 @@ next command: fyn pip install torch torchvision torchaudio --torch-backend=cu130 ``` -Use `--json` for scripting. `fyn torch doctor` reports the recommendation and current package -state, but does not modify `pyproject.toml`. +Use `--json` for scripting. `fyn torch doctor` reports the recommendation and current package state, +but does not modify `pyproject.toml`. ### Scripts @@ -367,18 +367,18 @@ behavior. Projects still need `[tool.uv]` renamed to `[tool.fyn]` and `uv.lock` See [MANIFESTO.md](MANIFESTO.md) for the fuller comparison, or the table below for some of the larger user-visible differences: -| Area | uv | fyn | -| ----------------------------- | ------------------------------------- | ----------------------------------- | -| Config namespace and lockfile | `[tool.uv]`, `uv.lock` | `[tool.fyn]`, `fyn.lock` | -| Package index User-Agent | `uv/` plus LineHaul metadata | Minimal `fyn/` | -| Task runner | No `[tool.uv.tasks]` | `[tool.fyn.tasks]` | -| `shell` command | No `uv shell` | `fyn shell` | -| `upgrade` command | No `uv upgrade` | `fyn upgrade` | -| `status` command | No `uv status` | `fyn status` | -| `torch doctor` command | No `uv torch doctor` | `fyn torch doctor` | -| Managed-project `pip` policy | No `pip-in-project` setting | `pip-in-project = warn|error|allow` | -| Cache size limit | No `UV_CACHE_MAX_SIZE` | `UV_CACHE_MAX_SIZE` | -| Custom lockfile name | No `UV_LOCKFILE` | `UV_LOCKFILE` | +| Area | uv | fyn | +| ----------------------------- | ------------------------------------- | ------------------------ | ----- | ------ | +| Config namespace and lockfile | `[tool.uv]`, `uv.lock` | `[tool.fyn]`, `fyn.lock` | +| Package index User-Agent | `uv/` plus LineHaul metadata | Minimal `fyn/` | +| Task runner | No `[tool.uv.tasks]` | `[tool.fyn.tasks]` | +| `shell` command | No `uv shell` | `fyn shell` | +| `upgrade` command | No `uv upgrade` | `fyn upgrade` | +| `status` command | No `uv status` | `fyn status` | +| `torch doctor` command | No `uv torch doctor` | `fyn torch doctor` | +| Managed-project `pip` policy | No `pip-in-project` setting | `pip-in-project = warn | error | allow` | +| Cache size limit | No `UV_CACHE_MAX_SIZE` | `UV_CACHE_MAX_SIZE` | +| Custom lockfile name | No `UV_LOCKFILE` | `UV_LOCKFILE` | ## Acknowledgements From 9dd87fe3472c3b84bb3c996fc80e6ca3416033b3 Mon Sep 17 00:00:00 2001 From: duriantaco Date: Wed, 8 Apr 2026 14:12:51 +0800 Subject: [PATCH 8/8] Backport uv direct URL hash and workspace conflict fixes --- crates/fyn-cli/src/compat.rs | 4 +- crates/fyn-distribution-types/src/hash.rs | 135 +++++-- crates/fyn-distribution-types/src/id.rs | 244 ++++++++++++- .../src/index/built_wheel_index.rs | 2 +- crates/fyn-distribution/src/source/mod.rs | 8 +- crates/fyn-pypi-types/src/conflicts.rs | 105 +++++- crates/fyn-requirements/src/lib.rs | 3 + crates/fyn-requirements/src/lookahead.rs | 11 +- crates/fyn-resolver/src/flat_index.rs | 19 +- crates/fyn-resolver/src/lock/export/mod.rs | 214 +++++------ ...__lock__tests__hash_optional_missing.snap} | 1 + ...__lock__tests__hash_optional_present.snap} | 1 + ...__lock__tests__hash_required_present.snap} | 1 + ...issing_dependency_source_unambiguous.snap} | 1 + ...ependency_source_version_unambiguous.snap} | 1 + ...__missing_dependency_version_dynamic.snap} | 1 + ...ssing_dependency_version_unambiguous.snap} | 1 + ...ock__tests__source_direct_has_subdir.snap} | 1 + ...lock__tests__source_direct_no_subdir.snap} | 1 + ...olver__lock__tests__source_directory.snap} | 1 + ...solver__lock__tests__source_editable.snap} | 1 + crates/fyn-resolver/src/universal_marker.rs | 56 ++- crates/fyn-types/src/hash.rs | 342 ++++++++++++++---- crates/fyn-workspace/src/workspace.rs | 18 +- crates/fyn/src/commands/pip/compile.rs | 2 +- crates/fyn/src/commands/pip/download.rs | 2 +- crates/fyn/src/commands/pip/install.rs | 4 +- crates/fyn/src/commands/pip/operations.rs | 21 +- crates/fyn/src/commands/pip/sync.rs | 4 +- crates/fyn/src/commands/pip/wheel.rs | 2 +- crates/fyn/src/commands/project/lock.rs | 4 +- .../fyn/src/commands/project/lock_target.rs | 6 +- crates/fyn/src/commands/project/mod.rs | 7 +- crates/fyn/tests/it/export.rs | 142 ++++++++ crates/fyn/tests/it/lock.rs | 160 ++++++++ crates/fyn/tests/it/pip_compile.rs | 11 +- crates/fyn/tests/it/pip_install.rs | 171 +++++++++ crates/fyn/tests/it/pip_sync.rs | 117 ++++-- crates/fyn/tests/it/sync.rs | 6 + 39 files changed, 1529 insertions(+), 302 deletions(-) rename crates/fyn-resolver/src/lock/snapshots/{uv_resolver__lock__tests__hash_optional_missing.snap => fyn_resolver__lock__tests__hash_optional_missing.snap} (99%) rename crates/fyn-resolver/src/lock/snapshots/{uv_resolver__lock__tests__hash_optional_present.snap => fyn_resolver__lock__tests__hash_optional_present.snap} (99%) rename crates/fyn-resolver/src/lock/snapshots/{uv_resolver__lock__tests__hash_required_present.snap => fyn_resolver__lock__tests__hash_required_present.snap} (99%) rename crates/fyn-resolver/src/lock/snapshots/{uv_resolver__lock__tests__missing_dependency_version_unambiguous.snap => fyn_resolver__lock__tests__missing_dependency_source_unambiguous.snap} (99%) rename crates/fyn-resolver/src/lock/snapshots/{uv_resolver__lock__tests__missing_dependency_source_version_unambiguous.snap => fyn_resolver__lock__tests__missing_dependency_source_version_unambiguous.snap} (99%) rename crates/fyn-resolver/src/lock/snapshots/{uv_resolver__lock__tests__missing_dependency_version_dynamic.snap => fyn_resolver__lock__tests__missing_dependency_version_dynamic.snap} (99%) rename crates/fyn-resolver/src/lock/snapshots/{uv_resolver__lock__tests__missing_dependency_source_unambiguous.snap => fyn_resolver__lock__tests__missing_dependency_version_unambiguous.snap} (99%) rename crates/fyn-resolver/src/lock/snapshots/{uv_resolver__lock__tests__source_direct_has_subdir.snap => fyn_resolver__lock__tests__source_direct_has_subdir.snap} (99%) rename crates/fyn-resolver/src/lock/snapshots/{uv_resolver__lock__tests__source_direct_no_subdir.snap => fyn_resolver__lock__tests__source_direct_no_subdir.snap} (99%) rename crates/fyn-resolver/src/lock/snapshots/{uv_resolver__lock__tests__source_directory.snap => fyn_resolver__lock__tests__source_directory.snap} (99%) rename crates/fyn-resolver/src/lock/snapshots/{uv_resolver__lock__tests__source_editable.snap => fyn_resolver__lock__tests__source_editable.snap} (99%) diff --git a/crates/fyn-cli/src/compat.rs b/crates/fyn-cli/src/compat.rs index 51370127a..2f3c561dd 100644 --- a/crates/fyn-cli/src/compat.rs +++ b/crates/fyn-cli/src/compat.rs @@ -100,7 +100,7 @@ impl CompatArgs for PipCompileCompatArgs { } Resolver::Legacy => { return Err(anyhow!( - "pip-compile's `--resolver=legacy` is unsupported (uv always backtracks)" + "pip-compile's `--resolver=legacy` is unsupported (fyn always backtracks)" )); } } @@ -108,7 +108,7 @@ impl CompatArgs for PipCompileCompatArgs { if self.max_rounds.is_some() { return Err(anyhow!( - "pip-compile's `--max-rounds` is unsupported (uv always resolves until convergence)" + "pip-compile's `--max-rounds` is unsupported (fyn always resolves until convergence)" )); } diff --git a/crates/fyn-distribution-types/src/hash.rs b/crates/fyn-distribution-types/src/hash.rs index fafba8c2e..603181dee 100644 --- a/crates/fyn-distribution-types/src/hash.rs +++ b/crates/fyn-distribution-types/src/hash.rs @@ -6,9 +6,13 @@ pub enum HashPolicy<'a> { None, /// Hashes should be generated (specifically, a SHA-256 hash), but not validated. Generate(HashGeneration), - /// Hashes should be validated against a pre-defined list of hashes. If necessary, hashes should - /// be generated so as to ensure that the archive is valid. - Validate(&'a [HashDigest]), + /// Hashes should be validated against a pre-defined list of hashes, and any matching digest is + /// sufficient. If necessary, hashes should be generated so as to ensure that the archive is + /// valid. + Any(&'a [HashDigest]), + /// Hashes should be validated against a pre-defined list of hashes, and every digest must + /// match. If necessary, hashes should be generated so as to ensure that the archive is valid. + All(&'a [HashDigest]), } impl HashPolicy<'_> { @@ -17,9 +21,9 @@ impl HashPolicy<'_> { matches!(self, Self::None) } - /// Returns `true` if the hash policy is `Validate`. - pub fn is_validate(&self) -> bool { - matches!(self, Self::Validate(_)) + /// Returns `true` if the hash policy requires hash validation. + pub fn requires_validation(&self) -> bool { + matches!(self, Self::Any(_) | Self::All(_)) } /// Returns `true` if the hash policy indicates that hashes should be generated. @@ -29,7 +33,7 @@ impl HashPolicy<'_> { Self::Generate(HashGeneration::All) => { dist.file().is_none_or(|file| file.hashes.is_empty()) } - Self::Validate(_) => false, + Self::Any(_) | Self::All(_) => false, Self::None => false, } } @@ -39,7 +43,7 @@ impl HashPolicy<'_> { match self { Self::None => vec![], Self::Generate(_) => vec![HashAlgorithm::Sha256], - Self::Validate(hashes) => { + Self::Any(hashes) | Self::All(hashes) => { let mut algorithms = hashes.iter().map(HashDigest::algorithm).collect::>(); algorithms.sort(); algorithms.dedup(); @@ -53,7 +57,47 @@ impl HashPolicy<'_> { match self { Self::None => &[], Self::Generate(_) => &[], - Self::Validate(hashes) => hashes, + Self::Any(hashes) | Self::All(hashes) => hashes, + } + } + + /// Returns `true` if the given hashes satisfy the policy. + pub fn matches(&self, hashes: &[HashDigest]) -> bool { + match self { + Self::None => true, + Self::Generate(_) => hashes + .iter() + .any(|hash| hash.algorithm == HashAlgorithm::Sha256), + Self::Any(required) => { + !required.is_empty() && hashes.iter().any(|hash| required.contains(hash)) + } + Self::All(required) => { + !required.is_empty() && required.iter().all(|hash| hashes.contains(hash)) + } + } + } + + /// Returns `true` if the given hashes include the algorithms required by the policy. + pub fn has_required_algorithms(&self, hashes: &[HashDigest]) -> bool { + match self { + Self::None => true, + Self::Generate(_) => hashes + .iter() + .any(|hash| hash.algorithm == HashAlgorithm::Sha256), + Self::Any(required) => { + !required.is_empty() + && required + .iter() + .map(HashDigest::algorithm) + .any(|algorithm| hashes.iter().any(|hash| hash.algorithm == algorithm)) + } + Self::All(required) => { + !required.is_empty() + && required + .iter() + .map(HashDigest::algorithm) + .all(|algorithm| hashes.iter().any(|hash| hash.algorithm == algorithm)) + } } } } @@ -74,28 +118,61 @@ pub trait Hashed { /// Returns `true` if the archive satisfies the given hash policy. fn satisfies(&self, hashes: HashPolicy) -> bool { - match hashes { - HashPolicy::None => true, - HashPolicy::Generate(_) => self - .hashes() - .iter() - .any(|hash| hash.algorithm == HashAlgorithm::Sha256), - HashPolicy::Validate(hashes) => self.hashes().iter().any(|hash| hashes.contains(hash)), - } + hashes.matches(self.hashes()) } - /// Returns `true` if the archive includes a hash for at least one of the given algorithms. + /// Returns `true` if the archive includes the algorithms required by the given hash policy. fn has_digests(&self, hashes: HashPolicy) -> bool { - match hashes { - HashPolicy::None => true, - HashPolicy::Generate(_) => self - .hashes() - .iter() - .any(|hash| hash.algorithm == HashAlgorithm::Sha256), - HashPolicy::Validate(hashes) => hashes - .iter() - .map(HashDigest::algorithm) - .any(|algorithm| self.hashes().iter().any(|hash| hash.algorithm == algorithm)), - } + hashes.has_required_algorithms(self.hashes()) + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use fyn_pypi_types::HashDigest; + + use super::HashPolicy; + + #[test] + fn validate_all_requires_every_digest() { + let sha256 = HashDigest::from_str( + "sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f", + ) + .unwrap(); + let sha512 = HashDigest::from_str( + "sha512:f30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2", + ) + .unwrap(); + let wrong_sha512 = HashDigest::from_str( + "sha512:e30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2", + ) + .unwrap(); + + let policy = HashPolicy::All(&[sha256.clone(), sha512.clone()]); + assert!(policy.matches(&[sha256.clone(), sha512])); + assert!(!policy.matches(std::slice::from_ref(&sha256))); + assert!(!policy.matches(&[sha256, wrong_sha512])); + } + + #[test] + fn validate_any_requires_one_digest() { + let sha256 = HashDigest::from_str( + "sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f", + ) + .unwrap(); + let sha512 = HashDigest::from_str( + "sha512:f30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2", + ) + .unwrap(); + let wrong_sha512 = HashDigest::from_str( + "sha512:e30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2", + ) + .unwrap(); + + let policy = HashPolicy::Any(&[sha256.clone(), sha512]); + assert!(policy.matches(&[sha256])); + assert!(!policy.matches(&[wrong_sha512])); } } diff --git a/crates/fyn-distribution-types/src/id.rs b/crates/fyn-distribution-types/src/id.rs index 85a29bcea..fe78575cf 100644 --- a/crates/fyn-distribution-types/src/id.rs +++ b/crates/fyn-distribution-types/src/id.rs @@ -1,11 +1,14 @@ use std::fmt::{Display, Formatter}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use fyn_cache_key::{CanonicalUrl, RepositoryUrl}; +use fyn_git_types::GitUrl; use fyn_normalize::PackageName; use fyn_pep440::Version; -use fyn_pypi_types::HashDigest; +use fyn_pypi_types::{ + HashDigest, ParsedArchiveUrl, ParsedDirectoryUrl, ParsedGitUrl, ParsedPathUrl, ParsedUrl, +}; use fyn_redacted::DisplaySafeUrl; /// A unique identifier for a package. A package can either be identified by a name (e.g., `black`) @@ -40,12 +43,33 @@ impl Display for PackageId { } /// A unique identifier for a package at a specific version (e.g., `black==23.10.0`). +/// +/// URL-based variants use kind-specific identity semantics. Archive URLs ignore hash fragments +/// while preserving semantic `subdirectory` information. Git URLs preserve semantic +/// `subdirectory` information while ignoring unrelated fragments. Local file URLs are keyed by +/// their resolved path and kind. #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum VersionId { /// The identifier consists of a package name and version. NameVersion(PackageName, Version), - /// The identifier consists of a URL. - Url(CanonicalUrl), + /// The identifier consists of an archive URL identified by its location and optional source + /// subdirectory. + ArchiveUrl { + location: CanonicalUrl, + subdirectory: Option, + }, + /// The identifier consists of a Git repository URL, its reference, and optional source + /// subdirectory. + Git { + url: GitUrl, + subdirectory: Option, + }, + /// The identifier consists of a local file path. + Path(PathBuf), + /// The identifier consists of a local directory path. + Directory(PathBuf), + /// The identifier consists of a URL whose source kind could not be determined. + Unknown(DisplaySafeUrl), } impl VersionId { @@ -54,9 +78,72 @@ impl VersionId { Self::NameVersion(name, version) } + /// Create a new [`VersionId`] from a parsed URL. + pub fn from_parsed_url(url: &ParsedUrl) -> Self { + match url { + ParsedUrl::Path(path) => Self::from_path_url(path), + ParsedUrl::Directory(directory) => Self::from_directory_url(directory), + ParsedUrl::Git(git) => Self::from_git_url(git), + ParsedUrl::Archive(archive) => Self::from_archive_url(archive), + } + } + /// Create a new [`VersionId`] from a URL. pub fn from_url(url: &DisplaySafeUrl) -> Self { - Self::Url(CanonicalUrl::new(url)) + match ParsedUrl::try_from(url.clone()) { + Ok(parsed) => Self::from_parsed_url(&parsed), + Err(_) => Self::Unknown(url.clone()), + } + } + + /// Create a new [`VersionId`] from an archive URL. + pub fn from_archive(location: &DisplaySafeUrl, subdirectory: Option<&Path>) -> Self { + Self::ArchiveUrl { + location: CanonicalUrl::new(location), + subdirectory: subdirectory.map(Path::to_path_buf), + } + } + + /// Create a new [`VersionId`] from a Git URL. + pub fn from_git(git: &GitUrl, subdirectory: Option<&Path>) -> Self { + let git = GitUrl::from_fields( + DisplaySafeUrl::from(CanonicalUrl::new(git.repository())), + git.reference().clone(), + git.precise(), + git.lfs(), + ) + .expect("canonical Git URLs should preserve supported schemes"); + + Self::Git { + url: git, + subdirectory: subdirectory.map(Path::to_path_buf), + } + } + + /// Create a new [`VersionId`] from a local file path. + pub fn from_path(path: &Path) -> Self { + Self::Path(path.to_path_buf()) + } + + /// Create a new [`VersionId`] from a local directory path. + pub fn from_directory(path: &Path) -> Self { + Self::Directory(path.to_path_buf()) + } + + fn from_archive_url(archive: &ParsedArchiveUrl) -> Self { + Self::from_archive(&archive.url, archive.subdirectory.as_deref()) + } + + fn from_path_url(path: &ParsedPathUrl) -> Self { + Self::from_path(path.install_path.as_ref()) + } + + fn from_directory_url(directory: &ParsedDirectoryUrl) -> Self { + Self::from_directory(directory.install_path.as_ref()) + } + + fn from_git_url(git: &ParsedGitUrl) -> Self { + Self::from_git(&git.url, git.subdirectory.as_deref()) } } @@ -64,7 +151,49 @@ impl Display for VersionId { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { Self::NameVersion(name, version) => write!(f, "{name}-{version}"), - Self::Url(url) => write!(f, "{url}"), + Self::ArchiveUrl { + location, + subdirectory, + } => { + let mut location = DisplaySafeUrl::from(location.clone()); + if let Some(subdirectory) = subdirectory { + location + .set_fragment(Some(&format!("subdirectory={}", subdirectory.display()))); + } + write!(f, "{location}") + } + Self::Git { url, subdirectory } => { + let mut git_url = DisplaySafeUrl::parse(&format!("git+{}", url.repository())) + .expect("canonical Git URLs should be display-safe"); + if let Some(precise) = url.precise() { + let path = format!("{}@{}", git_url.path(), precise); + git_url.set_path(&path); + } else if let Some(reference) = url.reference().as_str() { + let path = format!("{}@{}", git_url.path(), reference); + git_url.set_path(&path); + } + + let mut fragments = Vec::new(); + if let Some(subdirectory) = subdirectory { + fragments.push(format!("subdirectory={}", subdirectory.display())); + } + if url.lfs().enabled() { + fragments.push("lfs=true".to_string()); + } + if !fragments.is_empty() { + git_url.set_fragment(Some(&fragments.join("&"))); + } + + write!(f, "{git_url}") + } + Self::Path(path) | Self::Directory(path) => { + if let Ok(url) = DisplaySafeUrl::from_file_path(path) { + write!(f, "{url}") + } else { + write!(f, "{}", path.display()) + } + } + Self::Unknown(url) => write!(f, "{url}"), } } } @@ -121,3 +250,106 @@ impl From<&Self> for ResourceId { value.clone() } } + +#[cfg(test)] +mod tests { + use std::time::{SystemTime, UNIX_EPOCH}; + + use fs_err as fs; + + use super::VersionId; + use fyn_redacted::DisplaySafeUrl; + + #[test] + fn version_id_ignores_hash_fragments() { + let first = DisplaySafeUrl::parse( + "https://example.com/pkg-0.1.0.whl#sha256=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ) + .unwrap(); + let second = DisplaySafeUrl::parse( + "https://example.com/pkg-0.1.0.whl#sha512=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + ) + .unwrap(); + + assert_eq!(VersionId::from_url(&first), VersionId::from_url(&second)); + } + + #[test] + fn version_id_preserves_non_hash_fragments() { + let first = + DisplaySafeUrl::parse("https://example.com/pkg-0.1.0.tar.gz#subdirectory=foo").unwrap(); + let second = + DisplaySafeUrl::parse("https://example.com/pkg-0.1.0.tar.gz#subdirectory=bar").unwrap(); + + assert_ne!(VersionId::from_url(&first), VersionId::from_url(&second)); + } + + #[test] + fn version_id_ignores_hash_fragments_with_subdirectory() { + let first = DisplaySafeUrl::parse( + "https://example.com/pkg-0.1.0.tar.gz#subdirectory=foo&sha256=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ) + .unwrap(); + let second = DisplaySafeUrl::parse( + "https://example.com/pkg-0.1.0.tar.gz#sha512=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb&subdirectory=foo", + ) + .unwrap(); + + assert_eq!(VersionId::from_url(&first), VersionId::from_url(&second)); + } + + #[test] + fn version_id_preserves_non_archive_fragments() { + let first = + DisplaySafeUrl::parse("git+https://example.com/pkg.git#subdirectory=foo").unwrap(); + let second = + DisplaySafeUrl::parse("git+https://example.com/pkg.git#subdirectory=bar").unwrap(); + + assert_ne!(VersionId::from_url(&first), VersionId::from_url(&second)); + } + + #[test] + fn version_id_ignores_irrelevant_git_fragments() { + let first = + DisplaySafeUrl::parse("git+https://example.com/pkg.git@main#egg=pkg&subdirectory=foo") + .unwrap(); + let second = + DisplaySafeUrl::parse("git+https://example.com/pkg.git@main#subdirectory=foo").unwrap(); + + assert_eq!(VersionId::from_url(&first), VersionId::from_url(&second)); + } + + #[test] + fn version_id_uses_file_kinds() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!("uv-version-id-{nonce}")); + let file = root.join("pkg-0.1.0.whl"); + let directory = root.join("pkg"); + + fs::create_dir_all(&directory).unwrap(); + fs::write(&file, b"wheel").unwrap(); + + let file_url = DisplaySafeUrl::from_file_path(&file).unwrap(); + let directory_url = DisplaySafeUrl::from_file_path(&directory).unwrap(); + + assert!(matches!(VersionId::from_url(&file_url), VersionId::Path(_))); + assert!(matches!( + VersionId::from_url(&directory_url), + VersionId::Directory(_) + )); + + fs::remove_file(file).unwrap(); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn version_id_uses_unknown_for_invalid_git_like_urls() { + let url = + DisplaySafeUrl::parse("git+ftp://example.com/pkg.git@main#subdirectory=foo").unwrap(); + + assert!(matches!(VersionId::from_url(&url), VersionId::Unknown(_))); + } +} diff --git a/crates/fyn-distribution/src/index/built_wheel_index.rs b/crates/fyn-distribution/src/index/built_wheel_index.rs index 97a5d215a..bd6369ba1 100644 --- a/crates/fyn-distribution/src/index/built_wheel_index.rs +++ b/crates/fyn-distribution/src/index/built_wheel_index.rs @@ -194,7 +194,7 @@ impl<'a> BuiltWheelIndex<'a> { /// Return the most compatible [`CachedWheel`] for a given source distribution at a git URL. pub fn git(&self, source_dist: &GitSourceDist) -> Option { // Enforce hash-checking, which isn't supported for Git distributions. - if self.hasher.get(source_dist).is_validate() { + if self.hasher.get(source_dist).requires_validation() { return None; } diff --git a/crates/fyn-distribution/src/source/mod.rs b/crates/fyn-distribution/src/source/mod.rs index e8c392bb3..17c1121a9 100644 --- a/crates/fyn-distribution/src/source/mod.rs +++ b/crates/fyn-distribution/src/source/mod.rs @@ -1174,7 +1174,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { hashes: HashPolicy<'_>, ) -> Result { // Before running the build, check that the hashes match. - if hashes.is_validate() { + if hashes.requires_validation() { return Err(Error::HashesNotSupportedSourceTree(source.to_string())); } @@ -1278,7 +1278,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { credentials_cache: &CredentialsCache, ) -> Result { // Before running the build, check that the hashes match. - if hashes.is_validate() { + if hashes.requires_validation() { return Err(Error::HashesNotSupportedSourceTree(source.to_string())); } @@ -1566,7 +1566,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { client: &ManagedClient<'_>, ) -> Result { // Before running the build, check that the hashes match. - if hashes.is_validate() { + if hashes.requires_validation() { return Err(Error::HashesNotSupportedGit(source.to_string())); } @@ -1698,7 +1698,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { credentials_cache: &CredentialsCache, ) -> Result { // Before running the build, check that the hashes match. - if hashes.is_validate() { + if hashes.requires_validation() { return Err(Error::HashesNotSupportedGit(source.to_string())); } diff --git a/crates/fyn-pypi-types/src/conflicts.rs b/crates/fyn-pypi-types/src/conflicts.rs index d182b9237..86b620d82 100644 --- a/crates/fyn-pypi-types/src/conflicts.rs +++ b/crates/fyn-pypi-types/src/conflicts.rs @@ -6,6 +6,7 @@ use petgraph::{ use rustc_hash::{FxHashMap, FxHashSet}; #[cfg(feature = "schemars")] use std::borrow::Cow; +use std::fmt; use std::{collections::BTreeSet, hash::Hash, rc::Rc}; use crate::dependency_groups::{DependencyGroupSpecifier, DependencyGroups}; @@ -542,6 +543,29 @@ impl hashbrown::Equivalent for ConflictKindRef<'_> { } } +#[derive(Debug, Clone)] +pub struct ConflictEntry { + package: Option, + extra: Option, + group: Option, +} + +impl fmt::Display for ConflictEntry { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut parts = vec![]; + if let Some(package) = &self.package { + parts.push(format!("package = \"{package}\"")); + } + if let Some(extra) = &self.extra { + parts.push(format!("extra = \"{extra}\"")); + } + if let Some(group) = &self.group { + parts.push(format!("group = \"{group}\"")); + } + write!(f, "{{ {} }}", parts.join(", ")) + } +} + /// An error that occurs when the given conflicting set is invalid somehow. #[derive(Debug, thiserror::Error)] pub enum ConflictError { @@ -556,8 +580,8 @@ pub enum ConflictError { /// (This is only applicable when deserializing from the lock file. /// When deserializing from `pyproject.toml`, the `package` field is /// optional.) - #[error("Expected `package` field in conflicting entry")] - MissingPackage, + #[error("Expected `package` field in conflicting entry: {0}")] + MissingPackage(ConflictEntry), /// An error that occurs when all of `package`, `extra` and `group` are missing. #[error("Expected `package`, `extra` or `group` field in conflicting entry")] MissingPackageAndExtraAndGroup, @@ -582,21 +606,22 @@ pub enum ConflictError { pub struct SchemaConflicts(Vec); impl SchemaConflicts { - /// Convert the public schema "conflicting" type to our internal fully - /// resolved type. Effectively, this pairs the corresponding package name - /// with each conflict. - /// - /// If a conflict has an explicit package name (written by the end user), - /// then that takes precedence over the given package name, which is only - /// used when there is no explicit package name written. - pub fn to_conflicts_with_package_name(&self, package: &PackageName) -> Conflicts { + fn to_conflicts_with_default_package( + &self, + package: Option<&PackageName>, + ) -> Result { let mut conflicting = Conflicts::empty(); for tool_uv_set in &self.0 { let mut set = vec![]; for item in &tool_uv_set.0 { - let package = item.package.clone().unwrap_or_else(|| package.clone()); + let package = item + .package + .as_ref() + .or(package) + .ok_or_else(|| ConflictError::MissingPackage(ConflictEntry::from(item)))? + .clone(); set.push(ConflictItem { - package: package.clone(), + package, kind: item.kind.clone(), }); } @@ -606,7 +631,27 @@ impl SchemaConflicts { let set = ConflictSet::try_from(set).unwrap(); conflicting.push(set); } - conflicting + Ok(conflicting) + } + + /// Convert the public schema "conflicting" type to our internal fully + /// resolved type. + /// + /// Every conflict item must define `package` explicitly. + pub fn to_conflicts(&self) -> Result { + self.to_conflicts_with_default_package(None) + } + + /// Convert the public schema "conflicting" type to our internal fully + /// resolved type. Effectively, this pairs the corresponding package name + /// with each conflict. + /// + /// If a conflict has an explicit package name (written by the end user), + /// then that takes precedence over the given package name, which is only + /// used when there is no explicit package name written. + pub fn to_conflicts_with_package_name(&self, package: &PackageName) -> Conflicts { + self.to_conflicts_with_default_package(Some(package)) + .expect("default package name should satisfy schema conflicts") } } @@ -689,12 +734,44 @@ struct ConflictItemWire { group: Option, } +impl From<&SchemaConflictItem> for ConflictEntry { + fn from(item: &SchemaConflictItem) -> Self { + match &item.kind { + ConflictKind::Project => Self { + package: item.package.clone(), + extra: None, + group: None, + }, + ConflictKind::Extra(extra) => Self { + package: item.package.clone(), + extra: Some(extra.clone()), + group: None, + }, + ConflictKind::Group(group) => Self { + package: item.package.clone(), + extra: None, + group: Some(group.clone()), + }, + } + } +} + +impl From<&ConflictItemWire> for ConflictEntry { + fn from(item: &ConflictItemWire) -> Self { + Self { + package: item.package.clone(), + extra: item.extra.clone(), + group: item.group.clone(), + } + } +} + impl TryFrom for ConflictItem { type Error = ConflictError; fn try_from(wire: ConflictItemWire) -> Result { let Some(package) = wire.package else { - return Err(ConflictError::MissingPackage); + return Err(ConflictError::MissingPackage(ConflictEntry::from(&wire))); }; match (wire.extra, wire.group) { (Some(_), Some(_)) => Err(ConflictError::FoundExtraAndGroup), diff --git a/crates/fyn-requirements/src/lib.rs b/crates/fyn-requirements/src/lib.rs index 2720ad4b0..e37693863 100644 --- a/crates/fyn-requirements/src/lib.rs +++ b/crates/fyn-requirements/src/lib.rs @@ -32,6 +32,9 @@ pub enum Error { #[error(transparent)] DistributionTypes(#[from] fyn_distribution_types::Error), + #[error(transparent)] + HashStrategy(#[from] fyn_types::HashStrategyError), + #[error(transparent)] WheelFilename(#[from] fyn_distribution_filename::WheelFilenameError), diff --git a/crates/fyn-requirements/src/lookahead.rs b/crates/fyn-requirements/src/lookahead.rs index 6b4d9af3e..9234cfad5 100644 --- a/crates/fyn-requirements/src/lookahead.rs +++ b/crates/fyn-requirements/src/lookahead.rs @@ -82,10 +82,11 @@ impl<'a, Context: BuildContext> LookaheadResolver<'a, Context> { pub async fn resolve( self, env: &ResolverEnvironment, - ) -> Result, Error> { + ) -> Result<(Vec, HashStrategy), Error> { let mut results = Vec::new(); let mut futures = FuturesUnordered::new(); let mut seen = FxHashSet::default(); + let mut hasher = self.hasher.clone(); // Queue up the initial requirements. let mut queue: VecDeque<_> = self @@ -99,13 +100,14 @@ impl<'a, Context: BuildContext> LookaheadResolver<'a, Context> { while let Some(requirement) = queue.pop_front() { if !matches!(requirement.source, RequirementSource::Registry { .. }) { if seen.insert(requirement.clone()) { - futures.push(self.lookahead(requirement)); + futures.push(self.lookahead(requirement, hasher.clone())); } } } while let Some(result) = futures.next().await { if let Some(lookahead) = result? { + hasher = hasher.augment_with_requirements(lookahead.requirements().iter())?; for requirement in self .constraints .apply(self.overrides.apply(lookahead.requirements())) @@ -121,13 +123,14 @@ impl<'a, Context: BuildContext> LookaheadResolver<'a, Context> { } } - Ok(results) + Ok((results, hasher)) } /// Infer the package name for a given "unnamed" requirement. async fn lookahead( &self, requirement: Requirement, + hasher: HashStrategy, ) -> Result, Error> { trace!("Performing lookahead for {requirement}"); @@ -151,7 +154,7 @@ impl<'a, Context: BuildContext> LookaheadResolver<'a, Context> { // Run the PEP 517 build process to extract metadata from the source distribution. let archive = self .database - .get_or_build_wheel_metadata(&dist, self.hasher.get(&dist)) + .get_or_build_wheel_metadata(&dist, hasher.get(&dist)) .await .map_err(|err| Error::from_dist(dist, err))?; diff --git a/crates/fyn-resolver/src/flat_index.rs b/crates/fyn-resolver/src/flat_index.rs index f01d686c8..c59cc1320 100644 --- a/crates/fyn-resolver/src/flat_index.rs +++ b/crates/fyn-resolver/src/flat_index.rs @@ -8,9 +8,8 @@ use fyn_client::{FlatIndexEntries, FlatIndexEntry}; use fyn_configuration::BuildOptions; use fyn_distribution_filename::{DistFilename, SourceDistFilename, WheelFilename}; use fyn_distribution_types::{ - File, HashComparison, HashPolicy, IncompatibleSource, IncompatibleWheel, IndexUrl, - PrioritizedDist, RegistryBuiltWheel, RegistrySourceDist, SourceDistCompatibility, - WheelCompatibility, + File, HashComparison, IncompatibleSource, IncompatibleWheel, IndexUrl, PrioritizedDist, + RegistryBuiltWheel, RegistrySourceDist, SourceDistCompatibility, WheelCompatibility, }; use fyn_normalize::PackageName; use fyn_pep440::Version; @@ -184,12 +183,11 @@ impl FlatDistributions { } // Check if hashes line up - let hash = if let HashPolicy::Validate(required) = - hasher.get_package(&filename.name, &filename.version) - { + let hash_policy = hasher.get_package(&filename.name, &filename.version); + let hash = if hash_policy.requires_validation() { if hashes.is_empty() { HashComparison::Missing - } else if required.iter().any(|hash| hashes.contains(hash)) { + } else if hash_policy.matches(hashes) { HashComparison::Matched } else { HashComparison::Mismatched @@ -225,12 +223,11 @@ impl FlatDistributions { }; // Check if hashes line up. - let hash = if let HashPolicy::Validate(required) = - hasher.get_package(&filename.name, &filename.version) - { + let hash_policy = hasher.get_package(&filename.name, &filename.version); + let hash = if hash_policy.requires_validation() { if hashes.is_empty() { HashComparison::Missing - } else if required.iter().any(|hash| hashes.contains(hash)) { + } else if hash_policy.matches(hashes) { HashComparison::Matched } else { HashComparison::Mismatched diff --git a/crates/fyn-resolver/src/lock/export/mod.rs b/crates/fyn-resolver/src/lock/export/mod.rs index 39c937ae2..1e4c099bd 100644 --- a/crates/fyn-resolver/src/lock/export/mod.rs +++ b/crates/fyn-resolver/src/lock/export/mod.rs @@ -15,12 +15,12 @@ use fyn_normalize::{ExtraName, GroupName, PackageName}; use fyn_pep508::MarkerTree; use fyn_pypi_types::ConflictItem; -use crate::graph_ops::{Reachable, marker_reachability}; +use crate::graph_ops::Reachable; use crate::lock::LockErrorKind; pub(crate) use crate::lock::export::pylock_toml::PylockTomlPackage; pub use crate::lock::export::pylock_toml::{PylockToml, PylockTomlErrorKind}; pub use crate::lock::export::requirements_txt::RequirementsTxtExport; -use crate::universal_marker::resolve_conflicts; +use crate::universal_marker::resolve_activated_extras; use crate::{Installable, LockError, Package}; pub mod cyclonedx_json; @@ -55,16 +55,10 @@ impl<'lock> ExportableRequirements<'lock> { let size_guess = target.lock().packages.len(); let mut graph = Graph::, Edge<'lock>>::with_capacity(size_guess, size_guess); let mut inverse = FxHashMap::with_capacity_and_hasher(size_guess, FxBuildHasher); - let mut selected_extras: FxHashMap<_, Vec> = - FxHashMap::with_capacity_and_hasher(size_guess, FxBuildHasher); let mut queue: VecDeque<(&Package, Option<&ExtraName>)> = VecDeque::new(); let mut seen = FxHashSet::default(); - let mut conflicts = if target.lock().conflicts.is_empty() { - None - } else { - Some(FxHashMap::default()) - }; + let mut activated_items = FxHashMap::default(); let root = graph.add_node(Node::Root); @@ -85,39 +79,33 @@ impl<'lock> ExportableRequirements<'lock> { })?; // Track the activated package in the list of known conflicts. - if let Some(conflicts) = conflicts.as_mut() { - conflicts.insert(ConflictItem::from(dist.id.name.clone()), MarkerTree::TRUE); - } + activated_items.insert(ConflictItem::from(dist.id.name.clone()), MarkerTree::TRUE); if groups.prod() { // Add the workspace package to the graph. let index = *inverse .entry(&dist.id) .or_insert_with(|| graph.add_node(Node::Package(dist))); - graph.add_edge(root, index, Edge::Prod(MarkerTree::TRUE)); + graph.add_edge( + root, + index, + Edge::Prod { + marker: MarkerTree::TRUE, + dep_extras: Vec::new(), + }, + ); // Track the activated project in the list of known conflicts. - if let Some(conflicts) = conflicts.as_mut() { - conflicts.insert(ConflictItem::from(dist.id.name.clone()), MarkerTree::TRUE); - } + activated_items.insert(ConflictItem::from(dist.id.name.clone()), MarkerTree::TRUE); // Push its dependencies on the queue. queue.push_back((dist, None)); - let mut root_extras = Vec::new(); for extra in extras.extra_names(dist.optional_dependencies.keys()) { - root_extras.push(extra.clone()); queue.push_back((dist, Some(extra))); - - // Track the activated extra in the list of known conflicts. - if let Some(conflicts) = conflicts.as_mut() { - conflicts.insert( - ConflictItem::from((dist.id.name.clone(), extra.clone())), - MarkerTree::TRUE, - ); - } - } - if !root_extras.is_empty() { - selected_extras.insert(&dist.id, root_extras); + activated_items.insert( + ConflictItem::from((dist.id.name.clone(), extra.clone())), + MarkerTree::TRUE, + ); } } @@ -135,12 +123,10 @@ impl<'lock> ExportableRequirements<'lock> { .flatten() { // Track the activated group in the list of known conflicts. - if let Some(conflicts) = conflicts.as_mut() { - conflicts.insert( - ConflictItem::from((dist.id.name.clone(), group.clone())), - MarkerTree::TRUE, - ); - } + activated_items.insert( + ConflictItem::from((dist.id.name.clone(), group.clone())), + MarkerTree::TRUE, + ); if prune.contains(&dep.package_id.name) { continue; @@ -159,7 +145,11 @@ impl<'lock> ExportableRequirements<'lock> { graph.add_edge( root, dep_index, - Edge::Dev(group, dep.simplified_marker.as_simplified_marker_tree()), + Edge::Dev { + group, + marker: dep.simplified_marker.as_simplified_marker_tree(), + dep_extras: dep.extra.iter().collect(), + }, ); // Push its dependencies on the queue. @@ -243,7 +233,14 @@ impl<'lock> ExportableRequirements<'lock> { .or_insert_with(|| graph.add_node(Node::Package(dist))); // Add an edge from the root. - graph.add_edge(root, dep_index, Edge::Prod(marker)); + graph.add_edge( + root, + dep_index, + Edge::Prod { + marker, + dep_extras: requirement.extras.iter().collect(), + }, + ); // Push its dependencies on the queue. if seen.insert((&dist.id, None)) { @@ -287,50 +284,21 @@ impl<'lock> ExportableRequirements<'lock> { .entry(&dep.package_id) .or_insert_with(|| graph.add_node(Node::Package(dep_dist))); - // Add an edge from the dependency. - let mut active_extras = selected_extras - .get(&package.id) - .cloned() - .unwrap_or_default(); - if let Some(extra) = extra { - if !active_extras.contains(extra) { - active_extras.push((*extra).clone()); - } - } - if !active_extras.is_empty() { - match selected_extras.entry(&dep.package_id) { - Entry::Occupied(mut entry) => { - for extra in &active_extras { - if !entry.get().contains(extra) { - entry.get_mut().push(extra.clone()); - } - } - } - Entry::Vacant(entry) => { - entry.insert(active_extras.clone()); - } - } - } - + let dep_extras = dep.extra.iter().collect::>(); graph.add_edge( index, dep_index, if let Some(extra) = extra { - Edge::Optional( + Edge::Optional { extra, - dep.simplified_marker - .as_simplified_marker_tree() - .simplify_extras(std::slice::from_ref(extra)), - ) + marker: dep.simplified_marker.as_simplified_marker_tree(), + dep_extras, + } } else { - Edge::Prod(selected_extras.get(&package.id).map_or_else( - || dep.simplified_marker.as_simplified_marker_tree(), - |extras| { - dep.simplified_marker - .as_simplified_marker_tree() - .simplify_extras(extras) - }, - )) + Edge::Prod { + marker: dep.simplified_marker.as_simplified_marker_tree(), + dep_extras, + } }, ); @@ -347,11 +315,7 @@ impl<'lock> ExportableRequirements<'lock> { } // Determine the reachability of each node in the graph. - let mut reachability = if let Some(conflicts) = conflicts.as_ref() { - conflict_marker_reachability(&graph, &[], conflicts) - } else { - marker_reachability(&graph, &[]) - }; + let mut reachability = conflict_marker_reachability(&graph, &[], &activated_items); // Collect all packages. let nodes = graph @@ -403,18 +367,38 @@ enum Node<'lock> { /// An edge in the resolution graph, along with the marker that must be satisfied to traverse it. #[derive(Debug, Clone)] enum Edge<'lock> { - Prod(MarkerTree), - Optional(&'lock ExtraName, MarkerTree), - Dev(&'lock GroupName, MarkerTree), + Prod { + marker: MarkerTree, + dep_extras: Vec<&'lock ExtraName>, + }, + Optional { + extra: &'lock ExtraName, + marker: MarkerTree, + dep_extras: Vec<&'lock ExtraName>, + }, + Dev { + group: &'lock GroupName, + marker: MarkerTree, + dep_extras: Vec<&'lock ExtraName>, + }, } impl Edge<'_> { /// Return the [`MarkerTree`] for this edge. fn marker(&self) -> &MarkerTree { match self { - Self::Prod(marker) => marker, - Self::Optional(_, marker) => marker, - Self::Dev(_, marker) => marker, + Self::Prod { marker, .. } => marker, + Self::Optional { marker, .. } => marker, + Self::Dev { marker, .. } => marker, + } + } + + /// Return the dependency extras activated by traversing this edge. + fn dep_extras(&self) -> &[&ExtraName] { + match self { + Self::Prod { dep_extras, .. } => dep_extras, + Self::Optional { dep_extras, .. } => dep_extras, + Self::Dev { dep_extras, .. } => dep_extras, } } } @@ -493,7 +477,11 @@ fn conflict_marker_reachability<'lock>( // Resolve any conflicts in the parent marker. reachability.entry(parent_index).and_modify(|marker| { let conflict_map = conflict_maps.get(&parent_index).unwrap_or(known_conflicts); - *marker = resolve_conflicts(*marker, conflict_map); + let scope_package = match &graph[parent_index] { + Node::Package(package) => Some(package.name()), + Node::Root => None, + }; + *marker = resolve_activated_extras(*marker, scope_package, conflict_map); }); // When we see an edge like `parent [dotenv]> flask`, we should take the reachability @@ -508,10 +496,22 @@ fn conflict_marker_reachability<'lock>( .cloned() .unwrap_or_else(|| known_conflicts.clone()); + if let Node::Package(child) = graph[child_edge.target()] { + for extra in child_edge.weight().dep_extras() { + let item = ConflictItem::from((child.name().clone(), (*extra).clone())); + parent_map.insert(item, parent_marker); + } + } + + let scope_package = match &graph[parent_index] { + Node::Package(package) => Some(package.name()), + Node::Root => None, + }; + match child_edge.weight() { - Edge::Prod(marker) => { - // Resolve any conflicts on the edge. - let marker = resolve_conflicts(*marker, &parent_map); + Edge::Prod { marker, .. } => { + // Resolve any active extras on the edge. + let marker = resolve_activated_extras(*marker, scope_package, &parent_map); // Propagate the edge to the known conflicts. for value in parent_map.values_mut() { @@ -521,9 +521,16 @@ fn conflict_marker_reachability<'lock>( // Propagate the edge to the node itself. parent_marker.and(marker); } - Edge::Optional(extra, marker) => { - // Resolve any conflicts on the edge. - let marker = resolve_conflicts(*marker, &parent_map); + Edge::Optional { extra, marker, .. } => { + // The optional extra is active for this edge itself, so add it before + // resolving any active extras on the edge. + if let Node::Package(parent) = graph[parent_index] { + let item = ConflictItem::from((parent.name().clone(), (*extra).clone())); + parent_map.insert(item, parent_marker); + } + + // Resolve any active extras on the edge. + let marker = resolve_activated_extras(*marker, scope_package, &parent_map); // Propagate the edge to the known conflicts. for value in parent_map.values_mut() { @@ -532,16 +539,17 @@ fn conflict_marker_reachability<'lock>( // Propagate the edge to the node itself. parent_marker.and(marker); - - // Add a known conflict item for the extra. + } + Edge::Dev { group, marker, .. } => { + // The dependency group is active for this edge itself, so add it before + // resolving any active extras on the edge. if let Node::Package(parent) = graph[parent_index] { - let item = ConflictItem::from((parent.name().clone(), (*extra).clone())); + let item = ConflictItem::from((parent.name().clone(), (*group).clone())); parent_map.insert(item, parent_marker); } - } - Edge::Dev(group, marker) => { - // Resolve any conflicts on the edge. - let marker = resolve_conflicts(*marker, &parent_map); + + // Resolve any active extras on the edge. + let marker = resolve_activated_extras(*marker, scope_package, &parent_map); // Propagate the edge to the known conflicts. for value in parent_map.values_mut() { @@ -550,12 +558,6 @@ fn conflict_marker_reachability<'lock>( // Propagate the edge to the node itself. parent_marker.and(marker); - - // Add a known conflict item for the group. - if let Node::Package(parent) = graph[parent_index] { - let item = ConflictItem::from((parent.name().clone(), (*group).clone())); - parent_map.insert(item, parent_marker); - } } } diff --git a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__hash_optional_missing.snap b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__hash_optional_missing.snap similarity index 99% rename from crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__hash_optional_missing.snap rename to crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__hash_optional_missing.snap index a8940a29c..7dda1da60 100644 --- a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__hash_optional_missing.snap +++ b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__hash_optional_missing.snap @@ -1,5 +1,6 @@ --- source: crates/fyn-resolver/src/lock/mod.rs +assertion_line: 6806 expression: result --- Ok( diff --git a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__hash_optional_present.snap b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__hash_optional_present.snap similarity index 99% rename from crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__hash_optional_present.snap rename to crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__hash_optional_present.snap index b9d11e234..ae5f5b18a 100644 --- a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__hash_optional_present.snap +++ b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__hash_optional_present.snap @@ -1,5 +1,6 @@ --- source: crates/fyn-resolver/src/lock/mod.rs +assertion_line: 6822 expression: result --- Ok( diff --git a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__hash_required_present.snap b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__hash_required_present.snap similarity index 99% rename from crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__hash_required_present.snap rename to crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__hash_required_present.snap index 398ca84c8..00de26f96 100644 --- a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__hash_required_present.snap +++ b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__hash_required_present.snap @@ -1,5 +1,6 @@ --- source: crates/fyn-resolver/src/lock/mod.rs +assertion_line: 6838 expression: result --- Ok( diff --git a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__missing_dependency_version_unambiguous.snap b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__missing_dependency_source_unambiguous.snap similarity index 99% rename from crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__missing_dependency_version_unambiguous.snap rename to crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__missing_dependency_source_unambiguous.snap index 31f9fabb1..64fd7caf3 100644 --- a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__missing_dependency_version_unambiguous.snap +++ b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__missing_dependency_source_unambiguous.snap @@ -1,5 +1,6 @@ --- source: crates/fyn-resolver/src/lock/mod.rs +assertion_line: 6614 expression: result --- Ok( diff --git a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__missing_dependency_source_version_unambiguous.snap b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__missing_dependency_source_version_unambiguous.snap similarity index 99% rename from crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__missing_dependency_source_version_unambiguous.snap rename to crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__missing_dependency_source_version_unambiguous.snap index 31f9fabb1..1466eb743 100644 --- a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__missing_dependency_source_version_unambiguous.snap +++ b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__missing_dependency_source_version_unambiguous.snap @@ -1,5 +1,6 @@ --- source: crates/fyn-resolver/src/lock/mod.rs +assertion_line: 6665 expression: result --- Ok( diff --git a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__missing_dependency_version_dynamic.snap b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__missing_dependency_version_dynamic.snap similarity index 99% rename from crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__missing_dependency_version_dynamic.snap rename to crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__missing_dependency_version_dynamic.snap index 4cbfee69e..6f8afcab7 100644 --- a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__missing_dependency_version_dynamic.snap +++ b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__missing_dependency_version_dynamic.snap @@ -1,5 +1,6 @@ --- source: crates/fyn-resolver/src/lock/mod.rs +assertion_line: 6790 expression: result --- Ok( diff --git a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__missing_dependency_source_unambiguous.snap b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__missing_dependency_version_unambiguous.snap similarity index 99% rename from crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__missing_dependency_source_unambiguous.snap rename to crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__missing_dependency_version_unambiguous.snap index 31f9fabb1..ef6c1a82d 100644 --- a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__missing_dependency_source_unambiguous.snap +++ b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__missing_dependency_version_unambiguous.snap @@ -1,5 +1,6 @@ --- source: crates/fyn-resolver/src/lock/mod.rs +assertion_line: 6640 expression: result --- Ok( diff --git a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__source_direct_has_subdir.snap b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__source_direct_has_subdir.snap similarity index 99% rename from crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__source_direct_has_subdir.snap rename to crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__source_direct_has_subdir.snap index 5c208a0e5..ee1e56889 100644 --- a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__source_direct_has_subdir.snap +++ b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__source_direct_has_subdir.snap @@ -1,5 +1,6 @@ --- source: crates/fyn-resolver/src/lock/mod.rs +assertion_line: 6868 expression: result --- Ok( diff --git a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__source_direct_no_subdir.snap b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__source_direct_no_subdir.snap similarity index 99% rename from crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__source_direct_no_subdir.snap rename to crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__source_direct_no_subdir.snap index 11a0275da..50a91bf61 100644 --- a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__source_direct_no_subdir.snap +++ b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__source_direct_no_subdir.snap @@ -1,5 +1,6 @@ --- source: crates/fyn-resolver/src/lock/mod.rs +assertion_line: 6853 expression: result --- Ok( diff --git a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__source_directory.snap b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__source_directory.snap similarity index 99% rename from crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__source_directory.snap rename to crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__source_directory.snap index 41e04dbd6..4b74174bb 100644 --- a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__source_directory.snap +++ b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__source_directory.snap @@ -1,5 +1,6 @@ --- source: crates/fyn-resolver/src/lock/mod.rs +assertion_line: 6883 expression: result --- Ok( diff --git a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__source_editable.snap b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__source_editable.snap similarity index 99% rename from crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__source_editable.snap rename to crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__source_editable.snap index db65ad762..8f92d8221 100644 --- a/crates/fyn-resolver/src/lock/snapshots/uv_resolver__lock__tests__source_editable.snap +++ b/crates/fyn-resolver/src/lock/snapshots/fyn_resolver__lock__tests__source_editable.snap @@ -1,5 +1,6 @@ --- source: crates/fyn-resolver/src/lock/mod.rs +assertion_line: 6898 expression: result --- Ok( diff --git a/crates/fyn-resolver/src/universal_marker.rs b/crates/fyn-resolver/src/universal_marker.rs index e39d19971..b842f800c 100644 --- a/crates/fyn-resolver/src/universal_marker.rs +++ b/crates/fyn-resolver/src/universal_marker.rs @@ -710,8 +710,13 @@ impl<'a> ParsedRawExtra<'a> { /// /// If a conflict item isn't present in the map of known conflicts, it's assumed to be false in all /// environments. -pub(crate) fn resolve_conflicts( +/// +/// When `scope_package` is set, unencoded package extras like `extra == 'cpu'` are interpreted +/// relative to that package. Conflict-encoded extras and groups are resolved independent of +/// `scope_package`. +pub(crate) fn resolve_activated_extras( marker: MarkerTree, + scope_package: Option<&PackageName>, known_conflicts: &FxHashMap, ) -> MarkerTree { if marker.is_true() || marker.is_false() { @@ -805,6 +810,25 @@ pub(crate) fn resolve_conflicts( } } + // Search for an unencoded package extra in the current package scope. + if !found { + if let Some(package) = scope_package { + let conflict_item = ConflictItem::from((package.clone(), name.clone())); + if let Some(conflict_marker) = known_conflicts.get(&conflict_item) { + match operator { + ExtraOperator::Equal => { + or.and(*conflict_marker); + found = true; + } + ExtraOperator::NotEqual => { + or.and(conflict_marker.negate()); + found = true; + } + } + } + } + } + // If we didn't find the marker in the list of known conflicts, assume it's always // false. if !found { @@ -1014,7 +1038,7 @@ mod tests { fn resolve() { let known_conflicts = create_known_conflicts([("foo", "sys_platform == 'darwin'")]); let cm = MarkerTree::from_str("(python_version >= '3.10' and extra == 'extra-3-pkg-foo') or (python_version < '3.10' and extra != 'extra-3-pkg-foo')").unwrap(); - let cm = resolve_conflicts(cm, &known_conflicts); + let cm = resolve_activated_extras(cm, None, &known_conflicts); assert_eq!( cm.try_to_string().as_deref(), Some( @@ -1024,7 +1048,7 @@ mod tests { let cm = MarkerTree::from_str("python_version >= '3.10' and extra == 'extra-3-pkg-foo'") .unwrap(); - let cm = resolve_conflicts(cm, &known_conflicts); + let cm = resolve_activated_extras(cm, None, &known_conflicts); assert_eq!( cm.try_to_string().as_deref(), Some("python_full_version >= '3.10' and sys_platform == 'darwin'") @@ -1032,7 +1056,31 @@ mod tests { let cm = MarkerTree::from_str("python_version >= '3.10' and extra == 'extra-3-pkg-bar'") .unwrap(); - let cm = resolve_conflicts(cm, &known_conflicts); + let cm = resolve_activated_extras(cm, None, &known_conflicts); + assert!(cm.is_false()); + } + + #[test] + fn resolve_unencoded_package_extras() { + let known_conflicts = create_known_conflicts([("foo", "sys_platform == 'darwin'")]); + let package = create_package("pkg"); + + let cm = MarkerTree::from_str("python_version >= '3.10' and extra == 'foo'").unwrap(); + let cm = resolve_activated_extras(cm, Some(&package), &known_conflicts); + assert_eq!( + cm.try_to_string().as_deref(), + Some("python_full_version >= '3.10' and sys_platform == 'darwin'") + ); + + let cm = MarkerTree::from_str("python_version >= '3.10' and extra != 'foo'").unwrap(); + let cm = resolve_activated_extras(cm, Some(&package), &known_conflicts); + assert_eq!( + cm.try_to_string().as_deref(), + Some("python_full_version >= '3.10' and sys_platform != 'darwin'") + ); + + let cm = MarkerTree::from_str("python_version >= '3.10' and extra == 'bar'").unwrap(); + let cm = resolve_activated_extras(cm, Some(&package), &known_conflicts); assert!(cm.is_false()); } } diff --git a/crates/fyn-types/src/hash.rs b/crates/fyn-types/src/hash.rs index a8c6de855..507bd998c 100644 --- a/crates/fyn-types/src/hash.rs +++ b/crates/fyn-types/src/hash.rs @@ -1,3 +1,4 @@ +use std::fmt::Display; use std::str::FromStr; use std::sync::Arc; @@ -37,62 +38,57 @@ impl HashStrategy { Self::None => HashPolicy::None, Self::Generate(mode) => HashPolicy::Generate(*mode), Self::Verify(hashes) => { - if let Some(hashes) = hashes.get(&distribution.version_id()) { - HashPolicy::Validate(hashes.as_slice()) + let id = distribution.version_id(); + if let Some(hashes) = hashes.get(&id) { + hash_policy(&id, hashes.as_slice()) } else { HashPolicy::None } } - Self::Require(hashes) => HashPolicy::Validate( - hashes - .get(&distribution.version_id()) - .map(Vec::as_slice) - .unwrap_or_default(), - ), + Self::Require(hashes) => { + let id = distribution.version_id(); + hash_policy(&id, hashes.get(&id).map(Vec::as_slice).unwrap_or_default()) + } } } /// Return the [`HashPolicy`] for the given registry-based package. pub fn get_package(&self, name: &PackageName, version: &Version) -> HashPolicy<'_> { + let id = VersionId::from_registry(name.clone(), version.clone()); match self { Self::None => HashPolicy::None, Self::Generate(mode) => HashPolicy::Generate(*mode), Self::Verify(hashes) => { - if let Some(hashes) = - hashes.get(&VersionId::from_registry(name.clone(), version.clone())) - { - HashPolicy::Validate(hashes.as_slice()) + if let Some(hashes) = hashes.get(&id) { + HashPolicy::Any(hashes.as_slice()) } else { HashPolicy::None } } - Self::Require(hashes) => HashPolicy::Validate( - hashes - .get(&VersionId::from_registry(name.clone(), version.clone())) - .map(Vec::as_slice) - .unwrap_or_default(), - ), + Self::Require(hashes) => { + HashPolicy::Any(hashes.get(&id).map(Vec::as_slice).unwrap_or_default()) + } } } /// Return the [`HashPolicy`] for the given direct URL package. + /// + /// A direct URL identifies a single concrete artifact, so every provided digest must match. pub fn get_url(&self, url: &DisplaySafeUrl) -> HashPolicy<'_> { + let id = VersionId::from_url(url); match self { Self::None => HashPolicy::None, Self::Generate(mode) => HashPolicy::Generate(*mode), Self::Verify(hashes) => { - if let Some(hashes) = hashes.get(&VersionId::from_url(url)) { - HashPolicy::Validate(hashes.as_slice()) + if let Some(hashes) = hashes.get(&id) { + HashPolicy::All(hashes.as_slice()) } else { HashPolicy::None } } - Self::Require(hashes) => HashPolicy::Validate( - hashes - .get(&VersionId::from_url(url)) - .map(Vec::as_slice) - .unwrap_or_default(), - ), + Self::Require(hashes) => { + HashPolicy::All(hashes.get(&id).map(Vec::as_slice).unwrap_or_default()) + } } } @@ -118,6 +114,32 @@ impl HashStrategy { } } + /// Return a [`HashStrategy`] augmented with archive URL hashes discovered in additional + /// requirements after the initial command-line parse. + pub fn augment_with_requirements<'a>( + self, + requirements: impl Iterator, + ) -> Result { + Ok(match self { + Self::None => Self::None, + Self::Generate(mode) => Self::Generate(mode), + Self::Verify(existing) => { + if let Some(hashes) = Self::augment_hashes(existing.as_ref(), requirements)? { + Self::Verify(Arc::new(hashes)) + } else { + Self::Verify(existing) + } + } + Self::Require(existing) => { + if let Some(hashes) = Self::augment_hashes(existing.as_ref(), requirements)? { + Self::Require(Arc::new(hashes)) + } else { + Self::Require(existing) + } + } + }) + } + /// Generate the required hashes from a set of [`UnresolvedRequirement`] entries. /// /// When the environment is not given, this treats all marker expressions @@ -151,31 +173,24 @@ impl HashStrategy { continue; }; - let digests = if digests.is_empty() { - // If there are no hashes, and the distribution is URL-based, attempt to extract - // it from the fragment. - requirement - .hashes() - .map(HashDigests::from) - .map(|hashes| hashes.to_vec()) - .unwrap_or_default() - } else { - // Parse the hashes. - digests - .iter() - .map(|digest| HashDigest::from_str(digest)) - .collect::, _>>()? - }; + // Parse the hashes provided directly on the requirement, then merge in any hashes from + // the URL fragment. + let mut digests = digests + .iter() + .map(|digest| HashDigest::from_str(digest)) + .collect::, _>>()?; + if let Some(fragment_hashes) = requirement.hashes().map(HashDigests::from) { + merge_digests(&mut digests, fragment_hashes.iter(), requirement)?; + } if digests.is_empty() { continue; } - constraint_hashes.insert(id, digests); + merge_hashes(&mut constraint_hashes, id, digests, requirement)?; } - // For each requirement, map from name to allowed hashes. We use the last entry for each - // package. + // For each requirement, map from hash identity to allowed hashes. let mut requirement_hashes = FxHashMap::>::default(); for (requirement, digests) in requirements { if !requirement @@ -201,30 +216,28 @@ impl HashStrategy { } UnresolvedRequirement::Unnamed(requirement) => { // Direct URLs are always allowed. - VersionId::from_url(&requirement.url.verbatim) + VersionId::from_parsed_url(&requirement.url.parsed_url) } }; - let digests = if digests.is_empty() { - // If there are no hashes, and the distribution is URL-based, attempt to extract - // it from the fragment. - requirement - .hashes() - .map(HashDigests::from) - .map(|hashes| hashes.to_vec()) - .unwrap_or_default() - } else { - // Parse the hashes. - digests - .iter() - .map(|digest| HashDigest::from_str(digest)) - .collect::, _>>()? - }; + // Parse the hashes provided directly on the requirement, then merge in any hashes from + // the URL fragment. + let mut digests = digests + .iter() + .map(|digest| HashDigest::from_str(digest)) + .collect::, _>>()?; + if let Some(fragment_hashes) = requirement.hashes().map(HashDigests::from) { + merge_digests(&mut digests, fragment_hashes.iter(), requirement)?; + } let digests = if let Some(constraint) = constraint_hashes.remove(&id) { if digests.is_empty() { // If there are _only_ hashes on the constraints, use them. constraint + } else if matches!(id, VersionId::ArchiveUrl { .. }) { + let mut merged = digests; + merge_digests(&mut merged, &constraint, requirement)?; + merged } else { // If there are constraint and requirement hashes, take the intersection. let intersection: Vec<_> = digests @@ -254,7 +267,7 @@ impl HashStrategy { continue; } - requirement_hashes.insert(id, digests); + merge_hashes(&mut requirement_hashes, id, digests, requirement)?; } // Merge the hashes, preferring requirements over constraints, since overlapping @@ -296,7 +309,53 @@ impl HashStrategy { } } - /// Pin a [`Requirement`] to a [`PackageId`], if possible. + /// Augment an existing set of hashes with archive URL hashes discovered in additional + /// requirements. + /// + /// Archive URL requirements are keyed by a [`VersionId`] so that requirements that refer to + /// the same underlying archive but differ only in hash fragments are merged onto the same + /// digest set. + /// + /// Returns `Ok(None)` if no new hashes were added or updated. + fn augment_hashes<'a>( + existing: &FxHashMap>, + requirements: impl Iterator, + ) -> Result>>, HashStrategyError> { + let mut hashes = None; + + for requirement in requirements { + let Some((id, digests)) = Self::requirement_hashes(requirement) else { + continue; + }; + let current = hashes.as_ref().unwrap_or(existing); + let current_digests = current.get(&id); + let mut merged = current_digests.cloned().unwrap_or_default(); + merge_digests(&mut merged, &digests, requirement)?; + + if current_digests.map(Vec::as_slice) == Some(merged.as_slice()) { + continue; + } + + hashes + .get_or_insert_with(|| existing.clone()) + .insert(id, merged); + } + + Ok(hashes) + } + + /// Extract the archive URL hash target and digests for a requirement, if any. + fn requirement_hashes(requirement: &Requirement) -> Option<(VersionId, Vec)> { + let mut digests = HashDigests::from(requirement.hashes()?).to_vec(); + if digests.is_empty() { + return None; + } + digests.sort_unstable(); + let id = Self::pin(requirement)?; + Some((id, digests)) + } + + /// Pin a [`Requirement`] to a [`VersionId`], if possible. fn pin(requirement: &Requirement) -> Option { match &requirement.source { RequirementSource::Registry { specifier, .. } => { @@ -315,18 +374,98 @@ impl HashStrategy { specifier.version().clone(), )) } - RequirementSource::Url { url, .. } - | RequirementSource::Git { url, .. } - | RequirementSource::Path { url, .. } - | RequirementSource::Directory { url, .. } => Some(VersionId::from_url(url)), + RequirementSource::Url { + location, + subdirectory, + .. + } => Some(VersionId::from_archive(location, subdirectory.as_deref())), + RequirementSource::Git { + git, subdirectory, .. + } => Some(VersionId::from_git(git, subdirectory.as_deref())), + RequirementSource::Path { install_path, .. } => { + Some(VersionId::from_path(install_path)) + } + RequirementSource::Directory { install_path, .. } => { + Some(VersionId::from_directory(install_path)) + } } } } +fn hash_policy<'a>(id: &VersionId, digests: &'a [HashDigest]) -> HashPolicy<'a> { + match id { + VersionId::NameVersion(..) => HashPolicy::Any(digests), + VersionId::ArchiveUrl { .. } + | VersionId::Git { .. } + | VersionId::Path(..) + | VersionId::Directory(..) + | VersionId::Unknown(..) => HashPolicy::All(digests), + } +} + +/// Merge repeated hashes for a requirement or constraint into the hash map. +fn merge_hashes( + hashes: &mut FxHashMap>, + id: VersionId, + incoming: Vec, + requirement: impl Display, +) -> Result<(), HashStrategyError> { + if incoming.is_empty() { + return Ok(()); + } + + if !matches!(&id, VersionId::ArchiveUrl { .. }) { + hashes.insert(id, incoming); + return Ok(()); + } + + if let Some(existing) = hashes.get_mut(&id) { + return merge_digests(existing, &incoming, requirement); + } + + let mut merged = Vec::new(); + merge_digests(&mut merged, &incoming, requirement)?; + hashes.insert(id, merged); + Ok(()) +} + +/// Merge `incoming` digests into `existing`. +/// +/// Exact duplicates are ignored. Digests for different algorithms are accumulated. If the same +/// algorithm appears with two different values, returns +/// [`HashStrategyError::ConflictingArchiveUrlHashes`]. +fn merge_digests<'a>( + existing: &mut Vec, + incoming: impl IntoIterator, + requirement: impl Display, +) -> Result<(), HashStrategyError> { + for digest in incoming { + match existing + .iter() + .find(|candidate| candidate.algorithm == digest.algorithm) + { + Some(candidate) if candidate == digest => {} + Some(conflict) => { + return Err(HashStrategyError::ConflictingArchiveUrlHashes( + requirement.to_string(), + conflict.clone(), + digest.clone(), + )); + } + None => existing.push(digest.clone()), + } + } + existing.sort_unstable(); + + Ok(()) +} + #[derive(thiserror::Error, Debug)] pub enum HashStrategyError { #[error(transparent)] Hash(#[from] HashError), + #[error("Conflicting archive URL hashes for `{0}`: `{1}` conflicts with `{2}`")] + ConflictingArchiveUrlHashes(String, HashDigest, HashDigest), #[error( "In `{1}` mode, all requirements must have their versions pinned with `==`, but found: {0}" )] @@ -338,3 +477,74 @@ pub enum HashStrategyError { )] NoIntersection(String, HashCheckingMode), } + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use fyn_distribution_filename::DistExtension; + use fyn_distribution_types::{ + HashPolicy, Requirement, RequirementSource, UnresolvedRequirement, + }; + use fyn_pypi_types::HashDigest; + + use super::HashStrategy; + + fn requirement(url: &str) -> Requirement { + Requirement { + name: "anyio".parse().unwrap(), + extras: Box::default(), + groups: Box::default(), + marker: "python_version >= '3.8'".parse().unwrap(), + source: RequirementSource::Url { + location: "https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl" + .parse() + .unwrap(), + subdirectory: None, + ext: DistExtension::Wheel, + url: url.parse().unwrap(), + }, + origin: None, + } + } + + #[test] + fn from_requirements_merges_direct_url_hashes_across_fragments() { + let first = UnresolvedRequirement::Named(requirement( + "https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl#sha256=cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f", + )); + let second = UnresolvedRequirement::Named(requirement( + "https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl#sha512=f30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2", + )); + + let hasher = HashStrategy::from_requirements( + [(&first, &[][..]), (&second, &[][..])].into_iter(), + std::iter::empty(), + None, + fyn_configuration::HashCheckingMode::Require, + ) + .unwrap(); + + let mut expected = vec![ + HashDigest::from_str( + "sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f", + ) + .unwrap(), + HashDigest::from_str( + "sha512:f30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2", + ) + .unwrap(), + ]; + expected.sort_unstable(); + + for requirement in [&first, &second] { + let UnresolvedRequirement::Named(requirement) = requirement else { + panic!("expected named requirement"); + }; + let RequirementSource::Url { url, .. } = &requirement.source else { + panic!("expected direct URL requirement"); + }; + assert_eq!(hasher.get_url(url), HashPolicy::All(expected.as_slice())); + } + } +} diff --git a/crates/fyn-workspace/src/workspace.rs b/crates/fyn-workspace/src/workspace.rs index 800097195..1f32ba6db 100644 --- a/crates/fyn-workspace/src/workspace.rs +++ b/crates/fyn-workspace/src/workspace.rs @@ -15,7 +15,7 @@ use fyn_fs::{CWD, Simplified}; use fyn_normalize::{DEV_DEPENDENCIES, GroupName, PackageName}; use fyn_pep440::VersionSpecifiers; use fyn_pep508::{MarkerTree, VerbatimUrl}; -use fyn_pypi_types::{Conflicts, SupportedEnvironments, VerbatimParsedUrl}; +use fyn_pypi_types::{ConflictError, Conflicts, SupportedEnvironments, VerbatimParsedUrl}; use fyn_static::EnvVars; use fyn_warnings::warn_user_once; @@ -84,6 +84,8 @@ pub enum WorkspaceError { Io(#[from] std::io::Error), #[error("Failed to parse: `{}`", _0.user_display())] Toml(PathBuf, #[source] Box), + #[error(transparent)] + Conflicts(#[from] ConflictError), #[error("Failed to normalize workspace member path")] Normalize(#[source] std::io::Error), } @@ -535,12 +537,22 @@ impl Workspace { } /// Returns the set of conflicts for the workspace. - pub fn conflicts(&self) -> Conflicts { + pub fn conflicts(&self) -> Result { let mut conflicting = Conflicts::empty(); + if self.is_non_project() { + if let Some(root_conflicts) = self + .pyproject_toml + .tool_fyn() + .and_then(|fyn| fyn.conflicts.as_ref()) + { + let mut root_conflicts = root_conflicts.to_conflicts()?; + conflicting.append(&mut root_conflicts); + } + } for member in self.packages.values() { conflicting.append(&mut member.pyproject_toml.conflicts()); } - conflicting + Ok(conflicting) } /// Returns an iterator over the `requires-python` values for each member of the workspace. diff --git a/crates/fyn/src/commands/pip/compile.rs b/crates/fyn/src/commands/pip/compile.rs index bbad0e42d..96059d1a9 100644 --- a/crates/fyn/src/commands/pip/compile.rs +++ b/crates/fyn/src/commands/pip/compile.rs @@ -595,7 +595,7 @@ pub(crate) async fn pip_compile( ) .await { - Ok(resolution) => resolution, + Ok((resolution, _)) => resolution, Err(err) => { return diagnostics::OperationDiagnostic::native_tls(client_builder.is_native_tls()) .report(err) diff --git a/crates/fyn/src/commands/pip/download.rs b/crates/fyn/src/commands/pip/download.rs index d2814ea50..0e558e615 100644 --- a/crates/fyn/src/commands/pip/download.rs +++ b/crates/fyn/src/commands/pip/download.rs @@ -439,7 +439,7 @@ pub(crate) async fn pip_download( ) .await { - Ok(resolution) => Resolution::from(resolution), + Ok((resolution, _)) => Resolution::from(resolution), Err(err) => { return diagnostics::OperationDiagnostic::native_tls(client_builder.is_native_tls()) .report(err) diff --git a/crates/fyn/src/commands/pip/install.rs b/crates/fyn/src/commands/pip/install.rs index 5c493956b..0d6eee01f 100644 --- a/crates/fyn/src/commands/pip/install.rs +++ b/crates/fyn/src/commands/pip/install.rs @@ -574,7 +574,7 @@ pub(crate) async fn pip_install( .build(); // Resolve the requirements. - let resolution = match operations::resolve( + let (resolution, hasher) = match operations::resolve( requirements, constraints, overrides, @@ -605,7 +605,7 @@ pub(crate) async fn pip_install( ) .await { - Ok(graph) => Resolution::from(graph), + Ok((graph, hasher)) => (Resolution::from(graph), hasher), Err(err) => { return diagnostics::OperationDiagnostic::native_tls( client_builder.is_native_tls(), diff --git a/crates/fyn/src/commands/pip/operations.rs b/crates/fyn/src/commands/pip/operations.rs index 35ad94807..d46b1563f 100644 --- a/crates/fyn/src/commands/pip/operations.rs +++ b/crates/fyn/src/commands/pip/operations.rs @@ -132,7 +132,7 @@ pub(crate) async fn resolve( options: Options, logger: Box, printer: Printer, -) -> Result { +) -> Result<(ResolverOutput, HashStrategy), Error> { let start = std::time::Instant::now(); // Resolve the requirements from the provided sources. @@ -265,6 +265,11 @@ pub(crate) async fn resolve( requirements }; + // Incorporate hashes from requirements discovered while resolving source trees and groups. + let mut hasher = hasher + .clone() + .augment_with_requirements(requirements.iter())?; + // Resolve the overrides from the provided sources. let overrides = { // Partition the overrides into named and unnamed requirements. @@ -284,7 +289,7 @@ pub(crate) async fn resolve( if !unnamed.is_empty() { overrides.extend( NamedRequirementsResolver::new( - hasher, + &hasher, index, DistributionDatabase::new( client, @@ -315,11 +320,11 @@ pub(crate) async fn resolve( // Determine any lookahead requirements. let lookaheads = match options.dependency_mode { DependencyMode::Transitive => { - LookaheadResolver::new( + let (lookaheads, updated_hasher) = LookaheadResolver::new( &requirements, &constraints, &overrides, - hasher, + &hasher, index, DistributionDatabase::new( client, @@ -329,7 +334,9 @@ pub(crate) async fn resolve( ) .with_reporter(Arc::new(ResolverReporter::from(printer))) .resolve(&resolver_env) - .await? + .await?; + hasher = updated_hasher; + lookaheads } DependencyMode::Direct => Vec::new(), }; @@ -370,7 +377,7 @@ pub(crate) async fn resolve( tags, flat_index, index, - hasher, + &hasher, build_dispatch, installed_packages, DistributionDatabase::new( @@ -386,7 +393,7 @@ pub(crate) async fn resolve( logger.on_complete(resolution.len(), start, printer)?; - Ok(resolution) + Ok((resolution, hasher)) } #[derive(Debug, Clone, Copy)] diff --git a/crates/fyn/src/commands/pip/sync.rs b/crates/fyn/src/commands/pip/sync.rs index 948ca38af..d39d12d5d 100644 --- a/crates/fyn/src/commands/pip/sync.rs +++ b/crates/fyn/src/commands/pip/sync.rs @@ -465,7 +465,7 @@ pub(crate) async fn pip_sync( .build_options(build_options.clone()) .build(); - let resolution = match operations::resolve( + let (resolution, hasher) = match operations::resolve( requirements, constraints, overrides, @@ -496,7 +496,7 @@ pub(crate) async fn pip_sync( ) .await { - Ok(resolution) => Resolution::from(resolution), + Ok((resolution, hasher)) => (Resolution::from(resolution), hasher), Err(err) => { return diagnostics::OperationDiagnostic::native_tls( client_builder.is_native_tls(), diff --git a/crates/fyn/src/commands/pip/wheel.rs b/crates/fyn/src/commands/pip/wheel.rs index 2ad2ef8fb..7288d1240 100644 --- a/crates/fyn/src/commands/pip/wheel.rs +++ b/crates/fyn/src/commands/pip/wheel.rs @@ -438,7 +438,7 @@ pub(crate) async fn pip_wheel( ) .await { - Ok(resolution) => Resolution::from(resolution), + Ok((resolution, _)) => Resolution::from(resolution), Err(err) => { return diagnostics::OperationDiagnostic::native_tls(client_builder.is_native_tls()) .report(err) diff --git a/crates/fyn/src/commands/project/lock.rs b/crates/fyn/src/commands/project/lock.rs index 80ef849e6..65c0a37b4 100644 --- a/crates/fyn/src/commands/project/lock.rs +++ b/crates/fyn/src/commands/project/lock.rs @@ -543,7 +543,7 @@ async fn do_lock( .collect::, ProjectError>>()?; // Collect the conflicts. - let mut conflicts = target.conflicts(); + let mut conflicts = target.conflicts()?; if let LockTarget::Workspace(workspace) = target { if let Some(groups) = &workspace.pyproject_toml().dependency_groups { if let Some(project) = &workspace.pyproject_toml().project { @@ -915,7 +915,7 @@ async fn do_lock( ); // Resolve the requirements. - let resolution = pip::operations::resolve( + let (resolution, _) = pip::operations::resolve( ExtrasResolver::new(&hasher, state.index(), database) .with_reporter(Arc::new(ResolverReporter::from(printer))) .resolve(target.members_requirements()) diff --git a/crates/fyn/src/commands/project/lock_target.rs b/crates/fyn/src/commands/project/lock_target.rs index 58bce6ce6..cb2477e42 100644 --- a/crates/fyn/src/commands/project/lock_target.rs +++ b/crates/fyn/src/commands/project/lock_target.rs @@ -207,10 +207,10 @@ impl<'lock> LockTarget<'lock> { } /// Returns the set of conflicts for the [`LockTarget`]. - pub(crate) fn conflicts(self) -> Conflicts { + pub(crate) fn conflicts(self) -> Result { match self { - Self::Workspace(workspace) => workspace.conflicts(), - Self::Script(_) => Conflicts::empty(), + Self::Workspace(workspace) => Ok(workspace.conflicts()?), + Self::Script(_) => Ok(Conflicts::empty()), } } diff --git a/crates/fyn/src/commands/project/mod.rs b/crates/fyn/src/commands/project/mod.rs index 0dffe839f..b527e6347 100644 --- a/crates/fyn/src/commands/project/mod.rs +++ b/crates/fyn/src/commands/project/mod.rs @@ -2202,7 +2202,8 @@ pub(crate) async fn resolve_environment( logger, printer, ) - .await?) + .await? + .0) } /// Sync a [`PythonEnvironment`] with a set of resolved requirements. @@ -2568,7 +2569,7 @@ pub(crate) async fn update_environment( ); // Resolve the requirements. - let resolution = match pip::operations::resolve( + let (resolution, hasher) = match pip::operations::resolve( requirements, constraints, overrides, @@ -2599,7 +2600,7 @@ pub(crate) async fn update_environment( ) .await { - Ok(resolution) => Resolution::from(resolution), + Ok((resolution, hasher)) => (Resolution::from(resolution), hasher), Err(err) => return Err(err.into()), }; diff --git a/crates/fyn/tests/it/export.rs b/crates/fyn/tests/it/export.rs index 364db9f54..b71afd584 100644 --- a/crates/fyn/tests/it/export.rs +++ b/crates/fyn/tests/it/export.rs @@ -1900,6 +1900,9 @@ fn requirements_txt_relative_path() -> Result<()> { [tool.fyn.sources] dependency = { path = "../dependency" } + [tool.fyn.pip] + pip-in-project = "allow" + [build-system] requires = ["fyn-build>=0.7,<10000"] build-backend = "fyn_build" @@ -3673,6 +3676,145 @@ fn requirements_txt_cyclic_dependencies() -> Result<()> { Ok(()) } +#[test] +fn pylock_workspace_member_conflict_markers() -> Result<()> { + let context = fyn_test::test_context!("3.12").with_exclude_newer("2025-01-30T00:00:00Z"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [project] + name = "root" + version = "0.1.0" + requires-python = ">=3.12.0" + dependencies = ["member"] + + [project.optional-dependencies] + cpu = ["member[cpu]"] + cu124 = ["member[cu124]"] + + [tool.fyn.workspace] + members = ["packages/member"] + + [tool.fyn.sources] + member = { workspace = true } + "#, + )?; + + let member = context.temp_dir.child("packages/member"); + member.create_dir_all()?; + member.child("pyproject.toml").write_str( + r#" + [project] + name = "member" + version = "0.1.0" + requires-python = ">=3.12.0" + dependencies = [] + + [project.optional-dependencies] + cpu = ["torch>=2.6.0"] + cu124 = ["torch>=2.6.0"] + + [tool.fyn] + conflicts = [ + [ + { extra = "cpu" }, + { extra = "cu124" }, + ], + ] + + [tool.fyn.sources] + torch = [ + { index = "pytorch-cpu", extra = "cpu" }, + { index = "pytorch-cu124", extra = "cu124" }, + ] + + [[tool.fyn.index]] + name = "pytorch-cpu" + url = "https://astral-sh.github.io/pytorch-mirror/whl/cpu" + explicit = true + + [[tool.fyn.index]] + name = "pytorch-cu124" + url = "https://astral-sh.github.io/pytorch-mirror/whl/cu124" + explicit = true + "#, + )?; + + context.lock().assert().success(); + + let output = context + .export() + .arg("--format") + .arg("pylock.toml") + .arg("--extra") + .arg("cpu") + .assert() + .success() + .get_output() + .clone(); + + let stdout = apply_filters(String::from_utf8(output.stdout).unwrap(), context.filters()); + let stderr = apply_filters(String::from_utf8(output.stderr).unwrap(), context.filters()); + + assert_snapshot!(stderr, @"Resolved 28 packages in [TIME] +"); + + assert!( + !stdout.contains("https://astral-sh.github.io/pytorch-mirror/whl/cu124"), + "{stdout}" + ); + + let torch_blocks = stdout + .split("[[packages]]") + .filter_map(|block| { + let block = block.trim(); + if block.starts_with("name = \"member\"") || block.starts_with("name = \"torch\"") { + Some(block) + } else { + None + } + }) + .collect::>() + .join("\n\n[[packages]]\n"); + + assert_snapshot!( + torch_blocks, + @r#" + name = "member" + directory = { path = "packages/member", editable = true } + + [[packages]] + name = "torch" + version = "2.6.0" + marker = "sys_platform == 'darwin'" + index = "https://astral-sh.github.io/pytorch-mirror/whl/cpu" + wheels = [ + { url = "https://download.pytorch.org/whl/cpu/torch-2.6.0-cp312-none-macosx_11_0_arm64.whl", upload-time = 2025-01-29T22:50:59Z, hashes = {} }, + { url = "https://download.pytorch.org/whl/cpu/torch-2.6.0-cp313-none-macosx_11_0_arm64.whl", upload-time = 2025-01-29T22:50:59Z, hashes = {} }, + ] + + [[packages]] + name = "torch" + version = "2.6.0+cpu" + marker = "sys_platform != 'darwin'" + index = "https://astral-sh.github.io/pytorch-mirror/whl/cpu" + wheels = [ + { url = "https://download.pytorch.org/whl/cpu/torch-2.6.0%2Bcpu-cp312-cp312-linux_x86_64.whl", upload-time = 2025-01-29T22:50:59Z, hashes = {} }, + { url = "https://download.pytorch.org/whl/cpu/torch-2.6.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", upload-time = 2025-01-29T22:50:59Z, hashes = {} }, + { url = "https://download.pytorch.org/whl/cpu/torch-2.6.0%2Bcpu-cp312-cp312-win_amd64.whl", upload-time = 2025-01-29T22:50:59Z, hashes = {} }, + { url = "https://download.pytorch.org/whl/cpu/torch-2.6.0%2Bcpu-cp313-cp313-linux_x86_64.whl", upload-time = 2025-01-29T22:50:59Z, hashes = {} }, + { url = "https://download.pytorch.org/whl/cpu/torch-2.6.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", upload-time = 2025-01-29T22:50:59Z, hashes = {} }, + { url = "https://download.pytorch.org/whl/cpu/torch-2.6.0%2Bcpu-cp313-cp313-win_amd64.whl", upload-time = 2025-01-29T22:50:59Z, hashes = {} }, + { url = "https://download.pytorch.org/whl/cpu/torch-2.6.0%2Bcpu-cp313-cp313t-linux_x86_64.whl", upload-time = 2025-01-29T22:50:59Z, hashes = {} }, + { url = "https://download.pytorch.org/whl/cpu/torch-2.6.0%2Bcpu-cp313-cp313t-manylinux_2_28_aarch64.whl", upload-time = 2025-01-29T22:50:59Z, hashes = {} }, + ] + "# + ); + + Ok(()) +} + /// Export requirements in the presence of a cycle, with conflicts enabled. #[test] fn requirements_txt_cyclic_dependencies_conflict() -> Result<()> { diff --git a/crates/fyn/tests/it/lock.rs b/crates/fyn/tests/it/lock.rs index 676d78ab6..04459ecc2 100644 --- a/crates/fyn/tests/it/lock.rs +++ b/crates/fyn/tests/it/lock.rs @@ -18204,6 +18204,166 @@ fn lock_non_project_sources() -> Result<()> { Ok(()) } +/// Lock a non-project workspace root with conflicts declared between members. +#[test] +fn lock_non_project_member_conflicts() -> Result<()> { + let context = fyn_test::test_context!("3.12"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [tool.fyn.workspace] + members = ["member-a", "member-b"] + "#, + )?; + + let member_a = context.temp_dir.child("member-a"); + member_a.create_dir_all()?; + member_a.child("pyproject.toml").write_str( + r#" + [project] + name = "member-a" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = ["sortedcontainers==2.3.0"] + + [build-system] + requires = ["fyn-build>=0.7,<10000"] + build-backend = "fyn_build" + "#, + )?; + + let member_b = context.temp_dir.child("member-b"); + member_b.create_dir_all()?; + member_b.child("pyproject.toml").write_str( + r#" + [project] + name = "member-b" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = ["sortedcontainers==2.4.0"] + + [build-system] + requires = ["fyn-build>=0.7,<10000"] + build-backend = "fyn_build" + "#, + )?; + + fyn_snapshot!(context.filters(), context.lock(), @" + success: false + exit_code: 1 + ----- stdout ----- + + ----- stderr ----- + × No solution found when resolving dependencies: + ╰─▶ Because member-a depends on sortedcontainers==2.3.0 and member-b depends on sortedcontainers==2.4.0, we can conclude that member-a and member-b are incompatible. + And because your workspace requires member-a and member-b, we can conclude that your workspace's requirements are unsatisfiable. + "); + + pyproject_toml.write_str( + r#" + [tool.fyn.workspace] + members = ["member-a", "member-b"] + + [tool.fyn] + conflicts = [ + [ + { package = "member-a" }, + { package = "member-b" }, + ], + ] + "#, + )?; + + fyn_snapshot!(context.filters(), context.lock(), @" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + warning: Declaring conflicts for packages (`package = ...`) is experimental and may change without warning. Pass `--preview-features package-conflicts` to disable this warning. + Resolved 4 packages in [TIME] + "); + + let lock = context.read("fyn.lock"); + assert!(lock.contains("conflicts = [[")); + assert!(lock.contains("{ package = \"member-a\" }")); + assert!(lock.contains("{ package = \"member-b\" }")); + assert!(lock.contains("name = \"sortedcontainers\"\nversion = \"2.3.0\"")); + assert!(lock.contains("name = \"sortedcontainers\"\nversion = \"2.4.0\"")); + + fyn_snapshot!(context.filters(), context.lock().arg("--locked"), @" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + warning: Declaring conflicts for packages (`package = ...`) is experimental and may change without warning. Pass `--preview-features package-conflicts` to disable this warning. + Resolved 4 packages in [TIME] + "); + + fyn_snapshot!(context.filters(), context.sync().arg("--frozen").arg("--all-packages"), @" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + error: Package `member-a` and package `member-b` are incompatible with the declared conflicts: {member-a, member-b} + "); + + Ok(()) +} + +/// Locking a non-project workspace root should reject conflicts that omit +/// `package = ...`, since there is no root project name to infer. +#[test] +fn lock_non_project_member_conflicts_missing_package() -> Result<()> { + let context = fyn_test::test_context!("3.12"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [tool.fyn.workspace] + members = ["member-a"] + + [tool.fyn] + conflicts = [ + [ + { extra = "foo" }, + { package = "member-a" }, + ], + ] + "#, + )?; + + let member_a = context.temp_dir.child("member-a"); + member_a.create_dir_all()?; + member_a.child("pyproject.toml").write_str( + r#" + [project] + name = "member-a" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = [] + + [build-system] + requires = ["fyn-build>=0.7,<10000"] + build-backend = "fyn_build" + "#, + )?; + + fyn_snapshot!(context.filters(), context.lock(), @r#" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + error: Expected `package` field in conflicting entry: { extra = "foo" } + "#); + + Ok(()) +} + /// `coverage` defines a `toml` extra, but it doesn't enable any dependencies after Python 3.11. #[test] fn lock_dropped_dev_extra() -> Result<()> { diff --git a/crates/fyn/tests/it/pip_compile.rs b/crates/fyn/tests/it/pip_compile.rs index 95e811068..d19148c8e 100644 --- a/crates/fyn/tests/it/pip_compile.rs +++ b/crates/fyn/tests/it/pip_compile.rs @@ -4237,8 +4237,8 @@ fn override_dependency_from_specific_uv_toml() -> Result<()> { )?; let _ = context.temp_dir.child("fyn").create_dir_all(); - let uv_toml: assert_fs::fixture::ChildPath = context.temp_dir.child("uv").child("uv.toml"); - uv_toml.write_str( + let fyn_toml: assert_fs::fixture::ChildPath = context.temp_dir.child("fyn").child("fyn.toml"); + fyn_toml.write_str( r#" override-dependencies = [ "werkzeug==2.3.0" @@ -12710,7 +12710,7 @@ fn compile_index_url_first_match_base() -> Result<()> { × No solution found when resolving dependencies: ╰─▶ Because there is no version of jinja2==3.1.0 and you require jinja2==3.1.0, we can conclude that your requirements are unsatisfiable. - hint: `jinja2` was found on https://astral-sh.github.io/pytorch-mirror/whl/cpu, but not at the requested version (jinja2==3.1.0). A compatible version may be available on a subsequent index (e.g., https://pypi.org/simple). By default, uv will only consider versions that are published on the first index that contains a given package, to avoid dependency confusion attacks. If all indexes are equally trusted, use `--index-strategy unsafe-best-match` to consider all versions from all indexes, regardless of the order in which they were defined. + hint: `jinja2` was found on https://astral-sh.github.io/pytorch-mirror/whl/cpu, but not at the requested version (jinja2==3.1.0). A compatible version may be available on a subsequent index (e.g., https://pypi.org/simple). By default, fyn will only consider versions that are published on the first index that contains a given package, to avoid dependency confusion attacks. If all indexes are equally trusted, use `--index-strategy unsafe-best-match` to consider all versions from all indexes, regardless of the order in which they were defined. " ); @@ -12744,7 +12744,7 @@ fn compile_index_url_first_match_marker() -> Result<()> { × No solution found when resolving dependencies: ╰─▶ Because there is no version of jinja2{sys_platform == 'linux'}==3.1.0 and you require jinja2{sys_platform == 'linux'}==3.1.0, we can conclude that your requirements are unsatisfiable. - hint: `jinja2` was found on https://astral-sh.github.io/pytorch-mirror/whl/cpu, but not at the requested version (jinja2==3.1.0). A compatible version may be available on a subsequent index (e.g., https://pypi.org/simple). By default, uv will only consider versions that are published on the first index that contains a given package, to avoid dependency confusion attacks. If all indexes are equally trusted, use `--index-strategy unsafe-best-match` to consider all versions from all indexes, regardless of the order in which they were defined. + hint: `jinja2` was found on https://astral-sh.github.io/pytorch-mirror/whl/cpu, but not at the requested version (jinja2==3.1.0). A compatible version may be available on a subsequent index (e.g., https://pypi.org/simple). By default, fyn will only consider versions that are published on the first index that contains a given package, to avoid dependency confusion attacks. If all indexes are equally trusted, use `--index-strategy unsafe-best-match` to consider all versions from all indexes, regardless of the order in which they were defined. " ); @@ -16880,6 +16880,9 @@ fn pep_751_compile_directory() -> Result<()> { [tool.fyn.sources] foo = { path = "foo" } + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?; diff --git a/crates/fyn/tests/it/pip_install.rs b/crates/fyn/tests/it/pip_install.rs index ed873522e..d3ce4e1b8 100644 --- a/crates/fyn/tests/it/pip_install.rs +++ b/crates/fyn/tests/it/pip_install.rs @@ -10179,6 +10179,147 @@ fn static_metadata_already_installed() -> Result<()> { Ok(()) } +/// Regression test for: +#[test] +fn direct_url_hash_source_tree_dependency() -> Result<()> { + let context = fyn_test::test_context!("3.12"); + + context.temp_dir.child("pyproject.toml").write_str(indoc! {r#" + [project] + name = "pylock" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = [ + "protobug @ https://files.pythonhosted.org/packages/f2/cc/db26b91cddffbcf0c6df7834fd642578f737fe34197635ae8ea64643a35f/protobug-0.3.0-py3-none-any.whl#sha256=ee81583f376bb38e5e7af425d2453e5e8d4b57bfbf45e5dba1a75329c2026520", + ] + + [tool.fyn.pip] + pip-in-project = "allow" + + [build-system] + requires = ["fyn-build>=0.7,<10000"] + build-backend = "fyn_build" + "#})?; + context + .temp_dir + .child("src") + .child("pylock") + .child("__init__.py") + .touch()?; + + fyn_snapshot!(context.filters(), context.pip_install() + .arg("."), @" + success: false + exit_code: 1 + ----- stdout ----- + + ----- stderr ----- + Resolved 2 packages in [TIME] + × Failed to download `protobug @ https://files.pythonhosted.org/packages/f2/cc/db26b91cddffbcf0c6df7834fd642578f737fe34197635ae8ea64643a35f/protobug-0.3.0-py3-none-any.whl#sha256=ee81583f376bb38e5e7af425d2453e5e8d4b57bfbf45e5dba1a75329c2026520` + ╰─▶ Hash mismatch for `protobug @ https://files.pythonhosted.org/packages/f2/cc/db26b91cddffbcf0c6df7834fd642578f737fe34197635ae8ea64643a35f/protobug-0.3.0-py3-none-any.whl#sha256=ee81583f376bb38e5e7af425d2453e5e8d4b57bfbf45e5dba1a75329c2026520` + + Expected: + sha256:ee81583f376bb38e5e7af425d2453e5e8d4b57bfbf45e5dba1a75329c2026520 + + Computed: + sha256:ee81583f376bb38e5e7af425d2453e5e8d4b57bfbf45e5dba1a75329c202652e + help: `protobug` (v0.3.0) was included because `pylock` (v0.1.0) depends on `protobug` + " + ); + + Ok(()) +} + +#[test] +fn direct_url_hash_source_tree_dependency_conflict() -> Result<()> { + let context = fyn_test::test_context!("3.12"); + + context.temp_dir.child("pyproject.toml").write_str(indoc! {r#" + [project] + name = "pylock" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = [ + "anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl#sha256=cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f", + "anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl#sha256=f7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a", + ] + + [tool.fyn.pip] + pip-in-project = "allow" + + [build-system] + requires = ["fyn-build>=0.7,<10000"] + build-backend = "fyn_build" + "#})?; + context + .temp_dir + .child("src") + .child("pylock") + .child("__init__.py") + .touch()?; + + fyn_snapshot!(context.pip_install() + .arg("."), @" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + error: Conflicting archive URL hashes for `anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl#sha256=f7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a`: `sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f` conflicts with `sha256:f7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a` + " + ); + + Ok(()) +} + +#[test] +fn direct_url_hash_source_tree_dependency_multiple_hash_algorithms() -> Result<()> { + let context = fyn_test::test_context!("3.12"); + + context.temp_dir.child("pyproject.toml").write_str(indoc! {r#" + [project] + name = "pylock" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = [ + "anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl#sha256=cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f", + "anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl#sha512=f30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2", + ] + + [tool.fyn.pip] + pip-in-project = "allow" + + [build-system] + requires = ["fyn-build>=0.7,<10000"] + build-backend = "fyn_build" + "#})?; + context + .temp_dir + .child("src") + .child("pylock") + .child("__init__.py") + .touch()?; + + fyn_snapshot!(context.filters(), context.pip_install() + .arg("."), @" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 4 packages in [TIME] + Prepared 4 packages in [TIME] + Installed 4 packages in [TIME] + + anyio==4.0.0 (from https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl#sha512=f30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2) + + idna==3.6 + + pylock==0.1.0 (from file://[TEMP_DIR]/) + + sniffio==1.3.1 + " + ); + + Ok(()) +} + /// `circular-one` depends on `circular-two` as a build dependency, but `circular-two` depends on /// `circular-one` was a runtime dependency. #[test] @@ -11800,6 +11941,9 @@ fn pep_751_install_registry_wheel() -> Result<()> { version = "0.1.0" requires-python = ">=3.12" dependencies = ["iniconfig"] + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?; @@ -11853,6 +11997,9 @@ fn pep_751_install_registry_sdist() -> Result<()> { version = "0.1.0" requires-python = ">=3.12" dependencies = ["source-distribution"] + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?; @@ -11931,6 +12078,9 @@ fn pep_751_install_directory() -> Result<()> { [tool.fyn.sources] foo = { path = "foo" } + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?; @@ -11988,6 +12138,9 @@ fn pep_751_install_git() -> Result<()> { version = "0.1.0" requires-python = ">=3.12" dependencies = ["uv-public-pypackage @ git+https://github.com/astral-test/uv-public-pypackage.git@0.0.1"] + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?; @@ -12041,6 +12194,9 @@ fn pep_751_install_url_wheel() -> Result<()> { version = "0.1.0" requires-python = ">=3.12" dependencies = ["anyio @ https://files.pythonhosted.org/packages/14/fd/2f20c40b45e4fb4324834aea24bd4afdf1143390242c0b33774da0e2e34f/anyio-4.3.0-py3-none-any.whl"] + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?; @@ -12096,6 +12252,9 @@ fn pep_751_install_url_sdist() -> Result<()> { version = "0.1.0" requires-python = ">=3.12" dependencies = ["anyio @ https://files.pythonhosted.org/packages/db/4d/3970183622f0330d3c23d9b8a5f52e365e50381fd484d08e3285104333d3/anyio-4.3.0.tar.gz"] + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?; @@ -12161,6 +12320,9 @@ fn pep_751_install_path_wheel() -> Result<()> { [tool.fyn.sources] iniconfig = { path = "iniconfig-2.0.0-py3-none-any.whl" } + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?; @@ -12244,6 +12406,9 @@ fn pep_751_install_path_sdist() -> Result<()> { [tool.fyn.sources] iniconfig = { path = "iniconfig-2.0.0.tar.gz" } + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?; @@ -12345,6 +12510,9 @@ fn pep_751_mix() -> Result<()> { version = "0.1.0" requires-python = ">=3.12" dependencies = ["iniconfig"] + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?; @@ -12661,6 +12829,9 @@ fn pep_751_requires_python() -> Result<()> { version = "0.1.0" requires-python = ">=3.13" dependencies = ["iniconfig"] + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?; diff --git a/crates/fyn/tests/it/pip_sync.rs b/crates/fyn/tests/it/pip_sync.rs index 4c1e39b72..879970091 100644 --- a/crates/fyn/tests/it/pip_sync.rs +++ b/crates/fyn/tests/it/pip_sync.rs @@ -4514,37 +4514,45 @@ fn require_hashes_repeated_hash() -> Result<()> { " ); - // Use a different hash. The first hash is wrong, but that's fine, since we use the last hash. + // Use a different hash. The `sha512` is wrong, so validation should fail even though the + // `sha256` is still correct. let requirements_txt = context.temp_dir.child("requirements.txt"); requirements_txt .write_str(indoc::indoc! { r" - anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl --hash=sha256:a7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a - anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl --hash=md5:420d85e19168705cdf0223621b18831a + anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl --hash=sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f + anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl --hash=sha512:e30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2 " })?; fyn_snapshot!(context.pip_sync() .arg("requirements.txt") .arg("--require-hashes") .arg("--reinstall"), @" - success: true - exit_code: 0 + success: false + exit_code: 1 ----- stdout ----- ----- stderr ----- Resolved 1 package in [TIME] - Prepared 1 package in [TIME] - Uninstalled 1 package in [TIME] - Installed 1 package in [TIME] - ~ anyio==4.0.0 (from https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl) + × Failed to download `anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl` + ╰─▶ Hash mismatch for `anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl` + + Expected: + sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f + sha512:e30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2 + + Computed: + sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f + sha512:f30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2 " ); - // Use a different hash. The second hash is wrong. This should fail, since we use the last hash. + // Use different hashes, but both are wrong. This should fail because none of the merged + // hashes match. let requirements_txt = context.temp_dir.child("requirements.txt"); requirements_txt .write_str(indoc::indoc! { r" anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl --hash=sha256:f7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a - anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl --hash=md5:520d85e19168705cdf0223621b18831a + anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl --hash=sha512:e30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2 " })?; fyn_snapshot!(context.pip_sync() @@ -4561,10 +4569,51 @@ fn require_hashes_repeated_hash() -> Result<()> { ╰─▶ Hash mismatch for `anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl` Expected: - md5:520d85e19168705cdf0223621b18831a + sha256:f7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a + sha512:e30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2 Computed: - md5:420d85e19168705cdf0223621b18831a + sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f + sha512:f30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2 + " + ); + + Ok(()) +} + +/// Repeated direct URL requirements merge hashes across nested requirements files. +#[test] +fn require_hashes_repeated_hash_multiple_files() -> Result<()> { + let context = fyn_test::test_context!("3.12"); + + let requirements_a = context.temp_dir.child("requirements-a.txt"); + requirements_a.write_str(indoc::indoc! { r" + anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl --hash=sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f + " })?; + + let requirements_b = context.temp_dir.child("requirements-b.txt"); + requirements_b.write_str(indoc::indoc! { r" + anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl --hash=sha512:f30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2 + " })?; + + let requirements_txt = context.temp_dir.child("requirements.txt"); + requirements_txt.write_str(indoc::indoc! { r" + -r requirements-a.txt + -r requirements-b.txt + " })?; + + fyn_snapshot!(context.pip_sync() + .arg("requirements.txt") + .arg("--require-hashes"), @" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + Prepared 1 package in [TIME] + Installed 1 package in [TIME] + + anyio==4.0.0 (from https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl) " ); @@ -5220,32 +5269,27 @@ fn require_hashes_url_invalid() -> Result<()> { Ok(()) } -/// Ignore the (valid) hash on the fragment if (invalid) hashes are provided directly. +/// Merge the hash on the fragment with hashes provided directly. #[test] -fn require_hashes_url_ignore() -> Result<()> { +fn require_hashes_url_merge() -> Result<()> { let context = fyn_test::test_context!("3.12").with_exclude_newer("2025-01-29T00:00:00Z"); let requirements_txt = context.temp_dir.child("requirements.txt"); requirements_txt - .write_str("iniconfig @ https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl#sha256=b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 --hash sha256:c6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374")?; + .write_str("anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl#sha256=cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f --hash sha512:f30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2")?; fyn_snapshot!(context.pip_sync() .arg("requirements.txt") .arg("--require-hashes"), @" - success: false - exit_code: 1 + success: true + exit_code: 0 ----- stdout ----- ----- stderr ----- Resolved 1 package in [TIME] - × Failed to download `iniconfig @ https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl#sha256=b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374` - ╰─▶ Hash mismatch for `iniconfig @ https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl#sha256=b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374` - - Expected: - sha256:c6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 - - Computed: - sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 + Prepared 1 package in [TIME] + Installed 1 package in [TIME] + + anyio==4.0.0 (from https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl#sha256=cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f) " ); @@ -5854,6 +5898,9 @@ fn pep_751() -> Result<()> { version = "0.1.0" requires-python = ">=3.12" dependencies = ["anyio"] + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?; @@ -5900,6 +5947,9 @@ fn pep_751() -> Result<()> { version = "0.1.0" requires-python = ">=3.12" dependencies = ["iniconfig"] + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?; @@ -5947,6 +5997,9 @@ fn pep_751_wheel_only() -> Result<()> { version = "0.1.0" requires-python = ">=3.12.0" dependencies = ["torch"] + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?; @@ -6021,6 +6074,9 @@ fn pep_751_build_options() -> Result<()> { version = "0.1.0" requires-python = ">=3.12" dependencies = ["anyio"] + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?; @@ -6058,6 +6114,9 @@ fn pep_751_build_options() -> Result<()> { version = "0.1.0" requires-python = ">=3.12" dependencies = ["odrive"] + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?; @@ -6090,6 +6149,9 @@ fn pep_751_build_options() -> Result<()> { version = "0.1.0" requires-python = ">=3.12" dependencies = ["source-distribution"] + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?; @@ -6150,6 +6212,9 @@ fn pep_751_direct_url_tags() -> Result<()> { version = "0.1.0" requires-python = ">=3.12" dependencies = ["MarkupSafe @ https://files.pythonhosted.org/packages/6b/b0/18f76bba336fa5aecf79d45dcd6c806c280ec44538b3c13671d49099fdd0/MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl"] + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?; diff --git a/crates/fyn/tests/it/sync.rs b/crates/fyn/tests/it/sync.rs index 1af3108d5..db95b297e 100644 --- a/crates/fyn/tests/it/sync.rs +++ b/crates/fyn/tests/it/sync.rs @@ -1447,6 +1447,9 @@ fn sync_build_isolation() -> Result<()> { version = "0.1.0" requires-python = ">=3.12" dependencies = ["source-distribution @ https://files.pythonhosted.org/packages/10/1f/57aa4cce1b1abf6b433106676e15f9fa2c92ed2bd4cf77c3b50a9e9ac773/source_distribution-0.0.1.tar.gz"] + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?; @@ -1521,6 +1524,9 @@ fn sync_build_isolation_package() -> Result<()> { [tool.fyn] no-build-isolation-package = ["source-distribution"] + + [tool.fyn.pip] + pip-in-project = "allow" "#, )?;