Add separate compile and install commands (#17)

Closes #9.

Author: charliermarsh

crates/README.md

@@ -19,3 +19,7 @@ Types and functionality for working with Python packages, e.g., parsing wheel fi
 ## [puffin-platform](./puffin-platform)
 
 Functionality for detecting the current platform (operating system, architecture, etc.).
+
+## [puffin-resolve](./puffin-resolve)
+
+Functionality for resolving Python packages and their dependencies.

crates/puffin-cli/Cargo.toml

@@ -8,11 +8,11 @@ puffin-client = { path = "../puffin-client" }
 puffin-interpreter = { path = "../puffin-interpreter" }
 puffin-platform = { path = "../puffin-platform" }
 puffin-package = { path = "../puffin-package" }
+puffin-resolve = { path = "../puffin-resolve" }
 
 anyhow = { version = "1.0.75" }
 clap = { version = "4.4.6", features = ["derive"] }
 colored = { version = "2.0.4" }
-memchr = { version = "2.6.4" }
 async-std = { version = "1.12.0", features = [
     "attributes",
     "tokio1",

crates/puffin-cli/src/commands/compile.rs

@@ -0,0 +1,46 @@
+use std::path::Path;
+use std::str::FromStr;
+
+use anyhow::Result;
+use tracing::debug;
+
+use puffin_interpreter::PythonExecutable;
+use puffin_platform::Platform;
+use puffin_resolve::resolve;
+
+use crate::commands::ExitStatus;
+
+pub(crate) async fn compile(src: &Path, cache: Option<&Path>) -> Result<ExitStatus> {
+    // Read the `requirements.txt` from disk.
+    let requirements_txt = std::fs::read_to_string(src)?;
+
+    // Parse the `requirements.txt` into a list of requirements.
+    let requirements = puffin_package::requirements::Requirements::from_str(&requirements_txt)?;
+
+    // Detect the current Python interpreter.
+    let platform = Platform::current()?;
+    let python = PythonExecutable::from_env(&platform)?;
+    debug!(
+        "Using Python interpreter: {}",
+        python.executable().display()
+    );
+
+    // Resolve the dependencies.
+    let resolution = resolve(
+        &requirements,
+        python.version(),
+        python.markers(),
+        &platform,
+        cache,
+    )
+    .await?;
+
+    for (name, version) in resolution.iter() {
+        #[allow(clippy::print_stdout)]
+        {
+            println!("{name}=={version}");
+        }
+    }
+
+    Ok(ExitStatus::Success)
+}

crates/puffin-cli/src/commands/install.rs

@@ -1,35 +1,15 @@
-use std::collections::{HashMap, HashSet};
 use std::path::Path;
 use std::str::FromStr;
 
 use anyhow::Result;
-use futures::future::Either;
-use futures::{StreamExt, TryFutureExt};
-use pep440_rs::Version;
-use pep508_rs::{Requirement, VersionOrUrl};
 use tracing::debug;
 
-use puffin_client::{File, PypiClientBuilder, SimpleJson};
 use puffin_interpreter::PythonExecutable;
-use puffin_package::metadata::Metadata21;
-use puffin_package::package_name::PackageName;
-use puffin_package::wheel::WheelFilename;
 use puffin_platform::Platform;
+use puffin_resolve::resolve;
 
 use crate::commands::ExitStatus;
 
-#[derive(Debug)]
-enum Request {
-    Package(Requirement),
-    Version(Requirement, File),
-}
-
-#[derive(Debug)]
-enum Response {
-    Package(SimpleJson, Requirement),
-    Version(Metadata21, Requirement),
-}
-
 pub(crate) async fn install(src: &Path, cache: Option<&Path>) -> Result<ExitStatus> {
     // Read the `requirements.txt` from disk.
     let requirements_txt = std::fs::read_to_string(src)?;
@@ -45,134 +25,17 @@ pub(crate) async fn install(src: &Path, cache: Option<&Path>) -> Result<ExitStat
         python.executable().display()
     );
 
-    // Determine the compatible platform tags.
-    let tags = platform.compatible_tags(python.version())?;
-
-    // Instantiate a client.
-    let pypi_client = {
-        let mut pypi_client = PypiClientBuilder::default();
-        if let Some(cache) = cache {
-            pypi_client = pypi_client.cache(cache);
-        }
-        pypi_client.build()
-    };
-
-    // A channel to fetch package metadata (e.g., given `flask`, fetch all versions) and version
-    // metadata (e.g., given `flask==1.0.0`, fetch the metadata for that version).
-    let (package_sink, package_stream) = futures::channel::mpsc::unbounded();
-
-    // Initialize the package stream.
-    let mut package_stream = package_stream
-        .map(|request: Request| match request {
-            Request::Package(requirement) => Either::Left(
-                pypi_client
-                    .simple(requirement.name.clone())
-                    .map_ok(move |metadata| Response::Package(metadata, requirement)),
-            ),
-            Request::Version(requirement, file) => Either::Right(
-                pypi_client
-                    .file(file)
-                    .map_ok(move |metadata| Response::Version(metadata, requirement)),
-            ),
-        })
-        .buffer_unordered(32)
-        .ready_chunks(32);
-
-    // Push all the requirements into the package sink.
-    let mut in_flight: HashSet<PackageName> = HashSet::with_capacity(requirements.len());
-    for requirement in &*requirements {
-        debug!("--> adding root dependency: {}", requirement);
-        package_sink.unbounded_send(Request::Package(requirement.clone()))?;
-        in_flight.insert(PackageName::normalize(&requirement.name));
-    }
-
-    // Resolve the requirements.
-    let mut resolution: HashMap<PackageName, Version> = HashMap::with_capacity(requirements.len());
-
-    while let Some(chunk) = package_stream.next().await {
-        for result in chunk {
-            let result: Response = result?;
-            match result {
-                Response::Package(metadata, requirement) => {
-                    // TODO(charlie): Support URLs. Right now, we treat a URL as an unpinned dependency.
-                    let specifiers =
-                        requirement
-                            .version_or_url
-                            .as_ref()
-                            .and_then(|version_or_url| match version_or_url {
-                                VersionOrUrl::VersionSpecifier(specifiers) => Some(specifiers),
-                                VersionOrUrl::Url(_) => None,
-                            });
-
-                    // Pick a version that satisfies the requirement.
-                    let Some(file) = metadata.files.iter().rev().find(|file| {
-                        // We only support wheels for now.
-                        let Ok(name) = WheelFilename::from_str(file.filename.as_str()) else {
-                            return false;
-                        };
-
-                        let Ok(version) = Version::from_str(&name.version) else {
-                            return false;
-                        };
-
-                        if !name.is_compatible(&tags) {
-                            return false;
-                        }
-
-                        specifiers
-                            .iter()
-                            .all(|specifier| specifier.contains(&version))
-                    }) else {
-                        continue;
-                    };
-
-                    package_sink.unbounded_send(Request::Version(requirement, file.clone()))?;
-                }
-                Response::Version(metadata, requirement) => {
-                    debug!(
-                        "--> selected version {} for {}",
-                        metadata.version, requirement
-                    );
-
-                    // Add to the resolved set.
-                    let normalized_name = PackageName::normalize(&requirement.name);
-                    in_flight.remove(&normalized_name);
-                    resolution.insert(normalized_name, metadata.version);
-
-                    // Enqueue its dependencies.
-                    for dependency in metadata.requires_dist {
-                        if !dependency.evaluate_markers(
-                            python.markers(),
-                            requirement.extras.clone().unwrap_or_default(),
-                        ) {
-                            debug!("--> ignoring {dependency} due to environment mismatch");
-                            continue;
-                        }
-
-                        let normalized_name = PackageName::normalize(&dependency.name);
-
-                        if resolution.contains_key(&normalized_name) {
-                            continue;
-                        }
-
-                        if !in_flight.insert(normalized_name) {
-                            continue;
-                        }
-
-                        debug!("--> adding transitive dependency: {}", dependency);
-
-                        package_sink.unbounded_send(Request::Package(dependency))?;
-                    }
-                }
-            }
-        }
-
-        if in_flight.is_empty() {
-            break;
-        }
-    }
-
-    for (name, version) in resolution {
+    // Resolve the dependencies.
+    let resolution = resolve(
+        &requirements,
+        python.version(),
+        python.markers(),
+        &platform,
+        cache,
+    )
+    .await?;
+
+    for (name, version) in resolution.iter() {
         #[allow(clippy::print_stdout)]
         {
             println!("{name}=={version}");

crates/puffin-cli/src/commands/mod.rs

@@ -1,7 +1,9 @@
 use std::process::ExitCode;
 
+pub(crate) use compile::compile;
 pub(crate) use install::install;
 
+mod compile;
 mod install;
 
 #[derive(Copy, Clone)]

crates/puffin-cli/src/main.rs

@@ -20,13 +20,25 @@ struct Cli {
 
 #[derive(Subcommand)]
 enum Commands {
-    /// Install dependencies from a `requirements.text` file.
+    /// Compile a `requirements.in` file to a `requirements.txt` file.
+    Compile(CompileArgs),
+    /// Install dependencies from a `requirements.txt` file.
     Install(InstallArgs),
 }
 
+#[derive(Args)]
+struct CompileArgs {
+    /// Path to the `requirements.txt` file to compile.
+    src: PathBuf,
+
+    /// Avoid reading from or writing to the cache.
+    #[arg(long)]
+    no_cache: bool,
+}
+
 #[derive(Args)]
 struct InstallArgs {
-    /// Path to the `requirements.text` file to install.
+    /// Path to the `requirements.txt` file to install.
     src: PathBuf,
 
     /// Avoid reading from or writing to the cache.
@@ -43,12 +55,21 @@ async fn main() -> ExitCode {
     let dirs = ProjectDirs::from("", "", "puffin");
 
     let result = match &cli.command {
-        Commands::Install(install) => {
+        Commands::Compile(args) => {
+            commands::compile(
+                &args.src,
+                dirs.as_ref()
+                    .map(directories::ProjectDirs::cache_dir)
+                    .filter(|_| !args.no_cache),
+            )
+            .await
+        }
+        Commands::Install(args) => {
             commands::install(
-                &install.src,
+                &args.src,
                 dirs.as_ref()
                     .map(directories::ProjectDirs::cache_dir)
-                    .filter(|_| !install.no_cache),
+                    .filter(|_| !args.no_cache),
             )
             .await
         }

crates/puffin-interpreter/Cargo.toml

@@ -16,3 +16,4 @@ anyhow = { version = "1.0.75" }
 pep508_rs = { version = "0.2.3", features = ["serde"] }
 serde_json = { version = "1.0.107" }
 tracing = { version = "0.1.37" }
+pep440_rs = "0.3.12"

crates/puffin-interpreter/src/lib.rs

@@ -1,6 +1,7 @@
 use std::path::{Path, PathBuf};
 
 use anyhow::Result;
+use pep440_rs::Version;
 use pep508_rs::MarkerEnvironment;
 
 use puffin_platform::Platform;
@@ -42,13 +43,8 @@ impl PythonExecutable {
         &self.markers
     }
 
-    /// Returns the Python version as a tuple of (major, minor).
-    pub fn version(&self) -> (u8, u8) {
-        // TODO(charlie): Use `Version`.
-        let python_version = &self.markers.python_version;
-        (
-            u8::try_from(python_version.release[0]).expect("Python major version is too large"),
-            u8::try_from(python_version.release[1]).expect("Python minor version is too large"),
-        )
+    /// Returns the Python version.
+    pub fn version(&self) -> &Version {
+        &self.markers.python_version.version
     }
 }

crates/puffin-package/benches/parser.rs

@@ -2,7 +2,7 @@ use std::str::FromStr;
 
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 
-use puffin_package::Requirements;
+use puffin_package::requirements::Requirements;
 
 const REQUIREMENTS_TXT: &str = r"
 #

crates/puffin-platform/Cargo.toml

@@ -14,6 +14,7 @@ license.workspace = true
 [dependencies]
 glibc_version = "0.1.2"
 goblin = "0.6.0"
+pep440_rs = "0.3.12"
 platform-info = "2.0.2"
 plist = "1.5.0"
 regex = "1.9.6"