From 5fa949f929d97e515cc6fd1e04eb9b8fc0b14880 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Fri, 21 Jun 2024 14:53:41 +1000 Subject: [PATCH 01/11] Support for Poetry Environments --- Cargo.lock | 181 ++++++++++++++-- crates/pet-conda/Cargo.toml | 2 - crates/pet-core/src/manager.rs | 1 + crates/pet-core/src/python_environment.rs | 1 + crates/pet-global/Cargo.toml | 10 - crates/pet-global/src/lib.rs | 2 - crates/pet-poetry/Cargo.toml | 12 ++ crates/pet-poetry/src/config.rs | 146 +++++++++++++ crates/pet-poetry/src/env_variables.rs | 52 +++++ crates/pet-poetry/src/environment.rs | 36 ++++ .../pet-poetry/src/environment_locations.rs | 141 +++++++++++++ .../src/environment_locations_spawn.rs | 83 ++++++++ crates/pet-poetry/src/lib.rs | 197 +++++++++++++++++- crates/pet-poetry/src/manager.rs | 117 +++++++++++ crates/pet-poetry/src/pyproject_toml.rs | 78 +++++++ crates/pet-pyenv/Cargo.toml | 2 - crates/pet-python-utils/src/lib.rs | 1 + crates/pet-python-utils/src/platform_dirs.rs | 104 +++++++++ crates/pet-reporter/src/environment.rs | 1 + crates/pet-reporter/src/manager.rs | 1 + crates/pet-telemetry/Cargo.toml | 3 + crates/pet/Cargo.toml | 1 - crates/pet/src/find.rs | 12 +- crates/pet/src/lib.rs | 19 +- crates/pet/src/locators.rs | 2 + crates/pet/src/main.rs | 13 +- 26 files changed, 1168 insertions(+), 50 deletions(-) delete mode 100644 crates/pet-global/Cargo.toml delete mode 100644 crates/pet-global/src/lib.rs create mode 100644 crates/pet-poetry/src/config.rs create mode 100644 crates/pet-poetry/src/env_variables.rs create mode 100644 crates/pet-poetry/src/environment.rs create mode 100644 crates/pet-poetry/src/environment_locations.rs create mode 100644 crates/pet-poetry/src/environment_locations_spawn.rs create mode 100644 crates/pet-poetry/src/manager.rs create mode 100644 crates/pet-poetry/src/pyproject_toml.rs create mode 100644 crates/pet-python-utils/src/platform_dirs.rs diff --git a/Cargo.lock b/Cargo.lock index 3d536c01..0bb9bfed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,6 +60,21 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "cc" version = "1.0.99" @@ -118,6 +133,35 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "env_logger" version = "0.10.2" @@ -131,6 +175,28 @@ dependencies = [ "termcolor", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "heck" version = "0.5.0" @@ -149,6 +215,16 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "is-terminal" version = "0.4.12" @@ -217,7 +293,6 @@ dependencies = [ "pet-core", "pet-env-var-path", "pet-fs", - "pet-global", "pet-global-virtualenvs", "pet-homebrew", "pet-jsonrpc", @@ -308,14 +383,6 @@ dependencies = [ "msvc_spectre_libs", ] -[[package]] -name = "pet-global" -version = "0.1.0" -dependencies = [ - "log", - "msvc_spectre_libs", -] - [[package]] name = "pet-global-virtualenvs" version = "0.1.0" @@ -420,7 +487,19 @@ dependencies = [ name = "pet-poetry" version = "0.1.0" dependencies = [ + "base64", + "lazy_static", + "log", "msvc_spectre_libs", + "pet-core", + "pet-fs", + "pet-python-utils", + "pet-reporter", + "regex", + "serde", + "serde_json", + "sha2", + "toml", ] [[package]] @@ -473,6 +552,7 @@ dependencies = [ "env_logger", "lazy_static", "log", + "msvc_spectre_libs", "pet-core", "pet-fs", "pet-python-utils", @@ -547,9 +627,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -629,6 +709,26 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +dependencies = [ + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "strsim" version = "0.11.1" @@ -637,9 +737,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.66" +version = "2.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" +checksum = "ff8655ed1d86f3af4ee3fd3263786bc14245ad17c4c7e85ba7187fb3ae028c90" dependencies = [ "proc-macro2", "quote", @@ -655,6 +755,46 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "toml" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicode-ident" version = "1.0.12" @@ -667,6 +807,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "winapi-util" version = "0.1.8" @@ -815,6 +961,15 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +[[package]] +name = "winnow" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.52.0" diff --git a/crates/pet-conda/Cargo.toml b/crates/pet-conda/Cargo.toml index 949bc7f8..7af5ce76 100644 --- a/crates/pet-conda/Cargo.toml +++ b/crates/pet-conda/Cargo.toml @@ -18,8 +18,6 @@ log = "0.4.21" regex = "1.10.4" pet-reporter = { path = "../pet-reporter" } -[dev-dependencies] - [features] ci = [] \ No newline at end of file diff --git a/crates/pet-core/src/manager.rs b/crates/pet-core/src/manager.rs index 13108f62..e6e246ec 100644 --- a/crates/pet-core/src/manager.rs +++ b/crates/pet-core/src/manager.rs @@ -9,6 +9,7 @@ use std::path::PathBuf; #[derive(Debug, Hash)] pub enum EnvManagerType { Conda, + Poetry, Pyenv, } diff --git a/crates/pet-core/src/python_environment.rs b/crates/pet-core/src/python_environment.rs index 444e3cde..ae05c7f1 100644 --- a/crates/pet-core/src/python_environment.rs +++ b/crates/pet-core/src/python_environment.rs @@ -20,6 +20,7 @@ pub enum PythonEnvironmentCategory { PyenvVirtualEnv, // Pyenv virtualenvs. PyenvOther, // Such as pyston, stackless, nogil, etc. Pipenv, + Poetry, System, MacPythonOrg, MacCommandLineTools, diff --git a/crates/pet-global/Cargo.toml b/crates/pet-global/Cargo.toml deleted file mode 100644 index ad17f16b..00000000 --- a/crates/pet-global/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "pet-global" -version = "0.1.0" -edition = "2021" - -[target.'cfg(target_os = "windows")'.dependencies] -msvc_spectre_libs = { version = "0.1.1", features = ["error"] } - -[dependencies] -log = "0.4.21" diff --git a/crates/pet-global/src/lib.rs b/crates/pet-global/src/lib.rs deleted file mode 100644 index fc36ab24..00000000 --- a/crates/pet-global/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. diff --git a/crates/pet-poetry/Cargo.toml b/crates/pet-poetry/Cargo.toml index d262f60c..d1fe0e11 100644 --- a/crates/pet-poetry/Cargo.toml +++ b/crates/pet-poetry/Cargo.toml @@ -7,3 +7,15 @@ edition = "2021" msvc_spectre_libs = { version = "0.1.1", features = ["error"] } [dependencies] +serde = { version = "1.0.152", features = ["derive"] } +serde_json = "1.0.93" +lazy_static = "1.4.0" +pet-core = { path = "../pet-core" } +pet-python-utils = { path = "../pet-python-utils" } +pet-reporter = { path = "../pet-reporter" } +pet-fs = { path = "../pet-fs" } +log = "0.4.21" +regex = "1.10.4" +sha2 = "0.10.6" +base64 = "0.22.0" +toml = "0.8.14" diff --git a/crates/pet-poetry/src/config.rs b/crates/pet-poetry/src/config.rs new file mode 100644 index 00000000..0171fcb2 --- /dev/null +++ b/crates/pet-poetry/src/config.rs @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use log::trace; +use pet_python_utils::platform_dirs::Platformdirs; + +use crate::env_variables::EnvVariables; + +static _APP_NAME: &str = "pypoetry"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Config { + pub virtualenvs_in_project: bool, + pub virtualenvs_path: PathBuf, + pub file: PathBuf, +} + +impl Config { + fn new(file: PathBuf, virtualenvs_path: PathBuf, virtualenvs_in_project: bool) -> Self { + trace!( + "Poetry config file: {:?} with virtualenv.path {:?}", + file, + virtualenvs_path + ); + Config { + file, + virtualenvs_path, + virtualenvs_in_project, + } + } + pub fn find_global(env: &EnvVariables) -> Option { + let file = find_config_file(env)?; + create_config(&file, env) + } + pub fn find_local(path: &Path, env: &EnvVariables) -> Option { + let file = path.join("poetry.toml"); + if file.is_file() { + create_config(&file, env) + } else { + None + } + } +} + +fn create_config(file: &Path, env: &EnvVariables) -> Option { + let cfg = parse(file)?; + + if let Some(virtualenvs_path) = &cfg.virtualenvs_path { + return Some(Config::new( + file.to_path_buf(), + virtualenvs_path.clone(), + cfg.virtualenvs_in_project, + )); + } + + let cache_dir = match cfg.cache_dir { + Some(cache_dir) => { + if cache_dir.is_dir() { + Some(cache_dir) + } else { + get_default_cache_dir(env) + } + } + None => get_default_cache_dir(env), + }; + + if let Some(cache_dir) = cache_dir { + Some(Config::new( + file.to_path_buf(), + cache_dir.join("virtualenvs"), + cfg.virtualenvs_in_project, + )) + } else { + None + } +} +/// Maps to DEFAULT_CACHE_DIR in poetry +fn get_default_cache_dir(env: &EnvVariables) -> Option { + if let Some(cache_dir) = env.poetry_cache_dir.clone() { + Some(cache_dir) + } else { + Platformdirs::new(_APP_NAME.into(), false).user_cache_path() + } +} + +/// Maps to CONFIG_DIR in poetry +fn get_config_dir(env: &EnvVariables) -> Option { + if let Some(config) = env.poetry_config_dir.clone() { + return Some(config); + } + Platformdirs::new(_APP_NAME.into(), true).user_config_path() +} + +fn find_config_file(env: &EnvVariables) -> Option { + let config_dir = get_config_dir(env)?; + let file = config_dir.join("config.toml"); + if file.exists() { + Some(file) + } else { + None + } +} + +struct ConfigToml { + virtualenvs_in_project: bool, + cache_dir: Option, + virtualenvs_path: Option, +} + +fn parse(file: &Path) -> Option { + let contents = fs::read_to_string(file).ok()?; + + let mut virtualenvs_path = None; + let mut cache_dir = None; + let mut virtualenvs_in_project = false; + match toml::from_str::(&contents) { + Ok(value) => { + if let Some(virtualenvs) = value.get("virtualenvs") { + if let Some(path) = virtualenvs.get("path") { + virtualenvs_path = path.as_str().map(|s| s.trim()).map(PathBuf::from); + } + if let Some(in_project) = virtualenvs.get("in-project") { + virtualenvs_in_project = in_project.as_bool().unwrap_or_default(); + } + } + if let Some(value) = value.get("cache-dir") { + cache_dir = value.as_str().map(|s| s.trim()).map(PathBuf::from); + } + + Some(ConfigToml { + virtualenvs_in_project, + virtualenvs_path, + cache_dir, + }) + } + Err(e) => { + eprintln!("Error parsing toml file: {:?}", e); + None + } + } +} diff --git a/crates/pet-poetry/src/env_variables.rs b/crates/pet-poetry/src/env_variables.rs new file mode 100644 index 00000000..67f011f6 --- /dev/null +++ b/crates/pet-poetry/src/env_variables.rs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use pet_core::os_environment::Environment; +use std::path::PathBuf; + +#[derive(Debug, Clone)] +// NOTE: Do not implement Default trait, as we do not want to ever forget to set the values. +// Lets be explicit, this way we never miss a value (in Windows or Unix). +pub struct EnvVariables { + pub home: Option, + pub appdata: Option, + pub poetry_home: Option, + pub poetry_config_dir: Option, + pub poetry_cache_dir: Option, + pub poetry_virtualenvs_in_project: Option, + pub path: Option, +} + +impl EnvVariables { + pub fn from(env: &dyn Environment) -> Self { + let mut poetry_home = None; + let home = env.get_user_home(); + if let (Some(home), Some(poetry_home_value)) = + (&home, &env.get_env_var("POETRY_HOME".to_string())) + { + if poetry_home_value.starts_with('~') { + poetry_home = Some(PathBuf::from( + poetry_home_value.replace('~', home.to_str().unwrap()), + )); + } else { + poetry_home = Some(PathBuf::from(poetry_home_value)); + } + } + + EnvVariables { + home, + path: env.get_env_var("PATH".to_string()), + appdata: env.get_env_var("APPDATA".to_string()).map(PathBuf::from), + poetry_cache_dir: env + .get_env_var("POETRY_CACHE_DIR".to_string()) + .map(PathBuf::from), + poetry_config_dir: env + .get_env_var("POETRY_CONFIG_DIR".to_string()) + .map(PathBuf::from), + poetry_virtualenvs_in_project: env + .get_env_var("POETRY_VIRTUALENVS_IN_PROJECT".to_string()) + .map(|v| v == "1" || v.to_lowercase() == "true"), + poetry_home, + } + } +} diff --git a/crates/pet-poetry/src/environment.rs b/crates/pet-poetry/src/environment.rs new file mode 100644 index 00000000..66206cc5 --- /dev/null +++ b/crates/pet-poetry/src/environment.rs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::path::PathBuf; + +use pet_core::python_environment::{ + PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentCategory, +}; +use pet_python_utils::{executable::find_executables, version}; + +use crate::manager::PoetryManager; + +pub fn create_poetry_env( + prefix: &PathBuf, + project_dir: PathBuf, + manager: Option, +) -> Option { + if !prefix.exists() { + return None; + } + let executables = find_executables(prefix); + if executables.is_empty() { + return None; + } + let version = version::from_creator_for_virtual_env(prefix); + Some( + PythonEnvironmentBuilder::new(PythonEnvironmentCategory::Poetry) + .executable(Some(executables[0].clone())) + .prefix(Some(prefix.clone())) + .version(version) + .manager(manager.map(|m| m.to_manager())) + .project(Some(project_dir.clone())) + .symlinks(Some(executables)) + .build(), + ) +} diff --git a/crates/pet-poetry/src/environment_locations.rs b/crates/pet-poetry/src/environment_locations.rs new file mode 100644 index 00000000..1d109b39 --- /dev/null +++ b/crates/pet-poetry/src/environment_locations.rs @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use base64::{engine::general_purpose, Engine as _}; +use lazy_static::lazy_static; +use pet_core::python_environment::PythonEnvironment; +use pet_fs::path::norm_case; +use regex::Regex; +use sha2::{Digest, Sha256}; +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use crate::{ + config::Config, env_variables::EnvVariables, environment::create_poetry_env, + pyproject_toml::PyProjectToml, +}; + +lazy_static! { + static ref SANITIZE_NAME: Regex = Regex::new("[ $`!*@\"\\\r\n\t]") + .expect("Error generating RegEx for poetry file path hash generator"); +} + +pub fn list_environments( + env: &EnvVariables, + project_dirs: &Vec, +) -> Option> { + let mut envs = vec![]; + + let global_config = Config::find_global(env); + let mut global_envs = vec![]; + if let Some(config) = global_config.clone() { + global_envs = list_all_environments_from_config(&config).unwrap_or_default(); + } + + // We're only interested in directories that have a pyproject.toml + for project_dir in project_dirs { + if let Some(pyproject_toml) = PyProjectToml::find(project_dir) { + let virtualenv_prefix = generate_env_name(&pyproject_toml.name, project_dir); + + for virtual_env in + list_all_environments_from_project_config(&global_config, project_dir, env) + .unwrap_or(global_envs.clone()) + { + // Check if this virtual env belongs to this project + let name = virtual_env + .file_name() + .unwrap_or_default() + .to_str() + .unwrap_or_default(); + if name.starts_with(&virtualenv_prefix) { + if let Some(env) = create_poetry_env(&virtual_env, project_dir.clone(), None) { + envs.push(env); + } + } + } + } + } + + Some(envs) +} + +fn list_all_environments_from_project_config( + global: &Option, + path: &Path, + env: &EnvVariables, +) -> Option> { + let config = Config::find_local(path, env)?; + let mut envs = vec![]; + if let Some(project_envs) = list_all_environments_from_config(&config) { + envs.extend(project_envs); + } + + // Check if we're allowed to use .venv as a poetry env + // This can be configured in global, project or env variable. + if config.virtualenvs_in_project + || global + .clone() + .map(|config| config.virtualenvs_in_project) + .unwrap_or(false) + || env.poetry_virtualenvs_in_project.unwrap_or_default() + { + // If virtualenvs are in the project, then look for .venv + let venv = path.join(".venv"); + if venv.is_dir() { + envs.push(venv); + } + } + Some(envs) +} + +fn list_all_environments_from_config(cfg: &Config) -> Option> { + Some( + fs::read_dir(&cfg.virtualenvs_path) + .ok()? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.is_dir()) + .collect(), + ) +} + +// Source from https://github.com/python-poetry/poetry/blob/5bab98c9500f1050c6bb6adfb55580a23173f18d/src/poetry/utils/env/env_manager.py#L752C1-L757C63 +pub fn generate_env_name(name: &str, cwd: &PathBuf) -> String { + // name = name.lower() + // sanitized_name = re.sub(r'[ $`!*@"\\\r\n\t]', "_", name)[:42] + // normalized_cwd = os.path.normcase(os.path.realpath(cwd)) + // h_bytes = hashlib.sha256(encode(normalized_cwd)).digest() + // h_str = base64.urlsafe_b64encode(h_bytes).decode()[:8] + let sanitized_name = SANITIZE_NAME + .replace_all(&name.to_lowercase(), "_") + .chars() + .take(42) + .collect::(); + let normalized_cwd = norm_case(Path::new(cwd)); + let mut hasher = Sha256::new(); + hasher.update(normalized_cwd.to_str().unwrap().as_bytes()); + let h_bytes = hasher.finalize(); + let h_str = general_purpose::URL_SAFE + .encode(h_bytes) + .chars() + .take(8) + .collect::(); + format!("{}-{}-py", sanitized_name, h_str) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_hash_generation() { + let hashed_name = generate_env_name( + "poetry-demo", + &"/Users/donjayamanne/temp/poetry-sample1/poetry-demo".into(), + ); + + assert_eq!(hashed_name, "poetry-demo-gNT2WXAV-py"); + } +} diff --git a/crates/pet-poetry/src/environment_locations_spawn.rs b/crates/pet-poetry/src/environment_locations_spawn.rs new file mode 100644 index 00000000..23f4945a --- /dev/null +++ b/crates/pet-poetry/src/environment_locations_spawn.rs @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use lazy_static::lazy_static; +use log::{error, trace}; +use pet_core::python_environment::PythonEnvironment; +use regex::Regex; +use std::{path::PathBuf, time::SystemTime}; + +use crate::{environment::create_poetry_env, manager::PoetryManager}; + +lazy_static! { + static ref SANITIZE_NAME: Regex = Regex::new("[ $`!*@\"\\\r\n\t]") + .expect("Error generating RegEx for poetry file path hash generator"); +} + +pub fn list_environments( + executable: &PathBuf, + project_dirs: Vec, + manager: &PoetryManager, +) -> Vec { + let mut envs = vec![]; + for project_dir in project_dirs { + if let Some(project_envs) = get_environments(executable, &project_dir) { + for project_env in project_envs { + if let Some(env) = + create_poetry_env(&project_env, project_dir.clone(), Some(manager.clone())) + { + envs.push(env); + } + } + } + } + envs +} + +fn get_environments(executable: &PathBuf, project_dir: &PathBuf) -> Option> { + let start = SystemTime::now(); + let result = std::process::Command::new(executable) + .arg("env") + .arg("list") + .arg("--full-path") + .current_dir(project_dir) + .output(); + trace!( + "Executed Poetry ({}ms): {:?} env list --full-path for {:?}", + start.elapsed().unwrap_or_default().as_millis(), + executable, + project_dir + ); + match result { + Ok(output) => { + if output.status.success() { + let output = String::from_utf8_lossy(&output.stdout).to_string(); + Some( + output + .lines() + .map(|line| + // Remove the '(Activated)` suffix from the line + line.trim_end_matches(" (Activated)").trim()) + .filter(|line| !line.is_empty()) + .map(|line| + // Remove the '(Activated)` suffix from the line + PathBuf::from(line.trim_end_matches(" (Activated)").trim())) + .collect::>(), + ) + } else { + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + trace!( + "Failed to get Poetry Envs using exe {:?} ({:?}) {}", + executable, + output.status.code().unwrap_or_default(), + stderr + ); + None + } + } + Err(err) => { + error!("Failed to execute Poetry env list {:?}", err); + None + } + } +} diff --git a/crates/pet-poetry/src/lib.rs b/crates/pet-poetry/src/lib.rs index 7d12d9af..378a37b6 100644 --- a/crates/pet-poetry/src/lib.rs +++ b/crates/pet-poetry/src/lib.rs @@ -1,14 +1,193 @@ -pub fn add(left: usize, right: usize) -> usize { - left + right +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use env_variables::EnvVariables; +use environment_locations::list_environments; +use log::{error, warn}; +use manager::PoetryManager; +use pet_core::{ + os_environment::Environment, + python_environment::{PythonEnvironment, PythonEnvironmentCategory}, + reporter::Reporter, + Configuration, Locator, LocatorResult, +}; +use pet_python_utils::env::PythonEnv; +use std::{ + path::PathBuf, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, +}; + +mod config; +mod env_variables; +mod environment; +mod environment_locations; +mod environment_locations_spawn; +mod manager; +mod pyproject_toml; + +pub struct Poetry { + pub project_dirs: Arc>>, + pub env_vars: EnvVariables, + pub poetry_executable: Arc>>, + searched: AtomicBool, + environments: Arc>>, + manager: Arc>>, +} + +impl Poetry { + pub fn new(environment: &dyn Environment) -> Self { + Poetry { + searched: AtomicBool::new(false), + project_dirs: Arc::new(Mutex::new(vec![])), + env_vars: EnvVariables::from(environment), + poetry_executable: Arc::new(Mutex::new(None)), + environments: Arc::new(Mutex::new(vec![])), + manager: Arc::new(Mutex::new(None)), + } + } + pub fn from(environment: &dyn Environment) -> impl Locator { + Poetry::new(environment) + } + pub fn find_with_executable(&self) -> Option<()> { + let manager = manager::PoetryManager::find( + self.poetry_executable.lock().unwrap().clone(), + &self.env_vars, + )?; + + let environments_using_spawn = environment_locations_spawn::list_environments( + &manager.executable, + self.project_dirs.lock().unwrap().clone(), + &manager, + ) + .iter() + .filter_map(|env| env.prefix.clone()) + .collect::>(); + + // Get environments using the faster way. + if let Some(environments) = &self.find_with_cache() { + let environments = environments + .environments + .iter() + .filter_map(|env| env.prefix.clone()) + .collect::>(); + + for env in environments_using_spawn { + if !environments.contains(&env) { + warn!( + "Found a Poetry env {:?} using the poetry exe {:?}", + env, manager.executable + ); + // TODO: Send telemetry. + } + } + } else { + // TODO: Send telemetry. + for env in environments_using_spawn { + warn!( + "Found a Poetry env {:?} using the poetry exe {:?}", + env, manager.executable + ); + } + } + Some(()) + } + fn find_with_cache(&self) -> Option { + if let Ok(environments) = self.environments.lock() { + if !environments.is_empty() { + if let Ok(manager) = self.manager.lock() { + if let Some(manager) = manager.as_ref() { + return Some(LocatorResult { + managers: vec![manager.to_manager()], + environments: environments.clone(), + }); + } + } + } + if self.searched.load(Ordering::Relaxed) { + return None; + } + } + // First find the manager + let manager = manager::PoetryManager::find( + self.poetry_executable.lock().unwrap().clone(), + &self.env_vars, + ); + let mut managers = vec![]; + if let Some(manager) = manager { + let mut mgr = self.manager.lock().unwrap(); + mgr.replace(manager.clone()); + drop(mgr); + managers.push(manager.to_manager()); + } + let project_dirs = self.project_dirs.lock().unwrap().clone(); + + if let Some(result) = list_environments(&self.env_vars, &project_dirs) { + match self.environments.lock() { + Ok(mut environments) => { + environments.clear(); + environments.extend(result); + self.searched.store(true, Ordering::Relaxed); + Some(LocatorResult { + managers: managers.clone(), + environments: environments.clone(), + }) + } + Err(err) => { + error!("Failed to cache to Poetry environments: {:?}", err); + None + } + } + } else { + self.searched.store(true, Ordering::Relaxed); + None + } + } } -#[cfg(test)] -mod tests { - use super::*; +impl Locator for Poetry { + fn configure(&self, config: &Configuration) { + if let Some(search_paths) = &config.search_paths { + if !search_paths.is_empty() { + self.project_dirs.lock().unwrap().clear(); + self.project_dirs + .lock() + .unwrap() + .extend(search_paths.clone()); + } + } + if let Some(exe) = &config.poetry_executable { + self.poetry_executable.lock().unwrap().replace(exe.clone()); + } + } + + fn supported_categories(&self) -> Vec { + vec![PythonEnvironmentCategory::Poetry] + } + + fn from(&self, env: &PythonEnv) -> Option { + if let Some(result) = self.find_with_cache() { + for found_env in result.environments { + if let Some(symlinks) = &found_env.symlinks { + if symlinks.contains(&env.executable) { + return Some(found_env.clone()); + } + } + } + } + None + } - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); + fn find(&self, reporter: &dyn Reporter) { + if let Some(result) = self.find_with_cache() { + for found_env in result.environments { + if let Some(manager) = &found_env.manager { + reporter.report_manager(manager); + } + reporter.report_environment(&found_env); + } + } } } diff --git a/crates/pet-poetry/src/manager.rs b/crates/pet-poetry/src/manager.rs new file mode 100644 index 00000000..e06a4a56 --- /dev/null +++ b/crates/pet-poetry/src/manager.rs @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use pet_core::manager::{EnvManager, EnvManagerType}; +use std::{env, path::PathBuf}; + +use crate::env_variables::EnvVariables; + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct PoetryManager { + pub executable: PathBuf, +} + +impl PoetryManager { + pub fn find(executable: Option, env_variables: &EnvVariables) -> Option { + if let Some(executable) = executable { + if executable.is_file() { + return Some(PoetryManager { executable }); + } + } + + // Search in /.poetry/bin/python (as done in Python Extension) + + if let Some(home) = &env_variables.home { + let mut search_paths = vec![ + home.join(".poetry").join("bin").join("poetry"), + home.join(".local") + .join("pipx") + .join("venvs") + .join("poetry") + .join("bin") + .join("poetry"), + ]; + if let Some(poetry_home) = &env_variables.poetry_home { + if std::env::consts::OS == "windows" { + search_paths.push(poetry_home.join("bin").join("poetry.exe")); + } + search_paths.push(poetry_home.join("bin").join("poetry")); + } + if std::env::consts::OS == "windows" { + if let Some(app_data) = env_variables.appdata.clone() { + search_paths.push( + // https://python-poetry.org/docs/#installing-with-the-official-installer + app_data + .join("pypoetry") + .join("venv") + .join("Scripts") + .join("poetry.exe"), + ); + search_paths.push( + // https://python-poetry.org/docs/#installing-with-the-official-installer + app_data + .join("pypoetry") + .join("venv") + .join("Scripts") + .join("poetry"), + ); + search_paths.push( + app_data.join("Python").join("scripts").join("poetry.exe"), // https://python-poetry.org/docs/#installing-with-the-official-installer + ); + search_paths.push( + app_data.join("Python").join("scripts").join("poetry"), // https://python-poetry.org/docs/#installing-with-the-official-installer + ); + } + } else if std::env::consts::OS == "macos" { + search_paths.push( + // https://python-poetry.org/docs/#installing-with-the-official-installer + home.join("Library") + .join("Application Support") + .join("pypoetry") + .join("venv") + .join("bin") + .join("poetry"), + ); + search_paths.push( + home.join(".local").join("bin").join("poetry"), // https://python-poetry.org/docs/#installing-with-the-official-installer + ); + } else { + search_paths.push( + // https://python-poetry.org/docs/#installing-with-the-official-installer + home.join(".local") + .join("share") + .join("pypoetry") + .join("venv") + .join("bin") + .join("poetry"), + ); + search_paths.push( + home.join(".local").join("bin").join("poetry"), // https://python-poetry.org/docs/#installing-with-the-official-installer + ); + } + for executable in search_paths { + if executable.is_file() { + return Some(PoetryManager { executable }); + } + } + + // Look for poetry in current PATH. + if let Some(env_path) = &env_variables.path { + for each in env::split_paths(env_path) { + let executable = each.join("poetry"); + if executable.is_file() { + return Some(PoetryManager { executable }); + } + } + } + } + None + } + pub fn to_manager(&self) -> EnvManager { + EnvManager { + executable: self.executable.clone(), + version: None, + tool: EnvManagerType::Poetry, + } + } +} diff --git a/crates/pet-poetry/src/pyproject_toml.rs b/crates/pet-poetry/src/pyproject_toml.rs new file mode 100644 index 00000000..737505e8 --- /dev/null +++ b/crates/pet-poetry/src/pyproject_toml.rs @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use log::trace; + +pub struct PyProjectToml { + pub name: String, + pub file: PathBuf, +} + +impl PyProjectToml { + pub fn new(name: String, file: PathBuf) -> Self { + trace!("Poetry project: {:?} with name {:?}", file, name); + PyProjectToml { name, file } + } + pub fn find(path: &Path) -> Option { + parse(&path.join("pyproject.toml")) + } +} + +fn parse(file: &Path) -> Option { + let contents = fs::read_to_string(file).ok()?; + + match toml::from_str::(&contents) { + Ok(value) => { + let mut name = None; + if let Some(tool) = value.get("tool") { + if let Some(poetry) = tool.get("poetry") { + if let Some(name_value) = poetry.get("name") { + name = name_value.as_str().map(|s| s.to_string()); + } + } + } + name.map(|name| PyProjectToml::new(name, file.into())) + } + Err(e) => { + eprintln!("Error parsing toml file: {:?}", e); + None + } + } +} + +#[cfg(test)] +mod tests { + use serde_json::Value; + + #[test] + fn extract_name_from_pypoetry_toml() { + let cfg: Value = toml::from_str( + r#" +[tool.poetry] +name = "poetry-demo" +version = "0.1.0" +description = "" +authors = ["User Name "] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.12" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" +"#, + ) + .unwrap(); + assert_eq!( + cfg["tool"]["poetry"]["name"].as_str().unwrap(), + "poetry-demo" + ); + } +} diff --git a/crates/pet-pyenv/Cargo.toml b/crates/pet-pyenv/Cargo.toml index 4b709272..46c20a80 100644 --- a/crates/pet-pyenv/Cargo.toml +++ b/crates/pet-pyenv/Cargo.toml @@ -17,5 +17,3 @@ pet-fs = { path = "../pet-fs" } pet-conda = { path = "../pet-conda" } log = "0.4.21" regex = "1.10.4" - -[dev-dependencies] diff --git a/crates/pet-python-utils/src/lib.rs b/crates/pet-python-utils/src/lib.rs index b4644495..16872136 100644 --- a/crates/pet-python-utils/src/lib.rs +++ b/crates/pet-python-utils/src/lib.rs @@ -4,6 +4,7 @@ pub mod env; pub mod executable; mod headers; +pub mod platform_dirs; pub mod pyvenv_cfg; pub mod version; diff --git a/crates/pet-python-utils/src/platform_dirs.rs b/crates/pet-python-utils/src/platform_dirs.rs new file mode 100644 index 00000000..3b687e29 --- /dev/null +++ b/crates/pet-python-utils/src/platform_dirs.rs @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::{env, path::PathBuf}; + +/// Maps to platformdirs package in Python +pub struct Platformdirs { + app_name: String, + version: Option, + roaming: bool, +} + +impl Platformdirs { + pub fn new(app_name: String, roaming: bool) -> Self { + Self { + app_name, + version: None, + roaming, + } + } + + /// Maps to the user_cache_path function in platformdirs package (Python) + pub fn user_cache_path(&self) -> Option { + self.user_cache_dir() + } + + /// Maps to the user_cache_dir function in platformdirs package (Python) + pub fn user_cache_dir(&self) -> Option { + if cfg!(windows) { + env::var("CSIDL_LOCAL_APPDATA") + .ok() + .map(PathBuf::from) + .map(|app_data| { + self.append_app_name_and_version(app_data.join(&self.app_name).join("Cache")) + }) + } else if std::env::consts::OS == "macos" { + env::var("HOME") + .ok() + .map(PathBuf::from) + .map(|home| self.append_app_name_and_version(home.join("Library").join("Caches"))) + } else { + let mut path = env::var("XDG_CACHE_HOME").ok().map(PathBuf::from); + if path.is_none() { + path = env::var("HOME") + .ok() + .map(PathBuf::from) + .map(|home| home.join(".cache")); + } + path.map(|path| self.append_app_name_and_version(path)) + } + } + + /// Maps to the user_config_path function in platformdirs package (Python) + pub fn user_config_path(&self) -> Option { + if std::env::consts::OS == "windows" || std::env::consts::OS == "macos" { + self.user_data_dir() + } else { + let mut path = env::var("XDG_CONFIG_HOME").ok().map(PathBuf::from); + if path.is_none() { + path = env::var("HOME") + .ok() + .map(PathBuf::from) + .map(|home| home.join(".config")); + } + path.map(|path| self.append_app_name_and_version(path)) + } + } + /// Maps to the user_data_dir function in platformdirs package (Python) + pub fn user_data_dir(&self) -> Option { + if std::env::consts::OS == "windows" { + let var = if self.roaming { + "CSIDL_APPDATA" + } else { + "CSIDL_LOCAL_APPDATA" + }; + env::var(var) + .ok() + .map(PathBuf::from) + .map(|app_data| self.append_app_name_and_version(app_data)) + } else if std::env::consts::OS == "macos" { + env::var("HOME").ok().map(PathBuf::from).map(|home| { + self.append_app_name_and_version(home.join("Library").join("Application Support")) + }) + } else { + let mut path = env::var("XDG_DATA_HOME").ok().map(PathBuf::from); + if path.is_none() { + path = env::var("HOME") + .ok() + .map(PathBuf::from) + .map(|home| home.join(".local").join("share")); + } + path.map(|path| self.append_app_name_and_version(path)) + } + } + + fn append_app_name_and_version(&self, path: PathBuf) -> PathBuf { + let path = path.join(&self.app_name); + if let Some(version) = &self.version { + path.join(version) + } else { + path + } + } +} diff --git a/crates/pet-reporter/src/environment.rs b/crates/pet-reporter/src/environment.rs index 34e6a53e..5611d858 100644 --- a/crates/pet-reporter/src/environment.rs +++ b/crates/pet-reporter/src/environment.rs @@ -32,6 +32,7 @@ fn python_category_to_string(category: &PythonEnvironmentCategory) -> &'static s PythonEnvironmentCategory::Venv => "venv", PythonEnvironmentCategory::VirtualEnv => "virtualenv", PythonEnvironmentCategory::Unknown => "unknown", + PythonEnvironmentCategory::Poetry => "poetry", } } diff --git a/crates/pet-reporter/src/manager.rs b/crates/pet-reporter/src/manager.rs index a46cfe41..8760536d 100644 --- a/crates/pet-reporter/src/manager.rs +++ b/crates/pet-reporter/src/manager.rs @@ -9,6 +9,7 @@ fn tool_to_string(tool: &EnvManagerType) -> &'static str { match tool { EnvManagerType::Conda => "conda", EnvManagerType::Pyenv => "pyenv", + EnvManagerType::Poetry => "poery", } } diff --git a/crates/pet-telemetry/Cargo.toml b/crates/pet-telemetry/Cargo.toml index d42b1b98..38140aa0 100644 --- a/crates/pet-telemetry/Cargo.toml +++ b/crates/pet-telemetry/Cargo.toml @@ -3,6 +3,9 @@ name = "pet-telemetry" version = "0.1.0" edition = "2021" +[target.'cfg(target_os = "windows")'.dependencies] +msvc_spectre_libs = { version = "0.1.1", features = ["error"] } + [dependencies] pet-core = { path = "../pet-core" } pet-fs = { path = "../pet-fs" } diff --git a/crates/pet/Cargo.toml b/crates/pet/Cargo.toml index 2d50c5a7..9e8cb4cb 100644 --- a/crates/pet/Cargo.toml +++ b/crates/pet/Cargo.toml @@ -16,7 +16,6 @@ pet-homebrew = { path = "../pet-homebrew" } [dependencies] pet-core = { path = "../pet-core" } pet-conda = { path = "../pet-conda" } -pet-global = { path = "../pet-global" } pet-jsonrpc = { path = "../pet-jsonrpc" } pet-fs = { path = "../pet-fs" } pet-pyenv = { path = "../pet-pyenv" } diff --git a/crates/pet/src/find.rs b/crates/pet/src/find.rs index 2ef9a441..b82bb9a6 100644 --- a/crates/pet/src/find.rs +++ b/crates/pet/src/find.rs @@ -9,6 +9,7 @@ use pet_core::reporter::Reporter; use pet_core::{Configuration, Locator}; use pet_env_var_path::get_search_paths_from_env_variables; use pet_global_virtualenvs::list_global_virtual_envs_paths; +use pet_poetry::Poetry; use pet_python_utils::env::PythonEnv; use pet_python_utils::executable::{ find_executable, find_executables, should_search_for_environments_in_path, @@ -52,13 +53,22 @@ pub fn find_and_report_envs( }); // By now all conda envs have been found - // Get the conda info in a separate thread. + // Spawn conda in a separate thread. // & see if we can find more environments by spawning conda. // But we will not wait for this to complete. thread::spawn(move || { conda_locator.find_with_conda_executable(conda_executable); Some(()) }); + // By now all poetry envs have been found + // Spawn poetry exe in a separate thread. + // & see if we can find more environments by spawning poetry. + // But we will not wait for this to complete. + thread::spawn(move || { + let env = EnvironmentApi::new(); + Poetry::new(&env).find_with_executable(); + Some(()) + }); }); // Step 2: Search in PATH variable s.spawn(|| { diff --git a/crates/pet/src/lib.rs b/crates/pet/src/lib.rs index 48d6ac8a..06382b34 100644 --- a/crates/pet/src/lib.rs +++ b/crates/pet/src/lib.rs @@ -12,8 +12,12 @@ pub mod find; pub mod locators; pub mod resolve; -pub fn find_and_report_envs_stdio(print_list: bool, print_summary: bool) { - stdio::initialize_logger(log::LevelFilter::Info); +pub fn find_and_report_envs_stdio(print_list: bool, print_summary: bool, verbose: bool) { + stdio::initialize_logger(if verbose { + log::LevelFilter::Trace + } else { + log::LevelFilter::Info + }); let now = SystemTime::now(); let stdio_reporter = Arc::new(stdio::create_reporter(print_list)); @@ -25,13 +29,12 @@ pub fn find_and_report_envs_stdio(print_list: bool, print_summary: bool) { if let Ok(cwd) = env::current_dir() { config.search_paths = Some(vec![cwd]); } + let locators = create_locators(conda_locator.clone()); + for locator in locators.iter() { + locator.configure(&config); + } - find_and_report_envs( - &reporter, - config, - &create_locators(conda_locator.clone()), - conda_locator, - ); + find_and_report_envs(&reporter, config, &locators, conda_locator); if print_summary { let summary = stdio_reporter.get_summary(); diff --git a/crates/pet/src/locators.rs b/crates/pet/src/locators.rs index 761cdde5..fd8636d5 100644 --- a/crates/pet/src/locators.rs +++ b/crates/pet/src/locators.rs @@ -14,6 +14,7 @@ use pet_mac_commandlinetools::MacCmdLineTools; use pet_mac_python_org::MacPythonOrg; use pet_mac_xcode::MacXCode; use pet_pipenv::PipEnv; +use pet_poetry::Poetry; use pet_pyenv::PyEnv; use pet_python_utils::env::{PythonEnv, ResolvedPythonEnv}; use pet_venv::Venv; @@ -55,6 +56,7 @@ pub fn create_locators(conda_locator: Arc) -> Arc>> // 6. Support for Virtual Envs // The order of these matter. // Basically PipEnv is a superset of VirtualEnvWrapper, which is a superset of Venv, which is a superset of VirtualEnv. + locators.push(Arc::new(Poetry::from(&environment))); locators.push(Arc::new(PipEnv::from(&environment))); locators.push(Arc::new(VirtualEnvWrapper::from(&environment))); locators.push(Arc::new(Venv::new())); diff --git a/crates/pet/src/main.rs b/crates/pet/src/main.rs index 8e0826c4..eac05af3 100644 --- a/crates/pet/src/main.rs +++ b/crates/pet/src/main.rs @@ -22,6 +22,10 @@ enum Commands { Find { #[arg(short, long)] list: Option, + + // Whether to display verbose output (defaults to just info). + #[arg(short, long)] + verbose: bool, }, /// Starts the JSON RPC Server. Server, @@ -30,8 +34,13 @@ enum Commands { fn main() { let cli = Cli::parse(); - match cli.command.unwrap_or(Commands::Find { list: Some(true) }) { - Commands::Find { list } => find_and_report_envs_stdio(list.unwrap_or(true), true), + match cli.command.unwrap_or(Commands::Find { + list: Some(true), + verbose: false, + }) { + Commands::Find { list, verbose } => { + find_and_report_envs_stdio(list.unwrap_or(true), true, verbose) + } Commands::Server => start_jsonrpc_server(), } } From f62eecfbc2849dad901837992c1078c62971ea1b Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Fri, 21 Jun 2024 17:13:53 +1000 Subject: [PATCH 02/11] With fixes --- crates/pet-poetry/src/config.rs | 43 +++++--------- .../pet-poetry/src/environment_locations.rs | 59 ++++++++++++++++--- crates/pet-poetry/src/lib.rs | 5 ++ crates/pet-poetry/src/manager.rs | 16 +++-- crates/pet-python-utils/src/platform_dirs.rs | 55 ++++++++++------- 5 files changed, 116 insertions(+), 62 deletions(-) diff --git a/crates/pet-poetry/src/config.rs b/crates/pet-poetry/src/config.rs index 0171fcb2..4f67ac2c 100644 --- a/crates/pet-poetry/src/config.rs +++ b/crates/pet-poetry/src/config.rs @@ -17,11 +17,11 @@ static _APP_NAME: &str = "pypoetry"; pub struct Config { pub virtualenvs_in_project: bool, pub virtualenvs_path: PathBuf, - pub file: PathBuf, + pub file: Option, } impl Config { - fn new(file: PathBuf, virtualenvs_path: PathBuf, virtualenvs_in_project: bool) -> Self { + fn new(file: Option, virtualenvs_path: PathBuf, virtualenvs_in_project: bool) -> Self { trace!( "Poetry config file: {:?} with virtualenv.path {:?}", file, @@ -34,47 +34,31 @@ impl Config { } } pub fn find_global(env: &EnvVariables) -> Option { - let file = find_config_file(env)?; - create_config(&file, env) + let file = find_config_file(env); + create_config(file, env) } pub fn find_local(path: &Path, env: &EnvVariables) -> Option { let file = path.join("poetry.toml"); if file.is_file() { - create_config(&file, env) + create_config(Some(file), env) } else { None } } } -fn create_config(file: &Path, env: &EnvVariables) -> Option { - let cfg = parse(file)?; - - if let Some(virtualenvs_path) = &cfg.virtualenvs_path { +fn create_config(file: Option, env: &EnvVariables) -> Option { + let cfg = file.clone().and_then(|f| parse(&f)); + if let Some(virtualenvs_path) = &cfg.clone().and_then(|cfg| cfg.virtualenvs_path) { return Some(Config::new( - file.to_path_buf(), + file.clone(), virtualenvs_path.clone(), - cfg.virtualenvs_in_project, + cfg.map(|cfg| cfg.virtualenvs_in_project) + .unwrap_or_default(), )); } - - let cache_dir = match cfg.cache_dir { - Some(cache_dir) => { - if cache_dir.is_dir() { - Some(cache_dir) - } else { - get_default_cache_dir(env) - } - } - None => get_default_cache_dir(env), - }; - - if let Some(cache_dir) = cache_dir { - Some(Config::new( - file.to_path_buf(), - cache_dir.join("virtualenvs"), - cfg.virtualenvs_in_project, - )) + if let Some(cache_dir) = get_default_cache_dir(env) { + Some(Config::new(file, cache_dir.join("virtualenvs"), false)) } else { None } @@ -106,6 +90,7 @@ fn find_config_file(env: &EnvVariables) -> Option { } } +#[derive(Debug, Clone, PartialEq, Eq)] struct ConfigToml { virtualenvs_in_project: bool, cache_dir: Option, diff --git a/crates/pet-poetry/src/environment_locations.rs b/crates/pet-poetry/src/environment_locations.rs index 1d109b39..d890350f 100644 --- a/crates/pet-poetry/src/environment_locations.rs +++ b/crates/pet-poetry/src/environment_locations.rs @@ -3,6 +3,7 @@ use base64::{engine::general_purpose, Engine as _}; use lazy_static::lazy_static; +use log::trace; use pet_core::python_environment::PythonEnvironment; use pet_fs::path::norm_case; use regex::Regex; @@ -38,10 +39,19 @@ pub fn list_environments( for project_dir in project_dirs { if let Some(pyproject_toml) = PyProjectToml::find(project_dir) { let virtualenv_prefix = generate_env_name(&pyproject_toml.name, project_dir); + trace!( + "Found pyproject.toml ({}): {:?} in {:?}", + virtualenv_prefix, + pyproject_toml.name, + project_dir + ); - for virtual_env in + for virtual_env in [ list_all_environments_from_project_config(&global_config, project_dir, env) - .unwrap_or(global_envs.clone()) + .unwrap_or_default(), + global_envs.clone(), + ] + .concat() { // Check if this virtual env belongs to this project let name = virtual_env @@ -66,15 +76,21 @@ fn list_all_environments_from_project_config( path: &Path, env: &EnvVariables, ) -> Option> { - let config = Config::find_local(path, env)?; + let config = Config::find_local(path, env); let mut envs = vec![]; - if let Some(project_envs) = list_all_environments_from_config(&config) { - envs.extend(project_envs); + + if let Some(config) = &config { + if let Some(project_envs) = list_all_environments_from_config(config) { + envs.extend(project_envs); + } } // Check if we're allowed to use .venv as a poetry env // This can be configured in global, project or env variable. - if config.virtualenvs_in_project + if config + .clone() + .map(|c| c.virtualenvs_in_project) + .unwrap_or_default() || global .clone() .map(|config| config.virtualenvs_in_project) @@ -113,9 +129,13 @@ pub fn generate_env_name(name: &str, cwd: &PathBuf) -> String { .chars() .take(42) .collect::(); - let normalized_cwd = norm_case(Path::new(cwd)); + let normalized_cwd = if cfg!(windows) { + norm_case(cwd).to_str().unwrap_or_default().to_lowercase() + } else { + norm_case(cwd).to_str().unwrap_or_default().to_string() + }; let mut hasher = Sha256::new(); - hasher.update(normalized_cwd.to_str().unwrap().as_bytes()); + hasher.update(normalized_cwd.as_bytes()); let h_bytes = hasher.finalize(); let h_str = general_purpose::URL_SAFE .encode(h_bytes) @@ -130,6 +150,7 @@ mod tests { use super::*; #[test] + #[cfg(unix)] fn test_hash_generation() { let hashed_name = generate_env_name( "poetry-demo", @@ -138,4 +159,26 @@ mod tests { assert_eq!(hashed_name, "poetry-demo-gNT2WXAV-py"); } + + #[test] + #[cfg(unix)] + fn test_hash_generation_upper_case() { + let hashed_name = generate_env_name( + "new-project", + &"/Users/donjayamanne/temp/POETRY-UPPER/new-PROJECT".into(), + ); + + assert_eq!(hashed_name, "new-project-TbBV0MKD-py"); + } + + #[test] + #[cfg(windows)] + fn test_hash_generation_windows() { + let hashed_name = generate_env_name( + "demo-project1", + &"C:\\temp\\poetry-folders\\demo-project1".into(), + ); + + assert_eq!(hashed_name, "demo-project1-f7sQRtG5-py"); + } } diff --git a/crates/pet-poetry/src/lib.rs b/crates/pet-poetry/src/lib.rs index 378a37b6..4ea5a28c 100644 --- a/crates/pet-poetry/src/lib.rs +++ b/crates/pet-poetry/src/lib.rs @@ -182,6 +182,11 @@ impl Locator for Poetry { fn find(&self, reporter: &dyn Reporter) { if let Some(result) = self.find_with_cache() { + if let Ok(manager) = self.manager.lock() { + if let Some(manager) = manager.as_ref() { + reporter.report_manager(&manager.to_manager()); + } + } for found_env in result.environments { if let Some(manager) = &found_env.manager { reporter.report_manager(manager); diff --git a/crates/pet-poetry/src/manager.rs b/crates/pet-poetry/src/manager.rs index e06a4a56..c3d68f4d 100644 --- a/crates/pet-poetry/src/manager.rs +++ b/crates/pet-poetry/src/manager.rs @@ -48,12 +48,20 @@ impl PoetryManager { .join("poetry.exe"), ); search_paths.push( - // https://python-poetry.org/docs/#installing-with-the-official-installer + // Found after installing on windows using Poetry install notes app_data - .join("pypoetry") - .join("venv") + .join("Roaming") + .join("Python") .join("Scripts") - .join("poetry"), + .join("poetry.exe"), + ); + search_paths.push( + // https://python-poetry.org/docs/#installing-with-the-official-installer + app_data + .join("pypoetry") + .join("venv") + .join("Scripts") + .join("poetry"), ); search_paths.push( app_data.join("Python").join("scripts").join("poetry.exe"), // https://python-poetry.org/docs/#installing-with-the-official-installer diff --git a/crates/pet-python-utils/src/platform_dirs.rs b/crates/pet-python-utils/src/platform_dirs.rs index 3b687e29..5806a666 100644 --- a/crates/pet-python-utils/src/platform_dirs.rs +++ b/crates/pet-python-utils/src/platform_dirs.rs @@ -30,14 +30,14 @@ impl Platformdirs { env::var("CSIDL_LOCAL_APPDATA") .ok() .map(PathBuf::from) - .map(|app_data| { - self.append_app_name_and_version(app_data.join(&self.app_name).join("Cache")) - }) + .or(env::var("USERPROFILE") // Fallback for CSIDL_LOCAL_APPDATA + .ok() + .map(|user| PathBuf::from(user).join("AppData").join("Local"))) + .map(|app_data| self.append_app_name_and_version(app_data, Some("Cache"))) } else if std::env::consts::OS == "macos" { - env::var("HOME") - .ok() - .map(PathBuf::from) - .map(|home| self.append_app_name_and_version(home.join("Library").join("Caches"))) + env::var("HOME").ok().map(PathBuf::from).map(|home| { + self.append_app_name_and_version(home.join("Library").join("Caches"), None) + }) } else { let mut path = env::var("XDG_CACHE_HOME").ok().map(PathBuf::from); if path.is_none() { @@ -46,7 +46,7 @@ impl Platformdirs { .map(PathBuf::from) .map(|home| home.join(".cache")); } - path.map(|path| self.append_app_name_and_version(path)) + path.map(|path| self.append_app_name_and_version(path, None)) } } @@ -62,24 +62,32 @@ impl Platformdirs { .map(PathBuf::from) .map(|home| home.join(".config")); } - path.map(|path| self.append_app_name_and_version(path)) + path.map(|path| self.append_app_name_and_version(path, None)) } } /// Maps to the user_data_dir function in platformdirs package (Python) pub fn user_data_dir(&self) -> Option { if std::env::consts::OS == "windows" { - let var = if self.roaming { - "CSIDL_APPDATA" + let app_data = if self.roaming { + env::var("CSIDL_APPDATA").ok().map(PathBuf::from).or( + env::var("USERPROFILE") // Fallback for CSIDL_LOCAL_APPDATA + .ok() + .map(|user| PathBuf::from(user).join("AppData").join("Roaming")), + ) } else { - "CSIDL_LOCAL_APPDATA" + env::var("CSIDL_LOCAL_APPDATA").ok().map(PathBuf::from).or( + env::var("USERPROFILE") // Fallback for CSIDL_LOCAL_APPDATA + .ok() + .map(|user| PathBuf::from(user).join("AppData").join("Local")), + ) }; - env::var(var) - .ok() - .map(PathBuf::from) - .map(|app_data| self.append_app_name_and_version(app_data)) + app_data.map(|app_data| self.append_app_name_and_version(app_data, None)) } else if std::env::consts::OS == "macos" { env::var("HOME").ok().map(PathBuf::from).map(|home| { - self.append_app_name_and_version(home.join("Library").join("Application Support")) + self.append_app_name_and_version( + home.join("Library").join("Application Support"), + None, + ) }) } else { let mut path = env::var("XDG_DATA_HOME").ok().map(PathBuf::from); @@ -89,14 +97,19 @@ impl Platformdirs { .map(PathBuf::from) .map(|home| home.join(".local").join("share")); } - path.map(|path| self.append_app_name_and_version(path)) + path.map(|path| self.append_app_name_and_version(path, None)) } } - fn append_app_name_and_version(&self, path: PathBuf) -> PathBuf { - let path = path.join(&self.app_name); + fn append_app_name_and_version(&self, path: PathBuf, suffix: Option<&str>) -> PathBuf { + let mut path = path.join(&self.app_name); if let Some(version) = &self.version { - path.join(version) + path = path.join(version) + } else { + path = path + } + if let Some(suffix) = suffix { + path.join(suffix) } else { path } From 17a08cbf2efb4cf5896f6e846c86beed4d542d07 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Fri, 21 Jun 2024 17:16:43 +1000 Subject: [PATCH 03/11] Fixes --- crates/pet-poetry/src/manager.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/pet-poetry/src/manager.rs b/crates/pet-poetry/src/manager.rs index c3d68f4d..e5e1224d 100644 --- a/crates/pet-poetry/src/manager.rs +++ b/crates/pet-poetry/src/manager.rs @@ -58,10 +58,10 @@ impl PoetryManager { search_paths.push( // https://python-poetry.org/docs/#installing-with-the-official-installer app_data - .join("pypoetry") - .join("venv") - .join("Scripts") - .join("poetry"), + .join("pypoetry") + .join("venv") + .join("Scripts") + .join("poetry"), ); search_paths.push( app_data.join("Python").join("scripts").join("poetry.exe"), // https://python-poetry.org/docs/#installing-with-the-official-installer From e8e651ca8f2c31cf1a352e35ee40915f8d744063 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Fri, 21 Jun 2024 17:17:12 +1000 Subject: [PATCH 04/11] fixes --- crates/pet-python-utils/src/platform_dirs.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/pet-python-utils/src/platform_dirs.rs b/crates/pet-python-utils/src/platform_dirs.rs index 5806a666..b5696655 100644 --- a/crates/pet-python-utils/src/platform_dirs.rs +++ b/crates/pet-python-utils/src/platform_dirs.rs @@ -105,8 +105,6 @@ impl Platformdirs { let mut path = path.join(&self.app_name); if let Some(version) = &self.version { path = path.join(version) - } else { - path = path } if let Some(suffix) = suffix { path.join(suffix) From 16b1e63de1037694d6503e3a2d30cb9be92bdee1 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Fri, 21 Jun 2024 17:21:01 +1000 Subject: [PATCH 05/11] Fixes --- crates/pet-poetry/src/config.rs | 8 +++----- crates/pet-poetry/src/pyproject_toml.rs | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/crates/pet-poetry/src/config.rs b/crates/pet-poetry/src/config.rs index 4f67ac2c..1f4b570d 100644 --- a/crates/pet-poetry/src/config.rs +++ b/crates/pet-poetry/src/config.rs @@ -57,11 +57,9 @@ fn create_config(file: Option, env: &EnvVariables) -> Option { .unwrap_or_default(), )); } - if let Some(cache_dir) = get_default_cache_dir(env) { - Some(Config::new(file, cache_dir.join("virtualenvs"), false)) - } else { - None - } + + get_default_cache_dir(env) + .map(|cache_dir| Config::new(file, cache_dir.join("virtualenvs"), false)) } /// Maps to DEFAULT_CACHE_DIR in poetry fn get_default_cache_dir(env: &EnvVariables) -> Option { diff --git a/crates/pet-poetry/src/pyproject_toml.rs b/crates/pet-poetry/src/pyproject_toml.rs index 737505e8..b917afff 100644 --- a/crates/pet-poetry/src/pyproject_toml.rs +++ b/crates/pet-poetry/src/pyproject_toml.rs @@ -10,13 +10,12 @@ use log::trace; pub struct PyProjectToml { pub name: String, - pub file: PathBuf, } impl PyProjectToml { pub fn new(name: String, file: PathBuf) -> Self { trace!("Poetry project: {:?} with name {:?}", file, name); - PyProjectToml { name, file } + PyProjectToml { name } } pub fn find(path: &Path) -> Option { parse(&path.join("pyproject.toml")) From 8ccdfe10de114139907468e670cd2161c02e930c Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 24 Jun 2024 13:06:29 +1000 Subject: [PATCH 06/11] Fixes --- Cargo.lock | 1 + crates/pet-conda/src/conda_rc.rs | 13 +- crates/pet-conda/src/env_variables.rs | 1 + crates/pet-conda/src/utils.rs | 26 ++- crates/pet-conda/tests/conda_rc_test.rs | 1 + crates/pet-core/src/os_environment.rs | 4 +- crates/pet-poetry/Cargo.toml | 1 + crates/pet-poetry/src/config.rs | 177 ++++++++++++++++-- crates/pet-poetry/src/env_variables.rs | 13 +- .../pet-poetry/src/environment_locations.rs | 114 ++++++----- crates/pet-poetry/src/lib.rs | 88 ++++----- crates/pet-poetry/src/manager.rs | 3 +- crates/pet-poetry/src/pyproject_toml.rs | 23 ++- crates/pet-poetry/tests/common.rs | 58 ++++++ crates/pet-poetry/tests/config_test.rs | 128 +++++++++++++ .../user_home/config_dir/config.toml | 7 + .../project_dir/poetry.toml | 5 + .../user_home/config_dir/config.toml | 7 + crates/pet-pyenv/src/environments.rs | 79 +++++--- crates/pet-pyenv/src/lib.rs | 79 ++++++-- crates/pet/tests/ci_test.rs | 1 + 21 files changed, 646 insertions(+), 183 deletions(-) create mode 100644 crates/pet-poetry/tests/common.rs create mode 100644 crates/pet-poetry/tests/config_test.rs create mode 100644 crates/pet-poetry/tests/unix/global_config_with_values/user_home/config_dir/config.toml create mode 100644 crates/pet-poetry/tests/unix/local_config_with_values/project_dir/poetry.toml create mode 100644 crates/pet-poetry/tests/unix/local_config_with_values/user_home/config_dir/config.toml diff --git a/Cargo.lock b/Cargo.lock index 0bb9bfed..75953fae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -495,6 +495,7 @@ dependencies = [ "pet-fs", "pet-python-utils", "pet-reporter", + "pet-virtualenv", "regex", "serde", "serde_json", diff --git a/crates/pet-conda/src/conda_rc.rs b/crates/pet-conda/src/conda_rc.rs index 36964f72..a62c81f7 100644 --- a/crates/pet-conda/src/conda_rc.rs +++ b/crates/pet-conda/src/conda_rc.rs @@ -72,6 +72,8 @@ fn get_conda_rc_search_paths(env_vars: &EnvVariables) -> Vec { // Search paths documented here // https://conda.io/projects/conda/en/latest/user-guide/configuration/use-condarc.html#searching-for-condarc fn get_conda_rc_search_paths(env_vars: &EnvVariables) -> Vec { + use crate::utils::change_root_of_path; + let mut search_paths: Vec = [ "/etc/conda/.condarc", "/etc/conda/condarc", @@ -82,16 +84,7 @@ fn get_conda_rc_search_paths(env_vars: &EnvVariables) -> Vec { ] .iter() .map(PathBuf::from) - .map(|p| { - // This only applies in tests. - // We need this, as the root folder cannot be mocked. - if let Some(ref root) = env_vars.root { - // Strip the first `/` (this path is only for testing purposes) - root.join(&p.to_string_lossy()[1..]) - } else { - p - } - }) + .map(|p| change_root_of_path(&p, &env_vars.root)) .collect(); if let Some(ref conda_root) = env_vars.conda_root { diff --git a/crates/pet-conda/src/env_variables.rs b/crates/pet-conda/src/env_variables.rs index 08056ba6..aa1bebc8 100644 --- a/crates/pet-conda/src/env_variables.rs +++ b/crates/pet-conda/src/env_variables.rs @@ -10,6 +10,7 @@ use pet_core::os_environment::Environment; // Lets be explicit, this way we never miss a value (in Windows or Unix). pub struct EnvVariables { pub home: Option, + /// Only used in tests, None in production. pub root: Option, pub path: Option, pub userprofile: Option, diff --git a/crates/pet-conda/src/utils.rs b/crates/pet-conda/src/utils.rs index cbc95a21..aaf9e898 100644 --- a/crates/pet-conda/src/utils.rs +++ b/crates/pet-conda/src/utils.rs @@ -1,7 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -use std::{fs, path::Path}; +use std::{ + fs, + path::{Path, PathBuf}, +}; // conda-meta must exist as this contains a mandatory `history` file. pub fn is_conda_install(path: &Path) -> bool { @@ -17,3 +20,24 @@ pub fn is_conda_env(path: &Path) -> bool { false } } + +/// Only used in tests, noop in production. +/// +/// Change the root of the path to a new root. +/// Lets assume some config file is located in the root directory /etc/config/config.toml. +/// We cannot test this unless we create such a file on the root of the filesystem. +/// Thats very risky and not recommended (ideally we want to create stuff in separate test folders). +/// The solution is to change the root of the path to a test folder. +pub fn change_root_of_path(path: &Path, new_root: &Option) -> PathBuf { + if cfg!(windows) { + return path.to_path_buf(); + } + if let Some(new_root) = new_root { + // This only applies in tests. + // We need this, as the root folder cannot be mocked. + // Strip the first `/` (this path is only for testing purposes) + new_root.join(&path.to_string_lossy()[1..]) + } else { + path.to_path_buf() + } +} diff --git a/crates/pet-conda/tests/conda_rc_test.rs b/crates/pet-conda/tests/conda_rc_test.rs index 1590f8c5..1c060734 100644 --- a/crates/pet-conda/tests/conda_rc_test.rs +++ b/crates/pet-conda/tests/conda_rc_test.rs @@ -84,6 +84,7 @@ fn finds_conda_rc_from_root() { use pet_conda::conda_rc::Condarc; use std::path::PathBuf; + print!("root: {:?}", resolve_test_path(&["unix", "conda_rc_root", "root"])); let root = resolve_test_path(&["unix", "conda_rc_root", "root"]); let home = resolve_test_path(&["unix", "conda_rc_root", "user_home"]); let env = create_env_variables(home, root); diff --git a/crates/pet-core/src/os_environment.rs b/crates/pet-core/src/os_environment.rs index 0b7e0907..451496c1 100644 --- a/crates/pet-core/src/os_environment.rs +++ b/crates/pet-core/src/os_environment.rs @@ -7,9 +7,7 @@ use pet_fs::path::norm_case; pub trait Environment { fn get_user_home(&self) -> Option; - /** - * Only used in tests, this is the root `/`. - */ + /// Only used in tests, None in production. #[allow(dead_code)] fn get_root(&self) -> Option; fn get_env_var(&self, key: String) -> Option; diff --git a/crates/pet-poetry/Cargo.toml b/crates/pet-poetry/Cargo.toml index d1fe0e11..971a9380 100644 --- a/crates/pet-poetry/Cargo.toml +++ b/crates/pet-poetry/Cargo.toml @@ -12,6 +12,7 @@ serde_json = "1.0.93" lazy_static = "1.4.0" pet-core = { path = "../pet-core" } pet-python-utils = { path = "../pet-python-utils" } +pet-virtualenv = { path = "../pet-virtualenv" } pet-reporter = { path = "../pet-reporter" } pet-fs = { path = "../pet-fs" } log = "0.4.21" diff --git a/crates/pet-poetry/src/config.rs b/crates/pet-poetry/src/config.rs index 1f4b570d..8c2b0924 100644 --- a/crates/pet-poetry/src/config.rs +++ b/crates/pet-poetry/src/config.rs @@ -15,13 +15,17 @@ static _APP_NAME: &str = "pypoetry"; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Config { - pub virtualenvs_in_project: bool, + pub virtualenvs_in_project: Option, pub virtualenvs_path: PathBuf, pub file: Option, } impl Config { - fn new(file: Option, virtualenvs_path: PathBuf, virtualenvs_in_project: bool) -> Self { + fn new( + file: Option, + virtualenvs_path: PathBuf, + virtualenvs_in_project: Option, + ) -> Self { trace!( "Poetry config file: {:?} with virtualenv.path {:?}", file, @@ -53,13 +57,12 @@ fn create_config(file: Option, env: &EnvVariables) -> Option { return Some(Config::new( file.clone(), virtualenvs_path.clone(), - cfg.map(|cfg| cfg.virtualenvs_in_project) - .unwrap_or_default(), + cfg.and_then(|cfg| cfg.virtualenvs_in_project), )); } get_default_cache_dir(env) - .map(|cache_dir| Config::new(file, cache_dir.join("virtualenvs"), false)) + .map(|cache_dir| Config::new(file, cache_dir.join("virtualenvs"), None)) } /// Maps to DEFAULT_CACHE_DIR in poetry fn get_default_cache_dir(env: &EnvVariables) -> Option { @@ -73,7 +76,10 @@ fn get_default_cache_dir(env: &EnvVariables) -> Option { /// Maps to CONFIG_DIR in poetry fn get_config_dir(env: &EnvVariables) -> Option { if let Some(config) = env.poetry_config_dir.clone() { - return Some(config); + // Ensure we have a valid directory setup in the env variables. + if config.is_dir() { + return Some(config); + } } Platformdirs::new(_APP_NAME.into(), true).user_config_path() } @@ -90,29 +96,53 @@ fn find_config_file(env: &EnvVariables) -> Option { #[derive(Debug, Clone, PartialEq, Eq)] struct ConfigToml { - virtualenvs_in_project: bool, + virtualenvs_in_project: Option, cache_dir: Option, virtualenvs_path: Option, } fn parse(file: &Path) -> Option { let contents = fs::read_to_string(file).ok()?; + parse_contents(&contents) +} +fn parse_contents(contents: &str) -> Option { let mut virtualenvs_path = None; let mut cache_dir = None; - let mut virtualenvs_in_project = false; - match toml::from_str::(&contents) { + let mut virtualenvs_in_project = None; + match toml::from_str::(contents) { Ok(value) => { if let Some(virtualenvs) = value.get("virtualenvs") { if let Some(path) = virtualenvs.get("path") { - virtualenvs_path = path.as_str().map(|s| s.trim()).map(PathBuf::from); + // Can contain invalid toml, hence make no assumptions + // virtualenvs.in-project = null + // https://github.com/python-poetry/poetry/blob/5bab98c9500f1050c6bb6adfb55580a23173f18d/docs/configuration.md#L56 + if path.is_str() { + virtualenvs_path = path.as_str().map(|s| s.trim()).map(PathBuf::from); + } } if let Some(in_project) = virtualenvs.get("in-project") { - virtualenvs_in_project = in_project.as_bool().unwrap_or_default(); + // Can contain invalid toml, hence make no assumptions + // virtualenvs.in-project = null + // https://github.com/python-poetry/poetry/blob/5bab98c9500f1050c6bb6adfb55580a23173f18d/docs/configuration.md#L56 + if in_project.is_bool() { + virtualenvs_in_project = in_project.as_bool(); + } } } if let Some(value) = value.get("cache-dir") { - cache_dir = value.as_str().map(|s| s.trim()).map(PathBuf::from); + // Can contain invalid toml, hence make no assumptions + // virtualenvs.in-project = null + // https://github.com/python-poetry/poetry/blob/5bab98c9500f1050c6bb6adfb55580a23173f18d/docs/configuration.md#L56 + if value.is_str() { + cache_dir = value.as_str().map(|s| s.trim()).map(PathBuf::from); + + if let Some(cache_dir) = &cache_dir { + if virtualenvs_path.is_none() { + virtualenvs_path = Some(cache_dir.join("virtualenvs")); + } + } + } } Some(ConfigToml { @@ -122,8 +152,129 @@ fn parse(file: &Path) -> Option { }) } Err(e) => { - eprintln!("Error parsing toml file: {:?}", e); + eprintln!("Error parsing poetry toml file: {:?}", e); None } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_virtualenvs_in_poetry_toml() { + let cfg = r#" +[virtualenvs] +in-project = false +create = false + +"#; + + assert_eq!( + parse_contents(&cfg.to_string()) + .unwrap() + .virtualenvs_in_project + .unwrap_or_default(), + false + ); + + let cfg = r#" +[virtualenvs] +in-project = true +create = false + +"#; + assert_eq!( + parse_contents(&cfg.to_string()) + .unwrap() + .virtualenvs_in_project + .unwrap_or_default(), + true + ); + + let cfg = r#" +[virtualenvs] +create = false + +"#; + assert_eq!( + parse_contents(&cfg.to_string()) + .unwrap() + .virtualenvs_in_project + .unwrap_or_default(), + false + ); + + let cfg = r#" +virtualenvs.in-project = true # comment +"#; + assert_eq!( + parse_contents(&cfg.to_string()) + .unwrap() + .virtualenvs_in_project + .unwrap_or_default(), + true + ); + + let cfg = r#" +"#; + assert_eq!( + parse_contents(&cfg.to_string()) + .unwrap() + .virtualenvs_in_project + .unwrap_or_default(), + false + ); + } + + #[test] + fn parse_cache_dir_in_poetry_toml() { + let cfg = r#" +cache-dir = "/path/to/cache/directory" + +"#; + assert_eq!( + parse_contents(&cfg.to_string()).unwrap().cache_dir, + Some(PathBuf::from("/path/to/cache/directory".to_string())) + ); + + let cfg = r#" +some-other-value = 1234 + +"#; + assert_eq!(parse_contents(&cfg.to_string()).unwrap().cache_dir, None); + } + + #[test] + fn parse_virtualenvs_path_in_poetry_toml() { + let cfg = r#" +virtualenvs.path = "/path/to/virtualenvs" + +"#; + assert_eq!( + parse_contents(&cfg.to_string()).unwrap().virtualenvs_path, + Some(PathBuf::from("/path/to/virtualenvs".to_string())) + ); + + let cfg = r#" +some-other-value = 1234 + +"#; + assert_eq!( + parse_contents(&cfg.to_string()).unwrap().virtualenvs_path, + None + ); + } + + #[test] + fn use_cache_dir_to_build_virtualenvs_path() { + let cfg = r#" +cache-dir = "/path/to/cache/directory" +"#; + assert_eq!( + parse_contents(&cfg.to_string()).unwrap().virtualenvs_path, + Some(PathBuf::from("/path/to/cache/directory/virtualenvs")) + ); + } +} diff --git a/crates/pet-poetry/src/env_variables.rs b/crates/pet-poetry/src/env_variables.rs index 67f011f6..c76c572f 100644 --- a/crates/pet-poetry/src/env_variables.rs +++ b/crates/pet-poetry/src/env_variables.rs @@ -9,11 +9,19 @@ use std::path::PathBuf; // Lets be explicit, this way we never miss a value (in Windows or Unix). pub struct EnvVariables { pub home: Option, - pub appdata: Option, + /// Only used in tests, None in production. + pub root: Option, + /// Maps to env var `APPDATA` + pub app_data: Option, + /// Maps to env var `POETRY_HOME` pub poetry_home: Option, + /// Maps to env var `POETRY_CONFIG_DIR` pub poetry_config_dir: Option, + /// Maps to env var `POETRY_CACHE_DIR` pub poetry_cache_dir: Option, + /// Maps to env var `POETRY_VIRTUALENVS_IN_PROJECT` pub poetry_virtualenvs_in_project: Option, + /// Maps to env var `PATH` pub path: Option, } @@ -36,7 +44,8 @@ impl EnvVariables { EnvVariables { home, path: env.get_env_var("PATH".to_string()), - appdata: env.get_env_var("APPDATA".to_string()).map(PathBuf::from), + root: env.get_root(), + app_data: env.get_env_var("APPDATA".to_string()).map(PathBuf::from), poetry_cache_dir: env .get_env_var("POETRY_CACHE_DIR".to_string()) .map(PathBuf::from), diff --git a/crates/pet-poetry/src/environment_locations.rs b/crates/pet-poetry/src/environment_locations.rs index d890350f..4bcd3be0 100644 --- a/crates/pet-poetry/src/environment_locations.rs +++ b/crates/pet-poetry/src/environment_locations.rs @@ -25,8 +25,25 @@ lazy_static! { pub fn list_environments( env: &EnvVariables, - project_dirs: &Vec, + project_dirs: &[PathBuf], ) -> Option> { + if project_dirs.is_empty() { + return None; + } + + let project_dirs = project_dirs + .iter() + .map(|project_dir| (project_dir, PyProjectToml::find(project_dir))) + .filter_map(|(project_dir, pyproject_toml)| { + pyproject_toml.map(|pyproject_toml| (project_dir, pyproject_toml)) + }) + .collect::>(); + + // We're only interested in directories that have a pyproject.toml + if project_dirs.is_empty() { + return None; + } + let mut envs = vec![]; let global_config = Config::find_global(env); @@ -35,34 +52,31 @@ pub fn list_environments( global_envs = list_all_environments_from_config(&config).unwrap_or_default(); } - // We're only interested in directories that have a pyproject.toml - for project_dir in project_dirs { - if let Some(pyproject_toml) = PyProjectToml::find(project_dir) { - let virtualenv_prefix = generate_env_name(&pyproject_toml.name, project_dir); - trace!( - "Found pyproject.toml ({}): {:?} in {:?}", - virtualenv_prefix, - pyproject_toml.name, - project_dir - ); - - for virtual_env in [ - list_all_environments_from_project_config(&global_config, project_dir, env) - .unwrap_or_default(), - global_envs.clone(), - ] - .concat() - { - // Check if this virtual env belongs to this project - let name = virtual_env - .file_name() - .unwrap_or_default() - .to_str() - .unwrap_or_default(); - if name.starts_with(&virtualenv_prefix) { - if let Some(env) = create_poetry_env(&virtual_env, project_dir.clone(), None) { - envs.push(env); - } + for (project_dir, pyproject_toml) in project_dirs { + let virtualenv_prefix = generate_env_name(&pyproject_toml.name, project_dir); + trace!( + "Found pyproject.toml ({}): {:?} in {:?}", + virtualenv_prefix, + pyproject_toml.name, + project_dir + ); + + for virtual_env in [ + list_all_environments_from_project_config(&global_config, project_dir, env) + .unwrap_or_default(), + global_envs.clone(), + ] + .concat() + { + // Check if this virtual env belongs to this project + let name = virtual_env + .file_name() + .unwrap_or_default() + .to_str() + .unwrap_or_default(); + if name.starts_with(&virtualenv_prefix) { + if let Some(env) = create_poetry_env(&virtual_env, project_dir.clone(), None) { + envs.push(env); } } } @@ -76,27 +90,19 @@ fn list_all_environments_from_project_config( path: &Path, env: &EnvVariables, ) -> Option> { - let config = Config::find_local(path, env); + let local = Config::find_local(path, env); let mut envs = vec![]; - if let Some(config) = &config { - if let Some(project_envs) = list_all_environments_from_config(config) { + if let Some(local) = &local { + if let Some(project_envs) = list_all_environments_from_config(local) { envs.extend(project_envs); } } // Check if we're allowed to use .venv as a poetry env // This can be configured in global, project or env variable. - if config - .clone() - .map(|c| c.virtualenvs_in_project) - .unwrap_or_default() - || global - .clone() - .map(|config| config.virtualenvs_in_project) - .unwrap_or(false) - || env.poetry_virtualenvs_in_project.unwrap_or_default() - { + // Order of preference is Global, EnvVariable & Project (project wins) + if should_use_local_venv_as_poetry_env(global, &local, env) { // If virtualenvs are in the project, then look for .venv let venv = path.join(".venv"); if venv.is_dir() { @@ -106,6 +112,30 @@ fn list_all_environments_from_project_config( Some(envs) } +fn should_use_local_venv_as_poetry_env( + global: &Option, + local: &Option, + env: &EnvVariables, +) -> bool { + // Give preference to setting in local config file. + if let Some(poetry_virtualenvs_in_project) = + local.clone().and_then(|c| c.virtualenvs_in_project) + { + return poetry_virtualenvs_in_project; + } + + // Given preference to env variable. + if let Some(poetry_virtualenvs_in_project) = env.poetry_virtualenvs_in_project { + return poetry_virtualenvs_in_project; + } + + // Check global config setting. + global + .clone() + .and_then(|config| config.virtualenvs_in_project) + .unwrap_or_default() +} + fn list_all_environments_from_config(cfg: &Config) -> Option> { Some( fs::read_dir(&cfg.virtualenvs_path) diff --git a/crates/pet-poetry/src/lib.rs b/crates/pet-poetry/src/lib.rs index 4ea5a28c..ad101a1c 100644 --- a/crates/pet-poetry/src/lib.rs +++ b/crates/pet-poetry/src/lib.rs @@ -4,7 +4,6 @@ use env_variables::EnvVariables; use environment_locations::list_environments; use log::{error, warn}; -use manager::PoetryManager; use pet_core::{ os_environment::Environment, python_environment::{PythonEnvironment, PythonEnvironmentCategory}, @@ -12,6 +11,7 @@ use pet_core::{ Configuration, Locator, LocatorResult, }; use pet_python_utils::env::PythonEnv; +use pet_virtualenv::is_virtualenv; use std::{ path::PathBuf, sync::{ @@ -20,12 +20,12 @@ use std::{ }, }; -mod config; -mod env_variables; +pub mod config; +pub mod env_variables; mod environment; -mod environment_locations; +pub mod environment_locations; mod environment_locations_spawn; -mod manager; +pub mod manager; mod pyproject_toml; pub struct Poetry { @@ -33,19 +33,17 @@ pub struct Poetry { pub env_vars: EnvVariables, pub poetry_executable: Arc>>, searched: AtomicBool, - environments: Arc>>, - manager: Arc>>, + search_result: Arc>>, } impl Poetry { pub fn new(environment: &dyn Environment) -> Self { Poetry { searched: AtomicBool::new(false), + search_result: Arc::new(Mutex::new(None)), project_dirs: Arc::new(Mutex::new(vec![])), env_vars: EnvVariables::from(environment), poetry_executable: Arc::new(Mutex::new(None)), - environments: Arc::new(Mutex::new(vec![])), - manager: Arc::new(Mutex::new(None)), } } pub fn from(environment: &dyn Environment) -> impl Locator { @@ -95,54 +93,42 @@ impl Poetry { Some(()) } fn find_with_cache(&self) -> Option { - if let Ok(environments) = self.environments.lock() { - if !environments.is_empty() { - if let Ok(manager) = self.manager.lock() { - if let Some(manager) = manager.as_ref() { - return Some(LocatorResult { - managers: vec![manager.to_manager()], - environments: environments.clone(), - }); - } - } - } - if self.searched.load(Ordering::Relaxed) { - return None; - } + if self.searched.load(Ordering::Relaxed) { + return self.search_result.lock().unwrap().clone(); } // First find the manager let manager = manager::PoetryManager::find( self.poetry_executable.lock().unwrap().clone(), &self.env_vars, ); - let mut managers = vec![]; + let mut result = LocatorResult { + managers: vec![], + environments: vec![], + }; if let Some(manager) = manager { - let mut mgr = self.manager.lock().unwrap(); - mgr.replace(manager.clone()); - drop(mgr); - managers.push(manager.to_manager()); + result.managers.push(manager.to_manager()); + } + if let Ok(values) = self.project_dirs.lock() { + let project_dirs = values.clone(); + drop(values); + let envs = list_environments(&self.env_vars, &project_dirs.clone()).unwrap_or_default(); + result.environments.extend(envs.clone()); } - let project_dirs = self.project_dirs.lock().unwrap().clone(); - if let Some(result) = list_environments(&self.env_vars, &project_dirs) { - match self.environments.lock() { - Ok(mut environments) => { - environments.clear(); - environments.extend(result); - self.searched.store(true, Ordering::Relaxed); - Some(LocatorResult { - managers: managers.clone(), - environments: environments.clone(), - }) - } - Err(err) => { - error!("Failed to cache to Poetry environments: {:?}", err); + match self.search_result.lock().as_mut() { + Ok(search_result) => { + if result.managers.is_empty() && result.environments.is_empty() { + search_result.take(); None + } else { + search_result.replace(result.clone()); + Some(result) } } - } else { - self.searched.store(true, Ordering::Relaxed); - None + Err(err) => { + error!("Failed to cache to Poetry environments: {:?}", err); + None + } } } } @@ -168,6 +154,9 @@ impl Locator for Poetry { } fn from(&self, env: &PythonEnv) -> Option { + if !is_virtualenv(env) { + return None; + } if let Some(result) = self.find_with_cache() { for found_env in result.environments { if let Some(symlinks) = &found_env.symlinks { @@ -182,15 +171,10 @@ impl Locator for Poetry { fn find(&self, reporter: &dyn Reporter) { if let Some(result) = self.find_with_cache() { - if let Ok(manager) = self.manager.lock() { - if let Some(manager) = manager.as_ref() { - reporter.report_manager(&manager.to_manager()); - } + for manager in result.managers { + reporter.report_manager(&manager.clone()); } for found_env in result.environments { - if let Some(manager) = &found_env.manager { - reporter.report_manager(manager); - } reporter.report_environment(&found_env); } } diff --git a/crates/pet-poetry/src/manager.rs b/crates/pet-poetry/src/manager.rs index e5e1224d..db422504 100644 --- a/crates/pet-poetry/src/manager.rs +++ b/crates/pet-poetry/src/manager.rs @@ -24,6 +24,7 @@ impl PoetryManager { if let Some(home) = &env_variables.home { let mut search_paths = vec![ home.join(".poetry").join("bin").join("poetry"), + // Found after installing on Mac using pipx home.join(".local") .join("pipx") .join("venvs") @@ -38,7 +39,7 @@ impl PoetryManager { search_paths.push(poetry_home.join("bin").join("poetry")); } if std::env::consts::OS == "windows" { - if let Some(app_data) = env_variables.appdata.clone() { + if let Some(app_data) = env_variables.app_data.clone() { search_paths.push( // https://python-poetry.org/docs/#installing-with-the-official-installer app_data diff --git a/crates/pet-poetry/src/pyproject_toml.rs b/crates/pet-poetry/src/pyproject_toml.rs index b917afff..cef844d0 100644 --- a/crates/pet-poetry/src/pyproject_toml.rs +++ b/crates/pet-poetry/src/pyproject_toml.rs @@ -13,7 +13,7 @@ pub struct PyProjectToml { } impl PyProjectToml { - pub fn new(name: String, file: PathBuf) -> Self { + fn new(name: String, file: PathBuf) -> Self { trace!("Poetry project: {:?} with name {:?}", file, name); PyProjectToml { name } } @@ -24,8 +24,11 @@ impl PyProjectToml { fn parse(file: &Path) -> Option { let contents = fs::read_to_string(file).ok()?; + parse_contents(&contents, file) +} - match toml::from_str::(&contents) { +fn parse_contents(contents: &str, file: &Path) -> Option { + match toml::from_str::(contents) { Ok(value) => { let mut name = None; if let Some(tool) = value.get("tool") { @@ -46,12 +49,12 @@ fn parse(file: &Path) -> Option { #[cfg(test)] mod tests { - use serde_json::Value; + use super::*; + use std::path::Path; #[test] - fn extract_name_from_pypoetry_toml() { - let cfg: Value = toml::from_str( - r#" + fn extract_name_from_pyproject_toml() { + let cfg = r#" [tool.poetry] name = "poetry-demo" version = "0.1.0" @@ -66,11 +69,11 @@ python = "^3.12" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" -"#, - ) - .unwrap(); +"#; assert_eq!( - cfg["tool"]["poetry"]["name"].as_str().unwrap(), + parse_contents(&cfg.to_string(), Path::new("pyproject.toml")) + .unwrap() + .name, "poetry-demo" ); } diff --git a/crates/pet-poetry/tests/common.rs b/crates/pet-poetry/tests/common.rs new file mode 100644 index 00000000..be297f48 --- /dev/null +++ b/crates/pet-poetry/tests/common.rs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use pet_core::os_environment::Environment; +use pet_poetry::env_variables::EnvVariables; +use std::{collections::HashMap, path::PathBuf}; + +#[allow(dead_code)] +pub fn resolve_test_path(paths: &[&str]) -> PathBuf { + let mut root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests"); + + paths.iter().for_each(|p| root.push(p)); + + root +} + +#[allow(dead_code)] +pub fn create_env_variables(home: PathBuf, root: PathBuf) -> EnvVariables { + EnvVariables { + home: Some(home), + root: Some(root), + path: None, + app_data: None, + poetry_cache_dir: None, + poetry_config_dir: None, + poetry_home: None, + poetry_virtualenvs_in_project: None, + } +} + +#[allow(dead_code)] +pub struct TestEnvironment { + vars: HashMap, + home: Option, + root: Option, +} +#[allow(dead_code)] +pub fn create_test_environment( + vars: HashMap, + home: Option, + root: Option, +) -> TestEnvironment { + impl Environment for TestEnvironment { + fn get_env_var(&self, key: String) -> Option { + self.vars.get(&key).cloned() + } + fn get_root(&self) -> Option { + self.root.clone() + } + fn get_user_home(&self) -> Option { + self.home.clone() + } + fn get_know_global_search_locations(&self) -> Vec { + vec![] + } + } + TestEnvironment { vars, home, root } +} diff --git a/crates/pet-poetry/tests/config_test.rs b/crates/pet-poetry/tests/config_test.rs new file mode 100644 index 00000000..377ef076 --- /dev/null +++ b/crates/pet-poetry/tests/config_test.rs @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +mod common; + +#[cfg(unix)] +#[test] +fn global_config_with_defaults() { + use common::create_env_variables; + use common::resolve_test_path; + use pet_poetry::config::Config; + use pet_python_utils::platform_dirs::Platformdirs; + + let root = resolve_test_path(&["unix", "global_config_defaults", "root_empty"]); + let home = resolve_test_path(&["unix", "global_config_defaults", "user_home"]); + let mut env = create_env_variables(home, root); + env.poetry_config_dir = Some(resolve_test_path(&[ + "unix", + "global_config_defaults", + "user_home", + "config_dir", + ])); + + let config = Config::find_global(&env); + + assert!(config.clone().is_some()); + assert_eq!(config.clone().unwrap().file, None); + assert!(config.clone().unwrap().virtualenvs_in_project.is_none()); + assert_eq!( + config.clone().unwrap().virtualenvs_path, + Platformdirs::new("pypoetry".into(), false) + .user_cache_path() + .unwrap() + .join("virtualenvs") + ); +} + +#[cfg(unix)] +#[test] +fn global_config_with_specific_values() { + use std::path::PathBuf; + + use common::create_env_variables; + use common::resolve_test_path; + use pet_poetry::config::Config; + + let root = resolve_test_path(&["unix", "global_config_with_values", "root_empty"]); + let home = resolve_test_path(&["unix", "global_config_with_values", "user_home"]); + let mut env = create_env_variables(home, root); + env.poetry_config_dir = Some(resolve_test_path(&[ + "unix", + "global_config_with_values", + "user_home", + "config_dir", + ])); + + let config = Config::find_global(&env); + + assert!(config.clone().is_some()); + assert_eq!( + config.clone().unwrap().file, + Some(resolve_test_path(&[ + "unix", + "global_config_with_values", + "user_home", + "config_dir", + "config.toml" + ])) + ); + assert_eq!( + config + .clone() + .unwrap() + .virtualenvs_in_project + .unwrap_or_default(), + true + ); + assert_eq!( + config.clone().unwrap().virtualenvs_path, + PathBuf::from("some/path/virtualenvs".to_string()) + ); +} + +#[cfg(unix)] +#[test] +fn local_config_with_specific_values() { + use std::path::PathBuf; + + use common::create_env_variables; + use common::resolve_test_path; + use pet_poetry::config::Config; + + let root = resolve_test_path(&["unix", "local_config_with_values", "root_empty"]); + let home = resolve_test_path(&["unix", "local_config_with_values", "user_home"]); + let mut env = create_env_variables(home, root); + env.poetry_config_dir = Some(resolve_test_path(&[ + "unix", + "local_config_with_values", + "user_home", + "config_dir", + ])); + + let project_dir = resolve_test_path(&["unix", "local_config_with_values", "project_dir"]); + let config = Config::find_local(&project_dir, &env); + + assert!(config.clone().is_some()); + assert_eq!( + config.clone().unwrap().file, + Some(resolve_test_path(&[ + "unix", + "local_config_with_values", + "project_dir", + "poetry.toml" + ])) + ); + assert_eq!( + config + .clone() + .unwrap() + .virtualenvs_in_project + .unwrap_or_default(), + false + ); + assert_eq!( + config.clone().unwrap().virtualenvs_path, + PathBuf::from("/directory/virtualenvs".to_string()) + ); +} diff --git a/crates/pet-poetry/tests/unix/global_config_with_values/user_home/config_dir/config.toml b/crates/pet-poetry/tests/unix/global_config_with_values/user_home/config_dir/config.toml new file mode 100644 index 00000000..e4d898b2 --- /dev/null +++ b/crates/pet-poetry/tests/unix/global_config_with_values/user_home/config_dir/config.toml @@ -0,0 +1,7 @@ +cache-dir = "/path/to/cache/directory" + +[virtualenvs] +in-project = true # Some comments +create = false # Some comments +path = "some/path/virtualenvs" # Some comments + \ No newline at end of file diff --git a/crates/pet-poetry/tests/unix/local_config_with_values/project_dir/poetry.toml b/crates/pet-poetry/tests/unix/local_config_with_values/project_dir/poetry.toml new file mode 100644 index 00000000..bad238cf --- /dev/null +++ b/crates/pet-poetry/tests/unix/local_config_with_values/project_dir/poetry.toml @@ -0,0 +1,5 @@ +cache-dir = "/directory" + +[virtualenvs] +in-project = false # Some comments +create = false # Some comments diff --git a/crates/pet-poetry/tests/unix/local_config_with_values/user_home/config_dir/config.toml b/crates/pet-poetry/tests/unix/local_config_with_values/user_home/config_dir/config.toml new file mode 100644 index 00000000..e4d898b2 --- /dev/null +++ b/crates/pet-poetry/tests/unix/local_config_with_values/user_home/config_dir/config.toml @@ -0,0 +1,7 @@ +cache-dir = "/path/to/cache/directory" + +[virtualenvs] +in-project = true # Some comments +create = false # Some comments +path = "some/path/virtualenvs" # Some comments + \ No newline at end of file diff --git a/crates/pet-pyenv/src/environments.rs b/crates/pet-pyenv/src/environments.rs index 14d788ff..de6f774e 100644 --- a/crates/pet-pyenv/src/environments.rs +++ b/crates/pet-pyenv/src/environments.rs @@ -12,7 +12,12 @@ use pet_core::{ use pet_python_utils::executable::{find_executable, find_executables}; use pet_python_utils::version; use regex::Regex; -use std::{fs, path::Path, sync::Arc}; +use std::{ + fs, + path::Path, + sync::{Arc, Mutex}, + thread, +}; lazy_static! { // Stable Versions = like 3.10.10 @@ -34,37 +39,53 @@ pub fn list_pyenv_environments( versions_dir: &Path, conda_locator: &Arc, ) -> Option { - let mut envs: Vec = vec![]; - let mut managers: Vec = vec![]; + let envs = Arc::new(Mutex::new(vec![])); + let managers = Arc::new(Mutex::new(vec![])); - for path in fs::read_dir(versions_dir) - .ok()? - .filter_map(Result::ok) - .map(|e| e.path()) - { - if let Some(executable) = find_executable(&path) { - if is_conda_env(&path) { - if let Some(result) = conda_locator.find_in(&path) { - result.environments.iter().for_each(|e| { - envs.push(e.clone()); - }); - result.managers.iter().for_each(|e| { - managers.push(e.clone()); + thread::scope(|s| { + if let Ok(reader) = fs::read_dir(versions_dir) { + for path in reader.filter_map(Result::ok).map(|e| e.path()) { + if let Some(executable) = find_executable(&path) { + let path = path.clone(); + let executable = executable.clone(); + let conda_locator = conda_locator.clone(); + let manager = manager.clone(); + let envs = envs.clone(); + let managers = managers.clone(); + s.spawn(move || { + if is_conda_env(&path) { + if let Some(result) = conda_locator.find_in(&path) { + result.environments.iter().for_each(|e| { + envs.lock().unwrap().push(e.clone()); + }); + result.managers.iter().for_each(|e| { + managers.lock().unwrap().push(e.clone()); + }); + } + } else if let Some(env) = + get_pure_python_environment(&executable, &path, &manager) + { + envs.lock().unwrap().push(env); + } else if let Some(env) = + get_virtual_env_environment(&executable, &path, &manager) + { + envs.lock().unwrap().push(env); + } else if let Some(env) = + get_generic_environment(&executable, &path, &manager) + { + envs.lock().unwrap().push(env); + } }); } - } else if let Some(env) = get_pure_python_environment(&executable, &path, manager) { - envs.push(env); - } else if let Some(env) = get_virtual_env_environment(&executable, &path, manager) { - envs.push(env); - } else if let Some(env) = get_generic_environment(&executable, &path, manager) { - envs.push(env); } } - } + }); + let managers = managers.lock().unwrap(); + let envs = envs.lock().unwrap(); Some(LocatorResult { - managers, - environments: envs, + managers: managers.clone(), + environments: envs.clone(), }) } @@ -74,9 +95,8 @@ pub fn get_pure_python_environment( manager: &Option, ) -> Option { let file_name = path.file_name()?.to_string_lossy().to_string(); - let version = get_version(&file_name)?; // If we can get the version from the header files, thats more accurate. - let version = version::from_header_files(path).unwrap_or(version.clone()); + let version = version::from_header_files(path).or_else(|| get_version(&file_name)); let arch = if file_name.ends_with("-win32") { Some(Architecture::X86) @@ -87,7 +107,7 @@ pub fn get_pure_python_environment( Some( PythonEnvironmentBuilder::new(PythonEnvironmentCategory::Pyenv) .executable(Some(executable.to_path_buf())) - .version(Some(version)) + .version(version) .prefix(Some(path.to_path_buf())) .manager(manager.clone()) .arch(arch) @@ -118,11 +138,10 @@ pub fn get_generic_environment( path: &Path, manager: &Option, ) -> Option { - let version = version::from_header_files(path); Some( PythonEnvironmentBuilder::new(PythonEnvironmentCategory::PyenvOther) .executable(Some(executable.to_path_buf())) - .version(version) + .version(version::from_header_files(path)) .prefix(Some(path.to_path_buf())) .manager(manager.clone()) .symlinks(Some(find_executables(path))) diff --git a/crates/pet-pyenv/src/lib.rs b/crates/pet-pyenv/src/lib.rs index 52507dfe..5c595e6b 100644 --- a/crates/pet-pyenv/src/lib.rs +++ b/crates/pet-pyenv/src/lib.rs @@ -1,7 +1,10 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use std::sync::Arc; +use std::{ + path::PathBuf, + sync::{Arc, Mutex}, +}; use env_variables::EnvVariables; use environments::{ @@ -9,7 +12,7 @@ use environments::{ list_pyenv_environments, }; use manager::PyEnvInfo; -use pet_conda::CondaLocator; +use pet_conda::{utils::is_conda_env, CondaLocator}; use pet_core::{ manager::{EnvManager, EnvManagerType}, os_environment::Environment, @@ -27,6 +30,8 @@ mod manager; pub struct PyEnv { pub env_vars: EnvVariables, pub conda_locator: Arc, + manager: Arc>>, + versions_dir: Arc>>, } impl PyEnv { @@ -37,6 +42,8 @@ impl PyEnv { PyEnv { env_vars: EnvVariables::from(environment), conda_locator, + manager: Arc::new(Mutex::new(None)), + versions_dir: Arc::new(Mutex::new(None)), } } } @@ -51,29 +58,63 @@ impl Locator for PyEnv { } fn from(&self, env: &PythonEnv) -> Option { + if let Some(prefix) = &env.prefix { + if is_conda_env(prefix) { + return None; + } + } + // Possible this is a root conda env (hence parent directory is conda install dir). + if is_conda_env(env.executable.parent()?) { + return None; + } + // Possible this is a conda env (hence parent directory is Scripts/bin dir). + if is_conda_env(env.executable.parent()?.parent()?) { + return None; + } + // Env path must exists, // If exe is Scripts/python.exe or bin/python.exe // Then env path is parent of Scripts or bin // & in pyenv case thats a directory inside `versions` folder. - let pyenv_info = PyEnvInfo::from(&self.env_vars); - let mut manager: Option = None; - if let Some(ref exe) = pyenv_info.exe { - let version = pyenv_info.version.clone(); - manager = Some(EnvManager::new(exe.clone(), EnvManagerType::Pyenv, version)); + let mut binding_manager = self.manager.lock(); + let managers = binding_manager.as_mut().unwrap(); + let mut binding_versions = self.versions_dir.lock(); + let versions = binding_versions.as_mut().unwrap(); + if managers.is_none() || versions.is_none() { + let pyenv_info = PyEnvInfo::from(&self.env_vars); + let mut manager: Option = None; + if let Some(ref exe) = pyenv_info.exe { + let version = pyenv_info.version.clone(); + manager = Some(EnvManager::new(exe.clone(), EnvManagerType::Pyenv, version)); + } + if let Some(version_path) = &pyenv_info.versions { + versions.replace(version_path.clone()); + } else { + versions.take(); + } + if let Some(manager) = manager { + managers.replace(manager.clone()); + } else { + managers.take(); + } } - let versions = &pyenv_info.versions?; - if env.executable.starts_with(versions) { - let env_path = env.prefix.clone()?; - if let Some(env) = get_pure_python_environment(&env.executable, &env_path, &manager) { - return Some(env); - } else if let Some(env) = - get_virtual_env_environment(&env.executable, &env_path, &manager) - { - return Some(env); - } else if let Some(env) = get_generic_environment(&env.executable, &env_path, &manager) - { - return Some(env); + if let Some(versions) = versions.clone() { + let manager = managers.clone(); + if env.executable.starts_with(versions) { + let env_path = env.prefix.clone()?; + if let Some(env) = get_pure_python_environment(&env.executable, &env_path, &manager) + { + return Some(env); + } else if let Some(env) = + get_virtual_env_environment(&env.executable, &env_path, &manager) + { + return Some(env); + } else if let Some(env) = + get_generic_environment(&env.executable, &env_path, &manager) + { + return Some(env); + } } } None diff --git a/crates/pet/tests/ci_test.rs b/crates/pet/tests/ci_test.rs index 69f72e59..e151a9a1 100644 --- a/crates/pet/tests/ci_test.rs +++ b/crates/pet/tests/ci_test.rs @@ -217,6 +217,7 @@ fn get_conda_exe() -> &'static str { #[derive(Deserialize, Clone)] struct InterpreterInfo { sys_prefix: String, + #[allow(dead_code)] executable: String, sys_version: String, is64_bit: bool, From f617251d81bfb07190a6c610b0069554dd16184d Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 24 Jun 2024 13:10:34 +1000 Subject: [PATCH 07/11] Fixes --- crates/pet-conda/tests/conda_rc_test.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/pet-conda/tests/conda_rc_test.rs b/crates/pet-conda/tests/conda_rc_test.rs index 1c060734..cf1f33d1 100644 --- a/crates/pet-conda/tests/conda_rc_test.rs +++ b/crates/pet-conda/tests/conda_rc_test.rs @@ -84,7 +84,10 @@ fn finds_conda_rc_from_root() { use pet_conda::conda_rc::Condarc; use std::path::PathBuf; - print!("root: {:?}", resolve_test_path(&["unix", "conda_rc_root", "root"])); + print!( + "root: {:?}", + resolve_test_path(&["unix", "conda_rc_root", "root"]) + ); let root = resolve_test_path(&["unix", "conda_rc_root", "root"]); let home = resolve_test_path(&["unix", "conda_rc_root", "user_home"]); let env = create_env_variables(home, root); From d1bf4885ecd3472ba9081732b8f2ecc4f76770f9 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 24 Jun 2024 13:19:47 +1000 Subject: [PATCH 08/11] fix et --- crates/pet-core/src/python_environment.rs | 1 - crates/pet-pyenv/src/environments.rs | 24 ++--------------------- crates/pet-pyenv/src/lib.rs | 12 +++--------- crates/pet-pyenv/tests/pyenv_test.rs | 4 ++-- crates/pet-reporter/src/environment.rs | 1 - 5 files changed, 7 insertions(+), 35 deletions(-) diff --git a/crates/pet-core/src/python_environment.rs b/crates/pet-core/src/python_environment.rs index ae05c7f1..bcd1de16 100644 --- a/crates/pet-core/src/python_environment.rs +++ b/crates/pet-core/src/python_environment.rs @@ -18,7 +18,6 @@ pub enum PythonEnvironmentCategory { Pyenv, // Relates to Python installations in pyenv that are from Python org. GlobalPaths, // Python found in global locations like PATH, /usr/bin etc. PyenvVirtualEnv, // Pyenv virtualenvs. - PyenvOther, // Such as pyston, stackless, nogil, etc. Pipenv, Poetry, System, diff --git a/crates/pet-pyenv/src/environments.rs b/crates/pet-pyenv/src/environments.rs index de6f774e..8c9ae7ca 100644 --- a/crates/pet-pyenv/src/environments.rs +++ b/crates/pet-pyenv/src/environments.rs @@ -62,16 +62,12 @@ pub fn list_pyenv_environments( managers.lock().unwrap().push(e.clone()); }); } - } else if let Some(env) = - get_pure_python_environment(&executable, &path, &manager) - { - envs.lock().unwrap().push(env); } else if let Some(env) = get_virtual_env_environment(&executable, &path, &manager) { envs.lock().unwrap().push(env); } else if let Some(env) = - get_generic_environment(&executable, &path, &manager) + get_generic_python_environment(&executable, &path, &manager) { envs.lock().unwrap().push(env); } @@ -89,7 +85,7 @@ pub fn list_pyenv_environments( }) } -pub fn get_pure_python_environment( +pub fn get_generic_python_environment( executable: &Path, path: &Path, manager: &Option, @@ -133,22 +129,6 @@ pub fn get_virtual_env_environment( ) } -pub fn get_generic_environment( - executable: &Path, - path: &Path, - manager: &Option, -) -> Option { - Some( - PythonEnvironmentBuilder::new(PythonEnvironmentCategory::PyenvOther) - .executable(Some(executable.to_path_buf())) - .version(version::from_header_files(path)) - .prefix(Some(path.to_path_buf())) - .manager(manager.clone()) - .symlinks(Some(find_executables(path))) - .build(), - ) -} - fn get_version(folder_name: &str) -> Option { // Stable Versions = like 3.10.10 match PURE_PYTHON_VERSION.captures(folder_name) { diff --git a/crates/pet-pyenv/src/lib.rs b/crates/pet-pyenv/src/lib.rs index 5c595e6b..f9edc3eb 100644 --- a/crates/pet-pyenv/src/lib.rs +++ b/crates/pet-pyenv/src/lib.rs @@ -8,8 +8,7 @@ use std::{ use env_variables::EnvVariables; use environments::{ - get_generic_environment, get_pure_python_environment, get_virtual_env_environment, - list_pyenv_environments, + get_generic_python_environment, get_virtual_env_environment, list_pyenv_environments, }; use manager::PyEnvInfo; use pet_conda::{utils::is_conda_env, CondaLocator}; @@ -53,7 +52,6 @@ impl Locator for PyEnv { vec![ PythonEnvironmentCategory::Pyenv, PythonEnvironmentCategory::PyenvVirtualEnv, - PythonEnvironmentCategory::PyenvOther, ] } @@ -103,15 +101,11 @@ impl Locator for PyEnv { let manager = managers.clone(); if env.executable.starts_with(versions) { let env_path = env.prefix.clone()?; - if let Some(env) = get_pure_python_environment(&env.executable, &env_path, &manager) + if let Some(env) = get_virtual_env_environment(&env.executable, &env_path, &manager) { return Some(env); } else if let Some(env) = - get_virtual_env_environment(&env.executable, &env_path, &manager) - { - return Some(env); - } else if let Some(env) = - get_generic_environment(&env.executable, &env_path, &manager) + get_generic_python_environment(&env.executable, &env_path, &manager) { return Some(env); } diff --git a/crates/pet-pyenv/tests/pyenv_test.rs b/crates/pet-pyenv/tests/pyenv_test.rs index 4ce3d79f..548ad32b 100644 --- a/crates/pet-pyenv/tests/pyenv_test.rs +++ b/crates/pet-pyenv/tests/pyenv_test.rs @@ -266,7 +266,7 @@ fn find_pyenv_envs() { home.to_str().unwrap(), ".pyenv/versions/nogil-3.9.10-1/bin/python", ])), - category: PythonEnvironmentCategory::PyenvOther, + category: PythonEnvironmentCategory::Pyenv, version: Some("3.9.10".to_string()), prefix: Some(resolve_test_path(&[ home.to_str().unwrap(), @@ -288,7 +288,7 @@ fn find_pyenv_envs() { home.to_str().unwrap(), ".pyenv/versions/pypy3.9-7.3.15/bin/python", ])), - category: PythonEnvironmentCategory::PyenvOther, + category: PythonEnvironmentCategory::Pyenv, version: Some("3.9.18".to_string()), prefix: Some(resolve_test_path(&[ home.to_str().unwrap(), diff --git a/crates/pet-reporter/src/environment.rs b/crates/pet-reporter/src/environment.rs index 5611d858..86a74f74 100644 --- a/crates/pet-reporter/src/environment.rs +++ b/crates/pet-reporter/src/environment.rs @@ -24,7 +24,6 @@ fn python_category_to_string(category: &PythonEnvironmentCategory) -> &'static s PythonEnvironmentCategory::LinuxGlobal => "linux-global", PythonEnvironmentCategory::Pyenv => "pyenv", PythonEnvironmentCategory::PyenvVirtualEnv => "pyenv-virtualenv", - PythonEnvironmentCategory::PyenvOther => "pyenv-other", PythonEnvironmentCategory::WindowsStore => "windows-store", PythonEnvironmentCategory::WindowsRegistry => "windows-registry", PythonEnvironmentCategory::Pipenv => "pipenv", From 1e9f835207529708e6ad6d51cfbdac3de66529a1 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 24 Jun 2024 13:25:00 +1000 Subject: [PATCH 09/11] Fix testes --- crates/pet-pyenv/tests/pyenv_test.rs | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/crates/pet-pyenv/tests/pyenv_test.rs b/crates/pet-pyenv/tests/pyenv_test.rs index 548ad32b..d86d9646 100644 --- a/crates/pet-pyenv/tests/pyenv_test.rs +++ b/crates/pet-pyenv/tests/pyenv_test.rs @@ -385,7 +385,7 @@ fn resolve_pyenv_environment() { create_test_environment(HashMap::new(), Some(home.clone()), vec![homebrew_bin], None); let conda = Arc::new(Conda::from(&environment)); - let locator = PyEnv::from(&environment, conda); + let locator = PyEnv::from(&environment, conda.clone()); // let mut result = locator.find().unwrap(); let expected_manager = EnvManager { @@ -462,7 +462,7 @@ fn resolve_pyenv_environment() { assert_eq!(result.unwrap(), expected_virtual_env); - // Should resolve conda envs in pyenv + // Should not resolve conda envs in pyenv let result = locator.from(&PythonEnv::new( resolve_test_path(&[ home.to_str().unwrap(), @@ -476,4 +476,20 @@ fn resolve_pyenv_environment() { )); assert_eq!(result.is_some(), true); + + // Should not resolve conda envs using Conda Locator + let result = conda.from(&PythonEnv::new( + resolve_test_path(&[ + home.to_str().unwrap(), + ".pyenv/versions/anaconda-4.0.0/bin/python", + ]), + Some(resolve_test_path(&[ + home.to_str().unwrap(), + ".pyenv/versions/anaconda-4.0.0", + ])), + None, + )); + + assert_eq!(result.is_some(), true); + assert_eq!(result.unwrap().category, PythonEnvironmentCategory::Conda); } From a9c306aa4bc98dd53d9b05705a3156bf3567d020 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 24 Jun 2024 13:26:25 +1000 Subject: [PATCH 10/11] oops --- crates/pet-pyenv/tests/pyenv_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/pet-pyenv/tests/pyenv_test.rs b/crates/pet-pyenv/tests/pyenv_test.rs index d86d9646..795bbe46 100644 --- a/crates/pet-pyenv/tests/pyenv_test.rs +++ b/crates/pet-pyenv/tests/pyenv_test.rs @@ -475,7 +475,7 @@ fn resolve_pyenv_environment() { None, )); - assert_eq!(result.is_some(), true); + assert_eq!(result.is_none(), true); // Should not resolve conda envs using Conda Locator let result = conda.from(&PythonEnv::new( From 1081f4b68fded3bbe5710d7f93a560945e40770b Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 24 Jun 2024 13:43:22 +1000 Subject: [PATCH 11/11] oops --- crates/pet-conda/tests/conda_rc_test.rs | 4 ---- crates/pet-homebrew/src/lib.rs | 5 +++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/pet-conda/tests/conda_rc_test.rs b/crates/pet-conda/tests/conda_rc_test.rs index cf1f33d1..1590f8c5 100644 --- a/crates/pet-conda/tests/conda_rc_test.rs +++ b/crates/pet-conda/tests/conda_rc_test.rs @@ -84,10 +84,6 @@ fn finds_conda_rc_from_root() { use pet_conda::conda_rc::Condarc; use std::path::PathBuf; - print!( - "root: {:?}", - resolve_test_path(&["unix", "conda_rc_root", "root"]) - ); let root = resolve_test_path(&["unix", "conda_rc_root", "root"]); let home = resolve_test_path(&["unix", "conda_rc_root", "user_home"]); let env = create_env_variables(home, root); diff --git a/crates/pet-homebrew/src/lib.rs b/crates/pet-homebrew/src/lib.rs index 015a15bb..56b37010 100644 --- a/crates/pet-homebrew/src/lib.rs +++ b/crates/pet-homebrew/src/lib.rs @@ -32,6 +32,11 @@ impl Homebrew { } } +/// Deafult prefix paths for Homebrew +/// Below are from the docs `man brew` Display Homebrew’s install path. Default: +/// - macOS ARM: /opt/homebrew +/// - macOS Intel: /usr/local +/// - Linux: /home/linuxbrew/.linuxbrew fn from(env: &PythonEnv) -> Option { // Assume we create a virtual env from a homebrew python install, // Then the exe in the virtual env bin will be a symlink to the homebrew python install.