From 23df5468686ab411b1cabca2889ee52c993ed101 Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Thu, 13 Mar 2025 15:34:19 -0700 Subject: [PATCH 1/2] Fix env duplication on merged-usr systems On modern (merged-usr) Linux systems, `/bin`, `/sbin` and `/usr/sbin` are symlinks to `/usr/bin`. There is no point in reporting the same Python installation four times, so canonicalize search paths before searching. Before: Breakdown for finding Environments: ----------------------------------- GlobalVirtualEnvs : 17.412201ms Locators : 225.284494ms Path : 433.162905ms Workspaces : 2.161556ms Environments (41): ------------------ GlobalPaths : 14 LinuxGlobal : 16 VirtualEnvWrapper : 11 After: Breakdown for finding Environments: ----------------------------------- GlobalVirtualEnvs : 16.595382ms Locators : 223.759511ms Path : 313.276036ms Workspaces : 1.418024ms Environments (21): ------------------ GlobalPaths : 2 LinuxGlobal : 8 VirtualEnvWrapper : 11 --- crates/pet-env-var-path/src/lib.rs | 8 ++++++-- crates/pet-linux-global-python/src/lib.rs | 17 ++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/crates/pet-env-var-path/src/lib.rs b/crates/pet-env-var-path/src/lib.rs index 95740859..bd9aef9a 100644 --- a/crates/pet-env-var-path/src/lib.rs +++ b/crates/pet-env-var-path/src/lib.rs @@ -2,6 +2,8 @@ // Licensed under the MIT License. use pet_core::os_environment::Environment; +use std::collections::HashSet; +use std::fs; use std::path::PathBuf; pub fn get_search_paths_from_env_variables(environment: &dyn Environment) -> Vec { @@ -16,10 +18,12 @@ pub fn get_search_paths_from_env_variables(environment: &dyn Environment) -> Vec environment .get_know_global_search_locations() - .clone() + .into_iter() + .map(|p| fs::canonicalize(&p).unwrap_or(p)) + .collect::>() .into_iter() .filter(|p| !p.starts_with(apps_path.clone())) - .collect::>() + .collect() } else { Vec::new() } diff --git a/crates/pet-linux-global-python/src/lib.rs b/crates/pet-linux-global-python/src/lib.rs index 2fc76e94..c0d5b7af 100644 --- a/crates/pet-linux-global-python/src/lib.rs +++ b/crates/pet-linux-global-python/src/lib.rs @@ -2,7 +2,7 @@ // Licensed under the MIT License. use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, fs, path::{Path, PathBuf}, sync::{Arc, Mutex}, @@ -38,10 +38,17 @@ impl LinuxGlobalPython { return; } // Look through the /bin, /usr/bin, /usr/local/bin directories + let bin_dirs: HashSet<_> = [ + Path::new("/bin"), + Path::new("/usr/bin"), + Path::new("/usr/local/bin"), + ] + .map(|p| fs::canonicalize(p).unwrap_or(p.to_path_buf())) + .into(); thread::scope(|s| { - for bin in ["/bin", "/usr/bin", "/usr/local/bin"] { + for bin in bin_dirs { s.spawn(move || { - find_and_report_global_pythons_in(bin, reporter, &self.reported_executables); + find_and_report_global_pythons_in(&bin, reporter, &self.reported_executables); }); } }); @@ -103,11 +110,11 @@ impl Locator for LinuxGlobalPython { } fn find_and_report_global_pythons_in( - bin: &str, + bin: &Path, reporter: Option<&dyn Reporter>, reported_executables: &Arc>>, ) { - let python_executables = find_executables(Path::new(bin)); + let python_executables = find_executables(bin); for exe in python_executables.clone().iter() { if reported_executables.lock().unwrap().contains_key(exe) { From 2b593f1b8134eaa99ed581a142918d4ebb09ef63 Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Tue, 25 Mar 2025 22:23:31 -0700 Subject: [PATCH 2/2] Adjust test --- crates/pet/tests/ci_jupyter_container.rs | 37 +++--------------------- 1 file changed, 4 insertions(+), 33 deletions(-) diff --git a/crates/pet/tests/ci_jupyter_container.rs b/crates/pet/tests/ci_jupyter_container.rs index 4ecdbb18..c59aef24 100644 --- a/crates/pet/tests/ci_jupyter_container.rs +++ b/crates/pet/tests/ci_jupyter_container.rs @@ -70,30 +70,16 @@ fn verify_python_in_jupyter_contaner() { }), ..Default::default() }; - let codespace_python = PythonEnvironment { - kind: Some(PythonEnvironmentKind::GlobalPaths), - executable: Some(PathBuf::from("/home/codespace/.python/current/bin/python")), - prefix: Some(PathBuf::from("/usr/local/python/3.10.13")), - version: Some("3.10.13.final.0".to_string()), - arch: Some(Architecture::X64), - symlinks: Some(vec![ - PathBuf::from("/home/codespace/.python/current/bin/python"), - PathBuf::from("/home/codespace/.python/current/bin/python3"), - PathBuf::from("/home/codespace/.python/current/bin/python3.10"), - ]), - manager: None, - ..Default::default() - }; let current_python = PythonEnvironment { kind: Some(PythonEnvironmentKind::GlobalPaths), - executable: Some(PathBuf::from("/usr/local/python/current/bin/python")), + executable: Some(PathBuf::from("/usr/local/python/3.10.13/bin/python")), prefix: Some(PathBuf::from("/usr/local/python/3.10.13")), version: Some("3.10.13.final.0".to_string()), arch: Some(Architecture::X64), symlinks: Some(vec![ - PathBuf::from("/usr/local/python/current/bin/python"), - PathBuf::from("/usr/local/python/current/bin/python3"), - PathBuf::from("/usr/local/python/current/bin/python3.10"), + PathBuf::from("/usr/local/python/3.10.13/bin/python"), + PathBuf::from("/usr/local/python/3.10.13/bin/python3"), + PathBuf::from("/usr/local/python/3.10.13/bin/python3.10"), ]), manager: None, ..Default::default() @@ -111,26 +97,11 @@ fn verify_python_in_jupyter_contaner() { manager: None, ..Default::default() }; - let bin_python = PythonEnvironment { - kind: Some(PythonEnvironmentKind::LinuxGlobal), - executable: Some(PathBuf::from("/bin/python3")), - prefix: Some(PathBuf::from("/usr")), - version: Some("3.8.10.final.0".to_string()), - arch: Some(Architecture::X64), - symlinks: Some(vec![ - PathBuf::from("/bin/python3"), - PathBuf::from("/bin/python3.8"), - ]), - manager: None, - ..Default::default() - }; for env in [ conda, - codespace_python, current_python, usr_bin_python, - bin_python, ] .iter() {