Win tests, validate by resolving envs from all symlinks

Author: DonJayamanne

.github/workflows/pr-check.yml

@@ -28,9 +28,6 @@ jobs:
           - os: windows-latest
             target: x86_64-pc-windows-msvc
             run_cli: "yes"
-          - os: windows-latest
-            target: aarch64-pc-windows-msvc
-            run_cli: "no"
           - os: ubuntu-latest
             target: x86_64-unknown-linux-musl
             run_cli: "yes"
@@ -197,26 +194,35 @@ jobs:
     # Some of these tests are very specific and need to be run in isolation.
     # E.g. we need to ensure we have a poetry project setup correctly (without .venv created using `pip -m venv .venv`).
     # We can try to use the previous `tests` job, but that gets very complicated.
-    name: Other Tests
+    name: Env Tests
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         include:
-          - feature: ci-poetry-global
+          - feature: ci-poetry-global # Poetry tests with envs stored in standard location
             os: ubuntu-latest
             target: x86_64-unknown-linux-musl
-          - feature: ci-poetry-project
+          - feature: ci-poetry-project # Poetry tests, with poetry envs in project
             os: ubuntu-latest
             target: x86_64-unknown-linux-musl
-          - feature: ci-poetry-custom
+          - feature: ci-poetry-custom # Poetry tests with envs stored in a custom location
             os: ubuntu-latest
             target: x86_64-unknown-linux-musl
+          - feature: ci-poetry-global # Poetry tests with envs stored in standard location
+            os: windows-latest
+            target: x86_64-pc-windows-msvc
+          - feature: ci-poetry-project # Poetry tests, with poetry envs in project
+            os: windows-latest
+            target: x86_64-pc-windows-msvc
+          - feature: ci-poetry-custom # Poetry tests with envs stored in a custom location
+            os: windows-latest
+            target: x86_64-pc-windows-msvc
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      # Setup Poetry
+      # region Setup Poetry
       - name: Set Python 3.x to PATH
         if: startsWith( matrix.feature, 'ci-poetry')
         uses: actions/setup-python@v5
@@ -225,16 +231,33 @@ jobs:
 
       - name: Set Python 3.12 to PATH
         if: startsWith( matrix.feature, 'ci-poetry')
+        id: setupPython312
         uses: actions/setup-python@v5
         with:
           python-version: "3.12"
 
       - name: Set Python 3.11 to PATH
         if: startsWith( matrix.feature, 'ci-poetry')
+        id: setupPython311
         uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 
+      - name: Python 3.12 Path
+        if: startsWith( matrix.feature, 'ci-poetry') && startsWith( matrix.os, 'windows')
+        run: echo "PYTHON_3_12_PATH=${{ steps.setupPython312.outputs.python-path }}" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Python 3.12 Path
+        if: startsWith( matrix.feature, 'ci-poetry') && startsWith( matrix.os, 'windows')
+        run: echo $PYTHON_3_12_PATH
+        shell: bash
+
+      - name: Python 3.11 Path
+        if: startsWith( matrix.feature, 'ci-poetry') && startsWith( matrix.os, 'windows')
+        run: echo "PYTHON_3_11_PATH=${{ steps.setupPython311.outputs.python-path }}" >> $GITHUB_ENV
+        shell: bash
+
       - name: Install Poetry (envs globally)
         if: startsWith( matrix.feature, 'ci-poetry-global')
         uses: snok/install-poetry@93ada01c735cc8a383ce0ce2ae205a21c415379b
@@ -260,6 +283,11 @@ jobs:
           virtualenvs-path: ~/my-custom-path
           installer-parallel: true
 
+      - name: Petry exe
+        if: startsWith( matrix.feature, 'ci-poetry')
+        run: which poetry
+        shell: bash
+
       - name: Petry config
         if: startsWith( matrix.feature, 'ci-poetry')
         run: poetry config --list
@@ -272,13 +300,23 @@ jobs:
         shell: bash
 
       - name: Petry virtual env setup 3.12
-        if: startsWith( matrix.feature, 'ci-poetry')
-        run: poetry env use python3.12
+        if: startsWith( matrix.feature, 'ci-poetry') && startsWith( matrix.os, 'ubuntu')
+        run: poetry env use 3.12
+        shell: bash
+
+      - name: Petry virtual env setup 3.12
+        if: startsWith( matrix.feature, 'ci-poetry') && startsWith( matrix.os, 'windows')
+        run: poetry env use $PYTHON_3_12_PATH
         shell: bash
 
       - name: Petry virtual env setup 3.11
-        if: startsWith( matrix.feature, 'ci-poetry')
-        run: poetry env use python3.11
+        if: startsWith( matrix.feature, 'ci-poetry') && startsWith( matrix.os, 'ubuntu')
+        run: poetry env use 3.11
+        shell: bash
+
+      - name: Petry virtual env setup 3.11
+        if: startsWith( matrix.feature, 'ci-poetry') && startsWith( matrix.os, 'windows')
+        run: poetry env use $PYTHON_3_11_PATH
         shell: bash
 
       - name: Petry list envs
@@ -296,6 +334,8 @@ jobs:
       #   run: set
       #   shell: bash
 
+      # endregion
+
       # Rust
       - name: Rust Tool Chain setup
         uses: dtolnay/rust-toolchain@stable
@@ -366,7 +406,7 @@ jobs:
         shell: bash
 
       - name: Find Environments
-        run: cargo run --release --target ${{ matrix.target }} -- find -v
+        run: cargo run --release --target ${{ matrix.target }}
         shell: bash
 
       - name: Run Tests

Cargo.lock

@@ -7,6 +7,7 @@ use std::{
     sync::{Arc, Mutex},
 };
 
+use log::trace;
 use pet_fs::path::norm_case;
 
 pub trait Environment {
@@ -78,7 +79,7 @@ impl Environment for EnvironmentApi {
             let mut paths =
                 env::split_paths(&self.get_env_var("PATH".to_string()).unwrap_or_default())
                     .collect::<Vec<PathBuf>>();
-
+            trace!("Env PATH: {:?}", paths);
             vec![
                 PathBuf::from("/bin"),
                 PathBuf::from("/etc"),

crates/pet-core/src/os_environment.rs

@@ -48,8 +48,10 @@ impl Locator for MacCmdLineTools {
 
         if !env
             .executable
-            .to_string_lossy()
-            .starts_with("/Library/Developer/CommandLineTools/usr/bin/python")
+            .starts_with("/Library/Developer/CommandLineTools/usr/bin")
+            && !env.executable.starts_with(
+                "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions",
+            )
         {
             return None;
         }
@@ -70,6 +72,33 @@ impl Locator for MacCmdLineTools {
             symlinks.push(symlink);
         }
 
+        // Possible we got the file /Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/bin/python3.9
+        // We know that /Library/Developer/CommandLineTools/usr/bin/python3 is a symlink to the above.
+        if env
+            .executable
+            .starts_with("/Library/Developer/CommandLineTools/usr/bin")
+        {
+            let exe = PathBuf::from("/Library/Developer/CommandLineTools/usr/bin/python3");
+            if let Some(symlink) = resolve_symlink(&exe) {
+                if symlinks.contains(&symlink) {
+                    symlinks.push(symlink);
+
+                    // Rest of the files in this directory are also symlinks to the same exe.
+                    for exe in find_executables(PathBuf::from(
+                        "/Library/Developer/CommandLineTools/usr/bin",
+                    )) {
+                        if !symlinks.contains(&exe) {
+                            if let Some(symlink) = resolve_symlink(&exe) {
+                                if symlinks.contains(&symlink) {
+                                    symlinks.push(exe);
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
         // We know /usr/bin/python3 can end up pointing to this same Python exe as well
         // Hence look for those symlinks as well.
         // Unfortunately /usr/bin/python3 is not a real symlink
@@ -106,6 +135,17 @@ impl Locator for MacCmdLineTools {
         symlinks.sort();
         symlinks.dedup();
 
+        // Find other exes that are symlinks to the same exe in /Library/Developer/CommandLineTools/usr/bin
+        for exe in find_executables("/Library/Developer/CommandLineTools/usr/bin") {
+            if !symlinks.contains(&exe) {
+                if let Some(symlink) = resolve_symlink(&exe) {
+                    if symlinks.contains(&symlink) {
+                        symlinks.push(exe);
+                    }
+                }
+            }
+        }
+
         if prefix.is_none() {
             // We would have identified the symlinks by now.
             // Look for the one with the path `/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/bin/python3.9`

crates/pet-mac-commandlinetools/src/lib.rs

@@ -7,8 +7,11 @@ use pet_core::{
     Locator,
 };
 use pet_fs::path::resolve_symlink;
-use pet_python_utils::env::{PythonEnv, ResolvedPythonEnv};
 use pet_python_utils::version;
+use pet_python_utils::{
+    env::{PythonEnv, ResolvedPythonEnv},
+    executable::find_executables,
+};
 use pet_virtualenv::is_virtualenv;
 use std::path::PathBuf;
 
@@ -65,7 +68,34 @@ impl Locator for MacXCode {
         // /Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/bin/python3.9
         // Verify this and add that to the list of symlinks as well.
         if let Some(symlink) = resolve_symlink(&env.executable) {
-            symlinks.push(symlink);
+            symlinks.push(symlink.clone());
+
+            // All exes in the bin directory of the symlink are also symlinks (thats generally of the form /Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/bin/python3.9)
+            for exe in find_executables(symlink.parent().unwrap()) {
+                symlinks.push(exe);
+            }
+        }
+
+        // Possible the env.executable is "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/bin/python3.9"
+        // The symlink to the above exe is in /Applications/Xcode.app/Contents/Developer/usr/bin/python3
+        // Lets try to find that, because /usr/bin/python3 could also exist and when we run python, the sys.execuctable points to the file /Applications/Xcode.app/Contents/Developer/usr/bin/python3
+        // The name of the `Xcode.app` folder can be different on other machines, e.g. on CI it is `Xcode_15.0.1.app`
+        let xcode_folder_name = exe_str.split('/').nth(2).unwrap_or_default();
+
+        let bin = PathBuf::from(format!(
+            "/Applications/{}/Contents/Developer/usr/bin",
+            xcode_folder_name
+        ));
+        let exe = bin.join("python3");
+        if let Some(symlink) = resolve_symlink(&exe) {
+            if symlinks.contains(&symlink) {
+                symlinks.push(exe.clone());
+
+                // All exes in this directory are symlinks
+                for exe in find_executables(bin) {
+                    symlinks.push(exe);
+                }
+            }
         }
 
         // We know /usr/bin/python3 can end up pointing to this same Python exe as well

crates/pet-mac-xcode/src/lib.rs

@@ -15,7 +15,7 @@ use std::{
 
 use crate::{
     config::Config, env_variables::EnvVariables, environment::create_poetry_env,
-    pyproject_toml::PyProjectToml,
+    manager::PoetryManager, pyproject_toml::PyProjectToml,
 };
 
 lazy_static! {
@@ -26,6 +26,7 @@ lazy_static! {
 pub fn list_environments(
     env: &EnvVariables,
     project_dirs: &[PathBuf],
+    manager: Option<PoetryManager>,
 ) -> Option<Vec<PythonEnvironment>> {
     if project_dirs.is_empty() {
         return None;
@@ -76,7 +77,9 @@ pub fn list_environments(
                 .unwrap_or_default();
             // Look for .venv as well, in case we create the virtual envs in the local project folder.
             if name.starts_with(&virtualenv_prefix) || name.starts_with(".venv") {
-                if let Some(env) = create_poetry_env(&virtual_env, project_dir.clone(), None) {
+                if let Some(env) =
+                    create_poetry_env(&virtual_env, project_dir.clone(), manager.clone())
+                {
                     envs.push(env);
                 }
             }

crates/pet-poetry/src/environment_locations.rs

@@ -105,13 +105,14 @@ impl Poetry {
             managers: vec![],
             environments: vec![],
         };
-        if let Some(manager) = manager {
+        if let Some(manager) = &manager {
             result.managers.push(manager.to_manager());
         }
         if let Ok(values) = self.project_dirs.lock() {
             let project_dirs = values.clone();
             drop(values);
-            let envs = list_environments(&self.env_vars, &project_dirs.clone()).unwrap_or_default();
+            let envs = list_environments(&self.env_vars, &project_dirs.clone(), manager)
+                .unwrap_or_default();
             result.environments.extend(envs.clone());
         }
 

crates/pet-poetry/src/lib.rs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+use log::trace;
 use pet_core::manager::{EnvManager, EnvManagerType};
 use std::{env, path::PathBuf};
 
@@ -35,8 +36,10 @@ impl PoetryManager {
             if let Some(poetry_home) = &env_variables.poetry_home {
                 if std::env::consts::OS == "windows" {
                     search_paths.push(poetry_home.join("bin").join("poetry.exe"));
+                    search_paths.push(poetry_home.join("venv").join("bin").join("poetry.exe"));
                 }
                 search_paths.push(poetry_home.join("bin").join("poetry"));
+                search_paths.push(poetry_home.join("venv").join("bin").join("poetry"));
             }
             if std::env::consts::OS == "windows" {
                 if let Some(app_data) = env_variables.app_data.clone() {
@@ -71,6 +74,10 @@ impl PoetryManager {
                         app_data.join("Python").join("scripts").join("poetry"), // https://python-poetry.org/docs/#installing-with-the-official-installer
                     );
                 }
+                search_paths.push(
+                    // Found after installing on Windows via github actions.
+                    home.join(".local").join("bin").join("poetry"),
+                );
             } else if std::env::consts::OS == "macos" {
                 search_paths.push(
                     // https://python-poetry.org/docs/#installing-with-the-official-installer
@@ -111,9 +118,16 @@ impl PoetryManager {
                     if executable.is_file() {
                         return Some(PoetryManager { executable });
                     }
+                    if std::env::consts::OS == "windows" {
+                        let executable = each.join("poetry.exe");
+                        if executable.is_file() {
+                            return Some(PoetryManager { executable });
+                        }
+                    }
                 }
             }
         }
+        trace!("Poetry exe not found");
         None
     }
     pub fn to_manager(&self) -> EnvManager {

crates/pet-poetry/src/manager.rs

@@ -97,6 +97,11 @@ impl ResolvedPythonEnv {
         match result {
             Ok(output) => {
                 let output = String::from_utf8(output.stdout).unwrap().trim().to_string();
+                trace!(
+                    "Python Execution for {:?} produced an output {:?}",
+                    executable,
+                    output
+                );
                 if let Some((_, output)) = output.split_once(PYTHON_INFO_JSON_SEPARATOR) {
                     if let Ok(info) = serde_json::from_str::<InterpreterInfo>(output) {
                         Some(Self {

crates/pet-python-utils/src/env.rs

@@ -88,8 +88,10 @@ impl Locator for WindowsRegistry {
         if let Some(result) = self.find_with_cache() {
             // Find the same env here
             for found_env in result.environments {
-                if env.executable.to_str() == env.executable.to_str() {
-                    return Some(found_env);
+                if let Some(ref python_executable_path) = found_env.executable {
+                    if python_executable_path == &env.executable {
+                        return Some(found_env);
+                    }
                 }
             }
         }