fixup! Try switching to venv for isolation

Author: pradyunsg

src/pip/_internal/build_env.py

@@ -8,17 +8,25 @@
 import sys
 import venv
 import zipfile
-from sysconfig import get_paths
-from types import SimpleNamespace, TracebackType
-from typing import TYPE_CHECKING, Iterable, Iterator, List, Optional, Set, Tuple, Type
+from types import TracebackType
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    Type,
+)
 
 from pip._vendor.certifi import where
 from pip._vendor.packaging.requirements import Requirement
 from pip._vendor.packaging.version import Version
 
 from pip import __file__ as pip_location
 from pip._internal.cli.spinners import open_spinner
-from pip._internal.locations import get_platlib, get_prefixed_libs, get_purelib
 from pip._internal.metadata import get_environment
 from pip._internal.utils.subprocess import call_subprocess
 from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
@@ -29,17 +37,6 @@
 logger = logging.getLogger(__name__)
 
 
-class _Prefix:
-    def __init__(self, path: str) -> None:
-        self.path = path
-        self.setup = False
-        self.bin_dir = get_paths(
-            "nt" if os.name == "nt" else "posix_prefix",
-            vars={"base": path, "platbase": path},
-        )["scripts"]
-        self.lib_dirs = get_prefixed_libs(path)
-
-
 @contextlib.contextmanager
 def _create_standalone_pip() -> Iterator[str]:
     """Create a "standalone pip" zip file.
@@ -70,38 +67,54 @@ class BuildEnvironment:
     """Creates and manages an isolated environment to install build deps"""
 
     def __init__(self) -> None:
-        self._venv = venv.EnvBuilder(
-            system_site_packages=False,
-            clear=False,
-            symlinks=False,
-            upgrade=False,
-            with_pip=False,
-            prompt=None,
-            upgrade_deps=False,
+        self._temp_dir = TempDirectory(
+            kind=tempdir_kinds.BUILD_ENV, globally_managed=True
         )
-        self._temp_dir = None
-        self._env_executable_path = None
+        self._venv = venv.EnvBuilder()
+        context = self._venv.ensure_directories(self._temp_dir.path)
+        self._venv.create(self._temp_dir.path)
+
+        # Copy-pasted from venv/__init__.py
+        if sys.platform == "win32":
+            libpath = os.path.join(
+                self._temp_dir.path,
+                "Lib",
+                "site-packages",
+            )
+        else:
+            libpath = os.path.join(
+                self._temp_dir.path,
+                "lib",
+                f"python{sys.version_info.major}.{sys.version_info.minor}",
+                "site-packages",
+            )
+
+        self._lib_path = libpath
+        self._bin_path = context.bin_path
+        self._env_executable = context.env_exe
+        self._save_env: Dict[str, str] = {}
 
     def __enter__(self) -> None:
-        self._temp_dir = TempDirectory(
-            kind=tempdir_kinds.BUILD_ENV,
-            globally_managed=False,
-        )
+        self._save_env = {
+            name: os.environ.get(name, "") for name in ("PATH", "PYTHONPATH")
+        }
 
-        context = self._venv.ensure_directories(self._temp_dir)
-        self._env_executable_path = context.exe_name
+        old_path = self._save_env["PATH"]
+        new_path = os.pathsep.join(filter(None, [self._bin_path, old_path]))
 
-        self._venv.create(self._temp_dir)
+        os.environ.update({"PATH": new_path, "PYTHONPATH": self._lib_path})
 
     def __exit__(
         self,
         exc_type: Optional[Type[BaseException]],
         exc_val: Optional[BaseException],
         exc_tb: Optional[TracebackType],
     ) -> None:
-        self._temp_dir.cleanup()
-        self._temp_dir = None
-        self._env_executable_path = None
+        for varname, old_value in self._save_env.items():
+            if old_value is None:
+                os.environ.pop(varname, None)
+            else:
+                os.environ[varname] = old_value
 
     def check_requirements(
         self, reqs: Iterable[str]
@@ -113,7 +126,7 @@ def check_requirements(
         missing = set()
         conflicting = set()
         if reqs:
-            env = get_environment(self._lib_dirs)
+            env = get_environment([self._lib_path])
             for req_str in reqs:
                 req = Requirement(req_str)
                 dist = env.get_distribution(req.name)
@@ -136,11 +149,12 @@ def install_requirements(
         requirements: Iterable[str],
         message: str,
     ) -> None:
+        assert self._env_executable
         if not requirements:
             return
         with _create_standalone_pip() as pip_runnable:
             self._install_requirements(
-                self._env_executable_path,
+                self._env_executable,
                 pip_runnable,
                 finder,
                 requirements,
@@ -153,7 +167,6 @@ def _install_requirements(
         pip_runnable: str,
         finder: "PackageFinder",
         requirements: Iterable[str],
-        prefix: _Prefix,
         message: str,
     ) -> None:
         args: List[str] = [
@@ -162,8 +175,6 @@ def _install_requirements(
             "install",
             "--ignore-installed",
             "--no-user",
-            "--prefix",
-            prefix.path,
             "--no-warn-script-location",
         ]
         if logger.getEffectiveLevel() <= logging.DEBUG:
@@ -225,8 +236,8 @@ def check_requirements(
     def install_requirements(
         self,
         finder: "PackageFinder",
+        *,
         requirements: Iterable[str],
-        prefix_as_string: str,
         message: str,
     ) -> None:
         raise NotImplementedError()

tests/functional/test_build_env.py

@@ -79,29 +79,9 @@ def run_with_build_env(
 def test_build_env_allow_empty_requirements_install() -> None:
     finder = make_test_finder()
     build_env = BuildEnvironment()
-    for prefix in ("normal", "overlay"):
-        build_env.install_requirements(
-            finder, [], prefix, "Installing build dependencies"
-        )
-
-
-def test_build_env_allow_only_one_install(script: PipTestEnvironment) -> None:
-    create_basic_wheel_for_package(script, "foo", "1.0")
-    create_basic_wheel_for_package(script, "bar", "1.0")
-    finder = make_test_finder(find_links=[script.scratch_path])
-    build_env = BuildEnvironment()
-    for prefix in ("normal", "overlay"):
-        build_env.install_requirements(
-            finder, ["foo"], prefix, f"installing foo in {prefix}"
-        )
-        with pytest.raises(AssertionError):
-            build_env.install_requirements(
-                finder, ["bar"], prefix, f"installing bar in {prefix}"
-            )
-        with pytest.raises(AssertionError):
-            build_env.install_requirements(
-                finder, [], prefix, f"installing in {prefix}"
-            )
+    build_env.install_requirements(
+        finder, requirements=[], message="Installing build dependencies"
+    )
 
 
 def test_build_env_requirements_check(script: PipTestEnvironment) -> None:
@@ -130,54 +110,59 @@ def test_build_env_requirements_check(script: PipTestEnvironment) -> None:
     run_with_build_env(
         script,
         """
-        build_env.install_requirements(finder, ['foo', 'bar==3.0'], 'normal',
-                                       'installing foo in normal')
+        build_env.install_requirements(
+            finder, requirements=["foo", "bar==3.0"], message="installing foo"
+        )
 
-        r = build_env.check_requirements(['foo', 'bar', 'other'])
-        assert r == (set(), {'other'}), repr(r)
+        r = build_env.check_requirements(["foo", "bar", "other"])
+        assert r == (set(), {"other"}), repr(r)
 
-        r = build_env.check_requirements(['foo>1.0', 'bar==3.0'])
+        r = build_env.check_requirements(["foo>1.0", "bar==3.0"])
         assert r == (set(), set()), repr(r)
 
-        r = build_env.check_requirements(['foo>3.0', 'bar>=2.5'])
-        assert r == ({('foo==2.0', 'foo>3.0')}, set()), repr(r)
+        r = build_env.check_requirements(["foo>3.0", "bar>=2.5"])
+        assert r == ({("foo==2.0", "foo>3.0")}, set()), repr(r)
         """,
     )
 
     run_with_build_env(
         script,
         """
-        build_env.install_requirements(finder, ['foo', 'bar==3.0'], 'normal',
-                                       'installing foo in normal')
-        build_env.install_requirements(finder, ['bar==1.0'], 'overlay',
-                                       'installing foo in overlay')
+        build_env.install_requirements(
+            finder, requirements=["foo", "bar==3.0"], message="installing foo"
+        )
+        build_env.install_requirements(
+            finder, requirements=["1.0"], message="installing bar"
+        )
 
-        r = build_env.check_requirements(['foo', 'bar', 'other'])
-        assert r == (set(), {'other'}), repr(r)
+        r = build_env.check_requirements(["foo", "bar", "other"])
+        assert r == (set(), {"other"}), repr(r)
 
-        r = build_env.check_requirements(['foo>1.0', 'bar==3.0'])
-        assert r == ({('bar==1.0', 'bar==3.0')}, set()), repr(r)
+        r = build_env.check_requirements(["foo>1.0", "bar==3.0"])
+        assert r == ({("bar==1.0", "bar==3.0")}, set()), repr(r)
 
-        r = build_env.check_requirements(['foo>3.0', 'bar>=2.5'])
-        assert r == ({('bar==1.0', 'bar>=2.5'), ('foo==2.0', 'foo>3.0')}, \
-            set()), repr(r)
+        r = build_env.check_requirements(["foo>3.0", "bar>=2.5"])
+        assert r == ({("bar==1.0", "bar>=2.5"), ("foo==2.0", "foo>3.0")}, set()), \
+            repr(r)
         """,
     )
 
 
-def test_build_env_overlay_prefix_has_priority(script: PipTestEnvironment) -> None:
+def test_build_env_latest_install_has_priority(script: PipTestEnvironment) -> None:
     create_basic_wheel_for_package(script, "pkg", "2.0")
     create_basic_wheel_for_package(script, "pkg", "4.3")
     result = run_with_build_env(
         script,
         """
-        build_env.install_requirements(finder, ['pkg==2.0'], 'overlay',
-                                       'installing pkg==2.0 in overlay')
-        build_env.install_requirements(finder, ['pkg==4.3'], 'normal',
-                                       'installing pkg==4.3 in normal')
+        build_env.install_requirements(
+            finder, requirements=["pkg==4.3"], message="installing pkg==4.3"
+        )
+        build_env.install_requirements(
+            finder, requirements=["pkg==2.0"], message="installing pkg==2.0"
+        )
         """,
         """
-        print(__import__('pkg').__version__)
+        print(__import__("pkg").__version__)
         """,
     )
     assert result.stdout.strip() == "2.0", str(result)
@@ -208,7 +193,7 @@ def test_build_env_isolation(script: PipTestEnvironment) -> None:
     run_with_build_env(
         script,
         "",
-        r"""
+        R"""
         from distutils.sysconfig import get_python_lib
         import sys
 

tests/functional/test_pep517.py

@@ -36,7 +36,9 @@ def test_backend(tmpdir: Path, data: TestData) -> None:
     req.load_pyproject_toml()
     env = BuildEnvironment()
     finder = make_test_finder(find_links=[data.backends])
-    env.install_requirements(finder, ["dummy_backend"], "normal", "Installing")
+    env.install_requirements(
+        finder, requirements=["dummy_backend"], message="Installing"
+    )
     conflicting, missing = env.check_requirements(["dummy_backend"])
     assert not conflicting and not missing
     assert hasattr(req.pep517_backend, "build_wheel")
@@ -83,7 +85,9 @@ def test_backend_path_and_dep(tmpdir: Path, data: TestData) -> None:
     req.load_pyproject_toml()
     env = BuildEnvironment()
     finder = make_test_finder(find_links=[data.backends])
-    env.install_requirements(finder, ["dummy_backend"], "normal", "Installing")
+    env.install_requirements(
+        finder, requirements=["dummy_backend"], message="Installing"
+    )
 
     assert hasattr(req.pep517_backend, "build_wheel")
     with env: