Merge branch 'main' into python_option

Author: pfmoore

.github/workflows/ci.yml

@@ -221,6 +221,35 @@ jobs:
         env:
           TEMP: "R:\\Temp"
 
+  tests-zipapp:
+    name: tests / zipapp
+    runs-on: ubuntu-latest
+
+    needs: [pre-commit, packaging, determine-changes]
+    if: >-
+      needs.determine-changes.outputs.tests == 'true' ||
+      github.event_name != 'pull_request'
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.10"
+
+      - name: Install Ubuntu dependencies
+        run: sudo apt-get install bzr
+
+      - run: pip install nox 'virtualenv<20' 'setuptools != 60.6.0'
+
+      # Main check
+      - name: Run integration tests
+        run: >-
+          nox -s test-3.10 --
+          -m integration
+          --verbose --numprocesses auto --showlocals
+          --durations=5
+          --use-zipapp
+
   # TODO: Remove this when we add Python 3.11 to CI.
   tests-importlib-metadata:
     name: tests for importlib.metadata backend

docs/html/cli/pip_download.rst

@@ -43,7 +43,9 @@ match the constraint of the current interpreter (but not your target one), it
 is recommended to specify all of these options if you are specifying one of
 them. Generic dependencies (e.g. universal wheels, or dependencies with no
 platform, abi, or implementation constraints) will still match an over-
-constrained download requirement.
+constrained download requirement. If some of your dependencies are not
+available as binaries, you can build them manually for your target platform
+and let pip download know where to find them using ``--find-links``.
 
 
 

docs/html/user_guide.rst

@@ -2,6 +2,15 @@
 User Guide
 ==========
 
+.. Hello there!
+
+   If you're thinking of adding content to this page... please take a moment
+   to consider if this content can live on its own, within a topic guide or a
+   reference page.
+
+   There is active effort being put toward *reducing* the amount of content on
+   this specific page (https://github.com/pypa/pip/issues/9475) and moving it
+   into more focused single-page documents that cover that specific topic.
 
 Running pip
 ===========

news/10716.feature.rst

@@ -0,0 +1 @@
+Use ``shell=True`` for opening the editor with ``pip config edit``.

news/11250.feature.rst

@@ -0,0 +1 @@
+Add an option to run the test suite with pip built as a zipapp.

news/11314.bugfix.rst

@@ -0,0 +1 @@
+Avoid  ``AttributeError`` when removing the setuptools-provided ``_distutils_hack`` and it is missing its implementation.

news/11319.bugfix.rst

@@ -0,0 +1 @@
+Fix import error when reinstalling pip in user site.

src/pip/__pip-runner__.py

@@ -45,7 +45,6 @@ def check_python_version() -> None:
         )
 
 
-# TODO https://github.com/pypa/pip/issues/11294
 sys.meta_path.insert(0, PipImportRedirectingFinder())
 
 assert __name__ == "__main__", "Cannot run __pip-runner__.py as a non-main module"

src/pip/_internal/commands/configuration.py

@@ -228,9 +228,15 @@ def open_in_editor(self, options: Values, args: List[str]) -> None:
         fname = self.configuration.get_file_to_edit()
         if fname is None:
             raise PipError("Could not determine appropriate file.")
+        elif '"' in fname:
+            # This shouldn't happen, unless we see a username like that.
+            # If that happens, we'd appreciate a pull request fixing this.
+            raise PipError(
+                f'Can not open an editor for a file name containing "\n{fname}'
+            )
 
         try:
-            subprocess.check_call([editor, fname])
+            subprocess.check_call(f'{editor} "{fname}"', shell=True)
         except FileNotFoundError as e:
             if not e.filename:
                 e.filename = editor

src/pip/_internal/locations/__init__.py

@@ -60,6 +60,12 @@ def _should_use_sysconfig() -> bool:
 
 _USE_SYSCONFIG = _should_use_sysconfig()
 
+if not _USE_SYSCONFIG:
+    # Import distutils lazily to avoid deprecation warnings,
+    # but import it soon enough that it is in memory and available during
+    # a pip reinstall.
+    from . import _distutils
+
 # Be noisy about incompatibilities if this platforms "should" be using
 # sysconfig, but is explicitly opting out and using distutils instead.
 if _USE_SYSCONFIG_DEFAULT and not _USE_SYSCONFIG:
@@ -241,8 +247,6 @@ def get_scheme(
     if _USE_SYSCONFIG:
         return new
 
-    from . import _distutils
-
     old = _distutils.get_scheme(
         dist_name,
         user=user,
@@ -407,8 +411,6 @@ def get_bin_prefix() -> str:
     if _USE_SYSCONFIG:
         return new
 
-    from . import _distutils
-
     old = _distutils.get_bin_prefix()
     if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="bin_prefix"):
         _log_context()
@@ -442,8 +444,6 @@ def get_purelib() -> str:
     if _USE_SYSCONFIG:
         return new
 
-    from . import _distutils
-
     old = _distutils.get_purelib()
     if _looks_like_deb_system_dist_packages(old):
         return old
@@ -488,8 +488,6 @@ def get_prefixed_libs(prefix: str) -> List[str]:
     if _USE_SYSCONFIG:
         return _deduplicated(new_pure, new_plat)
 
-    from . import _distutils
-
     old_pure, old_plat = _distutils.get_prefixed_libs(prefix)
     old_lib_paths = _deduplicated(old_pure, old_plat)
 

src/pip/_internal/locations/_distutils.py

@@ -11,7 +11,7 @@
 # rationale for why this is done within pip.
 try:
     __import__("_distutils_hack").remove_shim()
-except ImportError:
+except (ImportError, AttributeError):
     pass
 
 import logging

tests/conftest.py

@@ -19,6 +19,7 @@
     Union,
 )
 from unittest.mock import patch
+from zipfile import ZipFile
 
 import pytest
 
@@ -33,6 +34,7 @@
 from installer.destinations import SchemeDictionaryDestination
 from installer.sources import WheelFile
 
+from pip import __file__ as pip_location
 from pip._internal.cli.main import main as pip_entry_point
 from pip._internal.locations import _USE_SYSCONFIG
 from pip._internal.utils.temp_dir import global_tempdir_manager
@@ -85,6 +87,12 @@ def pytest_addoption(parser: Parser) -> None:
         default=None,
         help="use given proxy in session network tests",
     )
+    parser.addoption(
+        "--use-zipapp",
+        action="store_true",
+        default=False,
+        help="use a zipapp when running pip in tests",
+    )
 
 
 def pytest_collection_modifyitems(config: Config, items: List[pytest.Function]) -> None:
@@ -513,10 +521,13 @@ def __call__(
 
 @pytest.fixture(scope="session")
 def script_factory(
-    virtualenv_factory: Callable[[Path], VirtualEnvironment], deprecated_python: bool
+    virtualenv_factory: Callable[[Path], VirtualEnvironment],
+    deprecated_python: bool,
+    zipapp: Optional[str],
 ) -> ScriptFactory:
     def factory(
-        tmpdir: Path, virtualenv: Optional[VirtualEnvironment] = None
+        tmpdir: Path,
+        virtualenv: Optional[VirtualEnvironment] = None,
     ) -> PipTestEnvironment:
         if virtualenv is None:
             virtualenv = virtualenv_factory(tmpdir.joinpath("venv"))
@@ -535,13 +546,64 @@ def factory(
             assert_no_temp=True,
             # Deprecated python versions produce an extra deprecation warning
             pip_expect_warning=deprecated_python,
+            # Tell the Test Environment if we want to run pip via a zipapp
+            zipapp=zipapp,
         )
 
     return factory
 
 
+ZIPAPP_MAIN = """\
+#!/usr/bin/env python
+
+import os
+import runpy
+import sys
+
+lib = os.path.join(os.path.dirname(__file__), "lib")
+sys.path.insert(0, lib)
+
+runpy.run_module("pip", run_name="__main__")
+"""
+
+
+def make_zipapp_from_pip(zipapp_name: Path) -> None:
+    pip_dir = Path(pip_location).parent
+    with zipapp_name.open("wb") as zipapp_file:
+        zipapp_file.write(b"#!/usr/bin/env python\n")
+        with ZipFile(zipapp_file, "w") as zipapp:
+            for pip_file in pip_dir.rglob("*"):
+                if pip_file.suffix == ".pyc":
+                    continue
+                if pip_file.name == "__pycache__":
+                    continue
+                rel_name = pip_file.relative_to(pip_dir.parent)
+                zipapp.write(pip_file, arcname=f"lib/{rel_name}")
+            zipapp.writestr("__main__.py", ZIPAPP_MAIN)
+
+
+@pytest.fixture(scope="session")
+def zipapp(
+    request: pytest.FixtureRequest, tmpdir_factory: pytest.TempPathFactory
+) -> Optional[str]:
+    """
+    If the user requested for pip to be run from a zipapp, build that zipapp
+    and return its location. If the user didn't request a zipapp, return None.
+
+    This fixture is session scoped, so the zipapp will only be created once.
+    """
+    if not request.config.getoption("--use-zipapp"):
+        return None
+
+    temp_location = tmpdir_factory.mktemp("zipapp")
+    pyz_file = temp_location / "pip.pyz"
+    make_zipapp_from_pip(pyz_file)
+    return str(pyz_file)
+
+
 @pytest.fixture
 def script(
+    request: pytest.FixtureRequest,
     tmpdir: Path,
     virtualenv: VirtualEnvironment,
     script_factory: ScriptFactory,

tests/functional/test_cli.py

@@ -16,6 +16,9 @@
     ],
 )
 def test_entrypoints_work(entrypoint: str, script: PipTestEnvironment) -> None:
+    if script.zipapp:
+        pytest.skip("Zipapp does not include entrypoints")
+
     fake_pkg = script.temp_path / "fake_pkg"
     fake_pkg.mkdir()
     fake_pkg.joinpath("setup.py").write_text(

tests/functional/test_completion.py

@@ -107,7 +107,12 @@ def test_completion_for_supported_shells(
     Test getting completion for bash shell
     """
     result = script_with_launchers.pip("completion", "--" + shell, use_module=False)
-    assert completion in result.stdout, str(result.stdout)
+    actual = str(result.stdout)
+    if script_with_launchers.zipapp:
+        # The zipapp reports its name as "pip.pyz", but the expected
+        # output assumes "pip"
+        actual = actual.replace("pip.pyz", "pip")
+    assert completion in actual, actual
 
 
 @pytest.fixture(scope="session")

tests/functional/test_pip_runner_script.py

@@ -12,7 +12,9 @@ def test_runner_work_in_environments_with_no_pip(
 
     # Ensure there's no pip installed in the environment
     script.pip("uninstall", "pip", "--yes", use_module=True)
-    script.pip("--version", expect_error=True)
+    # We don't use script.pip to check here, as when testing a
+    # zipapp, script.pip will run pip from the zipapp.
+    script.run("python", "-c", "import pip", expect_error=True)
 
     # The runner script should still invoke a usable pip
     result = script.run("python", os.fspath(runner), "--version")

tests/lib/__init__.py

@@ -505,6 +505,7 @@ def __init__(
         *args: Any,
         virtualenv: VirtualEnvironment,
         pip_expect_warning: bool = False,
+        zipapp: Optional[str] = None,
         **kwargs: Any,
     ) -> None:
         # Store paths related to the virtual environment
@@ -551,6 +552,9 @@ def __init__(
         # (useful for Python version deprecation)
         self.pip_expect_warning = pip_expect_warning
 
+        # The name of an (optional) zipapp to use when running pip
+        self.zipapp = zipapp
+
         # Call the TestFileEnvironment __init__
         super().__init__(base_path, *args, **kwargs)
 
@@ -585,6 +589,10 @@ def __init__(
     def _ignore_file(self, fn: str) -> bool:
         if fn.endswith("__pycache__") or fn.endswith(".pyc"):
             result = True
+        elif self.zipapp and fn.endswith("cacert.pem"):
+            # Temporary copies of cacert.pem are extracted
+            # when running from a zipapp
+            result = True
         else:
             result = super()._ignore_file(fn)
         return result
@@ -696,7 +704,10 @@ def pip(
         __tracebackhide__ = True
         if self.pip_expect_warning:
             kwargs["allow_stderr_warning"] = True
-        if use_module:
+        if self.zipapp:
+            exe = "python"
+            args = (self.zipapp,) + args
+        elif use_module:
             exe = "python"
             args = ("-m", "pip") + args
         else:

tests/lib/test_lib.py

@@ -41,6 +41,10 @@ def test_correct_pip_version(script: PipTestEnvironment) -> None:
     """
     Check we are running proper version of pip in run_pip.
     """
+
+    if script.zipapp:
+        pytest.skip("Test relies on the pip under test being in the filesystem")
+
     # output is like:
     # pip PIPVERSION from PIPDIRECTORY (python PYVERSION)
     result = script.pip("--version")