Fix broken Windows zipapp and drop 3.7 support (#2783)

gaborbernat · web-flow · commit f73a2f385316 · 2024-10-17T18:25:56.000-07:00
diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml
@@ -39,9 +39,6 @@ jobs:
           - { os: macos-latest, py: "brew@3.11" }
           - { os: macos-latest, py: "brew@3.10" }
           - { os: macos-latest, py: "brew@3.9" }
-          - { os: macos-latest, py: "brew@3.8" }
-          - { os: ubuntu-latest, py: "3.7" }
-          - { os: macos-13, py: "3.7" }
         exclude:
           - { os: windows-latest, py: "pypy-3.10" }
           - { os: windows-latest, py: "pypy-3.9" }
diff --git a/docs/changelog/2758.feature.rst b/docs/changelog/2758.feature.rst
@@ -0,0 +1 @@
+Drop 3.7 support as the CI environments no longer allow it running - by :user:`gaborbernat`.
diff --git a/docs/changelog/2783.bugfix.rst b/docs/changelog/2783.bugfix.rst
@@ -0,0 +1,8 @@
+Upgrade embedded wheels:
+
+* setuptools to ``75.2.0`` from ``75.1.0``
+* Removed pip of ``24.0``
+* Removed setuptools of ``68.0.0``
+* Removed wheel of ``0.42.0``
+
+- by :user:`gaborbernat`.
diff --git a/docs/changelog/2784.bugfix.rst b/docs/changelog/2784.bugfix.rst
@@ -0,0 +1 @@
+Fix zipapp is broken on Windows post distlib ``0.3.9`` - by :user:`gaborbernat`.
diff --git a/docs/installation.rst b/docs/installation.rst
@@ -97,6 +97,7 @@ request on our issue tracker.
 
 Note:
 
+- as of ``20.27.0`` -- ``2024-10-17`` -- we no longer support running under Python ``<=3.7``,
 - as of ``20.18.0`` -- ``2023-02-06`` -- we no longer support running under Python ``<=3.6``,
 - as of ``20.22.0`` -- ``2023-04-19`` -- we no longer support creating environments for Python ``<=3.6``.
 
@@ -120,4 +121,4 @@ In case of macOS we support:
 Windows
 ~~~~~~~
 - Installations from `python.org <https://www.python.org/downloads/>`_
-- Windows Store Python - note only `version 3.7+ <https://www.microsoft.com/en-us/p/python-38/9mssztt1n39l>`_
+- Windows Store Python - note only `version 3.8+ <https://www.microsoft.com/en-us/p/python-38/9mssztt1n39l>`_
diff --git a/pyproject.toml b/pyproject.toml
@@ -18,7 +18,7 @@ license = "MIT"
 maintainers = [
   { name = "Bernat Gabor", email = "gaborjbernat@gmail.com" },
 ]
-requires-python = ">=3.7"
+requires-python = ">=3.8"
 classifiers = [
   "Development Status :: 5 - Production/Stable",
   "Intended Audience :: Developers",
@@ -27,7 +27,6 @@ classifiers = [
   "Operating System :: Microsoft :: Windows",
   "Operating System :: POSIX",
   "Programming Language :: Python :: 3 :: Only",
-  "Programming Language :: Python :: 3.7",
   "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
@@ -106,7 +105,6 @@ build.targets.sdist.include = [
 version.source = "vcs"
 
 [tool.ruff]
-target-version = "py37"
 line-length = 120
 format.preview = true
 format.docstring-code-line-length = 100
@@ -128,6 +126,7 @@ lint.ignore = [
   "PLR0914", # Too many local variables
   "PLR0917", # Too many positional arguments
   "PLR6301", #  Method could be a function, class method, or static method
+  "PLW1510", # no need for check for subprocess
   "PTH",     # no pathlib, <=39 has problems on Windows with absolute/resolve, can revisit once we no longer need 39
   "S104",    # Possible binding to all interfaces
   "S404",    # Using subprocess is alright
diff --git a/src/virtualenv/run/plugin/base.py b/src/virtualenv/run/plugin/base.py
@@ -2,15 +2,9 @@
 
 import sys
 from collections import OrderedDict
+from importlib.metadata import entry_points
 
-if sys.version_info >= (3, 8):
-    from importlib.metadata import entry_points
-
-    importlib_metadata_version = ()
-else:
-    from importlib_metadata import entry_points, version
-
-    importlib_metadata_version = tuple(int(i) for i in version("importlib_metadata").split(".")[:2])
+importlib_metadata_version = ()
 
 
 class PluginLoader:
diff --git a/src/virtualenv/seed/wheels/embed/__init__.py b/src/virtualenv/seed/wheels/embed/__init__.py
@@ -6,48 +6,43 @@
 
 BUNDLE_FOLDER = Path(__file__).absolute().parent
 BUNDLE_SUPPORT = {
-    "3.7": {
-        "pip": "pip-24.0-py3-none-any.whl",
-        "setuptools": "setuptools-68.0.0-py3-none-any.whl",
-        "wheel": "wheel-0.42.0-py3-none-any.whl",
-    },
     "3.8": {
         "pip": "pip-24.2-py3-none-any.whl",
-        "setuptools": "setuptools-75.1.0-py3-none-any.whl",
+        "setuptools": "setuptools-75.2.0-py3-none-any.whl",
         "wheel": "wheel-0.44.0-py3-none-any.whl",
     },
     "3.9": {
         "pip": "pip-24.2-py3-none-any.whl",
-        "setuptools": "setuptools-75.1.0-py3-none-any.whl",
+        "setuptools": "setuptools-75.2.0-py3-none-any.whl",
         "wheel": "wheel-0.44.0-py3-none-any.whl",
     },
     "3.10": {
         "pip": "pip-24.2-py3-none-any.whl",
-        "setuptools": "setuptools-75.1.0-py3-none-any.whl",
+        "setuptools": "setuptools-75.2.0-py3-none-any.whl",
         "wheel": "wheel-0.44.0-py3-none-any.whl",
     },
     "3.11": {
         "pip": "pip-24.2-py3-none-any.whl",
-        "setuptools": "setuptools-75.1.0-py3-none-any.whl",
+        "setuptools": "setuptools-75.2.0-py3-none-any.whl",
         "wheel": "wheel-0.44.0-py3-none-any.whl",
     },
     "3.12": {
         "pip": "pip-24.2-py3-none-any.whl",
-        "setuptools": "setuptools-75.1.0-py3-none-any.whl",
+        "setuptools": "setuptools-75.2.0-py3-none-any.whl",
         "wheel": "wheel-0.44.0-py3-none-any.whl",
     },
     "3.13": {
         "pip": "pip-24.2-py3-none-any.whl",
-        "setuptools": "setuptools-75.1.0-py3-none-any.whl",
+        "setuptools": "setuptools-75.2.0-py3-none-any.whl",
         "wheel": "wheel-0.44.0-py3-none-any.whl",
     },
     "3.14": {
         "pip": "pip-24.2-py3-none-any.whl",
-        "setuptools": "setuptools-75.1.0-py3-none-any.whl",
+        "setuptools": "setuptools-75.2.0-py3-none-any.whl",
         "wheel": "wheel-0.44.0-py3-none-any.whl",
     },
 }
-MAX = "3.7"
+MAX = "3.8"
 
 
 def get_embed_wheel(distribution, for_py_version):
diff --git a/src/virtualenv/seed/wheels/embed/pip-24.0-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/pip-24.0-py3-none-any.whl
diff --git a/src/virtualenv/seed/wheels/embed/setuptools-68.0.0-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/setuptools-68.0.0-py3-none-any.whl
diff --git a/src/virtualenv/seed/wheels/embed/setuptools-75.2.0-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/setuptools-75.2.0-py3-none-any.whl
diff --git a/src/virtualenv/seed/wheels/embed/wheel-0.42.0-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/wheel-0.42.0-py3-none-any.whl
diff --git a/tasks/__main__zipapp.py b/tasks/__main__zipapp.py
@@ -4,22 +4,25 @@
 import os
 import sys
 import zipfile
+from functools import cached_property
+from importlib.abc import SourceLoader
+from importlib.util import spec_from_file_location
 
 ABS_HERE = os.path.abspath(os.path.dirname(__file__))
-NEW_IMPORT_SYSTEM = sys.version_info[0] >= 3  # noqa: PLR2004
 
 
 class VersionPlatformSelect:
     def __init__(self) -> None:
-        self.archive = ABS_HERE
-        self._zip_file = zipfile.ZipFile(ABS_HERE, "r")
+        zipapp = ABS_HERE
+        self.archive = zipapp
+        self._zip_file = zipfile.ZipFile(zipapp)
         self.modules = self._load("modules.json")
         self.distributions = self._load("distributions.json")
         self.__cache = {}
 
     def _load(self, of_file):
         version = ".".join(str(i) for i in sys.version_info[0:2])
-        per_version = json.loads(self.get_data(of_file).decode("utf-8"))
+        per_version = json.loads(self.get_data(of_file).decode())
         all_platforms = per_version[version] if version in per_version else per_version["3.9"]
         content = all_platforms.get("==any", {})  # start will all platforms
         not_us = f"!={sys.platform}"
@@ -65,25 +68,66 @@ def find_distributions(self, context):
     def __repr__(self) -> str:
         return f"{self.__class__.__name__}(path={ABS_HERE})"
 
-    def _register_distutils_finder(self):
+    def _register_distutils_finder(self):  # noqa: C901
         if "distlib" not in self.modules:
             return
 
+        class Resource:
+            def __init__(self, path: str, name: str, loader: SourceLoader) -> None:
+                self.path = os.path.join(path, name)
+                self._name = name
+                self.loader = loader
+
+            @cached_property
+            def name(self) -> str:
+                return os.path.basename(self._name)
+
+            @property
+            def bytes(self) -> bytes:
+                return self.loader.get_data(self._name)
+
+            @property
+            def is_container(self) -> bool:
+                return len(self.resources) > 1
+
+            @cached_property
+            def resources(self) -> list[str]:
+                return [
+                    i.filename
+                    for i in (
+                        (j for j in zip_file.filelist if j.filename.startswith(f"{self._name}/"))
+                        if self._name
+                        else zip_file.filelist
+                    )
+                ]
+
         class DistlibFinder:
             def __init__(self, path, loader) -> None:
                 self.path = path
                 self.loader = loader
 
             def find(self, name):
-                class Resource:
-                    def __init__(self, content) -> None:
-                        self.bytes = content
-
-                full_path = os.path.join(self.path, name)
-                return Resource(self.loader.get_data(full_path))
+                return Resource(self.path, name, self.loader)
+
+            def iterator(self, resource_name):
+                resource = self.find(resource_name)
+                if resource is not None:
+                    todo = [resource]
+                    while todo:
+                        resource = todo.pop(0)
+                        yield resource
+                        if resource.is_container:
+                            resource_name = resource.name
+                            for name in resource.resources:
+                                child = self.find(f"{resource_name}/{name}" if resource_name else name)
+                                if child.is_container:
+                                    todo.append(child)
+                                else:
+                                    yield child
 
         from distlib.resources import register_finder  # noqa: PLC0415
 
+        zip_file = self._zip_file
         register_finder(self, lambda module: DistlibFinder(os.path.dirname(module.__file__), self))
 
 
@@ -93,10 +137,7 @@ def __init__(self, content) -> None:
 def versioned_distribution_class():
     global _VER_DISTRIBUTION_CLASS  # noqa: PLW0603
     if _VER_DISTRIBUTION_CLASS is None:
-        if sys.version_info >= (3, 8):
-            from importlib.metadata import Distribution  # noqa: PLC0415
-        else:
-            from importlib_metadata import Distribution  # noqa: PLC0415
+        from importlib.metadata import Distribution  # noqa: PLC0415
 
         class VersionedDistribution(Distribution):
             def __init__(self, file_loader, dist_path) -> None:
@@ -113,41 +154,15 @@ def locate_file(self, path):
     return _VER_DISTRIBUTION_CLASS
 
 
-if NEW_IMPORT_SYSTEM:
-    from importlib.abc import SourceLoader
-    from importlib.util import spec_from_file_location
-
-    class VersionedFindLoad(VersionPlatformSelect, SourceLoader):
-        def find_spec(self, fullname, path, target=None):  # noqa: ARG002
-            zip_path = self.find_mod(fullname)
-            if zip_path is not None:
-                return spec_from_file_location(name=fullname, loader=self)
-            return None
-
-        def module_repr(self, module):
-            raise NotImplementedError
-
-else:
-    from imp import new_module
-
-    class VersionedFindLoad(VersionPlatformSelect):
-        def find_module(self, fullname, path=None):  # noqa: ARG002
-            return self if self.find_mod(fullname) else None
-
-        def load_module(self, fullname):
-            filename = self.get_filename(fullname)
-            code = self.get_data(filename)
-            mod = sys.modules.setdefault(fullname, new_module(fullname))
-            mod.__file__ = filename
-            mod.__loader__ = self
-            is_package = filename.endswith("__init__.py")
-            if is_package:
-                mod.__path__ = [os.path.dirname(filename)]
-                mod.__package__ = fullname
-            else:
-                mod.__package__ = fullname.rpartition(".")[0]
-            exec(code, mod.__dict__)  # noqa: S102
-            return mod
+class VersionedFindLoad(VersionPlatformSelect, SourceLoader):
+    def find_spec(self, fullname, path, target=None):  # noqa: ARG002
+        zip_path = self.find_mod(fullname)
+        if zip_path is not None:
+            return spec_from_file_location(name=fullname, loader=self)
+        return None
+
+    def module_repr(self, module):
+        raise NotImplementedError
 
 
 def run():
diff --git a/tasks/make_zipapp.py b/tasks/make_zipapp.py
@@ -23,7 +23,7 @@
 
 HERE = Path(__file__).parent.absolute()
 
-VERSIONS = [f"3.{i}" for i in range(10, 6, -1)]
+VERSIONS = [f"3.{i}" for i in range(13, 7, -1)]
 
 
 def main():
@@ -49,7 +49,7 @@ def create_zipapp(dest, packages):
         zip_app.writestr("__main__.py", (HERE / "__main__zipapp.py").read_bytes())
     bio.seek(0)
     zipapp.create_archive(bio, dest)
-    print(f"zipapp created at {dest}")  # noqa: T201
+    print(f"zipapp created at {dest} with size {os.path.getsize(dest) / 1024 / 1024:.2f}MB")  # noqa: T201
 
 
 def write_packages_to_zipapp(base, dist, modules, packages, zip_app):  # noqa: C901, PLR0912
diff --git a/tasks/upgrade_wheels.py b/tasks/upgrade_wheels.py
@@ -15,7 +15,7 @@
 STRICT = "UPGRADE_ADVISORY" not in os.environ
 
 BUNDLED = ["pip", "setuptools", "wheel"]
-SUPPORT = [(3, i) for i in range(7, 15)]
+SUPPORT = [(3, i) for i in range(8, 15)]
 DEST = Path(__file__).resolve().parents[1] / "src" / "virtualenv" / "seed" / "wheels" / "embed"
 
 
@@ -66,8 +66,8 @@ def run():  # noqa: C901
 
         added = collect_package_versions(new_packages)
         removed = collect_package_versions(remove_packages)
-
         outcome = (1 if STRICT else 0) if (added or removed) else 0
+        print(f"Outcome {outcome} added {added} removed {removed}")  # noqa: T201
         lines = ["Upgrade embedded wheels:", ""]
         for key, versions in added.items():
             text = f"* {key} to {fmt_version(versions)}"
@@ -119,15 +119,8 @@ def get_embed_wheel(distribution, for_py_version):
         )
         dest_target = DEST / "__init__.py"
         dest_target.write_text(msg, encoding="utf-8")
-
-        subprocess.run(
-            [sys.executable, "-m", "ruff", "check", str(dest_target), "--fix", "--unsafe-fixes"],
-            check=False,
-        )
-        subprocess.run(
-            [sys.executable, "-m", "ruff", "format", str(dest_target), "--preview"],
-            check=False,
-        )
+        subprocess.run([sys.executable, "-m", "ruff", "format", str(dest_target), "--preview"])
+        subprocess.run([sys.executable, "-m", "ruff", "check", str(dest_target), "--fix", "--unsafe-fixes"])
 
         raise SystemExit(outcome)
 
diff --git a/tests/unit/seed/embed/test_pip_invoke.py b/tests/unit/seed/embed/test_pip_invoke.py
@@ -25,7 +25,7 @@ def test_base_bootstrap_via_pip_invoke(tmp_path, coverage_env, mocker, current_f
     def _load_embed_wheel(app_data, distribution, for_py_version, version):  # noqa: ARG001
         return load_embed_wheel(app_data, distribution, old_ver, version)
 
-    old_ver = "3.7"
+    old_ver = "3.8"
     old = BUNDLE_SUPPORT[old_ver]
     mocker.patch("virtualenv.seed.wheels.bundle.load_embed_wheel", side_effect=_load_embed_wheel)
 
diff --git a/tox.ini b/tox.ini
@@ -10,7 +10,6 @@ env_list =
     3.10
     3.9
     3.8
-    3.7
     coverage
     readme
     docs

-Original file line number
+Diff line change
           - { os: macos-latest, py: "[email protected]" }
           - { os: macos-latest, py: "[email protected]" }
           - { os: macos-latest, py: "[email protected]" }
 -          - { os: macos-latest, py: "[email protected]" }
 -          - { os: ubuntu-latest, py: "3.7" }
 -          - { os: macos-13, py: "3.7" }
         exclude:
           - { os: windows-latest, py: "pypy-3.10" }
           - { os: windows-latest, py: "pypy-3.9" }
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Drop 3.7 support as the CI environments no longer allow it running - by :user:`gaborbernat`.
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Fix zipapp is broken on Windows post distlib ``0.3.9`` - by :user:`gaborbernat`.