Fix pkg_resources-style legacy namespaces in editable installs (#4041)

Author: abravalheri

newsfragments/4041.bugfix.1.rst

@@ -0,0 +1,2 @@
+Fix the name given to the ``*-nspkg.pth`` files in editable installs,
+ensuring they are unique per distribution.

newsfragments/4041.bugfix.2.rst

@@ -0,0 +1,2 @@
+Workaround some limitations on ``pkg_resources``-style legacy namespaces in
+the meta path finder for editable installations.

setuptools/command/editable_wheel.py

@@ -341,7 +341,7 @@ def _create_wheel_file(self, bdist_wheel):
         with unpacked_wheel as unpacked, build_lib as lib, build_tmp as tmp:
             unpacked_dist_info = Path(unpacked, Path(self.dist_info_dir).name)
             shutil.copytree(self.dist_info_dir, unpacked_dist_info)
-            self._install_namespaces(unpacked, dist_info.name)
+            self._install_namespaces(unpacked, dist_name)
             files, mapping = self._run_build_commands(dist_name, unpacked, lib, tmp)
             strategy = self._select_strategy(dist_name, tag, lib)
             with strategy, WheelFile(wheel_path, "w") as wheel_obj:
@@ -505,9 +505,19 @@ def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]
             )
         )
 
+        legacy_namespaces = {
+            pkg: find_package_path(pkg, roots, self.dist.src_root or "")
+            for pkg in self.dist.namespace_packages or []
+        }
+
+        mapping = {**roots, **legacy_namespaces}
+        # ^-- We need to explicitly add the legacy_namespaces to the mapping to be
+        #     able to import their modules even if another package sharing the same
+        #     namespace is installed in a conventional (non-editable) way.
+
         name = f"__editable__.{self.name}.finder"
         finder = _normalization.safe_identifier(name)
-        content = bytes(_finder_template(name, roots, namespaces_), "utf-8")
+        content = bytes(_finder_template(name, mapping, namespaces_), "utf-8")
         wheel.writestr(f"{finder}.py", content)
 
         content = _encode_pth(f"import {finder}; {finder}.install()")
@@ -752,9 +762,9 @@ def __init__(self, distribution, installation_dir, editable_name, src_root):
         self.outputs = []
         self.dry_run = False
 
-    def _get_target(self):
+    def _get_nspkg_file(self):
         """Installation target."""
-        return os.path.join(self.installation_dir, self.editable_name)
+        return os.path.join(self.installation_dir, self.editable_name + self.nspkg_ext)
 
     def _get_root(self):
         """Where the modules/packages should be loaded from."""
@@ -777,6 +787,8 @@ def _get_root(self):
 class _EditableFinder:  # MetaPathFinder
     @classmethod
     def find_spec(cls, fullname, path=None, target=None):
+        extra_path = []
+
         # Top-level packages and modules (we know these exist in the FS)
         if fullname in MAPPING:
             pkg_path = MAPPING[fullname]
@@ -787,7 +799,7 @@ def find_spec(cls, fullname, path=None, target=None):
         # to the importlib.machinery implementation.
         parent, _, child = fullname.rpartition(".")
         if parent and parent in MAPPING:
-            return PathFinder.find_spec(fullname, path=[MAPPING[parent]])
+            return PathFinder.find_spec(fullname, path=[MAPPING[parent], *extra_path])
 
         # Other levels of nesting should be handled automatically by importlib
         # using the parent path.

setuptools/namespaces.py

@@ -13,8 +13,7 @@ def install_namespaces(self):
         nsp = self._get_all_ns_packages()
         if not nsp:
             return
-        filename, ext = os.path.splitext(self._get_target())
-        filename += self.nspkg_ext
+        filename = self._get_nspkg_file()
         self.outputs.append(filename)
         log.info("Installing %s", filename)
         lines = map(self._gen_nspkg_line, nsp)
@@ -28,13 +27,16 @@ def install_namespaces(self):
             f.writelines(lines)
 
     def uninstall_namespaces(self):
-        filename, ext = os.path.splitext(self._get_target())
-        filename += self.nspkg_ext
+        filename = self._get_nspkg_file()
         if not os.path.exists(filename):
             return
         log.info("Removing %s", filename)
         os.remove(filename)
 
+    def _get_nspkg_file(self):
+        filename, _ = os.path.splitext(self._get_target())
+        return filename + self.nspkg_ext
+
     def _get_target(self):
         return self.target
 
@@ -75,7 +77,7 @@ def _gen_nspkg_line(self, pkg):
     def _get_all_ns_packages(self):
         """Return sorted list of all package namespaces"""
         pkgs = self.distribution.namespace_packages or []
-        return sorted(flatten(map(self._pkg_names, pkgs)))
+        return sorted(set(flatten(map(self._pkg_names, pkgs))))
 
     @staticmethod
     def _pkg_names(pkg):

setuptools/tests/namespaces.py

@@ -1,29 +1,54 @@
+import ast
+import json
 import textwrap
+from pathlib import Path
 
 
-def build_namespace_package(tmpdir, name):
+def iter_namespace_pkgs(namespace):
+    parts = namespace.split(".")
+    for i in range(len(parts)):
+        yield ".".join(parts[: i + 1])
+
+
+def build_namespace_package(tmpdir, name, version="1.0", impl="pkg_resources"):
     src_dir = tmpdir / name
     src_dir.mkdir()
     setup_py = src_dir / 'setup.py'
-    namespace, sep, rest = name.partition('.')
+    namespace, _, rest = name.rpartition('.')
+    namespaces = list(iter_namespace_pkgs(namespace))
+    setup_args = {
+        "name": name,
+        "version": version,
+        "packages": namespaces,
+    }
+
+    if impl == "pkg_resources":
+        tmpl = '__import__("pkg_resources").declare_namespace(__name__)'
+        setup_args["namespace_packages"] = namespaces
+    elif impl == "pkgutil":
+        tmpl = '__path__ = __import__("pkgutil").extend_path(__path__, __name__)'
+    else:
+        raise ValueError(f"Cannot recognise {impl=} when creating namespaces")
+
+    args = json.dumps(setup_args, indent=4)
+    assert ast.literal_eval(args)  # ensure it is valid Python
+
     script = textwrap.dedent(
-        """
+        """\
         import setuptools
-        setuptools.setup(
-            name={name!r},
-            version="1.0",
-            namespace_packages=[{namespace!r}],
-            packages=[{namespace!r}],
-        )
+        args = {args}
+        setuptools.setup(**args)
         """
-    ).format(**locals())
+    ).format(args=args)
     setup_py.write_text(script, encoding='utf-8')
-    ns_pkg_dir = src_dir / namespace
-    ns_pkg_dir.mkdir()
-    pkg_init = ns_pkg_dir / '__init__.py'
-    tmpl = '__import__("pkg_resources").declare_namespace({namespace!r})'
-    decl = tmpl.format(**locals())
-    pkg_init.write_text(decl, encoding='utf-8')
+
+    ns_pkg_dir = Path(src_dir, namespace.replace(".", "/"))
+    ns_pkg_dir.mkdir(parents=True)
+
+    for ns in namespaces:
+        pkg_init = src_dir / ns.replace(".", "/") / '__init__.py'
+        pkg_init.write_text(tmpl, encoding='utf-8')
+
     pkg_mod = ns_pkg_dir / (rest + '.py')
     some_functionality = 'name = {rest!r}'.format(**locals())
     pkg_mod.write_text(some_functionality, encoding='utf-8')
@@ -34,7 +59,7 @@ def build_pep420_namespace_package(tmpdir, name):
     src_dir = tmpdir / name
     src_dir.mkdir()
     pyproject = src_dir / "pyproject.toml"
-    namespace, sep, rest = name.rpartition(".")
+    namespace, _, rest = name.rpartition(".")
     script = f"""\
         [build-system]
         requires = ["setuptools"]
@@ -45,7 +70,7 @@ def build_pep420_namespace_package(tmpdir, name):
         version = "3.14159"
         """
     pyproject.write_text(textwrap.dedent(script), encoding='utf-8')
-    ns_pkg_dir = src_dir / namespace.replace(".", "/")
+    ns_pkg_dir = Path(src_dir, namespace.replace(".", "/"))
     ns_pkg_dir.mkdir(parents=True)
     pkg_mod = ns_pkg_dir / (rest + ".py")
     some_functionality = f"name = {rest!r}"

setuptools/tests/test_editable_install.py

@@ -11,6 +11,8 @@
 from unittest.mock import Mock
 from uuid import uuid4
 
+from distutils.core import run_setup
+
 import jaraco.envs
 import jaraco.path
 import pytest
@@ -31,6 +33,7 @@
 )
 from setuptools.dist import Distribution
 from setuptools.extension import Extension
+from setuptools.warnings import SetuptoolsDeprecationWarning
 
 
 @pytest.fixture(params=["strict", "lenient"])
@@ -230,30 +233,67 @@ def test_editable_with_single_module(tmp_path, venv, editable_opts):
 
 
 class TestLegacyNamespaces:
-    """Ported from test_develop"""
+    # legacy => pkg_resources.declare_namespace(...) + setup(namespace_packages=...)
 
-    def test_namespace_package_importable(self, venv, tmp_path, editable_opts):
+    def test_nspkg_file_is_unique(self, tmp_path, monkeypatch):
+        deprecation = pytest.warns(
+            SetuptoolsDeprecationWarning, match=".*namespace_packages parameter.*"
+        )
+        installation_dir = tmp_path / ".installation_dir"
+        installation_dir.mkdir()
+        examples = (
+            "myns.pkgA",
+            "myns.pkgB",
+            "myns.n.pkgA",
+            "myns.n.pkgB",
+        )
+
+        for name in examples:
+            pkg = namespaces.build_namespace_package(tmp_path, name, version="42")
+            with deprecation, monkeypatch.context() as ctx:
+                ctx.chdir(pkg)
+                dist = run_setup("setup.py", stop_after="config")
+                cmd = editable_wheel(dist)
+                cmd.finalize_options()
+                editable_name = cmd.get_finalized_command("dist_info").name
+                cmd._install_namespaces(installation_dir, editable_name)
+
+        files = list(installation_dir.glob("*-nspkg.pth"))
+        assert len(files) == len(examples)
+
+    @pytest.mark.parametrize(
+        "impl",
+        (
+            "pkg_resources",
+            #  "pkgutil",  => does not work
+        ),
+    )
+    @pytest.mark.parametrize("ns", ("myns.n",))
+    def test_namespace_package_importable(
+        self, venv, tmp_path, ns, impl, editable_opts
+    ):
         """
         Installing two packages sharing the same namespace, one installed
         naturally using pip or `--single-version-externally-managed`
         and the other installed in editable mode should leave the namespace
         intact and both packages reachable by import.
+        (Ported from test_develop).
         """
         build_system = """\
         [build-system]
         requires = ["setuptools"]
         build-backend = "setuptools.build_meta"
         """
-        pkg_A = namespaces.build_namespace_package(tmp_path, 'myns.pkgA')
-        pkg_B = namespaces.build_namespace_package(tmp_path, 'myns.pkgB')
+        pkg_A = namespaces.build_namespace_package(tmp_path, f"{ns}.pkgA", impl=impl)
+        pkg_B = namespaces.build_namespace_package(tmp_path, f"{ns}.pkgB", impl=impl)
         (pkg_A / "pyproject.toml").write_text(build_system, encoding="utf-8")
         (pkg_B / "pyproject.toml").write_text(build_system, encoding="utf-8")
         # use pip to install to the target directory
         opts = editable_opts[:]
         opts.append("--no-build-isolation")  # force current version of setuptools
         venv.run(["python", "-m", "pip", "install", str(pkg_A), *opts])
         venv.run(["python", "-m", "pip", "install", "-e", str(pkg_B), *opts])
-        venv.run(["python", "-c", "import myns.pkgA; import myns.pkgB"])
+        venv.run(["python", "-c", f"import {ns}.pkgA; import {ns}.pkgB"])
         # additionally ensure that pkg_resources import works
         venv.run(["python", "-c", "import pkg_resources"])