feat: data and pyi files in the venv

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data
 
 test --test_output=errors
 

MODULE.bazel

@@ -86,6 +86,7 @@ bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True)
 bazel_dep(name = "bazel_ci_rules", version = "1.0.0", dev_dependency = True)
 bazel_dep(name = "rules_pkg", version = "1.0.1", dev_dependency = True)
 bazel_dep(name = "other", version = "0", dev_dependency = True)
+bazel_dep(name = "another_module", version = "0", dev_dependency = True)
 
 # Extra gazelle plugin deps so that WORKSPACE.bzlmod can continue including it for e2e tests.
 # We use `WORKSPACE.bzlmod` because it is impossible to have dev-only local overrides.
@@ -116,6 +117,11 @@ local_path_override(
     path = "tests/modules/other",
 )
 
+local_path_override(
+    module_name = "another_module",
+    path = "tests/modules/another_module",
+)
+
 dev_python = use_extension(
     "//python/extensions:python.bzl",
     "python",

internal_dev_deps.bzl

@@ -48,6 +48,11 @@ def rules_python_internal_deps():
         path = "tests/modules/other",
     )
 
+    local_repository(
+        name = "another_module",
+        path = "tests/modules/another_module",
+    )
+
     http_archive(
         name = "bazel_skylib",
         sha256 = "bc283cdfcd526a52c3201279cda4bc298652efa898b10b4db0837dc51652756f",

python/private/BUILD.bazel

@@ -450,11 +450,13 @@ bzl_library(
         ":attributes_bzl",
         ":common_bzl",
         ":flags_bzl",
+        ":normalize_name_bzl",
         ":precompile_bzl",
         ":py_cc_link_params_info_bzl",
         ":py_internal_bzl",
         ":rule_builders_bzl",
         ":toolchain_types_bzl",
+        ":version_bzl",
         "@bazel_skylib//lib:dicts",
         "@bazel_skylib//rules:common_settings",
     ],

python/private/common.bzl

@@ -369,6 +369,19 @@ def collect_runfiles(ctx, files = depset()):
         collect_default = True,
     )
 
+def _third_party_first(targets):
+    """Sort targets to allow for deterministic depset merging.
+
+    First return the third party deps so that the depsets get merged in a way to allow
+    topological traversal so that the first dependency that is met during traversing
+    will be a third party dep.
+
+    This is because the DAG is going from first-party deps to third-party deps and usually
+    no third-party deps include first-party deps.
+    """
+    return targets
+    # return sorted(targets, lambda x: PyInfo in x and not x[PyInfo].package)
+
 def create_py_info(
         ctx,
         *,
@@ -378,6 +391,7 @@ def create_py_info(
         implicit_pyc_files,
         implicit_pyc_source_files,
         imports,
+        package = None,
         venv_symlinks = []):
     """Create PyInfo provider.
 
@@ -396,9 +410,9 @@ def create_py_info(
         implicit_pyc_files: {type}`depset[File]` Implicitly generated pyc files
             that a binary can choose to include.
         imports: depset of strings; the import path values to propagate.
-        venv_symlinks: {type}`list[tuple[str, str]]` tuples of
-            `(runfiles_path, site_packages_path)` for symlinks to create
-            in the consuming binary's venv site packages.
+        package: TODO
+        venv_symlinks: {type}`list[VenvSymlinkEntry]` instances for
+            symlinks to create in the consuming binary's venv.
 
     Returns:
         A tuple of the PyInfo instance and a depset of the
@@ -419,7 +433,7 @@ def create_py_info(
     py_info.merge_has_py2_only_sources(ctx.attr.srcs_version in ("PY2", "PY2ONLY"))
     py_info.merge_has_py3_only_sources(ctx.attr.srcs_version in ("PY3", "PY3ONLY"))
 
-    for target in ctx.attr.deps:
+    for target in _third_party_first(ctx.attr.deps):
         # PyInfo may not be present e.g. cc_library rules.
         if PyInfo in target or (BuiltinPyInfo != None and BuiltinPyInfo in target):
             py_info.merge(_get_py_info(target))
@@ -431,7 +445,7 @@ def create_py_info(
                 if f.extension == "py":
                     py_info.transitive_sources.add(f)
                 py_info.merge_uses_shared_libraries(cc_helper.is_valid_shared_library_artifact(f))
-    for target in ctx.attr.pyi_deps:
+    for target in _third_party_first(ctx.attr.pyi_deps):
         # PyInfo may not be present e.g. cc_library rules.
         if PyInfo in target or (BuiltinPyInfo != None and BuiltinPyInfo in target):
             py_info.merge(_get_py_info(target))
@@ -458,7 +472,7 @@ def create_py_info(
             if py_info.get_uses_shared_libraries():
                 break
 
-    return py_info.build(), deps_transitive_sources, py_info.build_builtin_py_info()
+    return py_info.build(package), deps_transitive_sources, py_info.build_builtin_py_info()
 
 def _get_py_info(target):
     return target[PyInfo] if PyInfo in target or BuiltinPyInfo == None else target[BuiltinPyInfo]

python/private/py_executable.bzl

@@ -680,43 +680,54 @@ def _create_venv_symlinks(ctx, venv_dir_map):
     return venv_files
 
 def _build_link_map(entries):
-    # dict[str kind, dict[str rel_path, str link_to_path]]
-    link_map = {}
+    # dict[str package, dict[str kind, dict[str rel_path, str link_to_path]]]
+    pkg_link_map = {}
+
+    # dict[str package, str version]
+    version_by_pkg = {}
+
     for entry in entries:
-        kind = entry.kind
-        kind_map = link_map.setdefault(kind, {})
-        if entry.venv_path in kind_map:
+        link_map = pkg_link_map.setdefault(entry.package, {})
+        kind_map = link_map.setdefault(entry.kind, {})
+
+        if version_by_pkg.setdefault(entry.package, entry.version) != entry.version:
             # We ignore duplicates by design. The dependency closer to the
             # binary gets precedence due to the topological ordering.
             continue
+        elif entry.venv_path in kind_map:
+            # NOTE @aignas 2025-06-05: This branch should be hit only for first party
+            # packages, maybe we should error here?
+            continue
         else:
             kind_map[entry.venv_path] = entry.link_to_path
 
     # An empty link_to value means to not create the site package symlink.
     # Because of the topological ordering, this allows binaries to remove
     # entries by having an earlier dependency produce empty link_to values.
-    for kind, kind_map in link_map.items():
-        for dir_path, link_to in kind_map.items():
-            if not link_to:
-                kind_map.pop(dir_path)
+    for link_map in pkg_link_map.values():
+        for kind, kind_map in link_map.items():
+            for dir_path, link_to in kind_map.items():
+                if not link_to:
+                    kind_map.pop(dir_path)
 
     # dict[str kind, dict[str rel_path, str link_to_path]]
     keep_link_map = {}
 
     # Remove entries that would be a child path of a created symlink.
     # Earlier entries have precedence to match how exact matches are handled.
-    for kind, kind_map in link_map.items():
-        keep_kind_map = keep_link_map.setdefault(kind, {})
-        for _ in range(len(kind_map)):
-            if not kind_map:
-                break
-            dirname, value = kind_map.popitem()
-            keep_kind_map[dirname] = value
-            prefix = dirname + "/"  # Add slash to prevent /X matching /XY
-            for maybe_suffix in kind_map.keys():
-                maybe_suffix += "/"  # Add slash to prevent /X matching /XY
-                if maybe_suffix.startswith(prefix) or prefix.startswith(maybe_suffix):
-                    kind_map.pop(maybe_suffix)
+    for link_map in pkg_link_map.values():
+        for kind, kind_map in link_map.items():
+            keep_kind_map = keep_link_map.setdefault(kind, {})
+            for _ in range(len(kind_map)):
+                if not kind_map:
+                    break
+                dirname, value = kind_map.popitem()
+                keep_kind_map[dirname] = value
+                prefix = dirname + "/"  # Add slash to prevent /X matching /XY
+                for maybe_suffix in kind_map.keys():
+                    maybe_suffix += "/"  # Add slash to prevent /X matching /XY
+                    if maybe_suffix.startswith(prefix) or prefix.startswith(maybe_suffix):
+                        kind_map.pop(maybe_suffix)
     return keep_link_map
 
 def _map_each_identity(v):

python/private/py_info.bzl

@@ -67,11 +67,24 @@ the venv to create the path under.
 
 A runfiles-root relative path that `venv_path` will symlink to. If `None`,
 it means to not create a symlink.
+""",
+        "package": """
+:type: str | None
+
+Represents the PyPI package name that the code originates from. It is normalized according to the
+PEP440 with all `-` replaced with `_`, i.e. the same as the package name in the hub repository that
+it would come from.
 """,
         "venv_path": """
 :type: str
 
 A path relative to the `kind` directory within the venv.
+""",
+        "version": """
+:type: str | None
+
+Represents the PyPI package version that the code originates from. It is normalized according to the
+PEP440 standard.
 """,
     },
 )
@@ -94,14 +107,15 @@ def _PyInfo_init(
         has_py2_only_sources = False,
         has_py3_only_sources = False,
         direct_pyc_files = depset(),
+        package = None,
         transitive_pyc_files = depset(),
         transitive_implicit_pyc_files = depset(),
         transitive_implicit_pyc_source_files = depset(),
         direct_original_sources = depset(),
         transitive_original_sources = depset(),
         direct_pyi_files = depset(),
         transitive_pyi_files = depset(),
-        venv_symlinks = depset()):
+        venv_symlinks = depset(order = "topological")):
     _check_arg_type("transitive_sources", "depset", transitive_sources)
 
     # Verify it's postorder compatible, but retain is original ordering.
@@ -129,6 +143,7 @@ def _PyInfo_init(
         "has_py2_only_sources": has_py2_only_sources,
         "has_py3_only_sources": has_py2_only_sources,
         "imports": imports,
+        "package": package,
         "transitive_implicit_pyc_files": transitive_implicit_pyc_files,
         "transitive_implicit_pyc_source_files": transitive_implicit_pyc_source_files,
         "transitive_original_sources": transitive_original_sources,
@@ -205,6 +220,7 @@ Python targets. These are accumulated from the transitive `deps`.
 The order of the depset is not guaranteed and may be changed in the future. It
 is recommended to use `default` order (the default).
 """,
+        "package": "TODO",
         "transitive_implicit_pyc_files": """
 :type: depset[File]
 
@@ -298,17 +314,6 @@ This field is currently unused in Bazel and may go away in the future.
 
 A depset with `topological` ordering.
 
-
-Tuples of `(runfiles_path, site_packages_path)`. Where
-* `runfiles_path` is a runfiles-root relative path. It is the path that
-  has the code to make importable. If `None` or empty string, then it means
-  to not create a site packages directory with the `site_packages_path`
-  name.
-* `site_packages_path` is a path relative to the site-packages directory of
-  the venv for whatever creates the venv (typically py_binary). It makes
-  the code in `runfiles_path` available for import. Note that this
-  is created as a "raw" symlink (via `declare_symlink`).
-
 :::{include} /_includes/experimental_api.md
 :::
 
@@ -623,11 +628,12 @@ def _PyInfoBuilder_merge_targets(self, targets):
         self.merge_target(t)
     return self
 
-def _PyInfoBuilder_build(self):
+def _PyInfoBuilder_build(self, package = None):
     """Builds into a {obj}`PyInfo` object.
 
     Args:
         self: implicitly added.
+        package: TODO
 
     Returns:
         {type}`PyInfo`
@@ -637,6 +643,7 @@ def _PyInfoBuilder_build(self):
             direct_original_sources = self.direct_original_sources.build(),
             direct_pyc_files = self.direct_pyc_files.build(),
             direct_pyi_files = self.direct_pyi_files.build(),
+            package = package,
             transitive_implicit_pyc_files = self.transitive_implicit_pyc_files.build(),
             transitive_implicit_pyc_source_files = self.transitive_implicit_pyc_source_files.build(),
             transitive_original_sources = self.transitive_original_sources.build(),