feat(bzlmod)!: Move each bzlmod extension into its own file (#1226)

This commit refactors the files that contain the bzlmod
extensions.

- All extensions are moved under the new extensions folder
- Private extensions are moved under extensions/private
- All extension files are renamed to remove the _extension suffix
- pip and internal_deps extensions are moved to their own file

This commit organizes the extensions better and also follows the
best practice of having a single extension per file. Having each
extension in its own file allows them to use some additional features
while helping avoid backwards incompatible changes.

## BREAKING CHANGES

This splits `//python:extensions.bzl`, which previously held the
`python`
and `pip` extensions, into separate files (`python.bzl` and `pip.bzl`,
respectively). Unfortunately, moving the location of the extensions is a
breaking change due to how bzlmod extension identity works (see
https://bazel.build/external/extension#extension_identity). Fortunately,
by moving to one extension per file, we shouldn't have to ever do this
again.

Users must update the file path in their `use_repo()` statements as
follows:

* `use_extension("@rules_python//python:extensions.bzl", "python")` ->
`use_extension("@rules_python//python/extensions:python.bzl", "python")`
* `use_extension("@rules_python//python:extensions.bzl", "pip")` ->
`use_extension("@rules_python//python/extensions:pip.bzl", "pip")`

The following `sed` commands should approximate the necessary changes:

```
sed 'sXuse_extension("@rules_python//python:extensions.bzl", "python")Xuse_extension("@rules_python//python/extensions:python.bzl", "python")X'`
sed 'sXuse_extension("@rules_python//python:extensions.bzl", "pip")Xuse_extension("@rules_python//python/extensions:pip.bzl", "pip")X'`

```

See `examples/bzlmod_build_file_generation/MODULE.bazel` for an example
of the new paths.

Author: chrislovecnm

MODULE.bazel

@@ -11,7 +11,7 @@ bazel_dep(name = "bazel_skylib", version = "1.3.0")
 bazel_dep(name = "rules_proto", version = "5.3.0-21.7")
 bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf")
 
-internal_deps = use_extension("@rules_python//python:extensions.bzl", "internal_deps")
+internal_deps = use_extension("@rules_python//python/extensions/private:internal_deps.bzl", "internal_deps")
 internal_deps.install()
 use_repo(
     internal_deps,
@@ -47,5 +47,5 @@ use_repo(
     "pypi__coverage_cp39_x86_64-unknown-linux-gnu",
 )
 
-python = use_extension("@rules_python//python:extensions.bzl", "python")
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
 use_repo(python, "pythons_hub")

examples/bzlmod/MODULE.bazel

@@ -10,7 +10,7 @@ local_path_override(
     path = "../..",
 )
 
-python = use_extension("@rules_python//python:extensions.bzl", "python")
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
 python.toolchain(
     name = "python3_9",
     configure_coverage_tool = True,
@@ -23,7 +23,7 @@ register_toolchains(
     "@python3_9_toolchains//:all",
 )
 
-pip = use_extension("@rules_python//python:extensions.bzl", "pip")
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
 pip.parse(
     name = "pip",
     requirements_lock = "//:requirements_lock.txt",

examples/bzlmod_build_file_generation/MODULE.bazel

@@ -42,7 +42,7 @@ bazel_dep(name = "gazelle", version = "0.30.0", repo_name = "bazel_gazelle")
 
 # The following stanze returns a proxy object representing a module extension;
 # its methods can be invoked to create module extension tags.
-python = use_extension("@rules_python//python:extensions.bzl", "python")
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
 
 # This name is passed into python.toolchain and it's use_repo statement.
 # We also use the same name for python.host_python_interpreter.
@@ -74,7 +74,7 @@ register_toolchains(
 # The interpreter extension discovers the platform specific Python binary.
 # It creates a symlink to the binary, and we pass the label to the following
 # pip.parse call.
-interpreter = use_extension("@rules_python//python:interpreter_extension.bzl", "interpreter")
+interpreter = use_extension("@rules_python//python/extensions:interpreter.bzl", "interpreter")
 interpreter.install(
     name = "interpreter_python3",
     python_name = PYTHON_NAME,
@@ -88,7 +88,7 @@ use_repo(interpreter, "interpreter_python3")
 # You can instead check this `requirements.bzl` file into your repo.
 # Because this project has different requirements for windows vs other
 # operating systems, we have requirements for each.
-pip = use_extension("@rules_python//python:extensions.bzl", "pip")
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
 pip.parse(
     name = "pip",
     # When using gazelle you must use set the following flag

examples/py_proto_library/MODULE.bazel

@@ -12,7 +12,7 @@ local_path_override(
     path = "../..",
 )
 
-python = use_extension("@rules_python//python:extensions.bzl", "python")
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
 python.toolchain(
     name = "python3_9",
     configure_coverage_tool = True,

python/extensions/BUILD.bazel

@@ -0,0 +1,23 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+filegroup(
+    name = "distribution",
+    srcs = glob(["**"]),
+    visibility = ["//extensions:__pkg__"],
+)

python/interpreter_extension.bzl renamed to python/extensions/interpreter.bzl

@@ -12,71 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"Module extensions for use with bzlmod"
+"pip module extension for use with bzlmod"
 
-load("@rules_python//python:repositories.bzl", "python_register_toolchains")
 load("@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_repository_attrs", "pip_repository_bzlmod", "use_isolated", "whl_library")
-load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies")
 load("@rules_python//python/pip_install:requirements_parser.bzl", parse_requirements = "parse")
-load("@rules_python//python/private:coverage_deps.bzl", "install_coverage_deps")
-load("@rules_python//python/private:interpreter_hub.bzl", "hub_repo")
-
-def _python_impl(module_ctx):
-    toolchains = []
-    for mod in module_ctx.modules:
-        for toolchain_attr in mod.tags.toolchain:
-            python_register_toolchains(
-                name = toolchain_attr.name,
-                python_version = toolchain_attr.python_version,
-                bzlmod = True,
-                # Toolchain registration in bzlmod is done in MODULE file
-                register_toolchains = False,
-                register_coverage_tool = toolchain_attr.configure_coverage_tool,
-                ignore_root_user_error = toolchain_attr.ignore_root_user_error,
-            )
-
-            # We collect all of the toolchain names to create
-            # the INTERPRETER_LABELS map.  This is used
-            # by interpreter_extensions.bzl
-            toolchains.append(toolchain_attr.name)
-
-    hub_repo(
-        name = "pythons_hub",
-        toolchains = toolchains,
-    )
-
-python = module_extension(
-    implementation = _python_impl,
-    tag_classes = {
-        "toolchain": tag_class(
-            attrs = {
-                "configure_coverage_tool": attr.bool(
-                    mandatory = False,
-                    doc = "Whether or not to configure the default coverage tool for the toolchains.",
-                ),
-                "ignore_root_user_error": attr.bool(
-                    default = False,
-                    doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.",
-                    mandatory = False,
-                ),
-                "name": attr.string(mandatory = True),
-                "python_version": attr.string(mandatory = True),
-            },
-        ),
-    },
-)
-
-# buildifier: disable=unused-variable
-def _internal_deps_impl(module_ctx):
-    pip_install_dependencies()
-    install_coverage_deps()
-
-internal_deps = module_extension(
-    implementation = _internal_deps_impl,
-    tag_classes = {
-        "install": tag_class(attrs = dict()),
-    },
-)
 
 def _pip_impl(module_ctx):
     for mod in module_ctx.modules:
@@ -134,6 +73,10 @@ def _pip_parse_ext_attrs():
     return attrs
 
 pip = module_extension(
+    doc = """\
+This extension is used to create a pip respository and create the various wheel libaries if
+provided in a requirements file.
+""",
     implementation = _pip_impl,
     tag_classes = {
         "parse": tag_class(attrs = _pip_parse_ext_attrs()),

python/extensions.bzl renamed to python/extensions/pip.bzl

@@ -0,0 +1,23 @@
+# Copyright 2022 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//visibility:private"])
+
+licenses(["notice"])
+
+filegroup(
+    name = "distribution",
+    srcs = glob(["**"]),
+    visibility = ["//python/extensions/private:__pkg__"],
+)

python/extensions/private/BUILD.bazel

@@ -0,0 +1,25 @@
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"Python toolchain module extension for internal rule use"
+
+load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies")
+load("@rules_python//python/private:coverage_deps.bzl", "install_coverage_deps")
+
+# buildifier: disable=unused-variable
+def _internal_deps_impl(module_ctx):
+    pip_install_dependencies()
+    install_coverage_deps()
+
+internal_deps = module_extension(
+    doc = "This extension to register internal rules_python dependecies.",
+    implementation = _internal_deps_impl,
+    tag_classes = {
+        "install": tag_class(attrs = dict()),
+    },
+)

python/extensions/private/internal_deps.bzl

@@ -0,0 +1,64 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"Python toolchain module extensions for use with bzlmod"
+
+load("@rules_python//python:repositories.bzl", "python_register_toolchains")
+load("@rules_python//python/extensions/private:interpreter_hub.bzl", "hub_repo")
+
+def _python_impl(module_ctx):
+    toolchains = []
+    for mod in module_ctx.modules:
+        for toolchain_attr in mod.tags.toolchain:
+            python_register_toolchains(
+                name = toolchain_attr.name,
+                python_version = toolchain_attr.python_version,
+                bzlmod = True,
+                # Toolchain registration in bzlmod is done in MODULE file
+                register_toolchains = False,
+                register_coverage_tool = toolchain_attr.configure_coverage_tool,
+                ignore_root_user_error = toolchain_attr.ignore_root_user_error,
+            )
+
+            # We collect all of the toolchain names to create
+            # the INTERPRETER_LABELS map.  This is used
+            # by interpreter_extensions.bzl
+            toolchains.append(toolchain_attr.name)
+
+    hub_repo(
+        name = "pythons_hub",
+        toolchains = toolchains,
+    )
+
+python = module_extension(
+    doc = "Bzlmod extension that is used to register a Python toolchain.",
+    implementation = _python_impl,
+    tag_classes = {
+        "toolchain": tag_class(
+            attrs = {
+                "configure_coverage_tool": attr.bool(
+                    mandatory = False,
+                    doc = "Whether or not to configure the default coverage tool for the toolchains.",
+                ),
+                "ignore_root_user_error": attr.bool(
+                    default = False,
+                    doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.",
+                    mandatory = False,
+                ),
+                "name": attr.string(mandatory = True),
+                "python_version": attr.string(mandatory = True),
+            },
+        ),
+    },
+)