ruby codegen: support generation of rbs files (#15633)

this introduces support for a new protoc option, `--rbs_out`, which points to a directory where ruby type definition files, defined in the RBS format, are stored.

[rbs](https://github.com/ruby/rbs) is the type signature syntax blessed by the ruby core team, used by static analysis tools such as [steep](https://github.com/soutaro/steep), which integrates with VS Code, the `irb` console (for features such as autocompletion). and [typeprof](https://github.com/ruby/typeprof).

It relies on type definitions written into `.rbs` files.

The `protobuf` library already exposes type definitions in [gem_rbs_collection](https://github.com/ruby/gem_rbs_collection/tree/main/gems/google-protobuf/3.22), which is used to source type definitions for libraries which do not want, or can't maintain type definitions themselves.

(`protobuf` could arguably import these into this repository, lmk if you're interested).

This should fix gaps such as better IDE integration, such as the ones described
[here](#9495).

The plan is to do roughly the same type of integration as was done for `.pyi` annotations, which also write to separate files: add the cli option and the rbs generator.

protobuf classes in ruby rely on dynamic attribution of the base class, which makes what I'm trying to achieve a bit difficult. Ideally the type hierarchy could be specified statically in the ruby source code.

```ruby
class Bar < AbstractMessage
class Bar <
Google::Protobuf::DescriptorPool.generated_pool.lookup("Bar").msgclass
```

```

Closes #15633

FUTURE_COPYBARA_INTEGRATE_REVIEW=#15633 from HoneyryderChuck:ruby-rbs-integration 569f032
PiperOrigin-RevId: 820790992

Author: HoneyryderChuck

.github/workflows/test_ruby.yml

@@ -213,3 +213,23 @@ jobs:
             bazel-bin/protoc --proto_path=src --proto_path=ruby/tests --proto_path=ruby --ruby_out=ruby tests/basic_test.proto;
             ${{ matrix.ffi == 'FFI' && 'PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI' || '' }} ruby ruby/tests/basic.rb;
             ${{ matrix.ffi == 'FFI' && 'PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI' || '' }} ruby ruby/tests/implementation.rb
+
+  typecheck:
+    name: Typecheck
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout pending changes
+        uses: protocolbuffers/protobuf-ci/checkout@v2
+        with:
+          ref: ${{ inputs.safe-checkout }}
+      - name: Run type checker
+        uses: protocolbuffers/protobuf-ci/bazel-docker@v2
+        with:
+          image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.6.1-e0df73e51131ccaf53451355d22577f377357604
+          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+          bazel-cache: ruby_typecheck
+          bash:
+            bazel build //:protoc
+            cd ruby
+            cp typecheck.gemfile.lock Gemfile.lock
+            bundle exec rake steep

protobuf.bzl

@@ -65,6 +65,9 @@ def _PyOuts(srcs, use_grpc_plugin = False):
 def _RubyOuts(srcs):
     return [s[:-len(".proto")] + "_pb.rb" for s in srcs]
 
+def _RBSOuts(srcs):
+    return [s[:-len(".proto")] + "_pb.rbs" for s in srcs]
+
 def _CsharpOuts(srcs):
     return [
         "".join([token.capitalize() for token in src[:-len(".proto")].split("_")]) + ".cs"
@@ -158,6 +161,8 @@ def _proto_gen_impl(ctx):
                 outs.extend(_PyOuts([src.basename], use_grpc_plugin = use_grpc_plugin))
             elif lang == "ruby":
                 outs.extend(_RubyOuts([src.basename]))
+            elif lang == "rbs":
+                outs.extend(_RBSOuts([src.basename]))
 
             # Otherwise, rely on user-supplied outs.
             args.append(("--%s_out=" + path_tpl) % (lang, gen_dir))
@@ -526,8 +531,6 @@ def internal_ruby_proto_library(
 
     """
 
-    # Note: we need to run the protoc build twice to get separate targets for
-    # the generated header and the source files.
     _proto_gen(
         name = name + "_genproto",
         srcs = srcs,
@@ -552,6 +555,58 @@ def internal_ruby_proto_library(
         **kwargs
     )
 
+def internal_rbs_proto_library(
+        name,
+        ruby_library,
+        srcs = [],
+        deps = [],
+        includes = ["."],
+        protoc = "@com_google_protobuf//:protoc",
+        testonly = None,
+        visibility = ["//visibility:public"],
+        **kwargs):
+    """Bazel rule to create an RBS type definitions for Ruby from proto source files
+
+    NOTE: the rule is only an internal workaround to generate protos. The
+    interface may change and the rule may be removed when bazel has introduced
+    the native rule.
+
+    Args:
+      name: the name of the ruby_proto_library.
+      ruby_library: the ruby library rules to use.
+      srcs: the .proto files to compile.
+      deps: a list of dependency labels; must be a internal_rbs_proto_library.
+      includes: a string indicating the include path of the .proto files.
+      protoc: the label of the protocol compiler to generate the sources.
+      testonly: common rule attribute (see:
+          https://bazel.build/reference/be/common-definitions#common-attributes)
+      visibility: the visibility of the generated files.
+      **kwargs: other keyword arguments that are passed to ruby_library.
+
+    """
+
+    _proto_gen(
+        name = name + "_genproto_rbs",
+        srcs = srcs,
+        deps = [s + "_genproto_rbs" for s in deps],
+        langs = ["rbs"],
+        includes = includes,
+        protoc = protoc,
+        testonly = testonly,
+        visibility = visibility,
+        tags = ["manual"],
+    )
+
+    ruby_library(
+        name = name,
+        srcs = [name + "_genproto_rbs"],
+        deps = [],
+        testonly = testonly,
+        visibility = visibility,
+        includes = includes,
+        **kwargs
+    )
+
 # When canonical labels are in use, use additional "@" prefix
 _canonical_label_prefix = "@" if str(Label("//:protoc")).startswith("@@") else ""
 

src/file_lists.cmake

@@ -472,6 +472,7 @@ set(libprotoc_srcs
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/python/helpers.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/python/pyi_generator.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/retention.cc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/ruby/rbs_generator.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/ruby/ruby_generator.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/accessor_case.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/accessors.cc
@@ -619,6 +620,7 @@ set(libprotoc_hdrs
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/python/helpers.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/python/pyi_generator.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/retention.h
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/ruby/rbs_generator.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/ruby/ruby_generator.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/accessor_case.h
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/accessors.h
@@ -1395,6 +1397,7 @@ set(compiler_test_files
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/php/generator_unittest.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/python/plugin_unittest.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/retention_unittest.cc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/ruby/rbs_generator_unittest.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/ruby/ruby_generator_unittest.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/versions_test.cc
 )

src/google/protobuf/compiler/main.cc

@@ -15,6 +15,7 @@
 #include "google/protobuf/compiler/php/php_generator.h"
 #include "google/protobuf/compiler/python/generator.h"
 #include "google/protobuf/compiler/python/pyi_generator.h"
+#include "google/protobuf/compiler/ruby/rbs_generator.h"
 #include "google/protobuf/compiler/ruby/ruby_generator.h"
 #include "google/protobuf/compiler/rust/generator.h"
 #ifdef GOOGLE_PROTOBUF_RUNTIME_INCLUDE_BASE
@@ -95,6 +96,10 @@ int ProtobufMain(int argc, char* argv[]) {
   cli.RegisterGenerator("--ruby_out", "--ruby_opt", &rb_generator,
                         "Generate Ruby source file.");
 
+  ruby::RBSGenerator rbs_generator;
+  cli.RegisterGenerator("--rbs_out", "--rbs_opt", &rbs_generator,
+                        "Generate RBS type definition.");
+
   // CSharp
   csharp::Generator csharp_generator;
   cli.RegisterGenerator("--csharp_out", "--csharp_opt", &csharp_generator,

src/google/protobuf/compiler/ruby/BUILD.bazel

@@ -20,8 +20,14 @@ cc_library(
 
 cc_library(
     name = "ruby",
-    srcs = ["ruby_generator.cc"],
-    hdrs = ["ruby_generator.h"],
+    srcs = [
+        "rbs_generator.cc",
+        "ruby_generator.cc",
+    ],
+    hdrs = [
+        "rbs_generator.h",
+        "ruby_generator.h",
+    ],
     copts = COPTS,
     strip_include_prefix = "/src",
     visibility = [
@@ -35,27 +41,37 @@ cc_library(
         "//src/google/protobuf/compiler:retention",
         "//src/google/protobuf/io",
         "//src/google/protobuf/io:printer",
+        "@abseil-cpp//absl/container:flat_hash_map",
         "@abseil-cpp//absl/container:flat_hash_set",
+        "@abseil-cpp//absl/log:absl_check",
         "@abseil-cpp//absl/log:absl_log",
         "@abseil-cpp//absl/strings",
     ],
 )
 
 cc_test(
     name = "generator_unittest",
-    srcs = ["ruby_generator_unittest.cc"],
+    srcs = [
+        "rbs_generator_unittest.cc",
+        "ruby_generator_unittest.cc",
+    ],
     data = [
         "ruby_generated_code.proto",
         "ruby_generated_code_pb.rb",
+        "ruby_generated_code_pb.rbs",
         "ruby_generated_code_proto2.proto",
         "ruby_generated_code_proto2_import.proto",
         "ruby_generated_code_proto2_pb.rb",
+        "ruby_generated_code_proto2_pb.rbs",
         "ruby_generated_pkg_explicit.proto",
         "ruby_generated_pkg_explicit_legacy.proto",
         "ruby_generated_pkg_explicit_legacy_pb.rb",
+        "ruby_generated_pkg_explicit_legacy_pb.rbs",
         "ruby_generated_pkg_explicit_pb.rb",
+        "ruby_generated_pkg_explicit_pb.rbs",
         "ruby_generated_pkg_implicit.proto",
         "ruby_generated_pkg_implicit_pb.rb",
+        "ruby_generated_pkg_implicit_pb.rbs",
         "//src/google/protobuf:testdata",
     ],
     deps = [