More client features (#157)

Author: cretz

.github/workflows/ci.yml

@@ -71,10 +71,19 @@ jobs:
         working-directory: ./temporalio
         run: cargo clippy && cargo fmt --check
 
-      # TODO(cretz): For checkTarget, regen protos and ensure no diff
+      - name: Install bundle
+        working-directory: ./temporalio
+        run: bundle install
+
+      - name: Check generated protos
+        if: ${{ matrix.checkTarget }}
+        working-directory: ./temporalio
+        run: |
+          bundle exec rake proto:generate
+          [[ -z $(git status --porcelain lib/temporalio/api) ]] || (git diff lib/temporalio/api; echo "Protos changed" 1>&2; exit 1)
 
       - name: Lint, compile, test Ruby
         working-directory: ./temporalio
-        run: bundle install && bundle exec rake TESTOPTS="--verbose"
+        run: bundle exec rake TESTOPTS="--verbose"
 
       # TODO(cretz): Build gem and smoke test against separate dir

README.md

@@ -21,7 +21,10 @@ Prerequisites:
 
 To build shared library for development use:
 
-    bundle exec rake compile:dev
+    bundle exec rake compile
+
+Note, this is not `compile:dev` because debug-mode in Rust has
+[an issue](https://github.com/rust-lang/rust/issues/34283) that causes runtime stack size problems.
 
 To build and test release:
 

temporalio/.rubocop.yml

@@ -34,7 +34,7 @@ Metrics/BlockLength:
 
 # The default is too small
 Metrics/ClassLength:
-  Max: 400
+  Max: 1000
 
 # The default is too small
 Metrics/CyclomaticComplexity:

temporalio/Rakefile

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+# rubocop:disable Metrics/BlockLength, Lint/MissingCopEnableDirective, Style/DocumentationMethod
+
 require 'bundler/gem_tasks'
 require 'rb_sys/cargo/metadata'
 require 'rb_sys/extensiontask'
@@ -34,57 +36,272 @@ require 'yard'
 YARD::Rake::YardocTask.new
 
 require 'fileutils'
+require 'google/protobuf'
 
 namespace :proto do
   desc 'Generate API and Core protobufs'
   task :generate do
     # Remove all existing
     FileUtils.rm_rf('lib/temporalio/api')
 
-    # Collect set of API protos with Google ones removed
-    api_protos = Dir.glob('ext/sdk-core/sdk-core-protos/protos/api_upstream/**/*.proto').reject do |proto|
-      proto.include?('google')
-    end
+    def generate_protos(api_protos)
+      # Generate API to temp dir and move
+      FileUtils.rm_rf('tmp-proto')
+      FileUtils.mkdir_p('tmp-proto')
+      sh 'bundle exec grpc_tools_ruby_protoc ' \
+         '--proto_path=ext/sdk-core/sdk-core-protos/protos/api_upstream ' \
+         '--proto_path=ext/sdk-core/sdk-core-protos/protos/api_cloud_upstream ' \
+         '--proto_path=ext/additional_protos ' \
+         '--ruby_out=tmp-proto ' \
+         "#{api_protos.join(' ')}"
+
+      # Walk all generated Ruby files and cleanup content and filename
+      Dir.glob('tmp-proto/temporal/api/**/*.rb') do |path|
+        # Fix up the import
+        content = File.read(path)
+        content.gsub!(%r{^require 'temporal/(.*)_pb'$}, "require 'temporalio/\\1'")
+        File.write(path, content)
+
+        # Remove _pb from the filename
+        FileUtils.mv(path, path.sub('_pb', ''))
+      end
 
-    # Generate API to temp dir and move
-    FileUtils.rm_rf('tmp-proto')
-    FileUtils.mkdir_p('tmp-proto')
-    sh 'bundle exec grpc_tools_ruby_protoc ' \
-       '--proto_path=ext/sdk-core/sdk-core-protos/protos/api_upstream ' \
-       '--ruby_out=tmp-proto ' \
-       "#{api_protos.join(' ')}"
-
-    # Walk all generated Ruby files and cleanup content and filename
-    Dir.glob('tmp-proto/temporal/api/**/*.rb') do |path|
-      # Fix up the import
-      content = File.read(path)
-      content.gsub!(%r{^require 'temporal/(.*)_pb'$}, "require 'temporalio/\\1'")
-      File.write(path, content)
-
-      # Remove _pb from the filename
-      FileUtils.mv(path, path.sub('_pb', ''))
+      # Move from temp dir and remove temp dir
+      FileUtils.cp_r('tmp-proto/temporal/api', 'lib/temporalio')
+      FileUtils.rm_rf('tmp-proto')
     end
 
-    # Move from temp dir and remove temp dir
-    FileUtils.mv('tmp-proto/temporal/api', 'lib/temporalio')
-    FileUtils.rm_rf('tmp-proto')
+    # Generate from API with Google ones removed
+    generate_protos(Dir.glob('ext/sdk-core/sdk-core-protos/protos/api_upstream/**/*.proto').reject do |proto|
+      proto.include?('google')
+    end)
+
+    # Generate from Cloud API
+    generate_protos(Dir.glob('ext/sdk-core/sdk-core-protos/protos/api_cloud_upstream/**/*.proto'))
+
+    # Generate additional protos
+    generate_protos(Dir.glob('ext/additional_protos/**/*.proto'))
 
     # Write files that will help with imports. We are requiring the
     # request_response and not the service because the service depends on Google
     # API annotations we don't want to have to depend on.
-    string_lit = "# frozen_string_literal: true\n\n"
+    File.write(
+      'lib/temporalio/api/cloud/cloudservice.rb',
+      <<~TEXT
+        # frozen_string_literal: true
+
+        require 'temporalio/api/cloud/cloudservice/v1/request_response'
+      TEXT
+    )
     File.write(
       'lib/temporalio/api/workflowservice.rb',
-      "#{string_lit}require 'temporalio/api/workflowservice/v1/request_response'\n"
+      <<~TEXT
+        # frozen_string_literal: true
+
+        require 'temporalio/api/workflowservice/v1/request_response'
+      TEXT
     )
     File.write(
       'lib/temporalio/api/operatorservice.rb',
-      "#{string_lit}require 'temporalio/api/operatorservice/v1/request_response'\n"
+      <<~TEXT
+        # frozen_string_literal: true
+
+        require 'temporalio/api/operatorservice/v1/request_response'
+      TEXT
     )
     File.write(
       'lib/temporalio/api.rb',
-      "#{string_lit}require 'temporalio/api/operatorservice'\nrequire 'temporalio/api/workflowservice'\n"
+      <<~TEXT
+        # frozen_string_literal: true
+
+        require 'temporalio/api/cloud/cloudservice'
+        require 'temporalio/api/common/v1/grpc_status'
+        require 'temporalio/api/errordetails/v1/message'
+        require 'temporalio/api/operatorservice'
+        require 'temporalio/api/workflowservice'
+
+        module Temporalio
+          # Raw protocol buffer models.
+          module Api
+          end
+        end
+      TEXT
     )
+
+    # Write the service classes that have the RPC calls
+    def write_service_file(qualified_service_name:, file_name:, class_name:, service_enum:)
+      # Do service lookup
+      desc = Google::Protobuf::DescriptorPool.generated_pool.lookup(qualified_service_name)
+      raise 'Failed finding service descriptor' unless desc
+
+      # Open file to generate Ruby code
+      File.open("lib/temporalio/client/connection/#{file_name}.rb", 'w') do |file|
+        file.puts <<~TEXT
+          # frozen_string_literal: true
+
+          # Generated code.  DO NOT EDIT!
+
+          require 'temporalio/api'
+          require 'temporalio/client/connection/service'
+          require 'temporalio/internal/bridge/client'
+
+          module Temporalio
+            class Client
+              class Connection
+                # #{class_name} API.
+                class #{class_name} < Service
+                  # @!visibility private
+                  def initialize(connection)
+                    super(connection, Internal::Bridge::Client::#{service_enum})
+                  end
+        TEXT
+
+        desc.each do |method|
+          # Camel case to snake case
+          rpc = method.name.gsub(/([A-Z])/, '_\1').downcase.delete_prefix('_')
+          file.puts <<-TEXT
+
+        # Calls #{class_name}.#{method.name} API call.
+        #
+        # @param request [#{method.input_type.msgclass}] API request.
+        # @param rpc_retry [Boolean] Whether to implicitly retry known retryable errors.
+        # @param rpc_metadata [Hash<String, String>, nil] Headers to include on the RPC call.
+        # @param rpc_timeout [Float, nil] Number of seconds before timeout.
+        # @return [#{method.output_type.msgclass}] API response.
+        def #{rpc}(request, rpc_retry: false, rpc_metadata: nil, rpc_timeout: nil)
+          invoke_rpc(
+            rpc: '#{rpc}',
+            request_class: #{method.input_type.msgclass},
+            response_class: #{method.output_type.msgclass},
+            request:,
+            rpc_retry:,
+            rpc_metadata:,
+            rpc_timeout:
+          )
+        end
+          TEXT
+        end
+
+        file.puts <<~TEXT
+                end
+              end
+            end
+          end
+        TEXT
+      end
+
+      # Open file to generate RBS code
+      # TODO(cretz): Improve this when RBS proto is supported
+      File.open("sig/temporalio/client/connection/#{file_name}.rbs", 'w') do |file|
+        file.puts <<~TEXT
+          # Generated code.  DO NOT EDIT!
+
+          module Temporalio
+            class Client
+              class Connection
+                class #{class_name} < Service
+                  def initialize: (Connection) -> void
+        TEXT
+
+        desc.each do |method|
+          # Camel case to snake case
+          rpc = method.name.gsub(/([A-Z])/, '_\1').downcase.delete_prefix('_')
+          file.puts <<-TEXT
+        def #{rpc}: (untyped request, ?rpc_retry: bool, ?rpc_metadata: Hash[String, String]?, ?rpc_timeout: Float?) -> untyped
+          TEXT
+        end
+
+        file.puts <<~TEXT
+                end
+              end
+            end
+          end
+        TEXT
+      end
+    end
+
+    require './lib/temporalio/api/workflowservice/v1/service'
+    write_service_file(
+      qualified_service_name: 'temporal.api.workflowservice.v1.WorkflowService',
+      file_name: 'workflow_service',
+      class_name: 'WorkflowService',
+      service_enum: 'SERVICE_WORKFLOW'
+    )
+    require './lib/temporalio/api/operatorservice/v1/service'
+    write_service_file(
+      qualified_service_name: 'temporal.api.operatorservice.v1.OperatorService',
+      file_name: 'operator_service',
+      class_name: 'OperatorService',
+      service_enum: 'SERVICE_OPERATOR'
+    )
+    require './lib/temporalio/api/cloud/cloudservice/v1/service'
+    write_service_file(
+      qualified_service_name: 'temporal.api.cloud.cloudservice.v1.CloudService',
+      file_name: 'cloud_service',
+      class_name: 'CloudService',
+      service_enum: 'SERVICE_CLOUD'
+    )
+
+    # Generate Rust code
+    def generate_rust_match_arm(file:, qualified_service_name:, service_enum:, trait:)
+      # Do service lookup
+      desc = Google::Protobuf::DescriptorPool.generated_pool.lookup(qualified_service_name)
+      file.puts <<~TEXT
+        #{service_enum} => match call.rpc.as_str() {
+      TEXT
+
+      desc.to_a.sort_by(&:name).each do |method|
+        # Camel case to snake case
+        rpc = method.name.gsub(/([A-Z])/, '_\1').downcase.delete_prefix('_')
+        file.puts <<~TEXT
+          "#{rpc}" => rpc_call!(self, block, call, #{trait}, #{rpc}),
+        TEXT
+      end
+      file.puts <<~TEXT
+          _ => Err(error!("Unknown RPC call {}", call.rpc)),
+        },
+      TEXT
+    end
+    File.open('ext/src/client_rpc_generated.rs', 'w') do |file|
+      file.puts <<~TEXT
+        // Generated code.  DO NOT EDIT!
+
+        use magnus::{block::Proc, value::Opaque, Error, Ruby};
+        use temporal_client::{CloudService, OperatorService, WorkflowService};
+
+        use super::{error, rpc_call};
+        use crate::client::{Client, RpcCall, SERVICE_CLOUD, SERVICE_OPERATOR, SERVICE_WORKFLOW};
+
+        impl Client {
+            pub fn invoke_rpc(&self, service: u8, block: Opaque<Proc>, call: RpcCall) -> Result<(), Error> {
+                match service {
+      TEXT
+      generate_rust_match_arm(
+        file:,
+        qualified_service_name: 'temporal.api.workflowservice.v1.WorkflowService',
+        service_enum: 'SERVICE_WORKFLOW',
+        trait: 'WorkflowService'
+      )
+      generate_rust_match_arm(
+        file:,
+        qualified_service_name: 'temporal.api.operatorservice.v1.OperatorService',
+        service_enum: 'SERVICE_OPERATOR',
+        trait: 'OperatorService'
+      )
+      generate_rust_match_arm(
+        file:,
+        qualified_service_name: 'temporal.api.cloud.cloudservice.v1.CloudService',
+        service_enum: 'SERVICE_CLOUD',
+        trait: 'CloudService'
+      )
+      file.puts <<~TEXT
+                    _ => Err(error!("Unknown service")),
+                }
+            }
+        }
+      TEXT
+    end
+    sh 'cargo', 'fmt', '--', 'ext/src/client_rpc_generated.rs'
   end
 end
 
@@ -104,4 +321,4 @@ Rake::Task[:build].enhance([:copy_parent_files]) do
   rm ['LICENSE', 'README.md']
 end
 
-task default: [:rubocop, 'rbs:install_collection', :steep, :compile, :test]
+task default: ['rubocop', 'compile', 'rbs:install_collection', 'steep', 'test']

temporalio/Steepfile

@@ -12,7 +12,7 @@ target :lib do
   library 'uri'
 
   configure_code_diagnostics do |hash|
+    # TODO(cretz): Fix as more protos are generated
     hash[D::Ruby::UnknownConstant] = :information
-    hash[D::Ruby::NoMethod] = :information
   end
 end

temporalio/ext/additional_protos/temporal/api/common/v1/grpc_status.proto

@@ -0,0 +1,15 @@
+syntax = "proto3";
+
+package temporal.api.common.v1;
+
+option ruby_package = "Temporalio::Api::Common::V1";
+
+import "google/protobuf/any.proto";
+
+// From https://github.com/grpc/grpc/blob/master/src/proto/grpc/status/status.proto
+// since we don't import grpc but still need the status info
+message GrpcStatus {
+  int32 code = 1;
+  string message = 2;
+  repeated google.protobuf.Any details = 3;
+}