feat: add workflow client updater for updating workflow client

src/Temporalio.Extensions.Hosting/README.md

@@ -99,4 +99,48 @@ start instead of expecting a `ITemporalClient` to be present on the service coll
 Some users may prefer to manually create the `TemporalWorker` without using host support, but still make their
 activities created via the service provider. `CreateTemporalActivityDefinitions` extension methods are present on
 `IServiceProvider` that will return a collection of `ActivityDefinition` instances for each activity on the type. These
-can be added to the `TemporalWorkerOptions` directly.
+can be added to the `TemporalWorkerOptions` directly.
+
+## Worker Client Refresh
+
+Some users may need to update the worker's connection to Temporal. It's desirable to do this without stopping the worker entirely, as that will evict the sticky workflow cache.
+
+This can be done by using the `IWorkerClientUpdater`.
+
+```csharp
+using Temporalio.Extensions.Hosting;
+
+var builder = Host.CreateApplicationBuilder(args);
+
+// Register a worker client updater.
+builder.Services.AddSingleton<IWorkerClientUpdater, WorkerClientUpdater>();
+
+// Add a hosted Temporal worker which returns a builder to add activities and workflows, along with the worker client updater.
+builder.Services.
+    AddHostedTemporalWorker(
+        "my-temporal-host:7233",
+        "my-namespace",
+        "my-task-queue").
+    AddScopedActivities<MyActivityClass>().
+    AddWorkflow<MyWorkflow>().
+    ConfigureOptions().
+    Configure<IWorkerClientUpdater>((options, workerClientUpdater) => options.WorkerClientUpdater = workerClientUpdater);
+
+var host = builder.Build();
+
+// You can have a BackgroundService periodically refresh the worker client like this.
+IWorkerClientUpdater workerClientUpdater = host.Services.GetRequiredService<IWorkerClientUpdater>();
+
+// Can update the TLS options if you need.
+TemporalClientConnectOptions clientConnectOptions = new("my-other-temporal-host:7233")
+{
+    Namespace = "default"
+};
+
+ITemporalClient updatedClient = await TemporalClient.ConnectAsync(clientConnectOptions).ConfigureAwait(false);
+
+workerClientUpdater.UpdateWorkerClient(updatedClient);
+
+// Make sure you use RunAsync and not Run, see https://github.com/temporalio/sdk-dotnet/issues/220
+await host.RunAsync();
+```

src/Temporalio/Worker/IWorkerClientUpdater.cs

@@ -0,0 +1,21 @@
+using System;
+
+namespace Temporalio.Worker
+{
+    /// <summary>
+    /// Represents a notification hub that can be used to push worker client updates to subscribing workers.
+    /// </summary>
+    public interface IWorkerClientUpdater
+    {
+        /// <summary>
+        /// The event for when a worker client update is broadcast.
+        /// </summary>
+        event EventHandler<WorkerClientUpdatedEventArgs> OnWorkerClientUpdated;
+
+        /// <summary>
+        /// Dispatches a notification to all subscribers that a new worker client should be used.
+        /// </summary>
+        /// <param name="client">The new <see cref="IWorkerClient"/> that should be pushed out to all subscribing workers.</param>
+        void UpdateWorkerClient(IWorkerClient client);
+    }
+}

src/Temporalio/Worker/TemporalWorker.cs

@@ -67,6 +67,11 @@ public TemporalWorker(IWorkerClient client, TemporalWorkerOptions options)
                 WorkflowTracingEventListener.Instance.Register();
             }
 
+            if (Options.WorkerClientUpdater != null)
+            {
+                Options.WorkerClientUpdater.OnWorkerClientUpdated += OnWorkerClientUpdated;
+            }
+
             // Create workers
             if (options.Activities.Count > 0)
             {
@@ -108,7 +113,7 @@ public TemporalWorker(IWorkerClient client, TemporalWorkerOptions options)
         /// </summary>
         /// <remarks>
         /// When this property is set, it actually replaces the underlying client that is being used
-        /// by the worker. This means the next calls by the worker to Temporal (e.g. responding
+        /// by the worker. This means subsequent calls by the worker to Temporal (e.g. responding
         /// task completion, activity heartbeat, etc) will be on this new client, but outstanding
         /// calls will not be immediately interrupted.
         /// </remarks>
@@ -245,6 +250,16 @@ public void Dispose()
             GC.SuppressFinalize(this);
         }
 
+        /// <summary>
+        /// Callback invoked when a worker client updated is pushed through the <see cref="IWorkerClientUpdater"/>.
+        /// </summary>
+        /// <param name="sender">The sender of the event.</param>
+        /// <param name="eventArgs">The <see cref="WorkerClientUpdatedEventArgs"/> of the event.</param>
+        internal void OnWorkerClientUpdated(object? sender, WorkerClientUpdatedEventArgs eventArgs)
+        {
+            Client = eventArgs.WorkerClient;
+        }
+
         /// <summary>
         /// Dispose the worker.
         /// </summary>
@@ -253,8 +268,14 @@ protected virtual void Dispose(bool disposing)
         {
             if (disposing)
             {
+                if (Options.WorkerClientUpdater != null)
+                {
+                    Options.WorkerClientUpdater.OnWorkerClientUpdated -= OnWorkerClientUpdated;
+                }
+
                 activityWorker?.Dispose();
                 BridgeWorker.Dispose();
+
                 // Remove task tracing if not disabled and there are workflows present
                 if (workflowTracingEventListenerEnabled)
                 {

src/Temporalio/Worker/TemporalWorkerOptions.cs

@@ -272,6 +272,11 @@ public TemporalWorkerOptions()
         /// </remarks>
         public WorkflowStackTrace WorkflowStackTrace { get; set; } = WorkflowStackTrace.None;
 
+        /// <summary>
+        /// Gets or sets the <see cref="IWorkerClientUpdater"/> that can be used to push updates to the worker client.
+        /// </summary>
+        public IWorkerClientUpdater? WorkerClientUpdater { get; set; }
+
         /// <summary>
         /// Gets the TEMPORAL_DEBUG environment variable.
         /// </summary>

src/Temporalio/Worker/WorkerClientUpdatedEventArgs.cs

@@ -0,0 +1,22 @@
+using System;
+using Temporalio.Client;
+
+namespace Temporalio.Worker
+{
+    /// <summary>
+    /// Event raised when a worker client is updated.
+    /// </summary>
+    public class WorkerClientUpdatedEventArgs : EventArgs
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="WorkerClientUpdatedEventArgs"/> class.
+        /// </summary>
+        /// <param name="workerClient">The client to update workers with.</param>
+        public WorkerClientUpdatedEventArgs(IWorkerClient workerClient) => WorkerClient = workerClient;
+
+        /// <summary>
+        /// Gets the <see cref="ITemporalClient"/> that will be propagated to all event listeners.
+        /// </summary>
+        public IWorkerClient WorkerClient { get; }
+    }
+}

src/Temporalio/Worker/WorkerClientUpdater.cs

@@ -0,0 +1,44 @@
+using System;
+
+namespace Temporalio.Worker
+{
+    /// <summary>
+    /// Implementation of the <see cref="IWorkerClientUpdater"/>.
+    /// </summary>
+    public class WorkerClientUpdater : IWorkerClientUpdater
+    {
+        private readonly object clientLock = new();
+
+        /// <summary>
+        /// The <see cref="EventHandler"/> used to dispatch notifications to subscribers that the client was updated.
+        /// </summary>
+        public event EventHandler<WorkerClientUpdatedEventArgs> OnWorkerClientUpdated
+        {
+            add
+            {
+                lock (clientLock)
+                {
+                    WorkerClientUpdatedEvent += value;
+                }
+            }
+
+            remove
+            {
+                lock (clientLock)
+                {
+                    WorkerClientUpdatedEvent -= value;
+                }
+            }
+        }
+
+        private event EventHandler<WorkerClientUpdatedEventArgs>? WorkerClientUpdatedEvent;
+
+        /// <inheritdoc />
+        public void UpdateWorkerClient(IWorkerClient client)
+        {
+            WorkerClientUpdatedEventArgs eventArgs = new WorkerClientUpdatedEventArgs(client);
+
+            WorkerClientUpdatedEvent?.Invoke(this, eventArgs);
+        }
+    }
+}

tests/Temporalio.Tests/Extensions/Hosting/TemporalWorkerServiceTests.cs

@@ -1,3 +1,6 @@
+using Temporalio.Tests.Worker;
+using Temporalio.Worker;
+
 #pragma warning disable SA1201, SA1204 // We want to have classes near their tests
 namespace Temporalio.Tests.Extensions.Hosting;
 
@@ -79,11 +82,11 @@ public async Task TemporalWorkerService_ExecuteAsync_SimpleWorker()
             // Add the rest of the services
             services.
                 AddSingleton<ILoggerFactory>(loggerFactory).
-                AddScoped<DatabaseClient>().
-                // We are also adding the DB client as a keyed service to demonstrate keyed service
-                // support for our DI logic. This used to break because newer DI library versions
-                // disallowed accessing certain properties on keyed services which we access
-                // internally for dupe checks.
+                AddScoped<DatabaseClient>().
+                // We are also adding the DB client as a keyed service to demonstrate keyed service
+                // support for our DI logic. This used to break because newer DI library versions
+                // disallowed accessing certain properties on keyed services which we access
+                // internally for dupe checks.
                 AddKeyedScoped<DatabaseClient>("client-keyed").
                 AddHostedTemporalWorker(taskQueue).
                 AddScopedActivities<DatabaseActivities>().
@@ -206,6 +209,70 @@ public async Task TemporalWorkerService_ExecuteAsync_MultipleWorkers()
                 ["scoped-other2"] = $"tq: {taskQueue2}, counter: 6",
             },
             result);
+    }
+
+    [Fact]
+    public async Task TemporalWorkerService_WorkerClientReplacement_UsesNewClient()
+    {
+        // We are going to start a second ephemeral server and then replace the client. So we will
+        // start a no-cache ticking workflow with the current client and confirm it has accomplished
+        // at least one task. Then we will start another on the other client, and confirm it gets
+        // started too. Then we will terminate both. We have to use a ticking workflow with only one
+        // poller to force a quick re-poll to recognize our client change quickly (as opposed to
+        // just waiting the minute for poll timeout).
+        await using var otherEnv = await Temporalio.Testing.WorkflowEnvironment.StartLocalAsync();
+
+        // Start both workflows on different servers
+        var taskQueue = $"tq-{Guid.NewGuid()}";
+        var handle1 = await Client.StartWorkflowAsync(
+            (WorkflowWorkerTests.TickingWorkflow wf) => wf.RunAsync(),
+            new(id: $"workflow-{Guid.NewGuid()}", taskQueue));
+        var handle2 = await otherEnv.Client.StartWorkflowAsync(
+            (WorkflowWorkerTests.TickingWorkflow wf) => wf.RunAsync(),
+            new(id: $"workflow-{Guid.NewGuid()}", taskQueue));
+
+        var bld = Host.CreateApplicationBuilder();
+
+        WorkerClientUpdater workerClientUpdater = new WorkerClientUpdater();
+
+        // Register the worker client updater.
+        bld.Services.AddSingleton<IWorkerClientUpdater>(workerClientUpdater);
+
+        // Add the first worker with the workflow and client already DI'd, and add the worker client updater.
+        bld.Services.
+            AddSingleton(Client).
+            AddHostedTemporalWorker(taskQueue).
+            AddWorkflow<WorkflowWorkerTests.TickingWorkflow>()
+            .ConfigureOptions()
+            .Configure<IWorkerClientUpdater>((options, updater) =>
+            {
+                options.WorkerClientUpdater = updater;
+                options.MaxCachedWorkflows = 0;
+                options.MaxConcurrentWorkflowTaskPolls = 1;
+            });
+
+        // Start the host
+        using var tokenSource = new CancellationTokenSource();
+        using var host = bld.Build();
+        var hostTask = Task.Run(() => host.RunAsync(tokenSource.Token));
+
+        // Confirm the first ticking workflow has completed a task but not the second workflow
+        await handle1.AssertHasEventEventuallyAsync(e => e.WorkflowTaskCompletedEventAttributes != null);
+        await foreach (var evt in handle2.FetchHistoryEventsAsync())
+        {
+            Assert.Null(evt.WorkflowTaskCompletedEventAttributes);
+        }
+
+        // Now replace the client, which should be used fairly quickly because we should have
+        // timer-done poll completions every 100ms
+        workerClientUpdater.UpdateWorkerClient(otherEnv.Client);
+
+        // Now confirm the other workflow has started
+        await handle2.AssertHasEventEventuallyAsync(e => e.WorkflowTaskCompletedEventAttributes != null);
+
+        // Terminate both
+        await handle1.TerminateAsync();
+        await handle2.TerminateAsync();
     }
 
     [Workflow("Workflow")]