feat: Add retry backoff to grpc connections (#1437)

marcozabel · web-flow · commit 4afe11acef56 · 2026-01-08T15:38:58.000-05:00
Signed-off-by: marcozabel &lt;marco.zabel@dynatrace.com&gt;
diff --git a/libs/providers/flagd/README.md b/libs/providers/flagd/README.md
@@ -28,7 +28,7 @@ Options can be defined in the constructor or as environment variables. Construct
 ### Available Configuration Options
 
 | Option name                            | Environment variable name      | Type    | Default                                                        | Supported values |
-| -------------------------------------- | ------------------------------ | ------- |----------------------------------------------------------------| ---------------- |
+| -------------------------------------- | ------------------------------ | ------- |----------------------------------------------------------------|------------------|
 | host                                   | FLAGD_HOST                     | string  | localhost                                                      |                  |
 | port                                   | FLAGD_PORT                     | number  | [resolver specific defaults](#resolver-type-specific-defaults) |                  |
 | tls                                    | FLAGD_TLS                      | boolean | false                                                          |                  |
@@ -40,7 +40,9 @@ Options can be defined in the constructor or as environment variables. Construct
 | cache                                  | FLAGD_CACHE                    | string  | lru                                                            | lru, disabled    |
 | maxCacheSize                           | FLAGD_MAX_CACHE_SIZE           | int     | 1000                                                           |                  |
 | defaultAuthority                       | FLAGD_DEFAULT_AUTHORITY        | string  | -                                                              | rpc, in-process  |
-| keepAliveTime                            | FLAGD_KEEP_ALIVE_TIME_MS       | number  | 0                                                              | rpc, in-process  |
+| keepAliveTime                          | FLAGD_KEEP_ALIVE_TIME_MS       | number  | 0                                                              | rpc, in-process  |
+| retryBackoffMs                         | FLAGD_RETRY_BACKOFF_MS         | int     | 1000                                                           | in-process       |
+| retryBackoffMaxMs                      | FLAGD_RETRY_BACKOFF_MAX_MS     | int     | 120000                                                         | in-process       |
 
 #### Resolver type-specific Defaults
 
diff --git a/libs/providers/flagd/src/lib/configuration.spec.ts b/libs/providers/flagd/src/lib/configuration.spec.ts
@@ -23,6 +23,8 @@ describe('Configuration', () => {
       maxCacheSize: DEFAULT_MAX_CACHE_SIZE,
       cache: 'lru',
       resolverType: 'rpc',
+      retryBackoffMaxMs: 120000,
+      retryBackoffMs: 1000,
       selector: '',
       deadlineMs: 500,
       contextEnricher: expect.any(Function),
@@ -118,6 +120,8 @@ describe('Configuration', () => {
       maxCacheSize: 1000,
       cache: 'lru',
       resolverType: 'rpc',
+      retryBackoffMaxMs: 120000,
+      retryBackoffMs: 1000,
       selector: '',
       defaultAuthority: '',
       deadlineMs: 500,
diff --git a/libs/providers/flagd/src/lib/configuration.ts b/libs/providers/flagd/src/lib/configuration.ts
@@ -90,6 +90,16 @@ export interface Config {
    */
   defaultAuthority?: string;
 
+  /**
+   * Initial retry backoff in milliseconds.
+   */
+  retryBackoffMs?: number;
+
+  /**
+   * Maximum retry backoff in milliseconds.
+   */
+  retryBackoffMaxMs?: number;
+
   /**
    * gRPC client KeepAlive in milliseconds. Disabled with 0.
    * Only applies to RPC and in-process resolvers.
@@ -131,6 +141,8 @@ const DEFAULT_CONFIG: Omit<FlagdConfig, 'port' | 'resolverType'> = {
   cache: 'lru',
   maxCacheSize: DEFAULT_MAX_CACHE_SIZE,
   contextEnricher: (syncContext: EvaluationContext | null) => syncContext ?? {},
+  retryBackoffMs: 1000,
+  retryBackoffMaxMs: 120000,
   keepAliveTime: 0,
   retryGracePeriod: DEFAULT_RETRY_GRACE_PERIOD,
 };
@@ -153,6 +165,8 @@ enum ENV_VAR {
   FLAGD_RESOLVER = 'FLAGD_RESOLVER',
   FLAGD_OFFLINE_FLAG_SOURCE_PATH = 'FLAGD_OFFLINE_FLAG_SOURCE_PATH',
   FLAGD_DEFAULT_AUTHORITY = 'FLAGD_DEFAULT_AUTHORITY',
+  FLAGD_RETRY_BACKOFF_MS = 'FLAGD_RETRY_BACKOFF_MS',
+  FLAGD_RETRY_BACKOFF_MAX_MS = 'FLAGD_RETRY_BACKOFF_MAX_MS',
   FLAGD_KEEP_ALIVE_TIME_MS = 'FLAGD_KEEP_ALIVE_TIME_MS',
   FLAGD_RETRY_GRACE_PERIOD = 'FLAGD_RETRY_GRACE_PERIOD',
 }
@@ -215,6 +229,12 @@ const getEnvVarConfig = (): Partial<Config> => {
     ...(process.env[ENV_VAR.FLAGD_DEFAULT_AUTHORITY] && {
       defaultAuthority: process.env[ENV_VAR.FLAGD_DEFAULT_AUTHORITY],
     }),
+    ...(Number(process.env[ENV_VAR.FLAGD_RETRY_BACKOFF_MS]) && {
+      retryBackoffMs: Number(process.env[ENV_VAR.FLAGD_RETRY_BACKOFF_MS]),
+    }),
+    ...(Number(process.env[ENV_VAR.FLAGD_RETRY_BACKOFF_MAX_MS]) && {
+      retryBackoffMaxMs: Number(process.env[ENV_VAR.FLAGD_RETRY_BACKOFF_MAX_MS]),
+    }),
     ...(Number(process.env[ENV_VAR.FLAGD_KEEP_ALIVE_TIME_MS]) && {
       keepAliveTime: Number(process.env[ENV_VAR.FLAGD_KEEP_ALIVE_TIME_MS]),
     }),
diff --git a/libs/providers/flagd/src/lib/service/common/grpc-util.spec.ts b/libs/providers/flagd/src/lib/service/common/grpc-util.spec.ts
@@ -1,4 +1,4 @@
-import { buildClientOptions } from './grpc-util';
+import { buildClientOptions, buildRetryPolicy } from './grpc-util';
 import type { Config } from '../../configuration';
 
 describe('buildClientOptions', () => {
@@ -8,10 +8,13 @@ describe('buildClientOptions', () => {
     tls: false,
     deadlineMs: 500,
     socketPath: '',
+    retryBackoffMs: 100,
+    retryBackoffMaxMs: 200,
   };
 
-  it('should return undefined when no relevant options are set', () => {
-    expect(buildClientOptions(baseConfig)).toBeUndefined();
+  it('should only return retry policy when no relevant options are set', () => {
+    expect(Object.keys(buildClientOptions(baseConfig)).length).toBe(1);
+    expect(Object.keys(buildClientOptions(baseConfig))).toEqual(['grpc.service_config']);
   });
 
   it.each([
@@ -24,22 +27,23 @@ describe('buildClientOptions', () => {
     { configKey: 'keepAliveTime', value: 10000, grpcKey: 'grpc.keepalive_time_ms', expected: 10000 },
   ])('should include $configKey when set to valid value', ({ configKey, value, grpcKey, expected }) => {
     const config = { ...baseConfig, [configKey]: value };
-    expect(buildClientOptions(config)).toEqual({ [grpcKey]: expected });
+    expect(buildClientOptions(config)).toMatchObject({ [grpcKey]: expected });
   });
 
   it.each([
     { configKey: 'keepAliveTime', value: 0, description: 'zero' },
     { configKey: 'keepAliveTime', value: -1, description: 'negative' },
   ])('should exclude $configKey when $description', ({ configKey, value }) => {
     const config = { ...baseConfig, [configKey]: value };
-    expect(buildClientOptions(config)).toBeUndefined();
+    expect(Object.keys(buildClientOptions(config))).not.toContain('grpc.keepalive_time_ms');
   });
 
   it('should combine multiple options', () => {
     const config: Config = { ...baseConfig, defaultAuthority: 'my-authority', keepAliveTime: 5000 };
     expect(buildClientOptions(config)).toEqual({
       'grpc.default_authority': 'my-authority',
       'grpc.keepalive_time_ms': 5000,
+      'grpc.service_config': buildRetryPolicy('flagd.service.v1.FlagService', 100, 200),
     });
   });
 });
diff --git a/libs/providers/flagd/src/lib/service/common/grpc-util.ts b/libs/providers/flagd/src/lib/service/common/grpc-util.ts
@@ -1,4 +1,4 @@
-import { credentials } from '@grpc/grpc-js';
+import { credentials, status } from '@grpc/grpc-js';
 import type { ClientReadableStream, ChannelCredentials, ClientOptions } from '@grpc/grpc-js';
 import { readFileSync, existsSync } from 'node:fs';
 import type { Config } from '../../configuration';
@@ -48,8 +48,14 @@ const CONFIG_TO_GRPC_OPTIONS: {
 /**
  * Builds gRPC client options from config.
  */
-export function buildClientOptions(config: Config): ClientOptions | undefined {
-  const options: Partial<ClientOptions> = {};
+export function buildClientOptions(config: Config): ClientOptions {
+  const options: Partial<ClientOptions> = {
+    'grpc.service_config': buildRetryPolicy(
+      'flagd.service.v1.FlagService',
+      config.retryBackoffMs,
+      config.retryBackoffMaxMs,
+    ),
+  };
 
   for (const { configKey, grpcKey, condition } of CONFIG_TO_GRPC_OPTIONS) {
     const value = config[configKey];
@@ -58,5 +64,33 @@ export function buildClientOptions(config: Config): ClientOptions | undefined {
     }
   }
 
-  return Object.keys(options).length > 0 ? options : undefined;
+  return options;
 }
+
+/**
+ * Builds RetryPolicy for gRPC client options.
+ * @param serviceName
+ * @param retryBackoffMs Initial backoff duration in milliseconds
+ * @param retryBackoffMaxMs Maximum backoff duration in milliseconds
+ * @returns gRPC client options with retry policy
+ */
+export const buildRetryPolicy = (serviceName: string, retryBackoffMs?: number, retryBackoffMaxMs?: number): string => {
+  const initialBackoff = retryBackoffMs ?? 1000;
+  const maxBackoff = retryBackoffMaxMs ?? 120000;
+
+  return JSON.stringify({
+    loadBalancingConfig: [],
+    methodConfig: [
+      {
+        name: [{ service: serviceName }],
+        retryPolicy: {
+          maxAttempts: 3,
+          initialBackoff: `${Math.round(initialBackoff / 1000).toFixed(2)}s`,
+          maxBackoff: `${Math.round(maxBackoff / 1000).toFixed(2)}s`,
+          backoffMultiplier: 2,
+          retryableStatusCodes: [status.UNAVAILABLE, status.UNKNOWN],
+        },
+      },
+    ],
+  });
+};
