Allow configurating history cache non-user context lock timeout (#4645)

Author: yycptt

common/dynamicconfig/constants.go

@@ -475,6 +475,12 @@ const (
 	HistoryCacheMaxSize = "history.cacheMaxSize"
 	// HistoryCacheTTL is TTL of history cache
 	HistoryCacheTTL = "history.cacheTTL"
+	// HistoryCacheNonUserContextLockTimeout controls how long non-user call (callerType != API or Operator)
+	// will wait on workflow lock acquisition. Requires service restart to take effect.
+	HistoryCacheNonUserContextLockTimeout = "history.cacheNonUserContextLockTimeout"
+	// HistoryStartupMembershipJoinDelay is the duration a history instance waits
+	// before joining membership after starting.
+	HistoryStartupMembershipJoinDelay = "history.startupMembershipJoinDelay"
 	// HistoryShutdownDrainDuration is the duration of traffic drain during shutdown
 	HistoryShutdownDrainDuration = "history.shutdownDrainDuration"
 	// EventsCacheInitialSize is initial size of events cache

service/history/configs/config.go

@@ -65,9 +65,10 @@ type Config struct {
 
 	// HistoryCache settings
 	// Change of these configs require shard restart
-	HistoryCacheInitialSize dynamicconfig.IntPropertyFn
-	HistoryCacheMaxSize     dynamicconfig.IntPropertyFn
-	HistoryCacheTTL         dynamicconfig.DurationPropertyFn
+	HistoryCacheInitialSize               dynamicconfig.IntPropertyFn
+	HistoryCacheMaxSize                   dynamicconfig.IntPropertyFn
+	HistoryCacheTTL                       dynamicconfig.DurationPropertyFn
+	HistoryCacheNonUserContextLockTimeout dynamicconfig.DurationPropertyFn
 
 	// EventsCache settings
 	// Change of these configs require shard restart
@@ -345,13 +346,16 @@ func NewConfig(
 		VisibilityDisableOrderByClause:    dc.GetBoolPropertyFnWithNamespaceFilter(dynamicconfig.VisibilityDisableOrderByClause, true),
 		VisibilityEnableManualPagination:  dc.GetBoolPropertyFnWithNamespaceFilter(dynamicconfig.VisibilityEnableManualPagination, true),
 
-		EmitShardLagLog:                      dc.GetBoolProperty(dynamicconfig.EmitShardLagLog, false),
-		HistoryCacheInitialSize:              dc.GetIntProperty(dynamicconfig.HistoryCacheInitialSize, 128),
-		HistoryCacheMaxSize:                  dc.GetIntProperty(dynamicconfig.HistoryCacheMaxSize, 512),
-		HistoryCacheTTL:                      dc.GetDurationProperty(dynamicconfig.HistoryCacheTTL, time.Hour),
-		EventsCacheInitialSize:               dc.GetIntProperty(dynamicconfig.EventsCacheInitialSize, 128*1024), // 128KB
-		EventsCacheMaxSize:                   dc.GetIntProperty(dynamicconfig.EventsCacheMaxSize, 512*1024),     // 512KB
-		EventsCacheTTL:                       dc.GetDurationProperty(dynamicconfig.EventsCacheTTL, time.Hour),
+		EmitShardLagLog:                       dc.GetBoolProperty(dynamicconfig.EmitShardLagLog, false),
+		HistoryCacheInitialSize:               dc.GetIntProperty(dynamicconfig.HistoryCacheInitialSize, 128),
+		HistoryCacheMaxSize:                   dc.GetIntProperty(dynamicconfig.HistoryCacheMaxSize, 512),
+		HistoryCacheTTL:                       dc.GetDurationProperty(dynamicconfig.HistoryCacheTTL, time.Hour),
+		HistoryCacheNonUserContextLockTimeout: dc.GetDurationProperty(dynamicconfig.HistoryCacheNonUserContextLockTimeout, 500*time.Millisecond),
+
+		EventsCacheInitialSize: dc.GetIntProperty(dynamicconfig.EventsCacheInitialSize, 128*1024), // 128KB
+		EventsCacheMaxSize:     dc.GetIntProperty(dynamicconfig.EventsCacheMaxSize, 512*1024),     // 512KB
+		EventsCacheTTL:         dc.GetDurationProperty(dynamicconfig.EventsCacheTTL, time.Hour),
+
 		RangeSizeBits:                        20, // 20 bits for sequencer, 2^20 sequence number for any range
 		AcquireShardInterval:                 dc.GetDurationProperty(dynamicconfig.AcquireShardInterval, time.Minute),
 		AcquireShardConcurrency:              dc.GetIntProperty(dynamicconfig.AcquireShardConcurrency, 10),

service/history/workflow/cache/cache.go

@@ -44,7 +44,6 @@ import (
 	"go.temporal.io/server/common/metrics"
 	"go.temporal.io/server/common/namespace"
 	"go.temporal.io/server/common/persistence"
-	"go.temporal.io/server/service/history/configs"
 	"go.temporal.io/server/service/history/consts"
 	"go.temporal.io/server/service/history/shard"
 	"go.temporal.io/server/service/history/workflow"
@@ -78,7 +77,8 @@ type (
 		shard          shard.Context
 		logger         log.Logger
 		metricsHandler metrics.Handler
-		config         *configs.Config
+
+		nonUserContextLockTimeout time.Duration
 	}
 
 	NewCacheFn func(shard shard.Context) Cache
@@ -87,10 +87,12 @@ type (
 var NoopReleaseFn ReleaseCacheFunc = func(err error) {}
 
 const (
-	cacheNotReleased            int32 = 0
-	cacheReleased               int32 = 1
-	workflowLockTimeoutTailTime       = 500 * time.Millisecond
-	nonApiContextLockTimeout          = 500 * time.Millisecond
+	cacheNotReleased int32 = 0
+	cacheReleased    int32 = 1
+)
+
+const (
+	workflowLockTimeoutTailTime = 500 * time.Millisecond
 )
 
 func NewCache(shard shard.Context) Cache {
@@ -101,11 +103,11 @@ func NewCache(shard shard.Context) Cache {
 	opts.Pin = true
 
 	return &CacheImpl{
-		Cache:          cache.New(config.HistoryCacheMaxSize(), opts),
-		shard:          shard,
-		logger:         log.With(shard.GetLogger(), tag.ComponentHistoryCache),
-		metricsHandler: shard.GetMetricsHandler().WithTags(metrics.CacheTypeTag(metrics.MutableStateCacheTypeTagValue)),
-		config:         config,
+		Cache:                     cache.New(config.HistoryCacheMaxSize(), opts),
+		shard:                     shard,
+		logger:                    log.With(shard.GetLogger(), tag.ComponentHistoryCache),
+		metricsHandler:            shard.GetMetricsHandler().WithTags(metrics.CacheTypeTag(metrics.MutableStateCacheTypeTagValue)),
+		nonUserContextLockTimeout: config.HistoryCacheNonUserContextLockTimeout(),
 	}
 }
 
@@ -220,7 +222,7 @@ func (c *CacheImpl) lockWorkflowExecution(ctx context.Context,
 	if deadline, ok := ctx.Deadline(); ok {
 		var cancel context.CancelFunc
 		if headers.GetCallerInfo(ctx).CallerType != headers.CallerTypeAPI {
-			newDeadline := time.Now().Add(nonApiContextLockTimeout)
+			newDeadline := time.Now().Add(c.nonUserContextLockTimeout)
 			if newDeadline.Before(deadline) {
 				ctx, cancel = context.WithDeadline(ctx, newDeadline)
 				defer cancel()