Add flag to enable execution scanner event Id validator (#4114)

* Add flag to enable execution scanner event id validator

Author: yux0

common/dynamicconfig/constants.go

@@ -707,6 +707,8 @@ const (
 	ExecutionDataDurationBuffer = "worker.executionDataDurationBuffer"
 	// ExecutionScannerWorkerCount is the execution scavenger worker count
 	ExecutionScannerWorkerCount = "worker.executionScannerWorkerCount"
+	// ExecutionScannerHistoryEventIdValidator is the flag to enable history event id validator
+	ExecutionScannerHistoryEventIdValidator = "worker.executionEnableHistoryEventIdValidator"
 	// TaskQueueScannerEnabled indicates if task queue scanner should be started as part of worker.Scanner
 	TaskQueueScannerEnabled = "worker.taskQueueScannerEnabled"
 	// HistoryScannerEnabled indicates if history scanner should be started as part of worker.Scanner

service/worker/scanner/executions/scavenger.go

@@ -56,16 +56,17 @@ type (
 		numHistoryShards int32
 		activityContext  context.Context
 
-		executionManager            persistence.ExecutionManager
-		registry                    namespace.Registry
-		historyClient               historyservice.HistoryServiceClient
-		adminClient                 adminservice.AdminServiceClient
-		executor                    executor.Executor
-		rateLimiter                 quotas.RateLimiter
-		perShardQPS                 dynamicconfig.IntPropertyFn
-		executionDataDurationBuffer dynamicconfig.DurationPropertyFn
-		metricsHandler              metrics.Handler
-		logger                      log.Logger
+		executionManager              persistence.ExecutionManager
+		registry                      namespace.Registry
+		historyClient                 historyservice.HistoryServiceClient
+		adminClient                   adminservice.AdminServiceClient
+		executor                      executor.Executor
+		rateLimiter                   quotas.RateLimiter
+		perShardQPS                   dynamicconfig.IntPropertyFn
+		executionDataDurationBuffer   dynamicconfig.DurationPropertyFn
+		enableHistoryEventIDValidator dynamicconfig.BoolPropertyFn
+		metricsHandler                metrics.Handler
+		logger                        log.Logger
 
 		stopC  chan struct{}
 		stopWG sync.WaitGroup
@@ -89,6 +90,7 @@ func NewScavenger(
 	perShardQPS dynamicconfig.IntPropertyFn,
 	executionDataDurationBuffer dynamicconfig.DurationPropertyFn,
 	executionTaskWorker dynamicconfig.IntPropertyFn,
+	enableHistoryEventIDValidator dynamicconfig.BoolPropertyFn,
 	executionManager persistence.ExecutionManager,
 	registry namespace.Registry,
 	historyClient historyservice.HistoryServiceClient,
@@ -112,10 +114,11 @@ func NewScavenger(
 		rateLimiter: quotas.NewDefaultOutgoingRateLimiter(
 			func() float64 { return float64(perHostQPS()) },
 		),
-		perShardQPS:                 perShardQPS,
-		executionDataDurationBuffer: executionDataDurationBuffer,
-		metricsHandler:              metricsHandler.WithTags(metrics.OperationTag(metrics.ExecutionsScavengerScope)),
-		logger:                      logger,
+		perShardQPS:                   perShardQPS,
+		executionDataDurationBuffer:   executionDataDurationBuffer,
+		enableHistoryEventIDValidator: enableHistoryEventIDValidator,
+		metricsHandler:                metricsHandler.WithTags(metrics.OperationTag(metrics.ExecutionsScavengerScope)),
+		logger:                        logger,
 
 		stopC: make(chan struct{}),
 	}
@@ -185,6 +188,7 @@ func (s *Scavenger) run() {
 				s.rateLimiter,
 			}),
 			s.executionDataDurationBuffer,
+			s.enableHistoryEventIDValidator,
 		))
 		if !submitted {
 			s.logger.Error("unable to submit task to executor", tag.ShardID(shardID))

service/worker/scanner/executions/task.go

@@ -66,10 +66,11 @@ type (
 		logger           log.Logger
 		scavenger        *Scavenger
 
-		ctx                         context.Context
-		rateLimiter                 quotas.RateLimiter
-		executionDataDurationBuffer dynamicconfig.DurationPropertyFn
-		paginationToken             []byte
+		ctx                           context.Context
+		rateLimiter                   quotas.RateLimiter
+		executionDataDurationBuffer   dynamicconfig.DurationPropertyFn
+		enableHistoryEventIDValidator dynamicconfig.BoolPropertyFn
+		paginationToken               []byte
 	}
 )
 
@@ -86,6 +87,7 @@ func newTask(
 	scavenger *Scavenger,
 	rateLimiter quotas.RateLimiter,
 	executionDataDurationBuffer dynamicconfig.DurationPropertyFn,
+	enableHistoryEventIDValidator dynamicconfig.BoolPropertyFn,
 ) executor.Task {
 	return &task{
 		shardID:          shardID,
@@ -98,9 +100,10 @@ func newTask(
 		logger:         logger,
 		scavenger:      scavenger,
 
-		ctx:                         ctx,
-		rateLimiter:                 rateLimiter,
-		executionDataDurationBuffer: executionDataDurationBuffer,
+		ctx:                           ctx,
+		rateLimiter:                   rateLimiter,
+		executionDataDurationBuffer:   executionDataDurationBuffer,
+		enableHistoryEventIDValidator: enableHistoryEventIDValidator,
 	}
 }
 
@@ -186,19 +189,21 @@ func (t *task) validate(
 		return results
 	}
 
-	if validationResults, err := NewHistoryEventIDValidator(
-		t.shardID,
-		t.executionManager,
-	).Validate(t.ctx, mutableState); err != nil {
-		t.logger.Error("unable to validate history event ID being contiguous",
-			tag.ShardID(t.shardID),
-			tag.WorkflowNamespaceID(mutableState.GetExecutionInfo().GetNamespaceId()),
-			tag.WorkflowID(mutableState.GetExecutionInfo().GetWorkflowId()),
-			tag.WorkflowRunID(mutableState.GetExecutionState().GetRunId()),
-			tag.Error(err),
-		)
-	} else {
-		results = append(results, validationResults...)
+	if t.enableHistoryEventIDValidator() {
+		if validationResults, err := NewHistoryEventIDValidator(
+			t.shardID,
+			t.executionManager,
+		).Validate(t.ctx, mutableState); err != nil {
+			t.logger.Error("unable to validate history event ID being contiguous",
+				tag.ShardID(t.shardID),
+				tag.WorkflowNamespaceID(mutableState.GetExecutionInfo().GetNamespaceId()),
+				tag.WorkflowID(mutableState.GetExecutionInfo().GetWorkflowId()),
+				tag.WorkflowRunID(mutableState.GetExecutionState().GetRunId()),
+				tag.Error(err),
+			)
+		} else {
+			results = append(results, validationResults...)
+		}
 	}
 
 	return results

service/worker/scanner/scanner.go

@@ -82,6 +82,8 @@ type (
 		ExecutionDataDurationBuffer dynamicconfig.DurationPropertyFn
 		// ExecutionScannerWorkerCount is the execution scavenger task worker number
 		ExecutionScannerWorkerCount dynamicconfig.IntPropertyFn
+		// ExecutionScannerHistoryEventIdValidator indicates if the execution scavenger to validate history event id.
+		ExecutionScannerHistoryEventIdValidator dynamicconfig.BoolPropertyFn
 	}
 
 	// scannerContext is the context object that get's

service/worker/scanner/workflow.go

@@ -196,6 +196,7 @@ func ExecutionsScavengerActivity(
 		ctx.cfg.ExecutionScannerPerShardQPS,
 		ctx.cfg.ExecutionDataDurationBuffer,
 		ctx.cfg.ExecutionScannerWorkerCount,
+		ctx.cfg.ExecutionScannerHistoryEventIdValidator,
 		ctx.executionManager,
 		ctx.namespaceRegistry,
 		ctx.historyClient,

service/worker/service.go

@@ -302,6 +302,10 @@ func NewConfig(dc *dynamicconfig.Collection, persistenceConfig *config.Persisten
 				dynamicconfig.ExecutionScannerWorkerCount,
 				8,
 			),
+			ExecutionScannerHistoryEventIdValidator: dc.GetBoolProperty(
+				dynamicconfig.ExecutionScannerHistoryEventIdValidator,
+				true,
+			),
 		},
 		EnableBatcher:      dc.GetBoolProperty(dynamicconfig.EnableBatcher, true),
 		BatcherRPS:         dc.GetIntPropertyFilteredByNamespace(dynamicconfig.BatcherRPS, batcher.DefaultRPS),