reduce unnecessary logging (#2549)

Author: yiminc

client/history/metricClient.go

@@ -27,6 +27,7 @@ package history
 import (
 	"context"
 
+	"go.temporal.io/api/serviceerror"
 	"google.golang.org/grpc"
 
 	"go.temporal.io/server/api/historyservice/v1"
@@ -627,7 +628,15 @@ func (c *metricClient) finishMetricsRecording(
 	err error,
 ) {
 	if err != nil {
-		c.throttledLogger.Error("history client encountered error", tag.Error(err), tag.ErrorType(err))
+		switch err.(type) {
+		case *serviceerror.Canceled,
+			*serviceerror.DeadlineExceeded,
+			*serviceerror.NotFound,
+			*serviceerror.WorkflowExecutionAlreadyStarted:
+			// noop - not interest and too many logs
+		default:
+			c.throttledLogger.Error("history client encountered error", tag.Error(err), tag.ErrorType(err))
+		}
 		scope.Tagged(metrics.ServiceErrorTypeTag(err)).IncCounter(metrics.ClientFailures)
 	}
 	stopwatch.Stop()

client/matching/metricClient.go

@@ -28,6 +28,7 @@ import (
 	"context"
 	"strings"
 
+	"go.temporal.io/api/serviceerror"
 	taskqueuepb "go.temporal.io/api/taskqueue/v1"
 	"google.golang.org/grpc"
 
@@ -256,7 +257,16 @@ func (c *metricClient) finishMetricsRecording(
 	err error,
 ) {
 	if err != nil {
-		c.throttledLogger.Error("matching client encountered error", tag.Error(err), tag.ErrorType(err))
+		switch err.(type) {
+		case *serviceerror.Canceled,
+			*serviceerror.DeadlineExceeded,
+			*serviceerror.NotFound,
+			*serviceerror.WorkflowExecutionAlreadyStarted:
+			// noop - not interest and too many logs
+		default:
+
+			c.throttledLogger.Error("matching client encountered error", tag.Error(err), tag.ErrorType(err))
+		}
 		scope.Tagged(metrics.ServiceErrorTypeTag(err)).IncCounter(metrics.ClientFailures)
 	}
 	stopwatch.Stop()

service/history/replicationTaskProcessor.go

@@ -493,7 +493,7 @@ func (p *ReplicationTaskProcessorImpl) cleanupReplicationTasks() error {
 		return nil
 	}
 
-	p.logger.Info("cleaning up replication task queue", tag.ReadLevel(*minAckedTaskID))
+	p.logger.Debug("cleaning up replication task queue", tag.ReadLevel(*minAckedTaskID))
 	p.metricsClient.Scope(metrics.ReplicationTaskCleanupScope).IncCounter(metrics.ReplicationTaskCleanupCount)
 	p.metricsClient.Scope(
 		metrics.ReplicationTaskFetcherScope,