Do not use Unvailable as error type for expected error cases (#4650)

<!-- Describe what has changed in this PR -->
**What changed?**
Do not use Unavailable error type for expected error cases.

<!-- Tell your future self why have you made these changes -->
**Why?**
To correctly reflect the error case.

<!-- How have you verified this change? Tested locally? Added a unit
test? Checked in staging env? -->
**How did you test it?**
eye_balls

<!-- Assuming the worst case, what can be broken when deploying this
change to production? -->
**Potential risks**
No

<!-- Is this PR a hotfix candidate or require that a notification be
sent to the broader community? (Yes/No) -->
**Is hotfix candidate?**
No

Author: yiminc

service/history/consts/const.go

@@ -70,13 +70,13 @@ var (
 	// ErrSignalsLimitExceeded is the error indicating limit reached for maximum number of signal events
 	ErrSignalsLimitExceeded = serviceerror.NewInvalidArgument("exceeded workflow execution limit for signal events")
 	// ErrWorkflowClosing is the error indicating requests to workflow got rejected due to workflow is closing
-	ErrWorkflowClosing = serviceerror.NewUnavailable("workflow operation rejected because workflow is closing")
+	ErrWorkflowClosing = serviceerror.NewWorkflowNotReady("workflow operation rejected because workflow is closing")
 	// ErrEventsAterWorkflowFinish is the error indicating server error trying to write events after workflow finish event
 	ErrEventsAterWorkflowFinish = serviceerror.NewInternal("error validating last event being workflow finish event")
 	// ErrQueryEnteredInvalidState is error indicating query entered invalid state
 	ErrQueryEnteredInvalidState = serviceerror.NewInvalidArgument("query entered invalid state, this should be impossible")
 	// ErrConsistentQueryBufferExceeded is error indicating that too many consistent queries have been buffered and until buffered queries are finished new consistent queries cannot be buffered
-	ErrConsistentQueryBufferExceeded = serviceerror.NewUnavailable("consistent query buffer is full, cannot accept new consistent queries")
+	ErrConsistentQueryBufferExceeded = serviceerror.NewWorkflowNotReady("consistent query buffer is full, this may be caused by too many queries and workflow not able to process query fast enough")
 	// ErrEmptyHistoryRawEventBatch indicate that one single batch of history raw events is of size 0
 	ErrEmptyHistoryRawEventBatch = serviceerror.NewInvalidArgument("encountered empty history batch")
 	// ErrHistorySizeExceedsLimit is error indicating workflow execution has exceeded system defined history size limit

tests/signal_workflow_test.go

@@ -40,6 +40,7 @@ import (
 	"go.temporal.io/api/serviceerror"
 	taskqueuepb "go.temporal.io/api/taskqueue/v1"
 	"go.temporal.io/api/workflowservice/v1"
+	"go.temporal.io/server/service/history/consts"
 
 	"go.temporal.io/server/common/convert"
 	"go.temporal.io/server/common/log/tag"
@@ -853,7 +854,7 @@ func (s *integrationSuite) TestSignalWorkflow_WorkflowCloseAttempted() {
 				RequestId:  uuid.New(),
 			})
 			s.Error(err)
-			s.IsType(&serviceerror.Unavailable{}, err)
+			s.Error(consts.ErrWorkflowClosing, err)
 		}
 
 		attemptCount++