Fail QueryWorkflow if last workflow task failed (#4099)

* Fail QueryWorkflow if last WorkflowTask failed

* fix linter issues

* pr comments

---------

Co-authored-by: Saman Barghi <[email protected]>
Co-authored-by: Yichao Yang <[email protected]>

Author: 2 people

service/history/api/queryworkflow/api.go

@@ -115,6 +115,12 @@ func Invoke(
 		return nil, consts.ErrWorkflowTaskNotScheduled
 	}
 
+	if mutableState.GetExecutionInfo().WorkflowTaskAttempt > 1 {
+		// while workflow task is failing, the query to that workflow will also fail. Failing fast here to prevent wasting
+		// resources to load history for a query that will fail.
+		return nil, serviceerror.NewFailedPrecondition("Cannot query workflow due to Workflow Task in failed state.")
+	}
+
 	// There are two ways in which queries get dispatched to workflow worker. First, queries can be dispatched on workflow tasks.
 	// These workflow tasks potentially contain new events and queries. The events are treated as coming before the query in time.
 	// The second way in which queries are dispatched to workflow worker is directly through matching; in this approach queries can be

tests/query_workflow_test.go

@@ -38,6 +38,7 @@ import (
 
 	"go.temporal.io/server/service/history/consts"
 
+	"go.temporal.io/api/serviceerror"
 	"go.temporal.io/server/common/log/tag"
 )
 
@@ -257,3 +258,41 @@ func (s *clientIntegrationSuite) TestQueryWorkflow_QueryBeforeStart() {
 	// wait query
 	wg.Wait()
 }
+
+func (s *clientIntegrationSuite) TestQueryWorkflow_QueryFailedWorkflowTask() {
+
+	workflowFn := func(ctx workflow.Context) (string, error) {
+		err := workflow.SetQueryHandler(ctx, "test", func() (string, error) {
+			return "", nil
+		})
+
+		if err != nil {
+			s.Logger.Fatal("SetQueryHandler failed: " + err.Error())
+		}
+		// force workflow task to fail
+		panic("Workflow failed")
+	}
+
+	s.worker.RegisterWorkflow(workflowFn)
+
+	id := "test-query-failed-workflow-task"
+	workflowOptions := sdkclient.StartWorkflowOptions{
+		ID:                 id,
+		TaskQueue:          s.taskQueue,
+		WorkflowRunTimeout: 20 * time.Second,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	workflowRun, err := s.sdkClient.ExecuteWorkflow(ctx, workflowOptions, workflowFn)
+	if err != nil {
+		s.Logger.Fatal("Start workflow failed with err", tag.Error(err))
+	}
+
+	s.NotNil(workflowRun)
+	s.True(workflowRun.GetRunID() != "")
+
+	// wait for workflow to fail
+	time.Sleep(time.Second * 5)
+	_, err = s.sdkClient.QueryWorkflow(ctx, id, "", "test")
+	s.IsType(&serviceerror.FailedPrecondition{}, err)
+}