eval status: enrich with related evals and placed allocs tables (#26156)

tgross · web-flow · commit aa3c08d069e9 · 2025-06-30T09:23:36.000-04:00
When debugging an evaluation, you almost always want to know about all the related evaluations and what allocations were placed by that evaluation (and where), not just failed placements. We can enrich the command by adding the `related` query parameter to the API, and having the command query for the evaluations allocations automatically. Emit this data as a pair of new tables and expose fields like quota limits, and previous/next/blocked eval without the `-verbose` flag. Update the docs to include the full output and remove references to long-removed behavior of the `-json` flag. Ref: https://hashicorp.atlassian.net/browse/NMD-818 Ref: https://go.hashi.co/rfc/nmd-212
diff --git a/.changelog/26156.txt b/.changelog/26156.txt
@@ -0,0 +1,7 @@
+```release-note:improvement
+cli: Added related evals and placed allocations tables to the eval status command, and exposed more fields without requiring the `-verbose` flag.
+```
+
+```release-note:improvement
+api: The `Evaluations.Info` method of the Go API now populates the `RelatedEvals` field.
+```
diff --git a/api/api_test.go b/api/api_test.go
@@ -161,7 +161,7 @@ func TestSetQueryOptions(t *testing.T) {
 	c, s := makeClient(t, nil, nil)
 	defer s.Stop()
 
-	r, _ := c.newRequest("GET", "/v1/jobs")
+	r, _ := c.newRequest("GET", "/v1/jobs?format=baz")
 	q := &QueryOptions{
 		Region:     "foo",
 		Namespace:  "bar",
@@ -188,6 +188,7 @@ func TestSetQueryOptions(t *testing.T) {
 	try("index", "1000")
 	try("wait", "100000ms")
 	try("reverse", "true")
+	try("format", "baz")
 }
 
 func TestQueryOptionsContext(t *testing.T) {
diff --git a/api/evaluations.go b/api/evaluations.go
@@ -46,7 +46,7 @@ func (e *Evaluations) Count(q *QueryOptions) (*EvalCountResponse, *QueryMeta, er
 // Info is used to query a single evaluation by its ID.
 func (e *Evaluations) Info(evalID string, q *QueryOptions) (*Evaluation, *QueryMeta, error) {
 	var resp Evaluation
-	qm, err := e.client.query("/v1/evaluation/"+evalID, &resp, q)
+	qm, err := e.client.query("/v1/evaluation/"+evalID+"?related=true", &resp, q)
 	if err != nil {
 		return nil, nil, err
 	}
diff --git a/command/eval_status.go b/command/eval_status.go
@@ -36,13 +36,15 @@ Eval Status Options:
     Monitor an outstanding evaluation
 
   -verbose
-    Show full information.
+    Show full-length IDs and exact timestamps.
 
   -json
-    Output the evaluation in its JSON format.
+    Output the evaluation in its JSON format. This format will not include
+    placed allocations.
 
   -t
-    Format and display evaluation using a Go template.
+    Format and display evaluation using a Go template. This format will not
+    include placed allocations.
 
   -ui
     Open the evaluation in the browser.
@@ -73,10 +75,6 @@ func (c *EvalStatusCommand) AutocompleteArgs() complete.Predictor {
 			return nil
 		}
 
-		if err != nil {
-			return nil
-		}
-
 		resp, _, err := client.Search().PrefixSearch(a.Last, contexts.Evals, nil)
 		if err != nil {
 			return []string{}
@@ -120,12 +118,6 @@ func (c *EvalStatusCommand) Run(args []string) int {
 
 	evalID := args[0]
 
-	// Truncate the id unless full length is requested
-	length := shortId
-	if verbose {
-		length = fullId
-	}
-
 	// Query the allocation info
 	if len(evalID) == 1 {
 		c.Ui.Error("Identifier must contain at least two characters.")
@@ -153,6 +145,12 @@ func (c *EvalStatusCommand) Run(args []string) int {
 		return 1
 	}
 
+	// Truncate the id unless full length is requested
+	length := shortId
+	if verbose {
+		length = fullId
+	}
+
 	// If we are in monitor mode, monitor and exit
 	if monitor {
 		mon := newMonitor(c.Ui, client, length)
@@ -178,6 +176,30 @@ func (c *EvalStatusCommand) Run(args []string) int {
 		return 0
 	}
 
+	placedAllocs, _, err := client.Evaluations().Allocations(eval.ID, nil)
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error querying related allocations: %s", err))
+		return 1
+	}
+
+	c.formatEvalStatus(eval, placedAllocs, verbose, length)
+
+	hint, _ := c.Meta.showUIPath(UIHintContext{
+		Command: "eval status",
+		PathParams: map[string]string{
+			"evalID": eval.ID,
+		},
+		OpenURL: openURL,
+	})
+	if hint != "" {
+		c.Ui.Warn(hint)
+	}
+
+	return 0
+}
+
+func (c *EvalStatusCommand) formatEvalStatus(eval *api.Evaluation, placedAllocs []*api.AllocationListStub, verbose bool, length int) {
+
 	failureString, failures := evalFailureStatus(eval)
 	triggerNoun, triggerSubj := getTriggerDetails(eval)
 	statusDesc := eval.StatusDescription
@@ -220,16 +242,27 @@ func (c *EvalStatusCommand) Run(args []string) int {
 		basic = append(basic,
 			fmt.Sprintf("Wait Until|%s", formatTime(eval.WaitUntil)))
 	}
-
-	if verbose {
-		// NextEval, PreviousEval, BlockedEval
+	if eval.QuotaLimitReached != "" {
 		basic = append(basic,
-			fmt.Sprintf("Previous Eval|%s", eval.PreviousEval),
-			fmt.Sprintf("Next Eval|%s", eval.NextEval),
-			fmt.Sprintf("Blocked Eval|%s", eval.BlockedEval))
+			fmt.Sprintf("Quota Limit Reached|%s", eval.QuotaLimitReached))
 	}
+	basic = append(basic,
+		fmt.Sprintf("Previous Eval|%s", limit(eval.PreviousEval, length)),
+		fmt.Sprintf("Next Eval|%s", limit(eval.NextEval, length)),
+		fmt.Sprintf("Blocked Eval|%s", limit(eval.BlockedEval, length)),
+	)
 	c.Ui.Output(formatKV(basic))
 
+	if len(eval.RelatedEvals) > 0 {
+		c.Ui.Output(c.Colorize().Color("\n[bold]Related Evaluations[reset]"))
+		c.Ui.Output(formatRelatedEvalStubs(eval.RelatedEvals, length))
+	}
+	if len(placedAllocs) > 0 {
+		c.Ui.Output(c.Colorize().Color("\n[bold]Placed Allocations[reset]"))
+		allocsOut := formatAllocListStubs(placedAllocs, false, length)
+		c.Ui.Output(allocsOut)
+	}
+
 	if failures {
 		c.Ui.Output(c.Colorize().Color("\n[bold]Failed Placements[reset]"))
 		sorted := sortedTaskGroupFromMetrics(eval.FailedTGAllocs)
@@ -240,29 +273,18 @@ func (c *EvalStatusCommand) Run(args []string) int {
 			if metrics.CoalescedFailures > 0 {
 				noun += "s"
 			}
-			c.Ui.Output(fmt.Sprintf("Task Group %q (failed to place %d %s):", tg, metrics.CoalescedFailures+1, noun))
+			c.Ui.Output(fmt.Sprintf("Task Group %q (failed to place %d %s):",
+				tg, metrics.CoalescedFailures+1, noun))
 			c.Ui.Output(formatAllocMetrics(metrics, false, "  "))
 			c.Ui.Output("")
 		}
 
 		if eval.BlockedEval != "" {
-			c.Ui.Output(fmt.Sprintf("Evaluation %q waiting for additional capacity to place remainder",
+			c.Ui.Output(fmt.Sprintf(
+				"Evaluation %q waiting for additional capacity to place remainder",
 				limit(eval.BlockedEval, length)))
 		}
 	}
-
-	hint, _ := c.Meta.showUIPath(UIHintContext{
-		Command: "eval status",
-		PathParams: map[string]string{
-			"evalID": eval.ID,
-		},
-		OpenURL: openURL,
-	})
-	if hint != "" {
-		c.Ui.Warn(hint)
-	}
-
-	return 0
 }
 
 func sortedTaskGroupFromMetrics(groups map[string]*api.AllocationMetric) []string {
@@ -284,3 +306,20 @@ func getTriggerDetails(eval *api.Evaluation) (noun, subject string) {
 		return "", ""
 	}
 }
+
+func formatRelatedEvalStubs(evals []*api.EvaluationStub, length int) string {
+	out := make([]string, len(evals)+1)
+	out[0] = "ID|Priority|Triggered By|Node ID|Status|Description"
+	for i, eval := range evals {
+		out[i+1] = fmt.Sprintf("%s|%d|%s|%s|%s|%s",
+			limit(eval.ID, length),
+			eval.Priority,
+			eval.TriggeredBy,
+			limit(eval.NodeID, length),
+			eval.Status,
+			eval.StatusDescription,
+		)
+	}
+
+	return formatList(out)
+}
diff --git a/command/eval_status_test.go b/command/eval_status_test.go
@@ -6,9 +6,13 @@ package command
 import (
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/cli"
+	"github.com/hashicorp/nomad/api"
 	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/helper/pointer"
+	"github.com/hashicorp/nomad/helper/uuid"
 	"github.com/hashicorp/nomad/nomad/mock"
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/posener/complete"
@@ -88,3 +92,119 @@ func TestEvalStatusCommand_AutocompleteArgs(t *testing.T) {
 	must.SliceLen(t, 1, res)
 	must.Eq(t, e.ID, res[0])
 }
+
+func TestEvalStatusCommand_Format(t *testing.T) {
+	now := time.Now().UTC()
+	ui := cli.NewMockUi()
+	cmd := &EvalStatusCommand{Meta: Meta{Ui: ui}}
+
+	eval := &api.Evaluation{
+		ID:                uuid.Generate(),
+		Priority:          50,
+		Type:              api.JobTypeService,
+		TriggeredBy:       structs.EvalTriggerAllocStop,
+		Namespace:         api.DefaultNamespace,
+		JobID:             "example",
+		JobModifyIndex:    0,
+		DeploymentID:      uuid.Generate(),
+		Status:            api.EvalStatusComplete,
+		StatusDescription: "complete",
+		NextEval:          "",
+		PreviousEval:      uuid.Generate(),
+		BlockedEval:       uuid.Generate(),
+		RelatedEvals: []*api.EvaluationStub{{
+			ID:                uuid.Generate(),
+			Priority:          50,
+			Type:              "service",
+			TriggeredBy:       "queued-allocs",
+			Namespace:         api.DefaultNamespace,
+			JobID:             "example",
+			DeploymentID:      "",
+			Status:            "pending",
+			StatusDescription: "",
+			WaitUntil:         time.Time{},
+			NextEval:          "",
+			PreviousEval:      uuid.Generate(),
+			BlockedEval:       "",
+			CreateIndex:       0,
+			ModifyIndex:       0,
+			CreateTime:        0,
+			ModifyTime:        0,
+		}},
+		FailedTGAllocs: map[string]*api.AllocationMetric{"web": {
+			NodesEvaluated:     6,
+			NodesFiltered:      4,
+			NodesInPool:        10,
+			NodesAvailable:     map[string]int{},
+			ClassFiltered:      map[string]int{},
+			ConstraintFiltered: map[string]int{"${attr.kernel.name} = linux": 2},
+			NodesExhausted:     2,
+			ClassExhausted:     map[string]int{},
+			DimensionExhausted: map[string]int{"memory": 2},
+			QuotaExhausted:     []string{},
+			ResourcesExhausted: map[string]*api.Resources{"web": {
+				Cores: pointer.Of(3),
+			}},
+			Scores:            map[string]float64{},
+			AllocationTime:    0,
+			CoalescedFailures: 0,
+			ScoreMetaData:     []*api.NodeScoreMeta{},
+		}},
+		ClassEligibility:     map[string]bool{},
+		EscapedComputedClass: true,
+		QuotaLimitReached:    "",
+		QueuedAllocations:    map[string]int{},
+		SnapshotIndex:        1001,
+		CreateIndex:          999,
+		ModifyIndex:          1003,
+		CreateTime:           now.UnixNano(),
+		ModifyTime:           now.Add(time.Second).UnixNano(),
+	}
+
+	placed := []*api.AllocationListStub{
+		{
+			ID:            uuid.Generate(),
+			NodeID:        uuid.Generate(),
+			TaskGroup:     "web",
+			DesiredStatus: "run",
+			JobVersion:    2,
+			ClientStatus:  "running",
+			CreateTime:    now.Add(-10 * time.Second).UnixNano(),
+			ModifyTime:    now.Add(-2 * time.Second).UnixNano(),
+		},
+		{
+			ID:            uuid.Generate(),
+			NodeID:        uuid.Generate(),
+			TaskGroup:     "web",
+			JobVersion:    2,
+			DesiredStatus: "run",
+			ClientStatus:  "pending",
+			CreateTime:    now.Add(-3 * time.Second).UnixNano(),
+			ModifyTime:    now.Add(-1 * time.Second).UnixNano(),
+		},
+		{
+			ID:            uuid.Generate(),
+			NodeID:        uuid.Generate(),
+			TaskGroup:     "web",
+			JobVersion:    2,
+			DesiredStatus: "run",
+			ClientStatus:  "pending",
+			CreateTime:    now.Add(-4 * time.Second).UnixNano(),
+			ModifyTime:    now.UnixNano(),
+		},
+	}
+
+	cmd.formatEvalStatus(eval, placed, false, shortId)
+	out := ui.OutputWriter.String()
+
+	// there isn't much logic here, so this is just a smoke test
+	must.StrContains(t, out, `
+Failed Placements
+Task Group "web" (failed to place 1 allocation):
+  * Constraint "${attr.kernel.name} = linux": 2 nodes excluded by filter
+  * Resources exhausted on 2 nodes
+  * Dimension "memory" exhausted on 2 nodes`)
+
+	must.StrContains(t, out, `Related Evaluations`)
+	must.StrContains(t, out, `Placed Allocations`)
+}
diff --git a/website/content/docs/commands/eval/status.mdx b/website/content/docs/commands/eval/status.mdx

Original file line number	Diff line number	Diff line change
`@@ -46,7 +46,7 @@ func (e Evaluations) Count(q QueryOptions) (EvalCountResponse, QueryMeta, er`
`46`	`46`	`// Info is used to query a single evaluation by its ID.`
`47`	`47`	`func (e Evaluations) Info(evalID string, q QueryOptions) (Evaluation, QueryMeta, error) {`
`48`	`48`	`var resp Evaluation`
`49`		`- qm, err := e.client.query("/v1/evaluation/"+evalID, &resp, q)`
	`49`	`+ qm, err := e.client.query("/v1/evaluation/"+evalID+"?related=true", &resp, q)`
`50`	`50`	`if err != nil {`
`51`	`51`	`return nil, nil, err`
`52`	`52`	`}`