Summary: Use lib.LegacySummary to avoid breaking the API

.github/workflows/xk6-tests/xk6-test.js

@@ -11,7 +11,7 @@ export let options = {
 
 export function handleSummary(data) {
     return {
-        'summary-results.txt': data.metrics.custom.foos.values.count.toString(),
+        'summary-results.txt': data.metrics.foos.values.count.toString(),
     };
 }
 

internal/cmd/run.go

@@ -192,6 +192,25 @@ func (c *cmdRun) run(cmd *cobra.Command, args []string) (err error) {
 
 	executionState := execScheduler.GetState()
 	if !testRunState.RuntimeOptions.NoSummary.Bool { //nolint:nestif
+		// Despite having the revamped [summary.Summary], we still keep the use of the
+		// [lib.LegacySummary] for multiple backwards compatibility options,
+		// to be deprecated by v1.0 and likely removed or replaced by v2.0:
+		// - the `legacy` summary mode (which keeps the old summary format/display).
+		// - the data structure for custom `handleSummary()` implementations.
+		// - the data structure for the JSON (--summary-export) output.
+		legacySummary := func() *lib.LegacySummary {
+			return &lib.LegacySummary{
+				Metrics:         metricsEngine.ObservedMetrics,
+				RootGroup:       testRunState.GroupSummary.Group(),
+				TestRunDuration: executionState.GetCurrentTestRunDuration(),
+				NoColor:         c.gs.Flags.NoColor,
+				UIState: lib.UIState{
+					IsStdOutTTY: c.gs.Stdout.IsTTY,
+					IsStdErrTTY: c.gs.Stderr.IsTTY,
+				},
+			}
+		}
+
 		sm, err := summary.ValidateMode(testRunState.RuntimeOptions.SummaryMode.String)
 		if err != nil {
 			logger.WithError(err).Warnf(
@@ -207,18 +226,7 @@ func (c *cmdRun) run(cmd *cobra.Command, args []string) (err error) {
 			defer func() {
 				logger.Debug("Generating the end-of-test summary...")
 
-				legacySummary := &lib.LegacySummary{
-					Metrics:         metricsEngine.ObservedMetrics,
-					RootGroup:       testRunState.GroupSummary.Group(),
-					TestRunDuration: executionState.GetCurrentTestRunDuration(),
-					NoColor:         c.gs.Flags.NoColor,
-					UIState: lib.UIState{
-						IsStdOutTTY: c.gs.Stdout.IsTTY,
-						IsStdErrTTY: c.gs.Stderr.IsTTY,
-					},
-				}
-
-				summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, legacySummary, nil)
+				summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, legacySummary(), nil)
 				if hsErr == nil {
 					hsErr = handleSummaryResult(c.gs.FS, c.gs.Stdout, c.gs.Stderr, summaryResult)
 				}
@@ -252,7 +260,7 @@ func (c *cmdRun) run(cmd *cobra.Command, args []string) (err error) {
 				summary.NoColor = c.gs.Flags.NoColor
 				summary.EnableColors = !summary.NoColor && c.gs.Stdout.IsTTY
 
-				summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, nil, summary)
+				summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, legacySummary(), summary)
 				if hsErr == nil {
 					hsErr = handleSummaryResult(c.gs.FS, c.gs.Stdout, c.gs.Stderr, summaryResult)
 				}

internal/cmd/tests/cmd_run_test.go

@@ -329,17 +329,17 @@ func TestMetricsAndThresholds(t *testing.T) {
 	var summary map[string]interface{}
 	require.NoError(t, json.Unmarshal(ts.Stdout.Bytes(), &summary))
 
-	thresholds, ok := summary["thresholds"].(map[string]interface{})
+	metrics, ok := summary["metrics"].(map[string]interface{})
 	require.True(t, ok)
 
-	teardownCounter, ok := thresholds["teardown_counter"].(map[string]interface{})
+	teardownCounter, ok := metrics["teardown_counter"].(map[string]interface{})
 	require.True(t, ok)
 
-	teardownCounterThresholds, ok := teardownCounter["thresholds"].([]interface{})
+	teardownThresholds, ok := teardownCounter["thresholds"].(map[string]interface{})
 	require.True(t, ok)
 
-	expected := []interface{}{map[string]interface{}{"source": "count == 1", "ok": true}}
-	require.Equal(t, expected, teardownCounterThresholds)
+	expected := map[string]interface{}{"count == 1": map[string]interface{}{"ok": true}}
+	require.Equal(t, expected, teardownThresholds)
 }
 
 func TestSSLKEYLOGFILEAbsolute(t *testing.T) {
@@ -2478,3 +2478,197 @@ func TestMultipleSecretSources(t *testing.T) {
 	assert.Contains(t, stderr, `level=info msg="***SECRET_REDACTED***" ***SECRET_REDACTED***=console`)
 	assert.Contains(t, stderr, `level=info msg="trigger exception on wrong key" ***SECRET_REDACTED***=console`)
 }
+
+func TestSummaryExport(t *testing.T) {
+	t.Parallel()
+
+	mainScript := `
+		import { check } from "k6";
+		import { Counter } from 'k6/metrics';
+
+		const customIter = new Counter("custom_iterations");
+
+		export default function () {
+			customIter.add(1);
+			check(true, { "TRUE is TRUE": (r) => r });
+		};
+	`
+
+	assertSummaryExport := func(t *testing.T, fs fsext.Fs) {
+		t.Helper()
+
+		rawSummaryExport, err := fsext.ReadFile(fs, "results.json")
+		require.NoError(t, err)
+
+		var summaryExport map[string]interface{}
+		require.NoError(t, json.Unmarshal(rawSummaryExport, &summaryExport))
+
+		assert.Equal(t, map[string]interface{}{
+			"groups": map[string]interface{}{},
+			"checks": map[string]interface{}{
+				"TRUE is TRUE": map[string]interface{}{
+					"fails":  float64(0),
+					"id":     "1bed1cc5e442054df516f1ca1076ac6a",
+					"name":   "TRUE is TRUE",
+					"passes": float64(1),
+					"path":   "::TRUE is TRUE",
+				},
+			},
+			"name": "",
+			"path": "",
+			"id":   "d41d8cd98f00b204e9800998ecf8427e",
+		}, summaryExport["root_group"])
+
+		metrics := summaryExport["metrics"].(map[string]interface{})
+
+		assert.Equal(t, 1.0, metrics["custom_iterations"].(map[string]interface{})["count"])
+		assert.Equal(t, 1.0, metrics["iterations"].(map[string]interface{})["count"])
+
+		checks := metrics["checks"].(map[string]interface{})
+		assert.Equal(t, 1.0, checks["passes"])
+		assert.Equal(t, 0.0, checks["fails"])
+		assert.Equal(t, 1.0, checks["value"])
+
+		// These metrics are created adhoc for visual end-of-test summary only,
+		// thus they shouldn't be present on the exported summary.
+		assert.NotContains(t, "checks_total", metrics)
+		assert.NotContains(t, "checks_succeeded", metrics)
+		assert.NotContains(t, "checks_failed", metrics)
+	}
+
+	for _, summaryMode := range []string{"compact", "full"} {
+		t.Run(summaryMode, func(t *testing.T) {
+			t.Parallel()
+
+			ts := NewGlobalTestState(t)
+			require.NoError(t, fsext.WriteFile(ts.FS, filepath.Join(ts.Cwd, "script.js"), []byte(mainScript), 0o644))
+
+			ts.CmdArgs = []string{
+				"k6", "run",
+				"--summary-export=results.json",
+				"--summary-mode=" + summaryMode,
+				"script.js",
+			}
+
+			cmd.ExecuteWithGlobalState(ts.GlobalState)
+
+			stdout := ts.Stdout.String()
+			t.Log(stdout)
+
+			assert.Contains(t, stdout, "checks_total.......................: 1")
+			assert.Contains(t, stdout, "checks_succeeded...................: 100.00% 1 out of 1")
+			assert.Contains(t, stdout, "checks_failed......................: 0.00%   0 out of 1")
+
+			assert.Contains(t, stdout, `CUSTOM
+    custom_iterations......................: 1`)
+			assert.Contains(t, stdout, "iterations.............................: 1")
+
+			assertSummaryExport(t, ts.FS)
+		})
+	}
+
+	t.Run("legacy", func(t *testing.T) {
+		t.Parallel()
+
+		ts := NewGlobalTestState(t)
+		require.NoError(t, fsext.WriteFile(ts.FS, filepath.Join(ts.Cwd, "script.js"), []byte(mainScript), 0o644))
+
+		ts.CmdArgs = []string{
+			"k6", "run",
+			"--summary-export=results.json",
+			"--summary-mode=legacy",
+			"script.js",
+		}
+
+		cmd.ExecuteWithGlobalState(ts.GlobalState)
+
+		stdout := ts.Stdout.String()
+		t.Log(stdout)
+
+		assert.Contains(t, stdout, "✓ TRUE is TRUE")
+		assert.Contains(t, stdout, "checks...............: 100.00% 1 out of 1")
+		assert.Contains(t, stdout, "custom_iterations....: 1")
+		assert.Contains(t, stdout, "iterations...........: 1")
+
+		assertSummaryExport(t, ts.FS)
+	})
+}
+
+func TestHandleSummary(t *testing.T) {
+	t.Parallel()
+	mainScript := `
+		import { check } from "k6";
+		import { Counter } from 'k6/metrics';
+
+		const customIter = new Counter("custom_iterations");
+
+		export default function () {
+			customIter.add(1);
+			check(true, { "TRUE is TRUE": (r) => r });
+		};
+
+		export function handleSummary(data) {
+	    return {
+		  'summary.json': JSON.stringify(data), //the default data object
+	    };
+	}
+	`
+
+	for _, summaryMode := range []string{"compact", "full", "legacy"} {
+		t.Run(summaryMode, func(t *testing.T) {
+			t.Parallel()
+
+			ts := NewGlobalTestState(t)
+			require.NoError(t, fsext.WriteFile(ts.FS, filepath.Join(ts.Cwd, "script.js"), []byte(mainScript), 0o644))
+
+			ts.CmdArgs = []string{"k6", "run", "script.js"}
+
+			cmd.ExecuteWithGlobalState(ts.GlobalState)
+
+			stdout := ts.Stdout.String()
+			t.Log(stdout)
+
+			rawSummaryExport, err := fsext.ReadFile(ts.FS, "summary.json")
+			require.NoError(t, err)
+
+			var summaryExport map[string]interface{}
+			require.NoError(t, json.Unmarshal(rawSummaryExport, &summaryExport))
+
+			assert.Equal(t, map[string]interface{}{
+				"groups": []interface{}{},
+				"checks": []interface{}{
+					map[string]interface{}{
+						"fails":  float64(0),
+						"id":     "1bed1cc5e442054df516f1ca1076ac6a",
+						"name":   "TRUE is TRUE",
+						"passes": float64(1),
+						"path":   "::TRUE is TRUE",
+					},
+				},
+				"name": "",
+				"path": "",
+				"id":   "d41d8cd98f00b204e9800998ecf8427e",
+			}, summaryExport["root_group"])
+
+			metrics := summaryExport["metrics"].(map[string]interface{})
+
+			assert.Equal(t, 1.0,
+				metrics["custom_iterations"].(map[string]interface{})["values"].(map[string]interface{})["count"],
+			)
+			assert.Equal(t, 1.0,
+				metrics["iterations"].(map[string]interface{})["values"].(map[string]interface{})["count"],
+			)
+
+			checks := metrics["checks"].(map[string]interface{})["values"].(map[string]interface{})
+			assert.Equal(t, 1.0, checks["rate"])
+			assert.Equal(t, 1.0, checks["passes"])
+			assert.Equal(t, 0.0, checks["fails"])
+
+			// These metrics are created adhoc for visual end-of-test summary only,
+			// thus they shouldn't be present on the custom `handleSummary()` data structure.
+			assert.NotContains(t, "checks_total", metrics)
+			assert.NotContains(t, "checks_succeeded", metrics)
+			assert.NotContains(t, "checks_failed", metrics)
+		})
+	}
+}

internal/js/runner.go

@@ -376,9 +376,9 @@ func (r *Runner) HandleSummary(
 	})
 	vu.moduleVUImpl.ctx = summaryCtx
 
-	noColor, enableColors, summaryDataForJS, summaryCode := prepareHandleSummaryCall(r, legacy, summary)
+	noColor, enableColors, legacyData, summaryData, summaryCode := prepareHandleSummaryCall(r, legacy, summary)
 
-	handleSummaryDataAsValue := vu.Runtime.ToValue(summaryDataForJS)
+	handleSummaryDataAsValue := vu.Runtime.ToValue(legacyData)
 	callbackResult, err := runUserProvidedHandleSummaryCallback(summaryCtx, vu, handleSummaryDataAsValue)
 	if err != nil {
 		return nil, err
@@ -394,7 +394,8 @@ func (r *Runner) HandleSummary(
 		return nil, fmt.Errorf("unexpected error did not get a callable summary wrapper")
 	}
 
-	wrapperArgs := prepareHandleWrapperArgs(vu, noColor, enableColors, callbackResult, handleSummaryDataAsValue)
+	wrapperArgs := prepareHandleWrapperArgs(
+		vu, noColor, enableColors, callbackResult, handleSummaryDataAsValue, vu.Runtime.ToValue(summaryData))
 	rawResult, _, _, err := vu.runFn(summaryCtx, false, handleSummaryWrapper, nil, wrapperArgs...)
 
 	if deadlineError := r.checkDeadline(summaryCtx, consts.HandleSummaryFn, rawResult, err); deadlineError != nil {
@@ -412,30 +413,29 @@ func prepareHandleSummaryCall(
 	r *Runner,
 	legacy *lib.LegacySummary,
 	summary *summary.Summary,
-) (bool, bool, interface{}, string) {
+) (bool, bool, interface{}, interface{}, string) {
 	var (
 		noColor          bool
 		enableColors     bool
+		legacyDataForJS  interface{}
 		summaryDataForJS interface{}
 		summaryCode      string
 	)
-
-	// TODO: Remove this code block once we stop supporting the legacy summary.
-	if legacy != nil {
-		noColor = legacy.NoColor
-		enableColors = !legacy.NoColor && legacy.UIState.IsStdOutTTY
-		summaryDataForJS = summarizeMetricsToObject(legacy, r.Bundle.Options, r.setupData)
-		summaryCode = jslibSummaryLegacyCode
-	}
-
 	if summary != nil {
 		noColor = summary.NoColor
 		enableColors = summary.EnableColors
+		legacyDataForJS = summarizeMetricsToObject(legacy, r.Bundle.Options, r.setupData)
 		summaryDataForJS = summary
 		summaryCode = jslibSummaryCode
+	} else { // TODO: Remove this code block once we stop supporting the legacy summary.
+		noColor = legacy.NoColor
+		enableColors = !legacy.NoColor && legacy.UIState.IsStdOutTTY
+		legacyDataForJS = summarizeMetricsToObject(legacy, r.Bundle.Options, r.setupData)
+		summaryDataForJS = legacyDataForJS
+		summaryCode = jslibSummaryLegacyCode
 	}
 
-	return noColor, enableColors, summaryDataForJS, summaryCode
+	return noColor, enableColors, legacyDataForJS, summaryDataForJS, summaryCode
 }
 
 func runUserProvidedHandleSummaryCallback(
@@ -467,8 +467,7 @@ func runUserProvidedHandleSummaryCallback(
 func prepareHandleWrapperArgs(
 	vu *VU,
 	noColor bool, enableColors bool,
-	callbackResult sobek.Value,
-	summaryDataForJS interface{},
+	callbackResult, handleSummaryDataAsValue, summaryDataAsValue sobek.Value,
 ) []sobek.Value {
 	options := map[string]interface{}{
 		// TODO: improve when we can easily export all option values, including defaults?
@@ -481,7 +480,8 @@ func prepareHandleWrapperArgs(
 	wrapperArgs := []sobek.Value{
 		callbackResult,
 		vu.Runtime.ToValue(vu.Runner.Bundle.preInitState.RuntimeOptions.SummaryExport.String),
-		vu.Runtime.ToValue(summaryDataForJS),
+		handleSummaryDataAsValue,
+		summaryDataAsValue,
 		vu.Runtime.ToValue(options),
 	}
 

internal/js/summary-wrapper.js

@@ -60,7 +60,7 @@
         return JSON.stringify(results, null, 4);
     };
 
-    return function (summaryCallbackResult, jsonSummaryPath, data, options) {
+    return function (summaryCallbackResult, jsonSummaryPath, legacyData, data, options) {
         let result = summaryCallbackResult;
         if (!result) {
             result = {
@@ -72,7 +72,7 @@
         // and if not, log an error and generate the default summary?
 
         if (jsonSummaryPath != '') {
-            result[jsonSummaryPath] = oldJSONSummary(data);
+            result[jsonSummaryPath] = oldJSONSummary(legacyData);
         }
 
         return result;