diff --git a/.github/workflows/xk6-tests/xk6-test.js b/.github/workflows/xk6-tests/xk6-test.js index a20de89a00..093bad76cc 100644 --- a/.github/workflows/xk6-tests/xk6-test.js +++ b/.github/workflows/xk6-tests/xk6-test.js @@ -11,7 +11,7 @@ export let options = { export function handleSummary(data) { return { - 'summary-results.txt': data.metrics.custom.foos.values.count.toString(), + 'summary-results.txt': data.metrics.foos.values.count.toString(), }; } diff --git a/internal/cmd/run.go b/internal/cmd/run.go index 2351489845..c6a7c59474 100644 --- a/internal/cmd/run.go +++ b/internal/cmd/run.go @@ -192,6 +192,25 @@ func (c *cmdRun) run(cmd *cobra.Command, args []string) (err error) { executionState := execScheduler.GetState() if !testRunState.RuntimeOptions.NoSummary.Bool { //nolint:nestif + // Despite having the revamped [summary.Summary], we still keep the use of the + // [lib.LegacySummary] for multiple backwards compatibility options, + // to be deprecated by v1.0 and likely removed or replaced by v2.0: + // - the `legacy` summary mode (which keeps the old summary format/display). + // - the data structure for custom `handleSummary()` implementations. + // - the data structure for the JSON (--summary-export) output. + legacySummary := func() *lib.LegacySummary { + return &lib.LegacySummary{ + Metrics: metricsEngine.ObservedMetrics, + RootGroup: testRunState.GroupSummary.Group(), + TestRunDuration: executionState.GetCurrentTestRunDuration(), + NoColor: c.gs.Flags.NoColor, + UIState: lib.UIState{ + IsStdOutTTY: c.gs.Stdout.IsTTY, + IsStdErrTTY: c.gs.Stderr.IsTTY, + }, + } + } + sm, err := summary.ValidateMode(testRunState.RuntimeOptions.SummaryMode.String) if err != nil { logger.WithError(err).Warnf( @@ -207,18 +226,7 @@ func (c *cmdRun) run(cmd *cobra.Command, args []string) (err error) { defer func() { logger.Debug("Generating the end-of-test summary...") - legacySummary := &lib.LegacySummary{ - Metrics: metricsEngine.ObservedMetrics, - RootGroup: testRunState.GroupSummary.Group(), - TestRunDuration: executionState.GetCurrentTestRunDuration(), - NoColor: c.gs.Flags.NoColor, - UIState: lib.UIState{ - IsStdOutTTY: c.gs.Stdout.IsTTY, - IsStdErrTTY: c.gs.Stderr.IsTTY, - }, - } - - summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, legacySummary, nil) + summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, legacySummary(), nil) if hsErr == nil { hsErr = handleSummaryResult(c.gs.FS, c.gs.Stdout, c.gs.Stderr, summaryResult) } @@ -252,7 +260,7 @@ func (c *cmdRun) run(cmd *cobra.Command, args []string) (err error) { summary.NoColor = c.gs.Flags.NoColor summary.EnableColors = !summary.NoColor && c.gs.Stdout.IsTTY - summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, nil, summary) + summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, legacySummary(), summary) if hsErr == nil { hsErr = handleSummaryResult(c.gs.FS, c.gs.Stdout, c.gs.Stderr, summaryResult) } diff --git a/internal/cmd/tests/cmd_run_test.go b/internal/cmd/tests/cmd_run_test.go index effd5f1b5e..d16ffd6edb 100644 --- a/internal/cmd/tests/cmd_run_test.go +++ b/internal/cmd/tests/cmd_run_test.go @@ -329,17 +329,17 @@ func TestMetricsAndThresholds(t *testing.T) { var summary map[string]interface{} require.NoError(t, json.Unmarshal(ts.Stdout.Bytes(), &summary)) - thresholds, ok := summary["thresholds"].(map[string]interface{}) + metrics, ok := summary["metrics"].(map[string]interface{}) require.True(t, ok) - teardownCounter, ok := thresholds["teardown_counter"].(map[string]interface{}) + teardownCounter, ok := metrics["teardown_counter"].(map[string]interface{}) require.True(t, ok) - teardownCounterThresholds, ok := teardownCounter["thresholds"].([]interface{}) + teardownThresholds, ok := teardownCounter["thresholds"].(map[string]interface{}) require.True(t, ok) - expected := []interface{}{map[string]interface{}{"source": "count == 1", "ok": true}} - require.Equal(t, expected, teardownCounterThresholds) + expected := map[string]interface{}{"count == 1": map[string]interface{}{"ok": true}} + require.Equal(t, expected, teardownThresholds) } func TestSSLKEYLOGFILEAbsolute(t *testing.T) { @@ -2478,3 +2478,197 @@ func TestMultipleSecretSources(t *testing.T) { assert.Contains(t, stderr, `level=info msg="***SECRET_REDACTED***" ***SECRET_REDACTED***=console`) assert.Contains(t, stderr, `level=info msg="trigger exception on wrong key" ***SECRET_REDACTED***=console`) } + +func TestSummaryExport(t *testing.T) { + t.Parallel() + + mainScript := ` + import { check } from "k6"; + import { Counter } from 'k6/metrics'; + + const customIter = new Counter("custom_iterations"); + + export default function () { + customIter.add(1); + check(true, { "TRUE is TRUE": (r) => r }); + }; + ` + + assertSummaryExport := func(t *testing.T, fs fsext.Fs) { + t.Helper() + + rawSummaryExport, err := fsext.ReadFile(fs, "results.json") + require.NoError(t, err) + + var summaryExport map[string]interface{} + require.NoError(t, json.Unmarshal(rawSummaryExport, &summaryExport)) + + assert.Equal(t, map[string]interface{}{ + "groups": map[string]interface{}{}, + "checks": map[string]interface{}{ + "TRUE is TRUE": map[string]interface{}{ + "fails": float64(0), + "id": "1bed1cc5e442054df516f1ca1076ac6a", + "name": "TRUE is TRUE", + "passes": float64(1), + "path": "::TRUE is TRUE", + }, + }, + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + }, summaryExport["root_group"]) + + metrics := summaryExport["metrics"].(map[string]interface{}) + + assert.Equal(t, 1.0, metrics["custom_iterations"].(map[string]interface{})["count"]) + assert.Equal(t, 1.0, metrics["iterations"].(map[string]interface{})["count"]) + + checks := metrics["checks"].(map[string]interface{}) + assert.Equal(t, 1.0, checks["passes"]) + assert.Equal(t, 0.0, checks["fails"]) + assert.Equal(t, 1.0, checks["value"]) + + // These metrics are created adhoc for visual end-of-test summary only, + // thus they shouldn't be present on the exported summary. + assert.NotContains(t, "checks_total", metrics) + assert.NotContains(t, "checks_succeeded", metrics) + assert.NotContains(t, "checks_failed", metrics) + } + + for _, summaryMode := range []string{"compact", "full"} { + t.Run(summaryMode, func(t *testing.T) { + t.Parallel() + + ts := NewGlobalTestState(t) + require.NoError(t, fsext.WriteFile(ts.FS, filepath.Join(ts.Cwd, "script.js"), []byte(mainScript), 0o644)) + + ts.CmdArgs = []string{ + "k6", "run", + "--summary-export=results.json", + "--summary-mode=" + summaryMode, + "script.js", + } + + cmd.ExecuteWithGlobalState(ts.GlobalState) + + stdout := ts.Stdout.String() + t.Log(stdout) + + assert.Contains(t, stdout, "checks_total.......................: 1") + assert.Contains(t, stdout, "checks_succeeded...................: 100.00% 1 out of 1") + assert.Contains(t, stdout, "checks_failed......................: 0.00% 0 out of 1") + + assert.Contains(t, stdout, `CUSTOM + custom_iterations......................: 1`) + assert.Contains(t, stdout, "iterations.............................: 1") + + assertSummaryExport(t, ts.FS) + }) + } + + t.Run("legacy", func(t *testing.T) { + t.Parallel() + + ts := NewGlobalTestState(t) + require.NoError(t, fsext.WriteFile(ts.FS, filepath.Join(ts.Cwd, "script.js"), []byte(mainScript), 0o644)) + + ts.CmdArgs = []string{ + "k6", "run", + "--summary-export=results.json", + "--summary-mode=legacy", + "script.js", + } + + cmd.ExecuteWithGlobalState(ts.GlobalState) + + stdout := ts.Stdout.String() + t.Log(stdout) + + assert.Contains(t, stdout, "✓ TRUE is TRUE") + assert.Contains(t, stdout, "checks...............: 100.00% 1 out of 1") + assert.Contains(t, stdout, "custom_iterations....: 1") + assert.Contains(t, stdout, "iterations...........: 1") + + assertSummaryExport(t, ts.FS) + }) +} + +func TestHandleSummary(t *testing.T) { + t.Parallel() + mainScript := ` + import { check } from "k6"; + import { Counter } from 'k6/metrics'; + + const customIter = new Counter("custom_iterations"); + + export default function () { + customIter.add(1); + check(true, { "TRUE is TRUE": (r) => r }); + }; + + export function handleSummary(data) { + return { + 'summary.json': JSON.stringify(data), //the default data object + }; + } + ` + + for _, summaryMode := range []string{"compact", "full", "legacy"} { + t.Run(summaryMode, func(t *testing.T) { + t.Parallel() + + ts := NewGlobalTestState(t) + require.NoError(t, fsext.WriteFile(ts.FS, filepath.Join(ts.Cwd, "script.js"), []byte(mainScript), 0o644)) + + ts.CmdArgs = []string{"k6", "run", "script.js"} + + cmd.ExecuteWithGlobalState(ts.GlobalState) + + stdout := ts.Stdout.String() + t.Log(stdout) + + rawSummaryExport, err := fsext.ReadFile(ts.FS, "summary.json") + require.NoError(t, err) + + var summaryExport map[string]interface{} + require.NoError(t, json.Unmarshal(rawSummaryExport, &summaryExport)) + + assert.Equal(t, map[string]interface{}{ + "groups": []interface{}{}, + "checks": []interface{}{ + map[string]interface{}{ + "fails": float64(0), + "id": "1bed1cc5e442054df516f1ca1076ac6a", + "name": "TRUE is TRUE", + "passes": float64(1), + "path": "::TRUE is TRUE", + }, + }, + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + }, summaryExport["root_group"]) + + metrics := summaryExport["metrics"].(map[string]interface{}) + + assert.Equal(t, 1.0, + metrics["custom_iterations"].(map[string]interface{})["values"].(map[string]interface{})["count"], + ) + assert.Equal(t, 1.0, + metrics["iterations"].(map[string]interface{})["values"].(map[string]interface{})["count"], + ) + + checks := metrics["checks"].(map[string]interface{})["values"].(map[string]interface{}) + assert.Equal(t, 1.0, checks["rate"]) + assert.Equal(t, 1.0, checks["passes"]) + assert.Equal(t, 0.0, checks["fails"]) + + // These metrics are created adhoc for visual end-of-test summary only, + // thus they shouldn't be present on the custom `handleSummary()` data structure. + assert.NotContains(t, "checks_total", metrics) + assert.NotContains(t, "checks_succeeded", metrics) + assert.NotContains(t, "checks_failed", metrics) + }) + } +} diff --git a/internal/js/runner.go b/internal/js/runner.go index 485a5f8528..baa595672d 100644 --- a/internal/js/runner.go +++ b/internal/js/runner.go @@ -376,9 +376,9 @@ func (r *Runner) HandleSummary( }) vu.moduleVUImpl.ctx = summaryCtx - noColor, enableColors, summaryDataForJS, summaryCode := prepareHandleSummaryCall(r, legacy, summary) + noColor, enableColors, legacyData, summaryData, summaryCode := prepareHandleSummaryCall(r, legacy, summary) - handleSummaryDataAsValue := vu.Runtime.ToValue(summaryDataForJS) + handleSummaryDataAsValue := vu.Runtime.ToValue(legacyData) callbackResult, err := runUserProvidedHandleSummaryCallback(summaryCtx, vu, handleSummaryDataAsValue) if err != nil { return nil, err @@ -394,7 +394,8 @@ func (r *Runner) HandleSummary( return nil, fmt.Errorf("unexpected error did not get a callable summary wrapper") } - wrapperArgs := prepareHandleWrapperArgs(vu, noColor, enableColors, callbackResult, handleSummaryDataAsValue) + wrapperArgs := prepareHandleWrapperArgs( + vu, noColor, enableColors, callbackResult, handleSummaryDataAsValue, vu.Runtime.ToValue(summaryData)) rawResult, _, _, err := vu.runFn(summaryCtx, false, handleSummaryWrapper, nil, wrapperArgs...) if deadlineError := r.checkDeadline(summaryCtx, consts.HandleSummaryFn, rawResult, err); deadlineError != nil { @@ -412,30 +413,29 @@ func prepareHandleSummaryCall( r *Runner, legacy *lib.LegacySummary, summary *summary.Summary, -) (bool, bool, interface{}, string) { +) (bool, bool, interface{}, interface{}, string) { var ( noColor bool enableColors bool + legacyDataForJS interface{} summaryDataForJS interface{} summaryCode string ) - - // TODO: Remove this code block once we stop supporting the legacy summary. - if legacy != nil { - noColor = legacy.NoColor - enableColors = !legacy.NoColor && legacy.UIState.IsStdOutTTY - summaryDataForJS = summarizeMetricsToObject(legacy, r.Bundle.Options, r.setupData) - summaryCode = jslibSummaryLegacyCode - } - if summary != nil { noColor = summary.NoColor enableColors = summary.EnableColors + legacyDataForJS = summarizeMetricsToObject(legacy, r.Bundle.Options, r.setupData) summaryDataForJS = summary summaryCode = jslibSummaryCode + } else { // TODO: Remove this code block once we stop supporting the legacy summary. + noColor = legacy.NoColor + enableColors = !legacy.NoColor && legacy.UIState.IsStdOutTTY + legacyDataForJS = summarizeMetricsToObject(legacy, r.Bundle.Options, r.setupData) + summaryDataForJS = legacyDataForJS + summaryCode = jslibSummaryLegacyCode } - return noColor, enableColors, summaryDataForJS, summaryCode + return noColor, enableColors, legacyDataForJS, summaryDataForJS, summaryCode } func runUserProvidedHandleSummaryCallback( @@ -467,8 +467,7 @@ func runUserProvidedHandleSummaryCallback( func prepareHandleWrapperArgs( vu *VU, noColor bool, enableColors bool, - callbackResult sobek.Value, - summaryDataForJS interface{}, + callbackResult, handleSummaryDataAsValue, summaryDataAsValue sobek.Value, ) []sobek.Value { options := map[string]interface{}{ // TODO: improve when we can easily export all option values, including defaults? @@ -481,7 +480,8 @@ func prepareHandleWrapperArgs( wrapperArgs := []sobek.Value{ callbackResult, vu.Runtime.ToValue(vu.Runner.Bundle.preInitState.RuntimeOptions.SummaryExport.String), - vu.Runtime.ToValue(summaryDataForJS), + handleSummaryDataAsValue, + summaryDataAsValue, vu.Runtime.ToValue(options), } diff --git a/internal/js/summary-wrapper.js b/internal/js/summary-wrapper.js index dc0d9e4814..3529adc1a5 100644 --- a/internal/js/summary-wrapper.js +++ b/internal/js/summary-wrapper.js @@ -60,7 +60,7 @@ return JSON.stringify(results, null, 4); }; - return function (summaryCallbackResult, jsonSummaryPath, data, options) { + return function (summaryCallbackResult, jsonSummaryPath, legacyData, data, options) { let result = summaryCallbackResult; if (!result) { result = { @@ -72,7 +72,7 @@ // and if not, log an error and generate the default summary? if (jsonSummaryPath != '') { - result[jsonSummaryPath] = oldJSONSummary(data); + result[jsonSummaryPath] = oldJSONSummary(legacyData); } return result;