Skip to content

Commit 8bdaebf

Browse files
joanlopezjoanlopez
authored
Summary: Use lib.LegacySummary to avoid breaking the API (#4649)
* Summary: Use lib.LegacySummary to avoid breaking the API * Summary: Add test coverage for handleSummary() and --summary-export
1 parent bc1329f commit 8bdaebf

File tree

5 files changed

+240
-38
lines changed

5 files changed

+240
-38
lines changed

‎.github/workflows/xk6-tests/xk6-test.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ export let options = {
1111

1212
export function handleSummary(data) {
1313
return {
14-
'summary-results.txt': data.metrics.custom.foos.values.count.toString(),
14+
'summary-results.txt': data.metrics.foos.values.count.toString(),
1515
};
1616
}
1717

‎internal/cmd/run.go‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,25 @@ func (c *cmdRun) run(cmd *cobra.Command, args []string) (err error) {
192192

193193
executionState := execScheduler.GetState()
194194
if !testRunState.RuntimeOptions.NoSummary.Bool { //nolint:nestif
195+
// Despite having the revamped [summary.Summary], we still keep the use of the
196+
// [lib.LegacySummary] for multiple backwards compatibility options,
197+
// to be deprecated by v1.0 and likely removed or replaced by v2.0:
198+
// - the `legacy` summary mode (which keeps the old summary format/display).
199+
// - the data structure for custom `handleSummary()` implementations.
200+
// - the data structure for the JSON (--summary-export) output.
201+
legacySummary := func() *lib.LegacySummary {
202+
return &lib.LegacySummary{
203+
Metrics: metricsEngine.ObservedMetrics,
204+
RootGroup: testRunState.GroupSummary.Group(),
205+
TestRunDuration: executionState.GetCurrentTestRunDuration(),
206+
NoColor: c.gs.Flags.NoColor,
207+
UIState: lib.UIState{
208+
IsStdOutTTY: c.gs.Stdout.IsTTY,
209+
IsStdErrTTY: c.gs.Stderr.IsTTY,
210+
},
211+
}
212+
}
213+
195214
sm, err := summary.ValidateMode(testRunState.RuntimeOptions.SummaryMode.String)
196215
if err != nil {
197216
logger.WithError(err).Warnf(
@@ -207,18 +226,7 @@ func (c *cmdRun) run(cmd *cobra.Command, args []string) (err error) {
207226
defer func() {
208227
logger.Debug("Generating the end-of-test summary...")
209228

210-
legacySummary := &lib.LegacySummary{
211-
Metrics: metricsEngine.ObservedMetrics,
212-
RootGroup: testRunState.GroupSummary.Group(),
213-
TestRunDuration: executionState.GetCurrentTestRunDuration(),
214-
NoColor: c.gs.Flags.NoColor,
215-
UIState: lib.UIState{
216-
IsStdOutTTY: c.gs.Stdout.IsTTY,
217-
IsStdErrTTY: c.gs.Stderr.IsTTY,
218-
},
219-
}
220-
221-
summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, legacySummary, nil)
229+
summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, legacySummary(), nil)
222230
if hsErr == nil {
223231
hsErr = handleSummaryResult(c.gs.FS, c.gs.Stdout, c.gs.Stderr, summaryResult)
224232
}
@@ -252,7 +260,7 @@ func (c *cmdRun) run(cmd *cobra.Command, args []string) (err error) {
252260
summary.NoColor = c.gs.Flags.NoColor
253261
summary.EnableColors = !summary.NoColor && c.gs.Stdout.IsTTY
254262

255-
summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, nil, summary)
263+
summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, legacySummary(), summary)
256264
if hsErr == nil {
257265
hsErr = handleSummaryResult(c.gs.FS, c.gs.Stdout, c.gs.Stderr, summaryResult)
258266
}

‎internal/cmd/tests/cmd_run_test.go‎

Lines changed: 199 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -329,17 +329,17 @@ func TestMetricsAndThresholds(t *testing.T) {
329329
var summary map[string]interface{}
330330
require.NoError(t, json.Unmarshal(ts.Stdout.Bytes(), &summary))
331331

332-
thresholds, ok := summary["thresholds"].(map[string]interface{})
332+
metrics, ok := summary["metrics"].(map[string]interface{})
333333
require.True(t, ok)
334334

335-
teardownCounter, ok := thresholds["teardown_counter"].(map[string]interface{})
335+
teardownCounter, ok := metrics["teardown_counter"].(map[string]interface{})
336336
require.True(t, ok)
337337

338-
teardownCounterThresholds, ok := teardownCounter["thresholds"].([]interface{})
338+
teardownThresholds, ok := teardownCounter["thresholds"].(map[string]interface{})
339339
require.True(t, ok)
340340

341-
expected := []interface{}{map[string]interface{}{"source": "count == 1", "ok": true}}
342-
require.Equal(t, expected, teardownCounterThresholds)
341+
expected := map[string]interface{}{"count == 1": map[string]interface{}{"ok": true}}
342+
require.Equal(t, expected, teardownThresholds)
343343
}
344344

345345
func TestSSLKEYLOGFILEAbsolute(t *testing.T) {
@@ -2478,3 +2478,197 @@ func TestMultipleSecretSources(t *testing.T) {
24782478
assert.Contains(t, stderr, `level=info msg="***SECRET_REDACTED***" ***SECRET_REDACTED***=console`)
24792479
assert.Contains(t, stderr, `level=info msg="trigger exception on wrong key" ***SECRET_REDACTED***=console`)
24802480
}
2481+
2482+
func TestSummaryExport(t *testing.T) {
2483+
t.Parallel()
2484+
2485+
mainScript := `
2486+
import { check } from "k6";
2487+
import { Counter } from 'k6/metrics';
2488+
2489+
const customIter = new Counter("custom_iterations");
2490+
2491+
export default function () {
2492+
customIter.add(1);
2493+
check(true, { "TRUE is TRUE": (r) => r });
2494+
};
2495+
`
2496+
2497+
assertSummaryExport := func(t *testing.T, fs fsext.Fs) {
2498+
t.Helper()
2499+
2500+
rawSummaryExport, err := fsext.ReadFile(fs, "results.json")
2501+
require.NoError(t, err)
2502+
2503+
var summaryExport map[string]interface{}
2504+
require.NoError(t, json.Unmarshal(rawSummaryExport, &summaryExport))
2505+
2506+
assert.Equal(t, map[string]interface{}{
2507+
"groups": map[string]interface{}{},
2508+
"checks": map[string]interface{}{
2509+
"TRUE is TRUE": map[string]interface{}{
2510+
"fails": float64(0),
2511+
"id": "1bed1cc5e442054df516f1ca1076ac6a",
2512+
"name": "TRUE is TRUE",
2513+
"passes": float64(1),
2514+
"path": "::TRUE is TRUE",
2515+
},
2516+
},
2517+
"name": "",
2518+
"path": "",
2519+
"id": "d41d8cd98f00b204e9800998ecf8427e",
2520+
}, summaryExport["root_group"])
2521+
2522+
metrics := summaryExport["metrics"].(map[string]interface{})
2523+
2524+
assert.Equal(t, 1.0, metrics["custom_iterations"].(map[string]interface{})["count"])
2525+
assert.Equal(t, 1.0, metrics["iterations"].(map[string]interface{})["count"])
2526+
2527+
checks := metrics["checks"].(map[string]interface{})
2528+
assert.Equal(t, 1.0, checks["passes"])
2529+
assert.Equal(t, 0.0, checks["fails"])
2530+
assert.Equal(t, 1.0, checks["value"])
2531+
2532+
// These metrics are created adhoc for visual end-of-test summary only,
2533+
// thus they shouldn't be present on the exported summary.
2534+
assert.NotContains(t, "checks_total", metrics)
2535+
assert.NotContains(t, "checks_succeeded", metrics)
2536+
assert.NotContains(t, "checks_failed", metrics)
2537+
}
2538+
2539+
for _, summaryMode := range []string{"compact", "full"} {
2540+
t.Run(summaryMode, func(t *testing.T) {
2541+
t.Parallel()
2542+
2543+
ts := NewGlobalTestState(t)
2544+
require.NoError(t, fsext.WriteFile(ts.FS, filepath.Join(ts.Cwd, "script.js"), []byte(mainScript), 0o644))
2545+
2546+
ts.CmdArgs = []string{
2547+
"k6", "run",
2548+
"--summary-export=results.json",
2549+
"--summary-mode=" + summaryMode,
2550+
"script.js",
2551+
}
2552+
2553+
cmd.ExecuteWithGlobalState(ts.GlobalState)
2554+
2555+
stdout := ts.Stdout.String()
2556+
t.Log(stdout)
2557+
2558+
assert.Contains(t, stdout, "checks_total.......................: 1")
2559+
assert.Contains(t, stdout, "checks_succeeded...................: 100.00% 1 out of 1")
2560+
assert.Contains(t, stdout, "checks_failed......................: 0.00% 0 out of 1")
2561+
2562+
assert.Contains(t, stdout, `CUSTOM
2563+
custom_iterations......................: 1`)
2564+
assert.Contains(t, stdout, "iterations.............................: 1")
2565+
2566+
assertSummaryExport(t, ts.FS)
2567+
})
2568+
}
2569+
2570+
t.Run("legacy", func(t *testing.T) {
2571+
t.Parallel()
2572+
2573+
ts := NewGlobalTestState(t)
2574+
require.NoError(t, fsext.WriteFile(ts.FS, filepath.Join(ts.Cwd, "script.js"), []byte(mainScript), 0o644))
2575+
2576+
ts.CmdArgs = []string{
2577+
"k6", "run",
2578+
"--summary-export=results.json",
2579+
"--summary-mode=legacy",
2580+
"script.js",
2581+
}
2582+
2583+
cmd.ExecuteWithGlobalState(ts.GlobalState)
2584+
2585+
stdout := ts.Stdout.String()
2586+
t.Log(stdout)
2587+
2588+
assert.Contains(t, stdout, "✓ TRUE is TRUE")
2589+
assert.Contains(t, stdout, "checks...............: 100.00% 1 out of 1")
2590+
assert.Contains(t, stdout, "custom_iterations....: 1")
2591+
assert.Contains(t, stdout, "iterations...........: 1")
2592+
2593+
assertSummaryExport(t, ts.FS)
2594+
})
2595+
}
2596+
2597+
func TestHandleSummary(t *testing.T) {
2598+
t.Parallel()
2599+
mainScript := `
2600+
import { check } from "k6";
2601+
import { Counter } from 'k6/metrics';
2602+
2603+
const customIter = new Counter("custom_iterations");
2604+
2605+
export default function () {
2606+
customIter.add(1);
2607+
check(true, { "TRUE is TRUE": (r) => r });
2608+
};
2609+
2610+
export function handleSummary(data) {
2611+
return {
2612+
'summary.json': JSON.stringify(data), //the default data object
2613+
};
2614+
}
2615+
`
2616+
2617+
for _, summaryMode := range []string{"compact", "full", "legacy"} {
2618+
t.Run(summaryMode, func(t *testing.T) {
2619+
t.Parallel()
2620+
2621+
ts := NewGlobalTestState(t)
2622+
require.NoError(t, fsext.WriteFile(ts.FS, filepath.Join(ts.Cwd, "script.js"), []byte(mainScript), 0o644))
2623+
2624+
ts.CmdArgs = []string{"k6", "run", "script.js"}
2625+
2626+
cmd.ExecuteWithGlobalState(ts.GlobalState)
2627+
2628+
stdout := ts.Stdout.String()
2629+
t.Log(stdout)
2630+
2631+
rawSummaryExport, err := fsext.ReadFile(ts.FS, "summary.json")
2632+
require.NoError(t, err)
2633+
2634+
var summaryExport map[string]interface{}
2635+
require.NoError(t, json.Unmarshal(rawSummaryExport, &summaryExport))
2636+
2637+
assert.Equal(t, map[string]interface{}{
2638+
"groups": []interface{}{},
2639+
"checks": []interface{}{
2640+
map[string]interface{}{
2641+
"fails": float64(0),
2642+
"id": "1bed1cc5e442054df516f1ca1076ac6a",
2643+
"name": "TRUE is TRUE",
2644+
"passes": float64(1),
2645+
"path": "::TRUE is TRUE",
2646+
},
2647+
},
2648+
"name": "",
2649+
"path": "",
2650+
"id": "d41d8cd98f00b204e9800998ecf8427e",
2651+
}, summaryExport["root_group"])
2652+
2653+
metrics := summaryExport["metrics"].(map[string]interface{})
2654+
2655+
assert.Equal(t, 1.0,
2656+
metrics["custom_iterations"].(map[string]interface{})["values"].(map[string]interface{})["count"],
2657+
)
2658+
assert.Equal(t, 1.0,
2659+
metrics["iterations"].(map[string]interface{})["values"].(map[string]interface{})["count"],
2660+
)
2661+
2662+
checks := metrics["checks"].(map[string]interface{})["values"].(map[string]interface{})
2663+
assert.Equal(t, 1.0, checks["rate"])
2664+
assert.Equal(t, 1.0, checks["passes"])
2665+
assert.Equal(t, 0.0, checks["fails"])
2666+
2667+
// These metrics are created adhoc for visual end-of-test summary only,
2668+
// thus they shouldn't be present on the custom `handleSummary()` data structure.
2669+
assert.NotContains(t, "checks_total", metrics)
2670+
assert.NotContains(t, "checks_succeeded", metrics)
2671+
assert.NotContains(t, "checks_failed", metrics)
2672+
})
2673+
}
2674+
}

‎internal/js/runner.go‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -376,9 +376,9 @@ func (r *Runner) HandleSummary(
376376
})
377377
vu.moduleVUImpl.ctx = summaryCtx
378378

379-
noColor, enableColors, summaryDataForJS, summaryCode := prepareHandleSummaryCall(r, legacy, summary)
379+
noColor, enableColors, legacyData, summaryData, summaryCode := prepareHandleSummaryCall(r, legacy, summary)
380380

381-
handleSummaryDataAsValue := vu.Runtime.ToValue(summaryDataForJS)
381+
handleSummaryDataAsValue := vu.Runtime.ToValue(legacyData)
382382
callbackResult, err := runUserProvidedHandleSummaryCallback(summaryCtx, vu, handleSummaryDataAsValue)
383383
if err != nil {
384384
return nil, err
@@ -394,7 +394,8 @@ func (r *Runner) HandleSummary(
394394
return nil, fmt.Errorf("unexpected error did not get a callable summary wrapper")
395395
}
396396

397-
wrapperArgs := prepareHandleWrapperArgs(vu, noColor, enableColors, callbackResult, handleSummaryDataAsValue)
397+
wrapperArgs := prepareHandleWrapperArgs(
398+
vu, noColor, enableColors, callbackResult, handleSummaryDataAsValue, vu.Runtime.ToValue(summaryData))
398399
rawResult, _, _, err := vu.runFn(summaryCtx, false, handleSummaryWrapper, nil, wrapperArgs...)
399400

400401
if deadlineError := r.checkDeadline(summaryCtx, consts.HandleSummaryFn, rawResult, err); deadlineError != nil {
@@ -412,30 +413,29 @@ func prepareHandleSummaryCall(
412413
r *Runner,
413414
legacy *lib.LegacySummary,
414415
summary *summary.Summary,
415-
) (bool, bool, interface{}, string) {
416+
) (bool, bool, interface{}, interface{}, string) {
416417
var (
417418
noColor bool
418419
enableColors bool
420+
legacyDataForJS interface{}
419421
summaryDataForJS interface{}
420422
summaryCode string
421423
)
422-
423-
// TODO: Remove this code block once we stop supporting the legacy summary.
424-
if legacy != nil {
425-
noColor = legacy.NoColor
426-
enableColors = !legacy.NoColor && legacy.UIState.IsStdOutTTY
427-
summaryDataForJS = summarizeMetricsToObject(legacy, r.Bundle.Options, r.setupData)
428-
summaryCode = jslibSummaryLegacyCode
429-
}
430-
431424
if summary != nil {
432425
noColor = summary.NoColor
433426
enableColors = summary.EnableColors
427+
legacyDataForJS = summarizeMetricsToObject(legacy, r.Bundle.Options, r.setupData)
434428
summaryDataForJS = summary
435429
summaryCode = jslibSummaryCode
430+
} else { // TODO: Remove this code block once we stop supporting the legacy summary.
431+
noColor = legacy.NoColor
432+
enableColors = !legacy.NoColor && legacy.UIState.IsStdOutTTY
433+
legacyDataForJS = summarizeMetricsToObject(legacy, r.Bundle.Options, r.setupData)
434+
summaryDataForJS = legacyDataForJS
435+
summaryCode = jslibSummaryLegacyCode
436436
}
437437

438-
return noColor, enableColors, summaryDataForJS, summaryCode
438+
return noColor, enableColors, legacyDataForJS, summaryDataForJS, summaryCode
439439
}
440440

441441
func runUserProvidedHandleSummaryCallback(
@@ -467,8 +467,7 @@ func runUserProvidedHandleSummaryCallback(
467467
func prepareHandleWrapperArgs(
468468
vu *VU,
469469
noColor bool, enableColors bool,
470-
callbackResult sobek.Value,
471-
summaryDataForJS interface{},
470+
callbackResult, handleSummaryDataAsValue, summaryDataAsValue sobek.Value,
472471
) []sobek.Value {
473472
options := map[string]interface{}{
474473
// TODO: improve when we can easily export all option values, including defaults?
@@ -481,7 +480,8 @@ func prepareHandleWrapperArgs(
481480
wrapperArgs := []sobek.Value{
482481
callbackResult,
483482
vu.Runtime.ToValue(vu.Runner.Bundle.preInitState.RuntimeOptions.SummaryExport.String),
484-
vu.Runtime.ToValue(summaryDataForJS),
483+
handleSummaryDataAsValue,
484+
summaryDataAsValue,
485485
vu.Runtime.ToValue(options),
486486
}
487487

‎internal/js/summary-wrapper.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@
6060
return JSON.stringify(results, null, 4);
6161
};
6262

63-
return function (summaryCallbackResult, jsonSummaryPath, data, options) {
63+
return function (summaryCallbackResult, jsonSummaryPath, legacyData, data, options) {
6464
let result = summaryCallbackResult;
6565
if (!result) {
6666
result = {
@@ -72,7 +72,7 @@
7272
// and if not, log an error and generate the default summary?
7373

7474
if (jsonSummaryPath != '') {
75-
result[jsonSummaryPath] = oldJSONSummary(data);
75+
result[jsonSummaryPath] = oldJSONSummary(legacyData);
7676
}
7777

7878
return result;

0 commit comments

Comments
 (0)