Revamp the machine-readable (JSON) end-of-test summary

.github/workflows/summary-code-generation.yml

@@ -0,0 +1,68 @@
+name: Summary code generation
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+jobs:
+  check-generated-code:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+        with:
+          persist-credentials: false
+
+      - name: Install cog
+        run: |
+          COG_VERSION="v0.0.46"
+          curl -sSL "https://github.com/grafana/cog/releases/download/${COG_VERSION}/cog_Linux_x86_64.tar.gz" \
+            | sudo tar -xz -C /usr/local/bin
+          cog --version
+
+      - name: Checkout schemas
+        run: |
+          ./internal/lib/summary/machinereadable/checkout.sh
+
+      - name: Generate code
+        run: |
+          ./internal/lib/summary/machinereadable/generate.sh
+
+      - name: Check for uncommitted changes
+        run: |
+          # Check if there are any changes in the Git working directory
+          if ! git diff --exit-code; then
+            echo "❌ Generated code is not up-to-date!"
+            echo ""
+            echo "The following files have uncommitted changes:"
+            git diff --name-only
+            echo ""
+            echo "Please run the following commands to update the generated code:"
+            echo "  ./internal/lib/summary/machinereadable/checkout.sh"
+            echo "  ./internal/lib/summary/machinereadable/generate.sh"
+            echo ""
+            echo "Then commit the changes."
+            exit 1
+          fi
+
+          # Also check for untracked files
+          if [ -n "$(git ls-files --others --exclude-standard internal/lib/summary/machinereadable/ | grep -v k6-summary/)" ]; then
+            echo "❌ There are untracked generated files!"
+            echo ""
+            echo "Untracked files:"
+            git ls-files --others --exclude-standard internal/lib/summary/machinereadable/ | grep -v k6-summary/
+            echo ""
+            echo "Please run the generation scripts and commit all generated files."
+            exit 1
+          fi
+
+          echo "✅ Generated code is up-to-date!"

.gitignore

@@ -2,6 +2,7 @@
 /k6.exe
 /dist
 /pkg-build
+/internal/lib/summary/machinereadable/k6-summary
 /internal/js/tc39/TestTC39
 /internal/js/modules/k6/experimental/streams/tests/wpt
 /internal/js/modules/k6/webcrypto/tests/wpt/

internal/cmd/run.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"slices"
 	"strings"
 	"sync"
 	"syscall"
@@ -26,6 +27,7 @@ import (
 	"go.k6.io/k6/internal/lib/summary"
 	"go.k6.io/k6/internal/lib/trace"
 	"go.k6.io/k6/internal/metrics/engine"
+	"go.k6.io/k6/internal/output/cloud"
 	summaryoutput "go.k6.io/k6/internal/output/summary"
 	"go.k6.io/k6/internal/ui/pb"
 	"go.k6.io/k6/js/common"
@@ -220,6 +222,14 @@ func (c *cmdRun) run(cmd *cobra.Command, args []string) (err error) {
 			}
 		}
 
+		summaryMeta := summary.Meta{
+			Script: string(test.source.Data),
+			IsCloud: slices.ContainsFunc(outputs, func(o output.Output) bool {
+				_, isCloud := o.(*cloud.Output)
+				return isCloud
+			}),
+		}
+
 		switch summaryMode {
 		// TODO(@joanlopez): remove by k6 v2.0, once we completely drop the support for --summary-mode=legacy.
 		case summary.ModeLegacy:
@@ -229,7 +239,7 @@ func (c *cmdRun) run(cmd *cobra.Command, args []string) (err error) {
 			defer func() {
 				logger.Debug("Generating the end-of-test summary...")
 
-				summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, legacySummary(), nil)
+				summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, legacySummary(), nil, summaryMeta)
 				if hsErr == nil {
 					hsErr = handleSummaryResult(c.gs.FS, c.gs.Stdout, c.gs.Stderr, summaryResult)
 				}
@@ -262,8 +272,10 @@ func (c *cmdRun) run(cmd *cobra.Command, args []string) (err error) {
 				// likely as an additional argument like options.
 				summary.NoColor = c.gs.Flags.NoColor
 				summary.EnableColors = !summary.NoColor && c.gs.Stdout.IsTTY
+				summary.NewMachineReadableSummary = testRunState.RuntimeOptions.NewMachineReadableSummary.Valid &&
+					testRunState.RuntimeOptions.NewMachineReadableSummary.Bool
 
-				summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, legacySummary(), summary)
+				summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, legacySummary(), summary, summaryMeta)
 				if hsErr == nil {
 					hsErr = handleSummaryResult(c.gs.FS, c.gs.Stdout, c.gs.Stderr, summaryResult)
 				}

internal/cmd/runtime_options.go

@@ -43,6 +43,9 @@ extended: base + sets "global" as alias for "globalThis"
 		"",
 		"output the end-of-test summary report to JSON file",
 	)
+	// TODO(@joanlopez): remove by k6 v2.0, once the new summary model is the default and the only one.
+	flags.Bool("new-machine-readable-summary", false, "enables the new machine-readable summary, "+
+		"which is used for summary exports and as handleSummary() argument")
 	flags.String("traces-output", "none",
 		"set the output for k6 traces, possible values are none,otel[=host:port]")
 	return flags
@@ -78,15 +81,16 @@ func getRuntimeOptions(
 
 func runtimeOptionsFromFlags(flags *pflag.FlagSet) lib.RuntimeOptions {
 	opts := lib.RuntimeOptions{
-		TestType:             getNullString(flags, "type"),
-		IncludeSystemEnvVars: getNullBool(flags, "include-system-env-vars"),
-		CompatibilityMode:    getNullString(flags, "compatibility-mode"),
-		NoThresholds:         getNullBool(flags, "no-thresholds"),
-		NoSummary:            getNullBool(flags, "no-summary"),
-		SummaryMode:          getNullString(flags, "summary-mode"),
-		SummaryExport:        getNullString(flags, "summary-export"),
-		TracesOutput:         getNullString(flags, "traces-output"),
-		Env:                  make(map[string]string),
+		TestType:                  getNullString(flags, "type"),
+		IncludeSystemEnvVars:      getNullBool(flags, "include-system-env-vars"),
+		CompatibilityMode:         getNullString(flags, "compatibility-mode"),
+		NoThresholds:              getNullBool(flags, "no-thresholds"),
+		NoSummary:                 getNullBool(flags, "no-summary"),
+		SummaryMode:               getNullString(flags, "summary-mode"),
+		SummaryExport:             getNullString(flags, "summary-export"),
+		NewMachineReadableSummary: getNullBool(flags, "new-machine-readable-summary"),
+		TracesOutput:              getNullString(flags, "traces-output"),
+		Env:                       make(map[string]string),
 	}
 	return opts
 }
@@ -133,6 +137,12 @@ func populateRuntimeOptionsFromEnv(opts lib.RuntimeOptions, environment map[stri
 		opts.SummaryExport = null.StringFrom(envVar)
 	}
 
+	if err := saveBoolFromEnv(
+		environment, "K6_NEW_MACHINE_READABLE_SUMMARY", &opts.NewMachineReadableSummary,
+	); err != nil {
+		return opts, err
+	}
+
 	if envVar, ok := environment["SSLKEYLOGFILE"]; !opts.KeyWriter.Valid && ok {
 		opts.KeyWriter = null.StringFrom(envVar)
 	}