Summary: Display thresholds values even when not in summaryTrendStats

Author: joanlopez

internal/cmd/tests/cmd_run_test.go

@@ -342,6 +342,38 @@ func TestMetricsAndThresholds(t *testing.T) {
 	require.Equal(t, expected, teardownThresholds)
 }
 
+func TestThresholdsWithCustomPercentile(t *testing.T) {
+	t.Parallel()
+	script := `
+		export const options = {
+			scenarios: {
+				sc1: {
+					executor: 'per-vu-iterations',
+					vus: 1,
+					iterations: 1,
+				},
+			},
+			thresholds: {
+				'iteration_duration': ['p(0)<50', 'p(90)<100', 'p(99.5)<150', 'p(99.99)<200'],
+			},
+		};
+
+		export default function () {}
+	`
+	ts := getSingleFileTestState(t, script, nil, 0)
+	cmd.ExecuteWithGlobalState(ts.GlobalState)
+
+	stdout := ts.Stdout.String()
+	t.Log(stdout)
+
+	// We want to make sure that all the thresholds expressions are accompanied
+	// by their values, despite those being present in `options.summaryTrendStats` or not.
+	assert.Regexp(t, `✓ 'p\(0\)<50' p\(0\)=\d+(\.\d+)?µs`, stdout)
+	assert.Regexp(t, `✓ 'p\(90\)<100' p\(90\)=\d+(\.\d+)?µs`, stdout)
+	assert.Regexp(t, `✓ 'p\(99.5\)<150' p\(99.5\)=\d+(\.\d+)?µs`, stdout)
+	assert.Regexp(t, `✓ 'p\(99.99\)<200' p\(99.99\)=\d+(\.\d+)?µs`, stdout)
+}
+
 func TestSSLKEYLOGFILEAbsolute(t *testing.T) {
 	t.Parallel()
 	ts := NewGlobalTestState(t)
@@ -2102,14 +2134,14 @@ func TestEventSystemError(t *testing.T) {
 					test.abort('oops!');
 				}
 		`, expLog: []string{
-				"got event Init with data '<nil>'",
-				"got event TestStart with data '<nil>'",
-				"got event IterStart with data '{Iteration:0 VUID:1 ScenarioName:default Error:<nil>}'",
-				"got event IterEnd with data '{Iteration:0 VUID:1 ScenarioName:default Error:test aborted: oops! at default (file:///-:11:16(5))}'",
-				"got event TestEnd with data '<nil>'",
-				"got event Exit with data '&{Error:test aborted: oops! at default (file:///-:11:16(5))}'",
-				"test aborted: oops! at default (file:///-:11:16(5))",
-			},
+			"got event Init with data '<nil>'",
+			"got event TestStart with data '<nil>'",
+			"got event IterStart with data '{Iteration:0 VUID:1 ScenarioName:default Error:<nil>}'",
+			"got event IterEnd with data '{Iteration:0 VUID:1 ScenarioName:default Error:test aborted: oops! at default (file:///-:11:16(5))}'",
+			"got event TestEnd with data '<nil>'",
+			"got event Exit with data '&{Error:test aborted: oops! at default (file:///-:11:16(5))}'",
+			"test aborted: oops! at default (file:///-:11:16(5))",
+		},
 			expExitCode: exitcodes.ScriptAborted,
 		},
 		{
@@ -2134,17 +2166,17 @@ func TestEventSystemError(t *testing.T) {
 					throw new Error('oops!');
 				}
 		`, expLog: []string{
-				"got event Init with data '<nil>'",
-				"got event TestStart with data '<nil>'",
-				"got event IterStart with data '{Iteration:0 VUID:1 ScenarioName:default Error:<nil>}'",
-				"got event IterEnd with data '{Iteration:0 VUID:1 ScenarioName:default Error:Error: oops!\n\tat default (file:///-:9:12(3))\n}'",
-				"Error: oops!\n\tat default (file:///-:9:12(3))\n",
-				"got event IterStart with data '{Iteration:1 VUID:1 ScenarioName:default Error:<nil>}'",
-				"got event IterEnd with data '{Iteration:1 VUID:1 ScenarioName:default Error:Error: oops!\n\tat default (file:///-:9:12(3))\n}'",
-				"Error: oops!\n\tat default (file:///-:9:12(3))\n",
-				"got event TestEnd with data '<nil>'",
-				"got event Exit with data '&{Error:<nil>}'",
-			},
+			"got event Init with data '<nil>'",
+			"got event TestStart with data '<nil>'",
+			"got event IterStart with data '{Iteration:0 VUID:1 ScenarioName:default Error:<nil>}'",
+			"got event IterEnd with data '{Iteration:0 VUID:1 ScenarioName:default Error:Error: oops!\n\tat default (file:///-:9:12(3))\n}'",
+			"Error: oops!\n\tat default (file:///-:9:12(3))\n",
+			"got event IterStart with data '{Iteration:1 VUID:1 ScenarioName:default Error:<nil>}'",
+			"got event IterEnd with data '{Iteration:1 VUID:1 ScenarioName:default Error:Error: oops!\n\tat default (file:///-:9:12(3))\n}'",
+			"Error: oops!\n\tat default (file:///-:9:12(3))\n",
+			"got event TestEnd with data '<nil>'",
+			"got event Exit with data '&{Error:<nil>}'",
+		},
 			expExitCode: 0,
 		},
 	}

internal/js/summary.js

@@ -839,7 +839,7 @@ function renderThresholdResults(
 			: formatter.decorate(failMark, 'red');
 
 		const sourceText = formatter.decorate(
-			`'${threshold.source}'`,
+			`'${threshold.source.trim()}'`,
 			'white',
 		);
 
@@ -941,13 +941,16 @@ function renderMetricValueForThresholds(
 	info,
 	formatter,
 ) {
-	const {trendStats, trendCols, nonTrendValues, nonTrendExtras} = info;
+	const {trendStats, trendCols, trendKeys, nonTrendValues, nonTrendExtras} = info;
 	const thresholdAgg = threshold.source.split(/[=><]/)[0].trim();
 
 	let value;
 	switch (metric.type) {
 		case 'trend':
-			value = trendCols[metric.name][trendStats.indexOf(thresholdAgg)]
+			const trendStatIndex = trendStats.indexOf(thresholdAgg);
+			value = (trendStatIndex !== -1)
+				? trendCols[metric.name]?.[trendStatIndex]
+				: trendKeys[metric.name]?.[thresholdAgg];
 			break;
 		case 'counter':
 			value = (thresholdAgg === 'count')
@@ -1042,6 +1045,7 @@ function renderTrendValue(value, stat, metric, options) {
  * @property {Object} nonTrendValues - The non-trend metric values.
  * @property {Object} nonTrendExtras - The non-trend metric extras.
  * @property {Object} trendCols - The trend columns.
+ * @property {Object} trendKeys - The trend keys (values that aren't included within `trendStats`).
  * @property {number[]} trendColMaxLens - The trend column maximum lengths.
  * @property {number} numTrendColumns - The number of trend columns.
  * @property {string[]} trendStats - The trend statistics.
@@ -1061,6 +1065,10 @@ function computeSummaryInfo(metrics, renderContext, options) {
 	const nonTrendExtras = {};
 	const trendCols = {};
 
+	// While "trendCols" contain the values for each "trendStats" aggregation (e.g. p(90) as a sorted array,
+	// "trendKeys" is used to store specific aggregation values that aren't part of "trendStats"; mainly for thresholds.
+	const trendKeys = {};
+
 	let maxNameWidth = 0;
 	let maxNonTrendValueLen = 0;
 	let nonTrendExtraMaxLens = []; // FIXME: "lens"?
@@ -1082,6 +1090,13 @@ function computeSummaryInfo(metrics, renderContext, options) {
 		maxNameWidth = Math.max(maxNameWidth, strWidth(displayName));
 
 		if (metric.type === 'trend') {
+			const keys = Object.keys(metric.values).reduce((acc, key) => {
+				if (!trendStats.includes(key)) {
+					acc[key] = renderTrendValue(metric.values[key], key, metric, options);
+				}
+				return acc;
+			}, {});
+
 			const cols = trendStats.map((stat) =>
 				renderTrendValue(metric.values[stat], stat, metric, options),
 			);
@@ -1094,6 +1109,7 @@ function computeSummaryInfo(metrics, renderContext, options) {
 				);
 			});
 			trendCols[name] = cols;
+			trendKeys[name] = keys;
 		} else {
 			const values = nonTrendMetricValueForSum(
 				metric,
@@ -1127,6 +1143,7 @@ function computeSummaryInfo(metrics, renderContext, options) {
 		nonTrendExtras,
 		trendCols,
 		trendColMaxLens,
+		trendKeys,
 		numTrendColumns,
 		trendStats,
 		maxNonTrendValueLen,

internal/output/summary/data.go

@@ -1,6 +1,8 @@
 package summary
 
 import (
+	"regexp"
+	"strconv"
 	"strings"
 	"sync/atomic"
 	"time"
@@ -245,6 +247,15 @@ func summaryThresholds(
 				Source: threshold.Source,
 				Ok:     !threshold.LastFailed,
 			})
+
+			// Additionally, if metric is a trend and the threshold source is a percentile,
+			// we may need to add the percentile value to the metric values, in case it's
+			// not one of [summaryTrendStats].
+			if trendSink, isTrend := mThresholds.Sink.(*metrics.TrendSink); isTrend {
+				if agg, percentile, isPercentile := extractPercentileThresholdSource(threshold.Source); isPercentile {
+					mt.Metric.Values[agg] = trendSink.P(percentile / 100)
+				}
+			}
 		}
 
 		rts[mName] = mt
@@ -393,3 +404,28 @@ func metricValueGetter(summaryTrendStats []string) func(metrics.Sink, time.Durat
 		return result
 	}
 }
+
+var percentileThresholdSourceRe = regexp.MustCompile(`^p\((\d+(?:\.\d+)?)\)\s*([<>=])`)
+
+func extractPercentileThresholdSource(source string) (agg string, percentile float64, isPercentile bool) {
+	// We capture the following three matches, in order to detect whether source is a percentile:
+	// 1. The percentile definition: p(...)
+	// 2. The percentile value: p(??)
+	// 3. The beginning of the operator: '<', '>', or '='
+	const expectedMatches = 3
+	matches := percentileThresholdSourceRe.FindStringSubmatch(strings.TrimSpace(source))
+
+	if len(matches) == expectedMatches {
+		var err error
+		percentile, err = strconv.ParseFloat(matches[1], 64)
+		if err != nil {
+			return "", 0, false
+		}
+
+		agg = "p(" + matches[1] + ")"
+		isPercentile = true
+		return
+	}
+
+	return "", 0, false
+}