Skip to content

Commit f9e1701

Browse files
authored
Show the number of test cases that ran at the end of a test (#972)
### Motivation: To have the number of test cases at the end of the test in for parameterized tests, so the logs don’t easily get buried. ### Modifications: Log the number of test cases for parameterized tests at the end of a test. ### Result: If the user runs the parameterized tests, an extra piece of information (with * test cases) would get bound to the end log. Resolves #943. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated.
1 parent c488e8f commit f9e1701

File tree

2 files changed

+69
-4
lines changed

2 files changed

+69
-4
lines changed

Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,9 @@ extension Event {
6262

6363
/// The number of known issues recorded for the test.
6464
var knownIssueCount = 0
65+
66+
/// The number of test cases for the test.
67+
var testCasesCount = 0
6568
}
6669

6770
/// Data tracked on a per-test basis.
@@ -281,6 +284,10 @@ extension Event.HumanReadableOutputRecorder {
281284
testData.issueCount[issue.severity] = issueCount + 1
282285
}
283286
context.testData[id] = testData
287+
288+
case .testCaseStarted:
289+
let test = test!
290+
context.testData[test.id.keyPathRepresentation]?.testCasesCount += 1
284291

285292
default:
286293
// These events do not manipulate the context structure.
@@ -366,18 +373,23 @@ extension Event.HumanReadableOutputRecorder {
366373
let testData = testDataGraph?.value ?? .init(startInstant: instant)
367374
let issues = _issueCounts(in: testDataGraph)
368375
let duration = testData.startInstant.descriptionOfDuration(to: instant)
376+
let testCasesCount = if test.isParameterized {
377+
" with \(testData.testCasesCount.counting("test case"))"
378+
} else {
379+
""
380+
}
369381
return if issues.errorIssueCount > 0 {
370382
CollectionOfOne(
371383
Message(
372384
symbol: .fail,
373-
stringValue: "\(_capitalizedTitle(for: test)) \(testName) failed after \(duration)\(issues.description)."
385+
stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCount) failed after \(duration)\(issues.description)."
374386
)
375387
) + _formattedComments(for: test)
376388
} else {
377389
[
378390
Message(
379391
symbol: .pass(knownIssueCount: issues.knownIssueCount),
380-
stringValue: "\(_capitalizedTitle(for: test)) \(testName) passed after \(duration)\(issues.description)."
392+
stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCount) passed after \(duration)\(issues.description)."
381393
)
382394
]
383395
}

Tests/TestingTests/EventRecorderTests.swift

Lines changed: 55 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,44 @@ struct EventRecorderTests {
178178
.first != nil
179179
)
180180
}
181+
182+
@available(_regexAPI, *)
183+
@Test(
184+
"Log the total number of test cases in parameterized tests at the end of the test run",
185+
arguments: [
186+
("f()", #".* Test f\(\) failed after .*"#),
187+
("h()", #".* Test h\(\) passed after .+"#),
188+
("l(_:)", #".* Test l\(_:\) with .+ test cases passed after.*"#),
189+
("m(_:)", #".* Test m\(_:\) with .+ test cases failed after.*"#),
190+
("n(_:)", #".* Test n\(_:\) with .+ test case passed after.*"#),
191+
("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"#),
192+
]
193+
)
194+
func numberOfTestCasesAtTestEnd(testName: String, expectedPattern: String) async throws {
195+
let stream = Stream()
196+
197+
var configuration = Configuration()
198+
let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write)
199+
configuration.eventHandler = { event, context in
200+
eventRecorder.record(event, in: context)
201+
}
202+
203+
await runTest(for: PredictablyFailingTests.self, configuration: configuration)
204+
205+
let buffer = stream.buffer.rawValue
206+
if testsWithSignificantIOAreEnabled {
207+
print(buffer, terminator: "")
208+
}
209+
210+
let aurgmentRegex = try Regex(expectedPattern)
211+
212+
#expect(
213+
(try buffer
214+
.split(whereSeparator: \.isNewline)
215+
.compactMap(aurgmentRegex.wholeMatch(in:))
216+
.first) != nil
217+
)
218+
}
181219

182220
@available(_regexAPI, *)
183221
@Test(
@@ -189,7 +227,7 @@ struct EventRecorderTests {
189227
("i()", #".* Test i\(\) failed after .+ seconds with 2 issues \(including 1 warning\)\."#),
190228
("j()", #".* Test j\(\) passed after .+ seconds with 1 warning and 1 known issue\."#),
191229
("k()", #".* Test k\(\) passed after .+ seconds with 1 known issue\."#),
192-
("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .+ seconds with 13 issues \(including 3 warnings and 6 known issues\)\."#),
230+
("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .+ seconds with 16 issues \(including 3 warnings and 6 known issues\)\."#),
193231
]
194232
)
195233
func issueCountSummingAtTestEnd(testName: String, expectedPattern: String) async throws {
@@ -284,7 +322,7 @@ struct EventRecorderTests {
284322
.compactMap(runFailureRegex.wholeMatch(in:))
285323
.first
286324
)
287-
#expect(match.output.1 == 9)
325+
#expect(match.output.1 == 12)
288326
#expect(match.output.2 == 5)
289327
}
290328

@@ -565,4 +603,19 @@ struct EventRecorderTests {
565603
Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record()
566604
}
567605
}
606+
607+
@Test(.hidden, arguments: [1, 2, 3])
608+
func l(_ arg: Int) {
609+
#expect(arg > 0)
610+
}
611+
612+
@Test(.hidden, arguments: [1, 2, 3])
613+
func m(_ arg: Int) {
614+
#expect(arg < 0)
615+
}
616+
617+
@Test(.hidden, arguments: [1])
618+
func n(_ arg: Int) {
619+
#expect(arg > 0)
620+
}
568621
}

0 commit comments

Comments
 (0)