Skip to content

Commit 2ba9d45

Browse files
jeanbzadmitshur
jeanbza
authored and
dmitshur
committed
[release-branch.go1.14] testing: reformat test chatty output
In #24929, we decided to stream chatty test output. It looks like, foo_test.go:138: TestFoo/sub-1: hello from subtest 1 foo_test.go:138: TestFoo/sub-2: hello from subtest 2 In this CL, we refactor the output to be grouped by === CONT lines, preserving the old test-file-before-log-line behavior: === CONT TestFoo/sub-1 foo_test.go:138 hello from subtest 1 === CONT TestFoo/sub-2 foo_test.go:138 hello from subtest 2 This should remove a layer of verbosity from tests, and make it easier to group together related lines. It also returns to a more familiar format (the pre-streaming format), whilst still preserving the streaming feature. Updates #38458. Fixes #39308. Change-Id: Iaef94c580d69cdd541b2ef055aa004f50d72d078 Reviewed-on: https://go-review.googlesource.com/c/go/+/229085 Run-TryBot: Emmanuel Odeke <[email protected]> TryBot-Result: Gobot Gobot <[email protected]> Reviewed-by: Bryan C. Mills <[email protected]> Reviewed-by: Andrew Bonventre <[email protected]> Reviewed-on: https://go-review.googlesource.com/c/go/+/242057 Reviewed-by: Jean de Klerk <[email protected]> Reviewed-by: Emmanuel Odeke <[email protected]> Run-TryBot: Jean de Klerk <[email protected]>
1 parent 399ce80 commit 2ba9d45

10 files changed

+370
-84
lines changed

‎src/cmd/go/testdata/script/test_benchmark_chatty_fail.txt

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Run chatty tests. Assert on CONT lines.
2+
! go test chatty_test.go -v -bench . chatty_bench
3+
4+
# Sanity check that output occurs.
5+
stdout -count=2 'this is sub-0'
6+
stdout -count=2 'this is sub-1'
7+
stdout -count=2 'this is sub-2'
8+
stdout -count=1 'error from sub-0'
9+
stdout -count=1 'error from sub-1'
10+
stdout -count=1 'error from sub-2'
11+
12+
# Benchmarks should not print CONT.
13+
! stdout CONT
14+
15+
-- chatty_test.go --
16+
package chatty_bench
17+
18+
import (
19+
"testing"
20+
"fmt"
21+
)
22+
23+
func BenchmarkChatty(b *testing.B) {
24+
for i := 0; i < 3; i++ {
25+
b.Run(fmt.Sprintf("sub-%d", i), func(b *testing.B) {
26+
for j := 0; j < 2; j++ {
27+
b.Logf("this is sub-%d", i)
28+
}
29+
b.Errorf("error from sub-%d", i)
30+
})
31+
}
32+
}

‎src/cmd/go/testdata/script/test_benchmark_chatty_success.txt

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Run chatty tests. Assert on CONT lines.
2+
go test chatty_test.go -v -bench . chatty_bench
3+
4+
# Sanity check that output happens. We don't provide -count because the amount
5+
# of output is variable.
6+
stdout 'this is sub-0'
7+
stdout 'this is sub-1'
8+
stdout 'this is sub-2'
9+
10+
# Benchmarks should not print CONT.
11+
! stdout CONT
12+
13+
-- chatty_test.go --
14+
package chatty_bench
15+
16+
import (
17+
"testing"
18+
"fmt"
19+
)
20+
21+
func BenchmarkChatty(b *testing.B) {
22+
for i := 0; i < 3; i++ {
23+
b.Run(fmt.Sprintf("sub-%d", i), func(b *testing.B) {
24+
for j := 0; j < 2; j++ {
25+
b.Logf("this is sub-%d", i)
26+
}
27+
})
28+
}
29+
}

‎src/cmd/go/testdata/script/test_chatty_fail.txt

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Run chatty tests. Assert on CONT lines.
2+
! go test chatty_test.go -v
3+
4+
# Sanity check that output occurs.
5+
stdout -count=2 'this is sub-0'
6+
stdout -count=2 'this is sub-1'
7+
stdout -count=2 'this is sub-2'
8+
stdout -count=1 'error from sub-0'
9+
stdout -count=1 'error from sub-1'
10+
stdout -count=1 'error from sub-2'
11+
12+
# Non-parallel tests should not print CONT.
13+
! stdout CONT
14+
15+
-- chatty_test.go --
16+
package chatty_test
17+
18+
import (
19+
"testing"
20+
"fmt"
21+
)
22+
23+
func TestChatty(t *testing.T) {
24+
for i := 0; i < 3; i++ {
25+
t.Run(fmt.Sprintf("sub-%d", i), func(t *testing.T) {
26+
for j := 0; j < 2; j++ {
27+
t.Logf("this is sub-%d", i)
28+
}
29+
t.Errorf("error from sub-%d", i)
30+
})
31+
}
32+
}

‎src/cmd/go/testdata/script/test_chatty_parallel_fail.txt

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Run parallel chatty tests. Assert on CONT lines. This test makes sure that
2+
# multiple parallel outputs have the appropriate CONT lines between them.
3+
! go test -parallel 3 chatty_parallel_test.go -v
4+
5+
stdout -count=1 '^=== CONT TestChattyParallel/sub-0\n chatty_parallel_test.go:38: error from sub-0$'
6+
stdout -count=1 '^=== CONT TestChattyParallel/sub-1\n chatty_parallel_test.go:38: error from sub-1$'
7+
stdout -count=1 '^=== CONT TestChattyParallel/sub-2\n chatty_parallel_test.go:38: error from sub-2$'
8+
9+
# Run parallel chatty tests with -json. Assert on CONT lines as above - make
10+
# sure there are CONT lines before each output line.
11+
! go test -json -parallel 3 chatty_parallel_test.go -v
12+
stdout -count=1 '{"Time":"[0-9-TZ:.]{20,40}","Action":"output","Package":"command-line-arguments","Test":"TestChattyParallel/sub-0","Output":"=== CONT TestChattyParallel/sub-0\\n"}\n{"Time":"[0-9-TZ:.]{20,40}","Action":"output","Package":"command-line-arguments","Test":"TestChattyParallel/sub-0","Output":" chatty_parallel_test.go:38: error from sub-0\\n"}'
13+
stdout -count=1 '{"Time":"[0-9-TZ:.]{20,40}","Action":"output","Package":"command-line-arguments","Test":"TestChattyParallel/sub-1","Output":"=== CONT TestChattyParallel/sub-1\\n"}\n{"Time":"[0-9-TZ:.]{20,40}","Action":"output","Package":"command-line-arguments","Test":"TestChattyParallel/sub-1","Output":" chatty_parallel_test.go:38: error from sub-1\\n"}'
14+
stdout -count=1 '{"Time":"[0-9-TZ:.]{20,40}","Action":"output","Package":"command-line-arguments","Test":"TestChattyParallel/sub-2","Output":"=== CONT TestChattyParallel/sub-2\\n"}\n{"Time":"[0-9-TZ:.]{20,40}","Action":"output","Package":"command-line-arguments","Test":"TestChattyParallel/sub-2","Output":" chatty_parallel_test.go:38: error from sub-2\\n"}'
15+
16+
-- chatty_parallel_test.go --
17+
package chatty_paralell_test
18+
19+
import (
20+
"testing"
21+
"fmt"
22+
"flag"
23+
)
24+
25+
// This test ensures the the order of CONT lines in parallel chatty tests.
26+
func TestChattyParallel(t *testing.T) {
27+
t.Parallel()
28+
29+
// The number of concurrent tests running. This is closely tied to the
30+
// -parallel test flag, so we grab it from the flag rather than setting it
31+
// to some constant.
32+
parallel := flag.Lookup("test.parallel").Value.(flag.Getter).Get().(int)
33+
34+
// ready is a synchronization mechanism that causes subtests to execute
35+
// round robin.
36+
ready := make([]chan bool, parallel)
37+
for i := range ready {
38+
ready[i] = make(chan bool, 1)
39+
}
40+
ready[0] <- true
41+
42+
for i := range ready {
43+
i := i
44+
t.Run(fmt.Sprintf("sub-%d", i), func(t *testing.T) {
45+
t.Parallel()
46+
47+
// Some basic log output to precede the failures.
48+
<-ready[i]
49+
t.Logf("this is sub-%d", i)
50+
ready[(i+1)%len(ready)] <- true
51+
52+
// The actual failure messages we care about.
53+
<-ready[i]
54+
t.Errorf("error from sub-%d", i)
55+
ready[(i+1)%len(ready)] <- true
56+
})
57+
}
58+
}

‎src/cmd/go/testdata/script/test_chatty_parallel_success.txt

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Run parallel chatty tests. Assert on CONT lines. This test makes sure that
2+
# multiple parallel outputs have the appropriate CONT lines between them.
3+
go test -parallel 3 chatty_parallel_test.go -v
4+
stdout -count=2 '^=== CONT TestChattyParallel/sub-0\n chatty_parallel_test.go:32: this is sub-0$'
5+
stdout -count=2 '^=== CONT TestChattyParallel/sub-1\n chatty_parallel_test.go:32: this is sub-1$'
6+
stdout -count=2 '^=== CONT TestChattyParallel/sub-2\n chatty_parallel_test.go:32: this is sub-2$'
7+
8+
# Run parallel chatty tests with -json. Assert on CONT lines as above - make
9+
# sure there are CONT lines before each output line.
10+
go test -json -parallel 3 chatty_parallel_test.go -v
11+
stdout -count=2 '{"Time":"[0-9-TZ:.]{20,40}","Action":"output","Package":"command-line-arguments","Test":"TestChattyParallel/sub-0","Output":"=== CONT TestChattyParallel/sub-0\\n"}\n{"Time":"[0-9-TZ:.]{20,40}","Action":"output","Package":"command-line-arguments","Test":"TestChattyParallel/sub-0","Output":" chatty_parallel_test.go:32: this is sub-0\\n"}'
12+
stdout -count=2 '{"Time":"[0-9-TZ:.]{20,40}","Action":"output","Package":"command-line-arguments","Test":"TestChattyParallel/sub-1","Output":"=== CONT TestChattyParallel/sub-1\\n"}\n{"Time":"[0-9-TZ:.]{20,40}","Action":"output","Package":"command-line-arguments","Test":"TestChattyParallel/sub-1","Output":" chatty_parallel_test.go:32: this is sub-1\\n"}'
13+
stdout -count=2 '{"Time":"[0-9-TZ:.]{20,40}","Action":"output","Package":"command-line-arguments","Test":"TestChattyParallel/sub-2","Output":"=== CONT TestChattyParallel/sub-2\\n"}\n{"Time":"[0-9-TZ:.]{20,40}","Action":"output","Package":"command-line-arguments","Test":"TestChattyParallel/sub-2","Output":" chatty_parallel_test.go:32: this is sub-2\\n"}'
14+
15+
-- chatty_parallel_test.go --
16+
package chatty_paralell_test
17+
18+
import (
19+
"testing"
20+
"fmt"
21+
"flag"
22+
)
23+
24+
// This test ensures the the order of CONT lines in parallel chatty tests.
25+
func TestChattyParallel(t *testing.T) {
26+
t.Parallel()
27+
28+
// The number of concurrent tests running. This is closely tied to the
29+
// -parallel test flag, so we grab it from the flag rather than setting it
30+
// to some constant.
31+
parallel := flag.Lookup("test.parallel").Value.(flag.Getter).Get().(int)
32+
33+
// ready is a synchronization mechanism that causes subtests to execute
34+
// round robin.
35+
ready := make([]chan bool, parallel)
36+
for i := range ready {
37+
ready[i] = make(chan bool, 1)
38+
}
39+
ready[0] <- true
40+
41+
for i := range ready {
42+
i := i
43+
t.Run(fmt.Sprintf("sub-%d", i), func(t *testing.T) {
44+
t.Parallel()
45+
for j := 0; j < 2; j++ {
46+
<-ready[i]
47+
t.Logf("this is sub-%d", i)
48+
ready[(i+1)%len(ready)] <- true
49+
}
50+
})
51+
}
52+
}

‎src/cmd/go/testdata/script/test_chatty_success.txt

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Run chatty tests. Assert on CONT lines.
2+
go test chatty_test.go -v
3+
4+
# Non-parallel tests should not print CONT.
5+
! stdout CONT
6+
7+
# The assertion is condensed into one line so that it precisely matches output,
8+
# rather than skipping lines and allow rogue CONT lines.
9+
stdout '=== RUN TestChatty\n=== RUN TestChatty/sub-0\n chatty_test.go:12: this is sub-0\n chatty_test.go:12: this is sub-0\n=== RUN TestChatty/sub-1\n chatty_test.go:12: this is sub-1\n chatty_test.go:12: this is sub-1\n=== RUN TestChatty/sub-2\n chatty_test.go:12: this is sub-2\n chatty_test.go:12: this is sub-2\n--- PASS: TestChatty'
10+
11+
-- chatty_test.go --
12+
package chatty_test
13+
14+
import (
15+
"testing"
16+
"fmt"
17+
)
18+
19+
func TestChatty(t *testing.T) {
20+
for i := 0; i < 3; i++ {
21+
t.Run(fmt.Sprintf("sub-%d", i), func(t *testing.T) {
22+
for j := 0; j < 2; j++ {
23+
t.Logf("this is sub-%d", i)
24+
}
25+
})
26+
}
27+
}

‎src/cmd/go/testdata/script/test_regexps.txt

+7-7
Original file line numberDiff line numberDiff line change
@@ -4,36 +4,36 @@ go test -cpu=1 -run=X/Y -bench=X/Y -count=2 -v testregexp
44

55
# TestX is run, twice
66
stdout -count=2 '^=== RUN TestX$'
7-
stdout -count=2 '^ TestX: x_test.go:6: LOG: X running$'
7+
stdout -count=2 '^ x_test.go:6: LOG: X running$'
88

99
# TestX/Y is run, twice
1010
stdout -count=2 '^=== RUN TestX/Y$'
11-
stdout -count=2 '^ TestX/Y: x_test.go:8: LOG: Y running$'
11+
stdout -count=2 '^ x_test.go:8: LOG: Y running$'
1212

1313
# TestXX is run, twice
1414
stdout -count=2 '^=== RUN TestXX$'
15-
stdout -count=2 '^ TestXX: z_test.go:10: LOG: XX running'
15+
stdout -count=2 '^ z_test.go:10: LOG: XX running'
1616

1717
# TestZ is not run
1818
! stdout '^=== RUN TestZ$'
1919

2020
# BenchmarkX is run with N=1 once, only to discover what sub-benchmarks it has,
2121
# and should not print a final summary line.
22-
stdout -count=1 '^\s+BenchmarkX: x_test.go:13: LOG: X running N=1$'
22+
stdout -count=1 '^ x_test.go:13: LOG: X running N=1$'
2323
! stdout '^\s+BenchmarkX: x_test.go:13: LOG: X running N=\d\d+'
2424
! stdout 'BenchmarkX\s+\d+'
2525

2626
# Same for BenchmarkXX.
27-
stdout -count=1 '^\s+BenchmarkXX: z_test.go:18: LOG: XX running N=1$'
28-
! stdout '^\s+BenchmarkXX: z_test.go:18: LOG: XX running N=\d\d+'
27+
stdout -count=1 '^ z_test.go:18: LOG: XX running N=1$'
28+
! stdout '^ z_test.go:18: LOG: XX running N=\d\d+'
2929
! stdout 'BenchmarkXX\s+\d+'
3030

3131
# BenchmarkX/Y is run in full twice due to -count=2.
3232
# "Run in full" means that it runs for approximately the default benchtime,
3333
# but may cap out at N=1e9.
3434
# We don't actually care what the final iteration count is, but it should be
3535
# a large number, and the last iteration count prints right before the results.
36-
stdout -count=2 '^\s+BenchmarkX/Y: x_test.go:15: LOG: Y running N=[1-9]\d{4,}\nBenchmarkX/Y\s+\d+'
36+
stdout -count=2 '^ x_test.go:15: LOG: Y running N=[1-9]\d{4,}\nBenchmarkX/Y\s+\d+'
3737

3838
-- testregexp/x_test.go --
3939
package x

‎src/testing/benchmark.go

+3
Original file line numberDiff line numberDiff line change
@@ -526,6 +526,7 @@ func runBenchmarks(importPath string, matchString func(pat, str string) (bool, e
526526
name: "Main",
527527
w: os.Stdout,
528528
chatty: *chatty,
529+
bench: true,
529530
},
530531
importPath: importPath,
531532
benchFunc: func(b *B) {
@@ -559,6 +560,7 @@ func (ctx *benchContext) processBench(b *B) {
559560
name: b.name,
560561
w: b.w,
561562
chatty: b.chatty,
563+
bench: true,
562564
},
563565
benchFunc: b.benchFunc,
564566
benchTime: b.benchTime,
@@ -624,6 +626,7 @@ func (b *B) Run(name string, f func(b *B)) bool {
624626
creator: pc[:n],
625627
w: b.w,
626628
chatty: b.chatty,
629+
bench: true,
627630
},
628631
importPath: b.importPath,
629632
benchFunc: f,

0 commit comments

Comments
 (0)