Partially support BenchmarkXXX functions with the -bench and -test.bench flags

.github/workflows/linux.yml

@@ -140,6 +140,8 @@ jobs:
           submodules: true
       - name: Install apt dependencies
         run: |
+          echo "Show cpuinfo; sometimes useful when troubleshooting"
+          cat /proc/cpuinfo
           sudo apt-get update
           sudo apt-get install --no-install-recommends \
               qemu-system-arm \

main.go

@@ -178,7 +178,7 @@ func Build(pkgName, outpath string, options *compileopts.Options) error {
 
 // Test runs the tests in the given package. Returns whether the test passed and
 // possibly an error if the test failed to run.
-func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options, testCompileOnly, testVerbose, testShort bool, testRunRegexp string, outpath string) (bool, error) {
+func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options, testCompileOnly, testVerbose, testShort bool, testRunRegexp string, testBenchRegexp string, outpath string) (bool, error) {
 	options.TestConfig.CompileTestBinary = true
 	config, err := builder.NewConfig(options)
 	if err != nil {
@@ -209,7 +209,7 @@ func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options
 		}()
 		start := time.Now()
 		var err error
-		passed, err = runPackageTest(config, stdout, stderr, result, testVerbose, testShort, testRunRegexp)
+		passed, err = runPackageTest(config, stdout, stderr, result, testVerbose, testShort, testRunRegexp, testBenchRegexp)
 		if err != nil {
 			return err
 		}
@@ -235,7 +235,7 @@ func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options
 // runPackageTest runs a test binary that was previously built. The return
 // values are whether the test passed and any errors encountered while trying to
 // run the binary.
-func runPackageTest(config *compileopts.Config, stdout, stderr io.Writer, result builder.BuildResult, testVerbose, testShort bool, testRunRegexp string) (bool, error) {
+func runPackageTest(config *compileopts.Config, stdout, stderr io.Writer, result builder.BuildResult, testVerbose, testShort bool, testRunRegexp string, testBenchRegexp string) (bool, error) {
 	var cmd *exec.Cmd
 	if len(config.Target.Emulator) == 0 {
 		// Run directly.
@@ -249,6 +249,9 @@ func runPackageTest(config *compileopts.Config, stdout, stderr io.Writer, result
 		if testRunRegexp != "" {
 			flags = append(flags, "-test.run="+testRunRegexp)
 		}
+		if testBenchRegexp != "" {
+			flags = append(flags, "-test.bench="+testBenchRegexp)
+		}
 		cmd = executeCommand(config.Options, result.Binary, flags...)
 	} else {
 		// Run in an emulator.
@@ -274,6 +277,9 @@ func runPackageTest(config *compileopts.Config, stdout, stderr io.Writer, result
 			if testRunRegexp != "" {
 				args = append(args, "-test.run="+testRunRegexp)
 			}
+			if testBenchRegexp != "" {
+				args = append(args, "-test.bench="+testBenchRegexp)
+			}
 		}
 		cmd = executeCommand(config.Options, config.Target.Emulator[0], args...)
 	}
@@ -1197,12 +1203,14 @@ func main() {
 		flag.StringVar(&outpath, "o", "", "output filename")
 	}
 	var testCompileOnlyFlag, testVerboseFlag, testShortFlag *bool
+	var testBenchRegexp *string
 	var testRunRegexp *string
 	if command == "help" || command == "test" {
 		testCompileOnlyFlag = flag.Bool("c", false, "compile the test binary but do not run it")
 		testVerboseFlag = flag.Bool("v", false, "verbose: print additional output")
 		testShortFlag = flag.Bool("short", false, "short: run smaller test suite to save time")
 		testRunRegexp = flag.String("run", "", "run: regexp of tests to run")
+		testBenchRegexp = flag.String("bench", "", "run: regexp of benchmarks to run")
 	}
 
 	// Early command processing, before commands are interpreted by the Go flag
@@ -1426,7 +1434,7 @@ func main() {
 				defer close(buf.done)
 				stdout := (*testStdout)(buf)
 				stderr := (*testStderr)(buf)
-				passed, err := Test(pkgName, stdout, stderr, options, *testCompileOnlyFlag, *testVerboseFlag, *testShortFlag, *testRunRegexp, outpath)
+				passed, err := Test(pkgName, stdout, stderr, options, *testCompileOnlyFlag, *testVerboseFlag, *testShortFlag, *testRunRegexp, *testBenchRegexp, outpath)
 				if err != nil {
 					printCompilerError(func(args ...interface{}) {
 						fmt.Fprintln(stderr, args...)

main_test.go

@@ -516,7 +516,7 @@ func TestTest(t *testing.T) {
 				defer out.Close()
 
 				opts := targ.opts
-				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/pass", out, out, &opts, false, false, false, "", "")
+				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/pass", out, out, &opts, false, false, false, "", "", "")
 				if err != nil {
 					t.Errorf("test error: %v", err)
 				}
@@ -537,7 +537,7 @@ func TestTest(t *testing.T) {
 				defer out.Close()
 
 				opts := targ.opts
-				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/fail", out, out, &opts, false, false, false, "", "")
+				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/fail", out, out, &opts, false, false, false, "", "", "")
 				if err != nil {
 					t.Errorf("test error: %v", err)
 				}
@@ -564,7 +564,7 @@ func TestTest(t *testing.T) {
 
 				var output bytes.Buffer
 				opts := targ.opts
-				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/nothing", io.MultiWriter(&output, out), out, &opts, false, false, false, "", "")
+				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/nothing", io.MultiWriter(&output, out), out, &opts, false, false, false, "", "", "")
 				if err != nil {
 					t.Errorf("test error: %v", err)
 				}
@@ -588,7 +588,7 @@ func TestTest(t *testing.T) {
 				defer out.Close()
 
 				opts := targ.opts
-				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/builderr", out, out, &opts, false, false, false, "", "")
+				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/builderr", out, out, &opts, false, false, false, "", "", "")
 				if err == nil {
 					t.Error("test did not error")
 				}

src/testing/benchmark.go

@@ -7,6 +7,7 @@
 package testing
 
 import (
+	"fmt"
 	"time"
 )
 
@@ -18,8 +19,26 @@ type benchTimeFlag struct {
 	d time.Duration
 }
 
-// B is a type passed to Benchmark functions to manage benchmark timing and to
-// specify the number of iterations to run.
+// InternalBenchmark is an internal type but exported because it is cross-package;
+// it is part of the implementation of the "go test" command.
+type InternalBenchmark struct {
+	Name string
+	F    func(b *B)
+}
+
+// B is a type passed to Benchmark functions to manage benchmark
+// timing and to specify the number of iterations to run.
+//
+// A benchmark ends when its Benchmark function returns or calls any of the methods
+// FailNow, Fatal, Fatalf, SkipNow, Skip, or Skipf. Those methods must be called
+// only from the goroutine running the Benchmark function.
+// The other reporting methods, such as the variations of Log and Error,
+// may be called simultaneously from multiple goroutines.
+//
+// Like in tests, benchmark logs are accumulated during execution
+// and dumped to standard output when done. Unlike in tests, benchmark logs
+// are always printed, so as not to hide output whose existence may be
+// affecting benchmark results.
 type B struct {
 	common
 	hasSub    bool          // TODO: should be in common, and atomic
@@ -32,49 +51,6 @@ type B struct {
 	result    BenchmarkResult
 }
 
-// InternalBenchmark is an internal type but exported because it is cross-package;
-// it is part of the implementation of the "go test" command.
-type InternalBenchmark struct {
-	Name string
-	F    func(b *B)
-}
-
-// BenchmarkResult contains the results of a benchmark run.
-type BenchmarkResult struct {
-	N int           // The number of iterations.
-	T time.Duration // The total time taken.
-}
-
-// NsPerOp returns the "ns/op" metric.
-func (r BenchmarkResult) NsPerOp() int64 {
-	if r.N <= 0 {
-		return 0
-	}
-	return r.T.Nanoseconds() / int64(r.N)
-}
-
-// AllocsPerOp returns the "allocs/op" metric,
-// which is calculated as r.MemAllocs / r.N.
-func (r BenchmarkResult) AllocsPerOp() int64 {
-	return 0 // Dummy version to allow running e.g. golang.org/test/fibo.go
-}
-
-// AllocedBytesPerOp returns the "B/op" metric,
-// which is calculated as r.MemBytes / r.N.
-func (r BenchmarkResult) AllocedBytesPerOp() int64 {
-	return 0 // Dummy version to allow running e.g. golang.org/test/fibo.go
-}
-
-func (b *B) SetBytes(n int64) {
-	panic("testing: unimplemented: B.SetBytes")
-}
-
-// ReportAllocs enables malloc statistics for this benchmark.
-func (b *B) ReportAllocs() {
-	// Dummy version to allow building e.g. golang.org/crypto/...
-	panic("testing: unimplemented: B.ReportAllocs")
-}
-
 // StartTimer starts timing a test. This function is called automatically
 // before a benchmark starts, but it can also be used to resume timing after
 // a call to StopTimer.
@@ -95,15 +71,28 @@ func (b *B) StopTimer() {
 	}
 }
 
-// ResetTimer zeroes the elapsed benchmark time.
-// It does not affect whether the timer is running.
+// ResetTimer zeroes the elapsed benchmark time and memory allocation counters
+// and deletes user-reported metrics.
 func (b *B) ResetTimer() {
 	if b.timerOn {
 		b.start = time.Now()
 	}
 	b.duration = 0
 }
 
+// SetBytes records the number of bytes processed in a single operation.
+// If this is called, the benchmark will report ns/op and MB/s.
+func (b *B) SetBytes(n int64) {
+	panic("testing: unimplemented: B.SetBytes")
+}
+
+// ReportAllocs enables malloc statistics for this benchmark.
+// It is equivalent to setting -test.benchmem, but it only affects the
+// benchmark function that calls ReportAllocs.
+func (b *B) ReportAllocs() {
+	panic("testing: unimplemented: B.ReportAllocs")
+}
+
 // runN runs a single benchmark for the specified number of iterations.
 func (b *B) runN(n int) {
 	b.N = n
@@ -173,6 +162,56 @@ func (b *B) launch() {
 	b.result = BenchmarkResult{b.N, b.duration}
 }
 
+// BenchmarkResult contains the results of a benchmark run.
+type BenchmarkResult struct {
+	N int           // The number of iterations.
+	T time.Duration // The total time taken.
+}
+
+// NsPerOp returns the "ns/op" metric.
+func (r BenchmarkResult) NsPerOp() int64 {
+	if r.N <= 0 {
+		return 0
+	}
+	return r.T.Nanoseconds() / int64(r.N)
+}
+
+// AllocsPerOp returns the "allocs/op" metric,
+// which is calculated as r.MemAllocs / r.N.
+func (r BenchmarkResult) AllocsPerOp() int64 {
+	return 0 // Dummy version to allow running e.g. golang.org/test/fibo.go
+}
+
+// AllocedBytesPerOp returns the "B/op" metric,
+// which is calculated as r.MemBytes / r.N.
+func (r BenchmarkResult) AllocedBytesPerOp() int64 {
+	return 0 // Dummy version to allow running e.g. golang.org/test/fibo.go
+}
+
+func runBenchmarks(benchmarks []InternalBenchmark) bool {
+	if len(benchmarks) == 0 {
+		return true
+	}
+	main := &B{
+		common: common{
+			name: "Main",
+		},
+		benchTime: benchTime,
+		benchFunc: func(b *B) {
+			for _, Benchmark := range benchmarks {
+				if flagVerbose {
+					fmt.Printf("=== RUN   %s\n", Benchmark.Name)
+				}
+				b.Run(Benchmark.Name, Benchmark.F)
+				fmt.Printf("--- Result: %d ns/op\n", b.result.NsPerOp())
+			}
+		},
+	}
+
+	main.runN(1)
+	return true
+}
+
 // Run benchmarks f as a subbenchmark with the given name. It reports
 // true if the subbenchmark succeeded.
 //
@@ -192,22 +231,6 @@ func (b *B) Run(name string, f func(b *B)) bool {
 	return !sub.failed
 }
 
-// Benchmark benchmarks a single function. It is useful for creating
-// custom benchmarks that do not use the "go test" command.
-//
-// If f calls Run, the result will be an estimate of running all its
-// subbenchmarks that don't call Run in sequence in a single benchmark.
-func Benchmark(f func(b *B)) BenchmarkResult {
-	b := &B{
-		benchFunc: f,
-		benchTime: benchTime,
-	}
-	if b.run1() {
-		b.run()
-	}
-	return b.result
-}
-
 // add simulates running benchmarks in sequence in a single iteration. It is
 // used to give some meaningful results in case func Benchmark is used in
 // combination with Run.
@@ -234,3 +257,19 @@ func (pb *PB) Next() bool {
 func (b *B) RunParallel(body func(*PB)) {
 	return
 }
+
+// Benchmark benchmarks a single function. It is useful for creating
+// custom benchmarks that do not use the "go test" command.
+//
+// If f calls Run, the result will be an estimate of running all its
+// subbenchmarks that don't call Run in sequence in a single benchmark.
+func Benchmark(f func(b *B)) BenchmarkResult {
+	b := &B{
+		benchFunc: f,
+		benchTime: benchTime,
+	}
+	if b.run1() {
+		b.run()
+	}
+	return b.result
+}