src/testing: support -bench option to run benchmarks matching the given pattern.

Author: dkegel-fastly

main.go

@@ -178,7 +178,7 @@ func Build(pkgName, outpath string, options *compileopts.Options) error {
 
 // Test runs the tests in the given package. Returns whether the test passed and
 // possibly an error if the test failed to run.
-func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options, testCompileOnly, testVerbose, testShort bool, testRunRegexp string, outpath string) (bool, error) {
+func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options, testCompileOnly, testVerbose, testShort bool, testRunRegexp string, testBenchRegexp string, outpath string) (bool, error) {
 	options.TestConfig.CompileTestBinary = true
 	config, err := builder.NewConfig(options)
 	if err != nil {
@@ -209,7 +209,7 @@ func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options
 		}()
 		start := time.Now()
 		var err error
-		passed, err = runPackageTest(config, stdout, stderr, result, testVerbose, testShort, testRunRegexp)
+		passed, err = runPackageTest(config, stdout, stderr, result, testVerbose, testShort, testRunRegexp, testBenchRegexp)
 		if err != nil {
 			return err
 		}
@@ -235,7 +235,7 @@ func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options
 // runPackageTest runs a test binary that was previously built. The return
 // values are whether the test passed and any errors encountered while trying to
 // run the binary.
-func runPackageTest(config *compileopts.Config, stdout, stderr io.Writer, result builder.BuildResult, testVerbose, testShort bool, testRunRegexp string) (bool, error) {
+func runPackageTest(config *compileopts.Config, stdout, stderr io.Writer, result builder.BuildResult, testVerbose, testShort bool, testRunRegexp string, testBenchRegexp string) (bool, error) {
 	var cmd *exec.Cmd
 	if len(config.Target.Emulator) == 0 {
 		// Run directly.
@@ -249,6 +249,9 @@ func runPackageTest(config *compileopts.Config, stdout, stderr io.Writer, result
 		if testRunRegexp != "" {
 			flags = append(flags, "-test.run="+testRunRegexp)
 		}
+		if testBenchRegexp != "" {
+			flags = append(flags, "-test.bench="+testBenchRegexp)
+		}
 		cmd = executeCommand(config.Options, result.Binary, flags...)
 	} else {
 		// Run in an emulator.
@@ -266,6 +269,9 @@ func runPackageTest(config *compileopts.Config, stdout, stderr io.Writer, result
 			if testRunRegexp != "" {
 				args = append(args, "-test.run="+testRunRegexp)
 			}
+			if testBenchRegexp != "" {
+				args = append(args, "-test.bench="+testBenchRegexp)
+			}
 		}
 		cmd = executeCommand(config.Options, config.Target.Emulator[0], args...)
 	}
@@ -1189,12 +1195,14 @@ func main() {
 		flag.StringVar(&outpath, "o", "", "output filename")
 	}
 	var testCompileOnlyFlag, testVerboseFlag, testShortFlag *bool
+	var testBenchRegexp *string
 	var testRunRegexp *string
 	if command == "help" || command == "test" {
 		testCompileOnlyFlag = flag.Bool("c", false, "compile the test binary but do not run it")
 		testVerboseFlag = flag.Bool("v", false, "verbose: print additional output")
 		testShortFlag = flag.Bool("short", false, "short: run smaller test suite to save time")
 		testRunRegexp = flag.String("run", "", "run: regexp of tests to run")
+		testBenchRegexp = flag.String("bench", "", "run: regexp of benchmarks to run")
 	}
 
 	// Early command processing, before commands are interpreted by the Go flag
@@ -1418,7 +1426,7 @@ func main() {
 				defer close(buf.done)
 				stdout := (*testStdout)(buf)
 				stderr := (*testStderr)(buf)
-				passed, err := Test(pkgName, stdout, stderr, options, *testCompileOnlyFlag, *testVerboseFlag, *testShortFlag, *testRunRegexp, outpath)
+				passed, err := Test(pkgName, stdout, stderr, options, *testCompileOnlyFlag, *testVerboseFlag, *testShortFlag, *testRunRegexp, *testBenchRegexp, outpath)
 				if err != nil {
 					printCompilerError(func(args ...interface{}) {
 						fmt.Fprintln(stderr, args...)

main_test.go

@@ -516,7 +516,7 @@ func TestTest(t *testing.T) {
 				defer out.Close()
 
 				opts := targ.opts
-				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/pass", out, out, &opts, false, false, false, "", "")
+				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/pass", out, out, &opts, false, false, false, "", "", "")
 				if err != nil {
 					t.Errorf("test error: %v", err)
 				}
@@ -537,7 +537,7 @@ func TestTest(t *testing.T) {
 				defer out.Close()
 
 				opts := targ.opts
-				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/fail", out, out, &opts, false, false, false, "", "")
+				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/fail", out, out, &opts, false, false, false, "", "", "")
 				if err != nil {
 					t.Errorf("test error: %v", err)
 				}
@@ -564,7 +564,7 @@ func TestTest(t *testing.T) {
 
 				var output bytes.Buffer
 				opts := targ.opts
-				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/nothing", io.MultiWriter(&output, out), out, &opts, false, false, false, "", "")
+				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/nothing", io.MultiWriter(&output, out), out, &opts, false, false, false, "", "", "")
 				if err != nil {
 					t.Errorf("test error: %v", err)
 				}
@@ -588,7 +588,7 @@ func TestTest(t *testing.T) {
 				defer out.Close()
 
 				opts := targ.opts
-				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/builderr", out, out, &opts, false, false, false, "", "")
+				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/builderr", out, out, &opts, false, false, false, "", "", "")
 				if err == nil {
 					t.Error("test did not error")
 				}

src/testing/benchmark.go

@@ -7,6 +7,7 @@
 package testing
 
 import (
+	"fmt"
 	"time"
 )
 
@@ -187,6 +188,30 @@ func (r BenchmarkResult) AllocedBytesPerOp() int64 {
 	return 0 // Dummy version to allow running e.g. golang.org/test/fibo.go
 }
 
+func runBenchmarks(benchmarks []InternalBenchmark) bool {
+	if len(benchmarks) == 0 {
+		return true
+	}
+	main := &B{
+		common: common{
+			name: "Main",
+		},
+		benchTime: benchTime,
+		benchFunc: func(b *B) {
+			for _, Benchmark := range benchmarks {
+				if flagVerbose {
+					fmt.Printf("=== RUN   %s\n", Benchmark.Name)
+				}
+				b.Run(Benchmark.Name, Benchmark.F)
+				fmt.Printf("--- Result: %d ns/op\n", b.result.NsPerOp())
+			}
+		},
+	}
+
+	main.runN(1)
+	return true
+}
+
 // Run benchmarks f as a subbenchmark with the given name. It reports
 // true if the subbenchmark succeeded.
 //
@@ -248,4 +273,3 @@ func Benchmark(f func(b *B)) BenchmarkResult {
 	}
 	return b.result
 }
-

src/testing/testing.go

@@ -18,9 +18,10 @@ import (
 
 // Testing flags.
 var (
-	flagVerbose   bool
-	flagShort     bool
-	flagRunRegexp string
+	flagVerbose     bool
+	flagShort       bool
+	flagRunRegexp   string
+	flagBenchRegexp string
 )
 
 var initRan bool
@@ -35,6 +36,7 @@ func Init() {
 	flag.BoolVar(&flagVerbose, "test.v", false, "verbose: print additional output")
 	flag.BoolVar(&flagShort, "test.short", false, "short: run smaller test suite to save time")
 	flag.StringVar(&flagRunRegexp, "test.run", "", "run: regexp of tests to run")
+	flag.StringVar(&flagBenchRegexp, "test.bench", "", "run: regexp of benchmarks to run")
 }
 
 // common holds the elements common between T and B and
@@ -243,7 +245,8 @@ type InternalTest struct {
 // M is a test suite.
 type M struct {
 	// tests is a list of the test names to execute
-	Tests []InternalTest
+	Tests      []InternalTest
+	Benchmarks []InternalBenchmark
 
 	deps testDeps
 }
@@ -275,8 +278,33 @@ func (m *M) Run() int {
 
 		m.Tests = filtered
 	}
+	if flagBenchRegexp != "" {
+		var filtered []InternalBenchmark
 
-	if len(m.Tests) == 0 {
+		// pre-test the regexp; we don't want to bother logging one failure for every test name if the regexp is broken
+		if _, err := m.deps.MatchString(flagBenchRegexp, "some-test-name"); err != nil {
+			fmt.Println("testing: invalid regexp for -test.bench:", err.Error())
+			failures++
+		}
+
+		// filter the list of tests before we try to run them
+		for _, test := range m.Benchmarks {
+			// ignore the error; we already tested that the regexp compiles fine above
+			if match, _ := m.deps.MatchString(flagBenchRegexp, test.Name); match {
+				filtered = append(filtered, test)
+			}
+		}
+
+		m.Benchmarks = filtered
+		flagVerbose = true
+		if flagRunRegexp == "" {
+			m.Tests = []InternalTest{}
+		}
+	} else {
+		m.Benchmarks = []InternalBenchmark{}
+	}
+
+	if len(m.Tests) == 0 && len(m.Benchmarks) == 0 {
 		fmt.Fprintln(os.Stderr, "testing: warning: no tests to run")
 	}
 
@@ -307,6 +335,8 @@ func (m *M) Run() int {
 		}
 	}
 
+	runBenchmarks(m.Benchmarks)
+
 	if failures > 0 {
 		fmt.Println("FAIL")
 	} else {
@@ -358,8 +388,9 @@ type testDeps interface {
 func MainStart(deps interface{}, tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) *M {
 	Init()
 	return &M{
-		Tests: tests,
-		deps:  deps.(testDeps),
+		Tests:      tests,
+		Benchmarks: benchmarks,
+		deps:       deps.(testDeps),
 	}
 }