Implement "iterations" directive for customer tests (flutter#124541)

Implement "iterations" directive for customer tests

Author: Hixie

dev/customer_testing/lib/customer_test.dart

@@ -14,6 +14,7 @@ class CustomerTest {
     final List<String> fetch = <String>[];
     final List<Directory> update = <Directory>[];
     final List<String> test = <String>[];
+    int? iterations;
     bool hasTests = false;
     for (final String line in testFile.readAsLinesSync().map((String line) => line.trim())) {
       if (line.isEmpty) {
@@ -26,6 +27,14 @@ class CustomerTest {
         fetch.add(line.substring(6));
       } else if (line.startsWith('update=')) {
         update.add(Directory(line.substring(7)));
+      } else if (line.startsWith('iterations=')) {
+        if (iterations != null) {
+          throw const FormatException('Cannot specify "iterations" directive multiple times.');
+        }
+        iterations = int.parse(line.substring(11));
+        if (iterations < 1) {
+          throw const FormatException('The "iterations" directive must have a positive integer value.');
+        }
       } else if (line.startsWith('test=')) {
         hasTests = true;
         test.add(line.substring(5));
@@ -84,10 +93,11 @@ class CustomerTest {
       List<String>.unmodifiable(fetch),
       List<Directory>.unmodifiable(update),
       List<String>.unmodifiable(test),
+      iterations,
     );
   }
 
-  const CustomerTest._(this.contacts, this.fetch, this.update, this.tests);
+  const CustomerTest._(this.contacts, this.fetch, this.update, this.tests, this.iterations);
 
   // (e-mail regexp from HTML standard)
   static final RegExp _email = RegExp(r"^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$");
@@ -98,4 +108,5 @@ class CustomerTest {
   final List<String> fetch;
   final List<Directory> update;
   final List<String> tests;
+  final int? iterations;
 }

dev/customer_testing/lib/runner.dart

@@ -133,6 +133,14 @@ Future<bool> runTests({
           if (verbose) {
             print('Running tests...');
           }
+          if (instructions.iterations != null && instructions.iterations! < repeat) {
+            if (verbose) {
+              final String s = instructions.iterations == 1 ? '' : 's';
+              print('Limiting to ${instructions.iterations} round$s rather than $repeat rounds because of "iterations" directive.');
+            }
+            repeat = instructions.iterations!;
+          }
+          final Stopwatch stopwatch = Stopwatch()..start();
           for (int iteration = 0; iteration < repeat; iteration += 1) {
             if (verbose && repeat > 1) {
               print('Round ${iteration + 1} of $repeat.');
@@ -146,8 +154,9 @@ Future<bool> runTests({
               }
             }
           }
+          stopwatch.stop();
           if (verbose && success) {
-            print('Tests finished.');
+            print('Tests finished in ${(stopwatch.elapsed.inSeconds / repeat).toStringAsFixed(2)} seconds per iteration.');
           }
         }
       }

dev/customer_testing/run_tests.dart

@@ -24,7 +24,7 @@ Future<bool> run(List<String> arguments) async {
     ..addOption(
       'repeat',
       defaultsTo: '1',
-      help: 'How many times to run each test. Set to a high value to look for flakes.',
+      help: 'How many times to run each test. Set to a high value to look for flakes. If a test specifies a number of iterations, the lower of the two values is used.',
       valueHelp: 'count',
     )
     ..addOption(