Reland "Fail tests on exceptions raised after test completed (#144706)" (#144980)

Reverts flutter/flutter#144970

No changes in this PR compared to the original. The test failure was fixed by adding missing awaits in flutter/flutter#144978.

Fixes flutter/flutter#144353.

Author: goderbauer

dev/automated_tests/test_smoke_test/fail_test_on_exception_after_test.dart

@@ -0,0 +1,29 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_test/flutter_test.dart';
+
+// This is a test to make sure that an asynchronous exception thrown after a
+// test ended actually causes a test failure.
+// See //flutter/dev/bots/test.dart
+
+void main() {
+  final Completer<void> complete = Completer<void>();
+
+  testWidgets('test smoke test -- this test SHOULD FAIL', (WidgetTester tester) async {
+    tester.runAsync(() async {
+      Timer.run(() {
+        complete.complete();
+        throw StateError('Exception thrown after test completed.');
+      });
+    });
+  });
+
+  tearDown(() async {
+    print('Waiting for asynchronous exception...');
+    await complete.future;
+  });
+}

dev/bots/test.dart

@@ -346,7 +346,26 @@ Future<void> _runTestHarnessTests() async {
           : 'Failed to find the stack trace for the pending Timer.\n\n'
             'stdout:\n${result.flattenedStdout}\n\n'
             'stderr:\n${result.flattenedStderr}';
-    }),
+      },
+    ),
+    () => _runFlutterTest(
+      automatedTests,
+      script: path.join('test_smoke_test', 'fail_test_on_exception_after_test.dart'),
+      expectFailure: true,
+      printOutput: false,
+      outputChecker: (CommandResult result) {
+        const String expectedError = '══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════════════════════════════════════════════════\n'
+            'The following StateError was thrown running a test (but after the test had completed):\n'
+            'Bad state: Exception thrown after test completed.';
+        if (result.flattenedStdout!.contains(expectedError)) {
+          return null;
+        }
+        return 'Failed to find expected output on stdout.\n\n'
+          'Expected output:\n$expectedError\n\n'
+          'Actual stdout:\n${result.flattenedStdout}\n\n'
+          'Actual stderr:\n${result.flattenedStderr}';
+      },
+    ),
     () => _runFlutterTest(
       automatedTests,
       script: path.join('test_smoke_test', 'crash1_test.dart'),

dev/bots/test/test_test.dart

@@ -128,13 +128,13 @@ void main() {
         <String, String>{'SHARD': kTestHarnessShardName, 'SUBSHARD': '1_3'},
       );
       expectExitCode(result, 0);
-      expect(result.stdout, contains('Selecting subshard 1 of 3 (tests 1-3 of 8)'));
+      expect(result.stdout, contains('Selecting subshard 1 of 3 (tests 1-3 of 9)'));
 
       result = await runScript(
         <String, String>{'SHARD': kTestHarnessShardName, 'SUBSHARD': '3_3'},
       );
       expectExitCode(result, 0);
-      expect(result.stdout, contains('Selecting subshard 3 of 3 (tests 7-8 of 8)'));
+      expect(result.stdout, contains('Selecting subshard 3 of 3 (tests 7-9 of 9)'));
     });
 
     test('exits with code 1 when SUBSHARD index greater than total', () async {

packages/flutter_test/lib/src/binding.dart

@@ -910,15 +910,14 @@ abstract class TestWidgetsFlutterBinding extends BindingBase
         // Ideally, once the test has failed we would stop getting errors from the test.
         // However, if someone tries hard enough they could get in a state where this happens.
         // If we silently dropped these errors on the ground, nobody would ever know. So instead
-        // we report them to the console. They don't cause test failures, but hopefully someone
-        // will see them in the logs at some point.
+        // we raise them and fail the test after it has already completed.
         debugPrint = debugPrintOverride; // just in case the test overrides it -- otherwise we won't see the error!
-        FlutterError.dumpErrorToConsole(FlutterErrorDetails(
+        reportTestException(FlutterErrorDetails(
           exception: exception,
           stack: stack,
           context: ErrorDescription('running a test (but after the test had completed)'),
           library: 'Flutter test framework',
-        ), forceReport: true);
+        ), description);
         return;
       }
       // This is where test failures, e.g. those in expect(), will end up.