Add exe compiler, supports running tests compiled to native executables

pkgs/test/CHANGELOG.md

@@ -8,6 +8,8 @@
     of compiling to kernel first.
   * If no given compiler is compatible for a platform, it will use its default
     compiler instead.
+* Add support for running tests as native executables (vm platform only).
+  * You can run tests this way with `--compiler exe`.
 * Support compiler identifiers in platform selectors.
 * List the supported compilers for each platform in the usage text.
 * Update all reporters to print the compiler along with the platform name

pkgs/test/test/runner/compiler_runtime_matrix_test.dart

@@ -0,0 +1,167 @@
+// Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+@TestOn('vm')
+import 'dart:io';
+
+import 'package:test/test.dart';
+import 'package:test_api/backend.dart'; // ignore: deprecated_member_use
+import 'package:test_descriptor/test_descriptor.dart' as d;
+
+import '../io.dart';
+
+void main() {
+  setUpAll(() async {
+    await precompileTestExecutable();
+  });
+
+  for (var runtime in Runtime.builtIn) {
+    for (var compiler in runtime.supportedCompilers) {
+      // Ignore the platforms we can't run on this OS.
+      if (runtime == Runtime.internetExplorer && !Platform.isWindows ||
+          runtime == Runtime.safari && !Platform.isMacOS) {
+        continue;
+      }
+      group('--runtime ${runtime.identifier} --compiler ${compiler.identifier}',
+          () {
+        final testArgs = [
+          'test.dart',
+          '-p',
+          runtime.identifier,
+          '-c',
+          compiler.identifier
+        ];
+
+        test('can run passing tests', () async {
+          await d.file('test.dart', _goodTest).create();
+          var test = await runTest(testArgs);
+
+          expect(test.stdout, emitsThrough(contains('+1: All tests passed!')));
+          await test.shouldExit(0);
+        });
+
+        test('fails gracefully for invalid code', () async {
+          await d.file('test.dart', _compileErrorTest).create();
+          var test = await runTest(testArgs);
+
+          expect(
+              test.stdout,
+              containsInOrder([
+                "Error: A value of type 'String' can't be assigned to a variable of type 'int'.",
+                "int x = 'hello';",
+              ]));
+
+          await test.shouldExit(1);
+        });
+
+        test('fails gracefully for test failures', () async {
+          await d.file('test.dart', _failingTest).create();
+          var test = await runTest(testArgs);
+
+          expect(
+              test.stdout,
+              containsInOrder([
+                'Expected: <2>',
+                'Actual: <1>',
+                'test.dart 5',
+                '+0 -1: Some tests failed.',
+              ]));
+
+          await test.shouldExit(1);
+        });
+
+        test('fails gracefully if a test file throws in main', () async {
+          await d.file('test.dart', _throwingTest).create();
+          var test = await runTest(testArgs);
+          var compileOrLoadMessage =
+              compiler == Compiler.dart2js ? 'compiling' : 'loading';
+
+          expect(
+              test.stdout,
+              containsInOrder([
+                '-1: [${runtime.name}, ${compiler.name}] $compileOrLoadMessage '
+                    'test.dart [E]',
+                'Failed to load "test.dart": oh no'
+              ]));
+          await test.shouldExit(1);
+        });
+
+        test('captures prints', () async {
+          await d.file('test.dart', _testWithPrints).create();
+          var test = await runTest([...testArgs, '-r', 'json']);
+
+          expect(
+              test.stdout,
+              containsInOrder([
+                '"messageType":"print","message":"hello","type":"print"',
+              ]));
+
+          await test.shouldExit(0);
+        });
+
+        if (runtime.isDartVM) {
+          test('forwards stdout/stderr', () async {
+            await d.file('test.dart', _testWithStdOutAndErr).create();
+            var test = await runTest(testArgs);
+
+            expect(test.stdout, emitsThrough('hello'));
+            expect(test.stderr, emits('world'));
+            await test.shouldExit(0);
+          });
+        }
+      },
+          skip: compiler == Compiler.dart2wasm
+              ? 'Wasm tests are experimental and require special setup'
+              : [Runtime.firefox, Runtime.nodeJS, Runtime.internetExplorer]
+                          .contains(runtime) &&
+                      Platform.isWindows
+                  ? 'https://github.com/dart-lang/test/issues/1942'
+                  : null);
+    }
+  }
+}
+
+final _goodTest = '''
+  import 'package:test/test.dart';
+
+  void main() {
+    test("success", () {});
+  }
+''';
+
+final _failingTest = '''
+  import 'package:test/test.dart';
+
+  void main() {
+    test("failure", () {
+      expect(1, 2);
+    });
+  }
+''';
+
+final _compileErrorTest = '''
+int x = 'hello';
+
+void main() {}
+''';
+
+final _throwingTest = "void main() => throw 'oh no';";
+
+final _testWithPrints = '''
+import 'package:test/test.dart';
+
+void main() {
+  print('hello');
+  test('success', () {});
+}''';
+
+final _testWithStdOutAndErr = '''
+import 'dart:io';
+import 'package:test/test.dart';
+
+void main() {
+  stdout.writeln('hello');
+  stderr.writeln('world');
+  test('success', () {});
+}''';

pkgs/test/test/runner/runner_test.dart

@@ -73,7 +73,7 @@ Running Tests:
                                       $_runtimes.
                                       Each platform supports the following compilers:
 $_runtimeCompilers
--c, --compiler                        The compiler(s) to use to run tests, supported compilers are [dart2js, dart2wasm, kernel, source].
+-c, --compiler                        The compiler(s) to use to run tests, supported compilers are [dart2js, dart2wasm, exe, kernel, source].
                                       Each platform has a default compiler but may support other compilers.
                                       You can target a compiler to a specific platform using arguments of the following form [<platform-selector>:]<compiler>.
                                       If a platform is specified but no given compiler is supported for that platform, then it will use its default compiler.
@@ -124,7 +124,7 @@ final _runtimes = '[vm (default), chrome, firefox'
     'experimental-chrome-wasm]';
 
 final _runtimeCompilers = [
-  '[vm]: kernel (default), source',
+  '[vm]: kernel (default), source, exe',
   '[chrome]: dart2js (default)',
   '[firefox]: dart2js (default)',
   if (Platform.isMacOS) '[safari]: dart2js (default)',

pkgs/test_api/lib/src/backend/compiler.dart

@@ -10,6 +10,9 @@ class Compiler {
   /// Experimental Dart to Wasm compiler.
   static const Compiler dart2wasm = Compiler._('Dart2WASM', 'dart2wasm');
 
+  /// Compiles dart code to a native executable.
+  static const Compiler exe = Compiler._('Exe', 'exe');
+
   /// The standard compiler for vm tests, compiles tests to kernel before
   /// running them on the VM.
   static const Compiler kernel = Compiler._('Kernel', 'kernel');
@@ -21,6 +24,7 @@ class Compiler {
   static const List<Compiler> builtIn = [
     Compiler.dart2js,
     Compiler.dart2wasm,
+    Compiler.exe,
     Compiler.kernel,
     Compiler.source,
   ];

pkgs/test_api/lib/src/backend/runtime.dart

@@ -10,8 +10,8 @@ class Runtime {
   // variable tests in test/backend/platform_selector/evaluate_test.
 
   /// The command-line Dart VM.
-  static const Runtime vm = Runtime(
-      'VM', 'vm', Compiler.kernel, [Compiler.kernel, Compiler.source],
+  static const Runtime vm = Runtime('VM', 'vm', Compiler.kernel,
+      [Compiler.kernel, Compiler.source, Compiler.exe],
       isDartVM: true);
 
   /// Google Chrome.

pkgs/test_core/CHANGELOG.md

@@ -8,6 +8,7 @@
     of compiling to kernel first.
   * If no given compiler is compatible for a platform, it will use its default
     compiler instead.
+* Add support for `-c exe` (the native executable compiler) to the vm platform.
 * Add `Compiler` class, exposed through `backend.dart`.
 * Support compiler identifiers in platform selectors.
 * List the supported compilers for each platform in the usage text.

pkgs/test_core/lib/src/bootstrap/vm.dart

@@ -3,14 +3,16 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'dart:developer';
+import 'dart:io';
 import 'dart:isolate';
 
 import 'package:stream_channel/isolate_channel.dart';
 import 'package:stream_channel/stream_channel.dart';
 
 import 'package:test_core/src/runner/plugin/remote_platform_helpers.dart';
+import 'package:test_core/src/runner/plugin/shared_platform_helpers.dart';
 
-/// Bootstraps a vm test to communicate with the test runner.
+/// Bootstraps a vm test to communicate with the test runner over an isolate.
 void internalBootstrapVmTest(Function Function() getMain, SendPort sendPort) {
   var platformChannel =
       MultiChannel(IsolateChannel<Object?>.connectSend(sendPort));
@@ -24,3 +26,24 @@ void internalBootstrapVmTest(Function Function() getMain, SendPort sendPort) {
     platformChannel.sink.add('done');
   });
 }
+
+/// Bootstraps a native executable test to communicate with the test runner over
+/// a socket.
+void internalBootstrapNativeTest(
+    Function Function() getMain, List<String> args) async {
+  if (args.length != 2) {
+    throw StateError(
+        'Expected exactly two args, a host and a port, but got $args');
+  }
+  var socket = await Socket.connect(args[0], int.parse(args[1]));
+  var platformChannel = MultiChannel<Object?>(jsonSocketStreamChannel(socket));
+  var testControlChannel = platformChannel.virtualChannel()
+    ..pipe(serializeSuite(getMain));
+  platformChannel.sink.add(testControlChannel.id);
+
+  platformChannel.stream.forEach((message) {
+    assert(message == 'debug');
+    debugger(message: 'Paused by test runner');
+    platformChannel.sink.add('done');
+  });
+}

pkgs/test_core/lib/src/runner/plugin/shared_platform_helpers.dart

@@ -0,0 +1,21 @@
+// Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:async/async.dart';
+import 'package:stream_channel/stream_channel.dart';
+
+/// Converts a raw [Socket] into a [StreamChannel] of JSON objects.
+///
+/// JSON messages are separated by newlines.
+StreamChannel<Object?> jsonSocketStreamChannel(Socket socket) =>
+    StreamChannel.withGuarantees(socket, socket)
+        .cast<List<int>>()
+        .transform(StreamChannelTransformer.fromCodec(utf8))
+        .transformStream(const LineSplitter())
+        .transformSink(StreamSinkTransformer.fromHandlers(
+            handleData: (original, sink) => sink.add('$original\n')))
+        .transform(jsonDocument);