From d1fed9fb18d361d0071fc19cc15f9e1364ea204f Mon Sep 17 00:00:00 2001 From: Jake Macdonald Date: Thu, 16 Feb 2023 21:46:22 +0000 Subject: [PATCH 01/13] add exe compiler and support it in the vm platform --- pkgs/test_api/lib/src/backend/compiler.dart | 4 + pkgs/test_api/lib/src/backend/runtime.dart | 4 +- pkgs/test_core/lib/src/bootstrap/vm.dart | 32 +++- .../test_core/lib/src/runner/vm/platform.dart | 145 +++++++++++++++--- 4 files changed, 162 insertions(+), 23 deletions(-) diff --git a/pkgs/test_api/lib/src/backend/compiler.dart b/pkgs/test_api/lib/src/backend/compiler.dart index da3c59a33..77a637619 100644 --- a/pkgs/test_api/lib/src/backend/compiler.dart +++ b/pkgs/test_api/lib/src/backend/compiler.dart @@ -10,6 +10,9 @@ class Compiler { /// Experimental Dart to Wasm compiler. static const Compiler dart2wasm = Compiler._('Dart2WASM', 'dart2wasm'); + /// Compiles dart code to a native executable. + static const Compiler exe = Compiler._('Exe', 'exe'); + /// The standard compiler for vm tests, compiles tests to kernel before /// running them on the VM. static const Compiler kernel = Compiler._('Kernel', 'kernel'); @@ -21,6 +24,7 @@ class Compiler { static const List builtIn = [ Compiler.dart2js, Compiler.dart2wasm, + Compiler.exe, Compiler.kernel, Compiler.source, ]; diff --git a/pkgs/test_api/lib/src/backend/runtime.dart b/pkgs/test_api/lib/src/backend/runtime.dart index 72c5a8dfd..2572402e7 100644 --- a/pkgs/test_api/lib/src/backend/runtime.dart +++ b/pkgs/test_api/lib/src/backend/runtime.dart @@ -10,8 +10,8 @@ class Runtime { // variable tests in test/backend/platform_selector/evaluate_test. /// The command-line Dart VM. - static const Runtime vm = Runtime( - 'VM', 'vm', Compiler.kernel, [Compiler.kernel, Compiler.source], + static const Runtime vm = Runtime('VM', 'vm', Compiler.kernel, + [Compiler.kernel, Compiler.source, Compiler.exe], isDartVM: true); /// Google Chrome. diff --git a/pkgs/test_core/lib/src/bootstrap/vm.dart b/pkgs/test_core/lib/src/bootstrap/vm.dart index d1eb59ada..9a241270a 100644 --- a/pkgs/test_core/lib/src/bootstrap/vm.dart +++ b/pkgs/test_core/lib/src/bootstrap/vm.dart @@ -2,15 +2,18 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:convert'; import 'dart:developer'; +import 'dart:io'; import 'dart:isolate'; +import 'package:async/async.dart'; import 'package:stream_channel/isolate_channel.dart'; import 'package:stream_channel/stream_channel.dart'; import 'package:test_core/src/runner/plugin/remote_platform_helpers.dart'; -/// Bootstraps a vm test to communicate with the test runner. +/// Bootstraps a vm test to communicate with the test runner over an isolate. void internalBootstrapVmTest(Function Function() getMain, SendPort sendPort) { var platformChannel = MultiChannel(IsolateChannel.connectSend(sendPort)); @@ -24,3 +27,30 @@ void internalBootstrapVmTest(Function Function() getMain, SendPort sendPort) { platformChannel.sink.add('done'); }); } + +/// Bootstraps a native executable test to communicate with the test runner over +/// a socket. +void internalBootstrapNativeTest( + Function Function() getMain, List args) async { + if (args.length != 2) { + throw StateError( + 'Expected exactly two args, a host and a port, but got $args'); + } + var socket = await Socket.connect(args[0], int.parse(args[1])); + var platformChannel = MultiChannel(StreamChannel(socket, socket) + .cast>() + .transform(StreamChannelTransformer.fromCodec(utf8)) + .transformStream(const LineSplitter()) + .transformSink(StreamSinkTransformer.fromHandlers( + handleData: (original, sink) => sink.add('$original\n'))) + .transform(jsonDocument)); + var testControlChannel = platformChannel.virtualChannel() + ..pipe(serializeSuite(getMain)); + platformChannel.sink.add(testControlChannel.id); + + platformChannel.stream.forEach((message) { + assert(message == 'debug'); + debugger(message: 'Paused by test runner'); + platformChannel.sink.add('done'); + }); +} diff --git a/pkgs/test_core/lib/src/runner/vm/platform.dart b/pkgs/test_core/lib/src/runner/vm/platform.dart index 9c7b4eaa4..060a19375 100644 --- a/pkgs/test_core/lib/src/runner/vm/platform.dart +++ b/pkgs/test_core/lib/src/runner/vm/platform.dart @@ -39,7 +39,7 @@ class VMPlatform extends PlatformPlugin { p.join(p.current, '.dart_tool', 'test', 'incremental_kernel')); final _closeMemo = AsyncMemoizer(); final _workingDirectory = Directory.current.uri; - final _tempDir = Directory.systemTemp.createTempSync('dart_test.kernel.'); + final _tempDir = Directory.systemTemp.createTempSync('dart_test.vm.'); @override Future load(String path, SuitePlatform platform, @@ -48,15 +48,40 @@ class VMPlatform extends PlatformPlugin { _setupPauseAfterTests(); - var receivePort = ReceivePort(); + MultiChannel outerChannel; + var cleanupFns = []; Isolate? isolate; - try { - isolate = await _spawnIsolate( - path, receivePort.sendPort, suiteConfig.metadata, platform.compiler); - if (isolate == null) return null; - } catch (error) { - receivePort.close(); - rethrow; + if (platform.compiler == Compiler.exe) { + var serverSocket = await ServerSocket.bind('localhost', 0); + var process = + await _spawnExecutable(path, suiteConfig.metadata, serverSocket); + process.stdout.map(stdout.add); + process.stderr.map(stderr.add); + var socket = await serverSocket.first; + outerChannel = MultiChannel(StreamChannel(socket, socket) + .cast>() + .transform(StreamChannelTransformer.fromCodec(utf8)) + .transformStream(const LineSplitter()) + .transformSink(StreamSinkTransformer.fromHandlers( + handleData: (original, sink) => sink.add('$original\n'))) + .transform(jsonDocument)); + cleanupFns + ..add(serverSocket.close) + ..add(process.kill); + } else { + var receivePort = ReceivePort(); + try { + isolate = await _spawnIsolate(path, receivePort.sendPort, + suiteConfig.metadata, platform.compiler); + if (isolate == null) return null; + } catch (error) { + receivePort.close(); + rethrow; + } + outerChannel = MultiChannel(IsolateChannel.connectReceive(receivePort)); + cleanupFns + ..add(receivePort.close) + ..add(isolate.kill); } VmService? client; @@ -64,7 +89,6 @@ class VMPlatform extends PlatformPlugin { // Typical test interaction will go across `channel`, `outerChannel` adds // additional communication directly between the test bootstrapping and this // platform to enable pausing after tests for debugging. - var outerChannel = MultiChannel(IsolateChannel.connectReceive(receivePort)); var outerQueue = StreamQueue(outerChannel.stream); var channelId = (await outerQueue.next) as int; var channel = outerChannel.virtualChannel(channelId).transformStream( @@ -73,8 +97,9 @@ class VMPlatform extends PlatformPlugin { outerChannel.sink.add('debug'); await outerQueue.next; } - receivePort.close(); - isolate!.kill(); + for (var fn in cleanupFns) { + fn(); + } eventSub?.cancel(); client?.dispose(); sink.close(); @@ -83,9 +108,14 @@ class VMPlatform extends PlatformPlugin { Environment? environment; IsolateRef? isolateRef; if (_config.debug) { + if (platform.compiler == Compiler.exe) { + throw UnsupportedError( + 'Unable to debug tests compiled to `exe` (tried to debug $path with ' + 'the `exe` compiler).'); + } var info = await Service.controlWebServer(enable: true, silenceOutput: true); - var isolateID = Service.getIsolateID(isolate)!; + var isolateID = Service.getIsolateID(isolate!)!; var libraryPath = _absolute(path).toString(); var serverUri = info.serverUri!; @@ -134,6 +164,47 @@ class VMPlatform extends PlatformPlugin { return _workingDirectory.resolveUri(uri); } + /// Compiles [path] to a native executable and spawns it as a process. + /// + /// Sets up a communication channel as well by passing command line arguments + /// for the host and port of [socket]. + Future _spawnExecutable( + String path, Metadata suiteMetadata, ServerSocket socket) async { + if (_config.suiteDefaults.precompiledPath != null) { + throw UnsupportedError( + 'Precompiled native executable tests are not supported at this time'); + } + var executable = await _compileToNative(path, suiteMetadata); + return await Process.start( + executable, [socket.address.host, socket.port.toString()]); + } + + /// Compiles [path] to a native executable using `dart compile exe`. + Future _compileToNative(String path, Metadata suiteMetadata) async { + var bootstrapPath = _bootstrapNativeTestFile( + path, + suiteMetadata.languageVersionComment ?? + await rootPackageLanguageVersionComment); + var output = File(p.setExtension(bootstrapPath, '.exe')); + var processResult = await Process.run(Platform.resolvedExecutable, [ + 'compile', + 'exe', + bootstrapPath, + '--output', + output.path, + '--packages', + (await packageConfigUri).path, + ]); + if (processResult.exitCode != 0 || !(await output.exists())) { + throw LoadException( + path, + 'exitCode: ${processResult.exitCode}\nstdout:\n' + '${processResult.stdout}\nstderr:\n' + '${processResult.stderr}.compilerOutput'); + } + return output.path; + } + /// Spawns an isolate with the current configuration and passes it [message]. /// /// This isolate connects an [IsolateChannel] to [message] and sends the @@ -157,7 +228,7 @@ class VMPlatform extends PlatformPlugin { await _compileToKernel(path, suiteMetadata), message); case Compiler.source: return _spawnIsolateWithUri( - _bootstrapTestFile( + _bootstrapIsolateTestFile( path, suiteMetadata.languageVersionComment ?? await rootPackageLanguageVersionComment), @@ -226,21 +297,41 @@ class VMPlatform extends PlatformPlugin { /// file. /// /// Returns the [Uri] to the created file. - Uri _bootstrapTestFile(String testPath, String languageVersionComment) { - var file = File( - p.join(_tempDir.path, p.setExtension(testPath, '.bootstrap.dart'))); + Uri _bootstrapIsolateTestFile( + String testPath, String languageVersionComment) { + var file = File(p.join( + _tempDir.path, p.setExtension(testPath, '.bootstrap.isolate.dart'))); if (!file.existsSync()) { file ..createSync(recursive: true) - ..writeAsStringSync(_bootstrapTestContents( + ..writeAsStringSync(_bootstrapIsolateTestContents( _absolute(testPath), languageVersionComment)); } return file.uri; } + + /// Bootstraps the test at [testPath] for native execution and writes its + /// contents to a temporary file. + /// + /// Returns the path to the created file. + String _bootstrapNativeTestFile( + String testPath, String languageVersionComment) { + var file = File(p.join( + _tempDir.path, p.setExtension(testPath, '.bootstrap.native.dart'))); + if (!file.existsSync()) { + file + ..createSync(recursive: true) + ..writeAsStringSync(_bootstrapNativeTestContents( + _absolute(testPath), languageVersionComment)); + } + return file.path; + } } -/// Creates bootstrap file contents for running [testUri] in the VM. -String _bootstrapTestContents(Uri testUri, String languageVersionComment) => ''' +/// Creates bootstrap file contents for running [testUri] in a VM isolate. +String _bootstrapIsolateTestContents( + Uri testUri, String languageVersionComment) => + ''' $languageVersionComment import "dart:isolate"; import "package:test_core/src/bootstrap/vm.dart"; @@ -250,6 +341,20 @@ String _bootstrapTestContents(Uri testUri, String languageVersionComment) => ''' } '''; +/// Creates bootstrap file contents for running [testUri] as a native +/// executable. +String _bootstrapNativeTestContents( + Uri testUri, String languageVersionComment) => + ''' + $languageVersionComment + import "dart:isolate"; + import "package:test_core/src/bootstrap/vm.dart"; + import "$testUri" as test; + void main(List args) { + internalBootstrapNativeTest(() => test.main, args); + } + '''; + Future> _gatherCoverage(Environment environment) async { final isolateId = Uri.parse(environment.observatoryUrl!.fragment) .queryParameters['isolateId']; From f0d90e93518386a1c54462c68a7879a8b397580d Mon Sep 17 00:00:00 2001 From: Jake Macdonald Date: Thu, 16 Feb 2023 21:49:33 +0000 Subject: [PATCH 02/13] update changelogs --- pkgs/test/CHANGELOG.md | 2 ++ pkgs/test_core/CHANGELOG.md | 1 + 2 files changed, 3 insertions(+) diff --git a/pkgs/test/CHANGELOG.md b/pkgs/test/CHANGELOG.md index f5743f848..3aea19f90 100644 --- a/pkgs/test/CHANGELOG.md +++ b/pkgs/test/CHANGELOG.md @@ -8,6 +8,8 @@ of compiling to kernel first. * If no given compiler is compatible for a platform, it will use its default compiler instead. +* Add support for running tests as native executables (vm platform only). + * You can run tests this way with `--compiler exe`. * Support compiler identifiers in platform selectors. * List the supported compilers for each platform in the usage text. * Update all reporters to print the compiler along with the platform name diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md index 7a8d8871d..376afec8b 100644 --- a/pkgs/test_core/CHANGELOG.md +++ b/pkgs/test_core/CHANGELOG.md @@ -8,6 +8,7 @@ of compiling to kernel first. * If no given compiler is compatible for a platform, it will use its default compiler instead. +* Add support for `-c exe` (the native executable compiler) to the vm platform. * Add `Compiler` class, exposed through `backend.dart`. * Support compiler identifiers in platform selectors. * List the supported compilers for each platform in the usage text. From 1f1c64427bb2c519733125d313f10c1cff5c28b9 Mon Sep 17 00:00:00 2001 From: Jake Macdonald Date: Thu, 16 Feb 2023 22:41:36 +0000 Subject: [PATCH 03/13] add a test covering the full supported matrix of runtimes and compilers --- .../runner/compiler_runtime_matrix_test.dart | 124 ++++++++++++++++++ .../test_core/lib/src/runner/vm/platform.dart | 19 ++- 2 files changed, 136 insertions(+), 7 deletions(-) create mode 100644 pkgs/test/test/runner/compiler_runtime_matrix_test.dart diff --git a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart new file mode 100644 index 000000000..91a67ff3e --- /dev/null +++ b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart @@ -0,0 +1,124 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +@TestOn('vm') +import 'dart:io'; + +import 'package:test/test.dart'; +import 'package:test_api/backend.dart'; // ignore: deprecated_member_use +import 'package:test_descriptor/test_descriptor.dart' as d; + +import '../io.dart'; + +void main() { + setUpAll(() async { + await precompileTestExecutable(); + await d.file('test.dart', _goodTest).create(); + }); + + for (var runtime in Runtime.builtIn) { + // Ignore the platforms we can't run on this OS. + if (runtime == Runtime.internetExplorer && !Platform.isWindows || + runtime == Runtime.safari && !Platform.isMacOS) { + continue; + } + group('--runtime ${runtime.identifier}', () { + for (var compiler in runtime.supportedCompilers) { + group('--compiler ${compiler.identifier}', () { + final testArgs = [ + 'test.dart', + '-p', + runtime.identifier, + '-c', + compiler.identifier + ]; + + test('can run passing tests', () async { + await d.file('test.dart', _goodTest).create(); + var test = await runTest(testArgs); + + expect( + test.stdout, emitsThrough(contains('+1: All tests passed!'))); + await test.shouldExit(0); + }); + + test('fails gracefully for invalid code', () async { + await d.file('test.dart', _compileErrorTest).create(); + var test = await runTest(testArgs); + + expect( + test.stdout, + containsInOrder([ + "Error: A value of type 'String' can't be assigned to a variable of type 'int'.", + "int x = 'hello';", + ])); + + await test.shouldExit(1); + }); + + test('fails gracefully for test failures', () async { + await d.file('test.dart', _failingTest).create(); + var test = await runTest(testArgs); + + expect( + test.stdout, + containsInOrder([ + 'Expected: <2>', + 'Actual: <1>', + 'test.dart 5', + '+0 -1: Some tests failed.', + ])); + + await test.shouldExit(1); + }); + + test('fails gracefully if a test file throws in main', () async { + await d.file('test.dart', _throwingTest).create(); + var test = await runTest(testArgs); + var compileOrLoadMessage = + compiler == Compiler.dart2js ? 'compiling' : 'loading'; + + expect( + test.stdout, + containsInOrder([ + '-1: [${runtime.name}, ${compiler.name}] $compileOrLoadMessage ' + 'test.dart [E]', + 'Failed to load "test.dart": oh no' + ])); + await test.shouldExit(1); + }); + }, + skip: compiler == Compiler.dart2wasm + ? 'Wasm tests are experimental and require special setup' + : null); + } + }); + } +} + +final _goodTest = ''' + import 'package:test/test.dart'; + + void main() { + test("success", () {}); + } +'''; + +final _failingTest = ''' + import 'package:test/test.dart'; + + void main() { + test("failure", () { + expect(1, 2); + }); + } +'''; + +final _compileErrorTest = ''' +int x = 'hello'; + +void main() {} +'''; + +final _throwingTest = "void main() => throw 'oh no';"; diff --git a/pkgs/test_core/lib/src/runner/vm/platform.dart b/pkgs/test_core/lib/src/runner/vm/platform.dart index 060a19375..2d0e51f73 100644 --- a/pkgs/test_core/lib/src/runner/vm/platform.dart +++ b/pkgs/test_core/lib/src/runner/vm/platform.dart @@ -53,8 +53,14 @@ class VMPlatform extends PlatformPlugin { Isolate? isolate; if (platform.compiler == Compiler.exe) { var serverSocket = await ServerSocket.bind('localhost', 0); - var process = - await _spawnExecutable(path, suiteConfig.metadata, serverSocket); + Process process; + try { + process = + await _spawnExecutable(path, suiteConfig.metadata, serverSocket); + } catch (error) { + serverSocket.close(); + rethrow; + } process.stdout.map(stdout.add); process.stderr.map(stderr.add); var socket = await serverSocket.first; @@ -196,11 +202,10 @@ class VMPlatform extends PlatformPlugin { (await packageConfigUri).path, ]); if (processResult.exitCode != 0 || !(await output.exists())) { - throw LoadException( - path, - 'exitCode: ${processResult.exitCode}\nstdout:\n' - '${processResult.stdout}\nstderr:\n' - '${processResult.stderr}.compilerOutput'); + throw LoadException(path, ''' +exitCode: ${processResult.exitCode} +stdout: ${processResult.stdout} +stderr: ${processResult.stderr}'''); } return output.path; } From 3cc6089a96bb5d6210a726118be569742df0f264 Mon Sep 17 00:00:00 2001 From: Jake Macdonald Date: Thu, 16 Feb 2023 22:44:14 +0000 Subject: [PATCH 04/13] clean up cruft line --- pkgs/test/test/runner/compiler_runtime_matrix_test.dart | 1 - 1 file changed, 1 deletion(-) diff --git a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart index 91a67ff3e..c43df9768 100644 --- a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart +++ b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart @@ -14,7 +14,6 @@ import '../io.dart'; void main() { setUpAll(() async { await precompileTestExecutable(); - await d.file('test.dart', _goodTest).create(); }); for (var runtime in Runtime.builtIn) { From 04e73625ac6facd91be529ba9d79d976ac4eba29 Mon Sep 17 00:00:00 2001 From: Jake Macdonald Date: Thu, 16 Feb 2023 23:05:18 +0000 Subject: [PATCH 05/13] update usage text in runner_test.dart --- pkgs/test/test/runner/runner_test.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/test/test/runner/runner_test.dart b/pkgs/test/test/runner/runner_test.dart index 4f5d7f467..6adf98200 100644 --- a/pkgs/test/test/runner/runner_test.dart +++ b/pkgs/test/test/runner/runner_test.dart @@ -73,7 +73,7 @@ Running Tests: $_runtimes. Each platform supports the following compilers: $_runtimeCompilers --c, --compiler The compiler(s) to use to run tests, supported compilers are [dart2js, dart2wasm, kernel, source]. +-c, --compiler The compiler(s) to use to run tests, supported compilers are [dart2js, dart2wasm, exe, kernel, source]. Each platform has a default compiler but may support other compilers. You can target a compiler to a specific platform using arguments of the following form [:]. If a platform is specified but no given compiler is supported for that platform, then it will use its default compiler. @@ -124,7 +124,7 @@ final _runtimes = '[vm (default), chrome, firefox' 'experimental-chrome-wasm]'; final _runtimeCompilers = [ - '[vm]: kernel (default), source', + '[vm]: kernel (default), source, exe', '[chrome]: dart2js (default)', '[firefox]: dart2js (default)', if (Platform.isMacOS) '[safari]: dart2js (default)', From 6ce8a2437c19ce9db4f9ebfbd4ae2c64145e274b Mon Sep 17 00:00:00 2001 From: Jake Macdonald Date: Thu, 16 Feb 2023 23:11:10 +0000 Subject: [PATCH 06/13] try `toFilePath()` instead of `path` for windows --- pkgs/test_core/lib/src/runner/vm/platform.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/test_core/lib/src/runner/vm/platform.dart b/pkgs/test_core/lib/src/runner/vm/platform.dart index 2d0e51f73..bc4ea92ec 100644 --- a/pkgs/test_core/lib/src/runner/vm/platform.dart +++ b/pkgs/test_core/lib/src/runner/vm/platform.dart @@ -199,7 +199,7 @@ class VMPlatform extends PlatformPlugin { '--output', output.path, '--packages', - (await packageConfigUri).path, + (await packageConfigUri).toFilePath(), ]); if (processResult.exitCode != 0 || !(await output.exists())) { throw LoadException(path, ''' From 9e9084b17cbf66e24fad605bcc722985e89b91bd Mon Sep 17 00:00:00 2001 From: Jake Macdonald Date: Fri, 17 Feb 2023 16:02:27 +0000 Subject: [PATCH 07/13] add tests for printing messages, fix forwarding of stdout/stderr --- .../runner/compiler_runtime_matrix_test.dart | 40 +++++++++++++++++++ .../test_core/lib/src/runner/vm/platform.dart | 4 +- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart index c43df9768..88d36b6cf 100644 --- a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart +++ b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart @@ -87,6 +87,28 @@ void main() { ])); await test.shouldExit(1); }); + + test('captures prints', () async { + await d.file('test.dart', _testWithPrints).create(); + var test = await runTest([...testArgs, '-r', 'json']); + + expect( + test.stdout, + containsInOrder([ + '"messageType":"print","message":"hello","type":"print"', + ])); + + await test.shouldExit(0); + }); + + test('forwards stdout/stderr', () async { + await d.file('test.dart', _testWithStdOutAndErr).create(); + var test = await runTest(testArgs); + + expect(test.stdout, emitsThrough('hello')); + expect(test.stderr, emits('world')); + await test.shouldExit(0); + }, testOn: '!browser'); }, skip: compiler == Compiler.dart2wasm ? 'Wasm tests are experimental and require special setup' @@ -121,3 +143,21 @@ void main() {} '''; final _throwingTest = "void main() => throw 'oh no';"; + +final _testWithPrints = ''' +import 'package:test/test.dart'; + +void main() { + print('hello'); + test('success', () {}); +}'''; + +final _testWithStdOutAndErr = ''' +import 'dart:io'; +import 'package:test/test.dart'; + +void main() { + stdout.writeln('hello'); + stderr.writeln('world'); + test('success', () {}); +}'''; diff --git a/pkgs/test_core/lib/src/runner/vm/platform.dart b/pkgs/test_core/lib/src/runner/vm/platform.dart index bc4ea92ec..5239f0d86 100644 --- a/pkgs/test_core/lib/src/runner/vm/platform.dart +++ b/pkgs/test_core/lib/src/runner/vm/platform.dart @@ -61,8 +61,8 @@ class VMPlatform extends PlatformPlugin { serverSocket.close(); rethrow; } - process.stdout.map(stdout.add); - process.stderr.map(stderr.add); + process.stdout.listen(stdout.add); + process.stderr.listen(stderr.add); var socket = await serverSocket.first; outerChannel = MultiChannel(StreamChannel(socket, socket) .cast>() From 80ef086d0e9cc4cb7bcdb5fd731cf7e7619a80ec Mon Sep 17 00:00:00 2001 From: Jake Macdonald Date: Fri, 17 Feb 2023 16:18:57 +0000 Subject: [PATCH 08/13] various code review fixes --- .../runner/compiler_runtime_matrix_test.dart | 182 +++++++++--------- pkgs/test_core/lib/src/bootstrap/vm.dart | 9 +- .../plugin/shared_platform_helpers.dart | 21 ++ .../test_core/lib/src/runner/vm/platform.dart | 17 +- 4 files changed, 120 insertions(+), 109 deletions(-) create mode 100644 pkgs/test_core/lib/src/runner/plugin/shared_platform_helpers.dart diff --git a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart index 88d36b6cf..9c07e2aac 100644 --- a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart +++ b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart @@ -17,90 +17,90 @@ void main() { }); for (var runtime in Runtime.builtIn) { - // Ignore the platforms we can't run on this OS. - if (runtime == Runtime.internetExplorer && !Platform.isWindows || - runtime == Runtime.safari && !Platform.isMacOS) { - continue; - } - group('--runtime ${runtime.identifier}', () { - for (var compiler in runtime.supportedCompilers) { - group('--compiler ${compiler.identifier}', () { - final testArgs = [ - 'test.dart', - '-p', - runtime.identifier, - '-c', - compiler.identifier - ]; - - test('can run passing tests', () async { - await d.file('test.dart', _goodTest).create(); - var test = await runTest(testArgs); - - expect( - test.stdout, emitsThrough(contains('+1: All tests passed!'))); - await test.shouldExit(0); - }); - - test('fails gracefully for invalid code', () async { - await d.file('test.dart', _compileErrorTest).create(); - var test = await runTest(testArgs); - - expect( - test.stdout, - containsInOrder([ - "Error: A value of type 'String' can't be assigned to a variable of type 'int'.", - "int x = 'hello';", - ])); - - await test.shouldExit(1); - }); - - test('fails gracefully for test failures', () async { - await d.file('test.dart', _failingTest).create(); - var test = await runTest(testArgs); - - expect( - test.stdout, - containsInOrder([ - 'Expected: <2>', - 'Actual: <1>', - 'test.dart 5', - '+0 -1: Some tests failed.', - ])); - - await test.shouldExit(1); - }); - - test('fails gracefully if a test file throws in main', () async { - await d.file('test.dart', _throwingTest).create(); - var test = await runTest(testArgs); - var compileOrLoadMessage = - compiler == Compiler.dart2js ? 'compiling' : 'loading'; - - expect( - test.stdout, - containsInOrder([ - '-1: [${runtime.name}, ${compiler.name}] $compileOrLoadMessage ' - 'test.dart [E]', - 'Failed to load "test.dart": oh no' - ])); - await test.shouldExit(1); - }); - - test('captures prints', () async { - await d.file('test.dart', _testWithPrints).create(); - var test = await runTest([...testArgs, '-r', 'json']); - - expect( - test.stdout, - containsInOrder([ - '"messageType":"print","message":"hello","type":"print"', - ])); - - await test.shouldExit(0); - }); - + for (var compiler in runtime.supportedCompilers) { + // Ignore the platforms we can't run on this OS. + if (runtime == Runtime.internetExplorer && !Platform.isWindows || + runtime == Runtime.safari && !Platform.isMacOS) { + continue; + } + group('--runtime ${runtime.identifier} --compiler ${compiler.identifier}', + () { + final testArgs = [ + 'test.dart', + '-p', + runtime.identifier, + '-c', + compiler.identifier + ]; + + test('can run passing tests', () async { + await d.file('test.dart', _goodTest).create(); + var test = await runTest(testArgs); + + expect(test.stdout, emitsThrough(contains('+1: All tests passed!'))); + await test.shouldExit(0); + }); + + test('fails gracefully for invalid code', () async { + await d.file('test.dart', _compileErrorTest).create(); + var test = await runTest(testArgs); + + expect( + test.stdout, + containsInOrder([ + "Error: A value of type 'String' can't be assigned to a variable of type 'int'.", + "int x = 'hello';", + ])); + + await test.shouldExit(1); + }); + + test('fails gracefully for test failures', () async { + await d.file('test.dart', _failingTest).create(); + var test = await runTest(testArgs); + + expect( + test.stdout, + containsInOrder([ + 'Expected: <2>', + 'Actual: <1>', + 'test.dart 5', + '+0 -1: Some tests failed.', + ])); + + await test.shouldExit(1); + }); + + test('fails gracefully if a test file throws in main', () async { + await d.file('test.dart', _throwingTest).create(); + var test = await runTest(testArgs); + var compileOrLoadMessage = + compiler == Compiler.dart2js ? 'compiling' : 'loading'; + + expect( + test.stdout, + containsInOrder([ + '-1: [${runtime.name}, ${compiler.name}] $compileOrLoadMessage ' + 'test.dart [E]', + 'Failed to load "test.dart": oh no' + ])); + await test.shouldExit(1); + }); + + test('captures prints', () async { + await d.file('test.dart', _testWithPrints).create(); + var test = await runTest([...testArgs, '-r', 'json']); + + expect( + test.stdout, + containsInOrder([ + '"messageType":"print","message":"hello","type":"print"', + ])); + + await test.shouldExit(0); + }); + + if (runtime.isDartVM) { test('forwards stdout/stderr', () async { await d.file('test.dart', _testWithStdOutAndErr).create(); var test = await runTest(testArgs); @@ -108,13 +108,13 @@ void main() { expect(test.stdout, emitsThrough('hello')); expect(test.stderr, emits('world')); await test.shouldExit(0); - }, testOn: '!browser'); - }, - skip: compiler == Compiler.dart2wasm - ? 'Wasm tests are experimental and require special setup' - : null); - } - }); + }); + } + }, + skip: compiler == Compiler.dart2wasm + ? 'Wasm tests are experimental and require special setup' + : null); + } } } diff --git a/pkgs/test_core/lib/src/bootstrap/vm.dart b/pkgs/test_core/lib/src/bootstrap/vm.dart index 9a241270a..b8e3a6b0c 100644 --- a/pkgs/test_core/lib/src/bootstrap/vm.dart +++ b/pkgs/test_core/lib/src/bootstrap/vm.dart @@ -12,6 +12,7 @@ import 'package:stream_channel/isolate_channel.dart'; import 'package:stream_channel/stream_channel.dart'; import 'package:test_core/src/runner/plugin/remote_platform_helpers.dart'; +import 'package:test_core/src/runner/plugin/shared_platform_helpers.dart'; /// Bootstraps a vm test to communicate with the test runner over an isolate. void internalBootstrapVmTest(Function Function() getMain, SendPort sendPort) { @@ -37,13 +38,7 @@ void internalBootstrapNativeTest( 'Expected exactly two args, a host and a port, but got $args'); } var socket = await Socket.connect(args[0], int.parse(args[1])); - var platformChannel = MultiChannel(StreamChannel(socket, socket) - .cast>() - .transform(StreamChannelTransformer.fromCodec(utf8)) - .transformStream(const LineSplitter()) - .transformSink(StreamSinkTransformer.fromHandlers( - handleData: (original, sink) => sink.add('$original\n'))) - .transform(jsonDocument)); + var platformChannel = MultiChannel(jsonSocketStreamChannel(socket)); var testControlChannel = platformChannel.virtualChannel() ..pipe(serializeSuite(getMain)); platformChannel.sink.add(testControlChannel.id); diff --git a/pkgs/test_core/lib/src/runner/plugin/shared_platform_helpers.dart b/pkgs/test_core/lib/src/runner/plugin/shared_platform_helpers.dart new file mode 100644 index 000000000..74b4b4c24 --- /dev/null +++ b/pkgs/test_core/lib/src/runner/plugin/shared_platform_helpers.dart @@ -0,0 +1,21 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:async/async.dart'; +import 'package:stream_channel/stream_channel.dart'; + +/// Converts a raw [Socket] into a [StreamChannel] of JSON objects. +/// +/// JSON messages are separated by newlines. +StreamChannel jsonSocketStreamChannel(Socket socket) => + StreamChannel(socket, socket) + .cast>() + .transform(StreamChannelTransformer.fromCodec(utf8)) + .transformStream(const LineSplitter()) + .transformSink(StreamSinkTransformer.fromHandlers( + handleData: (original, sink) => sink.add('$original\n'))) + .transform(jsonDocument); diff --git a/pkgs/test_core/lib/src/runner/vm/platform.dart b/pkgs/test_core/lib/src/runner/vm/platform.dart index 5239f0d86..bb43194a1 100644 --- a/pkgs/test_core/lib/src/runner/vm/platform.dart +++ b/pkgs/test_core/lib/src/runner/vm/platform.dart @@ -23,6 +23,7 @@ import '../../runner/environment.dart'; import '../../runner/load_exception.dart'; import '../../runner/platform.dart'; import '../../runner/plugin/platform_helpers.dart'; +import '../../runner/plugin/shared_platform_helpers.dart'; import '../../runner/runner_suite.dart'; import '../../runner/suite.dart'; import '../../util/package_config.dart'; @@ -49,7 +50,7 @@ class VMPlatform extends PlatformPlugin { _setupPauseAfterTests(); MultiChannel outerChannel; - var cleanupFns = []; + var cleanupCallbacks = []; Isolate? isolate; if (platform.compiler == Compiler.exe) { var serverSocket = await ServerSocket.bind('localhost', 0); @@ -64,14 +65,8 @@ class VMPlatform extends PlatformPlugin { process.stdout.listen(stdout.add); process.stderr.listen(stderr.add); var socket = await serverSocket.first; - outerChannel = MultiChannel(StreamChannel(socket, socket) - .cast>() - .transform(StreamChannelTransformer.fromCodec(utf8)) - .transformStream(const LineSplitter()) - .transformSink(StreamSinkTransformer.fromHandlers( - handleData: (original, sink) => sink.add('$original\n'))) - .transform(jsonDocument)); - cleanupFns + outerChannel = MultiChannel(jsonSocketStreamChannel(socket)); + cleanupCallbacks ..add(serverSocket.close) ..add(process.kill); } else { @@ -85,7 +80,7 @@ class VMPlatform extends PlatformPlugin { rethrow; } outerChannel = MultiChannel(IsolateChannel.connectReceive(receivePort)); - cleanupFns + cleanupCallbacks ..add(receivePort.close) ..add(isolate.kill); } @@ -103,7 +98,7 @@ class VMPlatform extends PlatformPlugin { outerChannel.sink.add('debug'); await outerQueue.next; } - for (var fn in cleanupFns) { + for (var fn in cleanupCallbacks) { fn(); } eventSub?.cancel(); From d24aa8c9b632841d6ae09fdf91434111e0ffe8c2 Mon Sep 17 00:00:00 2001 From: Jake Macdonald Date: Fri, 17 Feb 2023 16:26:01 +0000 Subject: [PATCH 09/13] remove unused imports --- pkgs/test_core/lib/src/bootstrap/vm.dart | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkgs/test_core/lib/src/bootstrap/vm.dart b/pkgs/test_core/lib/src/bootstrap/vm.dart index b8e3a6b0c..2166e7d8d 100644 --- a/pkgs/test_core/lib/src/bootstrap/vm.dart +++ b/pkgs/test_core/lib/src/bootstrap/vm.dart @@ -2,12 +2,10 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -import 'dart:convert'; import 'dart:developer'; import 'dart:io'; import 'dart:isolate'; -import 'package:async/async.dart'; import 'package:stream_channel/isolate_channel.dart'; import 'package:stream_channel/stream_channel.dart'; From 48a6cc5c569999a246d0310f607599c48049a778 Mon Sep 17 00:00:00 2001 From: Jake Macdonald Date: Fri, 17 Feb 2023 16:56:11 +0000 Subject: [PATCH 10/13] skip FF tests on windows also --- pkgs/test/test/runner/compiler_runtime_matrix_test.dart | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart index 9c07e2aac..7148ddca7 100644 --- a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart +++ b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart @@ -113,7 +113,9 @@ void main() { }, skip: compiler == Compiler.dart2wasm ? 'Wasm tests are experimental and require special setup' - : null); + : runtime == Runtime.firefox && Platform.isWindows + ? 'https://github.com/dart-lang/test/issues/1942' + : null); } } } From 0b6d621391d93ec328ce7f03538356b8a59aae71 Mon Sep 17 00:00:00 2001 From: Jake Macdonald Date: Fri, 17 Feb 2023 17:28:25 +0000 Subject: [PATCH 11/13] skip all non-chrome windows JS tests --- pkgs/test/test/runner/compiler_runtime_matrix_test.dart | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart index 7148ddca7..0442d74ce 100644 --- a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart +++ b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart @@ -113,7 +113,9 @@ void main() { }, skip: compiler == Compiler.dart2wasm ? 'Wasm tests are experimental and require special setup' - : runtime == Runtime.firefox && Platform.isWindows + : [Runtime.firefox, Runtime.nodeJS, Runtime.internetExplorer] + .contains(runtime) && + Platform.isWindows ? 'https://github.com/dart-lang/test/issues/1942' : null); } From 64a1758852a3de8a88694c3f76cbffd8f810b071 Mon Sep 17 00:00:00 2001 From: Jacob MacDonald Date: Fri, 17 Feb 2023 09:45:52 -0800 Subject: [PATCH 12/13] use withGuarantees --- .../lib/src/runner/plugin/shared_platform_helpers.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/test_core/lib/src/runner/plugin/shared_platform_helpers.dart b/pkgs/test_core/lib/src/runner/plugin/shared_platform_helpers.dart index 74b4b4c24..4fdcf23d1 100644 --- a/pkgs/test_core/lib/src/runner/plugin/shared_platform_helpers.dart +++ b/pkgs/test_core/lib/src/runner/plugin/shared_platform_helpers.dart @@ -12,7 +12,7 @@ import 'package:stream_channel/stream_channel.dart'; /// /// JSON messages are separated by newlines. StreamChannel jsonSocketStreamChannel(Socket socket) => - StreamChannel(socket, socket) + StreamChannel.withGuarantees(socket, socket) .cast>() .transform(StreamChannelTransformer.fromCodec(utf8)) .transformStream(const LineSplitter()) From 4d38e30312b4f7177cbc00483118d49b72f4e2ab Mon Sep 17 00:00:00 2001 From: Jake Macdonald Date: Fri, 17 Feb 2023 18:22:49 +0000 Subject: [PATCH 13/13] close the outer sink explicitly --- pkgs/test_core/lib/src/runner/vm/platform.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/test_core/lib/src/runner/vm/platform.dart b/pkgs/test_core/lib/src/runner/vm/platform.dart index bb43194a1..75e56dc5b 100644 --- a/pkgs/test_core/lib/src/runner/vm/platform.dart +++ b/pkgs/test_core/lib/src/runner/vm/platform.dart @@ -84,6 +84,7 @@ class VMPlatform extends PlatformPlugin { ..add(receivePort.close) ..add(isolate.kill); } + cleanupCallbacks.add(outerChannel.sink.close); VmService? client; StreamSubscription? eventSub;