Flexible Coverage API

pkgs/test/pubspec.yaml

@@ -31,7 +31,7 @@ dependencies:
   yaml: ^2.0.0
   # Use an exact version until the test_api and test_core package are stable.
   test_api: 0.2.14
-  test_core: 0.2.19
+  test_core: 0.3.0
 
 dev_dependencies:
   fake_async: ^1.0.0

pkgs/test_core/CHANGELOG.md

@@ -1,10 +1,11 @@
-## 0.2.19-dev
+## 0.3.0-dev
 
 * Bump minimum SDK to `2.4.0` for safer usage of for-loop elements.
 * Deprecate `PhantomJS` and provide warning when used. Support for `PhantomJS`
   will be removed in version `2.0.0`.
 * Differentiate between test-randomize-ordering-seed not set and 0 being chosen
   as the random seed.
+* `deserializeSuite` now takes an optional `gatherCoverage` callback.
 
 ## 0.2.18
 

pkgs/test_core/lib/src/runner/coverage.dart

@@ -5,29 +5,20 @@
 import 'dart:convert';
 import 'dart:io';
 
-import 'package:coverage/coverage.dart';
 import 'package:path/path.dart' as p;
 
 import 'live_suite_controller.dart';
 
-/// Collects coverage and outputs to the [coverage] path.
-Future<void> gatherCoverage(
-    String coverage, LiveSuiteController controller) async {
-  final suite = controller.liveSuite.suite;
-
-  if (!suite.platform.runtime.isDartVM) return;
-
-  final isolateId = Uri.parse(suite.environment.observatoryUrl.fragment)
-      .queryParameters['isolateId'];
-
-  final cov = await collect(
-      suite.environment.observatoryUrl, false, false, false, {},
-      isolateIds: {isolateId});
-
-  final outfile = File(p.join('$coverage', '${suite.path}.vm.json'))
+/// Collects coverage and outputs to the [coveragePath] path.
+Future<void> writeCoverage(
+    String coveragePath, LiveSuiteController controller) async {
+  var suite = controller.liveSuite.suite;
+  var coverage = await controller.liveSuite.suite.gatherCoverage();
+  final outfile = File(p.join(coveragePath,
+      '${suite.path}.${suite.platform.runtime.name.toLowerCase()}.json'))
     ..createSync(recursive: true);
   final out = outfile.openWrite();
-  out.write(json.encode(cov));
+  out.write(json.encode(coverage));
   await out.flush();
   await out.close();
 }

pkgs/test_core/lib/src/runner/coverage_stub.dart

@@ -4,6 +4,7 @@
 
 import 'live_suite_controller.dart';
 
-Future<void> gatherCoverage(String coverage, LiveSuiteController controller) =>
+Future<void> writeCoverage(
+        String coveragePath, LiveSuiteController controller) =>
     throw UnsupportedError(
         'Coverage is only supported through the test runner.');

pkgs/test_core/lib/src/runner/engine.dart

@@ -285,7 +285,7 @@ class Engine {
           if (_closed) return;
           await _runGroup(controller, controller.liveSuite.suite.group, []);
           controller.noMoreLiveTests();
-          if (_coverage != null) await gatherCoverage(_coverage, controller);
+          if (_coverage != null) await writeCoverage(_coverage, controller);
           loadResource.allowRelease(() => controller.close());
         });
       }());

pkgs/test_core/lib/src/runner/load_suite.dart

@@ -6,27 +6,23 @@ import 'dart:async';
 
 import 'package:stack_trace/stack_trace.dart';
 import 'package:stream_channel/stream_channel.dart';
-
 import 'package:test_api/src/backend/group.dart'; // ignore: implementation_imports
 import 'package:test_api/src/backend/invoker.dart'; // ignore: implementation_imports
 import 'package:test_api/src/backend/metadata.dart'; // ignore: implementation_imports
+import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports
 import 'package:test_api/src/backend/suite.dart'; // ignore: implementation_imports
 import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports
 import 'package:test_api/src/backend/test.dart'; // ignore: implementation_imports
-import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports
 import 'package:test_api/src/utils.dart'; // ignore: implementation_imports
 
-import 'runner_suite.dart';
-import 'suite.dart';
-
 import '../../test_core.dart';
-
-// ignore: uri_does_not_exist
 import '../util/io_stub.dart'
     // ignore: uri_does_not_exist
     if (dart.library.io) '../util/io.dart';
 import 'load_exception.dart';
 import 'plugin/environment.dart';
+import 'runner_suite.dart';
+import 'suite.dart';
 
 /// The timeout for loading a test suite.
 ///
@@ -214,4 +210,8 @@ class LoadSuite extends Suite implements RunnerSuite {
 
   @override
   Future close() async {}
+
+  @override
+  Future<Map<String, dynamic>> gatherCoverage() =>
+      throw UnsupportedError('Coverage is not supported for LoadSuite tests.');
 }

pkgs/test_core/lib/src/runner/plugin/platform_helpers.dart

@@ -7,19 +7,18 @@ import 'dart:io';
 
 import 'package:stack_trace/stack_trace.dart';
 import 'package:stream_channel/stream_channel.dart';
-
 import 'package:test_api/src/backend/group.dart'; // ignore: implementation_imports
 import 'package:test_api/src/backend/metadata.dart'; // ignore: implementation_imports
 import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports
 import 'package:test_api/src/backend/test.dart'; // ignore: implementation_imports
 import 'package:test_api/src/util/remote_exception.dart'; // ignore: implementation_imports
 
-import '../runner_suite.dart';
-import '../environment.dart';
-import '../suite.dart';
 import '../configuration.dart';
+import '../environment.dart';
 import '../load_exception.dart';
+import '../runner_suite.dart';
 import '../runner_test.dart';
+import '../suite.dart';
 
 /// A helper method for creating a [RunnerSuiteController] containing tests
 /// that communicate over [channel].
@@ -35,13 +34,17 @@ import '../runner_test.dart';
 ///
 /// If [mapper] is passed, it will be used to adjust stack traces for any errors
 /// emitted by tests.
+///
+/// [gatherCoverage] is a callback which returns a hit-map containing merged
+/// coverage report suitable for use with `package:coverage`.
 RunnerSuiteController deserializeSuite(
     String path,
     SuitePlatform platform,
     SuiteConfiguration suiteConfig,
     Environment environment,
     StreamChannel channel,
-    Object message) {
+    Object message,
+    {Future<Map<String, dynamic>> Function() gatherCoverage}) {
   var disconnector = Disconnector();
   var suiteChannel = MultiChannel(channel.transform(disconnector));
 
@@ -110,7 +113,8 @@ RunnerSuiteController deserializeSuite(
   return RunnerSuiteController(
       environment, suiteConfig, suiteChannel, completer.future, platform,
       path: path,
-      onClose: () => disconnector.disconnect().catchError(handleError));
+      onClose: () => disconnector.disconnect().catchError(handleError),
+      gatherCoverage: gatherCoverage);
 }
 
 /// A utility class for storing state while deserializing tests.

pkgs/test_core/lib/src/runner/runner_suite.dart

@@ -6,14 +6,13 @@ import 'dart:async';
 
 import 'package:async/async.dart';
 import 'package:stream_channel/stream_channel.dart';
-
 import 'package:test_api/src/backend/group.dart'; // ignore: implementation_imports
 import 'package:test_api/src/backend/suite.dart'; // ignore: implementation_imports
 import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports
 import 'package:test_api/src/backend/test.dart'; // ignore: implementation_imports
 
-import 'suite.dart';
 import 'environment.dart';
+import 'suite.dart';
 
 /// A suite produced and consumed by the test runner that has runner-specific
 /// logic and lifecycle management.
@@ -77,6 +76,13 @@ class RunnerSuite extends Suite {
 
   /// Closes the suite and releases any resources associated with it.
   Future close() => _controller._close();
+
+  /// Collects a hit-map containing merged coverage.
+  ///
+  /// Result is suitable for input to the coverage formatters provided by
+  /// `package:coverage`.
+  Future<Map<String, dynamic>> gatherCoverage() async =>
+      (await _controller._gatherCoverage?.call()) ?? {};
 }
 
 /// A class that exposes and controls a [RunnerSuite].
@@ -106,20 +112,28 @@ class RunnerSuiteController {
   /// The channel names that have already been used.
   final _channelNames = <String>{};
 
+  /// Collects a hit-map containing merged coverage.
+  final Future<Map<String, dynamic>> Function() _gatherCoverage;
+
   RunnerSuiteController(this._environment, this._config, this._suiteChannel,
       Future<Group> groupFuture, SuitePlatform platform,
-      {String path, Function() onClose})
-      : _onClose = onClose {
+      {String path,
+      Function() onClose,
+      Future<Map<String, dynamic>> Function() gatherCoverage})
+      : _onClose = onClose,
+        _gatherCoverage = gatherCoverage {
     _suite =
         groupFuture.then((group) => RunnerSuite._(this, group, path, platform));
   }
 
   /// Used by [new RunnerSuite] to create a runner suite that's not loaded from
   /// an external source.
   RunnerSuiteController._local(this._environment, this._config,
-      {Function() onClose})
+      {Function() onClose,
+      Future<Map<String, dynamic>> Function() gatherCoverage})
       : _suiteChannel = null,
-        _onClose = onClose;
+        _onClose = onClose,
+        _gatherCoverage = gatherCoverage;
 
   /// Sets whether the suite is paused for debugging.
   ///

pkgs/test_core/lib/src/runner/vm/platform.dart

@@ -7,23 +7,23 @@ import 'dart:developer';
 import 'dart:io';
 import 'dart:isolate';
 
+import 'package:coverage/coverage.dart';
 import 'package:path/path.dart' as p;
 import 'package:stream_channel/isolate_channel.dart';
 import 'package:stream_channel/stream_channel.dart';
 import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports
 import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports
-import 'package:test_core/src/runner/configuration.dart'; // ignore: implementation_imports
-import 'package:test_core/src/runner/load_exception.dart'; // ignore: implementation_imports
-import 'package:test_core/src/runner/platform.dart'; // ignore: implementation_imports
-import 'package:test_core/src/runner/plugin/environment.dart'; // ignore: implementation_imports
-import 'package:test_core/src/runner/plugin/platform_helpers.dart'; // ignore: implementation_imports
-import 'package:test_core/src/runner/runner_suite.dart'; // ignore: implementation_imports
-import 'package:test_core/src/runner/suite.dart'; // ignore: implementation_imports
-import 'package:test_core/src/util/dart.dart' // ignore: implementation_imports
-    as dart;
 import 'package:vm_service/vm_service.dart' hide Isolate;
 import 'package:vm_service/vm_service_io.dart';
 
+import '../../runner/configuration.dart';
+import '../../runner/environment.dart';
+import '../../runner/load_exception.dart';
+import '../../runner/platform.dart';
+import '../../runner/plugin/platform_helpers.dart';
+import '../../runner/runner_suite.dart';
+import '../../runner/suite.dart';
+import '../../util/dart.dart' as dart;
 import 'environment.dart';
 
 /// A platform that loads tests in isolates spawned within this Dart process.
@@ -61,7 +61,7 @@ class VMPlatform extends PlatformPlugin {
       sink.close();
     }));
 
-    VMEnvironment environment;
+    Environment environment;
     IsolateRef isolateRef;
     if (_config.debug) {
       // Print an empty line because the VM prints an "Observatory listening on"
@@ -87,8 +87,11 @@ class VMPlatform extends PlatformPlugin {
       environment = VMEnvironment(url, isolateRef, client);
     }
 
-    var controller = deserializeSuite(path, platform, suiteConfig,
-        environment ?? PluginEnvironment(), channel, message);
+    environment ??= PluginEnvironment();
+
+    var controller = deserializeSuite(
+        path, platform, suiteConfig, environment, channel, message,
+        gatherCoverage: () => _gatherCoverage(environment));
 
     if (isolateRef != null) {
       await client.streamListen('Debug');
@@ -152,6 +155,13 @@ Future<Isolate> _spawnPrecompiledIsolate(
       checked: true);
 }
 
+Future<Map<String, dynamic>> _gatherCoverage(Environment environment) async {
+  final isolateId = Uri.parse(environment.observatoryUrl.fragment)
+      .queryParameters['isolateId'];
+  return await collect(environment.observatoryUrl, false, false, false, {},
+      isolateIds: {isolateId});
+}
+
 Future<Isolate> _spawnPubServeIsolate(
     String testPath, SendPort message, Uri pubServeUrl) async {
   var url = pubServeUrl.resolveUri(

pkgs/test_core/pubspec.yaml

@@ -1,5 +1,5 @@
 name: test_core
-version: 0.2.19-dev
+version: 0.3.0-dev
 description: A basic library for writing tests and running them on the VM.
 homepage: https://github.com/dart-lang/test/blob/master/pkgs/test_core