diff --git a/lib/src/backend/metadata.dart b/lib/src/backend/metadata.dart index 299a6b391..013d91f32 100644 --- a/lib/src/backend/metadata.dart +++ b/lib/src/backend/metadata.dart @@ -19,6 +19,12 @@ import 'test_platform.dart'; /// This metadata comes from declarations on the test itself; it doesn't include /// configuration from the user. class Metadata { + /// Empty metadata with only default values. + /// + /// Using this is slightly more efficient than manually constructing a new + /// metadata with no arguments. + static final empty = new Metadata._(); + /// The selector indicating which platforms the suite supports. final PlatformSelector testOn; @@ -26,14 +32,16 @@ class Metadata { final Timeout timeout; /// Whether the test or suite should be skipped. - final bool skip; - - /// Whether to use verbose stack traces. - final bool verboseTrace; + bool get skip => _skip ?? false; + final bool _skip; /// The reason the test or suite should be skipped, if given. final String skipReason; + /// Whether to use verbose stack traces. + bool get verboseTrace => _verboseTrace ?? false; + final bool _verboseTrace; + /// The user-defined tags attached to the test or suite. final Set tags; @@ -122,8 +130,8 @@ class Metadata { /// If [forTag] contains metadata that applies to [tags], that metadata is /// included inline in the returned value. The values directly passed to the /// constructor take precedence over tag-specific metadata. - factory Metadata({PlatformSelector testOn, Timeout timeout, bool skip: false, - bool verboseTrace: false, String skipReason, Iterable tags, + factory Metadata({PlatformSelector testOn, Timeout timeout, bool skip, + bool verboseTrace, String skipReason, Iterable tags, Map onPlatform, Map forTag}) { // Returns metadata without forTag resolved at all. @@ -159,13 +167,14 @@ class Metadata { /// Creates new Metadata. /// /// Unlike [new Metadata], this assumes [forTag] is already resolved. - Metadata._({PlatformSelector testOn, Timeout timeout, bool skip: false, - this.verboseTrace: false, this.skipReason, Iterable tags, + Metadata._({PlatformSelector testOn, Timeout timeout, bool skip, + this.skipReason, bool verboseTrace, Iterable tags, Map onPlatform, Map forTag}) : testOn = testOn == null ? PlatformSelector.all : testOn, timeout = timeout == null ? const Timeout.factor(1) : timeout, - skip = skip, + _skip = skip, + _verboseTrace = verboseTrace, tags = new UnmodifiableSetView( tags == null ? new Set() : tags.toSet()), onPlatform = onPlatform == null @@ -182,13 +191,14 @@ class Metadata { /// /// Throws a [FormatException] if any field is invalid. Metadata.parse({String testOn, Timeout timeout, skip, - this.verboseTrace: false, Map onPlatform, + bool verboseTrace, Map onPlatform, tags}) : testOn = testOn == null ? PlatformSelector.all : new PlatformSelector.parse(testOn), timeout = timeout == null ? const Timeout.factor(1) : timeout, - skip = skip != null && skip != false, + _skip = skip == null ? null : skip != false, + _verboseTrace = verboseTrace, skipReason = skip is String ? skip : null, onPlatform = _parseOnPlatform(onPlatform), tags = _parseTags(tags), @@ -207,9 +217,9 @@ class Metadata { ? PlatformSelector.all : new PlatformSelector.parse(serialized['testOn']), timeout = _deserializeTimeout(serialized['timeout']), - skip = serialized['skip'], + _skip = serialized['skip'], skipReason = serialized['skipReason'], - verboseTrace = serialized['verboseTrace'], + _verboseTrace = serialized['verboseTrace'], tags = new Set.from(serialized['tags']), onPlatform = new Map.fromIterable(serialized['onPlatform'], key: (pair) => new PlatformSelector.parse(pair.first), @@ -252,9 +262,9 @@ class Metadata { new Metadata( testOn: testOn.intersection(other.testOn), timeout: timeout.merge(other.timeout), - skip: skip || other.skip, + skip: other._skip ?? _skip , skipReason: other.skipReason == null ? skipReason : other.skipReason, - verboseTrace: verboseTrace || other.verboseTrace, + verboseTrace: other._verboseTrace ?? _verboseTrace, tags: tags.union(other.tags), onPlatform: mergeMaps(onPlatform, other.onPlatform, value: (metadata1, metadata2) => metadata1.merge(metadata2)), @@ -273,8 +283,8 @@ class Metadata { Map forTag}) { testOn ??= this.testOn; timeout ??= this.timeout; - skip ??= this.skip; - verboseTrace ??= this.verboseTrace; + skip ??= this._skip; + verboseTrace ??= this._verboseTrace; skipReason ??= this.skipReason; onPlatform ??= this.onPlatform; tags ??= this.tags; diff --git a/lib/src/runner.dart b/lib/src/runner.dart index 67343efb1..04037bfe3 100644 --- a/lib/src/runner.dart +++ b/lib/src/runner.dart @@ -67,9 +67,7 @@ class Runner { /// Creates a new runner based on [configuration]. factory Runner(Configuration config) => config.asCurrent(() { - var engine = new Engine( - concurrency: config.concurrency, - runSkipped: config.runSkipped); + var engine = new Engine(concurrency: config.concurrency); var reporter; switch (config.reporter) { @@ -77,10 +75,9 @@ class Runner { reporter = ExpandedReporter.watch( engine, color: config.color, - verboseTrace: config.verboseTrace, printPath: config.paths.length > 1 || new Directory(config.paths.single).existsSync(), - printPlatform: config.platforms.length > 1); + printPlatform: config.suiteDefaults.platforms.length > 1); break; case "compact": @@ -124,9 +121,11 @@ class Runner { if (_closed) return false; - if (_engine.passed.length == 0 && _engine.failed.length == 0 && - _engine.skipped.length == 0 && _config.patterns.isNotEmpty) { - var patterns = toSentence(_config.patterns.map( + if (_engine.passed.length == 0 && + _engine.failed.length == 0 && + _engine.skipped.length == 0 && + _config.suiteDefaults.patterns.isNotEmpty) { + var patterns = toSentence(_config.suiteDefaults.patterns.map( (pattern) => pattern is RegExp ? 'regular expression "${pattern.pattern}"' : '"$pattern"')); @@ -142,11 +141,12 @@ class Runner { /// Emits a warning if the user is trying to run on a platform that's /// unsupported for the entire package. void _warnForUnsupportedPlatforms() { - if (_config.testOn == PlatformSelector.all) return; + var testOn = _config.suiteDefaults.metadata.testOn; + if (testOn == PlatformSelector.all) return; - var unsupportedPlatforms = _config.platforms.where((platform) { - return !_config.testOn.evaluate(platform, os: currentOS); - }).toList(); + var unsupportedPlatforms = _config.suiteDefaults.platforms + .where((platform) => !testOn.evaluate(platform, os: currentOS)) + .toList(); if (unsupportedPlatforms.isEmpty) return; // Human-readable names for all unsupported platforms. @@ -160,7 +160,7 @@ class Runner { if (unsupportedBrowsers.isNotEmpty) { var supportsAnyBrowser = TestPlatform.all .where((platform) => platform.isBrowser) - .any((platform) => _config.testOn.evaluate(platform)); + .any((platform) => testOn.evaluate(platform)); if (supportsAnyBrowser) { unsupportedNames.addAll( @@ -174,7 +174,7 @@ class Runner { // that's because of the current OS or whether the VM is unsupported. if (unsupportedPlatforms.contains(TestPlatform.vm)) { var supportsAnyOS = OperatingSystem.all.any((os) => - _config.testOn.evaluate(TestPlatform.vm, os: os)); + testOn.evaluate(TestPlatform.vm, os: os)); if (supportsAnyOS) { unsupportedNames.add(currentOS.name); @@ -232,29 +232,37 @@ class Runner { /// suites once they're loaded. Stream _loadSuites() { return StreamGroup.merge(_config.paths.map((path) { - if (new Directory(path).existsSync()) return _loader.loadDir(path); - if (new File(path).existsSync()) return _loader.loadFile(path); - - return new Stream.fromIterable([ - new LoadSuite.forLoadException( - new LoadException(path, 'Does not exist.')) - ]); + if (new Directory(path).existsSync()) { + return _loader.loadDir(path, _config.suiteDefaults); + } else if (new File(path).existsSync()) { + return _loader.loadFile(path, _config.suiteDefaults); + } else { + return new Stream.fromIterable([ + new LoadSuite.forLoadException( + new LoadException(path, 'Does not exist.'), + _config.suiteDefaults) + ]); + } })).map((loadSuite) { return loadSuite.changeSuite((suite) { _warnForUnknownTags(suite); return _shardSuite(suite.filter((test) { // Skip any tests that don't match all the given patterns. - if (!_config.patterns + if (!suite.config.patterns .every((pattern) => test.name.contains(pattern))) { return false; } // If the user provided tags, skip tests that don't match all of them. - if (!_config.includeTags.evaluate(test.metadata.tags)) return false; + if (!suite.config.includeTags.evaluate(test.metadata.tags)) { + return false; + } // Skip tests that do match any tags the user wants to exclude. - if (_config.excludeTags.evaluate(test.metadata.tags)) return false; + if (suite.config.excludeTags.evaluate(test.metadata.tags)) { + return false; + } return true; })); @@ -363,7 +371,7 @@ class Runner { /// Loads each suite in [suites] in order, pausing after load for platforms /// that support debugging. Future _loadThenPause(Stream suites) async { - if (_config.platforms.contains(TestPlatform.vm)) { + if (_config.suiteDefaults.platforms.contains(TestPlatform.vm)) { warn("Debugging is currently unsupported on the Dart VM.", color: _config.color); } diff --git a/lib/src/runner/browser/browser_manager.dart b/lib/src/runner/browser/browser_manager.dart index d8d0aa3cb..3de3b501d 100644 --- a/lib/src/runner/browser/browser_manager.dart +++ b/lib/src/runner/browser/browser_manager.dart @@ -10,10 +10,10 @@ import 'package:pool/pool.dart'; import 'package:stream_channel/stream_channel.dart'; import 'package:web_socket_channel/web_socket_channel.dart'; -import '../../backend/metadata.dart'; import '../../backend/test_platform.dart'; import '../../util/stack_trace_mapper.dart'; import '../application_exception.dart'; +import '../configuration/suite.dart'; import '../environment.dart'; import '../plugin/platform_helpers.dart'; import '../runner_suite.dart'; @@ -193,14 +193,14 @@ class BrowserManager { /// /// [url] should be an HTML page with a reference to the JS-compiled test /// suite. [path] is the path of the original test suite file, which is used - /// for reporting. [metadata] is the parsed metadata for the test suite. + /// for reporting. [suiteConfig] is the configuration for the test suite. /// /// If [mapper] is passed, it's used to map stack traces for errors coming /// from this test suite. - Future load(String path, Uri url, Metadata metadata, + Future load(String path, Uri url, SuiteConfiguration suiteConfig, {StackTraceMapper mapper}) async { url = url.replace(fragment: Uri.encodeFull(JSON.encode({ - "metadata": metadata.serialize(), + "metadata": suiteConfig.metadata.serialize(), "browser": _platform.identifier }))); @@ -235,7 +235,7 @@ class BrowserManager { try { controller = await deserializeSuite( - path, _platform, metadata, await _environment, suiteChannel, + path, _platform, suiteConfig, await _environment, suiteChannel, mapTrace: mapper?.mapStackTrace); _controllers.add(controller); return controller.suite; diff --git a/lib/src/runner/browser/compiler_pool.dart b/lib/src/runner/browser/compiler_pool.dart index 2c444a676..76dcfa1f8 100644 --- a/lib/src/runner/browser/compiler_pool.dart +++ b/lib/src/runner/browser/compiler_pool.dart @@ -13,6 +13,7 @@ import 'package:pool/pool.dart'; import '../../util/io.dart'; import '../configuration.dart'; +import '../configuration/suite.dart'; import '../load_exception.dart'; /// A regular expression matching the first status line printed by dart2js. @@ -47,7 +48,8 @@ class CompilerPool { /// /// The returned [Future] will complete once the `dart2js` process completes /// *and* all its output has been printed to the command line. - Future compile(String dartPath, String jsPath) { + Future compile(String dartPath, String jsPath, + SuiteConfiguration suiteConfig) { return _pool.withResource(() { if (_closed) return null; @@ -75,7 +77,7 @@ class CompilerPool { wrapperPath, "--out=$jsPath", await PackageResolver.current.processArgument - ]..addAll(_config.dart2jsArgs); + ]..addAll(suiteConfig.dart2jsArgs); if (_config.color) args.add("--enable-diagnostic-colors"); diff --git a/lib/src/runner/browser/platform.dart b/lib/src/runner/browser/platform.dart index 3661106d0..4a464ce67 100644 --- a/lib/src/runner/browser/platform.dart +++ b/lib/src/runner/browser/platform.dart @@ -19,7 +19,6 @@ import 'package:shelf_packages_handler/shelf_packages_handler.dart'; import 'package:stream_channel/stream_channel.dart'; import 'package:web_socket_channel/web_socket_channel.dart'; -import '../../backend/metadata.dart'; import '../../backend/test_platform.dart'; import '../../util/io.dart'; import '../../util/one_off_handler.dart'; @@ -27,6 +26,7 @@ import '../../util/path_handler.dart'; import '../../util/stack_trace_mapper.dart'; import '../../utils.dart'; import '../configuration.dart'; +import '../configuration/suite.dart'; import '../load_exception.dart'; import '../plugin/platform.dart'; import '../runner_suite.dart'; @@ -42,11 +42,11 @@ class BrowserPlatform extends PlatformPlugin { static Future start({String root}) async { var server = new shelf_io.IOServer(await HttpMultiServer.loopback(0)); - return new BrowserPlatform._(server, root: root); + return new BrowserPlatform._(server, Configuration.current, root: root); } /// The test runner configuration. - final _config = Configuration.current; + final Configuration _config; /// The underlying server. final shelf.Server _server; @@ -101,6 +101,14 @@ class BrowserPlatform extends PlatformPlugin { final _browserManagers = new Map>>(); + /// A cascade of handlers for suites' precompiled paths. + /// + /// This is `null` if there are no precompiled suites yet. + shelf.Cacade _precompiledCascade; + + /// The precompiled paths that have handlers in [_precompiledHandler]. + final _precompiledPaths = new Set(); + /// A map from test suite paths to Futures that will complete once those /// suites are finished compiling. /// @@ -111,14 +119,11 @@ class BrowserPlatform extends PlatformPlugin { /// Mappers for Dartifying stack traces, indexed by test path. final _mappers = new Map(); - BrowserPlatform._(this._server, {String root}) - : _root = root == null ? p.current : root, - _compiledDir = Configuration.current.pubServeUrl == null - ? createTempDir() - : null, - _http = Configuration.current.pubServeUrl == null - ? null - : new HttpClient(), + BrowserPlatform._(this._server, Configuration config, {String root}) + : _config = config, + _root = root == null ? p.current : root, + _compiledDir = config.pubServeUrl == null ? createTempDir() : null, + _http = config.pubServeUrl == null ? null : new HttpClient(), _compilers = new CompilerPool() { var cascade = new shelf.Cascade() .add(_webSocketHandler.handler); @@ -127,15 +132,13 @@ class BrowserPlatform extends PlatformPlugin { cascade = cascade .add(packagesDirHandler()) .add(_jsHandler.handler) - .add(createStaticHandler(_root)); - - // Add this before the wrapper handler so that its HTML takes precedence - // over the test runner's. - if (_config.precompiledPath != null) { - cascade = cascade.add(createStaticHandler(_config.precompiledPath)); - } + .add(createStaticHandler(_root)) - cascade = cascade.add(_wrapperHandler); + // Add this before the wrapper handler so that its HTML takes + // precedence over the test runner's. + .add((request) => _precompiledCascade?.handler(request) ?? + new shelf.Response.notFound(null)) + .add(_wrapperHandler); } var pipeline = new shelf.Pipeline() @@ -196,7 +199,9 @@ class BrowserPlatform extends PlatformPlugin { /// This will start a browser to load the suite if one isn't already running. /// Throws an [ArgumentError] if [browser] isn't a browser platform. Future load(String path, TestPlatform browser, - Metadata metadata) async { + SuiteConfiguration suiteConfig) async { + assert(suiteConfig.platforms.contains(browser)); + if (!browser.isBrowser) { throw new ArgumentError("$browser is not a browser."); } @@ -230,10 +235,21 @@ class BrowserPlatform extends PlatformPlugin { '$suitePrefix.dart.browser_test.dart'); } - await _pubServeSuite(path, dartUrl, browser); + await _pubServeSuite(path, dartUrl, browser, suiteConfig); suiteUrl = _config.pubServeUrl.resolveUri(p.toUri('$suitePrefix.html')); } else { - if (browser.isJS && !_precompiled(path)) await _compileSuite(path); + if (browser.isJS) { + if (_precompiled(suiteConfig, path)) { + if (_precompiledPaths.add(suiteConfig.precompiledPath)) { + _precompiledCascade ??= new shelf.Cascade(); + _precompiledCascade = _precompiledCascade.add( + createStaticHandler(suiteConfig.precompiledPath)); + } + } else { + await _compileSuite(path, suiteConfig); + } + } + if (_closed) return null; suiteUrl = url.resolveUri(p.toUri( p.withoutExtension(p.relative(path, from: _root)) + ".html")); @@ -245,18 +261,18 @@ class BrowserPlatform extends PlatformPlugin { var browserManager = await _browserManagerFor(browser); if (_closed) return null; - var suite = await browserManager.load(path, suiteUrl, metadata, + var suite = await browserManager.load(path, suiteUrl, suiteConfig, mapper: browser.isJS ? _mappers[path] : null); if (_closed) return null; return suite; } /// Returns whether the test at [path] has precompiled HTML available - /// underneath `_config.precompiledPath`. - bool _precompiled(String path) { - if (_config.precompiledPath == null) return false; + /// underneath [suiteConfig.precompiledPath]. + bool _precompiled(SuiteConfiguration suiteConfig, String path) { + if (suiteConfig.precompiledPath == null) return false; var htmlPath = p.join( - _config.precompiledPath, + suiteConfig.precompiledPath, p.relative(p.withoutExtension(path) + ".html", from: _root)); return new File(htmlPath).existsSync(); } @@ -268,7 +284,8 @@ class BrowserPlatform extends PlatformPlugin { /// /// This ensures that only one suite is loaded at a time, and that any errors /// are exposed as [LoadException]s. - Future _pubServeSuite(String path, Uri dartUrl, TestPlatform browser) { + Future _pubServeSuite(String path, Uri dartUrl, TestPlatform browser, + SuiteConfiguration suiteConfig) { return _pubServePool.withResource(() async { var timer = new Timer(new Duration(seconds: 1), () { print('"pub serve" is compiling $path...'); @@ -300,7 +317,7 @@ class BrowserPlatform extends PlatformPlugin { 'Make sure "pub serve" is serving the test/ directory.'); } - if (getSourceMap && !_config.jsTrace) { + if (getSourceMap && !suiteConfig.jsTrace) { _mappers[path] = new StackTraceMapper( await UTF8.decodeStream(response), mapUrl: url, @@ -332,12 +349,12 @@ class BrowserPlatform extends PlatformPlugin { /// /// Once the suite has been compiled, it's added to [_jsHandler] so it can be /// served. - Future _compileSuite(String dartPath) { + Future _compileSuite(String dartPath, SuiteConfiguration suiteConfig) { return _compileFutures.putIfAbsent(dartPath, () async { var dir = new Directory(_compiledDir).createTempSync('test_').path; var jsPath = p.join(dir, p.basename(dartPath) + ".browser_test.dart.js"); - await _compilers.compile(dartPath, jsPath); + await _compilers.compile(dartPath, jsPath, suiteConfig); if (_closed) return; var jsUrl = p.toUri(p.relative(dartPath, from: _root)).path + @@ -355,7 +372,7 @@ class BrowserPlatform extends PlatformPlugin { headers: {'Content-Type': 'application/json'}); }); - if (_config.jsTrace) return; + if (suiteConfig.jsTrace) return; var mapPath = jsPath + '.map'; _mappers[dartPath] = new StackTraceMapper( new File(mapPath).readAsStringSync(), diff --git a/lib/src/runner/configuration.dart b/lib/src/runner/configuration.dart index 337dd7b21..4c011081f 100644 --- a/lib/src/runner/configuration.dart +++ b/lib/src/runner/configuration.dart @@ -10,13 +10,13 @@ import 'package:collection/collection.dart'; import 'package:glob/glob.dart'; import 'package:path/path.dart' as p; -import '../backend/metadata.dart'; import '../backend/platform_selector.dart'; import '../backend/test_platform.dart'; import '../frontend/timeout.dart'; import '../util/io.dart'; import 'configuration/args.dart' as args; import 'configuration/load.dart'; +import 'configuration/suite.dart'; import 'configuration/values.dart'; /// The key used to look up [Configuration.current] in a zone. @@ -41,33 +41,6 @@ class Configuration { bool get version => _version ?? false; final bool _version; - /// Whether stack traces should be presented as-is or folded to remove - /// irrelevant packages. - bool get verboseTrace => _verboseTrace ?? false; - final bool _verboseTrace; - - /// Whether JavaScript stack traces should be left as-is or converted to - /// Dart-like traces. - bool get jsTrace => _jsTrace ?? false; - final bool _jsTrace; - - /// Whether tests should be skipped. - bool get skip => _skip ?? false; - final bool _skip; - - /// The reason tests or suites should be skipped, if given. - final String skipReason; - - /// Whether skipped tests should be run. - bool get runSkipped => _runSkipped ?? false; - final bool _runSkipped; - - /// The selector indicating which platforms the tests support. - /// - /// When [merge]d, this is intersected with the other configuration's - /// supported platforms. - final PlatformSelector testOn; - /// Whether to pause for debugging after loading each test suite. bool get pauseAfterLoad => _pauseAfterLoad ?? false; final bool _pauseAfterLoad; @@ -76,16 +49,6 @@ class Configuration { String get dart2jsPath => _dart2jsPath ?? p.join(sdkDir, 'bin', 'dart2js'); final String _dart2jsPath; - /// Additional arguments to pass to dart2js. - final List dart2jsArgs; - - /// The path to a mirror of this package containing HTML that points to - /// precompiled JS. - /// - /// This is used by the internal Google test runner so that test compilation - /// can more effectively make use of Google's build tools. - final String precompiledPath; - /// The name of the reporter to use to display results. String get reporter => _reporter ?? defaultReporter; final String _reporter; @@ -94,12 +57,6 @@ class Configuration { /// if tests should be loaded from the filesystem. final Uri pubServeUrl; - /// The default test timeout. - /// - /// When [merge]d, this combines with the other configuration's timeout using - /// [Timeout.merge]. - final Timeout timeout; - /// Whether to use command-line color escapes. bool get color => _color ?? canUseSpecialChars; final bool _color; @@ -143,16 +100,6 @@ class Configuration { Glob get filename => _filename ?? defaultFilename; final Glob _filename; - /// The patterns to match against test names to decide which to run, or `null` - /// if all tests should be run. - /// - /// All patterns must match in order for a test to be run. - final Set patterns; - - /// The set of platforms on which to run tests. - List get platforms => _platforms ?? [TestPlatform.vm]; - final List _platforms; - /// The set of presets to use. /// /// Any chosen presets for the parent configuration are added to the chosen @@ -161,54 +108,12 @@ class Configuration { /// Note that the order of this set matters. final Set chosenPresets; - /// Only run tests whose tags match this selector. - /// - /// When [merge]d, this is intersected with the other configuration's included - /// tags. - final BooleanSelector includeTags; - - /// Do not run tests whose tags match this selector. - /// - /// When [merge]d, this is unioned with the other configuration's - /// excluded tags. - final BooleanSelector excludeTags; - - /// Configuration for particular tags. - /// - /// The keys are tag selectors, and the values are configurations for tests - /// whose tags match those selectors. The configuration should only contain - /// test-level configuration fields, but that isn't enforced. - final Map tags; - - /// Tags that are added to the tests. - /// - /// This is usually only used for scoped configuration. - final Set addTags; - - /// The global test metadata derived from this configuration. - Metadata get metadata => new Metadata( - timeout: timeout, - verboseTrace: verboseTrace, - skip: skip, - skipReason: skipReason, - testOn: testOn, - tags: addTags, - forTag: mapMap(tags, value: (_, config) => config.metadata), - onPlatform: mapMap(onPlatform, value: (_, config) => config.metadata)); - /// The set of tags that have been declared in any way in this configuration. Set get knownTags { if (_knownTags != null) return _knownTags; - var known = includeTags.variables.toSet() - ..addAll(excludeTags.variables) - ..addAll(addTags); - - for (var selector in tags.keys) { - known.addAll(selector.variables); - } - - for (var configuration in _children) { + var known = suiteDefaults.knownTags.toSet(); + for (var configuration in presets.values) { known.addAll(configuration.knownTags); } @@ -217,13 +122,6 @@ class Configuration { } Set _knownTags; - /// Configuration for particular platforms. - /// - /// The keys are platform selectors, and the values are configurations for - /// those platforms. These configuration should only contain test-level - /// configuration fields, but that isn't enforced. - final Map onPlatform; - /// Configuration presets. /// /// These are configurations that can be explicitly selected by the user via @@ -241,7 +139,7 @@ class Configuration { if (_knownPresets != null) return _knownPresets; var known = presets.keys.toSet(); - for (var configuration in _children) { + for (var configuration in presets.values) { known.addAll(configuration.knownPresets); } @@ -250,13 +148,8 @@ class Configuration { } Set _knownPresets; - /// All child configurations of [this] that may be selected under various - /// circumstances. - Iterable get _children sync* { - yield* tags.values; - yield* onPlatform.values; - yield* presets.values; - } + /// The default suite-level configuration. + final SuiteConfiguration suiteDefaults; /// Returns the current configuration, or a default configuration if no /// current configuration is set. @@ -283,92 +176,74 @@ class Configuration { factory Configuration({ bool help, bool version, - bool verboseTrace, - bool jsTrace, - bool skip, - String skipReason, - bool runSkipped, - PlatformSelector testOn, bool pauseAfterLoad, bool color, String dart2jsPath, - Iterable dart2jsArgs, - String precompiledPath, String reporter, int pubServePort, int concurrency, int shardIndex, int totalShards, - Timeout timeout, - Iterable patterns, - Iterable platforms, Iterable paths, Glob filename, Iterable chosenPresets, + Map presets, + + // Suite-level configuration + bool jsTrace, + bool runSkipped, + Iterable dart2jsArgs, + String precompiledPath, + Iterable patterns, + Iterable platforms, BooleanSelector includeTags, BooleanSelector excludeTags, - Iterable addTags, - Map tags, - Map onPlatform, - Map presets}) { - _unresolved() => new Configuration._( + Map tags, + Map onPlatform, + + // Test-level configuration + Timeout timeout, + bool verboseTrace, + bool skip, + String skipReason, + PlatformSelector testOn, + Iterable addTags}) { + var chosenPresetSet = chosenPresets?.toSet(); + var configuration = new Configuration._( help: help, version: version, - verboseTrace: verboseTrace, - jsTrace: jsTrace, - skip: skip, - skipReason: skipReason, - runSkipped: runSkipped, - testOn: testOn, pauseAfterLoad: pauseAfterLoad, color: color, dart2jsPath: dart2jsPath, - dart2jsArgs: dart2jsArgs, - precompiledPath: precompiledPath, reporter: reporter, pubServePort: pubServePort, concurrency: concurrency, shardIndex: shardIndex, totalShards: totalShards, - timeout: timeout, - patterns: patterns, - platforms: platforms, paths: paths, filename: filename, - chosenPresets: chosenPresets, - includeTags: includeTags, - excludeTags: excludeTags, - addTags: addTags, - - // Make sure we pass [chosenPresets] to the child configurations as - // well. This ensures that tags and platforms can have preset-specific - // behavior. - tags: _withChosenPresets(tags, chosenPresets), - onPlatform: _withChosenPresets(onPlatform, chosenPresets), - presets: _withChosenPresets(presets, chosenPresets)); - - if (chosenPresets == null) return _unresolved(); - chosenPresets = new Set.from(chosenPresets); - - if (presets == null) return _unresolved(); - presets = new Map.from(presets); - - var knownPresets = presets.keys.toSet(); - - var merged = chosenPresets.fold(Configuration.empty, (merged, preset) { - if (!presets.containsKey(preset)) return merged; - return merged.merge(presets.remove(preset)); - }); - - var result = merged == Configuration.empty - ? _unresolved() - : _unresolved().merge(merged); - - // Make sure the configuration knows about presets that were selected and - // thus removed from [presets]. - result._knownPresets = result.knownPresets.union(knownPresets); - - return result; + chosenPresets: chosenPresetSet, + presets: _withChosenPresets(presets, chosenPresetSet), + suiteDefaults: new SuiteConfiguration( + jsTrace: jsTrace, + runSkipped: runSkipped, + dart2jsArgs: dart2jsArgs, + precompiledPath: precompiledPath, + patterns: patterns, + platforms: platforms, + includeTags: includeTags, + excludeTags: excludeTags, + tags: tags, + onPlatform: onPlatform, + + // Test-level configuration + timeout: timeout, + verboseTrace: verboseTrace, + skip: skip, + skipReason: skipReason, + testOn: testOn, + addTags: addTags)); + return configuration._resolvePresets(); } static Map _withChosenPresets( @@ -384,65 +259,38 @@ class Configuration { Configuration._({ bool help, bool version, - bool verboseTrace, - bool jsTrace, - bool skip, - this.skipReason, - bool runSkipped, - PlatformSelector testOn, bool pauseAfterLoad, bool color, String dart2jsPath, - Iterable dart2jsArgs, - this.precompiledPath, String reporter, int pubServePort, int concurrency, this.shardIndex, this.totalShards, - Timeout timeout, - Iterable patterns, - Iterable platforms, Iterable paths, Glob filename, Iterable chosenPresets, - BooleanSelector includeTags, - BooleanSelector excludeTags, - Iterable addTags, - Map tags, - Map onPlatform, - Map presets}) + Map presets, + SuiteConfiguration suiteDefaults}) : _help = help, _version = version, - _verboseTrace = verboseTrace, - _jsTrace = jsTrace, - _skip = skip, - _runSkipped = runSkipped, - testOn = testOn ?? PlatformSelector.all, _pauseAfterLoad = pauseAfterLoad, _color = color, _dart2jsPath = dart2jsPath, - dart2jsArgs = dart2jsArgs?.toList() ?? [], _reporter = reporter, pubServeUrl = pubServePort == null ? null : Uri.parse("http://localhost:$pubServePort"), _concurrency = concurrency, - timeout = (pauseAfterLoad ?? false) - ? Timeout.none - : (timeout == null ? new Timeout.factor(1) : timeout), - patterns = new UnmodifiableSetView(patterns?.toSet() ?? new Set()), - _platforms = _list(platforms), _paths = _list(paths), _filename = filename, chosenPresets = new UnmodifiableSetView( chosenPresets?.toSet() ?? new Set()), - includeTags = includeTags ?? BooleanSelector.all, - excludeTags = excludeTags ?? BooleanSelector.none, - addTags = new UnmodifiableSetView(addTags?.toSet() ?? new Set()), - tags = _map(tags), - onPlatform = _map(onPlatform), - presets = _map(presets) { + presets = _map(presets), + suiteDefaults = pauseAfterLoad == true + ? suiteDefaults?.change(timeout: Timeout.none) ?? + new SuiteConfiguration(timeout: Timeout.none) + : suiteDefaults ?? SuiteConfiguration.empty { if (_filename != null && _filename.context.style != p.style) { throw new ArgumentError( "filename's context must match the current operating system, was " @@ -458,10 +306,15 @@ class Configuration { } } - /// Returns a [input] as an unmodifiable list or `null`. + /// Creates a new [Configuration] that takes its configuration from + /// [SuiteConfiguration]. + factory Configuration.fromSuiteConfiguration( + SuiteConfiguration suiteConfig) => + new Configuration._(suiteDefaults: suiteConfig); + + /// Returns an unmodifiable copy of [input]. /// - /// If [input] is `null` or empty, this returns `null`. Otherwise, it returns - /// `input.toList()`. + /// If [input] is `null` or empty, this returns `null`. static List/**/ _list/**/(Iterable/**/ input) { if (input == null) return null; var list = new List/**/.unmodifiable(input); @@ -491,37 +344,23 @@ class Configuration { if (this == Configuration.empty) return other; if (other == Configuration.empty) return this; - var result = new Configuration( + var result = new Configuration._( help: other._help ?? _help, version: other._version ?? _version, - verboseTrace: other._verboseTrace ?? _verboseTrace, - jsTrace: other._jsTrace ?? _jsTrace, - skip: other._skip ?? _skip, - skipReason: other.skipReason ?? skipReason, - runSkipped: other._runSkipped ?? _runSkipped, - testOn: testOn.intersection(other.testOn), pauseAfterLoad: other._pauseAfterLoad ?? _pauseAfterLoad, color: other._color ?? _color, dart2jsPath: other._dart2jsPath ?? _dart2jsPath, - dart2jsArgs: dart2jsArgs.toList()..addAll(other.dart2jsArgs), - precompiledPath: other.precompiledPath ?? precompiledPath, reporter: other._reporter ?? _reporter, pubServePort: (other.pubServeUrl ?? pubServeUrl)?.port, concurrency: other._concurrency ?? _concurrency, shardIndex: other.shardIndex ?? shardIndex, totalShards: other.totalShards ?? totalShards, - timeout: timeout.merge(other.timeout), - patterns: patterns.union(other.patterns), - platforms: other._platforms ?? _platforms, paths: other._paths ?? _paths, filename: other._filename ?? _filename, chosenPresets: chosenPresets.union(other.chosenPresets), - includeTags: includeTags.intersection(other.includeTags), - excludeTags: excludeTags.union(other.excludeTags), - addTags: other.addTags.union(addTags), - tags: _mergeConfigMaps(tags, other.tags), - onPlatform: _mergeConfigMaps(onPlatform, other.onPlatform), - presets: _mergeConfigMaps(presets, other.presets)); + presets: _mergeConfigMaps(presets, other.presets), + suiteDefaults: suiteDefaults.merge(other.suiteDefaults)); + result = result._resolvePresets(); // Make sure the merged config preserves any presets that were chosen and // discarded. @@ -536,65 +375,71 @@ class Configuration { Configuration change({ bool help, bool version, - bool verboseTrace, - bool jsTrace, - bool skip, - String skipReason, - bool runSkipped, - PlatformSelector testOn, bool pauseAfterLoad, bool color, String dart2jsPath, - Iterable dart2jsArgs, - String precompiledPath, String reporter, int pubServePort, int concurrency, int shardIndex, int totalShards, - Timeout timeout, - Iterable patterns, - Iterable platforms, Iterable paths, Glob filename, Iterable chosenPresets, + Map presets, + + // Suite-level configuration + bool jsTrace, + bool runSkipped, + Iterable dart2jsArgs, + String precompiledPath, + Iterable patterns, + Iterable platforms, BooleanSelector includeTags, BooleanSelector excludeTags, - Iterable addTags, - Map tags, - Map onPlatform, - Map presets}) { - return new Configuration( + Map tags, + Map onPlatform, + + // Test-level configuration + Timeout timeout, + bool verboseTrace, + bool skip, + String skipReason, + PlatformSelector testOn, + Iterable addTags}) { + var config = new Configuration._( help: help ?? _help, version: version ?? _version, - verboseTrace: verboseTrace ?? _verboseTrace, - jsTrace: jsTrace ?? _jsTrace, - skip: skip ?? _skip, - skipReason: skipReason ?? this.skipReason, - runSkipped: runSkipped ?? _runSkipped, - testOn: testOn ?? this.testOn, pauseAfterLoad: pauseAfterLoad ?? _pauseAfterLoad, color: color ?? _color, dart2jsPath: dart2jsPath ?? _dart2jsPath, - dart2jsArgs: dart2jsArgs?.toList() ?? this.dart2jsArgs, - precompiledPath: precompiledPath ?? this.precompiledPath, reporter: reporter ?? _reporter, pubServePort: pubServePort ?? pubServeUrl?.port, concurrency: concurrency ?? _concurrency, shardIndex: shardIndex ?? this.shardIndex, totalShards: totalShards ?? this.totalShards, - timeout: timeout ?? this.timeout, - patterns: patterns ?? this.patterns, - platforms: platforms ?? _platforms, paths: paths ?? _paths, filename: filename ?? _filename, chosenPresets: chosenPresets ?? this.chosenPresets, - includeTags: includeTags ?? this.includeTags, - excludeTags: excludeTags ?? this.excludeTags, - addTags: addTags ?? this.addTags, - tags: tags ?? this.tags, - onPlatform: onPlatform ?? this.onPlatform, - presets: presets ?? this.presets); + presets: presets ?? this.presets, + suiteDefaults: suiteDefaults.change( + jsTrace: jsTrace, + runSkipped: runSkipped, + dart2jsArgs: dart2jsArgs, + precompiledPath: precompiledPath, + patterns: patterns, + platforms: platforms, + includeTags: includeTags, + excludeTags: excludeTags, + tags: tags, + onPlatform: onPlatform, + timeout: timeout, + verboseTrace: verboseTrace, + skip: skip, + skipReason: skipReason, + testOn: testOn, + addTags: addTags)); + return config._resolvePresets(); } /// Merges two maps whose values are [Configuration]s. @@ -605,4 +450,24 @@ class Configuration { Map map2) => mergeMaps(map1, map2, value: (config1, config2) => config1.merge(config2)); + + /// Returns a copy of this [Configuration] with all [chosenPresets] resolved + /// against [presets]. + Configuration _resolvePresets() { + if (chosenPresets.isEmpty || presets.isEmpty) return this; + + var newPresets = new Map.from(presets); + var merged = chosenPresets.fold(empty, (merged, preset) => + merged.merge(newPresets.remove(preset) ?? Configuration.empty)); + + if (merged == empty) return this; + var result = this.change(presets: newPresets).merge(merged); + + // Make sure the configuration knows about presets that were selected and + // thus removed from [newPresets]. + result._knownPresets = new UnmodifiableSetView( + result.knownPresets.toSet()..addAll(this.presets.keys)); + + return result; + } } diff --git a/lib/src/runner/configuration/load.dart b/lib/src/runner/configuration/load.dart index 4d0fb4844..c514fdde4 100644 --- a/lib/src/runner/configuration/load.dart +++ b/lib/src/runner/configuration/load.dart @@ -18,6 +18,7 @@ import '../../frontend/timeout.dart'; import '../../utils.dart'; import '../../util/io.dart'; import '../configuration.dart'; +import '../configuration/suite.dart'; import 'values.dart'; /// Loads configuration information from a YAML file at [path]. @@ -99,11 +100,12 @@ class _ConfigurationLoader { value: (valueNode) => _nestedConfig(valueNode, "presets value")); var config = new Configuration( - verboseTrace: verboseTrace, - jsTrace: jsTrace, - timeout: timeout, - onPlatform: onPlatform, - presets: presets); + verboseTrace: verboseTrace, + jsTrace: jsTrace, + timeout: timeout, + presets: presets) + .merge(_extractPresets/**/(onPlatform, + (map) => new Configuration(onPlatform: map))); var osConfig = onOS[currentOS]; return osConfig == null ? config : config.merge(osConfig); @@ -144,11 +146,12 @@ class _ConfigurationLoader { _nestedConfig(valueNode, "tag value", runnerConfig: false)); return new Configuration( - skip: skip, - skipReason: skipReason, - testOn: testOn, - addTags: addTags, - tags: tags); + skip: skip, + skipReason: skipReason, + testOn: testOn, + addTags: addTags) + .merge(_extractPresets/**/(tags, + (map) => new Configuration(tags: map))); } /// Loads runner configuration that's allowed in the global configuration @@ -387,6 +390,40 @@ class _ConfigurationLoader { return loader.load(); } + /// Takes a map that contains [Configuration]s and extracts any + /// preset-specific configuration into a parent [Configuration]. + /// + /// This is needed because parameters to [new Configuration] such as + /// `onPlatform` take maps to [SuiteConfiguration]s. [SuiteConfiguration] + /// doesn't support preset-specific configuration, so this extracts the preset + /// logic into a parent [Configuration], leaving only maps to + /// [SuiteConfiguration]s. The [create] function is used to construct + /// [Configuration]s from the resolved maps. + Configuration _extractPresets/**/( + Map/**/ map, + Configuration create(Map/**/ map)) { + if (map.isEmpty) return Configuration.empty; + + var base = /**/{}; + var presets = /*>*/{}; + map.forEach((key, config) { + base[key] = config.suiteDefaults; + config.presets.forEach((preset, presetConfig) { + presets.putIfAbsent(preset, () => {})[key] = presetConfig.suiteDefaults; + }); + }); + + if (presets.isEmpty) { + return base.isEmpty ? Configuration.empty : create(base); + } else { + var newPresets = + mapMap/*, String, Configuration>*/( + presets, + value: (_, map) => create(map)); + return create(base).change(presets: newPresets); + } + } + /// Throws an error if a field named [field] exists at this level. void _disallow(String field) { if (!_document.containsKey(field)) return; diff --git a/lib/src/runner/configuration/suite.dart b/lib/src/runner/configuration/suite.dart new file mode 100644 index 000000000..3afaf46e8 --- /dev/null +++ b/lib/src/runner/configuration/suite.dart @@ -0,0 +1,318 @@ +// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:boolean_selector/boolean_selector.dart'; +import 'package:collection/collection.dart'; + +import '../../backend/metadata.dart'; +import '../../backend/operating_system.dart'; +import '../../backend/platform_selector.dart'; +import '../../backend/test_platform.dart'; +import '../../frontend/timeout.dart'; + +/// Suite-level configuration. +/// +/// This tracks configuration that can differ from suite to suite. +class SuiteConfiguration { + /// Empty configuration with only default values. + /// + /// Using this is slightly more efficient than manually constructing a new + /// configuration with no arguments. + static final empty = new SuiteConfiguration._(); + + /// Whether JavaScript stack traces should be left as-is or converted to + /// Dart-like traces. + bool get jsTrace => _jsTrace ?? false; + final bool _jsTrace; + + /// Whether skipped tests should be run. + bool get runSkipped => _runSkipped ?? false; + final bool _runSkipped; + + /// The path to a mirror of this package containing HTML that points to + /// precompiled JS. + /// + /// This is used by the internal Google test runner so that test compilation + /// can more effectively make use of Google's build tools. + final String precompiledPath; + + /// Additional arguments to pass to dart2js. + /// + /// Note that this if multiple suites run the same JavaScript on different + /// platforms, and they have different [dart2jsArgs], only one (undefined) + /// suite's arguments will be used. + final List dart2jsArgs; + + /// The patterns to match against test names to decide which to run, or `null` + /// if all tests should be run. + /// + /// All patterns must match in order for a test to be run. + final Set patterns; + + /// The set of platforms on which to run tests. + List get platforms => _platforms ?? const [TestPlatform.vm]; + final List _platforms; + + /// Only run tests whose tags match this selector. + /// + /// When [merge]d, this is intersected with the other configuration's included + /// tags. + final BooleanSelector includeTags; + + /// Do not run tests whose tags match this selector. + /// + /// When [merge]d, this is unioned with the other configuration's + /// excluded tags. + final BooleanSelector excludeTags; + + /// Configuration for particular tags. + /// + /// The keys are tag selectors, and the values are configurations for tests + /// whose tags match those selectors. + final Map tags; + + /// Configuration for particular platforms. + /// + /// The keys are platform selectors, and the values are configurations for + /// those platforms. These configuration should only contain test-level + /// configuration fields, but that isn't enforced. + final Map onPlatform; + + /// The global test metadata derived from this configuration. + Metadata get metadata { + if (tags.isEmpty && onPlatform.isEmpty) return _metadata; + return _metadata.change( + forTag: mapMap(tags, value: (_, config) => config.metadata), + onPlatform: mapMap(onPlatform, value: (_, config) => config.metadata)); + } + final Metadata _metadata; + + /// The set of tags that have been declared in any way in this configuration. + Set get knownTags { + if (_knownTags != null) return _knownTags; + + var known = includeTags.variables.toSet() + ..addAll(excludeTags.variables) + ..addAll(_metadata.tags); + + for (var selector in tags.keys) { + known.addAll(selector.variables); + } + + for (var configuration in _children) { + known.addAll(configuration.knownTags); + } + + _knownTags = new UnmodifiableSetView(known); + return _knownTags; + } + Set _knownTags; + + /// All child configurations of [this] that may be selected under various + /// circumstances. + Iterable get _children sync* { + yield* tags.values; + yield* onPlatform.values; + } + + factory SuiteConfiguration({ + bool jsTrace, + bool runSkipped, + Iterable dart2jsArgs, + String precompiledPath, + Iterable patterns, + Iterable platforms, + BooleanSelector includeTags, + BooleanSelector excludeTags, + Map tags, + Map onPlatform, + + // Test-level configuration + Timeout timeout, + bool verboseTrace, + bool skip, + String skipReason, + PlatformSelector testOn, + Iterable addTags}) { + var config = new SuiteConfiguration._( + jsTrace: jsTrace, + runSkipped: runSkipped, + dart2jsArgs: dart2jsArgs, + precompiledPath: precompiledPath, + patterns: patterns, + platforms: platforms, + includeTags: includeTags, + excludeTags: excludeTags, + tags: tags, + onPlatform: onPlatform, + metadata: new Metadata( + timeout: timeout, + verboseTrace: verboseTrace, + skip: skip, + skipReason: skipReason, + testOn: testOn, + tags: addTags)); + return config._resolveTags(); + } + + /// Creates new SuiteConfiguration. + /// + /// Unlike [new SuiteConfiguration], this assumes [tags] is already + /// resolved. + SuiteConfiguration._({ + bool jsTrace, + bool runSkipped, + Iterable dart2jsArgs, + this.precompiledPath, + Iterable patterns, + Iterable platforms, + BooleanSelector includeTags, + BooleanSelector excludeTags, + Map tags, + Map onPlatform, + Metadata metadata}) + : _jsTrace = jsTrace, + _runSkipped = runSkipped, + dart2jsArgs = _list(dart2jsArgs) ?? const [], + patterns = new UnmodifiableSetView(patterns?.toSet() ?? new Set()), + _platforms = _list(platforms), + includeTags = includeTags ?? BooleanSelector.all, + excludeTags = excludeTags ?? BooleanSelector.none, + tags = _map(tags), + onPlatform = _map(onPlatform), + _metadata = metadata ?? Metadata.empty; + + /// Creates a new [SuiteConfiguration] that takes its configuration from + /// [metadata]. + factory SuiteConfiguration.fromMetadata(Metadata metadata) => + new SuiteConfiguration._( + tags: mapMap(metadata.forTag, + value: (_, child) => new SuiteConfiguration.fromMetadata(child)), + onPlatform: mapMap(metadata.onPlatform, + value: (_, child) => new SuiteConfiguration.fromMetadata(child)), + metadata: metadata.change(forTag: {}, onPlatform: {})); + + /// Returns an unmodifiable copy of [input]. + /// + /// If [input] is `null` or empty, this returns `null`. + static List/**/ _list/**/(Iterable/**/ input) { + if (input == null) return null; + var list = new List/**/.unmodifiable(input); + if (list.isEmpty) return null; + return list; + } + + /// Returns an unmodifiable copy of [input] or an empty unmodifiable map. + static Map/**/ _map/**/(Map/**/ input) { + if (input == null || input.isEmpty) return const {}; + return new Map.unmodifiable(input); + } + + /// Merges this with [other]. + /// + /// For most fields, if both configurations have values set, [other]'s value + /// takes precedence. However, certain fields are merged together instead. + /// This is indicated in those fields' documentation. + SuiteConfiguration merge(SuiteConfiguration other) { + if (this == SuiteConfiguration.empty) return other; + if (other == SuiteConfiguration.empty) return this; + + var config = new SuiteConfiguration._( + jsTrace: other._jsTrace ?? _jsTrace, + runSkipped: other._runSkipped ?? _runSkipped, + dart2jsArgs: dart2jsArgs.toList()..addAll(other.dart2jsArgs), + precompiledPath: other.precompiledPath ?? precompiledPath, + patterns: patterns.union(other.patterns), + platforms: other._platforms ?? _platforms, + includeTags: includeTags.intersection(other.includeTags), + excludeTags: excludeTags.union(other.excludeTags), + tags: _mergeConfigMaps(tags, other.tags), + onPlatform: _mergeConfigMaps(onPlatform, other.onPlatform), + metadata: metadata.merge(other.metadata)); + return config._resolveTags(); + } + + /// Returns a copy of this configuration with the given fields updated. + /// + /// Note that unlike [merge], this has no merging behavior—the old value is + /// always replaced by the new one. + SuiteConfiguration change({ + bool jsTrace, + bool runSkipped, + Iterable dart2jsArgs, + String precompiledPath, + Iterable patterns, + Iterable platforms, + BooleanSelector includeTags, + BooleanSelector excludeTags, + Map tags, + Map onPlatform, + + // Test-level configuration + Timeout timeout, + bool verboseTrace, + bool skip, + String skipReason, + PlatformSelector testOn, + Iterable addTags}) { + var config = new SuiteConfiguration._( + jsTrace: jsTrace ?? _jsTrace, + runSkipped: runSkipped ?? _runSkipped, + dart2jsArgs: dart2jsArgs?.toList() ?? this.dart2jsArgs, + precompiledPath: precompiledPath ?? this.precompiledPath, + patterns: patterns ?? this.patterns, + platforms: platforms ?? _platforms, + includeTags: includeTags ?? this.includeTags, + excludeTags: excludeTags ?? this.excludeTags, + tags: tags ?? this.tags, + onPlatform: onPlatform ?? this.onPlatform, + metadata: _metadata.change( + timeout: timeout, + verboseTrace: verboseTrace, + skip: skip, + skipReason: skipReason, + testOn: testOn, + tags: addTags)); + return config._resolveTags(); + } + + /// Returns a copy of [this] with all platform-specific configuration from + /// [onPlatform] resolved. + SuiteConfiguration forPlatform(TestPlatform platform, {OperatingSystem os}) { + if (onPlatform.isEmpty) return this; + + var config = this; + onPlatform.forEach((platformSelector, platformConfig) { + if (!platformSelector.evaluate(platform, os: os)) return; + config = config.merge(platformConfig); + }); + return config.change(onPlatform: {}); + } + + /// Merges two maps whose values are [SuiteConfiguration]s. + /// + /// Any overlapping keys in the maps have their configurations merged in the + /// returned map. + Map _mergeConfigMaps( + Map map1, + Map map2) => + mergeMaps(map1, map2, + value: (config1, config2) => config1.merge(config2)); + + SuiteConfiguration _resolveTags() { + // If there's no tag-specific configuration, or if none of it applies, just + // return the configuration as-is. + if (_metadata.tags.isEmpty || tags.isEmpty) return this; + + // Otherwise, resolve the tag-specific components. + var newTags = new Map.from(tags); + var merged = tags.keys.fold(empty, (merged, selector) { + if (!selector.evaluate(_metadata.tags)) return merged; + return merged.merge(newTags.remove(selector)); + }); + + if (merged == empty) return this; + return this.change(tags: newTags).merge(merged); + } +} diff --git a/lib/src/runner/engine.dart b/lib/src/runner/engine.dart index 69bdde80c..489a91224 100644 --- a/lib/src/runner/engine.dart +++ b/lib/src/runner/engine.dart @@ -51,9 +51,6 @@ import 'runner_suite.dart'; /// Load tests will always be emitted through [onTestStarted] so users can watch /// their event streams once they start running. class Engine { - /// Whether to run skipped tests. - final bool _runSkipped; - /// Whether [run] has been called yet. var _runCalled = false; @@ -197,12 +194,10 @@ class Engine { /// /// [concurrency] controls how many suites are run at once, and defaults to 1. /// [maxSuites] controls how many suites are *loaded* at once, and defaults to - /// four times [concurrency]. If [runSkipped] is `true`, skipped tests will be - /// run as though they weren't skipped. - Engine({int concurrency, int maxSuites, bool runSkipped: false}) + /// four times [concurrency]. + Engine({int concurrency, int maxSuites}) : _runPool = new Pool(concurrency ?? 1), - _loadPool = new Pool(maxSuites ?? (concurrency ?? 1) * 2), - _runSkipped = runSkipped { + _loadPool = new Pool(maxSuites ?? (concurrency ?? 1) * 2) { _group.future.then((_) { _onTestStartedGroup.close(); _onSuiteStartedController.close(); @@ -219,9 +214,8 @@ class Engine { /// /// [concurrency] controls how many suites are run at once. If [runSkipped] is /// `true`, skipped tests will be run as though they weren't skipped. - factory Engine.withSuites(List suites, {int concurrency, - bool runSkipped: false}) { - var engine = new Engine(concurrency: concurrency, runSkipped: runSkipped); + factory Engine.withSuites(List suites, {int concurrency}) { + var engine = new Engine(concurrency: concurrency); for (var suite in suites) engine.suiteSink.add(suite); engine.suiteSink.close(); return engine; @@ -283,7 +277,8 @@ class Engine { List parents) async { parents.add(group); try { - var skipGroup = !_runSkipped && group.metadata.skip; + var suiteConfig = suiteController.liveSuite.suite.config; + var skipGroup = !suiteConfig.runSkipped && group.metadata.skip; var setUpAllSucceeded = true; if (!skipGroup && group.setUpAll != null) { var liveTest = group.setUpAll.load(suiteController.liveSuite.suite, @@ -298,7 +293,7 @@ class Engine { if (entry is Group) { await _runGroup(suiteController, entry, parents); - } else if (!_runSkipped && entry.metadata.skip) { + } else if (!suiteConfig.runSkipped && entry.metadata.skip) { await _runSkippedTest(suiteController, entry, parents); } else { var test = entry as Test; diff --git a/lib/src/runner/load_suite.dart b/lib/src/runner/load_suite.dart index 242c6324d..7034de442 100644 --- a/lib/src/runner/load_suite.dart +++ b/lib/src/runner/load_suite.dart @@ -14,6 +14,7 @@ import '../backend/suite.dart'; import '../backend/test.dart'; import '../backend/test_platform.dart'; import '../utils.dart'; +import 'configuration/suite.dart'; import 'load_exception.dart'; import 'plugin/environment.dart'; import 'runner_suite.dart'; @@ -34,6 +35,7 @@ import 'runner_suite.dart'; /// prints will be emitted through the running [LiveTest]. class LoadSuite extends Suite implements RunnerSuite { final environment = const PluginEnvironment(); + final SuiteConfiguration config; final isDebugging = false; final onDebugging = new StreamController().stream; @@ -64,9 +66,10 @@ class LoadSuite extends Suite implements RunnerSuite { /// /// If the the load test is closed before [body] is complete, it will close /// the suite returned by [body] once it completes. - factory LoadSuite(String name, body(), {String path, TestPlatform platform}) { + factory LoadSuite(String name, SuiteConfiguration config, body(), + {String path, TestPlatform platform}) { var completer = new Completer>.sync(); - return new LoadSuite._(name, () { + return new LoadSuite._(name, config, () { var invoker = Invoker.current; invoker.addOutstandingCallback(); @@ -103,21 +106,25 @@ class LoadSuite extends Suite implements RunnerSuite { /// /// The suite's name will be based on [exception]'s path. factory LoadSuite.forLoadException(LoadException exception, - {StackTrace stackTrace, TestPlatform platform}) { + SuiteConfiguration config, {StackTrace stackTrace, + TestPlatform platform}) { if (stackTrace == null) stackTrace = new Trace.current(); - return new LoadSuite("loading ${exception.path}", () { - return new Future.error(exception, stackTrace); - }, path: exception.path, platform: platform); + return new LoadSuite( + "loading ${exception.path}", + config ?? SuiteConfiguration.empty, + () => new Future.error(exception, stackTrace), + path: exception.path, + platform: platform); } /// A utility constructor for a load suite that just emits [suite]. factory LoadSuite.forSuite(RunnerSuite suite) { - return new LoadSuite("loading ${suite.path}", () => suite, + return new LoadSuite("loading ${suite.path}", suite.config, () => suite, path: suite.path, platform: suite.platform); } - LoadSuite._(String name, void body(), this._suiteAndZone, + LoadSuite._(String name, this.config, void body(), this._suiteAndZone, {String path, TestPlatform platform}) : super(new Group.root([ new LocalTest(name, @@ -127,11 +134,13 @@ class LoadSuite extends Suite implements RunnerSuite { /// A constructor used by [changeSuite]. LoadSuite._changeSuite(LoadSuite old, this._suiteAndZone) - : super(old.group, path: old.path, platform: old.platform); + : config = old.config, + super(old.group, path: old.path, platform: old.platform); /// A constructor used by [filter]. LoadSuite._filtered(LoadSuite old, Group filtered) - : _suiteAndZone = old._suiteAndZone, + : config = old.config, + _suiteAndZone = old._suiteAndZone, super(old.group, path: old.path, platform: old.platform); /// Creates a new [LoadSuite] that's identical to this one, but that diff --git a/lib/src/runner/loader.dart b/lib/src/runner/loader.dart index 981053abc..f24ddd0dd 100644 --- a/lib/src/runner/loader.dart +++ b/lib/src/runner/loader.dart @@ -11,12 +11,12 @@ import 'package:path/path.dart' as p; import '../backend/group.dart'; import '../backend/invoker.dart'; -import '../backend/metadata.dart'; import '../backend/test_platform.dart'; import '../util/io.dart'; import '../utils.dart'; import 'browser/platform.dart'; import 'configuration.dart'; +import 'configuration/suite.dart'; import 'load_exception.dart'; import 'load_suite.dart'; import 'parse_metadata.dart'; @@ -81,20 +81,15 @@ class Loader { } } - /// Loads all test suites in [dir]. + /// Loads all test suites in [dir] according to [suiteConfig]. /// - /// This will load tests from files that match the configuration's filename - /// glob. Any tests that fail to load will be emitted as [LoadException]s. + /// This will load tests from files that match the global configuration's + /// filename glob. Any tests that fail to load will be emitted as + /// [LoadException]s. /// /// This emits [LoadSuite]s that must then be run to emit the actual /// [RunnerSuite]s defined in the file. - /// - /// If [platforms] is passed, these suites will only be loaded on those - /// platforms. It must be a subset of the current configuration's platforms. - /// Note that the suites aren't guaranteed to be loaded on all platforms in - /// [platforms]: their `@TestOn` declarations are still respected. - Stream loadDir(String dir, {Iterable platforms}) { - platforms = _validatePlatformSubset(platforms); + Stream loadDir(String dir, SuiteConfiguration suiteConfig) { return StreamGroup.merge(new Directory(dir).listSync(recursive: true) .map((entry) { if (entry is! File) return new Stream.fromIterable([]); @@ -107,69 +102,64 @@ class Loader { return new Stream.fromIterable([]); } - return loadFile(entry.path); + return loadFile(entry.path, suiteConfig); })); } - /// Loads a test suite from the file at [path]. + /// Loads a test suite from the file at [path] according to [suiteConfig]. /// /// This emits [LoadSuite]s that must then be run to emit the actual /// [RunnerSuite]s defined in the file. /// - /// If [platforms] is passed, these suites will only be loaded on those - /// platforms. It must be a subset of the current configuration's platforms. - /// Note that the suites aren't guaranteed to be loaded on all platforms in - /// [platforms]: their `@TestOn` declarations are still respected. - /// /// This will emit a [LoadException] if the file fails to load. - Stream loadFile(String path, {Iterable platforms}) => - // Ensure that the right config is current when invoking platform plugins. - _config.asCurrent(() async* { - platforms = _validatePlatformSubset(platforms); - - var suiteMetadata; + Stream loadFile(String path, SuiteConfiguration suiteConfig) + async* { try { - suiteMetadata = parseMetadata(path); + suiteConfig = suiteConfig.merge( + new SuiteConfiguration.fromMetadata(parseMetadata(path))); } on AnalyzerErrorGroup catch (_) { // Ignore the analyzer's error, since its formatting is much worse than // the VM's or dart2js's. - suiteMetadata = new Metadata(); } on FormatException catch (error, stackTrace) { yield new LoadSuite.forLoadException( - new LoadException(path, error), stackTrace: stackTrace); + new LoadException(path, error), suiteConfig, stackTrace: stackTrace); return; } - suiteMetadata = _config.metadata.merge(suiteMetadata); if (_config.pubServeUrl != null && !p.isWithin('test', path)) { - yield new LoadSuite.forLoadException(new LoadException( - path, 'When using "pub serve", all test files must be in test/.')); + yield new LoadSuite.forLoadException( + new LoadException( + path, 'When using "pub serve", all test files must be in test/.'), + suiteConfig); return; } - for (var platform in platforms ?? _config.platforms) { - if (!suiteMetadata.testOn.evaluate(platform, os: currentOS)) continue; + for (var platform in suiteConfig.platforms) { + if (!suiteConfig.metadata.testOn.evaluate(platform, os: currentOS)) { + continue; + } - var metadata = suiteMetadata.forPlatform(platform, os: currentOS); + var platformConfig = suiteConfig.forPlatform(platform, os: currentOS); // Don't load a skipped suite. - if (metadata.skip && !_config.runSkipped) { + if (platformConfig.metadata.skip && !platformConfig.runSkipped) { yield new LoadSuite.forSuite(new RunnerSuite( const PluginEnvironment(), + platformConfig, new Group.root( - [new LocalTest("(suite)", metadata, () {})], - metadata: metadata), + [new LocalTest("(suite)", platformConfig.metadata, () {})], + metadata: platformConfig.metadata), path: path, platform: platform)); continue; } var name = (platform.isJS ? "compiling " : "loading ") + path; - yield new LoadSuite(name, () async { + yield new LoadSuite(name, platformConfig, () async { var memo = _platformPlugins[platform]; try { var plugin = await memo.runOnce(_platformCallbacks[platform]); - var suite = await plugin.load(path, platform, metadata); + var suite = await plugin.load(path, platform, platformConfig); _suites.add(suite); return suite; } catch (error, stackTrace) { @@ -178,18 +168,6 @@ class Loader { } }, path: path, platform: platform); } - }); - - /// Asserts that [platforms] is a subset of [_config.platforms], and returns - /// it as a set. - /// - /// Returns `null` if [platforms] is `null`. - Set _validatePlatformSubset(Iterable platforms) { - if (platforms == null) return null; - platforms = platforms.toSet(); - if (platforms.every(_config.platforms.contains)) return platforms; - throw new ArgumentError.value(platforms, 'platforms', - "must be a subset of ${_config.platforms}."); } Future closeEphemeral() async { diff --git a/lib/src/runner/parse_metadata.dart b/lib/src/runner/parse_metadata.dart index b8dd1e3b5..7617c5063 100644 --- a/lib/src/runner/parse_metadata.dart +++ b/lib/src/runner/parse_metadata.dart @@ -86,7 +86,7 @@ class _Parser { return new Metadata( testOn: testOn, timeout: timeout, - skip: skip != null, + skip: skip == null ? null : true, skipReason: skip is String ? skip : null, onPlatform: onPlatform, tags: tags); diff --git a/lib/src/runner/plugin/platform.dart b/lib/src/runner/plugin/platform.dart index 820f4c037..afc87e501 100644 --- a/lib/src/runner/plugin/platform.dart +++ b/lib/src/runner/plugin/platform.dart @@ -8,6 +8,7 @@ import 'package:stream_channel/stream_channel.dart'; import '../../backend/metadata.dart'; import '../../backend/test_platform.dart'; +import '../configuration/suite.dart'; import '../environment.dart'; import '../runner_suite.dart'; import 'environment.dart'; @@ -45,7 +46,7 @@ abstract class PlatformPlugin { StreamChannel loadChannel(String path, TestPlatform platform); /// Loads the runner suite for the test file at [path] using [platform], with - /// [metadata] parsed from the test file's top-level annotations. + /// [suiteConfig] encoding the suite-specific configuration. /// /// By default, this just calls [loadChannel] and passes its result to /// [deserializeSuite]. However, it can be overridden to provide more @@ -56,12 +57,12 @@ abstract class PlatformPlugin { /// [deserializeSuite] in `platform_helpers.dart` to obtain a /// [RunnerSuiteController]. Future load(String path, TestPlatform platform, - Metadata metadata) async { + SuiteConfiguration suiteConfig) async { // loadChannel may throw an exception. That's fine; it will cause the // LoadSuite to emit an error, which will be presented to the user. var channel = loadChannel(path, platform); var controller = await deserializeSuite( - path, platform, metadata, new PluginEnvironment(), channel); + path, platform, suiteConfig, new PluginEnvironment(), channel); return controller.suite; } diff --git a/lib/src/runner/plugin/platform_helpers.dart b/lib/src/runner/plugin/platform_helpers.dart index b0c828df7..b961f5cc7 100644 --- a/lib/src/runner/plugin/platform_helpers.dart +++ b/lib/src/runner/plugin/platform_helpers.dart @@ -14,6 +14,7 @@ import '../../backend/test_platform.dart'; import '../../util/io.dart'; import '../../util/remote_exception.dart'; import '../configuration.dart'; +import '../configuration/suite.dart'; import '../environment.dart'; import '../load_exception.dart'; import '../runner_suite.dart'; @@ -32,8 +33,9 @@ typedef StackTrace _MapTrace(StackTrace trace); /// If [mapTrace] is passed, it will be used to adjust stack traces for any /// errors emitted by tests. Future deserializeSuite(String path, - TestPlatform platform, Metadata metadata, Environment environment, - StreamChannel channel, {StackTrace mapTrace(StackTrace trace)}) async { + TestPlatform platform, SuiteConfiguration suiteConfig, + Environment environment, StreamChannel channel, + {StackTrace mapTrace(StackTrace trace)}) async { if (mapTrace == null) mapTrace = (trace) => trace; var disconnector = new Disconnector(); @@ -41,7 +43,7 @@ Future deserializeSuite(String path, suiteChannel.sink.add({ 'platform': platform.identifier, - 'metadata': metadata.serialize(), + 'metadata': suiteConfig.metadata.serialize(), 'os': platform == TestPlatform.vm ? currentOS.identifier : null, 'collectTraces': Configuration.current.reporter == 'json' }); @@ -95,6 +97,7 @@ Future deserializeSuite(String path, return new RunnerSuiteController( environment, + suiteConfig, await completer.future, path: path, platform: platform, diff --git a/lib/src/runner/reporter/compact.dart b/lib/src/runner/reporter/compact.dart index 6de11d942..ed0157cef 100644 --- a/lib/src/runner/reporter/compact.dart +++ b/lib/src/runner/reporter/compact.dart @@ -204,7 +204,8 @@ class CompactReporter implements Reporter { if (error is! LoadException) { print(indent(error.toString())); - var chain = terseChain(stackTrace, verbose: _config.verboseTrace); + var chain = terseChain(stackTrace, + verbose: liveTest.test.metadata.verboseTrace); print(indent(chain.toString())); return; } @@ -350,7 +351,8 @@ class CompactReporter implements Reporter { name = "${liveTest.suite.path}: $name"; } - if (_config.platforms.length > 1 && liveTest.suite.platform != null) { + if (_config.suiteDefaults.platforms.length > 1 && + liveTest.suite.platform != null) { name = "[${liveTest.suite.platform.name}] $name"; } diff --git a/lib/src/runner/reporter/expanded.dart b/lib/src/runner/reporter/expanded.dart index 5fba7cf9b..961fe4249 100644 --- a/lib/src/runner/reporter/expanded.dart +++ b/lib/src/runner/reporter/expanded.dart @@ -53,9 +53,6 @@ class ExpandedReporter implements Reporter { /// this is Windows or not outputting to a terminal. final String _noColor; - /// Whether to use verbose stack traces. - final bool _verboseTrace; - /// The engine used to run the tests. final Engine _engine; @@ -101,25 +98,21 @@ class ExpandedReporter implements Reporter { /// terminal. /// /// If [color] is `true`, this will use terminal colors; if it's `false`, it - /// won't. If [verboseTrace] is `true`, this will print core library frames. - /// If [printPath] is `true`, this will print the path name as part of the - /// test description. Likewise, if [printPlatform] is `true`, this will print - /// the platform as part of the test description. + /// won't. If [printPath] is `true`, this will print the path name as part of + /// the test description. Likewise, if [printPlatform] is `true`, this will + /// print the platform as part of the test description. static ExpandedReporter watch(Engine engine, {bool color: true, - bool verboseTrace: false, bool printPath: true, - bool printPlatform: true}) { + bool printPath: true, bool printPlatform: true}) { return new ExpandedReporter._( engine, color: color, - verboseTrace: verboseTrace, printPath: printPath, printPlatform: printPlatform); } - ExpandedReporter._(this._engine, {bool color: true, bool verboseTrace: false, - bool printPath: true, bool printPlatform: true}) - : _verboseTrace = verboseTrace, - _printPath = printPath, + ExpandedReporter._(this._engine, {bool color: true, bool printPath: true, + bool printPlatform: true}) + : _printPath = printPath, _printPlatform = printPlatform, _color = color, _green = color ? '\u001b[32m' : '', @@ -214,7 +207,8 @@ class ExpandedReporter implements Reporter { if (error is! LoadException) { print(indent(error.toString())); - var chain = terseChain(stackTrace, verbose: _verboseTrace); + var chain = terseChain(stackTrace, + verbose: liveTest.test.metadata.verboseTrace); print(indent(chain.toString())); return; } diff --git a/lib/src/runner/reporter/json.dart b/lib/src/runner/reporter/json.dart index 94acb0cc2..52c002d32 100644 --- a/lib/src/runner/reporter/json.dart +++ b/lib/src/runner/reporter/json.dart @@ -15,16 +15,15 @@ import '../../backend/test_platform.dart'; import '../../frontend/expect.dart'; import '../../utils.dart'; import '../configuration.dart'; +import '../configuration/suite.dart'; import '../engine.dart'; import '../load_suite.dart'; import '../reporter.dart'; +import '../runner_suite.dart'; import '../version.dart'; /// A reporter that prints machine-readable JSON-formatted test results. class JsonReporter implements Reporter { - /// The test runner configuration. - final _config = Configuration.current; - /// The engine used to run the tests. final Engine _engine; @@ -121,15 +120,16 @@ class JsonReporter implements Reporter { ? [] : _idsForGroups(liveTest.groups, liveTest.suite); + var suiteConfig = _configFor(liveTest.suite); var id = _nextID++; _liveTestIDs[liveTest] = id; _emit("testStart", { - "test": _addFrameInfo({ + "test": _addFrameInfo(suiteConfig, { "id": id, "name": liveTest.test.name, "suiteID": suiteID, "groupIDs": groupIDs, - "metadata": _serializeMetadata(liveTest.test.metadata) + "metadata": _serializeMetadata(suiteConfig, liveTest.test.metadata) }, liveTest.test, liveTest.suite.platform) }); @@ -194,13 +194,14 @@ class JsonReporter implements Reporter { var id = _nextID++; _groupIDs[group] = id; + var suiteConfig = _configFor(suite); _emit("group", { - "group": _addFrameInfo({ + "group": _addFrameInfo(suiteConfig, { "id": id, "suiteID": _idForSuite(suite), "parentID": parentID, "name": group.name, - "metadata": _serializeMetadata(group.metadata), + "metadata": _serializeMetadata(suiteConfig, group.metadata), "testCount": group.testCount }, group, suite.platform) }); @@ -210,9 +211,10 @@ class JsonReporter implements Reporter { } /// Serializes [metadata] into a JSON-protocol-compatible map. - Map _serializeMetadata(Metadata metadata) => _config.runSkipped - ? {"skip": false, "skipReason": null} - : {"skip": metadata.skip, "skipReason": metadata.skipReason}; + Map _serializeMetadata(SuiteConfiguration suiteConfig, Metadata metadata) => + suiteConfig.runSkipped + ? {"skip": false, "skipReason": null} + : {"skip": metadata.skip, "skipReason": metadata.skipReason}; /// A callback called when [liveTest] finishes running. void _onComplete(LiveTest liveTest) { @@ -232,7 +234,8 @@ class JsonReporter implements Reporter { _emit("error", { "testID": _liveTestIDs[liveTest], "error": error.toString(), - "stackTrace": terseChain(stackTrace, verbose: _config.verboseTrace) + "stackTrace": + terseChain(stackTrace, verbose: liveTest.test.metadata.verboseTrace) .toString(), "isFailure": error is TestFailure }); @@ -249,6 +252,13 @@ class JsonReporter implements Reporter { _emit("done", {"success": success}); } + /// Returns the configuration for [suite]. + /// + /// If [suite] is a [RunnerSuite], this returns [RunnerSuite.config]. + /// Otherwise, it returns [SuiteConfiguration.empty]. + SuiteConfiguration _configFor(Suite suite) => + suite is RunnerSuite ? suite.config : SuiteConfiguration.empty; + /// Emits an event with the given type and attributes. void _emit(String type, Map attributes) { attributes["type"] = type; @@ -260,10 +270,10 @@ class JsonReporter implements Reporter { /// frame of [entry.trace]. /// /// Returns [map]. - Map _addFrameInfo(Map map, - GroupEntry entry, TestPlatform platform) { + Map _addFrameInfo(SuiteConfiguration suiteConfig, + Map map, GroupEntry entry, TestPlatform platform) { var frame = entry.trace?.frames?.first; - if (_config.jsTrace && platform.isJS) frame = null; + if (suiteConfig.jsTrace && platform.isJS) frame = null; map["line"] = frame?.line; map["column"] = frame?.column; diff --git a/lib/src/runner/runner_suite.dart b/lib/src/runner/runner_suite.dart index 1ab1de838..0f5612b92 100644 --- a/lib/src/runner/runner_suite.dart +++ b/lib/src/runner/runner_suite.dart @@ -12,6 +12,7 @@ import '../backend/suite.dart'; import '../backend/test.dart'; import '../backend/test_platform.dart'; import '../utils.dart'; +import 'configuration/suite.dart'; import 'environment.dart'; /// A suite produced and consumed by the test runner that has runner-specific @@ -29,6 +30,9 @@ class RunnerSuite extends Suite { /// The environment in which this suite runs. Environment get environment => _controller._environment; + /// The configuration for this suite. + SuiteConfiguration get config => _controller._config; + /// Whether the suite is paused for debugging. /// /// When using a dev inspector, this may also mean that the entire browser is @@ -43,9 +47,10 @@ class RunnerSuite extends Suite { /// A shortcut constructor for creating a [RunnerSuite] that never goes into /// debugging mode. - factory RunnerSuite(Environment environment, Group group, {String path, - TestPlatform platform, OperatingSystem os, AsyncFunction onClose}) { - var controller = new RunnerSuiteController(environment, group, + factory RunnerSuite(Environment environment, SuiteConfiguration config, + Group group, {String path, TestPlatform platform, OperatingSystem os, + AsyncFunction onClose}) { + var controller = new RunnerSuiteController(environment, config, group, path: path, platform: platform, os: os, onClose: onClose); return controller.suite; } @@ -73,6 +78,9 @@ class RunnerSuiteController { /// The backing value for [suite.environment]. final Environment _environment; + /// The configuration for this suite. + final SuiteConfiguration _config; + /// The function to call when the suite is closed. final AsyncFunction _onClose; @@ -82,8 +90,9 @@ class RunnerSuiteController { /// The controller for [suite.onDebugging]. final _onDebuggingController = new StreamController.broadcast(); - RunnerSuiteController(this._environment, Group group, {String path, - TestPlatform platform, OperatingSystem os, AsyncFunction onClose}) + RunnerSuiteController(this._environment, this._config, Group group, + {String path, TestPlatform platform, + OperatingSystem os, AsyncFunction onClose}) : _onClose = onClose { _suite = new RunnerSuite._(this, group, path, platform, os); } diff --git a/lib/test.dart b/lib/test.dart index 564f9363f..bb6ebd016 100644 --- a/lib/test.dart +++ b/lib/test.dart @@ -9,6 +9,7 @@ import 'package:path/path.dart' as p; import 'src/backend/declarer.dart'; import 'src/backend/test_platform.dart'; import 'src/frontend/timeout.dart'; +import 'src/runner/configuration/suite.dart'; import 'src/runner/engine.dart'; import 'src/runner/plugin/environment.dart'; import 'src/runner/reporter/expanded.dart'; @@ -52,6 +53,7 @@ Declarer get _declarer { scheduleMicrotask(() async { var suite = new RunnerSuite( const PluginEnvironment(), + SuiteConfiguration.empty, _globalDeclarer.build(), path: p.prettyUri(Uri.base), platform: TestPlatform.vm, diff --git a/pubspec.yaml b/pubspec.yaml index e2ae2ace9..5d934653d 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,5 +1,5 @@ name: test -version: 0.12.15+12 +version: 0.12.16-dev author: Dart Team description: A library for writing dart unit tests. homepage: https://github.com/dart-lang/test diff --git a/test/runner/browser/loader_test.dart b/test/runner/browser/loader_test.dart index 512732016..54bc75d27 100644 --- a/test/runner/browser/loader_test.dart +++ b/test/runner/browser/loader_test.dart @@ -11,6 +11,7 @@ import 'package:path/path.dart' as p; import 'package:test/src/backend/state.dart'; import 'package:test/src/backend/test_platform.dart'; import 'package:test/src/runner/configuration.dart'; +import 'package:test/src/runner/configuration/suite.dart'; import 'package:test/src/runner/loader.dart'; import 'package:test/src/util/io.dart'; import 'package:test/test.dart'; @@ -33,11 +34,13 @@ void main() { } """; +/// A configuration that loads suites on Chrome. +final _chrome = new SuiteConfiguration(platforms: [TestPlatform.chrome]); + void main() { setUp(() async { _sandbox = createTempDir(); - _loader = new Configuration(platforms: [TestPlatform.chrome]) - .asCurrent(() => new Loader(root: _sandbox)); + _loader = new Loader(root: _sandbox); /// TODO(nweiz): Use scheduled_test for this once it's compatible with this /// version of test. new File(p.join(_sandbox, 'a_test.dart')).writeAsStringSync(_tests); @@ -51,7 +54,8 @@ void main() { group(".loadFile()", () { var suite; setUp(() async { - var suites = await _loader.loadFile(p.join(_sandbox, 'a_test.dart')) + var suites = await _loader + .loadFile(p.join(_sandbox, 'a_test.dart'), _chrome) .toList(); expect(suites, hasLength(1)); @@ -112,7 +116,8 @@ Future main() { } """); - var suites = await _loader.loadFile(p.join(_sandbox, 'a_test.dart')) + var suites = await _loader + .loadFile(p.join(_sandbox, 'a_test.dart'), _chrome) .toList(); expect(suites, hasLength(1)); var loadSuite = suites.first; @@ -124,27 +129,22 @@ Future main() { }); test("loads a suite both in the browser and the VM", () async { - var loader = new Configuration( - platforms: [TestPlatform.vm, TestPlatform.chrome]) - .asCurrent(() => new Loader(root: _sandbox)); var path = p.join(_sandbox, 'a_test.dart'); - try { - var suites = await loader.loadFile(path) - .asyncMap((loadSuite) => loadSuite.getSuite()).toList(); - expect(suites[0].platform, equals(TestPlatform.vm)); - expect(suites[0].path, equals(path)); - expect(suites[1].platform, equals(TestPlatform.chrome)); - expect(suites[1].path, equals(path)); - - for (var suite in suites) { - expect(suite.group.entries, hasLength(3)); - expect(suite.group.entries[0].name, equals("success")); - expect(suite.group.entries[1].name, equals("failure")); - expect(suite.group.entries[2].name, equals("error")); - } - } finally { - await loader.close(); + var suites = await _loader + .loadFile(path, new SuiteConfiguration( + platforms: [TestPlatform.vm, TestPlatform.chrome])) + .asyncMap((loadSuite) => loadSuite.getSuite()).toList(); + expect(suites[0].platform, equals(TestPlatform.vm)); + expect(suites[0].path, equals(path)); + expect(suites[1].platform, equals(TestPlatform.chrome)); + expect(suites[1].path, equals(path)); + + for (var suite in suites) { + expect(suite.group.entries, hasLength(3)); + expect(suite.group.entries[0].name, equals("success")); + expect(suite.group.entries[1].name, equals("failure")); + expect(suite.group.entries[2].name, equals("error")); } }); @@ -154,7 +154,8 @@ void main() { print('print within test'); } """); - var suites = await _loader.loadFile(p.join(_sandbox, 'a_test.dart')) + var suites = await _loader + .loadFile(p.join(_sandbox, 'a_test.dart'), _chrome) .toList(); expect(suites, hasLength(1)); var loadSuite = suites.first; diff --git a/test/runner/configuration/configuration_test.dart b/test/runner/configuration/configuration_test.dart index 92c824d86..f0ee740f5 100644 --- a/test/runner/configuration/configuration_test.dart +++ b/test/runner/configuration/configuration_test.dart @@ -21,20 +21,13 @@ void main() { var merged = new Configuration().merge(new Configuration()); expect(merged.help, isFalse); expect(merged.version, isFalse); - expect(merged.verboseTrace, isFalse); - expect(merged.jsTrace, isFalse); - expect(merged.skip, isFalse); - expect(merged.skipReason, isNull); - expect(merged.runSkipped, isFalse); expect(merged.pauseAfterLoad, isFalse); expect(merged.color, equals(canUseSpecialChars)); - expect(merged.shardIndex, isNull); - expect(merged.totalShards, isNull); expect(merged.dart2jsPath, equals(p.join(sdkDir, 'bin', 'dart2js'))); - expect(merged.precompiledPath, isNull); expect(merged.reporter, equals(defaultReporter)); expect(merged.pubServeUrl, isNull); - expect(merged.platforms, equals([TestPlatform.vm])); + expect(merged.shardIndex, isNull); + expect(merged.totalShards, isNull); expect(merged.paths, equals(["test"])); }); @@ -42,39 +35,25 @@ void main() { var merged = new Configuration( help: true, version: true, - verboseTrace: true, - jsTrace: true, - skip: true, - skipReason: "boop", - runSkipped: true, pauseAfterLoad: true, color: true, - shardIndex: 3, - totalShards: 10, dart2jsPath: "/tmp/dart2js", - precompiledPath: "/tmp/js", reporter: "json", pubServePort: 1234, - platforms: [TestPlatform.chrome], + shardIndex: 3, + totalShards: 10, paths: ["bar"]) .merge(new Configuration()); expect(merged.help, isTrue); expect(merged.version, isTrue); - expect(merged.verboseTrace, isTrue); - expect(merged.jsTrace, isTrue); - expect(merged.skip, isTrue); - expect(merged.skipReason, equals("boop")); - expect(merged.runSkipped, isTrue); expect(merged.pauseAfterLoad, isTrue); expect(merged.color, isTrue); - expect(merged.shardIndex, equals(3)); - expect(merged.totalShards, equals(10)); expect(merged.dart2jsPath, equals("/tmp/dart2js")); - expect(merged.precompiledPath, equals("/tmp/js")); expect(merged.reporter, equals("json")); expect(merged.pubServeUrl.port, equals(1234)); - expect(merged.platforms, equals([TestPlatform.chrome])); + expect(merged.shardIndex, equals(3)); + expect(merged.totalShards, equals(10)); expect(merged.paths, equals(["bar"])); }); @@ -82,38 +61,24 @@ void main() { var merged = new Configuration().merge(new Configuration( help: true, version: true, - verboseTrace: true, - jsTrace: true, - skip: true, - skipReason: "boop", - runSkipped: true, pauseAfterLoad: true, color: true, - shardIndex: 3, - totalShards: 10, dart2jsPath: "/tmp/dart2js", - precompiledPath: "/tmp/js", reporter: "json", pubServePort: 1234, - platforms: [TestPlatform.chrome], + shardIndex: 3, + totalShards: 10, paths: ["bar"])); expect(merged.help, isTrue); expect(merged.version, isTrue); - expect(merged.verboseTrace, isTrue); - expect(merged.jsTrace, isTrue); - expect(merged.skip, isTrue); - expect(merged.skipReason, equals("boop")); - expect(merged.runSkipped, isTrue); expect(merged.pauseAfterLoad, isTrue); expect(merged.color, isTrue); - expect(merged.shardIndex, equals(3)); - expect(merged.totalShards, equals(10)); expect(merged.dart2jsPath, equals("/tmp/dart2js")); - expect(merged.precompiledPath, equals("/tmp/js")); expect(merged.reporter, equals("json")); expect(merged.pubServeUrl.port, equals(1234)); - expect(merged.platforms, equals([TestPlatform.chrome])); + expect(merged.shardIndex, equals(3)); + expect(merged.totalShards, equals(10)); expect(merged.paths, equals(["bar"])); }); @@ -122,331 +87,112 @@ void main() { var older = new Configuration( help: true, version: false, - verboseTrace: true, - jsTrace: false, - skip: true, - skipReason: "foo", - runSkipped: true, pauseAfterLoad: true, color: false, - shardIndex: 2, - totalShards: 4, dart2jsPath: "/tmp/dart2js", - precompiledPath: "/tmp/js", reporter: "json", pubServePort: 1234, - platforms: [TestPlatform.chrome], + shardIndex: 2, + totalShards: 4, paths: ["bar"]); var newer = new Configuration( help: false, version: true, - verboseTrace: false, - jsTrace: true, - skip: true, - skipReason: "bar", - runSkipped: false, pauseAfterLoad: false, color: true, - shardIndex: 3, - totalShards: 10, dart2jsPath: "../dart2js", - precompiledPath: "../js", reporter: "compact", pubServePort: 5678, - platforms: [TestPlatform.dartium], + shardIndex: 3, + totalShards: 10, paths: ["blech"]); var merged = older.merge(newer); expect(merged.help, isFalse); expect(merged.version, isTrue); - expect(merged.verboseTrace, isFalse); - expect(merged.jsTrace, isTrue); - expect(merged.skipReason, equals("bar")); - expect(merged.runSkipped, isFalse); expect(merged.pauseAfterLoad, isFalse); expect(merged.color, isTrue); - expect(merged.shardIndex, equals(3)); - expect(merged.totalShards, equals(10)); expect(merged.dart2jsPath, equals("../dart2js")); - expect(merged.precompiledPath, equals("../js")); expect(merged.reporter, equals("compact")); expect(merged.pubServeUrl.port, equals(5678)); - expect(merged.platforms, equals([TestPlatform.dartium])); + expect(merged.shardIndex, equals(3)); + expect(merged.totalShards, equals(10)); expect(merged.paths, equals(["blech"])); }); }); - group("for testOn", () { + group("for chosenPresets", () { test("if neither is defined, preserves the default", () { var merged = new Configuration().merge(new Configuration()); - expect(merged.testOn, equals(PlatformSelector.all)); - }); - - test("if only the old configuration's is defined, uses it", () { - var merged = new Configuration( - testOn: new PlatformSelector.parse("chrome")) - .merge(new Configuration()); - expect(merged.testOn, equals(new PlatformSelector.parse("chrome"))); - }); - - test("if only the new configuration's is defined, uses it", () { - var merged = new Configuration() - .merge(new Configuration( - testOn: new PlatformSelector.parse("chrome"))); - expect(merged.testOn, equals(new PlatformSelector.parse("chrome"))); - }); - - test("if both are defined, intersects them", () { - var older = new Configuration( - testOn: new PlatformSelector.parse("vm")); - var newer = new Configuration( - testOn: new PlatformSelector.parse("linux")); - var merged = older.merge(newer); - expect(merged.testOn, - equals(new PlatformSelector.parse("vm && linux"))); - }); - }); - - group("for include and excludeTags", () { - test("if neither is defined, preserves the default", () { - var merged = new Configuration().merge(new Configuration()); - expect(merged.includeTags, equals(BooleanSelector.all)); - expect(merged.excludeTags, equals(BooleanSelector.none)); - }); - - test("if only the old configuration's is defined, uses it", () { - var merged = new Configuration( - includeTags: new BooleanSelector.parse("foo || bar"), - excludeTags: new BooleanSelector.parse("baz || bang")) - .merge(new Configuration()); - - expect(merged.includeTags, - equals(new BooleanSelector.parse("foo || bar"))); - expect(merged.excludeTags, - equals(new BooleanSelector.parse("baz || bang"))); - }); - - test("if only the new configuration's is defined, uses it", () { - var merged = new Configuration().merge(new Configuration( - includeTags: new BooleanSelector.parse("foo || bar"), - excludeTags: new BooleanSelector.parse("baz || bang"))); - - expect(merged.includeTags, - equals(new BooleanSelector.parse("foo || bar"))); - expect(merged.excludeTags, - equals(new BooleanSelector.parse("baz || bang"))); - }); - - test("if both are defined, unions or intersects them", () { - var older = new Configuration( - includeTags: new BooleanSelector.parse("foo || bar"), - excludeTags: new BooleanSelector.parse("baz || bang")); - var newer = new Configuration( - includeTags: new BooleanSelector.parse("blip"), - excludeTags: new BooleanSelector.parse("qux")); - var merged = older.merge(newer); - - expect(merged.includeTags, - equals(new BooleanSelector.parse("(foo || bar) && blip"))); - expect(merged.excludeTags, - equals(new BooleanSelector.parse("(baz || bang) || qux"))); - }); - }); - - group("for sets", () { - test("if neither is defined, preserves the default", () { - var merged = new Configuration().merge(new Configuration()); - expect(merged.addTags, isEmpty); expect(merged.chosenPresets, isEmpty); - expect(merged.patterns, isEmpty); }); test("if only the old configuration's is defined, uses it", () { - var merged = new Configuration( - addTags: ["foo", "bar"], - chosenPresets: ["baz", "bang"], - patterns: ["beep", "boop"]) + var merged = new Configuration(chosenPresets: ["baz", "bang"]) .merge(new Configuration()); - - expect(merged.addTags, unorderedEquals(["foo", "bar"])); expect(merged.chosenPresets, equals(["baz", "bang"])); - expect(merged.patterns, equals(["beep", "boop"])); }); test("if only the new configuration's is defined, uses it", () { - var merged = new Configuration().merge(new Configuration( - addTags: ["foo", "bar"], - chosenPresets: ["baz", "bang"], - patterns: ["beep", "boop"])); - - expect(merged.addTags, unorderedEquals(["foo", "bar"])); + var merged = new Configuration() + .merge(new Configuration(chosenPresets: ["baz", "bang"])); expect(merged.chosenPresets, equals(["baz", "bang"])); - expect(merged.patterns, equals(["beep", "boop"])); }); test("if both are defined, unions them", () { - var older = new Configuration( - addTags: ["foo", "bar"], - chosenPresets: ["baz", "bang"], - patterns: ["beep", "boop"]); - var newer = new Configuration( - addTags: ["blip"], - chosenPresets: ["qux"], - patterns: ["bonk"]); - var merged = older.merge(newer); - - expect(merged.addTags, unorderedEquals(["foo", "bar", "blip"])); + var merged = new Configuration(chosenPresets: ["baz", "bang"]) + .merge(new Configuration(chosenPresets: ["qux"])); expect(merged.chosenPresets, equals(["baz", "bang", "qux"])); - expect(merged.patterns, unorderedEquals(["beep", "boop", "bonk"])); }); }); - group("for timeout", () { - test("if neither is defined, preserves the default", () { - var merged = new Configuration().merge(new Configuration()); - expect(merged.timeout, equals(new Timeout.factor(1))); - }); - - test("if only the old configuration's is defined, uses it", () { - var merged = new Configuration(timeout: new Timeout.factor(2)) - .merge(new Configuration()); - expect(merged.timeout, equals(new Timeout.factor(2))); - }); - - test("if only the new configuration's is defined, uses it", () { - var merged = new Configuration() - .merge(new Configuration(timeout: new Timeout.factor(2))); - expect(merged.timeout, equals(new Timeout.factor(2))); - }); - - test("if both are defined, merges them", () { - var older = new Configuration(timeout: new Timeout.factor(2)); - var newer = new Configuration(timeout: new Timeout.factor(3)); - var merged = older.merge(newer); - expect(merged.timeout, equals(new Timeout.factor(6))); - }); - - test("if the merge conflicts, prefers the new value", () { - var older = new Configuration( - timeout: new Timeout(new Duration(seconds: 1))); - var newer = new Configuration( - timeout: new Timeout(new Duration(seconds: 2))); - var merged = older.merge(newer); - expect(merged.timeout, equals(new Timeout(new Duration(seconds: 2)))); - }); - }); - - group("for dart2jsArgs", () { - test("if neither is defined, preserves the default", () { - var merged = new Configuration().merge(new Configuration()); - expect(merged.dart2jsArgs, isEmpty); - }); - - test("if only the old configuration's is defined, uses it", () { - var merged = new Configuration(dart2jsArgs: ["--foo", "--bar"]) - .merge(new Configuration()); - expect(merged.dart2jsArgs, equals(["--foo", "--bar"])); - }); - - test("if only the new configuration's is defined, uses it", () { - var merged = new Configuration() - .merge(new Configuration(dart2jsArgs: ["--foo", "--bar"])); - expect(merged.dart2jsArgs, equals(["--foo", "--bar"])); - }); - - test("if both are defined, concatenates them", () { - var older = new Configuration(dart2jsArgs: ["--foo", "--bar"]); - var newer = new Configuration(dart2jsArgs: ["--baz"]); - var merged = older.merge(newer); - expect(merged.dart2jsArgs, equals(["--foo", "--bar", "--baz"])); - }); - }); - - group("for config maps", () { + group("for presets", () { test("merges each nested configuration", () { - var merged = new Configuration( - tags: { - new BooleanSelector.parse("foo"): - new Configuration(verboseTrace: true), - new BooleanSelector.parse("bar"): new Configuration(jsTrace: true) - }, - onPlatform: { - new PlatformSelector.parse("vm"): - new Configuration(verboseTrace: true), - new PlatformSelector.parse("chrome"): - new Configuration(jsTrace: true) - }, - presets: { - "bang": new Configuration(verboseTrace: true), - "qux": new Configuration(jsTrace: true) - } - ).merge(new Configuration( - tags: { - new BooleanSelector.parse("bar"): new Configuration(jsTrace: false), - new BooleanSelector.parse("baz"): new Configuration(skip: true) - }, - onPlatform: { - new PlatformSelector.parse("chrome"): - new Configuration(jsTrace: false), - new PlatformSelector.parse("firefox"): new Configuration(skip: true) - }, - presets: { - "qux": new Configuration(jsTrace: false), - "zap": new Configuration(skip: true) - } - )); - - expect(merged.tags[new BooleanSelector.parse("foo")].verboseTrace, - isTrue); - expect(merged.tags[new BooleanSelector.parse("bar")].jsTrace, isFalse); - expect(merged.tags[new BooleanSelector.parse("baz")].skip, isTrue); + var merged = new Configuration(presets: { + "bang": new Configuration(pauseAfterLoad: true), + "qux": new Configuration(color: true) + }).merge(new Configuration(presets: { + "qux": new Configuration(color: false), + "zap": new Configuration(help: true) + })); - expect(merged.onPlatform[new PlatformSelector.parse("vm")].verboseTrace, - isTrue); - expect(merged.onPlatform[new PlatformSelector.parse("chrome")].jsTrace, - isFalse); - expect(merged.onPlatform[new PlatformSelector.parse("firefox")].skip, - isTrue); - - expect(merged.presets["bang"].verboseTrace, isTrue); - expect(merged.presets["qux"].jsTrace, isFalse); - expect(merged.presets["zap"].skip, isTrue); + expect(merged.presets["bang"].pauseAfterLoad, isTrue); + expect(merged.presets["qux"].color, isFalse); + expect(merged.presets["zap"].help, isTrue); }); - }); - group("for presets", () { test("automatically resolves a matching chosen preset", () { var configuration = new Configuration( - presets: {"foo": new Configuration(verboseTrace: true)}, + presets: {"foo": new Configuration(color: true)}, chosenPresets: ["foo"]); expect(configuration.presets, isEmpty); expect(configuration.chosenPresets, equals(["foo"])); expect(configuration.knownPresets, equals(["foo"])); - expect(configuration.verboseTrace, isTrue); + expect(configuration.color, isTrue); }); test("resolves a chosen presets in order", () { var configuration = new Configuration( presets: { - "foo": new Configuration(verboseTrace: true), - "bar": new Configuration(verboseTrace: false) + "foo": new Configuration(color: true), + "bar": new Configuration(color: false) }, chosenPresets: ["foo", "bar"]); expect(configuration.presets, isEmpty); expect(configuration.chosenPresets, equals(["foo", "bar"])); expect(configuration.knownPresets, unorderedEquals(["foo", "bar"])); - expect(configuration.verboseTrace, isFalse); + expect(configuration.color, isFalse); configuration = new Configuration( presets: { - "foo": new Configuration(verboseTrace: true), - "bar": new Configuration(verboseTrace: false) + "foo": new Configuration(color: true), + "bar": new Configuration(color: false) }, chosenPresets: ["bar", "foo"]); expect(configuration.presets, isEmpty); expect(configuration.chosenPresets, equals(["bar", "foo"])); expect(configuration.knownPresets, unorderedEquals(["foo", "bar"])); - expect(configuration.verboseTrace, isTrue); + expect(configuration.color, isTrue); }); test("ignores inapplicable chosen presets", () { @@ -460,25 +206,25 @@ void main() { test("resolves presets through merging", () { var configuration = new Configuration(presets: { - "foo": new Configuration(verboseTrace: true) + "foo": new Configuration(color: true) }).merge(new Configuration(chosenPresets: ["foo"])); expect(configuration.presets, isEmpty); expect(configuration.chosenPresets, equals(["foo"])); expect(configuration.knownPresets, equals(["foo"])); - expect(configuration.verboseTrace, isTrue); + expect(configuration.color, isTrue); }); test("preserves known presets through merging", () { - var configuration = new Configuration(presets: { - "foo": new Configuration(verboseTrace: true) - }, chosenPresets: ["foo"]) + var configuration = new Configuration( + presets: {"foo": new Configuration(color: true)}, + chosenPresets: ["foo"]) .merge(new Configuration()); expect(configuration.presets, isEmpty); expect(configuration.chosenPresets, equals(["foo"])); expect(configuration.knownPresets, equals(["foo"])); - expect(configuration.verboseTrace, isTrue); + expect(configuration.color, isTrue); }); }); }); diff --git a/test/runner/configuration/suite_test.dart b/test/runner/configuration/suite_test.dart new file mode 100644 index 000000000..05a58c58e --- /dev/null +++ b/test/runner/configuration/suite_test.dart @@ -0,0 +1,224 @@ +// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +@TestOn("vm") + +import 'package:boolean_selector/boolean_selector.dart'; +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; + +import 'package:test/src/backend/platform_selector.dart'; +import 'package:test/src/backend/test_platform.dart'; +import 'package:test/src/runner/configuration/suite.dart'; +import 'package:test/src/runner/configuration/values.dart'; +import 'package:test/src/util/io.dart'; + +void main() { + group("merge", () { + group("for most fields", () { + test("if neither is defined, preserves the default", () { + var merged = new SuiteConfiguration().merge(new SuiteConfiguration()); + expect(merged.jsTrace, isFalse); + expect(merged.runSkipped, isFalse); + expect(merged.precompiledPath, isNull); + expect(merged.platforms, equals([TestPlatform.vm])); + }); + + test("if only the old configuration's is defined, uses it", () { + var merged = new SuiteConfiguration( + jsTrace: true, + runSkipped: true, + precompiledPath: "/tmp/js", + platforms: [TestPlatform.chrome]) + .merge(new SuiteConfiguration()); + + expect(merged.jsTrace, isTrue); + expect(merged.runSkipped, isTrue); + expect(merged.precompiledPath, equals("/tmp/js")); + expect(merged.platforms, equals([TestPlatform.chrome])); + }); + + test("if only the new configuration's is defined, uses it", () { + var merged = new SuiteConfiguration().merge(new SuiteConfiguration( + jsTrace: true, + runSkipped: true, + precompiledPath: "/tmp/js", + platforms: [TestPlatform.chrome])); + + expect(merged.jsTrace, isTrue); + expect(merged.runSkipped, isTrue); + expect(merged.precompiledPath, equals("/tmp/js")); + expect(merged.platforms, equals([TestPlatform.chrome])); + }); + + test("if the two configurations conflict, uses the new configuration's " + "values", () { + var older = new SuiteConfiguration( + jsTrace: false, + runSkipped: true, + precompiledPath: "/tmp/js", + platforms: [TestPlatform.chrome]); + var newer = new SuiteConfiguration( + jsTrace: true, + runSkipped: false, + precompiledPath: "../js", + platforms: [TestPlatform.dartium]); + var merged = older.merge(newer); + + expect(merged.jsTrace, isTrue); + expect(merged.runSkipped, isFalse); + expect(merged.precompiledPath, equals("../js")); + expect(merged.platforms, equals([TestPlatform.dartium])); + }); + }); + + group("for include and excludeTags", () { + test("if neither is defined, preserves the default", () { + var merged = new SuiteConfiguration().merge(new SuiteConfiguration()); + expect(merged.includeTags, equals(BooleanSelector.all)); + expect(merged.excludeTags, equals(BooleanSelector.none)); + }); + + test("if only the old configuration's is defined, uses it", () { + var merged = new SuiteConfiguration( + includeTags: new BooleanSelector.parse("foo || bar"), + excludeTags: new BooleanSelector.parse("baz || bang")) + .merge(new SuiteConfiguration()); + + expect(merged.includeTags, + equals(new BooleanSelector.parse("foo || bar"))); + expect(merged.excludeTags, + equals(new BooleanSelector.parse("baz || bang"))); + }); + + test("if only the new configuration's is defined, uses it", () { + var merged = new SuiteConfiguration().merge(new SuiteConfiguration( + includeTags: new BooleanSelector.parse("foo || bar"), + excludeTags: new BooleanSelector.parse("baz || bang"))); + + expect(merged.includeTags, + equals(new BooleanSelector.parse("foo || bar"))); + expect(merged.excludeTags, + equals(new BooleanSelector.parse("baz || bang"))); + }); + + test("if both are defined, unions or intersects them", () { + var older = new SuiteConfiguration( + includeTags: new BooleanSelector.parse("foo || bar"), + excludeTags: new BooleanSelector.parse("baz || bang")); + var newer = new SuiteConfiguration( + includeTags: new BooleanSelector.parse("blip"), + excludeTags: new BooleanSelector.parse("qux")); + var merged = older.merge(newer); + + expect(merged.includeTags, + equals(new BooleanSelector.parse("(foo || bar) && blip"))); + expect(merged.excludeTags, + equals(new BooleanSelector.parse("(baz || bang) || qux"))); + }); + }); + + group("for sets", () { + test("if neither is defined, preserves the default", () { + var merged = new SuiteConfiguration().merge(new SuiteConfiguration()); + expect(merged.patterns, isEmpty); + }); + + test("if only the old configuration's is defined, uses it", () { + var merged = new SuiteConfiguration(patterns: ["beep", "boop"]) + .merge(new SuiteConfiguration()); + + expect(merged.patterns, equals(["beep", "boop"])); + }); + + test("if only the new configuration's is defined, uses it", () { + var merged = new SuiteConfiguration() + .merge(new SuiteConfiguration(patterns: ["beep", "boop"])); + + expect(merged.patterns, equals(["beep", "boop"])); + }); + + test("if both are defined, unions them", () { + var older = new SuiteConfiguration(patterns: ["beep", "boop"]); + var newer = new SuiteConfiguration(patterns: ["bonk"]); + var merged = older.merge(newer); + + expect(merged.patterns, unorderedEquals(["beep", "boop", "bonk"])); + }); + }); + + group("for dart2jsArgs", () { + test("if neither is defined, preserves the default", () { + var merged = new SuiteConfiguration().merge(new SuiteConfiguration()); + expect(merged.dart2jsArgs, isEmpty); + }); + + test("if only the old configuration's is defined, uses it", () { + var merged = new SuiteConfiguration(dart2jsArgs: ["--foo", "--bar"]) + .merge(new SuiteConfiguration()); + expect(merged.dart2jsArgs, equals(["--foo", "--bar"])); + }); + + test("if only the new configuration's is defined, uses it", () { + var merged = new SuiteConfiguration() + .merge(new SuiteConfiguration(dart2jsArgs: ["--foo", "--bar"])); + expect(merged.dart2jsArgs, equals(["--foo", "--bar"])); + }); + + test("if both are defined, concatenates them", () { + var older = new SuiteConfiguration(dart2jsArgs: ["--foo", "--bar"]); + var newer = new SuiteConfiguration(dart2jsArgs: ["--baz"]); + var merged = older.merge(newer); + expect(merged.dart2jsArgs, equals(["--foo", "--bar", "--baz"])); + }); + }); + + group("for config maps", () { + test("merges each nested configuration", () { + var merged = new SuiteConfiguration( + tags: { + new BooleanSelector.parse("foo"): + new SuiteConfiguration(precompiledPath: "path/"), + new BooleanSelector.parse("bar"): + new SuiteConfiguration(jsTrace: true) + }, + onPlatform: { + new PlatformSelector.parse("vm"): + new SuiteConfiguration(precompiledPath: "path/"), + new PlatformSelector.parse("chrome"): + new SuiteConfiguration(jsTrace: true) + } + ).merge(new SuiteConfiguration( + tags: { + new BooleanSelector.parse("bar"): + new SuiteConfiguration(jsTrace: false), + new BooleanSelector.parse("baz"): + new SuiteConfiguration(runSkipped: true) + }, + onPlatform: { + new PlatformSelector.parse("chrome"): + new SuiteConfiguration(jsTrace: false), + new PlatformSelector.parse("firefox"): + new SuiteConfiguration(runSkipped: true) + } + )); + + expect(merged.tags[new BooleanSelector.parse("foo")].precompiledPath, + equals("path/")); + expect(merged.tags[new BooleanSelector.parse("bar")].jsTrace, isFalse); + expect(merged.tags[new BooleanSelector.parse("baz")].runSkipped, + isTrue); + + expect( + merged.onPlatform[new PlatformSelector.parse("vm")].precompiledPath, + "path/"); + expect(merged.onPlatform[new PlatformSelector.parse("chrome")].jsTrace, + isFalse); + expect( + merged.onPlatform[new PlatformSelector.parse("firefox")].runSkipped, + isTrue); + }); + }); + }); +} diff --git a/test/runner/engine_test.dart b/test/runner/engine_test.dart index 833d680de..f2577bdab 100644 --- a/test/runner/engine_test.dart +++ b/test/runner/engine_test.dart @@ -27,8 +27,8 @@ void main() { }); var engine = new Engine.withSuites([ - new RunnerSuite(const PluginEnvironment(), new Group.root(tests.take(2))), - new RunnerSuite(const PluginEnvironment(), new Group.root(tests.skip(2))) + runnerSuite(new Group.root(tests.take(2))), + runnerSuite(new Group.root(tests.skip(2))) ]); await engine.run(); @@ -51,8 +51,7 @@ void main() { expect(testsRun, equals(4)); }), completes); - engine.suiteSink.add( - new RunnerSuite(const PluginEnvironment(), new Group.root(tests))); + engine.suiteSink.add(runnerSuite(new Group.root(tests))); engine.suiteSink.close(); }); @@ -175,9 +174,7 @@ void main() { test("test", () {}, skip: true); }); - var engine = new Engine.withSuites([ - new RunnerSuite(const PluginEnvironment(), new Group.root(tests)) - ]); + var engine = new Engine.withSuites([runnerSuite(new Group.root(tests))]); engine.onTestStarted.listen(expectAsync((liveTest) { expect(liveTest, same(engine.liveTests.single)); @@ -234,9 +231,8 @@ void main() { }, skip: true); }); - var engine = new Engine.withSuites([ - new RunnerSuite(const PluginEnvironment(), new Group.root(entries)) - ]); + var engine = new Engine.withSuites( + [runnerSuite(new Group.root(entries))]); engine.onTestStarted.listen(expectAsync((liveTest) { expect(liveTest, same(engine.liveTests.single)); diff --git a/test/runner/load_suite_test.dart b/test/runner/load_suite_test.dart index dc00f75f9..5c373218b 100644 --- a/test/runner/load_suite_test.dart +++ b/test/runner/load_suite_test.dart @@ -9,6 +9,7 @@ import 'dart:async'; import 'package:test/src/backend/group.dart'; import 'package:test/src/backend/state.dart'; import 'package:test/src/backend/test_platform.dart'; +import 'package:test/src/runner/configuration/suite.dart'; import 'package:test/src/runner/load_exception.dart'; import 'package:test/src/runner/load_suite.dart'; import 'package:test/src/runner/plugin/environment.dart'; @@ -20,11 +21,12 @@ import '../utils.dart'; void main() { var innerSuite; setUp(() { - innerSuite = new RunnerSuite(const PluginEnvironment(), new Group.root([])); + innerSuite = runnerSuite(new Group.root([])); }); test("running a load test causes LoadSuite.suite to emit a suite", () async { - var suite = new LoadSuite("name", () => new Future.value(innerSuite)); + var suite = new LoadSuite("name", SuiteConfiguration.empty, + () => new Future.value(innerSuite)); expect(suite.group.entries, hasLength(1)); expect(suite.suite, completion(equals(innerSuite))); @@ -34,7 +36,8 @@ void main() { }); test("running a load suite's body may be synchronous", () async { - var suite = new LoadSuite("name", () => innerSuite); + var suite = new LoadSuite("name", SuiteConfiguration.empty, + () => innerSuite); expect(suite.group.entries, hasLength(1)); expect(suite.suite, completion(equals(innerSuite))); @@ -45,7 +48,8 @@ void main() { test("a load test doesn't complete until the body returns", () async { var completer = new Completer(); - var suite = new LoadSuite("name", () => completer.future); + var suite = new LoadSuite("name", SuiteConfiguration.empty, + () => completer.future); expect(suite.group.entries, hasLength(1)); var liveTest = await suite.group.entries.single.load(suite); @@ -60,7 +64,8 @@ void main() { test("a load test forwards errors and completes LoadSuite.suite to null", () async { - var suite = new LoadSuite("name", () => fail("error")); + var suite = new LoadSuite("name", SuiteConfiguration.empty, + () => fail("error")); expect(suite.group.entries, hasLength(1)); expect(suite.suite, completion(isNull)); @@ -71,7 +76,8 @@ void main() { }); test("a load test completes early if it's closed", () async { - var suite = new LoadSuite("name", () => new Completer().future); + var suite = new LoadSuite("name", SuiteConfiguration.empty, + () => new Completer().future); expect(suite.group.entries, hasLength(1)); var liveTest = await suite.group.entries.single.load(suite); @@ -85,7 +91,8 @@ void main() { test("forLoadException() creates a suite that completes to a LoadException", () async { var exception = new LoadException("path", "error"); - var suite = new LoadSuite.forLoadException(exception); + var suite = new LoadSuite.forLoadException( + exception, SuiteConfiguration.empty); expect(suite.group.entries, hasLength(1)); expect(suite.suite, completion(isNull)); @@ -111,21 +118,23 @@ void main() { group("changeSuite()", () { test("returns a new load suite with the same properties", () { - var suite = new LoadSuite("name", () => innerSuite, + var suite = new LoadSuite("name", SuiteConfiguration.empty, + () => innerSuite, platform: TestPlatform.vm); expect(suite.group.entries, hasLength(1)); var newSuite = suite.changeSuite((suite) => suite); expect(newSuite.platform, equals(TestPlatform.vm)); - expect(newSuite.group.entries.single.name, equals(suite.group.entries.single.name)); + expect(newSuite.group.entries.single.name, + equals(suite.group.entries.single.name)); }); test("changes the inner suite", () async { - var suite = new LoadSuite("name", () => innerSuite); + var suite = new LoadSuite("name", SuiteConfiguration.empty, + () => innerSuite); expect(suite.group.entries, hasLength(1)); - var newInnerSuite = new RunnerSuite( - const PluginEnvironment(), new Group.root([])); + var newInnerSuite = runnerSuite(new Group.root([])); var newSuite = suite.changeSuite((suite) => newInnerSuite); expect(newSuite.suite, completion(equals(newInnerSuite))); @@ -135,7 +144,7 @@ void main() { }); test("doesn't run change() if the suite is null", () async { - var suite = new LoadSuite("name", () => null); + var suite = new LoadSuite("name", SuiteConfiguration.empty, () => null); expect(suite.group.entries, hasLength(1)); var newSuite = suite.changeSuite(expectAsync((_) {}, count: 0)); @@ -156,7 +165,8 @@ void main() { }); test("forwards errors to the future", () { - var suite = new LoadSuite("name", () => throw "error"); + var suite = new LoadSuite("name", SuiteConfiguration.empty, + () => throw "error"); expect(suite.group.entries, hasLength(1)); expect(suite.getSuite(), throwsA("error")); diff --git a/test/runner/loader_test.dart b/test/runner/loader_test.dart index cb77dad43..b8a968a5b 100644 --- a/test/runner/loader_test.dart +++ b/test/runner/loader_test.dart @@ -10,6 +10,7 @@ import 'package:path/path.dart' as p; import 'package:test/src/backend/state.dart'; import 'package:test/src/backend/test_platform.dart'; import 'package:test/src/runner/configuration.dart'; +import 'package:test/src/runner/configuration/suite.dart'; import 'package:test/src/runner/loader.dart'; import 'package:test/src/util/io.dart'; import 'package:test/test.dart'; @@ -49,7 +50,8 @@ void main() { /// TODO(nweiz): Use scheduled_test for this once it's compatible with /// this version of test. new File(p.join(_sandbox, 'a_test.dart')).writeAsStringSync(_tests); - var suites = await _loader.loadFile(p.join(_sandbox, 'a_test.dart')) + var suites = await _loader + .loadFile(p.join(_sandbox, 'a_test.dart'), SuiteConfiguration.empty) .toList(); expect(suites, hasLength(1)); var loadSuite = suites.first; @@ -90,19 +92,22 @@ void main() { group(".loadDir()", () { test("ignores non-Dart files", () { new File(p.join(_sandbox, 'a_test.txt')).writeAsStringSync(_tests); - expect(_loader.loadDir(_sandbox).toList(), completion(isEmpty)); + expect(_loader.loadDir(_sandbox, SuiteConfiguration.empty).toList(), + completion(isEmpty)); }); test("ignores files in packages/ directories", () { var dir = p.join(_sandbox, 'packages'); new Directory(dir).createSync(); new File(p.join(dir, 'a_test.dart')).writeAsStringSync(_tests); - expect(_loader.loadDir(_sandbox).toList(), completion(isEmpty)); + expect(_loader.loadDir(_sandbox, SuiteConfiguration.empty).toList(), + completion(isEmpty)); }); test("ignores files that don't end in _test.dart", () { new File(p.join(_sandbox, 'test.dart')).writeAsStringSync(_tests); - expect(_loader.loadDir(_sandbox).toList(), completion(isEmpty)); + expect(_loader.loadDir(_sandbox, SuiteConfiguration.empty).toList(), + completion(isEmpty)); }); group("with suites loaded from a directory", () { @@ -117,7 +122,7 @@ void main() { new File(p.join(_sandbox, 'dir/sub_test.dart')) .writeAsStringSync(_tests); - suites = await _loader.loadDir(_sandbox) + suites = await _loader.loadDir(_sandbox, SuiteConfiguration.empty) .asyncMap((loadSuite) => loadSuite.getSuite()) .toList(); }); @@ -145,7 +150,8 @@ void main() { print('print within test'); } """); - var suites = await _loader.loadFile(p.join(_sandbox, 'a_test.dart')) + var suites = await _loader + .loadFile(p.join(_sandbox, 'a_test.dart'), SuiteConfiguration.empty) .toList(); expect(suites, hasLength(1)); var loadSuite = suites.first; diff --git a/test/utils.dart b/test/utils.dart index 5bbba17b9..19ac49705 100644 --- a/test/utils.dart +++ b/test/utils.dart @@ -14,6 +14,7 @@ import 'package:test/src/backend/metadata.dart'; import 'package:test/src/backend/state.dart'; import 'package:test/src/backend/suite.dart'; import 'package:test/src/runner/application_exception.dart'; +import 'package:test/src/runner/configuration/suite.dart'; import 'package:test/src/runner/engine.dart'; import 'package:test/src/runner/load_exception.dart'; import 'package:test/src/runner/plugin/environment.dart'; @@ -308,6 +309,13 @@ List declare(void body()) { Engine declareEngine(void body(), {bool runSkipped: false}) { var declarer = new Declarer()..declare(body); return new Engine.withSuites([ - new RunnerSuite(const PluginEnvironment(), declarer.build()) - ], runSkipped: runSkipped); + new RunnerSuite( + const PluginEnvironment(), + new SuiteConfiguration(runSkipped: runSkipped), + declarer.build()) + ]); } + +/// Returns a [RunnerSuite] with a default environment and configuration. +RunnerSuite runnerSuite(Group root) => + new RunnerSuite(const PluginEnvironment(), SuiteConfiguration.empty, root);