Skip to content

Commit 9689f7f

Browse files
sealesjsealesj
authored
Refactor framework + test harness tests (#146213)
Refactor the framework + test harness test suites in order to reduce testing logic in test.dart and allow for later implementing package:test onto the existing tests The refactor of both suites included in this PR because they both depended on util functions and variables that also needed to be refactored. Part of flutter/flutter#145482
1 parent e34a9e3 commit 9689f7f

File tree

6 files changed

+504
-490
lines changed

6 files changed

+504
-490
lines changed

‎dev/bots/analyze.dart

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,6 @@ import 'custom_rules/render_box_intrinsics.dart';
2626
import 'run_command.dart';
2727
import 'utils.dart';
2828

29-
final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script))));
30-
final String flutter = path.join(flutterRoot, 'bin', Platform.isWindows ? 'flutter.bat' : 'flutter');
3129
final String flutterPackages = path.join(flutterRoot, 'packages');
3230
final String flutterExamples = path.join(flutterRoot, 'examples');
3331

‎dev/bots/suite_runners/run_framework_tests.dart

Lines changed: 307 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,307 @@
1+
// Copyright 2014 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'dart:convert';
6+
import 'dart:io' show Directory, File, FileSystemEntity, Platform, Process;
7+
import 'dart:typed_data';
8+
9+
import 'package:archive/archive.dart';
10+
import 'package:path/path.dart' as path;
11+
12+
import '../run_command.dart';
13+
import '../utils.dart';
14+
import 'run_test_harness_tests.dart';
15+
16+
Future<void> frameworkTestsRunner() async {
17+
final List<String> trackWidgetCreationAlternatives = <String>['--track-widget-creation', '--no-track-widget-creation'];
18+
19+
Future<void> runWidgets() async {
20+
printProgress('${green}Running packages/flutter tests $reset for ${cyan}test/widgets/$reset');
21+
for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
22+
await runFlutterTest(
23+
path.join(flutterRoot, 'packages', 'flutter'),
24+
options: <String>[trackWidgetCreationOption],
25+
tests: <String>[ path.join('test', 'widgets') + path.separator ],
26+
);
27+
}
28+
// Try compiling code outside of the packages/flutter directory with and without --track-widget-creation
29+
for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
30+
await runFlutterTest(
31+
path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery'),
32+
options: <String>[trackWidgetCreationOption],
33+
fatalWarnings: false, // until we've migrated video_player
34+
);
35+
}
36+
// Run release mode tests (see packages/flutter/test_release/README.md)
37+
await runFlutterTest(
38+
path.join(flutterRoot, 'packages', 'flutter'),
39+
options: <String>['--dart-define=dart.vm.product=true'],
40+
tests: <String>['test_release${path.separator}'],
41+
);
42+
// Run profile mode tests (see packages/flutter/test_profile/README.md)
43+
await runFlutterTest(
44+
path.join(flutterRoot, 'packages', 'flutter'),
45+
options: <String>['--dart-define=dart.vm.product=false', '--dart-define=dart.vm.profile=true'],
46+
tests: <String>['test_profile${path.separator}'],
47+
);
48+
}
49+
50+
Future<void> runImpeller() async {
51+
printProgress('${green}Running packages/flutter tests $reset in Impeller$reset');
52+
await runFlutterTest(
53+
path.join(flutterRoot, 'packages', 'flutter'),
54+
options: <String>['--enable-impeller'],
55+
);
56+
}
57+
58+
59+
Future<void> runLibraries() async {
60+
final List<String> tests = Directory(path.join(flutterRoot, 'packages', 'flutter', 'test'))
61+
.listSync(followLinks: false)
62+
.whereType<Directory>()
63+
.where((Directory dir) => !dir.path.endsWith('widgets'))
64+
.map<String>((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator)
65+
.toList();
66+
printProgress('${green}Running packages/flutter tests$reset for $cyan${tests.join(", ")}$reset');
67+
for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
68+
await runFlutterTest(
69+
path.join(flutterRoot, 'packages', 'flutter'),
70+
options: <String>[trackWidgetCreationOption],
71+
tests: tests,
72+
);
73+
}
74+
}
75+
76+
Future<void> runExampleTests() async {
77+
await runCommand(
78+
flutter,
79+
<String>['config', '--enable-${Platform.operatingSystem}-desktop'],
80+
workingDirectory: flutterRoot,
81+
);
82+
await runCommand(
83+
dart,
84+
<String>[path.join(flutterRoot, 'dev', 'tools', 'examples_smoke_test.dart')],
85+
workingDirectory: path.join(flutterRoot, 'examples', 'api'),
86+
);
87+
for (final FileSystemEntity entity in Directory(path.join(flutterRoot, 'examples')).listSync()) {
88+
if (entity is! Directory || !Directory(path.join(entity.path, 'test')).existsSync()) {
89+
continue;
90+
}
91+
await runFlutterTest(entity.path);
92+
}
93+
}
94+
95+
Future<void> runTracingTests() async {
96+
final String tracingDirectory = path.join(flutterRoot, 'dev', 'tracing_tests');
97+
98+
// run the tests for debug mode
99+
await runFlutterTest(tracingDirectory, options: <String>['--enable-vmservice']);
100+
101+
Future<List<String>> verifyTracingAppBuild({
102+
required String modeArgument,
103+
required String sourceFile,
104+
required Set<String> allowed,
105+
required Set<String> disallowed,
106+
}) async {
107+
try {
108+
await runCommand(
109+
flutter,
110+
<String>[
111+
'build', 'appbundle', '--$modeArgument', path.join('lib', sourceFile),
112+
],
113+
workingDirectory: tracingDirectory,
114+
);
115+
final Archive archive = ZipDecoder().decodeBytes(File(path.join(tracingDirectory, 'build', 'app', 'outputs', 'bundle', modeArgument, 'app-$modeArgument.aab')).readAsBytesSync());
116+
final ArchiveFile libapp = archive.findFile('base/lib/arm64-v8a/libapp.so')!;
117+
final Uint8List libappBytes = libapp.content as Uint8List; // bytes decompressed here
118+
final String libappStrings = utf8.decode(libappBytes, allowMalformed: true);
119+
await runCommand(flutter, <String>['clean'], workingDirectory: tracingDirectory);
120+
final List<String> results = <String>[];
121+
for (final String pattern in allowed) {
122+
if (!libappStrings.contains(pattern)) {
123+
results.add('When building with --$modeArgument, expected to find "$pattern" in libapp.so but could not find it.');
124+
}
125+
}
126+
for (final String pattern in disallowed) {
127+
if (libappStrings.contains(pattern)) {
128+
results.add('When building with --$modeArgument, expected to not find "$pattern" in libapp.so but did find it.');
129+
}
130+
}
131+
return results;
132+
} catch (error, stackTrace) {
133+
return <String>[
134+
error.toString(),
135+
...stackTrace.toString().trimRight().split('\n'),
136+
];
137+
}
138+
}
139+
140+
final List<String> results = <String>[];
141+
results.addAll(await verifyTracingAppBuild(
142+
modeArgument: 'profile',
143+
sourceFile: 'control.dart', // this is the control, the other two below are the actual test
144+
allowed: <String>{
145+
'TIMELINE ARGUMENTS TEST CONTROL FILE',
146+
'toTimelineArguments used in non-debug build', // we call toTimelineArguments directly to check the message does exist
147+
},
148+
disallowed: <String>{
149+
'BUILT IN DEBUG MODE', 'BUILT IN RELEASE MODE',
150+
},
151+
));
152+
results.addAll(await verifyTracingAppBuild(
153+
modeArgument: 'profile',
154+
sourceFile: 'test.dart',
155+
allowed: <String>{
156+
'BUILT IN PROFILE MODE', 'RenderTest.performResize called', // controls
157+
'BUILD', 'LAYOUT', 'PAINT', // we output these to the timeline in profile builds
158+
// (LAYOUT and PAINT also exist because of NEEDS-LAYOUT and NEEDS-PAINT in RenderObject.toStringShort)
159+
},
160+
disallowed: <String>{
161+
'BUILT IN DEBUG MODE', 'BUILT IN RELEASE MODE',
162+
'TestWidget.debugFillProperties called', 'RenderTest.debugFillProperties called', // debug only
163+
'toTimelineArguments used in non-debug build', // entire function should get dropped by tree shaker
164+
},
165+
));
166+
results.addAll(await verifyTracingAppBuild(
167+
modeArgument: 'release',
168+
sourceFile: 'test.dart',
169+
allowed: <String>{
170+
'BUILT IN RELEASE MODE', 'RenderTest.performResize called', // controls
171+
},
172+
disallowed: <String>{
173+
'BUILT IN DEBUG MODE', 'BUILT IN PROFILE MODE',
174+
'BUILD', 'LAYOUT', 'PAINT', // these are only used in Timeline.startSync calls that should not appear in release builds
175+
'TestWidget.debugFillProperties called', 'RenderTest.debugFillProperties called', // debug only
176+
'toTimelineArguments used in non-debug build', // not included in release builds
177+
},
178+
));
179+
if (results.isNotEmpty) {
180+
foundError(results);
181+
}
182+
}
183+
184+
Future<void> runFixTests(String package) async {
185+
final List<String> args = <String>[
186+
'fix',
187+
'--compare-to-golden',
188+
];
189+
await runCommand(
190+
dart,
191+
args,
192+
workingDirectory: path.join(flutterRoot, 'packages', package, 'test_fixes'),
193+
);
194+
}
195+
196+
Future<void> runPrivateTests() async {
197+
final List<String> args = <String>[
198+
'run',
199+
'bin/test_private.dart',
200+
];
201+
final Map<String, String> environment = <String, String>{
202+
'FLUTTER_ROOT': flutterRoot,
203+
if (Directory(pubCache).existsSync())
204+
'PUB_CACHE': pubCache,
205+
};
206+
adjustEnvironmentToEnableFlutterAsserts(environment);
207+
await runCommand(
208+
dart,
209+
args,
210+
workingDirectory: path.join(flutterRoot, 'packages', 'flutter', 'test_private'),
211+
environment: environment,
212+
);
213+
}
214+
215+
// Tests that take longer than average to run. This is usually because they
216+
// need to compile something large or make use of the analyzer for the test.
217+
// These tests need to be platform agnostic as they are only run on a linux
218+
// machine to save on execution time and cost.
219+
Future<void> runSlow() async {
220+
printProgress('${green}Running slow package tests$reset for directories other than packages/flutter');
221+
await runTracingTests();
222+
await runFixTests('flutter');
223+
await runFixTests('flutter_test');
224+
await runFixTests('integration_test');
225+
await runFixTests('flutter_driver');
226+
await runPrivateTests();
227+
}
228+
229+
Future<void> runMisc() async {
230+
printProgress('${green}Running package tests$reset for directories other than packages/flutter');
231+
await testHarnessTestsRunner();
232+
await runExampleTests();
233+
await runFlutterTest(
234+
path.join(flutterRoot, 'dev', 'a11y_assessments'),
235+
tests: <String>[ 'test' ],
236+
);
237+
await runDartTest(path.join(flutterRoot, 'dev', 'bots'));
238+
await runDartTest(path.join(flutterRoot, 'dev', 'devicelab'), ensurePrecompiledTool: false); // See https://github.com/flutter/flutter/issues/86209
239+
await runDartTest(path.join(flutterRoot, 'dev', 'conductor', 'core'), forceSingleCore: true);
240+
// TODO(gspencergoog): Remove the exception for fatalWarnings once https://github.com/flutter/flutter/issues/113782 has landed.
241+
await runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), fatalWarnings: false);
242+
await runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'ui'));
243+
await runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'));
244+
await runFlutterTest(path.join(flutterRoot, 'dev', 'tools'));
245+
await runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'));
246+
await runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'gen_defaults'));
247+
await runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'gen_keycodes'));
248+
await runFlutterTest(path.join(flutterRoot, 'dev', 'benchmarks', 'test_apps', 'stocks'));
249+
await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tests: <String>[path.join('test', 'src', 'real_tests')]);
250+
await runFlutterTest(path.join(flutterRoot, 'packages', 'integration_test'), options: <String>[
251+
'--enable-vmservice',
252+
// Web-specific tests depend on Chromium, so they run as part of the web_long_running_tests shard.
253+
'--exclude-tags=web',
254+
]);
255+
// Run java unit tests for integration_test
256+
//
257+
// Generate Gradle wrapper if it doesn't exist.
258+
Process.runSync(
259+
flutter,
260+
<String>['build', 'apk', '--config-only'],
261+
workingDirectory: path.join(flutterRoot, 'packages', 'integration_test', 'example', 'android'),
262+
);
263+
await runCommand(
264+
path.join(flutterRoot, 'packages', 'integration_test', 'example', 'android', 'gradlew$bat'),
265+
<String>[
266+
':integration_test:testDebugUnitTest',
267+
'--tests',
268+
'dev.flutter.plugins.integration_test.FlutterDeviceScreenshotTest',
269+
],
270+
workingDirectory: path.join(flutterRoot, 'packages', 'integration_test', 'example', 'android'),
271+
);
272+
await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_goldens'));
273+
await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'));
274+
await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'));
275+
await runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'));
276+
await runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable'));
277+
const String httpClientWarning =
278+
'Warning: At least one test in this suite creates an HttpClient. When running a test suite that uses\n'
279+
'TestWidgetsFlutterBinding, all HTTP requests will return status code 400, and no network request\n'
280+
'will actually be made. Any test expecting a real network connection and status code will fail.\n'
281+
'To test code that needs an HttpClient, provide your own HttpClient implementation to the code under\n'
282+
'test, so that your test can consistently provide a testable response to the code under test.';
283+
await runFlutterTest(
284+
path.join(flutterRoot, 'packages', 'flutter_test'),
285+
script: path.join('test', 'bindings_test_failure.dart'),
286+
expectFailure: true,
287+
printOutput: false,
288+
outputChecker: (CommandResult result) {
289+
final Iterable<Match> matches = httpClientWarning.allMatches(result.flattenedStdout!);
290+
if (matches.isEmpty || matches.length > 1) {
291+
return 'Failed to print warning about HttpClientUsage, or printed it too many times.\n\n'
292+
'stdout:\n${result.flattenedStdout}\n\n'
293+
'stderr:\n${result.flattenedStderr}';
294+
}
295+
return null;
296+
},
297+
);
298+
}
299+
300+
await selectSubshard(<String, ShardRunner>{
301+
'widgets': runWidgets,
302+
'libraries': runLibraries,
303+
'slow': runSlow,
304+
'misc': runMisc,
305+
'impeller': runImpeller,
306+
});
307+
}

0 commit comments

Comments
 (0)