Skip to content
This repository was archived by the owner on Feb 22, 2023. It is now read-only.

Commit 1b60d2f

Browse files
CareFCareF
committed
add new driver and perf
1 parent b4b6e9e commit 1b60d2f

File tree

6 files changed

+293
-5
lines changed

6 files changed

+293
-5
lines changed

‎packages/e2e/CHANGELOG.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
## 0.6.3
2+
3+
* Add customizable `flutter_driver` adaptor.
4+
* Add utilities for tracking frame performance in an e2e test.
5+
16
## 0.6.2+1
27

38
* Fix incorrect test results when one test passes then another fails

‎packages/e2e/example/test_driver/example_e2e_test.dart

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@ import 'dart:async';
22

33
import 'package:e2e/e2e_driver.dart' as e2e;
44

5-
Future<void> main() async => e2e.main();
5+
Future<void> main() async => e2e.e2eDriver();

‎packages/e2e/lib/e2e_driver.dart

Lines changed: 72 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,90 @@
1+
// Copyright 2014 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
15
import 'dart:async';
6+
import 'dart:convert';
27
import 'dart:io';
38

4-
import 'package:e2e/common.dart' as e2e;
59
import 'package:flutter_driver/flutter_driver.dart';
610

7-
Future<void> main() async {
11+
import 'package:e2e/common.dart' as e2e;
12+
import 'package:path/path.dart' as path;
13+
14+
/// This method remains for backword compatibility.
15+
Future<void> main() => e2eDriver();
16+
17+
/// Flutter Driver test output directory.
18+
///
19+
/// Tests should write any output files to this directory. Defaults to the path
20+
/// set in the FLUTTER_TEST_OUTPUTS_DIR environment variable, or `build` if
21+
/// unset.
22+
String testOutputsDirectory = Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? 'build';
23+
24+
/// The callback type to handle [e2e.Response.data] after the test succcess.
25+
typedef ResponseDataCallback = FutureOr<void> Function(Map<String, dynamic>);
26+
27+
/// Writes a json-serializable json data to to
28+
/// [testOutputsDirectory]/`testOutputFilename.json`.
29+
///
30+
/// This is the default `responseDataCallback` in [e2eDriver].
31+
Future<void> writeResponseData(Map<String, dynamic> data, {
32+
String testOutputFilename = 'e2e_perf_summary',
33+
}) async {
34+
assert(testOutputFilename != null);
35+
await fs.directory(testOutputsDirectory).create(recursive: true);
36+
final File file = fs.file(path.join(
37+
testOutputsDirectory,
38+
'$testOutputFilename.json'
39+
));
40+
final String resultString = _encodeJson(data, true);
41+
await file.writeAsString(resultString);
42+
}
43+
44+
/// Adaptor to run E2E test using `flutter drive`.
45+
///
46+
/// `timeout` controls the longest time waited before the test ends.
47+
/// It is not necessarily the execution time for the test app: the test may
48+
/// finish sooner than the `timeout`.
49+
///
50+
/// `responseDataCallback` is the handler for processing [e2e.Response.data].
51+
/// The default value is `writeResponseData`.
52+
///
53+
/// To an E2E test `<test_name>.dart` using `flutter drive`, put a file named
54+
/// `<test_name>_test.dart` in the app's `test_driver` directory:
55+
///
56+
/// ```dart
57+
/// import 'dart:async';
58+
///
59+
/// import 'package:e2e/e2e_driver.dart' as e2e;
60+
///
61+
/// Future<void> main() async => e2e.e2eDriver();
62+
///
63+
/// ```
64+
Future<void> e2eDriver({
65+
Duration timeout = const Duration(minutes: 1),
66+
ResponseDataCallback responseDataCallback = writeResponseData,
67+
}) async {
868
final FlutterDriver driver = await FlutterDriver.connect();
969
final String jsonResult =
10-
await driver.requestData(null, timeout: const Duration(minutes: 1));
70+
await driver.requestData(null, timeout: timeout);
1171
final e2e.Response response = e2e.Response.fromJson(jsonResult);
1272
await driver.close();
1373

1474
if (response.allTestsPassed) {
1575
print('All tests passed.');
76+
await responseDataCallback(response.data);
1677
exit(0);
1778
} else {
1879
print('Failure Details:\n${response.formattedFailureDetails}');
1980
exit(1);
2081
}
2182
}
83+
84+
const JsonEncoder _prettyEncoder = JsonEncoder.withIndent(' ');
85+
86+
String _encodeJson(Map<String, dynamic> jsonObject, bool pretty) {
87+
return pretty
88+
? _prettyEncoder.convert(jsonObject)
89+
: json.encode(jsonObject);
90+
}

‎packages/e2e/lib/e2e_perf.dart

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
// Copyright 2014 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'dart:async';
6+
import 'dart:ui';
7+
8+
import 'package:flutter/scheduler.dart';
9+
import 'package:flutter_test/flutter_test.dart';
10+
import 'package:flutter/widgets.dart';
11+
12+
import 'package:e2e/e2e.dart';
13+
14+
/// The maximum amount of time considered safe to spend for a frame's build
15+
/// phase. Anything past that is in the danger of missing the frame as 60FPS.
16+
///
17+
/// Changing this doesn't re-evaluate existing summary.
18+
Duration kBuildBudget = const Duration(milliseconds: 16);
19+
// TODO(CareF): Automatically calculate the refresh budget (#61958)
20+
21+
/// Usually it's recommended to limit callbacks of the test to [WidgetController]
22+
/// API so it can be more universally used.
23+
typedef ControlCallback = Future<void> Function(WidgetController controller);
24+
25+
bool _firstRun = true;
26+
27+
/// watches the [FrameTiming] of `action` and report it to the e2e binding.
28+
Future<void> watchPerformance(
29+
E2EWidgetsFlutterBinding binding,
30+
Future<void> action(),
31+
) async {
32+
assert(() {
33+
if (_firstRun) {
34+
debugPrint(kDebugWarning);
35+
_firstRun = false;
36+
}
37+
return true;
38+
}());
39+
final List<FrameTiming> frameTimings = <FrameTiming>[];
40+
final TimingsCallback watcher = frameTimings.addAll;
41+
binding.addTimingsCallback(watcher);
42+
await action();
43+
binding.removeTimingsCallback(watcher);
44+
// TODO(CareF): determine if it's running on firebase and report metric online
45+
final FrameTimingSummarizer frameTimes = FrameTimingSummarizer(frameTimings);
46+
binding.reportData = <String, dynamic>{'performance': frameTimes.summary};
47+
}
48+
49+
/// This class and summarizes a list of [FrameTiming] for the performance
50+
/// metrics.
51+
class FrameTimingSummarizer {
52+
/// Summarize `data` to frame build time and frame rasterizer time statistics.
53+
///
54+
/// See [TimelineSummary.summaryJson] for detail.
55+
factory FrameTimingSummarizer(List<FrameTiming> data) {
56+
assert(data != null);
57+
assert(data.isNotEmpty);
58+
final List<Duration> frameBuildTime = List<Duration>.unmodifiable(
59+
data.map<Duration>((FrameTiming datum) => datum.buildDuration),
60+
);
61+
final List<Duration> frameBuildTimeSorted = List<Duration>.from(frameBuildTime)..sort();
62+
final List<Duration> frameRasterizerTime = List<Duration>.unmodifiable(
63+
data.map<Duration>((FrameTiming datum) => datum.rasterDuration),
64+
);
65+
final List<Duration> frameRasterizerTimeSorted = List<Duration>.from(frameRasterizerTime)..sort();
66+
final Duration Function(Duration, Duration) add = (Duration a, Duration b) => a + b;
67+
return FrameTimingSummarizer._(
68+
frameBuildTime: frameBuildTime,
69+
frameRasterizerTime: frameRasterizerTime,
70+
// This avarage calculation is microsecond precision, which is fine
71+
// because typical values of these times are milliseconds.
72+
averageFrameBuildTime: frameBuildTime.reduce(add) ~/ data.length,
73+
p90FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.90),
74+
p99FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.99),
75+
worstFrameBuildTime: frameBuildTimeSorted.last,
76+
missedFrameBuildBudget: _countExceed(frameBuildTimeSorted, kBuildBudget),
77+
averageFrameRasterizerTime: frameRasterizerTime.reduce(add) ~/ data.length,
78+
p90FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.90),
79+
p99FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.99),
80+
worstFrameRasterizerTime: frameRasterizerTimeSorted.last,
81+
missedFrameRasterizerBudget: _countExceed(frameRasterizerTimeSorted, kBuildBudget),
82+
);
83+
}
84+
85+
const FrameTimingSummarizer._({
86+
@required this.frameBuildTime,
87+
@required this.frameRasterizerTime,
88+
@required this.averageFrameBuildTime,
89+
@required this.p90FrameBuildTime,
90+
@required this.p99FrameBuildTime,
91+
@required this.worstFrameBuildTime,
92+
@required this.missedFrameBuildBudget,
93+
@required this.averageFrameRasterizerTime,
94+
@required this.p90FrameRasterizerTime,
95+
@required this.p99FrameRasterizerTime,
96+
@required this.worstFrameRasterizerTime,
97+
@required this.missedFrameRasterizerBudget
98+
});
99+
100+
/// List of frame build time in microseconds
101+
final List<Duration> frameBuildTime;
102+
103+
/// List of frame rasterizer time in microseconds
104+
final List<Duration> frameRasterizerTime;
105+
106+
/// The average value of [frameBuildTime] in milliseconds.
107+
final Duration averageFrameBuildTime;
108+
109+
/// The 90-th percentile value of [frameBuildTime] in milliseconds
110+
final Duration p90FrameBuildTime;
111+
112+
/// The 99-th percentile value of [frameBuildTime] in milliseconds
113+
final Duration p99FrameBuildTime;
114+
115+
/// The largest value of [frameBuildTime] in milliseconds
116+
final Duration worstFrameBuildTime;
117+
118+
/// Number of items in [frameBuildTime] that's greater than [kBuildBudget]
119+
final int missedFrameBuildBudget;
120+
121+
/// The average value of [frameRasterizerTime] in milliseconds.
122+
final Duration averageFrameRasterizerTime;
123+
124+
/// The 90-th percentile value of [frameRasterizerTime] in milliseconds.
125+
final Duration p90FrameRasterizerTime;
126+
127+
/// The 99-th percentile value of [frameRasterizerTime] in milliseconds.
128+
final Duration p99FrameRasterizerTime;
129+
130+
/// The largest value of [frameRasterizerTime] in milliseconds.
131+
final Duration worstFrameRasterizerTime;
132+
133+
/// Number of items in [frameRasterizerTime] that's greater than [kBuildBudget]
134+
final int missedFrameRasterizerBudget;
135+
136+
/// Convert the summary result to a json object.
137+
///
138+
/// See [TimelineSummary.summaryJson] for detail.
139+
Map<String, dynamic> get summary => <String, dynamic>{
140+
'average_frame_build_time_millis':
141+
averageFrameBuildTime.inMicroseconds / 1E3,
142+
'90th_percentile_frame_build_time_millis':
143+
p90FrameBuildTime.inMicroseconds / 1E3,
144+
'99th_percentile_frame_build_time_millis':
145+
p99FrameBuildTime.inMicroseconds / 1E3,
146+
'worst_frame_build_time_millis':
147+
worstFrameBuildTime.inMicroseconds / 1E3,
148+
'missed_frame_build_budget_count': missedFrameBuildBudget,
149+
'average_frame_rasterizer_time_millis':
150+
averageFrameRasterizerTime.inMicroseconds / 1E3,
151+
'90th_percentile_frame_rasterizer_time_millis':
152+
p90FrameRasterizerTime.inMicroseconds / 1E3,
153+
'99th_percentile_frame_rasterizer_time_millis':
154+
p99FrameRasterizerTime.inMicroseconds / 1E3,
155+
'worst_frame_rasterizer_time_millis':
156+
worstFrameRasterizerTime.inMicroseconds / 1E3,
157+
'missed_frame_rasterizer_budget_count': missedFrameRasterizerBudget,
158+
'frame_count': frameBuildTime.length,
159+
'frame_build_times': frameBuildTime
160+
.map<int>((Duration datum) => datum.inMicroseconds).toList(),
161+
'frame_rasterizer_times': frameRasterizerTime
162+
.map<int>((Duration datum) => datum.inMicroseconds).toList(),
163+
};
164+
}
165+
166+
// The following helper functions require data sorted
167+
168+
// return the 100*p-th percentile of the data
169+
T _findPercentile<T>(List<T> data, double p) {
170+
assert(p >= 0 && p <= 1);
171+
return data[((data.length - 1) * p).round()];
172+
}
173+
174+
// return the number of items in data that > threshold
175+
int _countExceed<T extends Comparable<T>>(List<T> data, T threshold) {
176+
return data.length - data.indexWhere((T datum) => datum.compareTo(threshold) > 0);
177+
}

‎packages/e2e/pubspec.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name: e2e
22
description: Runs tests that use the flutter_test API as integration tests.
3-
version: 0.6.2+1
3+
version: 0.6.3
44
homepage: https://github.com/flutter/plugins/tree/master/packages/e2e
55

66
environment:

‎packages/e2e/test/frame_timing_summarizer_test.dart

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import 'dart:ui';
2+
3+
import 'package:flutter_test/flutter_test.dart';
4+
5+
import 'package:e2e/e2e_perf.dart';
6+
7+
void main() {
8+
test('Test FrameTimingSummarizer', () {
9+
List<int> buildTimes = <int>[
10+
for (int i = 1; i <= 100; i += 1)
11+
1000 * i,
12+
];
13+
buildTimes = buildTimes.reversed.toList();
14+
List<int> rasterTimes = <int>[
15+
for (int i = 1; i <= 100; i += 1)
16+
1000 * i + 1000,
17+
];
18+
rasterTimes = rasterTimes.reversed.toList();
19+
List<FrameTiming> inputData = <FrameTiming>[
20+
for (int i = 0; i < 100; i += 1)
21+
FrameTiming(<int>[0, buildTimes[i], 500, rasterTimes[i]]),
22+
];
23+
FrameTimingSummarizer summary = FrameTimingSummarizer(inputData);
24+
expect(summary.averageFrameBuildTime.inMicroseconds, 50500);
25+
expect(summary.p90FrameBuildTime.inMicroseconds, 90000);
26+
expect(summary.p99FrameBuildTime.inMicroseconds, 99000);
27+
expect(summary.worstFrameBuildTime.inMicroseconds, 100000);
28+
expect(summary.missedFrameBuildBudget, 84);
29+
30+
expect(summary.averageFrameRasterizerTime.inMicroseconds, 51000);
31+
expect(summary.p90FrameRasterizerTime.inMicroseconds, 90500);
32+
expect(summary.p99FrameRasterizerTime.inMicroseconds, 99500);
33+
expect(summary.worstFrameRasterizerTime.inMicroseconds, 100500);
34+
expect(summary.missedFrameRasterizerBudget, 85);
35+
expect(summary.frameBuildTime.length, 100);
36+
});
37+
}

0 commit comments

Comments
 (0)