From ab4b6513bb29e9d90b3a7e1f56b05e0657dfe1b5 Mon Sep 17 00:00:00 2001 From: Shu Chen Date: Mon, 23 Oct 2023 12:29:50 +0100 Subject: [PATCH 1/2] deps: Add dev:flutter_driver, dev:integration_test --- pubspec.lock | 39 +++++++++++++++++++++++++++++++++++++++ pubspec.yaml | 4 ++++ 2 files changed, 43 insertions(+) diff --git a/pubspec.lock b/pubspec.lock index ca6bec8ce1..1e25973755 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -414,6 +414,11 @@ packages: description: flutter source: sdk version: "0.0.0" + flutter_driver: + dependency: "direct dev" + description: flutter + source: sdk + version: "0.0.0" flutter_lints: dependency: "direct dev" description: @@ -477,6 +482,11 @@ packages: url: "https://pub.dev" source: hosted version: "3.2.0" + fuchsia_remote_debug_protocol: + dependency: transitive + description: flutter + source: sdk + version: "0.0.0" glob: dependency: transitive description: @@ -589,6 +599,11 @@ packages: url: "https://pub.dev" source: hosted version: "0.2.1+1" + integration_test: + dependency: "direct dev" + description: flutter + source: sdk + version: "0.0.0" intl: dependency: "direct main" description: @@ -821,6 +836,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.5.1" + process: + dependency: transitive + description: + name: process + sha256: "266ca5be5820feefc777793d0a583acfc8c40834893c87c00c6c09e2cf58ea42" + url: "https://pub.dev" + source: hosted + version: "5.0.1" pub_semver: dependency: transitive description: @@ -1002,6 +1025,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.2.0" + sync_http: + dependency: transitive + description: + name: sync_http + sha256: "7f0cd72eca000d2e026bcd6f990b81d0ca06022ef4e32fb257b30d3d1014a961" + url: "https://pub.dev" + source: hosted + version: "0.3.1" term_glyph: dependency: transitive description: @@ -1170,6 +1201,14 @@ packages: url: "https://pub.dev" source: hosted version: "2.4.0" + webdriver: + dependency: transitive + description: + name: webdriver + sha256: "003d7da9519e1e5f329422b36c4dcdf18d7d2978d1ba099ea4e45ba490ed845e" + url: "https://pub.dev" + source: hosted + version: "3.0.3" webkit_inspection_protocol: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index cd8c14a1ef..887691b973 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -66,8 +66,12 @@ dependencies: flutter_local_notifications: ^16.1.0 dev_dependencies: + flutter_driver: + sdk: flutter flutter_test: sdk: flutter + integration_test: + sdk: flutter # The "flutter_lints" package below contains a set of recommended lints to # encourage good coding practices. The lint set provided by the package is From 4efdc3a9ad4909122531b581ec0cfb38d429ebdb Mon Sep 17 00:00:00 2001 From: Shu Chen Date: Thu, 5 Oct 2023 14:51:40 +0100 Subject: [PATCH 2/2] test: Add integration test of _UnreadMarker animation Added an initial integration test to capture render performance of _UnreadMarker animation. This method was helpful for comparing across different implementations of the marker to see if any method was more efficient than others. The test driver `integration_test/perf_driver.dart` is derived from BSD licensed example code in Flutter documentation about integration tests profiling, see: https://docs.flutter.dev/cookbook/testing/integration/profiling#3-save-the-results-to-disk Also added a `docs/integration_tests.md` to capture notes on the process of gathering performance metrics on physical devices. --- docs/integration_tests.md | 62 ++++++++++++++++++++++++ integration_test/perf_driver.dart | 21 ++++++++ integration_test/unreadmarker_test.dart | 64 +++++++++++++++++++++++++ 3 files changed, 147 insertions(+) create mode 100644 docs/integration_tests.md create mode 100644 integration_test/perf_driver.dart create mode 100644 integration_test/unreadmarker_test.dart diff --git a/docs/integration_tests.md b/docs/integration_tests.md new file mode 100644 index 0000000000..d3497ecacf --- /dev/null +++ b/docs/integration_tests.md @@ -0,0 +1,62 @@ +# Integration Tests + +Integration tests in Flutter allow self-driving end-to-end +testing of app code running with the full GUI. + +This document is about using integration tests to capture +performance metrics on physical devices. For more +information on that topic see +[Flutter cookbook on integration profiling][profiling-cookbook]. + +For more background on integration testing in general +see [Flutter docs on integration testing][flutter-docs]. + +[profiling-cookbook]: https://docs.flutter.dev/cookbook/testing/integration/profiling +[flutter-docs]: https://docs.flutter.dev/testing/integration-tests + + +## Capturing performance metrics + +Capturing performance metrics involves two parts: an +integration test that runs on a device and driver code that +runs on the host. + +Integration test code is written in a similar style as +widget test code, using a `testWidgets` function as well as +a `WidgetTester` instance to arrange widgets and run +interactions. A difference is the usage of +`IntegrationTestWidgetsFlutterBinding` which provides a +`traceAction` method used to record Dart VM timelines. + +Driver code runs on the host and is useful to configure +output of captured timeline data. There is a baseline driver +at `integration_test/perf_driver.dart` that additionally +configures output of a timeline summary containing widget +build times and frame rendering performance. + + +## Obtaining performance metrics + +First, obtain a device ID using `flutter devices`. + +The command to run an integration test on a device: + +``` +$ flutter drive \ + --driver=integration_test/perf_driver.dart \ + --target=integration_test/unreadmarker_test.dart \ + --profile \ + --no-dds \ + -d +``` + +A data file with raw event timings will be produced in +`build/trace_output.timeline.json`. + +A more readily consumable file will also be produced in +`build/trace_output.timeline_summary.json`. This file +contains widget build and render timing data in a JSON +structure. See the fields `frame_build_times` and +`frame_rasterizer_times` as well as the provided percentile +scores of those. These values are useful for objective +comparison between different runs. diff --git a/integration_test/perf_driver.dart b/integration_test/perf_driver.dart new file mode 100644 index 0000000000..49bf21e255 --- /dev/null +++ b/integration_test/perf_driver.dart @@ -0,0 +1,21 @@ +// This integration driver configures output of timeline data +// and a summary thereof from integration tests. See +// docs/integration_tests.md for background. + +import 'package:flutter_driver/flutter_driver.dart' as driver; +import 'package:integration_test/integration_test_driver.dart'; + +Future main() { + // See cookbook recipe for this sort of driver: + // https://docs.flutter.dev/cookbook/testing/integration/profiling#3-save-the-results-to-disk + return integrationDriver( + responseDataCallback: (data) async { + if (data == null) return; + final timeline = driver.Timeline.fromJson(data['timeline']); + final summary = driver.TimelineSummary.summarize(timeline); + await summary.writeTimelineToFile( + 'trace_output', + pretty: true, + includeSummary: true); + }); +} diff --git a/integration_test/unreadmarker_test.dart b/integration_test/unreadmarker_test.dart new file mode 100644 index 0000000000..2db4222dc2 --- /dev/null +++ b/integration_test/unreadmarker_test.dart @@ -0,0 +1,64 @@ + +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart'; +import 'package:zulip/api/model/events.dart'; +import 'package:zulip/api/model/model.dart'; +import 'package:zulip/model/narrow.dart'; +import 'package:zulip/model/store.dart'; +import 'package:zulip/widgets/message_list.dart'; +import 'package:zulip/widgets/store.dart'; + +import '../test/api/fake_api.dart'; +import '../test/example_data.dart' as eg; +import '../test/model/binding.dart'; +import '../test/model/message_list_test.dart'; + +void main() { + final binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + TestZulipBinding.ensureInitialized(); + + late PerAccountStore store; + late FakeApiConnection connection; + + Future> setupMessageListPage(WidgetTester tester, int messageCount) async { + addTearDown(testBinding.reset); + await testBinding.globalStore.add(eg.selfAccount, eg.initialSnapshot()); + store = await testBinding.globalStore.perAccount(eg.selfAccount.id); + connection = store.connection as FakeApiConnection; + + // prepare message list data + final messages = List.generate(messageCount, + (i) => eg.streamMessage(flags: [MessageFlag.read])); + connection.prepare(json: + newestResult(foundOldest: true, messages: messages).toJson()); + + await tester.pumpWidget( + MaterialApp( + home: GlobalStoreWidget( + child: PerAccountStoreWidget( + accountId: eg.selfAccount.id, + child: const MessageListPage(narrow: AllMessagesNarrow()))))); + await tester.pumpAndSettle(); + return messages; + } + + testWidgets('_UnreadMarker animation performance test', (tester) async { + // This integration test is meant for measuring performance. + // See docs/integration_test.md for how to use it. + + final messages = await setupMessageListPage(tester, 500); + await binding.traceAction(() async { + store.handleEvent(eg.updateMessageFlagsRemoveEvent( + MessageFlag.read, + messages)); + await tester.pumpAndSettle(); + store.handleEvent(UpdateMessageFlagsAddEvent( + id: 1, + flag: MessageFlag.read, + messages: messages.map((e) => e.id).toList(), + all: false)); + await tester.pumpAndSettle(); + }); + }); +}