Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Commit 8682e51

Browse files
[canvaskit] Further improve overlay optimization by splitting pictures (#54878)
This enhances the overlay optimization by delaying combining pictures to get tighter bounds for the pictures that make up the scene, enabling more sophisticated optimization since we can determine if they intersect with platform views on a per-picture basis. Fixes flutter/flutter#149863 On a Macbook in Chrome in an example app with an infinite scrolling grid of platform views, this brings the ratio of dropped frames from 93% to 55% (roughly 4 fps to 30 fps). [C++, Objective-C, Java style guides]: https://github.com/flutter/engine/blob/main/CONTRIBUTING.md#style
1 parent 82f0224 commit 8682e51

File tree

10 files changed

+1005
-561
lines changed

10 files changed

+1005
-561
lines changed

ci/licenses_golden/licenses_flutter

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43589,6 +43589,7 @@ ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.da
4358943589
ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/layer.dart + ../../../flutter/LICENSE
4359043590
ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/layer_scene_builder.dart + ../../../flutter/LICENSE
4359143591
ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/layer_tree.dart + ../../../flutter/LICENSE
43592+
ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/layer_visitor.dart + ../../../flutter/LICENSE
4359243593
ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/mask_filter.dart + ../../../flutter/LICENSE
4359343594
ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/multi_surface_rasterizer.dart + ../../../flutter/LICENSE
4359443595
ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/n_way_canvas.dart + ../../../flutter/LICENSE
@@ -46469,6 +46470,7 @@ FILE: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart
4646946470
FILE: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/layer.dart
4647046471
FILE: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/layer_scene_builder.dart
4647146472
FILE: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/layer_tree.dart
46473+
FILE: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/layer_visitor.dart
4647246474
FILE: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/mask_filter.dart
4647346475
FILE: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/multi_surface_rasterizer.dart
4647446476
FILE: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/n_way_canvas.dart

lib/web_ui/lib/src/engine.dart

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ export 'engine/canvaskit/image_web_codecs.dart';
3333
export 'engine/canvaskit/layer.dart';
3434
export 'engine/canvaskit/layer_scene_builder.dart';
3535
export 'engine/canvaskit/layer_tree.dart';
36+
export 'engine/canvaskit/layer_visitor.dart';
3637
export 'engine/canvaskit/mask_filter.dart';
3738
export 'engine/canvaskit/multi_surface_rasterizer.dart';
3839
export 'engine/canvaskit/n_way_canvas.dart';

lib/web_ui/lib/src/engine/canvaskit/embedded_views.dart

Lines changed: 135 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import '../svg.dart';
1414
import '../util.dart';
1515
import '../vector_math.dart';
1616
import 'canvas.dart';
17+
import 'layer.dart';
1718
import 'overlay_scene_optimizer.dart';
1819
import 'painting.dart';
1920
import 'path.dart';
@@ -66,6 +67,9 @@ class HtmlViewEmbedder {
6667
/// Returns the most recent rendering. Only used in tests.
6768
Rendering get debugActiveRendering => _activeRendering;
6869

70+
/// If [debugOverlayOptimizationBounds] is true, this canvas will draw
71+
/// semitransparent rectangles showing the computed bounds of the platform
72+
/// views and pictures in the scene.
6973
DisplayCanvas? debugBoundsCanvas;
7074

7175
/// The size of the frame, in physical pixels.
@@ -75,27 +79,23 @@ class HtmlViewEmbedder {
7579
_frameSize = size;
7680
}
7781

78-
/// Returns a list of canvases which will be overlaid on top of the "base"
79-
/// canvas after a platform view is composited into the scene.
80-
///
81-
/// The engine asks for the overlay canvases immediately before the paint
82-
/// phase, after the preroll phase. In the preroll phase we must be
83-
/// conservative and assume that every platform view which is prerolled is
84-
/// also composited, and therefore requires an overlay canvas. However, not
85-
/// every platform view which is prerolled ends up being composited (it may be
86-
/// clipped out and not actually drawn). This means that we may end up
87-
/// overallocating canvases. This isn't a problem in practice, however, as
88-
/// unused recording canvases are simply deleted at the end of the frame.
89-
Iterable<CkCanvas> getOverlayCanvases() {
90-
return _context.pictureRecordersCreatedDuringPreroll
82+
/// Returns a list of recording canvases which the pictures in the upcoming
83+
/// paint step will be drawn into. These recording canvases are combined into
84+
/// an N-way canvas for the rasterizer to record clip and transform operations
85+
/// during the measure step.
86+
Iterable<CkCanvas> getPictureCanvases() {
87+
return _context.measuringPictureRecorders.values
9188
.map((CkPictureRecorder r) => r.recordingCanvas!);
9289
}
9390

94-
void prerollCompositeEmbeddedView(int viewId, EmbeddedViewParams params) {
95-
final CkPictureRecorder pictureRecorder = CkPictureRecorder();
96-
pictureRecorder.beginRecording(ui.Offset.zero & _frameSize.toSize());
97-
_context.pictureRecordersCreatedDuringPreroll.add(pictureRecorder);
91+
/// Returns a list of canvases for the optimized rendering. These are used in
92+
/// the paint step.
93+
Iterable<CkCanvas> getOptimizedCanvases() {
94+
return _context.optimizedCanvasRecorders!
95+
.map((CkPictureRecorder r) => r.recordingCanvas!);
96+
}
9897

98+
void prerollCompositeEmbeddedView(int viewId, EmbeddedViewParams params) {
9999
// Do nothing if the params didn't change.
100100
if (_currentCompositionParams[viewId] == params) {
101101
// If the view was prerolled but not composited, then it needs to be
@@ -109,30 +109,38 @@ class HtmlViewEmbedder {
109109
_viewsToRecomposite.add(viewId);
110110
}
111111

112+
/// Record that a picture recorder is needed for [picture] to be measured.
113+
void prerollPicture(PictureLayer picture) {
114+
final CkPictureRecorder pictureRecorder = CkPictureRecorder();
115+
pictureRecorder.beginRecording(ui.Offset.zero & _frameSize.toSize());
116+
_context.measuringPictureRecorders[picture] = pictureRecorder;
117+
}
118+
119+
/// Returns the canvas that was created to measure [picture].
120+
CkCanvas getMeasuringCanvasFor(PictureLayer picture) {
121+
return _context.measuringPictureRecorders[picture]!.recordingCanvas!;
122+
}
123+
124+
/// Adds the picture recorder associated with [picture] to the unoptimized
125+
/// scene.
126+
void addPictureToUnoptimizedScene(PictureLayer picture) {
127+
final CkPictureRecorder recorder =
128+
_context.measuringPictureRecorders[picture]!;
129+
_context.sceneElements.add(PictureSceneElement(picture, recorder));
130+
}
131+
112132
/// Prepares to composite [viewId].
113-
///
114-
/// If this returns a [CkCanvas], then that canvas should be the new leaf
115-
/// node. Otherwise, keep the same leaf node.
116-
CkCanvas? compositeEmbeddedView(int viewId) {
133+
void compositeEmbeddedView(int viewId) {
117134
// Ensure platform view with `viewId` is injected into the `rasterizer.view`.
118135
rasterizer.view.dom.injectPlatformView(viewId);
119136

120-
final int overlayIndex = _context.viewCount;
121137
_compositionOrder.add(viewId);
122-
_context.viewCount++;
123-
124-
CkPictureRecorder? recorderToUseForRendering;
125-
if (overlayIndex < _context.pictureRecordersCreatedDuringPreroll.length) {
126-
recorderToUseForRendering =
127-
_context.pictureRecordersCreatedDuringPreroll[overlayIndex];
128-
_context.pictureRecorders.add(recorderToUseForRendering);
129-
}
138+
_context.sceneElements.add(PlatformViewSceneElement(viewId));
130139

131140
if (_viewsToRecomposite.contains(viewId)) {
132141
_compositeWithParams(viewId, _currentCompositionParams[viewId]!);
133142
_viewsToRecomposite.remove(viewId);
134143
}
135-
return recorderToUseForRendering?.recordingCanvas;
136144
}
137145

138146
void _compositeWithParams(int platformViewId, EmbeddedViewParams params) {
@@ -355,14 +363,54 @@ class HtmlViewEmbedder {
355363
sceneHost.append(_svgPathDefs!);
356364
}
357365

358-
Future<void> submitFrame(CkPicture basePicture) async {
359-
final List<CkPicture> pictures = <CkPicture>[basePicture];
360-
for (final CkPictureRecorder recorder in _context.pictureRecorders) {
361-
pictures.add(recorder.endRecording());
362-
}
366+
/// Optimizes the scene to use the fewest possible canvases. This sets up
367+
/// the final paint pass to paint the pictures into the optimized canvases.
368+
void optimizeRendering() {
369+
final Map<CkPicture, PictureLayer> scenePictureToRawPicture =
370+
<CkPicture, PictureLayer>{};
371+
final Iterable<SceneElement> unoptimizedRendering =
372+
_context.sceneElements.map<SceneElement>((SceneElement element) {
373+
if (element is PictureSceneElement) {
374+
final CkPicture scenePicture = element.pictureRecorder.endRecording();
375+
element.scenePicture = scenePicture;
376+
scenePictureToRawPicture[scenePicture] = element.picture;
377+
return element;
378+
} else {
379+
return element;
380+
}
381+
});
363382
Rendering rendering = createOptimizedRendering(
364-
pictures, _compositionOrder, _currentCompositionParams);
383+
unoptimizedRendering, _currentCompositionParams);
365384
rendering = _modifyRenderingForMaxCanvases(rendering);
385+
_context.optimizedRendering = rendering;
386+
// Create new picture recorders for the optimized render canvases and record
387+
// which pictures go in which canvas.
388+
final List<CkPictureRecorder> optimizedCanvasRecorders =
389+
<CkPictureRecorder>[];
390+
final Map<PictureLayer, CkPictureRecorder> pictureToOptimizedCanvasMap =
391+
<PictureLayer, CkPictureRecorder>{};
392+
for (final RenderingRenderCanvas renderCanvas in rendering.canvases) {
393+
final CkPictureRecorder pictureRecorder = CkPictureRecorder();
394+
pictureRecorder.beginRecording(ui.Offset.zero & _frameSize.toSize());
395+
optimizedCanvasRecorders.add(pictureRecorder);
396+
for (final CkPicture picture in renderCanvas.pictures) {
397+
pictureToOptimizedCanvasMap[scenePictureToRawPicture[picture]!] =
398+
pictureRecorder;
399+
}
400+
}
401+
_context.optimizedCanvasRecorders = optimizedCanvasRecorders;
402+
_context.pictureToOptimizedCanvasMap = pictureToOptimizedCanvasMap;
403+
}
404+
405+
/// Returns the canvas that this picture layer should draw into in the
406+
/// optimized scene.
407+
CkCanvas getOptimizedCanvasFor(PictureLayer picture) {
408+
assert(_context.optimizedRendering != null);
409+
return _context.pictureToOptimizedCanvasMap![picture]!.recordingCanvas!;
410+
}
411+
412+
Future<void> submitFrame() async {
413+
final Rendering rendering = _context.optimizedRendering!;
366414
_updateDomForNewRendering(rendering);
367415
if (rendering.equalsForRendering(_activeRendering)) {
368416
// Copy the display canvases to the new rendering.
@@ -375,13 +423,17 @@ class HtmlViewEmbedder {
375423
_activeRendering = rendering;
376424

377425
final List<RenderingRenderCanvas> renderCanvases = rendering.canvases;
426+
int renderCanvasIndex = 0;
378427
for (final RenderingRenderCanvas renderCanvas in renderCanvases) {
428+
final CkPicture renderPicture = _context
429+
.optimizedCanvasRecorders![renderCanvasIndex++]
430+
.endRecording();
379431
await rasterizer.rasterizeToCanvas(
380-
renderCanvas.displayCanvas!, renderCanvas.pictures);
432+
renderCanvas.displayCanvas!, <CkPicture>[renderPicture]);
381433
}
382434

383435
for (final CkPictureRecorder recorder
384-
in _context.pictureRecordersCreatedDuringPreroll) {
436+
in _context.measuringPictureRecorders.values) {
385437
if (recorder.isRecording) {
386438
recorder.endRecording();
387439
}
@@ -393,11 +445,11 @@ class HtmlViewEmbedder {
393445
debugBoundsCanvas ??= rasterizer.displayFactory.getCanvas();
394446
final CkPictureRecorder boundsRecorder = CkPictureRecorder();
395447
final CkCanvas boundsCanvas = boundsRecorder.beginRecording(
396-
ui.Rect.fromLTWH(
397-
0,
398-
0,
399-
_frameSize.width.toDouble(),
400-
_frameSize.height.toDouble(),
448+
ui.Rect.fromLTWH(
449+
0,
450+
0,
451+
_frameSize.width.toDouble(),
452+
_frameSize.height.toDouble(),
401453
),
402454
);
403455
final CkPaint platformViewBoundsPaint = CkPaint()
@@ -903,20 +955,45 @@ class MutatorsStack extends Iterable<Mutator> {
903955
Iterable<Mutator> get reversed => _mutators;
904956
}
905957

906-
/// The state for the current frame.
907-
class EmbedderFrameContext {
908-
/// Picture recorders which were created during the preroll phase.
909-
///
910-
/// These picture recorders will be "claimed" in the paint phase by platform
911-
/// views being composited into the scene.
912-
final List<CkPictureRecorder> pictureRecordersCreatedDuringPreroll =
913-
<CkPictureRecorder>[];
958+
sealed class SceneElement {}
914959

915-
/// Picture recorders which were actually used in the paint phase.
916-
///
917-
/// This is a subset of [_pictureRecordersCreatedDuringPreroll].
918-
final List<CkPictureRecorder> pictureRecorders = <CkPictureRecorder>[];
960+
class PictureSceneElement extends SceneElement {
961+
PictureSceneElement(this.picture, this.pictureRecorder);
962+
963+
final PictureLayer picture;
964+
final CkPictureRecorder pictureRecorder;
965+
966+
/// The picture as it would be painted in the final scene, with clips and
967+
/// transforms applied. This is set by [optimizeRendering].
968+
CkPicture? scenePicture;
969+
}
919970

920-
/// The number of platform views in this frame.
921-
int viewCount = 0;
971+
class PlatformViewSceneElement extends SceneElement {
972+
PlatformViewSceneElement(this.viewId);
973+
974+
final int viewId;
975+
}
976+
977+
/// The state for the current frame.
978+
class EmbedderFrameContext {
979+
/// Picture recorders which were created d the final bounds of the picture in the scene.
980+
final Map<PictureLayer, CkPictureRecorder> measuringPictureRecorders =
981+
<PictureLayer, CkPictureRecorder>{};
982+
983+
/// List of picture recorders and platform view ids in the order they were
984+
/// painted.
985+
final List<SceneElement> sceneElements = <SceneElement>[];
986+
987+
/// The optimized rendering for this frame. This is set by calling
988+
/// [optimizeRendering].
989+
Rendering? optimizedRendering;
990+
991+
/// The picture recorders for the optimized rendering. This is set by calling
992+
/// [optimizeRendering].
993+
List<CkPictureRecorder>? optimizedCanvasRecorders;
994+
995+
/// A map from the original PictureLayer to the picture recorder it should go
996+
/// into in the optimized rendering. This is set by calling
997+
/// [optimizedRendering].
998+
Map<PictureLayer, CkPictureRecorder>? pictureToOptimizedCanvasMap;
922999
}

0 commit comments

Comments
 (0)