(MacOS) Add FlutterGLCompositor with support for rendering multiple layers (flutter#22782)

* Create FlutterGLCompositor.
* Add additional state to manage frame status and CALayers to FlutterGLCompositor

FlutterGLCompositor supports rendering multiple layers. The first layer is rendered using the FlutterView. Additional CALayers are created if there is more than one layer.

Platform view support will be added in following PR.

Author: RichardJCai

ci/licenses_golden/licenses_flutter

@@ -1053,6 +1053,8 @@ FILE: ../../../flutter/shell/platform/darwin/macos/framework/Headers/FlutterPlug
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Headers/FlutterViewController.h
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Info.plist
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterAppDelegate.mm
+FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterBackingStoreData.h
+FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterBackingStoreData.mm
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterDartProject.mm
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterDartProject_Internal.h
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm
@@ -1062,6 +1064,9 @@ FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterExter
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterExternalTextureGL.mm
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterFrameBufferProvider.h
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterFrameBufferProvider.mm
+FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterGLCompositor.h
+FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterGLCompositor.mm
+FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterGLCompositorUnittests.mm
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterIOSurfaceHolder.h
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterIOSurfaceHolder.mm
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterMouseCursorPlugin.h
@@ -1078,6 +1083,8 @@ FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterView.
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterView.mm
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController.mm
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm
+FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTestUtils.h
+FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTestUtils.mm
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController_Internal.h
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/MacOSGLContextSwitch.h
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/MacOSGLContextSwitch.mm

shell/platform/darwin/macos/BUILD.gn

@@ -46,6 +46,8 @@ source_set("flutter_framework_source") {
 
   sources = [
     "framework/Source/FlutterAppDelegate.mm",
+    "framework/Source/FlutterBackingStoreData.h",
+    "framework/Source/FlutterBackingStoreData.mm",
     "framework/Source/FlutterDartProject.mm",
     "framework/Source/FlutterDartProject_Internal.h",
     "framework/Source/FlutterEngine.mm",
@@ -54,6 +56,8 @@ source_set("flutter_framework_source") {
     "framework/Source/FlutterExternalTextureGL.mm",
     "framework/Source/FlutterFrameBufferProvider.h",
     "framework/Source/FlutterFrameBufferProvider.mm",
+    "framework/Source/FlutterGLCompositor.h",
+    "framework/Source/FlutterGLCompositor.mm",
     "framework/Source/FlutterIOSurfaceHolder.h",
     "framework/Source/FlutterIOSurfaceHolder.mm",
     "framework/Source/FlutterMouseCursorPlugin.h",
@@ -77,9 +81,12 @@ source_set("flutter_framework_source") {
   sources += _flutter_framework_headers
 
   deps = [
+    "//flutter/flow:flow",
+    "//flutter/fml",
     "//flutter/shell/platform/common/cpp:common_cpp_switches",
     "//flutter/shell/platform/darwin/common:framework_shared",
     "//flutter/shell/platform/embedder:embedder_as_internal_library",
+    "//third_party/skia",
   ]
 
   public_configs = [ "//flutter:config" ]
@@ -119,7 +126,10 @@ executable("flutter_desktop_darwin_unittests") {
 
   sources = [
     "framework/Source/FlutterEngineTest.mm",
+    "framework/Source/FlutterGLCompositorUnittests.mm",
     "framework/Source/FlutterViewControllerTest.mm",
+    "framework/Source/FlutterViewControllerTestUtils.h",
+    "framework/Source/FlutterViewControllerTestUtils.mm",
   ]
 
   cflags_objcc = [ "-fobjc-arc" ]

shell/platform/darwin/macos/framework/Source/FlutterBackingStoreData.h

@@ -0,0 +1,36 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterFrameBufferProvider.h"
+#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterIOSurfaceHolder.h"
+
+#import <Cocoa/Cocoa.h>
+#import <Foundation/Foundation.h>
+
+/**
+ * FlutterBackingStoreData holds data to be stored in the
+ * BackingStore's user_data.
+ */
+@interface FlutterBackingStoreData : NSObject
+
+- (nullable instancetype)initWithLayerId:(size_t)layerId
+                              fbProvider:(nonnull FlutterFrameBufferProvider*)fbProvider
+                         ioSurfaceHolder:(nonnull FlutterIOSurfaceHolder*)ioSurfaceHolder;
+
+/**
+ * The layer's key value in FlutterGLCompositor's ca_layer_map_.
+ */
+@property(nonatomic, readonly) size_t layerId;
+
+/**
+ * Provides the fbo for rendering the layer.
+ */
+@property(nonnull, nonatomic, readonly) FlutterFrameBufferProvider* frameBufferProvider;
+
+/**
+ * Contains the IOSurfaceRef with the layer contents.
+ */
+@property(nonnull, nonatomic, readonly) FlutterIOSurfaceHolder* ioSurfaceHolder;
+
+@end

shell/platform/darwin/macos/framework/Source/FlutterBackingStoreData.mm

@@ -0,0 +1,24 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterBackingStoreData.h"
+#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterFrameBufferProvider.h"
+#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterIOSurfaceHolder.h"
+
+#include <OpenGL/gl.h>
+
+@implementation FlutterBackingStoreData
+
+- (nullable instancetype)initWithLayerId:(size_t)layerId
+                              fbProvider:(nonnull FlutterFrameBufferProvider*)fbProvider
+                         ioSurfaceHolder:(nonnull FlutterIOSurfaceHolder*)ioSurfaceHolder {
+  if (self = [super init]) {
+    _layerId = layerId;
+    _frameBufferProvider = fbProvider;
+    _ioSurfaceHolder = ioSurfaceHolder;
+  }
+  return self;
+}
+
+@end

shell/platform/darwin/macos/framework/Source/FlutterEngine.mm

@@ -10,7 +10,7 @@
 
 #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterDartProject_Internal.h"
 #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterExternalTextureGL.h"
-#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController_Internal.h"
+#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterGLCompositor.h"
 #import "flutter/shell/platform/embedder/embedder.h"
 
 /**
@@ -197,6 +197,13 @@ @implementation FlutterEngine {
 
   // Pointer to the Dart AOT snapshot and instruction data.
   _FlutterEngineAOTData* _aotData;
+
+  // _macOSGLCompositor is created when the engine is created and
+  // it's destruction is handled by ARC when the engine is destroyed.
+  std::unique_ptr<flutter::FlutterGLCompositor> _macOSGLCompositor;
+
+  // FlutterCompositor is copied and used in embedder.cc.
+  FlutterCompositor _compositor;
 }
 
 - (instancetype)initWithName:(NSString*)labelPrefix project:(FlutterDartProject*)project {
@@ -306,6 +313,8 @@ - (BOOL)runWithEntrypoint:(NSString*)entrypoint {
     flutterArguments.aot_data = _aotData;
   }
 
+  flutterArguments.compositor = [self createFlutterCompositor];
+
   FlutterEngineResult result = _embedderAPI.Initialize(
       FLUTTER_ENGINE_VERSION, &rendererConfig, &flutterArguments, (__bridge void*)(self), &_engine);
   if (result != kSuccess) {
@@ -360,6 +369,54 @@ - (void)setViewController:(FlutterViewController*)controller {
   }
 }
 
+- (FlutterCompositor*)createFlutterCompositor {
+  // TODO(richardjcai): Add support for creating a FlutterGLCompositor
+  // with a nil _viewController for headless engines.
+  // https://github.com/flutter/flutter/issues/71606
+  if (_viewController == nullptr) {
+    return nullptr;
+  }
+
+  [_mainOpenGLContext makeCurrentContext];
+
+  _macOSGLCompositor = std::make_unique<flutter::FlutterGLCompositor>(_viewController);
+
+  _compositor = {};
+  _compositor.struct_size = sizeof(FlutterCompositor);
+  _compositor.user_data = _macOSGLCompositor.get();
+
+  _compositor.create_backing_store_callback = [](const FlutterBackingStoreConfig* config,  //
+                                                 FlutterBackingStore* backing_store_out,   //
+                                                 void* user_data                           //
+                                              ) {
+    return reinterpret_cast<flutter::FlutterGLCompositor*>(user_data)->CreateBackingStore(
+        config, backing_store_out);
+  };
+
+  _compositor.collect_backing_store_callback = [](const FlutterBackingStore* backing_store,  //
+                                                  void* user_data                            //
+                                               ) {
+    return reinterpret_cast<flutter::FlutterGLCompositor*>(user_data)->CollectBackingStore(
+        backing_store);
+  };
+
+  _compositor.present_layers_callback = [](const FlutterLayer** layers,  //
+                                           size_t layers_count,          //
+                                           void* user_data               //
+                                        ) {
+    return reinterpret_cast<flutter::FlutterGLCompositor*>(user_data)->Present(layers,
+                                                                               layers_count);
+  };
+
+  __weak FlutterEngine* weak_self = self;
+  _macOSGLCompositor->SetPresentCallback(
+      [weak_self]() { return [weak_self engineCallbackOnPresent]; });
+
+  _compositor.avoid_backing_store_cache = true;
+
+  return &_compositor;
+}
+
 - (id<FlutterBinaryMessenger>)binaryMessenger {
   // TODO(stuartmorgan): Switch to FlutterBinaryMessengerRelay to avoid plugins
   // keeping the engine alive.

shell/platform/darwin/macos/framework/Source/FlutterFrameBufferProvider.h

@@ -9,7 +9,7 @@
  */
 @interface FlutterFrameBufferProvider : NSObject
 
-- (nullable instancetype)initWithOpenGLContext:(nonnull NSOpenGLContext*)opengLContext;
+- (nullable instancetype)initWithOpenGLContext:(nonnull const NSOpenGLContext*)opengLContext;
 
 /**
  * Returns the id of the framebuffer.

shell/platform/darwin/macos/framework/Source/FlutterFrameBufferProvider.mm

@@ -15,7 +15,7 @@ @interface FlutterFrameBufferProvider () {
 @end
 
 @implementation FlutterFrameBufferProvider
-- (instancetype)initWithOpenGLContext:(NSOpenGLContext*)openGLContext {
+- (instancetype)initWithOpenGLContext:(const NSOpenGLContext*)openGLContext {
   if (self = [super init]) {
     MacOSGLContextSwitch context_switch(openGLContext);
 

shell/platform/darwin/macos/framework/Source/FlutterGLCompositor.h

@@ -0,0 +1,80 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <map>
+
+#include "flutter/fml/macros.h"
+#include "flutter/shell/platform/darwin/macos/framework/Source/FlutterSurfaceManager.h"
+#include "flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController_Internal.h"
+#include "flutter/shell/platform/embedder/embedder.h"
+#include "third_party/skia/include/gpu/GrDirectContext.h"
+
+namespace flutter {
+
+// FlutterGLCompositor creates and manages the backing stores used for
+// rendering Flutter content and presents Flutter content and Platform views.
+// Platform views are not yet supported.
+// FlutterGLCompositor is created and destroyed by FlutterEngine.
+class FlutterGLCompositor {
+ public:
+  FlutterGLCompositor(FlutterViewController* view_controller);
+
+  // Creates a BackingStore and saves updates the backing_store_out
+  // data with the new BackingStore data.
+  // If the backing store is being requested for the first time
+  // for a given frame, we do not create a new backing store but
+  // rather return the backing store associated with the
+  // FlutterView's FlutterSurfaceManager.
+  //
+  // Any additional state allocated for the backing store and
+  // saved as user_data in the backing store must be collected
+  // in the backing_store's desctruction_callback field which will
+  // be called when the embedder collects the backing store.
+  bool CreateBackingStore(const FlutterBackingStoreConfig* config,
+                          FlutterBackingStore* backing_store_out);
+
+  // Releases the memory for any state used by the backing store.
+  bool CollectBackingStore(const FlutterBackingStore* backing_store);
+
+  // Presents the FlutterLayers by updating FlutterView(s) using the
+  // layer content.
+  // Present sets frame_started_ to false.
+  bool Present(const FlutterLayer** layers, size_t layers_count);
+
+  using PresentCallback = std::function<bool()>;
+
+  // PresentCallback is called at the end of the Present function.
+  void SetPresentCallback(const PresentCallback& present_callback);
+
+ private:
+  const FlutterViewController* view_controller_;
+  const NSOpenGLContext* open_gl_context_;
+  PresentCallback present_callback_;
+
+  // Count for how many CALayers have been created for a frame.
+  // Resets when a frame is finished.
+  // ca_layer_count_ is also used as a layerId.
+  size_t ca_layer_count_ = 0;
+
+  // Maps a layer_id (size_t) to a CALayer.
+  // The layer_id starts at 0 for a given frame
+  // and increments by 1 for each new CALayer.
+  std::map<size_t, CALayer*> ca_layer_map_;
+
+  // frame_started_ keeps track of if a layer has been
+  // created for the frame.
+  bool frame_started_ = false;
+
+  // Set frame_started_ to true and reset all layer state.
+  void StartFrame();
+
+  // Creates a CALayer and adds it to ca_layer_map_ and increments
+  // ca_layer_count_; Returns the key value (size_t) for the layer in
+  // ca_layer_map_.
+  size_t CreateCALayer();
+
+  FML_DISALLOW_COPY_AND_ASSIGN(FlutterGLCompositor);
+};
+
+}  // namespace flutter