[iOS] Full keyboard access scrolling (flutter/engine#56606)

This PR adds basic FKA scrolling support: when the iOS focus (the focus state is maintained separately from the framework focus, see the previous PR) switches to an item in a scrollable container that is too close to the edge of the viewport, the container will scroll to make sure the next item is visible. 

Previous PR for context: flutter/engine#55964

https://github.com/user-attachments/assets/84ae5153-f955-4d23-9901-ce942c0e98ac

### Why the UIScrollView subclass in the focus hierarchy

The iOS focus system does not provide an API that allows apps to notify it of focus highlight changes. So if we were to keep using the transforms sent by the framework as-is and not introducing any UIViews in the focus hierarchy, the focus highlight will be positioned at the wrong location after scrolling (via FKA or via framework). That does not seem to be part of the public API and the focus system seems to only know how to properly highlight focusable UIViews.

### Things that currently may not work

1. Nested scroll views (have not tried to verify) 

The `UIScrollView`s are always subviews of the `FlutterView`. If there are nested scrollables the focus system may not be able to properly determine the focus hierarchy (in theory the iOS focus system should never depend on `UIView.parentView` but I haven't tried to verify that).

2. If the next item is too far below the bottom of the screen and there is a tab bar with focusable items, the focus will be transferred to tab bar instead of the next item in the list

Video demo (as you can see the scrolling is really finicky):

https://github.com/user-attachments/assets/51c2bfe4-d7b3-4614-aa49-4256214f8978

I've tried doing the same thing using a `UITableView` with similar configurations but it seems to have the same problem. I'll try to dig a bit deeper into this and see if there's a workaround.

[C++, Objective-C, Java style guides]: https://github.com/flutter/engine/blob/main/CONTRIBUTING.md#style

Author: LongCatIsLooong

engine/src/flutter/lib/ui/semantics.dart

@@ -45,6 +45,7 @@ class SemanticsAction {
   static const int _kMoveCursorBackwardByWordIndex = 1 << 20;
   static const int _kSetTextIndex = 1 << 21;
   static const int _kFocusIndex = 1 << 22;
+  static const int _kScrollToOffsetIndex = 1 << 23;
   // READ THIS: if you add an action here, you MUST update the
   // numSemanticsActions value in testing/dart/semantics_test.dart and
   // lib/web_ui/test/engine/semantics/semantics_api_test.dart, or tests
@@ -86,6 +87,17 @@ class SemanticsAction {
   /// scrollable.
   static const SemanticsAction scrollDown = SemanticsAction._(_kScrollDownIndex, 'scrollDown');
 
+  /// A request to scroll the scrollable container to a given scroll offset.
+  ///
+  /// The payload of this [SemanticsAction] is a flutter-standard-encoded
+  /// [Float64List] of length 2 containing the target horizontal and vertical
+  /// offsets (in logical pixels) the receiving scrollable container should
+  /// scroll to.
+  ///
+  /// This action is used by iOS Full Keyboard Access to reveal contents that
+  /// are currently not visible in the viewport.
+  static const SemanticsAction scrollToOffset = SemanticsAction._(_kScrollToOffsetIndex, 'scrollToOffset');
+
   /// A request to increase the value represented by the semantics node.
   ///
   /// For example, this action might be recognized by a slider control.
@@ -265,6 +277,7 @@ class SemanticsAction {
     _kScrollRightIndex: scrollRight,
     _kScrollUpIndex: scrollUp,
     _kScrollDownIndex: scrollDown,
+    _kScrollToOffsetIndex: scrollToOffset,
     _kIncreaseIndex: increase,
     _kDecreaseIndex: decrease,
     _kShowOnScreenIndex: showOnScreen,
@@ -764,7 +777,7 @@ base class LocaleStringAttribute extends StringAttribute {
     _initLocaleStringAttribute(this, range.start, range.end, locale.toLanguageTag());
   }
 
-  /// The lanuage of this attribute.
+  /// The language of this attribute.
   final Locale locale;
 
   @Native<Void Function(Handle, Int32, Int32, Handle)>(symbol: 'NativeStringAttribute::initLocaleStringAttribute')

engine/src/flutter/lib/ui/semantics/semantics_node.h

@@ -43,6 +43,7 @@ enum class SemanticsAction : int32_t {
   kMoveCursorBackwardByWord = 1 << 20,
   kSetText = 1 << 21,
   kFocus = 1 << 22,
+  kScrollToOffset = 1 << 23,
 };
 
 const int kVerticalScrollSemanticsActions =

engine/src/flutter/lib/web_ui/lib/semantics.dart

@@ -33,13 +33,15 @@ class SemanticsAction {
   static const int _kMoveCursorBackwardByWordIndex = 1 << 20;
   static const int _kSetTextIndex = 1 << 21;
   static const int _kFocusIndex = 1 << 22;
+  static const int _kScrollToOffsetIndex = 1 << 23;
 
   static const SemanticsAction tap = SemanticsAction._(_kTapIndex, 'tap');
   static const SemanticsAction longPress = SemanticsAction._(_kLongPressIndex, 'longPress');
   static const SemanticsAction scrollLeft = SemanticsAction._(_kScrollLeftIndex, 'scrollLeft');
   static const SemanticsAction scrollRight = SemanticsAction._(_kScrollRightIndex, 'scrollRight');
   static const SemanticsAction scrollUp = SemanticsAction._(_kScrollUpIndex, 'scrollUp');
   static const SemanticsAction scrollDown = SemanticsAction._(_kScrollDownIndex, 'scrollDown');
+  static const SemanticsAction scrollToOffset = SemanticsAction._(_kScrollToOffsetIndex, 'scrollToOffset');
   static const SemanticsAction increase = SemanticsAction._(_kIncreaseIndex, 'increase');
   static const SemanticsAction decrease = SemanticsAction._(_kDecreaseIndex, 'decrease');
   static const SemanticsAction showOnScreen = SemanticsAction._(_kShowOnScreenIndex, 'showOnScreen');
@@ -65,6 +67,7 @@ class SemanticsAction {
     _kScrollRightIndex: scrollRight,
     _kScrollUpIndex: scrollUp,
     _kScrollDownIndex: scrollDown,
+    _kScrollToOffsetIndex: scrollToOffset,
     _kIncreaseIndex: increase,
     _kDecreaseIndex: decrease,
     _kShowOnScreenIndex: showOnScreen,

engine/src/flutter/lib/web_ui/test/engine/semantics/semantics_api_test.dart

@@ -29,7 +29,7 @@ void testMain() {
   });
 
   // This must match the number of actions in lib/ui/semantics.dart
-  const int numSemanticsActions = 23;
+  const int numSemanticsActions = 24;
   test('SemanticsAction.values refers to all actions.', () async {
     expect(SemanticsAction.values.length, equals(numSemanticsActions));
     for (int index = 0; index < numSemanticsActions; ++index) {

engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java

@@ -2120,7 +2120,8 @@ public enum Action {
     MOVE_CURSOR_FORWARD_BY_WORD(1 << 19),
     MOVE_CURSOR_BACKWARD_BY_WORD(1 << 20),
     SET_TEXT(1 << 21),
-    FOCUS(1 << 22);
+    FOCUS(1 << 22),
+    SCROLL_TO_OFFSET(1 << 23);
 
     public final int value;
 

engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterSemanticsScrollView.h

@@ -18,10 +18,30 @@ NS_ASSUME_NONNULL_BEGIN
  * sends all of selector calls from accessibility services to the
  * owner SemanticsObject.
  */
-@interface FlutterSemanticsScrollView : UIScrollView
+@interface FlutterSemanticsScrollView : UIScrollView <UIScrollViewDelegate>
 
 @property(nonatomic, weak, nullable) SemanticsObject* semanticsObject;
 
+/// Whether this scroll view's content offset is actively being updated by UIKit
+/// or other the system services.
+///
+/// This flag is set by the `FlutterSemanticsScrollView` itself, typically in
+/// one of the `UIScrollViewDelegate` methods.
+///
+/// When this flag is true, the `SemanticsObject` implementation ignores all
+/// content offset updates coming from the Flutter framework, to prevent
+/// potential feedback loops (especially when the framework is only echoing
+/// the new content offset back to this scroll view).
+///
+/// For example, to scroll a scrollable container with iOS full keyboard access,
+/// the iOS focus system uses a display link to scroll the container to the
+/// desired offset animatedly. If the user changes the scroll offset during the
+/// animation, the display link will be invalidated and the scrolling animation
+/// will be interrupted. For simplicity, content offset updates coming from the
+/// framework will be ignored in the relatively short animation duration (~1s),
+/// allowing the scrolling animation to finish.
+@property(nonatomic, readonly) BOOL isDoingSystemScrolling;
+
 - (instancetype)init NS_UNAVAILABLE;
 - (instancetype)initWithFrame:(CGRect)frame NS_UNAVAILABLE;
 - (instancetype)initWithCoder:(NSCoder*)coder NS_UNAVAILABLE;

engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterSemanticsScrollView.mm

@@ -15,6 +15,8 @@ - (instancetype)initWithSemanticsObject:(SemanticsObject*)semanticsObject {
   self = [super initWithFrame:CGRectZero];
   if (self) {
     _semanticsObject = semanticsObject;
+    _isDoingSystemScrolling = NO;
+    self.delegate = self;
   }
   return self;
 }
@@ -105,4 +107,14 @@ - (NSInteger)accessibilityElementCount {
   return self.semanticsObject.children.count;
 }
 
+- (void)scrollViewWillEndDragging:(UIScrollView*)scrollView
+                     withVelocity:(CGPoint)velocity
+              targetContentOffset:(inout CGPoint*)targetContentOffset {
+  _isDoingSystemScrolling = YES;
+}
+
+- (void)scrollViewDidEndDecelerating:(UIScrollView*)scrollView {
+  _isDoingSystemScrolling = NO;
+}
+
 @end

engine/src/flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm

@@ -3,7 +3,10 @@
 // found in the LICENSE file.
 
 #import "SemanticsObject.h"
+#include "flutter/lib/ui/semantics/semantics_node.h"
+#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterCodecs.h"
 #import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h"
+#import "flutter/shell/platform/darwin/ios/framework/Source/FlutterSemanticsScrollView.h"
 
 FLUTTER_ASSERT_ARC
 
@@ -27,10 +30,19 @@
 // translated to calls such as -[NSObject accessibilityActivate]), while most
 // other key events are dispatched to the framework.
 @interface SemanticsObject (UIFocusSystem) <UIFocusItem, UIFocusItemContainer>
+/// The `UIFocusItem` that represents this SemanticsObject.
+///
+///  For regular `SemanticsObject`s, this method returns `self`,
+///  for `FlutterScrollableSemanticsObject`s, this method returns its scroll view.
+- (id<UIFocusItem>)focusItem;
 @end
 
 @implementation SemanticsObject (UIFocusSystem)
 
+- (id<UIFocusItem>)focusItem {
+  return self;
+}
+
 #pragma mark - UIFocusEnvironment Conformance
 
 - (void)setNeedsFocusUpdate {
@@ -49,7 +61,7 @@ - (void)didUpdateFocusInContext:(UIFocusUpdateContext*)context
 
 - (id<UIFocusEnvironment>)parentFocusEnvironment {
   // The root SemanticsObject node's parent is the FlutterView.
-  return self.parent ?: self.bridge->view();
+  return self.parent.focusItem ?: self.bridge->view();
 }
 
 - (NSArray<id<UIFocusEnvironment>>*)preferredFocusEnvironments {
@@ -71,8 +83,57 @@ - (BOOL)canBecomeFocused {
   return self.node.HasAction(flutter::SemanticsAction::kTap);
 }
 
+// The frame is described in the `coordinateSpace` of the
+// `parentFocusEnvironment` (all `parentFocusEnvironment`s are `UIFocusItem`s).
+//
+// See also the `coordinateSpace` implementation.
+// TODO(LongCatIsLooong): use CoreGraphics types.
 - (CGRect)frame {
-  return self.accessibilityFrame;
+  SkPoint quad[4] = {SkPoint::Make(self.node.rect.left(), self.node.rect.top()),
+                     SkPoint::Make(self.node.rect.left(), self.node.rect.bottom()),
+                     SkPoint::Make(self.node.rect.right(), self.node.rect.top()),
+                     SkPoint::Make(self.node.rect.right(), self.node.rect.bottom())};
+
+  SkM44 transform = self.node.transform;
+  FlutterSemanticsScrollView* scrollView;
+  for (SemanticsObject* ancestor = self.parent; ancestor; ancestor = ancestor.parent) {
+    if ([ancestor isKindOfClass:[FlutterScrollableSemanticsObject class]]) {
+      scrollView = ((FlutterScrollableSemanticsObject*)ancestor).scrollView;
+      break;
+    }
+    transform = ancestor.node.transform * transform;
+  }
+
+  for (auto& vertex : quad) {
+    SkV4 vector = transform.map(vertex.x(), vertex.y(), 0, 1);
+    vertex = SkPoint::Make(vector.x / vector.w, vector.y / vector.w);
+  }
+
+  SkRect rect;
+  rect.setBounds(quad, 4);
+  // If this UIFocusItemContainer's coordinateSpace is a UIScrollView, offset
+  // the rect by `contentOffset` because the contentOffset translation is
+  // incorporated into the paint transform at different node depth in UIKit
+  // and Flutter. In Flutter, the translation is added to the cells
+  // while in UIKit the viewport's bounds is manipulated (IOW, each cell's frame
+  // in the UIScrollView coordinateSpace does not change when the UIScrollView
+  // scrolls).
+  CGRect unscaledRect =
+      CGRectMake(rect.x() + scrollView.bounds.origin.x, rect.y() + scrollView.bounds.origin.y,
+                 rect.width(), rect.height());
+  if (scrollView) {
+    return unscaledRect;
+  }
+  // `rect` could be in physical pixels since the root RenderObject ("RenderView")
+  // applies a transform that turns logical pixels to physical pixels. Undo the
+  // transform by dividing the coordinates by the screen's scale factor, if this
+  // UIFocusItem's reported `coordinateSpace` is the root view (which means this
+  // UIFocusItem is not inside of a scroll view).
+  //
+  // Screen can be nil if the FlutterView is covered by another native view.
+  CGFloat scale = (self.bridge->view().window.screen ?: UIScreen.mainScreen).scale;
+  return CGRectMake(unscaledRect.origin.x / scale, unscaledRect.origin.y / scale,
+                    unscaledRect.size.width / scale, unscaledRect.size.height / scale);
 }
 
 #pragma mark - UIFocusItemContainer Conformance
@@ -87,16 +148,94 @@ - (CGRect)frame {
   //
   // This method is only supposed to return items within the given
   // rect but returning everything in the subtree seems to work fine.
-  NSMutableArray<SemanticsObject*>* reversedItems =
+  NSMutableArray<id<UIFocusItem>>* reversedItems =
       [[NSMutableArray alloc] initWithCapacity:self.childrenInHitTestOrder.count];
   for (NSUInteger i = 0; i < self.childrenInHitTestOrder.count; ++i) {
-    [reversedItems
-        addObject:self.childrenInHitTestOrder[self.childrenInHitTestOrder.count - 1 - i]];
+    SemanticsObject* child = self.childrenInHitTestOrder[self.childrenInHitTestOrder.count - 1 - i];
+    [reversedItems addObject:child.focusItem];
   }
   return reversedItems;
 }
 
 - (id<UICoordinateSpace>)coordinateSpace {
-  return self.bridge->view();
+  // A regular SemanticsObject uses the same coordinate space as its parent.
+  return self.parent.coordinateSpace ?: self.bridge->view();
+}
+
+@end
+
+/// Scrollable containers interact with the iOS focus engine using the
+/// `UIFocusItemScrollableContainer` protocol. The said protocol (and other focus-related protocols)
+/// does not provide means to inform the focus system of layout changes. In order for the focus
+/// highlight to update properly as the scroll view scrolls, this implementation incorporates a
+/// UIScrollView into the focus hierarchy to workaround the highlight update problem.
+///
+///  As a result, in the current implementation only scrollable containers and the root node
+///  establish their own `coordinateSpace`s. All other `UIFocusItemContainter`s use the same
+///  `coordinateSpace` as the containing UIScrollView, or the root `FlutterView`, whichever is
+///  closer.
+///
+/// See also the `frame` method implementation.
+#pragma mark - Scrolling
+
+@interface FlutterScrollableSemanticsObject (CoordinateSpace)
+@end
+
+@implementation FlutterScrollableSemanticsObject (CoordinateSpace)
+- (id<UICoordinateSpace>)coordinateSpace {
+  // A scrollable SemanticsObject uses the same coordinate space as the scroll view.
+  // This may not work very well in nested scroll views.
+  return self.scrollView;
+}
+
+- (id<UIFocusItem>)focusItem {
+  return self.scrollView;
+}
+
+@end
+
+@interface FlutterSemanticsScrollView (UIFocusItemScrollableContainer) <
+    UIFocusItemScrollableContainer>
+@end
+
+@implementation FlutterSemanticsScrollView (UIFocusItemScrollableContainer)
+
+#pragma mark - FlutterSemanticsScrollView UIFocusItemScrollableContainer Conformance
+
+- (CGSize)visibleSize {
+  return self.frame.size;
+}
+
+- (void)setContentOffset:(CGPoint)contentOffset {
+  [super setContentOffset:contentOffset];
+  // Do no send flutter::SemanticsAction::kScrollToOffset if it's triggered
+  // by a framework update.
+  if (![self.semanticsObject isAccessibilityBridgeAlive] || !self.isDoingSystemScrolling) {
+    return;
+  }
+
+  double offset[2] = {contentOffset.x, contentOffset.y};
+  FlutterStandardTypedData* offsetData = [FlutterStandardTypedData
+      typedDataWithFloat64:[NSData dataWithBytes:&offset length:sizeof(offset)]];
+  NSData* encoded = [[FlutterStandardMessageCodec sharedInstance] encode:offsetData];
+  self.semanticsObject.bridge->DispatchSemanticsAction(
+      self.semanticsObject.uid, flutter::SemanticsAction::kScrollToOffset,
+      fml::MallocMapping::Copy(encoded.bytes, encoded.length));
+}
+
+- (BOOL)canBecomeFocused {
+  return NO;
+}
+
+- (id<UIFocusEnvironment>)parentFocusEnvironment {
+  return self.semanticsObject.parentFocusEnvironment;
+}
+
+- (NSArray<id<UIFocusEnvironment>>*)preferredFocusEnvironments {
+  return nil;
+}
+
+- (NSArray<id<UIFocusItem>>*)focusItemsInRect:(CGRect)rect {
+  return [self.semanticsObject focusItemsInRect:rect];
 }
 @end

engine/src/flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.h

@@ -10,6 +10,7 @@
 #include "flutter/fml/macros.h"
 #include "flutter/fml/memory/weak_ptr.h"
 #include "flutter/lib/ui/semantics/semantics_node.h"
+#import "flutter/shell/platform/darwin/ios/framework/Source/FlutterSemanticsScrollView.h"
 #import "flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge_ios.h"
 
 constexpr int32_t kRootNodeId = 0;
@@ -186,7 +187,7 @@ constexpr float kScrollExtentMaxForInf = 1000;
 /// The semantics object for scrollable. This class creates an UIScrollView to interact with the
 /// iOS.
 @interface FlutterScrollableSemanticsObject : SemanticsObject
-
+@property(nonatomic, readonly) FlutterSemanticsScrollView* scrollView;
 @end
 
 /**

engine/src/flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.mm

@@ -154,6 +154,8 @@ - (instancetype)initWithBridge:(fml::WeakPtr<flutter::AccessibilityBridgeIos>)br
     _scrollView = [[FlutterSemanticsScrollView alloc] initWithSemanticsObject:self];
     [_scrollView setShowsHorizontalScrollIndicator:NO];
     [_scrollView setShowsVerticalScrollIndicator:NO];
+    [_scrollView setContentInset:UIEdgeInsetsZero];
+    [_scrollView setContentInsetAdjustmentBehavior:UIScrollViewContentInsetAdjustmentNever];
     [self.bridge->view() addSubview:_scrollView];
   }
   return self;
@@ -174,7 +176,10 @@ - (void)accessibilityBridgeDidFinishUpdate {
   // contentOffset is 0.0, only the scroll down action is available.
   self.scrollView.frame = self.accessibilityFrame;
   self.scrollView.contentSize = [self contentSizeInternal];
-  [self.scrollView setContentOffset:[self contentOffsetInternal] animated:NO];
+  // See the documentation on `isDoingSystemScrolling`.
+  if (!self.scrollView.isDoingSystemScrolling) {
+    [self.scrollView setContentOffset:self.contentOffsetInternal animated:NO];
+  }
 }
 
 - (id)nativeAccessibility {