From b67d0d505b284db3f8c9d859329fa18a128a3bc2 Mon Sep 17 00:00:00 2001 From: LongCatIsLooong <31859944+LongCatIsLooong@users.noreply.github.com> Date: Fri, 18 Oct 2024 13:54:01 -0700 Subject: [PATCH 1/6] A bare-bones implementation for iOS focus engine support, to enable basic full keyboard access (FKA), except for scrolling which will be implemented in a different patch. Partially fixes #76497 On iOS 15 and above, FKA, if enabled, always consumes relevant key events, so the Flutter framework can't see those key events as they won't be delivered via the UIResponder chain (https://developer.apple.com/documentation/uikit/uikeycommand/3780513-wantspriorityoversystembehavior). This patch provides the basic focus-related information to the iOS focus engine, based on the information that is already available in the accessibility tree, so the iOS focus engine can navigate the UI hierarchy and invoke `accessibilityActivate` on the current focus when the user presses the space key. This at the moment seems to be the best option: - There doesn't seem to be a way to reliably prevents FKA from consuming the key events and that seems to be by design. - The user can remap the FKA keys in iOS system settings, but that key mapping isn't available to apps, so even if the framework can get the key events it won't be able to honor custom key maps. - When FKA is on, `-[FlutterView isAccessibilityElement]` is called without user interaction (presumably it's called when the view appears), so when the user interacts with the app using FKA, it's likely that the accessibility is already enabled, we don't have to worry detecting whether FKA is on (at least for now). Scrolling using FKA currently does not work despite `FlutterScrollableSemanticsObject` conforms to `UIFocusItemScrollableContainer`. `setContentOffset:` must be implemented using a new API that informs the framework of the new contentOffset in the scroll view. `accessibilityScroll` does not work because it scrolls too much in most cases. --- shell/platform/darwin/ios/BUILD.gn | 1 + .../ios/framework/Source/FlutterView.mm | 27 ++++ .../Source/SemanticsObject+UIFocusSystem.mm | 129 ++++++++++++++++++ .../framework/Source/SemanticsObjectTest.mm | 53 +++++++ 4 files changed, 210 insertions(+) create mode 100644 shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm diff --git a/shell/platform/darwin/ios/BUILD.gn b/shell/platform/darwin/ios/BUILD.gn index 6bd0397b26b9f..244dfb5e0f664 100644 --- a/shell/platform/darwin/ios/BUILD.gn +++ b/shell/platform/darwin/ios/BUILD.gn @@ -106,6 +106,7 @@ source_set("flutter_framework_source_arc") { "framework/Source/FlutterViewResponder.h", "framework/Source/KeyCodeMap.g.mm", "framework/Source/KeyCodeMap_Internal.h", + "framework/Source/SemanticsObject+UIFocusSystem.mm", "framework/Source/SemanticsObject.h", "framework/Source/SemanticsObject.mm", "framework/Source/TextInputSemanticsObject.h", diff --git a/shell/platform/darwin/ios/framework/Source/FlutterView.mm b/shell/platform/darwin/ios/framework/Source/FlutterView.mm index 7e8e1108b13b5..137b1f712d828 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterView.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterView.mm @@ -3,6 +3,7 @@ // found in the LICENSE file. #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterView.h" +#import "flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.h" #include "flutter/fml/platform/darwin/cf_utils.h" @@ -226,4 +227,30 @@ - (BOOL)isAccessibilityElement { return NO; } +// Enables keyboard-based navigation when the user turns on +// full keyboard access (FKA), using existing accessibility information. +// +// iOS does not provide any API for monitoring or querying whether FKA is on, +// but it does call isAccessibilityElement if FKA is on, +// so the isAccessibilityElement implementation above will be called +// when the view appears and the accessibility information will most likely +// be available by the time the user starts to interact with the app using FKA. +// +// See SemanticsObject+UIFocusSystem.mm for more details. +- (NSArray>*)focusItemsInRect:(CGRect)rect { + NSObject* rootAccessibilityElement = + [self.accessibilityElements count] > 0 ? self.accessibilityElements[0] : nil; + return [rootAccessibilityElement isKindOfClass:[SemanticsObjectContainer class]] + ? @[ [rootAccessibilityElement accessibilityElementAtIndex:0] ] + : nil; +} + +- (NSArray>*)preferredFocusEnvironments { + // Occasionally we add subviews to FlutterView (text fields for example). + // These views shouldn't be directly visible to the iOS focus engine, instead + // the focus engine should only interact with the designated focus items + // (SemanticsObjects). + return nil; +} + @end diff --git a/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm b/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm new file mode 100644 index 0000000000000..beb832129df5f --- /dev/null +++ b/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm @@ -0,0 +1,129 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#import "SemanticsObject.h" +#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h" + +FLUTTER_ASSERT_ARC + +// The SemanticsObject class conforms to UIFocusItem and UIFocusItemContainer +// protocols, so the SemanticsObject tree can also be used to represent +// interactive UI components on screen that can receive UIFocusSystem focus. +// +// Typically, physical key events received by the FlutterViewController is +// first delivered to the framework, but that stopped working for navigation keys +// since iOS 15 when full keyboard access (FKA) is on, because those events are +// consumed by the UIFocusSystem and never dispatched to the UIResponders in the +// application (see +// https://developer.apple.com/documentation/uikit/uikeycommand/3780513-wantspriorityoversystembehavior +// ). FKA relies on the iOS focus engine, to enable FKA on iOS 15+, we use +// SemanticsObject to provide the iOS focus engine with the required hierarchical +// information and geometric context. +// +// The focus engine focus is different from accessibility focus, or even the +// currentFocus of the Flutter FocusManager in the framework. On iOS 15+, FKA +// key events are dispatched to the current iOS focus engine focus (and +// translated to calls such as -[NSObject accessibilityActivate]), while most +// other key events are dispatched to the framework. +@interface SemanticsObject (UIFocusSystem) +@end + +@implementation SemanticsObject (UIFocusSystem) + +#pragma mark - UIFocusEnvironment Conformance + +- (void)setNeedsFocusUpdate { +} + +- (void)updateFocusIfNeeded { +} + +- (BOOL)shouldUpdateFocusInContext:(UIFocusUpdateContext*)context { + return YES; +} + +- (void)didUpdateFocusInContext:(UIFocusUpdateContext*)context + withAnimationCoordinator:(UIFocusAnimationCoordinator*)coordinator { +} + +- (id)parentFocusEnvironment { + // The root SemanticsObject node's parent is the FlutterView. + return self.parent ?: self.bridge->view(); +} + +- (NSArray>*)preferredFocusEnvironments { + return nil; +} + +- (id)focusItemContainer { + return self; +} + +#pragma mark - UIFocusItem Conformance + +- (BOOL)canBecomeFocused { + if ((self.node.flags & static_cast(flutter::SemanticsFlags::kIsHidden)) != 0) { + return NO; + } + // Currently only supports SemanticsObjects that handle + // -[NSObject accessibilityActivate]. + return self.node.HasAction(flutter::SemanticsAction::kTap); +} + +- (CGRect)frame { + return self.accessibilityFrame; +} + +#pragma mark - UIFocusItemContainer Conformance + +- (NSArray>*)focusItemsInRect:(CGRect)rect { + // It seems the iOS focus system rely heavily on this method (instead of + // preferredFocusEnvironments) for directional navigation. + // The order of the items seems to be important, menus and dialogs become + // unreachable via FKA if the returned children are organized + // in hit-test order. + // + // Additionally, this method is only supposed to return items within the given + // rect but returning everything in the subtree seems to work fine. + NSMutableArray* reversedItems = + [[NSMutableArray alloc] initWithCapacity:self.childrenInHitTestOrder.count]; + for (NSUInteger i = 0; i < self.childrenInHitTestOrder.count; ++i) { + [reversedItems + addObject:self.childrenInHitTestOrder[self.childrenInHitTestOrder.count - 1 - i]]; + } + return reversedItems; +} + +- (id)coordinateSpace { + return self.bridge->view(); +} +@end + +@interface FlutterScrollableSemanticsObject () +@property(nonatomic, readonly) UIScrollView* scrollView; +@end + +@interface FlutterScrollableSemanticsObject (UIFocusItemScrollableContainer) < + UIFocusItemScrollableContainer> +@end + +@implementation FlutterScrollableSemanticsObject (UIFocusItemScrollableContainer) +- (CGPoint)contentOffset { + return self.scrollView.contentOffset; +} + +- (void)setContentOffset:(CGPoint)contentOffset { + // TODO(LongCatIsLooong): implement. This method is called by the focus engine + // to make the focused content visible in a scroll view when FKA is on. +} + +- (CGSize)contentSize { + return self.scrollView.contentSize; +} + +- (CGSize)visibleSize { + return self.scrollView.frame.size; +} + +@end diff --git a/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm b/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm index e95078a419cd6..1d56dcadd67c8 100644 --- a/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm +++ b/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm @@ -16,6 +16,9 @@ const float kFloatCompareEpsilon = 0.001; +@interface SemanticsObject (UIFocusSystem) +@end + @interface TextInputSemanticsObject (Test) - (UIView*)textInputSurrogate; @end @@ -1152,4 +1155,54 @@ - (void)testTextInputSemanticsObject_editActions { [self waitForExpectationsWithTimeout:1 handler:nil]; } +- (void)testUIFocusItemConformance { + fml::WeakPtrFactory factory( + new flutter::testing::MockAccessibilityBridge()); + fml::WeakPtr bridge = factory.GetWeakPtr(); + SemanticsObject* parent = [[SemanticsObject alloc] initWithBridge:bridge uid:0]; + SemanticsObject* child = [[SemanticsObject alloc] initWithBridge:bridge uid:1]; + parent.children = @[ child ]; + + // parentFocusEnvironment + XCTAssertTrue([parent.parentFocusEnvironment isKindOfClass:[UIView class]]); + XCTAssertEqual(child.parentFocusEnvironment, child.parent); + + // canBecomeFocused + flutter::SemanticsNode childNode; + childNode.flags = static_cast(flutter::SemanticsFlags::kIsHidden); + childNode.actions = static_cast(flutter::SemanticsAction::kTap); + [child setSemanticsNode:&childNode]; + XCTAssertFalse(child.canBecomeFocused); + childNode.flags = 0; + [child setSemanticsNode:&childNode]; + XCTAssertTrue(child.canBecomeFocused); + childNode.actions = 0; + [child setSemanticsNode:&childNode]; + XCTAssertFalse(child.canBecomeFocused); + + // frame + childNode.rect = SkRect::MakeXYWH(0, 0, 30, 30); + [child setSemanticsNode:&childNode]; + flutter::SemanticsNode parentNode; + parentNode.rect = SkRect::MakeXYWH(0, 0, 200, 200); + [parent setSemanticsNode:&parentNode]; + + XCTAssertTrue(CGRectEqualToRect(child.frame, CGRectMake(0, 0, 10, 10))); +} + +- (void)testUIFocusItemContainerConformance { + fml::WeakPtrFactory factory( + new flutter::testing::MockAccessibilityBridge()); + fml::WeakPtr bridge = factory.GetWeakPtr(); + SemanticsObject* parent = [[SemanticsObject alloc] initWithBridge:bridge uid:0]; + SemanticsObject* child1 = [[SemanticsObject alloc] initWithBridge:bridge uid:1]; + SemanticsObject* child2 = [[SemanticsObject alloc] initWithBridge:bridge uid:2]; + parent.childrenInHitTestOrder = @[ child1, child2 ]; + + // focusItemsInRect + NSArray>* itemsInRect = [parent focusItemsInRect:CGRectMake(0, 0, 100, 100)]; + XCTAssertEqual(itemsInRect.count, (unsigned long)2); + XCTAssertEqual(itemsInRect[0], child2); + XCTAssertEqual(itemsInRect[1], child1); +} @end From 28a21f7cf90b319448b0cc8c6a14b97eddc96391 Mon Sep 17 00:00:00 2001 From: LongCatIsLooong <31859944+LongCatIsLooong@users.noreply.github.com> Date: Fri, 18 Oct 2024 14:40:04 -0700 Subject: [PATCH 2/6] licenses --- ci/licenses_golden/licenses_flutter | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index f2873e025d53f..a3978a9265322 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -44373,6 +44373,7 @@ ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterViewT ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/IOKit.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/KeyCodeMap.g.mm + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/KeyCodeMap_Internal.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.mm + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm + ../../../flutter/LICENSE @@ -47261,6 +47262,7 @@ FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterViewTes FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/IOKit.h FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/KeyCodeMap.g.mm FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/KeyCodeMap_Internal.h +FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.h FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.mm FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm From ced64ca5f4ab00193395ec2528a2e9c783253580 Mon Sep 17 00:00:00 2001 From: LongCatIsLooong <31859944+LongCatIsLooong@users.noreply.github.com> Date: Fri, 18 Oct 2024 15:14:24 -0700 Subject: [PATCH 3/6] fix tests --- .../darwin/ios/framework/Source/SemanticsObjectTest.mm | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm b/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm index 1d56dcadd67c8..2f388ad224ddb 100644 --- a/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm +++ b/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm @@ -1180,14 +1180,15 @@ - (void)testUIFocusItemConformance { [child setSemanticsNode:&childNode]; XCTAssertFalse(child.canBecomeFocused); - // frame - childNode.rect = SkRect::MakeXYWH(0, 0, 30, 30); + CGFloat scale = ((bridge->view().window.screen ?: UIScreen.mainScreen)).scale; + + childNode.rect = SkRect::MakeXYWH(0, 0, 100 * scale, 100 * scale); [child setSemanticsNode:&childNode]; flutter::SemanticsNode parentNode; parentNode.rect = SkRect::MakeXYWH(0, 0, 200, 200); [parent setSemanticsNode:&parentNode]; - XCTAssertTrue(CGRectEqualToRect(child.frame, CGRectMake(0, 0, 10, 10))); + XCTAssertTrue(CGRectEqualToRect(child.frame, CGRectMake(0, 0, 100, 100))); } - (void)testUIFocusItemContainerConformance { From 3d55c09b7530ff1c2a4e2743b1820de7f2a31af1 Mon Sep 17 00:00:00 2001 From: LongCatIsLooong <31859944+LongCatIsLooong@users.noreply.github.com> Date: Sat, 19 Oct 2024 20:24:44 -0700 Subject: [PATCH 4/6] Remove unnecessary code --- .../Source/SemanticsObject+UIFocusSystem.mm | 40 +------------------ .../framework/Source/SemanticsObjectTest.mm | 4 +- 2 files changed, 4 insertions(+), 40 deletions(-) diff --git a/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm b/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm index beb832129df5f..a27bfcc95a60a 100644 --- a/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm +++ b/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm @@ -80,50 +80,14 @@ - (CGRect)frame { - (NSArray>*)focusItemsInRect:(CGRect)rect { // It seems the iOS focus system rely heavily on this method (instead of // preferredFocusEnvironments) for directional navigation. - // The order of the items seems to be important, menus and dialogs become - // unreachable via FKA if the returned children are organized - // in hit-test order. + // Whether the item order in the returned array matters is unknown. // // Additionally, this method is only supposed to return items within the given // rect but returning everything in the subtree seems to work fine. - NSMutableArray* reversedItems = - [[NSMutableArray alloc] initWithCapacity:self.childrenInHitTestOrder.count]; - for (NSUInteger i = 0; i < self.childrenInHitTestOrder.count; ++i) { - [reversedItems - addObject:self.childrenInHitTestOrder[self.childrenInHitTestOrder.count - 1 - i]]; - } - return reversedItems; + return self.childrenInHitTestOrder; } - (id)coordinateSpace { return self.bridge->view(); } @end - -@interface FlutterScrollableSemanticsObject () -@property(nonatomic, readonly) UIScrollView* scrollView; -@end - -@interface FlutterScrollableSemanticsObject (UIFocusItemScrollableContainer) < - UIFocusItemScrollableContainer> -@end - -@implementation FlutterScrollableSemanticsObject (UIFocusItemScrollableContainer) -- (CGPoint)contentOffset { - return self.scrollView.contentOffset; -} - -- (void)setContentOffset:(CGPoint)contentOffset { - // TODO(LongCatIsLooong): implement. This method is called by the focus engine - // to make the focused content visible in a scroll view when FKA is on. -} - -- (CGSize)contentSize { - return self.scrollView.contentSize; -} - -- (CGSize)visibleSize { - return self.scrollView.frame.size; -} - -@end diff --git a/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm b/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm index 2f388ad224ddb..d0fcaeab2bb31 100644 --- a/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm +++ b/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm @@ -1203,7 +1203,7 @@ - (void)testUIFocusItemContainerConformance { // focusItemsInRect NSArray>* itemsInRect = [parent focusItemsInRect:CGRectMake(0, 0, 100, 100)]; XCTAssertEqual(itemsInRect.count, (unsigned long)2); - XCTAssertEqual(itemsInRect[0], child2); - XCTAssertEqual(itemsInRect[1], child1); + XCTAssertTrue([itemsInRect containsObject:child1]); + XCTAssertTrue([itemsInRect containsObject:child2]); } @end From dcdea3177935618e557bd22bcd4dd671748b0841 Mon Sep 17 00:00:00 2001 From: LongCatIsLooong <31859944+LongCatIsLooong@users.noreply.github.com> Date: Mon, 21 Oct 2024 18:12:16 -0700 Subject: [PATCH 5/6] review --- .../ios/framework/Source/SemanticsObject+UIFocusSystem.mm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm b/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm index a27bfcc95a60a..f617ee5433c64 100644 --- a/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm +++ b/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm @@ -78,8 +78,8 @@ - (CGRect)frame { #pragma mark - UIFocusItemContainer Conformance - (NSArray>*)focusItemsInRect:(CGRect)rect { - // It seems the iOS focus system rely heavily on this method (instead of - // preferredFocusEnvironments) for directional navigation. + // It seems the iOS focus system relies heavily on focusItemsInRect + // (instead of preferredFocusEnvironments) for directional navigation. // Whether the item order in the returned array matters is unknown. // // Additionally, this method is only supposed to return items within the given From 6ca023f273a6384915818eba3860035ad62968dd Mon Sep 17 00:00:00 2001 From: LongCatIsLooong <31859944+LongCatIsLooong@users.noreply.github.com> Date: Mon, 21 Oct 2024 19:08:39 -0700 Subject: [PATCH 6/6] Make menu and dialogs reachable --- .../Source/SemanticsObject+UIFocusSystem.mm | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm b/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm index f617ee5433c64..3448b9280536b 100644 --- a/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm +++ b/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm @@ -80,11 +80,20 @@ - (CGRect)frame { - (NSArray>*)focusItemsInRect:(CGRect)rect { // It seems the iOS focus system relies heavily on focusItemsInRect // (instead of preferredFocusEnvironments) for directional navigation. - // Whether the item order in the returned array matters is unknown. // - // Additionally, this method is only supposed to return items within the given + // The order of the items seems to be important, menus and dialogs become + // unreachable via FKA if the returned children are organized + // in hit-test order. + // + // This method is only supposed to return items within the given // rect but returning everything in the subtree seems to work fine. - return self.childrenInHitTestOrder; + NSMutableArray* reversedItems = + [[NSMutableArray alloc] initWithCapacity:self.childrenInHitTestOrder.count]; + for (NSUInteger i = 0; i < self.childrenInHitTestOrder.count; ++i) { + [reversedItems + addObject:self.childrenInHitTestOrder[self.childrenInHitTestOrder.count - 1 - i]]; + } + return reversedItems; } - (id)coordinateSpace {