diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index f2873e025d53f..a3978a9265322 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -44373,6 +44373,7 @@ ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterViewT ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/IOKit.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/KeyCodeMap.g.mm + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/KeyCodeMap_Internal.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.mm + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm + ../../../flutter/LICENSE @@ -47261,6 +47262,7 @@ FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterViewTes FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/IOKit.h FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/KeyCodeMap.g.mm FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/KeyCodeMap_Internal.h +FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.h FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.mm FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm diff --git a/shell/platform/darwin/ios/BUILD.gn b/shell/platform/darwin/ios/BUILD.gn index 6bd0397b26b9f..244dfb5e0f664 100644 --- a/shell/platform/darwin/ios/BUILD.gn +++ b/shell/platform/darwin/ios/BUILD.gn @@ -106,6 +106,7 @@ source_set("flutter_framework_source_arc") { "framework/Source/FlutterViewResponder.h", "framework/Source/KeyCodeMap.g.mm", "framework/Source/KeyCodeMap_Internal.h", + "framework/Source/SemanticsObject+UIFocusSystem.mm", "framework/Source/SemanticsObject.h", "framework/Source/SemanticsObject.mm", "framework/Source/TextInputSemanticsObject.h", diff --git a/shell/platform/darwin/ios/framework/Source/FlutterView.mm b/shell/platform/darwin/ios/framework/Source/FlutterView.mm index 7e8e1108b13b5..137b1f712d828 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterView.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterView.mm @@ -3,6 +3,7 @@ // found in the LICENSE file. #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterView.h" +#import "flutter/shell/platform/darwin/ios/framework/Source/SemanticsObject.h" #include "flutter/fml/platform/darwin/cf_utils.h" @@ -226,4 +227,30 @@ - (BOOL)isAccessibilityElement { return NO; } +// Enables keyboard-based navigation when the user turns on +// full keyboard access (FKA), using existing accessibility information. +// +// iOS does not provide any API for monitoring or querying whether FKA is on, +// but it does call isAccessibilityElement if FKA is on, +// so the isAccessibilityElement implementation above will be called +// when the view appears and the accessibility information will most likely +// be available by the time the user starts to interact with the app using FKA. +// +// See SemanticsObject+UIFocusSystem.mm for more details. +- (NSArray>*)focusItemsInRect:(CGRect)rect { + NSObject* rootAccessibilityElement = + [self.accessibilityElements count] > 0 ? self.accessibilityElements[0] : nil; + return [rootAccessibilityElement isKindOfClass:[SemanticsObjectContainer class]] + ? @[ [rootAccessibilityElement accessibilityElementAtIndex:0] ] + : nil; +} + +- (NSArray>*)preferredFocusEnvironments { + // Occasionally we add subviews to FlutterView (text fields for example). + // These views shouldn't be directly visible to the iOS focus engine, instead + // the focus engine should only interact with the designated focus items + // (SemanticsObjects). + return nil; +} + @end diff --git a/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm b/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm new file mode 100644 index 0000000000000..3448b9280536b --- /dev/null +++ b/shell/platform/darwin/ios/framework/Source/SemanticsObject+UIFocusSystem.mm @@ -0,0 +1,102 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#import "SemanticsObject.h" +#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h" + +FLUTTER_ASSERT_ARC + +// The SemanticsObject class conforms to UIFocusItem and UIFocusItemContainer +// protocols, so the SemanticsObject tree can also be used to represent +// interactive UI components on screen that can receive UIFocusSystem focus. +// +// Typically, physical key events received by the FlutterViewController is +// first delivered to the framework, but that stopped working for navigation keys +// since iOS 15 when full keyboard access (FKA) is on, because those events are +// consumed by the UIFocusSystem and never dispatched to the UIResponders in the +// application (see +// https://developer.apple.com/documentation/uikit/uikeycommand/3780513-wantspriorityoversystembehavior +// ). FKA relies on the iOS focus engine, to enable FKA on iOS 15+, we use +// SemanticsObject to provide the iOS focus engine with the required hierarchical +// information and geometric context. +// +// The focus engine focus is different from accessibility focus, or even the +// currentFocus of the Flutter FocusManager in the framework. On iOS 15+, FKA +// key events are dispatched to the current iOS focus engine focus (and +// translated to calls such as -[NSObject accessibilityActivate]), while most +// other key events are dispatched to the framework. +@interface SemanticsObject (UIFocusSystem) +@end + +@implementation SemanticsObject (UIFocusSystem) + +#pragma mark - UIFocusEnvironment Conformance + +- (void)setNeedsFocusUpdate { +} + +- (void)updateFocusIfNeeded { +} + +- (BOOL)shouldUpdateFocusInContext:(UIFocusUpdateContext*)context { + return YES; +} + +- (void)didUpdateFocusInContext:(UIFocusUpdateContext*)context + withAnimationCoordinator:(UIFocusAnimationCoordinator*)coordinator { +} + +- (id)parentFocusEnvironment { + // The root SemanticsObject node's parent is the FlutterView. + return self.parent ?: self.bridge->view(); +} + +- (NSArray>*)preferredFocusEnvironments { + return nil; +} + +- (id)focusItemContainer { + return self; +} + +#pragma mark - UIFocusItem Conformance + +- (BOOL)canBecomeFocused { + if ((self.node.flags & static_cast(flutter::SemanticsFlags::kIsHidden)) != 0) { + return NO; + } + // Currently only supports SemanticsObjects that handle + // -[NSObject accessibilityActivate]. + return self.node.HasAction(flutter::SemanticsAction::kTap); +} + +- (CGRect)frame { + return self.accessibilityFrame; +} + +#pragma mark - UIFocusItemContainer Conformance + +- (NSArray>*)focusItemsInRect:(CGRect)rect { + // It seems the iOS focus system relies heavily on focusItemsInRect + // (instead of preferredFocusEnvironments) for directional navigation. + // + // The order of the items seems to be important, menus and dialogs become + // unreachable via FKA if the returned children are organized + // in hit-test order. + // + // This method is only supposed to return items within the given + // rect but returning everything in the subtree seems to work fine. + NSMutableArray* reversedItems = + [[NSMutableArray alloc] initWithCapacity:self.childrenInHitTestOrder.count]; + for (NSUInteger i = 0; i < self.childrenInHitTestOrder.count; ++i) { + [reversedItems + addObject:self.childrenInHitTestOrder[self.childrenInHitTestOrder.count - 1 - i]]; + } + return reversedItems; +} + +- (id)coordinateSpace { + return self.bridge->view(); +} +@end diff --git a/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm b/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm index e95078a419cd6..d0fcaeab2bb31 100644 --- a/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm +++ b/shell/platform/darwin/ios/framework/Source/SemanticsObjectTest.mm @@ -16,6 +16,9 @@ const float kFloatCompareEpsilon = 0.001; +@interface SemanticsObject (UIFocusSystem) +@end + @interface TextInputSemanticsObject (Test) - (UIView*)textInputSurrogate; @end @@ -1152,4 +1155,55 @@ - (void)testTextInputSemanticsObject_editActions { [self waitForExpectationsWithTimeout:1 handler:nil]; } +- (void)testUIFocusItemConformance { + fml::WeakPtrFactory factory( + new flutter::testing::MockAccessibilityBridge()); + fml::WeakPtr bridge = factory.GetWeakPtr(); + SemanticsObject* parent = [[SemanticsObject alloc] initWithBridge:bridge uid:0]; + SemanticsObject* child = [[SemanticsObject alloc] initWithBridge:bridge uid:1]; + parent.children = @[ child ]; + + // parentFocusEnvironment + XCTAssertTrue([parent.parentFocusEnvironment isKindOfClass:[UIView class]]); + XCTAssertEqual(child.parentFocusEnvironment, child.parent); + + // canBecomeFocused + flutter::SemanticsNode childNode; + childNode.flags = static_cast(flutter::SemanticsFlags::kIsHidden); + childNode.actions = static_cast(flutter::SemanticsAction::kTap); + [child setSemanticsNode:&childNode]; + XCTAssertFalse(child.canBecomeFocused); + childNode.flags = 0; + [child setSemanticsNode:&childNode]; + XCTAssertTrue(child.canBecomeFocused); + childNode.actions = 0; + [child setSemanticsNode:&childNode]; + XCTAssertFalse(child.canBecomeFocused); + + CGFloat scale = ((bridge->view().window.screen ?: UIScreen.mainScreen)).scale; + + childNode.rect = SkRect::MakeXYWH(0, 0, 100 * scale, 100 * scale); + [child setSemanticsNode:&childNode]; + flutter::SemanticsNode parentNode; + parentNode.rect = SkRect::MakeXYWH(0, 0, 200, 200); + [parent setSemanticsNode:&parentNode]; + + XCTAssertTrue(CGRectEqualToRect(child.frame, CGRectMake(0, 0, 100, 100))); +} + +- (void)testUIFocusItemContainerConformance { + fml::WeakPtrFactory factory( + new flutter::testing::MockAccessibilityBridge()); + fml::WeakPtr bridge = factory.GetWeakPtr(); + SemanticsObject* parent = [[SemanticsObject alloc] initWithBridge:bridge uid:0]; + SemanticsObject* child1 = [[SemanticsObject alloc] initWithBridge:bridge uid:1]; + SemanticsObject* child2 = [[SemanticsObject alloc] initWithBridge:bridge uid:2]; + parent.childrenInHitTestOrder = @[ child1, child2 ]; + + // focusItemsInRect + NSArray>* itemsInRect = [parent focusItemsInRect:CGRectMake(0, 0, 100, 100)]; + XCTAssertEqual(itemsInRect.count, (unsigned long)2); + XCTAssertTrue([itemsInRect containsObject:child1]); + XCTAssertTrue([itemsInRect containsObject:child2]); +} @end