Skip to content

Commit b45fe7e

Browse files
chunhtaichunhtai
authored
Support text editing voiceover feedback in macOS (flutter#25600)
1 parent 9502b6f commit b45fe7e

36 files changed

+1423
-145
lines changed

‎ci/licenses_golden/licenses_flutter

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1201,6 +1201,9 @@ FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterSurfa
12011201
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.h
12021202
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.mm
12031203
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPluginTest.mm
1204+
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObject.h
1205+
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObject.mm
1206+
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputSemanticsObjectTest.mm
12041207
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextureRegistrar.h
12051208
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterTextureRegistrar.mm
12061209
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterView.h

‎shell/platform/common/accessibility_bridge.cc

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,22 @@ AccessibilityBridge::GetPendingEvents() {
111111
return result;
112112
}
113113

114+
void AccessibilityBridge::UpdateDelegate(
115+
std::unique_ptr<AccessibilityBridgeDelegate> delegate) {
116+
delegate_ = std::move(delegate);
117+
// Recreate FlutterPlatformNodeDelegates since they may contain stale state
118+
// from the previous AccessibilityBridgeDelegate.
119+
for (const auto& [node_id, old_platform_node_delegate] : id_wrapper_map_) {
120+
std::shared_ptr<FlutterPlatformNodeDelegate> platform_node_delegate =
121+
delegate_->CreateFlutterPlatformNodeDelegate();
122+
platform_node_delegate->Init(
123+
std::static_pointer_cast<FlutterPlatformNodeDelegate::OwnerBridge>(
124+
shared_from_this()),
125+
old_platform_node_delegate->GetAXNode());
126+
id_wrapper_map_[node_id] = platform_node_delegate;
127+
}
128+
}
129+
114130
void AccessibilityBridge::OnNodeWillBeDeleted(ui::AXTree* tree,
115131
ui::AXNode* node) {}
116132

@@ -329,7 +345,7 @@ void AccessibilityBridge::SetBooleanAttributesFromFlutterUpdate(
329345
node_data.AddBoolAttribute(
330346
ax::mojom::BoolAttribute::kEditableRoot,
331347
flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField &&
332-
(flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly) > 0);
348+
(flags & FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly) == 0);
333349
}
334350

335351
void AccessibilityBridge::SetIntAttributesFromFlutterUpdate(

‎shell/platform/common/accessibility_bridge.h

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ class AccessibilityBridge
9797
/// by accessibility bridge whenever a new AXNode is created in
9898
/// AXTree. Each platform needs to implement this method in
9999
/// order to inject its subclass into the accessibility bridge.
100-
virtual std::unique_ptr<FlutterPlatformNodeDelegate>
100+
virtual std::shared_ptr<FlutterPlatformNodeDelegate>
101101
CreateFlutterPlatformNodeDelegate() = 0;
102102
};
103103
//-----------------------------------------------------------------------------
@@ -163,6 +163,11 @@ class AccessibilityBridge
163163
/// all pending events.
164164
const std::vector<ui::AXEventGenerator::TargetedEvent> GetPendingEvents();
165165

166+
//------------------------------------------------------------------------------
167+
/// @brief Update the AccessibilityBridgeDelegate stored in the
168+
/// accessibility bridge to a new one.
169+
void UpdateDelegate(std::unique_ptr<AccessibilityBridgeDelegate> delegate);
170+
166171
private:
167172
// See FlutterSemanticsNode in embedder.h
168173
typedef struct {

‎shell/platform/common/accessibility_bridge_unittests.cc

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,66 @@ TEST(AccessibilityBridgeTest, canFireChildrenChangedCorrectly) {
155155
actual_event.end());
156156
}
157157

158+
TEST(AccessibilityBridgeTest, canUpdateDelegate) {
159+
std::shared_ptr<AccessibilityBridge> bridge =
160+
std::make_shared<AccessibilityBridge>(
161+
std::make_unique<TestAccessibilityBridgeDelegate>());
162+
FlutterSemanticsNode root;
163+
root.id = 0;
164+
root.flags = static_cast<FlutterSemanticsFlag>(0);
165+
root.actions = static_cast<FlutterSemanticsAction>(0);
166+
root.text_selection_base = -1;
167+
root.text_selection_extent = -1;
168+
root.label = "root";
169+
root.hint = "";
170+
root.value = "";
171+
root.increased_value = "";
172+
root.decreased_value = "";
173+
root.child_count = 1;
174+
int32_t children[] = {1};
175+
root.children_in_traversal_order = children;
176+
root.custom_accessibility_actions_count = 0;
177+
bridge->AddFlutterSemanticsNodeUpdate(&root);
178+
179+
FlutterSemanticsNode child1;
180+
child1.id = 1;
181+
child1.flags = static_cast<FlutterSemanticsFlag>(0);
182+
child1.actions = static_cast<FlutterSemanticsAction>(0);
183+
child1.text_selection_base = -1;
184+
child1.text_selection_extent = -1;
185+
child1.label = "child 1";
186+
child1.hint = "";
187+
child1.value = "";
188+
child1.increased_value = "";
189+
child1.decreased_value = "";
190+
child1.child_count = 0;
191+
child1.custom_accessibility_actions_count = 0;
192+
bridge->AddFlutterSemanticsNodeUpdate(&child1);
193+
194+
bridge->CommitUpdates();
195+
196+
auto root_node = bridge->GetFlutterPlatformNodeDelegateFromID(0);
197+
auto child1_node = bridge->GetFlutterPlatformNodeDelegateFromID(1);
198+
EXPECT_FALSE(root_node.expired());
199+
EXPECT_FALSE(child1_node.expired());
200+
// Update Delegate
201+
bridge->UpdateDelegate(std::make_unique<TestAccessibilityBridgeDelegate>());
202+
203+
// Old tree is destroyed.
204+
EXPECT_TRUE(root_node.expired());
205+
EXPECT_TRUE(child1_node.expired());
206+
207+
// New tree still has the data.
208+
auto new_root_node = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock();
209+
auto new_child1_node = bridge->GetFlutterPlatformNodeDelegateFromID(1).lock();
210+
EXPECT_EQ(new_root_node->GetChildCount(), 1);
211+
EXPECT_EQ(new_root_node->GetData().child_ids[0], 1);
212+
EXPECT_EQ(new_root_node->GetName(), "root");
213+
214+
EXPECT_EQ(new_child1_node->GetChildCount(), 0);
215+
EXPECT_EQ(new_child1_node->GetName(), "child 1");
216+
}
217+
158218
TEST(AccessibilityBridgeTest, canHandleSelectionChangeCorrectly) {
159219
TestAccessibilityBridgeDelegate* delegate =
160220
new TestAccessibilityBridgeDelegate();
@@ -200,5 +260,34 @@ TEST(AccessibilityBridgeTest, canHandleSelectionChangeCorrectly) {
200260
ui::AXEventGenerator::Event::OTHER_ATTRIBUTE_CHANGED);
201261
}
202262

263+
TEST(AccessibilityBridgeTest, doesNotAssignEditableRootToSelectableText) {
264+
std::shared_ptr<AccessibilityBridge> bridge =
265+
std::make_shared<AccessibilityBridge>(
266+
std::make_unique<TestAccessibilityBridgeDelegate>());
267+
FlutterSemanticsNode root;
268+
root.id = 0;
269+
root.flags = static_cast<FlutterSemanticsFlag>(
270+
FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField |
271+
FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly);
272+
root.actions = static_cast<FlutterSemanticsAction>(0);
273+
root.text_selection_base = -1;
274+
root.text_selection_extent = -1;
275+
root.label = "root";
276+
root.hint = "";
277+
root.value = "";
278+
root.increased_value = "";
279+
root.decreased_value = "";
280+
root.child_count = 0;
281+
root.custom_accessibility_actions_count = 0;
282+
bridge->AddFlutterSemanticsNodeUpdate(&root);
283+
284+
bridge->CommitUpdates();
285+
286+
auto root_node = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock();
287+
288+
EXPECT_FALSE(root_node->GetData().GetBoolAttribute(
289+
ax::mojom::BoolAttribute::kEditableRoot));
290+
}
291+
203292
} // namespace testing
204293
} // namespace flutter

‎shell/platform/common/flutter_platform_node_delegate.cc

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,4 +107,9 @@ gfx::Rect FlutterPlatformNodeDelegate::GetBoundsRect(
107107
return gfx::ToEnclosingRect(bounds);
108108
}
109109

110+
std::weak_ptr<FlutterPlatformNodeDelegate::OwnerBridge>
111+
FlutterPlatformNodeDelegate::GetOwnerBridge() const {
112+
return bridge_;
113+
}
114+
110115
} // namespace flutter

‎shell/platform/common/flutter_platform_node_delegate.h

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,19 @@ class FlutterPlatformNodeDelegate : public ui::AXPlatformNodeDelegateBase {
3939
public:
4040
virtual ~OwnerBridge() = default;
4141

42+
//---------------------------------------------------------------------------
43+
/// @brief Gets the rectangular bounds of the ax node relative to
44+
/// global coordinate
45+
///
46+
/// @param[in] node The ax node to look up.
47+
/// @param[in] offscreen the bool reference to hold the result whether
48+
/// the ax node is outside of its ancestors' bounds.
49+
/// @param[in] clip_bounds whether to clip the result if the ax node cannot
50+
/// be fully contained in its ancestors' bounds.
51+
virtual gfx::RectF RelativeToGlobalBounds(const ui::AXNode* node,
52+
bool& offscreen,
53+
bool clip_bounds) = 0;
54+
4255
protected:
4356
friend class FlutterPlatformNodeDelegate;
4457

@@ -78,19 +91,6 @@ class FlutterPlatformNodeDelegate : public ui::AXPlatformNodeDelegateBase {
7891
///
7992
/// @param[in] node_id The id of the focused node.
8093
virtual void SetLastFocusedId(AccessibilityNodeId node_id) = 0;
81-
82-
//---------------------------------------------------------------------------
83-
/// @brief Gets the rectangular bounds of the ax node relative to
84-
/// global coordinate
85-
///
86-
/// @param[in] node The ax node to look up.
87-
/// @param[in] offscreen the bool reference to hold the result whether
88-
/// the ax node is outside of its ancestors' bounds.
89-
/// @param[in] clip_bounds whether to clip the result if the ax node cannot
90-
/// be fully contained in its ancestors' bounds.
91-
virtual gfx::RectF RelativeToGlobalBounds(const ui::AXNode* node,
92-
bool& offscreen,
93-
bool clip_bounds) = 0;
9494
};
9595

9696
FlutterPlatformNodeDelegate();
@@ -129,11 +129,18 @@ class FlutterPlatformNodeDelegate : public ui::AXPlatformNodeDelegateBase {
129129
/// Subclasses must call super.
130130
virtual void Init(std::weak_ptr<OwnerBridge> bridge, ui::AXNode* node);
131131

132-
protected:
133132
//------------------------------------------------------------------------------
134-
/// @brief Gets the underlying ax node for this accessibility node.
133+
/// @brief Gets the underlying ax node for this platform node delegate.
135134
ui::AXNode* GetAXNode() const;
136135

136+
//------------------------------------------------------------------------------
137+
/// @brief Gets the owner of this platform node delegate. This is useful
138+
/// when you want to get the information about surrounding nodes
139+
/// of this platform node delegate, e.g. the global rect of this
140+
/// platform node delegate. This pointer is only safe in the
141+
/// platform thread.
142+
std::weak_ptr<OwnerBridge> GetOwnerBridge() const;
143+
137144
private:
138145
ui::AXNode* ax_node_;
139146
std::weak_ptr<OwnerBridge> bridge_;

‎shell/platform/common/flutter_platform_node_delegate_unittests.cc

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,5 +174,51 @@ TEST(FlutterPlatformNodeDelegateTest, canCalculateOffScreenBoundsCorrectly) {
174174
EXPECT_EQ(result, ui::AXOffscreenResult::kOffscreen);
175175
}
176176

177+
TEST(FlutterPlatformNodeDelegateTest, canUseOwnerBridge) {
178+
std::shared_ptr<AccessibilityBridge> bridge =
179+
std::make_shared<AccessibilityBridge>(
180+
std::make_unique<TestAccessibilityBridgeDelegate>());
181+
FlutterSemanticsNode root;
182+
root.id = 0;
183+
root.label = "root";
184+
root.hint = "";
185+
root.value = "";
186+
root.increased_value = "";
187+
root.decreased_value = "";
188+
root.child_count = 1;
189+
int32_t children[] = {1};
190+
root.children_in_traversal_order = children;
191+
root.custom_accessibility_actions_count = 0;
192+
root.rect = {0, 0, 100, 100}; // LTRB
193+
root.transform = {1, 0, 0, 0, 1, 0, 0, 0, 1};
194+
bridge->AddFlutterSemanticsNodeUpdate(&root);
195+
196+
FlutterSemanticsNode child1;
197+
child1.id = 1;
198+
child1.label = "child 1";
199+
child1.hint = "";
200+
child1.value = "";
201+
child1.increased_value = "";
202+
child1.decreased_value = "";
203+
child1.child_count = 0;
204+
child1.custom_accessibility_actions_count = 0;
205+
child1.rect = {0, 0, 50, 50}; // LTRB
206+
child1.transform = {0.5, 0, 0, 0, 0.5, 0, 0, 0, 1};
207+
bridge->AddFlutterSemanticsNodeUpdate(&child1);
208+
209+
bridge->CommitUpdates();
210+
auto child1_node = bridge->GetFlutterPlatformNodeDelegateFromID(1).lock();
211+
auto owner_bridge = child1_node->GetOwnerBridge().lock();
212+
213+
bool result;
214+
gfx::RectF bounds = owner_bridge->RelativeToGlobalBounds(
215+
child1_node->GetAXNode(), result, true);
216+
EXPECT_EQ(bounds.x(), 0);
217+
EXPECT_EQ(bounds.y(), 0);
218+
EXPECT_EQ(bounds.width(), 25);
219+
EXPECT_EQ(bounds.height(), 25);
220+
EXPECT_EQ(result, false);
221+
}
222+
177223
} // namespace testing
178224
} // namespace flutter

‎shell/platform/common/test_accessibility_bridge.cc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
namespace flutter {
88

9-
std::unique_ptr<FlutterPlatformNodeDelegate>
9+
std::shared_ptr<FlutterPlatformNodeDelegate>
1010
TestAccessibilityBridgeDelegate::CreateFlutterPlatformNodeDelegate() {
1111
return std::make_unique<FlutterPlatformNodeDelegate>();
1212
};

‎shell/platform/common/test_accessibility_bridge.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ class TestAccessibilityBridgeDelegate
1919
void DispatchAccessibilityAction(AccessibilityNodeId target,
2020
FlutterSemanticsAction action,
2121
fml::MallocMapping data) override;
22-
std::unique_ptr<FlutterPlatformNodeDelegate>
23-
CreateFlutterPlatformNodeDelegate();
22+
std::shared_ptr<FlutterPlatformNodeDelegate>
23+
CreateFlutterPlatformNodeDelegate() override;
2424

2525
std::vector<ui::AXEventGenerator::TargetedEvent> accessibilitiy_events;
2626
std::vector<FlutterSemanticsAction> performed_actions;

‎shell/platform/darwin/macos/BUILD.gn

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,8 @@ source_set("flutter_framework_source") {
106106
"framework/Source/FlutterSurfaceManager.mm",
107107
"framework/Source/FlutterTextInputPlugin.h",
108108
"framework/Source/FlutterTextInputPlugin.mm",
109+
"framework/Source/FlutterTextInputSemanticsObject.h",
110+
"framework/Source/FlutterTextInputSemanticsObject.mm",
109111
"framework/Source/FlutterTextureRegistrar.h",
110112
"framework/Source/FlutterTextureRegistrar.mm",
111113
"framework/Source/FlutterView.h",
@@ -181,6 +183,7 @@ executable("flutter_desktop_darwin_unittests") {
181183
"framework/Source/FlutterOpenGLRendererTest.mm",
182184
"framework/Source/FlutterPlatformNodeDelegateMacTest.mm",
183185
"framework/Source/FlutterTextInputPluginTest.mm",
186+
"framework/Source/FlutterTextInputSemanticsObjectTest.mm",
184187
"framework/Source/FlutterViewControllerTest.mm",
185188
"framework/Source/FlutterViewControllerTestUtils.h",
186189
"framework/Source/FlutterViewControllerTestUtils.mm",

‎shell/platform/darwin/macos/framework/Source/AccessibilityBridgeMacDelegate.h

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "flutter/shell/platform/common/accessibility_bridge.h"
1111

1212
@class FlutterEngine;
13+
@class FlutterViewController;
1314

1415
namespace flutter {
1516

@@ -20,8 +21,10 @@ class AccessibilityBridgeMacDelegate : public AccessibilityBridge::Accessibility
2021
public:
2122
//---------------------------------------------------------------------------
2223
/// @brief Creates an AccessibilityBridgeMacDelegate.
23-
/// @param[in] flutterEngine The weak reference to the FlutterEngine.
24-
explicit AccessibilityBridgeMacDelegate(__weak FlutterEngine* flutter_engine);
24+
/// @param[in] flutter_engine The weak reference to the FlutterEngine.
25+
/// @param[in] view_controller The weak reference to the FlutterViewController.
26+
explicit AccessibilityBridgeMacDelegate(__weak FlutterEngine* flutter_engine,
27+
__weak FlutterViewController* view_controller);
2528
virtual ~AccessibilityBridgeMacDelegate() = default;
2629

2730
// |AccessibilityBridge::AccessibilityBridgeDelegate|
@@ -33,7 +36,7 @@ class AccessibilityBridgeMacDelegate : public AccessibilityBridge::Accessibility
3336
fml::MallocMapping data) override;
3437

3538
// |AccessibilityBridge::AccessibilityBridgeDelegate|
36-
std::unique_ptr<FlutterPlatformNodeDelegate> CreateFlutterPlatformNodeDelegate() override;
39+
std::shared_ptr<FlutterPlatformNodeDelegate> CreateFlutterPlatformNodeDelegate() override;
3740

3841
private:
3942
/// A wrapper structure to wraps macOS native accessibility events.
@@ -64,7 +67,7 @@ class AccessibilityBridgeMacDelegate : public AccessibilityBridge::Accessibility
6467

6568
//---------------------------------------------------------------------------
6669
/// @brief Whether the given event is in current pending events.
67-
/// @param[in] event_type The event you would like to look up.
70+
/// @param[in] event_type The event to look up.
6871
bool HasPendingEvent(ui::AXEventGenerator::Event event) const;
6972

7073
//---------------------------------------------------------------------------
@@ -76,6 +79,7 @@ class AccessibilityBridgeMacDelegate : public AccessibilityBridge::Accessibility
7679
const ui::AXNode& ax_node) const;
7780

7881
__weak FlutterEngine* flutter_engine_;
82+
__weak FlutterViewController* view_controller_;
7983
};
8084

8185
} // namespace flutter

0 commit comments

Comments
 (0)