Support customizing standard accessibility actions on Android. (#5823)

Author: Jonah Williams

lib/ui/semantics.dart

@@ -585,7 +585,9 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
     Float64List transform,
     Int32List childrenInTraversalOrder,
     Int32List childrenInHitTestOrder,
+    @Deprecated('use additionalActions instead')
     Int32List customAcccessibilityActions,
+    Int32List additionalActions,
   }) {
     if (transform.length != 16)
       throw new ArgumentError('transform argument must have 16 entries.');
@@ -611,7 +613,7 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
       transform,
       childrenInTraversalOrder,
       childrenInHitTestOrder,
-      customAcccessibilityActions,
+      additionalActions ?? customAcccessibilityActions,
     );
   }
   void _updateNode(
@@ -636,19 +638,30 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
     Float64List transform,
     Int32List childrenInTraversalOrder,
     Int32List childrenInHitTestOrder,
-    Int32List customAcccessibilityActions,
+    Int32List additionalActions,
   ) native 'SemanticsUpdateBuilder_updateNode';
 
-  /// Update the custom accessibility action associated with the given `id`.
+  /// Update the custom semantics action associated with the given `id`.
+  ///
+  /// The name of the action exposed to the user is the `label`. For overriden
+  /// standard actions this value is ignored.
+  ///
+  /// The `hint` should describe what happens when an action occurs, not the
+  /// manner in which a tap is accomplished. For example, use "delete" instead
+  /// of "double tap to delete".
+  ///
+  /// The text direction of the `hint` and `label` is the same as the global
+  /// window.
   ///
-  /// The name of the action exposed to the user is the `label`. The text
-  /// direction of this label is the same as the global window.
-  void updateCustomAction({int id, String label}) {
+  /// For overriden standard actions, `overrideId` corresponds with a
+  /// [SemanticsAction.index] value. For custom actions this argument should not be
+  /// provided.
+  void updateCustomAction({int id, String label, String hint, int overrideId = -1}) {
     assert(id != null);
-    assert(label != null && label != '');
-    _updateCustomAction(id, label);
+    assert(overrideId != null);
+    _updateCustomAction(id, label, hint, overrideId);
   }
-  void _updateCustomAction(int id, String label) native 'SemanticsUpdateBuilder_updateCustomAction';
+  void _updateCustomAction(int id, String label, String hint, int overrideId) native 'SemanticsUpdateBuilder_updateCustomAction';
 
   /// Creates a [SemanticsUpdate] object that encapsulates the updates recorded
   /// by this object.

lib/ui/semantics/custom_accessibility_action.h

@@ -20,7 +20,9 @@ struct CustomAccessibilityAction {
   ~CustomAccessibilityAction();
 
   int32_t id = 0;
+  int32_t overrideId = -1;
   std::string label;
+  std::string hint;
 };
 
 // Contains custom accessibility actions that need to be updated.

lib/ui/semantics/semantics_update_builder.cc

@@ -88,10 +88,15 @@ void SemanticsUpdateBuilder::updateNode(
   nodes_[id] = node;
 }
 
-void SemanticsUpdateBuilder::updateCustomAction(int id, std::string label) {
+void SemanticsUpdateBuilder::updateCustomAction(int id,
+                                                std::string label,
+                                                std::string hint,
+                                                int overrideId) {
   CustomAccessibilityAction action;
   action.id = id;
+  action.overrideId = overrideId;
   action.label = label;
+  action.hint = hint;
   actions_[id] = action;
 }
 

lib/ui/semantics/semantics_update_builder.h

@@ -47,7 +47,10 @@ class SemanticsUpdateBuilder
                   const tonic::Int32List& childrenInHitTestOrder,
                   const tonic::Int32List& customAccessibilityActions);
 
-  void updateCustomAction(int id, std::string label);
+  void updateCustomAction(int id,
+                          std::string label,
+                          std::string hint,
+                          int overrideId);
 
   fxl::RefPtr<SemanticsUpdate> build();
 

shell/platform/android/io/flutter/view/AccessibilityBridge.java

@@ -223,12 +223,24 @@ public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) {
                 !object.hasFlag(Flag.HAS_ENABLED_STATE) || object.hasFlag(Flag.IS_ENABLED));
 
         if (object.hasAction(Action.TAP)) {
-            result.addAction(AccessibilityNodeInfo.ACTION_CLICK);
-            result.setClickable(true);
+            if (Build.VERSION.SDK_INT >= 21 && object.onTapOverride != null) {
+                result.addAction(new AccessibilityNodeInfo.AccessibilityAction(
+                    AccessibilityNodeInfo.ACTION_CLICK, object.onTapOverride.hint));
+                result.setClickable(true);
+            } else {
+                result.addAction(AccessibilityNodeInfo.ACTION_CLICK);
+                result.setClickable(true);
+            }
         }
         if (object.hasAction(Action.LONG_PRESS)) {
-            result.addAction(AccessibilityNodeInfo.ACTION_LONG_CLICK);
-            result.setLongClickable(true);
+            if (Build.VERSION.SDK_INT >= 21 && object.onLongPressOverride != null) {
+                result.addAction(new AccessibilityNodeInfo.AccessibilityAction(AccessibilityNodeInfo.ACTION_LONG_CLICK,
+                    object.onLongPressOverride.hint));
+                result.setLongClickable(true);
+            } else {
+                result.addAction(AccessibilityNodeInfo.ACTION_LONG_CLICK);
+                result.setLongClickable(true);
+            }
         }
         if (object.hasAction(Action.SCROLL_LEFT) || object.hasAction(Action.SCROLL_UP)
                 || object.hasAction(Action.SCROLL_RIGHT) || object.hasAction(Action.SCROLL_DOWN)) {
@@ -288,8 +300,8 @@ public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) {
 
         // Actions on the local context menu
         if (Build.VERSION.SDK_INT >= 21) {
-            if (object.customAccessibilityAction != null) {
-                for (CustomAccessibilityAction action : object.customAccessibilityAction) {
+            if (object.customAccessibilityActions != null) {
+                for (CustomAccessibilityAction action : object.customAccessibilityActions) {
                     result.addAction(new AccessibilityNodeInfo.AccessibilityAction(
                             action.resourceId, action.label));
                 }
@@ -547,8 +559,11 @@ void updateCustomAccessibilityActions(ByteBuffer buffer, String[] strings) {
         while (buffer.hasRemaining()) {
             int id = buffer.getInt();
             CustomAccessibilityAction action = getOrCreateAction(id);
+            action.overrideId = buffer.getInt();
             int stringIndex = buffer.getInt();
             action.label = stringIndex == -1 ? null : strings[stringIndex];
+            stringIndex = buffer.getInt();
+            action.hint = stringIndex == -1 ? null : strings[stringIndex];
         }
     }
 
@@ -851,9 +866,17 @@ private class CustomAccessibilityAction {
         /// does not collide with existing Android accessibility actions.
         int resourceId = -1;
         int id = -1;
+        int overrideId = -1;
 
         /// The label is the user presented value which is displayed in the local context menu.
         String label;
+
+        /// The hint is the text used in overriden standard actions.
+        String hint;
+
+        boolean isStandardAction() {
+            return overrideId != -1;
+        }
     }
     /// Value is derived from ACTION_TYPE_MASK in AccessibilityNodeInfo.java
     static int firstResourceId = 267386881;
@@ -897,7 +920,9 @@ private class SemanticsObject {
         SemanticsObject parent;
         List<SemanticsObject> childrenInTraversalOrder;
         List<SemanticsObject> childrenInHitTestOrder;
-        List<CustomAccessibilityAction> customAccessibilityAction;
+        List<CustomAccessibilityAction> customAccessibilityActions;
+        CustomAccessibilityAction onTapOverride;
+        CustomAccessibilityAction onLongPressOverride;
 
         private boolean inverseTransformDirty = true;
         private float[] inverseTransform;
@@ -1030,17 +1055,27 @@ void updateWith(ByteBuffer buffer, String[] strings) {
             }
             final int actionCount = buffer.getInt();
             if (actionCount == 0) {
-                customAccessibilityAction = null;
+                customAccessibilityActions = null;
             } else {
-                if (customAccessibilityAction == null)
-                    customAccessibilityAction =
+                if (customAccessibilityActions == null)
+                    customAccessibilityActions =
                             new ArrayList<CustomAccessibilityAction>(actionCount);
                 else
-                    customAccessibilityAction.clear();
+                    customAccessibilityActions.clear();
 
                 for (int i = 0; i < actionCount; i++) {
                     CustomAccessibilityAction action = getOrCreateAction(buffer.getInt());
-                    customAccessibilityAction.add(action);
+                    if (action.overrideId == Action.TAP.value) {
+                        onTapOverride = action;
+                    } else if (action.overrideId == Action.LONG_PRESS.value) {
+                        onLongPressOverride = action;
+                    } else {
+                        // If we recieve a different overrideId it means that we were passed
+                        // a standard action to override that we don't yet support.
+                        assert action.overrideId == -1;
+                        customAccessibilityActions.add(action);
+                    }
+                    customAccessibilityActions.add(action);
                 }
             }
         }

shell/platform/android/platform_view_android.cc

@@ -183,7 +183,7 @@ void PlatformViewAndroid::UpdateSemantics(
     blink::CustomAccessibilityActionUpdates actions) {
   constexpr size_t kBytesPerNode = 36 * sizeof(int32_t);
   constexpr size_t kBytesPerChild = sizeof(int32_t);
-  constexpr size_t kBytesPerAction = 2 * sizeof(int32_t);
+  constexpr size_t kBytesPerAction = 4 * sizeof(int32_t);
 
   JNIEnv* env = fml::jni::AttachCurrentThread();
   {
@@ -284,12 +284,19 @@ void PlatformViewAndroid::UpdateSemantics(
       // sending.
       const blink::CustomAccessibilityAction& action = value.second;
       actions_buffer_int32[actions_position++] = action.id;
+      actions_buffer_int32[actions_position++] = action.overrideId;
       if (action.label.empty()) {
         actions_buffer_int32[actions_position++] = -1;
       } else {
         actions_buffer_int32[actions_position++] = action_strings.size();
         action_strings.push_back(action.label);
       }
+      if (action.hint.empty()) {
+        actions_buffer_int32[actions_position++] = -1;
+      } else {
+        actions_buffer_int32[actions_position++] = action_strings.size();
+        action_strings.push_back(action.hint);
+      }
     }
 
     // Calling NewDirectByteBuffer in API level 22 and below with a size of zero

shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm

@@ -542,6 +542,11 @@ - (BOOL)accessibilityScroll:(UIAccessibilityScrollDirection)direction {
           [[[NSMutableArray alloc] init] autorelease];
       for (int32_t action_id : node.customAccessibilityActions) {
         blink::CustomAccessibilityAction& action = actions_[action_id];
+        if (action.overrideId != -1) {
+          // iOS does not support overriding standard actions, so we ignore any
+          // custom actions that have an override id provided.
+          continue;
+        }
         NSString* label = @(action.label.data());
         SEL selector = @selector(onCustomAccessibilityAction:);
         FlutterCustomAccessibilityAction* customAction =