From a635eb37b90d0fcf1ab0e4b06ef86900f83795fe Mon Sep 17 00:00:00 2001 From: Bruno Leroux Date: Wed, 23 Nov 2022 14:19:04 +0100 Subject: [PATCH 1/3] [macos] Synthesize modifier keys events on pointer events --- .../Source/FlutterEmbedderKeyResponder.h | 9 ++++ .../Source/FlutterEmbedderKeyResponder.mm | 13 +++++ .../Source/FlutterEmbedderKeyResponderTest.mm | 51 +++++++++++++++++++ .../framework/Source/FlutterKeyboardManager.h | 9 ++++ .../Source/FlutterKeyboardManager.mm | 8 +++ .../framework/Source/FlutterViewController.mm | 2 + 6 files changed, 92 insertions(+) diff --git a/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.h b/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.h index ab2b1c107d785..7a3b9ee51bdca 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.h +++ b/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.h @@ -30,4 +30,13 @@ typedef void (^FlutterSendEmbedderKeyEvent)(const FlutterKeyEvent& /* event */, */ - (nonnull instancetype)initWithSendEvent:(_Nonnull FlutterSendEmbedderKeyEvent)sendEvent; +/** + * Synthesize modifier keys events. + * + * If needed, synthesize modifier keys up and down events by comparing their + * current pressing states with the given modifier flags. + */ +- (void)syncModifiersIfNeeded:(NSEventModifierFlags)modifierFlags + timestamp:(NSTimeInterval)timestamp; + @end diff --git a/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.mm b/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.mm index 5854bb02f35f6..4a5d14f4d847f 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.mm @@ -780,6 +780,19 @@ - (void)handleResponse:(BOOL)handled forId:(uint64_t)responseId { [_pendingResponses removeObjectForKey:@(responseId)]; } +- (void)syncModifiersIfNeeded:(NSEventModifierFlags)modifierFlags + timestamp:(NSTimeInterval)timestamp { + FlutterAsyncKeyCallback replyCallback = ^(BOOL handled) { + // Do nothing. + }; + FlutterKeyCallbackGuard* guardedCallback = + [[FlutterKeyCallbackGuard alloc] initWithCallback:replyCallback]; + [self synchronizeModifiers:modifierFlags + ignoringFlags:0 + timestamp:timestamp + guard:guardedCallback]; +} + @end namespace { diff --git a/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponderTest.mm b/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponderTest.mm index faa6703b050ab..d922081cf0e6d 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponderTest.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponderTest.mm @@ -5,6 +5,7 @@ #import #import +#import "KeyCodeMap_Internal.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.h" #include "flutter/shell/platform/embedder/test_utils/key_codes.g.h" #import "flutter/testing/testing.h" @@ -1216,4 +1217,54 @@ - (void)dealloc { [events removeAllObjects]; } +// Synthesize modifier keys events if needed. +TEST(FlutterEmbedderKeyResponderUnittests, SynchronizeModifiersIfNeeded) { + __block NSMutableArray* events = [[NSMutableArray alloc] init]; + + FlutterEmbedderKeyResponder* responder = [[FlutterEmbedderKeyResponder alloc] + initWithSendEvent:^(const FlutterKeyEvent& event, _Nullable FlutterKeyEventCallback callback, + _Nullable _VoidPtr user_data) { + [events addObject:[[TestKeyEvent alloc] initWithEvent:&event + callback:callback + userData:user_data]]; + }]; + + // Zeroed modifier flag should not synthesize events. + [responder syncModifiersIfNeeded:0x00 timestamp:0]; + EXPECT_EQ([events count], 0u); + [events removeAllObjects]; + + // For each modifier key, check that key events are synthesized. + [flutter::keyCodeToModifierFlag + enumerateKeysAndObjectsUsingBlock:^(NSNumber* keyCode, NSNumber* flag, BOOL* stop) { + FlutterKeyEvent* event; + NSNumber* logicalKey; + NSNumber* physicalKey; + + // Should synthesize down event. + [responder syncModifiersIfNeeded:[flag unsignedLongValue] timestamp:0]; + EXPECT_EQ([events count], 1u); + event = events[0].data; + logicalKey = [flutter::keyCodeToLogicalKey objectForKey:keyCode]; + physicalKey = [flutter::keyCodeToPhysicalKey objectForKey:keyCode]; + EXPECT_EQ(event->type, kFlutterKeyEventTypeDown); + EXPECT_EQ(event->logical, logicalKey.unsignedLongLongValue); + EXPECT_EQ(event->physical, physicalKey.unsignedLongLongValue); + EXPECT_EQ(event->synthesized, true); + + // Should synthesize up event. + [responder syncModifiersIfNeeded:0x00 timestamp:0]; + EXPECT_EQ([events count], 2u); + event = events[1].data; + logicalKey = [flutter::keyCodeToLogicalKey objectForKey:keyCode]; + physicalKey = [flutter::keyCodeToPhysicalKey objectForKey:keyCode]; + EXPECT_EQ(event->type, kFlutterKeyEventTypeUp); + EXPECT_EQ(event->logical, logicalKey.unsignedLongLongValue); + EXPECT_EQ(event->physical, physicalKey.unsignedLongLongValue); + EXPECT_EQ(event->synthesized, true); + + [events removeAllObjects]; + }]; +} + } // namespace flutter::testing diff --git a/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.h b/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.h index c8ba5058aabfb..64ca1a35f65b0 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.h +++ b/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.h @@ -48,4 +48,13 @@ */ - (BOOL)isDispatchingKeyEvent:(nonnull NSEvent*)event; +/** + * Synthesize modifier keys events. + * + * If needed, synthesize modifier keys up and down events by comparing their + * current pressing states with the given modifier flags. + */ +- (void)syncModifiersIfNeeded:(NSEventModifierFlags)modifierFlags + timestamp:(NSTimeInterval)timestamp; + @end diff --git a/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.mm b/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.mm index 116c94537d9be..78c18dd0b08e0 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.mm @@ -320,4 +320,12 @@ - (void)buildLayout { } } +- (void)syncModifiersIfNeeded:(NSEventModifierFlags)modifierFlags + timestamp:(NSTimeInterval)timestamp { + // The embedder responder is the first element in _primaryResponders. + FlutterEmbedderKeyResponder* embedderResponder = + (FlutterEmbedderKeyResponder*)_primaryResponders[0]; + [embedderResponder syncModifiersIfNeeded:modifierFlags timestamp:timestamp]; +} + @end diff --git a/shell/platform/darwin/macos/framework/Source/FlutterViewController.mm b/shell/platform/darwin/macos/framework/Source/FlutterViewController.mm index 9034e8deaa6c1..2c9e76e65990d 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterViewController.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterViewController.mm @@ -661,6 +661,8 @@ - (void)dispatchMouseEvent:(NSEvent*)event phase:(FlutterPointerPhase)phase { flutterEvent.scroll_delta_y = scaledDeltaY; } } + + [_keyboardManager syncModifiersIfNeeded:event.modifierFlags timestamp:event.timestamp]; [_engine sendPointerEvent:flutterEvent]; // Update tracking of state as reported to Flutter. From 470ba89f99febf312ec56ab4e7bafd6b0a2c5a6f Mon Sep 17 00:00:00 2001 From: Bruno Leroux Date: Wed, 11 Jan 2023 22:28:58 +0100 Subject: [PATCH 2/3] Move test to FlutterViewControllerTest --- .../Source/FlutterEmbedderKeyResponderTest.mm | 51 --------- .../Source/FlutterViewControllerTest.mm | 107 ++++++++++++++++++ 2 files changed, 107 insertions(+), 51 deletions(-) diff --git a/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponderTest.mm b/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponderTest.mm index d922081cf0e6d..faa6703b050ab 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponderTest.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponderTest.mm @@ -5,7 +5,6 @@ #import #import -#import "KeyCodeMap_Internal.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.h" #include "flutter/shell/platform/embedder/test_utils/key_codes.g.h" #import "flutter/testing/testing.h" @@ -1217,54 +1216,4 @@ - (void)dealloc { [events removeAllObjects]; } -// Synthesize modifier keys events if needed. -TEST(FlutterEmbedderKeyResponderUnittests, SynchronizeModifiersIfNeeded) { - __block NSMutableArray* events = [[NSMutableArray alloc] init]; - - FlutterEmbedderKeyResponder* responder = [[FlutterEmbedderKeyResponder alloc] - initWithSendEvent:^(const FlutterKeyEvent& event, _Nullable FlutterKeyEventCallback callback, - _Nullable _VoidPtr user_data) { - [events addObject:[[TestKeyEvent alloc] initWithEvent:&event - callback:callback - userData:user_data]]; - }]; - - // Zeroed modifier flag should not synthesize events. - [responder syncModifiersIfNeeded:0x00 timestamp:0]; - EXPECT_EQ([events count], 0u); - [events removeAllObjects]; - - // For each modifier key, check that key events are synthesized. - [flutter::keyCodeToModifierFlag - enumerateKeysAndObjectsUsingBlock:^(NSNumber* keyCode, NSNumber* flag, BOOL* stop) { - FlutterKeyEvent* event; - NSNumber* logicalKey; - NSNumber* physicalKey; - - // Should synthesize down event. - [responder syncModifiersIfNeeded:[flag unsignedLongValue] timestamp:0]; - EXPECT_EQ([events count], 1u); - event = events[0].data; - logicalKey = [flutter::keyCodeToLogicalKey objectForKey:keyCode]; - physicalKey = [flutter::keyCodeToPhysicalKey objectForKey:keyCode]; - EXPECT_EQ(event->type, kFlutterKeyEventTypeDown); - EXPECT_EQ(event->logical, logicalKey.unsignedLongLongValue); - EXPECT_EQ(event->physical, physicalKey.unsignedLongLongValue); - EXPECT_EQ(event->synthesized, true); - - // Should synthesize up event. - [responder syncModifiersIfNeeded:0x00 timestamp:0]; - EXPECT_EQ([events count], 2u); - event = events[1].data; - logicalKey = [flutter::keyCodeToLogicalKey objectForKey:keyCode]; - physicalKey = [flutter::keyCodeToPhysicalKey objectForKey:keyCode]; - EXPECT_EQ(event->type, kFlutterKeyEventTypeUp); - EXPECT_EQ(event->logical, logicalKey.unsignedLongLongValue); - EXPECT_EQ(event->physical, physicalKey.unsignedLongLongValue); - EXPECT_EQ(event->synthesized, true); - - [events removeAllObjects]; - }]; -} - } // namespace flutter::testing diff --git a/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm b/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm index 903065731a31a..8f0e8470825fd 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#import "KeyCodeMap_Internal.h" #import "flutter/shell/platform/darwin/macos/framework/Headers/FlutterViewController.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController_Internal.h" @@ -13,8 +14,27 @@ #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine_Internal.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterRenderer.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTestUtils.h" +#include "flutter/shell/platform/embedder/test_utils/key_codes.g.h" #import "flutter/testing/testing.h" +// A wrap to convert FlutterKeyEvent to a ObjC class. +@interface KeyEventWrapper : NSObject +@property(nonatomic) FlutterKeyEvent* data; +- (nonnull instancetype)initWithEvent:(const FlutterKeyEvent*)event; +@end + +@implementation KeyEventWrapper +- (instancetype)initWithEvent:(const FlutterKeyEvent*)event { + self = [super init]; + _data = new FlutterKeyEvent(*event); + return self; +} + +- (void)dealloc { + delete _data; +} +@end + @interface FlutterViewControllerTestObjC : NSObject - (bool)testKeyEventsAreSentToFramework; - (bool)testKeyEventsArePropagatedIfNotHandled; @@ -22,6 +42,7 @@ - (bool)testKeyEventsAreNotPropagatedIfHandled; - (bool)testFlagsChangedEventsArePropagatedIfNotHandled; - (bool)testKeyboardIsRestartedOnEngineRestart; - (bool)testTrackpadGesturesAreSentToFramework; +- (bool)testModifierKeysAreSynthesizedOnMouseMove; - (bool)testViewWillAppearCalledMultipleTimes; - (bool)testFlutterViewIsConfigured; @@ -30,6 +51,8 @@ + (void)respondFalseForSendEvent:(const FlutterKeyEvent&)event userData:(nullable void*)userData; @end +using namespace ::flutter::testing::keycodes; + namespace flutter::testing { namespace { @@ -69,6 +92,19 @@ id MockGestureEvent(NSEventType type, NSEventPhase phase, double magnification, OCMStub([mock flagsChanged:[OCMArg any]]).andDo(nil); return mock; } + +NSEvent* CreateMouseEvent(NSEventModifierFlags modifierFlags) { + return [NSEvent mouseEventWithType:NSEventTypeMouseMoved + location:NSZeroPoint + modifierFlags:modifierFlags + timestamp:0 + windowNumber:0 + context:nil + eventNumber:0 + clickCount:1 + pressure:1.0]; +} + } // namespace TEST(FlutterViewController, HasViewThatHidesOtherViewsInAccessibility) { @@ -161,6 +197,10 @@ id MockGestureEvent(NSEventType type, NSEventPhase phase, double magnification, ASSERT_TRUE([[FlutterViewControllerTestObjC alloc] testTrackpadGesturesAreSentToFramework]); } +TEST(FlutterViewControllerTest, TestModifierKeysAreSynthesizedOnMouseMove) { + ASSERT_TRUE([[FlutterViewControllerTestObjC alloc] testModifierKeysAreSynthesizedOnMouseMove]); +} + TEST(FlutterViewControllerTest, testViewWillAppearCalledMultipleTimes) { ASSERT_TRUE([[FlutterViewControllerTestObjC alloc] testViewWillAppearCalledMultipleTimes]); } @@ -763,4 +803,71 @@ - (bool)testViewWillAppearCalledMultipleTimes { return true; } +- (bool)testModifierKeysAreSynthesizedOnMouseMove { + id engineMock = OCMClassMock([FlutterEngine class]); + // Need to return a real renderer to allow view controller to load. + FlutterRenderer* renderer_ = [[FlutterRenderer alloc] initWithFlutterEngine:engineMock]; + OCMStub([engineMock renderer]).andReturn(renderer_); + + // Capture calls to sendKeyEvent + __block NSMutableArray* events = + [[NSMutableArray alloc] init]; + OCMStub([[engineMock ignoringNonObjectArgs] sendKeyEvent:FlutterKeyEvent {} + callback:nil + userData:nil]) + .andDo((^(NSInvocation* invocation) { + FlutterKeyEvent* event; + [invocation getArgument:&event atIndex:2]; + [events addObject:[[KeyEventWrapper alloc] initWithEvent:event]]; + })); + + FlutterViewController* viewController = [[FlutterViewController alloc] initWithEngine:engineMock + nibName:@"" + bundle:nil]; + [viewController loadView]; + [engineMock setViewController:viewController]; + [viewController viewWillAppear]; + + // Zeroed modifier flag should not synthesize events. + NSEvent* mouseEvent = flutter::testing::CreateMouseEvent(0x00); + [viewController mouseMoved:mouseEvent]; + EXPECT_EQ([events count], 0u); + + // For each modifier key, check that key events are synthesized. + [flutter::keyCodeToModifierFlag + enumerateKeysAndObjectsUsingBlock:^(NSNumber* keyCode, NSNumber* flag, BOOL* stop) { + FlutterKeyEvent* event; + NSNumber* logicalKey; + NSNumber* physicalKey; + + // Should synthesize down event. + NSEvent* mouseEvent = flutter::testing::CreateMouseEvent([flag unsignedLongValue]); + [viewController mouseMoved:mouseEvent]; + EXPECT_EQ([events count], 1u); + event = events[0].data; + logicalKey = [flutter::keyCodeToLogicalKey objectForKey:keyCode]; + physicalKey = [flutter::keyCodeToPhysicalKey objectForKey:keyCode]; + EXPECT_EQ(event->type, kFlutterKeyEventTypeDown); + EXPECT_EQ(event->logical, logicalKey.unsignedLongLongValue); + EXPECT_EQ(event->physical, physicalKey.unsignedLongLongValue); + EXPECT_EQ(event->synthesized, true); + + // Should synthesize up event. + mouseEvent = flutter::testing::CreateMouseEvent(0x00); + [viewController mouseMoved:mouseEvent]; + EXPECT_EQ([events count], 2u); + event = events[1].data; + logicalKey = [flutter::keyCodeToLogicalKey objectForKey:keyCode]; + physicalKey = [flutter::keyCodeToPhysicalKey objectForKey:keyCode]; + EXPECT_EQ(event->type, kFlutterKeyEventTypeUp); + EXPECT_EQ(event->logical, logicalKey.unsignedLongLongValue); + EXPECT_EQ(event->physical, physicalKey.unsignedLongLongValue); + EXPECT_EQ(event->synthesized, true); + + [events removeAllObjects]; + }]; + + return true; +} + @end From 6b3f193a8e980e2722dba2de147dd936fbb09040 Mon Sep 17 00:00:00 2001 From: Bruno Leroux Date: Fri, 20 Jan 2023 09:26:08 +0100 Subject: [PATCH 3/3] Simplify by using 'for in' --- .../Source/FlutterViewControllerTest.mm | 64 +++++++++---------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm b/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm index 8f0e8470825fd..dbc98870a0661 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm @@ -834,38 +834,38 @@ - (bool)testModifierKeysAreSynthesizedOnMouseMove { EXPECT_EQ([events count], 0u); // For each modifier key, check that key events are synthesized. - [flutter::keyCodeToModifierFlag - enumerateKeysAndObjectsUsingBlock:^(NSNumber* keyCode, NSNumber* flag, BOOL* stop) { - FlutterKeyEvent* event; - NSNumber* logicalKey; - NSNumber* physicalKey; - - // Should synthesize down event. - NSEvent* mouseEvent = flutter::testing::CreateMouseEvent([flag unsignedLongValue]); - [viewController mouseMoved:mouseEvent]; - EXPECT_EQ([events count], 1u); - event = events[0].data; - logicalKey = [flutter::keyCodeToLogicalKey objectForKey:keyCode]; - physicalKey = [flutter::keyCodeToPhysicalKey objectForKey:keyCode]; - EXPECT_EQ(event->type, kFlutterKeyEventTypeDown); - EXPECT_EQ(event->logical, logicalKey.unsignedLongLongValue); - EXPECT_EQ(event->physical, physicalKey.unsignedLongLongValue); - EXPECT_EQ(event->synthesized, true); - - // Should synthesize up event. - mouseEvent = flutter::testing::CreateMouseEvent(0x00); - [viewController mouseMoved:mouseEvent]; - EXPECT_EQ([events count], 2u); - event = events[1].data; - logicalKey = [flutter::keyCodeToLogicalKey objectForKey:keyCode]; - physicalKey = [flutter::keyCodeToPhysicalKey objectForKey:keyCode]; - EXPECT_EQ(event->type, kFlutterKeyEventTypeUp); - EXPECT_EQ(event->logical, logicalKey.unsignedLongLongValue); - EXPECT_EQ(event->physical, physicalKey.unsignedLongLongValue); - EXPECT_EQ(event->synthesized, true); - - [events removeAllObjects]; - }]; + for (NSNumber* keyCode in flutter::keyCodeToModifierFlag) { + FlutterKeyEvent* event; + NSNumber* logicalKey; + NSNumber* physicalKey; + NSNumber* flag = flutter::keyCodeToModifierFlag[keyCode]; + + // Should synthesize down event. + NSEvent* mouseEvent = flutter::testing::CreateMouseEvent([flag unsignedLongValue]); + [viewController mouseMoved:mouseEvent]; + EXPECT_EQ([events count], 1u); + event = events[0].data; + logicalKey = [flutter::keyCodeToLogicalKey objectForKey:keyCode]; + physicalKey = [flutter::keyCodeToPhysicalKey objectForKey:keyCode]; + EXPECT_EQ(event->type, kFlutterKeyEventTypeDown); + EXPECT_EQ(event->logical, logicalKey.unsignedLongLongValue); + EXPECT_EQ(event->physical, physicalKey.unsignedLongLongValue); + EXPECT_EQ(event->synthesized, true); + + // Should synthesize up event. + mouseEvent = flutter::testing::CreateMouseEvent(0x00); + [viewController mouseMoved:mouseEvent]; + EXPECT_EQ([events count], 2u); + event = events[1].data; + logicalKey = [flutter::keyCodeToLogicalKey objectForKey:keyCode]; + physicalKey = [flutter::keyCodeToPhysicalKey objectForKey:keyCode]; + EXPECT_EQ(event->type, kFlutterKeyEventTypeUp); + EXPECT_EQ(event->logical, logicalKey.unsignedLongLongValue); + EXPECT_EQ(event->physical, physicalKey.unsignedLongLongValue); + EXPECT_EQ(event->synthesized, true); + + [events removeAllObjects]; + }; return true; }