From 4c9ed8f893c4bbea8b4e3d4b1ed410570b85360a Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Fri, 27 Mar 2020 11:06:50 -0700 Subject: [PATCH 1/3] Add a gesture to open up a context menu; fix quitting --- webxr/openxr/input.rs | 82 +++++++++++++++++++++++++------ webxr/openxr/mod.rs | 112 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 172 insertions(+), 22 deletions(-) diff --git a/webxr/openxr/input.rs b/webxr/openxr/input.rs index 03b35a9a..276b92a3 100644 --- a/webxr/openxr/input.rs +++ b/webxr/openxr/input.rs @@ -75,23 +75,28 @@ pub struct OpenXRInput { action_grip_space: Space, action_click: Action, action_squeeze: Action, - hand: &'static str, + handedness: Handedness, click_state: ClickState, squeeze_state: ClickState, + menu_gesture_sustain: u8, +} + +fn hand_str(h: Handedness) -> &'static str { + match h { + Handedness::Right => "right", + Handedness::Left => "left", + _ => panic!("We don't support unknown handedness in openxr"), + } } impl OpenXRInput { pub fn new( id: InputId, - hand: Handedness, + handedness: Handedness, action_set: &ActionSet, session: &Session, ) -> Self { - let hand = match hand { - Handedness::Right => "right", - Handedness::Left => "left", - _ => panic!("We don't support unknown handedness in openxr"), - }; + let hand = hand_str(handedness); let action_aim_pose: Action = action_set .create_action( &format!("{}_hand_aim", hand), @@ -134,9 +139,10 @@ impl OpenXRInput { action_grip_space, action_click, action_squeeze, - hand, + handedness, click_state: ClickState::Done, squeeze_state: ClickState::Done, + menu_gesture_sustain: 0, } } @@ -182,23 +188,24 @@ impl OpenXRInput { select_name: &str, squeeze_name: Option<&str>, ) -> Vec { + let hand = hand_str(self.handedness); let path_aim_pose = instance - .string_to_path(&format!("/user/hand/{}/input/aim/pose", self.hand)) + .string_to_path(&format!("/user/hand/{}/input/aim/pose", hand)) .unwrap(); let binding_aim_pose = Binding::new(&self.action_aim_pose, path_aim_pose); let path_grip_pose = instance - .string_to_path(&format!("/user/hand/{}/input/grip/pose", self.hand)) + .string_to_path(&format!("/user/hand/{}/input/grip/pose", hand)) .unwrap(); let binding_grip_pose = Binding::new(&self.action_grip_pose, path_grip_pose); let path_click = instance - .string_to_path(&format!("/user/hand/{}/input/{}", self.hand, select_name)) + .string_to_path(&format!("/user/hand/{}/input/{}", hand, select_name)) .unwrap(); let binding_click = Binding::new(&self.action_click, path_click); let mut ret = vec![binding_aim_pose, binding_grip_pose, binding_click]; if let Some(squeeze_name) = squeeze_name { let path_squeeze = instance - .string_to_path(&format!("/user/hand/{}/input/{}", self.hand, squeeze_name)) + .string_to_path(&format!("/user/hand/{}/input/{}", hand, squeeze_name)) .unwrap(); let binding_squeeze = Binding::new(&self.action_squeeze, path_squeeze); ret.push(binding_squeeze); @@ -211,11 +218,53 @@ impl OpenXRInput { session: &Session, frame_state: &FrameState, base_space: &Space, - ) -> (InputFrame, Option, Option) { + ) -> ( + InputFrame, + /* clicked */ Option, + /* squeezed */ Option, + /* menu_selected */ bool, + ) { + use euclid::Vector3D; let target_ray_origin = pose_for(&self.action_aim_space, frame_state, base_space); let grip_origin = pose_for(&self.action_grip_space, frame_state, base_space); + let mut menu_selected = false; + // Check if the palm is facing up. This is our "menu" gesture. + if let Some(grip_origin) = grip_origin { + // The X axis of the grip is perpendicular to the palm, however its + // direction is the opposite for each hand + // + // We obtain a unit vector poking out of the palm + let x_dir = if let Handedness::Left = self.handedness { + 1.0 + } else { + -1.0 + }; + + // Rotate it by the grip to obtain the desired vector + let grip_x = grip_origin + .rotation + .transform_vector3d(Vector3D::new(x_dir, 0.0, 0.0)); + + // Dot product it with the "up" vector to see if it's pointing up + let angle = grip_x.dot(Vector3D::new(0.0, 1.0, 0.0)); + + // If the angle is close enough to 0, its cosine will be + // close to 1 + if angle > 0.9 { + self.menu_gesture_sustain += 1; + if self.menu_gesture_sustain > 60 { + menu_selected = true; + self.menu_gesture_sustain = 0; + } + } else { + self.menu_gesture_sustain = 0; + } + } else { + self.menu_gesture_sustain = 0; + } + let click = self.action_click.state(session, Path::NULL).unwrap(); let squeeze = self.action_squeeze.state(session, Path::NULL).unwrap(); @@ -232,7 +281,12 @@ impl OpenXRInput { grip_origin, }; - (input_frame, click_select_event, squeeze_select_event) + ( + input_frame, + click_select_event, + squeeze_select_event, + menu_selected, + ) } } diff --git a/webxr/openxr/mod.rs b/webxr/openxr/mod.rs index bccaa170..4fdec69d 100644 --- a/webxr/openxr/mod.rs +++ b/webxr/openxr/mod.rs @@ -65,19 +65,58 @@ pub trait SurfaceProviderRegistration: Send { fn clone(&self) -> Box; } +/// Provides a way to spawn and interact with context menus +pub trait ContextMenuProvider: Send { + /// Open a context menu, return a way to poll for the result + fn open_context_menu(&self) -> Box; + /// Clone self as a trait object + fn clone_object(&self) -> Box; +} + +/// A way to poll for the result of the context menu request +pub trait ContextMenuFuture { + fn poll(&self) -> ContextMenuResult; +} + +/// The result of polling on a context menu request +pub enum ContextMenuResult { + /// Session should exit + ExitSession, + /// Dialog was dismissed + Dismissed, + /// User has not acted on dialog + Pending, +} + +impl Drop for OpenXrDevice { + fn drop(&mut self) { + // This should be happening automatically in the destructors, + // but it isn't, presumably because there's an extra handle floating + // around somewhere + // XXXManishearth find out where that extra handle is + unsafe { + (self.instance.fp().destroy_session)(self.session.as_raw()); + (self.instance.fp().destroy_instance)(self.instance.as_raw()); + } + } +} + pub struct OpenXrDiscovery { gl_thread: Box, provider_registration: Box, + context_menu_provider: Box, } impl OpenXrDiscovery { pub fn new( gl_thread: Box, provider_registration: Box, + context_menu_provider: Box, ) -> Self { Self { gl_thread, provider_registration, + context_menu_provider, } } } @@ -132,6 +171,7 @@ impl DiscoveryAPI for OpenXrDiscovery { let provider_registration = self.provider_registration.clone(); let granted_features = init.validate(mode, &["local-floor".into()])?; let id = xr.id(); + let context_menu_provider = self.context_menu_provider.clone_object(); xr.spawn(move || { OpenXrDevice::new( gl_thread, @@ -139,6 +179,7 @@ impl DiscoveryAPI for OpenXrDiscovery { instance, granted_features, id, + context_menu_provider, ) }) } else { @@ -152,9 +193,9 @@ impl DiscoveryAPI for OpenXrDiscovery { } struct OpenXrDevice { + session: Session, instance: Instance, events: EventBuffer, - session: Session, frame_waiter: FrameWaiter, shared_data: Arc>, viewer_space: Space, @@ -167,6 +208,8 @@ struct OpenXrDevice { right_hand: OpenXRInput, left_hand: OpenXRInput, granted_features: Vec, + context_menu_provider: Box, + context_menu_future: Option>, } /// Data that is shared between the openxr thread and the @@ -368,6 +411,7 @@ impl OpenXrDevice { instance: Instance, granted_features: Vec, id: SessionId, + context_menu_provider: Box, ) -> Result { let (device_tx, device_rx) = crossbeam_channel::unbounded(); let (provider_tx, provider_rx) = crossbeam_channel::unbounded(); @@ -538,6 +582,8 @@ impl OpenXrDevice { right_hand, left_hand, granted_features, + context_menu_provider, + context_menu_future: None, }) } @@ -550,7 +596,8 @@ impl OpenXrDevice { match event { Some(SessionStateChanged(session_change)) => match session_change.state() { openxr::SessionState::EXITING | openxr::SessionState::LOSS_PENDING => { - break; + self.events.callback(Event::SessionEnd); + return false; } openxr::SessionState::STOPPING => { self.events @@ -581,7 +628,8 @@ impl OpenXrDevice { } }, Some(InstanceLossPending(_)) => { - break; + self.events.callback(Event::SessionEnd); + return false; } Some(_) => { // FIXME: Handle other events @@ -657,6 +705,17 @@ impl DeviceAPI for OpenXrDevice { // Session is not running anymore. return None; } + if let Some(ref context_menu_future) = self.context_menu_future { + match context_menu_future.poll() { + ContextMenuResult::ExitSession => { + self.quit(); + return None; + } + ContextMenuResult::Dismissed => self.context_menu_future = None, + ContextMenuResult::Pending => (), + } + } + let mut data = self.shared_data.lock().unwrap(); data.frame_state = self.frame_waiter.wait().expect("error waiting for frame"); let time_ns = time::precise_time_ns(); @@ -685,10 +744,10 @@ impl DeviceAPI for OpenXrDevice { self.session.sync_actions(&[active_action_set]).unwrap(); - let (right_input_frame, right_select, right_squeeze) = + let (mut right_input_frame, mut right_select, mut right_squeeze, right_menu) = self.right_hand .frame(&self.session, &data.frame_state, &data.space); - let (left_input_frame, left_select, left_squeeze) = + let (mut left_input_frame, mut left_select, mut left_squeeze, left_menu) = self.left_hand .frame(&self.session, &data.frame_state, &data.space); @@ -701,6 +760,22 @@ impl DeviceAPI for OpenXrDevice { vec![] }; + if (left_menu || right_menu) && self.context_menu_future.is_none() { + self.context_menu_future = Some(self.context_menu_provider.open_context_menu()); + } + + // Do not surface input info whilst the context menu is open + if self.context_menu_future.is_some() { + right_input_frame.target_ray_origin = None; + right_input_frame.grip_origin = None; + left_input_frame.target_ray_origin = None; + left_input_frame.grip_origin = None; + right_select = None; + right_squeeze = None; + left_select = None; + left_squeeze = None; + } + let frame = Frame { transform, inputs: vec![right_input_frame, left_input_frame], @@ -779,12 +854,33 @@ impl DeviceAPI for OpenXrDevice { fn quit(&mut self) { self.session.request_exit().unwrap(); + loop { + let mut buffer = openxr::EventDataBuffer::new(); + let event = self.instance.poll_event(&mut buffer).unwrap(); + match event { + Some(openxr::Event::SessionStateChanged(session_change)) => { + match session_change.state() { + openxr::SessionState::EXITING => { + self.events.callback(Event::SessionEnd); + break; + } + openxr::SessionState::STOPPING => { + self.session + .end() + .expect("Session failed to end on STOPPING"); + } + _ => (), + } + } + _ => (), + } + thread::sleep(Duration::from_millis(30)); + } } fn set_quitter(&mut self, _: Quitter) { - // Glwindow currently doesn't have any way to end its own session - // XXXManishearth add something for this that listens for the window - // being closed + // the quitter is only needed if we have anything from outside the render + // thread that can signal a quit. We don't. } fn update_clip_planes(&mut self, near: f32, far: f32) { From 82804a7bd70bcad1d4fe66c09dd193912dfe1b3b Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Mon, 30 Mar 2020 09:24:00 -0700 Subject: [PATCH 2/3] Return a struct from OpenXRInput.frame() --- webxr/openxr/input.rs | 30 ++++++++++++++++-------------- webxr/openxr/mod.rs | 41 ++++++++++++++++++++--------------------- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/webxr/openxr/input.rs b/webxr/openxr/input.rs index 276b92a3..967063cd 100644 --- a/webxr/openxr/input.rs +++ b/webxr/openxr/input.rs @@ -20,6 +20,14 @@ enum ClickState { Done, } +/// All the information on a single input frame +pub struct Frame { + pub frame: InputFrame, + pub select: Option, + pub squeeze: Option, + pub menu_selected: bool, +} + impl ClickState { fn update( &mut self, @@ -218,12 +226,7 @@ impl OpenXRInput { session: &Session, frame_state: &FrameState, base_space: &Space, - ) -> ( - InputFrame, - /* clicked */ Option, - /* squeezed */ Option, - /* menu_selected */ bool, - ) { + ) -> Frame { use euclid::Vector3D; let target_ray_origin = pose_for(&self.action_aim_space, frame_state, base_space); @@ -268,9 +271,8 @@ impl OpenXRInput { let click = self.action_click.state(session, Path::NULL).unwrap(); let squeeze = self.action_squeeze.state(session, Path::NULL).unwrap(); - let (click_is_active, click_select_event) = - self.click_state.update(&self.action_click, session); - let (squeeze_is_active, squeeze_select_event) = + let (click_is_active, click_event) = self.click_state.update(&self.action_click, session); + let (squeeze_is_active, squeeze_event) = self.squeeze_state.update(&self.action_squeeze, session); let input_frame = InputFrame { @@ -281,12 +283,12 @@ impl OpenXRInput { grip_origin, }; - ( - input_frame, - click_select_event, - squeeze_select_event, + Frame { + frame: input_frame, + select: click_event, + squeeze: squeeze_event, menu_selected, - ) + } } } diff --git a/webxr/openxr/mod.rs b/webxr/openxr/mod.rs index 4fdec69d..010025b2 100644 --- a/webxr/openxr/mod.rs +++ b/webxr/openxr/mod.rs @@ -744,12 +744,12 @@ impl DeviceAPI for OpenXrDevice { self.session.sync_actions(&[active_action_set]).unwrap(); - let (mut right_input_frame, mut right_select, mut right_squeeze, right_menu) = - self.right_hand - .frame(&self.session, &data.frame_state, &data.space); - let (mut left_input_frame, mut left_select, mut left_squeeze, left_menu) = - self.left_hand - .frame(&self.session, &data.frame_state, &data.space); + let mut right = self + .right_hand + .frame(&self.session, &data.frame_state, &data.space); + let mut left = self + .left_hand + .frame(&self.session, &data.frame_state, &data.space); // views() needs to reacquire the lock. drop(data); @@ -760,31 +760,31 @@ impl DeviceAPI for OpenXrDevice { vec![] }; - if (left_menu || right_menu) && self.context_menu_future.is_none() { + if (left.menu_selected || right.menu_selected) && self.context_menu_future.is_none() { self.context_menu_future = Some(self.context_menu_provider.open_context_menu()); } // Do not surface input info whilst the context menu is open if self.context_menu_future.is_some() { - right_input_frame.target_ray_origin = None; - right_input_frame.grip_origin = None; - left_input_frame.target_ray_origin = None; - left_input_frame.grip_origin = None; - right_select = None; - right_squeeze = None; - left_select = None; - left_squeeze = None; + right.frame.target_ray_origin = None; + right.frame.grip_origin = None; + left.frame.target_ray_origin = None; + left.frame.grip_origin = None; + right.select = None; + right.squeeze = None; + left.select = None; + left.squeeze = None; } let frame = Frame { transform, - inputs: vec![right_input_frame, left_input_frame], + inputs: vec![right.frame, left.frame], events, time_ns, sent_time: 0, }; - if let Some(right_select) = right_select { + if let Some(right_select) = right.select { self.events.callback(Event::Select( InputId(0), SelectKind::Select, @@ -792,7 +792,7 @@ impl DeviceAPI for OpenXrDevice { frame.clone(), )); } - if let Some(right_squeeze) = right_squeeze { + if let Some(right_squeeze) = right.squeeze { self.events.callback(Event::Select( InputId(0), SelectKind::Squeeze, @@ -800,7 +800,7 @@ impl DeviceAPI for OpenXrDevice { frame.clone(), )); } - if let Some(left_select) = left_select { + if let Some(left_select) = left.select { self.events.callback(Event::Select( InputId(1), SelectKind::Select, @@ -808,7 +808,7 @@ impl DeviceAPI for OpenXrDevice { frame.clone(), )); } - if let Some(left_squeeze) = left_squeeze { + if let Some(left_squeeze) = left.squeeze { self.events.callback(Event::Select( InputId(1), SelectKind::Squeeze, @@ -816,7 +816,6 @@ impl DeviceAPI for OpenXrDevice { frame.clone(), )); } - // todo use pose in input Some(frame) } From 8a53399b8742b84dac2f45ff363bad33cb7965a3 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Mon, 30 Mar 2020 09:26:25 -0700 Subject: [PATCH 3/3] Add constant for menu gesture sustain --- webxr/openxr/input.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/webxr/openxr/input.rs b/webxr/openxr/input.rs index 967063cd..8216332d 100644 --- a/webxr/openxr/input.rs +++ b/webxr/openxr/input.rs @@ -11,6 +11,10 @@ use webxr_api::InputId; use webxr_api::Native; use webxr_api::SelectEvent; +/// Number of frames to wait with the menu gesture before +/// opening the menu. +const MENU_GESTURE_SUSTAIN_THRESHOLD: u8 = 60; + #[derive(Copy, Clone, Debug, PartialEq, Eq)] enum ClickState { Clicking, @@ -257,7 +261,7 @@ impl OpenXRInput { // close to 1 if angle > 0.9 { self.menu_gesture_sustain += 1; - if self.menu_gesture_sustain > 60 { + if self.menu_gesture_sustain > MENU_GESTURE_SUSTAIN_THRESHOLD { menu_selected = true; self.menu_gesture_sustain = 0; }