OpenXR general hand interaction support (#10538)

* OpenXR general hand interaction support

* Update Assets/MRTK/Core/Definitions/Devices/ArticulatedHandDefinition.cs

Co-authored-by: Kurtis <[email protected]>

* Update Assets/MRTK/Providers/OpenXR/Scripts/MicrosoftArticulatedHand.cs

Co-authored-by: Kurtis <[email protected]>

* Update Assets/MRTK/Core/Definitions/Devices/ArticulatedHandDefinition.cs

Co-authored-by: Kurtis <[email protected]>

* pr feedback

* applied rotation to go from grip pose to source pose

* sanity check for palm joint retrieval

Co-authored-by: Kurtis <[email protected]>

Author: RogPodge

Assets/MRTK/Core/Definitions/Devices/ArticulatedHandDefinition.cs

@@ -86,8 +86,6 @@ public float ExitPinchDistance
             }
         }
 
-        private bool isPinching = false;
-
         /// <summary>
         /// The articulated hands default interactions.
         /// </summary>
@@ -120,8 +118,14 @@ public MixedRealityInteractionMapping[] DefaultInteractions
             new MixedRealityInputActionMapping("Teleport Pose", AxisType.DualAxis, DeviceInputType.ThumbStick),
         };
 
+
+        // Internal calculation of what the HandRay should be
+        // Useful as a fallback for hand ray data
+        protected virtual IHandRay HandRay { get; } = new HandRay();
+
         /// <summary>
         /// Calculates whether the current pose allows for pointing/distant interactions.
+        /// Equivalent to the HandRay's ShouldShowRay implementation <see cref="HandRay.ShouldShowRay"/>
         /// </summary>
         public bool IsInPointingPose
         {
@@ -168,13 +172,19 @@ protected bool IsInTeleportPose
 
                 Transform cameraTransform = mainCamera.transform;
 
+                Vector3 palmNormal = -palmPose.Up;
                 // We check if the palm up is roughly in line with the camera up
-                return Vector3.Dot(-palmPose.Up, cameraTransform.up) > 0.6f
+                return Vector3.Dot(palmNormal, cameraTransform.up) > 0.6f
                        // Thumb must be extended, and middle must be grabbing
                        && !isThumbGrabbing && isMiddleGrabbing;
             }
         }
 
+        /// <summary>
+        /// A bool tracking whether the hand definition is pinch or not
+        /// </summary>
+        private bool isPinching = false;
+
         /// <summary>
         /// Calculates whether the current the current joint pose is selecting (air tap gesture).
         /// </summary>
@@ -206,12 +216,31 @@ public bool IsPinching
             }
         }
 
+        public bool IsGrabbing => isIndexGrabbing && isMiddleGrabbing;
+
         private bool isIndexGrabbing;
         private bool isMiddleGrabbing;
         private bool isThumbGrabbing;
 
+        // Velocity internal states
+        private float deltaTimeStart;
+        private const int velocityUpdateInterval = 6;
+        private int frameOn = 0;
+
+        private readonly Vector3[] velocityPositionsCache = new Vector3[velocityUpdateInterval];
+        private readonly Vector3[] velocityNormalsCache = new Vector3[velocityUpdateInterval];
+        private Vector3 velocityPositionsSum = Vector3.zero;
+        private Vector3 velocityNormalsSum = Vector3.zero;
+
+        public Vector3 AngularVelocity { get; protected set; }
+
+        public Vector3 Velocity { get; protected set; }
+
+
         private static readonly ProfilerMarker UpdateHandJointsPerfMarker = new ProfilerMarker("[MRTK] ArticulatedHandDefinition.UpdateHandJoints");
 
+        #region Hand Definition Update functions
+
         /// <summary>
         /// Updates the current hand joints with new data.
         /// </summary>
@@ -257,6 +286,10 @@ public void UpdateCurrentIndexPose(MixedRealityInteractionMapping interactionMap
 
         private static readonly ProfilerMarker UpdateCurrentTeleportPosePerfMarker = new ProfilerMarker("[MRTK] ArticulatedHandDefinition.UpdateCurrentTeleportPose");
 
+        /// <summary>
+        /// Updates the MixedRealityInteractionMapping with the lastest teleport pose status and fires an event when appropriate
+        /// </summary>
+        /// <param name="interactionMapping">The teleport action's interaction mapping.</param>
         public void UpdateCurrentTeleportPose(MixedRealityInteractionMapping interactionMapping)
         {
             using (UpdateCurrentTeleportPosePerfMarker.Auto())
@@ -311,5 +344,80 @@ public void UpdateCurrentTeleportPose(MixedRealityInteractionMapping interaction
                 previousReadyToTeleport = isReadyForTeleport;
             }
         }
+
+        /// <summary>
+        /// Updates the MixedRealityInteractionMapping with the lastest pointer pose status and fires a corresponding pose event.
+        /// </summary>
+        /// <param name="interactionMapping">The pointer pose's interaction mapping.</param>
+        public void UpdatePointerPose(MixedRealityInteractionMapping interactionMapping)
+        {
+            if (!unityJointPoses.TryGetValue(TrackedHandJoint.IndexKnuckle, out var knucklePose)) return;
+            if (!unityJointPoses.TryGetValue(TrackedHandJoint.Palm, out var palmPose)) return;
+
+            Vector3 rayPosition = knucklePose.Position;
+            Vector3 palmNormal = -palmPose.Up;
+
+            HandRay.Update(rayPosition, palmNormal, CameraCache.Main.transform, Handedness);
+            Ray ray = HandRay.Ray;
+
+            MixedRealityPose pointerPose = new MixedRealityPose();
+            pointerPose.Position = ray.origin;
+            pointerPose.Rotation = Quaternion.LookRotation(ray.direction);
+
+            // Update the interaction data source
+            interactionMapping.PoseData = pointerPose;
+
+            // If our value changed raise it
+            if (interactionMapping.Changed)
+            {
+                // Raise input system event if it's enabled
+                CoreServices.InputSystem?.RaisePoseInputChanged(InputSource, Handedness, interactionMapping.MixedRealityInputAction, pointerPose);
+            }
+        }
+
+        /// <summary>
+        /// Updates the hand definition with its velocity
+        /// </summary>
+        public void UpdateVelocity()
+        {
+            if (unityJointPoses.TryGetValue(TrackedHandJoint.Palm, out var currentPalmPose))
+            {
+                Vector3 palmPosition = currentPalmPose.Position;
+                Vector3 palmNormal = -currentPalmPose.Up;
+
+                if (frameOn < velocityUpdateInterval)
+                {
+                    velocityPositionsCache[frameOn] = palmPosition;
+                    velocityPositionsSum += velocityPositionsCache[frameOn];
+                    velocityNormalsCache[frameOn] = palmNormal;
+                    velocityNormalsSum += velocityNormalsCache[frameOn];
+                }
+                else
+                {
+                    int frameIndex = frameOn % velocityUpdateInterval;
+
+                    float deltaTime = Time.unscaledTime - deltaTimeStart;
+
+                    Vector3 newPositionsSum = velocityPositionsSum - velocityPositionsCache[frameIndex] + palmPosition;
+                    Vector3 newNormalsSum = velocityNormalsSum - velocityNormalsCache[frameIndex] + palmNormal;
+
+                    Velocity = (newPositionsSum - velocityPositionsSum) / deltaTime / velocityUpdateInterval;
+
+                    Quaternion rotation = Quaternion.FromToRotation(velocityNormalsSum / velocityUpdateInterval, newNormalsSum / velocityUpdateInterval);
+                    Vector3 rotationRate = rotation.eulerAngles * Mathf.Deg2Rad;
+                    AngularVelocity = rotationRate / deltaTime;
+
+                    velocityPositionsCache[frameIndex] = palmPosition;
+                    velocityNormalsCache[frameIndex] = palmNormal;
+                    velocityPositionsSum = newPositionsSum;
+                    velocityNormalsSum = newNormalsSum;
+                }
+
+                deltaTimeStart = Time.unscaledTime;
+                frameOn++;
+            }
+        }
+
+        #endregion
     }
 }

Assets/MRTK/Providers/Oculus/XRSDK/MRTK-Quest/Scripts/Input/Controllers/OculusHand.cs

@@ -134,7 +134,6 @@ public void UpdateController(OVRHand hand, OVRSkeleton ovrSkeleton, Transform tr
                 currentPointerPose.Rotation = Quaternion.LookRotation(pointerForward, pointerUp);
 
                 currentGripPose = jointPoses[TrackedHandJoint.Palm];
-
                 CoreServices.InputSystem?.RaiseSourcePoseChanged(InputSource, this, currentGripPose);
 
                 UpdateVelocity();

Assets/MRTK/Providers/OpenXR/Scripts/MicrosoftArticulatedHand.cs

@@ -44,6 +44,11 @@ public MicrosoftArticulatedHand(TrackingState trackingState, Handedness controll
         private Quaternion currentPointerRotation = Quaternion.identity;
         private MixedRealityPose currentPointerPose = MixedRealityPose.ZeroIdentity;
 
+        // The rotation offset between the reported grip pose of a hand and the palm joint orientation.
+        // These values were calculated by comparing the platform's reported grip pose and palm pose.
+        private static readonly Quaternion rightPalmOffset = Quaternion.Inverse(new Quaternion(Mathf.Sqrt(0.125f), Mathf.Sqrt(0.125f), -Mathf.Sqrt(1.5f) / 2.0f, Mathf.Sqrt(1.5f) / 2.0f));
+        private static readonly Quaternion leftPalmOffset = Quaternion.Inverse(new Quaternion(Mathf.Sqrt(0.125f), -Mathf.Sqrt(0.125f), Mathf.Sqrt(1.5f) / 2.0f, Mathf.Sqrt(1.5f) / 2.0f));
+
         #region IMixedRealityHand Implementation
 
         /// <inheritdoc/>
@@ -54,8 +59,18 @@ public MicrosoftArticulatedHand(TrackingState trackingState, Handedness controll
         /// <inheritdoc/>
         public override bool IsInPointingPose => handDefinition.IsInPointingPose;
 
+        protected bool IsPinching => handDefinition.IsPinching;
+
+        // Pinch was also used as grab, we want to allow hand-curl grab not just pinch.
+        // Determine pinch and grab separately
+        protected bool IsGrabbing => handDefinition.IsGrabbing;
+
         private static readonly ProfilerMarker UpdateControllerPerfMarker = new ProfilerMarker("[MRTK] MicrosoftArticulatedHand.UpdateController");
 
+        // This bool is used to track whether or not we are recieving device data from the platform itself
+        // If we aren't we will attempt to infer some common input actions from the hand joint data (i.e. the pinch gesture, pointer positions etc)
+        private bool receivingDeviceInputs = false;
+
         /// <summary>
         /// The OpenXR plug-in uses extensions to expose all possible data, which might be surfaced through multiple input devices.
         /// This method is overridden to account for multiple input devices.
@@ -76,20 +91,87 @@ public override void UpdateController(InputDevice inputDevice)
                 if (inputDevice.TryGetFeatureValue(CommonUsages.devicePosition, out Vector3 _))
                 {
                     base.UpdateController(inputDevice);
+
+                    // We've gotten device data from the platform, don't attempt to infer other input actions
+                    // from the hand joint data
+                    receivingDeviceInputs = true;
                 }
                 else
                 {
                     UpdateHandData(inputDevice);
 
+                    // Updating the Index finger pose right after getting the hand data
+                    // regardless of whether device data is present
                     for (int i = 0; i < Interactions?.Length; i++)
                     {
-                        switch (Interactions[i].AxisType)
+                        var interactionMapping = Interactions[i];
+                        switch (interactionMapping.InputType)
                         {
-                            case AxisType.SixDof:
-                                UpdatePoseData(Interactions[i], inputDevice);
+                            case DeviceInputType.IndexFinger:
+                                handDefinition?.UpdateCurrentIndexPose(interactionMapping);
                                 break;
                         }
                     }
+
+                    // If we aren't getting device data, infer input actions, velocity, etc from hand joint data
+                    if (!receivingDeviceInputs)
+                    {
+                        for (int i = 0; i < Interactions?.Length; i++)
+                        {
+                            var interactionMapping = Interactions[i];
+                            switch (interactionMapping.InputType)
+                            {
+                                case DeviceInputType.SpatialGrip:
+                                    if (TryGetJoint(TrackedHandJoint.Palm, out MixedRealityPose currentGripPose))
+                                    {
+                                        interactionMapping.PoseData = currentGripPose;
+
+                                        if (interactionMapping.Changed)
+                                        {
+                                            CoreServices.InputSystem?.RaisePoseInputChanged(InputSource, ControllerHandedness, interactionMapping.MixedRealityInputAction, currentGripPose);
+
+                                            // Spatial Grip is also used as the basis for the source pose when device data is not provided
+                                            // We need to rotate it by an offset to properly represent the source pose.
+                                            MixedRealityPose CurrentControllerPose = currentGripPose;
+                                            CurrentControllerPose.Rotation *= (ControllerHandedness == Handedness.Left ? leftPalmOffset : rightPalmOffset);
+
+                                            CoreServices.InputSystem?.RaiseSourcePoseChanged(InputSource, this, CurrentControllerPose);
+                                            IsPositionAvailable = IsRotationAvailable = true;
+                                        }
+                                    }
+                                    break;
+                                case DeviceInputType.Select:
+                                case DeviceInputType.TriggerPress:
+                                case DeviceInputType.GripPress:
+                                    interactionMapping.BoolData = IsPinching || IsGrabbing;
+
+                                    if (interactionMapping.Changed)
+                                    {
+                                        if (interactionMapping.BoolData)
+                                        {
+                                            CoreServices.InputSystem?.RaiseOnInputDown(InputSource, ControllerHandedness, interactionMapping.MixedRealityInputAction);
+                                        }
+                                        else
+                                        {
+                                            CoreServices.InputSystem?.RaiseOnInputUp(InputSource, ControllerHandedness, interactionMapping.MixedRealityInputAction);
+                                        }
+                                    }
+                                    break;
+                                case DeviceInputType.SpatialPointer:
+                                    handDefinition?.UpdatePointerPose(interactionMapping);
+                                    break;
+                                // Gotta do this only for non-AR devices
+                                case DeviceInputType.ThumbStick:
+                                    handDefinition?.UpdateCurrentTeleportPose(interactionMapping);
+                                    break;
+                            }
+                        }
+
+                        // Update the controller velocity based on the hand definition's calculations
+                        handDefinition?.UpdateVelocity();
+                        Velocity = (handDefinition?.Velocity).Value;
+                        AngularVelocity = (handDefinition?.AngularVelocity).Value;
+                    }
                 }
             }
         }
@@ -186,9 +268,6 @@ protected override void UpdatePoseData(MixedRealityInteractionMapping interactio
             {
                 switch (interactionMapping.InputType)
                 {
-                    case DeviceInputType.IndexFinger:
-                        handDefinition?.UpdateCurrentIndexPose(interactionMapping);
-                        break;
                     case DeviceInputType.SpatialPointer:
                         if (inputDevice.TryGetFeatureValue(CustomUsages.PointerPosition, out currentPointerPosition))
                         {