Removing extra raycasts from eye gaze enabled objects (#10162)

* Initial commit for improving gaze provider

* Small finalizing removing EyeTrackingTarget Raycasts

* removing commented out code

* fixing gaps which caused unit tests to fail

* updated unit tests and added some safeguards

* fixed errors introduced by swapping profiles at runtime

* added checks to ensure eyetracking data is valid

* updated the gaze provider to update the hit point correctly

* updated focus provider comments

Author: RogPodge

Assets/MRTK/SDK/Features/Input/Handlers/EyeTrackingTarget.cs

@@ -170,15 +170,26 @@ public bool EyeCursorSnapToTargetCenter
         /// GameObject eye gaze is currently targeting, updated once per frame.
         /// null if no object with collider is currently being looked at.
         /// </summary>
-        public static GameObject LookedAtTarget { get; private set; }
+        public static GameObject LookedAtTarget => 
+            (CoreServices.InputSystem != null && 
+            CoreServices.InputSystem.EyeGazeProvider != null && 
+            CoreServices.InputSystem.EyeGazeProvider.IsEyeTrackingEnabledAndValid) ? CoreServices.InputSystem.EyeGazeProvider.GazeTarget : null;
+
+        /// <summary>
+        /// The point in space where the eye gaze hit. 
+        /// set to the origin if the EyeGazeProvider is not currently enabled
+        /// </summary>
+        public static Vector3 LookedAtPoint =>
+            (CoreServices.InputSystem != null &&
+            CoreServices.InputSystem.EyeGazeProvider != null &&
+            CoreServices.InputSystem.EyeGazeProvider.IsEyeTrackingEnabledAndValid) ? CoreServices.InputSystem.EyeGazeProvider.HitPosition : Vector3.zero;
 
         /// <summary>
         /// EyeTrackingTarget eye gaze is currently looking at.
         /// null if currently gazed at object has no EyeTrackingTarget, or if
         /// no object with collider is being looked at.
         /// </summary>
         public static EyeTrackingTarget LookedAtEyeTarget { get; private set; }
-        public static Vector3 LookedAtPoint { get; private set; }
 
         /// <summary>
         /// Most recently selected target, selected either using pointer
@@ -191,7 +202,6 @@ protected override void Start()
         {
             base.Start();
             IsLookedAt = false;
-            LookedAtTarget = null;
             LookedAtEyeTarget = null;
         }
 
@@ -252,28 +262,14 @@ private void UpdateHitTarget()
                     lastEyeSignalUpdateTimeFromET = (CoreServices.InputSystem?.EyeGazeProvider?.Timestamp).Value;
                     lastEyeSignalUpdateTimeLocal = DateTime.UtcNow;
 
-                    // ToDo: Handle raycasting layers
-                    var lookRay = new Ray(
-                        CoreServices.InputSystem.EyeGazeProvider.GazeOrigin,
-                        CoreServices.InputSystem.EyeGazeProvider.GazeDirection.normalized);
-                    bool isHit = UnityEngine.Physics.Raycast(lookRay, out RaycastHit hitInfo);
-
-                    if (isHit)
-                    {
-                        LookedAtEyeTarget = hitInfo.collider.transform.GetComponent<EyeTrackingTarget>();
-                        LookedAtTarget = hitInfo.collider.transform.gameObject;
-                        LookedAtPoint = hitInfo.point;
-                    }
-                    else
+                    if(LookedAtTarget != null)
                     {
-                        LookedAtTarget = null;
-                        LookedAtEyeTarget = null;
+                        LookedAtEyeTarget = LookedAtTarget.GetComponent<EyeTrackingTarget>();
                     }
                 }
             }
             else if ((DateTime.UtcNow - lastEyeSignalUpdateTimeLocal).TotalMilliseconds > EyeTrackingTimeoutInMilliseconds)
             {
-                LookedAtTarget = null;
                 LookedAtEyeTarget = null;
             }
         }
@@ -298,7 +294,7 @@ protected void OnEyeFocusStay()
         protected void OnEyeFocusDwell()
         {
             IsDwelledOn = true;
-            OnDwell.Invoke();
+            OnDwell?.Invoke();
         }
 
         protected void OnEyeFocusStop()

Assets/MRTK/Services/InputSystem/FocusProvider.cs

@@ -169,14 +169,6 @@ private bool IsSetupValid
             }
         }
 
-        /// <summary>
-        /// GazeProvider is a little special, so we keep track of it even if it's not a registered pointer. For the sake
-        /// of StabilizationPlaneModifier and potentially other components that care where the user's looking, we need
-        /// to do a gaze raycast even if gaze isn't used for focus.
-        /// </summary>
-        private PointerData gazeProviderPointingData;
-        private PointerHitResult gazeHitResult;
-
         /// <summary>
         /// Cached <see href="https://docs.unity3d.com/ScriptReference/Vector3.html">Vector3</see> reference to the new raycast position.
         /// </summary>
@@ -569,31 +561,42 @@ public override void Update()
 
         private static readonly ProfilerMarker UpdateGazeProviderPerfMarker = new ProfilerMarker("[MRTK] FocusProvider.UpdateGazeProvider");
 
+        /// <summary>
+        /// GazeProvider is a little special, so we keep track of it even if it's not a registered pointer. For the sake
+        /// of StabilizationPlaneModifier and potentially other components that care where the user's looking, we need
+        /// to do a gaze raycast even if gaze isn't used for focus.
+        /// </summary>
+        private PointerEventData gazeProviderPointingData;
+        private PointerHitResult gazeHitResult;
+
         /// <summary>
         /// Updates the gaze raycast provider even in scenarios where gaze isn't used for focus
         /// </summary>
         private void UpdateGazeProvider()
         {
             using (UpdateGazeProviderPerfMarker.Auto())
             {
-                // The gaze hit result may be populated from previous raycasts this frame, only recompute
+                // The gaze hit result may be populated from the UpdatePointers call. If it has not, then perform
                 // another raycast if it's not populated
                 if (gazeHitResult == null)
                 {
-                    if (gazeProviderPointingData?.Pointer != null)
+                    IMixedRealityPointer gazePointer = CoreServices.InputSystem.GazeProvider?.GazePointer;
+                    // Check that the gazePointer isn't null and that it has been properly registered as a pointer.
+                    if (gazePointer != null && gazeProviderPointingData != null)
                     {
                         // get 3d hit
+                        // This is unneccessary since the gaze pointer has been registered normally along with the other pointers(?)
                         hitResult3d.Clear();
                         var raycastProvider = CoreServices.InputSystem.RaycastProvider;
-                        LayerMask[] prioritizedLayerMasks = (gazeProviderPointingData.Pointer.PrioritizedLayerMasksOverride ?? FocusLayerMasks);
-                        QueryScene(gazeProviderPointingData.Pointer, raycastProvider, prioritizedLayerMasks,
+                        LayerMask[] prioritizedLayerMasks = (gazePointer.PrioritizedLayerMasksOverride ?? FocusLayerMasks);
+                        QueryScene(gazePointer, raycastProvider, prioritizedLayerMasks,
                             hitResult3d, maxQuerySceneResults, focusIndividualCompoundCollider);
 
                         if (shouldUseGraphicsRaycast)
                         {
                             // get ui hit
                             hitResultUi.Clear();
-                            RaycastGraphics(gazeProviderPointingData.Pointer, gazeProviderPointingData.GraphicEventData, prioritizedLayerMasks, hitResultUi);
+                            RaycastGraphics(gazePointer, gazeProviderPointingData, prioritizedLayerMasks, hitResultUi);
                         }
 
                         // set gaze hit according to distance and prioritization layer mask
@@ -605,7 +608,10 @@ private void UpdateGazeProvider()
                     }
                 }
 
-                CoreServices.InputSystem.GazeProvider.UpdateGazeInfoFromHit(gazeHitResult.raycastHit);
+                if (!CoreServices.InputSystem.GazeProvider.IsNull())
+                {
+                    CoreServices.InputSystem.GazeProvider.UpdateGazeInfoFromHit(gazeHitResult.raycastHit);
+                }
 
                 // Zero out value after every use to ensure the hit result is updated every frame.
                 gazeHitResult = null;
@@ -835,11 +841,12 @@ private void RegisterPointers(IMixedRealityInputSource inputSource)
                     RegisterPointer(inputSource.Pointers[i]);
 
                     // Special Registration for Gaze
+                    // Refreshes gazeProviderPointingData to a new reference to the current EventSystem 
                     if (!CoreServices.InputSystem.GazeProvider.IsNull()
                         && inputSource.SourceId == CoreServices.InputSystem.GazeProvider.GazeInputSource.SourceId
                         && gazeProviderPointingData == null)
                     {
-                        gazeProviderPointingData = new PointerData(inputSource.Pointers[i]);
+                        gazeProviderPointingData = new PointerEventData(EventSystem.current);
                     }
                 }
             }
@@ -1084,7 +1091,8 @@ private void UpdatePointer(PointerData pointerData)
                         pointerData.UpdateHit(hit);
 
                         // set gaze hit result - make sure to include unity ui hits
-                        if (gazeProviderPointingData?.Pointer != null && pointerData.Pointer.PointerId == gazeProviderPointingData.Pointer.PointerId)
+                        var gazePointer = CoreServices.InputSystem.GazeProvider.GazePointer;
+                        if (gazePointer != null && pointerData.Pointer.PointerId == gazePointer.PointerId)
                         {
                             gazeHitResult = hit;
                         }
@@ -1161,7 +1169,7 @@ private void ReconcilePointers()
         {
             using (ReconcilePointersPerfMarker.Auto())
             {
-                var gazePointer = gazeProviderPointingData?.Pointer as GenericPointer;
+                var gazePointer = CoreServices.InputSystem.GazeProvider?.GazePointer as GenericPointer;
                 NumFarPointersActive = 0;
                 NumNearPointersActive = 0;
                 int numFarPointersWithoutCursorActive = 0;
@@ -1559,18 +1567,19 @@ public void OnSourceLost(SourceStateEventData eventData)
 
                 pointerMediators.Remove(eventData.SourceId);
 
+
                 for (var i = 0; i < eventData.InputSource.Pointers.Length; i++)
                 {
+                    var gazePointer = CoreServices.InputSystem.GazeProvider?.GazePointer;
                     // Special unregistration for Gaze
-                    if (gazeProviderPointingData?.Pointer != null && eventData.InputSource.Pointers[i].PointerId == gazeProviderPointingData.Pointer.PointerId)
+                    if (gazePointer != null && eventData.InputSource.Pointers[i].PointerId == gazePointer.PointerId)
                     {
-                        // If the source lost is the gaze input source, then reset it.
+                        // If the source lost is the gaze input source, clear gazeProviderPointingData.
                         if (eventData.InputSource.SourceId == CoreServices.InputSystem.GazeProvider?.GazeInputSource.SourceId)
                         {
-                            gazeProviderPointingData.ResetFocusedObjects();
                             gazeProviderPointingData = null;
                         }
-                        // Otherwise, don't unregister the gaze pointer, since the gaze input source is still active.
+                        // Otherwise, don't clear gazeProviderPointingData, since the gaze input source is still active.
                         else
                         {
                             continue;

Assets/MRTK/Services/InputSystem/GazeProvider.cs

@@ -210,6 +210,7 @@ internal void SetGazeInputSourceParent(IMixedRealityInputSource gazeInputSource)
             private static readonly ProfilerMarker OnPreSceneQueryPerfMarker = new ProfilerMarker("[MRTK] InternalGazePointer.OnPreSceneQuery");
 
             /// <inheritdoc />
+            /// On pre-scene query, the gaze pointer will set up it's raycast ray to use either the eye gaze ray or the head gaze ray, depending on IsEyeTrackingEnabledAndValid
             public override void OnPreSceneQuery()
             {
                 using (OnPreSceneQueryPerfMarker.Auto())
@@ -252,8 +253,6 @@ public override void OnPreSceneQuery()
 
                     Vector3 endPoint = newGazeOrigin + (newGazeNormal * pointerExtent);
                     Rays[0].UpdateRayStep(ref newGazeOrigin, ref endPoint);
-
-                    gazeProvider.HitPosition = Rays[0].Origin + (gazeProvider.lastHitDistance * Rays[0].Direction);
                 }
             }
 
@@ -550,6 +549,12 @@ private async void RaiseSourceDetected()
         public void UpdateGazeInfoFromHit(MixedRealityRaycastHit raycastHit)
         {
             HitInfo = raycastHit;
+
+            if (IsEyeTrackingEnabledAndValid)
+            {
+                UpdateEyeGaze(null, GazePointer.Rays[0], DateTime.UtcNow);
+            }
+
             if (raycastHit.transform != null)
             {
                 GazeTarget = raycastHit.transform.gameObject;

Assets/MRTK/Tests/PlayModeTests/InputSystem/EyeTrackingTargetTest.cs

@@ -33,7 +33,8 @@ public class EyeTrackingTargetTest : BasePlayModeTests
         // do any kind of setup here that can't be done in playmode
         public override IEnumerator Setup()
         {
-            yield return base.Setup();
+            var eyeTrackingProfile = AssetDatabase.LoadAssetAtPath(eyeTrackingConfigurationProfilePath, typeof(MixedRealityToolkitConfigurationProfile)) as MixedRealityToolkitConfigurationProfile;
+            PlayModeTestUtilities.Setup(eyeTrackingProfile);
             TestUtilities.PlayspaceToOriginLookingForward();
             yield return null;
         }
@@ -46,17 +47,6 @@ public override IEnumerator Setup()
         [UnityTest]
         public IEnumerator TestEyeTrackingTarget()
         {
-            // Eye tracking configuration profile should set eye based gaze
-            var profile = AssetDatabase.LoadAssetAtPath(eyeTrackingConfigurationProfilePath, typeof(MixedRealityToolkitConfigurationProfile)) as MixedRealityToolkitConfigurationProfile;
-            MixedRealityToolkit.Instance.ResetConfiguration(profile);
-
-            // Reset view to origin
-            MixedRealityPlayspace.PerformTransformation(p =>
-            {
-                p.position = Vector3.zero;
-                p.LookAt(Vector3.forward);
-            });
-
             string targetName = "eyetrackingTargetObject";
 
             var eyetrackingTargetObject = GameObject.CreatePrimitive(PrimitiveType.Cube);
@@ -74,8 +64,9 @@ public IEnumerator TestEyeTrackingTarget()
 
             var isEyeGazeActive = InputRayUtils.TryGetEyeGazeRay(out var eyegazeRay);
             Assert.True(isEyeGazeActive);
-            yield return null;
+            yield return PlayModeTestUtilities.WaitForInputSystemUpdate();
             Assert.True(EyeTrackingTarget.LookedAtTarget != null);
+            Assert.True(EyeTrackingTarget.LookedAtEyeTarget != null);
             Assert.True(EyeTrackingTarget.LookedAtEyeTarget.name == targetName);
         }
         #endregion