Add layer mask prioritization for ui elements

Ended up needing a container to be able to store the layer mask index.
Also used the new generix MaxOrDefault

Author: Zod-

Assets/HoloToolkit/Utilities/Scripts/Extensions/EventSystemExtensions.cs

@@ -10,53 +10,32 @@ namespace HoloToolkit.Unity
     public static class EventSystemExtensions
     {
         private static readonly List<RaycastResult> RaycastResults = new List<RaycastResult>();
+        private static readonly List<ComparableRaycastResult> ComparableRaycastResults = new List<ComparableRaycastResult>();
         private static readonly RaycastResultComparer RaycastResultComparer = new RaycastResultComparer();
 
         /// <summary>
         /// Executes a raycast all and returns the closest element. Fixes the current issue with Unitys raycast sorting which does not 
         /// consider separate canvases.
         /// </summary>
         /// <returns>RaycastResult if hit, or an empty RaycastResult if nothing was hit</returns>
-        public static RaycastResult Raycast(this EventSystem eventSystem, PointerEventData pointerEventData, IEnumerable<LayerMask> layerMasks)
+        public static RaycastResult Raycast(this EventSystem eventSystem, PointerEventData pointerEventData, LayerMask[] layerMasks)
         {
             RaycastResults.Clear();
             eventSystem.RaycastAll(pointerEventData, RaycastResults);
-            return FindClosestRaycastHitInLayerMasks(layerMasks);
+            return PrioritizeRaycastResult(layerMasks);
         }
 
-        /// <summary>
-        /// Find the closest raycast hit in the list of RaycastResults that is also included in the LayerMask list.  
-        /// </summary>
-        /// <param name="layerMaskList">List of layers to support</param>
-        /// <returns>RaycastResult if hit, or an empty RaycastResult if nothing was hit</returns>
-        private static RaycastResult FindClosestRaycastHitInLayerMasks(IEnumerable<LayerMask> layerMaskList)
-        {
-            int combinedLayerMask = layerMaskList.Combine();
-
-            for (var i = RaycastResults.Count - 1; i >= 0; i--)
-            {
-                var candidate = RaycastResults[i];
-                if (candidate.gameObject && candidate.gameObject.layer.IsInLayerMask(combinedLayerMask)) continue;
-                RaycastResults.Remove(candidate);
-            }
-            ;
-            return RaycastResults.Count > 0 ? GetClosestRaycast() : default(RaycastResult);
-        }
-
-        private static RaycastResult GetClosestRaycast()
+        private static RaycastResult PrioritizeRaycastResult(LayerMask[] layerMaskPrio)
         {
-            var max = RaycastResults[0];
-
-            for (var i = 1; i < RaycastResults.Count; i++)
+            ComparableRaycastResults.Clear();
+            foreach (var raycastResult in RaycastResults)
             {
-                if (RaycastResultComparer.Compare(max, RaycastResults[i]) > 0)
-                {
-                    max = RaycastResults[i];
-                }
+                if (raycastResult.gameObject == null) { continue; }
+                var layerMaskIndex = raycastResult.gameObject.layer.FindLayerListIndex(layerMaskPrio);
+                if (layerMaskIndex == -1) { continue; }
+                ComparableRaycastResults.Add(new ComparableRaycastResult(raycastResult, layerMaskIndex));
             }
-
-            return max;
+            return ComparableRaycastResults.MaxOrDefault(RaycastResultComparer).RaycastResult;
         }
-
     }
 }

Assets/HoloToolkit/Utilities/Scripts/RaycastResultComparer.cs

@@ -7,17 +7,30 @@
 
 namespace HoloToolkit.Unity
 {
-    public class RaycastResultComparer : IComparer<RaycastResult>
+    public struct ComparableRaycastResult
     {
-        private static readonly List<Func<RaycastResult, RaycastResult, int>> Comparers = new List<Func<RaycastResult, RaycastResult, int>>
+        public readonly int LayerMaskIndex;
+        public readonly RaycastResult RaycastResult;
+
+        public ComparableRaycastResult(RaycastResult raycastResult, int layerMaskIndex = 0)
         {
+            RaycastResult = raycastResult;
+            LayerMaskIndex = layerMaskIndex;
+        }
+    }
+
+    public class RaycastResultComparer : IComparer<ComparableRaycastResult>
+    {
+        private static readonly List<Func<ComparableRaycastResult, ComparableRaycastResult, int>> Comparers = new List<Func<ComparableRaycastResult, ComparableRaycastResult, int>>
+        {
+            CompareRaycastsByLayerMaskPrioritization,
             CompareRaycastsBySortingLayer,
             CompareRaycastsBySortingOrder,
             CompareRaycastsByCanvasDepth,
             CompareRaycastsByDistance,
         };
 
-        public int Compare(RaycastResult left, RaycastResult right)
+        public int Compare(ComparableRaycastResult left, ComparableRaycastResult right)
         {
             for (var i = 0; i < Comparers.Count; i++)
             {
@@ -30,33 +43,39 @@ public int Compare(RaycastResult left, RaycastResult right)
             return 0;
         }
 
-        private static int CompareRaycastsBySortingOrder(RaycastResult left, RaycastResult right)
+        private static int CompareRaycastsByLayerMaskPrioritization(ComparableRaycastResult left, ComparableRaycastResult right)
+        {
+            //Lower is better, -1 is not relevant
+            return right.LayerMaskIndex.CompareTo(left.LayerMaskIndex);
+        }
+
+        private static int CompareRaycastsBySortingLayer(ComparableRaycastResult left, ComparableRaycastResult right)
         {
             //Higher is better
-            return right.sortingOrder.CompareTo(left.sortingOrder);
+            return left.RaycastResult.sortingLayer.CompareTo(right.RaycastResult.sortingLayer);
         }
 
-        private static int CompareRaycastsBySortingLayer(RaycastResult left, RaycastResult right)
+        private static int CompareRaycastsBySortingOrder(ComparableRaycastResult left, ComparableRaycastResult right)
         {
             //Higher is better
-            return right.sortingLayer.CompareTo(left.sortingLayer);
+            return left.RaycastResult.sortingOrder.CompareTo(right.RaycastResult.sortingOrder);
         }
 
-        private static int CompareRaycastsByCanvasDepth(RaycastResult left, RaycastResult right)
+        private static int CompareRaycastsByCanvasDepth(ComparableRaycastResult left, ComparableRaycastResult right)
         {
             //Module is the graphic raycaster on the canvases.
-            if (left.module.transform.IsParentOrChildOf(right.module.transform))
+            if (left.RaycastResult.module.transform.IsParentOrChildOf(right.RaycastResult.module.transform))
             {
                 //Higher is better
-                return right.depth.CompareTo(left.depth);
+                return left.RaycastResult.depth.CompareTo(right.RaycastResult.depth);
             }
             return 0;
         }
 
-        private static int CompareRaycastsByDistance(RaycastResult left, RaycastResult right)
+        private static int CompareRaycastsByDistance(ComparableRaycastResult left, ComparableRaycastResult right)
         {
             //Lower is better
-            return left.distance.CompareTo(right.distance);
+            return right.RaycastResult.distance.CompareTo(left.RaycastResult.distance);
         }
     }
 }