diff --git a/Packages/Tracking Preview/AggregationProviders/Runtime/Scripts/JointOcclusion.cs b/Packages/Tracking Preview/AggregationProviders/Runtime/Scripts/JointOcclusion.cs
index ff2f19c057..f1738ec8e0 100644
--- a/Packages/Tracking Preview/AggregationProviders/Runtime/Scripts/JointOcclusion.cs
+++ b/Packages/Tracking Preview/AggregationProviders/Runtime/Scripts/JointOcclusion.cs
@@ -1,3 +1,7 @@
+
+// Use this to log out the render colours seen by the camera in the render texture
+// #define LOGGING_RENDER_COLOURS
+
using Leap;
using Leap.Encoding;
using System.Collections.Generic;
@@ -17,12 +21,82 @@ public class JointOcclusion : MonoBehaviour
Camera cam;
Texture2D tex;
Rect regionToReadFrom;
+
+ // Colours used for the capsule hand joint spheres
Color[] occlusionSphereColorsLeft;
Color[] occlusionSphereColorsRight;
+
+ ///
+ /// For some reason the capsule hand colours that are rendered to the occlusion camera are not the original colours, but appear darker
+ /// This set is based on the colours as viewed in the render texture. The ComputMajorColour codeis used to log out the colours
+ /// found when looking for joints, when a hand is placed so all joints can be seen
+ ///
+ Color[] shiftedOcclusionSphereColorsLeft =
+ {
+ new Color(1, 0, 0, 1),
+ new Color(0.9294118f, 0.003921569f, 0, 1),
+ new Color(0.8627452f, 0.003921569f, 0, 1),
+ new Color(0.8000001f, 0.007843138f, 0, 1),
+ new Color(0.7372549f, 0.01568628f, 0, 1),
+ new Color(0.6784314f, 0.01960784f, 0, 1),
+ new Color(0.6235294f, 0.02745098f, 0, 1),
+ new Color(0.572549f, 0.03921569f, 0, 1),
+ new Color(0.0509804f, 0.5215687f, 0, 1),
+ new Color(0.4745098f, 0.0627451f, 0, 1),
+ new Color(0.4313726f, 0.07843138f, 0, 1),
+ new Color(0.3882353f, 0.09803922f, 0, 1),
+ new Color(0.0509804f, 0.5215687f, 0, 1),
+ new Color(0.3098039f, 0.1372549f, 0, 1),
+ new Color(0.2745098f, 0.1607843f, 0, 1),
+ new Color(0.2431373f, 0.1843137f, 0, 1),
+ new Color(0.2156863f, 0.2156863f, 0, 1),
+ new Color(0.1843137f, 0.2431373f, 0, 1),
+ new Color(0.1607843f, 0.2745098f, 0, 1),
+ new Color(0.1372549f, 0.3098039f, 0, 1)
+ };
+
+ ///
+ /// For some reason the capsule hand colours that are rendered to the occlusion camera are not the original colours, but appear darker
+ /// This set is based on the colours as viewed in the render texture. The ComputMajorColour codeis used to log out the colours
+ /// found when looking for joints, when a hand is placed so all joints can be seen
+ ///
+ Color[] shiftedOcclusionSphereColorsRight =
+ {
+ new Color(1f, 0.8313726f, 0f, 1f),
+ new Color(0.9294118f, 0.7725491f, 0.003921569f, 1f),
+ new Color(0.8627452f, 0.7176471f, 0.007843138f, 1f),
+ new Color(0.8000001f, 0.6666667f, 0.01176471f, 1f),
+ new Color(0.7372549f, 0.6156863f, 0.01568628f, 1f),
+ new Color(0.6784314f, 0.5647059f, 0.02352941f, 1f),
+ new Color(0.6235294f, 0.5215687f, 0.03137255f, 1f),
+ new Color(0.572549f, 0.4784314f, 0.04313726f, 1f),
+ new Color(0.0509804f, 0.04313726f, 0.5294118f, 1f),
+ new Color(0.4745098f, 0.3960785f, 0.07058824f, 1f),
+ new Color(0.4313726f, 0.3607843f, 0.08627451f, 1f),
+ new Color(0.3882353f, 0.3254902f, 0.1019608f, 1f),
+ new Color(0.3490196f, 0.2901961f, 0.1215686f, 1f),
+ new Color(0.3098039f, 0.2588235f, 0.145098f, 1f),
+ new Color(0.2745098f, 0.2313726f, 0.1686275f, 1f),
+ new Color(0.2431373f, 0.2039216f, 0.1921569f, 1f),
+ new Color(0.2156863f, 0.1803922f, 0.2196079f, 1f),
+ new Color(0.1843137f, 0.1568628f, 0.2509804f, 1f),
+ new Color(0.1607843f, 0.1333333f, 0.282353f, 1f),
+ new Color(0.1372549f, 0.1137255f, 0.3176471f, 1f),
+ };
+
Mesh cubeMesh;
Material cubeMaterial;
string layerName;
+ private bool setup = false;
+ public void Update()
+ {
+ if (!setup)
+ {
+ Setup();
+ }
+ }
+
///
/// this sets everything up, so that joint occlusion works (eg. rendering layers)
///
@@ -63,11 +137,9 @@ public void Setup()
occlusionHandLeft.SphereColors = occlusionSphereColorsLeft;
occlusionHandRight.SphereColors = occlusionSphereColorsRight;
-
-
cubeMesh = createCubeMesh();
cubeMaterial = new Material(Shader.Find("Standard"));
-
+ setup = true;
}
private Mesh createCubeMesh()
@@ -137,12 +209,12 @@ public float[] Confidence_JointOcclusion(float[] confidences, Transform deviceOr
tex.ReadPixels(regionToReadFrom, 0, 0);
tex.Apply();
-
// loop through all joints that are visible (all joints that are rendered on a capsule hand),
// and save how many pixels of a joint can be seen (in pixelsSeenCount)
// and how many pixels of a joint would be seen if the joint was not occluded at all (in optimalPixelsCount)
int[] pixelsSeenCount = new int[confidences.Length];
int[] optimalPixelsCount = new int[confidences.Length];
+ Color[] majorColours = new Color[occlusionSphereColorsLeft.Length];
foreach (var finger in hand.fingers)
{
for (int j = 0; j < 4; j++)
@@ -165,7 +237,6 @@ public float[] Confidence_JointOcclusion(float[] confidences, Transform deviceOr
optimalPixelsCount[key] = (int)(Mathf.PI * radius * radius);
-
// only count pixels around where the sphere is supposed to be (+5 pixel margin)
int margin = 5;
int x0 = Mathf.Clamp((int)(screenPosCenter.x - radius - margin), 0, tex.width);
@@ -177,11 +248,26 @@ public float[] Confidence_JointOcclusion(float[] confidences, Transform deviceOr
if (hand.IsLeft)
{
- pixelsSeenCount[key] = tempPixels.Where(x => DistanceBetweenColors(x, occlusionSphereColorsLeft[capsuleHandKey]) < 0.01f).Count();
+ pixelsSeenCount[key] = tempPixels.Where(x => DistanceBetweenColors(x, shiftedOcclusionSphereColorsLeft[capsuleHandKey]) < 0.01f).Count();
+
+#if LOGGING_RENDER_COLOURS
+ majorColours[capsuleHandKey] = FindDominantColourInTarget(tempPixels);
+
+ Debug.Log($"{capsuleHandKey} Original sphere colour {occlusionSphereColorsLeft[capsuleHandKey].r},{occlusionSphereColorsLeft[capsuleHandKey].g},{occlusionSphereColorsLeft[capsuleHandKey].b}, {occlusionSphereColorsLeft[capsuleHandKey].a} " +
+ $"Dominant colour actually found {majorColours[capsuleHandKey].r}, {majorColours[capsuleHandKey].g}, {majorColours[capsuleHandKey].b}, {majorColours[capsuleHandKey].a} " +
+ $"Difference {DistanceBetweenColors(majorColours[capsuleHandKey], occlusionSphereColorsLeft[capsuleHandKey])}");
+#endif
}
else
{
- pixelsSeenCount[key] = tempPixels.Where(x => DistanceBetweenColors(x, occlusionSphereColorsRight[capsuleHandKey]) < 0.01f).Count();
+ pixelsSeenCount[key] = tempPixels.Where(x => DistanceBetweenColors(x, shiftedOcclusionSphereColorsRight[capsuleHandKey]) < 0.01f).Count();
+
+#if LOGGING_RENDER_COLOURS
+ majorColours[capsuleHandKey] = FindDominantColourInTarget(tempPixels);
+ Debug.Log($"{capsuleHandKey} Original sphere colour {occlusionSphereColorsLeft[capsuleHandKey].r},{occlusionSphereColorsLeft[capsuleHandKey].g},{occlusionSphereColorsLeft[capsuleHandKey].b}, {occlusionSphereColorsLeft[capsuleHandKey].a} " +
+ $"Dominant colour actually found {majorColours[capsuleHandKey].r}, {majorColours[capsuleHandKey].g}, {majorColours[capsuleHandKey].b}, {majorColours[capsuleHandKey].a} " +
+ $"Difference {DistanceBetweenColors(majorColours[capsuleHandKey], occlusionSphereColorsLeft[capsuleHandKey])}");
+#endif
}
}
}
@@ -197,10 +283,48 @@ public float[] Confidence_JointOcclusion(float[] confidences, Transform deviceOr
return confidences;
}
- float DistanceBetweenColors(Color color1, Color color2)
+ private float DistanceBetweenColors(Color color1, Color color2)
{
Color colorDifference = color1 - color2;
Vector3 diffVetor = new Vector3(colorDifference.r, colorDifference.g, colorDifference.b);
return diffVetor.magnitude;
}
-}
\ No newline at end of file
+
+ private Color FindDominantColourInTarget(Color[] targetPixelRegion)
+ {
+ Dictionary colourStats = new Dictionary();
+
+ // Can get both black defined as 0,0,0,0 and 0,0,0,1
+ Color black = new Color(0, 0, 0, 0);
+
+ foreach (Color pixel in targetPixelRegion)
+ {
+ // Screen out white and black pixel(s)
+ if (pixel != Color.white && pixel != black && pixel != Color.black)
+ {
+ if (colourStats.ContainsKey(pixel))
+ {
+ colourStats[pixel] = colourStats[pixel] + 1;
+ }
+ else
+ {
+ colourStats.Add(pixel, 1);
+ }
+ }
+ }
+
+ int maxCount = 0;
+ Color mostCommonColor = Color.black;
+
+ foreach (Color color in colourStats.Keys)
+ {
+ if (colourStats[color] > maxCount)
+ {
+ maxCount = colourStats[color];
+ mostCommonColor = color;
+ }
+ }
+
+ return mostCommonColor;
+ }
+}
diff --git a/Packages/Tracking/CHANGELOG.md b/Packages/Tracking/CHANGELOG.md
index 2378d1fd32..8cec00fdb5 100644
--- a/Packages/Tracking/CHANGELOG.md
+++ b/Packages/Tracking/CHANGELOG.md
@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
- Fixed issue with LeapXRHandProvider not working on Quest devices when using XRI/XR Hands and Ultraleap tracking in direct (non OpenXR) mode, due to default constructor being optimized away.
+- Updated target colour palette values for the JointOcclusion confidence code to account for unexpected changes when rendered to the render texture
## [7.1.0] - 04/09/2024