Fixing keypoint bugs when visualizations are disabled (#290)

* Fixing keypoint bugs when visualizations are disabled (enabling async readback)

* Remove commented out code

* Pointing to stable ubuntu to work around issue in latest.

Author: JonathanHUnity

.yamato/environments.yml

@@ -53,7 +53,7 @@ test_platforms:
     standalone-platform: StandaloneOSX
   - name: ubuntu
     type: Unity::VM
-    image: package-ci/ubuntu:latest
+    image: package-ci/ubuntu:stable
     flavor: b1.large
 
 performance_platforms:
@@ -71,7 +71,7 @@ performance_platforms:
     standalone-platform: StandaloneOSX
   - name: ubuntu
     type: Unity::VM
-    image: package-ci/ubuntu:latest
+    image: package-ci/ubuntu:stable
     flavor: b1.large
 
 performance_suites:

com.unity.perception/Runtime/GroundTruth/Labelers/KeypointLabeler.cs

@@ -177,8 +177,9 @@ void OnInstanceSegmentationImageReadback(int frameCount, NativeArray<Color32> da
             {
                 if (InstanceIdToColorMapping.TryGetColorFromInstanceId(keypointSet.Key, out var idColor))
                 {
-                    foreach (var keypoint in keypointSet.Value.keypoints)
+                    for (var i = 0; i < keypointSet.Value.keypoints.Length; i++)
                     {
+                        var keypoint = keypointSet.Value.keypoints[i];
                         keypoint.state = DetermineKeypointState(keypoint, idColor, dimensions, data);
 
                         if (keypoint.state == 0)
@@ -191,6 +192,8 @@ void OnInstanceSegmentationImageReadback(int frameCount, NativeArray<Color32> da
                             keypoint.x = math.clamp(keypoint.x, 0, dimensions.width - .001f);
                             keypoint.y = math.clamp(keypoint.y, 0, dimensions.height - .001f);
                         }
+
+                        keypointSet.Value.keypoints[i] = keypoint;
                     }
                 }
             }
@@ -286,7 +289,7 @@ public class KeypointEntry
         /// The values of a specific keypoint
         /// </summary>
         [Serializable]
-        public class Keypoint
+        public struct Keypoint
         {
             /// <summary>
             /// The index of the keypoint in the template file
@@ -458,7 +461,17 @@ void ProcessLabel(Labeling labeledEntity)
                     cachedData.keypoints.pose = GetPose(cachedData.animator);
                 }
 
-                m_AsyncAnnotations[m_CurrentFrame].keypoints[labeledEntity.instanceId] = cachedData.keypoints;
+
+                var cachedKeypointEntry = cachedData.keypoints;
+                var keypointEntry = new KeypointEntry()
+                {
+                    instance_id = cachedKeypointEntry.instance_id,
+                    keypoints = cachedKeypointEntry.keypoints.ToArray(),
+                    label_id = cachedKeypointEntry.label_id,
+                    pose = cachedKeypointEntry.pose,
+                    template_guid = cachedKeypointEntry.template_guid
+                };
+                m_AsyncAnnotations[m_CurrentFrame].keypoints[labeledEntity.instanceId] = keypointEntry;
             }
         }
 
@@ -506,7 +519,7 @@ string GetPose(Animator animator)
             return "unset";
         }
 
-        Keypoint GetKeypointForJoint(KeypointEntry entry, int joint)
+        Keypoint? GetKeypointForJoint(KeypointEntry entry, int joint)
         {
             if (joint < 0 || joint >= entry.keypoints.Length) return null;
             return entry.keypoints[joint];
@@ -530,9 +543,9 @@ protected override void OnVisualize()
                     var joint1 = GetKeypointForJoint(entry, bone.joint1);
                     var joint2 = GetKeypointForJoint(entry, bone.joint2);
 
-                    if (joint1 != null && joint1.state == 2 && joint2 != null && joint2.state == 2)
+                    if (joint1 != null && joint1.Value.state == 2 && joint2 != null && joint2.Value.state == 2)
                     {
-                        VisualizationHelper.DrawLine(joint1.x, joint1.y, joint2.x, joint2.y, bone.color, 8, skeletonTexture);
+                        VisualizationHelper.DrawLine(joint1.Value.x, joint1.Value.y, joint2.Value.x, joint2.Value.y, bone.color, 8, skeletonTexture);
                     }
                 }
 

com.unity.perception/Tests/Runtime/GroundTruthTests/KeypointGroundTruthTests.cs

@@ -436,6 +436,76 @@ public IEnumerator Keypoint_TestAllOffScreen()
             }
         }
 
+        [UnityTest]
+        public IEnumerator Keypoint_TestMovingCube()
+        {
+            var incoming = new List<List<KeypointLabeler.KeypointEntry>>();
+
+            var keypoints = new[]
+            {
+                new KeypointDefinition
+                {
+                    label = "Center",
+                    associateToRig = false,
+                    color = Color.black
+                }
+            };
+            var template = ScriptableObject.CreateInstance<KeypointTemplate>();
+            template.templateID = Guid.NewGuid().ToString();
+            template.templateName = "label";
+            template.jointTexture = null;
+            template.skeletonTexture = null;
+            template.keypoints = keypoints;
+            template.skeleton = new SkeletonDefinition[0];
+
+            var texture = new RenderTexture(1024, 768, 16);
+            texture.Create();
+            var cam = SetupCamera(SetUpLabelConfig(), template, (frame, data) =>
+            {
+                incoming.Add(new List<KeypointLabeler.KeypointEntry>(data));
+            }, texture);
+            cam.GetComponent<PerceptionCamera>().showVisualizations = false;
+
+            var cube = TestHelper.CreateLabeledCube(scale: 6, z: 8);
+            SetupCubeJoint(cube, template, "Center", 0, 0, 0);
+
+            cube.SetActive(true);
+            cam.SetActive(true);
+
+            AddTestObjectForCleanup(cam);
+            AddTestObjectForCleanup(cube);
+
+            yield return null;
+            cube.transform.localPosition = new Vector3(-1, 0, 0);
+            yield return null;
+
+            //force all async readbacks to complete
+            DestroyTestObject(cam);
+            texture.Release();
+
+            var testCase = incoming[0];
+            Assert.AreEqual(1, testCase.Count);
+            var t = testCase.First();
+            Assert.NotNull(t);
+            Assert.AreEqual(1, t.instance_id);
+            Assert.AreEqual(1, t.label_id);
+            Assert.AreEqual(template.templateID.ToString(), t.template_guid);
+            Assert.AreEqual(1, t.keypoints.Length);
+
+            Assert.AreEqual(1024 / 2, t.keypoints[0].x);
+            Assert.AreEqual(768 / 2, t.keypoints[0].y);
+
+            Assert.AreEqual(0, t.keypoints[0].index);
+            Assert.AreEqual(2, t.keypoints[0].state);
+
+
+            var testCase2 = incoming[1];
+            Assert.AreEqual(1, testCase2.Count);
+            var t2 = testCase2.First();
+            Assert.AreEqual(445, t2.keypoints[0].x, 1);
+            Assert.AreEqual(768 / 2, t2.keypoints[0].y);
+        }
+
         [UnityTest]
         public IEnumerator Keypoint_TestPartialOffScreen([Values(1,5)] int framesToRunBeforeAsserting)
         {