Fixing labeling on per-material MaterialPropertyBlocks (#73)

* Fixing labeling when renderers have per-material MaterialPropertyBlocks. Also supporting refreshing labeling manually after making changes to renderers or the Labeling component.

* updating changelog

Author: JonathanHUnity

TestProjects/PerceptionURP/Assets/Scenes/SampleScene.meta

@@ -135,6 +135,7 @@ GameObject:
   - component: {fileID: 411238278}
   - component: {fileID: 411238277}
   - component: {fileID: 411238282}
+  - component: {fileID: 411238283}
   m_Layer: 0
   m_Name: Crate
   m_TagString: Untagged
@@ -243,6 +244,20 @@ MonoBehaviour:
   m_Name: 
   m_EditorClassIdentifier: 
   yDegreesPerSecond: 180
+--- !u!114 &411238283
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 411238276}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 695e410829600ff40bcdd76fa0818f6a, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  materialPropertyTarget: 1
+  color: {r: 0.745283, g: 0.40428087, b: 0.40428087, a: 0}
 --- !u!1 &464025704
 GameObject:
   m_ObjectHideFlags: 0
@@ -257,6 +272,7 @@ GameObject:
   - component: {fileID: 464025706}
   - component: {fileID: 464025705}
   - component: {fileID: 464025710}
+  - component: {fileID: 464025711}
   m_Layer: 0
   m_Name: Cube
   m_TagString: Untagged
@@ -310,6 +326,8 @@ MeshRenderer:
   m_RendererPriority: 0
   m_Materials:
   - {fileID: 2100000, guid: 31321ba15b8f8eb4c954353edc038b1d, type: 2}
+  - {fileID: 2100000, guid: 31321ba15b8f8eb4c954353edc038b1d, type: 2}
+  - {fileID: 2100000, guid: 31321ba15b8f8eb4c954353edc038b1d, type: 2}
   m_StaticBatchInfo:
     firstSubMesh: 0
     subMeshCount: 0
@@ -365,6 +383,20 @@ MonoBehaviour:
   m_Name: 
   m_EditorClassIdentifier: 
   yDegreesPerSecond: 180
+--- !u!114 &464025711
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 464025704}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 695e410829600ff40bcdd76fa0818f6a, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  materialPropertyTarget: 1
+  color: {r: 0.34109113, g: 0.42664438, b: 0.6886792, a: 0}
 --- !u!1 &705507993
 GameObject:
   m_ObjectHideFlags: 0
@@ -595,7 +627,7 @@ MonoBehaviour:
   - id: 1
   - id: 2
   - id: 3
-  showVisualizations: 0
+  showVisualizations: 1
   references:
     version: 1
     00000000:
@@ -762,4 +794,4 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 673a227032a8e4940b9828c5b6f852ab, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
-  yDegreesPerSecond: 180
+  yDegreesPerSecond: 180

TestProjects/PerceptionURP/Assets/Scenes/SampleScene.unity

@@ -0,0 +1,31 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+
+public enum MaterialPropertyTarget
+{
+    Renderer,
+    Material
+}
+[ExecuteInEditMode]
+public class TestMpbPerMaterial : MonoBehaviour
+{
+    public MaterialPropertyTarget materialPropertyTarget = MaterialPropertyTarget.Material;
+    public Color color;
+    // Update is called once per frame
+    void Start()
+    {
+        var meshRenderer = GetComponent<MeshRenderer>();
+        MaterialPropertyBlock mpb = new MaterialPropertyBlock();
+        mpb.SetColor("_BaseColor", color);
+        if (materialPropertyTarget == MaterialPropertyTarget.Renderer)
+            meshRenderer.SetPropertyBlock(mpb);
+        else
+        {
+            for (int i = 0; i < meshRenderer.sharedMaterials.Length; i++)
+            {
+                meshRenderer.SetPropertyBlock(mpb, i);
+            }
+        }
+    }
+}

TestProjects/PerceptionURP/Assets/TestMpbPerMaterial.cs

@@ -95,7 +95,7 @@ QualitySettings:
     skinWeights: 2
     textureQuality: 0
     anisotropicTextures: 1
-    antiAliasing: 0
+    antiAliasing: 2
     softParticles: 0
     softVegetation: 1
     realtimeReflectionProbes: 1

TestProjects/PerceptionURP/Assets/TestMpbPerMaterial.cs.meta

@@ -12,6 +12,10 @@ Added new randomization tools
 
 Added support for 2020.1
 
+Added Labeling.RefreshLabeling(), which can be used to update ground truth generators after the list of labels or the renderers is changed
+
+Added support for renderers with MaterialPropertyBlocks assigned to individual materials
+
 ### Changed
 
 Changed the way realtime visualizers rendered to avoid rendering conflicts

TestProjects/PerceptionURP/ProjectSettings/QualitySettings.asset

@@ -70,6 +70,20 @@ void InitGameObjectRecursive(GameObject gameObject, MaterialPropertyBlock mpb, L
                     pass.SetupMaterialProperties(mpb, renderer, labeling, instanceId);
 
                 renderer.SetPropertyBlock(mpb);
+
+                var materialCount = renderer.materials.Length;
+                for (int i = 0; i < materialCount; i++)
+                {
+                    renderer.GetPropertyBlock(mpb, i);
+                    //Only apply to individual materials if there is already a MaterialPropertyBlock on it
+                    if (!mpb.isEmpty)
+                    {
+                        foreach (var pass in m_ActiveGenerators)
+                            pass.SetupMaterialProperties(mpb, renderer, labeling, instanceId);
+
+                        renderer.SetPropertyBlock(mpb, i);
+                    }
+                }
             }
 
             for (var i = 0; i < gameObject.transform.childCount; i++)
@@ -106,5 +120,10 @@ public bool Deactivate(IGroundTruthGenerator generator)
         {
             return m_ActiveGenerators.Remove(generator);
         }
+
+        internal void RefreshLabeling(Entity labelingEntity)
+        {
+            EntityManager.RemoveComponent<GroundTruthInfo>(labelingEntity);
+        }
     }
 }

com.unity.perception/CHANGELOG.md

@@ -12,7 +12,7 @@ namespace UnityEngine.Perception.GroundTruth
     public class Labeling : MonoBehaviour
     {
         /// <summary>
-        /// The label names to associate with the GameObject. Modifications to this list after the Update() step of the frame the object is created in are 
+        /// The label names to associate with the GameObject. Modifications to this list after the Update() step of the frame the object is created in are
         /// not guaranteed to be reflected by labelers.
         /// </summary>
         [FormerlySerializedAs("classes")]
@@ -37,5 +37,14 @@ void OnDestroy()
             if (World.DefaultGameObjectInjectionWorld != null)
                 World.DefaultGameObjectInjectionWorld.EntityManager.DestroyEntity(m_Entity);
         }
+
+        /// <summary>
+        /// Refresh ground truth generation for the labeling of the attached GameObject. This is necessary when the
+        /// list of labels changes or when renderers or materials change on objects in the hierarchy.
+        /// </summary>
+        public void RefreshLabeling()
+        {
+            World.DefaultGameObjectInjectionWorld.GetOrCreateSystem<GroundTruthLabelSetupSystem>().RefreshLabeling(m_Entity);
+        }
     }
 }

com.unity.perception/Runtime/GroundTruth/GroundTruthLabelSetupSystem.cs

@@ -5,6 +5,7 @@
 using System.Linq;
 using NUnit.Framework;
 using Unity.Collections;
+using Unity.Simulation;
 using UnityEngine;
 using UnityEngine.Perception.GroundTruth;
 using UnityEngine.Rendering;
@@ -290,15 +291,70 @@ public IEnumerator SemanticSegmentationPass_WithTextureOverride_RendersToOverrid
             AddTestObjectForCleanup(TestHelper.CreateLabeledPlane());
 
             yield return null;
-            //NativeArray<Color32> readbackArray = new NativeArray<Color32>(targetTextureOverride.width * targetTextureOverride.height, Allocator.Temp);
-            var request = AsyncGPUReadback.Request(targetTextureOverride, callback: r =>
+            TestHelper.ReadRenderTextureRawData<Color32>(targetTextureOverride, data =>
             {
-                CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, targetTextureOverride.width * targetTextureOverride.height), r.GetData<Color32>());
+                CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, targetTextureOverride.width * targetTextureOverride.height), data);
             });
-            AsyncGPUReadback.WaitAllRequests();
-            //request.WaitForCompletion();
-            Assert.IsTrue(request.done);
-            Assert.IsFalse(request.hasError);
+        }
+
+
+        [UnityTest]
+        public IEnumerator SemanticSegmentationPass_WithMultiMaterial_ProducesCorrectValues([Values(true, false)] bool showVisualizations)
+        {
+            int timesSegmentationImageReceived = 0;
+            var expectedPixelValue = k_SemanticPixelValue;
+            void OnSegmentationImageReceived(NativeArray<Color32> data)
+            {
+                timesSegmentationImageReceived++;
+                CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data);
+            }
+
+            var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), false);
+
+            var plane = TestHelper.CreateLabeledPlane();
+            var meshRenderer = plane.GetComponent<MeshRenderer>();
+            var baseMaterial = meshRenderer.material;
+            meshRenderer.materials = new[] { baseMaterial, baseMaterial };
+            MaterialPropertyBlock mpb = new MaterialPropertyBlock();
+            mpb.SetFloat("float", 1f);
+            for (int i = 0; i < 2; i++)
+            {
+                meshRenderer.SetPropertyBlock(mpb, i);
+            }
+            AddTestObjectForCleanup(plane);
+            yield return null;
+            //destroy the object to force all pending segmented image readbacks to finish and events to be fired.
+            DestroyTestObject(cameraObject);
+            Assert.AreEqual(1, timesSegmentationImageReceived);
+        }
+
+
+        [UnityTest]
+        public IEnumerator SemanticSegmentationPass_WithChangingLabeling_ProducesCorrectValues([Values(true, false)] bool showVisualizations)
+        {
+            int timesSegmentationImageReceived = 0;
+            var expectedPixelValue = k_SemanticPixelValue;
+            void OnSegmentationImageReceived(NativeArray<Color32> data)
+            {
+                if (timesSegmentationImageReceived == 1)
+                {
+                    CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data);
+                }
+                timesSegmentationImageReceived++;
+            }
+
+            var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), false);
+
+            var plane = TestHelper.CreateLabeledPlane(label: "non-matching");
+            AddTestObjectForCleanup(plane);
+            yield return null;
+            var labeling = plane.GetComponent<Labeling>();
+            labeling.labels = new List<string> { "label" };
+            labeling.RefreshLabeling();
+            yield return null;
+            //destroy the object to force all pending segmented image readbacks to finish and events to be fired.
+            DestroyTestObject(cameraObject);
+            Assert.AreEqual(2, timesSegmentationImageReceived);
         }
 
         [UnityTest]

com.unity.perception/Runtime/GroundTruth/Labeling/Labeling.cs

@@ -1,6 +1,9 @@
 using System;
+using System.Diagnostics;
 using System.Text.RegularExpressions;
+using Unity.Collections;
 using UnityEngine;
+using UnityEngine.Experimental.Rendering;
 using UnityEngine.Perception.GroundTruth;
 
 namespace GroundTruthTests
@@ -18,6 +21,21 @@ public static GameObject CreateLabeledPlane(float scale = 10, string label = "la
             return planeObject;
         }
 
+        public static void ReadRenderTextureRawData<T>(RenderTexture renderTexture, Action<NativeArray<T>> callback) where T : struct
+        {
+            RenderTexture.active = renderTexture;
+
+            var cpuTexture = new Texture2D(renderTexture.width, renderTexture.height, renderTexture.graphicsFormat, TextureCreationFlags.None);
+
+            cpuTexture.ReadPixels(new Rect(
+                    Vector2.zero,
+                    new Vector2(renderTexture.width, renderTexture.height)),
+                0, 0);
+            RenderTexture.active = null;
+            var data = cpuTexture.GetRawTextureData<T>();
+            callback(data);
+        }
+
 #if UNITY_EDITOR
         public static void LoadAndStartRenderDocCapture(out UnityEditor.EditorWindow gameView)
         {