Branch synchronization: prerelease/2.6.0_stabilization --> mrtk_development (#9416)

* Fix issues associated with the new SU package

* Update Interactive Element Docs

* Remove Unity 2018 support for scene understanding

* Update SceneUnderstandingExample.unity

* Add comments

* add versioning to the shader import logic

* Update ReleaseNotes.md

* add versioning comment to shader source files

* Update SceneUnderstanding.md

* add documentation on updating shaders

* add doc comments

* undo changes to project prefs

* Update release notes

* Fix links

* Update TOC

* Update Assets/MRTK/StandardAssets/EditorUtilities/OnLoadUtilities.cs

Co-authored-by: Kurtis <[email protected]>

* update TMP version

* Explicitly set IsEnabled to false when the SU package is not present

Co-Authored-By: Kurtis <[email protected]>

* Update SceneUnderstanding.md

* Add null check for XR not initialized

* Update legacy and XR SDK implementation

* Fixed articulated hands being classified as HP Motion Controllers (#9410)

* Fixed articulated hands being classified as HP Motion Controllers

* small modification to ensure we only name interaction sources of kind controller

* small adjustment to the naming schemes

* implement ignore support for shader update

* update shader update documentation and images

* fix code validation error (missing doc comment contents)

* remove unneeded debug.log messages

* Add Scene Understanding to the ignore list

* fix test discovered error

* Add Pulse Shader Graduation to Release Notes

Co-authored-by: MaxWang-MS <[email protected]>
Co-authored-by: Catherine Diaz <[email protected]>
Co-authored-by: davidkline-ms <[email protected]>
Co-authored-by: David Kline <[email protected]>
Co-authored-by: Kurtis <[email protected]>
Co-authored-by: Kurtis <[email protected]>
Co-authored-by: RogPodge <[email protected]>
Co-authored-by: CDiaz-MS <[email protected]>

Author: 7 people

Assets/MRTK/Examples/Demos/ReadingMode/Scripts/ReadingModeSceneBehavior.cs

@@ -3,15 +3,9 @@
 
 using Microsoft.MixedReality.Toolkit.CameraSystem;
 using Microsoft.MixedReality.Toolkit.UI;
-using UnityEngine;
-
-#if UNITY_2019_3_OR_NEWER
 using Microsoft.MixedReality.Toolkit.Utilities;
-#endif // UNITY_2019_3_OR_NEWER
-
-#if !UNITY_2020_1_OR_NEWER
+using UnityEngine;
 using UnityEngine.XR;
-#endif // !UNITY_2020_1_OR_NEWER
 
 namespace Microsoft.MixedReality.Toolkit.Examples.Demos.ReadingMode
 {
@@ -35,20 +29,10 @@ private void Update()
 
             previousSliderValue = renderViewportScaleSlider.SliderValue;
 
-#if UNITY_2019_3_OR_NEWER
-            if (XRSubsystemHelpers.DisplaySubsystem != null)
-            {
-                XRSubsystemHelpers.DisplaySubsystem.scaleOfAllViewports = Mathf.Max(renderViewportScaleSlider.SliderValue, MinScale);
-                return;
-            }
-#endif // UNITY_2019_3_OR_NEWER
-
-#if !UNITY_2020_1_OR_NEWER
-            if (XRDevice.isPresent)
+            if (DeviceUtility.IsPresent)
             {
                 XRSettings.renderViewportScale = Mathf.Max(renderViewportScaleSlider.SliderValue, MinScale);
             }
-#endif // !UNITY_2020_1_OR_NEWER
         }
 
         public void EnableReadingMode()

Assets/MRTK/Examples/Experimental/SceneUnderstanding/Scenes/SceneUnderstandingExample.unity

@@ -3831,7 +3831,7 @@ MonoBehaviour:
   m_OnCullStateChanged:
     m_PersistentCalls:
       m_Calls: []
-  m_text: HoloLens 2 Only
+  m_text: HoloLens 2 & Unity 2019.4+ Only
   m_isRightToLeft: 0
   m_fontAsset: {fileID: 11400000, guid: f137eba12ee10834cb19632437cfdb2e, type: 2}
   m_sharedMaterial: {fileID: 21433621844796372, guid: f137eba12ee10834cb19632437cfdb2e,
@@ -3901,10 +3901,10 @@ MonoBehaviour:
   m_margin: {x: -0.4731984, y: 0.000020836515, z: -0.99050045, w: 1.3652543}
   m_textInfo:
     textComponent: {fileID: 1155140273}
-    characterCount: 15
+    characterCount: 31
     spriteCount: 0
-    spaceCount: 2
-    wordCount: 3
+    spaceCount: 5
+    wordCount: 6
     linkCount: 0
     lineCount: 1
     pageCount: 1
@@ -4926,22 +4926,22 @@ PrefabInstance:
     - target: {fileID: 2113875699234556495, guid: 18470d71939382448be2ecd06efa9662,
         type: 3}
       propertyPath: m_textInfo.characterCount
-      value: 699
+      value: 541
       objectReference: {fileID: 0}
     - target: {fileID: 2113875699234556495, guid: 18470d71939382448be2ecd06efa9662,
         type: 3}
       propertyPath: m_textInfo.spaceCount
-      value: 94
+      value: 76
       objectReference: {fileID: 0}
     - target: {fileID: 2113875699234556495, guid: 18470d71939382448be2ecd06efa9662,
         type: 3}
       propertyPath: m_textInfo.wordCount
-      value: 92
+      value: 76
       objectReference: {fileID: 0}
     - target: {fileID: 2113875699234556495, guid: 18470d71939382448be2ecd06efa9662,
         type: 3}
       propertyPath: m_textInfo.lineCount
-      value: 18
+      value: 13
       objectReference: {fileID: 0}
     - target: {fileID: 2113875699234556495, guid: 18470d71939382448be2ecd06efa9662,
         type: 3}
@@ -4961,14 +4961,10 @@ PrefabInstance:
 
         <size=24><b>Important - Required packages</b></size>
 
-        Use Nuget for Unity tool to get the Dot Net WinRT and SceneUnderstanding packages.
-        Please remember to Click Mixed Reality Toolkit -> Utilities -> Windows Mixed
-        Reality -> Check Configuration and Mixed Reality Toolkit -> Utilities -> Scene
-        Understanding -> Check Configuration after importing those two packages.
-
-
-        Uncheck ''Should Load From File'' in the DemoSceneUnderstandingObserverProfile
-        which can be accessed from MRTK''s Spatial Awareness profile.'
+        Use MixedReality Feature Tool to get the Dot Net WinRT and Scene Understanding
+        packages. The Scene Understanding package may not be available at MRTK 2.6
+        launch time but will be released soon after. Please check issue #9402 on the
+        MRTK-Unity GitHub repository for updates on its availability.'
       objectReference: {fileID: 0}
     - target: {fileID: 2113875699234556495, guid: 18470d71939382448be2ecd06efa9662,
         type: 3}
@@ -6250,8 +6246,8 @@ Transform:
   m_PrefabAsset: {fileID: 0}
   m_GameObject: {fileID: 2139668523}
   m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
-  m_LocalPosition: {x: 0.11993182, y: 0.08893815, z: -0.018035235}
-  m_LocalScale: {x: 0.08666847, y: 0.016785234, z: 0.04809894}
+  m_LocalPosition: {x: 0.0793, y: 0.08893815, z: -0.018}
+  m_LocalScale: {x: 0.168, y: 0.016785234, z: 0.04809894}
   m_Children: []
   m_Father: {fileID: 615958088}
   m_RootOrder: 1

Assets/MRTK/Providers/Experimental/WindowsSceneUnderstanding/Editor/MRTK.WSU.Editor.asmdef

@@ -15,5 +15,9 @@
     "overrideReferences": false,
     "precompiledReferences": [],
     "autoReferenced": true,
-    "defineConstraints": []
+    "defineConstraints": [
+        "UNITY_2019_4_OR_NEWER"
+    ],
+    "versionDefines": [],
+    "noEngineReferences": false
 }

Assets/MRTK/Providers/Experimental/WindowsSceneUnderstanding/Editor/WindowsSceneUnderstandingConfigChecker.cs

@@ -3,7 +3,9 @@
     "references": [
         "Microsoft.MixedReality.Toolkit",
         "Microsoft.MixedReality.Toolkit.Async",
-        "Microsoft.MixedReality.Toolkit.Providers.WindowsMixedReality.Shared"
+        "Microsoft.MixedReality.Toolkit.Providers.WindowsMixedReality.Shared",
+        "Microsoft.MixedReality.SceneUnderstanding.Projections.Editor",
+        "Microsoft.MixedReality.SceneUnderstanding.Projections.WSA"
     ],
     "optionalUnityReferences": [],
     "includePlatforms": [
@@ -15,5 +17,15 @@
     "overrideReferences": false,
     "precompiledReferences": [],
     "autoReferenced": true,
-    "defineConstraints": []
+    "defineConstraints": [
+        "UNITY_2019_4_OR_NEWER"
+    ],
+    "versionDefines": [
+        {
+            "name": "com.microsoft.mixedreality.sceneunderstanding",
+            "expression": "",
+            "define": "SCENE_UNDERSTANDING_PRESENT"
+        }
+    ],
+    "noEngineReferences": false
 }

Assets/MRTK/Providers/Experimental/WindowsSceneUnderstanding/Editor/WindowsSceneUnderstandingConfigChecker.cs.meta

@@ -31,13 +31,20 @@
 
 namespace Microsoft.MixedReality.Toolkit.WindowsSceneUnderstanding.Experimental
 {
+    /// <summary>
+    /// A Spatial Awareness observer with Scene Understanding capabilities. 
+    /// </summary>
+    /// <remarks>
+    /// Only works with HoloLens 2 and Unity 2019.4+
+    /// </remarks>
     [MixedRealityDataProvider(
         typeof(IMixedRealitySpatialAwarenessSystem),
         SupportedPlatforms.WindowsUniversal,
         "Windows Scene Understanding Observer",
         "Experimental/WindowsSceneUnderstanding/Profiles/DefaultSceneUnderstandingObserverProfile.asset",
         "MixedRealityToolkit.Providers",
         true)]
+    [HelpURL("https://docs.microsoft.com/windows/mixed-reality/mrtk-unity/features/spatial-awareness/scene-understanding")]
     public class WindowsSceneUnderstandingObserver :
         BaseSpatialObserver,
         IMixedRealitySceneUnderstandingObserver
@@ -97,25 +104,33 @@ private void ReadProfile()
             OrientScene = profile.OrientScene;
         }
 
-#if SCENE_UNDERSTANDING_PRESENT
-
         #region IMixedRealityService
 
+#if SCENE_UNDERSTANDING_PRESENT
+
         /// <inheritdoc />
         public override void Reset()
         {
             CleanupObserver();
             Initialize();
         }
 
+#endif // SCENE_UNDERSTANDING_PRESENT
+
         /// <inheritdoc />
         public override void Initialize()
         {
+#if !SCENE_UNDERSTANDING_PRESENT
+            if (Application.isPlaying)
+            {
+                Debug.LogWarning("The required package Microsoft.MixedReality.SceneUnderstanding is not installed or properly configured. Please visit https://docs.microsoft.com/windows/mixed-reality/mrtk-unity/features/spatial-awareness/scene-understanding for more information.");
+            }
+#else
             base.Initialize();
             sceneEventData = new MixedRealitySpatialAwarenessEventData<SpatialAwarenessSceneObject>(EventSystem.current);
             CreateQuadFromExtents(normalizedQuadMesh, 1, 1);
 
-            var accessStatus = SceneObserver.RequestAccessAsync().GetAwaiter().GetResult();
+            SceneObserverAccessStatus accessStatus = Task.Run(RequestAccess).GetAwaiter().GetResult();
             if (accessStatus == SceneObserverAccessStatus.Allowed)
             {
                 IsRunning = true;
@@ -131,20 +146,25 @@ public override void Initialize()
             {
                 observerState = ObserverState.GetScene;
             }
+#endif // SCENE_UNDERSTANDING_PRESENT
         }
 
         /// <inheritdoc />
         public override void Enable()
         {
+#if SCENE_UNDERSTANDING_PRESENT
+            base.Enable();
             // Terminate the background thread when we stop in editor.
             cancelToken = cancelTokenSource.Token;
 
-            // there is odd behavior with WaitForBackgroundTask
-            // it will sometimes run on the main thread unless we start it this way
-            task = RunObserverAsync(cancelToken);
-            task.ConfigureAwait(true);
+            task = Task.Run(() => RunObserverAsync(cancelToken));
+#else
+            IsEnabled = false;
+#endif // SCENE_UNDERSTANDING_PRESENT
         }
 
+#if SCENE_UNDERSTANDING_PRESENT
+
         /// <inheritdoc />
         public override void Update()
         {
@@ -171,8 +191,12 @@ public override void Destroy()
             CleanupObserver();
         }
 
+#endif // SCENE_UNDERSTANDING_PRESENT
+
         #endregion IMixedRealityService
 
+#if SCENE_UNDERSTANDING_PRESENT
+
         #region BaseService
 
         /// <inheritdoc />
@@ -280,7 +304,7 @@ public IReadOnlyDictionary<int, SpatialAwarenessSceneObject> SceneObjects
         public void SaveScene(string filenamePrefix)
         {
 #if WINDOWS_UWP && SCENE_UNDERSTANDING_PRESENT
-            SaveToFile(filenamePrefix);
+            Task.Run(() => SaveToFile(filenamePrefix));
 #else // WINDOWS_UWP
             Debug.LogWarning("SaveScene() only supported at runtime! Ignoring request.");
 #endif // WINDOWS_UWP
@@ -523,6 +547,11 @@ protected virtual void SendSceneObjectRemoved(int id)
             SpatialAwarenessSystem?.HandleEvent(sceneEventData, OnSceneObjectRemoved);
         }
 
+        private async Task<SceneObserverAccessStatus> RequestAccess()
+        {
+            return await SceneObserver.RequestAccessAsync();
+        }
+
         /// <summary>
         /// Sets up and starts update timers
         /// </summary>
@@ -630,7 +659,7 @@ private async Task RunObserverAsync(CancellationToken cancellationToken)
                         }
                         await new WaitForBackgroundThread();
                         {
-                            scene = GetSceneAsync(previousScene);
+                            scene = await GetSceneAsync(previousScene);
                             previousScene = scene;
                             sceneOriginId = scene.OriginSpatialGraphNodeId;
                         }
@@ -761,7 +790,7 @@ private System.Numerics.Matrix4x4 GetSceneToWorldTransform()
         /// </summary>
         /// <param name="previousScene">The previous scene</param>
         /// <returns>The retrieved scene</returns>
-        private Scene GetSceneAsync(Scene previousScene)
+        private async Task<Scene> GetSceneAsync(Scene previousScene)
         {
             Scene scene = null;
 
@@ -795,18 +824,13 @@ private Scene GetSceneAsync(Scene previousScene)
                     RequestedMeshLevelOfDetail = LevelOfDetailToMeshLOD(WorldMeshLevelOfDetail)
                 };
 
-                // Ideally you'd call SceneObserver.ComputeAsync() like this:
-                // scene = await SceneObserver.ComputeAsync(...);
-                // however this has been problematic (buggy?)
-                // For the time being we force it to be synchronous with the ...GetAwaiter().GetResult() pattern
-
                 if (UsePersistentObjects && previousScene != null)
                 {
-                    scene = SceneObserver.ComputeAsync(sceneQuerySettings, QueryRadius, previousScene).GetAwaiter().GetResult();
+                    scene = await SceneObserver.ComputeAsync(sceneQuerySettings, QueryRadius, previousScene);
                 }
                 else
                 {
-                    scene = SceneObserver.ComputeAsync(sceneQuerySettings, QueryRadius).GetAwaiter().GetResult();
+                    scene = await SceneObserver.ComputeAsync(sceneQuerySettings, QueryRadius);
                 }
             }
 
@@ -1301,7 +1325,7 @@ private async void SaveToFile(string prefix)
                 RequestedMeshLevelOfDetail = LevelOfDetailToMeshLOD(WorldMeshLevelOfDetail)
             };
 
-            var serializedScene = SceneObserver.ComputeSerializedAsync(sceneQuerySettings, QueryRadius).GetAwaiter().GetResult();
+            var serializedScene = await SceneObserver.ComputeSerializedAsync(sceneQuerySettings, QueryRadius);
             var bytes = new byte[serializedScene.Size];
             serializedScene.GetData(bytes);
             var timestamp = DateTime.Now.ToString("yyyyMMdd_hhmmss");

Assets/MRTK/Providers/Experimental/WindowsSceneUnderstanding/MRTK.WSU.asmdef

@@ -720,9 +720,13 @@ private BaseWindowsMixedRealitySource GetOrAddController(InteractionSource inter
 
             }
 
-            string nameModifier = controllingHand == Handedness.None ? interactionSource.kind.ToString() : controllingHand.ToString();
-            bool isHPController = !interactionSource.supportsTouchpad;
-            string inputSourceName = isHPController ? $"HP Motion Controller {nameModifier}" : $"Mixed Reality Controller {nameModifier}";
+            bool isHPController = !interactionSource.supportsTouchpad && interactionSource.kind == InteractionSourceKind.Controller;
+
+            string kindModifier = interactionSource.kind.ToString();
+            string handednessModifier = controllingHand == Handedness.None ? string.Empty : controllingHand.ToString();
+
+            string inputSourceName = isHPController ? $"HP Motion {kindModifier} {handednessModifier}" : $"Mixed Reality {kindModifier} {handednessModifier}";
+
             var inputSource = Service?.RequestNewGenericInputSource(inputSourceName, pointers, inputSourceType);
 
             BaseWindowsMixedRealitySource detectedController;