Merge pull request #9390 from MaxWang-MS/su_2.6

Fix issues associated with the new SU package

Author: MaxWang-MS

Assets/MRTK/Examples/Experimental/SceneUnderstanding/Scenes/SceneUnderstandingExample.unity

@@ -3831,7 +3831,7 @@ MonoBehaviour:
   m_OnCullStateChanged:
     m_PersistentCalls:
       m_Calls: []
-  m_text: HoloLens 2 Only
+  m_text: HoloLens 2 & Unity 2019.4+ Only
   m_isRightToLeft: 0
   m_fontAsset: {fileID: 11400000, guid: f137eba12ee10834cb19632437cfdb2e, type: 2}
   m_sharedMaterial: {fileID: 21433621844796372, guid: f137eba12ee10834cb19632437cfdb2e,
@@ -3901,10 +3901,10 @@ MonoBehaviour:
   m_margin: {x: -0.4731984, y: 0.000020836515, z: -0.99050045, w: 1.3652543}
   m_textInfo:
     textComponent: {fileID: 1155140273}
-    characterCount: 15
+    characterCount: 31
     spriteCount: 0
-    spaceCount: 2
-    wordCount: 3
+    spaceCount: 5
+    wordCount: 6
     linkCount: 0
     lineCount: 1
     pageCount: 1
@@ -4926,22 +4926,22 @@ PrefabInstance:
     - target: {fileID: 2113875699234556495, guid: 18470d71939382448be2ecd06efa9662,
         type: 3}
       propertyPath: m_textInfo.characterCount
-      value: 699
+      value: 541
       objectReference: {fileID: 0}
     - target: {fileID: 2113875699234556495, guid: 18470d71939382448be2ecd06efa9662,
         type: 3}
       propertyPath: m_textInfo.spaceCount
-      value: 94
+      value: 76
       objectReference: {fileID: 0}
     - target: {fileID: 2113875699234556495, guid: 18470d71939382448be2ecd06efa9662,
         type: 3}
       propertyPath: m_textInfo.wordCount
-      value: 92
+      value: 76
       objectReference: {fileID: 0}
     - target: {fileID: 2113875699234556495, guid: 18470d71939382448be2ecd06efa9662,
         type: 3}
       propertyPath: m_textInfo.lineCount
-      value: 18
+      value: 13
       objectReference: {fileID: 0}
     - target: {fileID: 2113875699234556495, guid: 18470d71939382448be2ecd06efa9662,
         type: 3}
@@ -4961,14 +4961,10 @@ PrefabInstance:
 
         <size=24><b>Important - Required packages</b></size>
 
-        Use Nuget for Unity tool to get the Dot Net WinRT and SceneUnderstanding packages.
-        Please remember to Click Mixed Reality Toolkit -> Utilities -> Windows Mixed
-        Reality -> Check Configuration and Mixed Reality Toolkit -> Utilities -> Scene
-        Understanding -> Check Configuration after importing those two packages.
-
-
-        Uncheck ''Should Load From File'' in the DemoSceneUnderstandingObserverProfile
-        which can be accessed from MRTK''s Spatial Awareness profile.'
+        Use MixedReality Feature Tool to get the Dot Net WinRT and Scene Understanding
+        packages. The Scene Understanding package may not be available at MRTK 2.6
+        launch time but will be released soon after. Please check issue #9402 on the
+        MRTK-Unity GitHub repository for updates on its availability.'
       objectReference: {fileID: 0}
     - target: {fileID: 2113875699234556495, guid: 18470d71939382448be2ecd06efa9662,
         type: 3}
@@ -6250,8 +6246,8 @@ Transform:
   m_PrefabAsset: {fileID: 0}
   m_GameObject: {fileID: 2139668523}
   m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
-  m_LocalPosition: {x: 0.11993182, y: 0.08893815, z: -0.018035235}
-  m_LocalScale: {x: 0.08666847, y: 0.016785234, z: 0.04809894}
+  m_LocalPosition: {x: 0.0793, y: 0.08893815, z: -0.018}
+  m_LocalScale: {x: 0.168, y: 0.016785234, z: 0.04809894}
   m_Children: []
   m_Father: {fileID: 615958088}
   m_RootOrder: 1

Assets/MRTK/Providers/Experimental/WindowsSceneUnderstanding/Editor/MRTK.WSU.Editor.asmdef

@@ -15,5 +15,9 @@
     "overrideReferences": false,
     "precompiledReferences": [],
     "autoReferenced": true,
-    "defineConstraints": []
+    "defineConstraints": [
+        "UNITY_2019_4_OR_NEWER"
+    ],
+    "versionDefines": [],
+    "noEngineReferences": false
 }

Assets/MRTK/Providers/Experimental/WindowsSceneUnderstanding/Editor/WindowsSceneUnderstandingConfigChecker.cs

@@ -3,7 +3,9 @@
     "references": [
         "Microsoft.MixedReality.Toolkit",
         "Microsoft.MixedReality.Toolkit.Async",
-        "Microsoft.MixedReality.Toolkit.Providers.WindowsMixedReality.Shared"
+        "Microsoft.MixedReality.Toolkit.Providers.WindowsMixedReality.Shared",
+        "Microsoft.MixedReality.SceneUnderstanding.Projections.Editor",
+        "Microsoft.MixedReality.SceneUnderstanding.Projections.WSA"
     ],
     "optionalUnityReferences": [],
     "includePlatforms": [
@@ -15,5 +17,15 @@
     "overrideReferences": false,
     "precompiledReferences": [],
     "autoReferenced": true,
-    "defineConstraints": []
+    "defineConstraints": [
+        "UNITY_2019_4_OR_NEWER"
+    ],
+    "versionDefines": [
+        {
+            "name": "com.microsoft.mixedreality.sceneunderstanding",
+            "expression": "",
+            "define": "SCENE_UNDERSTANDING_PRESENT"
+        }
+    ],
+    "noEngineReferences": false
 }

Assets/MRTK/Providers/Experimental/WindowsSceneUnderstanding/Editor/WindowsSceneUnderstandingConfigChecker.cs.meta

@@ -31,13 +31,20 @@
 
 namespace Microsoft.MixedReality.Toolkit.WindowsSceneUnderstanding.Experimental
 {
+    /// <summary>
+    /// A Spatial Awareness observer with Scene Understanding capabilities. 
+    /// </summary>
+    /// <remarks>
+    /// Only works with HoloLens 2 and Unity 2019.4+
+    /// </remarks>
     [MixedRealityDataProvider(
         typeof(IMixedRealitySpatialAwarenessSystem),
         SupportedPlatforms.WindowsUniversal,
         "Windows Scene Understanding Observer",
         "Experimental/WindowsSceneUnderstanding/Profiles/DefaultSceneUnderstandingObserverProfile.asset",
         "MixedRealityToolkit.Providers",
         true)]
+    [HelpURL("https://docs.microsoft.com/windows/mixed-reality/mrtk-unity/features/spatial-awareness/scene-understanding")]
     public class WindowsSceneUnderstandingObserver :
         BaseSpatialObserver,
         IMixedRealitySceneUnderstandingObserver
@@ -97,25 +104,33 @@ private void ReadProfile()
             OrientScene = profile.OrientScene;
         }
 
-#if SCENE_UNDERSTANDING_PRESENT
-
         #region IMixedRealityService
 
+#if SCENE_UNDERSTANDING_PRESENT
+
         /// <inheritdoc />
         public override void Reset()
         {
             CleanupObserver();
             Initialize();
         }
 
+#endif // SCENE_UNDERSTANDING_PRESENT
+
         /// <inheritdoc />
         public override void Initialize()
         {
+#if !SCENE_UNDERSTANDING_PRESENT
+            if (Application.isPlaying)
+            {
+                Debug.LogWarning("The required package Microsoft.MixedReality.SceneUnderstanding is not installed or properly configured. Please visit https://docs.microsoft.com/windows/mixed-reality/mrtk-unity/features/spatial-awareness/scene-understanding for more information.");
+            }
+#else
             base.Initialize();
             sceneEventData = new MixedRealitySpatialAwarenessEventData<SpatialAwarenessSceneObject>(EventSystem.current);
             CreateQuadFromExtents(normalizedQuadMesh, 1, 1);
 
-            var accessStatus = SceneObserver.RequestAccessAsync().GetAwaiter().GetResult();
+            SceneObserverAccessStatus accessStatus = Task.Run(RequestAccess).GetAwaiter().GetResult();
             if (accessStatus == SceneObserverAccessStatus.Allowed)
             {
                 IsRunning = true;
@@ -131,20 +146,25 @@ public override void Initialize()
             {
                 observerState = ObserverState.GetScene;
             }
+#endif // SCENE_UNDERSTANDING_PRESENT
         }
 
         /// <inheritdoc />
         public override void Enable()
         {
+#if SCENE_UNDERSTANDING_PRESENT
+            base.Enable();
             // Terminate the background thread when we stop in editor.
             cancelToken = cancelTokenSource.Token;
 
-            // there is odd behavior with WaitForBackgroundTask
-            // it will sometimes run on the main thread unless we start it this way
-            task = RunObserverAsync(cancelToken);
-            task.ConfigureAwait(true);
+            task = Task.Run(() => RunObserverAsync(cancelToken));
+#else
+            IsEnabled = false;
+#endif // SCENE_UNDERSTANDING_PRESENT
         }
 
+#if SCENE_UNDERSTANDING_PRESENT
+
         /// <inheritdoc />
         public override void Update()
         {
@@ -171,8 +191,12 @@ public override void Destroy()
             CleanupObserver();
         }
 
+#endif // SCENE_UNDERSTANDING_PRESENT
+
         #endregion IMixedRealityService
 
+#if SCENE_UNDERSTANDING_PRESENT
+
         #region BaseService
 
         /// <inheritdoc />
@@ -280,7 +304,7 @@ public IReadOnlyDictionary<int, SpatialAwarenessSceneObject> SceneObjects
         public void SaveScene(string filenamePrefix)
         {
 #if WINDOWS_UWP && SCENE_UNDERSTANDING_PRESENT
-            SaveToFile(filenamePrefix);
+            Task.Run(() => SaveToFile(filenamePrefix));
 #else // WINDOWS_UWP
             Debug.LogWarning("SaveScene() only supported at runtime! Ignoring request.");
 #endif // WINDOWS_UWP
@@ -523,6 +547,11 @@ protected virtual void SendSceneObjectRemoved(int id)
             SpatialAwarenessSystem?.HandleEvent(sceneEventData, OnSceneObjectRemoved);
         }
 
+        private async Task<SceneObserverAccessStatus> RequestAccess()
+        {
+            return await SceneObserver.RequestAccessAsync();
+        }
+
         /// <summary>
         /// Sets up and starts update timers
         /// </summary>
@@ -630,7 +659,7 @@ private async Task RunObserverAsync(CancellationToken cancellationToken)
                         }
                         await new WaitForBackgroundThread();
                         {
-                            scene = GetSceneAsync(previousScene);
+                            scene = await GetSceneAsync(previousScene);
                             previousScene = scene;
                             sceneOriginId = scene.OriginSpatialGraphNodeId;
                         }
@@ -761,7 +790,7 @@ private System.Numerics.Matrix4x4 GetSceneToWorldTransform()
         /// </summary>
         /// <param name="previousScene">The previous scene</param>
         /// <returns>The retrieved scene</returns>
-        private Scene GetSceneAsync(Scene previousScene)
+        private async Task<Scene> GetSceneAsync(Scene previousScene)
         {
             Scene scene = null;
 
@@ -795,18 +824,13 @@ private Scene GetSceneAsync(Scene previousScene)
                     RequestedMeshLevelOfDetail = LevelOfDetailToMeshLOD(WorldMeshLevelOfDetail)
                 };
 
-                // Ideally you'd call SceneObserver.ComputeAsync() like this:
-                // scene = await SceneObserver.ComputeAsync(...);
-                // however this has been problematic (buggy?)
-                // For the time being we force it to be synchronous with the ...GetAwaiter().GetResult() pattern
-
                 if (UsePersistentObjects && previousScene != null)
                 {
-                    scene = SceneObserver.ComputeAsync(sceneQuerySettings, QueryRadius, previousScene).GetAwaiter().GetResult();
+                    scene = await SceneObserver.ComputeAsync(sceneQuerySettings, QueryRadius, previousScene);
                 }
                 else
                 {
-                    scene = SceneObserver.ComputeAsync(sceneQuerySettings, QueryRadius).GetAwaiter().GetResult();
+                    scene = await SceneObserver.ComputeAsync(sceneQuerySettings, QueryRadius);
                 }
             }
 
@@ -1301,7 +1325,7 @@ private async void SaveToFile(string prefix)
                 RequestedMeshLevelOfDetail = LevelOfDetailToMeshLOD(WorldMeshLevelOfDetail)
             };
 
-            var serializedScene = SceneObserver.ComputeSerializedAsync(sceneQuerySettings, QueryRadius).GetAwaiter().GetResult();
+            var serializedScene = await SceneObserver.ComputeSerializedAsync(sceneQuerySettings, QueryRadius);
             var bytes = new byte[serializedScene.Size];
             serializedScene.GetData(bytes);
             var timestamp = DateTime.Now.ToString("yyyyMMdd_hhmmss");

Assets/MRTK/Providers/Experimental/WindowsSceneUnderstanding/MRTK.WSU.asmdef

@@ -45,6 +45,15 @@ see our updated [Input Simulation Service documentation](InputSimulation/InputSi
 
   ![Teleport Gesture](Images/ReleaseNotes/HandTeleport.gif)
 
+### Scene Understanding now available in MRTK as an experimental spatial awareness observer
+
+Experimental support of [Scene Understanding](https://docs.microsoft.com/en-us/windows/mixed-reality/scene-understanding) is introduced in MRTK 2.6. Users can incorporate the scene understanding capabilities of HoloLens 2 as a spatial awareness observer in MRTK based projects. Please read the [Scene Understanding documentation](SpatialAwareness/SceneUnderstanding.md) for more information.
+
+> [!IMPORTANT]
+> Scene Understanding is only supported on HoloLens 2 and Unity 2019.4 and higher.
+>
+> This feature requires the Scene Understanding package, which may not be available at the launch date of MRTK 2.6 but will be available soon after via the [Mixed Reality Feature Tool](https://aka.ms/MRFeatureTool). Please see [this issue](https://github.com/microsoft/MixedRealityToolkit-Unity/issues/9402) for updates on its availability.
+
 ### Runtime profile switching support
 
 MRTK now allows profile switching both before the initialization of the MRTK instance (i.e. Pre MRTK initialization profile switch) and after a profile has been in active use (i.e. Active profile switch). The former switch can be used to enable select components based on capabilities of the hardware, while the latter can be used to modify experience as the user enters a subpart of the application. Please read the [documentation on profile switching](MixedRealityConfigurationGuide.md#changing-profiles-at-runtime) for more information and code samples.

Assets/MRTK/Providers/Experimental/WindowsSceneUnderstanding/WindowsSceneUnderstandingObserver.cs

@@ -8,7 +8,7 @@ Some expected use cases of this technology are:
 * Provide physics engine friendly geometry as quads
 * Accelerate development by avoiding the need to write similar algorithms
 
-Currently Scene Understanding is available as an __experimental__ feature. It is integrated into MRTK as a [spatial observer](SpatialAwarenessGettingStarted.md#register-observers) called [`WindowsSceneUnderstandingObserver`](xref:Microsoft.MixedReality.Toolkit.WindowsSceneUnderstanding.Experimental.WindowsSceneUnderstandingObserver).
+Currently Scene Understanding is available as an __experimental__ feature. It is integrated into MRTK as a [spatial observer](SpatialAwarenessGettingStarted.md#register-observers) called [`WindowsSceneUnderstandingObserver`](xref:Microsoft.MixedReality.Toolkit.WindowsSceneUnderstanding.Experimental.WindowsSceneUnderstandingObserver). Scene Understanding works both with the Legacy XR pipeline and the XR SDK pipeline. In both cases the `WindowsSceneUnderstandingObserver` is used.
 
 ## Observer overview
 
@@ -17,10 +17,7 @@ When asked, the [`WindowsSceneUnderstandingObserver`](xref:Microsoft.MixedRealit
 ## Setup
 
 1. Ensure the platform is set to UWP in build settings.
-1. Ensure dotnetwinrt is installed.
-1. Acquire the SceneUnderstanding package that contains the binaries required for the feature
-1. In the menu bar click Mixed Reality Toolkit -> Utilities -> Windows Mixed Reality -> Check Configuration (this operation may take a while)
-1. In the menu bar click Mixed Reality Toolkit -> Utilities -> Scene Understanding -> Check Configuration (this operation may take a while)
+1. Acquire the Scene Understanding package via [Mixed Reality Feature Tool](https://aka.ms/MRFeatureTool). Note the package may not be available at the launch date of MRTK 2.6 but will be available soon after. Please see [this issue](https://github.com/microsoft/MixedRealityToolkit-Unity/issues/9402) for updates on its availability.
 
 ## Using Scene Understanding
 
@@ -37,6 +34,9 @@ The scene demonstrates the following:
 * Saving scene data to device for offline development
 * Loading previously saved scene data (.bytes files) to support in-editor development workflow
 
+> [!NOTE] 
+> The sample scene is based on the Legacy XR pipeline. If you are using the XR SDK pipeline you should modify the profiles accordingly. The provided Scene Understanding Spatial Awareness System profile (`DemoSceneUnderstandingSystemProfile`) and the Scene Understanding Observer profiles (`DefaultSceneUnderstandingObserverProfile` and `DemoSceneUnderstandingObserverProfile`) works for both pipelines.
+
 #### Configuring the observer service
 
 Select the 'MixedRealityToolkit' game object and check the inspector.