Merge pull request #10338 from MaxWang-MS/add-openxr-su-remoting

Enable Scene Understanding in remoting via OpenXR

Author: MaxWang-MS

Assets/MRTK/Providers/Experimental/WindowsSceneUnderstanding/WindowsSceneUnderstandingObserver.cs

@@ -14,14 +14,15 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.MixedReality.SceneUnderstanding;
-#if WINDOWS_UWP
-using Windows.Perception.Spatial;
-using Windows.Perception.Spatial.Preview;
 #if MSFT_OPENXR
 using Microsoft.MixedReality.OpenXR;
+using Microsoft.MixedReality.OpenXR.Remoting;
 using Microsoft.MixedReality.Toolkit.XRSDK;
 using UnityEngine.XR.OpenXR;
 #endif // MSFT_OPENXR
+#if WINDOWS_UWP
+using Windows.Perception.Spatial;
+using Windows.Perception.Spatial.Preview;
 #endif // WINDOWS_UWP
 using UnityEngine.Assertions;
 using UnityEngine.EventSystems;
@@ -129,13 +130,12 @@ public override void Initialize()
             }
 #else
             base.Initialize();
-#if WINDOWS_UWP
 #if MSFT_OPENXR
             isOpenXRLoaderActive = LoaderHelpers.IsLoaderActive<OpenXRLoaderBase>() ?? false;
-#else
+            isOpenXRRemotingConnected = AppRemoting.TryGetConnectionState(out ConnectionState state, out _) && state == ConnectionState.Connected;
+#elif WINDOWS_UWP
             isOpenXRLoaderActive = false;
 #endif // MSFT_OPENXR
-#endif // WINDOWS_UWP
             sceneEventData = new MixedRealitySpatialAwarenessEventData<SpatialAwarenessSceneObject>(EventSystem.current);
             CreateQuadFromExtents(normalizedQuadMesh, 1, 1);
 
@@ -419,9 +419,12 @@ private enum ObserverState
         private System.Numerics.Matrix4x4 sceneToWorldTransformMatrix;
         private List<SceneObject> filteredSelectedSurfaceTypesResult = new List<SceneObject>(128);
         private Texture defaultTexture;
-#if WINDOWS_UWP
+#if WINDOWS_UWP || MSFT_OPENXR
         private bool isOpenXRLoaderActive;
-#endif // WINDOWS_UWP
+#endif // WINDOWS_UWP || MSFT_OPENXR
+#if MSFT_OPENXR
+        private bool isOpenXRRemotingConnected;
+#endif // MSFT_OPENXR
 
         #endregion Private Fields
 
@@ -794,8 +797,13 @@ private async Task RunObserverAsync(CancellationToken cancellationToken)
         private System.Numerics.Matrix4x4? GetSceneToWorldTransform()
         {
             var result = System.Numerics.Matrix4x4.Identity;
-#if WINDOWS_UWP
+#if WINDOWS_UWP // On HoloLens 2 device
             if (isOpenXRLoaderActive)
+#elif MSFT_OPENXR // In editor and using OpenXR
+            if (isOpenXRLoaderActive && isOpenXRRemotingConnected && !ShouldLoadFromFile)
+#else // All other cases
+            if (false)
+#endif // WINDOWS_UWP
             {
 #if MSFT_OPENXR
                 SpatialGraphNode node = SpatialGraphNode.FromStaticNodeId(sceneOriginId);
@@ -811,6 +819,7 @@ private async Task RunObserverAsync(CancellationToken cancellationToken)
             }
             else
             {
+#if WINDOWS_UWP
                 SpatialCoordinateSystem sceneOrigin = SpatialGraphInteropPreview.CreateCoordinateSystemForNode(sceneOriginId);
                 SpatialCoordinateSystem worldOrigin = WindowsMixedReality.WindowsMixedRealityUtilities.SpatialCoordinateSystem;
 
@@ -824,8 +833,8 @@ private async Task RunObserverAsync(CancellationToken cancellationToken)
                 {
                     return null;
                 }
-            }
 #endif // WINDOWS_UWP
+            }
             return result;
         }