Merge pull request #9550 from keveleigh/xrsdk-controller-models

Add support for motion controller models at runtime in WMR/XRSDK

Author: keveleigh

Assets/MRTK/Core/Definitions/Devices/MixedRealityControllerVisualizationSetting.cs

@@ -21,6 +21,7 @@ public struct MixedRealityControllerVisualizationSetting
         /// <param name="controllerType">Controller Type to instantiate at runtime.</param>
         /// <param name="handedness">The designated hand that the device is managing.</param>
         /// <param name="overrideModel">The controller model prefab to be rendered.</param>
+        [Obsolete("This constructor doesn't need to be called directly.")]
         public MixedRealityControllerVisualizationSetting(string description, Type controllerType, Handedness handedness = Handedness.None, GameObject overrideModel = null) : this()
         {
             this.description = description;

Assets/MRTK/Core/Providers/BaseController.cs

@@ -234,14 +234,14 @@ public void AssignControllerMappings(MixedRealityInteractionMapping[] mappings)
         /// <returns>True if a model was successfully loaded or model rendering is disabled. False if a model tried to load but failed.</returns>
         protected virtual bool TryRenderControllerModel(Type controllerType, InputSourceType inputSourceType)
         {
-            GameObject controllerModel = null;
-
             if (GetControllerVisualizationProfile() == null ||
                 !GetControllerVisualizationProfile().RenderMotionControllers)
             {
                 return true;
             }
 
+            GameObject controllerModel = null;
+
             // If a specific controller template wants to override the global model, assign that instead.
             if (IsControllerMappingEnabled() &&
                 GetControllerVisualizationProfile() != null &&

Assets/MRTK/Providers/WindowsMixedReality/Shared/Extensions/WindowsExtensions.cs

@@ -2,10 +2,13 @@
 // Licensed under the MIT License.
 
 #if (UNITY_WSA && DOTNETWINRT_PRESENT) || WINDOWS_UWP
+using Microsoft.MixedReality.Toolkit.Input;
 using Microsoft.MixedReality.Toolkit.Utilities;
 #if WINDOWS_UWP
+using Windows.Perception;
 using Windows.UI.Input.Spatial;
 #elif DOTNETWINRT_PRESENT
+using Microsoft.Windows.Perception;
 using Microsoft.Windows.UI.Input.Spatial;
 #endif
 #endif // (UNITY_WSA && DOTNETWINRT_PRESENT) || WINDOWS_UWP
@@ -37,6 +40,61 @@ public static Handedness ToMRTKHandedness(this SpatialInteractionSourceHandednes
                     return Handedness.None;
             }
         }
+
+        /// <summary>
+        /// Tries to get an active SpatialInteractionSource with the corresponding handedness and input source type.
+        /// </summary>
+        /// <param name="handedness">The handedness of the source to get.</param>
+        /// <param name="inputSourceType">The input source type of the source to get.</param>
+        /// <returns>The input source or null if none could be found.</returns>
+        public static SpatialInteractionSource GetSpatialInteractionSource(Handedness handedness, InputSourceType inputSourceType)
+        {
+            SpatialInteractionSourceHandedness sourceHandedness;
+            switch (handedness)
+            {
+                default:
+                    sourceHandedness = SpatialInteractionSourceHandedness.Unspecified;
+                    break;
+                case Handedness.Left:
+                    sourceHandedness = SpatialInteractionSourceHandedness.Left;
+                    break;
+                case Handedness.Right:
+                    sourceHandedness = SpatialInteractionSourceHandedness.Right;
+                    break;
+            }
+
+            SpatialInteractionSourceKind sourceKind;
+            switch (inputSourceType)
+            {
+                default:
+                    sourceKind = SpatialInteractionSourceKind.Other;
+                    break;
+                case InputSourceType.Controller:
+                    sourceKind = SpatialInteractionSourceKind.Controller;
+                    break;
+                case InputSourceType.Hand:
+                    sourceKind = SpatialInteractionSourceKind.Hand;
+                    break;
+            }
+
+            System.Collections.Generic.IReadOnlyList<SpatialInteractionSourceState> sourceStates =
+                WindowsMixedRealityUtilities.SpatialInteractionManager?.GetDetectedSourcesAtTimestamp(PerceptionTimestampHelper.FromHistoricalTargetTime(System.DateTimeOffset.UtcNow));
+
+            if (sourceStates == null)
+            {
+                return null;
+            }
+
+            foreach (SpatialInteractionSourceState sourceState in sourceStates)
+            {
+                if (sourceState.Source.Handedness == sourceHandedness && sourceState.Source.Kind == sourceKind)
+                {
+                    return sourceState.Source;
+                }
+            }
+
+            return null;
+        }
 #endif // (UNITY_WSA && DOTNETWINRT_PRESENT) || WINDOWS_UWP
     }
 }

Assets/MRTK/Providers/WindowsMixedReality/Shared/MRTK.WMR.Shared.asmdef

@@ -1,7 +1,8 @@
 {
     "name": "Microsoft.MixedReality.Toolkit.Providers.WindowsMixedReality.Shared",
     "references": [
-        "Microsoft.MixedReality.Toolkit"
+        "Microsoft.MixedReality.Toolkit",
+        "Microsoft.MixedReality.Toolkit.Gltf"
     ],
     "includePlatforms": [
         "Editor",

Assets/MRTK/Providers/WindowsMixedReality/Shared/WindowsMixedRealityControllerModelProvider.cs

@@ -0,0 +1,93 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using Microsoft.MixedReality.Toolkit.Utilities;
+
+#if WINDOWS_UWP
+using Microsoft.MixedReality.Toolkit.Input;
+using Microsoft.MixedReality.Toolkit.Utilities.Gltf.Serialization;
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using UnityEngine;
+using Windows.Storage.Streams;
+using Windows.UI.Input.Spatial;
+#endif
+
+namespace Microsoft.MixedReality.Toolkit.WindowsMixedReality
+{
+    /// <summary>
+    /// Queries the WinRT APIs for a renderable controller model.
+    /// </summary>
+    public class WindowsMixedRealityControllerModelProvider
+    {
+        public WindowsMixedRealityControllerModelProvider(Handedness handedness)
+        {
+#if WINDOWS_UWP
+            spatialInteractionSource = WindowsExtensions.GetSpatialInteractionSource(handedness, InputSourceType.Controller);
+#endif // WINDOWS_UWP
+        }
+
+#if WINDOWS_UWP
+        private readonly SpatialInteractionSource spatialInteractionSource;
+
+        private static readonly Dictionary<string, GameObject> ControllerModelDictionary = new Dictionary<string, GameObject>(2);
+
+        /// <summary>
+        /// Attempts to load the glTF controller model from the Windows SDK.
+        /// </summary>
+        /// <returns>The controller model as a GameObject or null if it was unobtainable.</returns>
+        public async Task<GameObject> TryGenerateControllerModelFromPlatformSDK()
+        {
+            if (spatialInteractionSource == null)
+            {
+                return null;
+            }
+
+            // See if we've generated this model before and if we can return it
+            if (ControllerModelDictionary.TryGetValue(GenerateKey(spatialInteractionSource), out GameObject controllerModel))
+            {
+                controllerModel.SetActive(true);
+                return controllerModel;
+            }
+
+            Debug.Log("Trying to load controller model from platform SDK");
+            byte[] fileBytes = null;
+
+            var controllerModelStream = await spatialInteractionSource.Controller.TryGetRenderableModelAsync();
+            if (controllerModelStream == null ||
+                controllerModelStream.Size == 0)
+            {
+                Debug.LogError("Failed to obtain controller model from driver");
+            }
+            else
+            {
+                fileBytes = new byte[controllerModelStream.Size];
+                using (DataReader reader = new DataReader(controllerModelStream))
+                {
+                    await reader.LoadAsync((uint)controllerModelStream.Size);
+                    reader.ReadBytes(fileBytes);
+                }
+            }
+
+            GameObject gltfGameObject = null;
+            if (fileBytes != null)
+            {
+                Utilities.Gltf.Schema.GltfObject gltfObject = GltfUtility.GetGltfObjectFromGlb(fileBytes);
+                gltfGameObject = await gltfObject.ConstructAsync();
+                if (gltfGameObject != null)
+                {
+                    ControllerModelDictionary.Add(GenerateKey(spatialInteractionSource), gltfGameObject);
+                }
+            }
+
+            return gltfGameObject;
+        }
+
+        private string GenerateKey(SpatialInteractionSource spatialInteractionSource)
+        {
+            return spatialInteractionSource.Controller.VendorId + "/" + spatialInteractionSource.Controller.ProductId + "/" + spatialInteractionSource.Controller.Version + "/" + spatialInteractionSource.Handedness;
+        }
+#endif // WINDOWS_UWP
+    }
+}