Skip to content

Commit 9984eb9

Browse files
keveleighkeveleigh
authored
Merge pull request #7146 from keveleigh/refactor-out-shared-wmr-hand-definition
Refactor out shared WMR articulated hand definition
2 parents 15fbc47 + ff2657e commit 9984eb9

File tree

4 files changed

+311
-459
lines changed

4 files changed

+311
-459
lines changed

‎Assets/MixedRealityToolkit.Providers/WindowsMixedReality/Shared/WindowsMixedRealityArticulatedHandDefinition.cs‎

Lines changed: 241 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,241 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using Microsoft.MixedReality.Toolkit.Input;
5+
using Microsoft.MixedReality.Toolkit.Utilities;
6+
using System;
7+
using System.Collections.Generic;
8+
using UnityEngine;
9+
10+
#if WINDOWS_UWP
11+
using Windows.Perception.People;
12+
using Windows.UI.Input.Spatial;
13+
#endif // WINDOWS_UWP
14+
15+
namespace Microsoft.MixedReality.Toolkit.WindowsMixedReality
16+
{
17+
public class WindowsMixedRealityArticulatedHandDefinition
18+
{
19+
public WindowsMixedRealityArticulatedHandDefinition(IMixedRealityInputSource source, Handedness handedness)
20+
{
21+
inputSource = source;
22+
this.handedness = handedness;
23+
}
24+
25+
private readonly IMixedRealityInputSource inputSource;
26+
private readonly Handedness handedness;
27+
private readonly float cursorBeamBackwardTolerance = 0.5f;
28+
private readonly float cursorBeamUpTolerance = 0.8f;
29+
30+
private Dictionary<TrackedHandJoint, MixedRealityPose> unityJointPoses = new Dictionary<TrackedHandJoint, MixedRealityPose>();
31+
private MixedRealityPose currentIndexPose = MixedRealityPose.ZeroIdentity;
32+
33+
/// <summary>
34+
/// The Windows Mixed Reality articulated hands default interactions.
35+
/// </summary>
36+
/// <remarks>A single interaction mapping works for both left and right articulated hands.</remarks>
37+
public MixedRealityInteractionMapping[] DefaultInteractions => new[]
38+
{
39+
new MixedRealityInteractionMapping(0, "Spatial Pointer", AxisType.SixDof, DeviceInputType.SpatialPointer),
40+
new MixedRealityInteractionMapping(1, "Spatial Grip", AxisType.SixDof, DeviceInputType.SpatialGrip),
41+
new MixedRealityInteractionMapping(2, "Select", AxisType.Digital, DeviceInputType.Select),
42+
new MixedRealityInteractionMapping(3, "Grab", AxisType.SingleAxis, DeviceInputType.TriggerPress),
43+
new MixedRealityInteractionMapping(4, "Index Finger Pose", AxisType.SixDof, DeviceInputType.IndexFinger)
44+
};
45+
46+
/// <summary>
47+
/// Calculates whether the current pose allows for pointing/distant interactions.
48+
/// </summary>
49+
public bool IsInPointingPose
50+
{
51+
get
52+
{
53+
MixedRealityPose palmJoint;
54+
if (unityJointPoses.TryGetValue(TrackedHandJoint.Palm, out palmJoint))
55+
{
56+
Vector3 palmNormal = palmJoint.Rotation * (-1 * Vector3.up);
57+
if (cursorBeamBackwardTolerance >= 0)
58+
{
59+
Vector3 cameraBackward = -CameraCache.Main.transform.forward;
60+
if (Vector3.Dot(palmNormal.normalized, cameraBackward) > cursorBeamBackwardTolerance)
61+
{
62+
return false;
63+
}
64+
}
65+
if (cursorBeamUpTolerance >= 0)
66+
{
67+
if (Vector3.Dot(palmNormal, Vector3.up) > cursorBeamUpTolerance)
68+
{
69+
return false;
70+
}
71+
}
72+
}
73+
return true;
74+
}
75+
}
76+
77+
public void UpdateHandJoints(Dictionary<TrackedHandJoint, MixedRealityPose> jointPoses)
78+
{
79+
unityJointPoses = jointPoses;
80+
CoreServices.InputSystem?.RaiseHandJointsUpdated(inputSource, handedness, unityJointPoses);
81+
}
82+
83+
public void UpdateCurrentIndexPose(MixedRealityInteractionMapping interactionMapping)
84+
{
85+
if (unityJointPoses.TryGetValue(TrackedHandJoint.IndexTip, out currentIndexPose))
86+
{
87+
// Update the interaction data source
88+
interactionMapping.PoseData = currentIndexPose;
89+
90+
// If our value changed raise it
91+
if (interactionMapping.Changed)
92+
{
93+
// Raise input system event if it's enabled
94+
CoreServices.InputSystem?.RaisePoseInputChanged(inputSource, handedness, interactionMapping.MixedRealityInputAction, currentIndexPose);
95+
}
96+
}
97+
}
98+
99+
#if WINDOWS_UWP
100+
private Vector2[] handMeshUVs = null;
101+
private HandMeshObserver handMeshObserver = null;
102+
private int[] handMeshTriangleIndices = null;
103+
private bool hasRequestedHandMeshObserver = false;
104+
105+
private async void SetHandMeshObserver(SpatialInteractionSourceState sourceState)
106+
{
107+
handMeshObserver = await sourceState.Source.TryCreateHandMeshObserverAsync();
108+
}
109+
110+
private void InitializeUVs(Vector3[] neutralPoseVertices)
111+
{
112+
if (neutralPoseVertices.Length == 0)
113+
{
114+
Debug.LogError("Loaded 0 verts for neutralPoseVertices");
115+
}
116+
117+
float minY = neutralPoseVertices[0].y;
118+
float maxY = minY;
119+
120+
for (int ix = 1; ix < neutralPoseVertices.Length; ix++)
121+
{
122+
Vector3 p = neutralPoseVertices[ix];
123+
124+
if (p.y < minY)
125+
{
126+
minY = p.y;
127+
}
128+
else if (p.y > maxY)
129+
{
130+
maxY = p.y;
131+
}
132+
}
133+
134+
float scale = 1.0f / (maxY - minY);
135+
136+
handMeshUVs = new Vector2[neutralPoseVertices.Length];
137+
138+
for (int ix = 0; ix < neutralPoseVertices.Length; ix++)
139+
{
140+
Vector3 p = neutralPoseVertices[ix];
141+
142+
handMeshUVs[ix] = new Vector2(p.x * scale + 0.5f, (p.y - minY) * scale);
143+
}
144+
}
145+
146+
public void UpdateHandMesh(SpatialInteractionSourceState sourceState)
147+
{
148+
MixedRealityHandTrackingProfile handTrackingProfile = null;
149+
MixedRealityInputSystemProfile inputSystemProfile = CoreServices.InputSystem?.InputSystemProfile;
150+
if (inputSystemProfile != null)
151+
{
152+
handTrackingProfile = inputSystemProfile.HandTrackingProfile;
153+
}
154+
155+
if (handTrackingProfile == null || !handTrackingProfile.EnableHandMeshVisualization)
156+
{
157+
// If hand mesh visualization is disabled make sure to destroy our hand mesh observer if it has already been created
158+
if (handMeshObserver != null)
159+
{
160+
// Notify that hand mesh has been updated (cleared)
161+
HandMeshInfo handMeshInfo = new HandMeshInfo();
162+
CoreServices.InputSystem?.RaiseHandMeshUpdated(inputSource, handedness, handMeshInfo);
163+
hasRequestedHandMeshObserver = false;
164+
handMeshObserver = null;
165+
}
166+
return;
167+
}
168+
169+
HandPose handPose = sourceState.TryGetHandPose();
170+
171+
// Accessing the hand mesh data involves copying quite a bit of data, so only do it if application requests it.
172+
if (handMeshObserver == null && !hasRequestedHandMeshObserver)
173+
{
174+
SetHandMeshObserver(sourceState);
175+
hasRequestedHandMeshObserver = true;
176+
}
177+
178+
if (handMeshObserver != null && handMeshTriangleIndices == null)
179+
{
180+
uint indexCount = handMeshObserver.TriangleIndexCount;
181+
ushort[] indices = new ushort[indexCount];
182+
handMeshObserver.GetTriangleIndices(indices);
183+
handMeshTriangleIndices = new int[indexCount];
184+
Array.Copy(indices, handMeshTriangleIndices, (int)handMeshObserver.TriangleIndexCount);
185+
186+
// Compute neutral pose
187+
Vector3[] neutralPoseVertices = new Vector3[handMeshObserver.VertexCount];
188+
HandPose neutralPose = handMeshObserver.NeutralPose;
189+
var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
190+
HandMeshVertexState handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose);
191+
handMeshVertexState.GetVertices(vertexAndNormals);
192+
193+
for (int i = 0; i < handMeshObserver.VertexCount; i++)
194+
{
195+
neutralPoseVertices[i] = vertexAndNormals[i].Position.ToUnityVector3();
196+
}
197+
198+
// Compute UV mapping
199+
InitializeUVs(neutralPoseVertices);
200+
}
201+
202+
if (handPose != null && handMeshObserver != null && handMeshTriangleIndices != null)
203+
{
204+
var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
205+
var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose);
206+
handMeshVertexState.GetVertices(vertexAndNormals);
207+
208+
var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(WindowsMixedRealityUtilities.SpatialCoordinateSystem);
209+
if (meshTransform.HasValue)
210+
{
211+
System.Numerics.Vector3 scale;
212+
System.Numerics.Quaternion rotation;
213+
System.Numerics.Vector3 translation;
214+
System.Numerics.Matrix4x4.Decompose(meshTransform.Value, out scale, out rotation, out translation);
215+
216+
var handMeshVertices = new Vector3[handMeshObserver.VertexCount];
217+
var handMeshNormals = new Vector3[handMeshObserver.VertexCount];
218+
219+
for (int i = 0; i < handMeshObserver.VertexCount; i++)
220+
{
221+
handMeshVertices[i] = vertexAndNormals[i].Position.ToUnityVector3();
222+
handMeshNormals[i] = vertexAndNormals[i].Normal.ToUnityVector3();
223+
}
224+
225+
HandMeshInfo handMeshInfo = new HandMeshInfo
226+
{
227+
vertices = handMeshVertices,
228+
normals = handMeshNormals,
229+
triangles = handMeshTriangleIndices,
230+
uvs = handMeshUVs,
231+
position = translation.ToUnityVector3(),
232+
rotation = rotation.ToUnityQuaternion()
233+
};
234+
235+
CoreServices.InputSystem?.RaiseHandMeshUpdated(inputSource, handedness, handMeshInfo);
236+
}
237+
}
238+
}
239+
#endif // WINDOWS_UWP
240+
}
241+
}

‎Assets/MixedRealityToolkit.Providers/WindowsMixedReality/Shared/WindowsMixedRealityArticulatedHandDefinition.cs.meta‎

Lines changed: 11 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)