fix(hand tracking): fix issue when hand lose tracking and stays visible

Fixes #273

Me a @saitonakamura checked how the events were sent and found some issues with code not cleaning up, not disposing used resources and not hiding hands when they are not tracking. We decided to add OculusHandModel and XRHandMeshModel to this library so the fix is ready for usage as soon as possible.

Author: Ledzz

CONTRIBUTING.md

@@ -12,7 +12,7 @@
 
 ## Run locally
 
-* Run `yarn dev` to be able to to run examples
+* Run `yarn dev` to be able to run examples
 
 ## PR checklist
 
@@ -27,4 +27,4 @@
 ### Test examples locally
 
 Using real device is recommended, otherwise you can use https://github.com/meta-quest/immersive-web-emulator/.
-See [Run locally](#run-locally)
+See [Run locally](#run-locally)

examples/src/demos/Teleport.tsx

@@ -1,5 +1,5 @@
 import { Canvas } from '@react-three/fiber'
-import { Hands, XR, VRButton, TeleportationPlane, Controllers } from '@react-three/xr'
+import { XR, VRButton, TeleportationPlane, Controllers } from '@react-three/xr'
 
 export default function () {
   return (

src/Controllers.tsx

@@ -64,10 +64,16 @@ class ControllerModel extends THREE.Group {
   }
 
   private _onConnected(event: XRControllerEvent) {
+    if (event.data?.hand) {
+      return
+    }
     modelFactory.initializeControllerModel(this.xrControllerModel, event)
   }
 
-  private _onDisconnected(_event: XRControllerEvent) {
+  private _onDisconnected(event: XRControllerEvent) {
+    if (event.data?.hand) {
+      return
+    }
     this.xrControllerModel.disconnect()
   }
 

src/Hands.tsx

@@ -1,6 +1,6 @@
 import * as React from 'react'
 import { Object3DNode, extend, createPortal } from '@react-three/fiber'
-import { OculusHandModel } from 'three-stdlib'
+import { OculusHandModel } from './OculusHandModel'
 import { useXR } from './XR'
 import { useIsomorphicLayoutEffect } from './utils'
 

src/OculusHandModel.ts

@@ -0,0 +1,117 @@
+import { Object3D, Sphere, Box3, Mesh, Texture, Vector3, EventListener, Event } from 'three'
+import { XRHandMeshModel } from './XRHandMeshModel'
+const TOUCH_RADIUS = 0.01
+const POINTING_JOINT = 'index-finger-tip'
+
+export interface XRButton extends Object3D {
+  onPress(): void
+  onClear(): void
+  isPressed(): boolean
+  whilePressed(): void
+}
+
+class OculusHandModel extends Object3D {
+  controller: Object3D
+  motionController: XRHandMeshModel | null
+  envMap: Texture | null
+  mesh: Mesh | null
+  xrInputSource: XRInputSource | null
+
+  leftModelPath?: string
+  rightModelPath?: string
+
+  constructor(controller: Object3D, leftModelPath?: string, rightModelPath?: string) {
+    super()
+
+    this.controller = controller
+    this.motionController = null
+    this.envMap = null
+    this.leftModelPath = leftModelPath
+    this.rightModelPath = rightModelPath
+
+    this.mesh = null
+    this.xrInputSource = null
+
+    controller.addEventListener('connected', this._onConnected)
+    controller.addEventListener('disconnected', this._onDisconnected)
+  }
+
+  private _onConnected: EventListener<Event, 'connected', Object3D<Event>> = (event) => {
+    const xrInputSource = event.data
+
+    if (xrInputSource.hand && !this.motionController) {
+      this.xrInputSource = xrInputSource
+
+      this.motionController = new XRHandMeshModel(
+        this,
+        this.controller,
+        undefined,
+        xrInputSource.handedness,
+        xrInputSource.handedness === 'left' ? this.leftModelPath : this.rightModelPath
+      )
+    }
+  }
+
+  private _onDisconnected: EventListener<Event, 'disconnected', Object3D<Event>> = () => {
+    if (!this.xrInputSource?.hand) {
+      return;
+    }
+    this.motionControllerCleanup()
+  }
+
+  private motionControllerCleanup(): void {
+    this.clear()
+    this.motionController?.dispose()
+    this.motionController = null
+  }
+
+  updateMatrixWorld(force?: boolean): void {
+    super.updateMatrixWorld(force)
+
+    if (this.motionController) {
+      this.motionController.updateMesh()
+    }
+  }
+
+  getPointerPosition(): Vector3 | null {
+    // @ts-ignore XRController needs to extend Group
+    const indexFingerTip = this.controller.joints[POINTING_JOINT]
+    if (indexFingerTip) {
+      return indexFingerTip.position
+    } else {
+      return null
+    }
+  }
+
+  intersectBoxObject(boxObject: Object3D): boolean {
+    const pointerPosition = this.getPointerPosition()
+    if (pointerPosition) {
+      const indexSphere = new Sphere(pointerPosition, TOUCH_RADIUS)
+      const box = new Box3().setFromObject(boxObject)
+      return indexSphere.intersectsBox(box)
+    } else {
+      return false
+    }
+  }
+
+  checkButton(button: XRButton): void {
+    if (this.intersectBoxObject(button)) {
+      button.onPress()
+    } else {
+      button.onClear()
+    }
+
+    if (button.isPressed()) {
+      button.whilePressed()
+    }
+  }
+
+  dispose(): void {
+    this.motionControllerCleanup()
+
+    this.controller.removeEventListener('connected', this._onConnected)
+    this.controller.removeEventListener('disconnected', this._onDisconnected)
+  }
+}
+
+export { OculusHandModel }

src/XRHandMeshModel.ts

@@ -0,0 +1,113 @@
+import { Object3D } from 'three'
+import { GLTFLoader } from 'three-stdlib'
+
+const DEFAULT_HAND_PROFILE_PATH = 'https://cdn.jsdelivr.net/npm/@webxr-input-profiles/[email protected]/dist/profiles/generic-hand/'
+
+class XRHandMeshModel {
+  controller: Object3D
+  handModel: Object3D
+  bones: Object3D[]
+  scene?: Object3D
+
+  constructor(
+    handModel: Object3D,
+    controller: Object3D,
+    path: string = DEFAULT_HAND_PROFILE_PATH,
+    handedness: string,
+    customModelPath?: string
+  ) {
+    this.controller = controller
+    this.handModel = handModel
+
+    this.bones = []
+
+    const loader = new GLTFLoader()
+    if (!customModelPath) loader.setPath(path)
+    loader.load(customModelPath ?? `${handedness}.glb`, (gltf: { scene: Object3D }) => {
+      const object = gltf.scene.children[0]
+      this.handModel.add(object)
+      this.scene = object
+
+      const mesh = object.getObjectByProperty('type', 'SkinnedMesh')!
+      mesh.frustumCulled = false
+      mesh.castShadow = true
+      mesh.receiveShadow = true
+
+      const joints = [
+        'wrist',
+        'thumb-metacarpal',
+        'thumb-phalanx-proximal',
+        'thumb-phalanx-distal',
+        'thumb-tip',
+        'index-finger-metacarpal',
+        'index-finger-phalanx-proximal',
+        'index-finger-phalanx-intermediate',
+        'index-finger-phalanx-distal',
+        'index-finger-tip',
+        'middle-finger-metacarpal',
+        'middle-finger-phalanx-proximal',
+        'middle-finger-phalanx-intermediate',
+        'middle-finger-phalanx-distal',
+        'middle-finger-tip',
+        'ring-finger-metacarpal',
+        'ring-finger-phalanx-proximal',
+        'ring-finger-phalanx-intermediate',
+        'ring-finger-phalanx-distal',
+        'ring-finger-tip',
+        'pinky-finger-metacarpal',
+        'pinky-finger-phalanx-proximal',
+        'pinky-finger-phalanx-intermediate',
+        'pinky-finger-phalanx-distal',
+        'pinky-finger-tip'
+      ]
+
+      joints.forEach((jointName) => {
+        const bone = object.getObjectByName(jointName) as any
+
+        if (bone !== undefined) {
+          bone.jointName = jointName
+        } else {
+          console.warn(`Couldn't find ${jointName} in ${handedness} hand mesh`)
+        }
+
+        this.bones.push(bone)
+      })
+    })
+  }
+
+  updateMesh(): void {
+    // XR Joints
+    const XRJoints = (this.controller as any).joints
+    let allInvisible = true
+
+    for (let i = 0; i < this.bones.length; i++) {
+      const bone = this.bones[i]
+
+      if (bone) {
+        const XRJoint = XRJoints[(bone as any).jointName]
+
+        if (XRJoint.visible) {
+          const position = XRJoint.position
+          bone.position.copy(position)
+          bone.quaternion.copy(XRJoint.quaternion)
+          allInvisible = false
+        }
+      }
+    }
+
+    // Hide hand mesh if all joints are invisible in case hand loses tracking
+    if (allInvisible && this.scene) {
+      this.scene.visible = false
+    } else if (this.scene) {
+      this.scene.visible = true
+    }
+  }
+
+  dispose(): void {
+    if (this.scene) {
+      this.handModel.remove(this.scene)
+    }
+  }
+}
+
+export { XRHandMeshModel }

yarn.lock

@@ -10,7 +10,6 @@
     "@jridgewell/gen-mapping" "^0.1.0"
     "@jridgewell/trace-mapping" "^0.3.9"
 
-
 "@babel/code-frame@^7.18.6":
   version "7.18.6"
   resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz"
@@ -755,6 +754,11 @@
   resolved "https://registry.npmjs.org/@types/chai/-/chai-4.3.4.tgz"
   integrity sha512-KnRanxnpfpjUTqTCXslZSEdLfXExwgNxYPdiO2WGUj8+HDjFi8R3k5RVKPeSCzLjCcshCAtVO2QBbVuAV4kTnw==
 
+"@types/draco3d@^1.4.0":
+  version "1.4.2"
+  resolved "https://registry.yarnpkg.com/@types/draco3d/-/draco3d-1.4.2.tgz#7faccb809db2a5e19b9efb97c5f2eb9d64d527ea"
+  integrity sha512-goh23EGr6CLV6aKPwN1p8kBD/7tT5V/bLpToSbarKrwVejqNrspVrv8DhliteYkkhZYrlq/fwKZRRUzH4XN88w==
+
 "@types/json-schema@^7.0.9":
   version "7.0.11"
   resolved "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz"
@@ -827,6 +831,11 @@
   resolved "https://registry.npmjs.org/@types/webxr/-/webxr-0.4.0.tgz"
   integrity sha512-LQvrACV3Pj17GpkwHwXuTd733gfY+D7b9mKdrTmLdO7vo7P/o6209Qqtk63y/FCv/lspdmi0pWz6Qe/ull9kQg==
 
+"@types/webxr@^0.5.2":
+  version "0.5.2"
+  resolved "https://registry.yarnpkg.com/@types/webxr/-/webxr-0.5.2.tgz#5d9627b0ffe223aa3b166de7112ac8a9460dc54f"
+  integrity sha512-szL74BnIcok9m7QwYtVmQ+EdIKwbjPANudfuvDrAF8Cljg9MKUlIoc1w5tjj9PMpeSH3U1Xnx//czQybJ0EfSw==
+
 "@typescript-eslint/eslint-plugin@^5.11.0":
   version "5.28.0"
   resolved "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.28.0.tgz"
@@ -2337,11 +2346,6 @@ jsonc-parser@^3.2.0:
     array-includes "^3.1.4"
     object.assign "^4.1.2"
 
-ktx-parse@^0.2.1:
-  version "0.2.2"
-  resolved "https://registry.npmjs.org/ktx-parse/-/ktx-parse-0.2.2.tgz"
-  integrity sha512-cFBc1jnGG2WlUf52NbDUXK2obJ+Mo9WUkBRvr6tP6CKxRMvZwDDFNV3JAS4cewETp5KyexByfWm9sm+O8AffiQ==
-
 ktx-parse@^0.4.5:
   version "0.4.5"
   resolved "https://registry.npmjs.org/ktx-parse/-/ktx-parse-0.4.5.tgz"
@@ -3187,16 +3191,17 @@ three-mesh-bvh@^0.5.10:
   integrity sha512-IMNHrAnsLCIxcFmAGkA4Wibw1QEpFQlkR72XUxZFOatNSpfMRUhJXQwQ5jPxbrX0W+OR838t/IR3laMOvQnT/g==
 
 three-stdlib@^2.10.2:
-  version "2.12.1"
-  resolved "https://registry.npmjs.org/three-stdlib/-/three-stdlib-2.12.1.tgz"
-  integrity sha512-G3SSsCOBiWa0sjjPt+K28ikQ84Plm/ZVUozMfWagK59kZqBWcaPVXpOThkAgvdBpm2zCWLW3edAoW/4XIbljVQ==
+  version "2.23.10"
+  resolved "https://registry.yarnpkg.com/three-stdlib/-/three-stdlib-2.23.10.tgz#94907a558a00da327bd74308c92078fea72f77fc"
+  integrity sha512-y0DlxaN5HZXI9hKjEtqO2xlCEt7XyDCOMvD2M3JJFBmYjwbU+PbJ1n3Z+7Hr/6BeVGE6KZYcqPMnfKrTK5WTJg==
   dependencies:
-    "@babel/runtime" "^7.16.7"
-    "@webgpu/glslang" "^0.0.15"
+    "@types/draco3d" "^1.4.0"
+    "@types/offscreencanvas" "^2019.6.4"
+    "@types/webxr" "^0.5.2"
     chevrotain "^10.1.2"
     draco3d "^1.4.1"
     fflate "^0.6.9"
-    ktx-parse "^0.2.1"
+    ktx-parse "^0.4.5"
     mmd-parser "^1.0.4"
     opentype.js "^1.3.3"
     potpack "^1.0.1"