feat: anchor, iwer emulate, hit test, dom overlay

fix: xr screen input, teleport pointer enabled by default

Author: bbohlender

packages/xr/package.json

@@ -26,8 +26,13 @@
     "fix:prettier": "prettier --write src",
     "fix:eslint": "eslint 'src/**/*.ts' --fix"
   },
+  "peerDependencies": {
+    "three": "*"
+  },
   "dependencies": {
+    "@iwer/devui": "^0.1.0",
     "@pmndrs/pointer-events": "workspace:^",
+    "iwer": "^1.0.3",
     "meshline": "^3.3.1",
     "zustand": "^4.5.2"
   },

packages/xr/src/anchor.ts

@@ -0,0 +1,126 @@
+import { Matrix4, Quaternion, Vector3 } from 'three'
+import { XRStore } from './store.js'
+
+const OneVector = new Vector3(1, 1, 1)
+const ZeroVector = new Vector3(1, 1, 1)
+const NeutralQuaternion = new Quaternion()
+
+const matrixHelper1 = new Matrix4()
+const matrixHelper2 = new Matrix4()
+const quaternionHelper = new Quaternion()
+const positionHelper = new Vector3()
+
+const directionHelper = new Vector3()
+
+/*
+export async function loadXRPersistentAnchor(session: XRSession, id: string): Promise<XRAnchor | undefined> {
+  const anchorId = localStorage.getItem(id)
+  if (anchorId == null || session == null) {
+    return undefined
+  }
+  if (!('restorePersistentAnchor' in session)) {
+    console.warn(`"restorePersistentAnchor" not supported`)
+    return undefined
+  }
+  return (session.restorePersistentAnchor as (id: string) => Promise<XRAnchor>)(anchorId)
+}
+
+export async function requestXRPersistentAnchor(
+  store: XRStore<any>,
+  id: string,
+  options: XRAnchorOptions,
+  abortRef?: { current: boolean },
+): Promise<XRAnchor | undefined> {
+  const anchor = await requestXRAnchor(store, options)
+  if (anchor == null || abortRef?.current === true) {
+    return undefined
+  }
+  if (!('requestPersistentHandle' in anchor)) {
+    console.warn(`"requestPersistentHandle" not supported`)
+    return undefined
+  }
+  const anchorHandle = await (anchor.requestPersistentHandle as () => Promise<string>)()
+  if (anchor == null || (abortRef?.current as boolean | undefined) === true) {
+    return undefined
+  }
+  localStorage.setItem(id, anchorHandle)
+  return anchor
+}
+
+export async function deleteXRPersistentAnchor(store: XRStore<any>, id: string) {
+  const { session } = store.getState()
+  if (session == null) {
+    return
+  }
+  if (!('deletePersistentAnchor' in session)) {
+    console.warn(`"deletePersistentAnchor" not supported`)
+    return undefined
+  }
+  return (session.deletePersistentAnchor as (id: string) => Promise<undefined>)(id)
+}*/
+
+export type XRAnchorOptions = XRAnchorWorldOptions | XRAnchorSpaceOptions | XRAnchorHitTestResultOptions
+
+export type XRAnchorWorldOptions = {
+  relativeTo: 'world'
+  worldPosition: Vector3
+  worldQuaternion: Quaternion
+  frame?: XRFrame
+}
+
+export type XRAnchorHitTestResultOptions = {
+  relativeTo: 'hit-test-result'
+  hitTestResult: XRHitTestResult
+  offsetPosition?: Vector3
+  offsetQuaternion?: Quaternion
+}
+export type XRAnchorSpaceOptions = {
+  relativeTo: 'space'
+  space: XRSpace
+  offsetPosition?: Vector3
+  offsetQuaternion?: Quaternion
+  frame?: XRFrame
+}
+
+export async function requestXRAnchor(store: XRStore<any>, options: XRAnchorOptions): Promise<XRAnchor | undefined> {
+  if (options.relativeTo === 'hit-test-result') {
+    directionHelper.set(0, 0, 1)
+    if (options.offsetQuaternion != null) {
+      directionHelper.applyQuaternion(options.offsetQuaternion)
+    }
+    return options.hitTestResult.createAnchor?.(
+      new XRRigidTransform({ ...options.offsetPosition }, { ...directionHelper }),
+    )
+  }
+  let frame: XRFrame
+  let space: XRSpace
+  if (options.relativeTo === 'world') {
+    frame = options.frame ?? (await store.requestFrame())
+    const { origin, originReferenceSpace } = store.getState()
+    if (originReferenceSpace == null) {
+      return undefined
+    }
+    space = originReferenceSpace
+    const { worldPosition, worldQuaternion } = options
+    if (origin != null) {
+      //compute vectorHelper and quaternionHelper in the local space of the origin
+      matrixHelper1.copy(origin.matrixWorld).invert()
+      matrixHelper2.compose(worldPosition, worldQuaternion, OneVector).multiply(matrixHelper1)
+      matrixHelper2.decompose(positionHelper, quaternionHelper, directionHelper)
+
+      quaternionHelper.setFromRotationMatrix(matrixHelper2)
+    } else {
+      positionHelper.copy(worldPosition)
+      quaternionHelper.copy(worldQuaternion)
+    }
+  } else {
+    frame = options.frame ?? (await store.requestFrame())
+    space = options.space
+    const { offsetPosition, offsetQuaternion } = options
+    positionHelper.copy(offsetPosition ?? ZeroVector)
+    quaternionHelper.copy(offsetQuaternion ?? NeutralQuaternion)
+  }
+  directionHelper.set(0, 0, 1)
+  directionHelper.applyQuaternion(quaternionHelper)
+  return frame.createAnchor?.(new XRRigidTransform({ ...positionHelper }, { ...directionHelper }), space)
+}

packages/xr/src/controller/layout.ts

@@ -50,7 +50,15 @@ type XRControllerProfilesList = Record<string, { path: string }>
 const DefaultDefaultControllerProfileId = 'generic-trigger'
 
 export type XRControllerLayoutLoaderOptions = {
+  /**
+   * where to load the controller profiles and models from
+   * @default 'https://cdn.jsdelivr.net/npm/@webxr-input-profiles/[email protected]/dist/profiles/'
+   */
   baseAssetPath?: string
+  /**
+   * profile id that is used if no matching profile id is found
+   * @default 'generic-trigger'
+   */
   defaultControllerProfileId?: string
 }
 

packages/xr/src/controller/model.ts

@@ -9,7 +9,15 @@ export async function loadXRControllerModel(layout: XRControllerLayout, loader:
 }
 
 export type XRControllerModelOptions = {
+  /**
+   * allows to configure whether the controller is rendered to the color buffer
+   * can be used to show the real controller in AR passthrough mode
+   */
   colorWrite?: boolean
+  /**
+   * allows to configure the render order of the controller model
+   * @default undefined
+   */
   renderOrder?: number
 }
 

packages/xr/src/controller/state.ts

@@ -1,23 +1,17 @@
-import { Object3D } from 'three'
 import { XRControllerGamepadState, updateXRControllerGamepadState } from './gamepad.js'
-import { XRControllerLayout, XRControllerLayoutLoader } from './layout.js'
-
-export type XRControllerState = {
-  type: 'controller'
-  inputSource: XRInputSource
-  gamepad: XRControllerGamepadState
-  layout: XRControllerLayout
-  object?: Object3D
-}
+import { XRControllerLayoutLoader } from './layout.js'
+import { XRControllerState } from '../input.js'
 
 export async function createXRControllerState(
   inputSource: XRInputSource,
   layoutLoader: XRControllerLayoutLoader,
+  events: ReadonlyArray<XRInputSourceEvent>,
 ): Promise<XRControllerState> {
   const layout = await layoutLoader.load(inputSource.profiles, inputSource.handedness)
   const gamepad: XRControllerGamepadState = {}
   updateXRControllerGamepadState(gamepad, inputSource, layout)
   return {
+    events,
     type: 'controller',
     inputSource,
     gamepad,

packages/xr/src/default.ts

@@ -22,9 +22,29 @@ export type DefaultXRHandTouchPointerOptions = TouchPointerOptions & {
 }
 
 export type DefaultXRControllerOptions = {
+  /**
+   * provide options to the <XRControllerModel/>
+   * `false` disables the model
+   * @default true
+   */
   model?: boolean | XRControllerModelOptions
+  /**
+   * provide options to the <DefaultXRInputSourceGrabPointer/>
+   * `false` disables the grab pointer
+   * @default true
+   */
   grabPointer?: boolean | DefaultXRInputSourceGrabPointerOptions
+  /**
+   * provide options to the <DefaultXRInputSourceRayPointer/>
+   * `false` disables the ray pointer
+   * @default true
+   */
   rayPointer?: boolean | DefaultXRInputSourceRayPointerOptions
+  /**
+   * provide options to the <DefaultXRInputSourceTeleportPointer/>
+   * `false` disables the teleport pointer
+   * @default false
+   */
   teleportPointer?: boolean | DefaultXRInputSourceTeleportPointerOptions
 }
 

packages/xr/src/emulate.ts

@@ -0,0 +1,13 @@
+import { XRDevice, metaQuest3, metaQuest2, metaQuestPro, oculusQuest1 } from 'iwer'
+import { DevUI } from '@iwer/devui'
+
+const configurations = { metaQuest3, metaQuest2, metaQuestPro, oculusQuest1 }
+
+export type EmulatorType = keyof typeof configurations
+
+export function emulate(type: EmulatorType) {
+  const xrdevice = new XRDevice(configurations[type])
+  xrdevice.ipd = 0
+  xrdevice.installRuntime()
+  new DevUI(xrdevice)
+}

packages/xr/src/hand/model.ts

@@ -8,7 +8,15 @@ import { clone as cloneSkeleton } from 'three/examples/jsm/utils/SkeletonUtils.j
 const DefaultDefaultXRHandProfileId = 'generic-hand'
 
 export type XRHandLoaderOptions = {
+  /**
+   * where to load the hand models from
+   * @default 'https://cdn.jsdelivr.net/npm/@webxr-input-profiles/[email protected]/dist/profiles/'
+   */
   baseAssetPath?: string
+  /**
+   * profile id that is used if no matching profile id is found
+   * @default 'generic-hand'
+   */
   defaultXRHandProfileId?: string
 }
 

packages/xr/src/hand/state.ts

@@ -1,27 +1,25 @@
-import { Object3D, WebXRManager } from 'three'
-import { XRHandPoseState, createHandPoseState, updateXRHandPoseState } from './pose.js'
+import { WebXRManager } from 'three'
+import { createHandPoseState, updateXRHandPoseState } from './pose.js'
 import { XRHandLoaderOptions, getXRHandAssetPath } from './model.js'
+import type { XRHandState } from '../input.js'
 
 export type XRHandInputSource = XRInputSource & { hand: XRHand }
 
 export function isXRHandInputSource(inputSource: XRInputSource): inputSource is XRHandInputSource {
   return inputSource.hand != null
 }
 
-export type XRHandState = {
-  type: 'hand'
-  inputSource: XRHandInputSource
-  pose: XRHandPoseState
-  assetPath: string
-  object?: Object3D
-}
-
-export function createXRHandState(inputSource: XRInputSource, options: XRHandLoaderOptions | undefined): XRHandState {
+export function createXRHandState(
+  inputSource: XRInputSource,
+  options: XRHandLoaderOptions | undefined,
+  events: ReadonlyArray<XRInputSourceEvent>,
+): XRHandState {
   return {
     type: 'hand',
     inputSource: inputSource as XRHandInputSource,
     pose: createHandPoseState(inputSource.hand!),
     assetPath: getXRHandAssetPath(inputSource.handedness, options),
+    events,
   }
 }
 

packages/xr/src/hand/visual.ts

@@ -1,5 +1,4 @@
 import { Object3D } from 'three'
-import { GetXRSpace } from '../space.js'
 
 const joints: Array<XRHandJoint> = [
   'wrist',
@@ -32,7 +31,7 @@ const joints: Array<XRHandJoint> = [
 export function createUpdateXRHandVisuals(
   hand: XRHand,
   handModel: Object3D,
-  referenceSpace: GetXRSpace,
+  referenceSpace: XRSpace | (() => XRSpace | undefined),
 ): (frame: XRFrame | undefined) => void {
   const buffer = new Float32Array(hand.size * 16)
   const jointObjects = joints.map((joint) => {
@@ -44,11 +43,8 @@ export function createUpdateXRHandVisuals(
     return jointObject
   })
   return (frame) => {
-    if (frame == null) {
-      return
-    }
     const resolvedReferenceSpace = typeof referenceSpace === 'function' ? referenceSpace() : referenceSpace
-    if (resolvedReferenceSpace == null) {
+    if (frame == null || resolvedReferenceSpace == null) {
       return
     }
     frame.fillPoses(hand.values(), resolvedReferenceSpace, buffer)