feat(Controller): environment map (#282)

Author: saitonakamura

README.md

@@ -160,6 +160,10 @@ Controllers can be added with `<Controllers />` for [motion-controllers](https:/
 />
 ```
 
+### Environment map
+
+You can set environment map and/or it's intensity on controller models via props on `<Controllers />`. See [ControllerEnvMap](./examples/src/demos/ControllersEnvMap.tsx) to find out how to do it.
+
 ### useController
 
 `useController` references an `XRController` by handedness, exposing position and orientation info.
@@ -172,14 +176,15 @@ const gazeController = useController('none')
 
 ### XRController
 
-`XRController` is an `Object3D` that represents an [`XRInputSource`](https://developer.mozilla.org/en-US/docs/Web/API/XRInputSource) with the following properties:
+`XRController` is an long-living `Object3D` that represents an [`XRInputSource`](https://developer.mozilla.org/en-US/docs/Web/API/XRInputSource) with the following properties:
 
 ```jsx
 index: number
 controller: THREE.XRTargetRaySpace
 grip: THREE.XRGripSpace
 hand: THREE.XRHandSpace
-inputSource: XRInputSource
+inputSource: XRInputSource | null
+xrControllerModel: XRControllerModel | null
 ```
 
 ## Interactions

examples/src/assets/brown_photostudio_04_256.hdr

@@ -0,0 +1,41 @@
+import { Canvas, useThree } from '@react-three/fiber'
+import { XR, VRButton, Controllers } from '@react-three/xr'
+import { PMREMGenerator, Texture } from 'three'
+import { RGBELoader } from 'three-stdlib'
+import { useEffect, useState } from 'react'
+import EnvMap from '../assets/brown_photostudio_04_256.hdr'
+
+function ControllersWithEnvMap() {
+  const renderer = useThree(({ gl }) => gl)
+  const [envMap, setEnvMap] = useState<Texture>()
+
+  useEffect(() => {
+    const generateEnvMap = async () => {
+      const rgbeLoader = new RGBELoader()
+      const dataTexture = await rgbeLoader.loadAsync(EnvMap)
+      const pmremGenerator = new PMREMGenerator(renderer)
+      pmremGenerator.compileEquirectangularShader()
+      const rt = pmremGenerator.fromEquirectangular(dataTexture)
+      const radianceMap = rt.texture
+      setEnvMap(radianceMap)
+      pmremGenerator.dispose()
+    }
+
+    generateEnvMap()
+  }, [renderer])
+
+  return <Controllers envMap={envMap} envMapIntensity={1} />
+}
+
+export default function () {
+  return (
+    <>
+      <VRButton onError={(e) => console.error(e)} />
+      <Canvas>
+        <XR>
+          <ControllersWithEnvMap />
+        </XR>
+      </Canvas>
+    </>
+  )
+}

examples/src/demos/ControllersEnvMap.tsx

@@ -5,7 +5,8 @@ const HitTest = { Component: lazy(() => import('./HitTest')) }
 const Player = { Component: lazy(() => import('./Player')) }
 const Text = { Component: lazy(() => import('./Text')) }
 const Hands = { Component: lazy(() => import('./Hands')) }
+const ControllersEnvMap = { Component: lazy(() => import('./ControllersEnvMap')) }
 const Teleport = { Component: lazy(() => import('./Teleport')) }
 const CameraLinkedObject = { Component: lazy(() => import('./CameraLinkedObject')) }
 
-export { Interactive, HitTest, Player, Text, Hands, Teleport, CameraLinkedObject }
+export { Interactive, HitTest, Player, Text, Hands, Teleport, CameraLinkedObject, ControllersEnvMap }

examples/src/demos/index.tsx

@@ -0,0 +1,4 @@
+declare module '*.hdr' {
+  const path: string
+  export = path
+}

examples/src/global.d.ts

@@ -634,10 +634,10 @@
     "@jridgewell/resolve-uri" "3.1.0"
     "@jridgewell/sourcemap-codec" "1.4.14"
 
-"@mediapipe/[email protected].2-rc2":
-  version "0.10.2-rc2"
-  resolved "https://registry.yarnpkg.com/@mediapipe/tasks-vision/-/tasks-vision-0.10.2-rc2.tgz#e3fa5d84d58b9031a0e975d1e5ef8eb8e4a6fc11"
-  integrity sha512-b9ar6TEUo8I07n/jXSuKDu5HgzkDah9pe4H8BYpcubhCEahlfDD5ixE+9SQyJM4HXHXdF9nN/wRQT7rEnLz7Gg==
+"@mediapipe/tasks-vision@^0.10.0":
+  version "0.10.1"
+  resolved "https://registry.yarnpkg.com/@mediapipe/tasks-vision/-/tasks-vision-0.10.1.tgz#68047459352019cc141dc9c1d15c05b8ab689423"
+  integrity sha512-/zIKjOAIABx+KVfqe8hA6X2pxBGsBYlEtvD7/gpXecvzKefo/JQO6XaggmJul7+noaqiPYM0CVGZxmFJ2oTdSQ==
 
 "@nicolo-ribaudo/semver-v6@^6.3.3":
   version "6.3.3"
@@ -718,9 +718,9 @@
     zustand "^3.5.13"
 
 "@react-three/fiber@^8.10.0":
-  version "8.13.4"
-  resolved "https://registry.yarnpkg.com/@react-three/fiber/-/fiber-8.13.4.tgz#27cf964bd1d353884fb9555e21b0460736d173b5"
-  integrity sha512-OmyRKt9JU2i/Rc3uw4A+zERXKkFdu8slJjWQZfacoFNHIzGP9QVQ9XxlJWgTbgTLIOD39cUgnmH3RZZGWJqAoQ==
+  version "8.13.7"
+  resolved "https://registry.yarnpkg.com/@react-three/fiber/-/fiber-8.13.7.tgz#809f63c85effc7dddd3001ee10c2256c53a82b16"
+  integrity sha512-fH1wYi8+A2YZX8uYd9N4hfbAV+kHE565s7f62+SMNmpeynaUsN8NzXACmmJ6BpVKAKdxfvOde6dBGwG1BrWOKQ==
   dependencies:
     "@babel/runtime" "^7.17.8"
     "@types/react-reconciler" "^0.26.7"
@@ -930,10 +930,10 @@ cac@^6.7.14:
   resolved "https://registry.yarnpkg.com/cac/-/cac-6.7.14.tgz#804e1e6f506ee363cb0e3ccbb09cad5dd9870959"
   integrity sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==
 
-camera-controls@^2.4.2:
-  version "2.7.0"
-  resolved "https://registry.yarnpkg.com/camera-controls/-/camera-controls-2.7.0.tgz#13e2895375fbd8fb3353baeada6c8bc267a60d09"
-  integrity sha512-HONMoMYHieOCQOoweS639bdWHP/P/fvVGR08imnECGVUp04mqGfsX/zp1ZufLeiAA5hA6i1JhP6SrnOwh01C0w==
+camera-controls@^2.3.1:
+  version "2.4.2"
+  resolved "https://registry.yarnpkg.com/camera-controls/-/camera-controls-2.4.2.tgz#815aa5d7c4c43054fc55fb8b6cc685a56540fea2"
+  integrity sha512-blYDPECYFT/4egDMNWqKc2lBrpOfIAjPPRUNVswQELPi8naGBXUvZM3sDJSNuIRaHqid+JKPtlcoZk+Cb+X5qg==
 
 caniuse-lite@^1.0.30001503:
   version "1.0.30001512"
@@ -1035,10 +1035,10 @@ deepmerge@^4.2.2:
   resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.3.1.tgz#44b5f2147cd3b00d4b56137685966f26fd25dd4a"
   integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==
 
-detect-gpu@^5.0.28:
-  version "5.0.31"
-  resolved "https://registry.yarnpkg.com/detect-gpu/-/detect-gpu-5.0.31.tgz#5749bea3aa56bc2ec41383b585f6cbd965619fee"
-  integrity sha512-+ZZr/deA5OvuBxod6kKFUvpZA9YR2r4fRYlAJGL7N5aUSLrY3Xgi+K4U5NHmeuk2mNC044n1YJwsq2Aw6hPmUw==
+detect-gpu@^5.0.14:
+  version "5.0.27"
+  resolved "https://registry.yarnpkg.com/detect-gpu/-/detect-gpu-5.0.27.tgz#821d9331c87e32568c483d85e12a9adee43d7bb2"
+  integrity sha512-IDjjqTkS+f0xm/ntbD21IPYiF0srzpePC/hhUMmctEsoklZwJwStJiMi/KN0pnH0LjSsgjwbP+QwW7y+Qf4/SQ==
   dependencies:
     webgl-constants "^1.1.1"
 
@@ -1251,10 +1251,10 @@ lru-cache@^5.1.1:
   dependencies:
     yallist "^3.0.2"
 
-maath@^0.6.0:
-  version "0.6.0"
-  resolved "https://registry.yarnpkg.com/maath/-/maath-0.6.0.tgz#7841d0fb95bbb37d19b08b7c5458ef70190950d2"
-  integrity sha512-dSb2xQuP7vDnaYqfoKzlApeRcR2xtN8/f7WV/TMAkBC8552TwTLtOO0JTcSygkYMjNDPoo6V01jTw/aPi4JrMw==
+maath@^0.5.2:
+  version "0.5.3"
+  resolved "https://registry.yarnpkg.com/maath/-/maath-0.5.3.tgz#777a1f9b8463c6ffb199ea43406874a357c0cd58"
+  integrity sha512-ut63A4zTd9abtpi+sOHW1fPWPtAFrjK0E17eAthx1k93W/T2cWLKV5oaswyotJVDvvW1EXSdokAqhK5KOu0Qdw==
 
 magic-string@^0.27.0:
   version "0.27.0"
@@ -1532,20 +1532,25 @@ supports-color@^7.1.0:
   dependencies:
     has-flag "^4.0.0"
 
+suspend-react@^0.0.8:
+  version "0.0.8"
+  resolved "https://registry.yarnpkg.com/suspend-react/-/suspend-react-0.0.8.tgz#b0740c1386b4eb652f17affe4339915ee268bd31"
+  integrity sha512-ZC3r8Hu1y0dIThzsGw0RLZplnX9yXwfItcvaIzJc2VQVi8TGyGDlu92syMB5ulybfvGLHAI5Ghzlk23UBPF8xg==
+
 suspend-react@^0.1.3:
   version "0.1.3"
   resolved "https://registry.yarnpkg.com/suspend-react/-/suspend-react-0.1.3.tgz#a52f49d21cfae9a2fb70bd0c68413d3f9d90768e"
   integrity sha512-aqldKgX9aZqpoDp3e8/BZ8Dm7x1pJl+qI3ZKxDN0i/IQTWUwBx/ManmlVJ3wowqbno6c2bmiIfs+Um6LbsjJyQ==
 
-three-mesh-bvh@^0.6.0:
-  version "0.6.0"
-  resolved "https://registry.yarnpkg.com/three-mesh-bvh/-/three-mesh-bvh-0.6.0.tgz#15523c335383df658dc60063a783fdd52d045dc5"
-  integrity sha512-4/oXeqVMLuN9/P0M3L5ezIVrFiXQXKvjVTErkiSYMjSaPoWfNPAwqulSgLf4bIUPn8/Lq3rmIJwxbCuD8qDobA==
+three-mesh-bvh@^0.5.23:
+  version "0.5.23"
+  resolved "https://registry.yarnpkg.com/three-mesh-bvh/-/three-mesh-bvh-0.5.23.tgz#08e5b629144b48b11acbd433519680e457d398ed"
+  integrity sha512-nyk+MskdyDgECqkxdv57UjazqqhrMi+Al9PxJN6yFtx1CTW4r0eCQ27FtyYKY5gCIWhxjtNfWYDPVy8lzx6LkA==
 
-three-stdlib@^2.23.9:
-  version "2.23.12"
-  resolved "https://registry.yarnpkg.com/three-stdlib/-/three-stdlib-2.23.12.tgz#f269398e3125c77bcd374d87f4c1da8d550e7f21"
-  integrity sha512-YFpuCu/ZVHBiK42bzEihZTA3tvEPQhaKE5tYej41AlNYXbwIWxO93fxYYrX7vs275s0yCKr6Zp6y7kI+mOklRQ==
+three-stdlib@^2.23.5:
+  version "2.23.9"
+  resolved "https://registry.yarnpkg.com/three-stdlib/-/three-stdlib-2.23.9.tgz#09c74fc6acced3d124e4f9d695156136c587a355"
+  integrity sha512-fYBClVGQptD7UZcoRZGNlR3sKcUW37hVPoEW1v68E4XuiwD0Ml/VqDUJ0yEMVE2DlooDvqgqv/rIcHC/B4N5pg==
   dependencies:
     "@types/draco3d" "^1.4.0"
     "@types/offscreencanvas" "^2019.6.4"
@@ -1574,7 +1579,7 @@ to-fast-properties@^2.0.0:
   resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e"
   integrity sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==
 
-troika-three-text@^0.47.2:
+troika-three-text@^0.47.1:
   version "0.47.2"
   resolved "https://registry.yarnpkg.com/troika-three-text/-/troika-three-text-0.47.2.tgz#fdf89059c010563bb829262b20c41f69ca79b712"
   integrity sha512-qylT0F+U7xGs+/PEf3ujBdJMYWbn0Qci0kLqI5BJG2kW1wdg4T1XSxneypnF05DxFqJhEzuaOR9S2SjiyknMng==

examples/yarn.lock

@@ -60,7 +60,7 @@
     "three": "^0.141.0",
     "typescript": "^4.5.5",
     "vite": "^3.0.5",
-    "vitest": "^0.29.1"
+    "vitest": "^0.34.3"
   },
   "dependencies": {
     "@types/webxr": "*",

package.json

@@ -8,6 +8,7 @@ import { XRControllerModel } from './XRControllerModel'
 import { XRControllerModelFactoryMock } from './mocks/XRControllerModelFactoryMock'
 import { XRInputSourceMock } from './mocks/XRInputSourceMock'
 import { act } from '@react-three/test-renderer'
+import { Texture } from 'three'
 
 vi.mock('./XRControllerModelFactory', async () => {
   const { XRControllerModelFactoryMock } = await vi.importActual<typeof import('./mocks/XRControllerModelFactoryMock')>(
@@ -141,4 +142,86 @@ describe('Controllers', () => {
     expect(disconnectSpy).not.toBeCalled()
     expect(xrControllerModelFactory?.initializeControllerModel).toBeCalledTimes(1)
   })
+
+  describe('envMap', () => {
+    it("should not set env map if it's not provided in props", async () => {
+      const store = createStoreMock()
+      const xrControllerMock = new XRControllerMock(0)
+      store.setState({ controllers: [xrControllerMock] })
+
+      await render(<Controllers />, { wrapper: createStoreProvider(store) })
+
+      const xrControllerModel = xrControllerMock.xrControllerModel
+
+      expect(xrControllerModel!.envMap).toBeNull()
+      expect(xrControllerModel!.envMapIntensity).toBe(1)
+    })
+
+    it("should set env map if it's provided in props", async () => {
+      const store = createStoreMock()
+      const xrControllerMock = new XRControllerMock(0)
+      store.setState({ controllers: [xrControllerMock] })
+      const envMap = new Texture()
+
+      await render(<Controllers envMap={envMap} />, { wrapper: createStoreProvider(store) })
+
+      const xrControllerModel = xrControllerMock.xrControllerModel
+
+      expect(xrControllerModel!.envMap).toBe(envMap)
+      expect(xrControllerModel!.envMapIntensity).toBe(1)
+    })
+
+    it("should only set env map intensity if it's provided in props", async () => {
+      const store = createStoreMock()
+      const xrControllerMock = new XRControllerMock(0)
+      store.setState({ controllers: [xrControllerMock] })
+
+      await render(<Controllers envMapIntensity={0.5} />, { wrapper: createStoreProvider(store) })
+
+      const xrControllerModel = xrControllerMock.xrControllerModel
+
+      expect(xrControllerModel!.envMap).toBeNull()
+      expect(xrControllerModel!.envMapIntensity).toBe(0.5)
+    })
+
+    it("should change env map intensity if it's provided in props then updated to a different value", async () => {
+      const store = createStoreMock()
+      const xrControllerMock = new XRControllerMock(0)
+      store.setState({ controllers: [xrControllerMock] })
+
+      const { rerender } = await render(<Controllers envMapIntensity={0.5} />, { wrapper: createStoreProvider(store) })
+      const xrControllerModel = xrControllerMock.xrControllerModel
+      await rerender(<Controllers envMapIntensity={0.6} />)
+
+      expect(xrControllerModel!.envMap).toBeNull()
+      expect(xrControllerModel!.envMapIntensity).toBe(0.6)
+    })
+
+    it("should remove env map if it's provided in props first, and then removed", async () => {
+      const store = createStoreMock()
+      const xrControllerMock = new XRControllerMock(0)
+      store.setState({ controllers: [xrControllerMock] })
+      const envMap = new Texture()
+
+      const { rerender } = await render(<Controllers envMap={envMap} />, { wrapper: createStoreProvider(store) })
+      const xrControllerModel = xrControllerMock.xrControllerModel
+      await rerender(<Controllers />)
+
+      expect(xrControllerModel!.envMap).toBeNull()
+    })
+
+    it("should change env map intensity if it's provided in props then updated to a different value but envMap stays the same", async () => {
+      const store = createStoreMock()
+      const xrControllerMock = new XRControllerMock(0)
+      store.setState({ controllers: [xrControllerMock] })
+      const envMap = new Texture()
+
+      const { rerender } = await render(<Controllers envMap={envMap} envMapIntensity={0.5} />, { wrapper: createStoreProvider(store) })
+      const xrControllerModel = xrControllerMock.xrControllerModel
+      await rerender(<Controllers envMap={envMap} envMapIntensity={0.6} />)
+
+      expect(xrControllerModel!.envMap).toBe(envMap)
+      expect(xrControllerModel!.envMapIntensity).toBe(0.6)
+    })
+  })
 })