Fix missing camera pose handling and skybox gamma correction (#199)

Author: slimbuck

src/camera-manager.ts

@@ -44,9 +44,12 @@ class CameraManager {
     constructor(global: Global, bbox: BoundingBox, collider: VoxelCollider | null = null) {
         const { config, events, settings, state } = global;
 
-        const camera0 = settings.cameras[0].initial;
-        const frameCamera = createFrameCamera(bbox, camera0.fov);
-        const resetCamera = createCamera(new Vec3(camera0.position), new Vec3(camera0.target), camera0.fov);
+        const camera0 = settings.cameras[0]?.initial;
+        const defaultFov = camera0?.fov ?? 75;
+        const frameCamera = createFrameCamera(bbox, defaultFov);
+        const resetCamera = camera0 ?
+            createCamera(new Vec3(camera0.position), new Vec3(camera0.target), camera0.fov) :
+            frameCamera;
 
         const getAnimTrack = (initial: Camera, isObjectExperience: boolean) => {
             const { animTracks } = settings;
@@ -65,7 +68,7 @@ class CameraManager {
 
         // object experience starts outside the bounding box
         const isObjectExperience = !bbox.containsPoint(resetCamera.position);
-        const animTrack = getAnimTrack(settings.hasStartPose ? resetCamera : frameCamera, isObjectExperience);
+        const animTrack = getAnimTrack(resetCamera, isObjectExperience);
 
         const controllers = {
             orbit: new OrbitController(),

src/schemas/v2.ts

@@ -78,9 +78,7 @@ type ExperienceSettings = {
     cameras: Camera[],
     annotations: Annotation[],
 
-    startMode: 'default' | 'animTrack' | 'annotation',
-
-    hasStartPose?: boolean
+    startMode: 'default' | 'animTrack' | 'annotation'
 };
 
 export type { AnimTrack, Camera, Annotation, PostEffectSettings, ExperienceSettings };

src/settings.ts

@@ -87,16 +87,15 @@ const migrateV2 = (v1: V1): V2 => {
         animTracks: v1.animTracks.map((animTrackV1: AnimTrackV1) => {
             return migrateAnimTrackV2(animTrackV1, v1.camera.fov || 60);
         }),
-        cameras: [{
+        cameras: (v1.camera.position && v1.camera.target) ? [{
             initial: {
-                position: (v1.camera.position || [0, 0, 5]) as [number, number, number],
-                target: (v1.camera.target || [0, 0, 0]) as [number, number, number],
+                position: v1.camera.position as [number, number, number],
+                target: v1.camera.target as [number, number, number],
                 fov: v1.camera.fov || 75
             }
-        }],
+        }] : [],
         annotations: [],
-        startMode: v1.camera.startAnim === 'animTrack' ? 'animTrack' : 'default',
-        hasStartPose: !!(v1.camera.position && v1.camera.target)
+        startMode: v1.camera.startAnim === 'animTrack' ? 'animTrack' : 'default'
     };
 };
 

src/viewer.ts

@@ -133,10 +133,12 @@ class Viewer {
 
     origChunks: {
         glsl: {
-            gsplatOutputVS: string
+            gsplatOutputVS: string,
+            skyboxPS: string
         },
         wgsl: {
-            gsplatOutputVS: string
+            gsplatOutputVS: string,
+            skyboxPS: string
         }
     };
 
@@ -149,22 +151,24 @@ class Viewer {
         // enable anonymous CORS for image loading in safari
         (app.loader.getHandler('texture') as TextureHandler).imgParser.crossOrigin = 'anonymous';
 
-        this.origChunks = {
-            glsl: {
-                gsplatOutputVS: ShaderChunks.get(graphicsDevice, 'glsl').get('gsplatOutputVS')
-            },
-            wgsl: {
-                gsplatOutputVS: ShaderChunks.get(graphicsDevice, 'wgsl').get('gsplatOutputVS')
-            }
-        };
-
         // render skybox as plain equirect
         const glsl = ShaderChunks.get(graphicsDevice, 'glsl');
         glsl.set('skyboxPS', glsl.get('skyboxPS').replace('mapRoughnessUv(uv, mipLevel)', 'uv'));
 
         const wgsl = ShaderChunks.get(graphicsDevice, 'wgsl');
         wgsl.set('skyboxPS', wgsl.get('skyboxPS').replace('mapRoughnessUv(uv, uniform.mipLevel)', 'uv'));
 
+        this.origChunks = {
+            glsl: {
+                gsplatOutputVS: glsl.get('gsplatOutputVS'),
+                skyboxPS: glsl.get('skyboxPS')
+            },
+            wgsl: {
+                gsplatOutputVS: wgsl.get('gsplatOutputVS'),
+                skyboxPS: wgsl.get('skyboxPS')
+            }
+        };
+
         // disable auto render, we'll render only when camera changes
         app.autoRender = false;
 
@@ -476,6 +480,21 @@ class Viewer {
             ShaderChunks.get(app.graphicsDevice, 'glsl').set('gsplatOutputVS', gammaChunkGlsl);
             ShaderChunks.get(app.graphicsDevice, 'wgsl').set('gsplatOutputVS', gammaChunkWgsl);
 
+            // force skybox shader to write gamma-space colors (inline pow replaces the
+            // gammaCorrectOutput call which is a no-op under CameraFrame's GAMMA_NONE)
+            ShaderChunks.get(app.graphicsDevice, 'glsl').set('skyboxPS',
+                this.origChunks.glsl.skyboxPS.replace(
+                    'gammaCorrectOutput(toneMap(processEnvironment(linear)))',
+                    'pow(toneMap(processEnvironment(linear)) + 0.0000001, vec3(1.0 / 2.2))'
+                )
+            );
+            ShaderChunks.get(app.graphicsDevice, 'wgsl').set('skyboxPS',
+                this.origChunks.wgsl.skyboxPS.replace(
+                    'gammaCorrectOutput(toneMap(processEnvironment(linear)))',
+                    'pow(toneMap(processEnvironment(linear)) + 0.0000001, vec3f(1.0 / 2.2))'
+                )
+            );
+
             // ensure the final compose blit doesn't perform linear->gamma conversion.
             RenderTarget.prototype.isColorBufferSrgb = function (index) {
                 return this === app.graphicsDevice.backBuffer ? true : origIsColorBufferSrgb.call(this, index);
@@ -489,9 +508,11 @@ class Viewer {
                 this.cameraFrame = null;
             }
 
-            // restore gsplat output shader chunks to engine defaults
+            // restore shader chunks to engine defaults
             ShaderChunks.get(app.graphicsDevice, 'glsl').set('gsplatOutputVS', this.origChunks.glsl.gsplatOutputVS);
             ShaderChunks.get(app.graphicsDevice, 'wgsl').set('gsplatOutputVS', this.origChunks.wgsl.gsplatOutputVS);
+            ShaderChunks.get(app.graphicsDevice, 'glsl').set('skyboxPS', this.origChunks.glsl.skyboxPS);
+            ShaderChunks.get(app.graphicsDevice, 'wgsl').set('skyboxPS', this.origChunks.wgsl.skyboxPS);
 
             // restore original isColorBufferSrgb behavior
             RenderTarget.prototype.isColorBufferSrgb = origIsColorBufferSrgb;