GS SH View Direction (#16804)

CedricGuillemet · web-flow · commit 551754766b9c · 2025-06-25T10:41:51.000+02:00
Change space for compressed PLY and flip SH direction. followup https://forum.babylonjs.com/t/gaussian-splatting-seems-to-not-work-since-version-8/57903/37
diff --git a/packages/dev/core/src/Materials/GaussianSplatting/gaussianSplattingMaterial.ts b/packages/dev/core/src/Materials/GaussianSplatting/gaussianSplattingMaterial.ts
@@ -201,6 +201,7 @@ export class GaussianSplattingMaterial extends PushMaterial {
                 "focal",
                 "eyePosition",
                 "kernelSize",
+                "viewDirectionFactor",
             ];
             const samplers = ["covariancesATexture", "covariancesBTexture", "centersTexture", "colorsTexture", "shTexture0", "shTexture1", "shTexture2"];
             const uniformBuffers = ["Scene", "Mesh"];
@@ -292,6 +293,7 @@ export class GaussianSplattingMaterial extends PushMaterial {
         }
 
         effect.setFloat2("focal", focal, focal);
+        effect.setVector3("viewDirectionFactor", gsMesh.viewDirectionFactor);
         effect.setFloat("kernelSize", gsMaterial && gsMaterial.kernelSize ? gsMaterial.kernelSize : GaussianSplattingMaterial.KernelSize);
         scene.bindEyePosition(effect, "eyePosition", true);
 
diff --git a/packages/dev/core/src/Materials/Node/Blocks/GaussianSplatting/gaussianSplattingBlock.ts b/packages/dev/core/src/Materials/Node/Blocks/GaussianSplatting/gaussianSplattingBlock.ts
@@ -133,6 +133,7 @@ export class GaussianSplattingBlock extends NodeMaterialBlock {
         state._emitUniformFromString("invViewport", NodeMaterialBlockConnectionPointTypes.Vector2);
         state._emitUniformFromString("kernelSize", NodeMaterialBlockConnectionPointTypes.Float);
         state._emitUniformFromString("eyePosition", NodeMaterialBlockConnectionPointTypes.Vector3);
+        state._emitUniformFromString("viewDirectionFactor", NodeMaterialBlockConnectionPointTypes.Vector3);
         state.attributes.push(VertexBuffer.PositionKind);
         state.sharedData.nodeMaterial.backFaceCulling = false;
 
@@ -163,13 +164,14 @@ export class GaussianSplattingBlock extends NodeMaterialBlock {
                 state.compilationString += `let worldRot: mat3x3f =  mat3x3f(${world.associatedVariableName}[0].xyz, ${world.associatedVariableName}[1].xyz, ${world.associatedVariableName}[2].xyz);`;
                 state.compilationString += `let normWorldRot: mat3x3f = inverseMat3(worldRot);`;
                 state.compilationString += `var dir: vec3f = normalize(normWorldRot * (${splatPosition.associatedVariableName}.xyz - uniforms.eyePosition));\n`;
+                state.compilationString += `dir *= uniforms.viewDirectionFactor;\n`;
             } else {
                 state.compilationString += `mat3 worldRot = mat3(${world.associatedVariableName});`;
                 state.compilationString += `mat3 normWorldRot = inverseMat3(worldRot);`;
                 state.compilationString += `vec3 dir = normalize(normWorldRot * (${splatPosition.associatedVariableName}.xyz - eyePosition));\n`;
+                state.compilationString += `dir *= viewDirectionFactor;\n`;
             }
 
-            state.compilationString += `dir *= vec3${addF}(1.,1.,-1.);\n`;
             state.compilationString += `${state._declareOutput(sh)} = computeSH(splat, dir);\n`;
             state.compilationString += `#else\n`;
             state.compilationString += `${state._declareOutput(sh)} = vec3${addF}(0.,0.,0.);\n`;
diff --git a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts
@@ -307,6 +307,14 @@ export class GaussianSplattingMesh extends Mesh {
     // batch size between 2 yield calls during the PLY to splat conversion.
     private static _PlyConversionBatchSize = 32768;
     private _shDegree = 0;
+    private _viewDirectionFactor = new Vector3(1, 1, -1);
+
+    /**
+     * View direction factor used to compute the SH view direction in the shader.
+     */
+    public get viewDirectionFactor() {
+        return this._viewDirectionFactor;
+    }
 
     /**
      * SH degree. 0 = no sh (default). 1 = 3 parameters. 2 = 8 parameters. 3 = 15 parameters.
@@ -931,8 +939,8 @@ export class GaussianSplattingMesh extends Mesh {
                         const compressedChunk = compressedChunks![chunkIndex];
                         Unpack111011(value, temp3);
                         position[0] = Scalar.Lerp(compressedChunk.min.x, compressedChunk.max.x, temp3.x);
-                        position[1] = -Scalar.Lerp(compressedChunk.min.y, compressedChunk.max.y, temp3.y);
-                        position[2] = -Scalar.Lerp(compressedChunk.min.z, compressedChunk.max.z, temp3.z);
+                        position[1] = Scalar.Lerp(compressedChunk.min.y, compressedChunk.max.y, temp3.y);
+                        position[2] = Scalar.Lerp(compressedChunk.min.z, compressedChunk.max.z, temp3.z);
                     }
                     break;
                 case PLYValue.PACKED_ROTATION:
@@ -941,8 +949,8 @@ export class GaussianSplattingMesh extends Mesh {
 
                         r0 = q.x;
                         r1 = q.y;
-                        r2 = -q.z;
-                        r3 = -q.w;
+                        r2 = q.z;
+                        r3 = q.w;
                     }
                     break;
                 case PLYValue.PACKED_SCALE:
diff --git a/packages/dev/core/src/Shaders/gaussianSplatting.vertex.fx b/packages/dev/core/src/Shaders/gaussianSplatting.vertex.fx
@@ -19,6 +19,7 @@ uniform vec2 dataTextureSize;
 uniform vec2 focal;
 uniform float kernelSize;
 uniform vec3 eyePosition;
+uniform vec3 viewDirectionFactor;
 
 uniform sampler2D covariancesATexture;
 uniform sampler2D covariancesBTexture;
@@ -56,7 +57,7 @@ void main () {
     mat3 normWorldRot = inverseMat3(worldRot);
 
     vec3 dir = normalize(normWorldRot * (worldPos.xyz - eyePosition));
-    dir *= vec3(1.,1.,-1.); // convert to Babylon Space
+    dir *= viewDirectionFactor;
     vColor.xyz = splat.color.xyz + computeSH(splat, dir);
 #endif
 
diff --git a/packages/dev/core/src/ShadersWGSL/gaussianSplatting.vertex.fx b/packages/dev/core/src/ShadersWGSL/gaussianSplatting.vertex.fx
@@ -16,6 +16,7 @@ uniform dataTextureSize: vec2f;
 uniform focal: vec2f;
 uniform kernelSize: f32;
 uniform eyePosition: vec3f;
+uniform viewDirectionFactor: vec3f;
 
 // textures
 var covariancesATexture: texture_2d<f32>;
@@ -53,7 +54,7 @@ fn main(input : VertexInputs) -> FragmentInputs {
     let normWorldRot: mat3x3f = inverseMat3(worldRot);
 
     var dir: vec3f = normalize(normWorldRot * (worldPos.xyz - uniforms.eyePosition.xyz));
-    dir *= vec3f(1.,1.,-1.); // convert to Babylon Space
+    dir *= viewDirectionFactor;
     vertexOutputs.vColor = vec4f(splat.color.xyz + computeSH(splat, dir), splat.color.w);
 #else
     vertexOutputs.vColor = splat.color;
diff --git a/packages/dev/loaders/src/SPLAT/splatFileLoader.ts b/packages/dev/loaders/src/SPLAT/splatFileLoader.ts
@@ -47,6 +47,7 @@ interface IParsedPLY {
     hasVertexColors?: boolean;
     sh?: Uint8Array[];
     trainedWithAntialiasing?: boolean;
+    compressed?: boolean;
 }
 
 /**
@@ -380,6 +381,9 @@ export class SPLATFileLoader implements ISceneLoaderPluginAsync, ISceneLoaderPlu
                                     gaussianSplatting._parentContainer = this._assetContainer;
                                     babylonMeshesArray.push(gaussianSplatting);
                                     gaussianSplatting.updateData(parsedPLY.data, parsedPLY.sh);
+                                    if (parsedPLY.compressed) {
+                                        gaussianSplatting.viewDirectionFactor.set(-1, -1, 1);
+                                    }
                                 }
                                 break;
                             case Mode.PointCloud:
@@ -577,7 +581,7 @@ export class SPLATFileLoader implements ISceneLoaderPluginAsync, ISceneLoaderPlu
             // early exit for chunked/quantized ply
             if (chunkCount) {
                 return await new Promise((resolve) => {
-                    resolve({ mode: Mode.Splat, data: splatsData.buffer, sh: splatsData.sh, faces: faces, hasVertexColors: false });
+                    resolve({ mode: Mode.Splat, data: splatsData.buffer, sh: splatsData.sh, faces: faces, hasVertexColors: false, compressed: true });
                 });
             }
             // count available properties. if all necessary are present then it's a splat. Otherwise, it's a point cloud
@@ -599,7 +603,7 @@ export class SPLATFileLoader implements ISceneLoaderPluginAsync, ISceneLoaderPlu
             const currentMode = faceCount ? Mode.Mesh : hasMandatoryProperties ? Mode.Splat : Mode.PointCloud;
             // parsed ready ready to be used as a splat
             return await new Promise((resolve) => {
-                resolve({ mode: currentMode, data: splatsData.buffer, sh: splatsData.sh, faces: faces, hasVertexColors: !!propertyColorCount });
+                resolve({ mode: currentMode, data: splatsData.buffer, sh: splatsData.sh, faces: faces, hasVertexColors: !!propertyColorCount, compressed: false });
             });
         });
     }
