fix: planeNormal used to aligned floddfill and brush was wrong if the local coord space was not aligned with the global (e.g. x and z inverted for example, which is common).

Author: briossant

src/layer/vox/index.ts

@@ -39,6 +39,7 @@ import { TrackableValue, WatchableValue } from "#src/trackable_value.js";
 import type { UserLayerWithAnnotations } from "#src/ui/annotations.js";
 import { randomUint64 } from "#src/util/bigint.js";
 import { RefCounted } from "#src/util/disposable.js";
+import { vec3 } from "#src/util/geom.js";
 import {
   parseUint64,
   verifyFiniteFloat,
@@ -141,6 +142,29 @@ export class VoxelEditingContext
     if (!ok) return undefined;
     return this.cachedVoxelPosition;
   }
+
+  transformGlobalToVoxelNormal(globalNormal: vec3): vec3 {
+    const chunkTransform = this.cachedChunkTransform;
+    if (chunkTransform === undefined)
+      throw new Error("Chunk transform not computed");
+    const { modelTransform, layerToChunkTransform, layerRank } = chunkTransform;
+    const { globalToRenderLayerDimensions } = modelTransform;
+    const globalRank = globalToRenderLayerDimensions.length;
+    const voxelNormal = vec3.create();
+
+    for (let chunkDim = 0; chunkDim < 3; ++chunkDim) {
+      let sum = 0;
+      for (let globalDim = 0; globalDim < Math.min(globalRank, 3); ++globalDim) {
+        const layerDim = globalToRenderLayerDimensions[globalDim];
+        if (layerDim !== -1) {
+          sum += layerToChunkTransform[chunkDim + layerDim * (layerRank + 1)] * globalNormal[globalDim];
+        }
+      }
+      voxelNormal[chunkDim] = sum;
+    }
+    vec3.normalize(voxelNormal, voxelNormal);
+    return voxelNormal;
+  }
 }
 
 export declare abstract class UserLayerWithVoxelEditing extends UserLayer {

src/ui/voxel_annotations.ts

@@ -48,7 +48,8 @@ abstract class BaseVoxelTool extends LayerTool<UserLayerWithVoxelEditing> {
       | Float32Array
       | undefined;
     if (!mouseState?.active || !vox) return undefined;
-    const planeNormal = mouseState?.planeNormal;
+    if(!mouseState.planeNormal) return;
+    const planeNormal =  editContext.transformGlobalToVoxelNormal(mouseState.planeNormal);
     if (!mouseState?.active || !vox || !planeNormal) return undefined;
     const CHUNK_POSITION_EPSILON = 1e-3;
     const shiftedVox = new Float32Array(3);
@@ -288,10 +289,14 @@ export class VoxelBrushTool extends BaseVoxelTool {
       1,
       Math.floor(this.layer.voxBrushRadius.value ?? 3),
     );
+    const editContext = this.getEditingContext();
+    if (editContext === undefined) {
+      throw new Error("editContext is undefined");
+    }
     const shapeEnum = this.layer.voxBrushShape.value;
     let basis = undefined as undefined | { u: Float32Array; v: Float32Array };
     if (shapeEnum === BrushShape.DISK && this.currentMouseState?.planeNormal) {
-      const n = this.currentMouseState.planeNormal;
+      const n =  editContext.transformGlobalToVoxelNormal(this.currentMouseState.planeNormal);
       const u = vec3.create();
       const tempVec =
         Math.abs(vec3.dot(n, vec3.fromValues(1, 0, 0))) < 0.9
@@ -304,10 +309,7 @@ export class VoxelBrushTool extends BaseVoxelTool {
       basis = { u, v };
     }
 
-    const editContext = this.getEditingContext();
-    if (editContext === undefined) {
-      throw new Error("editContext is undefined");
-    }
+
     for (const p of points)
       editContext.controller?.paintBrushWithShape(
         p,
@@ -361,7 +363,8 @@ export class VoxelFloodFillTool extends BaseVoxelTool {
       return;
     }
     const seed = this.getPoint(this.mouseState);
-    const planeNormal = this.mouseState.planeNormal;
+    if(!this.mouseState.planeNormal) return;
+    const planeNormal =  editContext.transformGlobalToVoxelNormal(this.mouseState.planeNormal);
     if (!seed || !planeNormal) return;
     try {
       const value = this.layer.getVoxelPaintValue(