Fix loading ad-hoc meshes for transformed layers (#8992)

### URL of deployed dev instance (used for testing):
- https://fixadhocmeshesfortransformedlayers.webknossos.xyz/

### Steps to test:
- Test this with multiple datasets: one where all layers are
transformed, and one where not all layers are transformed
- make sure the datasets have a segmentation layer

- register segments, load ad hoc meshes, and load precomputed meshes:
- register some segments, scroll within the viewport, and then click on
some segments in the segment list on the right. the viewport sshould
jump to the right segment. do this with and without rendered
transformations, and also toggle transformation rendering after
registering the segments in the list
    - load some ad hoc meshes by selecting this option...
    - a) from the context menu in the segment list
    - b) from the context menu in the viewport
    - same for precomputed meshes
-> again, this should be working with and without rendered
transformations

### To do
- [x] I think I found another bug (reproduced on wk.org): click segment
with transformed layers, scroll away, turn rendering with
transformations off, click segment in segment list -> wrong place is
focused
- [x] testing!!!

### Issues:
- fixes #8913 

------
(Please delete unneeded items, merge only when none are left open)
- [x] Added changelog entry (create a `$PR_NUMBER.md` file in
`unreleased_changes` or use `./tools/create-changelog-entry.py`)
- [ ] Added migration guide entry if applicable (edit the same file as
for the changelog)
- [ ] Updated [documentation](../blob/master/docs) if applicable
- [ ] Adapted [wk-libs python
client](https://github.com/scalableminds/webknossos-libs/tree/master/webknossos/webknossos/client)
if relevant API parts change
- [ ] Removed dev-only changes like prints and application.conf edits
- [x] Considered [common edge
cases](../blob/master/.github/common_edge_cases.md)
- [ ] Needs datastore update after deployment

Author: knollengewaechs

frontend/javascripts/viewer/controller/combinations/segmentation_handlers.ts

@@ -4,7 +4,11 @@ import {
   getSegmentIdForPosition,
   getSegmentIdForPositionAsync,
 } from "viewer/controller/combinations/volume_handlers";
-import { getMappingInfo } from "viewer/model/accessors/dataset_accessor";
+import {
+  getMappingInfo,
+  getVisibleSegmentationLayer,
+} from "viewer/model/accessors/dataset_accessor";
+import { globalToLayerTransformedPosition } from "viewer/model/accessors/dataset_layer_transformation_accessor";
 import { getTreeNameForAgglomerateSkeleton } from "viewer/model/accessors/skeletontracing_accessor";
 import { calculateGlobalPos } from "viewer/model/accessors/view_mode_accessor";
 import {
@@ -79,9 +83,22 @@ export function handleClickSegment(clickPosition: Point2) {
   const state = Store.getState();
   const globalPosition = calculateGlobalPos(state, clickPosition);
   const segmentId = getSegmentIdForPosition(globalPosition.rounded);
+  const visibleSegmentationLayer = getVisibleSegmentationLayer(state);
+  const positionInSegmentationLayerSpace =
+    visibleSegmentationLayer != null
+      ? (globalToLayerTransformedPosition(
+          globalPosition.rounded,
+          visibleSegmentationLayer.name,
+          "segmentation",
+          state,
+        ).map(Math.floor) as Vector3)
+      : null;
+
   const { additionalCoordinates } = state.flycam;
 
-  if (segmentId > 0) {
-    Store.dispatch(clickSegmentAction(segmentId, globalPosition.rounded, additionalCoordinates));
+  if (segmentId > 0 && positionInSegmentationLayerSpace != null) {
+    Store.dispatch(
+      clickSegmentAction(segmentId, positionInSegmentationLayerSpace, additionalCoordinates),
+    );
   }
 }

frontend/javascripts/viewer/controller/combinations/skeleton_handlers.ts

@@ -136,10 +136,10 @@ export function handleOpenContextMenu(
   meshIntersectionPosition?: Vector3 | null | undefined,
   unmappedSegmentId?: number | null | undefined,
 ) {
-  const { activeViewport } = Store.getState().viewModeData.plane;
+  const state = Store.getState();
+  const { activeViewport } = state.viewModeData.plane;
 
   const nodeId = maybeGetNodeIdFromPosition(planeView, position, plane, isTouch);
-  const state = Store.getState();
   // Use calculateMaybeGlobalPos instead of calculateGlobalPos, since calculateGlobalPos
   // only works for the data viewports, but this function is also called for the 3d viewport.
   const globalPosition = calculateMaybeGlobalPos(state, position);

frontend/javascripts/viewer/model/accessors/dataset_layer_transformation_accessor.ts

@@ -504,3 +504,24 @@ export function globalToLayerTransformedPosition(
   }
   return globalPos;
 }
+
+export function layerToGlobalTransformedPosition(
+  layerPos: Vector3,
+  layerName: string,
+  layerCategory: APIDataLayer["category"] | "skeleton",
+  state: WebknossosState,
+): Vector3 {
+  const layerDescriptor =
+    layerCategory !== "skeleton"
+      ? getLayerByName(state.dataset, layerName, true)
+      : ({ name: "skeleton", category: "skeleton" } as APISkeletonLayer);
+  const layerTransforms = getTransformsForLayerOrNull(
+    state.dataset,
+    layerDescriptor,
+    state.datasetConfiguration.nativelyRenderedLayerName,
+  );
+  if (layerTransforms) {
+    return transformPointUnscaled(layerTransforms)(layerPos);
+  }
+  return layerPos;
+}

frontend/javascripts/viewer/view/context_menu.tsx

@@ -154,6 +154,7 @@ import type {
   VolumeTracing,
 } from "viewer/store";
 
+import { globalToLayerTransformedPosition } from "viewer/model/accessors/dataset_layer_transformation_accessor";
 import { deleteNodeAsUserAction } from "viewer/model/actions/skeletontracing_actions_with_effects";
 import { type MutableNode, type Tree, TreeMap } from "viewer/model/types/tree_types";
 import Store from "viewer/store";
@@ -1055,9 +1056,24 @@ function getNoNodeContextMenuOptions(props: NoNodeContextMenuProps): ItemType[]
   const segmentOrSuperVoxel =
     isProofreadingActive && maybeUnmappedSegmentId != null ? "Supervoxel" : "Segment";
   Store.dispatch(maybeFetchMeshFilesAction(visibleSegmentationLayer, dataset, false));
+  const positionInLayerSpace =
+    globalPosition != null && visibleSegmentationLayer != null
+      ? globalToLayerTransformedPosition(
+          globalPosition,
+          visibleSegmentationLayer.name,
+          "segmentation",
+          Store.getState(),
+        )
+      : null;
 
   const loadPrecomputedMesh = async () => {
-    if (!currentMeshFile || !visibleSegmentationLayer || globalPosition == null) return;
+    if (
+      !currentMeshFile ||
+      !visibleSegmentationLayer ||
+      globalPosition == null ||
+      positionInLayerSpace == null
+    )
+      return;
     // Ensure that the segment ID is loaded, since a mapping might have been activated
     // shortly before
     const segmentId = await getSegmentIdForPositionAsync(globalPosition);
@@ -1070,7 +1086,7 @@ function getNoNodeContextMenuOptions(props: NoNodeContextMenuProps): ItemType[]
     Store.dispatch(
       loadPrecomputedMeshAction(
         segmentId,
-        globalPosition,
+        positionInLayerSpace,
         additionalCoordinates,
         currentMeshFile.name,
         undefined,
@@ -1156,7 +1172,7 @@ function getNoNodeContextMenuOptions(props: NoNodeContextMenuProps): ItemType[]
   };
 
   const computeMeshAdHoc = () => {
-    if (!visibleSegmentationLayer || globalPosition == null) {
+    if (!visibleSegmentationLayer || globalPosition == null || positionInLayerSpace == null) {
       return;
     }
 
@@ -1167,7 +1183,7 @@ function getNoNodeContextMenuOptions(props: NoNodeContextMenuProps): ItemType[]
       return;
     }
 
-    Store.dispatch(loadAdHocMeshAction(segmentId, globalPosition, additionalCoordinates));
+    Store.dispatch(loadAdHocMeshAction(segmentId, positionInLayerSpace, additionalCoordinates));
   };
 
   const showAutomatedSegmentationServicesModal = (errorMessage: string, entity: string) =>

frontend/javascripts/viewer/view/right-border-tabs/segments_tab/segments_view.tsx

@@ -58,6 +58,7 @@ import {
   getMaybeSegmentIndexAvailability,
   getVisibleSegmentationLayer,
 } from "viewer/model/accessors/dataset_accessor";
+import { layerToGlobalTransformedPosition } from "viewer/model/accessors/dataset_layer_transformation_accessor";
 import { getAdditionalCoordinatesAsString } from "viewer/model/accessors/flycam_accessor";
 import { AnnotationTool } from "viewer/model/accessors/tool_accessor";
 import {
@@ -807,7 +808,13 @@ class SegmentsView extends React.Component<Props, State> {
       );
       return;
     }
-    this.props.setPosition(segment.somePosition);
+    const transformedPosition = layerToGlobalTransformedPosition(
+      segment.somePosition,
+      visibleSegmentationLayer.name,
+      "segmentation",
+      Store.getState(),
+    );
+    this.props.setPosition(transformedPosition);
     const segmentAdditionalCoordinates = segment.someAdditionalCoordinates;
     if (segmentAdditionalCoordinates != null) {
       this.props.setAdditionalCoordinates(segmentAdditionalCoordinates);

unreleased_changes/8992.md

@@ -0,0 +1,2 @@
+### Fixed
+- Fixed ad-hoc mesh loading and jumping to registered segments for transformed datasets.