Merge branch 'master' into s3-upload

normanrz · web-flow · commit cc0509d90206 · 2025-09-16T13:07:43.000+02:00
diff --git a/app/controllers/AnnotationController.scala b/app/controllers/AnnotationController.scala
@@ -276,17 +276,6 @@ class AnnotationController @Inject()(
       } yield JsonOk(Messages("annotation.edit.success"))
   }
 
-  def editAnnotationLayer(typ: String, id: ObjectId, tracingId: String): Action[JsValue] =
-    sil.SecuredAction.async(parse.json) { implicit request =>
-      for {
-        annotation <- provider.provideAnnotation(typ, id, request.identity) ~> NOT_FOUND
-        restrictions <- provider.restrictionsFor(typ, id) ?~> "restrictions.notFound" ~> NOT_FOUND
-        _ <- restrictions.allowUpdate(request.identity) ?~> "notAllowed" ~> FORBIDDEN
-        newLayerName = (request.body \ "name").as[String]
-        _ <- annotationLayerDAO.updateName(annotation._id, tracingId, newLayerName) ?~> "annotation.edit.failed"
-      } yield JsonOk(Messages("annotation.edit.success"))
-    }
-
   def annotationsForTask(taskId: ObjectId): Action[AnyContent] =
     sil.SecuredAction.async { implicit request =>
       for {
diff --git a/app/controllers/WKRemoteTracingStoreController.scala b/app/controllers/WKRemoteTracingStoreController.scala
@@ -61,11 +61,11 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore
           layerIdsToUpdate = existingLayerIds.intersect(newLayerIds)
           layerIdsToInsert = newLayerIds.diff(existingLayerIds)
           _ <- Fox.serialCombined(layerIdsToDelete.toList)(annotationLayerDAO.deleteOneByTracingId(annotationId, _))
-          _ <- Fox.serialCombined(newLayersProto.filter(l => layerIdsToInsert.contains(l.tracingId))) { layerProto =>
-            annotationLayerDAO.insertOne(annotationId, AnnotationLayer.fromProto(layerProto))
-          }
-          _ <- Fox.serialCombined(newLayersProto.filter(l => layerIdsToUpdate.contains(l.tracingId)))(l =>
-            annotationLayerDAO.updateName(annotationId, l.tracingId, l.name))
+          _ <- annotationLayerDAO.updateLayers(
+            annotationId,
+            newLayersProto.filter(l => layerIdsToInsert.contains(l.tracingId)).map(AnnotationLayer.fromProto),
+            newLayersProto.filter(l => layerIdsToUpdate.contains(l.tracingId)).map(AnnotationLayer.fromProto)
+          )
           // Layer stats are ignored here, they are sent eagerly when saving updates
           _ <- annotationDAO.updateDescription(annotationId, request.body.description)
         } yield Ok
diff --git a/app/models/annotation/Annotation.scala b/app/models/annotation/Annotation.scala
@@ -16,6 +16,7 @@ import slick.jdbc.TransactionIsolation.Serializable
 import slick.lifted.Rep
 import slick.sql.SqlAction
 import com.scalableminds.util.objectid.ObjectId
+import slick.dbio.DBIO
 import utils.sql.{SQLDAO, SimpleSQLDAO, SqlClient, SqlToken}
 
 import javax.inject.Inject
@@ -131,8 +132,7 @@ class AnnotationLayerDAO @Inject()(SQLClient: SqlClient)(implicit ec: ExecutionC
       _ <- run(insertOneQuery(annotationId, annotationLayer))
     } yield ()
 
-  def insertLayerQueries(annotationId: ObjectId,
-                         layers: List[AnnotationLayer]): List[SqlAction[Int, NoStream, Effect]] =
+  def insertLayerQueries(annotationId: ObjectId, layers: Seq[AnnotationLayer]): Seq[SqlAction[Int, NoStream, Effect]] =
     layers.map { annotationLayer =>
       insertOneQuery(annotationId, annotationLayer)
     }
@@ -161,13 +161,20 @@ class AnnotationLayerDAO @Inject()(SQLClient: SqlClient)(implicit ec: ExecutionC
       head <- rList.headOption.toFox
     } yield head
 
-  def updateName(annotationId: ObjectId, tracingId: String, newName: String): Fox[Unit] =
+  def updateLayers(annotationId: ObjectId,
+                   newLayers: Seq[AnnotationLayer],
+                   updatedLayers: Seq[AnnotationLayer]): Fox[Unit] = {
+    val insertQueries = insertLayerQueries(annotationId, newLayers)
+    val updateQueries = updatedLayers.map { updatedLayer =>
+      q"""UPDATE webknossos.annotation_layers
+                         SET name = ${updatedLayer.name}
+                         WHERE _annotation = $annotationId
+                         AND tracingId = ${updatedLayer.tracingId}""".asUpdate
+    }
     for {
-      _ <- run(q"""UPDATE webknossos.annotation_layers
-                   SET name = $newName
-                   WHERE _annotation = $annotationId
-                   AND tracingId = $tracingId""".asUpdate)
+      _ <- run(DBIO.sequence(insertQueries ++ updateQueries).transactionally)
     } yield ()
+  }
 
   def deleteAllForAnnotationQuery(annotationId: ObjectId): SqlAction[Int, NoStream, Effect] =
     q"DELETE FROM webknossos.annotation_layers WHERE _annotation = $annotationId".asUpdate
diff --git a/conf/evolutions/140-annotation-layer-name-check-deferrable.sql b/conf/evolutions/140-annotation-layer-name-check-deferrable.sql
@@ -0,0 +1,10 @@
+START TRANSACTION;
+
+do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 139, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
+
+ALTER TABLE webknossos.annotation_layers DROP CONSTRAINT annotation_layers_name__annotation_key;
+ALTER TABLE webknossos.annotation_layers ADD CONSTRAINT annotation_layers_name__annotation_key UNIQUE (name, _annotation) DEFERRABLE INITIALLY DEFERRED;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 140;
+
+COMMIT TRANSACTION;
diff --git a/conf/evolutions/reversions/140-annotation-layer-name-check-deferrable.sql b/conf/evolutions/reversions/140-annotation-layer-name-check-deferrable.sql
@@ -0,0 +1,10 @@
+START TRANSACTION;
+
+do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 140, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
+
+ALTER TABLE webknossos.annotation_layers DROP CONSTRAINT annotation_layers_name__annotation_key;
+ALTER TABLE webknossos.annotation_layers ADD CONSTRAINT annotation_layers_name__annotation_key UNIQUE (name, _annotation);
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 139;
+
+COMMIT TRANSACTION;
diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes
@@ -156,7 +156,6 @@ POST          /userToken/generate
 POST          /annotations/upload                                                               controllers.AnnotationIOController.upload()
 POST          /annotations/:typ/:id/duplicate                                                   controllers.AnnotationController.duplicate(typ: String, id: ObjectId)
 PATCH         /annotations/:typ/:id/edit                                                        controllers.AnnotationController.editAnnotation(typ: String, id: ObjectId)
-PATCH         /annotations/:typ/:id/editLayer/:tracingId                                        controllers.AnnotationController.editAnnotationLayer(typ: String, id: ObjectId, tracingId: String)
 
 PATCH         /annotations/:typ/:id/finish                                                      controllers.AnnotationController.finish(typ: String, id: ObjectId, timestamp: Long)
 PATCH         /annotations/:typ/finish                                                          controllers.AnnotationController.finishAll(typ: String, timestamp: Long)
diff --git a/docs/data/concepts.md b/docs/data/concepts.md
@@ -105,7 +105,7 @@ The term "magnifications" is used synonymously for mags throughout the UI.
 At the moment, WEBKNOSSOS guarantees correct rendering of data with non-uniform mag factors only if the z-component between two mags changes by a factor of 1 or 2.
 
 Most users do not create these metadata files manually.
-When using the [WEBKNOSSOS CLI](https://docs.webknossos.org/cli), a metadata file is automatically generated. Alternatively, you can create and edit WEBKNOSSOS datasets using the [WEBKNOSSOS Python library](https://github.com/scalableminds/webknossos-libs/).
+When using the [WEBKNOSSOS CLI](https://docs.webknossos.org/cli/), a metadata file is automatically generated. Alternatively, you can create and edit WEBKNOSSOS datasets using the [WEBKNOSSOS Python library](https://github.com/scalableminds/webknossos-libs/).
 During the data import process, WEBKNOSSOS will ask for the necessary properties.
 
 [See below for the full specification](#dataset-metadata-specification).
diff --git a/docs/data/export_python.md b/docs/data/export_python.md
@@ -1,5 +1,5 @@
 ## Data Export through Python
 
-The [WEBKNOSSOS Python library](https://docs.webknossos.org/webknossos-py/index.html) makes it very easy to download/upload any of your organization's datasets, and annotations. For detailed info, please refer to the `webknossos-libs` [documentation website for guides and tutorials](https://docs.webknossos.org/webknossos-py/index.html).
+The [WEBKNOSSOS Python library](https://docs.webknossos.org/webknossos-py/) makes it very easy to download/upload any of your organization's datasets, and annotations. For detailed info, please refer to the `webknossos-libs` [documentation website for guides and tutorials](https://docs.webknossos.org/webknossos-py/).
 
 There are also quick start instructions for Python download available directly from the [WEBKNOSSOS UI](./export_ui.md).
diff --git a/docs/data/image_stacks.md b/docs/data/image_stacks.md
@@ -54,8 +54,8 @@ WEBKNOSSOS understands most modern bio-imaging file formats and uses the [BioFor
 
 You can manually convert image stacks through:
 
-- [WEBKNOSSOS CLI](https://docs.webknossos.org/cli)
-- [WEBKNOSSOS Python library](https://docs.webknossos.org/webknossos-py)
+- [WEBKNOSSOS CLI](https://docs.webknossos.org/cli/)
+- [WEBKNOSSOS Python library](https://docs.webknossos.org/webknossos-py/)
 
 ### Conversion with CLI
 You can easily convert image stacks manually with the WEBKNOSSOS CLI.
@@ -75,11 +75,11 @@ This snippet converts an image stack that is located in directory called `data/s
 It will create a so called `color` layer containing your raw greyscale/color image.
 The supplied `--voxel-size` is specified in nanometers.
 
-Read the full documentation at [WEBKNOSSOS CLI](https://docs.webknossos.org/cli).
+Read the full documentation at [WEBKNOSSOS CLI](https://docs.webknossos.org/cli/).
 
 ### Conversion with Python
 
-You can use the free [WEBKNOSSOS Python library](https://docs.webknossos.org/webknossos-py) to convert image stacks to Zarr v3 or integrate the conversion as part of an existing workflow. 
+You can use the free [WEBKNOSSOS Python library](https://docs.webknossos.org/webknossos-py/) to convert image stacks to Zarr v3 or integrate the conversion as part of an existing workflow. 
 
 ```python
 import webknossos as wk
diff --git a/docs/data/index.md b/docs/data/index.md
@@ -12,6 +12,6 @@ WEBKNOSSOS natively supports loading and streaming data in the following formats
 
 The Zarr3 format is used for all internal voxel data representations - both for the raw (microscopy) image datasets and segmentations. Skeleton annotations are saved as NML files. 
 
-Any dataset uploaded to webknossos.org will automatically be converted to Zarr3 on upload - given its source file format is supported by WEBKNOSSOS. Alternatively, you can manually convert your datasets using the [WEBKNOSSOS CLI tool](https://docs.webknossos.org/cli) or use a custom script based on the [WEBKNOSSOS Python library](https://docs.webknossos.org/webknossos-py/index.html).
+Any dataset uploaded to webknossos.org will automatically be converted to Zarr3 on upload - given its source file format is supported by WEBKNOSSOS. Alternatively, you can manually convert your datasets using the [WEBKNOSSOS CLI tool](https://docs.webknossos.org/cli/) or use a custom script based on the [WEBKNOSSOS Python library](https://docs.webknossos.org/webknossos-py/).
 
 Read more about uploading and configuring datasets on the [datasets page](../datasets/settings.md).
diff --git a/docs/data/upload_python.md b/docs/data/upload_python.md
@@ -1,2 +1,2 @@
 ### Uploading through Python
-For those wishing to automate dataset upload or to do it programmatically, check out the WEBKNOSSOS [Python library](https://docs.webknossos.org/webknossos-py). You can create, manage and upload datasets with the Python lib. 
+For those wishing to automate dataset upload or to do it programmatically, check out the WEBKNOSSOS [Python library](https://docs.webknossos.org/webknossos-py/). You can create, manage and upload datasets with the Python lib. 
diff --git a/docs/data/upload_ui.md b/docs/data/upload_ui.md
@@ -21,8 +21,8 @@ Depending on the size of the dataset, the conversion will take some time.
 You can check the progress at the [`Jobs`](../automation/jobs.md) page or the "Datasets" tab in the dashboard.
 WEBKNOSSOS will also send you an email notification.
 - Convert your data manually to WKW. For this, we provide the following software tools and libraries:
-    - The [WEBKNOSSOS CLI](https://docs.webknossos.org/cli) is a CLI tool that can convert many formats to WKW. 
-    - For other file formats, the [WEBKNOSSOS Python library](https://docs.webknossos.org/webknossos-py/index.html) can be an option for custom scripting.
+    - The [WEBKNOSSOS CLI](https://docs.webknossos.org/cli/) is a CLI tool that can convert many formats to WKW. 
+    - For other file formats, the [WEBKNOSSOS Python library](https://docs.webknossos.org/webknossos-py/) can be an option for custom scripting.
 
 In particular, the following file formats are supported for uploading (and conversion):
 
diff --git a/docs/data/zarr.md b/docs/data/zarr.md
@@ -110,7 +110,7 @@ For OME-Zarr (v0.5) datasets, the structure is slightly different (See [OME-Zarr
 
 ## Conversion to Zarr
 
-You can easily convert image stacks manually with the [WEBKNOSSOS CLI](https://docs.webknossos.org/cli).
+You can easily convert image stacks manually with the [WEBKNOSSOS CLI](https://docs.webknossos.org/cli/).
 The CLI tool expects a single file or all image files in a single folder with numbered file names.
 After installing, you can convert image stacks to Zarr datasets with the following command:
 
@@ -132,11 +132,11 @@ This example will create a sharded Zarr v3 dataset with a voxel size of (11.24,
 A maximum of 4 parallel jobs will be used to parallelize the conversion, compression and downsampling.
 Using the `--data-format zarr` argument will produce unsharded Zarr v2 datasets.
 
-Read the full documentation at [WEBKNOSSOS CLI](https://docs.webknossos.org/cli).
+Read the full documentation at [WEBKNOSSOS CLI](https://docs.webknossos.org/cli/).
 
 ### Conversion with Python
 
-You can use the free [WEBKNOSSOS Python library](https://docs.webknossos.org/webknossos-py) to convert image stacks to Zarr3 or integrate the conversion as part of an existing workflow. 
+You can use the free [WEBKNOSSOS Python library](https://docs.webknossos.org/webknossos-py/) to convert image stacks to Zarr3 or integrate the conversion as part of an existing workflow. 
 
 ```python
 import webknossos as wk
diff --git a/docs/faq.md b/docs/faq.md
@@ -27,13 +27,13 @@ We have years of experience with automated machine learning analysis and [offer
 We are also always interested in new collaborations. 
 Get in touch if you want to work together on a project resulting in new classifiers.
 
-WEBKNOSSOS does not allow you to run custom machine learning models on your data yet. As a work-around you can download your annotations from WEBKNOSSOS - either manually or scripted [through our Python library](https://docs.webknossos.org/webknossos-py) - and do your ML analysis offline and use WEBKNOSSOS to inspect the results. 
+WEBKNOSSOS does not allow you to run custom machine learning models on your data yet. As a work-around you can download your annotations from WEBKNOSSOS - either manually or scripted [through our Python library](https://docs.webknossos.org/webknossos-py/) - and do your ML analysis offline and use WEBKNOSSOS to inspect the results. 
 
 ## How can I use my dataset with WEBKNOSSOS?
 
 WEBKNOSSOS supports [WKW (optimized), OME-Zarr (NGFF), Neuroglancer Precomputed, N5, KNOSSOS cubes](./data/index.md), and image stacks (converted on upload). You can also connect to Neuroglancer Precomputed, N5, and Zarr datasets hosted in the cloud (Google Cloud Storage, AWS S3).
 
-Smaller datasets (up to multiple GB) can be uploaded directly through the web interface. For larger datasets, we recommend converting them to the standard WKW format using the [WEBKNOSSOS CLI](https://docs.webknossos.org/cli) CLI tool and uploading it via the [WEBKNOSSOS Python package](https://docs.webknossos.org/webknossos-py/examples/upload_image_data.html).
+Smaller datasets (up to multiple GB) can be uploaded directly through the web interface. For larger datasets, we recommend converting them to the standard WKW format using the [WEBKNOSSOS CLI](https://docs.webknossos.org/cli/) CLI tool and uploading it via the [WEBKNOSSOS Python package](https://docs.webknossos.org/webknossos-py/examples/upload_image_data.html).
 
 ## Can I host the WEBKNOSSOS data in my own compute cluster (on-premise installation)?
 
@@ -53,7 +53,7 @@ For example, the WEBKNOSSOS main component could be hosted on commercial cloud i
 ## Can I further analyze my annotations outside of WEBKNOSSOS with Python/MATLAB?
 Yes, you can. WEBKNOSSOS allows the download and export of skeleton annotations as NML files and segmentations/volume data as binary/wkw files.
 
-Use our free [Python library](https://docs.webknossos.org/webknossos-py) to work with the WEBKNOSSOS standard formats.
+Use our free [Python library](https://docs.webknossos.org/webknossos-py/) to work with the WEBKNOSSOS standard formats.
 
 ## Newly registered users don't show up
 
diff --git a/docs/index.md b/docs/index.md
@@ -26,7 +26,7 @@ WEBKNOSSOS is also a platform for [showcasing datasets](https://webknossos.org)
 - Supports Time Series datasets
 - [3D Mesh Visualization](./meshes/index.md)
 - [Integrated Synapse and Connectome Viewer](./connectome_viewer.md)
-- [Python library with documentation for API access and integration in custom analysis workflows](https://docs.webknossos.org/webknossos-py/index.html)
+- [Python library with documentation for API access and integration in custom analysis workflows](https://docs.webknossos.org/webknossos-py/)
 - [Frontend API for user scripting](https://webknossos.org/assets/docs/frontend-api/index.html)
 - Developed as an open-source project with [automated testing](https://circleci.com/gh/scalableminds/webknossos)
 - [Deployable with Docker for production and development](https://hub.docker.com/r/scalableminds/webknossos/)
diff --git a/docs/skeleton_annotation/import_export.md b/docs/skeleton_annotation/import_export.md
@@ -23,7 +23,7 @@ Importing a skeleton annotation can be achieved using one of two ways:
 
 ![Skeletons can be imported by drag and drop in the annotation view or from the dashboard](../images/tracing_ui_import.jpeg)
 
-If you are looking to import/export annotations through Python code, check out our [WEBKNOSSOS Python library](https://docs.webknossos.org/webknossos-py).
+If you are looking to import/export annotations through Python code, check out our [WEBKNOSSOS Python library](https://docs.webknossos.org/webknossos-py/).
 
 ## Merging skeleton annotations
 
diff --git a/frontend/javascripts/test/global_mocks.ts b/frontend/javascripts/test/global_mocks.ts
@@ -22,6 +22,7 @@ vi.mock("libs/toast", () => ({
     success: vi.fn(),
     info: vi.fn(),
   },
+  showToastOnce: vi.fn(),
 }));
 
 vi.mock("libs/user_local_storage", () => ({
diff --git a/frontend/javascripts/viewer/model/sagas/meshes/ad_hoc_mesh_saga.ts b/frontend/javascripts/viewer/model/sagas/meshes/ad_hoc_mesh_saga.ts
@@ -429,8 +429,12 @@ function* maybeLoadMeshChunk(
 ): Saga<Vector3[]> {
   const additionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates);
   const threeDMap = getOrAddMapForSegment(layer.name, segmentId, additionalCoordinates);
+  const mag = magInfo.getMagByIndexOrThrow(zoomStep);
+  const paddedPosition = V3.toArray(V3.sub(clippedPosition, mag));
+  const paddedPositionWithinLayer =
+    layer.cube.boundingBox.clipPositionIntoBoundingBox(paddedPosition);
 
-  if (threeDMap.get(clippedPosition)) {
+  if (threeDMap.get(paddedPositionWithinLayer)) {
     return [];
   }
 
@@ -439,7 +443,7 @@ function* maybeLoadMeshChunk(
   }
 
   batchCounterPerSegment[segmentId]++;
-  threeDMap.set(clippedPosition, true);
+  threeDMap.set(paddedPositionWithinLayer, true);
   const scaleFactor = yield* select((state) => state.dataset.dataSource.scale.factor);
   const dataStoreHost = yield* select((state) => state.dataset.dataStore.url);
   const datasetId = yield* select((state) => state.dataset.id);
@@ -449,8 +453,6 @@ function* maybeLoadMeshChunk(
   }`;
   const tracingStoreUrl = `${tracingStoreHost}/tracings/volume/${layer.name}`;
 
-  const mag = magInfo.getMagByIndexOrThrow(zoomStep);
-
   if (isInitialRequest) {
     sendAnalyticsEvent("request_isosurface", {
       mode: useDataStore ? "view" : "annotation",
@@ -462,9 +464,6 @@ function* maybeLoadMeshChunk(
   const { segmentMeshController } = getSceneController();
 
   const cubeSize = marchingCubeSizeInTargetMag();
-  const paddedPosition = V3.toArray(V3.sub(clippedPosition, mag));
-  const paddedPositionWithinLayer =
-    layer.cube.boundingBox.clipPositionIntoBoundingBox(paddedPosition);
 
   while (retryCount < MAX_RETRY_COUNT) {
     try {
diff --git a/frontend/javascripts/viewer/model/sagas/saving/save_saga.ts b/frontend/javascripts/viewer/model/sagas/saving/save_saga.ts
@@ -98,6 +98,7 @@ function* watchForSaveConflicts(): Saga<void> {
       annotationId,
       versionOnClient + 1,
       undefined,
+      false,
       true,
     );
 
diff --git a/tools/postgres/schema.sql b/tools/postgres/schema.sql
diff --git a/unreleased_changes/8896.md b/unreleased_changes/8896.md
diff --git a/unreleased_changes/8915.md b/unreleased_changes/8915.md
diff --git a/unreleased_changes/8919.md b/unreleased_changes/8919.md
