Reupload exported editable mapping annotation zip (#8969)

### URL of deployed dev instance (used for testing):
- https://uploadeditablemapping.webknossos.xyz

### Steps to test:
- Proofread some
- Download annotation, reupload
- Should see the same proofread state, history should be inaccessible.
- ~Test with multiple proofreading layers in one annotation~ ←
unsupported case
- Try uploading multiple annotations, assertions should block if one has
proofreading.

### TODOs:
- [x] Handle history version numbers correctly
- [x] Do we need to find the initial largestAgglomerateId? how does this
work in create;duplicate?
- [x] Add assertions when uploading multiple annotations (or fully build
it?)
- [x] Unzip the zarr arrays
- [x] Create update actions from them
- [x] Apply update actions?
- [x] Make sure the result is a consistent state.
- [x] tracingstore now needs chunkContentsCache
- [x] propagate correct boolean fill_value
- [x] double-check update actions have correct ordering
- [x] test with multiple proofreading layers, are versions dense?
- [x] test with multiple layers only one of which has proofreading, do
versions look right?
- [x] clean up

### Issues:
- fixes #8826

------
- [x] Added changelog entry (create a `$PR_NUMBER.md` file in
`unreleased_changes` or use `./tools/create-changelog-entry.py`)
- [x] Removed dev-only changes like prints and application.conf edits
- [x] Considered [common edge
cases](../blob/master/.github/common_edge_cases.md)
- [x] Needs datastore update after deployment

Author: fm3

app/Startup.scala

@@ -98,7 +98,7 @@ class Startup @Inject()(actorSystem: ActorSystem,
   }
 
   private def ensurePostgresSchema(): Unit = {
-    logger.info("Checking database schema…")
+    logger.info("Checking database schema...")
 
     val errorMessageBuilder = mutable.ListBuffer[String]()
     val capturingProcessLogger =
@@ -115,7 +115,7 @@ class Startup @Inject()(actorSystem: ActorSystem,
   }
 
   private def ensurePostgresDatabase(): Unit = {
-    logger.info(s"Ensuring Postgres database…")
+    logger.info(s"Ensuring Postgres database...")
     val processLogger =
       ProcessLogger((o: String) => logger.info(s"dbtool: $o"), (e: String) => logger.error(s"dbtool: $e"))
 

app/controllers/AnnotationIOController.scala

@@ -132,25 +132,25 @@ class AnnotationIOController @Inject()(
           volumeLayersGrouped <- adaptVolumeTracingsToFallbackLayer(volumeLayersGroupedRaw, dataset, usableDataSource)
           tracingStoreClient <- tracingStoreService.clientFor(dataset)
           newAnnotationId = ObjectId.generate
-          mergedVolumeLayers <- mergeAndSaveVolumeLayers(newAnnotationId,
-                                                         volumeLayersGrouped,
-                                                         tracingStoreClient,
-                                                         parsedFiles.otherFiles,
-                                                         usableDataSource,
-                                                         dataset._id)
+          (mergedVolumeLayers, earliestAccessibleVersion) <- mergeAndSaveVolumeLayers(newAnnotationId,
+                                                                                      volumeLayersGrouped,
+                                                                                      tracingStoreClient,
+                                                                                      parsedFiles.otherFiles,
+                                                                                      usableDataSource,
+                                                                                      dataset._id)
           mergedSkeletonLayers <- mergeAndSaveSkeletonLayers(skeletonTracings, tracingStoreClient)
           annotation <- annotationService.createFrom(request.identity,
                                                      dataset,
                                                      mergedSkeletonLayers ::: mergedVolumeLayers,
                                                      AnnotationType.Explorational,
                                                      name,
                                                      description,
-                                                     ObjectId.generate)
+                                                     newAnnotationId)
           annotationProto = AnnotationProto(
             description = annotation.description,
             version = 0L,
             annotationLayers = annotation.annotationLayers.map(_.toProto),
-            earliestAccessibleVersion = 0L
+            earliestAccessibleVersion = earliestAccessibleVersion
           )
           _ <- tracingStoreClient.saveAnnotationProto(annotation._id, annotationProto)
           _ <- annotationDAO.insertOne(annotation)
@@ -163,37 +163,64 @@ class AnnotationIOController @Inject()(
       }
   }
 
+  private def layersHaveDuplicateFallbackLayer(annotationLayers: Seq[UploadedVolumeLayer]) = {
+    val withFallbackLayer = annotationLayers.filter(_.tracing.fallbackLayer.isDefined)
+    withFallbackLayer.length > withFallbackLayer.distinctBy(_.tracing.fallbackLayer).length
+  }
+
   private def mergeAndSaveVolumeLayers(newAnnotationId: ObjectId,
                                        volumeLayersGrouped: Seq[List[UploadedVolumeLayer]],
                                        client: WKRemoteTracingStoreClient,
                                        otherFiles: Map[String, File],
                                        dataSource: UsableDataSource,
-                                       datasetId: ObjectId): Fox[List[AnnotationLayer]] =
+                                       datasetId: ObjectId): Fox[(List[AnnotationLayer], Long)] =
     if (volumeLayersGrouped.isEmpty)
-      Fox.successful(List())
+      Fox.successful(List(), 0L)
+    else if (volumeLayersGrouped.exists(layersHaveDuplicateFallbackLayer(_)))
+      Fox.failure("Cannot save annotation with multiple volume layers that have the same fallback segmentation layer.")
     else if (volumeLayersGrouped.length > 1 && volumeLayersGrouped.exists(_.length > 1))
       Fox.failure("Cannot merge multiple annotations that each have multiple volume layers.")
-    else if (volumeLayersGrouped.length == 1) { // Just one annotation was uploaded, keep its layers separate
-      Fox.serialCombined(volumeLayersGrouped.toList.flatten.zipWithIndex) { volumeLayerWithIndex =>
-        val uploadedVolumeLayer = volumeLayerWithIndex._1
-        val idx = volumeLayerWithIndex._2
-        val newTracingId = TracingId.generate
-        for {
-          _ <- client.saveVolumeTracing(newAnnotationId,
-                                        newTracingId,
-                                        uploadedVolumeLayer.tracing,
-                                        uploadedVolumeLayer.getDataZipFrom(otherFiles),
-                                        dataSource = dataSource,
-                                        datasetId = datasetId)
-        } yield
-          AnnotationLayer(
-            newTracingId,
-            AnnotationLayerType.Volume,
-            uploadedVolumeLayer.name.getOrElse(AnnotationLayer.defaultVolumeLayerName + idx.toString),
-            AnnotationLayerStatistics.unknown
-          )
-      }
-    } else { // Multiple annotations with volume layers (but at most one each) was uploaded merge those volume layers into one
+    else if (volumeLayersGrouped.length > 1 && volumeLayersGrouped.exists(
+               _.exists(_.editedMappingEdgesLocation.isDefined))) {
+      Fox.failure("Cannot merge multiple annotations with editable mapping (proofreading) edges.")
+    } else if (volumeLayersGrouped.length == 1) { // Just one annotation was uploaded, keep its layers separate
+      var layerUpdatesStartVersionMutable = 1L
+      for {
+        annotationLayers <- Fox.serialCombined(volumeLayersGrouped.toList.flatten.zipWithIndex) {
+          volumeLayerWithIndex =>
+            val uploadedVolumeLayer = volumeLayerWithIndex._1
+            val idx = volumeLayerWithIndex._2
+            val newTracingId = TracingId.generate
+            for {
+              numberOfSavedVersions <- client.saveEditableMappingIfPresent(
+                newAnnotationId,
+                newTracingId,
+                uploadedVolumeLayer.getEditableMappingEdgesZipFrom(otherFiles),
+                uploadedVolumeLayer.editedMappingBaseMappingName,
+                startVersion = layerUpdatesStartVersionMutable
+              )
+              // The next layer’s update actions then need to start after this one
+              _ = layerUpdatesStartVersionMutable = layerUpdatesStartVersionMutable + numberOfSavedVersions
+              mappingName = if (uploadedVolumeLayer.editedMappingEdgesLocation.isDefined) Some(newTracingId)
+              else uploadedVolumeLayer.tracing.mappingName
+              _ <- client.saveVolumeTracing(
+                newAnnotationId,
+                newTracingId,
+                uploadedVolumeLayer.tracing.copy(mappingName = mappingName),
+                uploadedVolumeLayer.getDataZipFrom(otherFiles),
+                dataSource = dataSource,
+                datasetId = datasetId
+              )
+            } yield
+              AnnotationLayer(
+                newTracingId,
+                AnnotationLayerType.Volume,
+                uploadedVolumeLayer.name.getOrElse(AnnotationLayer.defaultVolumeLayerName + idx.toString),
+                AnnotationLayerStatistics.unknown
+              )
+        }
+      } yield (annotationLayers, layerUpdatesStartVersionMutable)
+    } else { // Multiple annotations with volume layers (but at most one each) were uploaded, they have no editable mappings. Merge those volume layers into one
       val uploadedVolumeLayersFlat = volumeLayersGrouped.toList.flatten
       val newTracingId = TracingId.generate
       for {
@@ -206,13 +233,14 @@ class AnnotationIOController @Inject()(
           uploadedVolumeLayersFlat.map(v => v.getDataZipFrom(otherFiles))
         )
       } yield
-        List(
-          AnnotationLayer(
-            newTracingId,
-            AnnotationLayerType.Volume,
-            AnnotationLayer.defaultVolumeLayerName,
-            AnnotationLayerStatistics.unknown
-          ))
+        (List(
+           AnnotationLayer(
+             newTracingId,
+             AnnotationLayerType.Volume,
+             AnnotationLayer.defaultVolumeLayerName,
+             AnnotationLayerStatistics.unknown
+           )),
+         0L)
     }
 
   private def mergeAndSaveSkeletonLayers(skeletonTracings: List[SkeletonTracing],

app/models/annotation/AnnotationUploadService.scala

@@ -21,9 +21,17 @@ import play.api.i18n.MessagesProvider
 
 import scala.concurrent.{ExecutionContext, Future}
 
-case class UploadedVolumeLayer(tracing: VolumeTracing, dataZipLocation: String, name: Option[String]) {
+case class UploadedVolumeLayer(tracing: VolumeTracing,
+                               dataZipLocation: String,
+                               name: Option[String],
+                               editedMappingEdgesLocation: Option[String],
+                               editedMappingBaseMappingName: Option[String]) {
   def getDataZipFrom(otherFiles: Map[String, File]): Option[File] =
     otherFiles.get(dataZipLocation)
+
+  def getEditableMappingEdgesZipFrom(otherFiles: Map[String, File]): Option[File] =
+    editedMappingEdgesLocation.flatMap(otherFiles.get)
+
 }
 
 case class SharedParsingParameters(useZipName: Boolean,

app/models/annotation/WKRemoteTracingStoreClient.scala

@@ -249,6 +249,23 @@ class WKRemoteTracingStoreClient(
     } yield ()
   }
 
+  def saveEditableMappingIfPresent(annotationId: ObjectId,
+                                   newTracingId: String,
+                                   editedMappingEdgesZip: Option[File],
+                                   editedMappingBaseMappingName: Option[String],
+                                   startVersion: Long): Fox[Long] =
+    (editedMappingEdgesZip, editedMappingBaseMappingName) match {
+      case (Some(zipfile), Some(baseMappingName)) =>
+        rpc(s"${tracingStore.url}/tracings/mapping/$newTracingId/save").withLongTimeout
+          .addQueryParam("token", RpcTokenHolder.webknossosToken)
+          .addQueryParam("annotationId", annotationId)
+          .addQueryParam("baseMappingName", baseMappingName)
+          .addQueryParam("startVersion", startVersion)
+          .postFileWithJsonResponse[Long](zipfile)
+      case (None, None) => Fox.successful(0L)
+      case _            => Fox.failure("annotation.upload.editableMappingIncompleteInformation")
+    }
+
   def getVolumeTracing(annotationId: ObjectId,
                        annotationLayer: AnnotationLayer,
                        version: Option[Long],

app/models/annotation/nml/NmlParser.scala

@@ -91,10 +91,13 @@ class NmlParser @Inject()(datasetDAO: DatasetDAO)
             segmentGroups = v.segmentGroups,
             hasSegmentIndex = None, // Note: this property may be adapted later in adaptPropertiesToFallbackLayer
             editPositionAdditionalCoordinates = nmlParams.editPositionAdditionalCoordinates,
-            additionalAxes = nmlParams.additionalAxisProtos
+            additionalAxes = nmlParams.additionalAxisProtos,
+            hasEditableMapping = if (v.editedMappingEdgesLocation.isDefined) Some(true) else None
           ),
           basePath.getOrElse("") + v.dataZipPath,
           v.name,
+          v.editedMappingEdgesLocation.map(location => basePath.getOrElse("") + location),
+          v.editedMappingBaseMappingName
         )
       }
       skeletonTracing: SkeletonTracing = SkeletonTracing(
@@ -220,7 +223,9 @@ class NmlParser @Inject()(datasetDAO: DatasetDAO)
           getSingleAttributeOpt(node, "name"),
           parseVolumeSegmentMetadata(node \ "segments" \ "segment"),
           getSingleAttributeOpt(node, "largestSegmentId").flatMap(_.toLongOpt),
-          extractSegmentGroups(node \ "groups").getOrElse(List())
+          extractSegmentGroups(node \ "groups").getOrElse(List()),
+          getSingleAttributeOpt(node, "editedMappingEdgesLocation"),
+          getSingleAttributeOpt(node, "editedMappingBaseMappingName")
         )
       }
     )

app/models/annotation/nml/NmlVolumeTag.scala

@@ -9,4 +9,6 @@ case class NmlVolumeTag(dataZipPath: String,
                         name: Option[String],
                         segments: Seq[Segment],
                         largestSegmentId: Option[Long],
-                        segmentGroups: Seq[SegmentGroup]) {}
+                        segmentGroups: Seq[SegmentGroup],
+                        editedMappingEdgesLocation: Option[String],
+                        editedMappingBaseMappingName: Option[String])

conf/application.conf

@@ -200,6 +200,7 @@ tracingstore {
     address = "localhost"
     port = 6379
   }
+  cache.chunkCacheMaxSizeBytes = 20000000 # 20 MB
 }
 
 # Serve image data. Only active if the corresponding play module is enabled

conf/messages

@@ -267,6 +267,7 @@ annotation.idForTracing.failed=Could not find the annotation id for this tracing
 annotation.editableMapping.getAgglomerateGraph.failed=Could not look up an agglomerate graph for requested agglomerate.
 annotation.editableMapping.getAgglomerateIdsForSegments.failed=Could not look up agglomerate ids for requested segments.
 annotation.duplicate.failed=Failed to duplicate annotation
+annotation.upload.editableMappingIncompleteInformation=Could not store editable mapping, either file or baseMappingName is missing.
 
 mesh.file.listChunks.failed=Failed to load chunk list for segment {0} from mesh file “{1}”
 mesh.file.loadChunk.failed=Failed to load mesh chunk for segment

frontend/javascripts/viewer/view/version_entry.tsx

@@ -167,7 +167,7 @@ const descriptionFns: Record<
         ? `at position ${action.value.segmentPosition1}`
         : (action.value.segmentId1 ?? "unknown");
     const segment2Description =
-      action.value.segmentPosition2 ?? action.value.segmentId1 ?? "unknown";
+      action.value.segmentPosition2 ?? action.value.segmentId2 ?? "unknown";
     const description = `Split agglomerate ${action.value.agglomerateId} by separating the segments ${segment1Description} and ${segment2Description}.`;
     return {
       description,
@@ -180,7 +180,7 @@ const descriptionFns: Record<
         ? `at position ${action.value.segmentPosition1}`
         : (action.value.segmentId1 ?? "unknown");
     const segment2Description =
-      action.value.segmentPosition2 ?? action.value.segmentId1 ?? "unknown";
+      action.value.segmentPosition2 ?? action.value.segmentId2 ?? "unknown";
     const description = `Merged agglomerates ${action.value.agglomerateId1} and ${action.value.agglomerateId2} by combining the segments  ${segment1Description} and ${segment2Description}.`;
     return {
       description,

unreleased_changes/8969.md

@@ -0,0 +1,2 @@
+### Added
+- Editable mapping (aka proofreading) annotations can now be downloaded as zipfile and re-uploaded.