Show task bounding boxes in compound annotations (#8735)

fm3 · web-flow · commit 7b90571e4f5d · 2025-07-07T09:10:30.000+02:00
### URL of deployed dev instance (used for testing): - https://compoundincludetaskbboxes.webknossos.xyz ### Steps to test: - Create task with bbox - Trace it + finish - View compound. Bbox should be in user bounding boxes ### TODOs: - [x] No more unique-i-fying. Leave in all instances’ boxes - [x] Add both task id and annotation id to box name ### Issues: - fixes #7835 ------ - [x] Added changelog entry (create a `$PR_NUMBER.md` file in `unreleased_changes` or use `./tools/create-changelog-entry.py`) - [x] Considered [common edge cases](../blob/master/.github/common_edge_cases.md)
diff --git a/app/models/annotation/AnnotationMerger.scala b/app/models/annotation/AnnotationMerger.scala
@@ -10,6 +10,7 @@ import models.annotation.AnnotationType.AnnotationType
 import models.dataset.DatasetDAO
 import models.user.User
 import com.scalableminds.util.objectid.ObjectId
+import com.scalableminds.webknossos.tracingstore.tracings.NamedBoundingBox
 
 import scala.concurrent.ExecutionContext
 
@@ -30,7 +31,8 @@ class AnnotationMerger @Inject()(datasetDAO: DatasetDAO, tracingStoreService: Tr
       annotationB._dataset,
       annotationB._team,
       AnnotationType.Explorational,
-      List(annotationA, annotationB)
+      List(annotationA, annotationB),
+      Seq.empty
     )
 
   def mergeN(
@@ -40,7 +42,8 @@ class AnnotationMerger @Inject()(datasetDAO: DatasetDAO, tracingStoreService: Tr
       datasetId: ObjectId,
       teamId: ObjectId,
       typ: AnnotationType,
-      annotations: List[Annotation]
+      annotations: List[Annotation],
+      additionalBoundingBoxes: Seq[NamedBoundingBox]
   )(implicit ctx: DBAccessContext): Fox[Annotation] =
     if (annotations.isEmpty)
       Fox.empty
@@ -51,7 +54,8 @@ class AnnotationMerger @Inject()(datasetDAO: DatasetDAO, tracingStoreService: Tr
           datasetId,
           newId,
           userId,
-          toTemporaryStore) ?~> "Failed to merge annotations in tracingstore."
+          toTemporaryStore,
+          additionalBoundingBoxes) ?~> "Failed to merge annotations in tracingstore."
       } yield {
         Annotation(
           newId,
@@ -70,15 +74,17 @@ class AnnotationMerger @Inject()(datasetDAO: DatasetDAO, tracingStoreService: Tr
       datasetId: ObjectId,
       newAnnotationId: ObjectId,
       requestingUserId: ObjectId,
-      toTemporaryStore: Boolean)(implicit ctx: DBAccessContext): Fox[List[AnnotationLayer]] =
+      toTemporaryStore: Boolean,
+      additionalBoundingBoxes: Seq[NamedBoundingBox])(implicit ctx: DBAccessContext): Fox[List[AnnotationLayer]] =
     for {
       dataset <- datasetDAO.findOne(datasetId)
       tracingStoreClient: WKRemoteTracingStoreClient <- tracingStoreService.clientFor(dataset)
       mergedAnnotationProto <- tracingStoreClient.mergeAnnotationsByIds(annotations.map(_._id),
                                                                         annotations.map(_._user),
                                                                         newAnnotationId,
                                                                         toTemporaryStore,
-                                                                        requestingUserId)
+                                                                        requestingUserId,
+                                                                        additionalBoundingBoxes)
       layers = mergedAnnotationProto.annotationLayers.map(AnnotationLayer.fromProto)
     } yield layers.toList
 
diff --git a/app/models/annotation/WKRemoteTracingStoreClient.scala b/app/models/annotation/WKRemoteTracingStoreClient.scala
@@ -22,7 +22,7 @@ import com.scalableminds.webknossos.datastore.models.annotation.{
 import com.scalableminds.webknossos.datastore.models.datasource.DataSourceLike
 import com.scalableminds.webknossos.datastore.rpc.RPC
 import com.scalableminds.webknossos.tracingstore.controllers.MergedFromIdsRequest
-import com.scalableminds.webknossos.tracingstore.tracings.TracingSelector
+import com.scalableminds.webknossos.tracingstore.tracings.{NamedBoundingBox, TracingSelector}
 import com.scalableminds.webknossos.tracingstore.tracings.volume.MagRestrictions
 import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeDataZipFormat.VolumeDataZipFormat
 import com.typesafe.scalalogging.LazyLogging
@@ -176,15 +176,16 @@ class WKRemoteTracingStoreClient(
                             ownerIds: List[ObjectId],
                             newAnnotationId: ObjectId,
                             toTemporaryStore: Boolean,
-                            requestingUserId: ObjectId): Fox[AnnotationProto] = {
+                            requestingUserId: ObjectId,
+                            additionalBoundingBoxes: Seq[NamedBoundingBox]): Fox[AnnotationProto] = {
     logger.debug(s"Called to merge ${annotationIds.length} annotations by ids." + baseInfo)
     rpc(s"${tracingStore.url}/tracings/annotation/mergedFromIds").withLongTimeout
       .addQueryString("token" -> RpcTokenHolder.webknossosToken)
       .addQueryString("toTemporaryStore" -> toTemporaryStore.toString)
       .addQueryString("newAnnotationId" -> newAnnotationId.toString)
       .addQueryString("requestingUserId" -> requestingUserId.toString)
-      .postJsonWithProtoResponse[MergedFromIdsRequest, AnnotationProto](MergedFromIdsRequest(annotationIds, ownerIds))(
-        AnnotationProto)
+      .postJsonWithProtoResponse[MergedFromIdsRequest, AnnotationProto](
+        MergedFromIdsRequest(annotationIds, ownerIds, additionalBoundingBoxes))(AnnotationProto)
   }
 
   def mergeSkeletonTracingsByContents(newTracingId: String, tracings: SkeletonTracings): Fox[Unit] = {
diff --git a/app/models/annotation/handler/ProjectInformationHandler.scala b/app/models/annotation/handler/ProjectInformationHandler.scala
@@ -2,17 +2,19 @@ package models.annotation.handler
 
 import com.scalableminds.util.accesscontext.DBAccessContext
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
+
 import javax.inject.Inject
 import models.annotation._
 import models.project.ProjectDAO
 import models.user.{User, UserService}
-
 import com.scalableminds.util.objectid.ObjectId
+import models.task.TaskDAO
 
 import scala.concurrent.ExecutionContext
 
 class ProjectInformationHandler @Inject()(annotationDAO: AnnotationDAO,
                                           projectDAO: ProjectDAO,
+                                          taskDAO: TaskDAO,
                                           userService: UserService,
                                           annotationMerger: AnnotationMerger)(implicit val ec: ExecutionContext)
     extends AnnotationInformationHandler
@@ -28,13 +30,15 @@ class ProjectInformationHandler @Inject()(annotationDAO: AnnotationDAO,
       _ <- assertAllOnSameDataset(annotations)
       _ <- assertNonEmpty(annotations) ?~> "project.noAnnotations"
       datasetId <- annotations.headOption.map(_._dataset).toFox
+      taskBoundingBoxes <- taskDAO.findTaskBoundingBoxesByAnnotationIds(annotations.map(_._id))
       mergedAnnotation <- annotationMerger.mergeN(projectId,
                                                   toTemporaryStore = true,
                                                   user._id,
                                                   datasetId,
                                                   project._team,
                                                   AnnotationType.CompoundProject,
-                                                  annotations) ?~> "annotation.merge.failed.compound"
+                                                  annotations,
+                                                  taskBoundingBoxes) ?~> "annotation.merge.failed.compound"
     } yield mergedAnnotation
 
   override def restrictionsFor(projectId: ObjectId)(implicit ctx: DBAccessContext): Fox[AnnotationRestrictions] =
diff --git a/app/models/annotation/handler/TaskInformationHandler.scala b/app/models/annotation/handler/TaskInformationHandler.scala
@@ -31,13 +31,15 @@ class TaskInformationHandler @Inject()(taskDAO: TaskDAO,
       user <- userOpt.toFox ?~> "user.notAuthorised"
       project <- projectDAO.findOne(task._project)
       datasetId <- finishedAnnotations.headOption.map(_._dataset).toFox
+      taskBoundingBoxes <- taskDAO.findTaskBoundingBoxesByAnnotationIds(annotations.map(_._id))
       mergedAnnotation <- annotationMerger.mergeN(task._id,
                                                   toTemporaryStore = true,
                                                   user._id,
                                                   datasetId,
                                                   project._team,
                                                   AnnotationType.CompoundTask,
-                                                  finishedAnnotations) ?~> "annotation.merge.failed.compound"
+                                                  finishedAnnotations,
+                                                  taskBoundingBoxes) ?~> "annotation.merge.failed.compound"
     } yield mergedAnnotation
 
   def restrictionsFor(taskId: ObjectId)(implicit ctx: DBAccessContext): Fox[AnnotationRestrictions] =
diff --git a/app/models/annotation/handler/TaskTypeInformationHandler.scala b/app/models/annotation/handler/TaskTypeInformationHandler.scala
@@ -32,13 +32,15 @@ class TaskTypeInformationHandler @Inject()(taskTypeDAO: TaskTypeDAO,
       _ <- assertNonEmpty(finishedAnnotations) ?~> "taskType.noAnnotations"
       user <- userOpt.toFox ?~> "user.notAuthorised"
       datasetId <- finishedAnnotations.headOption.map(_._dataset).toFox
+      taskBoundingBoxes <- taskDAO.findTaskBoundingBoxesByAnnotationIds(annotations.map(_._id))
       mergedAnnotation <- annotationMerger.mergeN(taskTypeId,
                                                   toTemporaryStore = true,
                                                   user._id,
                                                   datasetId,
                                                   taskType._team,
                                                   AnnotationType.CompoundTaskType,
-                                                  finishedAnnotations) ?~> "annotation.merge.failed.compound"
+                                                  finishedAnnotations,
+                                                  taskBoundingBoxes) ?~> "annotation.merge.failed.compound"
     } yield mergedAnnotation
 
   override def restrictionsFor(taskTypeId: ObjectId)(implicit ctx: DBAccessContext): Fox[AnnotationRestrictions] =
diff --git a/app/models/task/Task.scala b/app/models/task/Task.scala
@@ -6,6 +6,7 @@ import com.scalableminds.util.objectid.ObjectId
 import com.scalableminds.util.time.Instant
 import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.schema.Tables._
+import com.scalableminds.webknossos.tracingstore.tracings.NamedBoundingBox
 
 import javax.inject.Inject
 import models.annotation._
@@ -59,7 +60,7 @@ class TaskDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
         r.totalinstances,
         r.pendinginstances,
         r.tracingtime,
-        r.boundingbox.map(b => parseArrayLiteral(b).map(_.toInt)).flatMap(BoundingBox.fromSQL),
+        parseBboxOpt(r.boundingbox),
         editPosition,
         editRotation,
         r.creationinfo,
@@ -68,6 +69,9 @@ class TaskDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
       )
     }
 
+  private def parseBboxOpt(bboxLiteral: Option[String]): Option[BoundingBox] =
+    bboxLiteral.map(b => parseArrayLiteral(b).map(_.toInt)).flatMap(BoundingBox.fromSQL)
+
   override protected def readAccessQ(requestingUserId: ObjectId) =
     q"""((SELECT _team FROM webknossos.projects p WHERE _project = p._id) IN (SELECT _team FROM webknossos.user_team_roles WHERE _user = $requestingUserId)
       or ((SELECT _organization FROM webknossos.teams WHERE webknossos.teams._id = (SELECT _team FROM webknossos.projects p WHERE _project = p._id))
@@ -282,6 +286,22 @@ class TaskDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
         q"SELECT domain FROM webknossos.experienceDomains WHERE _organization = $organizationId".as[String])
     } yield rowsRaw.toList
 
+  def findTaskBoundingBoxesByAnnotationIds(annotationIds: Seq[ObjectId]): Fox[Seq[NamedBoundingBox]] =
+    for {
+      rowsRaw <- run(q"""SELECT t.boundingBox, t._id, a._id
+                         FROM webknossos.tasks_ t
+                         JOIN webknossos.annotations_ a on a._task = t._id
+                         WHERE a._id IN ${SqlToken.tupleFromList(annotationIds)}
+                         AND t.boundingBox IS NOT NULL
+                         ORDER BY t._id
+                         """.as[(String, ObjectId, ObjectId)])
+      namedBboxes = rowsRaw.flatMap {
+        case (bboxLiteral, taskId, annotationId) =>
+          parseBboxOpt(Some(bboxLiteral)).map(bbox =>
+            NamedBoundingBox(0, Some(s"Task bounding box of instance $annotationId of task $taskId"), None, None, bbox))
+      }
+    } yield namedBboxes
+
   def insertOne(t: Task): Fox[Unit] =
     for {
       _ <- run(q"""INSERT INTO webknossos.tasks(_id, _project, _script, _taskType,
diff --git a/conf/messages b/conf/messages
@@ -216,14 +216,9 @@ annotation.notActive=The annotation is not active and cannot be finished
 annotation.reset.failed=Could not reset annotation to base
 annotation.merge.success=Merging successfully done
 annotation.merge.failed=Merging failed
-annotation.merge.failed.compound=Could not merge annotations for compound view
-annotation.transferee.noDatasetAccess=Cannot transfer annotation to a user who has no access to the dataset
-annotation.timelogging.read.failed=Time
 annotation.write.failed=Could not convert annotation to json
 annotation.merge.failed.compound=Couldn’t merge annotations for compound view
 annotation.transferee.noDatasetAccess=Cannot transfer annotation to a user who has no access to the dataset
-annotation.timelogging.read.failed=Time
-annotation.write.failed=Could not convert annotation to json
 annotation.needsEitherSkeletonOrVolume=Annotation needs at least one of skeleton or volume
 annotation.addLayer.explorationalsOnly=Could not add annotation layer because it is only allowed for explorational annotations.
 annotation.addLayer.nameInUse=An annotation layer with this name already exists in this annotation. Please change it to prevent duplicates.
diff --git a/unreleased_changes/8735.md b/unreleased_changes/8735.md
@@ -0,0 +1,2 @@
+### Changed
+- Compound annotations (merged views of finished task annotations) now show the task bounding boxes.
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala
@@ -108,7 +108,9 @@ class SkeletonTracingController @Inject()(skeletonTracingService: SkeletonTracin
         accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) {
           val tracings: List[Option[SkeletonTracing]] = request.body
           for {
-            mergedTracing <- skeletonTracingService.merge(tracings.flatten, newVersion = 0L).toFox
+            mergedTracing <- skeletonTracingService
+              .merge(tracings.flatten, newVersion = 0L, additionalBoundingBoxes = Seq.empty)
+              .toFox
             processedTracing = skeletonTracingService.remapTooLargeTreeIds(mergedTracing)
             _ <- skeletonTracingService.saveSkeleton(newTracingId, processedTracing.version, processedTracing)
           } yield Ok
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala
@@ -35,7 +35,9 @@ import play.api.mvc.{Action, AnyContent, PlayBodyParsers}
 
 import scala.concurrent.ExecutionContext
 
-case class MergedFromIdsRequest(annotationIds: Seq[ObjectId], ownerIds: Seq[ObjectId])
+case class MergedFromIdsRequest(annotationIds: Seq[ObjectId],
+                                ownerIds: Seq[ObjectId],
+                                additionalBoundingBoxes: Seq[NamedBoundingBox])
 
 object MergedFromIdsRequest {
   implicit val jsonFormat: OFormat[MergedFromIdsRequest] = Json.format[MergedFromIdsRequest]
@@ -243,7 +245,11 @@ class TSAnnotationController @Inject()(
                                                                       toTemporaryStore) ?~> "mergeVolumeData.failed"
             mergedVolumeOpt <- Fox.runIf(volumeTracings.nonEmpty)(
               volumeTracingService
-                .merge(volumeTracings, mergedVolumeStats, newMappingName, newVersion = newTargetVersion)
+                .merge(volumeTracings,
+                       mergedVolumeStats,
+                       newMappingName,
+                       newVersion = newTargetVersion,
+                       additionalBoundingBoxes = request.body.additionalBoundingBoxes)
                 .toFox) ?~> "mergeVolume.failed"
             _ <- Fox.runOptional(mergedVolumeOpt)(
               volumeTracingService.saveVolume(newVolumeId, version = newTargetVersion, _, toTemporaryStore))
@@ -254,7 +260,11 @@ class TSAnnotationController @Inject()(
             }
             skeletonTracings = skeletonTracingsAdaptedNested.flatten
             mergedSkeletonOpt <- Fox.runIf(skeletonTracings.nonEmpty)(
-              skeletonTracingService.merge(skeletonTracings, newVersion = newTargetVersion).toFox)
+              skeletonTracingService
+                .merge(skeletonTracings,
+                       newVersion = newTargetVersion,
+                       additionalBoundingBoxes = request.body.additionalBoundingBoxes)
+                .toFox)
             _ <- Fox.runOptional(mergedSkeletonOpt)(
               skeletonTracingService
                 .saveSkeleton(newSkeletonId, version = newTargetVersion, _, toTemporaryStore = toTemporaryStore))
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -144,7 +144,11 @@ class VolumeTracingController @Inject()(
           val shouldCreateSegmentIndex = volumeSegmentIndexService.shouldCreateSegmentIndexForMerged(tracingsFlat)
           for {
             mergedTracingRaw <- volumeTracingService
-              .merge(tracingsFlat, MergedVolumeStats.empty(shouldCreateSegmentIndex), None, newVersion = 0L)
+              .merge(tracingsFlat,
+                     MergedVolumeStats.empty(shouldCreateSegmentIndex),
+                     None,
+                     newVersion = 0L,
+                     additionalBoundingBoxes = Seq.empty)
               .toFox
             // segment lists for multi-volume uploads are not supported yet, compare https://github.com/scalableminds/webknossos/issues/6887
             mergedTracing = mergedTracingRaw.copy(segments = List.empty)
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/BoundingBoxMerger.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/BoundingBoxMerger.scala
@@ -1,7 +1,7 @@
 package com.scalableminds.webknossos.tracingstore.tracings
 
 import com.scalableminds.util.enumeration.ExtendedEnumeration
-import com.scalableminds.webknossos.datastore.geometry.{NamedBoundingBoxProto => ProtoNamedBoundingBox}
+import com.scalableminds.webknossos.datastore.geometry.NamedBoundingBoxProto
 import com.scalableminds.webknossos.datastore.geometry.{BoundingBoxProto => ProtoBoundingBox}
 import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits
 
@@ -26,18 +26,18 @@ trait BoundingBoxMerger extends ProtoGeometryImplicits {
 
   protected def combineUserBoundingBoxes(singleBoundingBoxAOpt: Option[ProtoBoundingBox],
                                          singleBoundingBoxBOpt: Option[ProtoBoundingBox],
-                                         userBoundingBoxesA: Seq[ProtoNamedBoundingBox],
-                                         userBoundingBoxesB: Seq[ProtoNamedBoundingBox],
-  ): (Seq[ProtoNamedBoundingBox], UserBboxIdMap, UserBboxIdMap) = {
+                                         userBoundingBoxesA: Seq[NamedBoundingBoxProto],
+                                         userBoundingBoxesB: Seq[NamedBoundingBoxProto],
+  ): (Seq[NamedBoundingBoxProto], UserBboxIdMap, UserBboxIdMap) = {
     // note that the singleBoundingBox field is deprecated but still supported here to avoid database evolutions
     val singleBoundingBoxANamed =
-      singleBoundingBoxAOpt.map(bb => (ProtoNamedBoundingBox(0, boundingBox = bb), BoundingBoxSource.singleA))
+      singleBoundingBoxAOpt.map(bb => (NamedBoundingBoxProto(0, boundingBox = bb), BoundingBoxSource.singleA))
     val singleBoundingBoxBNamed =
-      singleBoundingBoxBOpt.map(bb => (ProtoNamedBoundingBox(0, boundingBox = bb), BoundingBoxSource.singleB))
+      singleBoundingBoxBOpt.map(bb => (NamedBoundingBoxProto(0, boundingBox = bb), BoundingBoxSource.singleB))
 
-    val allBoundingBoxes: Seq[(ProtoNamedBoundingBox, BoundingBoxSource.Value)] = userBoundingBoxesA.map(
+    val allBoundingBoxes: Seq[(NamedBoundingBoxProto, BoundingBoxSource.Value)] = userBoundingBoxesA.map(
       (_, BoundingBoxSource.userA)) ++ userBoundingBoxesB.map((_, BoundingBoxSource.userB)) ++ singleBoundingBoxANamed ++ singleBoundingBoxBNamed
-    val newBoxesAndPrevIds: Seq[(ProtoNamedBoundingBox, BoundingBoxSource.Value, Int)] =
+    val newBoxesAndPrevIds: Seq[(NamedBoundingBoxProto, BoundingBoxSource.Value, Int)] =
       allBoundingBoxes.distinctBy(_._1.copy(id = 0)).zipWithIndex.map {
         case ((uBB, source), idx) => (uBB.copy(id = idx), source, uBB.id)
       }
@@ -54,4 +54,14 @@ trait BoundingBoxMerger extends ProtoGeometryImplicits {
     (newBoxes, idMapA, idMapB)
   }
 
+  protected def addAdditionalBoundingBoxes(
+      originalBoundingBoxes: Seq[NamedBoundingBoxProto],
+      additionalBoundingBoxes: Seq[NamedBoundingBox]): Seq[NamedBoundingBoxProto] = {
+    val idOffset = originalBoundingBoxes.map(_.id).maxOption.getOrElse(-1) + 1
+    val additionalAdapted = additionalBoundingBoxes.zipWithIndex.map {
+      case (bb, idx) => bb.copy(id = idx + idOffset).toProto
+    }
+    originalBoundingBoxes ++ additionalAdapted
+  }
+
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+### Changed`
	`2`	`+- Compound annotations (merged views of finished task annotations) now show the task bounding boxes.`