Make Realpath Scans More Resilient

app/controllers/WKRemoteDataStoreController.scala

@@ -193,7 +193,7 @@ class WKRemoteDataStoreController @Inject()(
       }
     }
 
-  def updatePaths(name: String, key: String): Action[List[DataSourcePathInfo]] =
+  def updateRealPaths(name: String, key: String): Action[List[DataSourcePathInfo]] =
     Action.async(validateJson[List[DataSourcePathInfo]]) { implicit request =>
       dataStoreService.validateAccess(name, key) { _ =>
         for {

app/models/dataset/Dataset.scala

@@ -31,7 +31,7 @@ import com.scalableminds.webknossos.datastore.models.datasource.{
   ThinPlateSplineCorrespondences,
   DataLayerAttachments => AttachmentWrapper
 }
-import com.scalableminds.webknossos.datastore.services.MagPathInfo
+import com.scalableminds.webknossos.datastore.services.RealPathInfo
 import com.scalableminds.webknossos.schema.Tables._
 import controllers.DatasetUpdateParameters
 
@@ -824,16 +824,15 @@ class DatasetMagsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionConte
     replaceSequentiallyAsTransaction(clearQuery, insertQueries)
   }
 
-  def updateMagPathsForDataset(datasetId: ObjectId, magPathInfos: List[MagPathInfo]): Fox[Unit] =
+  // Note: also see attachments
+  def updateMagRealPathsForDataset(datasetId: ObjectId, realPathInfos: Seq[RealPathInfo]): Fox[Unit] =
     for {
       _ <- Fox.successful(())
-      updateQueries = magPathInfos.map(magPathInfo => {
-        val magLiteral = s"(${magPathInfo.mag.x}, ${magPathInfo.mag.y}, ${magPathInfo.mag.z})"
+      updateQueries = realPathInfos.map(realPathInfo => {
         q"""UPDATE webknossos.dataset_mags
-                 SET path = ${magPathInfo.path}, realPath = ${magPathInfo.realPath}, hasLocalData = ${magPathInfo.hasLocalData}
-                 WHERE _dataset = $datasetId
-                  AND dataLayerName = ${magPathInfo.layerName}
-                  AND mag = CAST($magLiteral AS webknossos.vector3)""".asUpdate
+            SET realPath = ${realPathInfo.realPath}, hasLocalData = ${realPathInfo.hasLocalData}
+            WHERE _dataset = $datasetId
+            AND path = ${realPathInfo.path}""".asUpdate
       })
       composedQuery = DBIO.sequence(updateQueries)
       _ <- run(
@@ -1130,7 +1129,8 @@ class DatasetLayerAttachmentsDAO @Inject()(sqlClient: SqlClient)(implicit ec: Ex
 
   def findAllForDatasetAndDataLayerName(datasetId: ObjectId, layerName: String): Fox[AttachmentWrapper] =
     for {
-      rows <- run(q"""SELECT _dataset, layerName, name, path, type, dataFormat, uploadToPathIsPending
+      rows <- run(
+        q"""SELECT _dataset, layerName, name, path, realpath, hasLocalData, type, dataFormat, uploadToPathIsPending
                 FROM webknossos.dataset_layer_attachments
                 WHERE _dataset = $datasetId
                 AND layerName = $layerName
@@ -1170,6 +1170,24 @@ class DatasetLayerAttachmentsDAO @Inject()(sqlClient: SqlClient)(implicit ec: Ex
     replaceSequentiallyAsTransaction(clearQuery, insertQueries)
   }
 
+  // Note: also see mags.
+  def updateAttachmentRealPathsForDataset(datasetId: ObjectId, realPathInfos: Seq[RealPathInfo]): Fox[Unit] =
+    for {
+      _ <- Fox.successful(())
+      updateQueries = realPathInfos.map(realPathInfo => {
+        q"""UPDATE webknossos.dataset_layer_attachments
+            SET realPath = ${realPathInfo.realPath}, hasLocalData = ${realPathInfo.hasLocalData}
+            WHERE _dataset = $datasetId
+            AND path = ${realPathInfo.path}""".asUpdate
+      })
+      composedQuery = DBIO.sequence(updateQueries)
+      _ <- run(
+        composedQuery.transactionally.withTransactionIsolation(Serializable),
+        retryCount = 50,
+        retryIfErrorContains = List(transactionSerializationError)
+      )
+    } yield ()
+
   def insertPending(datasetId: ObjectId,
                     layerName: String,
                     attachmentName: String,

app/models/dataset/DatasetService.scala

@@ -42,6 +42,7 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
                                datasetLastUsedTimesDAO: DatasetLastUsedTimesDAO,
                                datasetDataLayerDAO: DatasetLayerDAO,
                                datasetMagsDAO: DatasetMagsDAO,
+                               datasetLayerAttachmentsDAO: DatasetLayerAttachmentsDAO,
                                teamDAO: TeamDAO,
                                folderDAO: FolderDAO,
                                multiUserDAO: MultiUserDAO,
@@ -470,26 +471,26 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
       _ <- datasetDAO.updateUploader(dataset._id, Some(_uploader)) ?~> "dataset.uploader.forbidden"
     } yield ()
 
-  private def updateRealPath(pathInfo: DataSourcePathInfo)(implicit ctx: DBAccessContext): Fox[Unit] =
-    if (pathInfo.magPathInfos.isEmpty) {
-      Fox.successful(())
-    } else {
-      val dataset = datasetDAO.findOneByDataSourceId(pathInfo.dataSourceId).shiftBox
-      dataset.flatMap {
-        case Full(dataset) if !dataset.isVirtual =>
-          datasetMagsDAO.updateMagPathsForDataset(dataset._id, pathInfo.magPathInfos)
-        case Full(_) => // Dataset is virtual, no updates from datastore are accepted.
-          Fox.successful(())
-        case Empty => // Dataset reported but ignored (non-existing/forbidden org)
-          Fox.successful(())
-        case e: EmptyBox =>
-          Fox.failure("dataset.notFound", e)
-      }
+  private def updateRealPathsForDataSource(pathInfo: DataSourcePathInfo)(implicit ctx: DBAccessContext): Fox[Unit] = {
+    val datasetBox = datasetDAO.findOneByDataSourceId(pathInfo.dataSourceId).shiftBox
+    datasetBox.flatMap {
+      case Full(dataset) if !dataset.isVirtual =>
+        for {
+          _ <- datasetMagsDAO.updateMagRealPathsForDataset(dataset._id, pathInfo.magPathInfos)
+          _ <- datasetLayerAttachmentsDAO.updateAttachmentRealPathsForDataset(dataset._id, pathInfo.attachmentPathInfos)
+        } yield ()
+      case Full(_) => // Dataset is virtual, no updates from datastore are accepted.
+        Fox.successful(())
+      case Empty => // Dataset reported but ignored (non-existing/forbidden org)
+        Fox.successful(())
+      case e: EmptyBox =>
+        Fox.failure("dataset.notFound", e)
     }
+  }
 
   def updateRealPaths(pathInfos: List[DataSourcePathInfo])(implicit ctx: DBAccessContext): Fox[Unit] =
     for {
-      _ <- Fox.serialCombined(pathInfos)(updateRealPath)
+      _ <- Fox.serialCombined(pathInfos)(updateRealPathsForDataSource)
     } yield ()
 
   /**

app/models/dataset/DatasetUploadToPathsService.scala

@@ -173,7 +173,7 @@ class DatasetUploadToPathsService @Inject()(datasetService: DatasetService,
     } yield layerUpdated
 
   private def addPathToMag(mag: MagLocator, layerPath: UPath): MagLocator =
-    mag.copy(path = Some(layerPath / mag.mag.toMagLiteral()))
+    mag.copy(path = Some(layerPath / mag.mag.toMagLiteral(allowScalar = true)))
 
   private def addPathsToAttachments(attachmentsOpt: Option[DataLayerAttachments], layerPath: UPath)(
       implicit ec: ExecutionContext): Fox[Option[DataLayerAttachments]] =

app/models/dataset/WKRemoteDataStoreClient.scala

@@ -40,7 +40,7 @@ class WKRemoteDataStoreClient(dataStore: DataStore, rpc: RPC) extends LazyLoggin
       s"Thumbnail called for: ${dataset._id}, organization: ${dataset._organization}, directoryName: ${dataset.directoryName}, Layer: $dataLayerName")
     rpc(s"${dataStore.url}/data/datasets/${dataset._id}/layers/$dataLayerName/thumbnail.jpg")
       .addQueryParam("token", RpcTokenHolder.webknossosToken)
-      .addQueryParam("mag", mag.toMagLiteral())
+      .addQueryParam("mag", mag.toMagLiteral(allowScalar = false))
       .addQueryParam("x", mag1BoundingBox.topLeft.x)
       .addQueryParam("y", mag1BoundingBox.topLeft.y)
       .addQueryParam("z", mag1BoundingBox.topLeft.z)

conf/evolutions/145-attachment-realpaths.sql

@@ -0,0 +1,10 @@
+START TRANSACTION;
+
+do $$ begin if (select schemaVersion from webknossos.releaseInformation) <> 144 then raise exception 'Previous schema version mismatch'; end if; end; $$ language plpgsql;
+
+ALTER TABLE webknossos.dataset_layer_attachments ADD COLUMN realPath TEXT;
+ALTER TABLE webknossos.dataset_layer_attachments ADD COLUMN hasLocalData BOOLEAN NOT NULL DEFAULT false;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 145;
+
+COMMIT TRANSACTION;

conf/evolutions/reversions/145-attachment-realpaths.sql

@@ -0,0 +1,10 @@
+START TRANSACTION;
+
+do $$ begin if (select schemaVersion from webknossos.releaseInformation) <> 145 then raise exception 'Previous schema version mismatch'; end if; end; $$ language plpgsql;
+
+ALTER TABLE webknossos.dataset_layer_attachments DROP COLUMN realPath;
+ALTER TABLE webknossos.dataset_layer_attachments DROP COLUMN hasLocalData;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 144;
+
+COMMIT TRANSACTION;

conf/webknossos.latest.routes

@@ -125,7 +125,7 @@ DELETE        /folders/:id
 GET           /datastores                                                                       controllers.DataStoreController.list()
 PUT           /datastores/:name/datasource                                                      controllers.WKRemoteDataStoreController.updateOne(name: String, key: String)
 PUT           /datastores/:name/datasources                                                     controllers.WKRemoteDataStoreController.updateAll(name: String, key: String, organizationId: Option[String])
-PUT           /datastores/:name/datasources/paths                                               controllers.WKRemoteDataStoreController.updatePaths(name: String, key: String)
+PUT           /datastores/:name/datasources/paths                                               controllers.WKRemoteDataStoreController.updateRealPaths(name: String, key: String)
 GET           /datastores/:name/datasources/:datasetId/paths                                    controllers.WKRemoteDataStoreController.getPaths(name: String, key: String, datasetId: ObjectId)
 GET           /datastores/:name/datasources/:datasetId                                          controllers.WKRemoteDataStoreController.getDataSource(name: String, key: String, datasetId: ObjectId)
 PUT           /datastores/:name/datasources/:datasetId                                          controllers.WKRemoteDataStoreController.updateDataSource(name: String, key: String, datasetId: ObjectId)

tools/postgres/schema.sql

@@ -21,7 +21,7 @@ CREATE TABLE webknossos.releaseInformation (
   schemaVersion BIGINT NOT NULL
 );
 
-INSERT INTO webknossos.releaseInformation(schemaVersion) values(144);
+INSERT INTO webknossos.releaseInformation(schemaVersion) values(145);
 COMMIT TRANSACTION;
 
 
@@ -166,14 +166,16 @@ CREATE TABLE webknossos.dataset_layer_additionalAxes(
 CREATE TYPE webknossos.LAYER_ATTACHMENT_TYPE AS ENUM ('agglomerate', 'connectome', 'segmentIndex', 'mesh', 'cumsum');
 CREATE TYPE webknossos.LAYER_ATTACHMENT_DATAFORMAT AS ENUM ('hdf5', 'zarr3', 'json', 'neuroglancerPrecomputed');
 CREATE TABLE webknossos.dataset_layer_attachments(
-   _dataset TEXT CONSTRAINT _dataset_objectId CHECK (_dataset ~ '^[0-9a-f]{24}$') NOT NULL,
-   layerName TEXT NOT NULL,
-   name TEXT NOT NULL,
-   path TEXT NOT NULL,
-   type webknossos.LAYER_ATTACHMENT_TYPE NOT NULL,
-   dataFormat webknossos.LAYER_ATTACHMENT_DATAFORMAT NOT NULL,
-   uploadToPathIsPending BOOLEAN NOT NULL DEFAULT FALSE,
-   PRIMARY KEY(_dataset, layerName, name, type)
+  _dataset TEXT CONSTRAINT _dataset_objectId CHECK (_dataset ~ '^[0-9a-f]{24}$') NOT NULL,
+  layerName TEXT NOT NULL,
+  name TEXT NOT NULL,
+  path TEXT NOT NULL,
+  realPath TEXT,
+  hasLocalData BOOLEAN NOT NULL DEFAULT FALSE,
+  type webknossos.LAYER_ATTACHMENT_TYPE NOT NULL,
+  dataFormat webknossos.LAYER_ATTACHMENT_DATAFORMAT NOT NULL,
+  uploadToPathIsPending BOOLEAN NOT NULL DEFAULT FALSE,
+  PRIMARY KEY(_dataset, layerName, name, type)
 );
 
 CREATE TABLE webknossos.dataset_allowedTeams(

unreleased_changes/9019.md

@@ -0,0 +1,8 @@
+### Added
+- Added registering attachment realpaths next to mag realpaths.
+
+### Fixed
+- Fixed that scanning realpaths of dataset mags would stop if a single mag didn’t exist.
+
+### Postgres Evolutions
+- [145-attachment-realpaths.sql](conf/evolutions/145-attachment-realpaths.sql)

util/src/main/scala/com/scalableminds/util/geometry/Vec3Int.scala

@@ -37,7 +37,7 @@ case class Vec3Int(x: Int, y: Int, z: Int) {
 
   override def toString: String = s"($x, $y, $z)"
 
-  def toMagLiteral(allowScalar: Boolean = false): String =
+  def toMagLiteral(allowScalar: Boolean): String =
     if (allowScalar && isIsotropic) s"$x" else s"$x-$y-$z"
 
   def toUriLiteral: String = s"$x,$y,$z"

webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSLegacyApiController.scala

@@ -555,7 +555,8 @@ class DSLegacyApiController @Inject()(
       // Dataset is not present in DB. This can be because reload was called after a dataset was written into the directory
       val dataSource = dataSourceService.dataSourceFromDir(
         dataSourceService.dataBaseDir.resolve(organizationId).resolve(datasetDirectoryName),
-        organizationId)
+        organizationId,
+        resolveMagPaths = true)
       dataSource match {
         case UsableDataSource(_, _, _, _, _) =>
           for {

webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala

@@ -708,7 +708,9 @@ class DataSourceController @Inject()(
         Some(
           dataSourceService.dataSourceFromDir(
             dataSourceService.dataBaseDir.resolve(dataSourceId.organizationId).resolve(dataSourceId.directoryName),
-            dataSourceId.organizationId))
+            dataSourceId.organizationId,
+            resolveMagPaths = true
+          ))
       } else None
       _ <- Fox.runOptional(dataSourceFromDirOpt)(ds => dsRemoteWebknossosClient.updateDataSource(ds, datasetId))
       _ = datasetCache.invalidateCache(datasetId)

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebknossosClient.scala

@@ -5,7 +5,6 @@ import com.google.inject.Inject
 import com.google.inject.name.Named
 import com.scalableminds.util.accesscontext.TokenContext
 import com.scalableminds.util.cache.AlfuCache
-import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.objectid.ObjectId
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.DataStoreConfig
@@ -38,16 +37,20 @@ object TracingStoreInfo {
   implicit val jsonFormat: OFormat[TracingStoreInfo] = Json.format[TracingStoreInfo]
 }
 
-case class DataSourcePathInfo(dataSourceId: DataSourceId, magPathInfos: List[MagPathInfo])
+case class DataSourcePathInfo(dataSourceId: DataSourceId,
+                              magPathInfos: Seq[RealPathInfo],
+                              attachmentPathInfos: Seq[RealPathInfo]) {
+  def nonEmpty: Boolean = magPathInfos.nonEmpty || attachmentPathInfos.nonEmpty
+}
 
 object DataSourcePathInfo {
   implicit val jsonFormat: OFormat[DataSourcePathInfo] = Json.format[DataSourcePathInfo]
 }
 
-case class MagPathInfo(layerName: String, mag: Vec3Int, path: UPath, realPath: UPath, hasLocalData: Boolean)
+case class RealPathInfo(path: UPath, realPath: UPath, hasLocalData: Boolean)
 
-object MagPathInfo {
-  implicit val jsonFormat: OFormat[MagPathInfo] = Json.format[MagPathInfo]
+object RealPathInfo {
+  implicit val jsonFormat: OFormat[RealPathInfo] = Json.format[RealPathInfo]
 }
 
 trait RemoteWebknossosClient {
@@ -108,7 +111,7 @@ class DSRemoteWebknossosClient @Inject()(
       .silent
       .putJson(dataSources)
 
-  def reportRealPaths(dataSourcePaths: List[DataSourcePathInfo]): Fox[_] =
+  def reportRealPaths(dataSourcePaths: Seq[DataSourcePathInfo]): Fox[_] =
     rpc(s"$webknossosUri/api/datastores/$dataStoreName/datasources/paths")
       .addQueryParam("key", dataStoreKey)
       .silent