Use user token when requesting from HttpsDataVault on same DS (#8322)

Author: frcroth

CHANGELOG.unreleased.md

@@ -13,6 +13,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 ### Added
 
 ### Changed
+- When using a zarr link to a wk-served data layer as another layer’s source, the user’s token is used to access the data. [#8322](https://github.com/scalableminds/webknossos/pull/8322/)
 
 ### Fixed
 - Fixed a bug that would lock a non existing mapping to an empty segmentation layer under certain conditions. [#8401](https://github.com/scalableminds/webknossos/pull/8401)

test/backend/DataVaultTestSuite.scala

@@ -1,5 +1,6 @@
 package backend
 
+import com.scalableminds.util.accesscontext.TokenContext
 import com.scalableminds.util.tools.Fox
 import org.scalatestplus.play.PlaySpec
 
@@ -26,6 +27,8 @@ import scala.concurrent.ExecutionContext.{global => globalExecutionContext}
 class DataVaultTestSuite extends PlaySpec {
 
   val handleFoxJustification = "Handling Fox in Unit Test Context"
+  val emptyTokenContext: TokenContext = TokenContext(None)
+  val dummyDataStoreHost = "example.com"
 
   "Data vault" when {
     "using Range requests" when {
@@ -36,10 +39,11 @@ class DataVaultTestSuite extends PlaySpec {
         "return correct response" in {
           WsTestClient.withClient { ws =>
             val uri = new URI("http://storage.googleapis.com/")
-            val vaultPath = new VaultPath(uri, HttpsDataVault.create(RemoteSourceDescriptor(uri, None), ws))
+            val vaultPath =
+              new VaultPath(uri, HttpsDataVault.create(RemoteSourceDescriptor(uri, None), ws, dummyDataStoreHost))
             val bytes =
               (vaultPath / s"neuroglancer-fafb-data/fafb_v14/fafb_v14_orig/$dataKey")
-                .readBytes(Some(range))(globalExecutionContext)
+                .readBytes(Some(range))(globalExecutionContext, emptyTokenContext)
                 .get(handleFoxJustification)
 
             assert(bytes.length == range.length)
@@ -53,7 +57,9 @@ class DataVaultTestSuite extends PlaySpec {
         val vaultPath = new VaultPath(uri, GoogleCloudDataVault.create(RemoteSourceDescriptor(uri, None)))
         "return correct response" in {
 
-          val bytes = (vaultPath / dataKey).readBytes(Some(range))(globalExecutionContext).get(handleFoxJustification)
+          val bytes = (vaultPath / dataKey)
+            .readBytes(Some(range))(globalExecutionContext, emptyTokenContext)
+            .get(handleFoxJustification)
 
           assert(bytes.length == range.length)
           assert(bytes.take(10).sameElements(Array(-1, -40, -1, -32, 0, 16, 74, 70, 73, 70)))
@@ -63,15 +69,15 @@ class DataVaultTestSuite extends PlaySpec {
           "requesting a non-existent object" in {
             val result =
               (vaultPath / s"non-existent-key${UUID.randomUUID}")
-                .readBytes()(globalExecutionContext)
+                .readBytes()(globalExecutionContext, emptyTokenContext)
                 .await(handleFoxJustification)
             assertBoxEmpty(result)
           }
         }
         "return failure" when {
           "requesting invalid range" in {
             val result = (vaultPath / dataKey)
-              .readBytes(Some(Range.Long(-5, -10, 1)))(globalExecutionContext)
+              .readBytes(Some(Range.Long(-5, -10, 1)))(globalExecutionContext, emptyTokenContext)
               .await(handleFoxJustification)
             assertBoxFailure(result)
           }
@@ -83,7 +89,7 @@ class DataVaultTestSuite extends PlaySpec {
                                 uri,
                                 Some(GoogleServiceAccountCredential("name", JsString("secret"), "user", "org")))))
             val result = (vaultPath / dataKey)
-              .readBytes(Some(Range.Long(-10, 10, 1)))(globalExecutionContext)
+              .readBytes(Some(Range.Long(-10, 10, 1)))(globalExecutionContext, emptyTokenContext)
               .await(handleFoxJustification)
             assertBoxFailure(result)
           }
@@ -97,7 +103,9 @@ class DataVaultTestSuite extends PlaySpec {
             val vaultPath =
               new VaultPath(uri, S3DataVault.create(RemoteSourceDescriptor(uri, None), ws)(globalExecutionContext))
             val bytes =
-              (vaultPath / "s0/5/5/5").readBytes(Some(range))(globalExecutionContext).get(handleFoxJustification)
+              (vaultPath / "s0/5/5/5")
+                .readBytes(Some(range))(globalExecutionContext, emptyTokenContext)
+                .get(handleFoxJustification)
             assert(bytes.length == range.length)
             assert(bytes.take(10).sameElements(Array(0, 0, 0, 3, 0, 0, 0, 64, 0, 0)))
           }
@@ -113,9 +121,10 @@ class DataVaultTestSuite extends PlaySpec {
         "return correct response" in {
           WsTestClient.withClient { ws =>
             val uri = new URI("http://storage.googleapis.com/")
-            val vaultPath = new VaultPath(uri, HttpsDataVault.create(RemoteSourceDescriptor(uri, None), ws))
+            val vaultPath =
+              new VaultPath(uri, HttpsDataVault.create(RemoteSourceDescriptor(uri, None), ws, dummyDataStoreHost))
             val bytes = (vaultPath / s"neuroglancer-fafb-data/fafb_v14/fafb_v14_orig/$dataKey")
-              .readBytes()(globalExecutionContext)
+              .readBytes()(globalExecutionContext, emptyTokenContext)
               .get(handleFoxJustification)
 
             assert(bytes.length == dataLength)
@@ -128,7 +137,8 @@ class DataVaultTestSuite extends PlaySpec {
         "return correct response" in {
           val uri = new URI("gs://neuroglancer-fafb-data/fafb_v14/fafb_v14_orig")
           val vaultPath = new VaultPath(uri, GoogleCloudDataVault.create(RemoteSourceDescriptor(uri, None)))
-          val bytes = (vaultPath / dataKey).readBytes()(globalExecutionContext).get(handleFoxJustification)
+          val bytes =
+            (vaultPath / dataKey).readBytes()(globalExecutionContext, emptyTokenContext).get(handleFoxJustification)
 
           assert(bytes.length == dataLength)
           assert(bytes.take(10).sameElements(Array(-1, -40, -1, -32, 0, 16, 74, 70, 73, 70)))
@@ -143,7 +153,7 @@ class DataVaultTestSuite extends PlaySpec {
               new VaultPath(uri, S3DataVault.create(RemoteSourceDescriptor(uri, None), ws)(globalExecutionContext))
             val bytes =
               (vaultPath / "33792-34304_29696-30208_3216-3232")
-                .readBytes()(globalExecutionContext)
+                .readBytes()(globalExecutionContext, emptyTokenContext)
                 .get(handleFoxJustification)
             assert(bytes.take(10).sameElements(Array(-87, -95, -85, -94, -101, 124, 115, 100, 113, 111)))
           }
@@ -155,7 +165,8 @@ class DataVaultTestSuite extends PlaySpec {
             WsTestClient.withClient { ws =>
               val s3DataVault = S3DataVault.create(RemoteSourceDescriptor(uri, None), ws)(globalExecutionContext)
               val vaultPath = new VaultPath(uri, s3DataVault)
-              val result = vaultPath.readBytes()(globalExecutionContext).await(handleFoxJustification)
+              val result =
+                vaultPath.readBytes()(globalExecutionContext, emptyTokenContext).await(handleFoxJustification)
               assertBoxEmpty(result)
             }
           }
@@ -167,7 +178,8 @@ class DataVaultTestSuite extends PlaySpec {
             WsTestClient.withClient { ws =>
               val s3DataVault = S3DataVault.create(RemoteSourceDescriptor(uri, None), ws)(globalExecutionContext)
               val vaultPath = new VaultPath(uri, s3DataVault)
-              val result = vaultPath.readBytes()(globalExecutionContext).await(handleFoxJustification)
+              val result =
+                vaultPath.readBytes()(globalExecutionContext, emptyTokenContext).await(handleFoxJustification)
               assertBoxEmpty(result)
             }
           }
@@ -207,7 +219,8 @@ class DataVaultTestSuite extends PlaySpec {
     "using vault path" when {
       class MockDataVault extends DataVault {
         override def readBytesAndEncoding(path: VaultPath, range: RangeSpecifier)(
-            implicit ec: ExecutionContext): Fox[(Array[Byte], Encoding.Value)] = ???
+            implicit ec: ExecutionContext,
+            tc: TokenContext): Fox[(Array[Byte], Encoding.Value)] = ???
 
         override def listDirectory(path: VaultPath,
                                    maxItems: Int)(implicit ec: ExecutionContext): Fox[List[VaultPath]] = ???

webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala

@@ -1,6 +1,7 @@
 package com.scalableminds.webknossos.datastore.controllers
 
 import com.google.inject.Inject
+import com.scalableminds.util.accesscontext.TokenContext
 import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.image.{Color, JPEGWriter}
 import com.scalableminds.util.time.Instant
@@ -102,14 +103,14 @@ class BinaryDataController @Inject()(
                                                                                   datasetDirectoryName,
                                                                                   dataLayerName) ~> NOT_FOUND
         magParsed <- Vec3Int.fromMagLiteral(mag).toFox ?~> "malformedMag"
-        request = DataRequest(
+        dataRequest = DataRequest(
           VoxelPosition(x, y, z, magParsed),
           width,
           height,
           depth,
           DataServiceRequestSettings(halfByte = halfByte, appliedAgglomerate = mappingName)
         )
-        (data, indices) <- requestData(dataSource, dataLayer, request)
+        (data, indices) <- requestData(dataSource, dataLayer, dataRequest)
       } yield Ok(data).withHeaders(createMissingBucketsHeaders(indices): _*)
     }
   }
@@ -147,13 +148,13 @@ class BinaryDataController @Inject()(
         (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId,
                                                                                   datasetDirectoryName,
                                                                                   dataLayerName) ~> NOT_FOUND
-        request = DataRequest(
+        dataRequest = DataRequest(
           VoxelPosition(x * cubeSize * mag, y * cubeSize * mag, z * cubeSize * mag, Vec3Int(mag, mag, mag)),
           cubeSize,
           cubeSize,
           cubeSize
         )
-        (data, indices) <- requestData(dataSource, dataLayer, request)
+        (data, indices) <- requestData(dataSource, dataLayer, dataRequest)
       } yield Ok(data).withHeaders(createMissingBucketsHeaders(indices): _*)
     }
   }
@@ -180,14 +181,14 @@ class BinaryDataController @Inject()(
                                                                                   dataLayerName) ?~> Messages(
           "dataSource.notFound") ~> NOT_FOUND
         magParsed <- Vec3Int.fromMagLiteral(mag).toFox ?~> "malformedMag"
-        request = DataRequest(
+        dataRequest = DataRequest(
           VoxelPosition(x, y, z, magParsed),
           width,
           height,
           depth = 1,
           DataServiceRequestSettings(appliedAgglomerate = mappingName)
         )
-        (data, _) <- requestData(dataSource, dataLayer, request)
+        (data, _) <- requestData(dataSource, dataLayer, dataRequest)
         intensityRange: Option[(Double, Double)] = intensityMin.flatMap(min => intensityMax.map(max => (min, max)))
         layerColor = color.flatMap(Color.fromHTML)
         params = ImageCreatorParameters(
@@ -252,10 +253,11 @@ class BinaryDataController @Inject()(
             request.body.cuboid(dataLayer),
             request.body.segmentId,
             request.body.voxelSizeFactorInUnit,
+            tokenContextForRequest(request),
             request.body.mapping,
             request.body.mappingType,
             request.body.additionalCoordinates,
-            request.body.findNeighbors
+            request.body.findNeighbors,
           )
           // The client expects the ad-hoc mesh as a flat float-array. Three consecutive floats form a 3D point, three
           // consecutive 3D points (i.e., nine floats) form a triangle.
@@ -308,7 +310,7 @@ class BinaryDataController @Inject()(
       dataSource: DataSource,
       dataLayer: DataLayer,
       dataRequests: DataRequestCollection
-  ): Fox[(Array[Byte], List[Int])] = {
+  )(implicit tc: TokenContext): Fox[(Array[Byte], List[Int])] = {
     val requests =
       dataRequests.map(r => DataServiceDataRequest(dataSource, dataLayer, r.cuboid(dataLayer), r.settings))
     binaryDataService.handleDataRequests(requests)

webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala

@@ -261,7 +261,7 @@ class ZarrStreamingController @Inject()(
       dataLayerName: String,
       mag: String,
       coordinates: String,
-  )(implicit m: MessagesProvider): Fox[Result] =
+  )(implicit m: MessagesProvider, tc: TokenContext): Fox[Result] =
     for {
       (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId,
                                                                                 datasetDirectoryName,

webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/BucketProvider.scala

@@ -1,9 +1,11 @@
 package com.scalableminds.webknossos.datastore.dataformats
 
+import com.scalableminds.util.accesscontext.TokenContext
 import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.datastore.models.requests.DataReadInstruction
+
 import scala.concurrent.ExecutionContext
 
 trait BucketProvider {
-  def load(readInstruction: DataReadInstruction)(implicit ec: ExecutionContext): Fox[Array[Byte]]
+  def load(readInstruction: DataReadInstruction)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Array[Byte]]
 }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala

@@ -1,5 +1,6 @@
 package com.scalableminds.webknossos.datastore.dataformats
 
+import com.scalableminds.util.accesscontext.TokenContext
 import com.scalableminds.util.cache.AlfuCache
 import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.time.Instant
@@ -16,6 +17,7 @@ import com.scalableminds.webknossos.datastore.models.requests.DataReadInstructio
 import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService
 import com.typesafe.scalalogging.LazyLogging
 import net.liftweb.common.Empty
+
 import scala.concurrent.duration._
 import ucar.ma2.{Array => MultiArray}
 
@@ -32,7 +34,7 @@ class DatasetArrayBucketProvider(dataLayer: DataLayer,
   // Cache the DatasetArrays of all mags of this layer
   private lazy val datasetArrayCache = AlfuCache[Vec3Int, DatasetArray](maxCapacity = 50)
 
-  def load(readInstruction: DataReadInstruction)(implicit ec: ExecutionContext): Fox[Array[Byte]] =
+  def load(readInstruction: DataReadInstruction)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Array[Byte]] =
     for {
       datasetArray <- datasetArrayCache.getOrLoad(readInstruction.bucket.mag,
                                                   _ => openDatasetArrayWithTimeLogging(readInstruction))
@@ -45,8 +47,8 @@ class DatasetArrayBucketProvider(dataLayer: DataLayer,
                                                                     dataLayer.elementClass == ElementClass.uint24)
     } yield bucketData
 
-  private def openDatasetArrayWithTimeLogging(readInstruction: DataReadInstruction)(
-      implicit ec: ExecutionContext): Fox[DatasetArray] = {
+  private def openDatasetArrayWithTimeLogging(
+      readInstruction: DataReadInstruction)(implicit ec: ExecutionContext, tc: TokenContext): Fox[DatasetArray] = {
     val before = Instant.now
     for {
       result <- openDatasetArray(readInstruction).futureBox
@@ -59,8 +61,8 @@ class DatasetArrayBucketProvider(dataLayer: DataLayer,
     } yield result
   }
 
-  private def openDatasetArray(readInstruction: DataReadInstruction)(
-      implicit ec: ExecutionContext): Fox[DatasetArray] = {
+  private def openDatasetArray(readInstruction: DataReadInstruction)(implicit ec: ExecutionContext,
+                                                                     tc: TokenContext): Fox[DatasetArray] = {
     val magLocatorOpt: Option[MagLocator] =
       dataLayer.mags.find(_.mag == readInstruction.bucket.mag)
 

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkReader.scala

@@ -1,5 +1,6 @@
 package com.scalableminds.webknossos.datastore.datareaders
 
+import com.scalableminds.util.accesscontext.TokenContext
 import com.scalableminds.util.tools.Fox
 import com.scalableminds.util.tools.Fox.box2Fox
 import com.scalableminds.webknossos.datastore.datavault.VaultPath
@@ -18,7 +19,7 @@ class ChunkReader(header: DatasetHeader) {
   def read(path: VaultPath,
            chunkShapeFromMetadata: Array[Int],
            range: Option[NumericRange[Long]],
-           useSkipTypingShortcut: Boolean)(implicit ec: ExecutionContext): Fox[MultiArray] =
+           useSkipTypingShortcut: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[MultiArray] =
     for {
       chunkBytesAndShapeBox: Box[(Array[Byte], Option[Array[Int]])] <- readChunkBytesAndShape(path, range).futureBox
       chunkShape: Array[Int] = chunkBytesAndShapeBox.toOption.flatMap(_._2).getOrElse(chunkShapeFromMetadata)
@@ -39,7 +40,8 @@ class ChunkReader(header: DatasetHeader) {
   // Returns bytes (optional, Fox.empty may later be replaced with fill value)
   // and chunk shape (optional, only for data formats where each chunk reports its own shape, e.g. N5)
   protected def readChunkBytesAndShape(path: VaultPath, range: Option[NumericRange[Long]])(
-      implicit ec: ExecutionContext): Fox[(Array[Byte], Option[Array[Int]])] =
+      implicit ec: ExecutionContext,
+      tc: TokenContext): Fox[(Array[Byte], Option[Array[Int]])] =
     for {
       bytes <- path.readBytes(range)
       decompressed <- tryo(header.compressorImpl.decompress(bytes)).toFox ?~> "chunk.decompress.failed"