Skip to content

Commit 63f1b69

Browse files
frcrothfrcroth
committed
Implement rudimentary upload of datasets to S3
1 parent 903acef commit 63f1b69

File tree

11 files changed

+403
-42
lines changed

11 files changed

+403
-42
lines changed

‎app/controllers/DatasetController.scala‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,20 +7,17 @@ import com.scalableminds.util.objectid.ObjectId
77
import com.scalableminds.util.time.Instant
88
import com.scalableminds.util.tools.{Fox, TristateOptionJsonHelper}
99
import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate
10-
import com.scalableminds.webknossos.datastore.models.datasource.{DataSource, ElementClass}
10+
import com.scalableminds.webknossos.datastore.models.datasource.ElementClass
1111
import mail.{MailchimpClient, MailchimpTag}
1212
import models.analytics.{AnalyticsService, ChangeDatasetSettingsEvent, OpenDatasetEvent}
1313
import models.dataset._
14-
import models.dataset.explore.{
15-
ExploreAndAddRemoteDatasetParameters,
16-
WKExploreRemoteLayerParameters,
17-
WKExploreRemoteLayerService
18-
}
14+
import models.dataset.explore.{ExploreAndAddRemoteDatasetParameters, WKExploreRemoteLayerParameters, WKExploreRemoteLayerService}
1915
import models.folder.FolderService
2016
import models.organization.OrganizationDAO
2117
import models.team.{TeamDAO, TeamService}
2218
import models.user.{User, UserDAO, UserService}
2319
import com.scalableminds.util.tools.{Empty, Failure, Full}
20+
import com.scalableminds.webknossos.datastore.services.DataSourceRegistrationInfo
2421
import play.api.i18n.{Messages, MessagesProvider}
2522
import play.api.libs.functional.syntax._
2623
import play.api.libs.json._
@@ -71,12 +68,6 @@ object SegmentAnythingMaskParameters {
7168
implicit val jsonFormat: Format[SegmentAnythingMaskParameters] = Json.format[SegmentAnythingMaskParameters]
7269
}
7370

74-
case class DataSourceRegistrationInfo(dataSource: DataSource, folderId: Option[String], dataStoreName: String)
75-
76-
object DataSourceRegistrationInfo {
77-
implicit val jsonFormat: OFormat[DataSourceRegistrationInfo] = Json.format[DataSourceRegistrationInfo]
78-
}
79-
8071
class DatasetController @Inject()(userService: UserService,
8172
userDAO: UserDAO,
8273
datasetService: DatasetService,

‎app/controllers/WKRemoteDataStoreController.scala‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import com.scalableminds.webknossos.datastore.helpers.{LayerMagLinkInfo, MagLink
99
import com.scalableminds.webknossos.datastore.models.UnfinishedUpload
1010
import com.scalableminds.webknossos.datastore.models.datasource.{AbstractDataLayer, DataSource, DataSourceId}
1111
import com.scalableminds.webknossos.datastore.models.datasource.inbox.{InboxDataSourceLike => InboxDataSource}
12-
import com.scalableminds.webknossos.datastore.services.{DataSourcePathInfo, DataStoreStatus}
12+
import com.scalableminds.webknossos.datastore.services.{DataSourcePathInfo, DataSourceRegistrationInfo, DataStoreStatus}
1313
import com.scalableminds.webknossos.datastore.services.uploading.{
1414
LinkedLayerIdentifier,
1515
ReserveAdditionalInformation,
@@ -317,6 +317,14 @@ class WKRemoteDataStoreController @Inject()(
317317
"organization.notFound",
318318
organizationId) ~> NOT_FOUND
319319
_ <- Fox.fromBool(organization._id == user._organization) ?~> "notAllowed" ~> FORBIDDEN
320+
existingDatasetOpt <- Fox.fromFuture(
321+
datasetDAO
322+
.findOneByDirectoryNameAndOrganization(directoryName, organization._id)(GlobalAccessContext)
323+
.toFutureOption)
324+
// Uploading creates an unusable dataset first, here we delete it if it exists.
325+
_ <- existingDatasetOpt
326+
.map(existingDataset => datasetDAO.deleteDataset(existingDataset._id, onlyMarkAsDeleted = false))
327+
.getOrElse(Fox.successful(()))
320328
dataset <- datasetService.createVirtualDataset(
321329
directoryName,
322330
dataStore,

‎conf/webknossos.latest.routes‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ PUT /datastores/:name/datasources/paths
124124
GET /datastores/:name/datasources/:datasetId/paths controllers.WKRemoteDataStoreController.getPaths(name: String, key: String, datasetId: ObjectId)
125125
GET /datastores/:name/datasources/:datasetId controllers.WKRemoteDataStoreController.getDataSource(name: String, key: String, datasetId: ObjectId)
126126
PUT /datastores/:name/datasources/:datasetId controllers.WKRemoteDataStoreController.updateDataSource(name: String, key: String, datasetId: ObjectId, allowNewPaths: Boolean)
127+
POST /datastores/:name/datasources/:organizationId/:directoryName controllers.WKRemoteDataStoreController.registerDataSource(name: String, key: String, organizationId: String, directoryName: String, token: String)
127128
PATCH /datastores/:name/status controllers.WKRemoteDataStoreController.statusUpdate(name: String, key: String)
128129
POST /datastores/:name/reserveUpload controllers.WKRemoteDataStoreController.reserveDatasetUpload(name: String, key: String, token: String)
129130
GET /datastores/:name/getUnfinishedUploadsForUser controllers.WKRemoteDataStoreController.getUnfinishedUploadsForUser(name: String, key: String, token: String, organizationName: String)
@@ -287,7 +288,7 @@ GET /jobs/:id/export
287288

288289
# AI Models
289290
POST /aiModels/runNeuronModelTraining controllers.AiModelController.runNeuronTraining
290-
POST /aiModels/runInstanceModelTraining controllers.AiModelController.runInstanceTraining
291+
POST /aiModels/runInstanceModelTraining controllers.AiModelController.runInstanceTraining
291292
POST /aiModels/inferences/runCustomNeuronModelInference controllers.AiModelController.runCustomNeuronInference
292293
POST /aiModels/inferences/runCustomInstanceModelInference controllers.AiModelController.runCustomInstanceModelInference
293294
GET /aiModels/inferences/:id controllers.AiModelController.readAiInferenceInfo(id: ObjectId)

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreConfig.scala‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,13 @@ class DataStoreConfig @Inject()(configuration: Configuration) extends ConfigRead
6060
object DataVaults {
6161
val credentials: List[Config] = getList[Config]("datastore.dataVaults.credentials")
6262
}
63+
object S3Upload {
64+
val enabled: Boolean = get[Boolean]("datastore.s3Upload.enabled")
65+
val endpoint: String = get[String]("datastore.s3Upload.endpoint")
66+
val bucketName: String = get[String]("datastore.s3Upload.bucketName")
67+
val objectKeyPrefix: String = get[String]("datastore.s3Upload.objectKeyPrefix")
68+
val credentialName: String = get[String]("datastore.s3Upload.credentialName")
69+
}
6370
val children = List(WebKnossos, WatchFileSystem, Cache, AdHocMesh, Redis, AgglomerateSkeleton)
6471
}
6572

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import com.scalableminds.util.objectid.ObjectId
77
import com.scalableminds.util.time.Instant
88
import com.scalableminds.util.tools.{Box, Empty, Failure, Fox, FoxImplicits, Full}
99
import com.scalableminds.webknossos.datastore.ListOfLong.ListOfLong
10+
import com.scalableminds.webknossos.datastore.datavault.S3DataVault
1011
import com.scalableminds.webknossos.datastore.explore.{
1112
ExploreRemoteDatasetRequest,
1213
ExploreRemoteDatasetResponse,
@@ -196,6 +197,17 @@ class DataSourceController @Inject()(
196197
totalChunkCount,
197198
chunkNumber,
198199
new File(chunkFile.ref.path.toString))
200+
201+
/*_ <- uploadService.handleUploadChunkAws(
202+
uploadFileId,
203+
chunkSize,
204+
currentChunkSize,
205+
totalChunkCount,
206+
chunkNumber,
207+
new File(chunkFile.ref.path.toString),
208+
"webknossos-test",
209+
s"upload-tests/upload-test-${uploadFileId}",
210+
)*/
199211
} yield Ok
200212
}
201213
} yield result

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/Encoding.scala‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,12 @@ object Encoding extends ExtendedEnumeration {
1212
// List of possible entries: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding
1313
def fromRfc7231String(s: String): Box[Encoding] =
1414
s match {
15-
case "gzip" => Full(gzip)
16-
case "x-gzip" => Full(gzip)
17-
case "br" => Full(brotli)
18-
case "identity" => Full(identity)
19-
case "" => Full(identity)
20-
case _ => Failure(s"Unsupported encoding: $s")
15+
case "gzip" => Full(gzip)
16+
case "x-gzip" => Full(gzip)
17+
case "br" => Full(brotli)
18+
case "identity" => Full(identity)
19+
case "" => Full(identity)
20+
case "aws-chunked" => Full(identity) // TODO: Does this work?
21+
case _ => Failure(s"Unsupported encoding: $s")
2122
}
2223
}

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/DirectoryConstants.scala‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,5 @@ trait DirectoryConstants {
44
val forConversionDir = ".forConversion"
55
val trashDir = ".trash"
66
val uploadingDir: String = ".uploading"
7+
val uploadToS3Dir = ".cloud"
78
}

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala‎

Lines changed: 22 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -412,7 +412,8 @@ trait DataLayerWithMagLocators extends DataLayer {
412412
defaultViewConfigurationMapping: Option[LayerViewConfiguration] => Option[LayerViewConfiguration] = l => l,
413413
magMapping: MagLocator => MagLocator = m => m,
414414
name: String = this.name,
415-
coordinateTransformations: Option[List[CoordinateTransformation]] = this.coordinateTransformations)
415+
coordinateTransformations: Option[List[CoordinateTransformation]] = this.coordinateTransformations,
416+
attachmentMapping: DatasetLayerAttachments => DatasetLayerAttachments = a => a)
416417
: DataLayerWithMagLocators =
417418
this match {
418419
case l: ZarrDataLayer =>
@@ -421,79 +422,89 @@ trait DataLayerWithMagLocators extends DataLayer {
421422
defaultViewConfiguration = defaultViewConfigurationMapping(l.defaultViewConfiguration),
422423
mags = l.mags.map(magMapping),
423424
name = name,
424-
coordinateTransformations = coordinateTransformations
425+
coordinateTransformations = coordinateTransformations,
426+
attachments = l.attachments.map(attachmentMapping)
425427
)
426428
case l: ZarrSegmentationLayer =>
427429
l.copy(
428430
boundingBox = boundingBoxMapping(l.boundingBox),
429431
defaultViewConfiguration = defaultViewConfigurationMapping(l.defaultViewConfiguration),
430432
mags = l.mags.map(magMapping),
431433
name = name,
432-
coordinateTransformations = coordinateTransformations
434+
coordinateTransformations = coordinateTransformations,
435+
attachments = l.attachments.map(attachmentMapping)
433436
)
434437
case l: N5DataLayer =>
435438
l.copy(
436439
boundingBox = boundingBoxMapping(l.boundingBox),
437440
defaultViewConfiguration = defaultViewConfigurationMapping(l.defaultViewConfiguration),
438441
mags = l.mags.map(magMapping),
439442
name = name,
440-
coordinateTransformations = coordinateTransformations
443+
coordinateTransformations = coordinateTransformations,
444+
attachments = l.attachments.map(attachmentMapping)
441445
)
442446
case l: N5SegmentationLayer =>
443447
l.copy(
444448
boundingBox = boundingBoxMapping(l.boundingBox),
445449
defaultViewConfiguration = defaultViewConfigurationMapping(l.defaultViewConfiguration),
446450
mags = l.mags.map(magMapping),
447451
name = name,
448-
coordinateTransformations = coordinateTransformations
452+
coordinateTransformations = coordinateTransformations,
453+
attachments = l.attachments.map(attachmentMapping)
449454
)
450455
case l: PrecomputedDataLayer =>
451456
l.copy(
452457
boundingBox = boundingBoxMapping(l.boundingBox),
453458
defaultViewConfiguration = defaultViewConfigurationMapping(l.defaultViewConfiguration),
454459
mags = l.mags.map(magMapping),
455460
name = name,
456-
coordinateTransformations = coordinateTransformations
461+
coordinateTransformations = coordinateTransformations,
462+
attachments = l.attachments.map(attachmentMapping)
457463
)
458464
case l: PrecomputedSegmentationLayer =>
459465
l.copy(
460466
boundingBox = boundingBoxMapping(l.boundingBox),
461467
defaultViewConfiguration = defaultViewConfigurationMapping(l.defaultViewConfiguration),
462468
mags = l.mags.map(magMapping),
463469
name = name,
464-
coordinateTransformations = coordinateTransformations
470+
coordinateTransformations = coordinateTransformations,
471+
attachments = l.attachments.map(attachmentMapping)
465472
)
466473
case l: Zarr3DataLayer =>
467474
l.copy(
468475
boundingBox = boundingBoxMapping(l.boundingBox),
469476
defaultViewConfiguration = defaultViewConfigurationMapping(l.defaultViewConfiguration),
470477
mags = l.mags.map(magMapping),
471478
name = name,
472-
coordinateTransformations = coordinateTransformations
479+
coordinateTransformations = coordinateTransformations,
480+
attachments = l.attachments.map(attachmentMapping)
473481
)
474482
case l: Zarr3SegmentationLayer =>
475483
l.copy(
476484
boundingBox = boundingBoxMapping(l.boundingBox),
477485
defaultViewConfiguration = defaultViewConfigurationMapping(l.defaultViewConfiguration),
478486
mags = l.mags.map(magMapping),
479487
name = name,
480-
coordinateTransformations = coordinateTransformations
488+
coordinateTransformations = coordinateTransformations,
489+
attachments = l.attachments.map(attachmentMapping)
481490
)
482491
case l: WKWDataLayer =>
483492
l.copy(
484493
boundingBox = boundingBoxMapping(l.boundingBox),
485494
defaultViewConfiguration = defaultViewConfigurationMapping(l.defaultViewConfiguration),
486495
mags = l.mags.map(magMapping),
487496
name = name,
488-
coordinateTransformations = coordinateTransformations
497+
coordinateTransformations = coordinateTransformations,
498+
attachments = l.attachments.map(attachmentMapping)
489499
)
490500
case l: WKWSegmentationLayer =>
491501
l.copy(
492502
boundingBox = boundingBoxMapping(l.boundingBox),
493503
defaultViewConfiguration = defaultViewConfigurationMapping(l.defaultViewConfiguration),
494504
mags = l.mags.map(magMapping),
495505
name = name,
496-
coordinateTransformations = coordinateTransformations
506+
coordinateTransformations = coordinateTransformations,
507+
attachments = l.attachments.map(attachmentMapping)
497508
)
498509
case _ => throw new Exception("Encountered unsupported layer format")
499510
}

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebknossosClient.scala‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,12 @@ object MagPathInfo {
5050
implicit val jsonFormat: OFormat[MagPathInfo] = Json.format[MagPathInfo]
5151
}
5252

53+
case class DataSourceRegistrationInfo(dataSource: DataSource, folderId: Option[String], dataStoreName: String)
54+
55+
object DataSourceRegistrationInfo {
56+
implicit val jsonFormat: OFormat[DataSourceRegistrationInfo] = Json.format[DataSourceRegistrationInfo]
57+
}
58+
5359
trait RemoteWebknossosClient {
5460
def requestUserAccess(accessRequest: UserAccessRequest)(implicit tc: TokenContext): Fox[UserAccessAnswer]
5561
}
@@ -144,6 +150,19 @@ class DSRemoteWebknossosClient @Inject()(
144150
.withTokenFromContext
145151
.putJson(dataSource)
146152

153+
def registerDataSource(dataSource: DataSource, dataSourceId: DataSourceId, folderId: Option[String])(
154+
implicit tc: TokenContext): Fox[ObjectId] =
155+
for {
156+
_ <- Fox.successful(())
157+
info = DataSourceRegistrationInfo(dataSource, folderId, dataStoreName)
158+
response <- rpc(
159+
s"$webknossosUri/api/datastores/$dataStoreName/datasources/${dataSourceId.organizationId}/${dataSourceId.directoryName}")
160+
.addQueryString("key" -> dataStoreKey)
161+
.withTokenFromContext
162+
.postJson[DataSourceRegistrationInfo](info)
163+
datasetId <- ObjectId.fromString(response.body)
164+
} yield datasetId
165+
147166
def deleteDataSource(id: DataSourceId): Fox[_] =
148167
rpc(s"$webknossosUri/api/datastores/$dataStoreName/deleteDataset")
149168
.addQueryString("key" -> dataStoreKey)

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -365,6 +365,54 @@ class DataSourceService @Inject()(
365365
}
366366
}
367367

368+
// Replace relative paths with absolute paths
369+
// TODO: Rename method
370+
def replacePaths(dataSource: InboxDataSource, newBasePath: String): Fox[DataSource] = {
371+
val replaceUri = (uri: URI) => {
372+
val isRelativeFilePath = (uri.getScheme == null || uri.getScheme.isEmpty || uri.getScheme == DataVaultService.schemeFile) && !uri.isAbsolute
373+
uri.getPath match {
374+
// TODO: Does this make sense?
375+
case pathStr if isRelativeFilePath =>
376+
new URI(uri.getScheme,
377+
uri.getUserInfo,
378+
uri.getHost,
379+
uri.getPort,
380+
newBasePath + pathStr,
381+
uri.getQuery,
382+
uri.getFragment)
383+
case _ => uri
384+
}
385+
}
386+
387+
dataSource.toUsable match {
388+
case Some(usableDataSource) =>
389+
val updatedDataLayers = usableDataSource.dataLayers.map {
390+
case layerWithMagLocators: DataLayerWithMagLocators =>
391+
layerWithMagLocators.mapped(
392+
identity,
393+
identity,
394+
mag =>
395+
mag.path match {
396+
case Some(pathStr) => mag.copy(path = Some(replaceUri(new URI(pathStr)).toString))
397+
case _ => mag
398+
},
399+
attachmentMapping = attachment =>
400+
DatasetLayerAttachments(
401+
attachment.meshes.map(a => a.copy(path = replaceUri(a.path))),
402+
attachment.agglomerates.map(a => a.copy(path = replaceUri(a.path))),
403+
attachment.segmentIndex.map(a => a.copy(path = replaceUri(a.path))),
404+
attachment.connectomes.map(a => a.copy(path = replaceUri(a.path))),
405+
attachment.cumsum.map(a => a.copy(path = replaceUri(a.path)))
406+
)
407+
)
408+
case layer => layer
409+
}
410+
Fox.successful(usableDataSource.copy(dataLayers = updatedDataLayers))
411+
case None =>
412+
Fox.failure("Cannot replace paths of unusable datasource")
413+
}
414+
}
415+
368416
private def scanForAttachedFiles(dataSourcePath: Path, dataSource: DataSource) =
369417
dataSource.dataLayers.map(dataLayer => {
370418
val dataLayerPath = dataSourcePath.resolve(dataLayer.name)

0 commit comments

Comments
 (0)