Skip to content

Commit b5641d8

Browse files
lmarinilmarini
authored
Merge branch 'develop' into 232-upload-file-to-folder
2 parents f65d505 + b97b1c1 commit b5641d8

File tree

7 files changed

+198
-140
lines changed

7 files changed

+198
-140
lines changed

‎CHANGELOG.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,18 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
55
and this project adheres to [Semantic Versioning](http://semver.org/).
66

77
## Unreleased
8+
- Added folder and folder id to api datasets files list [#34](https://github.com/clowder-framework/clowder/issues/34)
89

910
### Fixed
1011
- When uploading a file, it would ignore any extractors marked disabled at the space level. [#246](https://github.com/clowder-framework/clowder/issues/246)
1112
- Added index for comments, will speed up index creation
13+
- If using S3 storage in docker, it was not reflected correctly in the docker-compose file
14+
15+
### Added
16+
- Status endpoint will now show what storage is used
17+
18+
### Fixed
19+
- Docker image for mongo-init now based on python:3.7-slim reduces size
1220

1321
## 1.17.0 - 2021-04-29
1422

‎app/api/Datasets.scala‎

Lines changed: 80 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,34 @@
11
package api
22

3-
import java.io._
4-
import java.io.{File => JFile}
5-
import java.net.URL
6-
import java.security.{DigestInputStream, MessageDigest}
7-
import java.text.SimpleDateFormat
8-
import java.util.{Calendar, Date}
9-
3+
import _root_.util._
104
import api.Permission.Permission
11-
import java.util.zip._
12-
13-
import javax.inject.{Inject, Singleton}
5+
import controllers.Utils.https
146
import controllers.{Previewers, Utils}
157
import jsonutils.JsonUtil
168
import models._
179
import org.apache.commons.codec.binary.Hex
1810
import org.json.JSONObject
1911
import play.api.Logger
20-
import play.api.Play.{configuration, current, routes}
12+
import play.api.Play.{configuration, current}
2113
import play.api.i18n.Messages
14+
import play.api.libs.Files
2215
import play.api.libs.concurrent.Execution.Implicits._
2316
import play.api.libs.iteratee.Enumerator
24-
import play.api.libs.json._
2517
import play.api.libs.json.Json._
26-
import play.api.mvc.{Action, AnyContent, MultipartFormData, SimpleResult}
27-
import services._
28-
import _root_.util._
29-
import controllers.Utils.https
30-
import org.json.simple.{JSONArray, JSONObject => SimpleJSONObject}
31-
import org.json.simple.parser.JSONParser
32-
import play.api.libs.Files
33-
import play.api.libs.Files.TemporaryFile
18+
import play.api.libs.json._
19+
import play.api.mvc.{AnyContent, MultipartFormData, SimpleResult}
3420
import scalax.file.Path.createTempFile
21+
import services._
3522

36-
import scala.concurrent.{ExecutionContext, Future}
23+
import java.io._
24+
import java.net.URL
25+
import java.security.{DigestInputStream, MessageDigest}
26+
import java.text.SimpleDateFormat
27+
import java.util.zip._
28+
import java.util.{Calendar, Date}
29+
import javax.inject.{Inject, Singleton}
3730
import scala.collection.mutable.{ListBuffer, Map => MutaMap}
31+
import scala.concurrent.{ExecutionContext, Future}
3832

3933
/**
4034
* Dataset API.
@@ -933,6 +927,7 @@ class Datasets @Inject()(
933927
}
934928
}
935929

930+
936931
def getMetadataDefinitions(id: UUID, currentSpace: Option[String]) = PermissionAction(Permission.AddMetadata, Some(ResourceRef(ResourceRef.dataset, id))) { implicit request =>
937932
implicit val user = request.user
938933
datasets.get(id) match {
@@ -1134,47 +1129,84 @@ class Datasets @Inject()(
11341129
private def getFilesWithinFolders(id: UUID, serveradmin: Boolean=false, max: Int = -1): List[JsValue] = {
11351130
val output = new ListBuffer[JsValue]()
11361131
var resultCount = 0
1132+
var current_folder : Option[Folder] = None
11371133
datasets.get(id) match {
11381134
case Some(dataset) => {
1139-
folders.findByParentDatasetId(id).map { folder =>
1135+
folders.findByParentDatasetId(id).foreach { folder =>
1136+
current_folder = Some(folder)
11401137
files.get(folder.files).found.foreach(file => {
11411138
if (max < 0 || resultCount < max) {
1142-
output += jsonFile(file, serveradmin)
1139+
output += jsonFile(file, serveradmin, Some(folder))
11431140
resultCount += 1
11441141
}
11451142
})
1143+
print("done with folder")
11461144
}
11471145
}
11481146
case None => Logger.error(s"Error getting dataset $id")
11491147
}
11501148
output.toList
11511149
}
11521150

1153-
def jsonFile(file: models.File, serverAdmin: Boolean = false): JsValue = {
1154-
val defaultMap = Map(
1155-
"id" -> file.id.toString,
1156-
"filename" -> file.filename,
1157-
"contentType" -> file.contentType,
1158-
"date-created" -> file.uploadDate.toString(),
1159-
"size" -> file.length.toString)
1160-
1161-
// Only include filepath if using DiskByte storage and user is serverAdmin
1162-
val jsonMap = file.loader match {
1163-
case "services.filesystem.DiskByteStorageService" => {
1164-
if (serverAdmin)
1165-
Map(
1166-
"id" -> file.id.toString,
1167-
"filename" -> file.filename,
1168-
"filepath" -> file.loader_id,
1169-
"contentType" -> file.contentType,
1170-
"date-created" -> file.uploadDate.toString(),
1171-
"size" -> file.length.toString)
1172-
else
1173-
defaultMap
1174-
}
1175-
case _ => defaultMap
1176-
}
1177-
toJson(jsonMap)
1151+
1152+
def jsonFile(file: models.File, serverAdmin: Boolean = false, folder : Option[Folder] = None): JsValue = {
1153+
folder match {
1154+
case Some(f) => {
1155+
val folderMap : JsValue = Json.obj("id"->f.id, "name"->f.name)
1156+
val defaultMap : JsValue = Json.obj(
1157+
"id" -> file.id.toString,
1158+
"filename" -> file.filename,
1159+
"contentType" -> file.contentType,
1160+
"date-created" -> file.uploadDate.toString(),
1161+
"folders"->folderMap,
1162+
"size" -> file.length.toString)
1163+
1164+
// Only include filepath if using DiskByte storage and user is serverAdmin
1165+
val jsonMap = file.loader match {
1166+
case "services.filesystem.DiskByteStorageService" => {
1167+
if (serverAdmin)
1168+
Json.obj(
1169+
"id" -> file.id.toString,
1170+
"filename" -> file.filename,
1171+
"filepath" -> file.loader_id,
1172+
"contentType" -> file.contentType,
1173+
"date-created" -> file.uploadDate.toString(),
1174+
"folders"->folderMap,
1175+
"size" -> file.length.toString)
1176+
else
1177+
defaultMap
1178+
}
1179+
case _ => defaultMap
1180+
}
1181+
toJson(jsonMap)
1182+
}
1183+
case None => {
1184+
val defaultMap = Map(
1185+
"id" -> file.id.toString,
1186+
"filename" -> file.filename,
1187+
"contentType" -> file.contentType,
1188+
"date-created" -> file.uploadDate.toString(),
1189+
"size" -> file.length.toString)
1190+
1191+
// Only include filepath if using DiskByte storage and user is serverAdmin
1192+
val jsonMap = file.loader match {
1193+
case "services.filesystem.DiskByteStorageService" => {
1194+
if (serverAdmin)
1195+
Map(
1196+
"id" -> file.id.toString,
1197+
"filename" -> file.filename,
1198+
"filepath" -> file.loader_id,
1199+
"contentType" -> file.contentType,
1200+
"date-created" -> file.uploadDate.toString(),
1201+
"size" -> file.length.toString)
1202+
else
1203+
defaultMap
1204+
}
1205+
case _ => defaultMap
1206+
}
1207+
toJson(jsonMap)
1208+
}
1209+
}
11781210
}
11791211

11801212
//Update Dataset Information code starts

‎app/api/Status.scala‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
package api
22

33
import javax.inject.Inject
4-
5-
import play.api.Logger
4+
import play.api.{Logger, Play}
65
import models.User
76
import play.api.Play._
87
import play.api.libs.json.{JsValue, Json}
98
import services._
109
import services.mongodb.MongoSalatPlugin
10+
import services.s3.S3ByteStorageService
1111

1212
import scala.collection.mutable
1313

@@ -35,6 +35,7 @@ class Status @Inject()(spaces: SpaceService,
3535
"version" -> getVersionInfo,
3636
"counts" -> getCounts(request.user),
3737
"plugins" -> getPlugins(request.user),
38+
"storage" -> getStorage(request.user),
3839
"extractors" -> Json.toJson(extractors.getExtractorNames(List.empty))))
3940
}
4041

@@ -138,6 +139,35 @@ class Status @Inject()(spaces: SpaceService,
138139
Json.toJson(result.toMap[String, JsValue])
139140
}
140141

142+
def getStorage(user: Option[User]): JsValue = {
143+
val config = Play.current.configuration
144+
val result = new mutable.HashMap[String, String]()
145+
146+
ByteStorageService.storage.getClass.getName match {
147+
case "services.mongodb.MongoDBByteStorage" => {
148+
result.put("location", "mongo")
149+
}
150+
case "services.filesystem.DiskByteStorageService" => {
151+
result.put("location", "disk")
152+
if (Permission.checkServerAdmin(user)) {
153+
result.put("path", config.getString("clowder.diskStorage.path").getOrElse[String]("unknown"))
154+
}
155+
}
156+
case "services.s3.S3ByteStorageService" => {
157+
result.put("location", "s3")
158+
if (Permission.checkServerAdmin(user)) {
159+
result.put("endpoint", config.getString(S3ByteStorageService.ServiceEndpoint).getOrElse[String]("unknown"))
160+
result.put("region", config.getString(S3ByteStorageService.Region).getOrElse[String]("unknown"))
161+
result.put("bucket", config.getString(S3ByteStorageService.BucketName).getOrElse[String]("unknown"))
162+
}
163+
}
164+
case name => {
165+
result.put("location", name)
166+
}
167+
}
168+
Json.toJson(result.toMap[String, String])
169+
}
170+
141171
def getCounts(user: Option[User]): JsValue = {
142172
val counts = appConfig.getIndexCounts()
143173
// TODO: Revisit this check as it is currently too slow

0 commit comments

Comments
 (0)