Skip to content

Commit ee1b820

Browse files
tcnicholtcnichol
committed
Merge remote-tracking branch 'origin/232-upload-file-to-folder' into 232-upload-file-to-folder
2 parents cb941ea + b5641d8 commit ee1b820

File tree

9 files changed

+203
-144
lines changed

9 files changed

+203
-144
lines changed

‎CHANGELOG.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,18 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
55
and this project adheres to [Semantic Versioning](http://semver.org/).
66

77
## Unreleased
8+
- Added folder and folder id to api datasets files list [#34](https://github.com/clowder-framework/clowder/issues/34)
89

910
### Fixed
11+
- When uploading a file, it would ignore any extractors marked disabled at the space level. [#246](https://github.com/clowder-framework/clowder/issues/246)
1012
- Added index for comments, will speed up index creation
13+
- If using S3 storage in docker, it was not reflected correctly in the docker-compose file
14+
15+
### Added
16+
- Status endpoint will now show what storage is used
17+
18+
### Fixed
19+
- Docker image for mongo-init now based on python:3.7-slim reduces size
1120

1221
### Added
1322
- Endpoint '/api/files/uploadToDataset' now allows folder_id for uploading file to folder. [#232](https://github.com/clowder-framework/clowder/issues/232)

‎CONTRIBUTING.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
# How to contribute
22

33
We would love to get contributions from you! To keep the process manageable we please ask you to follow the workflow
4-
below. By participating in this project, you agree to abide by the [code of conduct](https://clowderframework.org/docs/Clowder-CoC.pdf).
5-
Before your code can be accepted, you will have to sign a [CLA](https://clowderframework.org/docs/Clowder-CLA.pdf)
4+
below. By participating in this project, you agree to abide by the [code of conduct](https://clowderframework.org/pdf/Clowder-CoC.pdf).
5+
Before your code can be accepted, you will have to sign a [CLA](https://clowderframework.org/pdf/Clowder-CLA.pdf)
66
and mail it to [email protected].
77

88
Most of the core development happens on [NCSA Bitbucket][bitbucket]. The `master` and `develop` branches are pushed to
@@ -81,4 +81,4 @@ Developer documentation is available in [Confluence][confluence]. User documenta
8181
[bitbucket]: https://opensource.ncsa.illinois.edu/bitbucket/projects/CATS
8282
[bamboo]: https://opensource.ncsa.illinois.edu/bamboo/browse/CATS
8383
[userdocs]: https://clowderframework.org/docs/
84-
[confluence]: https://opensource.ncsa.illinois.edu/confluence/display/CATS/Home
84+
[confluence]: https://opensource.ncsa.illinois.edu/confluence/display/CATS/Home

‎app/api/Datasets.scala‎

Lines changed: 80 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,34 @@
11
package api
22

3-
import java.io._
4-
import java.io.{File => JFile}
5-
import java.net.URL
6-
import java.security.{DigestInputStream, MessageDigest}
7-
import java.text.SimpleDateFormat
8-
import java.util.{Calendar, Date}
9-
3+
import _root_.util._
104
import api.Permission.Permission
11-
import java.util.zip._
12-
13-
import javax.inject.{Inject, Singleton}
5+
import controllers.Utils.https
146
import controllers.{Previewers, Utils}
157
import jsonutils.JsonUtil
168
import models._
179
import org.apache.commons.codec.binary.Hex
1810
import org.json.JSONObject
1911
import play.api.Logger
20-
import play.api.Play.{configuration, current, routes}
12+
import play.api.Play.{configuration, current}
2113
import play.api.i18n.Messages
14+
import play.api.libs.Files
2215
import play.api.libs.concurrent.Execution.Implicits._
2316
import play.api.libs.iteratee.Enumerator
24-
import play.api.libs.json._
2517
import play.api.libs.json.Json._
26-
import play.api.mvc.{Action, AnyContent, MultipartFormData, SimpleResult}
27-
import services._
28-
import _root_.util._
29-
import controllers.Utils.https
30-
import org.json.simple.{JSONArray, JSONObject => SimpleJSONObject}
31-
import org.json.simple.parser.JSONParser
32-
import play.api.libs.Files
33-
import play.api.libs.Files.TemporaryFile
18+
import play.api.libs.json._
19+
import play.api.mvc.{AnyContent, MultipartFormData, SimpleResult}
3420
import scalax.file.Path.createTempFile
21+
import services._
3522

36-
import scala.concurrent.{ExecutionContext, Future}
23+
import java.io._
24+
import java.net.URL
25+
import java.security.{DigestInputStream, MessageDigest}
26+
import java.text.SimpleDateFormat
27+
import java.util.zip._
28+
import java.util.{Calendar, Date}
29+
import javax.inject.{Inject, Singleton}
3730
import scala.collection.mutable.{ListBuffer, Map => MutaMap}
31+
import scala.concurrent.{ExecutionContext, Future}
3832

3933
/**
4034
* Dataset API.
@@ -933,6 +927,7 @@ class Datasets @Inject()(
933927
}
934928
}
935929

930+
936931
def getMetadataDefinitions(id: UUID, currentSpace: Option[String]) = PermissionAction(Permission.AddMetadata, Some(ResourceRef(ResourceRef.dataset, id))) { implicit request =>
937932
implicit val user = request.user
938933
datasets.get(id) match {
@@ -1134,47 +1129,84 @@ class Datasets @Inject()(
11341129
private def getFilesWithinFolders(id: UUID, serveradmin: Boolean=false, max: Int = -1): List[JsValue] = {
11351130
val output = new ListBuffer[JsValue]()
11361131
var resultCount = 0
1132+
var current_folder : Option[Folder] = None
11371133
datasets.get(id) match {
11381134
case Some(dataset) => {
1139-
folders.findByParentDatasetId(id).map { folder =>
1135+
folders.findByParentDatasetId(id).foreach { folder =>
1136+
current_folder = Some(folder)
11401137
files.get(folder.files).found.foreach(file => {
11411138
if (max < 0 || resultCount < max) {
1142-
output += jsonFile(file, serveradmin)
1139+
output += jsonFile(file, serveradmin, Some(folder))
11431140
resultCount += 1
11441141
}
11451142
})
1143+
print("done with folder")
11461144
}
11471145
}
11481146
case None => Logger.error(s"Error getting dataset $id")
11491147
}
11501148
output.toList
11511149
}
11521150

1153-
def jsonFile(file: models.File, serverAdmin: Boolean = false): JsValue = {
1154-
val defaultMap = Map(
1155-
"id" -> file.id.toString,
1156-
"filename" -> file.filename,
1157-
"contentType" -> file.contentType,
1158-
"date-created" -> file.uploadDate.toString(),
1159-
"size" -> file.length.toString)
1160-
1161-
// Only include filepath if using DiskByte storage and user is serverAdmin
1162-
val jsonMap = file.loader match {
1163-
case "services.filesystem.DiskByteStorageService" => {
1164-
if (serverAdmin)
1165-
Map(
1166-
"id" -> file.id.toString,
1167-
"filename" -> file.filename,
1168-
"filepath" -> file.loader_id,
1169-
"contentType" -> file.contentType,
1170-
"date-created" -> file.uploadDate.toString(),
1171-
"size" -> file.length.toString)
1172-
else
1173-
defaultMap
1174-
}
1175-
case _ => defaultMap
1176-
}
1177-
toJson(jsonMap)
1151+
1152+
def jsonFile(file: models.File, serverAdmin: Boolean = false, folder : Option[Folder] = None): JsValue = {
1153+
folder match {
1154+
case Some(f) => {
1155+
val folderMap : JsValue = Json.obj("id"->f.id, "name"->f.name)
1156+
val defaultMap : JsValue = Json.obj(
1157+
"id" -> file.id.toString,
1158+
"filename" -> file.filename,
1159+
"contentType" -> file.contentType,
1160+
"date-created" -> file.uploadDate.toString(),
1161+
"folders"->folderMap,
1162+
"size" -> file.length.toString)
1163+
1164+
// Only include filepath if using DiskByte storage and user is serverAdmin
1165+
val jsonMap = file.loader match {
1166+
case "services.filesystem.DiskByteStorageService" => {
1167+
if (serverAdmin)
1168+
Json.obj(
1169+
"id" -> file.id.toString,
1170+
"filename" -> file.filename,
1171+
"filepath" -> file.loader_id,
1172+
"contentType" -> file.contentType,
1173+
"date-created" -> file.uploadDate.toString(),
1174+
"folders"->folderMap,
1175+
"size" -> file.length.toString)
1176+
else
1177+
defaultMap
1178+
}
1179+
case _ => defaultMap
1180+
}
1181+
toJson(jsonMap)
1182+
}
1183+
case None => {
1184+
val defaultMap = Map(
1185+
"id" -> file.id.toString,
1186+
"filename" -> file.filename,
1187+
"contentType" -> file.contentType,
1188+
"date-created" -> file.uploadDate.toString(),
1189+
"size" -> file.length.toString)
1190+
1191+
// Only include filepath if using DiskByte storage and user is serverAdmin
1192+
val jsonMap = file.loader match {
1193+
case "services.filesystem.DiskByteStorageService" => {
1194+
if (serverAdmin)
1195+
Map(
1196+
"id" -> file.id.toString,
1197+
"filename" -> file.filename,
1198+
"filepath" -> file.loader_id,
1199+
"contentType" -> file.contentType,
1200+
"date-created" -> file.uploadDate.toString(),
1201+
"size" -> file.length.toString)
1202+
else
1203+
defaultMap
1204+
}
1205+
case _ => defaultMap
1206+
}
1207+
toJson(jsonMap)
1208+
}
1209+
}
11781210
}
11791211

11801212
//Update Dataset Information code starts

‎app/api/Status.scala‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
package api
22

33
import javax.inject.Inject
4-
5-
import play.api.Logger
4+
import play.api.{Logger, Play}
65
import models.User
76
import play.api.Play._
87
import play.api.libs.json.{JsValue, Json}
98
import services._
109
import services.mongodb.MongoSalatPlugin
10+
import services.s3.S3ByteStorageService
1111

1212
import scala.collection.mutable
1313

@@ -35,6 +35,7 @@ class Status @Inject()(spaces: SpaceService,
3535
"version" -> getVersionInfo,
3636
"counts" -> getCounts(request.user),
3737
"plugins" -> getPlugins(request.user),
38+
"storage" -> getStorage(request.user),
3839
"extractors" -> Json.toJson(extractors.getExtractorNames(List.empty))))
3940
}
4041

@@ -138,6 +139,35 @@ class Status @Inject()(spaces: SpaceService,
138139
Json.toJson(result.toMap[String, JsValue])
139140
}
140141

142+
def getStorage(user: Option[User]): JsValue = {
143+
val config = Play.current.configuration
144+
val result = new mutable.HashMap[String, String]()
145+
146+
ByteStorageService.storage.getClass.getName match {
147+
case "services.mongodb.MongoDBByteStorage" => {
148+
result.put("location", "mongo")
149+
}
150+
case "services.filesystem.DiskByteStorageService" => {
151+
result.put("location", "disk")
152+
if (Permission.checkServerAdmin(user)) {
153+
result.put("path", config.getString("clowder.diskStorage.path").getOrElse[String]("unknown"))
154+
}
155+
}
156+
case "services.s3.S3ByteStorageService" => {
157+
result.put("location", "s3")
158+
if (Permission.checkServerAdmin(user)) {
159+
result.put("endpoint", config.getString(S3ByteStorageService.ServiceEndpoint).getOrElse[String]("unknown"))
160+
result.put("region", config.getString(S3ByteStorageService.Region).getOrElse[String]("unknown"))
161+
result.put("bucket", config.getString(S3ByteStorageService.BucketName).getOrElse[String]("unknown"))
162+
}
163+
}
164+
case name => {
165+
result.put("location", name)
166+
}
167+
}
168+
Json.toJson(result.toMap[String, String])
169+
}
170+
141171
def getCounts(user: Option[User]): JsValue = {
142172
val counts = appConfig.getIndexCounts()
143173
// TODO: Revisit this check as it is currently too slow

‎app/services/ExtractorRoutingService.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ class ExtractorRoutingService {
7575
private def getMatchingExtractors(extractorIds: List[String], operation: String, resourceType: ResourceType.Value): List[String] = {
7676
val extractorsService = DI.injector.getInstance(classOf[ExtractorService])
7777

78-
extractorsService.getEnabledExtractors().flatMap(exId =>
78+
extractorIds.flatMap(exId =>
7979
extractorsService.getExtractorInfo(exId)).filter(exInfo =>
8080
resourceType match {
8181
case ResourceType.dataset =>

0 commit comments

Comments
 (0)