Skip to content

Commit 612eb29

Browse files
max-zillamax-zilla
authored
Merge branch 'develop' into 232-upload-file-to-folder
2 parents ca4d9d8 + e993504 commit 612eb29

File tree

6 files changed

+245
-65
lines changed

6 files changed

+245
-65
lines changed

‎CHANGELOG.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
44
The format is based on [Keep a Changelog](http://keepachangelog.com/)
55
and this project adheres to [Semantic Versioning](http://semver.org/).
66

7+
## Unreleased
8+
### Added
9+
- Add "when" parameter in a few GET API endpoints to enable pagination [#266](https://github.com/clowder-framework/clowder/issues/266)
10+
11+
712
## 1.18.1 - 2021-08-16
813

914
This release fixes a critical issue where invalid zip files could result in the files not being uploaded correctly. To check to see if you are affected, please use the following query:
@@ -19,6 +24,7 @@ If any files are returned, you should check to see if these files affected and a
1924
- Updated swagger documentation
2025
- Return 404 not found when calling file/dataset/space api endpoints with an invalid ID [#251](https://github.com/clowder-framework/clowder/issues/251)
2126
- Line breaks in welcome message breaks swagger build [#187](https://github.com/clowder-framework/clowder/issues/187)
27+
- Collections created using api route are now indexed upon creation. [#257](https://github.com/clowder-framework/clowder/issues/257)
2228

2329
### Changed
2430
- Added more information when writing files to make sure files are written correctly

‎app/api/Collections.scala‎

Lines changed: 70 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,28 @@
11
package api
22

3-
import java.io.{ByteArrayInputStream, InputStream, ByteArrayOutputStream}
4-
import java.security.{DigestInputStream, MessageDigest}
5-
import java.text.SimpleDateFormat
6-
import java.util.zip.{ZipEntry, ZipOutputStream, Deflater}
7-
83
import Iterators.RootCollectionIterator
9-
import _root_.util.JSONLD
4+
import util.SearchUtils
105
import api.Permission.Permission
11-
import org.apache.commons.codec.binary.Hex
6+
import controllers.Utils
7+
import models._
128
import play.api.Logger
139
import play.api.Play.current
14-
import models._
10+
import play.api.libs.concurrent.Execution.Implicits._
1511
import play.api.libs.iteratee.Enumerator
16-
import services._
17-
import play.api.libs.json._
18-
import play.api.libs.json.{JsObject, JsValue}
1912
import play.api.libs.json.Json.toJson
20-
import javax.inject.{ Singleton, Inject}
21-
import scala.collection.mutable.ListBuffer
22-
import scala.concurrent.{Future, ExecutionContext}
23-
import play.api.libs.concurrent.Execution.Implicits._
24-
import scala.util.parsing.json.JSONArray
25-
import scala.util.{Try, Success, Failure}
26-
import java.util.{Calendar, Date}
27-
import controllers.Utils
13+
import play.api.libs.json.{JsObject, JsValue, _}
14+
import services._
2815

16+
17+
import java.io.ByteArrayOutputStream
18+
import java.security.MessageDigest
19+
import java.util.zip.{Deflater, ZipOutputStream}
20+
import java.util.{Calendar, Date}
21+
import javax.inject.{Inject, Singleton}
2922
import scala.collection.immutable.List
23+
import scala.collection.mutable.ListBuffer
24+
import scala.concurrent.{ExecutionContext, Future}
25+
import scala.util.{Failure, Success, Try}
3026

3127

3228
/**
@@ -72,6 +68,13 @@ class Collections @Inject() (datasets: DatasetService,
7268
collections.addToRootSpaces(c.id, s.id)
7369
events.addSourceEvent(request.user, c.id, c.name, s.id, s.name, EventType.ADD_COLLECTION_SPACE.toString)
7470
})
71+
// index collection
72+
current.plugin[ElasticsearchPlugin].foreach{
73+
_.index(SearchUtils.getElasticsearchObject(c))
74+
}
75+
//Add to Events Table
76+
val option_user = userService.findByIdentity(identity)
77+
events.addObjectEvent(option_user, c.id, c.name, EventType.CREATE_COLLECTION.toString)
7578
Ok(toJson(Map("id" -> id)))
7679
}
7780
case None => Ok(toJson(Map("status" -> "error")))
@@ -237,15 +240,15 @@ class Collections @Inject() (datasets: DatasetService,
237240
}
238241
}
239242

240-
def list(title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
241-
Ok(toJson(listCollections(title, date, limit, Set[Permission](Permission.ViewCollection), false, request.user, request.user.fold(false)(_.superAdminMode), exact)))
243+
def list(when: Option[String], title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
244+
Ok(toJson(listCollections(when, title, date, limit, Set[Permission](Permission.ViewCollection), false, request.user, request.user.fold(false)(_.superAdminMode), exact)))
242245
}
243246

244-
def listCanEdit(title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
245-
Ok(toJson(listCollections(title, date, limit, Set[Permission](Permission.AddResourceToCollection, Permission.EditCollection), false, request.user, request.user.fold(false)(_.superAdminMode), exact)))
247+
def listCanEdit(when: Option[String], title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
248+
Ok(toJson(listCollections(when, title, date, limit, Set[Permission](Permission.AddResourceToCollection, Permission.EditCollection), false, request.user, request.user.fold(false)(_.superAdminMode), exact)))
246249
}
247250

248-
def addDatasetToCollectionOptions(datasetId: UUID, title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
251+
def addDatasetToCollectionOptions(when: Option[String], datasetId: UUID, title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
249252
implicit val user = request.user
250253
var listAll = false
251254
var collectionList: List[Collection] = List.empty
@@ -265,7 +268,7 @@ class Collections @Inject() (datasets: DatasetService,
265268
}
266269
}
267270
if(listAll) {
268-
collectionList = listCollections(title, date, limit, Set[Permission](Permission.AddResourceToCollection, Permission.EditCollection), false, request.user, request.user.fold(false)(_.superAdminMode), exact)
271+
collectionList = listCollections(when, title, date, limit, Set[Permission](Permission.AddResourceToCollection, Permission.EditCollection), false, request.user, request.user.fold(false)(_.superAdminMode), exact)
269272
}
270273
Ok(toJson(collectionList))
271274
}
@@ -274,10 +277,10 @@ class Collections @Inject() (datasets: DatasetService,
274277
collections.get(current_collections.map(_.child_collection_ids).flatten).found
275278
}
276279

277-
def listPossibleParents(currentCollectionId : String, title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
280+
def listPossibleParents(when: Option[String], currentCollectionId : String, title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
278281
val selfAndAncestors = collections.getSelfAndAncestors(UUID(currentCollectionId))
279282
val descendants = collections.getAllDescendants(UUID(currentCollectionId)).toList
280-
val allCollections = listCollections(title, date, limit, Set[Permission](Permission.AddResourceToCollection, Permission.EditCollection), false,
283+
val allCollections = listCollections(when, title, date, limit, Set[Permission](Permission.AddResourceToCollection, Permission.EditCollection), false,
281284
request.user, request.user.fold(false)(_.superAdminMode), exact)
282285
val possibleNewParents = allCollections.filter(c =>
283286
if(play.api.Play.current.plugin[services.SpaceSharingPlugin].isDefined) {
@@ -303,29 +306,55 @@ class Collections @Inject() (datasets: DatasetService,
303306
* Returns list of collections based on parameters and permissions.
304307
* TODO this needs to be cleaned up when do permissions for adding to a resource
305308
*/
306-
private def listCollections(title: Option[String], date: Option[String], limit: Int, permission: Set[Permission], mine: Boolean, user: Option[User], superAdmin: Boolean, exact: Boolean) : List[Collection] = {
309+
private def listCollections(when: Option[String], title: Option[String], date: Option[String], limit: Int, permission: Set[Permission], mine: Boolean, user: Option[User], superAdmin: Boolean, exact: Boolean) : List[Collection] = {
307310
if (mine && user.isEmpty) return List.empty[Collection]
308311

309-
(title, date) match {
310-
case (Some(t), Some(d)) => {
312+
(when, title, date) match {
313+
case (Some(w), Some(t), Some(d)) => {
314+
if (mine)
315+
collections.listUser(d, nextPage=(w=="a"), limit, t, user, superAdmin, user.get, exact)
316+
else
317+
collections.listAccess(d, nextPage=(w=="a"), limit, t, permission, user, superAdmin, true,false, exact)
318+
}
319+
case (Some(w), Some(t), None) => {
320+
if (mine)
321+
collections.listUser(limit, t, user, superAdmin, user.get, exact)
322+
else
323+
collections.listAccess(limit, t, permission, user, superAdmin, true,false, exact)
324+
}
325+
case (Some(w), None, Some(d)) => {
326+
if (mine)
327+
collections.listUser(d, nextPage=(w=="a"), limit, user, superAdmin, user.get)
328+
else
329+
collections.listAccess(d, nextPage=(w=="a"), limit, permission, user, superAdmin, true,false)
330+
}
331+
case (Some(w), None, None) => {
332+
if (mine)
333+
collections.listUser(limit, user, superAdmin, user.get)
334+
else
335+
collections.listAccess(limit, permission, user, superAdmin, true,false)
336+
}
337+
338+
// default when to be "after" if not present in parameters. i.e. nextPage=true
339+
case (None, Some(t), Some(d)) => {
311340
if (mine)
312341
collections.listUser(d, true, limit, t, user, superAdmin, user.get, exact)
313342
else
314343
collections.listAccess(d, true, limit, t, permission, user, superAdmin, true,false, exact)
315344
}
316-
case (Some(t), None) => {
345+
case (None, Some(t), None) => {
317346
if (mine)
318347
collections.listUser(limit, t, user, superAdmin, user.get, exact)
319348
else
320349
collections.listAccess(limit, t, permission, user, superAdmin, true,false, exact)
321350
}
322-
case (None, Some(d)) => {
351+
case (None, None, Some(d)) => {
323352
if (mine)
324353
collections.listUser(d, true, limit, user, superAdmin, user.get)
325354
else
326355
collections.listAccess(d, true, limit, permission, user, superAdmin, true,false)
327356
}
328-
case (None, None) => {
357+
case (None, None, None) => {
329358
if (mine)
330359
collections.listUser(limit, user, superAdmin, user.get)
331360
else
@@ -572,6 +601,14 @@ class Collections @Inject() (datasets: DatasetService,
572601
events.addSourceEvent(request.user, c.id, c.name, s.id, s.name, EventType.ADD_COLLECTION_SPACE.toString)
573602
}
574603

604+
// index collection
605+
current.plugin[ElasticsearchPlugin].foreach{
606+
_.index(SearchUtils.getElasticsearchObject(c))
607+
}
608+
//Add to Events Table
609+
val option_user = userService.findByIdentity(identity)
610+
events.addObjectEvent(option_user, c.id, c.name, EventType.CREATE_COLLECTION.toString)
611+
575612
//do stuff with parent here
576613
(request.body \"parentId").asOpt[String] match {
577614
case Some(parentId) => {

‎app/api/Datasets.scala‎

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -65,19 +65,19 @@ class Datasets @Inject()(
6565
}
6666
}
6767

68-
def list(title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
69-
Ok(toJson(listDatasets(title, date, limit, Set[Permission](Permission.ViewDataset), request.user, request.user.fold(false)(_.superAdminMode), exact)))
68+
def list(when: Option[String], title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
69+
Ok(toJson(listDatasets(when, title, date, limit, Set[Permission](Permission.ViewDataset), request.user, request.user.fold(false)(_.superAdminMode), exact)))
7070
}
7171

72-
def listCanEdit(title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
73-
Ok(toJson(listDatasets(title, date, limit, Set[Permission](Permission.AddResourceToDataset, Permission.EditDataset), request.user, request.user.fold(false)(_.superAdminMode), exact)))
72+
def listCanEdit(when: Option[String], title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
73+
Ok(toJson(listDatasets(when, title, date, limit, Set[Permission](Permission.AddResourceToDataset, Permission.EditDataset), request.user, request.user.fold(false)(_.superAdminMode), exact)))
7474
}
7575

7676
def listMoveFileToDataset(file_id: UUID, title: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
7777
if (play.Play.application().configuration().getBoolean("datasetFileWithinSpace")) {
7878
Ok(toJson(listDatasetsInSpace(file_id, title, limit, Set[Permission](Permission.AddResourceToDataset, Permission.EditDataset), request.user, request.user.fold(false)(_.superAdminMode), exact)))
7979
} else {
80-
Ok(toJson(listDatasets(title, None, limit, Set[Permission](Permission.AddResourceToDataset, Permission.EditDataset), request.user, request.user.fold(false)(_.superAdminMode), exact)))
80+
Ok(toJson(listDatasets(None, title, None, limit, Set[Permission](Permission.AddResourceToDataset, Permission.EditDataset), request.user, request.user.fold(false)(_.superAdminMode), exact)))
8181
}
8282
}
8383

@@ -152,18 +152,31 @@ class Datasets @Inject()(
152152
/**
153153
* Returns list of datasets based on parameters and permissions.
154154
*/
155-
private def listDatasets(title: Option[String], date: Option[String], limit: Int, permission: Set[Permission], user: Option[User], superAdmin: Boolean, exact: Boolean) : List[Dataset] = {
156-
(title, date) match {
157-
case (Some(t), Some(d)) => {
155+
private def listDatasets(when: Option[String], title: Option[String], date: Option[String], limit: Int, permission: Set[Permission], user: Option[User], superAdmin: Boolean, exact: Boolean) : List[Dataset] = {
156+
(when, title, date) match {
157+
case (Some(w), Some(t), Some(d)) => {
158+
datasets.listAccess(d, nextPage=(w=="a"), limit, t, permission, user, superAdmin, true,false, exact)
159+
}
160+
case (Some(w), Some(t), None) => {
161+
datasets.listAccess(limit, t, permission, user, superAdmin, true,false, exact)
162+
}
163+
case (Some(w), None, Some(d)) => {
164+
datasets.listAccess(d, nextPage=(w=="a"), limit, permission, user, superAdmin, true,false)
165+
}
166+
case (Some(w), None, None) => {
167+
datasets.listAccess(limit, permission, user, superAdmin, true,false)
168+
}
169+
// default when to be "after" if not present in parameters. i.e. nextPage=true
170+
case (None, Some(t), Some(d)) => {
158171
datasets.listAccess(d, true, limit, t, permission, user, superAdmin, true,false, exact)
159172
}
160-
case (Some(t), None) => {
173+
case (None, Some(t), None) => {
161174
datasets.listAccess(limit, t, permission, user, superAdmin, true,false, exact)
162175
}
163-
case (None, Some(d)) => {
176+
case (None, None, Some(d)) => {
164177
datasets.listAccess(d, true, limit, permission, user, superAdmin, true,false)
165178
}
166-
case (None, None) => {
179+
case (None, None, None) => {
167180
datasets.listAccess(limit, permission, user, superAdmin, true,false)
168181
}
169182
}

‎app/api/Spaces.scala‎

Lines changed: 37 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -86,45 +86,70 @@ class Spaces @Inject()(spaces: SpaceService,
8686
}
8787
}
8888

89-
def list(title: Option[String], date: Option[String], limit: Int) = UserAction(needActive=false) { implicit request =>
90-
Ok(toJson(listSpaces(title, date, limit, Set[Permission](Permission.ViewSpace), false, request.user, request.user.fold(false)(_.superAdminMode), true).map(spaceToJson)))
89+
def list(when: Option[String], title: Option[String], date: Option[String], limit: Int) = UserAction(needActive=false) { implicit request =>
90+
Ok(toJson(listSpaces(when, title, date, limit, Set[Permission](Permission.ViewSpace), false, request.user, request.user.fold(false)(_.superAdminMode), true).map(spaceToJson)))
9191
}
9292

93-
def listCanEdit(title: Option[String], date: Option[String], limit: Int) = UserAction(needActive=true) { implicit request =>
94-
Ok(toJson(listSpaces(title, date, limit, Set[Permission](Permission.AddResourceToSpace, Permission.EditSpace), false, request.user, request.user.fold(false)(_.superAdminMode), true).map(spaceToJson)))
93+
def listCanEdit(when: Option[String], title: Option[String], date: Option[String], limit: Int) = UserAction(needActive=true) { implicit request =>
94+
Ok(toJson(listSpaces(when, title, date, limit, Set[Permission](Permission.AddResourceToSpace, Permission.EditSpace), false, request.user, request.user.fold(false)(_.superAdminMode), true).map(spaceToJson)))
9595
}
9696

97-
def listCanEditNotAlreadyIn(collectionId : UUID, title: Option[String], date: Option[String], limit: Int) = UserAction(needActive=true ){ implicit request =>
98-
Ok(toJson(listSpaces(title, date, limit, Set[Permission](Permission.AddResourceToSpace, Permission.EditSpace), false, request.user, request.user.fold(false)(_.superAdminMode), true).map(spaceToJson)))
97+
def listCanEditNotAlreadyIn(when: Option[String], collectionId : UUID, title: Option[String], date: Option[String], limit: Int) = UserAction(needActive=true ){ implicit request =>
98+
Ok(toJson(listSpaces(when, title, date, limit, Set[Permission](Permission.AddResourceToSpace, Permission.EditSpace), false, request.user, request.user.fold(false)(_.superAdminMode), true).map(spaceToJson)))
9999
}
100100

101101
/**
102102
* Returns list of collections based on parameters and permissions.
103103
* TODO this needs to be cleaned up when do permissions for adding to a resource
104104
*/
105-
private def listSpaces(title: Option[String], date: Option[String], limit: Int, permission: Set[Permission], mine: Boolean, user: Option[User], superAdmin: Boolean, showPublic: Boolean, onlyTrial: Boolean = false) : List[ProjectSpace] = {
105+
private def listSpaces(when: Option[String], title: Option[String], date: Option[String], limit: Int, permission: Set[Permission], mine: Boolean, user: Option[User], superAdmin: Boolean, showPublic: Boolean, onlyTrial: Boolean = false) : List[ProjectSpace] = {
106106
if (mine && user.isEmpty) return List.empty[ProjectSpace]
107107

108-
(title, date) match {
109-
case (Some(t), Some(d)) => {
108+
(when, title, date) match {
109+
case (Some(w), Some(t), Some(d)) => {
110+
if (mine)
111+
spaces.listUser(d, nextPage=(w=="a"), limit, t, user, superAdmin, user.get)
112+
else
113+
spaces.listAccess(d, nextPage=(w=="a"), limit, t, permission, user, superAdmin, showPublic, showOnlyShared = false)
114+
}
115+
case (Some(w), Some(t), None) => {
116+
if (mine)
117+
spaces.listUser(limit, t, user, superAdmin, user.get)
118+
else
119+
spaces.listAccess(limit, t, permission, user, superAdmin, showPublic, showOnlyShared = false)
120+
}
121+
case (Some(w), None, Some(d)) => {
122+
if (mine)
123+
spaces.listUser(d, nextPage=(w=="a"), limit, user, superAdmin, user.get)
124+
else
125+
spaces.listAccess(d, nextPage=(w=="a"), limit, permission, user, superAdmin, showPublic, onlyTrial, showOnlyShared = false)
126+
}
127+
case (Some(w), None, None) => {
128+
if (mine)
129+
spaces.listUser(limit, user, superAdmin, user.get)
130+
else
131+
spaces.listAccess(limit, permission, user, superAdmin, showPublic, onlyTrial, showOnlyShared = false)
132+
}
133+
// default when to be "after" if not present in parameters. i.e. nextPage=true
134+
case (None, Some(t), Some(d)) => {
110135
if (mine)
111136
spaces.listUser(d, true, limit, t, user, superAdmin, user.get)
112137
else
113138
spaces.listAccess(d, true, limit, t, permission, user, superAdmin, showPublic, showOnlyShared = false)
114139
}
115-
case (Some(t), None) => {
140+
case (None, Some(t), None) => {
116141
if (mine)
117142
spaces.listUser(limit, t, user, superAdmin, user.get)
118143
else
119144
spaces.listAccess(limit, t, permission, user, superAdmin, showPublic, showOnlyShared = false)
120145
}
121-
case (None, Some(d)) => {
146+
case (None, None, Some(d)) => {
122147
if (mine)
123148
spaces.listUser(d, true, limit, user, superAdmin, user.get)
124149
else
125150
spaces.listAccess(d, true, limit, permission, user, superAdmin, showPublic, onlyTrial, showOnlyShared = false)
126151
}
127-
case (None, None) => {
152+
case (None, None, None) => {
128153
if (mine)
129154
spaces.listUser(limit, user, superAdmin, user.get)
130155
else

0 commit comments

Comments
 (0)