Skip to content

Commit 041c9d0

Browse files
tcnichollmarini
tcnichol
and
lmarini
authored
adding a bulk delete files method (#153)
* adding a bulk delete files method * changelog entry * preparing to check permissions in FileService method * adding check to see if user can delete file needs to be improved, moved to separate method * checks if user can delete file this might not be the best approach * checks permission inside method only deletes files with proper permission * using the new checkPermissions method for list of resource ref instead of changing code in the fileservice method * removing redundant permissions check * matching develop * adding swagger entry * Updated CHANGELOG.md. Co-authored-by: Luigi Marini <[email protected]>
1 parent 8dfd1aa commit 041c9d0

File tree

6 files changed

+113
-78
lines changed

6 files changed

+113
-78
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
1313
- Log an event each time that a user archives or unarchives a file.
1414
- Endpoint `/api/datasets/createfrombag` to ingest datasets in BagIt format. Includes basic dataset metadata, files,
1515
folders and technical metadata. Downloading datasets now includes extra Datacite and Clowder metadata.
16+
- Endpoint /api/files/bulkRemove to delete multiple files in one call. [#12](https://github.com/clowder-framework/clowder/issues/12)
1617

1718
## 1.16.0 - 2021-03-31
1819

‎app/api/Files.scala‎

Lines changed: 35 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,8 @@
11
package api
22

3-
import scala.annotation.tailrec
4-
import java.io.FileInputStream
5-
import java.net.{URL, URLEncoder}
6-
7-
import javax.inject.Inject
8-
import javax.mail.internet.MimeUtility
9-
import _root_.util.{FileUtils, JSONLD, Parsers, RequestUtils, SearchUtils}
3+
import _root_.util._
104
import com.mongodb.casbah.Imports._
11-
import controllers.Previewers
5+
import controllers.{Previewers, Utils}
126
import jsonutils.JsonUtil
137
import models._
148
import play.api.Logger
@@ -18,16 +12,16 @@ import play.api.libs.concurrent.Execution.Implicits._
1812
import play.api.libs.iteratee.Enumerator
1913
import play.api.libs.json.Json._
2014
import play.api.libs.json._
21-
import play.api.mvc.{Action, ResponseHeader, Result, SimpleResult}
15+
import play.api.mvc.{ResponseHeader, SimpleResult}
2216
import services._
17+
import services.s3.S3ByteStorageService
2318

24-
import scala.collection.mutable.ListBuffer
25-
import scala.util.parsing.json.JSONArray
26-
import java.text.SimpleDateFormat
19+
import java.io.FileInputStream
20+
import java.net.URL
2721
import java.util.Date
28-
29-
import controllers.Utils
30-
import services.s3.S3ByteStorageService
22+
import javax.inject.Inject
23+
import scala.annotation.tailrec
24+
import scala.collection.mutable.ListBuffer
3125

3226
/**
3327
* Json API for files.
@@ -1616,6 +1610,32 @@ class Files @Inject()(
16161610
}
16171611
}
16181612

1613+
def bulkDeleteFiles() = PrivateServerAction (parse.json) {implicit request=>
1614+
request.user match {
1615+
case Some(user) => {
1616+
val fileIds = request.body.\("fileIds").asOpt[List[String]].getOrElse(List.empty[String])
1617+
if (fileIds.isEmpty){
1618+
BadRequest("No file ids supplied")
1619+
} else {
1620+
var resourceRefList: ListBuffer[ResourceRef] = ListBuffer.empty[ResourceRef]
1621+
for (fileId <- fileIds) {
1622+
if (UUID.isValid(fileId)) {
1623+
val current_resource_ref = ResourceRef(ResourceRef.file, UUID(fileId))
1624+
resourceRefList += current_resource_ref
1625+
}
1626+
}
1627+
val filesIdsCanDelete = Permission.checkPermissions(request.user, Permission.DeleteFile, resourceRefList.toList).approved.map(_.id)
1628+
for (id <- filesIdsCanDelete) {
1629+
files.removeFile(id,Utils.baseUrl(request), request.apiKey, request.user)
1630+
}
1631+
Ok(toJson(Map("status" -> "success")))
1632+
}
1633+
}
1634+
case None => {
1635+
BadRequest("No user supplied")
1636+
}
1637+
}
1638+
}
16191639

16201640
def removeFile(id: UUID) = PermissionAction(Permission.DeleteFile, Some(ResourceRef(ResourceRef.file, id))) { implicit request =>
16211641
files.get(id) match {

‎app/controllers/Application.scala‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,14 @@
11
package controllers
22

3-
import java.net.URL
4-
5-
import javax.inject.{Inject, Singleton}
6-
import api.Permission
7-
import api.Permission._
8-
import play.api.{Logger, Play, Routes}
3+
import models.{Event, UUID, UserStatus}
4+
import play.api.Play.current
95
import play.api.mvc.Action
6+
import play.api.{Logger, Play, Routes}
107
import services._
11-
import models.{Event, UUID, User, UserStatus}
12-
import org.owasp.html.Sanitizers
13-
import play.api.Logger
14-
import play.api.libs.concurrent.Execution.Implicits._
15-
import play.api.Play.current
168
import util.Formatters.sanitizeHTML
179

10+
import java.net.URL
11+
import javax.inject.{Inject, Singleton}
1812
import scala.collection.immutable.List
1913
import scala.collection.mutable.ListBuffer
2014

@@ -368,6 +362,7 @@ class Application @Inject() (files: FileService, collections: CollectionService,
368362
api.routes.javascript.Files.updateFileName,
369363
api.routes.javascript.Files.updateDescription,
370364
api.routes.javascript.Files.extract,
365+
api.routes.javascript.Files.bulkDeleteFiles,
371366
api.routes.javascript.Files.removeFile,
372367
api.routes.javascript.Files.follow,
373368
api.routes.javascript.Files.unfollow,

‎app/services/mongodb/MongoDBFileService.scala‎

Lines changed: 52 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -806,67 +806,67 @@ class MongoDBFileService @Inject() (
806806
def removeFile(id: UUID, host: String, apiKey: Option[String], user: Option[User]){
807807
get(id) match{
808808
case Some(file) => {
809-
if(!file.isIntermediate){
810-
val fileDatasets = datasets.findByFileIdDirectlyContain(file.id)
811-
for(fileDataset <- fileDatasets){
812-
datasets.removeFile(fileDataset.id, id)
813-
if(!file.xmlMetadata.isEmpty){
814-
datasets.index(fileDataset.id)
815-
}
809+
if(!file.isIntermediate){
810+
val fileDatasets = datasets.findByFileIdDirectlyContain(file.id)
811+
for(fileDataset <- fileDatasets){
812+
datasets.removeFile(fileDataset.id, id)
813+
if(!file.xmlMetadata.isEmpty){
814+
datasets.index(fileDataset.id)
815+
}
816816

817-
if(!file.thumbnail_id.isEmpty && !fileDataset.thumbnail_id.isEmpty){
818-
if(file.thumbnail_id.get.equals(fileDataset.thumbnail_id.get)){
819-
datasets.newThumbnail(fileDataset.id)
820-
collections.get(fileDataset.collections).found.foreach(collection => {
821-
if(!collection.thumbnail_id.isEmpty){
822-
if(collection.thumbnail_id.get.equals(fileDataset.thumbnail_id.get)){
823-
collections.createThumbnail(collection.id)
817+
if(!file.thumbnail_id.isEmpty && !fileDataset.thumbnail_id.isEmpty){
818+
if(file.thumbnail_id.get.equals(fileDataset.thumbnail_id.get)){
819+
datasets.newThumbnail(fileDataset.id)
820+
collections.get(fileDataset.collections).found.foreach(collection => {
821+
if(!collection.thumbnail_id.isEmpty){
822+
if(collection.thumbnail_id.get.equals(fileDataset.thumbnail_id.get)){
823+
collections.createThumbnail(collection.id)
824+
}
824825
}
825-
}
826-
})
827-
}
826+
})
827+
}
828+
}
829+
828830
}
829-
830-
}
831831

832-
val fileFolders = folders.findByFileId(file.id)
833-
for(fileFolder <- fileFolders) {
834-
folders.removeFile(fileFolder.id, file.id)
835-
}
836-
for(section <- sections.findByFileId(file.id)){
837-
sections.removeSection(section)
838-
}
839-
for(comment <- comments.findCommentsByFileId(id)){
840-
comments.removeComment(comment)
841-
}
842-
for(texture <- threeD.findTexturesByFileId(file.id)){
843-
ThreeDTextureDAO.removeById(new ObjectId(texture.id.stringify))
844-
}
845-
for (follower <- file.followers) {
846-
userService.unfollowFile(follower, id)
832+
val fileFolders = folders.findByFileId(file.id)
833+
for(fileFolder <- fileFolders) {
834+
folders.removeFile(fileFolder.id, file.id)
835+
}
836+
for(section <- sections.findByFileId(file.id)){
837+
sections.removeSection(section)
838+
}
839+
for(comment <- comments.findCommentsByFileId(id)){
840+
comments.removeComment(comment)
841+
}
842+
for(texture <- threeD.findTexturesByFileId(file.id)){
843+
ThreeDTextureDAO.removeById(new ObjectId(texture.id.stringify))
844+
}
845+
for (follower <- file.followers) {
846+
userService.unfollowFile(follower, id)
847+
}
847848
}
848-
}
849849

850-
// delete the actual file
851-
if(isLastPointingToLoader(file.loader, file.loader_id)) {
852-
for(preview <- previews.findByFileId(file.id)){
853-
previews.removePreview(preview)
850+
// delete the actual file
851+
if(isLastPointingToLoader(file.loader, file.loader_id)) {
852+
for(preview <- previews.findByFileId(file.id)){
853+
previews.removePreview(preview)
854+
}
855+
if(!file.thumbnail_id.isEmpty)
856+
thumbnails.remove(UUID(file.thumbnail_id.get))
857+
ByteStorageService.delete(file.loader, file.loader_id, FileDAO.COLLECTION)
854858
}
855-
if(!file.thumbnail_id.isEmpty)
856-
thumbnails.remove(UUID(file.thumbnail_id.get))
857-
ByteStorageService.delete(file.loader, file.loader_id, FileDAO.COLLECTION)
858-
}
859859

860-
import UUIDConversions._
861-
FileDAO.removeById(file.id)
862-
appConfig.incrementCount('files, -1)
863-
appConfig.incrementCount('bytes, -file.length)
864-
current.plugin[ElasticsearchPlugin].foreach {
865-
_.delete(id.stringify)
866-
}
860+
import UUIDConversions._
861+
FileDAO.removeById(file.id)
862+
appConfig.incrementCount('files, -1)
863+
appConfig.incrementCount('bytes, -file.length)
864+
current.plugin[ElasticsearchPlugin].foreach {
865+
_.delete(id.stringify)
866+
}
867867

868-
// finally remove metadata - if done before file is deleted, document metadataCounts won't match
869-
metadatas.removeMetadataByAttachTo(ResourceRef(ResourceRef.file, id), host, apiKey, user)
868+
// finally remove metadata - if done before file is deleted, document metadataCounts won't match
869+
metadatas.removeMetadataByAttachTo(ResourceRef(ResourceRef.file, id), host, apiKey, user)
870870
}
871871
case None => Logger.debug("File not found")
872872
}

‎conf/routes‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,7 @@ GET /api/files/:id/blob
388388

389389
# deprecrated use DELETE
390390
POST /api/files/:id/remove @api.Files.removeFile(id: UUID)
391+
POST /api/files/bulkRemove @api.Files.bulkDeleteFiles()
391392
GET /api/files/:id/metadata @api.Files.get(id: UUID)
392393
POST /api/files/:id/metadata @api.Files.addMetadata(id: UUID)
393394
GET /api/files/:id/metadataDefinitions @api.Files.getMetadataDefinitions(id: UUID, space: Option[String] ?= None)

‎public/swagger.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,24 @@ paths:
189189
'401':
190190
$ref: '#/components/responses/Disabled'
191191

192+
/files/bulkRemove:
193+
post:
194+
tags:
195+
- files
196+
summary: Deletes files
197+
descriptions: Deletes a list of files by fileIds
198+
responses:
199+
'200':
200+
description: Returns Status Success
201+
content:
202+
application/json:
203+
schema:
204+
type: array
205+
items:
206+
$ref: '#components/parameters/fileIds'
207+
'401':
208+
$ref: '#components/responses/Disabled'
209+
192210
/files/{id}:
193211
parameters:
194212
- $ref: '#/components/parameters/fileId'

0 commit comments

Comments
 (0)