Merge pull request #212 from clowder-framework/release/1.17.0

1.17.0 release

Author: lmarini

CHANGELOG.md

@@ -4,6 +4,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## 1.17.0 - 2021-04-29
+
+### Fixed
+- Close channel after submitting events to RabbitMQMessageService.
+
+### Added
+- Endpoint `/api/datasets/createfrombag` to ingest datasets in BagIt format. Includes basic dataset metadata, files,
+  folders and technical metadata. Downloading datasets now includes extra Datacite and Clowder metadata.
+- Endpoint `/api/files/bulkRemove` to delete multiple files in one call. [#12](https://github.com/clowder-framework/clowder/issues/12)
+- Log an event each time that a user archives or unarchives a file.
+- Persist name of message bus response queue, preventing status messages from getting lost after a reboot.
+
+### Changed
+- Updated Sphinx dependencies due to security and changes in required packages.
+
 ## 1.16.0 - 2021-03-31
 
 ### Fixed

app/Global.scala

@@ -45,6 +45,9 @@ object Global extends WithFilters(new GzipFilter(), new Jsonp(), CORSFilter()) w
 
     val users: UserService = DI.injector.getInstance(classOf[UserService])
 
+    // get clowder unique ID
+    Logger.info(s"Starting clowder with id = ${AppConfiguration.getInstance}")
+
     // set the default ToS version
     AppConfiguration.setDefaultTermsOfServicesVersion()
 

app/api/Datasets.scala

@@ -1,14 +1,8 @@
 package api
 
-import scala.annotation.tailrec
-import java.io.FileInputStream
-import java.net.{URL, URLEncoder}
-
-import javax.inject.Inject
-import javax.mail.internet.MimeUtility
-import _root_.util.{FileUtils, JSONLD, Parsers, RequestUtils, SearchUtils}
+import _root_.util._
 import com.mongodb.casbah.Imports._
-import controllers.Previewers
+import controllers.{Previewers, Utils}
 import jsonutils.JsonUtil
 import models._
 import play.api.Logger
@@ -18,16 +12,16 @@ import play.api.libs.concurrent.Execution.Implicits._
 import play.api.libs.iteratee.Enumerator
 import play.api.libs.json.Json._
 import play.api.libs.json._
-import play.api.mvc.{Action, ResponseHeader, Result, SimpleResult}
+import play.api.mvc.{ResponseHeader, SimpleResult}
 import services._
+import services.s3.S3ByteStorageService
 
-import scala.collection.mutable.ListBuffer
-import scala.util.parsing.json.JSONArray
-import java.text.SimpleDateFormat
+import java.io.FileInputStream
+import java.net.URL
 import java.util.Date
-
-import controllers.Utils
-import services.s3.S3ByteStorageService
+import javax.inject.Inject
+import scala.annotation.tailrec
+import scala.collection.mutable.ListBuffer
 
 /**
  * Json API for files.
@@ -1616,6 +1610,32 @@ class Files @Inject()(
     }
   }
 
+  def bulkDeleteFiles() = PrivateServerAction (parse.json) {implicit request=>
+    request.user match {
+      case Some(user) => {
+        val fileIds = request.body.\("fileIds").asOpt[List[String]].getOrElse(List.empty[String])
+        if (fileIds.isEmpty){
+          BadRequest("No file ids supplied")
+        } else {
+          var resourceRefList: ListBuffer[ResourceRef] = ListBuffer.empty[ResourceRef]
+          for (fileId <- fileIds) {
+            if (UUID.isValid(fileId)) {
+              val current_resource_ref = ResourceRef(ResourceRef.file, UUID(fileId))
+              resourceRefList += current_resource_ref
+            }
+          }
+          val filesIdsCanDelete = Permission.checkPermissions(request.user, Permission.DeleteFile, resourceRefList.toList).approved.map(_.id)
+          for (id <- filesIdsCanDelete) {
+            files.removeFile(id,Utils.baseUrl(request), request.apiKey, request.user)
+          }
+          Ok(toJson(Map("status" -> "success")))
+        }
+      }
+      case None => {
+        BadRequest("No user supplied")
+      }
+    }
+  }
 
   def removeFile(id: UUID) = PermissionAction(Permission.DeleteFile, Some(ResourceRef(ResourceRef.file, id))) { implicit request =>
     files.get(id) match {
@@ -1856,9 +1876,11 @@ class Files @Inject()(
   }
 
   def archive(id: UUID) = PermissionAction(Permission.ArchiveFile, Some(ResourceRef(ResourceRef.file, id))) { implicit request =>
+    implicit val user = request.user
     files.get(id) match {
       case Some(file) => {
         files.setStatus(id, FileStatus.ARCHIVED)
+        sinkService.logFileArchiveEvent(file, user)
         Ok(toJson(Map("status" -> "success")))
       }
       case None => {
@@ -1869,9 +1891,11 @@ class Files @Inject()(
   }
 
   def unarchive(id: UUID) = PermissionAction(Permission.ArchiveFile, Some(ResourceRef(ResourceRef.file, id))) { implicit request =>
+    implicit val user = request.user
     files.get(id) match {
       case Some(file) => {
         files.setStatus(id, FileStatus.PROCESSED)
+        sinkService.logFileUnarchiveEvent(file, user)
         Ok(toJson(Map("status" -> "success")))
       }
       case None => {

app/api/Files.scala

@@ -31,7 +31,8 @@ class Status @Inject()(spaces: SpaceService,
 
   def status = UserAction(needActive=false) { implicit request =>
 
-    Ok(Json.obj("version" -> getVersionInfo,
+    Ok(Json.obj("instance" -> AppConfiguration.getInstance,
+      "version" -> getVersionInfo,
       "counts" -> getCounts(request.user),
       "plugins" -> getPlugins(request.user),
       "extractors" -> Json.toJson(extractors.getExtractorNames(List.empty))))

app/api/Status.scala

@@ -1,20 +1,14 @@
 package controllers
 
-import java.net.URL
-
-import javax.inject.{Inject, Singleton}
-import api.Permission
-import api.Permission._
-import play.api.{Logger, Play, Routes}
+import models.{Event, UUID, UserStatus}
+import play.api.Play.current
 import play.api.mvc.Action
+import play.api.{Logger, Play, Routes}
 import services._
-import models.{Event, UUID, User, UserStatus}
-import org.owasp.html.Sanitizers
-import play.api.Logger
-import play.api.libs.concurrent.Execution.Implicits._
-import play.api.Play.current
 import util.Formatters.sanitizeHTML
 
+import java.net.URL
+import javax.inject.{Inject, Singleton}
 import scala.collection.immutable.List
 import scala.collection.mutable.ListBuffer
 
@@ -368,6 +362,7 @@ class Application @Inject() (files: FileService, collections: CollectionService,
         api.routes.javascript.Files.updateFileName,
         api.routes.javascript.Files.updateDescription,
         api.routes.javascript.Files.extract,
+        api.routes.javascript.Files.bulkDeleteFiles,
         api.routes.javascript.Files.removeFile,
         api.routes.javascript.Files.follow,
         api.routes.javascript.Files.unfollow,

app/controllers/Application.scala

@@ -70,6 +70,18 @@ trait AppConfigurationService {
 object AppConfiguration {
   val appConfig: AppConfigurationService = DI.injector.getInstance(classOf[AppConfigurationService])
 
+  // ----------------------------------------------------------------------
+  def getInstance: String = {
+    appConfig.getProperty[String]("instance") match {
+      case Some(id) => id
+      case None => {
+        val id = scala.util.Random.alphanumeric.take(10).mkString
+        appConfig.setProperty("instance", id)
+        id
+      }
+    }
+  }
+
   // ----------------------------------------------------------------------
 
   /** Set the default theme */

app/services/AppConfigurationService.scala

@@ -32,7 +32,7 @@ class EventSinkService {
   def logEvent(message: JsValue) = {
     // Inject timestamp before logging the event
     val event = message.as[JsObject] + ("created" -> Json.toJson(java.util.Date.from(Instant.now())))
-    Logger.info("Submitting message to event sink exchange: " + Json.stringify(event))
+    Logger.debug("Submitting message to event sink exchange: " + Json.stringify(event))
     try {
       messageService.submit(exchangeName, queueName, event, "fanout")
     } catch {
@@ -261,6 +261,34 @@ class EventSinkService {
       "size" -> (dataset.files.length + dataset.folders.length)
     ))
   }
+
+  def logFileArchiveEvent(file: File, archiver: Option[User]) = {
+    logEvent(Json.obj(
+      "category" -> "archive",
+      "type" -> "file",
+      "resource_id" -> file.id,
+      "resource_name" -> file.filename,
+      "author_id" -> file.author.id,
+      "author_name" -> file.author.fullName,
+      "user_id" -> archiver.get.id,
+      "user_name" -> archiver.get.getMiniUser.fullName,
+      "size" -> file.length
+    ))
+  }
+
+  def logFileUnarchiveEvent(file: File, unarchiver: Option[User]) = {
+    logEvent(Json.obj(
+      "category" -> "unarchive",
+      "type" -> "file",
+      "resource_id" -> file.id,
+      "resource_name" -> file.filename,
+      "author_id" -> file.author.id,
+      "author_name" -> file.author.fullName,
+      "user_id" -> unarchiver.get.id,
+      "user_name" -> unarchiver.get.getMiniUser.fullName,
+      "size" -> file.length
+    ))
+  }
 }
 
 //case class EventSinkMessage(created: Long, category: String, metadata: JsValue)

app/services/EventSinkService.scala

@@ -806,67 +806,67 @@ class MongoDBFileService @Inject() (
   def removeFile(id: UUID, host: String, apiKey: Option[String], user: Option[User]){
     get(id) match{
       case Some(file) => {
-        if(!file.isIntermediate){
-          val fileDatasets = datasets.findByFileIdDirectlyContain(file.id)
-          for(fileDataset <- fileDatasets){
-            datasets.removeFile(fileDataset.id, id)
-            if(!file.xmlMetadata.isEmpty){
-              datasets.index(fileDataset.id)
-            }
+          if(!file.isIntermediate){
+            val fileDatasets = datasets.findByFileIdDirectlyContain(file.id)
+            for(fileDataset <- fileDatasets){
+              datasets.removeFile(fileDataset.id, id)
+              if(!file.xmlMetadata.isEmpty){
+                datasets.index(fileDataset.id)
+              }
 
-            if(!file.thumbnail_id.isEmpty && !fileDataset.thumbnail_id.isEmpty){            
-              if(file.thumbnail_id.get.equals(fileDataset.thumbnail_id.get)){ 
-                datasets.newThumbnail(fileDataset.id)
-                collections.get(fileDataset.collections).found.foreach(collection => {
-                  if(!collection.thumbnail_id.isEmpty){
-                    if(collection.thumbnail_id.get.equals(fileDataset.thumbnail_id.get)){
-                      collections.createThumbnail(collection.id)
+              if(!file.thumbnail_id.isEmpty && !fileDataset.thumbnail_id.isEmpty){
+                if(file.thumbnail_id.get.equals(fileDataset.thumbnail_id.get)){
+                  datasets.newThumbnail(fileDataset.id)
+                  collections.get(fileDataset.collections).found.foreach(collection => {
+                    if(!collection.thumbnail_id.isEmpty){
+                      if(collection.thumbnail_id.get.equals(fileDataset.thumbnail_id.get)){
+                        collections.createThumbnail(collection.id)
+                      }
                     }
-                  }
-                })
-		          }
+                  })
+                }
+              }
+
             }
-                     
-          }
 
-          val fileFolders = folders.findByFileId(file.id)
-          for(fileFolder <- fileFolders) {
-            folders.removeFile(fileFolder.id, file.id)
-          }
-          for(section <- sections.findByFileId(file.id)){
-            sections.removeSection(section)
-          }
-          for(comment <- comments.findCommentsByFileId(id)){
-            comments.removeComment(comment)
-          }
-          for(texture <- threeD.findTexturesByFileId(file.id)){
-            ThreeDTextureDAO.removeById(new ObjectId(texture.id.stringify))
-          }
-          for (follower <- file.followers) {
-            userService.unfollowFile(follower, id)
+            val fileFolders = folders.findByFileId(file.id)
+            for(fileFolder <- fileFolders) {
+              folders.removeFile(fileFolder.id, file.id)
+            }
+            for(section <- sections.findByFileId(file.id)){
+              sections.removeSection(section)
+            }
+            for(comment <- comments.findCommentsByFileId(id)){
+              comments.removeComment(comment)
+            }
+            for(texture <- threeD.findTexturesByFileId(file.id)){
+              ThreeDTextureDAO.removeById(new ObjectId(texture.id.stringify))
+            }
+            for (follower <- file.followers) {
+              userService.unfollowFile(follower, id)
+            }
           }
-        }
 
-        // delete the actual file
-        if(isLastPointingToLoader(file.loader, file.loader_id)) {
-          for(preview <- previews.findByFileId(file.id)){
-            previews.removePreview(preview)
+          // delete the actual file
+          if(isLastPointingToLoader(file.loader, file.loader_id)) {
+            for(preview <- previews.findByFileId(file.id)){
+              previews.removePreview(preview)
+            }
+            if(!file.thumbnail_id.isEmpty)
+              thumbnails.remove(UUID(file.thumbnail_id.get))
+            ByteStorageService.delete(file.loader, file.loader_id, FileDAO.COLLECTION)
           }
-          if(!file.thumbnail_id.isEmpty)
-            thumbnails.remove(UUID(file.thumbnail_id.get))
-          ByteStorageService.delete(file.loader, file.loader_id, FileDAO.COLLECTION)
-        }
 
-        import UUIDConversions._
-        FileDAO.removeById(file.id)
-        appConfig.incrementCount('files, -1)
-        appConfig.incrementCount('bytes, -file.length)
-        current.plugin[ElasticsearchPlugin].foreach {
-          _.delete(id.stringify)
-        }
+          import UUIDConversions._
+          FileDAO.removeById(file.id)
+          appConfig.incrementCount('files, -1)
+          appConfig.incrementCount('bytes, -file.length)
+          current.plugin[ElasticsearchPlugin].foreach {
+            _.delete(id.stringify)
+          }
 
-        // finally remove metadata - if done before file is deleted, document metadataCounts won't match
-        metadatas.removeMetadataByAttachTo(ResourceRef(ResourceRef.file, id), host, apiKey, user)
+          // finally remove metadata - if done before file is deleted, document metadataCounts won't match
+          metadatas.removeMetadataByAttachTo(ResourceRef(ResourceRef.file, id), host, apiKey, user)
       }
       case None => Logger.debug("File not found")
     }