Ability to queue archive / unarchive for full datasets (#234)

* Ability to queue up archive / unarchive for full datasets

* Add to CHANGELOG

* Call incrementDownloads when unarchiving, add optional flag to incrementDownload to only update the date (don't increment the count)

* fix: dateOnly should default to false to preserve current default behavior

* Default value changed false -> true

Co-authored-by: Luigi Marini <[email protected]>

Author: bodom0015

CHANGELOG.md

@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ## Unreleased
 - Added folder and folder id to api datasets files list [#34](https://github.com/clowder-framework/clowder/issues/34)
 
+### Added
+- Ability to queue archive / unarchive for full datasets.
+
 ### Fixed
 - RabbitMQ will not use connection if None exists
 - previews returns 404 if preview is not found

app/api/Datasets.scala

@@ -3108,6 +3108,51 @@ class  Datasets @Inject()(
     }
 
   }
+
+  /**
+   * Recursively submit requests to archive the contents of the given dataset
+   * @param id dataset to archive
+   * @return
+   */
+  def queueArchival(id: UUID) = PermissionAction(Permission.ArchiveDataset, Some(ResourceRef(ResourceRef.dataset, id)))(parse.json) { implicit request =>
+    val reqParams = (request.body \ "parameters").asOpt[JsObject].getOrElse(JsObject(Seq.empty[(String, JsValue)]))
+    val parameters = reqParams + FileService.ARCHIVE_PARAMETER
+    datasets.get(id) match {
+      case Some(ds) => {
+        val host = Utils.baseUrl(request)
+        datasets.recursiveArchive(ds, host, parameters, request.apiKey, request.user)
+        sinkService.logDatasetArchiveEvent(ds, request.user)
+        Ok(toJson(Map("status" -> "success")))
+      }
+      case None => {
+        Logger.error("Error getting dataset " + id)
+        NotFound
+      }
+    }
+  }
+
+
+  /**
+   * Recursively submit requests to unarchive the contents of the given dataset
+   * @param id dataset to unarchive
+   * @return
+   */
+  def queueUnarchival(id: UUID) = PermissionAction(Permission.ArchiveDataset, Some(ResourceRef(ResourceRef.dataset, id)))(parse.json) { implicit request =>
+    val reqParams = (request.body \ "parameters").asOpt[JsObject].getOrElse(JsObject(Seq.empty[(String, JsValue)]))
+    val parameters = reqParams + FileService.UNARCHIVE_PARAMETER
+    datasets.get(id) match {
+      case Some(ds) => {
+        val host = Utils.baseUrl(request)
+        datasets.recursiveArchive(ds, host, parameters, request.apiKey, request.user)
+        sinkService.logDatasetUnarchiveEvent(ds, request.user)
+        Ok(toJson(Map("status" -> "success")))
+      }
+      case None => {
+        Logger.error("Error getting dataset " + id)
+        NotFound
+      }
+    }
+  }
 }
 
 object ActivityFound extends Exception {}

app/api/Files.scala

@@ -1885,7 +1885,7 @@ class Files @Inject()(
       }
       case None => {
         Logger.error("Error getting file " + id)
-        InternalServerError
+        NotFound
       }
     }
   }
@@ -1896,11 +1896,14 @@ class Files @Inject()(
       case Some(file) => {
         files.setStatus(id, FileStatus.PROCESSED)
         sinkService.logFileUnarchiveEvent(file, user)
+
+        // Only increment download date, to avoid immediate auto-archive
+        files.incrementDownloads(id, user, true)
         Ok(toJson(Map("status" -> "success")))
       }
       case None => {
         Logger.error("Error getting file " + id)
-        InternalServerError
+        NotFound
       }
     }
   }
@@ -1916,7 +1919,7 @@ class Files @Inject()(
       }
       case None => {
         Logger.error("Error getting file " + id)
-        InternalServerError
+        NotFound
       }
     }
   }
@@ -1932,7 +1935,7 @@ class Files @Inject()(
       }
       case None => {
         Logger.error("Error getting file " + id)
-        InternalServerError
+        NotFound
       }
     }
   }

app/api/Permissions.scala

@@ -29,6 +29,7 @@ object Permission extends Enumeration {
     DeleteDataset,
     EditDataset,
     PublicDataset,
+    ArchiveDataset,
     AddResourceToDataset,
     RemoveResourceFromDataset,
     ExecuteOnDataset,
@@ -120,7 +121,8 @@ object Permission extends Enumeration {
     CreateSensor, DeleteSensor, AddGeoStream, DeleteGeoStream, AddDatapoints,
     CreateRelation, ViewRelation, DeleteRelation,
     CreateVocabulary, DeleteVocabulary, EditVocabulary,
-    CreateVocabularyTerm, DeleteVocabularyTerm, EditVocabularyTerm
+    CreateVocabularyTerm, DeleteVocabularyTerm, EditVocabularyTerm,
+    ArchiveFile, ArchiveDataset
   )
 
   lazy val files: FileService = DI.injector.getInstance(classOf[FileService])

app/controllers/Application.scala

@@ -347,6 +347,8 @@ class Application @Inject() (files: FileService, collections: CollectionService,
         api.routes.javascript.Datasets.users,
         api.routes.javascript.Datasets.restoreDataset,
         api.routes.javascript.Datasets.emptyTrash,
+        api.routes.javascript.Datasets.queueArchival,
+        api.routes.javascript.Datasets.queueUnarchival,
         api.routes.javascript.Extractions.submitFilesToExtractor,
         api.routes.javascript.Files.download,
         api.routes.javascript.Files.archive,

app/services/DatasetService.scala

@@ -1,10 +1,10 @@
 package services
 
 import java.util.Date
-
 import api.Permission.Permission
 import com.mongodb.casbah.Imports._
 import models.{File, _}
+import play.api.libs.json.JsObject
 
 /**
  * Generic dataset service.
@@ -386,4 +386,6 @@ trait DatasetService {
   def getIterator(space: Option[String], since: Option[String], until: Option[String]): Iterator[Dataset]
 
   def getTrashedIds(): List[UUID]
+
+  def recursiveArchive(dataset: Dataset, host: String, parameters: JsObject, apiKey: Option[String], user: Option[User])
 }

app/services/EventSinkService.scala

@@ -289,6 +289,34 @@ class EventSinkService {
       "size" -> file.length
     ))
   }
+
+  def logDatasetArchiveEvent(ds: Dataset, archiver: Option[User]) = {
+    logEvent(Json.obj(
+      "category" -> "archive",
+      "type" -> "dataset",
+      "resource_id" -> ds.id,
+      "resource_name" -> ds.name,
+      "author_id" -> ds.author.id,
+      "author_name" -> ds.author.fullName,
+      "user_id" -> archiver.get.id,
+      "user_name" -> archiver.get.getMiniUser.fullName,
+      "size" -> (ds.files.length + ds.folders.length)
+    ))
+  }
+
+  def logDatasetUnarchiveEvent(ds: Dataset, unarchiver: Option[User]) = {
+    logEvent(Json.obj(
+      "category" -> "unarchive",
+      "type" -> "dataset",
+      "resource_id" -> ds.id,
+      "resource_name" -> ds.name,
+      "author_id" -> ds.author.id,
+      "author_name" -> ds.author.fullName,
+      "user_id" -> unarchiver.get.id,
+      "user_name" -> unarchiver.get.getMiniUser.fullName,
+      "size" -> (ds.files.length + ds.folders.length)
+    ))
+  }
 }
 
 //case class EventSinkMessage(created: Long, category: String, metadata: JsValue)

app/services/FileService.scala

@@ -244,7 +244,7 @@ trait FileService {
 
   def incrementViews(id: UUID, user: Option[User]): (Int, Date)
 
-  def incrementDownloads(id: UUID, user: Option[User])
+  def incrementDownloads(id: UUID, user: Option[User], dateOnly: Boolean = false)
 
   def getIterator(space: Option[String], since: Option[String], until: Option[String]): Iterator[File]
 

app/services/FolderService.scala

@@ -1,6 +1,7 @@
 package services
 
-import models.{DBResult, TypedID, UUID, Folder}
+import models.{DBResult, Folder, TypedID, UUID, User}
+import play.api.libs.json.JsObject
 /**
  * Generic Folder Service
  */
@@ -96,4 +97,9 @@ trait FolderService {
     * Get all the folders in a list of parent datasettIds. It helps to identify the files that a user has access to.
     */
   def findByParentDatasetIds(parentIds: List[UUID]): List[Folder]
+
+  /**
+   * Archive all files and subfolders within this folder.
+   */
+  def recursiveArchive(folder: Folder, host: String, parameters: JsObject, apiKey: Option[String], user: Option[User]): Unit
 }

app/services/mongodb/MongoDBDatasetService.scala

@@ -3,11 +3,10 @@ package services.mongodb
 import java.io._
 import java.text.SimpleDateFormat
 import java.util.{ArrayList, Date}
-
 import javax.inject.{Inject, Singleton}
 import Transformation.LidoToCidocConvertion
 import util.{Formatters, Parsers}
-import api.Permission
+import api.{Permission, UserRequest}
 import api.Permission.Permission
 import com.mongodb.DBObject
 import com.mongodb.casbah.Imports._
@@ -16,6 +15,7 @@ import com.mongodb.casbah.commons.MongoDBList
 import com.mongodb.casbah.commons.MongoDBObject
 import com.mongodb.util.JSON
 import com.novus.salat.dao.{ModelCompanion, SalatDAO}
+import controllers.Utils
 import jsonutils.JsonUtil
 import models.{File, _}
 import org.apache.commons.io.FileUtils
@@ -24,7 +24,8 @@ import org.json.JSONObject
 import play.api.Logger
 import play.api.Play._
 import play.api.libs.json.Json._
-import play.api.libs.json.{JsArray, JsValue, Json}
+import play.api.libs.json.{JsArray, JsObject, JsValue, Json}
+import play.api.mvc.AnyContent
 import services._
 import services.mongodb.MongoContext.context
 
@@ -1658,6 +1659,21 @@ class MongoDBDatasetService @Inject() (
     })
     trashedIds.toList
   }
+
+  /**
+  * Recursively submit requests to archive or unarchive the contents of the given dataset.
+  * NOTE: "parameters" includes "operation", which supports both archiving and unarchiving
+  */
+  def recursiveArchive(ds: Dataset, host: String, parameters: JsObject, apiKey: Option[String], user: Option[User]) = {
+    ds.files.foreach(fileId => files.get(fileId) match {
+      case None => Logger.error("Error getting file " + fileId)
+      case Some(file) => files.submitArchivalOperation(file, fileId, host, parameters, apiKey, user)
+    })
+    ds.folders.foreach(folderId => folders.get(folderId) match {
+      case None => Logger.error("Error getting folder " + folderId)
+      case Some(folder) => folders.recursiveArchive(folder, host, parameters, apiKey, user)
+    })
+  }
 }
 
 object Dataset extends ModelCompanion[Dataset, ObjectId] {