Merge branch 'develop' into release/1.12.0

Author: lmarini

CHANGELOG.md

@@ -4,7 +4,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
-## Unreleased
+## [Unreleased]
+
+### Added
+- Track user_id with every extraction event. [#94](https://github.com/clowder-framework/clowder/issues/94)
+- Added a new storage report at `GET api/reports/storage/spaces/:id` for auditing user storage usage on a space basis.
+- The file and dataset metrics reports also have support for since and until ISO8601 date parameters.
 
 ### Fixed
 - Fixed permissions checks on search results for search interfaces that would cause misleading counts. [#60](https://github.com/clowder-framework/clowder/issues/60)

app/api/Datasets.scala

@@ -2751,7 +2751,7 @@ class  Datasets @Inject()(
         // Setup userList, add all users of all spaces associated with the dataset
         dataset.spaces.foreach { spaceId =>
           spaces.get(spaceId) match {
-            case Some(spc) => userList = spaces.getUsersInSpace(spaceId) ::: userList
+            case Some(spc) => userList = spaces.getUsersInSpace(spaceId, None) ::: userList
             case None => NotFound(s"Error: No $spaceTitle found for $id.")
           }
         }

app/api/Files.scala

@@ -1832,7 +1832,7 @@ class Files @Inject()(
         datasets.findByFileIdDirectlyContain(id).foreach(dataset => {
           dataset.spaces.foreach { spaceId =>
             spaces.get(spaceId) match {
-              case Some(spc) => userList = spaces.getUsersInSpace(spaceId) ::: userList
+              case Some(spc) => userList = spaces.getUsersInSpace(spaceId, None) ::: userList
               case None => NotFound(s"Error: No $spaceTitle found for $id.")
             }
           }

app/api/Reporting.scala

@@ -2,14 +2,19 @@ package api
 
 import api.Permission._
 import play.api.libs.iteratee.Enumerator
+
 import scala.concurrent.{ExecutionContext, Future}
 import play.api.libs.concurrent.Execution.Implicits._
 import play.api.mvc.Controller
 import play.api.Logger
 import javax.inject.Inject
-import java.util.{TimeZone, Date}
+import java.util.{Date, TimeZone}
+
 import services._
-import models.{File, Dataset, Collection, ProjectSpace, User, UserStatus}
+import models.{Collection, Dataset, File, ProjectSpace, UUID, User, UserStatus}
+import util.Parsers
+
+import scala.collection.mutable.ListBuffer
 
 
 /**
@@ -25,10 +30,10 @@ class Reporting @Inject()(selections: SelectionService,
   val dateFormat = new java.text.SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
   dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"))
 
-  def fileMetrics() = ServerAdminAction { implicit request =>
+  def fileMetrics(since: Option[String], until: Option[String]) = ServerAdminAction { implicit request =>
     Logger.debug("Generating file metrics report")
 
-    val results = files.getMetrics()
+    val results = files.getIterator(None, since, until)
     var headerRow = true
     val enum = Enumerator.generateM({
       val chunk = if (headerRow) {
@@ -58,10 +63,10 @@ class Reporting @Inject()(selections: SelectionService,
     )
   }
 
-  def datasetMetrics() = ServerAdminAction { implicit request =>
+  def datasetMetrics(since: Option[String], until: Option[String]) = ServerAdminAction { implicit request =>
     Logger.debug("Generating dataset metrics report")
 
-    val results = datasets.getMetrics()
+    val results = datasets.getIterator(None, since, until)
     var headerRow = true
     val enum = Enumerator.generateM({
       val chunk = if (headerRow) {
@@ -135,10 +140,10 @@ class Reporting @Inject()(selections: SelectionService,
     collections.getMetrics().foreach(coll => {
       contents += _buildCollectionRow(coll, true)
     })
-    datasets.getMetrics().foreach(ds => {
+    datasets.getIterator(None, None, None).foreach(ds => {
       contents += _buildDatasetRow(ds, true)
     })
-    files.getMetrics().foreach(f => {
+    files.getIterator(None, None, None).foreach(f => {
       contents += _buildFileRow(f)
     })
 
@@ -223,7 +228,7 @@ class Reporting @Inject()(selections: SelectionService,
     )
   }
 
-  def _buildFileRow(f: File): String = {
+  private def _buildFileRow(f: File): String = {
     var contents = ""
 
     // Parent datasets, collections & spaces are sublists within the columns
@@ -287,7 +292,7 @@ class Reporting @Inject()(selections: SelectionService,
     return contents
   }
 
-  def _buildDatasetRow(ds: Dataset, returnAllColums: Boolean = false): String = {
+  private def _buildDatasetRow(ds: Dataset, returnAllColums: Boolean = false): String = {
     """returnAllColumns will include empty columns to align with file rows on report"""
     var contents = ""
 
@@ -341,7 +346,7 @@ class Reporting @Inject()(selections: SelectionService,
     return contents
   }
 
-  def _buildCollectionRow(coll: Collection, returnAllColums: Boolean = false): String = {
+  private def _buildCollectionRow(coll: Collection, returnAllColums: Boolean = false): String = {
     """returnAllColumns will include empty columns to align with file rows on report"""
 
     var contents = ""
@@ -389,4 +394,110 @@ class Reporting @Inject()(selections: SelectionService,
     return contents
   }
 
+  def spaceStorage(id: UUID, since: Option[String], until: Option[String]) = ServerAdminAction { implicit request =>
+    // Iterate over the files of every dataset in the space
+    val results = datasets.getIterator(Some(id), None, None) // TODO: Can't use time filters here if user intends files
+
+    var headerRow = true
+    val enum = Enumerator.generateM({
+      val chunk = if (headerRow) {
+        val header = "file_type,id,name,owner,owner_email,owner_id,size_kb,uploaded,location,parent_datasets,parent_collections,parent_spaces,space_owners,space_admins\n"
+        headerRow = false
+        Some(header.getBytes("UTF-8"))
+      } else {
+        scala.concurrent.blocking {
+          if (results.hasNext) {
+            try {
+              val ds = results.next
+
+              // Each file in the dataset inherits same parent info from dataset
+              val ds_list = ds.id.stringify
+              var coll_list = ""
+              var space_list = ""
+              val space_ids: ListBuffer[UUID] = ListBuffer.empty
+              var j = 1
+              var k = 1
+              ds.collections.foreach(coll => {
+                if (!coll_list.contains(coll.uuid)) {
+                  coll_list += (if (j>1) ", " else "") + coll.uuid
+                  j += 1
+                }
+              })
+              ds.spaces.foreach(sp => {
+                if (!space_list.contains(sp.uuid)) {
+                  space_list += (if (k>1) ", " else "") + sp.uuid
+                  space_ids += sp
+                  k += 1
+                }
+              })
+
+              // Get admin and owner of space(s)
+              // TODO: Should we include email and/or name too?
+              var space_owner_list = ""
+              var space_admin_list = ""
+              var l = 1
+              var m = 1
+              spaces.get(space_ids.toList).found.foreach(sp => {
+                space_owner_list += (if (l>1) ", " else "") + sp.creator.uuid
+                l += 1
+                spaces.getUsersInSpace(sp.id, Some("Admin")).foreach(spadmin => {
+                  space_admin_list += (if (m>1) ", " else "") + spadmin.id.uuid
+                  m += 1
+                })
+              })
+
+              var contents = ""
+              files.get(ds.files).found.foreach(f => {
+                // TODO: Need to redesign File model because this is gonna be so slow...
+                val sinceOK = {
+                  since match {
+                    case None => true
+                    case Some(t) => (Parsers.fromISO8601(t).before(f.uploadDate))
+                  }
+                }
+                val untilOK = {
+                  until match {
+                    case None => true
+                    case Some(t) => (Parsers.fromISO8601(t).after(f.uploadDate))
+                  }
+                }
+
+                if (sinceOK && untilOK) {
+                  // build next row of storage report
+                  contents += "\""+f.contentType+"\","
+                  contents += "\""+f.id.toString+"\","
+                  contents += "\""+f.filename+"\","
+                  contents += "\""+f.author.fullName+"\","
+                  contents += "\""+f.author.email.getOrElse("")+"\","
+                  contents += "\""+f.author.id+"\","
+                  contents += (f.length/1000).toInt.toString+","
+                  contents += dateFormat.format(f.uploadDate)+","
+                  contents += "\""+f.loader_id+"\","
+                  contents += "\""+ds_list+"\","
+                  contents += "\""+coll_list+"\","
+                  contents += "\""+space_list+"\","
+                  contents += "\""+space_owner_list+"\","
+                  contents += "\""+space_admin_list+"\""
+                  contents += "\n"
+                }
+              })
+              // Submit all file rows for this dataset at once
+              Some(contents.getBytes("UTF-8"))
+            }
+            catch {
+              case _ => Some("".getBytes("UTF-8"))
+            }
+          }
+          else None
+        }
+      }
+
+      Future(chunk)
+    })
+
+    Ok.chunked(enum.andThen(Enumerator.eof)).withHeaders(
+      "Content-Type" -> "text/csv",
+      "Content-Disposition" -> ("attachment; filename=SpaceStorage"+id.stringify+".csv")
+    )
+  }
 }

app/api/Spaces.scala

@@ -451,7 +451,7 @@ class Spaces @Inject()(spaces: SpaceService,
       aResult match {
         case aMap: JsSuccess[Map[String, String]] => {
           //Set up a map of existing users to check against
-          val existingUsers = spaces.getUsersInSpace(spaceId)
+          val existingUsers = spaces.getUsersInSpace(spaceId, None)
           var existUserRole: Map[String, String] = Map.empty
           for (aUser <- existingUsers) {
             spaces.getRoleForUserInSpace(spaceId, aUser.id) match {
@@ -527,8 +527,8 @@ class Spaces @Inject()(spaces: SpaceService,
                   case None => Logger.debug("A role was sent up that doesn't exist. It is " + k)
                 }
               }
-              if(space.userCount != spaces.getUsersInSpace(space.id).length){
-                spaces.updateUserCount(space.id, spaces.getUsersInSpace(space.id).length)
+              if(space.userCount != spaces.getUsersInSpace(space.id, None).length){
+                spaces.updateUserCount(space.id, spaces.getUsersInSpace(space.id, None).length)
               }
 
               Ok(Json.obj("status" -> "success"))
@@ -699,7 +699,7 @@ class Spaces @Inject()(spaces: SpaceService,
               }
             }
 
-            userService.listUsersInSpace(s.id).map { member =>
+            userService.listUsersInSpace(s.id, None).map { member =>
               val theHtml = views.html.spaces.verifySpaceEmail(s.id.stringify, s.name, member.getMiniUser.fullName)
               Mail.sendEmail("Space Status update", request.user, member, theHtml)
             }

app/controllers/Collections.scala

@@ -642,7 +642,7 @@ class Collections @Inject() (datasets: DatasetService, collections: CollectionSe
         // Setup userList, add all users of all spaces associated with the collection
         collection.spaces.foreach { spaceId =>
           spaceService.get(spaceId) match {
-            case Some(spc) => userList = spaceService.getUsersInSpace(spaceId) ::: userList
+            case Some(spc) => userList = spaceService.getUsersInSpace(spaceId, None) ::: userList
             case None => Redirect (routes.Collections.collection(id)).flashing ("error" -> s"Error: No $spaceTitle found for collection $id.");
           }
         }
@@ -653,7 +653,7 @@ class Collections @Inject() (datasets: DatasetService, collections: CollectionSe
         collection.spaces.foreach { spaceId =>
           spaceService.get(spaceId) match {
             case Some(spc) => {
-              val usersInCurrSpace: List[User] = spaceService.getUsersInSpace(spaceId)
+              val usersInCurrSpace: List[User] = spaceService.getUsersInSpace(spaceId, None)
               if (usersInCurrSpace.nonEmpty) {
 
                 usersInCurrSpace.foreach { usr =>

app/controllers/Datasets.scala

@@ -786,7 +786,7 @@ class Datasets @Inject() (
         // Setup userList, add all users of all spaces associated with the dataset
         dataset.spaces.foreach { spaceId =>
           spaceService.get(spaceId) match {
-            case Some(spc) => userList = spaceService.getUsersInSpace(spaceId) ::: userList
+            case Some(spc) => userList = spaceService.getUsersInSpace(spaceId, None) ::: userList
             case None => Redirect(routes.Datasets.dataset(id)).flashing("error" -> s"Error: No $spaceTitle found for $Messages('dataset.title') $id.")
           }
         }
@@ -797,7 +797,7 @@ class Datasets @Inject() (
         dataset.spaces.foreach { spaceId =>
           spaceService.get(spaceId) match {
             case Some(spc) => {
-              val usersInCurrSpace: List[User] = spaceService.getUsersInSpace(spaceId)
+              val usersInCurrSpace: List[User] = spaceService.getUsersInSpace(spaceId, None)
               if (usersInCurrSpace.nonEmpty) {
 
                 usersInCurrSpace.foreach { usr =>

app/controllers/Spaces.scala

@@ -177,7 +177,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
         val collectionsInSpace = spaces.getCollectionsInSpace(Some(id.stringify), Some(size))
         val datasetsInSpace = datasets.listSpace(size, id.toString(), user)
         val publicDatasetsInSpace = datasets.listSpaceStatus(size, id.toString(), "publicAll", user)
-        val usersInSpace = spaces.getUsersInSpace(id)
+        val usersInSpace = spaces.getUsersInSpace(id, None)
         var curationObjectsInSpace: List[CurationObject] = List()
         var inSpaceBuffer = usersInSpace.to[ArrayBuffer]
         creator match {
@@ -250,7 +250,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
       case Some(s) => {
         val creator = users.findById(s.creator)
         var creatorActual: User = null
-        val usersInSpace = spaces.getUsersInSpace(id)
+        val usersInSpace = spaces.getUsersInSpace(id, None)
         var inSpaceBuffer = usersInSpace.to[ArrayBuffer]
         creator match {
           case Some(theCreator) => {
@@ -376,7 +376,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
               val subject: String = "Request for access from " + AppConfiguration.getDisplayName
               val body = views.html.spaces.requestemail(user, id.toString, s.name)
 
-              for (requestReceiver <- spaces.getUsersInSpace(s.id)) {
+              for (requestReceiver <- spaces.getUsersInSpace(s.id, None)) {
                 spaces.getRoleForUserInSpace(s.id, requestReceiver.id) match {
                   case Some(aRole) => {
                     if (aRole.permissions.contains(Permission.EditSpace.toString)) {

app/models/Extraction.scala

@@ -19,7 +19,9 @@ case class Extraction(
   extractor_id: String,
   status: String = "N/A",
   start: Date,
-  end: Option[Date])
+  end: Option[Date],
+  user_id: UUID = User.anonymous.id
+)
 
 /**
  * Currently running extractor name

app/services/DatasetService.scala

@@ -383,6 +383,6 @@ trait DatasetService {
 
   def incrementDownloads(id: UUID, user: Option[User])
 
-  def getMetrics(): Iterator[Dataset]
+  def getIterator(space: Option[UUID], since: Option[String], until: Option[String]): Iterator[Dataset]
 
 }