199 show space bytes (#263)

* adding parameter spaceBytes inside getSpace

* add method getBytesPerSpace
changing html

* adding to space statistics

need cleanup

* statistics changed - replaced with more data modeled on the index page

We now see Collections, Datasets, Files, Bytes per space and Users in right column

* changelog

* adding spaceBytes parameter, does not work yet

* new method, get bytes for dataset

* methods need to match

* adding and removing datasets from space will increment spaceBytes.

* increment bytes when file added to dataset

* decrement space bytes on remove file

* remove commented out code

* use cached value of bytes per space not method call

* adding update space bytes method

* added space bytes update

* fixig test

* fix few issues with migration

- no datasets in space would result in not writing out count
- very large databsae would have cursor timeout
- simplified logic to count bytes

* Space layout with statistics (#283)

* make space layout more like datasets/files

* update changelog

- undo removal
- add comment + link to issue for space stats
- comment about new layout

* Lowered margin from 20px to 15px for button links.

Co-authored-by: Luigi Marini <[email protected]>

Co-authored-by: Luigi Marini <[email protected]>
Co-authored-by: Max Burnette <[email protected]>
Co-authored-by: Rob Kooper <[email protected]>
Co-authored-by: Luigi Marini <[email protected]>

Author: 4 people

CHANGELOG.md

@@ -14,6 +14,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 - Adding mime type for geojson
 - Add "when" parameter in a few GET API endpoints to enable pagination [#266](https://github.com/clowder-framework/clowder/issues/266)
+- Show statistics of spaces (bytes, users. etc) [#119](https://github.com/clowder-framework/clowder/issues/119)
+
+### Changed
 - Add "id" in GET metadata.jsonld endpoints [#278](https://github.com/clowder-framework/clowder/issues/278)
 
 ## 1.18.1 - 2021-08-16
@@ -71,7 +74,6 @@ If any files are returned, you should check to see if these files affected and a
 
 ### Changed
 - Updated Sphinx dependencies due to security and changes in required packages.
-
 - Updated the three.js libraries for the FBX previewer
 
 ## 1.16.0 - 2021-03-31

app/api/Files.scala

@@ -1675,6 +1675,10 @@ class Files @Inject()(
         // notify rabbitmq
         datasets.findByFileIdAllContain(file.id).foreach { ds =>
           routing.fileRemovedFromDataset(file, ds, Utils.baseUrl(request), request.apiKey)
+          val ds_spaces = ds.spaces
+          for (ds_s <- ds_spaces) {
+            spaces.decrementSpaceBytes(ds_s, file.length)
+          }
         }
 
         //this stmt has to be before files.removeFile

app/api/Spaces.scala

@@ -41,7 +41,7 @@ class Spaces @Inject()(spaces: SpaceService,
         val userId = request.user.get.id
         val c = ProjectSpace(name = name, description = description, created = new Date(), creator = userId,
           homePage = List.empty, logoURL = None, bannerURL = None, collectionCount = 0,
-          datasetCount = 0, userCount = 0, metadata = List.empty)
+          datasetCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty)
         spaces.insert(c) match {
           case Some(id) => {
             appConfig.incrementCount('spaces, 1)

app/controllers/SecuredController.scala

@@ -2,14 +2,11 @@ package controllers
 
 import api.Permission.Permission
 import api.{Permission, UserRequest}
-import models.{ClowderUser, RequestResource, ResourceRef, User, UserStatus}
-import org.apache.commons.lang.StringEscapeUtils._
+import models.{ClowderUser, ResourceRef, User, UserStatus}
 import play.api.i18n.Messages
 import play.api.mvc._
 import securesocial.core.{Authenticator, SecureSocial, UserService}
 import services._
-import securesocial.core.IdentityProvider
-import securesocial.core.providers.utils.RoutesHelper
 
 import scala.concurrent.Future
 
@@ -156,7 +153,7 @@ trait SecuredController extends Controller {
         val spaces: SpaceService = DI.injector.getInstance(classOf[SpaceService])
         spaces.get(id) match {
           case None => Future.successful(BadRequest(views.html.notFound(spaceTitle + " does not exist.")(user)))
-          case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List())(user)))
+          case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List(),0,0)(user)))
         }
       }
 

app/controllers/Spaces.scala

@@ -1,31 +1,22 @@
 package controllers
 
-import java.net.URL
-import java.util.{ Calendar, Date }
-import javax.inject.Inject
-
 import api.Permission
 import api.Permission._
 import models._
-import play.api.{ Logger, Play }
+import org.joda.time.DateTime
 import play.api.data.Forms._
-import play.api.data.{ Form, Forms }
-import play.api.libs.json.JsValue
-import play.api.libs.json.Json
+import play.api.data.{Form, Forms}
 import play.api.i18n.Messages
+import play.api.{Logger, Play}
+import securesocial.core.providers.{Token, UsernamePasswordProvider}
 import services._
-import securesocial.core.providers.{ Token, UsernamePasswordProvider }
-import org.joda.time.DateTime
-import play.api.i18n.Messages
-import play.api.libs.ws._
-import services.AppConfiguration
-import util.{ Formatters, Mail, Publications }
+import util.{Formatters, Mail, Publications}
 
+import java.net.URL
+import java.util.{Calendar, Date}
+import javax.inject.Inject
 import scala.collection.immutable.List
-import scala.collection.mutable.{ ArrayBuffer, ListBuffer }
-import scala.concurrent.{ Future, Await }
-import scala.concurrent.duration._
-import org.apache.commons.lang.StringEscapeUtils.escapeJava
+import scala.collection.mutable.{ArrayBuffer, ListBuffer}
 
 /**
  * Spaces allow users to partition the data into realms only accessible to users with the right permissions.
@@ -176,6 +167,8 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
         var creatorActual: User = null
         val collectionsInSpace = spaces.getCollectionsInSpace(Some(id.stringify), Some(size))
         val datasetsInSpace = datasets.listSpace(size, id.toString(), user)
+        val spaceBytes : Long = s.spaceBytes
+        val spaceFiles : Integer = getFilesPerSpace(id, user.get)
         val publicDatasetsInSpace = datasets.listSpaceStatus(size, id.toString(), "publicAll", user)
         val usersInSpace = spaces.getUsersInSpace(id, None)
         var curationObjectsInSpace: List[CurationObject] = List()
@@ -224,7 +217,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
           case None => List.empty
         }
         sinkService.logSpaceViewEvent(s, user)
-        Ok(views.html.spaces.space(Utils.decodeSpaceElements(s), collectionsInSpace, publicDatasetsInSpace, datasetsInSpace, rs, play.Play.application().configuration().getString("SEADservices.uri"), userRoleMap, userSelections))
+        Ok(views.html.spaces.space(Utils.decodeSpaceElements(s), collectionsInSpace, publicDatasetsInSpace, datasetsInSpace, rs, play.Play.application().configuration().getString("SEADservices.uri"), userRoleMap, userSelections, spaceBytes, spaceFiles))
       }
       case None => BadRequest(views.html.notFound(spaceTitle + " does not exist."))
     }
@@ -421,7 +414,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
                       val newSpace = ProjectSpace(name = formData.name, description = formData.description,
                         created = new Date, creator = userId, homePage = formData.homePage,
                         logoURL = formData.logoURL, bannerURL = formData.bannerURL,
-                        collectionCount = 0, datasetCount = 0, userCount = 0, metadata = List.empty,
+                        collectionCount = 0, datasetCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty,
                         resourceTimeToLive = formData.resourceTimeToLive * 60 * 60 * 1000L, isTimeToLiveEnabled = formData.isTimeToLiveEnabled,
                         status = formData.access,
                         affiliatedSpaces = formData.affSpace)
@@ -648,4 +641,14 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
       }
   }
 
+  private def getFilesPerSpace(spaceId: UUID, user: models.User) : Integer = {
+    var spaceFiles: Integer = 0
+    val allDatasetsInSpace = datasets.listSpace(0, spaceId.toString(), Some(user))
+    for (ds <- allDatasetsInSpace) {
+      val files_in_ds = ds.files.length
+      spaceFiles += files_in_ds
+    }
+    spaceFiles
+  }
+
 }

app/models/Space.scala

@@ -23,6 +23,7 @@ case class ProjectSpace (
   collectionCount: Integer,
   datasetCount: Integer,
   userCount: Integer,
+  spaceBytes: Long, 
   metadata: List[Metadata],
   resourceTimeToLive: Long = SpaceConfig.getTimeToLive(),
   isTimeToLiveEnabled: Boolean = SpaceConfig.getIsTimeToLiveEnabled(),
@@ -52,7 +53,8 @@ case class UserSpace (
    bannerURL: Option[URL],
    collectionCount: Integer,
    datasetCount: Integer,
-   userCount: Integer)
+   userCount: Integer,
+   spaceBytes: Long)
 
 case class SpaceInvite(
     id: UUID = UUID.generate,

app/services/DatasetService.scala

@@ -295,6 +295,8 @@ trait DatasetService {
 
   def findMetadataChangedDatasets(): List[Dataset]
 
+  def getBytesForDataset(datasetId: UUID) : Long
+
   /**
    * Check recursively whether a dataset's user-input metadata match a requested search tree.
    */

app/services/SpaceService.scala

@@ -94,6 +94,10 @@ trait SpaceService {
 
   def incrementCollectionCounter(collection: UUID, space: UUID, increment: Int)
 
+  def incrementSpaceBytes(space: UUID, increment: Long)
+
+  def decrementSpaceBytes(space: UUID, decrement: Long)
+
   def addDataset(dataset: UUID, space: UUID)
 
   def removeDataset(dataset:UUID, space: UUID)

app/services/mongodb/MongoDBDatasetService.scala

@@ -1068,6 +1068,17 @@ class MongoDBDatasetService @Inject() (
     Dataset.find(MongoDBObject("userMetadataWasModified" -> true)).toList
   }
 
+  def getBytesForDataset(datasetId: UUID) : Long = {
+    val dataset = Dataset.findOneById(new ObjectId(datasetId.stringify)).get
+    val datasetFiles = dataset.files
+    var datasetBytes : Long = 0
+    datasetFiles.foreach{ f => {
+      val currentFileBytes = files.get(f).get.length
+      datasetBytes += currentFileBytes
+    }}
+    datasetBytes
+  }
+
   def removeTag(id: UUID, tagId: UUID) {
     Logger.debug("Removing tag " + tagId)
     val result = Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $pull("tags" -> MongoDBObject("_id" -> new ObjectId(tagId.stringify))), false, false, WriteConcern.Safe)

app/services/mongodb/MongoDBSpaceService.scala

@@ -378,6 +378,14 @@ class MongoDBSpaceService @Inject() (
     ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("collectionCount" -> -1), upsert=false, multi=false, WriteConcern.Safe)
   }
 
+  def incrementSpaceBytes(space: UUID, increment: Long ): Unit = {
+    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> increment), upsert=false, multi=false, WriteConcern.Safe)
+  }
+
+  def decrementSpaceBytes(space: UUID, decrement: Long): Unit = {
+    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> decrement), upsert=false, multi=false, WriteConcern.Safe)
+  }
+
   def removeCollection(collection:UUID, space:UUID): Unit = {
     log.debug(s"Space Service - removing $collection from $space")
     collections.removeFromSpace(collection, space)
@@ -390,7 +398,9 @@ class MongoDBSpaceService @Inject() (
    */
   def addDataset(dataset: UUID, space: UUID): Unit = {
     log.debug(s"Space Service - Adding $dataset to $space")
+    val datasetBytes = datasets.getBytesForDataset(dataset)
     datasets.addToSpace(dataset, space)
+    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> datasetBytes), upsert=false, multi=false, WriteConcern.Safe)
     ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> 1), upsert=false, multi=false, WriteConcern.Safe)
 
   }
@@ -404,6 +414,8 @@ class MongoDBSpaceService @Inject() (
   def removeDataset(dataset:UUID, space:UUID): Unit = {
     log.debug(s"Space Service - removing $dataset from $space")
     datasets.removeFromSpace(dataset, space)
+    val datasetBytes = datasets.getBytesForDataset(dataset)
+    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> -datasetBytes), upsert=false, multi=false, WriteConcern.Safe)
     ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> -1), upsert=false, multi=false, WriteConcern.Safe)
   }