cache files in space to speed up rendering (#299)

robkooper · web-flow · commit cb32180f391f · 2021-11-09T08:56:01.000-06:00
* cache files in space to speed up rendering

for spaces with large number of datasets this would result in compute times of minutes. Now data is cached and will speed up processing.

* fix test

* decreate counts of files/bytes when file deleted
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## Unreleased
+
+See [fix-counts.js](scripts/updates/fix-count.js) for a script that can be run before this update (as well as the
+update for 1.19.0) to pre populate the migration. This will speed up the update and will not impact the running
+instance.
+
+### Fixed
+- If a space has a lot of datasets, rendering the space page is very slow. Files in a space is now cached.
+
 ## 1.19.2 - 2021-10-20
 
 ### Fixed
diff --git a/app/api/Files.scala b/app/api/Files.scala
@@ -1678,6 +1678,7 @@ class Files @Inject()(
           val ds_spaces = ds.spaces
           for (ds_s <- ds_spaces) {
             spaces.decrementSpaceBytes(ds_s, file.length)
+            spaces.decrementFileCounter(ds_s, 1)
           }
         }
 
diff --git a/app/api/Spaces.scala b/app/api/Spaces.scala
@@ -41,7 +41,7 @@ class Spaces @Inject()(spaces: SpaceService,
         val userId = request.user.get.id
         val c = ProjectSpace(name = name, description = description, created = new Date(), creator = userId,
           homePage = List.empty, logoURL = None, bannerURL = None, collectionCount = 0,
-          datasetCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty)
+          datasetCount = 0, fileCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty)
         spaces.insert(c) match {
           case Some(id) => {
             appConfig.incrementCount('spaces, 1)
diff --git a/app/controllers/SecuredController.scala b/app/controllers/SecuredController.scala
@@ -153,7 +153,7 @@ trait SecuredController extends Controller {
         val spaces: SpaceService = DI.injector.getInstance(classOf[SpaceService])
         spaces.get(id) match {
           case None => Future.successful(BadRequest(views.html.notFound(spaceTitle + " does not exist.")(user)))
-          case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List(),0,0)(user)))
+          case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List())(user)))
         }
       }
 
diff --git a/app/controllers/Spaces.scala b/app/controllers/Spaces.scala
@@ -167,8 +167,6 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
         var creatorActual: User = null
         val collectionsInSpace = spaces.getCollectionsInSpace(Some(id.stringify), Some(size))
         val datasetsInSpace = datasets.listSpace(size, id.toString(), user)
-        val spaceBytes : Long = s.spaceBytes
-        val spaceFiles : Integer = getFilesPerSpace(id, user)
         val publicDatasetsInSpace = datasets.listSpaceStatus(size, id.toString(), "publicAll", user)
         val usersInSpace = spaces.getUsersInSpace(id, None)
         var curationObjectsInSpace: List[CurationObject] = List()
@@ -217,7 +215,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
           case None => List.empty
         }
         sinkService.logSpaceViewEvent(s, user)
-        Ok(views.html.spaces.space(Utils.decodeSpaceElements(s), collectionsInSpace, publicDatasetsInSpace, datasetsInSpace, rs, play.Play.application().configuration().getString("SEADservices.uri"), userRoleMap, userSelections, spaceBytes, spaceFiles))
+        Ok(views.html.spaces.space(Utils.decodeSpaceElements(s), collectionsInSpace, publicDatasetsInSpace, datasetsInSpace, rs, play.Play.application().configuration().getString("SEADservices.uri"), userRoleMap, userSelections))
       }
       case None => BadRequest(views.html.notFound(spaceTitle + " does not exist."))
     }
@@ -414,7 +412,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
                       val newSpace = ProjectSpace(name = formData.name, description = formData.description,
                         created = new Date, creator = userId, homePage = formData.homePage,
                         logoURL = formData.logoURL, bannerURL = formData.bannerURL,
-                        collectionCount = 0, datasetCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty,
+                        collectionCount = 0, datasetCount = 0, fileCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty,
                         resourceTimeToLive = formData.resourceTimeToLive * 60 * 60 * 1000L, isTimeToLiveEnabled = formData.isTimeToLiveEnabled,
                         status = formData.access,
                         affiliatedSpaces = formData.affSpace)
@@ -640,15 +638,4 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
         case None => BadRequest(views.html.notFound(spaceTitle + " does not exist."))
       }
   }
-
-  private def getFilesPerSpace(spaceId: UUID, user: Option[User]) : Integer = {
-    var spaceFiles: Integer = 0
-    val allDatasetsInSpace = datasets.listSpace(0, spaceId.toString(), user)
-    for (ds <- allDatasetsInSpace) {
-      val files_in_ds = ds.files.length
-      spaceFiles += files_in_ds
-    }
-    spaceFiles
-  }
-
 }
diff --git a/app/models/Space.scala b/app/models/Space.scala
@@ -22,6 +22,7 @@ case class ProjectSpace (
   bannerURL: Option[URL],
   collectionCount: Integer,
   datasetCount: Integer,
+  fileCount: Integer,
   userCount: Integer,
   spaceBytes: Long, 
   metadata: List[Metadata],
diff --git a/app/services/SpaceService.scala b/app/services/SpaceService.scala
@@ -94,6 +94,10 @@ trait SpaceService {
 
   def incrementCollectionCounter(collection: UUID, space: UUID, increment: Int)
 
+  def incrementFileCounter(space: UUID, increment: Long)
+
+  def decrementFileCounter(space: UUID, decrement: Long)
+
   def incrementSpaceBytes(space: UUID, increment: Long)
 
   def decrementSpaceBytes(space: UUID, decrement: Long)
diff --git a/app/services/mongodb/MongoDBDatasetService.scala b/app/services/mongodb/MongoDBDatasetService.scala
@@ -1399,6 +1399,15 @@ class MongoDBDatasetService @Inject() (
         }
         for (f <- dataset.files) {
           val notTheDataset = for (currDataset <- findByFileIdDirectlyContain(f) if !dataset.id.toString.equals(currDataset.id.toString)) yield currDataset
+          files.get(f) match {
+            case Some(file) => {
+              for(space <- dataset.spaces) {
+                spaces.decrementFileCounter(space, 1)
+                spaces.decrementSpaceBytes(space, file.length)
+              }
+            }
+            case None => Logger.error(s"Error file with with id ${f} no longer exists")
+          }
           if (notTheDataset.size == 0)
             files.removeFile(f, host, apiKey, user)
         }
diff --git a/app/services/mongodb/MongoDBSpaceService.scala b/app/services/mongodb/MongoDBSpaceService.scala
@@ -375,15 +375,23 @@ class MongoDBSpaceService @Inject() (
   }
 
   def decrementCollectionCounter(collection: UUID, space: UUID, decrement: Int): Unit = {
-    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("collectionCount" -> -1), upsert=false, multi=false, WriteConcern.Safe)
+    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("collectionCount" -> -1 * decrement), upsert=false, multi=false, WriteConcern.Safe)
+  }
+
+  def incrementFileCounter(space: UUID, increment: Long ): Unit = {
+    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> increment), upsert=false, multi=false, WriteConcern.Safe)
+  }
+
+  def decrementFileCounter(space: UUID, decrement: Long): Unit = {
+    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> -1 * decrement), upsert=false, multi=false, WriteConcern.Safe)
   }
 
   def incrementSpaceBytes(space: UUID, increment: Long ): Unit = {
     ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> increment), upsert=false, multi=false, WriteConcern.Safe)
   }
 
   def decrementSpaceBytes(space: UUID, decrement: Long): Unit = {
-    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> decrement), upsert=false, multi=false, WriteConcern.Safe)
+    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> -1 * decrement), upsert=false, multi=false, WriteConcern.Safe)
   }
 
   def removeCollection(collection:UUID, space:UUID): Unit = {
@@ -398,11 +406,15 @@ class MongoDBSpaceService @Inject() (
    */
   def addDataset(dataset: UUID, space: UUID): Unit = {
     log.debug(s"Space Service - Adding $dataset to $space")
-    val datasetBytes = datasets.getBytesForDataset(dataset)
-    datasets.addToSpace(dataset, space)
-    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> datasetBytes), upsert=false, multi=false, WriteConcern.Safe)
-    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> 1), upsert=false, multi=false, WriteConcern.Safe)
-
+    datasets.get(dataset) match {
+      case Some(x) => {
+        val datasetBytes = datasets.getBytesForDataset(dataset)
+        datasets.addToSpace(dataset, space)
+        ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> datasetBytes), upsert=false, multi=false, WriteConcern.Safe)
+        ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> x.files.length), upsert=false, multi=false, WriteConcern.Safe)
+        ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> 1), upsert=false, multi=false, WriteConcern.Safe)
+      }
+    }
   }
 
   /**
@@ -414,9 +426,15 @@ class MongoDBSpaceService @Inject() (
   def removeDataset(dataset:UUID, space:UUID): Unit = {
     log.debug(s"Space Service - removing $dataset from $space")
     datasets.removeFromSpace(dataset, space)
-    val datasetBytes = datasets.getBytesForDataset(dataset)
-    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> -datasetBytes), upsert=false, multi=false, WriteConcern.Safe)
-    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> -1), upsert=false, multi=false, WriteConcern.Safe)
+    datasets.get(dataset) match {
+      case Some(x) => {
+        val datasetBytes = datasets.getBytesForDataset(dataset)
+        datasets.addToSpace(dataset, space)
+        ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> -1 * datasetBytes), upsert=false, multi=false, WriteConcern.Safe)
+        ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> -1 * x.files.length), upsert=false, multi=false, WriteConcern.Safe)
+        ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> -1), upsert=false, multi=false, WriteConcern.Safe)
+      }
+    }
   }
 
   /**
diff --git a/app/services/mongodb/MongoSalatPlugin.scala b/app/services/mongodb/MongoSalatPlugin.scala
@@ -452,6 +452,7 @@ class MongoSalatPlugin(app: Application) extends Plugin {
 
     // Adds space bytes to space
     updateMongo(updateKey = "update-space-bytes", updateSpaceBytes)
+    updateMongo(updateKey = "update-space-files", updateSpaceFiles)
   }
 
   private def updateMongo(updateKey: String, block: () => Unit): Unit = {
@@ -513,7 +514,8 @@ class MongoSalatPlugin(app: Application) extends Plugin {
         val spacename = java.net.InetAddress.getLocalHost.getHostName
         val newspace = new ProjectSpace(name = spacename, description = "", created = new Date(), creator = UUID("000000000000000000000000"),
           homePage = List.empty[URL], logoURL = None, bannerURL = None, metadata = List.empty[Metadata],
-          collectionCount = collections.toInt, datasetCount = datasets.toInt, userCount = users.toInt, spaceBytes = 0)
+          collectionCount = collections.toInt, datasetCount = datasets.toInt, userCount = users.toInt, fileCount = 0,
+          spaceBytes = 0)
         ProjectSpaceDAO.save(newspace)
         val spaceId = new ObjectId(newspace.id.stringify)
 
@@ -1707,4 +1709,16 @@ class MongoSalatPlugin(app: Application) extends Plugin {
       collection("spaces.projects").update(MongoDBObject("_id" -> spaceId), $set("spaceBytes" -> currentSpaceBytes))
     }
   }
+
+  private def updateSpaceFiles(): Unit = {
+    collection("spaces.projects").find().toList.foreach{ space =>
+      var fileCount: Integer = 0
+      val spaceId = space.get("_id")
+      val spaceDatasets = collection("datasets").find(MongoDBObject("spaces" -> spaceId)).toList
+      spaceDatasets.foreach{ spaceDataset =>
+        fileCount += spaceDataset.getAsOrElse[MongoDBList]("files", MongoDBList.empty).length
+      }
+      collection("spaces.projects").update(MongoDBObject("_id" -> spaceId), $set("fileCount" -> fileCount))
+    }
+  }
 }
diff --git a/app/util/FileUtils.scala b/app/util/FileUtils.scala
@@ -595,6 +595,7 @@ object FileUtils {
           val datasetSpaces = dataset.get.spaces
           for (s <- datasetSpaces){
             spaceService.incrementSpaceBytes(s, file.length)
+            spaceService.incrementFileCounter(s, 1)
           }
         }
       }
diff --git a/app/views/spaces/space.scala.html b/app/views/spaces/space.scala.html
@@ -1,4 +1,4 @@
-@(space: ProjectSpace, collections: List[Collection], publicDatasets: List[Dataset], datasets: List[Dataset], publishedData: List[play.api.libs.json.JsValue], servicesUrl: String, userRoleMap: Map[User, String], userSelections: List[String], spaceBytes: Long, spaceFiles: Integer)(implicit user: Option[models.User])
+@(space: ProjectSpace, collections: List[Collection], publicDatasets: List[Dataset], datasets: List[Dataset], publishedData: List[play.api.libs.json.JsValue], servicesUrl: String, userRoleMap: Map[User, String], userSelections: List[String])(implicit user: Option[models.User])
 @import play.api.libs.json._
 
 @import play.api.Play.current
@@ -119,10 +119,10 @@ <h3>@Messages("collections.title")</h3>
         @* right column, space statistics, actions *@
         <div class="col-md-4 col-lg-4 col-sm-4">
             @if(user.isDefined) {
-                @spaces.statistics(space, spaceBytes, None, spaceFiles )
+                @spaces.statistics(space, None )
                 @spaces.otherActions(space)
             } else {
-                @spaces.statistics(space, spaceBytes, None, spaceFiles)
+                @spaces.statistics(space, None)
             }
             @spaces.externalLinks(space.homePage, space, "row ds-section-sm")
             @if(play.Play.application().configuration().getBoolean("enablePublic")) {
diff --git a/app/views/spaces/statistics.scala.html b/app/views/spaces/statistics.scala.html
@@ -1,4 +1,4 @@
-@(space: ProjectSpace, spaceBytes: Long, isPublic: Option[Boolean], spaceFiles: Integer)(implicit user: Option[models.User])
+@(space: ProjectSpace, isPublic: Option[Boolean])(implicit user: Option[models.User])
 
 @import play.api.i18n.Messages
 @import _root_.util.Formatters._
@@ -12,23 +12,23 @@ <h3>Statistics</h3>
             <tbody>
                 <tr>
                     <td><b><a href="@routes.Collections.sortedListInSpace(space.id.stringify, 0,12,isPublic.getOrElse(true))">@Messages("collections.title")</a></b></td>
-                    <td><span class="pull-right">@space.collectionCount</span></td>
+                    <td><span class="pull-right">@humanReadableNumber(space.collectionCount)</span></td>
                 </tr>
                 <tr>
                     <td><b><a href="@routes.Datasets.sortedListInSpace(space.id.stringify, 0,12,isPublic.getOrElse(true))">@Messages("datasets.title")</a></b></td>
-                    <td><span class="pull-right">@space.datasetCount</span></td>
+                    <td><span class="pull-right">@humanReadableNumber(space.datasetCount)</span></td>
                 </tr>
                 <tr>
                     <td><b>Files:</b></td>
-                    <td><span class="pull-right">@spaceFiles</span></td>
+                    <td><span class="pull-right">@humanReadableNumber(space.fileCount)</span></td>
                 </tr>
                 <tr>
                     <td><b>Bytes:</b></td>
-                    <td><span class="pull-right">@humanReadableByteCount(spaceBytes)</span></td>
+                    <td><span class="pull-right">@humanReadableByteCount(space.spaceBytes)</span></td>
                 </tr>
                 <tr>
                     <td><b>Users:</b></td>
-                    <td><span class="pull-right">@space.userCount</span></td>
+                    <td><span class="pull-right">@humanReadableNumber(space.userCount)</span></td>
                 </tr>
             </tbody>
         </table>
diff --git a/scripts/updates/fix-counts.js b/scripts/updates/fix-counts.js
@@ -1,7 +1,28 @@
+db.app.configuration.update({"key" : "mongodb.updates"}, { $addToSet: {"value": "update-space-bytes"}})
+db.spaces.projects.update({"spaceBytes": {$exists : false}}, {$set: {'spaceBytes': NumberLong(0)}})
+db.app.configuration.update({"key" : "mongodb.updates"}, { $addToSet: {"value": "update-space-files"}})
+db.spaces.projects.update({"fileCount": {$exists : false}}, {$set: {'fileCount': 0}})
+
 db.app.configuration.update({"key" : "countof.users"}, {$set: { "value": NumberLong(db.social.users.count())}})
 db.app.configuration.update({"key" : "countof.datasets"}, {$set: { "value": NumberLong(db.datasets.count())}})
 db.app.configuration.update({"key" : "countof.files"}, {$set: { "value": NumberLong(db.uploads.count())}})
 db.app.configuration.update({"key" : "countof.bytes"}, {$set: { "value": db.uploads.aggregate({$group: {_id: "bytes", "sum": {$sum : "$length"}}}).next().sum}})
 db.app.configuration.update({"key" : "countof.collections"}, {$set: { "value": NumberLong(db.collections.count())}})
 db.app.configuration.update({"key" : "countof.spaces"}, {$set: { "value": NumberLong(db.spaces.projects.count())}})
 
+db.spaces.projects.find().sort({"datasetCount": 1}).forEach(function(s) {
+	var spaceBytes = 0;
+	var filecount = 0;
+	db.datasets.find({"spaces": s._id}).forEach(function(d) {
+		d.files.forEach(function(fid) {
+			db.uploads.find({"_id": fid}).forEach(function(f) {
+				spaceBytes += f.length;
+				filecount += 1;
+			});
+		});
+	});
+	s.spaceBytes = NumberLong(spaceBytes);
+	s.fileCount = filecount;
+	db.spaces.projects.save(s);
+	print(s["_id"] + " " + s.name + " = " + s.spaceBytes + " in " + s.fileCount + " files");
+})
diff --git a/test/integration/spaces/SpaceMongoDBSpec.scala b/test/integration/spaces/SpaceMongoDBSpec.scala
@@ -24,6 +24,7 @@ class SpaceMongoDBSpec extends PlaySpec with OneServerPerSuite {
     bannerURL = None,
     collectionCount = 0,
     datasetCount = 0,
+    fileCount = 0,
     userCount = 0,
     spaceBytes = 0,
     metadata = List.empty

Original file line number	Diff line number	Diff line change
`@@ -1678,6 +1678,7 @@ class Files @Inject()(`
`1678`	`1678`	`val ds_spaces = ds.spaces`
`1679`	`1679`	`for (ds_s <- ds_spaces) {`
`1680`	`1680`	`spaces.decrementSpaceBytes(ds_s, file.length)`
	`1681`	`+ spaces.decrementFileCounter(ds_s, 1)`
`1681`	`1682`	`}`
`1682`	`1683`	`}`
`1683`	`1684`
Original file line number	Diff line number	Diff line change
`@@ -153,7 +153,7 @@ trait SecuredController extends Controller {`
`153`	`153`	`val spaces: SpaceService = DI.injector.getInstance(classOf[SpaceService])`
`154`	`154`	`spaces.get(id) match {`
`155`	`155`	`case None => Future.successful(BadRequest(views.html.notFound(spaceTitle + " does not exist.")(user)))`
`156`		`- case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List(),0,0)(user)))`
	`156`	`+ case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List())(user)))`
`157`	`157`	`}`
`158`	`158`	`}`
`159`	`159`