Do not index datasets & collections in trash (#131)

* Do not index datasets & collections in trash

* Omit trashed files from query

Co-authored-by: Luigi Marini <[email protected]>

Author: max-zilla

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Changed
 - api/reports/storage/spaces endpoint now accepts a space parameter for ID rather than requiring a space filter.
+- Datasets and collections in the trash are no longer indexed for discovery in search services.
 
 ## 1.13.0 - 2020-12-02
 
@@ -40,7 +41,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ### Changed
 - Docker Images are now pushed to [github container registry](https://github.com/orgs/clowder-framework/packages)
 
-
 ## 1.12.0 - 2020-10-19
 **_Warning:_**
 - This update modifies the MongoDB schema. Make sure to start the application with `-DMONGOUPDATE=1`.

app/services/DatasetService.scala

@@ -385,4 +385,5 @@ trait DatasetService {
 
   def getIterator(space: Option[String], since: Option[String], until: Option[String]): Iterator[Dataset]
 
+  def getTrashedIds(): List[UUID]
 }

app/services/mongodb/MongoDBCollectionService.scala

@@ -894,7 +894,7 @@ class MongoDBCollectionService @Inject() (
 
   def indexAll(idx: Option[String] = None) = {
     // Bypass Salat in case any of the file records are malformed to continue past them
-    Collection.dao.collection.find(MongoDBObject(), MongoDBObject("_id" -> 1)).foreach(c => {
+    Collection.dao.collection.find(MongoDBObject("trash" -> false), MongoDBObject("_id" -> 1)).foreach(c => {
       index(new UUID(c.get("_id").toString), idx)
     })
   }

app/services/mongodb/MongoDBDatasetService.scala

@@ -1412,7 +1412,7 @@ class MongoDBDatasetService @Inject() (
 
   def indexAll(idx: Option[String] = None) = {
     // Bypass Salat in case any of the file records are malformed to continue past them
-    Dataset.dao.collection.find(MongoDBObject(), MongoDBObject("_id" -> 1)).foreach(d => {
+    Dataset.dao.collection.find(MongoDBObject("trash" -> false), MongoDBObject("_id" -> 1)).foreach(d => {
       index(new UUID(d.get("_id").toString), idx)
     })
   }
@@ -1648,6 +1648,16 @@ class MongoDBDatasetService @Inject() (
     until.foreach(t => query = query ++ ("created" $lte Parsers.fromISO8601(t)))
     Dataset.find(query)
   }
+
+  // Get a list of all trashed dataset and file ids for comparison
+  def getTrashedIds(): List[UUID] = {
+    val trashedIds = ListBuffer[UUID]()
+    Dataset.find(MongoDBObject("trash" -> true)).map(ds => {
+      ds.files.foreach(fid => trashedIds += fid)
+      trashedIds += ds.id
+    })
+    trashedIds.toList
+  }
 }
 
 object Dataset extends ModelCompanion[Dataset, ObjectId] {

app/services/mongodb/MongoDBFileService.scala

@@ -326,8 +326,11 @@ class MongoDBFileService @Inject() (
 
   def indexAll(idx: Option[String] = None) = {
     // Bypass Salat in case any of the file records are malformed to continue past them
+    val trashedIds = datasets.getTrashedIds()
     FileDAO.dao.collection.find(MongoDBObject(), MongoDBObject("_id" -> 1)).foreach(f => {
-      index(new UUID(f.get("_id").toString), idx)
+      val fid = new UUID(f.get("_id").toString)
+      if (!trashedIds.contains(fid))
+        index(fid, idx)
     })
   }