Merge pull request #106 from clowder-framework/extractor-catalog-labels

Extractor Catalog: Extractor Labels features

Author: max-zilla

CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## Unreleased
+
+### Fixed
+- Fixed permissions checks on search results for search interfaces that would cause misleading counts. [#60](https://github.com/clowder-framework/clowder/issues/60)
+
 ## 1.12.0 - 2020-10-19
 **_Warning:_**
 - This update modifies the MongoDB schema. Make sure to start the application with `-DMONGOUPDATE=1`.
@@ -24,7 +29,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - Ignore the `update` field when posting to `/api/extractors`. [#89](https://github.com/clowder-framework/clowder/issues/89)
 - Search results were hardcoded to be in batches of 2.
 
-# 1.11.2 - 2020-10-13
+## 1.11.2 - 2020-10-13
 
 ### Fixed
 - Clowder healthcheck was not correct, resulting in docker-compose never thinking it was healthy. This could also result 

app/api/Extractions.scala

@@ -3,8 +3,8 @@ package api
 import java.io.{FileInputStream, InputStream}
 import java.net.URL
 import java.util.Calendar
-import javax.inject.Inject
 
+import javax.inject.Inject
 import controllers.Utils
 import fileutils.FilesUtils
 import models._
@@ -475,7 +475,32 @@ class Extractions @Inject()(
       },
       info => {
         extractors.updateExtractorInfo(info) match {
-          case Some(u) => Ok(Json.obj("status" -> "OK", "message" -> ("Extractor info updated. ID = " + u.id)))
+          case Some(u) => {
+            // Create/assign any default labels for this extractor
+            u.defaultLabels.foreach(labelStr => {
+              val segments = labelStr.split("/")
+              val (labelName, labelCategory) = if (segments.length > 1) {
+                (segments(1), segments(0))
+              } else {
+                (segments(0), "Other")
+              }
+              extractors.getExtractorsLabel(labelName) match {
+                case None => {
+                  // Label does not exist - create and assign it
+                  val createdLabel = extractors.createExtractorsLabel(labelName, Some(labelCategory), List[String](u.name))
+                }
+                case Some(lbl) => {
+                  // Label already exists, assign it
+                  if (!lbl.extractors.contains(u.name)) {
+                    val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](u.name))
+                    val updatedLabel = extractors.updateExtractorsLabel(label)
+                  }
+                }
+              }
+            })
+
+            Ok(Json.obj("status" -> "OK", "message" -> ("Extractor info updated. ID = " + u.id)))
+          }
           case None => BadRequest(Json.obj("status" -> "KO", "message" -> "Error updating extractor info"))
         }
       }
@@ -674,4 +699,71 @@ class Extractions @Inject()(
     Ok(toJson("added new event"))
   }
 
+  def createExtractorsLabel() = ServerAdminAction(parse.json) { implicit request =>
+    // Fetch parameters from request body
+    val (name, category, assignedExtractors) = parseExtractorsLabel(request)
+
+    // Validate that name is not empty
+    if (name.isEmpty) {
+      BadRequest("Label Name cannot be empty")
+    } else {
+      // Validate that name is unique
+      extractors.getExtractorsLabel(name) match {
+        case Some(lbl) => Conflict("Label name is already in use: " + lbl.name)
+        case None => {
+          // Create the new label
+          val label = extractors.createExtractorsLabel(name, category, assignedExtractors)
+          Ok(Json.toJson(label))
+        }
+      }
+    }
+  }
+
+  def updateExtractorsLabel(id: UUID) = ServerAdminAction(parse.json) { implicit request =>
+    // Fetch parameters from request body
+    val (name, category, assignedExtractors) = parseExtractorsLabel(request)
+
+    // Validate that name is not empty
+    if (name.isEmpty) {
+      BadRequest("Label Name cannot be empty")
+    } else {
+      // Validate that name is still unique
+      extractors.getExtractorsLabel(name) match {
+        case Some(lbl) => {
+          // Exclude current id (in case name hasn't changed)
+          if (lbl.id != id) {
+            Conflict("Label name is already in use: " + lbl.name)
+          } else {
+            // Update the label
+            val updatedLabel = extractors.updateExtractorsLabel(ExtractorsLabel(id, name, category, assignedExtractors))
+            Ok(Json.toJson(updatedLabel))
+          }
+        }
+        case None => {
+          // Update the label
+          val updatedLabel = extractors.updateExtractorsLabel(ExtractorsLabel(id, name, category, assignedExtractors))
+          Ok(Json.toJson(updatedLabel))
+        }
+      }
+    }
+  }
+
+  def deleteExtractorsLabel(id: UUID) = ServerAdminAction { implicit request =>
+    // Fetch existing label
+    extractors.getExtractorsLabel(id) match {
+      case Some(lbl) => {
+        val deleted = extractors.deleteExtractorsLabel(lbl)
+        Ok(Json.toJson(deleted))
+      }
+      case None => BadRequest("Failed to delete label: " + id)
+    }
+  }
+
+  def parseExtractorsLabel(request: UserRequest[JsValue]): (String, Option[String], List[String]) = {
+    val name = (request.body \ "name").as[String]
+    val category = (request.body \ "category").asOpt[String]
+    val assignedExtractors = (request.body \ "extractors").as[List[String]]
+
+    (name, category, assignedExtractors)
+  }
 }

app/api/Search.scala

@@ -1,8 +1,7 @@
 package api
 
 import api.Permission._
-import services.{RdfSPARQLService, DatasetService, FileService, CollectionService, PreviewService, SpaceService,
-MultimediaQueryService, ElasticsearchPlugin}
+import services.{RdfSPARQLService, PreviewService, SpaceService, MultimediaQueryService, ElasticsearchPlugin}
 import play.Logger
 import scala.collection.mutable.{ListBuffer, HashMap}
 import util.{SearchUtils, SearchResult}
@@ -15,9 +14,6 @@ import models._
 
 @Singleton
 class Search @Inject() (
-   files: FileService,
-   datasets: DatasetService,
-   collections: CollectionService,
    previews: PreviewService,
    queries: MultimediaQueryService,
    spaces: SpaceService,
@@ -49,10 +45,16 @@ class Search @Inject() (
           (tag match {case Some(x) => s"&tag=$x" case None => ""})
 
         // Add space filter to search here as a simple permissions check
-        val permitted = spaces.listAccess(0, Set[Permission](Permission.ViewSpace), request.user, true, true, false, false).map(sp => sp.id)
+        val superAdmin = request.user match {
+          case Some(u) => u.superAdminMode
+          case None => false
+        }
+        val permitted = if (superAdmin)
+          List[UUID]()
+        else
+          spaces.listAccess(0, Set[Permission](Permission.ViewSpace), request.user, true, true, false, false).map(sp => sp.id)
 
         val response = plugin.search(query, resource_type, datasetid, collectionid, spaceid, folderid, field, tag, from_index, size, permitted, request.user)
-
         val result = SearchUtils.prepareSearchResponse(response, source_url, request.user)
         Ok(toJson(result))
       }
@@ -70,8 +72,19 @@ class Search @Inject() (
 
       current.plugin[ElasticsearchPlugin] match {
         case Some(plugin) => {
+          // Add space filter to search here as a simple permissions check
+          val superAdmin = request.user match {
+            case Some(u) => u.superAdminMode
+            case None => false
+          }
+          val permitted = if (superAdmin)
+            List[UUID]()
+          else
+            spaces.listAccess(0, Set[Permission](Permission.ViewSpace), request.user, true, true, false, false).map(sp => sp.id)
+
+
           val queryList = Json.parse(query).as[List[JsValue]]
-          val response = plugin.search(queryList, grouping, from, size, user)
+          val response = plugin.search(queryList, grouping, from, size, permitted, user)
 
           // TODO: Better way to build a URL?
           val source_url = s"/api/search?query=$query&grouping=$grouping"

app/controllers/Application.scala

@@ -496,6 +496,9 @@ class Application @Inject() (files: FileService, collections: CollectionService,
         api.routes.javascript.Extractions.addExtractorInfo,
         api.routes.javascript.Extractions.getExtractorInfo,
         api.routes.javascript.Extractions.deleteExtractor,
+        api.routes.javascript.Extractions.createExtractorsLabel,
+        api.routes.javascript.Extractions.updateExtractorsLabel,
+        api.routes.javascript.Extractions.deleteExtractorsLabel,
         api.routes.javascript.Folders.createFolder,
         api.routes.javascript.Folders.deleteFolder,
         api.routes.javascript.Folders.updateFolderName,
@@ -523,6 +526,7 @@ class Application @Inject() (files: FileService, collections: CollectionService,
         controllers.routes.javascript.Collections.newCollectionWithParent,
         controllers.routes.javascript.Spaces.stagingArea,
         controllers.routes.javascript.Extractors.selectExtractors,
+        controllers.routes.javascript.Extractors.manageLabels,
         controllers.routes.javascript.Extractors.showJobHistory,
         controllers.routes.javascript.CurationObjects.submit,
         controllers.routes.javascript.CurationObjects.getCurationObject,

app/controllers/Extractors.scala

@@ -55,6 +55,7 @@ class Extractors  @Inject() (extractions: ExtractionService,
     implicit val user = request.user
     // Filter extractors by the trigger type if necessary
     var runningExtractors: List[ExtractorInfo] = extractorService.listExtractorsInfo(List.empty)
+
     request.getQueryString("processTriggerSearchFilter") match {
       case Some("file/*") => runningExtractors = runningExtractors.filter(re => re.process.file.length > 0)
       case Some("dataset/*") => runningExtractors = runningExtractors.filter(re => re.process.dataset.length > 0)
@@ -63,7 +64,26 @@ class Extractors  @Inject() (extractions: ExtractionService,
     }
     val selectedExtractors: List[String] = extractorService.getEnabledExtractors()
     val groups = extractions.groupByType(extractions.findAll())
-    Ok(views.html.updateExtractors(runningExtractors, selectedExtractors, groups))
+    val allLabels = extractorService.listExtractorsLabels()
+    val categorizedLabels = allLabels.groupBy(_.category.getOrElse("Other"))
+    request.getQueryString("labelFilter") match {
+      case None => {}
+      case Some(lblName) => allLabels.find(lbl => lblName == lbl.name) match {
+        case None => {}
+        case Some(label) => runningExtractors = runningExtractors.filter(re => label.extractors.contains(re.name))
+      }
+    }
+
+    Ok(views.html.updateExtractors(runningExtractors, selectedExtractors, groups, categorizedLabels))
+  }
+
+  def manageLabels = ServerAdminAction { implicit request =>
+    implicit val user = request.user
+    val categories = List[String]("EXTRACT")
+    val extractors = extractorService.listExtractorsInfo(categories)
+    val labels = extractorService.listExtractorsLabels()
+
+    Ok(views.html.extractorLabels(labels, extractors))
   }
 
   /**
@@ -107,7 +127,10 @@ class Extractors  @Inject() (extractions: ExtractionService,
     implicit val user = request.user
     val targetExtractor = extractorService.listExtractorsInfo(List.empty).find(p => p.name == extractorName)
     targetExtractor match {
-      case Some(extractor) => Ok(views.html.extractorDetails(extractor))
+      case Some(extractor) => {
+        val labels = extractorService.getLabelsForExtractor(extractor.name)
+        Ok(views.html.extractorDetails(extractor, labels))
+      }
       case None => InternalServerError("Extractor not found: " + extractorName)
     }
   }

app/models/Extraction.scala

@@ -72,6 +72,10 @@ case class ExtractorDetail(
  * @param external_services external services used by the extractor
  * @param libraries libraries on which the code depends
  * @param bibtex bibtext formatted citation of relevant papers
+ * @param maturity indicates whether this extractor is ready for public consumption
+ *             * For example: "Development" (default), "Staging", or "Production"
+ * @param defaultLabels the categorization label names that were imported by default
+ *             * For example: "Image", "Video", "Audio", etc
  * @param process events that should trigger this extractor to process
  * @param categories list of categories that apply to the extractor
  * @param parameters JSON schema representing allowed parameters
@@ -94,6 +98,7 @@ case class ExtractorInfo(
   libraries: List[String],
   bibtex: List[String],
   maturity: String = "Development",
+  defaultLabels: List[String] = List[String](),
   process: ExtractorProcessTriggers = new ExtractorProcessTriggers(),
   categories: List[String] = List[String](ExtractorCategory.EXTRACT.toString),
   parameters: JsValue = JsObject(Seq())
@@ -146,6 +151,7 @@ object ExtractorInfo {
       (JsPath \ "libraries").read[List[String]].orElse(Reads.pure(List.empty)) and
       (JsPath \ "bibtex").read[List[String]].orElse(Reads.pure(List.empty)) and
       (JsPath \ "maturity").read[String].orElse(Reads.pure("Development")) and
+      (JsPath \ "labels").read[List[String]].orElse(Reads.pure(List.empty)) and
       (JsPath \ "process").read[ExtractorProcessTriggers].orElse(Reads.pure(new ExtractorProcessTriggers())) and
       (JsPath \ "categories").read[List[String]].orElse(Reads.pure(List[String](ExtractorCategory.EXTRACT.toString))) and
       (JsPath \ "parameters").read[JsValue].orElse(Reads.pure(JsObject(Seq())))
@@ -185,3 +191,27 @@ case class ExtractionGroup(
                           latestMsg: String,
                           allMsgs: Map[UUID, List[Extraction]]
                           )
+
+case class ExtractorsLabel(
+                            id: UUID,
+                            name: String,
+                            category: Option[String],
+                            extractors: List[String]
+                         )
+
+object ExtractorsLabel {
+  implicit val writes = new Writes[ExtractorsLabel] {
+    def writes(label: ExtractorsLabel) = Json.obj(
+      "id" -> label.id,
+      "name" -> label.name,
+      "category" -> label.category,
+      "extractors" -> label.extractors
+    )
+  }
+  implicit val reads = (
+    (JsPath \ "id").read[UUID] and
+      (JsPath \ "name").read[String] and
+      (JsPath \ "category").readNullable[String] and
+      (JsPath \ "extractors").read[List[String]].orElse(Reads.pure(List.empty))
+    )(ExtractorsLabel.apply _)
+}