Merge pull request #147 from clowder-framework/release/1.13.0

v1.13.0

Author: lmarini

CHANGELOG.md

@@ -5,6 +5,19 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
 
+## 1.13.0 - 2020-12-02
+
+### Added
+- Ability to submit multiple selected files within a dataset to an extractor.
+- Support for Amplitude clickstream tracking. See Admin -> Customize to configure Amplitude apikey.
+- UpdateUserId.js to scripts/updates. This code adds user_id to each document in extractions collection in mongodb. 
+  user_id is taken from author id in uploads.files if exists, else it taken from author id in datasets collection.
+
+### Fixed
+- An extractor with file matching set to `*/*` (all file types) would incorrectly send out dataset events.
+- Space Editors can now delete tags on files, datasets and sections.
+- GeospatialViewer previewer no longer shows if file does not contain geospatial layers.
+
 ## 1.12.2 - 2020-11-19
 
 ### Changed

CONTRIBUTORS.md

@@ -21,6 +21,7 @@ Following is a list of contributors in alphabetical order:
 - Mario Felarca
 - Max Burnette
 - Michal Ondrejcek
+- Michael Johnson
 - Michelle Pitcel
 - Mike Bobak
 - Mike Lambert

app/api/Admin.scala

@@ -80,6 +80,8 @@ class Admin @Inject() (userService: UserService,
     getValueString(request.body, "sensor").foreach(AppConfiguration.setSensorTitle(_))
     getValueString(request.body, "parameters").foreach(AppConfiguration.setParametersTitle(_))
     getValueString(request.body, "parameter").foreach(AppConfiguration.setParameterTitle(_))
+    getValueString(request.body, "amplitudeApikey").foreach(AppConfiguration.setAmplitudeApiKey(_))
+
     getValueString(request.body, "tosText").foreach { tos =>
       events.addEvent(Event(request.user.get, event_type = EventType.TOS_UPDATE.toString))
       AppConfiguration.setTermsOfServicesText(tos)

app/api/Datasets.scala

@@ -1395,18 +1395,16 @@ class  Datasets @Inject()(
 
     val error_str = tagCheck.error_str
     val not_found = tagCheck.not_found
-    val userOpt = tagCheck.userOpt
-    val extractorOpt = tagCheck.extractorOpt
     val tags = tagCheck.tags
 
     // Now the real work: removing the tags.
     if ("" == error_str) {
       // Clean up leading, trailing and multiple contiguous white spaces.
       val tagsCleaned = tags.get.map(_.trim().replaceAll("\\s+", " "))
       (obj_type) match {
-        case TagCheck_File => files.removeTags(id, userOpt, extractorOpt, tagsCleaned)
+        case TagCheck_File => files.removeTags(id, tagsCleaned)
         case TagCheck_Dataset => {
-          datasets.removeTags(id, userOpt, extractorOpt, tagsCleaned)
+          datasets.removeTags(id, tagsCleaned)
           datasets.get(id) match {
             case Some(dataset) => {
               events.addObjectEvent(request.user, id, dataset.name, EventType.REMOVE_TAGS_DATASET.toString)
@@ -1416,7 +1414,7 @@ class  Datasets @Inject()(
 
         }
 
-        case TagCheck_Section => sections.removeTags(id, userOpt, extractorOpt, tagsCleaned)
+        case TagCheck_Section => sections.removeTags(id, tagsCleaned)
       }
       Ok(Json.obj("status" -> "success"))
     } else {

app/api/Extractions.scala

@@ -22,6 +22,7 @@ import services._
 
 import scala.collection.mutable.ListBuffer
 import scala.concurrent.Future
+import scala.util.parsing.json.JSONObject
 
 
 /**
@@ -585,6 +586,81 @@ class Extractions @Inject()(
     }
   }
 
+  def submitFilesToExtractor(ds_id: UUID, file_ids: String)= PermissionAction(Permission.EditDataset, Some(ResourceRef(ResourceRef.dataset,
+    ds_id)))(parse.json) { implicit request =>
+    current.plugin[RabbitmqPlugin] match {
+      case Some(p) => {
+        var results = Map[String, String]()
+        files.get(file_ids.split(",").map(fid => UUID(fid)).toList).found.foreach(fi => {
+          createFileSubmission(request, fi) match {
+            case Some(jobid) => results += (fi.id.stringify -> jobid.stringify)
+            case None => Logger.error("File not submitted: "+fi.id)
+          }
+        })
+        Ok(Json.obj("status" -> "success", "jobs" -> results))
+      }
+      case None =>
+        Ok(Json.obj("status" -> "error", "msg"-> "RabbitmqPlugin disabled"))
+    }
+  }
+
+  private def createFileSubmission(request: UserRequest[JsValue], file: File): Option[UUID] = {
+    current.plugin[RabbitmqPlugin] match {
+      case Some(p) => {
+        val id = file.id
+        val fileType = file.contentType
+        val idAndFlags = ""
+
+        // check that the file is ready for processing
+        if (file.status.equals(models.FileStatus.PROCESSED.toString)) {
+          // parameters for execution
+          val parameters = (request.body \ "parameters").asOpt[JsObject].getOrElse(JsObject(Seq.empty[(String, JsValue)]))
+
+          // Log request
+          val clientIP = request.remoteAddress
+          val serverIP = request.host
+          dtsrequests.insertRequest(serverIP, clientIP, file.filename, id, fileType, file.length, file.uploadDate)
+
+          val extra = Map("filename" -> file.filename,
+            "parameters" -> parameters,
+            "action" -> "manual-submission")
+          val showPreviews = file.showPreviews
+
+          val newFlags = if (showPreviews.equals("FileLevel"))
+            idAndFlags + "+filelevelshowpreviews"
+          else if (showPreviews.equals("None"))
+            idAndFlags + "+nopreviews"
+          else
+            idAndFlags
+
+          val originalId = if (!file.isIntermediate) {
+            file.id.toString()
+          } else {
+            idAndFlags
+          }
+
+          var datasetId: UUID = null
+          // search datasets containning this file, either directly under dataset or indirectly.
+          val datasetslists:List[Dataset] = datasets.findByFileIdAllContain(file.id)
+          // Note, we assume only at most one dataset will contain a given file.
+          if (0 != datasetslists.length) {
+            datasetId = datasetslists.head.id
+          }
+          // if extractor_id is not specified default to execution of all extractors matching mime type
+          (request.body \ "extractor").asOpt[String] match {
+            case Some(extractorId) => p.submitFileManually(new UUID(originalId), file, Utils.baseUrl(request), extractorId, extra,
+                datasetId, newFlags, request.apiKey, request.user)
+            case None => p.fileCreated(file, None, Utils.baseUrl(request), request.apiKey)
+          }
+        } else None
+      }
+      case None => {
+        Logger.error("RabbitMQ disabled.")
+        None
+      }
+    }
+  }
+
   def submitDatasetToExtractor(ds_id: UUID) = PermissionAction(Permission.EditDataset, Some(ResourceRef(ResourceRef.dataset,
     ds_id)))(parse.json) { implicit request =>
     Logger.debug(s"Submitting dataset for extraction with body $request.body")

app/controllers/Admin.scala

@@ -29,7 +29,8 @@ class Admin @Inject() (sectionIndexInfo: SectionIndexInfoService, userService: U
     Ok(views.html.admin.customize(theme,
       AppConfiguration.getDisplayName,
       AppConfiguration.getWelcomeMessage,
-      AppConfiguration.getGoogleAnalytics))
+      AppConfiguration.getGoogleAnalytics,
+      AppConfiguration.getAmplitudeApiKey))
   }
 
   def tos = ServerAdminAction { implicit request =>

app/controllers/Application.scala

@@ -353,6 +353,7 @@ class Application @Inject() (files: FileService, collections: CollectionService,
         api.routes.javascript.Datasets.users,
         api.routes.javascript.Datasets.restoreDataset,
         api.routes.javascript.Datasets.emptyTrash,
+        api.routes.javascript.Extractions.submitFilesToExtractor,
         api.routes.javascript.Files.download,
         api.routes.javascript.Files.archive,
         api.routes.javascript.Files.sendArchiveRequest,
@@ -528,6 +529,7 @@ class Application @Inject() (files: FileService, collections: CollectionService,
         controllers.routes.javascript.Extractors.selectExtractors,
         controllers.routes.javascript.Extractors.manageLabels,
         controllers.routes.javascript.Extractors.showJobHistory,
+        controllers.routes.javascript.Extractors.submitSelectedExtractions,
         controllers.routes.javascript.CurationObjects.submit,
         controllers.routes.javascript.CurationObjects.getCurationObject,
         controllers.routes.javascript.CurationObjects.getUpdatedFilesAndFolders,

app/controllers/Extractors.scala

@@ -350,6 +350,26 @@ class Extractors  @Inject() (extractions: ExtractionService,
     }
   }
 
+  def submitSelectedExtractions(ds_id: UUID) = PermissionAction(Permission.EditDataset, Some(ResourceRef(ResourceRef.dataset, ds_id))) { implicit request =>
+    implicit val user = request.user
+    val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"))
+    val extractors = all_extractors.filter(!_.process.file.isEmpty)
+    datasets.get(ds_id) match {
+      case Some(dataset) => {
+        val allDecodedDatasets = List(dataset)
+        val decodedSpacesContaining = ListBuffer.empty[models.ProjectSpace]
+        dataset.spaces.map{ sp =>
+          spaces.get(sp) match {
+            case Some(s) => decodedSpacesContaining += Utils.decodeSpaceElements(s)
+            case None => Logger.error("Dataset "+dataset.id.stringify+" space not found: "+sp.stringify)
+          }
+        }
+        Ok(views.html.extractions.submitSelectedExtraction(extractors, dataset))
+      }
+      case None => InternalServerError("Dataset not found")
+    }
+  }
+
   def submitDatasetExtraction(ds_id: UUID) = PermissionAction(Permission.EditDataset, Some(ResourceRef(ResourceRef.dataset, ds_id))) { implicit request =>
     implicit val user = request.user
     val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"))

app/models/ResourceType.scala

@@ -7,7 +7,7 @@ import play.api.libs.json._
  */
 object ResourceType extends Enumeration {
 //  type ResourceType = Value
-  val dataset, file, collection, user, sensor, stream, folder = Value
+  val dataset, file, collection, user, sensor, stream, folder, metadata = Value
 
   def isWorkingDay(d: ResourceType.Value) = ! (d == Dataset || d == File)
 }

app/services/AppConfigurationService.scala

@@ -116,11 +116,23 @@ object AppConfiguration {
   /** Set the google analytics code */
   def setGoogleAnalytics(gacode: String) = appConfig.setProperty("google.analytics", gacode)
 
-  /** Get the welcome message */
+  /** Get the google analytics code */
   def getGoogleAnalytics: String = appConfig.getProperty("google.analytics", "")
 
   // ----------------------------------------------------------------------
 
+  /** Set the Amplitude clickstream/analytics configuration */
+  def setAmplitudeApiKey(ampApiKey: String) = {
+    appConfig.setProperty("amplitude.apikey", ampApiKey)
+  }
+
+  /** Get the Amplitude clickstream/analytics configuration */
+  def getAmplitudeApiKey: String = {
+    appConfig.getProperty("amplitude.apikey", "")
+  }
+
+  // ----------------------------------------------------------------------
+
   /** Set the Sensors title */
   def setSensorsTitle(sensorsTitle: String) = appConfig.setProperty("sensors.title", sensorsTitle)