Merge branch '2.0-work-in-progress' into for-2.0-rabbitmq-service

# Conflicts:
#	app/api/Datasets.scala
#	app/api/Files.scala
#	app/api/Indexes.scala
#	app/api/Metadata.scala
#	app/api/Status.scala
#	app/controllers/Files.scala
#	app/services/DI.scala
#	app/services/mongodb/MongoDBFileService.scala
#	app/services/mongodb/MongoDBMetadataService.scala
#	doc/src/sphinx/admin/customizing.rst
#	docker/play.plugins

Author: toddn

CHANGELOG.md

@@ -33,8 +33,16 @@ Clowder runs in a location that doesn't not have access to https://clowderframew
 
 ### Changed
 - Include collection prefix in path when saving to S3.
+- Include length of file in `FileService` when saving the bytes to any backend service. This helps optimize S3 implementation.
 - Upgraded sbt from 0.13.0 to 0.13.6 to fix build failures.
   [CATS-1038](https://opensource.ncsa.illinois.edu/jira/browse/CATS-1038)
+- Remove Elasticsearch plugin and replace it with a `SearchService` framework and implementation. This separates out the 
+  logic that handles queuing and the logic that handles searching and indexing to be more modular and includes a memory-
+  based queuing service in addition to the MongoDB service.
+
+### Fixed
+- Calling api/Files.removeFile should no longer decrement related counters twice.
+  [CATS-929](https://opensource.ncsa.illinois.edu/jira/browse/CATS-929)
 
 ## 1.8.0 - 2019-11-06
 **_Warning:_ This update adds a new permission for archiving files and adds it to the Admin role. Please make sure

app/api/Admin.scala

@@ -5,7 +5,6 @@ import java.util.Date
 import javax.inject.Inject
 import models._
 import org.apache.commons.lang3.StringEscapeUtils
-import play.api.libs.concurrent.Akka
 import play.api.mvc.Controller
 import play.api.Play.current
 import play.api.libs.json.Json.toJson
@@ -15,8 +14,6 @@ import services.mongodb.MongoSalatPlugin
 import play.api.Logger
 import util.Mail
 
-import scala.concurrent.duration._
-import play.api.libs.concurrent.Execution.Implicits._
 import play.api.libs.json.{JsString, JsUndefined, JsValue}
 
 /**
@@ -27,14 +24,14 @@ class Admin @Inject() (userService: UserService,
     collections: CollectionService,
     files: FileService,
     events: EventService,
-    esqueue: ElasticsearchQueue) extends Controller with ApiController {
+    searches: SearchService) extends Controller with ApiController {
 
   /**
    * DANGER: deletes all data, keep users.
    */
   def deleteAllData(resetAll: Boolean) = ServerAdminAction { implicit request =>
     current.plugin[MongoSalatPlugin].map(_.dropAllData(resetAll))
-    current.plugin[ElasticsearchPlugin].map(_.deleteAll)
+    searches.deleteAll
 
     Ok(toJson("done"))
   }
@@ -174,8 +171,7 @@ class Admin @Inject() (userService: UserService,
   }
 
   def reindex = ServerAdminAction { implicit request =>
-    val success = esqueue.queue("index_all")
-    if (success) Ok(toJson(Map("status" -> "reindex successfully queued")))
-    else BadRequest(toJson(Map("status" -> "reindex queuing failed, Elasticsearch may be disabled")))
+    val msg = searches.indexAll()
+    Ok(toJson(Map("status" -> msg)))
   }
 }

app/api/Collections.scala

@@ -43,8 +43,7 @@ class Collections @Inject() (datasets: DatasetService,
                              folders : FolderService,
                              files: FileService,
                              metadataService : MetadataService,
-                             adminsNotifierService: AdminsNotifierService,
-                             esqueue: ElasticsearchQueue) extends ApiController {
+                             adminsNotifierService: AdminsNotifierService) extends ApiController {
 
   def createCollection() = PermissionAction(Permission.CreateCollection) (parse.json) { implicit request =>
     Logger.debug("Creating new collection")
@@ -116,9 +115,8 @@ class Collections @Inject() (datasets: DatasetService,
   def reindex(id: UUID, recursive: Boolean) = PermissionAction(Permission.CreateCollection, Some(ResourceRef(ResourceRef.collection, id))) {  implicit request =>
       collections.get(id) match {
         case Some(coll) => {
-          val success = esqueue.queue("index_collection", new ResourceRef('collection, id), new ElasticsearchParameters(recursive=recursive))
-          if (success) Ok(toJson(Map("status" -> "reindex successfully queued")))
-          else BadRequest(toJson(Map("status" -> "reindex queuing failed, Elasticsearch may be disabled")))
+          collections.index(id)
+          Ok(toJson(s"Collection $id reindexed"))
         }
         case None => {
           Logger.error("Error getting collection" + id)

app/api/Comments.scala

@@ -16,7 +16,11 @@ import play.api.i18n.Messages
  * Comments on datasets.
  *
  */
-class Comments @Inject()(datasets: DatasetService, comments: CommentService, events: EventService, users: UserService) extends ApiController {
+class Comments @Inject()(datasets: DatasetService,
+												 comments: CommentService,
+												 events: EventService,
+												 users: UserService,
+												 searches: SearchService) extends ApiController {
 
   def comment(id: UUID) = PermissionAction(Permission.AddComment, Some(ResourceRef(ResourceRef.comment, id)))(parse.json) { implicit request =>
       Logger.trace("Adding comment")
@@ -37,9 +41,7 @@ class Comments @Inject()(datasets: DatasetService, comments: CommentService, eve
                   if (parent.dataset_id.isDefined) {
                     datasets.get(parent.dataset_id.get) match {
                       case Some(dataset) => {
-                        current.plugin[ElasticsearchPlugin].foreach {
-                          _.index(dataset, false)
-                        }
+                        searches.index(dataset, false)
                       }
                       case None => Logger.error("Dataset not found: " + id)
                     }

app/api/Extractions.scala

@@ -85,7 +85,9 @@ class Extractions @Inject()(
             val fid = for {response <- futureResponse} yield {
               if (response.status == 200) {
                 val inputStream: InputStream = response.ahcResponse.getResponseBodyAsStream()
-                val file = files.save(inputStream, filename, response.header("Content-Type"), user, null)
+                val contentLengthStr = response.header("Content-Length").getOrElse("-1")
+                val contentLength = Integer.parseInt(contentLengthStr).toLong
+                val file = files.save(inputStream, filename, contentLength, response.header("Content-Type"), user, null)
                 file match {
                   case Some(f) => {
                     // Add new file & byte count to appConfig

app/api/Files.scala

@@ -507,20 +507,6 @@ class Files @Inject()(
       }
     }
 
-  /**
-    * Upload file using multipart form enconding.
-    */
-  @deprecated
-  def upload(showPreviews: String = "DatasetLevel", originalZipFile: String = "", flagsFromPrevious: String = "") = PermissionAction(Permission.AddFile)(parse.multipartFormData) { implicit request =>
-    val uploadedFiles = FileUtils.uploadFilesMultipart(request, showPreviews = showPreviews, originalZipFile = originalZipFile,
-      flagsFromPrevious = flagsFromPrevious, apiKey = request.apiKey)
-    uploadedFiles.length match {
-      case 0 => BadRequest("No files uploaded")
-      case 1 => Ok(Json.obj("id" -> uploadedFiles.head.id))
-      case _ => Ok(Json.obj("ids" -> uploadedFiles.toList))
-    }
-  }
-
   /**
     * Upload a file to a specific dataset
     */

app/api/Geometry.scala

@@ -78,7 +78,8 @@ class Logos @Inject()(logos: LogoService) extends ApiController {
           request.body.file("image") match {
             case Some(f) => {
               val ct = util.FileUtils.getContentType(f.filename, f.contentType)
-              logos.save(new FileInputStream(f.ref.file), p, name, showText, Some(ct), user) match {
+              logos.save(new FileInputStream(f.ref.file), p, name, showText,
+                f.ref.file.length, Some(ct), user) match {
                 case Some(logo) => {
                   // delete old images
                   logos.list(Some(p), Some(name)).foreach { l =>

app/api/Logos.scala

@@ -99,7 +99,7 @@ class Previews @Inject()(previews: PreviewService, tiles: TileService) extends A
   /**
    * Upload a preview.
    */
-  def upload(iipKey: String = "") =
+  def upload() =
     PermissionAction(Permission.AddFile)(parse.multipartFormData) { implicit request =>
         request.body.file("File").map { f =>
           try {
@@ -110,26 +110,8 @@ class Previews @Inject()(previews: PreviewService, tiles: TileService) extends A
             if (f.contentType.getOrElse("application/octet-stream").equals("application/xml"))
               realContentType = Some("application/dzi")
 
-            val id = UUID(previews.save(new FileInputStream(f.ref.file), f.filename, realContentType))
+            val id = UUID(previews.save(new FileInputStream(f.ref.file), f.filename, f.ref.file.length, realContentType))
             Logger.debug("ctp: " + realContentType)
-            // for IIP server references, store the IIP URL, key and filename on the IIP server for possible later deletion of the previewed file
-            if (f.filename.endsWith(".imageurl")) {
-              val iipRefReader = new BufferedReader(new FileReader(f.ref.file));
-
-              val serverLine = iipRefReader.readLine()
-              var urlEnd = serverLine.indexOf("/", serverLine.indexOf("://") + 3)
-              if (urlEnd == -1) {
-                urlEnd = serverLine.length()
-              }
-              val iipURL = serverLine.substring(8, urlEnd)
-
-              val imageLine = iipRefReader.readLine()
-              val iipImage = imageLine.substring(imageLine.lastIndexOf("/") + 1)
-
-              iipRefReader.close()
-
-              previews.setIIPReferences(id, iipURL, iipImage, iipKey)
-            }
 
             // Check whether a title for the preview was sent
             request.body.dataParts.get("title") match {
@@ -259,60 +241,6 @@ class Previews @Inject()(previews: PreviewService, tiles: TileService) extends A
         }
     }
 
-  /**
-   * Add annotation to 3D model preview.
-   */
-  def attachAnnotation(preview_id: UUID) = PermissionAction(Permission.AddFile, Some(ResourceRef(ResourceRef.preview, preview_id)))(parse.json) { implicit request =>
-        val x_coord = (request.body \ "x_coord").asOpt[String].getOrElse("0.0")
-        val y_coord = (request.body \ "y_coord").asOpt[String].getOrElse("0.0")
-        val z_coord = (request.body \ "z_coord").asOpt[String].getOrElse("0.0")
-        val description = (request.body \ "description").asOpt[String].getOrElse("")
-
-        previews.get(preview_id) match {
-          case Some(preview) => {
-            val annotation = ThreeDAnnotation(x_coord, y_coord, z_coord, description)
-            previews.annotation(preview_id, annotation)
-            Ok(toJson(Map("status" -> "success")))
-          }
-          case None => BadRequest(toJson("Preview not found " + preview_id))
-        }
-    }
-
-  def editAnnotation(preview_id: UUID) = PermissionAction(Permission.AddFile, Some(ResourceRef(ResourceRef.preview, preview_id)))(parse.json) { implicit request =>
-        Logger.debug("thereq: " + request.body.toString)
-        val x_coord = (request.body \ "x_coord").asOpt[String].getOrElse("0.0")
-        val y_coord = (request.body \ "y_coord").asOpt[String].getOrElse("0.0")
-        val z_coord = (request.body \ "z_coord").asOpt[String].getOrElse("0.0")
-        val description = (request.body \ "description").asOpt[String].getOrElse("")
-
-        previews.get(preview_id) match {
-          case Some(preview) => {
-            previews.findAnnotation(preview_id, x_coord, y_coord, z_coord) match {
-              case Some(annotation) => {
-                previews.updateAnnotation(preview_id, annotation.id, description)
-                Ok(toJson(Map("status" -> "success")))
-              }
-              case None => Ok(toJson(Map("status" -> "success"))) //What the user sees locally must not change if an annotation is deleted after the user loads the dataset
-              //but before attempting to modify the selected annotation's description.
-              //BadRequest(toJson("Annotation for preview " + preview_id + " not found: " + x_coord + "," + y_coord + "," + z_coord))
-            }
-          }
-          case None => BadRequest(toJson("Preview not found " + preview_id))
-        }
-    }
-
-  def listAnnotations(preview_id: UUID) = PermissionAction(Permission.ViewFile, Some(ResourceRef(ResourceRef.preview, preview_id))) { implicit request =>
-        previews.get(preview_id) match {
-          case Some(preview) => {
-            val annotationsOfPreview = previews.listAnnotations(preview_id)
-            val list = for (annotation <- annotationsOfPreview) yield jsonAnnotation(annotation)
-            Logger.debug("thelist: " + toJson(list))
-            Ok(toJson(list))
-          }
-          case None => BadRequest(toJson("Preview not found " + preview_id))
-        }
-    }
-
   /**
     * Update the title field of a preview to change what is displayed on preview tab
     * @param preview_id UUID of preview to change

app/api/Previews.scala

@@ -2,10 +2,10 @@ package api
 
 import api.Permission._
 import services.{DatasetService, FileService, CollectionService, PreviewService, SpaceService,
-MultimediaQueryService, ElasticsearchPlugin}
+MultimediaQueryService, SearchService}
 import play.Logger
 import scala.collection.mutable.{ListBuffer, HashMap}
-import util.{SearchUtils, SearchResult}
+import util.SearchResult
 import play.api.libs.json.{JsObject, Json, JsValue}
 import play.api.libs.json.Json.toJson
 import javax.inject.{Inject, Singleton}
@@ -20,45 +20,32 @@ class Search @Inject() (
    collections: CollectionService,
    previews: PreviewService,
    queries: MultimediaQueryService,
-   spaces: SpaceService)  extends ApiController {
+   spaces: SpaceService,
+   searches: SearchService)  extends ApiController {
 
   /** Search using a simple text string with filters */
   def search(query: String, resource_type: Option[String], datasetid: Option[String], collectionid: Option[String],
              spaceid: Option[String], folderid: Option[String], field: Option[String], tag: Option[String],
              from: Option[Int], size: Option[Int], page: Option[Int]) = PermissionAction(Permission.ViewDataset) { implicit request =>
-    current.plugin[ElasticsearchPlugin] match {
-      case Some(plugin) => {
-        // If from is specified, use it. Otherwise use page * size of page if possible, otherwise use 0.
-        val from_index = from match {
-          case Some(f) => from
-          case None => page match {
-            case Some(p) => Some(size.getOrElse(0) * p)
-            case None => None
-          }
+    if (searches.isEnabled) {
+      // If from is specified, use it. Otherwise use page * size of page if possible, otherwise use 0.
+      val from_index = from match {
+        case Some(f) => from
+        case None => page match {
+          case Some(p) => Some(size.getOrElse(0) * p)
+          case None => None
         }
+      }
 
-        // TODO: Better way to build a URL?
-        val source_url = s"/api/search?query=$query" +
-          (resource_type match {case Some(x) => s"&resource_type=$x" case None => ""}) +
-          (datasetid match {case Some(x) => s"&datasetid=$x" case None => ""}) +
-          (collectionid match {case Some(x) => s"&collectionid=$x" case None => ""}) +
-          (spaceid match {case Some(x) => s"&spaceid=$x" case None => ""}) +
-          (folderid match {case Some(x) => s"&folderid=$x" case None => ""}) +
-          (field match {case Some(x) => s"&field=$x" case None => ""}) +
-          (tag match {case Some(x) => s"&tag=$x" case None => ""})
-
-        // Add space filter to search here as a simple permissions check
-        val permitted = spaces.listAccess(0, Set[Permission](Permission.ViewSpace), request.user, true, true, false, false).map(sp => sp.id)
+      // Add space filter to search here as a simple permissions check
+      val permitted = spaces.listAccess(0, Set[Permission](Permission.ViewSpace), request.user, true, true, false, false).map(sp => sp.id)
 
-        val response = plugin.search(query, resource_type, datasetid, collectionid, spaceid, folderid, field, tag, from_index, size, permitted, request.user)
+      val result = searches.search(query, resource_type, datasetid, collectionid, spaceid, folderid, field, tag, from_index, size, permitted, request.user)
 
-        val result = SearchUtils.prepareSearchResponse(response, source_url, request.user)
-        Ok(toJson(result))
-      }
-      case None => {
-        Logger.debug("Search plugin not enabled")
-        Ok(views.html.pluginNotEnabled("Text search"))
-      }
+      Ok(toJson(result))
+    } else {
+      Logger.debug("Search plugin not enabled")
+      Ok(views.html.pluginNotEnabled("Text search"))
     }
   }
 
@@ -67,21 +54,14 @@ class Search @Inject() (
     implicit request =>
       implicit val user = request.user
 
-      current.plugin[ElasticsearchPlugin] match {
-        case Some(plugin) => {
-          val queryList = Json.parse(query).as[List[JsValue]]
-          val response = plugin.search(queryList, grouping, from, size, user)
-
-          // TODO: Better way to build a URL?
-          val source_url = s"/api/search?query=$query&grouping=$grouping"
-
-          val result = SearchUtils.prepareSearchResponse(response, source_url, user)
-          Ok(toJson(result))
-        }
-        case None => {
-          BadRequest("Elasticsearch plugin could not be reached")
-        }
+      if (searches.isEnabled) {
+        val queryList = Json.parse(query).as[List[JsValue]]
+        val result = searches.search(queryList, grouping, from, size, user)
+        Ok(toJson(result))
+      } else {
+        BadRequest("Elasticsearch plugin could not be reached")
       }
+
   }
 
   /**