changed added from 2.0 WIP - prepare for merge

Author: toddn

app/api/Datasets.scala

@@ -48,8 +48,8 @@ class  Datasets @Inject()(
   userService: UserService,
   thumbnailService : ThumbnailService,
   appConfig: AppConfigurationService,
-  esqueue: ElasticsearchQueue,
   adminsNotifierService: AdminsNotifierService,
+  searches: SearchService,
   extractionBusService: ExtractionBusService) extends ApiController {
 
   def get(id: UUID) = PermissionAction(Permission.ViewDataset, Some(ResourceRef(ResourceRef.dataset, id))) { implicit request =>
@@ -224,14 +224,9 @@ class  Datasets @Inject()(
                   if (!file.xmlMetadata.isEmpty) {
                     val xmlToJSON = files.getXMLMetadataJSON(UUID(file_id))
                     datasets.addXMLMetadata(UUID(id), UUID(file_id), xmlToJSON)
-                    current.plugin[ElasticsearchPlugin].foreach {
-                      _.index(SearchUtils.getElasticsearchObject(d))
                     }
-                  } else {
-                    current.plugin[ElasticsearchPlugin].foreach {
-                      _.index(SearchUtils.getElasticsearchObject(d))
-                    }
-                  }
+                  searches.index(d, true)
+
 
                   adminsNotifierService.sendAdminsNotification(Utils.baseUrl(request), "Dataset", "added", id, name)
 
@@ -390,9 +385,8 @@ class  Datasets @Inject()(
   def reindex(id: UUID, recursive: Boolean) = PermissionAction(Permission.CreateDataset, Some(ResourceRef(ResourceRef.dataset, id))) { implicit request =>
     datasets.get(id) match {
       case Some(ds) => {
-        val success = esqueue.queue("index_dataset", new ResourceRef('dataset, id), new ElasticsearchParameters(recursive=recursive))
-        if (success) Ok(toJson(Map("status" -> "reindex successfully queued")))
-        else BadRequest(toJson(Map("status" -> "reindex queuing failed, Elasticsearch may be disabled")))
+        datasets.index(id)
+        Ok(toJson(s"Dataset $id reindexed"))
       }
       case None => {
         Logger.error("Error getting dataset" + id)
@@ -1617,16 +1611,8 @@ class  Datasets @Inject()(
   }
 
   def jsonPreview(pvId: String, pId: String, pPath: String, pMain: String, pvRoute: String, pvContentType: String, pvLength: Long): JsValue = {
-    if (pId.equals("X3d"))
-      toJson(Map("pv_id" -> pvId, "p_id" -> pId,
-        "p_path" -> controllers.routes.Assets.at(pPath).toString,
-        "p_main" -> pMain, "pv_route" -> pvRoute,
-        "pv_contenttype" -> pvContentType, "pv_length" -> pvLength.toString,
-        "pv_annotationsEditPath" -> api.routes.Previews.editAnnotation(UUID(pvId)).toString,
-        "pv_annotationsListPath" -> api.routes.Previews.listAnnotations(UUID(pvId)).toString,
-        "pv_annotationsAttachPath" -> api.routes.Previews.attachAnnotation(UUID(pvId)).toString))
-    else
-      toJson(Map("pv_id" -> pvId, "p_id" -> pId, "p_path" -> controllers.routes.Assets.at(pPath).toString, "p_main" -> pMain, "pv_route" -> pvRoute, "pv_contenttype" -> pvContentType, "pv_length" -> pvLength.toString))
+    toJson(Map("pv_id" -> pvId, "p_id" -> pId, "p_path" -> controllers.routes.Assets.at(pPath).toString,
+      "p_main" -> pMain, "pv_route" -> pvRoute, "pv_contenttype" -> pvContentType, "pv_length" -> pvLength.toString))
   }
 
   def getPreviews(id: UUID) = PermissionAction(Permission.ViewDataset, Some(ResourceRef(ResourceRef.dataset, id))) { implicit request =>

app/api/Files.scala

@@ -6,7 +6,7 @@ import java.net.{URL, URLEncoder}
 
 import javax.inject.Inject
 import javax.mail.internet.MimeUtility
-import _root_.util.{FileUtils, JSONLD, Parsers, RequestUtils, SearchUtils}
+import _root_.util.{FileUtils, JSONLD, Parsers, RequestUtils}
 import com.mongodb.casbah.Imports._
 import controllers.Previewers
 import jsonutils.JsonUtil
@@ -42,7 +42,6 @@ class Files @Inject()(
   extractions: ExtractionService,
   dtsrequests:ExtractionRequestsService,
   previews: PreviewService,
-  threeD: ThreeDService,
   metadataService: MetadataService,
   contextService: ContextLDService,
   thumbnails: ThumbnailService,
@@ -53,7 +52,8 @@ class Files @Inject()(
   appConfig: AppConfigurationService,
   adminsNotifierService: AdminsNotifierService,
   extractionBusService: ExtractionBusService,
-  esqueue: ElasticsearchQueue) extends ApiController {
+  versusService: VersusService,
+  searches: SearchService) extends ApiController {
 
   def get(id: UUID) = PermissionAction(Permission.ViewFile, Some(ResourceRef(ResourceRef.file, id))) { implicit request =>
     Logger.debug("GET file with id " + id)
@@ -560,9 +560,8 @@ class Files @Inject()(
   def reindex(id: UUID) = PermissionAction(Permission.AddFile, Some(ResourceRef(ResourceRef.file, id))) { implicit request =>
     files.get(id) match {
       case Some(file) => {
-        val success = esqueue.queue("index_file", new ResourceRef('file, id))
-        if (success) Ok(toJson(Map("status" -> "reindex successfully queued")))
-        else BadRequest(toJson(Map("status" -> "reindex queuing failed, Elasticsearch may be disabled")))
+        files.index(id)
+        Ok(toJson(s"File $id reindexed"))
       }
       case None => {
         Logger.error("Error getting file" + id)
@@ -771,52 +770,6 @@ class Files @Inject()(
     }
   }
 
-  /**
-    * Add 3D geometry file to file.
-    */
-  def attachGeometry(file_id: UUID, geometry_id: UUID) = PermissionAction(Permission.AddFile, Some(ResourceRef(ResourceRef.file, file_id)))(parse.json) { implicit request =>
-    request.body match {
-      case JsObject(fields) => {
-        files.get(file_id) match {
-          case Some(file) => {
-            threeD.getGeometry(geometry_id) match {
-              case Some(geometry) =>
-                threeD.updateGeometry(file_id, geometry_id, fields)
-                Ok(toJson(Map("status" -> "success")))
-              case None => BadRequest(toJson("Geometry file not found"))
-            }
-          }
-          case None => BadRequest(toJson("File not found " + file_id))
-        }
-      }
-      case _ => Ok("received something else: " + request.body + '\n')
-    }
-  }
-
-
-  /**
-    * Add 3D texture to file.
-    */
-  def attachTexture(file_id: UUID, texture_id: UUID) = PermissionAction(Permission.AddFile, Some(ResourceRef(ResourceRef.file, file_id)))(parse.json) { implicit request =>
-    request.body match {
-      case JsObject(fields) => {
-        files.get((file_id)) match {
-          case Some(file) => {
-            threeD.getTexture(texture_id) match {
-              case Some(texture) => {
-                threeD.updateTexture(file_id, texture_id, fields)
-                Ok(toJson(Map("status" -> "success")))
-              }
-              case None => BadRequest(toJson("Texture file not found"))
-            }
-          }
-          case None => BadRequest(toJson("File not found " + file_id))
-        }
-      }
-      case _ => Ok("received something else: " + request.body + '\n')
-    }
-  }
-
   /**
     * Add thumbnail to file.
     */
@@ -871,113 +824,6 @@ class Files @Inject()(
     }
   }
 
-
-  /**
-    * Find geometry file for given 3D file and geometry filename.
-    */
-  def getGeometry(three_d_file_id: UUID, filename: String) = PermissionAction(Permission.ViewFile, Some(ResourceRef(ResourceRef.file, three_d_file_id))) { implicit request =>
-    threeD.findGeometry(three_d_file_id, filename) match {
-      case Some(geometry) => {
-
-        threeD.getGeometryBlob(geometry.id) match {
-
-          case Some((inputStream, filename, contentType, contentLength)) => {
-            request.headers.get(RANGE) match {
-              case Some(value) => {
-                val range: (Long, Long) = value.substring("bytes=".length).split("-") match {
-                  case x if x.length == 1 => (x.head.toLong, contentLength - 1)
-                  case x => (x(0).toLong, x(1).toLong)
-                }
-                range match {
-                  case (start, end) =>
-
-                    inputStream.skip(start)
-                    import play.api.mvc.{ResponseHeader, SimpleResult}
-                    SimpleResult(
-                      header = ResponseHeader(PARTIAL_CONTENT,
-                        Map(
-                          CONNECTION -> "keep-alive",
-                          ACCEPT_RANGES -> "bytes",
-                          CONTENT_RANGE -> "bytes %d-%d/%d".format(start, end, contentLength),
-                          CONTENT_LENGTH -> (end - start + 1).toString,
-                          CONTENT_TYPE -> contentType
-                        )
-                      ),
-                      body = Enumerator.fromStream(inputStream)
-                    )
-                }
-              }
-              case None => {
-                //IMPORTANT: Setting CONTENT_LENGTH header here introduces bug!
-                Ok.chunked(Enumerator.fromStream(inputStream))
-                  .withHeaders(CONTENT_TYPE -> contentType)
-                  .withHeaders(CONTENT_DISPOSITION -> (FileUtils.encodeAttachment(filename, request.headers.get("user-agent").getOrElse(""))))
-
-              }
-            }
-          }
-          case None => Logger.error("No geometry file found: " + geometry.id); InternalServerError("No geometry file found")
-
-        }
-      }
-      case None => Logger.error("Geometry file not found"); InternalServerError
-    }
-  }
-
-
-  /**
-    * Find texture file for given 3D file and texture filename.
-    */
-  def getTexture(three_d_file_id: UUID, filename: String) = PermissionAction(Permission.ViewFile, Some(ResourceRef(ResourceRef.file, three_d_file_id))) { implicit request =>
-    threeD.findTexture(three_d_file_id, filename) match {
-      case Some(texture) => {
-
-        threeD.getBlob(texture.id) match {
-
-          case Some((inputStream, filename, contentType, contentLength)) => {
-            request.headers.get(RANGE) match {
-              case Some(value) => {
-                val range: (Long, Long) = value.substring("bytes=".length).split("-") match {
-                  case x if x.length == 1 => (x.head.toLong, contentLength - 1)
-                  case x => (x(0).toLong, x(1).toLong)
-                }
-                range match {
-                  case (start, end) =>
-
-                    inputStream.skip(start)
-
-                    SimpleResult(
-                      header = ResponseHeader(PARTIAL_CONTENT,
-                        Map(
-                          CONNECTION -> "keep-alive",
-                          ACCEPT_RANGES -> "bytes",
-                          CONTENT_RANGE -> "bytes %d-%d/%d".format(start, end, contentLength),
-                          CONTENT_LENGTH -> (end - start + 1).toString,
-                          CONTENT_TYPE -> contentType
-                        )
-                      ),
-                      body = Enumerator.fromStream(inputStream)
-                    )
-                }
-              }
-              case None => {
-                //IMPORTANT: Setting CONTENT_LENGTH header here introduces bug!
-                Ok.chunked(Enumerator.fromStream(inputStream))
-                  .withHeaders(CONTENT_TYPE -> contentType)
-                  //.withHeaders(CONTENT_LENGTH -> contentLength.toString)
-                  .withHeaders(CONTENT_DISPOSITION -> (FileUtils.encodeAttachment(filename, request.headers.get("user-agent").getOrElse(""))))
-
-              }
-            }
-          }
-          case None => Logger.error("No texture file found: " + texture.id.toString()); InternalServerError("No texture found")
-
-        }
-      }
-      case None => Logger.error("Texture file not found"); InternalServerError
-    }
-  }
-
   /**
     * REST endpoint: PUT: update or change the filename
     * args
@@ -1445,13 +1291,6 @@ class Files @Inject()(
   }
 
   def jsonPreview(pvId: UUID, pId: String, pPath: String, pMain: String, pvRoute: java.lang.String, pvContentType: String, pvLength: Long): JsValue = {
-    if (pId.equals("X3d"))
-      toJson(Map("pv_id" -> pvId.stringify, "p_id" -> pId, "p_path" -> controllers.routes.Assets.at(pPath).toString,
-        "p_main" -> pMain, "pv_route" -> pvRoute, "pv_contenttype" -> pvContentType, "pv_length" -> pvLength.toString,
-        "pv_annotationsEditPath" -> api.routes.Previews.editAnnotation(pvId).toString,
-        "pv_annotationsListPath" -> api.routes.Previews.listAnnotations(pvId).toString,
-        "pv_annotationsAttachPath" -> api.routes.Previews.attachAnnotation(pvId).toString))
-    else
       toJson(Map("pv_id" -> pvId.stringify, "p_id" -> pId, "p_path" -> controllers.routes.Assets.at(pPath).toString,
         "p_main" -> pMain, "pv_route" -> pvRoute, "pv_contenttype" -> pvContentType, "pv_length" -> pvLength.toString))
   }
@@ -1600,17 +1439,11 @@ class Files @Inject()(
 
         //this stmt has to be before files.removeFile
         Logger.debug("Deleting file from indexes " + file.filename)
-        current.plugin[VersusPlugin].foreach {
-          _.removeFromIndexes(id)
-        }
+        versusService.removeFromIndexes(id)
         Logger.debug("Deleting file: " + file.filename)
         files.removeFile(id, Utils.baseUrl(request), request.apiKey, request.user)
-        appConfig.incrementCount('files, -1)
-        appConfig.incrementCount('bytes, -file.length)
 
-        current.plugin[ElasticsearchPlugin].foreach {
-          _.delete("data", "file", id.stringify)
-        }
+        searches.delete(id.stringify, "file")
 
         adminsNotifierService.sendAdminsNotification(Utils.baseUrl(request), "File", "removed", id.stringify, file.filename)
 
@@ -1689,11 +1522,7 @@ class Files @Inject()(
 
   def index(id: UUID) {
     files.get(id) match {
-      case Some(file) => {
-        current.plugin[ElasticsearchPlugin].foreach {
-          _.index(SearchUtils.getElasticsearchObject(file))
-        }
-      }
+      case Some(file) => searches.index(file)
       case None => Logger.error("File not found: " + id)
     }
   }

app/api/Indexes.scala

@@ -14,7 +14,7 @@ import services.{ExtractorMessage, _}
  * Index data.
  */
 @Inject
-class Indexes @Inject() (multimediaSearch: MultimediaQueryService, previews: PreviewService, extractionBusService: ExtractionBusService) extends Controller with ApiController {
+class Indexes @Inject() (multimediaSearch: MultimediaQueryService, previews: PreviewService, versusService: VersusService, extractionBusService: ExtractionBusService) extends Controller with ApiController {
 
   /**
    * Submit section, preview, file for indexing.
@@ -26,9 +26,7 @@ class Indexes @Inject() (multimediaSearch: MultimediaQueryService, previews: Pre
       	      case Some(p) =>
                 extractionBusService.submitSectionPreviewManually(p, new UUID(section_id), Utils.baseUrl(request), request.apiKey)
                 val fileType = p.contentType
-                current.plugin[VersusPlugin].foreach{
-                  _.indexPreview(p.id,fileType)
-                }
+                versusService.indexPreview(p.id,fileType)
                 Ok(toJson("success"))
       	      case None =>
                 BadRequest(toJson("Missing parameter [preview_id]"))

app/api/Metadata.scala

@@ -40,6 +40,7 @@ class Metadata @Inject() (
   vocabs: StandardVocabularyService,
   events: EventService,
   spaceService: SpaceService,
+  searches: SearchService,
   extractionBusService: ExtractionBusService) extends ApiController {
 
   def getDefinitions() = PermissionAction(Permission.ViewDataset) {
@@ -71,22 +72,19 @@ class Metadata @Inject() (
     }
 
     // Next get Elasticsearch metadata fields if plugin available
-    current.plugin[ElasticsearchPlugin] match {
-      case Some(plugin) => {
-        val mdTerms = plugin.getAutocompleteMetadataFields(query)
+    if (searches.isEnabled()) {
+      val mdTerms = searches.getAutocompleteMetadataFields(query)
         for (term <- mdTerms) {
           // e.g. "metadata.http://localhost:9000/clowder/api/extractors/terra.plantcv.angle",
           //      "metadata.Jane Doe.Alternative Title"
           if (!(listOfTerms contains term))
             listOfTerms.append(term)
         }
         Ok(toJson(listOfTerms.distinct))
-      }
-      case None => {
+    } else {
         BadRequest("Elasticsearch plugin is not enabled")
       }
     }
-  }
 
   def getDefinitionsByDataset(id: UUID) = PermissionAction(Permission.AddMetadata, Some(ResourceRef(ResourceRef.dataset, id))) { implicit request =>
     implicit val user = request.user
@@ -548,20 +546,15 @@ class Metadata @Inject() (
 
               m.attachedTo.resourceType match {
                 case ResourceRef.file => {
-                  current.plugin[ElasticsearchPlugin].foreach { p =>
-                    // Delete existing index entry and re-index
-                    p.delete(m.attachedTo.id.stringify)
-                    files.index(m.attachedTo.id)
-                  }
+                  searches.delete(m.attachedTo.id.stringify)
+                  files.index(m.attachedTo.id)
                   extractionBusService.metadataRemovedFromResource(id, m.attachedTo, Utils.baseUrl(request),
                     request.apiKey, request.user)
                 }
                 case ResourceRef.dataset => {
-                  current.plugin[ElasticsearchPlugin].foreach { p =>
-                    // Delete existing index entry and re-index
-                    p.delete(m.attachedTo.id.stringify)
-                    datasets.index(m.attachedTo.id)
-                  }
+                  // Delete existing index entry and re-index
+                  searches.delete(m.attachedTo.id.stringify)
+                  datasets.index(m.attachedTo.id)
                   extractionBusService.metadataRemovedFromResource(id, m.attachedTo, Utils.baseUrl(request), request.apiKey, request.user)
                 }
                 case _ => {