Address feedback, support backwards compatibility with job_id and pyclowder

Author: bodom0015

app/api/Extractions.scala

@@ -142,7 +142,7 @@ class Extractions @Inject()(
               // FIXME dataset not available?
               plugin.fileCreated(file, None, Utils.baseUrl(request), request.apiKey) match {
                 case Some(jobId) => {
-                  Ok(Json.obj("status" -> "Ok", "job_id" -> jobId))
+                  Ok(Json.obj("status" -> "OK", "job_id" -> jobId))
                 }
                 case None => {
                   val message = "No jobId found for Extraction"

app/models/Extraction.scala

@@ -12,7 +12,7 @@ import play.api.libs.functional.syntax._
 case class Extraction(
   id: UUID = UUID.generate,
   file_id: UUID,
-  job_id: UUID,
+  job_id: Option[UUID],
   extractor_id: String,
   status: String = "N/A",
   start: Date,

app/services/RabbitmqPlugin.scala

@@ -46,7 +46,7 @@ import scala.util.Try
 case class ExtractorMessage(
   msgid: UUID,
   fileId: UUID,
-  jobId: UUID,
+  jobId: Option[UUID],
   notifies: List[String],
   intermediateId: UUID,
   host: String,
@@ -431,13 +431,13 @@ class RabbitmqPlugin(application: Application) extends Plugin {
     * @param file_id the UUID of file
     * @param extractor_id the extractor queue name to be submitted
     */
-  def postSubmissionEven(file_id: UUID, extractor_id: String): (UUID, UUID) = {
+  def postSubmissionEven(file_id: UUID, extractor_id: String): (UUID, Option[UUID]) = {
     val extractions: ExtractionService = DI.injector.getInstance(classOf[ExtractionService])
 
     import java.text.SimpleDateFormat
     val dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssX")
     val submittedDateConvert = new java.util.Date()
-    val job_id = UUID.generate()
+    val job_id = Some(UUID.generate())
     extractions.insert(Extraction(UUID.generate(), file_id, job_id, extractor_id, "SUBMITTED", submittedDateConvert, None)) match {
       case Some(objectid) => (UUID(objectid.toString), job_id)
       case None => (UUID(""), job_id)
@@ -487,7 +487,7 @@ class RabbitmqPlugin(application: Application) extends Plugin {
           val msg = ExtractorMessage(id, file.id, job_id, notifies, file.id, host, queue, extraInfo, file.length.toString,
             d.id, "", apiKey, routingKey, source, "created", None)
           extractWorkQueue(msg)
-          jobId = Option(job_id)
+          jobId = job_id
         }
         jobId
       }
@@ -609,7 +609,7 @@ class RabbitmqPlugin(application: Application) extends Plugin {
     * @param newFlags
     */
   def submitFileManually(originalId: UUID, file: File, host: String, queue: String, extraInfo: Map[String, Any],
-    datasetId: UUID, newFlags: String, requestAPIKey: Option[String], user: Option[User]): UUID = {
+    datasetId: UUID, newFlags: String, requestAPIKey: Option[String], user: Option[User]): Option[UUID] = {
     Logger.debug(s"Sending message to $queue from $host with extraInfo $extraInfo")
     val apiKey = getApiKey(requestAPIKey, user)
     val sourceExtra = JsObject((Seq("filename" -> JsString(file.filename))))
@@ -632,7 +632,7 @@ class RabbitmqPlugin(application: Application) extends Plugin {
     * @param newFlags
     */
   def submitDatasetManually(host: String, queue: String, extraInfo: Map[String, Any], datasetId: UUID, newFlags: String,
-    requestAPIKey: Option[String], user: Option[User]): UUID = {
+    requestAPIKey: Option[String], user: Option[User]): Option[UUID] = {
     Logger.debug(s"Sending message $queue from $host with extraInfo $extraInfo")
     val apiKey = getApiKey(requestAPIKey, user)
     val source = Entity(ResourceRef(ResourceRef.dataset, datasetId), None, JsObject(Seq.empty))
@@ -964,13 +964,13 @@ class PendingRequestCancellationActor(exchange: String, connection: Option[Conne
       val extractions: ExtractionService = DI.injector.getInstance(classOf[ExtractionService])
       val dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssX")
       var startDate = new java.util.Date()
-      val job_id: UUID = extractions.get(msg_id) match {
+      val job_id: Option[UUID] = extractions.get(msg_id) match {
         case Some(extraction) => {
           extraction.job_id
         }
         case None => {
           Logger.warn("Failed to lookup jobId.. no extraction message found with id=" + msg_id)
-          UUID("")
+          None
         }
       }
       extractions.insert(Extraction(UUID.generate(), id, job_id, queueName, "Cancel Requested", startDate, None))
@@ -1111,7 +1111,7 @@ class PublishDirectActor(channel: Channel, replyQueueName: String) extends Actor
         "notifies" -> Json.toJson(notifies),
         "msgid" -> Json.toJson(msgid.stringify),
         "id" -> Json.toJson(fileid.stringify),
-        "jobid" -> Json.toJson(jobid.stringify),
+        "jobid" -> Json.toJson(jobid.get.stringify),
         "intermediateId" -> Json.toJson(intermediateId.stringify),
         "fileSize" -> Json.toJson(fileSize),
         "host" -> Json.toJson(actualHost),
@@ -1189,7 +1189,10 @@ class EventFilter(channel: Channel, queue: String) extends Actor {
       Logger.debug("Received extractor status: " + statusBody)
       val json = Json.parse(statusBody)
       val file_id = UUID((json \ "file_id").as[String])
-      val job_id = UUID((json \ "job_id").as[String])
+      val job_id: Option[UUID] = (json \ "job_id").asOpt[String] match {
+        case Some(jid) => { Some(UUID(jid)) }
+        case None => { None }
+      }
       val extractor_id = (json \ "extractor_id").as[String]
       val status = (json \ "status").as[String]
       val startDate = (json \ "start").asOpt[String].map(x =>

app/services/mongodb/MongoDBExtractionService.scala

@@ -85,7 +85,7 @@ class MongoDBExtractionService extends ExtractionService {
     var groupings = Map[String, ExtractionGroup]()
 
     for (e <- extraction_list) {
-
+      var job_id = e.job_id.getOrElse(UUID(""))
       // Update timestamp and groupings
       if (groupings.contains(e.extractor_id)) {
         // Update entry in the Map
@@ -95,14 +95,11 @@ class MongoDBExtractionService extends ExtractionService {
         var grp_endmsg = groupings(e.extractor_id).latestMsg
 
         // For each job_id, keep track of all extraction events for that job
-        var grp_map = groupings(e.extractor_id).allMsgs
-        if(grp_map.contains(e.job_id)) {
-          val existing: List[Extraction] = grp_map(e.job_id)
-          val updated = List(e) ++ existing
-          // Re-sort list by timestamp (newest first)
-          grp_map = grp_map + (e.job_id -> updated)
+        val init_grp_map = groupings(e.extractor_id).allMsgs
+        val grp_map = if(init_grp_map.contains(job_id)) {
+          init_grp_map + (job_id -> (List(e) ++ init_grp_map(job_id)))
         } else {
-          grp_map = grp_map + (e.job_id -> List(e))
+          init_grp_map + (job_id -> List(e))
         }
 
         // Use start time for time groupings as backup because end is frequently empty
@@ -140,7 +137,7 @@ class MongoDBExtractionService extends ExtractionService {
           case Some(e) => e.toString
           case None => start
         }
-        groupings = groupings + (e.extractor_id -> ExtractionGroup(start, end, e.status, Map(e.job_id -> List(e))))
+        groupings = groupings + (e.extractor_id -> ExtractionGroup(start, end, e.status, Map(job_id -> List(e))))
       }
 
     }

app/views/extractions/extractionsList.scala.html

@@ -1,4 +1,5 @@
 @(extractions: Map[String, ExtractionGroup], extractType: String)
+<link rel="stylesheet" href="@routes.Assets.at("stylesheets/extractions.css")">
 <div class="well">
     @if(extractions.size == 0) {
         <p>No extraction events recorded.</p>
@@ -36,7 +37,7 @@
                             @for((job_id, eventList) <- group.allMsgs.toSeq.sortWith(_._1.stringify > _._1.stringify)) {
                                 <tr class="job-list-accordion-toggle clickable collapsed" data-toggle="collapse" data-target="#extraction-job-@job_id">
                                     <td width="35%" title="@job_id"><a class="btn btn-link">@eventList.head.start</a></td>
-                                    <td width="60%" class="accordion-status" style="vertical-align:middle;">@eventList.head.status</td>
+                                    <td width="60%" class="accordion-status" style="vertical-align:middle;">@eventList.head.status.replace("StatusMessage.start", "START").replace("StatusMessage.processing", "PROCESS").replace("StatusMessage.error", "ERROR")</td>
                                 </tr>
                                 <tr class="collapse" id="extraction-job-@job_id">
                                     <td colspan="3" class="extractor-log">
@@ -52,7 +53,7 @@
                                                     <td width="60%">
                                                         @evt.status.replace("StatusMessage.start", "START").replace("StatusMessage.processing", "PROCESS").replace("StatusMessage.error", "ERROR")
                                                         @if(evt.status == "SUBMITTED") {
-                                                            <a class="btn btn-link" id='@(evt.id)' onclick="cancelSubmission('@(evt.file_id)', '@(evt.id)', '@(evt.extractor_id)', '@(extractType)')">(Cancel submission)</a>
+                                                            <a class="btn btn-link btn-xs" id='@(evt.id)' onclick="cancelSubmission('@(evt.file_id)', '@(evt.id)', '@(evt.extractor_id)', '@(extractType)')">(Cancel submission)</a>
                                                         }
                                                     </td>
                                                 </tr>
@@ -100,32 +101,3 @@
             });
         }
 </script>
-<style>
-    .clickable {
-        cursor: pointer;
-    }
-    .job-list-accordion-toggle:before, .extractor-accordion-toggle:before {
-        /* symbol for "opening" panels */
-        font-family: 'Glyphicons Halflings';  /* essential for enabling glyphicon */
-        content: "\e114";    /* adjust as needed, taken from bootstrap.css */
-        float: right;        /* adjust as needed */
-        color: grey;         /* adjust as needed */
-    }
-    .job-list-accordion-toggle.collapsed:before, .extractor-accordion-toggle.collapsed:before {
-        /* symbol for "collapsed" panels */
-        content: "\e080";    /* adjust as needed, taken from bootstrap.css */
-    }
-    .job-list-accordion-toggle:before {
-        padding: 14px 12px;
-    }
-    .extractor-accordion-toggle:before {
-        padding: 16px 12px;
-    }
-    .job-list-accordion-status {
-        padding: 12px 8px !important;
-    }
-    .extractor-log {
-        padding: 10px 0px 10px 50px !important;
-        border-top: none !important;
-    }
-</style>

public/stylesheets/extractions.css

@@ -0,0 +1,33 @@
+.clickable {
+    cursor: pointer;
+}
+
+.job-list-accordion-toggle:before, .extractor-accordion-toggle:before {
+    /* symbol for "opening" panels */
+    font-family: 'Glyphicons Halflings';  /* essential for enabling glyphicon */
+    content: "\e114";    /* adjust as needed, taken from bootstrap.css */
+    float: right;        /* adjust as needed */
+    color: grey;         /* adjust as needed */
+}
+
+.job-list-accordion-toggle.collapsed:before, .extractor-accordion-toggle.collapsed:before {
+    /* symbol for "collapsed" panels */
+    content: "\e080";    /* adjust as needed, taken from bootstrap.css */
+}
+
+.job-list-accordion-toggle:before {
+    padding: 14px 12px;
+}
+
+.extractor-accordion-toggle:before {
+    padding: 16px 12px;
+}
+
+.job-list-accordion-status {
+    padding: 12px 8px !important;
+}
+
+.extractor-log {
+    padding: 10px 0px 10px 50px !important;
+    border-top: none !important;
+}