Skip to content

Commit eb16223

Browse files
max-zillalmarini
andauthored
Submit marked files in dataset for extraction (#137)
* stub implementation * add selection lookup on submission page * Change path of cookie for marked files * Enable Mark All option * Clearer names * Only show Submit if RMQ enabled * Added to CHANGELOG.md Co-authored-by: Luigi Marini <[email protected]>
1 parent 6054aec commit eb16223

File tree

8 files changed

+267
-7
lines changed

8 files changed

+267
-7
lines changed

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
88

99
### Added
1010
- Added support for Amplitude clickstream tracking. See Admin -> Customize to configure Amplitude apikey.
11+
- Ability to submit multiple selected files within a dataset to an extractor.
1112

1213
## 1.12.2 - 2020-11-19
1314

app/api/Extractions.scala

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import services._
2222

2323
import scala.collection.mutable.ListBuffer
2424
import scala.concurrent.Future
25+
import scala.util.parsing.json.JSONObject
2526

2627

2728
/**
@@ -585,6 +586,81 @@ class Extractions @Inject()(
585586
}
586587
}
587588

589+
def submitFilesToExtractor(ds_id: UUID, file_ids: String)= PermissionAction(Permission.EditDataset, Some(ResourceRef(ResourceRef.dataset,
590+
ds_id)))(parse.json) { implicit request =>
591+
current.plugin[RabbitmqPlugin] match {
592+
case Some(p) => {
593+
var results = Map[String, String]()
594+
files.get(file_ids.split(",").map(fid => UUID(fid)).toList).found.foreach(fi => {
595+
createFileSubmission(request, fi) match {
596+
case Some(jobid) => results += (fi.id.stringify -> jobid.stringify)
597+
case None => Logger.error("File not submitted: "+fi.id)
598+
}
599+
})
600+
Ok(Json.obj("status" -> "success", "jobs" -> results))
601+
}
602+
case None =>
603+
Ok(Json.obj("status" -> "error", "msg"-> "RabbitmqPlugin disabled"))
604+
}
605+
}
606+
607+
private def createFileSubmission(request: UserRequest[JsValue], file: File): Option[UUID] = {
608+
current.plugin[RabbitmqPlugin] match {
609+
case Some(p) => {
610+
val id = file.id
611+
val fileType = file.contentType
612+
val idAndFlags = ""
613+
614+
// check that the file is ready for processing
615+
if (file.status.equals(models.FileStatus.PROCESSED.toString)) {
616+
// parameters for execution
617+
val parameters = (request.body \ "parameters").asOpt[JsObject].getOrElse(JsObject(Seq.empty[(String, JsValue)]))
618+
619+
// Log request
620+
val clientIP = request.remoteAddress
621+
val serverIP = request.host
622+
dtsrequests.insertRequest(serverIP, clientIP, file.filename, id, fileType, file.length, file.uploadDate)
623+
624+
val extra = Map("filename" -> file.filename,
625+
"parameters" -> parameters,
626+
"action" -> "manual-submission")
627+
val showPreviews = file.showPreviews
628+
629+
val newFlags = if (showPreviews.equals("FileLevel"))
630+
idAndFlags + "+filelevelshowpreviews"
631+
else if (showPreviews.equals("None"))
632+
idAndFlags + "+nopreviews"
633+
else
634+
idAndFlags
635+
636+
val originalId = if (!file.isIntermediate) {
637+
file.id.toString()
638+
} else {
639+
idAndFlags
640+
}
641+
642+
var datasetId: UUID = null
643+
// search datasets containning this file, either directly under dataset or indirectly.
644+
val datasetslists:List[Dataset] = datasets.findByFileIdAllContain(file.id)
645+
// Note, we assume only at most one dataset will contain a given file.
646+
if (0 != datasetslists.length) {
647+
datasetId = datasetslists.head.id
648+
}
649+
// if extractor_id is not specified default to execution of all extractors matching mime type
650+
(request.body \ "extractor").asOpt[String] match {
651+
case Some(extractorId) => p.submitFileManually(new UUID(originalId), file, Utils.baseUrl(request), extractorId, extra,
652+
datasetId, newFlags, request.apiKey, request.user)
653+
case None => p.fileCreated(file, None, Utils.baseUrl(request), request.apiKey)
654+
}
655+
} else None
656+
}
657+
case None => {
658+
Logger.error("RabbitMQ disabled.")
659+
None
660+
}
661+
}
662+
}
663+
588664
def submitDatasetToExtractor(ds_id: UUID) = PermissionAction(Permission.EditDataset, Some(ResourceRef(ResourceRef.dataset,
589665
ds_id)))(parse.json) { implicit request =>
590666
Logger.debug(s"Submitting dataset for extraction with body $request.body")

app/controllers/Application.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -353,6 +353,7 @@ class Application @Inject() (files: FileService, collections: CollectionService,
353353
api.routes.javascript.Datasets.users,
354354
api.routes.javascript.Datasets.restoreDataset,
355355
api.routes.javascript.Datasets.emptyTrash,
356+
api.routes.javascript.Extractions.submitFilesToExtractor,
356357
api.routes.javascript.Files.download,
357358
api.routes.javascript.Files.archive,
358359
api.routes.javascript.Files.sendArchiveRequest,
@@ -528,6 +529,7 @@ class Application @Inject() (files: FileService, collections: CollectionService,
528529
controllers.routes.javascript.Extractors.selectExtractors,
529530
controllers.routes.javascript.Extractors.manageLabels,
530531
controllers.routes.javascript.Extractors.showJobHistory,
532+
controllers.routes.javascript.Extractors.submitSelectedExtractions,
531533
controllers.routes.javascript.CurationObjects.submit,
532534
controllers.routes.javascript.CurationObjects.getCurationObject,
533535
controllers.routes.javascript.CurationObjects.getUpdatedFilesAndFolders,

app/controllers/Extractors.scala

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -350,6 +350,26 @@ class Extractors @Inject() (extractions: ExtractionService,
350350
}
351351
}
352352

353+
def submitSelectedExtractions(ds_id: UUID) = PermissionAction(Permission.EditDataset, Some(ResourceRef(ResourceRef.dataset, ds_id))) { implicit request =>
354+
implicit val user = request.user
355+
val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"))
356+
val extractors = all_extractors.filter(!_.process.file.isEmpty)
357+
datasets.get(ds_id) match {
358+
case Some(dataset) => {
359+
val allDecodedDatasets = List(dataset)
360+
val decodedSpacesContaining = ListBuffer.empty[models.ProjectSpace]
361+
dataset.spaces.map{ sp =>
362+
spaces.get(sp) match {
363+
case Some(s) => decodedSpacesContaining += Utils.decodeSpaceElements(s)
364+
case None => Logger.error("Dataset "+dataset.id.stringify+" space not found: "+sp.stringify)
365+
}
366+
}
367+
Ok(views.html.extractions.submitSelectedExtraction(extractors, dataset))
368+
}
369+
case None => InternalServerError("Dataset not found")
370+
}
371+
}
372+
353373
def submitDatasetExtraction(ds_id: UUID) = PermissionAction(Permission.EditDataset, Some(ResourceRef(ResourceRef.dataset, ds_id))) { implicit request =>
354374
implicit val user = request.user
355375
val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"))

app/services/mongodb/MongoDBSelectionService.scala

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,10 @@ class MongoDBSelectionService @Inject() (datasets: DatasetService) extends Sele
5252
def deleteAll(user: String) = {}
5353

5454
def downloadAll(user: String) = {}
55+
56+
def submitAll() = {
57+
58+
}
5559
}
5660

5761
object SelectedDAO extends ModelCompanion[Selected, ObjectId] {

app/views/datasets/filesAndFolders.scala.html

Lines changed: 46 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -81,22 +81,51 @@
8181
var selected = $.cookie('[email protected]');
8282
if (selected) {
8383
var fileUUIDs = selected.split(',');
84-
//$('#selected-count-box').html('Marked Files (<b>'+fileUUIDs.length+'</b>)')
8584
$('#selected-count-box').html('<div class="dropdown"><a href="#" class="dropdown-toggle" ' +
8685
'data-toggle="dropdown" aria-expanded="false"> Marked Files (<b>'+fileUUIDs.length+'</b>)' +
8786
'<b class="caret"></b></a><ul class="dropdown-menu">' +
8887
@* TODO: Need better solution than cookies to support 'Show Only Marked' request
8988
'<li><a onclick="showOnlyMarked();"><span class="glyphicon glyphicon-ok"/>&nbsp;Show Only Marked</a></li>' +
9089
'<li><a onclick="showAllFiles();"><span class="glyphicon glyphicon-ok"/>&nbsp;Show All Files</a></li>' +
9190
*@
92-
'<li><a onclick="downloadMarked();"><span class="glyphicon glyphicon-download-alt"/>&nbsp;Download All</a></li>' +
93-
'<li><a onclick="confirmTagMarked();"><span class="glyphicon glyphicon-tag"/>&nbsp;Tag All</a></li>' +
94-
'<li><a onclick="confirmDeleteMarked();"><span class="glyphicon glyphicon-trash"/>&nbsp;Delete All</a></li>' +
95-
'<li><a onclick="clearMarked();"><span class="glyphicon glyphicon-erase"/>&nbsp;Clear All</a></li></ul>' +
91+
'<li><a onclick="markAllFiles();"><span class="glyphicon glyphicon-check"/>&nbsp;Mark All</a></li>' +
92+
'<li><a onclick="downloadMarked();"><span class="glyphicon glyphicon-download-alt"/>&nbsp;Download</a></li>' +
93+
@if(play.api.Play.current.plugin[services.RabbitmqPlugin].isDefined) {
94+
'<li><a onclick="submitMarked();"><span class="glyphicon glyphicon-send"/>&nbsp;Submit</a></li>' +
95+
}
96+
'<li><a onclick="confirmTagMarked();"><span class="glyphicon glyphicon-tag"/>&nbsp;Tag</a></li>' +
97+
'<li><a onclick="confirmDeleteMarked();"><span class="glyphicon glyphicon-trash"/>&nbsp;Delete</a></li>' +
98+
'<li><a onclick="clearMarked();"><span class="glyphicon glyphicon-erase"/>&nbsp;Unmark All</a></li></ul>' +
9699
'</div>')
97100
} else {
98-
$('#selected-count-box').html('Marked Files (<b>0</b>)')
101+
$('#selected-count-box').html('<div class="dropdown"><a href="#" class="dropdown-toggle" ' +
102+
'data-toggle="dropdown" aria-expanded="false"> Marked Files (<b>0</b>)' +
103+
'<b class="caret"></b></a><ul class="dropdown-menu">' +
104+
@* TODO: Need better solution than cookies to support 'Show Only Marked' request
105+
'<li><a onclick="showOnlyMarked();"><span class="glyphicon glyphicon-ok"/>&nbsp;Show Only Marked</a></li>' +
106+
'<li><a onclick="showAllFiles();"><span class="glyphicon glyphicon-ok"/>&nbsp;Show All Files</a></li>' +
107+
*@
108+
'<li><a onclick="markAllFiles();"><span class="glyphicon glyphicon-check"/>&nbsp;Mark All</a></li>' +
109+
//'<li><a onclick="downloadMarked();"><span class="glyphicon glyphicon-download-alt"/>&nbsp;Download</a></li>' +
110+
//'<li><a onclick="submitMarked();"><span class="glyphicon glyphicon-send"/>&nbsp;Submit</a></li>' +
111+
//'<li><a onclick="confirmTagMarked();"><span class="glyphicon glyphicon-tag"/>&nbsp;Tag</a></li>' +
112+
//'<li><a onclick="confirmDeleteMarked();"><span class="glyphicon glyphicon-trash"/>&nbsp;Delete</a></li>' +
113+
//'<li><a onclick="clearMarked();"><span class="glyphicon glyphicon-erase"/>&nbsp;Unmark All</a></li></ul>' +
114+
'</div>')
115+
}
116+
}
117+
118+
function markAllFiles() {
119+
var selected = "@{dataset.files.mkString(",")}";
120+
console.log(selected)
121+
$.cookie('[email protected]', selected);
122+
var fileUUIDs = selected.split(',');
123+
for (idx in fileUUIDs) {
124+
file_id = fileUUIDs[idx];
125+
$("a[data-id='"+file_id+"'] span.glyphicon").removeClass('glyphicon-plus');
126+
$("a[data-id='"+file_id+"'] span.glyphicon").addClass('glyphicon-ok');
99127
}
128+
updateSelectedFileCount();
100129
}
101130

102131
function confirmDeleteMarked() {
@@ -172,6 +201,16 @@
172201
}
173202
}
174203

204+
// Submit selected files to extractor
205+
function submitMarked() {
206+
var selected = $.cookie('[email protected]');
207+
if (selected) {
208+
window.open("@routes.Extractors.submitSelectedExtractions(dataset.id).url", "_self");
209+
} else {
210+
notify("No files selected.")
211+
}
212+
}
213+
175214
function confirmTagMarked() {
176215
var selected = $.cookie('[email protected]');
177216
if (selected) {
@@ -310,8 +349,8 @@ <h4 class="modal-title">Tag All Marked Files</h4>
310349
$("a[data-id='"+file_id+"'] span.glyphicon").removeClass('glyphicon-plus');
311350
$("a[data-id='"+file_id+"'] span.glyphicon").addClass('glyphicon-ok');
312351
}
313-
updateSelectedFileCount();
314352
}
353+
updateSelectedFileCount();
315354
});
316355
</script>
317356

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
@(extractors: List[ExtractorInfo], ds: Dataset)(implicit user: Option[models.User])
2+
@import _root_.util.Formatters._
3+
@import play.api.libs.json._
4+
5+
@main("Extractions") {
6+
<ol class="breadcrumb">
7+
<li><span class="glyphicon glyphicon-user"></span> <a href = "@routes.Profile.viewProfileUUID(ds.author.id)"> @ds.author.fullName</a></li>
8+
<li> <span class="glyphicon glyphicon-briefcase"></span> <a href="@routes.Datasets.dataset(ds.id)" title="@ds.name"> @Html(ellipsize(ds.name, 18))</a></li>
9+
<li><span class="glyphicon glyphicon-fullscreen"></span> Marked Files</li>
10+
11+
</ol>
12+
<div class="row">
13+
<div id="submission_header" class="col-xs-12">
14+
<h1>Submit marked files for extraction</h1>
15+
</div>
16+
</div>
17+
<div class="row">
18+
<div class="col-xs-12">
19+
<p>Submit these files to a specific extractor below by providing parameters and clicking
20+
the submit button. Some parameters may be left empty.</p>
21+
</div>
22+
</div>
23+
<div class="row">
24+
<div class="col-xs-12">
25+
<table class="table">
26+
<thead>
27+
<tr>
28+
<th>Extractor's Name</th>
29+
<th>Description</th>
30+
<th>Parameters</th>
31+
<th>Submit</th>
32+
</tr>
33+
</thead>
34+
<tbody>
35+
<script src="https://cdn.jsdelivr.net/npm/jsonform@@2.1.5/deps/underscore.js"></script>
36+
<script src="https://cdn.jsdelivr.net/npm/jsonform@@2.1.5/lib/jsonform.min.js"></script>
37+
@for(e <- extractors) {
38+
<tr>
39+
<td>
40+
<a href="@routes.Extractors.showExtractorInfo(e.name)">@e.name</a>
41+
</td>
42+
<td>@e.description</td>
43+
<td>
44+
<form id="@(e.name.replaceAll("\\.", "_"))_parameters"></form>
45+
</td>
46+
<td><button id="@e.id" class="btn btn-primary" onclick="submit('@e.name','@(e.name.replaceAll("\\.", "_"))_parameters','@ds.id','@e.id')">Submit</button></td>
47+
</tr>
48+
49+
<script>
50+
var params = JSON.parse("@Json.stringify(e.parameters)".replace(/&quot;/g, "\""));
51+
var selector = "#@(e.name.replaceAll("\\.", "_"))_parameters";
52+
if (Object.keys(params).length != 0) {
53+
$(selector).jsonForm(params);
54+
}
55+
</script>
56+
}
57+
</tbody>
58+
</table>
59+
</div>
60+
</div>
61+
<script type="text/javascript">
62+
function disableSubmit(btn) {
63+
btn.attr('disabled', true);
64+
btn.addClass('disabled');
65+
btn.removeClass('btn-primary');
66+
btn.addClass('btn-success');
67+
btn.html('Submitted');
68+
}
69+
70+
function enableSubmit(btn) {
71+
btn.html('Submit');
72+
btn.removeClass('disabled');
73+
btn.removeClass('btn-success');
74+
btn.addClass('btn-primary');
75+
btn.attr('disabled', false);
76+
}
77+
78+
function submit(extractor_name, textbox_id, file_id, submit_id) {
79+
var clickedBtn = $('#' + submit_id);
80+
81+
var selected = $.cookie('[email protected]');
82+
83+
// Throttle submissions to one every 3 seconds
84+
disableSubmit(clickedBtn);
85+
setTimeout(function() {
86+
enableSubmit(clickedBtn);
87+
}, 3000);
88+
89+
var params = $('#'+textbox_id).jsonFormValue();
90+
if (params === "") params = "{}";
91+
var dataBody = {'extractor': extractor_name, 'parameters': params};
92+
var request = jsRoutes.api.Extractions.submitFilesToExtractor("@ds.id", selected).ajax({
93+
data: JSON.stringify(dataBody),
94+
type: 'POST',
95+
contentType: "application/json",
96+
});
97+
98+
request.done(function (response, textStatus, jqXHR){
99+
notify("Submitted successfully", "success");
100+
});
101+
102+
request.fail(function (jqXHR, textStatus, errorThrown){
103+
console.error("The following error occured: " + textStatus, errorThrown);
104+
});
105+
}
106+
107+
$(document).ready(function() {
108+
var selected = $.cookie('[email protected]');
109+
if (selected) {
110+
var fileUUIDs = selected.split(',');
111+
$('#submission_header')[0].innerHTML = "<h1>Submit "+fileUUIDs.length+" marked files for extraction</h1>"
112+
}
113+
});
114+
</script>
115+
}
116+

conf/routes

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -728,13 +728,15 @@ POST /api/comment/:id/editComment
728728
# SELECTIONS API
729729
# ----------------------------------------------------------------------
730730
GET /selected @controllers.Selected.get
731+
GET /datasets/submit/:ds_id @controllers.Extractors.submitSelectedExtractions(ds_id: UUID)
731732
GET /api/selected @api.Selected.get
732733
POST /api/selected @api.Selected.add
733734
POST /api/selected/remove @api.Selected.remove
734735
DELETE /api/selected/files @api.Selected.deleteAll
735736
GET /api/selected/files @api.Selected.downloadAll
736737
POST /api/selected/clear @api.Selected.clearAll
737738
POST /api/selected/tag @api.Selected.tagAll(tags: List[String])
739+
POST /api/selected/submit/:ds_id/:file_ids @api.Extractions.submitFilesToExtractor(ds_id: UUID, file_ids: String)
738740

739741
# ----------------------------------------------------------------------
740742
# RELATIONS API

0 commit comments

Comments
 (0)