Skip to content

Commit 975fe76

Browse files
hvarghvarg
committed
Storing full query results
1 parent dd4e491 commit 975fe76

File tree

11 files changed

+203
-154
lines changed

11 files changed

+203
-154
lines changed

‎server/src/main/java/org/diskproject/server/db/DiskDB.java‎

Lines changed: 58 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import java.net.URL;
44
import java.text.SimpleDateFormat;
55
import java.util.ArrayList;
6+
import java.util.Arrays;
67
import java.util.Date;
78
import java.util.List;
89

@@ -20,6 +21,7 @@
2021
import org.diskproject.shared.classes.hypothesis.GoalResult;
2122
import org.diskproject.shared.classes.loi.DataQueryResult;
2223
import org.diskproject.shared.classes.loi.DataQueryTemplate;
24+
import org.diskproject.shared.classes.loi.LOICommon;
2325
import org.diskproject.shared.classes.loi.LineOfInquiry;
2426
import org.diskproject.shared.classes.loi.TriggeredLOI;
2527
import org.diskproject.shared.classes.question.Question;
@@ -507,8 +509,10 @@ private KBObject _writeDataQueryTemplate (DataQueryTemplate dataQuery, KBObject
507509
domainKB.setPropertyValue(dq, DISKOnt.getProperty(DISK.HAS_DATA_SOURCE), domainKB.getIndividual(dataQuery.getEndpoint().getId()));
508510
if (dataQuery.getTemplate() != null)
509511
domainKB.setPropertyValue(dq, DISKOnt.getProperty(DISK.HAS_QUERY_TEMPLATE), domainKB.createLiteral(dataQuery.getTemplate()));
510-
if (dataQuery.getVariablesToShow() != null)
511-
domainKB.setPropertyValue(dq, DISKOnt.getProperty(DISK.HAS_TABLE_VARIABLES), domainKB.createLiteral(dataQuery.getVariablesToShow()));
512+
if (dataQuery.getVariablesToShow() != null) {
513+
String arrString = "[" + String.join(",",dataQuery.getVariablesToShow()) + "]";
514+
domainKB.setPropertyValue(dq, DISKOnt.getProperty(DISK.HAS_TABLE_VARIABLES), domainKB.createLiteral(arrString));
515+
}
512516
if (dataQuery.getFootnote() != null)
513517
domainKB.setPropertyValue(dq, DISKOnt.getProperty(DISK.HAS_TABLE_DESCRIPTION), domainKB.createLiteral(dataQuery.getFootnote()));
514518
return dq;
@@ -521,8 +525,14 @@ private DataQueryTemplate loadDataQueryTemplate (KBObject objTemplate) {
521525
if (domainKB.getComment(objTemplate) != null)
522526
dataQuery.setDescription(domainKB.getComment(objTemplate));
523527
KBObject objVars = domainKB.getPropertyValue(objTemplate, DISKOnt.getProperty(DISK.HAS_TABLE_VARIABLES));
524-
if (objVars != null)
525-
dataQuery.setVariablesToShow(objVars.getValueAsString());
528+
if (objVars != null) {
529+
String raw = objVars.getValueAsString();
530+
if (raw.startsWith("[") && raw.endsWith("]")) {
531+
dataQuery.setVariablesToShow(Arrays.asList(raw.substring(1, raw.length()-1).split(",")));
532+
} else {
533+
System.out.println("Could not read table variables: " + raw);
534+
}
535+
}
526536
KBObject objFootnotes = domainKB.getPropertyValue(objTemplate, DISKOnt.getProperty(DISK.HAS_TABLE_DESCRIPTION));
527537
if (objFootnotes != null)
528538
dataQuery.setFootnote(objFootnotes.getValueAsString());
@@ -583,7 +593,24 @@ public boolean writeLOI(LineOfInquiry loi) {
583593
this.rdf.startWrite();
584594

585595
KBObject loiItem = writeCommonResource(loi, loiId, DISKOnt.getClass(DISK.LINE_OF_INQUIRY));
586-
writeLOIExtras(loi, loiItem);
596+
writeLOICommon(loi, loiItem);
597+
if (loi.getDataQueryTemplate() != null) {
598+
KBObject dqt = writeDataQueryTemplate(loi.getDataQueryTemplate());
599+
domainKB.setPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_DATA_QUERY), dqt);
600+
}
601+
List<WorkflowSeed> wf = loi.getWorkflowSeeds(), mwf = loi.getMetaWorkflowSeeds();
602+
if (wf != null && wf.size() > 0) {
603+
for (WorkflowSeed wfSeed: wf) {
604+
domainKB.addPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_WORKFLOW_SEED),
605+
writeWorkflowSeed(wfSeed, loi.getId()));
606+
}
607+
}
608+
if (mwf != null && mwf.size() > 0) {
609+
for (WorkflowSeed wfSeed: mwf) {
610+
domainKB.addPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_META_WORKFLOW_SEED),
611+
writeWorkflowSeed(wfSeed, loi.getId()));
612+
}
613+
}
587614
this.rdf.save(domainKB);
588615
this.rdf.end();
589616

@@ -602,71 +629,56 @@ public LineOfInquiry loadLOI(String id) {
602629
}
603630

604631
LineOfInquiry loi = new LineOfInquiry(loadCommonResource(loiItem));
605-
loadLOIExtras(loi, loiItem);
632+
loadLOICommon(loi, loiItem);
633+
KBObject dataQueryObj = domainKB.getPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_DATA_QUERY));
634+
if (dataQueryObj != null)
635+
loi.setDataQueryTemplate(loadDataQueryTemplate(dataQueryObj));
636+
List<KBObject> wfSeeds = domainKB.getPropertyValues(loiItem, DISKOnt.getProperty(DISK.HAS_WORKFLOW_SEED));
637+
List<KBObject> mwfSeeds = domainKB.getPropertyValues(loiItem, DISKOnt.getProperty(DISK.HAS_META_WORKFLOW_SEED));
638+
List<WorkflowSeed> wList = new ArrayList<WorkflowSeed>(), mList = new ArrayList<WorkflowSeed>();
639+
640+
if (wfSeeds != null && wfSeeds.size() > 0) {
641+
for (KBObject t: wfSeeds) {
642+
wList.add(loadWorkflowSeed(t));
643+
}
644+
}
645+
if (mwfSeeds != null && mwfSeeds.size() > 0) {
646+
for (KBObject t: mwfSeeds) {
647+
mList.add(loadWorkflowSeed(t));
648+
}
649+
}
650+
loi.setWorkflowSeeds(wList);
651+
loi.setMetaWorkflowSeeds(mList);
652+
606653
this.rdf.end();
607654
return loi;
608655
}
609656

610-
private void writeLOIExtras (LineOfInquiry loi, KBObject loiItem) {
657+
private void writeLOICommon (LOICommon loi, KBObject loiItem) {
611658
domainKB.setPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_UPDATE_CONDITION),
612659
domainKB.createLiteral(loi.getUpdateCondition()));
613660
if (loi.getGoalQuery() != null) {
614661
domainKB.setPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_GOAL_QUERY),
615662
domainKB.createLiteral(loi.getGoalQuery()));
616663
}
617-
if (loi.getDataQueryTemplate() != null) {
618-
KBObject dqt = writeDataQueryTemplate(loi.getDataQueryTemplate());
619-
domainKB.setPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_DATA_QUERY), dqt);
620-
}
621664
String questionId = loi.getQuestion().getId();
622665
if (questionId != null)
623666
domainKB.setPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_QUESTION),
624667
domainKB.createLiteral(questionId));
625-
List<WorkflowSeed> wf = loi.getWorkflowSeeds(), mwf = loi.getMetaWorkflowSeeds();
626-
if (wf != null && wf.size() > 0) {
627-
for (WorkflowSeed wfSeed: wf) {
628-
domainKB.addPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_WORKFLOW_SEED),
629-
writeWorkflowSeed(wfSeed, loi.getId()));
630-
}
631-
}
632-
if (mwf != null && mwf.size() > 0) {
633-
for (WorkflowSeed wfSeed: mwf) {
634-
domainKB.addPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_META_WORKFLOW_SEED),
635-
writeWorkflowSeed(wfSeed, loi.getId()));
636-
}
637-
}
668+
638669
}
639670

640-
private void loadLOIExtras (LineOfInquiry loi, KBObject loiItem) {
671+
private void loadLOICommon (LOICommon loi, KBObject loiItem) {
641672
KBObject goalQueryObj = domainKB.getPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_GOAL_QUERY));
642673
if (goalQueryObj != null)
643674
loi.setGoalQuery(goalQueryObj.getValueAsString());
644-
KBObject dataQueryObj = domainKB.getPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_DATA_QUERY));
645-
if (dataQueryObj != null)
646-
loi.setDataQueryTemplate(loadDataQueryTemplate(dataQueryObj));
647675
KBObject questionobj = domainKB.getPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_QUESTION));
648676
if (questionobj != null)
649677
loi.setQuestion(new Question(questionobj.getValueAsString()));
650678
KBObject updateCondObj = domainKB.getPropertyValue(loiItem, DISKOnt.getProperty(DISK.HAS_UPDATE_CONDITION));
651679
if (updateCondObj != null)
652680
loi.setUpdateCondition( Integer.parseInt(updateCondObj.getValueAsString()) );
653681

654-
List<KBObject> wfSeeds = domainKB.getPropertyValues(loiItem, DISKOnt.getProperty(DISK.HAS_WORKFLOW_SEED));
655-
List<KBObject> mwfSeeds = domainKB.getPropertyValues(loiItem, DISKOnt.getProperty(DISK.HAS_META_WORKFLOW_SEED));
656-
List<WorkflowSeed> wList = new ArrayList<WorkflowSeed>(), mList = new ArrayList<WorkflowSeed>();
657-
658-
if (wfSeeds != null && wfSeeds.size() > 0) {
659-
for (KBObject t: wfSeeds) {
660-
wList.add(loadWorkflowSeed(t));
661-
}
662-
}
663-
if (mwfSeeds != null && mwfSeeds.size() > 0) {
664-
for (KBObject t: mwfSeeds) {
665-
mList.add(loadWorkflowSeed(t));
666-
}
667-
}
668-
loi.setWorkflowSeeds(wList);
669-
loi.setMetaWorkflowSeeds(mList);
670682
}
671683

672684
public boolean deleteLOI(String id) {
@@ -1058,7 +1070,7 @@ public boolean writeTLOI(TriggeredLOI tloi) {
10581070

10591071
this.rdf.startWrite();
10601072
KBObject tloiItem = writeCommonResource(tloi, tloiId, DISKOnt.getClass(DISK.TRIGGERED_LINE_OF_INQUIRY));
1061-
writeLOIExtras(tloi, tloiItem);
1073+
writeLOICommon(tloi, tloiItem);
10621074
if (tloi.getParentLoi() != null) {
10631075
KBObject loiObj = domainKB.getResource(tloi.getParentLoi().getId());
10641076
domainKB.setPropertyValue(tloiItem, DISKOnt.getProperty(DISK.HAS_LINE_OF_INQUIRY), loiObj);
@@ -1103,7 +1115,7 @@ public TriggeredLOI loadTLOI(String id) {
11031115
KBObject obj = domainKB.getIndividual(tloiId);
11041116
if (obj != null && obj.getName() != null) {
11051117
TriggeredLOI tloi = new TriggeredLOI(loadCommonResource(obj));
1106-
loadLOIExtras(tloi, obj);
1118+
loadLOICommon(tloi, obj);
11071119

11081120
KBObject parentLOI = domainKB.getPropertyValue(obj, DISKOnt.getProperty(DISK.HAS_LINE_OF_INQUIRY));
11091121
if (parentLOI != null) // We do not load the LOI

‎server/src/main/java/org/diskproject/server/managers/MethodAdapterManager.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ public String toString () {
118118

119119
/**
120120
* Create records for each adapter into the RDF database.
121-
* @param db Diskdb where to register the adapters.
121+
* @param db DiskDB where to register the adapters.
122122
*/
123123
public void registerAdapters (DiskDB db) {
124124
this.byId = new HashMap<String, MethodAdapter>();

‎server/src/main/java/org/diskproject/server/quering/Match.java‎

Lines changed: 42 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,10 @@ public class Match {
3232
public final Question question;
3333
public final DataAdapter dataSource;
3434
public boolean fullCSV, valid, ready;
35-
public Set<String> seedVariables, arrayVariables; // These are the variables required from the seeds.
35+
public Set<String> selectVariables, seedVariables, arrayVariables; // These are the variables required from the seeds.
3636
public String csvURL, querySend;
3737
public List<DataResult> queryResult;
38-
public Set<String> queryVariables; //this ones dont have the ?
38+
public Set<String> queryVariables; //this ones do not have the `?`
3939

4040
public Match (Goal goal, LineOfInquiry loi, Question question, DataAdapter dataSource) {
4141
this.goal = goal;
@@ -58,7 +58,7 @@ public Match (Goal goal, LineOfInquiry loi, Question question, DataAdapter dataS
5858
private boolean analyseWorkflows () {
5959
boolean isCSV = false;
6060
List<WorkflowSeed> allSeeds = Stream.concat(loi.getWorkflowSeeds().stream(), loi.getMetaWorkflowSeeds().stream()).collect(Collectors.toList());
61-
Set<String> reqVariables = new HashSet<String>(), arrayVars = new HashSet<String>(), nonArrayVars = new HashSet<String>();
61+
Set<String> reqVariables = new HashSet<String>(), arrayVars = new HashSet<String>(), selectVars = new HashSet<String>();
6262
for (WorkflowSeed seed: allSeeds) {
6363
Endpoint source = seed.getSource();
6464
if (source == null || source.getId() == null || source.getId().equals("")) {
@@ -76,14 +76,14 @@ private boolean analyseWorkflows () {
7676
} else {
7777
//This should not happen.
7878
System.out.println("Workflow seed configuration sets two variables for a single parameter/input");
79-
nonArrayVars.add(raw);
8079
return false;
8180
}
8281
} else {
8382
raw = vb.getSingleBinding();
8483
}
8584
if (raw.startsWith("?")) {
8685
reqVariables.add(raw);
86+
selectVars.add(raw);
8787
} else if (raw.equals(SPECIAL.CSV)) {
8888
isCSV = true;
8989
}
@@ -94,9 +94,16 @@ private boolean analyseWorkflows () {
9494
System.out.println("Workflow seeds must require at least one variable from the data query ");
9595
return false;
9696
}
97+
98+
if (loi.getDataQueryTemplate() != null) {
99+
for (String v: loi.getDataQueryTemplate().getVariablesToShow()) {
100+
selectVars.add(v);
101+
}
102+
}
97103
this.fullCSV = isCSV;
98104
this.seedVariables = reqVariables;
99105
this.arrayVariables = arrayVars;
106+
this.selectVariables = selectVars;
100107
return true;
101108
}
102109

@@ -179,7 +186,15 @@ public boolean createWorkflowInstances () {
179186
}
180187

181188
public String getResultsAsCSV () {
182-
return "";
189+
String csv = String.join(",", queryVariables);
190+
for (DataResult line: queryResult) {
191+
List<String> arrLine = new ArrayList<String>();
192+
for (String varName: queryVariables) {
193+
arrLine.add(line.getValue(varName));
194+
}
195+
csv += "\n" + String.join(",", arrLine);
196+
}
197+
return csv;
183198
}
184199

185200
public TriggeredLOI createTLOI () {
@@ -203,14 +218,15 @@ private List<WorkflowInstantiation> createWorkflowInstance (WorkflowSeed seed) {
203218
if (queryResult == null || queryVariables == null || !ready || queryResult.size() == 0 || queryVariables.size() == 0) {
204219
return null;
205220
}
221+
List<DataResult> filteredResults = filterQueryResults(selectVariables);
206222
// One seed can create multiple instances. As the results are a table, we need to aggregate the results.
207223
int runs = 0;
208224
Map<String,Integer> ticks = new HashMap<String,Integer>();
209225
for (String name: this.queryVariables) {
210226
ticks.put(name, 0);
211227
String lastValue = null;
212228
boolean isArray = arrayVariables.contains("?" + name);
213-
for (DataResult cell: this.queryResult) {
229+
for (DataResult cell: filteredResults) {
214230
String currentValue = cell.getValue(name);
215231
if (currentValue != lastValue) {
216232
int newMax = ticks.get(name) + 1;
@@ -223,15 +239,11 @@ private List<WorkflowInstantiation> createWorkflowInstance (WorkflowSeed seed) {
223239
}
224240
}
225241

226-
//for (String name: queryVariables) {
227-
// System.out.println(name + (arrayVariables.contains("?" + name) ? " [array]" : " [single]") + " = " + ticks.get(name));
228-
//}
229-
230242
//Separate the table depending of the runs.
231243
List<List<DataResult>> independentResults = new ArrayList<List<DataResult>>();
232244
List<DataResult> lastList = new ArrayList<DataResult>();
233-
int count = 0, splitSize = queryResult.size()/runs;
234-
for (DataResult cell: queryResult) {
245+
int count = 0, splitSize = filteredResults.size()/runs;
246+
for (DataResult cell: filteredResults) {
235247
count += 1;
236248
if (count >= splitSize) {
237249
count = 0;
@@ -240,7 +252,6 @@ private List<WorkflowInstantiation> createWorkflowInstance (WorkflowSeed seed) {
240252
}
241253
lastList.add(cell);
242254
}
243-
//System.out.println(independentResults.size());
244255

245256
List<WorkflowInstantiation> inst = new ArrayList<WorkflowInstantiation>();
246257
for (List<DataResult> resultsToBind: independentResults) {
@@ -267,7 +278,7 @@ private List<WorkflowInstantiation> createWorkflowInstance (WorkflowSeed seed) {
267278
}
268279
} else {
269280
// Required variable is not on the query results.
270-
// Should break the outer loop, FIXME;
281+
// Should break the outer loop?
271282
continue;
272283
}
273284
}
@@ -277,10 +288,26 @@ private List<WorkflowInstantiation> createWorkflowInstance (WorkflowSeed seed) {
277288
}
278289
}
279290
current.setDataBindings(dataBindings);
280-
//System.out.println(current);
281291
inst.add(current);
282292
}
283293
return inst;
284294
}
285295

296+
private List<DataResult> filterQueryResults (Set<String> allowedVariables) {
297+
// If the query results includes variables not used on the workflow, these are removed if the contents are the same.
298+
List<DataResult> list = new ArrayList<DataResult>();
299+
String lastLine = "";
300+
for (DataResult cell: queryResult) {
301+
String currentLine = "";
302+
for (String v: allowedVariables) {
303+
String varName = v.startsWith("?") ? v.substring(1) : v;
304+
currentLine += cell.getValue(varName) + ",";
305+
}
306+
if (!currentLine.equals(lastLine)) {
307+
lastLine = currentLine;
308+
list.add(cell);
309+
}
310+
}
311+
return list;
312+
}
286313
}

‎server/src/main/java/org/diskproject/server/repository/DiskRepository.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -544,7 +544,7 @@ public List<TriggeredLOI> queryGoal (String id) throws Exception, QueryParseExce
544544

545545
private TriggeredLOI uploadData (TriggeredLOI tloi) {
546546
//Check and upload files.
547-
DataAdapter dataAdapter = dataAdapters.getMethodAdapterByEndpoint(tloi.getDataQueryTemplate().getEndpoint());
547+
DataAdapter dataAdapter = dataAdapters.getMethodAdapterByEndpoint(tloi.getQueryResults().getEndpoint());
548548
List<WorkflowInstantiation> wf = new ArrayList<WorkflowInstantiation>(),
549549
mwf = new ArrayList<WorkflowInstantiation>();
550550
for (WorkflowInstantiation i: tloi.getWorkflows()) {

‎server/src/main/java/org/diskproject/server/threads/MonitorThread.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -154,9 +154,5 @@ private void updateWorkflowStatus() {
154154
}
155155
}
156156
}
157-
System.out.println("Update wf status:");
158-
for (WorkflowInstantiation inst: list) {
159-
System.out.println(inst.getStatus());
160-
}
161157
}
162158
}

0 commit comments

Comments
 (0)