Skip to content

Commit 1cdc077

Browse files
committed
Merge remote-tracking branch 'origin/master' into packed-workflows
# Conflicts: # src/main/java/org/commonwl/view/workflow/WorkflowFormValidator.java # src/main/java/org/commonwl/view/workflow/WorkflowService.java
2 parents a3e4796 + 055d20c commit 1cdc077

File tree

12 files changed

+115
-106
lines changed

12 files changed

+115
-106
lines changed

src/main/java/org/commonwl/view/cwl/CWLService.java

Lines changed: 42 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -19,20 +19,27 @@
1919

2020
package org.commonwl.view.cwl;
2121

22-
import com.fasterxml.jackson.databind.JsonNode;
23-
import com.fasterxml.jackson.databind.ObjectMapper;
24-
import com.fasterxml.jackson.databind.node.ArrayNode;
25-
import com.fasterxml.jackson.databind.node.ObjectNode;
26-
import com.fasterxml.jackson.databind.node.TextNode;
22+
import static org.apache.commons.io.FileUtils.readFileToString;
23+
24+
import java.io.ByteArrayInputStream;
25+
import java.io.File;
26+
import java.io.IOException;
27+
import java.io.StringWriter;
28+
import java.util.ArrayList;
29+
import java.util.HashMap;
30+
import java.util.Iterator;
31+
import java.util.List;
32+
import java.util.Map;
33+
2734
import org.apache.commons.io.FileUtils;
2835
import org.apache.commons.io.FilenameUtils;
36+
import org.apache.jena.iri.IRI;
37+
import org.apache.jena.iri.IRIFactory;
2938
import org.apache.jena.ontology.OntModelSpec;
3039
import org.apache.jena.query.QuerySolution;
3140
import org.apache.jena.query.ResultSet;
3241
import org.apache.jena.rdf.model.Model;
3342
import org.apache.jena.rdf.model.ModelFactory;
34-
import org.apache.jena.rdf.model.Statement;
35-
import org.apache.jena.rdf.model.StmtIterator;
3643
import org.apache.jena.riot.RiotException;
3744
import org.commonwl.view.docker.DockerService;
3845
import org.commonwl.view.git.GitDetails;
@@ -47,15 +54,11 @@
4754
import org.springframework.stereotype.Service;
4855
import org.yaml.snakeyaml.Yaml;
4956

50-
import java.io.ByteArrayInputStream;
51-
import java.io.File;
52-
import java.io.IOException;
53-
import java.io.StringWriter;
54-
import java.nio.file.Path;
55-
import java.nio.file.Paths;
56-
import java.util.*;
57-
58-
import static org.apache.commons.io.FileUtils.readFileToString;
57+
import com.fasterxml.jackson.databind.JsonNode;
58+
import com.fasterxml.jackson.databind.ObjectMapper;
59+
import com.fasterxml.jackson.databind.node.ArrayNode;
60+
import com.fasterxml.jackson.databind.node.ObjectNode;
61+
import com.fasterxml.jackson.databind.node.TextNode;
5962

6063
/**
6164
* Provides CWL parsing for workflows to gather an overview
@@ -65,6 +68,7 @@
6568
public class CWLService {
6669

6770
private final Logger logger = LoggerFactory.getLogger(this.getClass());
71+
private final IRIFactory iriFactory = IRIFactory.iriImplementation();
6872

6973
// Autowired properties/services
7074
private final RDFService rdfService;
@@ -273,42 +277,18 @@ public Workflow parseWorkflowWithCwltool(Workflow basicModel,
273277
String inputName = rdfService.stepNameFromURI(gitPath, input.get("name").toString());
274278

275279
CWLElement wfInput = new CWLElement();
276-
277-
// Array types
278280
if (input.contains("type")) {
279-
StmtIterator itr = input.get("type").asResource().listProperties();
280-
if (itr.hasNext()) {
281-
while (itr.hasNext()) {
282-
Statement complexType = itr.nextStatement();
283-
if (complexType.getPredicate().toString()
284-
.equals("https://w3id.org/cwl/salad#items")) {
285-
if (wfInputs.containsKey(inputName)
286-
&& wfInputs.get(inputName).getType().equals("?")) {
287-
wfInput.setType(typeURIToString(complexType.getObject().toString()) + "[]?");
288-
} else {
289-
wfInput.setType(typeURIToString(complexType.getObject().toString()) + "[]");
290-
}
291-
}
292-
}
281+
String type;
282+
if (input.get("type").toString().equals("https://w3id.org/cwl/salad#array")) {
283+
type = typeURIToString(input.get("items").toString()) + "[]";
293284
} else {
294-
// Optional types
295-
if (input.get("type").toString().equals("https://w3id.org/cwl/salad#null")) {
296-
if (wfInputs.containsKey(inputName)) {
297-
CWLElement inputInMap = wfInputs.get(inputName);
298-
inputInMap.setType(inputInMap.getType() + "?");
299-
} else {
300-
wfInput.setType("?");
301-
}
302-
} else if (wfInput.getType() != null && wfInput.getType().equals("?")
303-
&& !wfInput.getType().endsWith("[]")) {
304-
wfInput.setType(typeURIToString(input.get("type").toString()) + "?");
305-
} else {
306-
// Standard type
307-
wfInput.setType(typeURIToString(input.get("type").toString()));
308-
}
285+
type = typeURIToString(input.get("type").toString());
286+
}
287+
if (input.contains("null")) {
288+
type += " (Optional)";
309289
}
290+
wfInput.setType(type);
310291
}
311-
312292
if (input.contains("format")) {
313293
String format = input.get("format").toString();
314294
setFormat(wfInput, format);
@@ -330,41 +310,17 @@ public Workflow parseWorkflowWithCwltool(Workflow basicModel,
330310
CWLElement wfOutput = new CWLElement();
331311

332312
String outputName = rdfService.stepNameFromURI(gitPath, output.get("name").toString());
333-
334-
// Array types
335313
if (output.contains("type")) {
336-
StmtIterator itr = output.get("type").asResource().listProperties();
337-
if (itr.hasNext()) {
338-
while (itr.hasNext()) {
339-
Statement complexType = itr.nextStatement();
340-
if (complexType.getPredicate().toString()
341-
.equals("https://w3id.org/cwl/salad#items")) {
342-
if (wfOutputs.containsKey(outputName)
343-
&& wfOutputs.get(outputName).getType().equals("?")) {
344-
wfOutput.setType(typeURIToString(complexType.getObject().toString()) + "[]?");
345-
} else {
346-
wfOutput.setType(typeURIToString(complexType.getObject().toString()) + "[]");
347-
}
348-
}
349-
}
314+
String type;
315+
if (output.get("type").toString().equals("https://w3id.org/cwl/salad#array")) {
316+
type = typeURIToString(output.get("items").toString()) + "[]";
350317
} else {
351-
// Standard types
352-
if (wfOutput.getType() != null && wfOutput.getType().equals("?")) {
353-
wfOutput.setType(typeURIToString(output.get("type").toString()) + "?");
354-
} else {
355-
wfOutput.setType(typeURIToString(output.get("type").toString()));
356-
}
357-
358-
// Optional types
359-
if (output.get("type").toString().equals("https://w3id.org/cwl/salad#null")) {
360-
if (wfOutputs.containsKey(outputName)) {
361-
CWLElement outputInMap = wfOutputs.get(outputName);
362-
outputInMap.setType(outputInMap.getType() + "?");
363-
} else {
364-
wfOutput.setType("?");
365-
}
366-
}
318+
type = typeURIToString(output.get("type").toString());
367319
}
320+
if (output.contains("null")) {
321+
type += " (Optional)";
322+
}
323+
wfOutput.setType(type);
368324
}
369325

370326
if (output.contains("src")) {
@@ -408,8 +364,10 @@ public Workflow parseWorkflowWithCwltool(Workflow basicModel,
408364
// Add new step
409365
CWLStep wfStep = new CWLStep();
410366

411-
Path workflowPath = Paths.get(step.get("wf").toString()).getParent();
412-
Path runPath = Paths.get(step.get("run").toString());
367+
IRI wfStepUri = iriFactory.construct(step.get("wf").asResource().getURI());
368+
IRI workflowPath = wfStepUri.resolve("./");
369+
370+
IRI runPath = iriFactory.construct(step.get("run").asResource().getURI());
413371
wfStep.setRun(workflowPath.relativize(runPath).toString());
414372
wfStep.setRunType(rdfService.strToRuntype(step.get("runtype").toString()));
415373

@@ -538,9 +496,9 @@ private void setFormat(CWLElement inputOutput, String format) {
538496
rdfService.addToOntologies(ontModel);
539497
}
540498
String formatLabel = rdfService.getOntLabel(format);
541-
inputOutput.setType(inputOutput.getType() + " (" + formatLabel + ")");
499+
inputOutput.setType(inputOutput.getType() + " [" + formatLabel + "]");
542500
} catch (RiotException ex) {
543-
inputOutput.setType(inputOutput.getType() + " (format)");
501+
inputOutput.setType(inputOutput.getType() + " [format]");
544502
}
545503
}
546504

src/main/java/org/commonwl/view/cwl/CWLTool.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ private String runCwltoolOnWorkflow(String argument, String workflowUrl) throws
117117
inputGobbler.join();
118118
return inputGobbler.getContent();
119119
} else {
120+
errorGobbler.join();
120121
throw new CWLValidationException(errorGobbler.getContent());
121122
}
122123
} catch (IOException|InterruptedException e) {

src/main/java/org/commonwl/view/cwl/RDFService.java

Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -158,15 +158,29 @@ public String getOntLabel(String ontologyURI) {
158158
public ResultSet getInputs(String path, String workflowURI) {
159159
ParameterizedSparqlString inputsQuery = new ParameterizedSparqlString();
160160
inputsQuery.setCommandText(queryCtx +
161-
"SELECT ?name ?type ?format ?label ?doc\n" +
161+
"SELECT ?name ?type ?items ?null ?format ?label ?doc\n" +
162162
"WHERE {\n" +
163163
" GRAPH ?graphName {" +
164164
" ?wf rdf:type cwl:Workflow .\n" +
165165
" ?wf cwl:inputs ?name .\n" +
166-
" OPTIONAL { ?name sld:type ?type }\n" +
166+
" OPTIONAL {\n" +
167+
" { \n" +
168+
" ?name sld:type ?type\n" +
169+
" FILTER(?type != sld:null) \n" +
170+
" FILTER (!isBlank(?type))\n" +
171+
" } UNION { \n" +
172+
" ?name sld:type ?arraytype .\n" +
173+
" ?arraytype sld:type ?type .\n" +
174+
" ?arraytype sld:items ?items \n" +
175+
" }\n" +
176+
" }\n" +
177+
" OPTIONAL { \n" +
178+
" ?name sld:type ?null\n" +
179+
" FILTER(?null = sld:null)\n" +
180+
" }\n" +
167181
" OPTIONAL { ?name cwl:format ?format }\n" +
168182
" OPTIONAL { ?name sld:label|rdfs:label ?label }\n" +
169-
" OPTIONAL { ?name sld:doc|rdfs:comment ?doc }\n" +
183+
" OPTIONAL { ?name sld:doc|rdfs:comment ?doc }" +
170184
" FILTER(regex(str(?wf), ?wfFilter, \"i\" ))" +
171185
" }" +
172186
"}");
@@ -184,15 +198,29 @@ public ResultSet getInputs(String path, String workflowURI) {
184198
public ResultSet getOutputs(String path, String workflowURI) {
185199
ParameterizedSparqlString outputsQuery = new ParameterizedSparqlString();
186200
outputsQuery.setCommandText(queryCtx +
187-
"SELECT ?name ?type ?format ?label ?doc\n" +
201+
"SELECT ?name ?type ?items ?null ?format ?label ?doc\n" +
188202
"WHERE {\n" +
189203
" GRAPH ?graphName {" +
190204
" ?wf rdf:type cwl:Workflow .\n" +
191205
" ?wf cwl:outputs ?name .\n" +
192-
" OPTIONAL { ?name sld:type ?type }\n" +
206+
" OPTIONAL {\n" +
207+
" { \n" +
208+
" ?name sld:type ?type\n" +
209+
" FILTER(?type != sld:null) \n" +
210+
" FILTER (!isBlank(?type))\n" +
211+
" } UNION { \n" +
212+
" ?name sld:type ?arraytype .\n" +
213+
" ?arraytype sld:type ?type .\n" +
214+
" ?arraytype sld:items ?items \n" +
215+
" }\n" +
216+
" }\n" +
217+
" OPTIONAL { \n" +
218+
" ?name sld:type ?null\n" +
219+
" FILTER(?null = sld:null)\n" +
220+
" }\n" +
193221
" OPTIONAL { ?name cwl:format ?format }\n" +
194222
" OPTIONAL { ?name sld:label|rdfs:label ?label }\n" +
195-
" OPTIONAL { ?name sld:doc|rdfs:comment ?doc }\n" +
223+
" OPTIONAL { ?name sld:doc|rdfs:comment ?doc }" +
196224
" FILTER(regex(str(?wf), ?wfFilter, \"i\" ))" +
197225
" }" +
198226
"}");

src/main/java/org/commonwl/view/workflow/WorkflowController.java

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -135,22 +135,24 @@ public ModelAndView createWorkflow(@Valid WorkflowForm workflowForm, BindingResu
135135
return new ModelAndView("redirect:" + gitInfo.getInternalUrl());
136136
}
137137
} catch (TransportException ex) {
138+
logger.warn("git.sshError " + workflowForm , ex);
138139
bindingResult.rejectValue("url", "git.sshError");
139140
return new ModelAndView("index");
140141
} catch (GitAPIException ex) {
142+
logger.error("git.retrievalError " + workflowForm , ex);
141143
bindingResult.rejectValue("url", "git.retrievalError");
142-
logger.error("Git API Error", ex);
143144
return new ModelAndView("index");
144145
} catch (WorkflowNotFoundException ex) {
146+
logger.warn("git.pathTraversal " + workflowForm , ex);
145147
bindingResult.rejectValue("url", "git.pathTraversal");
146148
return new ModelAndView("index");
147149
} catch (Exception ex) {
150+
logger.warn("url.parsingError " + workflowForm , ex);
148151
bindingResult.rejectValue("url", "url.parsingError");
149152
return new ModelAndView("index");
150153
}
151154
}
152155
gitInfo = workflow.getRetrievedFrom();
153-
154156
// Redirect to the workflow
155157
return new ModelAndView("redirect:" + gitInfo.getInternalUrl());
156158
}
@@ -498,12 +500,16 @@ private ModelAndView getWorkflow(GitDetails gitDetails, RedirectAttributes redir
498500
}
499501
}
500502
} catch (TransportException ex) {
503+
logger.warn("git.sshError " + workflowForm , ex);
501504
errors.rejectValue("url", "git.sshError", "SSH URLs are not supported, please provide a HTTPS URL for the repository or submodules");
502505
} catch (GitAPIException ex) {
506+
logger.error("git.retrievalError " + workflowForm, ex);
503507
errors.rejectValue("url", "git.retrievalError", "The workflow could not be retrieved from the Git repository using the details given");
504508
} catch (WorkflowNotFoundException ex) {
509+
logger.warn("git.pathTraversal " + workflowForm, ex);
505510
errors.rejectValue("url", "git.pathTraversal", "The path given did not resolve to a location within the repository");
506511
} catch (IOException ex) {
512+
logger.warn("git.parsingError " + workflowForm, ex);
507513
errors.rejectValue("url", "url.parsingError", "The workflow could not be parsed from the given URL");
508514
}
509515
}

src/main/java/org/commonwl/view/workflow/WorkflowForm.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,4 +75,11 @@ public void setPath(String path) {
7575
private String trimTrailingSlashes(String url) {
7676
return url.replaceAll("\\/+$", "");
7777
}
78+
79+
@Override
80+
public String toString() {
81+
return "WorkflowForm [" + (url != null ? "url=" + url + ", " : "")
82+
+ (branch != null ? "branch=" + branch + ", " : "") + (path != null ? "path=" + path : "") + "]";
83+
}
84+
7885
}

src/main/java/org/commonwl/view/workflow/WorkflowFormValidator.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,15 +98,15 @@ public GitDetails validateAndParse(WorkflowForm form, Errors e) {
9898

9999
// Github Dir URL
100100
m = githubDirPattern.matcher(form.getUrl());
101-
if (m.find()) {
101+
if (m.find() && ! m.group(2).endsWith(".git")) {
102102
repoUrl = "https://github.com/" + m.group(1) + "/" + m.group(2) + ".git";
103103
if (branch == null) branch = m.group(3);
104104
if (path == null) path = m.group(4);
105105
}
106106

107107
// Gitlab Dir URL
108108
m = gitlabDirPattern.matcher(form.getUrl());
109-
if (m.find()) {
109+
if (m.find() && ! m.group(2).endsWith(".git")) {
110110
repoUrl = "https://gitlab.com/" + m.group(1) + "/" + m.group(2) + ".git";
111111
if (branch == null) branch = m.group(3);
112112
if (path == null) path = m.group(4);

src/main/java/org/commonwl/view/workflow/WorkflowService.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,9 @@
4141

4242
import java.io.File;
4343
import java.io.IOException;
44+
import java.nio.file.Files;
4445
import java.nio.file.Path;
46+
import java.nio.file.Paths;
4547
import java.util.ArrayList;
4648
import java.util.Calendar;
4749
import java.util.Date;
@@ -212,7 +214,8 @@ public List<WorkflowOverview> getWorkflowsFromDirectory(GitDetails gitInfo) thro
212214

213215
Path localPath = repo.getRepository().getWorkTree().toPath();
214216
Path pathToDirectory = localPath.resolve(gitInfo.getPath()).normalize().toAbsolutePath();
215-
if (pathToDirectory.toString().equals("/")) {
217+
Path root = Paths.get("/").toAbsolutePath();
218+
if (pathToDirectory.equals(root)) {
216219
pathToDirectory = localPath;
217220
} else if (!pathToDirectory.startsWith(localPath.normalize().toAbsolutePath())) {
218221
// Prevent path traversal attacks
@@ -307,6 +310,9 @@ public QueuedWorkflow createQueuedWorkflow(GitDetails gitInfo)
307310
}
308311

309312
File workflowFile = new File(pathToWorkflowFile.toString());
313+
if (! Files.isReadable(workflowFile.toPath())) {
314+
throw new WorkflowNotFoundException();
315+
}
310316

311317
// Handling of packed workflows
312318
String packedWorkflowId = gitInfo.getPackedId();

src/main/resources/application.properties

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,14 @@
66
applicationName = Common Workflow Language Viewer
77
applicationURL = https://view.commonwl.org
88

9-
# Path to a directory in which the RO Bundles will be stored
10-
bundleStorage = /tmp
9+
# Path to a directory in which the RO Bundles will be stored, e.g. /tmp
10+
bundleStorage = ${java.io.tmpdir}
1111

12-
# Path to a directory in which graphviz images will be stored
13-
graphvizStorage = /tmp
12+
# Path to a directory in which graphviz images will be stored, e.g. /tmp
13+
graphvizStorage = ${java.io.tmpdir}
1414

15-
# Path to a directory in which git repositories will be checked out into
16-
gitStorage = /tmp
15+
# Path to a directory in which git repositories will be checked out into, e.g. /tmp
16+
gitStorage = ${java.io.tmpdir}
1717

1818
# How long to cache workflows in days before checking for changes via Github
1919
cacheDays = 1

src/main/resources/templates/workflow.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
<meta name="twitter:site" content="@commonwl" />
2727
<meta name="twitter:title" th:content="${'CWL Workflow: ' + workflow.label}" />
2828
<meta name="twitter:description" th:if="${workflow.doc != null}" th:content="${workflow.doc}" />
29-
<meta name="twitter:image" th:content="${appURL + '/workflows/' + workflow.id + '/graph/png'}" />
29+
<meta name="twitter:image" th:content="@{${workflow.getVisualisationPng()}}" />
3030
<title th:text="${workflow.label + ' - Common Workflow Language Viewer'}">Common Workflow Language Viewer</title>
3131
<link rel="stylesheet" th:href="@{/bower_components/bootstrap/dist/css/bootstrap.min.css}" href="../static/bower_components/bootstrap/dist/css/bootstrap.min.css" />
3232
<link rel="stylesheet" type="text/css" th:href="@{/css/main-20170616.css}" href="../static/css/main-20170616.css" />

0 commit comments

Comments
 (0)