Skip to content

Commit 9ee66d8

Browse files
stainstain
authored
Merge branch 'master' into hotfix-twitter
2 parents b15a490 + f44689e commit 9ee66d8

20 files changed

+634
-120
lines changed

‎src/main/java/org/commonwl/view/cwl/CWLElement.java

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -73,14 +73,6 @@ public void setFormat(String format) {
7373
this.format = format;
7474
}
7575

76-
public List<String> getSourceID() {
77-
return sourceID;
78-
}
79-
80-
public void setSourceID(List<String> sourceID) {
81-
this.sourceID = sourceID;
82-
}
83-
8476
public List<String> getSourceIDs() {
8577
return sourceID;
8678
}

‎src/main/java/org/commonwl/view/cwl/CWLService.java

Lines changed: 74 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,22 @@
1919

2020
package org.commonwl.view.cwl;
2121

22-
import com.fasterxml.jackson.databind.JsonNode;
23-
import com.fasterxml.jackson.databind.ObjectMapper;
24-
import com.fasterxml.jackson.databind.node.ArrayNode;
25-
import com.fasterxml.jackson.databind.node.ObjectNode;
26-
import com.fasterxml.jackson.databind.node.TextNode;
22+
import static org.apache.commons.io.FileUtils.readFileToString;
23+
24+
import java.io.ByteArrayInputStream;
25+
import java.io.File;
26+
import java.io.IOException;
27+
import java.io.StringWriter;
28+
import java.util.ArrayList;
29+
import java.util.HashMap;
30+
import java.util.Iterator;
31+
import java.util.List;
32+
import java.util.Map;
33+
2734
import org.apache.commons.io.FileUtils;
2835
import org.apache.commons.io.FilenameUtils;
36+
import org.apache.jena.iri.IRI;
37+
import org.apache.jena.iri.IRIFactory;
2938
import org.apache.jena.ontology.OntModelSpec;
3039
import org.apache.jena.query.QuerySolution;
3140
import org.apache.jena.query.ResultSet;
@@ -39,22 +48,19 @@
3948
import org.commonwl.view.graphviz.ModelDotWriter;
4049
import org.commonwl.view.graphviz.RDFDotWriter;
4150
import org.commonwl.view.workflow.Workflow;
51+
import org.commonwl.view.workflow.WorkflowOverview;
4252
import org.slf4j.Logger;
4353
import org.slf4j.LoggerFactory;
4454
import org.springframework.beans.factory.annotation.Autowired;
4555
import org.springframework.beans.factory.annotation.Value;
4656
import org.springframework.stereotype.Service;
4757
import org.yaml.snakeyaml.Yaml;
4858

49-
import java.io.ByteArrayInputStream;
50-
import java.io.File;
51-
import java.io.IOException;
52-
import java.io.StringWriter;
53-
import java.nio.file.Path;
54-
import java.nio.file.Paths;
55-
import java.util.*;
56-
57-
import static org.apache.commons.io.FileUtils.readFileToString;
59+
import com.fasterxml.jackson.databind.JsonNode;
60+
import com.fasterxml.jackson.databind.ObjectMapper;
61+
import com.fasterxml.jackson.databind.node.ArrayNode;
62+
import com.fasterxml.jackson.databind.node.ObjectNode;
63+
import com.fasterxml.jackson.databind.node.TextNode;
5864

5965
/**
6066
* Provides CWL parsing for workflows to gather an overview
@@ -64,6 +70,7 @@
6470
public class CWLService {
6571

6672
private final Logger logger = LoggerFactory.getLogger(this.getClass());
73+
private final IRIFactory iriFactory = IRIFactory.iriImplementation();
6774

6875
// Autowired properties/services
6976
private final RDFService rdfService;
@@ -367,8 +374,10 @@ public Workflow parseWorkflowWithCwltool(Workflow basicModel,
367374
// Add new step
368375
CWLStep wfStep = new CWLStep();
369376

370-
Path workflowPath = Paths.get(step.get("wf").toString()).getParent();
371-
Path runPath = Paths.get(step.get("run").toString());
377+
IRI wfStepUri = iriFactory.construct(step.get("wf").asResource().getURI());
378+
IRI workflowPath = wfStepUri.resolve("./");
379+
380+
IRI runPath = iriFactory.construct(step.get("run").asResource().getURI());
372381
wfStep.setRun(workflowPath.relativize(runPath).toString());
373382
wfStep.setRunType(rdfService.strToRuntype(step.get("runtype").toString()));
374383

@@ -428,6 +437,55 @@ public Workflow parseWorkflowWithCwltool(Workflow basicModel,
428437

429438
}
430439

440+
/**
441+
* Get an overview of a workflow
442+
* @param file A file, potentially a workflow
443+
* @return A constructed WorkflowOverview of the workflow
444+
* @throws IOException Any API errors which may have occurred
445+
*/
446+
public WorkflowOverview getWorkflowOverview(File file) throws IOException {
447+
448+
// Get the content of this file from Github
449+
long fileSizeBytes = file.length();
450+
451+
// Check file size limit before parsing
452+
if (fileSizeBytes <= singleFileSizeLimit) {
453+
454+
// Parse file as yaml
455+
JsonNode cwlFile = yamlStringToJson(readFileToString(file));
456+
457+
// If the CWL file is packed there can be multiple workflows in a file
458+
if (cwlFile.has(DOC_GRAPH)) {
459+
// Packed CWL, find the first subelement which is a workflow and take it
460+
for (JsonNode jsonNode : cwlFile.get(DOC_GRAPH)) {
461+
if (extractProcess(jsonNode) == CWLProcess.WORKFLOW) {
462+
cwlFile = jsonNode;
463+
}
464+
}
465+
}
466+
467+
// Can only make an overview if this is a workflow
468+
if (extractProcess(cwlFile) == CWLProcess.WORKFLOW) {
469+
// Use filename for label if there is no defined one
470+
String label = extractLabel(cwlFile);
471+
if (label == null) {
472+
label = file.getName();
473+
}
474+
475+
// Return the constructed overview
476+
return new WorkflowOverview(file.getName(), label, extractDoc(cwlFile));
477+
} else {
478+
// Return null if not a workflow file
479+
return null;
480+
}
481+
} else {
482+
throw new IOException("File '" + file.getName() + "' is over singleFileSizeLimit - " +
483+
FileUtils.byteCountToDisplaySize(fileSizeBytes) + "/" +
484+
FileUtils.byteCountToDisplaySize(singleFileSizeLimit));
485+
}
486+
487+
}
488+
431489
/**
432490
* Set the format for an input or output, handling ontologies
433491
* @param inputOutput The input or output CWL Element

‎src/main/java/org/commonwl/view/git/GitDetails.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -139,12 +139,13 @@ public String getUrl() {
139139
* @return The URL
140140
*/
141141
public String getInternalUrl() {
142+
String pathPart = path.equals("/") ? "" : "/" + path;
142143
switch (getType()) {
143144
case GITHUB:
144145
case GITLAB:
145-
return "/workflows/" + normaliseUrl(repoUrl).replace(".git", "") + "/blob/" + branch + "/" + path;
146+
return "/workflows/" + normaliseUrl(repoUrl).replace(".git", "") + "/blob/" + branch + pathPart;
146147
default:
147-
return "/workflows/" + normaliseUrl(repoUrl) + "/" + branch + "/" + path;
148+
return "/workflows/" + normaliseUrl(repoUrl) + "/" + branch + pathPart;
148149
}
149150
}
150151

‎src/main/java/org/commonwl/view/workflow/WorkflowController.java

Lines changed: 35 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.commonwl.view.git.GitDetails;
2525
import org.commonwl.view.graphviz.GraphVizService;
2626
import org.eclipse.jgit.api.errors.GitAPIException;
27+
import org.eclipse.jgit.api.errors.TransportException;
2728
import org.slf4j.Logger;
2829
import org.slf4j.LoggerFactory;
2930
import org.springframework.beans.factory.annotation.Autowired;
@@ -45,6 +46,7 @@
4546
import javax.validation.Valid;
4647
import java.io.File;
4748
import java.io.IOException;
49+
import java.util.List;
4850

4951
@Controller
5052
public class WorkflowController {
@@ -119,21 +121,30 @@ public ModelAndView createWorkflow(@Valid WorkflowForm workflowForm, BindingResu
119121
Workflow workflow = workflowService.getWorkflow(gitInfo);
120122
if (workflow == null) {
121123
try {
122-
workflow = workflowService.createQueuedWorkflow(gitInfo).getTempRepresentation();
124+
if (gitInfo.getPath().endsWith(".cwl")) {
125+
workflow = workflowService.createQueuedWorkflow(gitInfo).getTempRepresentation();
126+
} else {
127+
return new ModelAndView("redirect:" + gitInfo.getInternalUrl());
128+
}
129+
} catch (TransportException ex) {
130+
logger.warn("git.sshError " + workflowForm , ex);
131+
bindingResult.rejectValue("url", "git.sshError");
132+
return new ModelAndView("index");
123133
} catch (GitAPIException ex) {
134+
logger.error("git.retrievalError " + workflowForm , ex);
124135
bindingResult.rejectValue("url", "git.retrievalError");
125-
logger.error("Git API Error", ex);
126136
return new ModelAndView("index");
127137
} catch (WorkflowNotFoundException ex) {
138+
logger.warn("git.pathTraversal " + workflowForm , ex);
128139
bindingResult.rejectValue("url", "git.pathTraversal");
129140
return new ModelAndView("index");
130141
} catch (Exception ex) {
142+
logger.warn("url.parsingError " + workflowForm , ex);
131143
bindingResult.rejectValue("url", "url.parsingError");
132144
return new ModelAndView("index");
133145
}
134146
}
135147
gitInfo = workflow.getRetrievedFrom();
136-
137148
// Redirect to the workflow
138149
return new ModelAndView("redirect:" + gitInfo.getInternalUrl());
139150
}
@@ -444,18 +455,36 @@ private ModelAndView getWorkflow(GitDetails gitDetails, RedirectAttributes redir
444455
queued = workflowService.getQueuedWorkflow(gitDetails);
445456
if (queued == null) {
446457
// Validation
447-
WorkflowForm workflowForm = new WorkflowForm(gitDetails.getUrl());
458+
WorkflowForm workflowForm = new WorkflowForm(gitDetails.getUrl(), gitDetails.getBranch(), gitDetails.getPath());
448459
BeanPropertyBindingResult errors = new BeanPropertyBindingResult(workflowForm, "errors");
449460
workflowFormValidator.validateAndParse(workflowForm, errors);
450461
if (!errors.hasErrors()) {
451462
try {
452-
queued = workflowService.createQueuedWorkflow(gitDetails);
463+
if (gitDetails.getPath().endsWith(".cwl")) {
464+
queued = workflowService.createQueuedWorkflow(gitDetails);
465+
} else {
466+
List<WorkflowOverview> workflowOverviews = workflowService.getWorkflowsFromDirectory(gitDetails);
467+
if (workflowOverviews.size() > 1) {
468+
return new ModelAndView("selectworkflow", "workflowOverviews", workflowOverviews)
469+
.addObject("gitDetails", gitDetails);
470+
} else if (workflowOverviews.size() == 1) {
471+
return new ModelAndView("redirect:" + gitDetails.getInternalUrl() +
472+
"/" + workflowOverviews.get(0).getFileName());
473+
} else {
474+
errors.rejectValue("url", "url.noWorkflowsInDirectory", "No workflow files were found in the given directory");
475+
}
476+
}
477+
} catch (TransportException ex) {
478+
logger.warn("git.sshError " + workflowForm , ex);
479+
errors.rejectValue("url", "git.sshError", "SSH URLs are not supported, please provide a HTTPS URL for the repository or submodules");
453480
} catch (GitAPIException ex) {
481+
logger.error("git.retrievalError " + workflowForm, ex);
454482
errors.rejectValue("url", "git.retrievalError", "The workflow could not be retrieved from the Git repository using the details given");
455-
logger.error("Git API Error", ex);
456483
} catch (WorkflowNotFoundException ex) {
484+
logger.warn("git.pathTraversal " + workflowForm, ex);
457485
errors.rejectValue("url", "git.pathTraversal", "The path given did not resolve to a location within the repository");
458486
} catch (IOException ex) {
487+
logger.warn("git.parsingError " + workflowForm, ex);
459488
errors.rejectValue("url", "url.parsingError", "The workflow could not be parsed from the given URL");
460489
}
461490
}

‎src/main/java/org/commonwl/view/workflow/WorkflowForm.java

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,13 @@ public class WorkflowForm {
3232
public WorkflowForm() {}
3333

3434
public WorkflowForm(String url) {
35-
if (url != null) {
36-
this.url = trimTrailingSlashes(url);
37-
}
35+
setUrl(url);
36+
}
37+
38+
public WorkflowForm(String url, String branch, String path) {
39+
setUrl(url);
40+
this.branch = branch;
41+
this.path = path;
3842
}
3943

4044
public String getUrl() {
@@ -71,4 +75,11 @@ public void setPath(String path) {
7175
private String trimTrailingSlashes(String url) {
7276
return url.replaceAll("\\/+$", "");
7377
}
78+
79+
@Override
80+
public String toString() {
81+
return "WorkflowForm [" + (url != null ? "url=" + url + ", " : "")
82+
+ (branch != null ? "branch=" + branch + ", " : "") + (path != null ? "path=" + path : "") + "]";
83+
}
84+
7485
}

‎src/main/java/org/commonwl/view/workflow/WorkflowFormValidator.java

Lines changed: 55 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.commonwl.view.workflow;
2121

22+
import org.apache.commons.lang.StringUtils;
2223
import org.commonwl.view.git.GitDetails;
2324
import org.slf4j.Logger;
2425
import org.slf4j.LoggerFactory;
@@ -41,11 +42,19 @@ public class WorkflowFormValidator {
4142
private static final String GITHUB_CWL_REGEX = "^https?:\\/\\/github\\.com\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:tree|blob)\\/([^/]+)(?:\\/(.+\\.cwl))$";
4243
private static final Pattern githubCwlPattern = Pattern.compile(GITHUB_CWL_REGEX);
4344

45+
// URL validation for directories on Github
46+
private static final String GITHUB_DIR_REGEX = "^https?:\\/\\/github\\.com\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:(?:tree|blob)\\/([^/]+)\\/?(.*)?)?$";
47+
private static final Pattern githubDirPattern = Pattern.compile(GITHUB_DIR_REGEX);
48+
4449
// URL validation for cwl files on Gitlab
4550
private static final String GITLAB_CWL_REGEX = "^https?:\\/\\/gitlab\\.com\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:tree|blob)\\/([^/]+)(?:\\/(.+\\.cwl))$";
4651
private static final Pattern gitlabCwlPattern = Pattern.compile(GITLAB_CWL_REGEX);
4752

48-
// Git URL validation
53+
// URL validation for directories on Gitlab
54+
private static final String GITLAB_DIR_REGEX = "^https?:\\/\\/gitlab\\.com\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:(?:tree|blob)\\/([^/]+)\\/?(.*)?)?$";
55+
private static final Pattern gitlabDirPattern = Pattern.compile(GITLAB_DIR_REGEX);
56+
57+
// Generic Git URL validation
4958
private static final String GIT_REPO_REGEX = "^((git|ssh|http(s)?)|(git@[\\w\\.]+))(:(//)?)([\\w\\.@\\:/\\-~]+)(\\.git)(/)?$";
5059
private static final Pattern gitRepoPattern = Pattern.compile(GIT_REPO_REGEX);
5160

@@ -60,23 +69,54 @@ public GitDetails validateAndParse(WorkflowForm form, Errors e) {
6069
// If not null and isn't just whitespace
6170
if (!e.hasErrors()) {
6271

72+
// Override if specific branch or path is given in the form
73+
String branch = null;
74+
String path = null;
75+
if (!isEmptyOrWhitespace(form.getBranch())) {
76+
branch = form.getBranch();
77+
}
78+
if (!isEmptyOrWhitespace(form.getPath())) {
79+
path = form.getPath();
80+
}
81+
6382
// Github URL
6483
Matcher m = githubCwlPattern.matcher(form.getUrl());
6584
if (m.find()) {
6685
String repoUrl = "https://github.com/" + m.group(1) + "/" + m.group(2) + ".git";
67-
return new GitDetails(repoUrl, m.group(3), m.group(4));
86+
if (branch == null) branch = m.group(3);
87+
if (path == null) path = m.group(4);
88+
return new GitDetails(repoUrl, branch, path);
6889
}
6990

7091
// Gitlab URL
7192
m = gitlabCwlPattern.matcher(form.getUrl());
7293
if (m.find()) {
7394
String repoUrl = "https://gitlab.com/" + m.group(1) + "/" + m.group(2) + ".git";
74-
return new GitDetails(repoUrl, m.group(3), m.group(4));
95+
if (branch == null) branch = m.group(3);
96+
if (path == null) path = m.group(4);
97+
return new GitDetails(repoUrl, branch, path);
98+
}
99+
100+
// Github Dir URL
101+
m = githubDirPattern.matcher(form.getUrl());
102+
if (m.find() && ! m.group(2).endsWith(".git")) {
103+
String repoUrl = "https://github.com/" + m.group(1) + "/" + m.group(2) + ".git";
104+
if (branch == null) branch = m.group(3);
105+
if (path == null) path = m.group(4);
106+
return new GitDetails(repoUrl, branch, path);
107+
}
108+
109+
// Gitlab Dir URL
110+
m = gitlabDirPattern.matcher(form.getUrl());
111+
if (m.find() && ! m.group(2).endsWith(".git")) {
112+
String repoUrl = "https://gitlab.com/" + m.group(1) + "/" + m.group(2) + ".git";
113+
if (branch == null) branch = m.group(3);
114+
if (path == null) path = m.group(4);
115+
return new GitDetails(repoUrl, branch, path);
75116
}
76117

77118
// General Git details if didn't match the above
78119
ValidationUtils.rejectIfEmptyOrWhitespace(e, "branch", "branch.emptyOrWhitespace");
79-
ValidationUtils.rejectIfEmptyOrWhitespace(e, "path", "path.emptyOrWhitespace");
80120
if (!e.hasErrors()) {
81121
m = gitRepoPattern.matcher(form.getUrl());
82122
if (m.find()) {
@@ -89,4 +129,15 @@ public GitDetails validateAndParse(WorkflowForm form, Errors e) {
89129
// Errors will stop this being used anyway
90130
return null;
91131
}
132+
133+
/**
134+
* Checks if a string is empty or whitespace
135+
* @param str The string to be checked
136+
* @return Whether the string is empty or whitespace
137+
*/
138+
private boolean isEmptyOrWhitespace(String str) {
139+
return (str == null ||
140+
str.length() == 0 ||
141+
StringUtils.isWhitespace(str));
142+
}
92143
}

0 commit comments

Comments
 (0)