Skip to content

Commit 27a4c1e

Browse files
author
Mark Robinson
committed
Add listing of workflow files when a directory is given
1 parent c10d473 commit 27a4c1e

File tree

7 files changed

+225
-35
lines changed

7 files changed

+225
-35
lines changed

‎src/main/java/org/commonwl/view/cwl/CWLService.java

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.commonwl.view.github.GitHubService;
3131
import org.commonwl.view.github.GithubDetails;
3232
import org.commonwl.view.workflow.Workflow;
33+
import org.commonwl.view.workflow.WorkflowOverview;
3334
import org.slf4j.Logger;
3435
import org.slf4j.LoggerFactory;
3536
import org.springframework.beans.factory.annotation.Autowired;
@@ -119,7 +120,7 @@ public Workflow parseWorkflow(GithubDetails githubInfo, String latestCommit) thr
119120
// Packed CWL, find the first subelement which is a workflow and take it
120121
for (JsonNode jsonNode : cwlFile.get(DOC_GRAPH)) {
121122
packedFiles.put(jsonNode.get(ID).asText(), jsonNode);
122-
if (jsonNode.has(CLASS) && jsonNode.get(CLASS).asText().equals(WORKFLOW)) {
123+
if (extractProcess(jsonNode) == CWLProcess.WORKFLOW) {
123124
cwlFile = jsonNode;
124125
}
125126
}
@@ -146,6 +147,57 @@ public Workflow parseWorkflow(GithubDetails githubInfo, String latestCommit) thr
146147

147148
}
148149

150+
/**
151+
* Get an overview of a workflow
152+
* @param githubInfo The details to access the workflow
153+
* @return A constructed WorkflowOverview of the workflow
154+
* @throws IOException Any API errors which may have occurred
155+
*/
156+
public WorkflowOverview getWorkflowOverview(GithubDetails githubInfo) throws IOException {
157+
158+
// Get the content of this file from Github
159+
String fileContent = githubService.downloadFile(githubInfo);
160+
int fileSizeBytes = fileContent.getBytes("UTF-8").length;
161+
162+
// Check file size limit before parsing
163+
if (fileSizeBytes <= singleFileSizeLimit) {
164+
165+
// Parse file as yaml
166+
JsonNode cwlFile = yamlStringToJson(fileContent);
167+
168+
// If the CWL file is packed there can be multiple workflows in a file
169+
if (cwlFile.has(DOC_GRAPH)) {
170+
// Packed CWL, find the first subelement which is a workflow and take it
171+
for (JsonNode jsonNode : cwlFile.get(DOC_GRAPH)) {
172+
if (extractProcess(jsonNode) == CWLProcess.WORKFLOW) {
173+
cwlFile = jsonNode;
174+
}
175+
}
176+
}
177+
178+
// Can only make an overview if this is a workflow
179+
if (extractProcess(cwlFile) == CWLProcess.WORKFLOW) {
180+
// Use filename for label if there is no defined one
181+
String path = FilenameUtils.getName(githubInfo.getPath());
182+
String label = extractLabel(cwlFile);
183+
if (label == null) {
184+
label = path;
185+
}
186+
187+
// Return the constructed overview
188+
return new WorkflowOverview(path, label, extractDoc(cwlFile));
189+
190+
} else {
191+
return null;
192+
}
193+
} else {
194+
throw new IOException("File '" + githubInfo.getPath() + "' is over singleFileSizeLimit - " +
195+
FileUtils.byteCountToDisplaySize(fileSizeBytes) + "/" +
196+
FileUtils.byteCountToDisplaySize(singleFileSizeLimit));
197+
}
198+
199+
}
200+
149201
/**
150202
* Converts a yaml String to JsonNode
151203
* @param yaml A String containing the yaml content

‎src/main/java/org/commonwl/view/github/GitHubService.java

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,12 @@ public class GitHubService {
5353
private final CommitService commitService;
5454

5555
// URL validation for directory links
56-
private final String GITHUB_URL_REGEX = "^https?:\\/\\/github\\.com\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:(?:tree|blob)\\/([^/]+)\\/?(.*)?)?$";
57-
private final Pattern githubUrlPattern = Pattern.compile(GITHUB_URL_REGEX);
56+
private final String GITHUB_DIR_REGEX = "^https?:\\/\\/github\\.com\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:(?:tree|blob)\\/([^/]+)\\/?(.*)?)?$";
57+
private final Pattern githubDirPattern = Pattern.compile(GITHUB_DIR_REGEX);
58+
59+
// URL validation for cwl files
60+
private final String GITHUB_CWL_REGEX = "^https?:\\/\\/github\\.com\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:tree|blob)\\/([^/]+)(?:\\/(.+\\.cwl))$";
61+
private final Pattern githubCwlPattern = Pattern.compile(GITHUB_CWL_REGEX);
5862

5963
@Autowired
6064
public GitHubService(@Value("${githubAPI.authentication}") String authSetting,
@@ -72,12 +76,25 @@ public GitHubService(@Value("${githubAPI.authentication}") String authSetting,
7276
}
7377

7478
/**
75-
* Extract the details of a Github URL using a regular expression
79+
* Extract the details of a Github cwl file URL using a regular expression
80+
* @param url The Github URL to a directory or file
81+
* @return A list with the groups of the regex match, [owner, repo, branch, path]
82+
*/
83+
public GithubDetails detailsFromFileURL(String url) {
84+
Matcher m = githubCwlPattern.matcher(url);
85+
if (m.find()) {
86+
return new GithubDetails(m.group(1), m.group(2), m.group(3), m.group(4));
87+
}
88+
return null;
89+
}
90+
91+
/**
92+
* Extract the details of a Github directory URL using a regular expression
7693
* @param url The Github URL to a directory or file
7794
* @return A list with the groups of the regex match, [owner, repo, branch, path]
7895
*/
79-
public GithubDetails detailsFromURL(String url) {
80-
Matcher m = githubUrlPattern.matcher(url);
96+
public GithubDetails detailsFromDirURL(String url) {
97+
Matcher m = githubDirPattern.matcher(url);
8198
if (m.find()) {
8299
return new GithubDetails(m.group(1), m.group(2), m.group(3), m.group(4));
83100
}

‎src/main/java/org/commonwl/view/workflow/WorkflowController.java

Lines changed: 43 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import javax.validation.Valid;
4444
import java.io.File;
4545
import java.io.IOException;
46+
import java.util.List;
4647

4748
@Controller
4849
public class WorkflowController {
@@ -97,23 +98,24 @@ public ModelAndView newWorkflowFromGithubURL(@Valid WorkflowForm workflowForm, B
9798
// Go back to index if there are validation errors
9899
return new ModelAndView("index");
99100
} else {
100-
// Get workflow or create if does not exist
101-
Workflow workflow = workflowService.getWorkflow(githubInfo);
102-
if (workflow == null) {
103-
workflow = workflowService.createWorkflow(githubInfo);
104-
105-
// Runtime error if workflow could not be generated
101+
if (githubInfo.getPath().endsWith(".cwl")) {
102+
// Get workflow or create if does not exist
103+
Workflow workflow = workflowService.getWorkflow(githubInfo);
106104
if (workflow == null) {
107-
bindingResult.rejectValue("githubURL", "githubURL.parsingError");
108-
return new ModelAndView("index");
105+
workflow = workflowService.createWorkflow(githubInfo);
106+
107+
// Runtime error if workflow could not be generated
108+
if (workflow == null) {
109+
bindingResult.rejectValue("githubURL", "githubURL.parsingError");
110+
return new ModelAndView("index");
111+
}
109112
}
113+
githubInfo = workflow.getRetrievedFrom();
110114
}
111-
112-
// Redirect to the workflow
113-
GithubDetails githubDetails = workflow.getRetrievedFrom();
114-
return new ModelAndView("redirect:/workflows/github.com/" + githubDetails.getOwner()
115-
+ "/" + githubDetails.getRepoName() + "/tree/" + githubDetails.getBranch()
116-
+ "/" + githubDetails.getPath());
115+
// Redirect to the workflow or choice of files
116+
return new ModelAndView("redirect:/workflows/github.com/" + githubInfo.getOwner()
117+
+ "/" + githubInfo.getRepoName() + "/tree/" + githubInfo.getBranch()
118+
+ "/" + githubInfo.getPath());
117119
}
118120
}
119121

@@ -138,7 +140,7 @@ public ModelAndView getWorkflowByGithubDetails(@Value("${applicationURL}") Strin
138140
if (pathStartIndex > -1 && pathStartIndex < path.length() - 1) {
139141
path = path.substring(pathStartIndex + 1).replaceAll("\\/$", "");
140142
} else {
141-
path = null;
143+
path = "/";
142144
}
143145

144146
// Construct a GithubDetails object to search for in the database
@@ -151,12 +153,36 @@ public ModelAndView getWorkflowByGithubDetails(@Value("${applicationURL}") Strin
151153
WorkflowForm workflowForm = new WorkflowForm(githubDetails.getURL());
152154
BeanPropertyBindingResult errors = new BeanPropertyBindingResult(workflowForm, "errors");
153155
workflowFormValidator.validateAndParse(workflowForm, errors);
156+
if (!errors.hasErrors()) {
157+
if (githubDetails.getPath().endsWith(".cwl")) {
158+
workflowModel = workflowService.createWorkflow(githubDetails);
159+
} else {
160+
// If this is a directory, get a list of workflows and return the view for it
161+
try {
162+
List<WorkflowOverview> workflowOverviews = workflowService.getWorkflowsFromDirectory(githubDetails);
163+
if (workflowOverviews.size() > 1) {
164+
return new ModelAndView("selectworkflow", "workflowOverviews", workflowOverviews)
165+
.addObject("githubDetails", githubDetails);
166+
} else if (workflowOverviews.size() == 1) {
167+
return new ModelAndView("redirect:/workflows/github.com/" + githubDetails.getOwner()
168+
+ "/" + githubDetails.getRepoName() + "/tree/" + githubDetails.getBranch()
169+
+ "/" + githubDetails.getPath() + "/" + workflowOverviews.get(0).getFileName());
170+
} else {
171+
logger.error("No .cwl files were found in the given directory");
172+
errors.rejectValue("githubURL", "githubURL.invalid", "You must enter a valid Github URL to a .cwl file");
173+
}
174+
} catch (IOException ex) {
175+
logger.error("Contents of Github directory could not be found", ex);
176+
errors.rejectValue("githubURL", "githubURL.invalid", "You must enter a valid Github URL to a .cwl file");
177+
}
178+
}
179+
}
180+
181+
// Redirect to main page with errors if they occurred
154182
if (errors.hasErrors()) {
155183
redirectAttrs.addFlashAttribute("errors", errors);
156184
return new ModelAndView("redirect:/?url=https://github.com/" +
157185
owner + "/" + repoName + "/tree/" + branch + "/" + path);
158-
} else {
159-
workflowModel = workflowService.createWorkflow(githubDetails);
160186
}
161187
}
162188

‎src/main/java/org/commonwl/view/workflow/WorkflowFormValidator.java

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -56,14 +56,11 @@ public WorkflowFormValidator(GitHubService githubService) {
5656
public GithubDetails validateAndParse(WorkflowForm form, Errors e) {
5757
ValidationUtils.rejectIfEmptyOrWhitespace(e, "githubURL", "githubURL.emptyOrWhitespace");
5858

59-
// Only continue if not null and isn't just whitespace
59+
// If not null and isn't just whitespace
6060
if (!e.hasErrors()) {
61-
GithubDetails githubInfo = githubService.detailsFromURL(form.getGithubURL());
62-
63-
// If the URL is valid and details could be extracted
61+
// Check for valid CWL file
62+
GithubDetails githubInfo = githubService.detailsFromFileURL(form.getGithubURL());
6463
if (githubInfo != null) {
65-
66-
// Check the repository exists and get content to ensure that branch/path exist
6764
try {
6865
// Downloads the workflow file to check for existence
6966
if (githubService.downloadFile(githubInfo) != null) {
@@ -74,8 +71,14 @@ public GithubDetails validateAndParse(WorkflowForm form, Errors e) {
7471
e.rejectValue("githubURL", "githubURL.missingWorkflow", "Workflow was not found at the given Github URL");
7572
}
7673
} else {
77-
logger.error("The Github URL " + form.getGithubURL() + " is not valid");
78-
e.rejectValue("githubURL", "githubURL.invalid", "You must enter a valid Github URL to a .cwl file");
74+
// Check for valid Github directory
75+
githubInfo = githubService.detailsFromDirURL(form.getGithubURL());
76+
if (githubInfo != null) {
77+
return githubInfo;
78+
} else {
79+
logger.error("The Github URL " + form.getGithubURL() + " is not valid");
80+
e.rejectValue("githubURL", "githubURL.invalid", "You must enter a valid Github URL to a .cwl file");
81+
}
7982
}
8083
} else {
8184
logger.error("Github URL is empty");

‎src/main/java/org/commonwl/view/workflow/WorkflowOverview.java

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.commonwl.view.workflow;
21+
22+
/**
23+
* Gives an overview of a workflow
24+
*/
25+
public class WorkflowOverview {
26+
27+
private String fileName;
28+
private String label;
29+
private String doc;
30+
31+
public WorkflowOverview(String fileName, String label, String doc) {
32+
this.fileName = fileName;
33+
this.label = label;
34+
this.doc = doc;
35+
}
36+
37+
public String getFileName() {
38+
return fileName;
39+
}
40+
41+
public void setFileName(String fileName) {
42+
this.fileName = fileName;
43+
}
44+
45+
public String getLabel() {
46+
return label;
47+
}
48+
49+
public void setLabel(String label) {
50+
this.label = label;
51+
}
52+
53+
public String getDoc() {
54+
return doc;
55+
}
56+
57+
public void setDoc(String doc) {
58+
this.doc = doc;
59+
}
60+
61+
}

‎src/main/java/org/commonwl/view/workflow/WorkflowService.java

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.commonwl.view.graphviz.GraphVizService;
2626
import org.commonwl.view.researchobject.ROBundleFactory;
2727
import org.commonwl.view.researchobject.ROBundleNotFoundException;
28+
import org.eclipse.egit.github.core.RepositoryContents;
2829
import org.slf4j.Logger;
2930
import org.slf4j.LoggerFactory;
3031
import org.springframework.beans.factory.annotation.Autowired;
@@ -34,8 +35,11 @@
3435
import org.springframework.stereotype.Service;
3536

3637
import java.io.File;
38+
import java.io.IOException;
39+
import java.util.ArrayList;
3740
import java.util.Calendar;
3841
import java.util.Date;
42+
import java.util.List;
3943

4044
@Service
4145
public class WorkflowService {
@@ -82,6 +86,30 @@ public Workflow getWorkflow(String id) {
8286
return workflowRepository.findOne(id);
8387
}
8488

89+
/**
90+
* Get a list of workflows from a directory in Github
91+
* @param githubInfo Github information for the workflow
92+
* @return The list of workflow names
93+
*/
94+
public List<WorkflowOverview> getWorkflowsFromDirectory(GithubDetails githubInfo) throws IOException {
95+
List<WorkflowOverview> workflowsInDir = new ArrayList<>();
96+
for (RepositoryContents content : githubService.getContents(githubInfo)) {
97+
int eIndex = content.getName().lastIndexOf('.') + 1;
98+
if (eIndex > 0) {
99+
String extension = content.getName().substring(eIndex);
100+
if (extension.equals("cwl")) {
101+
GithubDetails githubFile = new GithubDetails(githubInfo.getOwner(),
102+
githubInfo.getRepoName(), githubInfo.getBranch(), content.getPath());
103+
WorkflowOverview overview = cwlService.getWorkflowOverview(githubFile);
104+
if (overview != null) {
105+
workflowsInDir.add(overview);
106+
}
107+
}
108+
}
109+
}
110+
return workflowsInDir;
111+
}
112+
85113
/**
86114
* Get a workflow from the database, refreshing it if cache has expired
87115
* @param githubInfo Github information for the workflow

‎src/main/resources/templates/selectworkflow.html

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,14 +33,17 @@
3333
<div class="container">
3434
<div class="row">
3535
<div class="col-md-12" role="main" id="main">
36-
<h1>Choose Workflow</h1>
36+
<h1 th:text="'Choose Workflow from ' + ${githubDetails.path}">Choose Workflow</h1>
3737
<div class="alert alert-info">
3838
<p>Multiple CWL files were found, please select the workflow you would like to view below</p>
3939
</div>
4040
<div class="list-group">
41-
<a href="#" class="list-group-item" th:each="workflowName : ${workflowNames}">
42-
<h4 class="list-group-item-heading" th:text="${workflowName}">ExampleWorkflow.cwl</h4>
43-
<p class="list-group-item-text">Place any details with minimal overhead here, maybe label/doc</p>
41+
<a href="#" class="list-group-item" th:each="workflowOverview : ${workflowOverviews}" th:href="@{'/workflows/github.com/' + ${githubDetails.owner} + '/' + ${githubDetails.repoName} + '/tree/' + ${githubDetails.branch} + '/' + ${githubDetails.path} + '/' + ${workflowOverview.fileName}}">
42+
<h4 class="list-group-item-heading" th:text="${workflowOverview.fileName}">ExampleWorkflow.cwl</h4>
43+
<p class="list-group-item-text">
44+
<span th:if="${workflowOverview.label != null}" th:text="${workflowOverview.label}">Label</span>
45+
<i th:if="${workflowOverview.doc != null}" th:text="' - ' + ${workflowOverview.doc}">Doc</i>
46+
</p>
4447
</a>
4548
</div>
4649
</div>

0 commit comments

Comments
 (0)