Skip to content

Commit a0781f0

Browse files
MarkRobboMarkRobbo
committed
Correct Research Object Bundle generation
1 parent 177c311 commit a0781f0

File tree

7 files changed

+158
-115
lines changed

7 files changed

+158
-115
lines changed

‎src/main/java/org/commonwl/view/cwl/CWLToolRunner.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,9 @@ public void createWorkflowFromQueued(QueuedWorkflow queuedWorkflow, File workflo
8181
newWorkflow.setRetrievedOn(new Date());
8282
newWorkflow.setLastCommit(tempWorkflow.getLastCommit());
8383
newWorkflow.setGitRepoPath(tempWorkflow.getGitRepoPath());
84+
newWorkflow.setPackedWorkflowID(tempWorkflow.getPackedWorkflowID());
8485
newWorkflow.setCwltoolVersion(cwlToolVersion);
86+
8587
workflowRepository.save(newWorkflow);
8688

8789
// Generate RO bundle

‎src/main/java/org/commonwl/view/researchobject/ROBundleService.java

Lines changed: 131 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@
2525
import org.apache.taverna.robundle.Bundles;
2626
import org.apache.taverna.robundle.manifest.*;
2727
import org.commonwl.view.cwl.CWLTool;
28+
import org.commonwl.view.cwl.CWLValidationException;
2829
import org.commonwl.view.git.GitDetails;
30+
import org.commonwl.view.git.GitType;
2931
import org.commonwl.view.graphviz.GraphVizService;
3032
import org.commonwl.view.workflow.Workflow;
3133
import org.eclipse.jgit.api.Git;
@@ -128,8 +130,9 @@ public Bundle createBundle(Workflow workflow, GitDetails gitInfo) throws IOExcep
128130
// Add the files from the Github repo to this workflow
129131
Set<HashableAgent> authors = new HashSet<>();
130132
Git gitRepo = Git.open(new File(workflow.getGitRepoPath()));
131-
Path gitPath = gitRepo.getRepository().getDirectory().toPath();
132-
addFilesToBundle(bundle, bundlePath, gitRepo, gitPath, authors);
133+
Path relativePath = Paths.get(FilenameUtils.getPath(gitInfo.getPath()));
134+
Path gitPath = gitRepo.getRepository().getWorkTree().toPath().resolve(relativePath);
135+
addFilesToBundle(gitInfo, bundle, bundlePath, gitRepo, gitPath, authors);
133136

134137
// Add combined authors
135138
manifest.setAuthoredBy(new ArrayList<>(authors));
@@ -148,13 +151,32 @@ public Bundle createBundle(Workflow workflow, GitDetails gitInfo) throws IOExcep
148151
// Add annotation files
149152
GitDetails wfDetails = workflow.getRetrievedFrom();
150153

151-
// Run cwltool
154+
// Get URL to run cwltool
152155
String rawUrl = wfDetails.getRawUrl();
156+
String packedWorkflowID = workflow.getPackedWorkflowID();
157+
if (packedWorkflowID != null) {
158+
if (packedWorkflowID.charAt(0) != '#') {
159+
rawUrl += "#";
160+
}
161+
rawUrl += packedWorkflowID;
162+
}
163+
164+
// Run cwltool for annotations
153165
List<PathAnnotation> manifestAnnotations = new ArrayList<>();
154-
addAggregation(bundle, manifestAnnotations,
155-
"merged.cwl", cwlTool.getPackedVersion(rawUrl));
156-
addAggregation(bundle, manifestAnnotations,
157-
"workflow.ttl", cwlTool.getRDF(rawUrl));
166+
try {
167+
addAggregation(bundle, manifestAnnotations,
168+
"merged.cwl", cwlTool.getPackedVersion(rawUrl));
169+
} catch (CWLValidationException ex) {
170+
logger.error("Could not pack workflow when creating Research Object", ex.getMessage());
171+
}
172+
if (gitInfo.getType() != GitType.GENERIC) {
173+
try {
174+
addAggregation(bundle, manifestAnnotations,
175+
"workflow.ttl", cwlTool.getRDF(rawUrl));
176+
} catch (CWLValidationException ex) {
177+
logger.error("Could not get RDF for workflow from raw URL", ex.getMessage());
178+
}
179+
}
158180
bundle.getManifest().setAnnotations(manifestAnnotations);
159181

160182
// Git2prov history
@@ -174,117 +196,130 @@ public Bundle createBundle(Workflow workflow, GitDetails gitInfo) throws IOExcep
174196

175197
/**
176198
* Add files to this bundle from a list of Github repository contents
199+
* @param gitDetails The Git information for the repository
177200
* @param bundle The RO bundle to add files/directories to
178201
* @param bundlePath The current path within the RO bundle
179202
* @param gitRepo The Git repository
180203
* @param repoPath The current path within the Git repository
181204
* @param authors The combined set of authors for al the files
182205
*/
183-
private void addFilesToBundle(Bundle bundle, Path bundlePath,
184-
Git gitRepo, Path repoPath,
185-
Set<HashableAgent> authors) throws IOException {
186-
File[] files = gitRepo.getRepository().getDirectory().listFiles();
206+
private void addFilesToBundle(GitDetails gitDetails, Bundle bundle, Path bundlePath,
207+
Git gitRepo, Path repoPath, Set<HashableAgent> authors)
208+
throws IOException {
209+
File[] files = repoPath.toFile().listFiles();
187210
for (File file : files) {
188-
if (file.isDirectory()) {
211+
if (!file.getName().equals(".git")) {
212+
if (file.isDirectory()) {
189213

190-
// Create a new folder in the RO for this directory
191-
Path newBundlePath = bundlePath.resolve(file.getName());
192-
Files.createDirectory(newBundlePath);
214+
// Create a new folder in the RO for this directory
215+
Path newBundlePath = bundlePath.resolve(file.getName());
216+
Files.createDirectory(newBundlePath);
193217

194-
// Add all files in the subdirectory to this new folder
195-
addFilesToBundle(bundle, newBundlePath, gitRepo,
196-
repoPath.resolve(file.getName()), authors);
218+
// Create git details object for subfolder
219+
GitDetails subfolderGitDetails = new GitDetails(gitDetails.getRepoUrl(), gitDetails.getBranch(),
220+
Paths.get(gitDetails.getPath()).resolve(file.getName()).toString());
197221

198-
} else {
199-
try {
200-
// Where to store the new file in bundle
201-
Path bundleFilePath = bundlePath.resolve(file.getName());
222+
// Add all files in the subdirectory to this new folder
223+
addFilesToBundle(subfolderGitDetails, bundle, newBundlePath, gitRepo,
224+
repoPath.resolve(file.getName()), authors);
202225

203-
// Get direct URL
204-
URI rawURI = new URI("http://example.com");
205-
206-
// Variable to store file contents and aggregation
207-
String fileContent = null;
208-
PathMetadata aggregation;
209-
210-
// Download or externally link if oversized
211-
if (file.length() <= singleFileSizeLimit) {
212-
// Save file to research object bundle
213-
fileContent = readFileToString(file);
214-
Bundles.setStringValue(bundleFilePath, fileContent);
215-
216-
// Set retrieved information for this file in the manifest
217-
aggregation = bundle.getManifest().getAggregation(bundleFilePath);
218-
aggregation.setRetrievedFrom(rawURI);
219-
aggregation.setRetrievedBy(appAgent);
220-
aggregation.setRetrievedOn(aggregation.getCreatedOn());
221-
} else {
222-
logger.info("File " + file.getName() + " is too large to download - " +
223-
FileUtils.byteCountToDisplaySize(file.length()) + "/" +
224-
FileUtils.byteCountToDisplaySize(singleFileSizeLimit) +
225-
", linking externally to RO bundle");
226-
227-
// Set information for this file in the manifest
228-
aggregation = bundle.getManifest().getAggregation(rawURI);
229-
Proxy bundledAs = new Proxy();
230-
bundledAs.setURI();
231-
bundledAs.setFolder(repoPath);
232-
aggregation.setBundledAs(bundledAs);
233-
}
226+
} else {
227+
try {
228+
// Where to store the new file in bundle
229+
Path bundleFilePath = bundlePath.resolve(file.getName());
230+
231+
// Create git details object for file
232+
GitDetails fileGitDetails = new GitDetails(gitDetails.getRepoUrl(), gitDetails.getBranch(),
233+
Paths.get(gitDetails.getPath()).resolve(file.getName()).toString());
234+
235+
// Get direct URL
236+
URI rawURI = new URI(fileGitDetails.getRawUrl());
237+
238+
// Variable to store file contents and aggregation
239+
String fileContent = null;
240+
PathMetadata aggregation;
241+
242+
// Download or externally link if oversized
243+
if (file.length() <= singleFileSizeLimit) {
244+
// Save file to research object bundle
245+
fileContent = readFileToString(file);
246+
Bundles.setStringValue(bundleFilePath, fileContent);
247+
248+
// Set retrieved information for this file in the manifest
249+
aggregation = bundle.getManifest().getAggregation(bundleFilePath);
250+
if (gitDetails.getType() != GitType.GENERIC) {
251+
aggregation.setRetrievedFrom(rawURI);
252+
aggregation.setRetrievedBy(appAgent);
253+
aggregation.setRetrievedOn(aggregation.getCreatedOn());
254+
}
255+
} else {
256+
logger.info("File " + file.getName() + " is too large to download - " +
257+
FileUtils.byteCountToDisplaySize(file.length()) + "/" +
258+
FileUtils.byteCountToDisplaySize(singleFileSizeLimit) +
259+
", linking externally to RO bundle");
260+
261+
// Set information for this file in the manifest
262+
aggregation = bundle.getManifest().getAggregation(rawURI);
263+
Proxy bundledAs = new Proxy();
264+
bundledAs.setURI();
265+
bundledAs.setFolder(repoPath);
266+
aggregation.setBundledAs(bundledAs);
267+
}
234268

235-
// Special handling for cwl files
236-
if (FilenameUtils.getExtension(file.getName()).equals("cwl")) {
237-
// Correct mime type (no official standard for yaml)
238-
aggregation.setMediatype("text/x-yaml");
269+
// Special handling for cwl files
270+
if (FilenameUtils.getExtension(file.getName()).equals("cwl")) {
271+
// Correct mime type (no official standard for yaml)
272+
aggregation.setMediatype("text/x-yaml");
239273

240-
// Add conformsTo for version extracted from regex
241-
if (fileContent != null) {
242-
Matcher m = cwlVersionPattern.matcher(fileContent);
243-
if (m.find()) {
244-
aggregation.setConformsTo(new URI("https://w3id.org/cwl/" + m.group(1)));
274+
// Add conformsTo for version extracted from regex
275+
if (fileContent != null) {
276+
Matcher m = cwlVersionPattern.matcher(fileContent);
277+
if (m.find()) {
278+
aggregation.setConformsTo(new URI("https://w3id.org/cwl/" + m.group(1)));
279+
}
245280
}
246281
}
247-
}
248282

249-
// Add authors from git commits to the file
250-
try {
251-
Set<HashableAgent> fileAuthors = new HashSet<>();
252-
Iterable<RevCommit> logs = gitRepo.log()
253-
.addPath(bundlePath.toString())
254-
.call();
255-
for (RevCommit rev : logs) {
256-
// Use author first with backup of committer
257-
PersonIdent author = rev.getAuthorIdent();
258-
if (author == null) {
259-
author = rev.getCommitterIdent();
260-
}
261-
// Create a new agent and add as much detail as possible
262-
if (author != null) {
263-
HashableAgent newAgent = new HashableAgent();
264-
String name = author.getName();
265-
if (name != null && name.length() > 0) {
266-
newAgent.setName(author.getName());
283+
// Add authors from git commits to the file
284+
try {
285+
Set<HashableAgent> fileAuthors = new HashSet<>();
286+
Iterable<RevCommit> logs = gitRepo.log()
287+
.addPath(Paths.get(gitDetails.getPath()).resolve(file.getName()).toString())
288+
.call();
289+
for (RevCommit rev : logs) {
290+
// Use author first with backup of committer
291+
PersonIdent author = rev.getAuthorIdent();
292+
if (author == null) {
293+
author = rev.getCommitterIdent();
267294
}
268-
String email = author.getEmailAddress();
269-
if (email != null && email.length() > 0) {
270-
newAgent.setUri(new URI("mailto:" + author.getEmailAddress()));
295+
// Create a new agent and add as much detail as possible
296+
if (author != null) {
297+
HashableAgent newAgent = new HashableAgent();
298+
String name = author.getName();
299+
if (name != null && name.length() > 0) {
300+
newAgent.setName(author.getName());
301+
}
302+
String email = author.getEmailAddress();
303+
if (email != null && email.length() > 0) {
304+
newAgent.setUri(new URI("mailto:" + author.getEmailAddress()));
305+
}
306+
fileAuthors.add(newAgent);
271307
}
272-
fileAuthors.add(newAgent);
273308
}
309+
authors.addAll(fileAuthors);
310+
aggregation.setAuthoredBy(new ArrayList<>(fileAuthors));
311+
} catch (GitAPIException ex) {
312+
logger.error("Could not get commits for file " + repoPath, ex);
274313
}
275-
authors.addAll(fileAuthors);
276-
aggregation.setAuthoredBy(new ArrayList<>(fileAuthors));
277-
} catch (GitAPIException ex) {
278-
logger.error("Could not get commits for file " + repoPath, ex);
279-
}
280314

281-
// Set retrieved information for this file in the manifest
282-
aggregation.setRetrievedFrom(rawURI);
283-
aggregation.setRetrievedBy(appAgent);
284-
aggregation.setRetrievedOn(aggregation.getCreatedOn());
315+
// Set retrieved information for this file in the manifest
316+
aggregation.setRetrievedFrom(rawURI);
317+
aggregation.setRetrievedBy(appAgent);
318+
aggregation.setRetrievedOn(aggregation.getCreatedOn());
285319

286-
} catch (URISyntaxException ex) {
287-
logger.error("Error creating URI for RO Bundle", ex);
320+
} catch (URISyntaxException ex) {
321+
logger.error("Error creating URI for RO Bundle", ex);
322+
}
288323
}
289324
}
290325
}

‎src/main/java/org/commonwl/view/workflow/Workflow.java

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
* Representation of a workflow
3737
*/
3838
@JsonInclude(JsonInclude.Include.NON_NULL)
39-
@JsonIgnoreProperties(value = {"id", "roBundlePath", "gitRepoPath"})
39+
@JsonIgnoreProperties(value = {"id", "roBundlePath", "gitRepoPath", "roBundleLink"})
4040
@Document
4141
public class Workflow {
4242

@@ -222,11 +222,15 @@ public String getVisualisationSvg() {
222222
return retrievedFrom.getInternalUrl().replaceFirst("/workflows", "/graph/svg");
223223
}
224224

225-
public String getROBundle() {
225+
public String getRoBundle() {
226226
if (roBundlePath != null) {
227-
return retrievedFrom.getInternalUrl().replaceFirst("/workflows", "/robundle");
227+
return getRoBundleLink();
228228
} else {
229229
return null;
230230
}
231231
}
232+
233+
public String getRoBundleLink() {
234+
return retrievedFrom.getInternalUrl().replaceFirst("/workflows", "/robundle");
235+
}
232236
}

‎src/main/java/org/commonwl/view/workflow/WorkflowController.java

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121

2222
import org.apache.commons.lang.StringUtils;
2323
import org.commonwl.view.cwl.CWLToolStatus;
24-
import org.commonwl.view.cwl.CWLValidationException;
2524
import org.commonwl.view.git.GitDetails;
2625
import org.commonwl.view.graphviz.GraphVizService;
2726
import org.eclipse.jgit.api.errors.GitAPIException;
@@ -121,12 +120,14 @@ public ModelAndView newWorkflowFromGithubURL(@Valid WorkflowForm workflowForm, B
121120
if (workflow == null) {
122121
try {
123122
workflow = workflowService.createQueuedWorkflow(gitInfo).getTempRepresentation();
124-
} catch (CWLValidationException ex) {
125-
bindingResult.rejectValue("url", "cwltool.validationError", ex.getMessage());
123+
} catch (GitAPIException ex) {
124+
bindingResult.rejectValue("url", "git.retrievalError");
125+
return new ModelAndView("index");
126+
} catch (WorkflowNotFoundException ex) {
127+
bindingResult.rejectValue("url", "git.pathTraversal");
126128
return new ModelAndView("index");
127129
} catch (Exception ex) {
128-
bindingResult.rejectValue("url", "githubURL.parsingError");
129-
logger.error(ex.getMessage(), ex);
130+
bindingResult.rejectValue("url", "url.parsingError");
130131
return new ModelAndView("index");
131132
}
132133
}
@@ -449,11 +450,11 @@ private ModelAndView getWorkflow(GitDetails gitDetails, RedirectAttributes redir
449450
try {
450451
queued = workflowService.createQueuedWorkflow(gitDetails);
451452
} catch (GitAPIException ex) {
452-
errors.rejectValue("url", "git.retrievalError", ex.getMessage());
453-
} catch (CWLValidationException ex) {
454-
errors.rejectValue("url", "cwltool.validationError", ex.getMessage());
453+
errors.rejectValue("url", "git.retrievalError", "The workflow could not be retrieved from the Git repository using the details given");
454+
} catch (WorkflowNotFoundException ex) {
455+
errors.rejectValue("url", "git.pathTraversal", "The path given did not resolve to a location within the repository");
455456
} catch (IOException ex) {
456-
errors.rejectValue("url", "githubURL.parsingError", "The workflow could not be parsed from the given URL");
457+
errors.rejectValue("url", "url.parsingError", "The workflow could not be parsed from the given URL");
457458
}
458459
}
459460
// Redirect to main page with errors if they occurred

0 commit comments

Comments
 (0)