Skip to content

Commit 8e12c2f

Browse files
MarkRobboMarkRobbo
committed
Clone repo from Github for queued workflows with native parsing
1 parent 75d0caf commit 8e12c2f

30 files changed

+551
-573
lines changed

‎src/main/java/org/commonwl/view/cwl/CWLService.java

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@
3535
import org.apache.jena.rdf.model.StmtIterator;
3636
import org.apache.jena.riot.RiotException;
3737
import org.commonwl.view.docker.DockerService;
38-
import org.commonwl.view.github.GitHubService;
39-
import org.commonwl.view.github.GithubDetails;
38+
import org.commonwl.view.github.GitDetails;
39+
import org.commonwl.view.github.GitService;
4040
import org.commonwl.view.graphviz.ModelDotWriter;
4141
import org.commonwl.view.graphviz.RDFDotWriter;
4242
import org.commonwl.view.workflow.Workflow;
@@ -49,12 +49,15 @@
4949
import org.yaml.snakeyaml.Yaml;
5050

5151
import java.io.ByteArrayInputStream;
52+
import java.io.File;
5253
import java.io.IOException;
5354
import java.io.StringWriter;
5455
import java.nio.file.Path;
5556
import java.nio.file.Paths;
5657
import java.util.*;
5758

59+
import static org.apache.commons.io.FileUtils.readFileToString;
60+
5861
/**
5962
* Provides CWL parsing for workflows to gather an overview
6063
* for display and visualisation
@@ -65,7 +68,7 @@ public class CWLService {
6568
private final Logger logger = LoggerFactory.getLogger(this.getClass());
6669

6770
// Autowired properties/services
68-
private final GitHubService githubService;
71+
private final GitService githubService;
6972
private final RDFService rdfService;
7073
private final CWLTool cwlTool;
7174
private final int singleFileSizeLimit;
@@ -101,7 +104,7 @@ public class CWLService {
101104
* @param singleFileSizeLimit The file size limit for single files
102105
*/
103106
@Autowired
104-
public CWLService(GitHubService githubService,
107+
public CWLService(GitService githubService,
105108
RDFService rdfService,
106109
CWLTool cwlTool,
107110
@Value("${singleFileSizeLimit}") int singleFileSizeLimit) {
@@ -113,21 +116,17 @@ public CWLService(GitHubService githubService,
113116

114117
/**
115118
* Gets the Workflow object from internal parsing
116-
* @param githubInfo The Github repository information
117-
* @param latestCommit The latest commit ID
119+
* @param workflowFile The workflow file to be parsed
118120
* @return The constructed workflow object
119121
*/
120-
public Workflow parseWorkflowNative(GithubDetails githubInfo, String latestCommit) throws IOException {
121-
122-
// Get the content of this file from Github
123-
String fileContent = githubService.downloadFile(githubInfo, latestCommit);
124-
int fileSizeBytes = fileContent.getBytes("UTF-8").length;
122+
public Workflow parseWorkflowNative(File workflowFile) throws IOException {
125123

126124
// Check file size limit before parsing
125+
long fileSizeBytes = workflowFile.length();
127126
if (fileSizeBytes <= singleFileSizeLimit) {
128127

129128
// Parse file as yaml
130-
JsonNode cwlFile = yamlStringToJson(fileContent);
129+
JsonNode cwlFile = yamlStringToJson(readFileToString(workflowFile));
131130

132131
// If the CWL file is packed there can be multiple workflows in a file
133132
Map<String, JsonNode> packedFiles = new HashMap<>();
@@ -144,7 +143,7 @@ public Workflow parseWorkflowNative(GithubDetails githubInfo, String latestCommi
144143
// Use filename for label if there is no defined one
145144
String label = extractLabel(cwlFile);
146145
if (label == null) {
147-
label = FilenameUtils.getName(githubInfo.getPath());
146+
label = FilenameUtils.getName(workflowFile.getPath());
148147
}
149148

150149
// Construct the rest of the workflow model
@@ -170,7 +169,7 @@ public Workflow parseWorkflowNative(GithubDetails githubInfo, String latestCommi
170169
return workflowModel;
171170

172171
} else {
173-
throw new IOException("File '" + githubInfo.getPath() + "' is over singleFileSizeLimit - " +
172+
throw new IOException("File '" + workflowFile.getName() + "' is over singleFileSizeLimit - " +
174173
FileUtils.byteCountToDisplaySize(fileSizeBytes) + "/" +
175174
FileUtils.byteCountToDisplaySize(singleFileSizeLimit));
176175
}
@@ -184,7 +183,7 @@ public Workflow parseWorkflowNative(GithubDetails githubInfo, String latestCommi
184183
* @param packedWorkflowID The workflow ID if the file has multiple objects
185184
* @return The constructed workflow object
186185
*/
187-
public Workflow parseWorkflowWithCwltool(GithubDetails githubInfo,
186+
public Workflow parseWorkflowWithCwltool(GitDetails githubInfo,
188187
String latestCommit,
189188
String packedWorkflowID) throws CWLValidationException {
190189

@@ -432,7 +431,7 @@ public Workflow parseWorkflowWithCwltool(GithubDetails githubInfo,
432431
* @return A constructed WorkflowOverview of the workflow
433432
* @throws IOException Any API errors which may have occurred
434433
*/
435-
public WorkflowOverview getWorkflowOverview(GithubDetails githubInfo) throws IOException {
434+
public WorkflowOverview getWorkflowOverview(GitDetails githubInfo) throws IOException {
436435

437436
// Get the content of this file from Github
438437
String fileContent = githubService.downloadFile(githubInfo);

‎src/main/java/org/commonwl/view/cwl/CWLToolRunner.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
package org.commonwl.view.cwl;
2121

22-
import org.commonwl.view.github.GitHubService;
22+
import org.commonwl.view.github.GitService;
2323
import org.commonwl.view.researchobject.ROBundleFactory;
2424
import org.commonwl.view.workflow.QueuedWorkflow;
2525
import org.commonwl.view.workflow.QueuedWorkflowRepository;
@@ -47,7 +47,7 @@ public class CWLToolRunner {
4747
private final WorkflowRepository workflowRepository;
4848
private final QueuedWorkflowRepository queuedWorkflowRepository;
4949
private final CWLService cwlService;
50-
private final GitHubService githubService;
50+
private final GitService githubService;
5151
private final ROBundleFactory roBundleFactory;
5252
private final String cwlToolVersion;
5353

@@ -56,7 +56,7 @@ public CWLToolRunner(WorkflowRepository workflowRepository,
5656
QueuedWorkflowRepository queuedWorkflowRepository,
5757
CWLService cwlService,
5858
CWLTool cwlTool,
59-
GitHubService githubService,
59+
GitService githubService,
6060
ROBundleFactory roBundleFactory) {
6161
this.workflowRepository = workflowRepository;
6262
this.queuedWorkflowRepository = queuedWorkflowRepository;

‎src/main/java/org/commonwl/view/github/GitDetails.java

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.commonwl.view.github;
21+
22+
import java.io.Serializable;
23+
24+
/**
25+
* Represents all the parameters necessary to access a file/directory with Git
26+
*/
27+
public class GitDetails implements Serializable {
28+
29+
private String repoUrl;
30+
private String branch;
31+
private String path;
32+
private GitType type;
33+
34+
public GitDetails(String repoUrl, String branch, String path, GitType type) {
35+
this.repoUrl = repoUrl;
36+
37+
// Default to the master branch
38+
if (branch == null || branch.isEmpty()) {
39+
// TODO: get default branch name for this rather than assuming master
40+
this.branch = "master";
41+
} else {
42+
this.branch = branch;
43+
}
44+
45+
// Default to root path
46+
if (path == null || path.isEmpty()) {
47+
this.path = "/";
48+
} else {
49+
this.path = path;
50+
}
51+
52+
this.type = type;
53+
}
54+
55+
56+
public String getRepoUrl() {
57+
return repoUrl;
58+
}
59+
60+
public void setRepoUrl(String repoUrl) {
61+
this.repoUrl = repoUrl;
62+
}
63+
64+
public String getBranch() {
65+
return branch;
66+
}
67+
68+
public void setBranch(String branch) {
69+
this.branch = branch;
70+
}
71+
72+
public String getPath() {
73+
return path;
74+
}
75+
76+
public void setPath(String path) {
77+
this.path = path;
78+
}
79+
80+
public GitType getType() {
81+
return type;
82+
}
83+
84+
public void setType(GitType type) {
85+
this.type = type;
86+
}
87+
88+
/**
89+
* Get the URL to the external resource representing this workflow
90+
* @return The URL
91+
*/
92+
public String getUrl() {
93+
switch (this.type) {
94+
case GENERIC:
95+
return repoUrl;
96+
case GITHUB:
97+
case GITLAB:
98+
return "https://" + normaliseURL(repoUrl).replace(".git", "") + "/" + branch + "/" + path;
99+
default:
100+
return null;
101+
}
102+
}
103+
104+
/**
105+
* Get the URL to the page containing this workflow
106+
* @return The URL
107+
*/
108+
public String getInternalUrl() {
109+
switch (this.type) {
110+
case GENERIC:
111+
return "/workflows/" + normaliseURL(repoUrl) + "/" + branch + "/" + path;
112+
case GITHUB:
113+
case GITLAB:
114+
return "/workflows/" + normaliseURL(repoUrl).replace(".git", "") + "/" + branch + "/" + path;
115+
default:
116+
return null;
117+
}
118+
}
119+
120+
/**
121+
* Normalises the URL removing protocol and www.
122+
* @param url The URL to be normalised
123+
* @return The normalised URL
124+
*/
125+
private String normaliseURL(String url) {
126+
return url.replace("http://", "")
127+
.replace("https://", "")
128+
.replace("ssh://", "")
129+
.replace("git://", "")
130+
.replace("www.", "");
131+
}
132+
133+
}

0 commit comments

Comments
 (0)