Skip to content

Commit d8fb843

Browse files
MarkRobboMarkRobbo
committed
Rework workflow cache refreshes
1 parent 4333dbd commit d8fb843

File tree

6 files changed

+54
-59
lines changed

6 files changed

+54
-59
lines changed

‎src/main/java/org/commonwl/view/cwl/CWLService.java

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -176,17 +176,19 @@ public Workflow parseWorkflowNative(File workflowFile) throws IOException {
176176
}
177177

178178
/**
179-
* Create a workflow model from cwltool rdf output
179+
* Create a workflow model using cwltool rdf output
180+
* @param basicModel The basic workflow object created thus far
180181
* @param workflowFile The workflow file to run cwltool on
181-
* @param packedWorkflowID The workflow ID if the file has multiple objects
182182
* @return The constructed workflow object
183183
*/
184-
public Workflow parseWorkflowWithCwltool(GitDetails gitDetails,
185-
File workflowFile,
186-
String packedWorkflowID) throws CWLValidationException {
184+
public Workflow parseWorkflowWithCwltool(Workflow basicModel,
185+
File workflowFile) throws CWLValidationException {
186+
GitDetails gitDetails = basicModel.getRetrievedFrom();
187+
String latestCommit = basicModel.getLastCommit();
188+
String packedWorkflowID = basicModel.getPackedWorkflowID();
187189

188190
// Get paths to workflow
189-
String url = gitDetails.getUrl().replace("https://", "");
191+
String url = gitDetails.getUrl(latestCommit).replace("https://", "");
190192
String localPath = workflowFile.toPath().toAbsolutePath().toString();
191193
String gitPath = gitDetails.getPath();
192194
if (packedWorkflowID != null) {

‎src/main/java/org/commonwl/view/cwl/CWLToolRunner.java

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -72,15 +72,13 @@ public void createWorkflowFromQueued(QueuedWorkflow queuedWorkflow, File workflo
7272
// Parse using cwltool and replace in database
7373
try {
7474
Workflow newWorkflow = cwlService.parseWorkflowWithCwltool(
75-
tempWorkflow.getRetrievedFrom(),
76-
workflowFile,
77-
tempWorkflow.getPackedWorkflowID());
75+
tempWorkflow,
76+
workflowFile);
7877

7978
// Success
8079
newWorkflow.setRetrievedFrom(tempWorkflow.getRetrievedFrom());
8180
newWorkflow.setRetrievedOn(new Date());
8281
newWorkflow.setLastCommit(tempWorkflow.getLastCommit());
83-
newWorkflow.setGitRepoPath(tempWorkflow.getGitRepoPath());
8482
newWorkflow.setPackedWorkflowID(tempWorkflow.getPackedWorkflowID());
8583
newWorkflow.setCwltoolVersion(cwlToolVersion);
8684

‎src/main/java/org/commonwl/view/git/GitService.java

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,14 +54,15 @@ public GitService(@Value("${gitStorage}") Path gitStorage,
5454
}
5555

5656
/**
57-
* Clone a repository into a local directory
57+
* Gets a repository, cloning into a local directory or
5858
* @param gitDetails The details of the Git repository
59+
* @returns The git object for the repository
5960
*/
60-
public Git cloneRepository(GitDetails gitDetails)
61+
public Git getRepository(GitDetails gitDetails)
6162
throws GitAPIException {
6263
Git repo = null;
6364
try {
64-
// Get base directory based on configuration
65+
// Base dir from configuration, name from hash of repository URL
6566
File baseDir = new File(gitStorage.toString());
6667
String baseName = DigestUtils.shaHex(GitDetails.normaliseUrl(gitDetails.getRepoUrl()));
6768

‎src/main/java/org/commonwl/view/researchobject/ROBundleService.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.commonwl.view.cwl.CWLTool;
2828
import org.commonwl.view.cwl.CWLValidationException;
2929
import org.commonwl.view.git.GitDetails;
30+
import org.commonwl.view.git.GitService;
3031
import org.commonwl.view.git.GitType;
3132
import org.commonwl.view.graphviz.GraphVizService;
3233
import org.commonwl.view.workflow.Workflow;
@@ -66,6 +67,7 @@ public class ROBundleService {
6667

6768
// Services
6869
private GraphVizService graphVizService;
70+
private GitService gitService;
6971
private CWLTool cwlTool;
7072

7173
// Configuration variables
@@ -91,12 +93,14 @@ public ROBundleService(@Value("${bundleStorage}") Path bundleStorage,
9193
@Value("${applicationURL}") String appURL,
9294
@Value("${singleFileSizeLimit}") int singleFileSizeLimit,
9395
GraphVizService graphVizService,
96+
GitService gitService,
9497
CWLTool cwlTool) throws URISyntaxException {
9598
this.bundleStorage = bundleStorage;
9699
this.appAgent = new Agent(appName);
97100
appAgent.setUri(new URI(appURL));
98101
this.singleFileSizeLimit = singleFileSizeLimit;
99102
this.graphVizService = graphVizService;
103+
this.gitService = gitService;
100104
this.cwlTool = cwlTool;
101105
}
102106

@@ -129,7 +133,7 @@ public Bundle createBundle(Workflow workflow, GitDetails gitInfo) throws IOExcep
129133

130134
// Add the files from the Github repo to this workflow
131135
Set<HashableAgent> authors = new HashSet<>();
132-
Git gitRepo = Git.open(new File(workflow.getGitRepoPath()));
136+
Git gitRepo = gitService.getRepository(workflow.getRetrievedFrom());
133137
Path relativePath = Paths.get(FilenameUtils.getPath(gitInfo.getPath()));
134138
Path gitPath = gitRepo.getRepository().getWorkTree().toPath().resolve(relativePath);
135139
addFilesToBundle(gitInfo, bundle, bundlePath, gitRepo, gitPath, authors);
@@ -187,6 +191,8 @@ public Bundle createBundle(Workflow workflow, GitDetails gitInfo) throws IOExcep
187191

188192
} catch (URISyntaxException ex) {
189193
logger.error("Error creating URI for RO Bundle", ex);
194+
} catch (GitAPIException ex) {
195+
logger.error("Error getting repository to create RO Bundle", ex);
190196
}
191197

192198
// Return the completed bundle

‎src/main/java/org/commonwl/view/workflow/Workflow.java

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
* Representation of a workflow
3737
*/
3838
@JsonInclude(JsonInclude.Include.NON_NULL)
39-
@JsonIgnoreProperties(value = {"id", "roBundlePath", "gitRepoPath", "roBundleLink"})
39+
@JsonIgnoreProperties(value = {"id", "roBundlePath", "roBundleLink"})
4040
@Document
4141
public class Workflow {
4242

@@ -57,9 +57,6 @@ public class Workflow {
5757
// If schema salad packed, the workflow ID
5858
private String packedWorkflowID;
5959

60-
// Directory in which the git repository is held while processing
61-
private String gitRepoPath;
62-
6360
// A String which represents the path to a RO bundle
6461
// Path types cannot be stored using Spring Data, unfortunately
6562
private String roBundlePath;
@@ -199,14 +196,6 @@ public void setVisualisationDot(String visualisationDot) {
199196
this.visualisationDot = visualisationDot;
200197
}
201198

202-
public String getGitRepoPath() {
203-
return gitRepoPath;
204-
}
205-
206-
public void setGitRepoPath(String gitRepoPath) {
207-
this.gitRepoPath = gitRepoPath;
208-
}
209-
210199
// The following are here for Jackson message converter for the REST API
211200
// Include links to related resources
212201

‎src/main/java/org/commonwl/view/workflow/WorkflowService.java

Lines changed: 32 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,7 @@ public QueuedWorkflow createQueuedWorkflow(GitDetails gitInfo)
223223
Git repo = null;
224224
while (repo == null) {
225225
try {
226-
repo = gitService.cloneRepository(gitInfo);
226+
repo = gitService.getRepository(gitInfo);
227227
} catch (RefNotFoundException ex) {
228228
// Attempt slashes in branch fix
229229
GitDetails correctedForSlash = transferPathToBranch(gitInfo);
@@ -250,7 +250,6 @@ public QueuedWorkflow createQueuedWorkflow(GitDetails gitInfo)
250250
basicModel.setRetrievedOn(new Date());
251251
basicModel.setRetrievedFrom(gitInfo);
252252
basicModel.setLastCommit(latestCommit);
253-
basicModel.setGitRepoPath(localPath.toString());
254253

255254
// Save the queued workflow to database
256255
QueuedWorkflow queuedWorkflow = new QueuedWorkflow();
@@ -327,40 +326,40 @@ private void removeWorkflow(Workflow workflow) {
327326
* @return Whether or not there are new commits
328327
*/
329328
private boolean cacheExpired(Workflow workflow) {
330-
try {
331-
// Calculate expiration
332-
Calendar expireCal = Calendar.getInstance();
333-
expireCal.setTime(workflow.getRetrievedOn());
334-
expireCal.add(Calendar.DATE, cacheDays);
335-
Date expirationDate = expireCal.getTime();
336-
337-
// Check cached retrievedOn date
338-
if (expirationDate.before(new Date())) {
339-
// Cache expiry time has elapsed
340-
// Check current head of the branch with the cached head
341-
logger.debug("Time has expired for caching, checking commits...");
342-
String repoPath = workflow.getGitRepoPath();
343-
Git repo = Git.open(new File(repoPath));
344-
String currentHead = repo.getRepository().findRef("HEAD").getObjectId().getName();
345-
logger.debug("Current: " + workflow.getLastCommit() + ", HEAD: " + currentHead);
346-
347-
// Reset date in database if there are still no changes
348-
boolean expired = !workflow.getLastCommit().equals(currentHead);
349-
if (!expired) {
350-
workflow.setRetrievedOn(new Date());
351-
workflowRepository.save(workflow);
329+
// If this is a branch and not added by commit ID
330+
if (!workflow.getRetrievedFrom().getBranch().equals(workflow.getLastCommit())) {
331+
try {
332+
// Calculate expiration
333+
Calendar expireCal = Calendar.getInstance();
334+
expireCal.setTime(workflow.getRetrievedOn());
335+
expireCal.add(Calendar.DATE, cacheDays);
336+
Date expirationDate = expireCal.getTime();
337+
338+
// Check cached retrievedOn date
339+
if (expirationDate.before(new Date())) {
340+
// Cache expiry time has elapsed
341+
// Check current head of the branch with the cached head
342+
logger.info("Time has expired for caching, checking commits...");
343+
Git repo = gitService.getRepository(workflow.getRetrievedFrom());
344+
String currentHead = repo.getRepository().findRef("HEAD").getObjectId().getName();
345+
logger.info("Current: " + workflow.getLastCommit() + ", HEAD: " + currentHead);
346+
347+
// Reset date in database if there are still no changes
348+
boolean expired = !workflow.getLastCommit().equals(currentHead);
349+
if (!expired) {
350+
workflow.setRetrievedOn(new Date());
351+
workflowRepository.save(workflow);
352+
}
353+
354+
// Return whether the cache has expired
355+
return expired;
352356
}
353-
354-
// Return whether the cache has expired
355-
return expired;
356-
} else {
357-
// Cache expiry time has not elapsed yet
358-
return false;
357+
} catch(Exception ex){
358+
// Default to no expiry if there was an API error
359+
logger.error("API Error when checking for latest commit ID for caching", ex);
359360
}
360-
} catch (Exception ex) {
361-
// Default to no expiry if there was an API error
362-
return false;
363361
}
362+
return false;
364363
}
365364

366365
/**

0 commit comments

Comments
 (0)