Merge branch 'main' into fix-queued-workflow-delete

Author: tetron

.github/workflows/codeql-analysis.yml

@@ -2,7 +2,9 @@ name: "Code scanning - action"
 
 on:
   push:
+    branches: [main]
   pull_request:
+    branches: [main]
   schedule:
     - cron: '0 9 * * 0'
 

README.md

@@ -38,19 +38,19 @@ This application can be started with [Docker](https://www.docker.com/) and [Dock
 
 Then run the following commands to clone the project in your local system.
 
-   ``` 
+   ```
     git clone https://github.com/common-workflow-language/cwlviewer.git
-    cd cwlviewer 
+    cd cwlviewer
   ```
 
 In the project directory, to start CWLViewer exposed on port `8080`, run:
 
     docker-compose up
-     
-The web server will connect to a local host, you'll see the message saying "Tomacat started on port(s):8080". 
 
-To see the locally running CWL Viewer app, visit http://localhost:8080/ in your web browser.  
-   
+The web server will connect to a local host, you'll see the message saying "Tomacat started on port(s):8080".
+
+To see the locally running CWL Viewer app, visit http://localhost:8080/ in your web browser.
+
 To stop and remove:
 
     docker-compose down
@@ -72,6 +72,44 @@ If you have modified the source code, then you may want to build the docker imag
 
     docker build -t commonworkflowlanguage/cwlviewer .
 
+## Running Spring Boot locally for development, with MongoDB and Jena Fuseki in Docker
+
+Create `docker-compose.override.yml`:
+
+```
+version: '3.2'
+services:
+  mongo:
+    ports:
+     - "27017:27017"
+  sparql:
+    ports:
+     - "3030:3030"
+```
+
+Then start the containers:
+
+```
+docker-compose up
+```
+
+Then start Spring Boot locally:
+
+```
+mvn spring-boot:run -Dserver.port=7999
+```
+
+Now you can connect to http://localhost:7999 in your browser.
+
+## Deleting the data volumes to reset state
+
+To completely reset the state, you must delete the data volumes:
+
+```
+docker-compose down
+docker volume rm  cwlviewer_bundle cwlviewer_git cwlviewer_graphviz cwlviewer_mongo cwlviewer_sparql
+```
+
 ## Running without Docker
 
 ### Requirements

pom.xml

@@ -23,10 +23,10 @@
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
 		<java.version>1.8</java.version>
-		<jena.version>4.0.0</jena.version>
+		<jena.version>4.1.0</jena.version>
 		<!-- BEGIN should correspond to https://repo1.maven.org/maven2/org/apache/jena/jena/3.9.0/jena-3.9.0.pom -->
 		<ver.jsonldjava>0.13.3</ver.jsonldjava>
-		<ver.jackson>2.12.3</ver.jackson>
+		<ver.jackson>2.12.4</ver.jackson>
 		<!-- END -->
 	</properties>
 
@@ -70,7 +70,7 @@
 		<dependency>
 			<groupId>org.yaml</groupId>
 			<artifactId>snakeyaml</artifactId>
-			<version>1.28</version>
+			<version>1.29</version>
 		</dependency>
 		<dependency>
 			<groupId>com.github.jabbalaci</groupId>
@@ -106,7 +106,7 @@
 		<dependency>
 			<groupId>org.eclipse.jgit</groupId>
 			<artifactId>org.eclipse.jgit</artifactId>
-			<version>5.11.0.202103091610-r</version>
+			<version>5.12.0.202106070339-r</version>
 		</dependency>
 		<dependency>
 			<groupId>org.mockito</groupId>
@@ -117,7 +117,7 @@
 		<dependency>
 			<groupId>org.apache.commons</groupId>
 			<artifactId>commons-compress</artifactId>
-			<version>1.20</version>
+			<version>1.21</version>
 		</dependency>
 	</dependencies>
 

src/main/java/org/commonwl/view/CwlViewerApplication.java

@@ -23,10 +23,12 @@
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
 import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.scheduling.annotation.EnableScheduling;
 
 @SpringBootApplication
 @EnableMongoRepositories
 @EnableAsync
+@EnableScheduling
 public class CwlViewerApplication {
 
 	public static void main(String[] args) {

src/main/java/org/commonwl/view/Scheduler.java

@@ -0,0 +1,58 @@
+package org.commonwl.view;
+
+
+import org.commonwl.view.workflow.QueuedWorkflowRepository;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import java.util.Calendar;
+import java.util.Date;
+
+/**
+ * Scheduler class for recurrent processes.
+ */
+@Component
+public class Scheduler {
+
+    private final Logger logger = LoggerFactory.getLogger(this.getClass());
+    private final QueuedWorkflowRepository queuedWorkflowRepository;
+
+    @Value("${queuedWorkflowAgeLimitHours}")
+    private Integer QUEUED_WORKFLOW_AGE_LIMIT_HOURS;
+
+    @Autowired
+    public Scheduler(QueuedWorkflowRepository queuedWorkflowRepository) {
+        this.queuedWorkflowRepository = queuedWorkflowRepository;
+    }
+
+
+    /**
+     * A Scheduled function to delete old queued workflow entries
+     * from the queue. Age is determined by QUEUED_WORKFLOW_AGE_LIMIT_HOURS
+     */
+    @Scheduled(cron = "${cron.deleteOldQueuedWorkflows}")
+    public void removeOldQueuedWorkflowEntries() {
+        Calendar calendar = Calendar.getInstance();
+        Date now = new Date();
+        calendar.setTime(now);
+
+        if (QUEUED_WORKFLOW_AGE_LIMIT_HOURS == null) {
+            QUEUED_WORKFLOW_AGE_LIMIT_HOURS = 24;
+        }
+
+        // calculate time QUEUED_WORKFLOW_AGE_LIMIT_HOURS before now
+        calendar.add(Calendar.HOUR, -QUEUED_WORKFLOW_AGE_LIMIT_HOURS);
+        Date removeTime = calendar.getTime();
+
+        logger.info("The time is " + now);
+        logger.info("Delete time interval is : OLDER THAN " + QUEUED_WORKFLOW_AGE_LIMIT_HOURS + " HOURS");
+        logger.info("Deleting queued workflows older than or equal to " + removeTime);
+
+        logger.info(queuedWorkflowRepository.deleteByTempRepresentation_RetrievedOnLessThanEqual(removeTime)
+                + " Old queued workflows removed");
+    }
+}

src/main/java/org/commonwl/view/workflow/QueuedWorkflowRepository.java

@@ -4,6 +4,9 @@
 import org.springframework.data.mongodb.repository.Query;
 import org.springframework.data.repository.PagingAndSortingRepository;
 
+import java.util.Date;
+import java.util.List;
+
 /**
  * Stores workflows in the queue waiting for cwltool
  */
@@ -23,4 +26,20 @@ public interface QueuedWorkflowRepository extends PagingAndSortingRepository<Que
      */
     void deleteByTempRepresentation_RetrievedFrom(GitDetails retrievedFrom);
 
+    /**
+     * Deletes all queued workflows with date retrieved on older or equal to the Date argument passed.
+     * @param retrievedOn Date of when the queued workflow was retrieved
+     * @return The number of queued workflows deleted
+     */
+    Long deleteByTempRepresentation_RetrievedOnLessThanEqual(Date retrievedOn);
+
+
+    /**
+     * Finds and returns all queued workflows with date retrieved on older or equal to the Date argument passed.
+     * @param retrievedOn Details of where the queued workflow is from
+     * @return A list of queued workflows
+     */
+    List<QueuedWorkflow> findByTempRepresentation_RetrievedOnLessThanEqual(Date retrievedOn);
+
+
 }

src/main/resources/application.properties

@@ -45,4 +45,17 @@ spring.data.mongodb.port = 27017
 #=======================
 # SPARQL endpoint
 #=======================
-sparql.endpoint = http://localhost:3030/cwlviewer/
+sparql.endpoint = http://localhost:3030/cwlviewer/
+
+
+#=======================
+# Scheduler settings
+#=======================
+
+# Cron expression that specifies the interval for running deletes of old queued workflow
+# For more info see https://spring.io/blog/2020/11/10/new-in-spring-5-3-improved-cron-expressions#usage
+# The expression below implies every hour at the 0th second and 0th minute i.e (01:00:00, 02:00::00,... etc)
+cron.deleteOldQueuedWorkflows = 0 0 * * * ?
+
+# Age limit for queued workflows in hours.
+queuedWorkflowAgeLimitHours = 24

src/main/resources/static/bower_components/bootstrap/Gemfile

@@ -2,5 +2,5 @@ source 'https://rubygems.org'
 
 group :development, :test do
   gem 'jekyll', '~> 3.9.0'
-  gem 'jekyll-sitemap', '~> 0.11.0'
+  gem 'jekyll-sitemap', '~> 0.12.0'
 end

src/main/resources/static/bower_components/bootstrap/Gemfile.lock

@@ -1,21 +1,22 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.4.0)
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
     colorator (1.1.0)
-    concurrent-ruby (1.1.7)
+    concurrent-ruby (1.1.9)
     em-websocket (0.5.2)
       eventmachine (>= 0.12.9)
       http_parser.rb (~> 0.6.0)
     eventmachine (1.2.7)
     eventmachine (1.2.7-x64-mingw32)
-    ffi (1.13.1)
-    ffi (1.13.1-x64-mingw32)
+    ffi (1.15.3)
+    ffi (1.15.3-x64-mingw32)
     forwardable-extended (2.6.0)
     http_parser.rb (0.6.0)
     i18n (0.9.5)
       concurrent-ruby (~> 1.0)
-    jekyll (3.9.0)
+    jekyll (3.9.1)
       addressable (~> 2.4)
       colorator (~> 1.0)
       em-websocket (~> 0.5)
@@ -30,24 +31,25 @@ GEM
       safe_yaml (~> 1.0)
     jekyll-sass-converter (1.5.2)
       sass (~> 3.4)
-    jekyll-sitemap (0.11.0)
-      addressable (~> 2.4.0)
+    jekyll-sitemap (0.12.0)
+      jekyll (~> 3.3)
     jekyll-watch (2.2.1)
       listen (~> 3.0)
-    kramdown (2.3.0)
+    kramdown (2.3.1)
       rexml
     liquid (4.0.3)
-    listen (3.3.3)
+    listen (3.5.1)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
     mercenary (0.3.6)
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
-    rb-fsevent (0.10.4)
+    public_suffix (4.0.6)
+    rb-fsevent (0.11.0)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
-    rexml (3.2.4)
-    rouge (3.25.0)
+    rexml (3.2.5)
+    rouge (3.26.0)
     safe_yaml (1.0.5)
     sass (3.7.4)
       sass-listen (~> 4.0.0)
@@ -61,7 +63,7 @@ PLATFORMS
 
 DEPENDENCIES
   jekyll (~> 3.9.0)
-  jekyll-sitemap (~> 0.11.0)
+  jekyll-sitemap (~> 0.12.0)
 
 BUNDLED WITH
    1.17.3