Merge pull request #326 from obasekiosa/purge-old-queuedworkflows

Schedule recurrent CWL Viewer maintained cron-job for purging of old queued workflows from database

Author: tetron

src/main/java/org/commonwl/view/CwlViewerApplication.java

@@ -23,10 +23,12 @@
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
 import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.scheduling.annotation.EnableScheduling;
 
 @SpringBootApplication
 @EnableMongoRepositories
 @EnableAsync
+@EnableScheduling
 public class CwlViewerApplication {
 
 	public static void main(String[] args) {

src/main/java/org/commonwl/view/Scheduler.java

@@ -0,0 +1,58 @@
+package org.commonwl.view;
+
+
+import org.commonwl.view.workflow.QueuedWorkflowRepository;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import java.util.Calendar;
+import java.util.Date;
+
+/**
+ * Scheduler class for recurrent processes.
+ */
+@Component
+public class Scheduler {
+
+    private final Logger logger = LoggerFactory.getLogger(this.getClass());
+    private final QueuedWorkflowRepository queuedWorkflowRepository;
+
+    @Value("${queuedWorkflowAgeLimitHours}")
+    private Integer QUEUED_WORKFLOW_AGE_LIMIT_HOURS;
+
+    @Autowired
+    public Scheduler(QueuedWorkflowRepository queuedWorkflowRepository) {
+        this.queuedWorkflowRepository = queuedWorkflowRepository;
+    }
+
+
+    /**
+     * A Scheduled function to delete old queued workflow entries
+     * from the queue. Age is determined by QUEUED_WORKFLOW_AGE_LIMIT_HOURS
+     */
+    @Scheduled(cron = "${cron.deleteOldQueuedWorkflows}")
+    public void removeOldQueuedWorkflowEntries() {
+        Calendar calendar = Calendar.getInstance();
+        Date now = new Date();
+        calendar.setTime(now);
+
+        if (QUEUED_WORKFLOW_AGE_LIMIT_HOURS == null) {
+            QUEUED_WORKFLOW_AGE_LIMIT_HOURS = 24;
+        }
+
+        // calculate time QUEUED_WORKFLOW_AGE_LIMIT_HOURS before now
+        calendar.add(Calendar.HOUR, -QUEUED_WORKFLOW_AGE_LIMIT_HOURS);
+        Date removeTime = calendar.getTime();
+
+        logger.info("The time is " + now);
+        logger.info("Delete time interval is : OLDER THAN " + QUEUED_WORKFLOW_AGE_LIMIT_HOURS + " HOURS");
+        logger.info("Deleting queued workflows older than or equal to " + removeTime);
+
+        logger.info(queuedWorkflowRepository.deleteByTempRepresentation_RetrievedOnLessThanEqual(removeTime)
+                + " Old queued workflows removed");
+    }
+}

src/main/java/org/commonwl/view/workflow/QueuedWorkflowRepository.java

@@ -4,6 +4,9 @@
 import org.springframework.data.mongodb.repository.Query;
 import org.springframework.data.repository.PagingAndSortingRepository;
 
+import java.util.Date;
+import java.util.List;
+
 /**
  * Stores workflows in the queue waiting for cwltool
  */
@@ -24,4 +27,20 @@ public interface QueuedWorkflowRepository extends PagingAndSortingRepository<Que
     @Query("{tempRepresentation.retrievedFrom: ?0}")
     void deleteByRetrievedFrom(GitDetails retrievedFrom);
 
+    /**
+     * Deletes all queued workflows with date retrieved on older or equal to the Date argument passed.
+     * @param retrievedOn Date of when the queued workflow was retrieved
+     * @return The number of queued workflows deleted
+     */
+    Long deleteByTempRepresentation_RetrievedOnLessThanEqual(Date retrievedOn);
+
+
+    /**
+     * Finds and returns all queued workflows with date retrieved on older or equal to the Date argument passed.
+     * @param retrievedOn Details of where the queued workflow is from
+     * @return A list of queued workflows
+     */
+    List<QueuedWorkflow> findByTempRepresentation_RetrievedOnLessThanEqual(Date retrievedOn);
+
+
 }

src/main/resources/application.properties

@@ -45,4 +45,17 @@ spring.data.mongodb.port = 27017
 #=======================
 # SPARQL endpoint
 #=======================
-sparql.endpoint = http://localhost:3030/cwlviewer/
+sparql.endpoint = http://localhost:3030/cwlviewer/
+
+
+#=======================
+# Scheduler settings
+#=======================
+
+# Cron expression that specifies the interval for running deletes of old queued workflow
+# For more info see https://spring.io/blog/2020/11/10/new-in-spring-5-3-improved-cron-expressions#usage
+# The expression below implies every hour at the 0th second and 0th minute i.e (01:00:00, 02:00::00,... etc)
+cron.deleteOldQueuedWorkflows = 0 0 * * * ?
+
+# Age limit for queued workflows in hours.
+queuedWorkflowAgeLimitHours = 24