Feature/scheduled automations (#44)

* change cron job algorithm
* update readme

Author: yooper

README.md

@@ -9,6 +9,12 @@ in json to provide additional functionality. The functionality aids in adding fl
 custom code or complexity. This product is meant to simplify digital archiving by providing tools that can automate most
 of the collection process. 
 
+# Privacy Policy
+We do not collect your data, share your data , store your data or even handle your data. All your data stays on your computer. Please see our privacy policy and feel free to reach 
+out with any suggestions on improvements. https://bakerstreet.llc/your-rapport-privacy-policy/
+
+**This is an open source project, which strives to prioritize your privacy.**
+
 ### Getting Started 
 After installing the Chrome extension from https://chromewebstore.google.com/detail/your-rapport/clkaalonjdkliiaadkgodlfbiipidjmn, 
 "Your Rapport" will automatically be ready to collect.
@@ -40,7 +46,7 @@ Check out the wiki for more in depth information. Or review the source code in t
 
 # Technical and Product Roadmap 
 
-Below are the stages used to process collected information, commonly referred to as an ETL pipeline. This workflow is 
+Below are the stages used to process collected information, that stays locally on your machine, commonly referred to as an ETL pipeline. This workflow is 
 designed to run entirely inside a Chrome extension. Discovery Plugins connect events to actions. For example, you might 
 save a web page, an MHTML file, an image, or a snippet of JavaScript and then trigger a scan by a remote service to 
 extract people, places, or names from the text, or run OCR to pull text from an image. The extension collects the 

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "your-rapport",
-  "version": "0.21.0",
+  "version": "0.22.0",
   "description": "A chrome extension for saving screenshots and making them searchable.",
   "license": "Commercial",
   "scripts": {

package.json

@@ -45,7 +45,7 @@ export async function complete(queuedJob: BulkAutomationUrl) {
   queuedJob.status = 'done';
   queuedJob.leaseUntil = null;
   queuedJob.completedOn = new Date().getTime();
-  queuedJob.description = 'Completed Successfully: '+ queuedJob.description ?? '';
+  queuedJob.description = 'Completed Successfully';
   await db.bulkAutomation.put(queuedJob);
 }
 

src/backgrounds/automation-queue.ts

@@ -11,7 +11,7 @@ import { db } from '../models/db/dexieDb';
 import BulkAutomationUrl from '../models/schemas/BulkAutomationUrl';
 import { NoChangeDetectedError } from '../errors/NoChangeDetectedError';
 import { Tag } from '../models/schemas/Tag';
-import { getUtcNow } from '../utilities/transformers';
+import { getUtc, getUtcNow } from '../utilities/transformers';
 
 
 let processing: boolean = false;
@@ -50,19 +50,20 @@ export function initializeAutomationRunner() {
 }
 
 async function queueScheduledAutomations(){
+
   // upon trigger, we need to check for any active scheduled automations that must be run by checking their crontab evaluation and generate
   // any associated bulk automations
   db.transaction('rw', db.bulkAutomation, db.scheduledAutomation, async () => {
       processing = true;
       const scheduledAutomations: ScheduledAutomation[] = await db.scheduledAutomation.filter(scheduled => scheduled.active).toArray();
-      const utcNow: Date = new Date();
+      const utcNow: number = getUtcNow();
       debug('Scheduling automations', {scheduledAutomations});
-      utcNow.setSeconds(0, 0);
       const automations: BulkAutomationUrl[] = [];
       for (const scheduledAutomation of scheduledAutomations) {
         const interval = CronExpressionParser.parse(scheduledAutomation.crontab);
-        if(interval.includesDate(utcNow)){
-          // TODO: queue the job
+        // run automation
+        if(!scheduledAutomation?.nextRunOn || scheduledAutomation.nextRunOn <= utcNow) {
+          // add bulk automation url
           const automation = BulkAutomationUrl.createBulkAutomationJob(scheduledAutomation.url, {
             keepTabOpen: scheduledAutomation.keepTabOpen ?? false,
             isDeepSave: scheduledAutomation.isDeepSave ?? true,
@@ -73,7 +74,14 @@ async function queueScheduledAutomations(){
           // MUST be set to active to trigger running in the automation queue
           automation.active = true;
           automations.push(automation)
-          scheduledAutomation.lastRanOn = getUtcNow();
+          scheduledAutomation.prevRanOn = utcNow;
+          if(interval.hasNext()){
+            scheduledAutomation.nextRunOn = getUtc(interval.next().toDate());
+          }
+          else{
+            // deactivate the scheduled automation if no future time exists
+            scheduledAutomation.active = false;
+          }
           await db.scheduledAutomation.put(scheduledAutomation);
         }
       }
@@ -82,6 +90,7 @@ async function queueScheduledAutomations(){
         await db.bulkAutomation.bulkAdd(automations);
         return automations
       }
+      return []
     }).then((automations) => {
       if(automations?.length ?? [].length > 0){
         debug('The following bulk automations were scheduled', {automations})
@@ -173,7 +182,9 @@ async function processQueue() {
         ; // do nothing, no change was detected
         job.description = 'No Change Detected';
         await complete(job)
-      } else {
+      }
+      else {
+        await ('automationRunner:failed ' + String(e), {job})
         await fail(job, String(e?.message ?? e));
       }
     } finally {

src/backgrounds/automation-runner.ts

@@ -271,6 +271,7 @@ export default function ScheduledAutomationDataTable(): JSX.Element {
                     try {
                       const now = new Date()
                       const interval = CronExpressionParser.parse(record.crontab);
+                      scheduledAutomation.nextRunOn = interval.next().getTime();
                       debug(interval.fields.minute);
                       if(interval.includesDate(now)){
                         debug('now '+ now.toISOString());

src/components/tables/ScheduledAutomationDataTable.tsx

@@ -9,7 +9,7 @@
     "default_title": "Your Rapport is a screenshot tool with auto scroll capabilities. Right click, select Collect or Alt+S to save. Alt+A for auto scrolling captures. Alt+X open the dashboard. Alt+Q for doing a quick scan.",
     "default_popup": "popup.html"
   },
-  "version": "0.21.0",
+  "version": "0.22.0",
   "icons": {
     "128": "icon-128.png",
     "48": "icon-48.png",

src/manifest.json

@@ -1,6 +1,7 @@
 import { IBulkAutomationRecord } from '../../types';
 import { ScheduledAutomation } from './ScheduledAutomation';
 import { DiscoveryPlugin } from './DiscoveryPlugin';
+import { getUtc, getUtcNow } from '../../utilities/transformers';
 
 export default class BulkAutomationUrl implements IBulkAutomationRecord {
   uuid: string;
@@ -9,7 +10,7 @@ export default class BulkAutomationUrl implements IBulkAutomationRecord {
   value: number;
   screenShotsCollected: number;
   keepTabOpen: boolean;
-  createdOn: Date;
+  createdOn: number;
   ranOn: number | null;
   completedOn: number | null;
   description: string | null;
@@ -33,7 +34,7 @@ export default class BulkAutomationUrl implements IBulkAutomationRecord {
     this.value = 0;                // sane default
     this.screenShotsCollected = 0;
     this.keepTabOpen = false;
-    this.createdOn = new Date();   // matches type Date
+    this.createdOn = getUtcNow();
     this.ranOn = null;
     this.completedOn = null;
     this.description = null;

src/models/schemas/BulkAutomationUrl.ts

@@ -1,8 +1,9 @@
-import { getUtcNow } from '../../utilities/transformers';
+import { getUtc, getUtcNow } from '../../utilities/transformers';
 import { ChangeDetection } from '../../types';
 import { DiscoveryPlugin } from './DiscoveryPlugin';
 import { db } from '../db/dexieDb';
 import { debug } from '../../services/logger_services';
+import { CronExpressionParser } from 'cron-parser';
 
 export class ScheduledAutomation {
   uuid: string;
@@ -20,7 +21,8 @@ export class ScheduledAutomation {
   enableSelectorChangeDetector: boolean;
   enableTextChangeDetector: boolean;
   onlySaveOnChange: boolean;
-  lastRanOn: number | null;
+  prevRanOn: number | null;
+  nextRunOn: number | null;
   lastError: string | null;
   discoveryPlugin: DiscoveryPlugin | null;
   tags: string[];
@@ -31,18 +33,19 @@ export class ScheduledAutomation {
     this.unit = 'count';
     this.value = 100;
     this.keepTabOpen = false;
-    this.createdOn = getUtcNow();   // matches type Date
+    this.createdOn = getUtcNow();
     this.active = true;
     this.isDeepSave = true;
     this.changeDetectors = [];
     this.onlySaveOnChange = true;
     this.enableImageChangeDetector = true;
     this.enableSelectorChangeDetector = true;
     this.enableTextChangeDetector = true;
-    this.lastRanOn = null;
+    this.prevRanOn = null;
+    this.nextRunOn = null;
     this.runInServiceWorker = false;
     this.lastError = null
-    this.crontab = '0 * * * * *';
+    this.crontab = '0 0 * * * *';
     this.discoveryPlugin = null;
     this.tags = [];
     this.caseManagementUuid = "30583002-f730-4383-bf28-fdd8aadcf387"
@@ -59,6 +62,11 @@ export class ScheduledAutomation {
     const scheduledAutomation = new ScheduledAutomation();
     scheduledAutomation.url = url;
     scheduledAutomation.crontab = crontab;
+    const interval = CronExpressionParser.parse(crontab);
+
+    if(interval.hasNext()){
+      scheduledAutomation.nextRunOn = getUtc(interval.next().toDate());
+    }
 
     await db.transaction("rw", db.scheduledAutomation, async () => {
       try {

src/models/schemas/ScheduledAutomation.ts

@@ -15,7 +15,6 @@ export async function debug(
   try {
 
     const tsMessage = `[${new Date().toISOString()}] `+message
-
     const value: boolean = await Configuration.getConfigurationValue<boolean>(
       'debugMessagesEnabled',
       true