Merge pull request #1903 from floccusaddon/feat/indexeddb-logs

feat(Logger): Use IndexedDB to store logs in order to store more

Author: marcelklehr

package-lock.json

@@ -97,6 +97,7 @@
     "buffer": "^6.0.3",
     "cheerio": "^1.0.0-rc.12",
     "core-js": "3.x",
+    "dexie": "^4.0.11",
     "fast-xml-parser": "^4.2.7",
     "humanize-duration": "^3.25.1",
     "intl-messageformat": "^9.9.1",

package.json

@@ -198,7 +198,6 @@ export default class Account {
               Logger.log(
                 'Resource is locked, trying again soon'
               )
-              await Logger.persist()
               return
             }
           } else {
@@ -348,7 +347,6 @@ export default class Account {
         await this.init()
       }
     }
-    await Logger.persist()
   }
 
   static async stringifyError(er:any):Promise<string> {

src/lib/Account.ts

@@ -0,0 +1,43 @@
+import Dexie, { type EntityTable } from 'dexie'
+
+interface LogMessage {
+  id: number;
+  dateTime: number;
+  message: string;
+}
+
+const db = new Dexie('floccus') as Dexie & {
+  logs: EntityTable<
+    LogMessage,
+    'id' // primary key "id" (for the typings only)
+  >;
+}
+
+db.version(1).stores({
+  logs: '++id, dateTime, message'
+})
+
+export { db }
+export { LogMessage }
+
+export async function freeStorageIfNecessary() {
+  if (navigator.storage && navigator.storage.estimate) {
+    let {usage, quota} = await navigator.storage.estimate()
+    if (usage / quota > 0.9) {
+      const oneWeekAgo = Date.now() - 60 * 60 * 1000 * 24 * 7
+
+      await db.logs
+        .where('dateTime').below(oneWeekAgo)
+        .delete()
+    }
+
+    ({usage, quota} = await navigator.storage.estimate())
+    if (usage / quota > 0.6) {
+      const oneDayAgo = Date.now() - 60 * 60 * 1000 * 24
+
+      await db.logs
+        .where('dateTime').below(oneDayAgo)
+        .delete()
+    }
+  }
+}

src/lib/IndexedDB.ts

@@ -6,38 +6,31 @@ import Crypto from './Crypto'
 import { Share } from '@capacitor/share'
 import { Filesystem, Directory, Encoding } from '@capacitor/filesystem'
 import { Capacitor } from '@capacitor/core'
+import { db } from './IndexedDB'
 
 export default class Logger {
   static log() {
-    const logMsg = [new Date().toISOString(), ...arguments]
+    const dateTime = Date.now()
+    const logMsg = [...arguments]
+    const message = util.format.apply(util, logMsg)
 
     // log to console
     DEBUG && console.log(util.format.apply(util, logMsg))
-    this.messages.push(util.format.apply(util, logMsg)) // TODO: Use a linked list here to get O(n)
-  }
-
-  static async persist() {
-    const Storage = (Capacitor.getPlatform() === 'web') ? await import('./browser/BrowserAccountStorage') : await import('./native/NativeAccountStorage')
-    await Storage.default.changeEntry(
-      'logs',
-      log => {
-        const messages = this.messages
-        this.messages = []
-        return messages // only save the last sync run
-      },
-      []
-    )
+    db.logs.add({dateTime, message})
+      .catch(e => {
+        console.error('Failed to log to IndexedDB: ', e)
+        console.error(e)
+      })
   }
 
   static async getLogs() {
-    const Storage = (Capacitor.getPlatform() === 'web') ? await import('./browser/BrowserAccountStorage') : await import('./native/NativeAccountStorage')
-    return Storage.default.getEntry('logs', [])
+    return db.logs.orderBy('dateTime').toArray()
   }
 
   static async anonymizeLogs(logs) {
     const regex = /\[(.*?)\]\((.*?)\)|\[(.*?)\]/g
-    const newLogs = await Parallel.map(logs, async(entry) => {
-      return Logger.replaceAsync(entry, regex, async(match, p1, p2, p3) => {
+    await Parallel.map(logs, async(logMessage) => {
+      logMessage.message = await Logger.replaceAsync(logMessage.message, regex, async(match, p1, p2, p3) => {
         if (p1 && p2) {
           const hash1 = await Crypto.sha256(p1)
           const hash2 = await Crypto.sha256(p2)
@@ -50,8 +43,11 @@ export default class Logger {
     }, 1)
     const regex2 = /url=https?%3A%2F%2F.*$|url=https?%3A%2F%2F[^ ]*/
     const regex3 = /https?:\/\/[^ /]*\//
-    return newLogs
-      .map(line => line.replace(regex2, '###url###').replace(regex3, '###server###'))
+    logs
+      .forEach(logMessage => {
+        logMessage.message = logMessage.message.replace(regex2, '###url###').replace(regex3, '###server###')
+      })
+    return logs
   }
 
   static async replaceAsync(str, regex, asyncFn) {
@@ -75,7 +71,12 @@ export default class Logger {
     if (anonymous) {
       logs = await Logger.anonymizeLogs(logs)
     }
-    let blob = new Blob([logs.join('\n')], {
+    logs = logs
+      .map(logMessage => {
+        return new Date(logMessage.dateTime).toISOString() + ' ' + logMessage.message
+      })
+      .join('\n')
+    let blob = new Blob([logs], {
       type: 'text/plain',
       endings: 'native'
     })
@@ -121,4 +122,3 @@ export default class Logger {
     }
   }
 }
-Logger.messages = []

src/lib/Logger.js

@@ -9,6 +9,7 @@ import uniqBy from 'lodash/uniqBy'
 import Account from '../Account'
 import { STATUS_ALLGOOD, STATUS_DISABLED, STATUS_ERROR, STATUS_SYNCING } from '../interfaces/Controller'
 import * as Sentry from '@sentry/browser'
+import { freeStorageIfNecessary } from '../IndexedDB'
 
 const INACTIVITY_TIMEOUT = 7 * 1000 // 7 seconds
 const MAX_BACKOFF_INTERVAL = 1000 * 60 * 60 // 1 hour
@@ -23,19 +24,18 @@ class AlarmManager {
 
   async checkSync() {
     const accounts = await BrowserAccountStorage.getAllAccounts()
-    const promises = []
     for (let accountId of accounts) {
       const account = await Account.get(accountId)
       const data = account.getData()
       const lastSync = data.lastSync || 0
       const interval = data.syncInterval || DEFAULT_SYNC_INTERVAL
       if (data.scheduled) {
-        promises.push(this.ctl.scheduleSync(accountId))
+        await this.ctl.scheduleSync(accountId)
         continue
       }
       if (data.error && data.errorCount > 1) {
         if (Date.now() > this.getBackoffInterval(interval, data.errorCount, lastSync) + lastSync) {
-          promises.push(this.ctl.scheduleSync(accountId))
+          await this.ctl.scheduleSync(accountId)
           continue
         }
         continue
@@ -44,10 +44,9 @@ class AlarmManager {
         Date.now() >
         interval * 1000 * 60 + lastSync
       ) {
-        promises.push(this.ctl.scheduleSync(accountId))
+        await this.ctl.scheduleSync(accountId)
       }
     }
-    await Promise.all(promises)
   }
 
   /**
@@ -121,6 +120,17 @@ export default class BrowserController {
       }
     })
 
+    // Remove old logs
+
+    BrowserAccountStorage.changeEntry(
+      'logs',
+      log => {
+        return []
+      },
+      []
+    )
+    freeStorageIfNecessary()
+
     // do some cleaning if this is a new version
 
     browser.storage.local.get(['currentVersion', 'lastInterventionAt']).then(async d => {

src/lib/browser/BrowserController.js

@@ -4,6 +4,7 @@ import Cryptography from '../Crypto'
 import NativeAccountStorage from './NativeAccountStorage'
 import Account from '../Account'
 import { STATUS_ALLGOOD, STATUS_DISABLED, STATUS_ERROR, STATUS_SYNCING } from '../interfaces/Controller'
+import { freeStorageIfNecessary } from '../IndexedDB'
 
 const INACTIVITY_TIMEOUT = 1000 * 7
 const MAX_BACKOFF_INTERVAL = 1000 * 60 * 60 // 1 hour
@@ -35,12 +36,12 @@ class AlarmManager {
       const lastSync = data.lastSync || 0
       const interval = data.syncInterval || DEFAULT_SYNC_INTERVAL
       if (data.scheduled) {
-        this.ctl.scheduleSync(accountId)
+        await this.ctl.scheduleSync(accountId)
         continue
       }
       if (data.error && data.errorCount > 1) {
         if (Date.now() > this.getBackoffInterval(interval, data.errorCount, lastSync) + lastSync) {
-          this.ctl.scheduleSync(accountId)
+          await this.ctl.scheduleSync(accountId)
           continue
         }
         continue
@@ -49,7 +50,7 @@ class AlarmManager {
         Date.now() >
         interval * 1000 * 60 + data.lastSync
       ) {
-        this.ctl.scheduleSync(accountId)
+        await this.ctl.scheduleSync(accountId)
       }
     }
   }
@@ -83,6 +84,17 @@ export default class NativeController {
 
     this.alarms = new AlarmManager(this)
 
+    // Remove old logs
+
+    NativeAccountStorage.changeEntry(
+      'logs',
+      log => {
+        return []
+      },
+      []
+    )
+    freeStorageIfNecessary()
+
     // lock accounts when locking is enabled
 
     Storage.get({key: 'accountsLocked' }).then(async({value: accountsLocked}) => {
@@ -133,6 +145,7 @@ export default class NativeController {
       if (this.schedule[accountId]) {
         clearTimeout(this.schedule[accountId])
       }
+      console.log('scheduleSync: setting a timeout in ms :', INACTIVITY_TIMEOUT)
       this.schedule[accountId] = setTimeout(
         () => this.scheduleSync(accountId),
         INACTIVITY_TIMEOUT