Improving handling of logs in sqlite

andrewshell · andrewshell · commit 557635db9e48 · 2018-10-03T08:12:28.000-05:00
diff --git a/.jshintignore b/.jshintignore
@@ -0,0 +1 @@
+node_modules/*
diff --git a/.jshintrc b/.jshintrc
@@ -0,0 +1 @@
+{ "esversion":6 }
diff --git a/controllers/home.js b/controllers/home.js
@@ -0,0 +1,19 @@
+(function () {
+    "use strict";
+
+    var express = require('express'),
+        router = express.Router();
+
+    router.get('/', function (req, res) {
+        switch (req.accepts('html')) {
+            case 'html':
+                res.render('home');
+                break;
+            default:
+                res.status(406).send('Not Acceptable');
+                break;
+        }
+    });
+
+    module.exports = router;
+}());
diff --git a/controllers/index.js b/controllers/index.js
@@ -4,6 +4,7 @@
     var express = require('express'),
         router = express.Router();
 
+    router.use('/', require('./home'));
     router.use('/pleaseNotify', require('./please-notify'));
     router.use('/ping', require('./ping'));
     router.use('/pingForm', require('./ping-form'));
diff --git a/controllers/ping-form.js b/controllers/ping-form.js
@@ -6,12 +6,12 @@
 
     router.get('/', function (req, res) {
         switch (req.accepts('html')) {
-        case 'html':
-            res.render('ping-form');
-            break;
-        default:
-            res.status(406).send('Not Acceptable');
-            break;
+            case 'html':
+                res.render('ping-form');
+                break;
+            default:
+                res.status(406).send('Not Acceptable');
+                break;
         }
     });
 
diff --git a/controllers/ping.js b/controllers/ping.js
@@ -14,20 +14,20 @@
 
     function processResponse(req, res, result) {
         switch (req.accepts('xml', 'json')) {
-        case 'xml':
-            res.set('Content-Type', 'text/xml');
-            res.send(restReturnSuccess(
-                result.success,
-                result.msg,
-                'result'
-            ));
-            break;
-        case 'json':
-            res.json(result);
-            break;
-        default:
-            res.status(406).send('Not Acceptable');
-            break;
+            case 'xml':
+                res.set('Content-Type', 'text/xml');
+                res.send(restReturnSuccess(
+                    result.success,
+                    result.msg,
+                    'result'
+                ));
+                break;
+            case 'json':
+                res.json(result);
+                break;
+            default:
+                res.status(406).send('Not Acceptable');
+                break;
         }
     }
 
diff --git a/controllers/please-notify.js b/controllers/please-notify.js
@@ -14,20 +14,20 @@
 
     function processResponse(req, res, result) {
         switch (req.accepts('xml', 'json')) {
-        case 'xml':
-            res.set('Content-Type', 'text/xml');
-            res.send(restReturnSuccess(
-                result.success,
-                result.msg,
-                'notifyResult'
-            ));
-            break;
-        case 'json':
-            res.json(result);
-            break;
-        default:
-            res.status(406).send('Not Acceptable');
-            break;
+            case 'xml':
+                res.set('Content-Type', 'text/xml');
+                res.send(restReturnSuccess(
+                    result.success,
+                    result.msg,
+                    'notifyResult'
+                ));
+                break;
+            case 'json':
+                res.json(result);
+                break;
+            default:
+                res.status(406).send('Not Acceptable');
+                break;
         }
     }
 
diff --git a/controllers/view-log.js b/controllers/view-log.js
@@ -8,33 +8,33 @@
         logEmitter = require('../services/log-emitter'),
         router = express.Router();
 
-    function fetchVals(db, id, callback) {
+    function fetchVals(db, callback) {
         var vals = {
             'eventlog': []
         };
 
-        db.serialize(function() {
-            db.each("SELECT * FROM log_events ORDER BY time DESC LIMIT 1000", function(err, row) {
+        db.serialize(() => {
+            db.each("SELECT * FROM log_events ORDER BY time DESC LIMIT 1000", (err, row) => {
                 row.headers = JSON.parse(row.headers);
                 vals.eventlog.push(row);
-            }, function () {
+            }, () => {
                 callback(null, vals);
             });
         });
     }
 
     function processResponse(req, res, vals) {
         switch (req.accepts('html', 'json')) {
-        case 'html':
-            vals.wshost = res.app.locals.host + ':' + res.app.locals.port;
-            res.render('view-log', vals);
-            break;
-        case 'json':
-            res.json(vals.eventlog);
-            break;
-        default:
-            res.status(406).send('Not Acceptable');
-            break;
+            case 'html':
+                vals.wshost = res.app.locals.host + ':' + res.app.locals.port;
+                res.render('view-log', vals);
+                break;
+            case 'json':
+                res.json(vals.eventlog);
+                break;
+            default:
+                res.status(406).send('Not Acceptable');
+                break;
         }
     }
 
@@ -44,26 +44,24 @@
 
     router.get('/', function (req, res) {
         async.waterfall([
-            function (callback) {
+            (callback) => {
                 data.getDb(callback);
             },
-            function (db, callback) {
-                fetchVals(db, 0, callback);
+            (db, callback) => {
+                fetchVals(db, callback);
             },
-            function (vals) {
+            (vals) => {
                 processResponse(req, res, vals);
             }
-        ], function (errorMessage) {
+        ], (errorMessage) => {
             handleError(req, res, errorMessage);
         });
     });
 
-    router.ws('/', function(ws, req) {
-        var id = 0;
-
+    router.ws('/', (ws, req) => {
         function sendLogEvent(logEvent) {
             ws.send(logEvent);
-        };
+        }
 
         logEmitter.on('logged-event', sendLogEvent);
 
diff --git a/migrations/20170530180056-add-log-events.js b/migrations/20170530180056-add-log-events.js
@@ -1,34 +1,31 @@
-'use strict';
+(function () {
+    "use strict";
 
-var dbm;
-var type;
-var seed;
+    var dbm, type, seed;
 
-/**
-  * We receive the dbmigrate dependency from dbmigrate initially.
-  * This enables us to not have to rely on NODE_PATH.
-  */
-exports.setup = function(options, seedLink) {
-  dbm = options.dbmigrate;
-  type = dbm.dataType;
-  seed = seedLink;
-};
+    /**
+      * We receive the dbmigrate dependency from dbmigrate initially.
+      * This enables us to not have to rely on NODE_PATH.
+      */
+    module.exports.setup = (options, seedLink) => {
+        dbm = options.dbmigrate;
+        type = dbm.dataType;
+        seed = seedLink;
+    };
 
-exports.up = function(db, callback) {
-    db.createTable('log_events', {
-        id: { type: 'int', primaryKey: true },
-        eventtype: 'text',
-        htmltext: 'text',
-        secs: 'int',
-        time: 'text',
-        headers: 'text'
-    }, callback);
-};
+    module.exports.up = (db, callback) => {
+        db.createTable('log_events', {
+            id: { type: 'int', primaryKey: true },
+            eventtype: 'text',
+            htmltext: 'text',
+            secs: 'int',
+            time: 'text',
+            headers: 'text'
+        }, callback);
+    };
 
-exports.down = function(db, callback) {
-    db.dropTable('log_events', callback);
-};
+    module.exports.down = (db, callback) => {
+        db.dropTable('log_events', callback);
+    };
 
-exports._meta = {
-  "version": 1
-};
+}());
diff --git a/package.json b/package.json
@@ -11,40 +11,37 @@
     "test": "echo '=> testing' && istanbul cover _mocha -- --recursive -R spec -r should",
     "posttest": "istanbul check-coverage"
   },
-  "engines": {
-    "node": "^6.10.3"
-  },
   "author": "Andrew Shell <andrew@andrewshell.org>",
   "license": "MIT",
   "dependencies": {
-    "async": "^2.4.1",
-    "body-parser": "^1.17.2",
-    "console-stamp": "^0.2.5",
-    "cors": "^2.8.3",
-    "db-migrate": "^0.10.0-beta.20",
-    "db-migrate-sqlite3": "^0.2.1",
-    "express": "^4.15.3",
+    "async": "^2.6.1",
+    "body-parser": "^1.18.3",
+    "console-stamp": "^0.2.7",
+    "cors": "^2.8.4",
+    "db-migrate": "^0.11.3",
+    "db-migrate-sqlite3": "^0.3.1",
+    "express": "^4.16.3",
     "express-handlebars": "^3.0.0",
-    "express-ws": "^3.0.0",
-    "moment": "^2.18.1",
-    "morgan": "^1.8.2",
-    "nconf": "^0.8.4",
-    "request": "^2.81.0",
+    "express-ws": "^4.0.0",
+    "moment": "^2.22.2",
+    "morgan": "^1.9.1",
+    "nconf": "^0.10.0",
+    "request": "^2.88.0",
     "sprintf-js": "^1.1.1",
-    "sqlite3": "^3.1.8",
-    "xmlbuilder": "^9.0.0"
+    "sqlite3": "^4.0.2",
+    "xmlbuilder": "^10.0.0"
   },
   "repository": {
     "type": "git",
     "url": "https://github.com/andrewshell/rsscloud-server.git"
   },
   "devDependencies": {
-    "istanbul": "^0.4.0",
-    "jscs": "^2.5.1",
-    "jshint": "^2.8.0",
-    "mocha": "^2.3.3",
-    "nock": "^2.17.0",
-    "should": "^7.1.1",
-    "sinon": "^1.17.2"
+    "istanbul": "^0.4.5",
+    "jscs": "^3.0.7",
+    "jshint": "^2.9.6",
+    "mocha": "^5.2.0",
+    "nock": "^9.6.1",
+    "should": "^13.2.3",
+    "sinon": "^6.3.4"
   }
 }
diff --git a/services/data/init.js b/services/data/init.js
@@ -3,6 +3,7 @@
 
     var async = require('async'),
         DBMigrate = require('db-migrate'),
+        moment = require('moment'),
         sqlite3 = require('sqlite3');
 
     function connectToDatabase(filename, callback) {
@@ -18,19 +19,47 @@
         );
     }
 
+    function flushOldLogs(db) {
+        var stmt = db.prepare(`
+            DELETE FROM log_events
+            WHERE time < ?
+        `);
+
+        var time = moment().subtract(1, 'day');
+
+        console.log('Flushing old logs: ' + time.toISOString());
+
+        stmt.run(
+            time.toISOString()
+        );
+
+        stmt.finalize();
+    }
+
     function init(callback) {
         var dbmigrate;
         async.waterfall([
             function asyncDoMigration(callback) {
                 dbmigrate = DBMigrate.getInstance(true);
-                dbmigrate.up(callback);
+                dbmigrate.up(() => {
+                    callback();
+                });
             },
             function asyncConnectToDatabase(callback) {
                 if ('sqlite3' === dbmigrate.config.getCurrent().settings.driver) {
                     connectToDatabase(dbmigrate.config.getCurrent().settings.filename, callback);
                 } else {
                     callback('This application currently requires sqlite3 as the driver');
                 }
+            },
+            function asyncFlushOldLogs(db, callback) {
+                flushOldLogs(db);
+
+                setInterval(() => {
+                    flushOldLogs(db);
+                }, 3600000);
+
+                callback(null, db);
             }
         ], callback);
     }
diff --git a/services/log-event.js b/services/log-event.js
@@ -21,7 +21,7 @@
                 return;
             }
 
-            db.serialize(function() {
+            db.serialize(() => {
 
                 var stmt = db.prepare(`
                         INSERT INTO log_events (
diff --git a/services/parse-notify-params.js b/services/parse-notify-params.js
diff --git a/views/home.handlebars b/views/home.handlebars
diff --git a/yarn.lock b/yarn.lock
