Merge pull request #41 from relekang/sse

Add realtime events with SSE

Author: mxstbr

README.md

@@ -50,18 +50,70 @@ If you just want to get the views for an id and don't want to increment the view
 
 If you want to get all views for all ids, set the `all` query parameter to `true` on a root request. (i.e. `/?all=true`) If you pass the `all` parameter to an id, all ids starting with that pathname will be included. E.g. `/x?all=true` will match views for `/x`, `/xyz` but not `/y`.
 
+### Options
+
+```
+$ micro-analytics --help
+Usage: micro-analytics [options] [command]
+
+Commands:
+
+  help  Display help
+
+Options:
+
+  -a, --adapter [value]  Database adapter used (defaults to "flat-file-db")
+  -h, --help             Output usage information
+  -H, --host [value]     Host to listen on (defaults to "0.0.0.0")
+  -p, --port <n>         Port to listen on (defaults to 3000)
+  -v, --version          Output the version number
+```
+
 ### Database adapters
 
 By default, `micro-analytics` uses `flat-file-db`, a fast in-process flat file database, which makes for easy setup and backups.
 
-This works fine for side-project usage, but for a production application with bajillions of visitors you might want to use a real database with a _database adapter_. Install the necessary npm package (e.g. `micro-analytics-adapter-xyz`) and then specify the `DB_ADAPTER` environment variable: `$ DB_ADAPTER=xyz micro-analytics`
+This works fine for side-project usage, but for a production application with bajillions of visitors you might want to use a real database with a _database adapter_. Install the necessary npm package (e.g. `micro-analytics-adapter-xyz`) and then specify the `DB_ADAPTER` environment variable: `$ DB_ADAPTER=xyz micro-analytics` or use the `--adapter` cli option.
 
 These are the available database adapters, made by the community:
 
 - [`micro-analytics-adapter-redis`](https://github.com/relekang/micro-analytics-adapter-redis)
 
 Don't see your favorite database here? Writing your own adapter is super easy! See [`writing-adapters.md`](writing-adapters.md) for a simple step-by-step guide.
 
+### Live updates
+
+micro-analytics also let's you listen into updates live with [server-sent events][].
+That means you can e.g. build a realtime dashboard for your analytics!
+
+Note: Make sure your database adapter supports this feature. If not, bug them to implement it!
+micro-analytics will tell you when it starts up if it is supported, so the easiest way to find
+out is just to start it up.
+
+The example below shows how you can listen for events in the browser, just swap
+micro-analytics.now.sh with your own domain and give it a try.
+
+```es6
+const sse = new EventSource('https://micro-analytics.now.sh/realtime')
+sse.onopen = function () { console.log('[sse] open') }
+sse.onerror = function (error) { console.error('[sse error]', error) }
+sse.addEventListener('micro-analytics-ping', function (e) { console.log('[sse]', e) })
+```
+
+#### Browser support
+
+Server-sent events is not supported in all browsers. This can easily be fixed by using a polyfill.
+Take a look at [the caniuse table][] for server-sent events if you need one. Polyfills that are
+supported(disclaimer this list is from the documentation of the sse library we use [rexxars/sse-channel][]):
+
+* [amvtek/EventSource](https://github.com/amvtek/EventSource)
+* [Yaffle/EventSource)](https://github.com/Yaffle/EventSource)
+* [remy/polyfills/EventSource.js](https://github.com/remy/polyfills/blob/master/EventSource.js)
+
+[server-sent events]: https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events
+[the caniuse table]: http://caniuse.com/#feat=eventsource
+[rexxars/sse-channel]: https://github.com/rexxars/sse-channel
+
 ## License
 
 Copyright ©️ 2017 Maximilian Stoiber, licensed under the MIT License. See [`license.md`](./license.md) for more information.

package.json

@@ -15,22 +15,25 @@
     "node": ">=6.0.0"
   },
   "scripts": {
-    "start": "micro dist/index.js",
+    "start": "node dist/index.js",
     "build": "./node_modules/.bin/async-to-gen src --out-dir dist",
     "prepublish": "npm run build",
-    "dev": "NODE_ENV=development nodemon --config package.json src/index.js",
+    "dev": "NODE_ENV=development nodemon --config package.json --exec async-node src/index.js",
     "test": "jest"
   },
   "author": "Max Stoiber <[email protected]> (http://mxstbr.com/)",
   "license": "MIT",
   "repository": "https://github.com/mxstbr/micro-analytics",
   "dependencies": {
+    "args": "^2.3.0",
     "flat-file-db": "^1.0.0",
     "micro": "6.1.0",
-    "micro-analytics-adapter-flat-file-db": "^1.0.3",
+    "micro-analytics-adapter-flat-file-db": "^1.2.1",
     "promise": "^7.1.1",
     "shelljs": "^0.7.6",
-    "update-notifier": "^1.0.3"
+    "sse-channel": "^2.0.6",
+    "update-notifier": "^1.0.3",
+    "zen-observable": "^0.4.0"
   },
   "devDependencies": {
     "async-to-gen": "^1.3.0",
@@ -39,6 +42,7 @@
     "babel-polyfill": "^6.20.0",
     "babel-preset-node6": "^11.0.0",
     "jest": "^19.0.2",
+    "nodemon": "^1.11.0",
     "request-promise": "^4.1.1"
   },
   "execMap": {

src/db.js

@@ -1,26 +1,33 @@
 const promise = require('promise')
 
-let adapter
-const adapterName = `micro-analytics-adapter-${process.env.DB_ADAPTER || 'flat-file-db'}`
+function initDbAdapter(adapterName) {
+  let adapter
+  const repeatCharacter = (char, n) => `${Array(n + 1).join(char)}`
 
-const repeatCharacter = (char, n) => `${Array(n + 1).join(char)}`
-
-try {
-  adapter = require(adapterName)
-} catch (err) {
-  if (err.code === 'MODULE_NOT_FOUND') {
-    // Console.error a warning message, but normally exit the process to avoid printing ugly npm ERR lines and stack trace.
-    console.error(`\n${repeatCharacter(' ', 22)}⚠️ ERROR ⚠️\n${repeatCharacter('-', 55)}\nYou specified "${process.env.DB_ADAPTER}" as the DB_ADAPTER, but no package\ncalled "${adapterName}" was found.\n\nPlease make sure you spelled the name correctly and\nhave "npm install"ed the necessary adapter package!\n${repeatCharacter('-', 55)}\n`)
-    process.exit(0)
-  } else {
-    throw err
+  try {
+    adapter = require(`micro-analytics-adapter-${adapterName}`)
+  } catch (err) {
+    if (err.code === 'MODULE_NOT_FOUND') {
+      // Console.error a warning message, but normally exit the process to avoid printing ugly npm ERR lines and stack trace.
+      console.error(`\n${repeatCharacter(' ', 22)}⚠️ ERROR ⚠️\n${repeatCharacter('-', 55)}\nYou specified "${adapterName}" as the DB_ADAPTER, but no package\ncalled "micro-analytics-adapter-${adapterName}" was found.\n\nPlease make sure you spelled the name correctly and\nhave "npm install"ed the necessary adapter package!\n${repeatCharacter('-', 55)}\n`)
+      process.exit(0)
+    } else {
+      throw err
+    }
   }
+
+
+  module.exports.get = adapter.get;
+  module.exports.getAll = adapter.getAll;
+  module.exports.put = adapter.put;
+  module.exports.has = adapter.has;
+  module.exports.keys = adapter.keys;
+  module.exports.subscribe = adapter.subscribe;
+  module.exports.hasFeature = (feature) => typeof adapter[feature] === "function";
+
 }
 
 module.exports = {
-  get: adapter.get,
-  getAll: adapter.getAll,
-  put: adapter.put,
-  has: adapter.has,
-  keys: adapter.keys,
+  initDbAdapter: initDbAdapter,
+  hasFeature: (feature) => false,
 }

src/handler.js

@@ -0,0 +1,69 @@
+const url = require('url')
+const { send, createError, sendError } = require('micro')
+
+const db = require('./db')
+const { pushView } = require('./utils')
+
+let sse;
+
+if (db.hasFeature("subscribe")) {
+  const SseChannel = require('sse-channel')
+  const sseHandler = require('./sse')
+  sse = new SseChannel({ cors: { origins: ['*'] } })
+  sseHandler(sse)
+}
+
+module.exports = async function (req, res) {
+  const { pathname, query } = url.parse(req.url, /* parseQueryString */ true)
+
+  if (pathname === '/realtime') {
+    if (sse) {
+      sse.addClient(req, res);
+    } else {
+      send(res, 400, {error: 'The current database adapter does not support live updates.'})
+    }
+  }
+
+  res.setHeader('Access-Control-Allow-Origin', '*')
+  // Send all views down if "?all" is true
+  if (String(query.all) === 'true') {
+    try {
+      const data = {
+        data: await db.getAll({
+          pathname: pathname,
+          before: parseInt(query.before, 10),
+          after: parseInt(query.after, 10),
+        }),
+        time: Date.now()
+      }
+      send(res, 200, data)
+      return
+    } catch (err) {
+      console.log(err)
+      throw createError(500, 'Internal server error.')
+    }
+  }
+  // Check that a page is provided
+  if (pathname.length <= 1) {
+    throw createError(400, 'Please include a path to a page.')
+  }
+  if (req.method !== 'GET' && req.method !== 'POST') {
+    throw createError(400, 'Please make a GET or a POST request.')
+  }
+  const shouldIncrement = String(query.inc) !== 'false'
+  try {
+    const currentViews = await db.has(pathname) ? (await db.get(pathname)).views.length : 0
+    // Add a view and send the total views back to the client
+    if (shouldIncrement) {
+      await pushView(pathname, { time: Date.now() })
+    }
+    if (req.method === 'GET') {
+      send(res, 200, { views: shouldIncrement ? currentViews + 1 : currentViews })
+    } else {
+      send(res, 200)
+    }
+  } catch (err) {
+    console.log(err)
+    throw createError(500, 'Internal server error.')
+  }
+}

src/index.js

@@ -1,51 +1,24 @@
-const url = require('url')
-const { send, createError, sendError } = require('micro')
+const micro = require('micro')
 
-const db = require('./db')
-const { pushView } = require('./utils')
+const parseArgs = require('./parseArgs')
+const db = require('./db');
 
-module.exports = async function (req, res) {
-  const { pathname, query } = url.parse(req.url, /* parseQueryString */ true)
-  res.setHeader('Access-Control-Allow-Origin', '*')
-  // Send all views down if "?all" is true
-  if (String(query.all) === 'true') {
-    try {
-      const data = {
-        data: await db.getAll({
-          pathname: pathname,
-          before: parseInt(query.before, 10),
-          after: parseInt(query.after, 10),
-        }),
-        time: Date.now()
-      }
-      send(res, 200, data)
-      return
-    } catch (err) {
-      console.log(err)
-      throw createError(500, 'Internal server error.')
-    }
-  }
-  // Check that a page is provided
-  if (pathname.length <= 1) {
-    throw createError(400, 'Please include a path to a page.')
-  }
-  if (req.method !== 'GET' && req.method !== 'POST') {
-    throw createError(400, 'Please make a GET or a POST request.')
-  }
-  const shouldIncrement = String(query.inc) !== 'false'
-  try {
-    const currentViews = await db.has(pathname) ? (await db.get(pathname)).views.length : 0
-    // Add a view and send the total views back to the client
-    if (shouldIncrement) {
-      await pushView(pathname, { time: Date.now() })
-    }
-    if (req.method === 'GET') {
-      send(res, 200, { views: shouldIncrement ? currentViews + 1 : currentViews })
-    } else {
-      send(res, 200)
-    }
-  } catch (err) {
-    console.log(err)
-    throw createError(500, 'Internal server error.')
+const flags = parseArgs(process.argv)
+
+db.initDbAdapter(flags.adapter)
+
+const handler = require('./handler')
+const server = micro(handler)
+
+server.listen(flags.port, flags.host, (error) => {
+  if (error) {
+    console.error(error)
+    process.exit(1)
   }
-}
+
+  console.log(
+    'micro-analytics listening on ' + flags.host + ':' + flags.port + '\n' +
+    '  with adapter ' + flags.adapter +
+    (db.hasFeature("subscribe") ? '\n  with server side events' : '')
+  )
+})

src/parseArgs.js

@@ -0,0 +1,10 @@
+
+const args = require('args');
+
+module.exports = function parseArgs(argv) {
+  return args
+    .option(['p', 'port'], 'Port to listen on', process.env.PORT || 3000, Number)
+    .option(['H', 'host'], 'Host to listen on', process.env.HOST ||'0.0.0.0')
+    .option(['a', 'adapter'], 'Database adapter used', process.env.DB_ADAPTER || 'flat-file-db')
+    .parse(argv, { name: 'micro-analytics' })
+}

src/sse.js

@@ -0,0 +1,11 @@
+const db = require('./db')
+
+module.exports = function handleSseConnection(connection) {
+  const subscription = db.subscribe((event) => {
+    connection.send({ event: 'micro-analytics-ping', data: JSON.stringify(event) })
+  })
+
+  connection.on('close', function () {
+    subscription.unsubscribe()
+  })
+}

tests/__snapshots__/parseArgs.test.js.snap

@@ -0,0 +1,12 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`parseArgs should have correct defaults 1`] = `
+Object {
+  "H": "0.0.0.0",
+  "a": "flat-file-db",
+  "adapter": "flat-file-db",
+  "host": "0.0.0.0",
+  "p": 3000,
+  "port": 3000,
+}
+`;

tests/__snapshots__/sse.test.js.snap

@@ -0,0 +1,12 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`sse should publish events with correct structure 1`] = `
+Array [
+  Array [
+    Object {
+      "data": "{\\"key\\":\\"superview\\",\\"value\\":{\\"views\\":[{\\"time\\":1490432584312}]}}",
+      "event": "micro-analytics-ping",
+    },
+  ],
+]
+`;

tests/atomicity.test.js

@@ -2,9 +2,12 @@ const request = require('request-promise')
 const { listen, mockDb } = require('./utils')
 
 jest.mock('flat-file-db', () => mockDb)
-const service = require('../src')
+const db = require('../src/db')
+const service = require('../src/handler')
 let url
 
+db.initDbAdapter('flat-file-db');
+
 beforeEach(async () => {
   url = await listen(service)
   mockDb._setDelay(10)