Merge pull request #14 from relekang/move-flat-file-db-to-adapter

Add db adapters

Author: mxstbr

README.md

@@ -24,6 +24,12 @@ See [`server-setup.md`](./server-setup.md) for instructions on acquiring a serve
 
 > **Note**: You can pass any option to the `micro-analytics` command that you can pass to [`micro`](https://github.com/zeit/micro). As an example, to change the host you'd do `micro-analytics -H 127.0.0.1`
 
+### Database adapters
+
+micro-analytics supports custom database adapters. They can be configured with the environment
+variable `DB_ADAPTER`. Setting it to `redis` will make it require `micro-analytics-adapter-redis`.
+Leaving it unset will make micro-analytics use the builtin flat-file-db adapter.
+
 ## Usage
 
 ### Tracking views

src/db.js

@@ -1,18 +1,28 @@
-const flatfile = require('flat-file-db')
 const promise = require('promise')
 
-const db = flatfile.sync(process.env.DB_NAME || 'views.db')
+let adapter
 
-const promisifiedDb = {
-  // Promisify async operations
-  put: promise.denodeify(db.put.bind(db)),
-  del: promise.denodeify(db.del.bind(db)),
-  clear: promise.denodeify(db.clear.bind(db)),
+const repeatCharacter = (char, n) => `${Array(n + 1).join(char)}`
 
-  get: db.get.bind(db),
-  has: db.has.bind(db),
-  keys: db.keys.bind(db),
-  close: db.close.bind(db),
+if (process.env.DB_ADAPTER) {
+  const adapterName = `micro-analytics-adapter-${process.env.DB_ADAPTER}`
+  try {
+    adapter = require(adapterName)
+  } catch (err) {
+    if (err.code === 'MODULE_NOT_FOUND') {
+      // Console.error a warning message, but normally exit the process to avoid printing ugly npm ERR lines and stack trace.
+      console.error(`\n${repeatCharacter(' ', 22)}⚠️ ERROR ⚠️\n${repeatCharacter('-', 55)}\nYou specified "${process.env.DB_ADAPTER}" as the DB_ADAPTER, but no package\ncalled "${adapterName}" was found.\n\nPlease make sure you spelled the name correctly and\nhave "npm install"ed the necessary adapter package!\n${repeatCharacter('-', 55)}\n`)
+      process.exit(0)
+    }
+  }
+} else {
+  adapter = require('./flat-file-adapter')
 }
 
-module.exports = promisifiedDb
+module.exports = {
+  get: adapter.get,
+  getAll: adapter.getAll,
+  put: adapter.put,
+  has: adapter.has,
+  keys: adapter.keys,
+}

src/flat-file-adapter.js

@@ -0,0 +1,35 @@
+const flatfile = require('flat-file-db')
+const promise = require('promise')
+
+const db = flatfile.sync(process.env.DB_NAME || 'views.db')
+
+module.exports = {
+  put: promise.denodeify(db.put.bind(db)),
+
+  has: (key) => Promise.resolve(db.has(key)),
+  get: (key, options) => {
+    const value = db.get(key) || { views: [] }
+
+    return {
+      views: value.views.filter(view => {
+        if (options && options.before && view.time > options.before) return false
+        if (options && options.after && view.time < options.after) return false
+        return true
+      })
+    }
+  },
+  keys: () => Promise.resolve(db.keys()),
+
+  getAll: async function getAll(options) {
+    const data = {}
+    const keys = await module.exports.keys()
+
+    keys
+      .filter(key => key.startsWith(options.pathname))
+      .forEach((key) => {
+        data[key] = module.exports.get(key, { before: options.before, after: options.after })
+      })
+
+    return data
+  }
+}

src/index.js

@@ -9,15 +9,21 @@ module.exports = async function (req, res) {
   res.setHeader('Access-Control-Allow-Origin', '*')
   // Send all views down if "?all" is true
   if (String(query.all) === 'true') {
-    const data = {
-      data: {},
-      time: Date.now()
+    try {
+      const data = {
+        data: await db.getAll({
+          pathname: pathname,
+          before: parseInt(query.before, 10),
+          after: parseInt(query.after, 10),
+        }),
+        time: Date.now()
+      }
+      send(res, 200, data)
+      return
+    } catch (err) {
+      console.log(err)
+      throw createError(500, 'Internal server error.')
     }
-    for (let key of db.keys().filter(key => String(query.filter) === 'false' ? true : key.startsWith(pathname))) {
-      data.data[key] = db.get(key)
-    }
-    send(res, 200, data)
-    return
   }
   // Check that a page is provided
   if (pathname.length <= 1) {
@@ -28,7 +34,7 @@ module.exports = async function (req, res) {
   }
   const shouldIncrement = String(query.inc) !== 'false'
   try {
-    const currentViews = db.has(pathname) ? db.get(pathname).views.length : 0
+    const currentViews = await db.has(pathname) ? (await db.get(pathname)).views.length : 0
     // Add a view and send the total views back to the client
     if (shouldIncrement) {
       await pushView(pathname, { time: Date.now() })

src/utils.js

@@ -10,8 +10,8 @@ const pushView = async (key, view) => {
     if (locks[key]) return setImmediate(async () => { await push() })
     locks[key] = true
 
-    const views = db.has(key)
-      ? db.get(key).views
+    const views = await db.has(key)
+      ? (await db.get(key)).views
       : []
 
     try {

tests/atomicity.test.js

@@ -1,7 +1,7 @@
 const request = require('request-promise')
 const { listen, mockDb } = require('./utils')
 
-jest.mock('../src/db', () => mockDb)
+jest.mock('flat-file-db', () => mockDb)
 const service = require('../src')
 let url
 

tests/items.test.js

@@ -1,7 +1,7 @@
 const request = require('request-promise')
 const { listen, mockDb } = require('./utils')
 
-jest.mock('../src/db', () => mockDb)
+jest.mock('flat-file-db', () => mockDb)
 const service = require('../src')
 let url
 
@@ -91,12 +91,26 @@ describe('all', () => {
       expect(body.data['/rover2'].views.length).toBe(1)
     })
 
-    it('should not filter if filter is set to false', async () => {
-      await request(`${url}/rover`)
-      await request(`${url}/rover2`)
-      await request(`${url}/route`)
-      const body = JSON.parse(await request(`${url}/rover?all=true&filter=false`))
-      expect(Object.keys(body.data).length).toBe(3)
+    it('should filter based on before after', async () => {
+      const after = new Date('2017-01-01T09:11:00.000Z').getTime()
+      const before = new Date('2017-01-01T09:41:00.000Z').getTime()
+
+      mockDb._put('/rover', { views: [
+        { time: new Date('2017-01-01T09:00:00.000Z').getTime() },
+        { time: new Date('2017-01-01T09:10:00.000Z').getTime() },
+        { time: new Date('2017-01-01T09:20:00.000Z').getTime() },
+        { time: new Date('2017-01-01T09:30:00.000Z').getTime() },
+        { time: new Date('2017-01-01T09:40:00.000Z').getTime() },
+        { time: new Date('2017-01-01T09:50:00.000Z').getTime() },
+      ]})
+
+      const mapToIsoString = view => new Date(view.time).toISOString()
+      const body = JSON.parse(await request(`${url}/rover?all=true&before=${before}&after=${after}`))
+      expect(body.data['/rover'].views.map(mapToIsoString)).toEqual([
+        '2017-01-01T09:20:00.000Z',
+        '2017-01-01T09:30:00.000Z',
+        '2017-01-01T09:40:00.000Z'
+      ])
     })
   })
 })

tests/utils.js

@@ -6,18 +6,22 @@ const DB = () => {
   let DELAY = 1
 
   return {
-    get: (key) => data[key],
-    put: (key, val) => new Promise((res, rej) => {
-      setTimeout(() => {
-        data[key] = val
-        res()
-      }, DELAY)
+    sync: () => ({
+      get: (key) => data[key],
+      put: (key, val, cb) => {
+        setTimeout(() => {
+          data[key] = val
+          cb()
+        }, DELAY)
+      },
+      has: (key) => ({}.hasOwnProperty.call(data, key)),
+      keys: () => Object.keys(data),
     }),
-    has: (key) => !!data[key],
-    keys: () => Object.keys(data),
+
     // Custom methods used in tests
     _reset: () => { data = {} },
-    _setDelay: (ms) => { DELAY = ms || 1 }
+    _setDelay: (ms) => { DELAY = ms || 1 },
+    _put: (key, value) => { data[key] = value }
   }
 }