TTN MQTT & device management (#8)

config.js

@@ -13,4 +13,16 @@ module.exports = {
    * trace: show intermediate decoding results
    */
   loglevel: e['TTN_OSEM_loglevel'] || 'info',
+
+  /**
+   * commaseparated list of auth keys that are allowed to request
+   * authenticated routes when sent in the 'authorization' header
+   * eg GET /v1.1/ttndevices/:boxId
+   */
+  authTokens: e['TTN_OSEM_authtoken'] ? e['TTN_OSEM_authtoken'].split(',') : [],
+
+  ttn: {
+    appId: e['TTN_OSEM_ttn_app'],
+    key: e['TTN_OSEM_ttn_key'], // the key requires full rights (settings, devices, messages)
+  },
 };

lib/database.js

@@ -0,0 +1,46 @@
+'use strict';
+
+const { Box } = require('openSenseMapAPI').models;
+const { BoxNotFoundError } = require('./errors');
+
+/**
+ * look up a Box in the database for a given ttn configuration
+ * @param {string} app_id
+ * @param {string} dev_id
+ * @param {string} [port]
+ */
+const boxFromDevId = function boxFromDevId (app_id, dev_id, port) {
+  return Box.find({
+    'integrations.ttn.app_id': app_id,
+    'integrations.ttn.dev_id': dev_id
+  })
+    .then(boxes => {
+      if (!boxes.length) {
+        throw new BoxNotFoundError(`for dev_id '${dev_id}' and app_id '${app_id}'`);
+      }
+
+      // filter the boxes by their configured port.
+      // also include boxes with undefined port.
+      const box = boxes.filter(box => {
+        const p = box.integrations.ttn.port;
+
+        return (p === port || p === undefined);
+      })[0];
+
+      if (!box) {
+        throw new BoxNotFoundError(`for port ${port}`);
+      }
+
+      return box;
+    });
+};
+
+const boxFromBoxId = function boxFromBoxId (id) {
+  return Box.findBoxById(id, { populate: false })
+    .catch(err => Promise.reject(new BoxNotFoundError(err.message)));
+};
+
+module.exports = {
+  boxFromBoxId,
+  boxFromDevId,
+};

lib/decoding/debug.js

@@ -11,6 +11,7 @@
  */
 
 const { bytesToInt } = require('./helpers');
+const { DecodingError } = require('../errors');
 
 /**
  * returns a bufferTransfomer for transformation of a buffer to measurements.
@@ -29,11 +30,11 @@ const createBufferTransformer = function createBufferTransformer (box) {
     transformer = [];
 
   if (!byteMask) {
-    throw new Error('profile \'debug\' requires a valid byteMask');
+    throw new DecodingError('box requires a valid byteMask', 'debug');
   }
 
   if (box.sensors.length < byteMask.length) {
-    throw new Error(`box requires at least ${byteMask.length} sensors`);
+    throw new DecodingError(`box requires at least ${byteMask.length} sensors`, 'debug');
   }
 
   for (let i = 0; i < byteMask.length; i++) {

lib/decoding/index.js

@@ -12,6 +12,7 @@
 
 const { transformAndValidateArray, json } = require('openSenseMapAPI').decoding;
 const { createLogger } = require('../logging');
+const { DecodingError } = require('../errors');
 
 const profiles = {
   /* eslint-disable global-require */
@@ -50,7 +51,7 @@ const bufferToMeasurements = function bufferToMeasurements (buffer, bufferTransf
   }
 
   if (maskLength !== buffer.length) {
-    throw new Error(`incorrect amount of bytes: got ${buffer.length}, should be ${maskLength}`);
+    throw new DecodingError(`incorrect amount of bytes: got ${buffer.length}, should be ${maskLength}`);
   }
 
   // feed each bufferTransformer element
@@ -88,18 +89,18 @@ const decodeBuffer = function decodeBuffer (buffer, box, timestamp) {
   return Promise.resolve().then(function () {
     // should never be thrown, as we find a box by it's ttn config
     if (!buffer.length) {
-      throw new Error('payload may not be empty');
+      throw new DecodingError('payload may not be empty');
     }
 
     if (!box.integrations || !box.integrations.ttn) {
-      throw new Error('box has no TTN configuration');
+      throw new DecodingError('box has no TTN configuration');
     }
 
     // select bufferTransformer according to profile
     const profile = profiles[box.integrations.ttn.profile];
 
     if (!profile) {
-      throw new Error(`profile '${box.integrations.ttn.profile}' is not supported`);
+      throw new DecodingError(`profile '${box.integrations.ttn.profile}' is not supported`);
     }
 
     const bufferTransformer = profile.createBufferTransformer(box);
@@ -149,32 +150,33 @@ const decodeJSON = json.decodeMessage;
  * decodes the request payload in `req.body` to validated measurements,
  * selects the decoder from `req.box` and applies `req.time`
  * @see module:decoding~decodeBuffer
- * @param {Request} req
+ * @param {TTNUplink} payload A valid uplink payload object as received from TTN
+ * @param {Box} box The senseBox to match the decoding
+ * @param {string} [time] An ISODate as fallback for message timestamps
  * @return {Promise} Once fulfilled returns a validated array of measurements
  */
-const decodeRequest = function decodeRequest (req) {
+const decodeRequest = function decodeRequest (payload, box, time) {
   // extract time from payload: if available use time from
   // gateway, else use TTN API time, else use local request time
-  let time = req.time.toISOString();
   let timeSource = 'local';
-  if (req.body.metadata) {
-    time = req.body.metadata.time;
+  if (payload.metadata) {
+    time = payload.metadata.time;
     timeSource = 'TTN api';
-    if (req.body.metadata.gateways && req.body.metadata.gateways[0].time) {
-      time = req.body.metadata.gateways[0].time;
+    if (payload.metadata.gateways && payload.metadata.gateways[0].time) {
+      time = payload.metadata.gateways[0].time;
       timeSource = 'gateway';
     }
   }
 
-  log.trace(`using ${timeSource} time & ${req.box.integrations.ttn.profile} decoder`);
+  log.trace(`using ${timeSource} time & ${box.integrations.ttn.profile} decoder`);
 
-  if (req.box.integrations.ttn.profile === 'json') {
-    return req.body.payload_fields
-      ? decodeJSON(req.body.payload_fields)
-      : Promise.reject('no payload for profile `json` provided');
+  if (box.integrations.ttn.profile === 'json') {
+    return payload.payload_fields
+      ? decodeJSON(payload.payload_fields)
+      : Promise.reject(new DecodingError('no payload for profile `json` provided'));
   }
 
-  return decodeBase64(req.body.payload_raw, req.box, time);
+  return decodeBase64(payload.payload_raw, box, time);
 };
 
 module.exports = {

lib/decoding/lora-serialization.js

@@ -8,6 +8,7 @@
 
 const loraSerialization = require('lora-serialization').decoder,
   { createLogger } = require('../logging'),
+  { LoraError } = require('../errors'),
   { findSensorIds, applyValueFromMeasurement } = require('./helpers');
 
 const log = createLogger('decoder/lora-serialization');
@@ -41,7 +42,7 @@ const createBufferTransformer = function createBufferTransformer (box) {
     byteMask.constructor !== Array ||
     !byteMask.every(opts => typeof opts === 'object')
   ) {
-    throw new Error('profile \'lora-serialization\' requires valid decodeOptions');
+    throw new LoraError('profile \'lora-serialization\' requires valid decodeOptions');
   }
 
   let expectedSensorCount = byteMask.length;
@@ -67,7 +68,7 @@ const createBufferTransformer = function createBufferTransformer (box) {
     if (['unixtime', 'latLng'].includes(el.decoder)) {
       expectedSensorCount--;
     } else if (!Object.keys(match).length) {
-      throw new Error('invalid decodeOptions. requires at least one of [sensor_id, sensor_title, sensor_type]');
+      throw new LoraError('invalid decodeOptions. requires at least one of [sensor_id, sensor_title, sensor_type]');
     } else {
       sensorMatchings.push(match);
     }
@@ -82,7 +83,7 @@ const createBufferTransformer = function createBufferTransformer (box) {
   }, 'matched sensors');
 
   if (Object.keys(sensorIds).length !== expectedSensorCount) {
-    throw new Error('box does not contain sensors mentioned in byteMask');
+    throw new LoraError('box does not contain sensors mentioned in byteMask');
   }
 
   // create the transformer elements for each measurement.
@@ -95,7 +96,7 @@ const createBufferTransformer = function createBufferTransformer (box) {
       typeof transformer !== 'function' ||
       byteMask[i].decoder === 'decode'
     ) {
-      throw new Error(`'${byteMask[i].decoder}' is not a supported transformer`);
+      throw new LoraError(`'${byteMask[i].decoder}' is not a supported transformer`);
     }
 
     const mask = {

lib/decoding/sensebox_home.js

@@ -9,6 +9,7 @@
  */
 
 const { bytesToInt, findSensorIds } = require('./helpers');
+const { DecodingError } = require('../errors');
 
 // alternative titles recognized for the sensors
 const sensorMatchings = {
@@ -39,7 +40,7 @@ const createBufferTransformer = function createBufferTransformer (box) {
   const sensorMap = findSensorIds(box.sensors, sensorMatchings);
 
   if (Object.keys(sensorMap).length !== Object.keys(sensorMatchings).length) {
-    throw new Error('box does not contain valid sensors for this profile');
+    throw new DecodingError('box does not contain valid sensors for this profile', 'sensebox/home');
   }
 
   const transformer = [

lib/errors.js

@@ -0,0 +1,43 @@
+'use strict';
+
+/**
+ * All errors handled by the app must inherit from TTNError
+ * and should be defined here for easier discovery.
+ */
+class TTNError extends Error { }
+
+class BoxNotFoundError extends TTNError {
+  constructor (message) {
+    super(`No box found ${message}`);
+    this.code = 404;
+  }
+}
+
+class PayloadError extends TTNError {
+  constructor (message) {
+    super(`Invalid payload: ${message}`);
+    this.code = 422;
+  }
+}
+
+class DecodingError extends PayloadError {
+  constructor (message, decoder = 'generic') {
+    super(`${message} (${decoder} decoder)`);
+    this.component = decoder;
+  }
+}
+
+class LoraError extends DecodingError {
+  constructor (message) {
+    super(message, 'lora-serialization');
+  }
+}
+
+
+module.exports = {
+  TTNError,
+  BoxNotFoundError,
+  PayloadError,
+  DecodingError,
+  LoraError,
+};

lib/index.js

@@ -5,14 +5,15 @@ const express = require('express'),
   server = express(),
   { connect, mongoose } = require('openSenseMapAPI').db,
   cfg = require('../config'),
+  { getApp, getMqtt } = require('./ttn'),
   { createLogger } = require('./logging'),
   v11Router = require('./routes/v1.1');
 
-const log = createLogger('server');
+const httpLog = createLogger('http');
 
 server.use(function reqLogger (req, res, next) {
   req.time = new Date();
-  log.debug({ req }, `${req.method} ${req.url} from ${req.ip}`);
+  httpLog.debug({ req }, `${req.method} ${req.url} from ${req.ip}`);
   next();
 });
 
@@ -24,8 +25,14 @@ server.use('/v1.1', v11Router);
 
 const msg = `404 Not Found. Available routes:
 
-POST  /v1.1    webhook for messages from the TTN HTTP Integration
-               payload format: https://www.thethingsnetwork.org/docs/applications/http/
+POST  /v1.1
+      webhook for messages from the TTN HTTP Integration
+      payload format: https://www.thethingsnetwork.org/docs/applications/http/
+
+GET   /1.1/ttndevice/:boxId
+      get the TTN device for a given box. must be called once to enable the feature
+      for a box. (alternative to the webhook feature)
+      requires auth.
 `;
 
 server.use(function notFoundHandler (req, res, next) {
@@ -38,14 +45,20 @@ server.use(function notFoundHandler (req, res, next) {
   next();
 });
 
-// launch server once connected to DB
+// launch server & connect to TTN once connected to DB
 mongoose.set('debug', false);
-connect().then(function onDBConnection () {
-  server.listen(cfg.port, (err) => {
-    if (!err) {
-      log.info(`server listening on port ${cfg.port}`);
-    } else {
-      log.error(err);
-    }
-  });
-});
+connect()
+  .then(function onDBConnection () {
+    server.listen(cfg.port, (err) => {
+      if (!err) {
+        httpLog.info(`HTTP API listening on port ${cfg.port}`);
+      } else {
+        httpLog.error(err);
+      }
+    });
+
+    // app doesnt have to be initialized at startup,
+    // but makes first request faster & catches errors earlier
+    return Promise.all([getApp(), getMqtt()]);
+  })
+  .catch(httpLog.fatal);

lib/logging.js

@@ -8,16 +8,26 @@ const logger = bunyan.createLogger({
   level: cfg.loglevel,
   serializers: {
     req: bunyan.stdSerializers.req,
-    res: bunyan.stdSerializers.res,
     err: bunyan.stdSerializers.err,
+    box (box) {
+      return box
+        ? { id: box._id, sensors: box.sensors, ttn: box.integrations.ttn }
+        : {};
+    },
+    res (res) {
+      const { responseTime, result: { code } } = res.locals;
+
+      return { responseTime, code };
+    },
   }
 });
 
 /**
  * Creates a bunyan logger that inherits some global settings
  * @private
  * @param {string} component name for the logger, stored in the logfield `component`
- * @param {*} options bunyan options to be passed to the logger
+ * @param {*} [options] bunyan options to be passed to the logger
+ * @return a bunyan logger
  */
 const createLogger = function createLogger (component, options) {
   return logger.child(Object.assign({ component }, options));