Feat download by grouptag (#660)

* download measurements by grouptag

* allow multiple grouptags and clean code

* tests and docs for download by grouptag


Co-authored-by: Matthias Pfeil <[email protected]>

Author: umut0

docker-compose.yml

@@ -15,4 +15,5 @@ services:
       # - ./dumps/measurements:/exports/measurements
       - ./.scripts/mongodb/osem_admin.sh:/docker-entrypoint-initdb.d/osem_admin.sh
       # - ./.scripts/mongodb/osem_seed_boxes.sh:/docker-entrypoint-initdb.d/osem_seed_boxes.sh
-      # - ./.scripts/mongodb/osem_seed_measurements.sh:/docker-entrypoint-initdb.d/osem_seed_measurements.sh
+      # - ./.scripts/mongodb/osem_seed_measurements.sh:/docker-entrypoint-initdb.d/osem_seed_measurements.sh
+      

packages/api/lib/controllers/measurementsController.js

@@ -233,6 +233,46 @@ const getDataMulti = async function getDataMulti (req, res, next) {
   }
 };
 
+
+/**
+ * @api {get,post} /boxes/data?grouptag=:grouptag Get latest measurements for a grouptag as JSON
+ * @apiDescription Download data of a given grouptag from multiple senseBoxes as JSON
+ * @apiGroup Measurements
+ * @apiName getDataByGroupTag
+ * @apiParam {String} grouptag The grouptag to search by.
+ */
+const getDataByGroupTag = async function getDataByGroupTag (req, res, next) {
+  const { grouptag, format } = req._userParams;
+  const queryTags = grouptag.split(',');
+  // build query
+  const queryParams = {};
+  if (grouptag) {
+    queryParams['grouptag'] = { '$all': queryTags };
+  }
+
+  try {
+    let stream = await Box.findMeasurementsOfBoxesByTagStream({
+      query: queryParams
+    });
+    stream = stream
+      .on('error', function (err) {
+        return handleError(err, next);
+      });
+    switch (format) {
+    case 'json':
+      res.header('Content-Type', 'application/json');
+      stream = stream
+        .pipe(jsonstringify({ open: '[', close: ']' }));
+      break;
+    }
+
+    stream
+      .pipe(res);
+  } catch (err) {
+    handleError(err, next);
+  }
+};
+
 /**
  * @api {post} /boxes/:senseBoxId/:sensorId Post new measurement
  * @apiDescription Posts a new measurement to a specific sensor of a box.
@@ -433,6 +473,13 @@ module.exports = {
     validateFromToTimeParams,
     getDataMulti
   ],
+  getDataByGroupTag: [
+    retrieveParameters([
+      { name: 'grouptag', required: true },
+      { name: 'format', defaultValue: 'json', allowedValues: ['json'] }
+    ]),
+    getDataByGroupTag
+  ],
   getLatestMeasurements: [
     retrieveParameters([
       { predef: 'boxId', required: true },

packages/api/lib/routes.js

@@ -78,6 +78,7 @@ const routes = {
     { path: `${statisticsPath}/descriptive`, method: 'get', handler: statisticsController.descriptiveStatisticsHandler, reference: 'api-Statistics-descriptive' },
     { path: `${boxesPath}`, method: 'get', handler: boxesController.getBoxes, reference: 'api-Boxes-getBoxes' },
     { path: `${boxesPath}/data`, method: 'get', handler: measurementsController.getDataMulti, reference: 'api-Measurements-getDataMulti' },
+    { path: `${boxesPath}/data/bytag`, method: 'get', handler: measurementsController.getDataByGroupTag, reference: 'api-Measurements-getDataByGroupTag' },
     { path: `${boxesPath}/:boxId`, method: 'get', handler: boxesController.getBox, reference: 'api-Boxes-getBox' },
     { path: `${boxesPath}/:boxId/sensors`, method: 'get', handler: measurementsController.getLatestMeasurements, reference: 'api-Measurements-getLatestMeasurements' },
     { path: `${boxesPath}/:boxId/sensors/:sensorId`, method: 'get', handler: measurementsController.getLatestMeasurements, reference: 'api-Measurements-getLatestMeasurementOfSensor' },

packages/models/src/box/box.js

@@ -767,6 +767,46 @@ boxSchema.statics.findMeasurementsOfBoxesStream = function findMeasurementsOfBox
     });
 };
 
+boxSchema.statics.findMeasurementsOfBoxesByTagStream = function findMeasurementsOfBoxesByTagStream (opts) {
+  const { query } = opts;
+
+  return this.find(query, BOX_PROPS_FOR_POPULATION)
+    .lean()
+    .then(function (boxData) {
+      if (boxData.length === 0) {
+        throw new ModelError('No senseBoxes found', { type: 'NotFoundError' });
+      }
+
+      const sensors = Object.create(null);
+      // store all matching sensors under sensors[sensorId]
+      for (let i = 0, len = boxData.length; i < len; i++) {
+        for (let j = 0, sensorslen = boxData[i].sensors.length; j < sensorslen; j++) {
+          const sensor = boxData[i].sensors[j];
+
+          sensor.boxId = boxData[i]._id.toString();
+          sensor.sensorId = sensor._id.toString();
+
+          sensors[boxData[i].sensors[j]['_id']] = sensor;
+        }
+      }
+
+      // // construct a stream transformer applied to queried measurements
+      // // that augments each measure with queried columns (location, ...)
+      // // and applies transformations to timestamps
+      const transformer = measurementTransformer(['sensorId', 'boxId'], sensors, undefined);
+
+      const measureQuery = {
+        'sensor_id': { '$in': Object.keys(sensors) },
+        // 'createdAt': { '$gt': from, '$lt': to }
+      };
+
+      return Measurement.find(measureQuery, { 'createdAt': 1, 'value': 1, 'location': 1, '_id': 0, 'sensor_id': 1 })
+        .cursor({ lean: true, order: 1 })
+        .map(transformer);
+    });
+};
+
+
 // try to add sensors defined in addons to the box. If the sensors already exist,
 // nothing is done.
 boxSchema.methods.addAddon = function addAddon (addon) {

tests/tests/005-create-boxes-test.js

@@ -595,6 +595,41 @@ describe('openSenseMap API Routes: /boxes', function () {
         return chakram.wait();
       });
   });
+  it('should allow to update the box via PUT with array as grouptags', function () {
+    const update_payload = { name: 'neuername', exposure: 'outdoor', grouptag: ['newgroup'], description: 'total neue beschreibung', location: { lat: 54.2, lng: 21.1 }, weblink: 'http://www.google.de', image: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII=' };
+
+    return chakram.put(`${BASE_URL}/boxes/${boxIds[2]}`, update_payload, { headers: { 'Authorization': `Bearer ${jwt2}` } })
+      .then(function (response) {
+        expect(response).to.have.status(200);
+        expect(response).to.comprise.of.json('data.name', update_payload.name);
+        expect(response).to.comprise.of.json('data.exposure', update_payload.exposure);
+        expect(response.body.data.grouptag).to.have.same.members(update_payload.grouptag);
+        expect(response).to.comprise.of.json('data.description', update_payload.description);
+        expect(response).to.comprise.of.json('data.currentLocation', {
+          type: 'Point',
+          coordinates: [update_payload.location.lng, update_payload.location.lat]
+        });
+
+        // loc field with old schema (backwards compat):
+        expect(response).to.comprise.of.json('data.loc', [{
+          type: 'Feature',
+          geometry: {
+            type: 'Point',
+            coordinates: [
+              update_payload.location.lng,
+              update_payload.location.lat
+            ]
+          }
+        }]);
+
+        // image should contain timestamp. request duration should be less than 1s.
+        expect(response).to.comprise.of.json('data.image', function (image) {
+          return expect(moment().diff(moment(parseInt(image.split('_')[1].slice(0, -4), 36) * 1000))).to.be.below(1000);
+        });
+
+        return chakram.wait();
+      });
+  });
 
   it('should deny to update a box of other users', function () {
     let otherJwt, otherBoxId;
@@ -677,7 +712,7 @@ describe('openSenseMap API Routes: /boxes', function () {
         expect(response).to.have.status(200);
         expect(response).to.have.header('content-type', 'application/json; charset=utf-8');
         expect(Array.isArray(response.body)).to.be.true;
-        expect(response.body.length).to.be.equal(1);
+        expect(response.body.length).to.be.equal(2);
 
         return chakram.wait();
       });

tests/tests/007-download-data-test.js

@@ -584,4 +584,20 @@ describe('downloading data', function () {
 
   });
 
+  describe('/boxes/data/bytag?grouptag=newgroup', function () {
+
+    const expectedMeasurementsCount = 40;
+
+    it('should allow download data by Grouptag /boxes/data/bytag=newgroup as json', function () {
+      return chakram.get(`${BASE_URL}/boxes/data/bytag?grouptag=newgroup`)
+        .then(function (response) {
+          expect(response).to.have.status(200);
+          expect(response.body.length).to.be.equal(expectedMeasurementsCount);
+          expect(response).to.have.header('content-type', 'application/json');
+
+          return chakram.wait();
+        });
+    });
+  });
+
 });