feat(medic#9885): adds new meta audit database (medic#9909)

Adds new mechanism to audit document changes, through a new shared library, audit. The library is injected into every call that handles requests (PouchDb fetch and couch-request).
When a monitored API is called (POST to create a doc, PUT to update a doc or _bulk_docs), the response is analyzed and entries are added into a new database medic-audit.
In medic-audit, each document has an audit doc which contains the last 10 entries of received changes. Each entry includes the rev of the change in question, the user that made the change, the request_id (in case this was a direct client request), the service that made the change (api or sentinel) and the date when the change was registered.
When an audit doc has 10 history entries, a rotation mechanism will save the old audit entry in a new doc, and the existing document trail will only contain the new change.

Moves getDeployInfo into its own shared library to avoid circular dependencies.

closes medic#9885

Author: dianabarsan

api/src/controllers/bulk-docs.js

@@ -26,9 +26,7 @@ const invalidRequest = req => {
 };
 
 const interceptResponse = (requestDocs, req, res, response) => {
-  response = JSON.parse(response);
-  const formattedResults = bulkDocs.formatResults(requestDocs, req.body.docs, response);
-  res.json(formattedResults);
+  return bulkDocs.formatResults(requestDocs, req.body.docs, response);
 };
 
 module.exports = {

api/src/db.js

@@ -2,11 +2,13 @@ const PouchDB = require('pouchdb-core');
 const logger = require('@medic/logger');
 const environment = require('@medic/environment');
 const request = require('@medic/couch-request');
+
 PouchDB.plugin(require('pouchdb-adapter-http'));
 PouchDB.plugin(require('pouchdb-session-authentication'));
 PouchDB.plugin(require('pouchdb-find'));
 PouchDB.plugin(require('pouchdb-mapreduce'));
 const asyncLocalStorage = require('./services/async-storage');
+const audit = require('@medic/audit');
 const { REQUEST_ID_HEADER } = require('./server-utils');
 
 const { UNIT_TEST_ENV } = process.env;
@@ -73,31 +75,37 @@ if (UNIT_TEST_ENV) {
     module.exports[fn] = () => notStubbed(fn);
   });
 } else {
-  const fetch = (url, opts) => {
+  const service = 'api';
+  environment.setService(service);
+
+  const fetchFn = (url, opts) => {
     // Adding audit flag (haproxy) Service that made the request initially.
-    opts.headers.set('X-Medic-Service', 'api');
-    const requestId = asyncLocalStorage.getRequestId();
-    if (requestId) {
-      opts.headers.set(REQUEST_ID_HEADER, requestId);
+    opts.headers.set('X-Medic-Service', service);
+    const requestMetadata = asyncLocalStorage.getRequest();
+    if (requestMetadata.requestId) {
+      opts.headers.set(REQUEST_ID_HEADER, requestMetadata.requestId);
     }
-    return PouchDB.fetch(url, opts);
+    return PouchDB.fetch(url, opts).then(response => {
+      void audit.fetchCallback(url, opts, response, requestMetadata);
+      return response;
+    });
   };
 
-  const DB = new PouchDB(environment.couchUrl, { fetch });
+  const DB = new PouchDB(environment.couchUrl, { fetch: fetchFn });
   const getDbUrl = name => `${environment.serverUrl}/${name}`;
 
   DB.setMaxListeners(0);
   module.exports.medic = DB;
-  module.exports.medicUsersMeta = new PouchDB(`${environment.couchUrl}-users-meta`, { fetch });
-  module.exports.medicLogs = new PouchDB(`${environment.couchUrl}-logs`, { fetch });
-  module.exports.sentinel = new PouchDB(`${environment.couchUrl}-sentinel`, { fetch });
-  module.exports.vault = new PouchDB(`${environment.couchUrl}-vault`, { fetch });
+  module.exports.medicUsersMeta = new PouchDB(`${environment.couchUrl}-users-meta`, { fetch: fetchFn });
+  module.exports.medicLogs = new PouchDB(`${environment.couchUrl}-logs`, { fetch: fetchFn });
+  module.exports.sentinel = new PouchDB(`${environment.couchUrl}-sentinel`, { fetch: fetchFn });
+  module.exports.vault = new PouchDB(`${environment.couchUrl}-vault`, { fetch: fetchFn });
   module.exports.createVault = () => module.exports.vault.info();
-  module.exports.users = new PouchDB(getDbUrl('_users'), { fetch });
+  module.exports.users = new PouchDB(getDbUrl('_users'), { fetch: fetchFn });
   module.exports.builds = new PouchDB(environment.buildsUrl);
 
   // Get the DB with the given name
-  module.exports.get = name => new PouchDB(getDbUrl(name), { fetch });
+  module.exports.get = name => new PouchDB(getDbUrl(name), { fetch: fetchFn });
   module.exports.close = db => {
     if (!db || db._destroyed || db._closed) {
       return;
@@ -112,7 +120,7 @@ if (UNIT_TEST_ENV) {
 
   // Resolves with the PouchDB object if the DB with the given name exists
   module.exports.exists = name => {
-    const db = new PouchDB(getDbUrl(name), { skip_setup: true, fetch });
+    const db = new PouchDB(getDbUrl(name), { skip_setup: true, fetch: fetchFn });
     return db
       .info()
       .then(result => {

api/src/routing.js

@@ -13,6 +13,7 @@ const auth = require('./auth');
 const prometheusMiddleware = require('prometheus-api-metrics');
 const rateLimiterMiddleware = require('./middleware/rate-limiter');
 const logger = require('@medic/logger');
+const audit = require('@medic/audit');
 const isClientHuman = require('./is-client-human');
 
 const port = typeof environment.port !== 'undefined' && environment.port !== null ? `:${environment.port}` : '';
@@ -849,12 +850,12 @@ const canEdit = (req, res) => {
     .check(req, 'can_edit')
     .then(userCtx => {
       if (!userCtx || !userCtx.name) {
-        serverUtils.serverError('not-authorized', req, res);
+        serverUtils.serverError({ code: 401, message: 'not-authorized' }, req, res);
         return;
       }
       proxyForAuth.web(req, res);
     })
-    .catch(() => serverUtils.serverError('not-authorized', req, res));
+    .catch((err) => serverUtils.error(err, req, res));
 };
 
 const editPath = routePrefix + '*';
@@ -895,14 +896,18 @@ proxyForAuth.on('proxyRes', (proxyRes, req, res) => {
   }
 
   copyProxyHeaders(proxyRes, res);
+  let body = Buffer.from('');
+  proxyRes.on('data', data => (body = Buffer.concat([body, data])));
 
-  if (res.interceptResponse) {
-    let body = Buffer.from('');
-    proxyRes.on('data', data => (body = Buffer.concat([body, data])));
-    proxyRes.on('end', () => res.interceptResponse(req, res, body.toString()));
-  } else {
-    proxyRes.pipe(res);
-  }
+  proxyRes.on('end', () => {
+    body = JSON.parse(body.toString());
+    if (res.interceptResponse) {
+      body = res.interceptResponse(req, res, body);
+    }
+    res.json(body);
+
+    audit.expressCallback(req, body, asyncLocalStorage.getRequest());
+  });
 });
 
 proxyForAuth.on('proxyRes', infodoc.update);

api/src/services/async-storage.js

@@ -12,6 +12,13 @@ module.exports = {
     const localStorage = asyncLocalStorage.getStore();
     return localStorage?.clientRequest?.id;
   },
+  getRequest: () => {
+    const localStorage = asyncLocalStorage.getStore();
+    return {
+      user: localStorage?.clientRequest?.userCtx?.name,
+      requestId: localStorage?.clientRequest?.id,
+    };
+  }
 };
 
-request.initialize(module.exports, REQUEST_ID_HEADER);
+request.setStore(module.exports, REQUEST_ID_HEADER);

api/src/services/deploy-info.js

@@ -1,12 +1,12 @@
-const environment = require('@medic/environment');
+const serverInfo = require('@medic/server-info');
 const fs = require('fs');
 const path = require('path');
 const resources = require('../resources');
 
 const webappPath = resources.webappPath;
 const DEPLOY_INFO_OUTPUT_PATH = path.join(webappPath, 'deploy-info.json');
 
-const getDeployInfo = () => environment.getDeployInfo();
+const getDeployInfo = () => serverInfo.getDeployInfo();
 
 const store = async () => {
   const deployInfo = await getDeployInfo();

api/src/services/setup/upgrade-steps.js

@@ -3,7 +3,7 @@ const viewIndexerProgress = require('./view-indexer-progress');
 const upgradeLogService = require('./upgrade-log');
 const viewIndexer = require('./view-indexer');
 const logger = require('@medic/logger');
-const environment = require('@medic/environment');
+const serverInfo = require('@medic/server-info');
 const startupLog = require('./startup-log');
 
 /**
@@ -21,7 +21,7 @@ const finalize = async () => {
   await upgradeUtils.deleteStagedDdocs();
   await upgradeLogService.setFinalized();
   await upgradeUtils.cleanup();
-  await environment.getDeployInfo(true);
+  await serverInfo.getDeployInfo(true);
 };
 
 /**

api/tests/mocha/controllers/bulk-docs.spec.js

@@ -81,16 +81,16 @@ describe('Bulk Docs controller', () => {
           docs: ['filteredDocs'],
           new_edits: 'something'
         };
-        controller._interceptResponse(['requestDocs'], testReq, testRes, JSON.stringify(['results']));
+        const newResponse = controller._interceptResponse(['requestDocs'], testReq, testRes, ['results']);
         service.formatResults.callCount.should.equal(1);
         service.formatResults.args[0].should.deep.equal([
           ['requestDocs'],
           ['filteredDocs'],
           ['results']
         ]);
 
-        testRes.json.callCount.should.equal(1);
-        testRes.json.args[0].should.deep.equal([['formatted', 'results']]);
+        newResponse.should.deep.equal(['formatted', 'results']);
+        testRes.json.callCount.should.equal(0);
       });
     });
 

api/tests/mocha/db.spec.js

@@ -413,7 +413,7 @@ describe('db', () => {
         json: sinon.stub().resolves({ result: true }),
         ok: true,
       });
-      sinon.stub(asyncLocalStorage, 'getRequestId').returns('the_id');
+      sinon.stub(asyncLocalStorage, 'getRequest').returns({ requestId: 'the_id' });
       db = rewire('../../src/db');
 
       await db.medic.info();

api/tests/mocha/services/async-storage.spec.js

@@ -8,7 +8,7 @@ describe('async-storage', () => {
   let service;
 
   beforeEach(() => {
-    sinon.stub(request, 'initialize');
+    sinon.stub(request, 'setStore');
   });
 
   afterEach(() => {
@@ -17,7 +17,7 @@ describe('async-storage', () => {
 
   it('should initialize async storage and initialize couch-request', async () => {
     service = rewire('../../../src/services/async-storage');
-    expect(request.initialize.args).to.deep.equal([[
+    expect(request.setStore.args).to.deep.equal([[
       service,
       serverUtils.REQUEST_ID_HEADER
     ]]);

api/tests/mocha/services/deploy-info.spec.js

@@ -4,7 +4,7 @@ const rewire = require('rewire');
 
 const fs = require('fs');
 
-const environment = require('@medic/environment');
+const serverInfo = require('@medic/server-info');
 const resources = require('../../../src/resources');
 
 let service;
@@ -29,25 +29,25 @@ describe('deploy info', () => {
         time: 'human readable',
       };
 
-      sinon.stub(environment, 'getDeployInfo').resolves(deployInfo);
+      sinon.stub(serverInfo, 'getDeployInfo').resolves(deployInfo);
 
       const result = await service.get();
 
       expect(result).to.deep.equal(deployInfo);
-      expect(environment.getDeployInfo.callCount).to.equal(1);
+      expect(serverInfo.getDeployInfo.callCount).to.equal(1);
     });
 
     it('should work with undefined deploy info and build info', async () => {
       const deployInfo = { version: '20000' };
 
-      sinon.stub(environment, 'getDeployInfo').resolves(deployInfo);
+      sinon.stub(serverInfo, 'getDeployInfo').resolves(deployInfo);
 
       expect(await service.get()).to.deep.equal(deployInfo);
     });
 
     it('should throw error when getDeployInfo fails', async () => {
       const error = { error: 'whatever' };
-      sinon.stub(environment, 'getDeployInfo').rejects(error);
+      sinon.stub(serverInfo, 'getDeployInfo').rejects(error);
 
       try {
         await service.get();
@@ -71,7 +71,7 @@ describe('deploy info', () => {
         version: '4.10.0-beta.1',
       };
 
-      sinon.stub(environment, 'getDeployInfo').resolves(deployInfo);
+      sinon.stub(serverInfo, 'getDeployInfo').resolves(deployInfo);
 
       await service.store();