Periodically prune reviewed and rejected messages (#141)

* Prune processed msgs

Signed-off-by: Miroslav Kovar <[email protected]>
Co-authored-by: Patrik Stas <[email protected]>

Author: mirgee

dbutils/migrations_scheme_application/20220210204408-create-events.js

@@ -0,0 +1,53 @@
+'use strict';
+
+var dbm;
+var type;
+var seed;
+var fs = require('fs');
+var path = require('path');
+var Promise;
+
+/**
+  * We receive the dbmigrate dependency from dbmigrate initially.
+  * This enables us to not have to rely on NODE_PATH.
+  */
+exports.setup = function(options, seedLink) {
+  dbm = options.dbmigrate;
+  type = dbm.dataType;
+  seed = seedLink;
+  Promise = options.Promise;
+};
+
+exports.up = function(db) {
+  var filePath = path.join(__dirname, 'sqls', '20220210204408-create-events-up.sql');
+  return new Promise( function( resolve, reject ) {
+    fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
+      if (err) return reject(err);
+      console.log('received data: ' + data);
+
+      resolve(data);
+    });
+  })
+  .then(function(data) {
+    return db.runSql(data);
+  });
+};
+
+exports.down = function(db) {
+  var filePath = path.join(__dirname, 'sqls', '20220210204408-create-events-down.sql');
+  return new Promise( function( resolve, reject ) {
+    fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
+      if (err) return reject(err);
+      console.log('received data: ' + data);
+
+      resolve(data);
+    });
+  })
+  .then(function(data) {
+    return db.runSql(data);
+  });
+};
+
+exports._meta = {
+  "version": 1
+};

dbutils/migrations_scheme_application/sqls/20220210204408-create-events-down.sql

@@ -0,0 +1 @@
+/* Replace with your SQL commands */

dbutils/migrations_scheme_application/sqls/20220210204408-create-events-up.sql

@@ -0,0 +1,12 @@
+CREATE PROCEDURE prune_msgs()
+BEGIN
+    DELETE LOW_PRIORITY FROM messages WHERE status_code = 'MS-105' OR status_code = 'MS-106';
+END;
+
+
+CREATE EVENT msgs_cleanup
+    ON SCHEDULE EVERY 10 MINUTE
+    ON COMPLETION PRESERVE
+    COMMENT 'Cleanup processed messages (in states MS-105 or MS-106)'
+    DO
+        CALL prune_msgs();

dbutils/src/db-schemas.js

@@ -20,12 +20,13 @@ async function _runSingleCmd(dbConfig, cmd) {
   }
 }
 
-async function runSingleCmd(user, password, host, port, cmd) {
+async function runSingleCmd(user, password, host, port, cmd, database) {
   const dbConfig = {
     host,
     port,
     user,
     password,
+    database,
     multipleStatements: false
   }
   await _runSingleCmd(dbConfig, cmd)
@@ -110,4 +111,5 @@ module.exports = {
   createMysqlDatabase,
   assureMysqlDatabase,
   enableConsoleLogging,
+  runSingleCmd
 }

dbutils/src/db-testutils.js

@@ -18,15 +18,16 @@
 
 const uuid = require('uuid')
 const { migrateSchemaData, migrateSchemaWallet } = require('./migration')
-const { dropMysqlDatabase, createMysqlDatabase } = require('./db-schemas')
+const { dropMysqlDatabase, createMysqlDatabase, runSingleCmd } = require('./db-schemas')
+
+const user = 'root'
+const password = 'mysecretpassword'
+const host = 'localhost'
+const port = 3306
 
 async function createDbSchemaApplication (dbNameId) {
   const dbId = dbNameId || uuid.v4().split('-').join("")
   const database = `agencytest_data_${dbId}`
-  const user = 'root'
-  const password = 'mysecretpassword'
-  const host = 'localhost'
-  const port = 3306
 
   await createMysqlDatabase(user, password, host, port, database)
   await migrateSchemaData(user, password, host, port, database)
@@ -45,10 +46,6 @@ async function createDbSchemaApplication (dbNameId) {
 async function createDbSchemaWallets (dbNameId) {
   const dbId = dbNameId || uuid.v4().split('-').join("")
   const database = `agencytest_wallet_${dbId}`
-  const user = 'root'
-  const password = 'mysecretpassword'
-  const host = 'localhost'
-  const port = 3306
 
   await createMysqlDatabase(user, password, host, port, database)
   await migrateSchemaWallet(user, password, host, port, database)
@@ -64,7 +61,13 @@ async function createDbSchemaWallets (dbNameId) {
   }
 }
 
+async function pruneMsgs (dbName) {
+  const cmd = 'CALL prune_msgs()'
+  await runSingleCmd(user, password, host, port, cmd, dbName)
+}
+
 module.exports = {
   createDbSchemaApplication,
-  createDbSchemaWallets
+  createDbSchemaWallets,
+  pruneMsgs
 }

vcxagency-node/test/unit/messaging/agent-msgs.spec.js

@@ -34,7 +34,7 @@ const uuid = require('uuid')
 const rimraf = require('rimraf')
 const os = require('os')
 const { getBaseAppConfig } = require('./common')
-const { createDbSchemaApplication, createDbSchemaWallets } = require('dbutils')
+const { createDbSchemaApplication, createDbSchemaWallets, pruneMsgs } = require('dbutils')
 const { setupVcxLogging } = require('../../utils')
 const { buildApplication, cleanUpApplication } = require('../../../src/setup/app')
 const { buildAgencyClientVirtual } = require('./common')
@@ -369,24 +369,41 @@ describe('onboarding', () => {
     expect(updated2.uids.length).toBe(1)
     expect(updated2.uids.includes(msg6Id)).toBeTruthy()
 
-    const msgReply = await vcxFlowGetMsgsFromAgent(agencyUserWh, sendToAgency, agent1Did, agent1Verkey, agencyUserVerkey, [aconn1UserPwDid, aconn2UserPwDid], [], [])
-    const { msgsByConns } = msgReply
+    let msgReply = await vcxFlowGetMsgsFromAgent(agencyUserWh, sendToAgency, agent1Did, agent1Verkey, agencyUserVerkey, [aconn1UserPwDid, aconn2UserPwDid], [], [])
+    let { msgsByConns } = msgReply
     expect(Array.isArray(msgsByConns)).toBeTruthy()
     expect(msgsByConns.length).toBe(2)
 
-    const msgsByConn1 = msgsByConns.find(msgsByConn => msgsByConn.pairwiseDID === aconn1UserPwDid)
+    let msgsByConn1 = msgsByConns.find(msgsByConn => msgsByConn.pairwiseDID === aconn1UserPwDid)
     expect(msgsByConn1).toBeDefined()
     expect(msgsByConn1.msgs.length).toBe(3)
     assertMsgsByConHasMessageWithStatus(msgsByConn1, msg1Id, 'MS-106')
     assertMsgsByConHasMessageWithStatus(msgsByConn1, msg2Id, 'MS-106')
     assertMsgsByConHasMessageWithStatus(msgsByConn1, msg3Id, 'MS-104')
 
-    const msgsByConn2 = msgsByConns.find(msgsByConn => msgsByConn.pairwiseDID === aconn2UserPwDid)
+    let msgsByConn2 = msgsByConns.find(msgsByConn => msgsByConn.pairwiseDID === aconn2UserPwDid)
     expect(msgsByConn2).toBeDefined()
     expect(msgsByConn2.msgs.length).toBe(3)
     assertMsgsByConHasMessageWithStatus(msgsByConn2, msg4Id, 'MS-103')
     assertMsgsByConHasMessageWithStatus(msgsByConn2, msg5Id, 'MS-103')
     assertMsgsByConHasMessageWithStatus(msgsByConn2, msg6Id, 'MS-106')
+
+    await pruneMsgs(tmpDbData.info.database)
+    msgReply = await vcxFlowGetMsgsFromAgent(agencyUserWh, sendToAgency, agent1Did, agent1Verkey, agencyUserVerkey, [aconn1UserPwDid, aconn2UserPwDid], [], []);
+    ({ msgsByConns } = msgReply)
+    expect(Array.isArray(msgsByConns)).toBeTruthy()
+    expect(msgsByConns.length).toBe(2)
+
+    msgsByConn1 = msgsByConns.find(msgsByConn => msgsByConn.pairwiseDID === aconn1UserPwDid)
+    expect(msgsByConn1).toBeDefined()
+    expect(msgsByConn1.msgs.length).toBe(1)
+    assertMsgsByConHasMessageWithStatus(msgsByConn1, msg3Id, 'MS-104')
+
+    msgsByConn2 = msgsByConns.find(msgsByConn => msgsByConn.pairwiseDID === aconn2UserPwDid)
+    expect(msgsByConn2).toBeDefined()
+    expect(msgsByConn2.msgs.length).toBe(2)
+    assertMsgsByConHasMessageWithStatus(msgsByConn2, msg4Id, 'MS-103')
+    assertMsgsByConHasMessageWithStatus(msgsByConn2, msg5Id, 'MS-103')
   })
 
   it('should not update any message statusCode of no uids are specified in update request', async () => {