Merge remote-tracking branch 'origin/improvement/BB-728/filter-out-internal-buckets' into w/9.1/improvement/BB-728/filter-out-internal-buckets

Author: fredmnl

extensions/notification/NotificationQueuePopulator.js

@@ -2,7 +2,7 @@ const assert = require('assert');
 const util = require('util');
 
 const { isMasterKey } = require('arsenal').versioning;
-const { usersBucket, mpuBucketPrefix, supportedNotificationEvents } = require('arsenal').constants;
+const { mpuBucketPrefix, supportedNotificationEvents } = require('arsenal').constants;
 const VID_SEPERATOR = require('arsenal').versioning.VersioningConstants.VersionId.Separator;
 const configUtil = require('./utils/config');
 const safeJsonParse = require('./utils/safeJsonParse');
@@ -332,8 +332,13 @@ class NotificationQueuePopulator extends QueuePopulatorExtension {
             this.log.error('could not parse log entry', { value, error });
             return cb();
         }
-        // ignore bucket op, mpu's or if the entry has no bucket
-        if (!bucket || bucket === usersBucket || (key && key.startsWith(mpuBucketPrefix))) {
+        // ignore mpu's or if the entry has no bucket
+        if (!bucket || (key && key.startsWith(mpuBucketPrefix))) {
+            return cb();
+        }
+        // ignore internal buckets
+        if (bucket.includes('..')) {
+            this.log.trace('skipping internal bucket entry', { bucket });
             return cb();
         }
         // bucket notification configuration updates

extensions/replication/ReplicationQueuePopulator.js

@@ -18,9 +18,14 @@ class ReplicationQueuePopulator extends QueuePopulatorExtension {
             // bucket updates have no key in raft log
             return undefined;
         }
+        // users..bucket has a special role for "echo mode"
         if (entry.bucket === usersBucket) {
             return this._filterBucketOp(entry);
         }
+        // internal buckets (other than users..bucket) are ignored
+        if (entry.bucket.includes('..')) {
+            return undefined;
+        }
         return this._filterKeyOp(entry);
     }
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "backbeat",
-  "version": "9.1.2",
+  "version": "9.1.3",
   "description": "Asynchronous queue and job manager",
   "main": "index.js",
   "scripts": {

tests/unit/notification/NotificationQueuePopulator.js

@@ -404,6 +404,26 @@ describe('NotificationQueuePopulator ::', () => {
             });
         });
 
+        it('should ignore internal bucket operations', done => {
+            const processObjectEntryStub = sinon.stub(notificationQueuePopulator, '_processObjectEntry');
+            const processBucketEntryStub = sinon.stub(notificationQueuePopulator, '_processBucketEntry');
+            const entry = {
+                bucket: 'internal..backupIndex',
+                key: 'example-key',
+                type: 'put',
+                value: '{}',
+                overheadFields: {
+                    opTimestamp: new Date().toISOString(),
+                },
+            };
+            notificationQueuePopulator.filterAsync(entry, err => {
+                assert.ifError(err);
+                assert(processObjectEntryStub.notCalled);
+                assert(processBucketEntryStub.notCalled);
+                return done();
+            });
+        });
+
         it('should ignore mpu bucket operations', done => {
             const processObjectEntryStub = sinon.stub(notificationQueuePopulator, '_processObjectEntry');
             const processBucketEntryStub = sinon.stub(notificationQueuePopulator, '_processBucketEntry');

tests/unit/replication/ReplicationQueuePopulator.spec.js

@@ -325,4 +325,35 @@ describe('replication queue populator', () => {
         const publishedMessage = rqp.getState();
         assert(!publishedMessage.key);
     });
+
+    it('should ignore internal buckets except users..bucket', () => {
+        const entry = {
+            bucket: 'internal..backupIndex',
+            key: 'key',
+            value: '{}',
+            type: 'put',
+            logReader: {
+                getMetricLabels: () => {},
+            },
+        };
+        rqp.filter(entry);
+        const publishedMessage = rqp.getState();
+        assert.deepStrictEqual(publishedMessage, {});
+    });
+
+    it('should not ignore users..bucket', () => {
+        const entry = {
+            bucket: 'users..bucket',
+            type: 'put',
+            key: 'owner..|..bucket',
+            value: '{}',
+            logReader: {
+                getMetricLabels: () => {},
+            },
+        };
+        rqp.filter(entry);
+        const publishedMessage = rqp.getState();
+        assert(publishedMessage.key);
+        assert.strictEqual(decodeURIComponent(publishedMessage.key), 'users..bucket');
+    });
 });