add support for reserved bucket tags

Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com>

fix put_bucket_tagging test - pass dummy objectsdk

Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com>

Author: tangledbytes

config.js

@@ -956,6 +956,31 @@ config.NSFS_GLACIER_FORCE_EXPIRE_ON_GET = false;
 // interval
 config.NSFS_GLACIER_MIGRATE_LOG_THRESHOLD = 50 * 1024;
 
+/** 
+ * NSFS_GLACIER_RESERVED_BUCKET_TAGS defines an object of bucket tags which will be reserved
+ * by the system and PUT operations for them via S3 API would be disallowed even by the bucket
+ * owners
+ * 
+ * @type {Record<string, {
+ *  schema: Record<any, any> & { $id: string },
+ *  immutable: true | false | 'if-data',
+ *  default: any
+ * }>}
+ * 
+ * @example
+ * {
+    'deep-archive-copies': {
+        schema: {
+            $id: 'deep-archive-copies-schema-v0',
+            enum: ['1', '2']
+        }, // JSON Schema
+        immutable: 'if-data',
+        default: '1',
+    }
+ * }
+ */
+config.NSFS_GLACIER_RESERVED_BUCKET_TAGS = {};
+
 // anonymous account name
 config.ANONYMOUS_ACCOUNT_NAME = 'anonymous';
 

src/cmd/manage_nsfs.js

@@ -39,6 +39,7 @@ const { throw_cli_error, get_bucket_owner_account_by_name,
 const manage_nsfs_validations = require('../manage_nsfs/manage_nsfs_validations');
 const nc_mkm = require('../manage_nsfs/nc_master_key_manager').get_instance();
 const notifications_util = require('../util/notifications_util');
+const BucketSpaceFS = require('../sdk/bucketspace_fs');
 
 ///////////////
 //// GENERAL //
@@ -135,7 +136,6 @@ async function fetch_bucket_data(action, user_input) {
         force_md5_etag: user_input.force_md5_etag === undefined || user_input.force_md5_etag === '' ? user_input.force_md5_etag : get_boolean_or_string_value(user_input.force_md5_etag),
         notifications: user_input.notifications
     };
-
     if (user_input.bucket_policy !== undefined) {
         if (typeof user_input.bucket_policy === 'string') {
             // bucket_policy deletion specified with empty string ''
@@ -154,6 +154,15 @@ async function fetch_bucket_data(action, user_input) {
         data = await merge_new_and_existing_config_data(data);
     }
 
+    if ((action === ACTIONS.UPDATE && user_input.tag) || (action === ACTIONS.ADD)) {
+        const tags = JSON.parse(user_input.tag || '[]');
+        data.tag = BucketSpaceFS._merge_reserved_tags(
+            data.tag || BucketSpaceFS._default_bucket_tags(),
+            tags,
+            action === ACTIONS.ADD ? true : await _is_bucket_empty(data),
+        );
+    }
+
     //if we're updating the owner, needs to override owner in file with the owner from user input.
     //if we're adding a bucket, need to set its owner id field
     if ((action === ACTIONS.UPDATE && user_input.owner) || (action === ACTIONS.ADD)) {
@@ -256,25 +265,14 @@ async function update_bucket(data) {
  */
 async function delete_bucket(data, force) {
     try {
-        const temp_dir_name = native_fs_utils.get_bucket_tmpdir_name(data._id);
+        const bucket_empty = await _is_bucket_empty(data);
+        if (!bucket_empty && !force) {
+            throw_cli_error(ManageCLIError.BucketDeleteForbiddenHasObjects, data.name);
+        }
+
         const bucket_temp_dir_path = native_fs_utils.get_bucket_tmpdir_full_path(data.path, data._id);
-        // fs_contexts for bucket temp dir (storage path)
         const fs_context_fs_backend = native_fs_utils.get_process_fs_context(data.fs_backend);
-        let entries;
-        try {
-            entries = await nb_native().fs.readdir(fs_context_fs_backend, data.path);
-        } catch (err) {
-            dbg.warn(`delete_bucket: bucket name ${data.name},` +
-                `got an error on readdir with path: ${data.path}`, err);
-            // if the bucket's path was deleted first (encounter ENOENT error) - continue deletion
-            if (err.code !== 'ENOENT') throw err;
-        }
-        if (entries) {
-            const object_entries = entries.filter(element => !element.name.endsWith(temp_dir_name));
-            if (object_entries.length > 0 && !force) {
-                throw_cli_error(ManageCLIError.BucketDeleteForbiddenHasObjects, data.name);
-            }
-        }
+
         await native_fs_utils.folder_delete(bucket_temp_dir_path, fs_context_fs_backend, true);
         await config_fs.delete_bucket_config_file(data.name);
         return { code: ManageCLIResponse.BucketDeleted, detail: { name: data.name }, event_arg: { bucket: data.name } };
@@ -340,6 +338,27 @@ async function list_bucket_config_files(wide, filters = {}) {
     return config_files_list;
 }
 
+async function _is_bucket_empty(data) {
+    const temp_dir_name = native_fs_utils.get_bucket_tmpdir_name(data._id);
+    // fs_contexts for bucket temp dir (storage path)
+    const fs_context_fs_backend = native_fs_utils.get_process_fs_context(data.fs_backend);
+    let entries;
+    try {
+        entries = await nb_native().fs.readdir(fs_context_fs_backend, data.path);
+    } catch (err) {
+        dbg.warn(`_is_bucket_empty: bucket name ${data.name},` +
+            `got an error on readdir with path: ${data.path}`, err);
+        // if the bucket's path was deleted first (encounter ENOENT error) - continue deletion
+        if (err.code !== 'ENOENT') throw err;
+    }
+    if (entries) {
+        const object_entries = entries.filter(element => !element.name.endsWith(temp_dir_name));
+        return object_entries.length === 0;
+    }
+
+    return true;
+}
+
 /**
  * bucket_management does the following - 
  * 1. fetches the bucket data if this is not a list operation

src/manage_nsfs/manage_nsfs_constants.js

@@ -62,7 +62,7 @@ const VALID_OPTIONS_ANONYMOUS_ACCOUNT = {
 
 const VALID_OPTIONS_BUCKET = {
     'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'force_md5_etag', 'notifications', FROM_FILE, ...CLI_MUTUAL_OPTIONS]),
-    'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', 'notifications', ...CLI_MUTUAL_OPTIONS]),
+    'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', 'notifications', 'tag', ...CLI_MUTUAL_OPTIONS]),
     'delete': new Set(['name', 'force', ...CLI_MUTUAL_OPTIONS]),
     'list': new Set(['wide', 'name', ...CLI_MUTUAL_OPTIONS]),
     'status': new Set(['name', ...CLI_MUTUAL_OPTIONS]),
@@ -171,6 +171,8 @@ const OPTION_TYPE = {
     key: 'string',
     value: 'string',
     remove_key: 'boolean',
+    // bucket tagging
+    tag: 'string'
 };
 
 const BOOLEAN_STRING_VALUES = ['true', 'false'];

src/sdk/bucketspace_fs.js

@@ -4,6 +4,7 @@
 const _ = require('lodash');
 const util = require('util');
 const path = require('path');
+const { default: Ajv } = require('ajv');
 const P = require('../util/promise');
 const config = require('../../config');
 const RpcError = require('../rpc/rpc_error');
@@ -35,6 +36,7 @@ const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEve
 const dbg = require('../util/debug_module')(__filename);
 const bucket_semaphore = new KeysSemaphore(1);
 
+const ajv = new Ajv();
 
 class BucketSpaceFS extends BucketSpaceSimpleFS {
     constructor({ config_root }, stats) {
@@ -339,7 +341,7 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
         return {
             _id: mongo_utils.mongoObjectId(),
             name,
-            tag: js_utils.default_value(tag, undefined),
+            tag: js_utils.default_value(tag, BucketSpaceFS._default_bucket_tags()),
             owner_account: account.owner ? account.owner : account._id, // The account is the owner of the buckets that were created by it or by its users.
             creator: account._id,
             versioning: config.NSFS_VERSIONING_ENABLED && lock_enabled ? 'ENABLED' : 'DISABLED',
@@ -485,12 +487,21 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
     // BUCKET TAGGING //
     ////////////////////
 
-    async put_bucket_tagging(params) {
+    /**
+     * @param {*} params 
+     * @param {nb.ObjectSDK} object_sdk 
+     */
+    async put_bucket_tagging(params, object_sdk) {
         try {
             const { name, tagging } = params;
             dbg.log0('BucketSpaceFS.put_bucket_tagging: Bucket name, tagging', name, tagging);
             const bucket = await this.config_fs.get_bucket_by_name(name);
-            bucket.tag = tagging;
+            const { ns } = await object_sdk.read_bucket_full_info(name);
+            const is_bucket_empty = await BucketSpaceFS.#_is_bucket_empty(name, null, ns, object_sdk);
+
+            bucket.tag = BucketSpaceFS._merge_reserved_tags(
+                bucket.tag || BucketSpaceFS._default_bucket_tags(), tagging, is_bucket_empty
+            );
             await this.config_fs.update_bucket_config_file(bucket);
         } catch (error) {
             throw translate_error_codes(error, entity_enum.BUCKET);
@@ -502,7 +513,16 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
             const { name } = params;
             dbg.log0('BucketSpaceFS.delete_bucket_tagging: Bucket name', name);
             const bucket = await this.config_fs.get_bucket_by_name(name);
-            delete bucket.tag;
+
+            const preserved_tags = [];
+            for (const tag of bucket.tag) {
+                const tag_info = config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[tag.key];
+                if (tag_info?.immutable) {
+                    preserved_tags.push(tag);
+                }
+            }
+
+            bucket.tag = preserved_tags;
             await this.config_fs.update_bucket_config_file(bucket);
         } catch (error) {
             throw translate_error_codes(error, entity_enum.BUCKET);
@@ -520,6 +540,40 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
         }
     }
 
+    static _default_bucket_tags() {
+        return Object.keys(
+            config.NSFS_GLACIER_RESERVED_BUCKET_TAGS
+        ).map(key => ({ key, value: config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[key].default }));
+    }
+
+    static _merge_reserved_tags(original_tags, new_tags, is_bucket_empty) {
+        const merged_tags = original_tags.reduce((curr, tag) => {
+            if (!config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[tag.key]) return curr;
+            return Object.assign(curr, { [tag.key]: tag.value });
+        }, {});
+
+        for (const tag of new_tags) {
+            const tag_info = config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[tag.key];
+            if (!tag_info) {
+                merged_tags[tag.key] = tag.value;
+                continue;
+            }
+            if (!tag_info.immutable || (tag_info.immutable === 'if-data' && is_bucket_empty)) {
+                let validator = ajv.getSchema(tag_info.schema.$id);
+                if (!validator) {
+                    ajv.addSchema(tag_info.schema);
+                    validator = ajv.getSchema(tag_info.schema.$id);
+                }
+
+                if (validator(tag.value)) {
+                    merged_tags[tag.key] = tag.value;
+                }
+            }
+        }
+
+        return Object.keys(merged_tags).map(key => ({ key, value: merged_tags[key] }));
+    }
+
     ////////////////////
     // BUCKET LOGGING //
     ////////////////////
@@ -933,6 +987,25 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
 
         return storage_classes;
     }
+
+    static async #_is_bucket_empty(name, params, ns, object_sdk) {
+        params = params || {};
+
+        let list;
+        try {
+            if (ns._is_versioning_disabled()) {
+                list = await ns.list_objects({ ...params, bucket: name, limit: 1 }, object_sdk);
+            } else {
+                list = await ns.list_object_versions({ ...params, bucket: name, limit: 1 }, object_sdk);
+            }
+        } catch (err) {
+            dbg.warn('#_is_bucket_empty: bucket name', name, 'got an error while trying to list_objects', err);
+            // in case the ULS was deleted - we will continue
+            if (err.rpc_code !== 'NO_SUCH_BUCKET') throw err;
+        }
+
+        return !(list && list.objects && list.objects.length > 0);
+    }
 }
 
 module.exports = BucketSpaceFS;

src/sdk/bucketspace_simple_fs.js

@@ -168,7 +168,12 @@ class BucketSpaceSimpleFS {
     // BUCKET TAGGING //
     ////////////////////
 
-    async put_bucket_tagging(params) {
+    /**
+     * 
+     * @param {*} params 
+     * @param {nb.ObjectSDK} object_sdk
+     */
+    async put_bucket_tagging(params, object_sdk) {
         // TODO
     }
 

src/sdk/nb.d.ts

@@ -846,7 +846,7 @@ interface BucketSpace {
 
     set_bucket_versioning(params: object, object_sdk: ObjectSDK): Promise<any>;
 
-    put_bucket_tagging(params: object): Promise<any>;
+    put_bucket_tagging(params: object, object_sdk: ObjectSDK): Promise<any>;
     delete_bucket_tagging(params: object): Promise<any>;
     get_bucket_tagging(params: object): Promise<any>;
 

src/sdk/object_sdk.js

@@ -986,7 +986,7 @@ class ObjectSDK {
 
     async put_bucket_tagging(params) {
         const bs = this._get_bucketspace();
-        return bs.put_bucket_tagging(params);
+        return bs.put_bucket_tagging(params, this);
     }
 
     async delete_bucket_tagging(params) {

src/test/unit_tests/test_bucketspace_fs.js

@@ -930,7 +930,7 @@ mocha.describe('bucketspace_fs', function() {
     mocha.describe('bucket tagging operations', function() {
         mocha.it('put_bucket_tagging', async function() {
             const param = { name: test_bucket, tagging: [{ key: 'k1', value: 'v1' }] };
-            await bucketspace_fs.put_bucket_tagging(param);
+            await bucketspace_fs.put_bucket_tagging(param, dummy_object_sdk);
             const tag = await bucketspace_fs.get_bucket_tagging(param);
             assert.deepEqual(tag, { tagging: param.tagging });
         });