add support for reserved bucket tags

Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com>

fix put_bucket_tagging test - pass dummy objectsdk

Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com>

add support for events, race safe tag manipulation and --merge_tag flag

Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com>

Author: tangledbytes

config.js

@@ -956,6 +956,33 @@ config.NSFS_GLACIER_FORCE_EXPIRE_ON_GET = false;
 // interval
 config.NSFS_GLACIER_MIGRATE_LOG_THRESHOLD = 50 * 1024;
 
+/** 
+ * NSFS_GLACIER_RESERVED_BUCKET_TAGS defines an object of bucket tags which will be reserved
+ * by the system and PUT operations for them via S3 API would be disallowed even by the bucket
+ * owners
+ * 
+ * @type {Record<string, {
+ *  schema: Record<any, any> & { $id: string },
+ *  immutable: true | false | 'if-data',
+ *  default: any,
+ *  event: boolean
+ * }>}
+ * 
+ * @example
+ * {
+    'deep-archive-copies': {
+        schema: {
+            $id: 'deep-archive-copies-schema-v0',
+            enum: ['1', '2']
+        }, // JSON Schema
+        immutable: 'if-data',
+        default: '1',
+        event: true
+    }
+ * }
+ */
+config.NSFS_GLACIER_RESERVED_BUCKET_TAGS = {};
+
 // anonymous account name
 config.ANONYMOUS_ACCOUNT_NAME = 'anonymous';
 

docs/NooBaaNonContainerized/Events.md

@@ -43,6 +43,11 @@ The following list includes events that indicate on a normal / successful operat
 - Arguments: `whitelist_ips`
 - Description: Whitelist Server IPs updated successfully using NooBaa CLI.  
 
+#### 7. `noobaa_bucket_reserved_tag_modified`
+- Arguments:
+    - `bucket_name`
+    - `<tag_value>` (if `event` is `true` for the reserved tag)
+- Description: NooBaa bucket reserved tag was modified successfully using NooBaa CLI or S3.
 
 ### Error Indicating Events
 

src/cmd/manage_nsfs.js

@@ -39,6 +39,8 @@ const { throw_cli_error, get_bucket_owner_account_by_name,
 const manage_nsfs_validations = require('../manage_nsfs/manage_nsfs_validations');
 const nc_mkm = require('../manage_nsfs/nc_master_key_manager').get_instance();
 const notifications_util = require('../util/notifications_util');
+const BucketSpaceFS = require('../sdk/bucketspace_fs');
+const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent;
 
 ///////////////
 //// GENERAL //
@@ -135,7 +137,6 @@ async function fetch_bucket_data(action, user_input) {
         force_md5_etag: user_input.force_md5_etag === undefined || user_input.force_md5_etag === '' ? user_input.force_md5_etag : get_boolean_or_string_value(user_input.force_md5_etag),
         notifications: user_input.notifications
     };
-
     if (user_input.bucket_policy !== undefined) {
         if (typeof user_input.bucket_policy === 'string') {
             // bucket_policy deletion specified with empty string ''
@@ -154,6 +155,27 @@ async function fetch_bucket_data(action, user_input) {
         data = await merge_new_and_existing_config_data(data);
     }
 
+    if ((action === ACTIONS.UPDATE && user_input.tag) || (action === ACTIONS.ADD)) {
+        const tags = JSON.parse(user_input.tag || '[]');
+        data.tag = BucketSpaceFS._merge_reserved_tags(
+            data.tag || BucketSpaceFS._default_bucket_tags(),
+            tags,
+            action === ACTIONS.ADD ? true : await _is_bucket_empty(data),
+        );
+    }
+
+    if ((action === ACTIONS.UPDATE && user_input.merge_tag) || (action === ACTIONS.ADD)) {
+        const merge_tags = JSON.parse(user_input.merge_tag || '[]');
+        data.tag = _.merge(
+            data.tag,
+            BucketSpaceFS._merge_reserved_tags(
+                data.tag || BucketSpaceFS._default_bucket_tags(),
+                merge_tags,
+                action === ACTIONS.ADD ? true : await _is_bucket_empty(data),
+            )
+        );
+    }
+
     //if we're updating the owner, needs to override owner in file with the owner from user input.
     //if we're adding a bucket, need to set its owner id field
     if ((action === ACTIONS.UPDATE && user_input.owner) || (action === ACTIONS.ADD)) {
@@ -256,25 +278,14 @@ async function update_bucket(data) {
  */
 async function delete_bucket(data, force) {
     try {
-        const temp_dir_name = native_fs_utils.get_bucket_tmpdir_name(data._id);
+        const bucket_empty = await _is_bucket_empty(data);
+        if (!bucket_empty && !force) {
+            throw_cli_error(ManageCLIError.BucketDeleteForbiddenHasObjects, data.name);
+        }
+
         const bucket_temp_dir_path = native_fs_utils.get_bucket_tmpdir_full_path(data.path, data._id);
-        // fs_contexts for bucket temp dir (storage path)
         const fs_context_fs_backend = native_fs_utils.get_process_fs_context(data.fs_backend);
-        let entries;
-        try {
-            entries = await nb_native().fs.readdir(fs_context_fs_backend, data.path);
-        } catch (err) {
-            dbg.warn(`delete_bucket: bucket name ${data.name},` +
-                `got an error on readdir with path: ${data.path}`, err);
-            // if the bucket's path was deleted first (encounter ENOENT error) - continue deletion
-            if (err.code !== 'ENOENT') throw err;
-        }
-        if (entries) {
-            const object_entries = entries.filter(element => !element.name.endsWith(temp_dir_name));
-            if (object_entries.length > 0 && !force) {
-                throw_cli_error(ManageCLIError.BucketDeleteForbiddenHasObjects, data.name);
-            }
-        }
+
         await native_fs_utils.folder_delete(bucket_temp_dir_path, fs_context_fs_backend, true);
         await config_fs.delete_bucket_config_file(data.name);
         return { code: ManageCLIResponse.BucketDeleted, detail: { name: data.name }, event_arg: { bucket: data.name } };
@@ -340,6 +351,27 @@ async function list_bucket_config_files(wide, filters = {}) {
     return config_files_list;
 }
 
+async function _is_bucket_empty(data) {
+    const temp_dir_name = native_fs_utils.get_bucket_tmpdir_name(data._id);
+    // fs_contexts for bucket temp dir (storage path)
+    const fs_context_fs_backend = native_fs_utils.get_process_fs_context(data.fs_backend);
+    let entries;
+    try {
+        entries = await nb_native().fs.readdir(fs_context_fs_backend, data.path);
+    } catch (err) {
+        dbg.warn(`_is_bucket_empty: bucket name ${data.name},` +
+            `got an error on readdir with path: ${data.path}`, err);
+        // if the bucket's path was deleted first (encounter ENOENT error) - continue deletion
+        if (err.code !== 'ENOENT') throw err;
+    }
+    if (entries) {
+        const object_entries = entries.filter(element => !element.name.endsWith(temp_dir_name));
+        return object_entries.length === 0;
+    }
+
+    return true;
+}
+
 /**
  * bucket_management does the following - 
  * 1. fetches the bucket data if this is not a list operation
@@ -361,7 +393,36 @@ async function bucket_management(action, user_input) {
     } else if (action === ACTIONS.STATUS) {
         response = await get_bucket_status(data);
     } else if (action === ACTIONS.UPDATE) {
-        response = await update_bucket(data);
+        const bucket_path = config_fs.get_bucket_path_by_name(user_input.name);
+        const bucket_lock_file = `${bucket_path}.lock`;
+        await native_fs_utils.lock_and_run(config_fs.fs_context, bucket_lock_file, async () => {
+            const prev_bucket_info = await fetch_bucket_data(action, _.omit(user_input, ['tag', 'merge_tag']));
+            const bucket_info = await fetch_bucket_data(action, user_input);
+
+            const tagging_object = prev_bucket_info.tag?.reduce((curr, tag) => {
+                return Object.assign(curr, { [tag.key]: tag.value });
+            }, {});
+
+            let reserved_tag_modified = false;
+            const reserved_tag_event_args = bucket_info.tag?.reduce((curr, tag) => {
+                const tag_info = config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[tag.key];
+
+                // If not a reserved tag - skip
+                if (!tag_info) return curr;
+
+                // If no event is requested - skip
+                if (!tag_info.event) return curr;
+
+                // If value didn't change - skip
+                if (_.isEqual(tagging_object[tag.key], tag.value)) return curr;
+
+                reserved_tag_modified = true;
+                return Object.assign(curr, { [tag.key]: tag.value });
+            }, {});
+
+            response = await update_bucket(bucket_info);
+            if (reserved_tag_modified) new NoobaaEvent(NoobaaEvent.BUCKET_RESERVED_TAG_MODIFIED).create_event(undefined, { ...reserved_tag_event_args, bucket_name: user_input.name });
+        });
     } else if (action === ACTIONS.DELETE) {
         const force = get_boolean_or_string_value(user_input.force);
         response = await delete_bucket(data, force);

src/manage_nsfs/manage_nsfs_constants.js

@@ -62,7 +62,7 @@ const VALID_OPTIONS_ANONYMOUS_ACCOUNT = {
 
 const VALID_OPTIONS_BUCKET = {
     'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'force_md5_etag', 'notifications', FROM_FILE, ...CLI_MUTUAL_OPTIONS]),
-    'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', 'notifications', ...CLI_MUTUAL_OPTIONS]),
+    'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', 'notifications', 'tag', 'merge_tag', ...CLI_MUTUAL_OPTIONS]),
     'delete': new Set(['name', 'force', ...CLI_MUTUAL_OPTIONS]),
     'list': new Set(['wide', 'name', ...CLI_MUTUAL_OPTIONS]),
     'status': new Set(['name', ...CLI_MUTUAL_OPTIONS]),
@@ -171,6 +171,9 @@ const OPTION_TYPE = {
     key: 'string',
     value: 'string',
     remove_key: 'boolean',
+    // bucket tagging
+    tag: 'string',
+    merge_tag: 'string',
 };
 
 const BOOLEAN_STRING_VALUES = ['true', 'false'];

src/manage_nsfs/manage_nsfs_events_utils.js

@@ -322,6 +322,16 @@ NoobaaEvent.UNAUTHORIZED = Object.freeze({
     severity: 'ERROR',
     state: 'HEALTHY',
 });
+NoobaaEvent.BUCKET_RESERVED_TAG_MODIFIED = Object.freeze({
+    event_code: 'noobaa_bucket_reserved_tag_modified',
+    message: 'Bucket reserved tag modified',
+    description: 'Noobaa bucket reserved tag modified',
+    entity_type: 'NODE',
+    event_type: 'INFO',
+    scope: 'NODE',
+    severity: 'INFO',
+    state: 'HEALTHY',
+});
 
 NoobaaEvent.IO_STREAM_ITEM_TIMEOUT = Object.freeze({
     event_code: 'bucket_io_stream_item_timeout',