NC | add bypass governance flag

Signed-off-by: nadav mizrahi <nadav.mizrahi16@gmail.com>

Author: nadavMiz

src/cmd/manage_nsfs.js

@@ -505,6 +505,7 @@ async function fetch_account_data(action, user_input) {
             new_buckets_path: user_input.new_buckets_path,
             fs_backend: user_input.fs_backend ? String(user_input.fs_backend) : config.NSFS_NC_STORAGE_BACKEND,
             custom_bucket_path_allowed_list: user_input.custom_bucket_path_allowed_list,
+            allow_bypass_governance: user_input.allow_bypass_governance === '' ? user_input.allow_bypass_governance : get_boolean_or_string_value(user_input.allow_bypass_governance),
         },
         default_connection: user_input.default_connection === undefined ? undefined : String(user_input.default_connection)
     };
@@ -545,6 +546,7 @@ async function fetch_account_data(action, user_input) {
     }
     // custom_bucket_path_allowed_list deletion specified with empty string ''
     data.nsfs_account_config.custom_bucket_path_allowed_list = data.nsfs_account_config.custom_bucket_path_allowed_list || undefined;
+    data.nsfs_account_config.allow_bypass_governance = data.nsfs_account_config.allow_bypass_governance === '' ? undefined : data.nsfs_account_config.allow_bypass_governance;
 
     return data;
 }

src/manage_nsfs/manage_nsfs_constants.js

@@ -46,8 +46,8 @@ const FROM_FILE = 'from_file';
 const ANONYMOUS = 'anonymous';
 
 const VALID_OPTIONS_ACCOUNT = {
-    'add': new Set(['name', 'uid', 'gid', 'supplemental_groups', 'new_buckets_path', 'custom_bucket_path_allowed_list', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', 'force_md5_etag', 'iam_operate_on_root_account', 'default_connection', FROM_FILE, ...CLI_MUTUAL_OPTIONS]),
-    'update': new Set(['name', 'uid', 'gid', 'supplemental_groups', 'new_buckets_path', 'custom_bucket_path_allowed_list', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', 'force_md5_etag', 'iam_operate_on_root_account', 'new_name', 'regenerate', 'default_connection', ...CLI_MUTUAL_OPTIONS]),
+    'add': new Set(['name', 'uid', 'gid', 'supplemental_groups', 'new_buckets_path', 'custom_bucket_path_allowed_list', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', 'force_md5_etag', 'iam_operate_on_root_account', 'default_connection', 'allow_bypass_governance', FROM_FILE, ...CLI_MUTUAL_OPTIONS]),
+    'update': new Set(['name', 'uid', 'gid', 'supplemental_groups', 'new_buckets_path', 'custom_bucket_path_allowed_list', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', 'force_md5_etag', 'iam_operate_on_root_account', 'new_name', 'regenerate', 'default_connection', 'allow_bypass_governance', ...CLI_MUTUAL_OPTIONS]),
     'delete': new Set(['name', ...CLI_MUTUAL_OPTIONS]),
     'list': new Set(['wide', 'show_secrets', 'gid', 'uid', 'user', 'name', 'access_key', ...CLI_MUTUAL_OPTIONS]),
     'status': new Set(['name', 'access_key', 'show_secrets', ...CLI_MUTUAL_OPTIONS]),
@@ -177,12 +177,13 @@ const OPTION_TYPE = {
     // bucket tagging
     tag: 'string',
     merge_tag: 'string',
+    allow_bypass_governance: 'boolean',
 };
 
 const BOOLEAN_STRING_VALUES = ['true', 'false'];
 const BOOLEAN_STRING_OPTIONS = new Set(['allow_bucket_creation', 'regenerate', 'wide', 'show_secrets', 'force',
     'force_md5_etag', 'iam_operate_on_root_account', 'all_account_details', 'all_bucket_details', 'anonymous',
-    'disable_service_validation', 'disable_runtime_validation', 'short_status', 'skip_verification', 'continue']);
+    'disable_service_validation', 'disable_runtime_validation', 'short_status', 'skip_verification', 'continue', 'allow_bypass_governance']);
 
 // CLI UNSET VALUES
 const CLI_EMPTY_STRING = '';
@@ -199,6 +200,7 @@ const UNSETTABLE_OPTIONS_OBJ = Object.freeze({
     'new_buckets_path': CLI_EMPTY_STRING,
     'custom_bucket_path_allowed_list': CLI_EMPTY_STRING,
     'ips': CLI_EMPTY_STRING_ARRAY,
+    'allow_bypass_governance': CLI_EMPTY_STRING,
 });
 
 const LIST_ACCOUNT_FILTERS = ['uid', 'gid', 'user', 'name', 'access_key'];

src/manage_nsfs/manage_nsfs_help_utils.js

@@ -144,6 +144,7 @@ Flags:
     --iam_operate_on_root_account <true | false>          (optional)        Set the account to create root accounts instead of IAM users in IAM API requests.
     --from_file <string>                                  (optional)        Use details from the JSON file, there is no need to mention all the properties individually in the CLI
     --custom_bucket_path_allowed_list <string>            (optional)        Set the list of allowed custom bucket paths, separated by colons (:) example: '/gpfs/data/custom1/:/gpfs/data/custom2/'
+    --allow_bypass_governance <true | false>              (optional)        Set the account to allow bypassing governance mode object lock for object deletion and retention configuration (unset with '')
 `;
 
 const ACCOUNT_FLAGS_UPDATE = `
@@ -172,6 +173,7 @@ Flags:
     --force_md5_etag <true | false>                       (optional)        Update the account to force md5 etag calculation (unset with '') (will override default config.NSFS_NC_STORAGE_BACKEND)
     --iam_operate_on_root_account <true | false>          (optional)        Update the account to create root accounts instead of IAM users in IAM API requests.
     --custom_bucket_path_allowed_list <string>            (optional)        Update the list of allowed custom bucket paths, separated by colons (:) example: '/gpfs/data/custom1/:/gpfs/data/custom2/' (override;unset with '')
+    --allow_bypass_governance <true | false>              (optional)        Update the account to allow bypassing governance mode object lock for object deletion and retention configuration (unset with '')
 `;
 
 const ACCOUNT_FLAGS_DELETE = `

src/sdk/accountspace_fs.js

@@ -609,6 +609,7 @@ class AccountSpaceFS {
                 new_buckets_path: requesting_account.nsfs_account_config.new_buckets_path,
                 fs_backend: requesting_account.nsfs_account_config.fs_backend,
                 custom_bucket_path_allowed_list: requesting_account.nsfs_account_config.custom_bucket_path_allowed_list,
+                allow_bypass_governance: requesting_account.nsfs_account_config.allow_bypass_governance,
             }
         };
         if (requesting_account.iam_operate_on_root_account) {

src/sdk/namespace_fs.js

@@ -2317,18 +2317,22 @@ class NamespaceFS {
         return { retention: { mode: default_retention.mode, retain_until_date } };
     }
 
+    _check_bypass_governance_permission(fs_context) {
+        return fs_context?.allow_bypass_governance;
+    }
+
     /**
      * check if the object deletion should be blocked by retention lock. if the object is blocked will throw AccessDenied error
      * @param {Object} retention - object retention lock settings
      * @param {boolean} bypass_governance - if true, and user has permission to use this flag, will allow to bypass governance mode retention lock. compliance mode retention lock cannot be bypassed.
      * @throws {S3Error.AccessDenied} if the object is protected by object lock and the user does not have permission to bypass the lock
      */
-    _check_object_retention(retention, bypass_governance) {
+    _check_object_retention(fs_context, retention, bypass_governance) {
         if (retention) {
             const retain_until_date = new Date(retention.retain_until_date);
             const now = new Date();
             if (now < retain_until_date) {
-                //TODO should check for bypass_governance permission
+                bypass_governance = bypass_governance && this._check_bypass_governance_permission(fs_context);
                 if (retention.mode === 'COMPLIANCE' ||
                     (retention.mode === 'GOVERNANCE' && !bypass_governance)) {
                     const err = new S3Error(S3Error.AccessDenied);
@@ -2349,13 +2353,13 @@ class NamespaceFS {
      * @param {boolean} bypass_governance - if true, and user has permission to use this flag, will allow to bypass governance mode retention lock. compliance mode retention lock cannot be bypassed.
      * @throws {S3Error.AccessDenied} if the object is protected by object lock and the user does not have permission to bypass the lock
      */
-    _compare_object_retention(current_retention, new_retention, bypass_governance) {
+    _compare_object_retention(fs_context, current_retention, new_retention, bypass_governance) {
         if (current_retention) {
             const retain_until_date = new Date(current_retention.retain_until_date);
             const new_date = new Date(new_retention.retain_until_date);
             //can always increase retention time
-            if (new_date >= retain_until_date && new_retention.mode === current_retention.mode) return;
-            //TODO should check for bypass_governance permission
+            if (new_date > retain_until_date && new_retention.mode === current_retention.mode) return;
+            bypass_governance = bypass_governance && this._check_bypass_governance_permission(fs_context);
             if (current_retention.mode === 'COMPLIANCE' ||
                 (current_retention.mode === 'GOVERNANCE' && !bypass_governance)) {
                 const err = new S3Error(S3Error.AccessDenied);
@@ -2371,13 +2375,13 @@ class NamespaceFS {
      * @param {Object} params - request params, contains bypass_governance flag
      * @throws {S3Error.AccessDenied} if the object is protected by object lock and the user does not have permission to bypass the lock
      */
-    _check_object_lock(version_id, params) {
+    _check_object_lock(fs_context, version_id, params) {
         if (version_id.legal_hold === 'ON') {
             const err = new S3Error(S3Error.AccessDenied);
             err.message = 'Access Denied because object protected by object lock.';
             throw err;
         }
-        this._check_object_retention(version_id.retention, params.bypass_governance);
+        this._check_object_retention(fs_context, version_id.retention, params.bypass_governance);
     }
 
 
@@ -2468,7 +2472,7 @@ class NamespaceFS {
         try {
             const stat = await nb_native().fs.stat(fs_context, file_path);
             const current_retention = this._get_retention_mode_from_xattr(stat.xattr);
-            this._compare_object_retention(current_retention, params.retention, params.bypass_governance);
+            this._compare_object_retention(fs_context, current_retention, params.retention, params.bypass_governance);
             const fs_xattr = {};
             fs_xattr[XATTR_RETENTION_MODE] = params.retention.mode;
             fs_xattr[XATTR_RETENTION_DATE] = params.retention.retain_until_date.toISOString();
@@ -3420,7 +3424,7 @@ class NamespaceFS {
                 await this._check_path_in_bucket_boundaries(fs_context, file_path);
                 const version_info = await this._get_version_info(fs_context, file_path);
                 if (!version_info) return;
-                this._check_object_lock(version_info, params);
+                this._check_object_lock(fs_context, version_info, params);
 
                 const deleted_latest = file_path === latest_version_path;
                 if (deleted_latest) {

src/server/system_services/schemas/nsfs_account_schema.js

@@ -89,6 +89,7 @@ module.exports = {
                         $ref: 'common_api#/definitions/fs_backend'
                     },
                     custom_bucket_path_allowed_list: { type: 'string' },
+                    allow_bypass_governance: { type: 'boolean' },
                 }
             }, {
                 type: 'object',
@@ -100,6 +101,7 @@ module.exports = {
                         $ref: 'common_api#/definitions/fs_backend'
                     },
                     custom_bucket_path_allowed_list: { type: 'string' },
+                    allow_bypass_governance: { type: 'boolean' },
                 }
             }]
         },

src/test/integration_tests/api/s3/test_s3_worm.js

@@ -44,6 +44,7 @@ mocha.describe('s3 worm', function() {
     let version_id3;
     let version_id4;
     const user_a = 'alicia';
+    const user_a_mail = 'alicia@test.com';
     let s3_owner;
     //let s3_a;
     mocha.before(async function() {
@@ -70,19 +71,23 @@ mocha.describe('s3 worm', function() {
             account.nsfs_account_config = {
                 uid: process.getuid(),
                 gid: process.getgid(),
-                new_buckets_path: tmp_fs_root
+                new_buckets_path: tmp_fs_root,
+                allow_bypass_governance: 'true',
             };
         }
         const admin_keys = (await rpc_client.account.read_account({ email: EMAIL, })).access_keys;
         account.name = user_a;
         account.email = user_a;
         const user_a_keys = (await rpc_client.account.create_account(account)).access_keys;
-        s3_creds.credentials.accessKeyId = user_a_keys[0].access_key.unwrap();
-        s3_creds.credentials.secretAccessKey = user_a_keys[0].secret_key.unwrap();
-        //s3_a = new AWS.S3(s3_creds);
-        s3_creds.credentials.accessKeyId = admin_keys[0].access_key.unwrap();
-        s3_creds.credentials.secretAccessKey = admin_keys[0].secret_key.unwrap();
-        s3_owner = new S3(s3_creds);
+        if (is_nc_coretest) {
+            s3_creds.credentials.accessKeyId = user_a_keys[0].access_key.unwrap();
+            s3_creds.credentials.secretAccessKey = user_a_keys[0].secret_key.unwrap();
+            s3_owner = new S3(s3_creds);
+        } else {
+            s3_creds.credentials.accessKeyId = admin_keys[0].access_key.unwrap();
+            s3_creds.credentials.secretAccessKey = admin_keys[0].secret_key.unwrap();
+            s3_owner = new S3(s3_creds);
+        }
     });
     mocha.describe('buckets creation', function() {
         mocha.it('create bucket BKT & enable lock', async function() {
@@ -977,4 +982,43 @@ mocha.describe('s3 worm', function() {
             }), 'NoSuchObjectLockConfiguration', 'The specified object does not have a ObjectLock configuration');
         });
     });
+
+    mocha.describe('no bypass permissions for user', async function() {
+        let version_id;
+        mocha.before(async function() {
+            const update_conf = {
+                email: user_a_mail,
+                nsfs_account_config: {allow_bypass_governance: 'false'}};
+            await rpc_client.account.update_account_s3_access(update_conf);
+
+            const conf = await s3_owner.putObject({
+                Bucket: BKT,
+                Key: OBJ1,
+                Body: file_body,
+                ContentType: 'text/plain',
+                ObjectLockMode: 'GOVERNANCE',
+                ObjectLockRetainUntilDate: tomorrow
+            });
+            version_id = conf.VersionId;
+        });
+
+        mocha.it('should fail to put retention without bypass flag', async function() {
+            await assert_throws_async(s3_owner.putObjectRetention({
+                Bucket: BKT,
+                Key: OBJ1,
+                Retention: { Mode: 'COMPLIANCE', RetainUntilDate: tomorrow },
+                VersionId: version_id,
+                BypassGovernanceRetention: true
+            }), 'AccessDenied', is_nc_coretest ? 'Access Denied because object protected by object lock.' : 'Access Denied');
+        });
+
+        mocha.it('should fail to delete object with retention without bypass flag', async function() {
+            await assert_throws_async(s3_owner.deleteObject({
+                Bucket: BKT,
+                Key: OBJ1,
+                VersionId: version_id1,
+                BypassGovernanceRetention: true,
+            }), 'AccessDenied', is_nc_coretest ? 'Access Denied because object protected by object lock.' : 'Access Denied');
+        });
+    });
 });

src/test/integration_tests/nc/cli/test_nc_account_cli.test.js

@@ -31,6 +31,7 @@ const quoted_type = Object.freeze({
 
 // eslint-disable-next-line max-lines-per-function
 describe('manage nsfs cli account flow', () => {
+    /* eslint-disable max-statements */
     describe('cli create account', () => {
         const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs');
         const config_fs = new ConfigFS(config_root);
@@ -671,6 +672,20 @@ describe('manage nsfs cli account flow', () => {
             assert_account(account, account_options, false);
             expect(account.nsfs_account_config.custom_bucket_path_allowed_list).toBe(custom_bucket_path_allowed_list);
         });
+
+        it('cli account add - with bypass governance lock flag', async function() {
+            const action = ACTIONS.ADD;
+            const { type, name, new_buckets_path, uid, gid } = defaults;
+            const allow_bypass_governance = 'true';
+            const account_options = { config_root, name, new_buckets_path, uid, gid, allow_bypass_governance };
+            await fs_utils.create_fresh_path(new_buckets_path);
+            await fs_utils.file_must_exist(new_buckets_path);
+            await set_path_permissions_and_owner(new_buckets_path, account_options, 0o700);
+            await exec_manage_cli(type, action, account_options);
+            const account = await config_fs.get_account_by_name(name, config_fs_account_options);
+            assert_account(account, account_options, false);
+            expect(account.nsfs_account_config.allow_bypass_governance).toBe(true);
+        });
     });
 
     describe('cli update account', () => {
@@ -1256,6 +1271,25 @@ describe('manage nsfs cli account flow', () => {
                 new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options);
                 expect(new_account_details.nsfs_account_config.custom_bucket_path_allowed_list).toBe("/some/path/:/another/path/");
             });
+
+            it('cli update account with bypass governance lock flag', async function() {
+                const { name } = defaults;
+                const account_options = { config_root, name, allow_bypass_governance: 'true'};
+                const action = ACTIONS.UPDATE;
+                await exec_manage_cli(type, action, account_options);
+                let new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options);
+                expect(new_account_details.nsfs_account_config.allow_bypass_governance).toBe(true);
+
+                account_options.allow_bypass_governance = 'false';
+                await exec_manage_cli(type, action, account_options);
+                new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options);
+                expect(new_account_details.nsfs_account_config.allow_bypass_governance).toBe(false);
+
+                account_options.allow_bypass_governance = CLI_UNSET_EMPTY_STRING;
+                await exec_manage_cli(type, action, account_options);
+                new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options);
+                expect(new_account_details.nsfs_account_config.allow_bypass_governance).toBeUndefined();
+            });
         });
 
         describe('cli update account (has distinguished name)', () => {

src/test/utils/coretest/nc_coretest.js

@@ -344,6 +344,7 @@ async function create_account_manage(options) {
         access_key: options.access_key,
         secret_key: options.secret_key,
         custom_bucket_path_allowed_list: options.nsfs_account_config.custom_bucket_path_allowed_list,
+        allow_bypass_governance: options.nsfs_account_config.allow_bypass_governance,
     };
     const res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, cli_options);
     const json_account = JSON.parse(res);
@@ -441,6 +442,7 @@ async function update_account_s3_access_manage(options) {
         uid: options.nsfs_account_config.uid,
         gid: options.nsfs_account_config.gid,
         custom_bucket_path_allowed_list: options.nsfs_account_config.custom_bucket_path_allowed_list,
+        allow_bypass_governance: options.nsfs_account_config.allow_bypass_governance,
     };
     await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.UPDATE, cli_options);
 }