NC | add bypass governance flag

Signed-off-by: nadav mizrahi <nadav.mizrahi16@gmail.com>

Author: nadavMiz

docs/NooBaaNonContainerized/AccountsAndBuckets.md

@@ -38,7 +38,9 @@ See all available account properties - [NC Account Schema](../../src/server/syst
 
   - `new_buckets_path` - When an account creates a bucket using the S3 protocol, NooBaa will create the underlying file system directory. This directory will be created under new_buckets_path. Note that the account must have read and write access to its `new_buckets_path`.  Must be an absolute path.  
 
-  - `custom_bucket_path_allowed_list` - When an account creates a bucket using the S3 protocol, He can override the default bucket path location (under new_buckets_path) using `x-noobaa-custom-bucket-path` HTTP header. This directory will be created only if this path will be under one of the provided allowed list paths in custom_bucket_path_allowed_list. Must be a list of absolute paths (divided by colons).  
+  - `custom_bucket_path_allowed_list` - When an account creates a bucket using the S3 protocol, He can override the default bucket path location (under new_buckets_path) using `x-noobaa-custom-bucket-path` HTTP header. This directory will be created only if this path will be under one of the provided allowed list paths in custom_bucket_path_allowed_list. Must be a list of absolute paths (divided by colons). 
+
+  - `allow_bypass_governance` - give permission to the user to bypass governance mode retention object lock. allowing him to set `x-amz-bypass-governance-retention` header for deleting the object and modifying retention lock. see https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html#object-lock-retention-modes 
 
 ### Account configuration  
 Currently, an account can be configured via NooBaa CLI, see - [NooBaa CLI](./NooBaaCLI.md).  

docs/NooBaaNonContainerized/NooBaaCLI.md

@@ -145,6 +145,10 @@ noobaa-cli account add --name <account_name> --uid <uid> --gid <gid> [--user]
     - Description: Specifies an allowed list where this account can create buckets in using 
     x-noobaa-custom-bucket-path header in create_bucket
 
+- `allow_bypass_governance`
+    - Type: Boolean
+    - Description: allow user to bypass governance-mode retention object lock when deleting or modifying locked object
+
 ### Update Account
 
 The `account update` command is used to update an existing account with customizable options.
@@ -154,7 +158,7 @@ The `account update` command is used to update an existing account with customiz
 noobaa-cli account update --name <account_name> [--new_name][--uid][--gid][--user]
 [--new_buckets_path][--access_key][--secret_key][--regenerate][--fs_backend]
 [--allow_bucket_creation][--force_md5_etag][--anonymous][--iam_operate_on_root_account][--default_connection]
-[--custom_bucket_path_allowed_list]
+[--custom_bucket_path_allowed_list][--allow_bypass_governance]
 ```
 #### Flags -
 - `name` (Required)
@@ -227,6 +231,10 @@ noobaa-cli account update --name <account_name> [--new_name][--uid][--gid][--use
     - Description: Specifies an allowed list where this account can create buckets in using 
     x-noobaa-custom-bucket-path header in create_bucket
 
+- `allow_bypass_governance`
+    - Type: Boolean
+    - Description: allow user to bypass governance-mode retention object lock when deleting or modifying locked object
+
 ### Account Status
 
 The `account status` command is used to print the status of the account.

src/cmd/manage_nsfs.js

@@ -505,6 +505,7 @@ async function fetch_account_data(action, user_input) {
             new_buckets_path: user_input.new_buckets_path,
             fs_backend: user_input.fs_backend ? String(user_input.fs_backend) : config.NSFS_NC_STORAGE_BACKEND,
             custom_bucket_path_allowed_list: user_input.custom_bucket_path_allowed_list,
+            allow_bypass_governance: user_input.allow_bypass_governance === undefined || user_input.allow_bypass_governance === '' ? user_input.allow_bypass_governance : get_boolean_or_string_value(user_input.allow_bypass_governance),
         },
         default_connection: user_input.default_connection === undefined ? undefined : String(user_input.default_connection)
     };
@@ -545,6 +546,7 @@ async function fetch_account_data(action, user_input) {
     }
     // custom_bucket_path_allowed_list deletion specified with empty string ''
     data.nsfs_account_config.custom_bucket_path_allowed_list = data.nsfs_account_config.custom_bucket_path_allowed_list || undefined;
+    data.nsfs_account_config.allow_bypass_governance = data.nsfs_account_config.allow_bypass_governance === '' ? undefined : data.nsfs_account_config.allow_bypass_governance;
 
     return data;
 }

src/manage_nsfs/manage_nsfs_constants.js

@@ -46,8 +46,8 @@ const FROM_FILE = 'from_file';
 const ANONYMOUS = 'anonymous';
 
 const VALID_OPTIONS_ACCOUNT = {
-    'add': new Set(['name', 'uid', 'gid', 'supplemental_groups', 'new_buckets_path', 'custom_bucket_path_allowed_list', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', 'force_md5_etag', 'iam_operate_on_root_account', 'default_connection', FROM_FILE, ...CLI_MUTUAL_OPTIONS]),
-    'update': new Set(['name', 'uid', 'gid', 'supplemental_groups', 'new_buckets_path', 'custom_bucket_path_allowed_list', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', 'force_md5_etag', 'iam_operate_on_root_account', 'new_name', 'regenerate', 'default_connection', ...CLI_MUTUAL_OPTIONS]),
+    'add': new Set(['name', 'uid', 'gid', 'supplemental_groups', 'new_buckets_path', 'custom_bucket_path_allowed_list', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', 'force_md5_etag', 'iam_operate_on_root_account', 'default_connection', 'allow_bypass_governance', FROM_FILE, ...CLI_MUTUAL_OPTIONS]),
+    'update': new Set(['name', 'uid', 'gid', 'supplemental_groups', 'new_buckets_path', 'custom_bucket_path_allowed_list', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', 'force_md5_etag', 'iam_operate_on_root_account', 'new_name', 'regenerate', 'default_connection', 'allow_bypass_governance', ...CLI_MUTUAL_OPTIONS]),
     'delete': new Set(['name', ...CLI_MUTUAL_OPTIONS]),
     'list': new Set(['wide', 'show_secrets', 'gid', 'uid', 'user', 'name', 'access_key', ...CLI_MUTUAL_OPTIONS]),
     'status': new Set(['name', 'access_key', 'show_secrets', ...CLI_MUTUAL_OPTIONS]),
@@ -177,12 +177,13 @@ const OPTION_TYPE = {
     // bucket tagging
     tag: 'string',
     merge_tag: 'string',
+    allow_bypass_governance: 'boolean',
 };
 
 const BOOLEAN_STRING_VALUES = ['true', 'false'];
 const BOOLEAN_STRING_OPTIONS = new Set(['allow_bucket_creation', 'regenerate', 'wide', 'show_secrets', 'force',
     'force_md5_etag', 'iam_operate_on_root_account', 'all_account_details', 'all_bucket_details', 'anonymous',
-    'disable_service_validation', 'disable_runtime_validation', 'short_status', 'skip_verification', 'continue']);
+    'disable_service_validation', 'disable_runtime_validation', 'short_status', 'skip_verification', 'continue', 'allow_bypass_governance']);
 
 // CLI UNSET VALUES
 const CLI_EMPTY_STRING = '';
@@ -199,6 +200,7 @@ const UNSETTABLE_OPTIONS_OBJ = Object.freeze({
     'new_buckets_path': CLI_EMPTY_STRING,
     'custom_bucket_path_allowed_list': CLI_EMPTY_STRING,
     'ips': CLI_EMPTY_STRING_ARRAY,
+    'allow_bypass_governance': CLI_EMPTY_STRING,
 });
 
 const LIST_ACCOUNT_FILTERS = ['uid', 'gid', 'user', 'name', 'access_key'];

src/manage_nsfs/manage_nsfs_help_utils.js

@@ -144,6 +144,7 @@ Flags:
     --iam_operate_on_root_account <true | false>          (optional)        Set the account to create root accounts instead of IAM users in IAM API requests.
     --from_file <string>                                  (optional)        Use details from the JSON file, there is no need to mention all the properties individually in the CLI
     --custom_bucket_path_allowed_list <string>            (optional)        Set the list of allowed custom bucket paths, separated by colons (:) example: '/gpfs/data/custom1/:/gpfs/data/custom2/'
+    --allow_bypass_governance <true | false>              (optional)        Set the account to allow bypassing governance mode object lock for object deletion and retention configuration (unset with '')
 `;
 
 const ACCOUNT_FLAGS_UPDATE = `
@@ -172,6 +173,7 @@ Flags:
     --force_md5_etag <true | false>                       (optional)        Update the account to force md5 etag calculation (unset with '') (will override default config.NSFS_NC_STORAGE_BACKEND)
     --iam_operate_on_root_account <true | false>          (optional)        Update the account to create root accounts instead of IAM users in IAM API requests.
     --custom_bucket_path_allowed_list <string>            (optional)        Update the list of allowed custom bucket paths, separated by colons (:) example: '/gpfs/data/custom1/:/gpfs/data/custom2/' (override;unset with '')
+    --allow_bypass_governance <true | false>              (optional)        Update the account to allow bypassing governance mode object lock for object deletion and retention configuration (unset with '')
 `;
 
 const ACCOUNT_FLAGS_DELETE = `

src/sdk/accountspace_fs.js

@@ -609,6 +609,7 @@ class AccountSpaceFS {
                 new_buckets_path: requesting_account.nsfs_account_config.new_buckets_path,
                 fs_backend: requesting_account.nsfs_account_config.fs_backend,
                 custom_bucket_path_allowed_list: requesting_account.nsfs_account_config.custom_bucket_path_allowed_list,
+                allow_bypass_governance: requesting_account.nsfs_account_config.allow_bypass_governance,
             }
         };
         if (requesting_account.iam_operate_on_root_account) {

src/sdk/namespace_fs.js

@@ -2317,18 +2317,22 @@ class NamespaceFS {
         return { retention: { mode: default_retention.mode, retain_until_date } };
     }
 
+    _check_bypass_governance_permission(fs_context) {
+        return fs_context?.allow_bypass_governance;
+    }
+
     /**
      * check if the object deletion should be blocked by retention lock. if the object is blocked will throw AccessDenied error
      * @param {Object} retention - object retention lock settings
      * @param {boolean} bypass_governance - if true, and user has permission to use this flag, will allow to bypass governance mode retention lock. compliance mode retention lock cannot be bypassed.
      * @throws {S3Error.AccessDenied} if the object is protected by object lock and the user does not have permission to bypass the lock
      */
-    _check_object_retention(retention, bypass_governance) {
+    _check_object_retention(fs_context, retention, bypass_governance) {
         if (retention) {
             const retain_until_date = new Date(retention.retain_until_date);
             const now = new Date();
             if (now < retain_until_date) {
-                //TODO should check for bypass_governance permission
+                bypass_governance = bypass_governance && this._check_bypass_governance_permission(fs_context);
                 if (retention.mode === 'COMPLIANCE' ||
                     (retention.mode === 'GOVERNANCE' && !bypass_governance)) {
                     const err = new S3Error(S3Error.AccessDenied);
@@ -2349,13 +2353,13 @@ class NamespaceFS {
      * @param {boolean} bypass_governance - if true, and user has permission to use this flag, will allow to bypass governance mode retention lock. compliance mode retention lock cannot be bypassed.
      * @throws {S3Error.AccessDenied} if the object is protected by object lock and the user does not have permission to bypass the lock
      */
-    _compare_object_retention(current_retention, new_retention, bypass_governance) {
+    _compare_object_retention(fs_context, current_retention, new_retention, bypass_governance) {
         if (current_retention) {
             const retain_until_date = new Date(current_retention.retain_until_date);
             const new_date = new Date(new_retention.retain_until_date);
-            //can always increase retention time
+            //can always increase retention time when mode is unchanged
             if (new_date >= retain_until_date && new_retention.mode === current_retention.mode) return;
-            //TODO should check for bypass_governance permission
+            bypass_governance = bypass_governance && this._check_bypass_governance_permission(fs_context);
             if (current_retention.mode === 'COMPLIANCE' ||
                 (current_retention.mode === 'GOVERNANCE' && !bypass_governance)) {
                 const err = new S3Error(S3Error.AccessDenied);
@@ -2371,13 +2375,13 @@ class NamespaceFS {
      * @param {Object} params - request params, contains bypass_governance flag
      * @throws {S3Error.AccessDenied} if the object is protected by object lock and the user does not have permission to bypass the lock
      */
-    _check_object_lock(version_id, params) {
+    _check_object_lock(fs_context, version_id, params) {
         if (version_id.legal_hold === 'ON') {
             const err = new S3Error(S3Error.AccessDenied);
             err.message = 'Access Denied because object protected by object lock.';
             throw err;
         }
-        this._check_object_retention(version_id.retention, params.bypass_governance);
+        this._check_object_retention(fs_context, version_id.retention, params.bypass_governance);
     }
 
 
@@ -2468,7 +2472,7 @@ class NamespaceFS {
         try {
             const stat = await nb_native().fs.stat(fs_context, file_path);
             const current_retention = this._get_retention_mode_from_xattr(stat.xattr);
-            this._compare_object_retention(current_retention, params.retention, params.bypass_governance);
+            this._compare_object_retention(fs_context, current_retention, params.retention, params.bypass_governance);
             const fs_xattr = {};
             fs_xattr[XATTR_RETENTION_MODE] = params.retention.mode;
             fs_xattr[XATTR_RETENTION_DATE] = params.retention.retain_until_date.toISOString();
@@ -3420,7 +3424,7 @@ class NamespaceFS {
                 await this._check_path_in_bucket_boundaries(fs_context, file_path);
                 const version_info = await this._get_version_info(fs_context, file_path);
                 if (!version_info) return;
-                this._check_object_lock(version_info, params);
+                this._check_object_lock(fs_context, version_info, params);
 
                 const deleted_latest = file_path === latest_version_path;
                 if (deleted_latest) {

src/server/system_services/schemas/nsfs_account_schema.js

@@ -89,6 +89,7 @@ module.exports = {
                         $ref: 'common_api#/definitions/fs_backend'
                     },
                     custom_bucket_path_allowed_list: { type: 'string' },
+                    allow_bypass_governance: { type: 'boolean' },
                 }
             }, {
                 type: 'object',
@@ -100,6 +101,7 @@ module.exports = {
                         $ref: 'common_api#/definitions/fs_backend'
                     },
                     custom_bucket_path_allowed_list: { type: 'string' },
+                    allow_bypass_governance: { type: 'boolean' },
                 }
             }]
         },

src/test/integration_tests/api/s3/test_s3_worm.js

@@ -44,6 +44,7 @@ mocha.describe('s3 worm', function() {
     let version_id3;
     let version_id4;
     const user_a = 'alicia';
+    const user_a_mail = 'alicia@test.com';
     let s3_owner;
     //let s3_a;
     mocha.before(async function() {
@@ -70,19 +71,23 @@ mocha.describe('s3 worm', function() {
             account.nsfs_account_config = {
                 uid: process.getuid(),
                 gid: process.getgid(),
-                new_buckets_path: tmp_fs_root
+                new_buckets_path: tmp_fs_root,
+                allow_bypass_governance: 'true',
             };
         }
         const admin_keys = (await rpc_client.account.read_account({ email: EMAIL, })).access_keys;
         account.name = user_a;
         account.email = user_a;
         const user_a_keys = (await rpc_client.account.create_account(account)).access_keys;
-        s3_creds.credentials.accessKeyId = user_a_keys[0].access_key.unwrap();
-        s3_creds.credentials.secretAccessKey = user_a_keys[0].secret_key.unwrap();
-        //s3_a = new AWS.S3(s3_creds);
-        s3_creds.credentials.accessKeyId = admin_keys[0].access_key.unwrap();
-        s3_creds.credentials.secretAccessKey = admin_keys[0].secret_key.unwrap();
-        s3_owner = new S3(s3_creds);
+        if (is_nc_coretest) {
+            s3_creds.credentials.accessKeyId = user_a_keys[0].access_key.unwrap();
+            s3_creds.credentials.secretAccessKey = user_a_keys[0].secret_key.unwrap();
+            s3_owner = new S3(s3_creds);
+        } else {
+            s3_creds.credentials.accessKeyId = admin_keys[0].access_key.unwrap();
+            s3_creds.credentials.secretAccessKey = admin_keys[0].secret_key.unwrap();
+            s3_owner = new S3(s3_creds);
+        }
     });
     mocha.describe('buckets creation', function() {
         mocha.it('create bucket BKT & enable lock', async function() {
@@ -977,4 +982,43 @@ mocha.describe('s3 worm', function() {
             }), 'NoSuchObjectLockConfiguration', 'The specified object does not have a ObjectLock configuration');
         });
     });
+
+    mocha.describe('no bypass permissions for user', async function() {
+        let version_id;
+        mocha.before(async function() {
+            const update_conf = {
+                email: user_a_mail,
+                nsfs_account_config: {allow_bypass_governance: 'false'}};
+            await rpc_client.account.update_account_s3_access(update_conf);
+
+            const conf = await s3_owner.putObject({
+                Bucket: BKT,
+                Key: OBJ1,
+                Body: file_body,
+                ContentType: 'text/plain',
+                ObjectLockMode: 'GOVERNANCE',
+                ObjectLockRetainUntilDate: tomorrow
+            });
+            version_id = conf.VersionId;
+        });
+
+        mocha.it('should fail to put retention without bypass flag', async function() {
+            await assert_throws_async(s3_owner.putObjectRetention({
+                Bucket: BKT,
+                Key: OBJ1,
+                Retention: { Mode: 'COMPLIANCE', RetainUntilDate: tomorrow },
+                VersionId: version_id,
+                BypassGovernanceRetention: true
+            }), 'AccessDenied', is_nc_coretest ? 'Access Denied because object protected by object lock.' : 'Access Denied');
+        });
+
+        mocha.it('should fail to delete object with retention without bypass flag', async function() {
+            await assert_throws_async(s3_owner.deleteObject({
+                Bucket: BKT,
+                Key: OBJ1,
+                VersionId: version_id,
+                BypassGovernanceRetention: true,
+            }), 'AccessDenied', is_nc_coretest ? 'Access Denied because object protected by object lock.' : 'Access Denied');
+        });
+    });
 });