Merge pull request ceph#65480 from nzliu-dev/wip-admin-rest-api-for-setting-account-quota

rgw: Implement Admin REST APIs for Setting Account Quota

Reviewed-by: Ville Ojamo <[email protected]>
Reviewed-by: Casey Bodley <[email protected]>

Author: cbodley

doc/radosgw/adminops.rst

@@ -29,14 +29,10 @@ def rgwadmin_rest(connection, cmd, params=None, headers=None, raw=False):
     """
     log.info('radosgw-admin-rest: %s %s' % (cmd, params))
     put_cmds = ['create', 'link', 'add', 'set']
-    post_cmds = ['unlink', 'modify']
+    post_cmds = ['unlink', 'modify', 'post']
     delete_cmds = ['trim', 'rm', 'process']
     get_cmds = ['check', 'info', 'show', 'list', 'get', '']
 
-    bucket_sub_resources = ['object', 'policy', 'index']
-    user_sub_resources = ['subuser', 'key', 'caps', 'quota']
-    zone_sub_resources = ['pool', 'log', 'garbage']
-
     def get_cmd_method_and_handler(cmd):
         """
         Get the rest command and handler from information in cmd and
@@ -53,29 +49,21 @@ def get_cmd_method_and_handler(cmd):
 
     def get_resource(cmd):
         """
-        Get the name of the resource from information in cmd.
+        Always return (resource, subresource) as a 2-tuple.
+        Accepts:
+        ['user','info']                  -> ('user','')
+        [('user','key'),'create']        -> ('user','key')
+        ['info','']                      -> ('info','')
         """
-        if cmd[0] == 'bucket' or cmd[0] in bucket_sub_resources:
-            if cmd[0] == 'bucket':
-                return 'bucket', ''
-            else:
-                return 'bucket', cmd[0]
-        elif cmd[0] == 'user' or cmd[0] in user_sub_resources:
-            if cmd[0] == 'user':
-                return 'user', ''
-            else:
-                return 'user', cmd[0]
-        elif cmd[0] == 'usage':
-            return 'usage', ''
-        elif cmd[0] == 'info':
-            return 'info', ''
-        elif cmd[0] == 'ratelimit':
-            return 'ratelimit', ''
-        elif cmd[0] == 'zone' or cmd[0] in zone_sub_resources:
-            if cmd[0] == 'zone':
-                return 'zone', ''
+        r = cmd[0]
+        if isinstance(r, (list, tuple)):
+            if len(r) == 1:
+                return r[0], ''
             else:
-                return 'zone', cmd[0]
+                return r
+        if isinstance(r, str):
+            return r, ''
+        raise TypeError(f'Unsupported resource type for {str(r)}')
 
     def build_admin_request(conn, method, resource = '', headers=None, data='',
             query_args=None, params=None):
@@ -302,7 +290,7 @@ def task(ctx, config):
     admin_display_name = 'Ms. Admin User'
     admin_access_key = 'MH1WC2XQ1S8UISFDZC8W'
     admin_secret_key = 'dQyrTPA0s248YeN5bBv4ukvKU0kh54LWWywkrpoG'
-    admin_caps = 'users=read, write; usage=read, write; buckets=read, write; zone=read, write; info=read;ratelimit=read, write'
+    admin_caps = 'users=read, write; usage=read, write; buckets=read, write; zone=read, write; info=read; ratelimit=read, write; accounts=read, write'
 
     user1 = 'foo'
     user2 = 'fud'
@@ -320,6 +308,7 @@ def task(ctx, config):
     swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'
 
     bucket_name = 'myfoo'
+    account_id = 'RGW00000000000000001'
 
     # legend (test cases can be easily grep-ed out)
     # TESTCASE 'testname','object','method','operation','assertion'
@@ -476,7 +465,7 @@ def task(ctx, config):
 
     # TESTCASE 'add-keys','key','create','w/valid info','succeeds'
     (ret, out) = rgwadmin_rest(admin_conn,
-            ['key', 'create'],
+            [('user', 'key'), 'create'],
             {'uid' : user1,
              'access-key' : access_key2,
              'secret-key' : secret_key2
@@ -494,7 +483,7 @@ def task(ctx, config):
 
     # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
     (ret, out) = rgwadmin_rest(admin_conn,
-            ['key', 'rm'],
+            [('user', 'key'), 'rm'],
             {'uid' : user1,
              'access-key' : access_key2
             })
@@ -509,7 +498,7 @@ def task(ctx, config):
 
     # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
     (ret, out) = rgwadmin_rest(admin_conn,
-            ['subuser', 'create'],
+            [('user', 'subuser'), 'create'],
             {'subuser' : subuser1,
              'secret-key' : swift_secret1,
              'key-type' : 'swift'
@@ -526,7 +515,7 @@ def task(ctx, config):
 
     # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
     (ret, out) = rgwadmin_rest(admin_conn,
-            ['subuser', 'create'],
+            [('user', 'subuser'), 'create'],
             {'subuser' : subuser2,
              'secret-key' : swift_secret2,
              'key-type' : 'swift'
@@ -543,7 +532,7 @@ def task(ctx, config):
 
     # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
     (ret, out) = rgwadmin_rest(admin_conn,
-            ['key', 'rm'],
+            [('user', 'key'), 'rm'],
             {'subuser' : subuser1,
              'key-type' :'swift'
             })
@@ -555,7 +544,7 @@ def task(ctx, config):
 
     # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
     (ret, out) = rgwadmin_rest(admin_conn,
-            ['subuser', 'rm'],
+            [('user', 'subuser'), 'rm'],
             {'subuser' : subuser1
             })
 
@@ -566,7 +555,7 @@ def task(ctx, config):
 
     # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
     (ret, out) = rgwadmin_rest(admin_conn,
-            ['subuser', 'rm'],
+            [('user', 'subuser'), 'rm'],
             {'subuser' : subuser2,
              'key-type' : 'swift',
              '{purge-keys' :True
@@ -712,7 +701,7 @@ def task(ctx, config):
     key.set_contents_from_string(object_name)
 
     # now delete it
-    (ret, out) = rgwadmin_rest(admin_conn, ['object', 'rm'], {'bucket' : bucket_name, 'object' : object_name})
+    (ret, out) = rgwadmin_rest(admin_conn, [('bucket', 'object'), 'rm'], {'bucket' : bucket_name, 'object' : object_name})
     assert ret == 200
 
     # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
@@ -837,14 +826,14 @@ def task(ctx, config):
     # should be private already but guarantee it
     key.set_acl('private')
 
-    (ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], {'bucket' : bucket.name, 'object' : key.key})
+    (ret, out) = rgwadmin_rest(admin_conn, [('bucket', 'policy'), 'show'], {'bucket' : bucket.name, 'object' : key.key})
     assert ret == 200
     assert len(out['acl']['grant_map']) == 1
 
     # add another grantee by making the object public read
     key.set_acl('public-read')
 
-    (ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], {'bucket' : bucket.name, 'object' : key.key})
+    (ret, out) = rgwadmin_rest(admin_conn, [('bucket', 'policy'), 'show'], {'bucket' : bucket.name, 'object' : key.key})
     assert ret == 200
     assert len(out['acl']['grant_map']) == 2
 
@@ -865,12 +854,12 @@ def task(ctx, config):
 
     # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
     caps = 'usage=read'
-    (ret, out) = rgwadmin_rest(admin_conn, ['caps', 'add'], {'uid' :  user1, 'user-caps' : caps})
+    (ret, out) = rgwadmin_rest(admin_conn, [('user', 'caps'), 'add'], {'uid' :  user1, 'user-caps' : caps})
     assert ret == 200
     assert out[0]['perm'] == 'read'
 
     # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
-    (ret, out) = rgwadmin_rest(admin_conn, ['caps', 'rm'], {'uid' :  user1, 'user-caps' : caps})
+    (ret, out) = rgwadmin_rest(admin_conn, [('user', 'caps'), 'rm'], {'uid' :  user1, 'user-caps' : caps})
     assert ret == 200
     assert not out
 
@@ -1053,3 +1042,148 @@ def task(ctx, config):
     assert user_ratelimit['max_list_ops'] == 600
     assert user_ratelimit['max_delete_ops'] == 400
 
+    # TESTCASE 'create account' 'account' 'post' 'creating a new account' 'succeeds'
+    (ret, out) = rgwadmin_rest(admin_conn, ['account', 'post'], {'id' : account_id, 'name': account_id})
+    assert ret == 200
+
+    # TESTCASE 'account-info' 'account' 'info' 'getting the account info, including its quota, and checking for default values' 'succeeds'
+    (ret, out) = rgwadmin_rest(admin_conn, ['account', 'info'], {'id' : account_id})
+    assert ret == 200
+    account_quota = out['quota']
+    assert not account_quota["enabled"]
+    assert account_quota["max_size"] == -1
+    assert account_quota["max_objects"] == -1
+    bucket_quota = out['bucket_quota']
+    assert not bucket_quota["enabled"]
+    assert bucket_quota["max_size"] == -1
+    assert bucket_quota["max_objects"] == -1
+    assert out["max_users"] == 1000
+    assert out["max_roles"] == 1000
+    assert out["max_groups"] == 1000
+    assert out["max_buckets"] == 1000
+    assert out["max_access_keys"] == 4
+
+    # TESTCASE 'bucket-level-account-quota-set' 'account-quota-set' 'setting account quota at the bucket level. Other values should remain the same' 'succeeds'
+    (ret, out) = rgwadmin_rest(admin_conn, [('account', 'quota'), 'set'], {'id': account_id, 'quota-type' : 'bucket', 'max-size': 12345, 'max-objects' : 54321, 'enabled': True})
+    assert ret == 200
+    (ret, out) = rgwadmin_rest(admin_conn, ['account', 'info'], {'id' : account_id})
+    assert ret == 200
+    # verifying bucket-level quota has been changed
+    bucket_quota = out['bucket_quota']
+    assert bucket_quota["enabled"]
+    assert bucket_quota["max_size"] == 12345
+    assert bucket_quota["max_objects"] == 54321
+    # verifying the rest of the values remain the same
+    account_quota = out['quota']
+    assert not account_quota["enabled"]
+    assert account_quota["max_size"] == -1
+    assert account_quota["max_objects"] == -1
+    assert out["max_users"] == 1000
+    assert out["max_roles"] == 1000
+    assert out["max_groups"] == 1000
+    assert out["max_buckets"] == 1000
+    assert out["max_access_keys"] == 4
+    # cleanup the account
+    (ret, out) = rgwadmin_rest(admin_conn, ['account', 'rm'], {'id' : account_id})
+    assert ret == 200
+
+
+    # TESTCASE 'account-level-quota-set' 'account-quota-set' 'setting account quota at the account level. Other values should remain the same' 'succeeds'
+    # re-create the account
+    (ret, out) = rgwadmin_rest(admin_conn, ['account', 'post'], {'id' : account_id, 'name': account_id})
+    assert ret == 200
+    (ret, out) = rgwadmin_rest(admin_conn, [('account', 'quota'), 'set'], {'id': account_id, 'quota-type' : 'account', 'max-size': 12345, 'max-objects' : 54321, 'enabled': True})
+    assert ret == 200
+    (ret, out) = rgwadmin_rest(admin_conn, ['account', 'info'], {'id' : account_id})
+    assert ret == 200
+    # verifying account-level quota has been changed
+    account_quota = out['quota']
+    assert account_quota["enabled"]
+    assert account_quota["max_size"] == 12345
+    assert account_quota["max_objects"] == 54321
+    # verifying the rest of the values remain the same
+    bucket_quota = out['bucket_quota']
+    assert not bucket_quota["enabled"]
+    assert bucket_quota["max_size"] == -1
+    assert bucket_quota["max_objects"] == -1
+    assert out["max_users"] == 1000
+    assert out["max_roles"] == 1000
+    assert out["max_groups"] == 1000
+    assert out["max_buckets"] == 1000
+    assert out["max_access_keys"] == 4
+    # cleanup the account
+    (ret, out) = rgwadmin_rest(admin_conn, ['account', 'rm'], {'id' : account_id})
+    assert ret == 200
+
+    # TESTCASE 'account-quota-set-invalid-args' 'account-quota-set' 'verify set quota requests with invalid args fail' 'fails'
+    # re-create the account
+    (ret, out) = rgwadmin_rest(admin_conn, ['account', 'post'], {'id' : account_id, 'name': account_id})
+    assert ret == 200
+
+    # verify invalid quota-type fails
+    (ret, out) = rgwadmin_rest(admin_conn, [('account', 'quota'), 'set'], {'id': account_id, 'quota-type' : 'invalid', 'max-size': 12345, 'max-objects' : 54321, 'enabled': True})
+    assert ret == 400
+
+    # verify empty quota-type fails
+    (ret, out) = rgwadmin_rest(admin_conn, [('account', 'quota'), 'set'], {'id': account_id, 'max-size': 12345, 'max-objects' : 54321, 'enabled': True})
+    assert ret == 400
+
+    # verify request with no account id fails
+    (ret, out) = rgwadmin_rest(admin_conn, [('account', 'quota'), 'set'], {'quota-type' : 'bucket', 'max-size': 12345, 'max-objects' : 54321, 'enabled': True})
+    assert ret == 400
+
+    # cleanup the account
+    (ret, out) = rgwadmin_rest(admin_conn, ['account', 'rm'], {'id' : account_id})
+    assert ret == 200
+
+    # TESTCASE 'rm account caps' 'user-caps-rm' 'verify access after removing user caps for account read and write' 'fails'
+    (err, out) = rgwadmin(ctx, client, [
+            'caps', 'rm',
+            '--uid', admin_user,
+            '--caps', 'accounts=read, write'
+            ])
+    logging.error(out)
+    logging.error(err)
+    assert not err
+
+    # checking failed access to create account
+    (ret, out) = rgwadmin_rest(admin_conn, ['account', 'post'], {'id' : account_id, 'name': account_id})
+    assert ret == 403
+
+    # checking failed access to set quota
+    (ret, out) = rgwadmin_rest(admin_conn, [('account', 'quota'), 'set'], {'id': account_id, 'quota-type' : 'account', 'max-size': -1, 'max-objects' : -1, 'enabled': True})
+    assert ret == 403
+
+    # checking failed access to get account info
+    (ret, out) = rgwadmin_rest(admin_conn, ['account', 'info'], {'id' : account_id})
+    assert ret == 403
+
+    # checking failed access to remove account
+    (ret, out) = rgwadmin_rest(admin_conn, ['account', 'rm'], {'id' : account_id})
+    assert ret == 403
+
+    # TESTCASE 'add back account caps' 'user-caps-add' 'verify access after adding back user caps for account read and write' 'succeeds'
+    (err, out) = rgwadmin(ctx, client, [
+            'caps', 'add',
+            '--uid', admin_user,
+            '--caps', 'accounts=read, write'
+            ])
+    logging.error(out)
+    logging.error(err)
+    assert not err
+
+    # checking create account access
+    (ret, out) = rgwadmin_rest(admin_conn, ['account', 'post'], {'id' : account_id, 'name': account_id})
+    assert ret == 200
+
+    # checking set quota access
+    (ret, out) = rgwadmin_rest(admin_conn, [('account', 'quota'), 'set'], {'id': account_id, 'quota-type' : 'account', 'max-size': -1, 'max-objects' : -1, 'enabled': True})
+    assert ret == 200
+
+    # checking get account info access
+    (ret, out) = rgwadmin_rest(admin_conn, ['account', 'info'], {'id' : account_id})
+    assert ret == 200
+
+    # checking remove account access
+    (ret, out) = rgwadmin_rest(admin_conn, ['account', 'rm'], {'id' : account_id})
+    assert ret == 200