Update quota apis with keystone limits and usage

This makes use of the keystone APIs to get limits from Keystone when
showing the user the limits on their project.

Note we also change the default in_use amount from -1, which is what the
no op driver originally used, to 0, which matches what the db driver
typically returns for deprecated quota values, like floating ip limits.
This seems a more sane value to respond with, given we don't count the
usage for those values.

blueprint unified-limits-nova

Change-Id: I933dc135a364b14ddadc8eee67b42d8e1278a9ae

Author: JohnGarbutt

nova/quota.py

@@ -29,6 +29,7 @@
 from nova.db.main import api as main_db_api
 from nova import exception
 from nova.limit import local as local_limit
+from nova.limit import placement as placement_limit
 from nova import objects
 from nova.scheduler.client import report
 from nova import utils
@@ -806,17 +807,17 @@ def get_class_quotas(self, context, resources, quota_class):
 
     def get_defaults(self, context, resources):
         local_limits = local_limit.get_legacy_default_limits()
-        # TODO(melwitt): This is temporary when we are in a state where cores,
-        # ram, and instances quota limits are not known/enforced with unified
-        # limits yet. This will occur in later patches and when it does, we
-        # will change the default to 0 to signal to operators that they need to
-        # register a limit for a resource before that resource will be
-        # allocated.
-        # Default to unlimited, as per no-op for everything that isn't
-        # a local limit
+        # Note we get 0 if there is no registered limit,
+        # to mirror oslo_limit behaviour when there is no registered limit
+        placement_limits = placement_limit.get_legacy_default_limits()
         quotas = {}
         for resource in resources.values():
-            quotas[resource.name] = local_limits.get(resource.name, -1)
+            if resource.name in placement_limits:
+                quotas[resource.name] = placement_limits[resource.name]
+            else:
+                # return -1 for things like security_group_rules
+                # that are neither a keystone limit or a local limit
+                quotas[resource.name] = local_limits.get(resource.name, -1)
 
         return quotas
 
@@ -829,21 +830,39 @@ def get_project_quotas(self, context, resources, project_id,
         if remains:
             raise NotImplementedError("remains")
 
-        local_limits = self.get_class_quotas(context, resources, quota_class)
-        local_in_use = {}
-        if usages:
-            local_in_use = local_limit.get_in_use(context, project_id)
+        local_limits = local_limit.get_legacy_default_limits()
+        # keystone limits always returns core, ram and instances
+        # if nothing set in keystone, we get back 0, i.e. don't allow
+        placement_limits = placement_limit.get_legacy_project_limits(
+            project_id)
 
-        quotas = {}
-        # As we only apply limits to resources we know about,
-        # we return unlimited (-1) for all other resources
+        project_quotas = {}
         for resource in resources.values():
-            quota = {"limit": local_limits.get(resource.name, -1)}
-            if usages:
-                quota["in_use"] = local_in_use.get(resource.name, -1)
-            quotas[resource.name] = quota
+            if resource.name in placement_limits:
+                limit = placement_limits[resource.name]
+            else:
+                # return -1 for things like security_group_rules
+                # that are neither a keystone limit or a local limit
+                limit = local_limits.get(resource.name, -1)
+            project_quotas[resource.name] = {"limit": limit}
 
-        return quotas
+        if usages:
+            local_in_use = local_limit.get_in_use(context, project_id)
+            p_in_use = placement_limit.get_legacy_counts(context, project_id)
+
+            for resource in resources.values():
+                # default to 0 for resources that are deprecated,
+                # i.e. not in keystone or local limits, such that we
+                # are API compatible with what was returned with
+                # the db driver, even though noop driver returned -1
+                usage_count = 0
+                if resource.name in local_in_use:
+                    usage_count = local_in_use[resource.name]
+                if resource.name in p_in_use:
+                    usage_count = p_in_use[resource.name]
+                project_quotas[resource.name]["in_use"] = usage_count
+
+        return project_quotas
 
     def get_user_quotas(self, context, resources, project_id, user_id,
                         quota_class=None, usages=True):

nova/tests/unit/api/openstack/compute/test_limits.py

@@ -31,6 +31,7 @@
 import nova.context
 from nova import exception
 from nova.limit import local as local_limit
+from nova.limit import placement as placement_limit
 from nova import objects
 from nova.policies import limits as l_policies
 from nova import quota
@@ -564,8 +565,12 @@ def setUp(self):
                      local_limit.SERVER_GROUP_MEMBERS: 10}
         self.useFixture(limit_fixture.LimitFixture(reglimits, {}))
 
+    @mock.patch.object(placement_limit, "get_legacy_counts")
+    @mock.patch.object(placement_limit, "get_legacy_project_limits")
     @mock.patch.object(objects.InstanceGroupList, "get_counts")
-    def test_index_v21(self, mock_count):
+    def test_index_v21(self, mock_count, mock_proj, mock_kcount):
+        mock_proj.return_value = {"instances": 1, "cores": 2, "ram": 3}
+        mock_kcount.return_value = {"instances": 4, "cores": 5, "ram": 6}
         mock_count.return_value = {'project': {'server_groups': 9}}
         req = fakes.HTTPRequest.blank("/")
         response = self.controller.index(req)
@@ -581,24 +586,28 @@ def test_index_v21(self, mock_count):
                     'maxServerGroupMembers': 10,
                     'maxServerGroups': 12,
                     'maxServerMeta': 128,
-                    'maxTotalCores': -1,
+                    'maxTotalCores': 2,
                     'maxTotalFloatingIps': -1,
-                    'maxTotalInstances': -1,
+                    'maxTotalInstances': 1,
                     'maxTotalKeypairs': 100,
-                    'maxTotalRAMSize': -1,
-                    'totalCoresUsed': -1,
-                    'totalFloatingIpsUsed': -1,
-                    'totalInstancesUsed': -1,
-                    'totalRAMUsed': -1,
-                    'totalSecurityGroupsUsed': -1,
+                    'maxTotalRAMSize': 3,
+                    'totalCoresUsed': 5,
+                    'totalFloatingIpsUsed': 0,
+                    'totalInstancesUsed': 4,
+                    'totalRAMUsed': 6,
+                    'totalSecurityGroupsUsed': 0,
                     'totalServerGroupsUsed': 9,
                 },
             },
         }
         self.assertEqual(expected_response, response)
 
+    @mock.patch.object(placement_limit, "get_legacy_counts")
+    @mock.patch.object(placement_limit, "get_legacy_project_limits")
     @mock.patch.object(objects.InstanceGroupList, "get_counts")
-    def test_index_v275(self, mock_count):
+    def test_index_v275(self, mock_count, mock_proj, mock_kcount):
+        mock_proj.return_value = {"instances": 1, "cores": 2, "ram": 3}
+        mock_kcount.return_value = {"instances": 4, "cores": 5, "ram": 6}
         mock_count.return_value = {'project': {'server_groups': 9}}
         req = fakes.HTTPRequest.blank("/?tenant_id=faketenant",
                                       version='2.75')
@@ -610,13 +619,13 @@ def test_index_v275(self, mock_count):
                     'maxServerGroupMembers': 10,
                     'maxServerGroups': 12,
                     'maxServerMeta': 128,
-                    'maxTotalCores': -1,
-                    'maxTotalInstances': -1,
+                    'maxTotalCores': 2,
+                    'maxTotalInstances': 1,
                     'maxTotalKeypairs': 100,
-                    'maxTotalRAMSize': -1,
-                    'totalCoresUsed': -1,
-                    'totalInstancesUsed': -1,
-                    'totalRAMUsed': -1,
+                    'maxTotalRAMSize': 3,
+                    'totalCoresUsed': 5,
+                    'totalInstancesUsed': 4,
+                    'totalRAMUsed': 6,
                     'totalServerGroupsUsed': 9,
                 },
             },

nova/tests/unit/api/openstack/compute/test_quota_classes.py

@@ -21,6 +21,7 @@
        as quota_classes_v21
 from nova import exception
 from nova.limit import local as local_limit
+from nova.limit import placement as placement_limit
 from nova import objects
 from nova import test
 from nova.tests.unit.api.openstack import fakes
@@ -279,20 +280,22 @@ def setUp(self):
                      local_limit.SERVER_GROUP_MEMBERS: 10}
         self.useFixture(limit_fixture.LimitFixture(reglimits, {}))
 
-    def test_show_v21(self):
+    @mock.patch.object(placement_limit, "get_legacy_default_limits")
+    def test_show_v21(self, mock_default):
+        mock_default.return_value = {"instances": 1, "cores": 2, "ram": 3}
         req = fakes.HTTPRequest.blank("")
         response = self.controller.show(req, "test_class")
         expected_response = {
             'quota_class_set': {
                 'id': 'test_class',
-                'cores': -1,
+                'cores': 2,
                 'fixed_ips': -1,
                 'floating_ips': -1,
-                'ram': -1,
+                'ram': 3,
                 'injected_file_content_bytes': 10240,
                 'injected_file_path_bytes': 255,
                 'injected_files': 5,
-                'instances': -1,
+                'instances': 1,
                 'key_pairs': 100,
                 'metadata_items': 128,
                 'security_group_rules': -1,
@@ -301,15 +304,17 @@ def test_show_v21(self):
         }
         self.assertEqual(expected_response, response)
 
-    def test_show_v257(self):
+    @mock.patch.object(placement_limit, "get_legacy_default_limits")
+    def test_show_v257(self, mock_default):
+        mock_default.return_value = {"instances": 1, "cores": 2, "ram": 3}
         req = fakes.HTTPRequest.blank("", version='2.57')
         response = self.controller.show(req, "default")
         expected_response = {
             'quota_class_set': {
                 'id': 'default',
-                'cores': -1,
-                'instances': -1,
-                'ram': -1,
+                'cores': 2,
+                'instances': 1,
+                'ram': 3,
                 'key_pairs': 100,
                 'metadata_items': 128,
                 'server_group_members': 10,
@@ -325,23 +330,25 @@ def test_update_still_rejects_badrequests(self):
         self.assertRaises(exception.ValidationError, self.controller.update,
                           req, 'test_class', body=body)
 
+    @mock.patch.object(placement_limit, "get_legacy_default_limits")
     @mock.patch.object(objects.Quotas, "update_class")
-    def test_update_v21(self, mock_update):
+    def test_update_v21(self, mock_update, mock_default):
+        mock_default.return_value = {"instances": 1, "cores": 2, "ram": 3}
         req = fakes.HTTPRequest.blank("")
         body = {'quota_class_set': {'ram': 51200}}
         response = self.controller.update(req, 'default', body=body)
         expected_response = {
             'quota_class_set': {
-                'cores': -1,
+                'cores': 2,
                 'fixed_ips': -1,
                 'floating_ips': -1,
                 'injected_file_content_bytes': 10240,
                 'injected_file_path_bytes': 255,
                 'injected_files': 5,
-                'instances': -1,
+                'instances': 1,
                 'key_pairs': 100,
                 'metadata_items': 128,
-                'ram': -1,
+                'ram': 3,
                 'security_group_rules': -1,
                 'security_groups': -1
             }
@@ -350,16 +357,18 @@ def test_update_v21(self, mock_update):
         # TODO(johngarbutt) we should be proxying to keystone
         self.assertEqual(0, mock_update.call_count)
 
+    @mock.patch.object(placement_limit, "get_legacy_default_limits")
     @mock.patch.object(objects.Quotas, "update_class")
-    def test_update_v257(self, mock_update):
+    def test_update_v257(self, mock_update, mock_default):
+        mock_default.return_value = {"instances": 1, "cores": 2, "ram": 3}
         req = fakes.HTTPRequest.blank("", version='2.57')
         body = {'quota_class_set': {'ram': 51200}}
         response = self.controller.update(req, 'default', body=body)
         expected_response = {
             'quota_class_set': {
-                'cores': -1,
-                'instances': -1,
-                'ram': -1,
+                'cores': 2,
+                'instances': 1,
+                'ram': 3,
                 'key_pairs': 100,
                 'metadata_items': 128,
                 'server_group_members': 10,