Merge "[Exporter] Add metrics for instances in verify_resize state"

Author: MOS CI

rockoon/exporter/collectors/openstack/nova.py

@@ -171,6 +171,11 @@ def init_families(self):
                 "Total number of instances in active state",
                 labels=[],
             ),
+            "verify_resize_instances": GaugeMetricFamily(
+                f"{self._name}_verify_resize_instances",
+                "Total number of instances in VERIFY_RESIZE status",
+                labels=[],
+            ),
             "hypervisor_instances": GaugeMetricFamily(
                 f"{self._name}_hypervisor_instances",
                 "Total number of instances per hypervisor",
@@ -206,6 +211,11 @@ def init_families(self):
                 "Total number of instances on all compute hosts in host aggregate.",
                 labels=["name"],
             ),
+            "instance_status": GaugeMetricFamily(
+                f"{self._name}_instance_status",
+                "Instance status (currently only VERIFY_RESIZE instances are exposed)",
+                labels=["name", "id", "status"],
+            ),
         }
         for resource_class in self.hypervisor_resource_classes:
             res[f"hypervisor_{resource_class}"] = GaugeMetricFamily(
@@ -414,9 +424,15 @@ def update_service_samples(self):
 
     @utils.timeit
     def update_instances_samples(self):
-        instances = {"total": 0, "active": 0, "error": 0}
+        instances = {
+            "total": 0,
+            "active": 0,
+            "error": 0,
+            "verify_resize": 0,
+        }
         hypervisor_instances = {}
         availability_zone_instances_total = {}
+        instance_status_samples = []
         for zone in self.cache.get("availability_zones", []):
             availability_zone_instances_total[zone["name"]] = 0
         for instance in self.oc.oc.compute.servers(all_projects=True):
@@ -428,13 +444,28 @@ def update_instances_samples(self):
                 instances[status] += 1
                 hypervisor_instances.setdefault(host, {"total": 0})
                 hypervisor_instances[host]["total"] += 1
+            if status == "verify_resize":
+                instance_status_samples.append(
+                    (
+                        [
+                            instance["name"],
+                            instance["id"],
+                            status,
+                        ],
+                        1,
+                    )
+                )
             if zone:
                 availability_zone_instances_total.setdefault(zone, 0)
                 availability_zone_instances_total[zone] += 1
 
         self.set_samples("instances", [([], instances["total"])])
-        for key in ["error", "active"]:
+        for key in ["error", "active", "verify_resize"]:
             self.set_samples(f"{key}_instances", [([], instances[key])])
+        self.set_samples(
+            "instance_status",
+            instance_status_samples,
+        )
 
         availability_zone_instances_samples = []
         for zone, total in availability_zone_instances_total.items():

rockoon/tests/functional/base.py

@@ -284,6 +284,19 @@ def server_reset_state(self, server, status, wait=True):
         if wait is True:
             waiters.wait_for_server_status(self.ocm, server, status=status)
 
+    def get_flavor_id(self, flavor_name):
+        return self.ocm.oc.compute.find_flavor(flavor_name).id
+
+    def resize_server(self, server, flavor_id, wait=True):
+        self.ocm.oc.compute.resize_server(server.id, flavor_id)
+        if wait is True:
+            waiters.wait_for_server_status(
+                self.ocm,
+                server,
+                status="VERIFY_RESIZE",
+                timeout=CONF.SERVER_RESIZE_TIMEOUT,
+            )
+
     @classmethod
     def lb_bundle_create(
         cls,

rockoon/tests/functional/config.py

@@ -25,6 +25,7 @@ def __init__(self):
         self.CIRROS_TEST_IMAGE_NAME = self.get_cirros_image()
         self.UBUNTU_TEST_IMAGE_NAME = "Ubuntu-18.04"
         self.TEST_FLAVOR_SMALL_NAME = "m1.small"
+        self.TEST_FLAVOR_TINY_NAME = "m1.tiny"
         self.TEST_FLAVOR_NAME = "m1.extra_tiny_test"
         self.TEST_SUBNET_RANGE = "10.20.30.0/24"
         self.TEST_SUBNET_RANGE_ALT = "10.20.31.0/24"
@@ -50,6 +51,8 @@ def __init__(self):
 
         # Time in seconds to wait for a server to change a status. Default is 60 seconds.
         self.SERVER_TIMEOUT = 60
+        # Time in seconds to wait for a server to resize. Default is 180 seconds.
+        self.SERVER_RESIZE_TIMEOUT = 60 * 3
         # Interval in seconds to check the server status. Default is 1 second.
         self.SERVER_READY_INTERVAL = 1
 

rockoon/tests/functional/exporter/parallel/test_nova_collector.py

@@ -7,6 +7,9 @@
 from rockoon.tests.functional.exporter import base
 from rockoon.tests.functional import data_utils
 from rockoon.tests.functional.base import LOG
+from rockoon.tests.functional import config
+
+CONF = config.Config()
 
 
 class NovaCollectorFunctionalTestCase(base.BaseFunctionalExporterTestCase):
@@ -16,6 +19,10 @@ class NovaCollectorFunctionalTestCase(base.BaseFunctionalExporterTestCase):
         "osdpl_nova_instances": {"labels": []},
         "osdpl_nova_active_instances": {"labels": []},
         "osdpl_nova_error_instances": {"labels": []},
+        "osdpl_nova_verify_resize_instances": {"labels": []},
+        # "osdpl_nova_instance_status": {
+        #     "labels": ["name", "id", "status"]
+        # },
         "osdpl_nova_hypervisor_instances": {"labels": ["host", "zone"]},
         # "osdpl_nova_aggregate_hosts": {"labels": ["name"]},
         # "osdpl_nova_host_aggregate_info": {"labels": ["hosts", "name"]},
@@ -218,6 +225,136 @@ def test_nova_error_instances(self):
             f"Expected numbers of error servers: {len(error_servers)}.",
         )
 
+    def test_nova_verify_resize_instances(self):
+        """Total number of instances in VERIFY_RESIZE status.
+
+        **Steps**
+
+        #. Get exporter metric `osdpl_nova_verify_resize_instances` with initial value
+        #. Create test instance
+        #. Resize instance and wait until it reaches VERIFY_RESIZE status
+        #. Check that number of VERIFY_RESIZE instances increased in exporter metrics
+        #. Confirm resize and delete test instance
+        #. Check that number of VERIFY_RESIZE instances decreased in exporter metrics
+        """
+
+        metric_name = "osdpl_nova_verify_resize_instances"
+
+        initial_metric = self.get_metric_after_refresh(
+            metric_name, self.scrape_collector
+        )
+        initial_servers = self.ocm.compute_get_all_servers(
+            status="VERIFY_RESIZE"
+        )
+
+        self.assertEqual(
+            int(initial_metric.samples[0].value),
+            len(initial_servers),
+            f"Current number of VERIFY_RESIZE servers in exporter's metrics is "
+            f"{int(initial_metric.samples[0].value)}. "
+            f"Expected number of VERIFY_RESIZE servers: {len(initial_servers)}.",
+        )
+        # Create a server with a smaller flavor to test resize
+        flavor_id = self.get_flavor_id(CONF.TEST_FLAVOR_TINY_NAME)
+        resize_flavor_id = self.get_flavor_id(CONF.TEST_FLAVOR_NAME)
+        server = self.server_create(flavorRef=flavor_id)
+        self.resize_server(server, resize_flavor_id)
+
+        metric_after_resize = self.get_metric_after_refresh(
+            metric_name, self.scrape_collector
+        )
+        servers_after_resize = self.ocm.compute_get_all_servers(
+            status="VERIFY_RESIZE"
+        )
+
+        self.assertEqual(
+            int(metric_after_resize.samples[0].value),
+            len(servers_after_resize),
+            f"Current number of VERIFY_RESIZE servers in exporter's metrics is "
+            f"{int(metric_after_resize.samples[0].value)}. "
+            f"Expected number of VERIFY_RESIZE servers: {len(servers_after_resize)}.",
+        )
+
+        self.server_delete(server)
+
+        metric = self.get_metric_after_refresh(
+            metric_name, self.scrape_collector
+        )
+        self.assertEqual(
+            int(metric.samples[0].value),
+            len(servers_after_resize) - 1,
+            f"Current number of VERIFY_RESIZE servers in exporter's metrics is "
+            f"{int(metric.samples[0].value)}. "
+            f"Expected number of VERIFY_RESIZE servers: {len(servers_after_resize) - 1}.",
+        )
+
+    def test_instance_status(self):
+        """Check that instance_status metric appears when VM is in VERIFY_RESIZE status.
+
+        **Steps:**
+
+        #. Create a test instance
+        #. Get initial samples for metric `osdpl_nova_instance_status`
+        #. Check that metric doesn't have sample for the test instance
+        #. Resize test instance
+        #. Refresh metric and check that a new sample with the VM's name, id and status appears
+        #. Delete the test instance
+        #. Refresh metric and check that the sample disappears
+        """
+        metric_name = "osdpl_nova_instance_status"
+
+        # Create a server with a smaller flavor to test resize
+        flavor_id = self.get_flavor_id(CONF.TEST_FLAVOR_TINY_NAME)
+        resize_flavor_id = self.get_flavor_id(CONF.TEST_FLAVOR_NAME)
+        server = self.server_create(flavorRef=flavor_id)
+        metric = self.get_metric_after_refresh(
+            metric_name, self.scrape_collector
+        )
+        labels = {
+            "id": server["id"],
+            "name": server["name"],
+            "status": server["status"].lower(),
+        }
+        samples = self.filter_metric_samples(metric, labels)
+        self.assertEqual(
+            len(samples),
+            0,
+            f"Instance_status metric contain sample for VM {server['name']}",
+        )
+
+        self.resize_server(server, resize_flavor_id)
+        server = self.ocm.oc.get_server(server.id)
+        labels = {
+            "id": server["id"],
+            "name": server["name"],
+            "status": server["status"].lower(),
+        }
+        metric = self.get_metric_after_refresh(
+            metric_name, self.scrape_collector
+        )
+        samples = self.filter_metric_samples(metric, labels)
+        self.assertEqual(
+            len(samples),
+            1,
+            f"Instance_status metric does not contain sample for VM {server['name']}",
+        )
+        self.assertCountEqual(
+            ["name", "id", "status"],
+            samples[0].labels.keys(),
+        )
+        self.server_delete(server)
+        metric_after_delete = self.get_metric_after_refresh(
+            metric_name, self.scrape_collector
+        )
+        deleted_samples = self.filter_metric_samples(
+            metric_after_delete, labels
+        )
+        self.assertEqual(
+            len(deleted_samples),
+            0,
+            f"Instance_status metric sample for VM {server['name']} still exists after deletion",
+        )
+
 
 @pytest.mark.xdist_group("exporter-compute-network")
 class NovaAvailabilityZonesTestCase(base.BaseFunctionalExporterTestCase):

rockoon/tests/functional/waiters.py

@@ -48,9 +48,10 @@ def wait_for_service_status_state(
             raise TimeoutError(message)
 
 
-def wait_for_server_status(openstack_client, server, status):
+def wait_for_server_status(
+    openstack_client, server, status, timeout=CONF.SERVER_TIMEOUT
+):
     start_time = int(time.time())
-    timeout = CONF.SERVER_TIMEOUT
     while True:
         server = openstack_client.oc.get_server(server.id)
         if server.status.upper() == status.upper():