Add MonitorResourceAPIServerRequests, release 0.1.1.2

Author: sarahbx

README.md

@@ -2,12 +2,12 @@
 
 WIP
 
-# ocp_scale_utilites.threaded
+## ocp_scale_utilities.threaded
 * https://docs.python.org/3/library/concurrent.futures.html
 * https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor
 
 
-## Usage
+### Usage
 ```
 from ocp_resources.virtual_machine import VirtualMachine
 from ocp_scale_utilities.threaded.utils import (
@@ -35,3 +35,27 @@ def funcB():
     with ThreadedScaleResources(resources=vms, wait_for_status=VirtualMachine.Status.RUNNING):
         yield vms
 ```
+
+## ocp_scale_utilities.monitoring
+
+### Usage
+
+```
+from ocp_resources.virtual_machine import VirtualMachine
+from ocp_scale_utilities.monitoring import MonitorResourceAPIServerRequests
+from ocp_scale_utilities.threaded.scale import ThreadedScaleResources
+from ocp_utilities.monitoring import Prometheus
+
+monitor_api_requests = MonitorResourceAPIServerRequests(
+    prometheus=Prometheus(...),
+    resource_class=VirtualMachine,
+    idle_requests_value=int(virtual_machine_resource_idle),
+)
+
+monitor_api_requests.wait_for_idle()
+with ThreadedScaleResources(resources=vms):
+    monitor_api_requests.wait_for_idle()
+    yield vms
+monitor_api_requests.wait_for_idle()
+
+```

ocp_scale_utilities/constants.py

@@ -1 +1,4 @@
+TIMEOUT_30SEC = 30
+
 TIMEOUT_2MIN = 60 * 2
+TIMEOUT_5MIN = 60 * 5

ocp_scale_utilities/monitoring.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import logging
+
+from ocp_resources.resource import Resource
+from ocp_utilities.monitoring import Prometheus
+from timeout_sampler import TimeoutExpiredError, TimeoutSampler
+
+from ocp_scale_utilities.constants import TIMEOUT_5MIN, TIMEOUT_30SEC
+
+LOGGER = logging.getLogger(__name__)
+
+
+class MonitorResourceAPIServerRequests:
+    def __init__(
+        self,
+        prometheus: Prometheus,
+        resource_class: type[Resource],
+        idle_requests_value: float,
+        time_duration_seconds: int = TIMEOUT_5MIN,
+    ):
+        """
+        Monitor API Server Requests for a particular Resource
+
+        Args:
+            prometheus (Prometheus): Prometheus object from ocp_utilities
+            resource_class (Resource): Resource class to monitor
+            time_duration_seconds (int, optional): Time duration to use with prometheus query
+            idle_requests_value (float, optional): Minimum value indicating an 'idle' state
+        """
+        self.prometheus = prometheus
+        self.resource_class = resource_class
+        self.time_duration_seconds = time_duration_seconds
+        self.idle_requests_value = idle_requests_value
+
+        self.apiserver_requests_query = (
+            "sum by (resource) (rate(apiserver_request_total{"
+            f'group="{self.resource_class.api_group}",'
+            f'resource="{self.resource_class.kind.lower()}s"'
+            f"}}[{self.time_duration_seconds}s]))"
+        )
+
+    def _initial_wait(self) -> None:
+        """
+        The initial state is unknown, and will be unknown, it cannot be guaranteed.
+        The code calling this could be the first code that upsets the cluster, or not.
+        Wait for the cluster to either stay silent or become noisy.
+        """
+        initial_silence_count = 0
+        initial_noise_count = 0
+        sampler = TimeoutSampler(
+            wait_timeout=TIMEOUT_30SEC,
+            sleep=5,
+            func=self.prometheus.query_sampler,
+            query=self.apiserver_requests_query,
+        )
+        try:
+            for sample in sampler:
+                if sample:
+                    value = float(sample[0]["value"][1])
+                    if value < self.idle_requests_value:
+                        initial_noise_count = 0
+                        initial_silence_count += 1
+                    else:
+                        initial_silence_count = 0
+                        initial_noise_count += 1
+
+                    if initial_silence_count > 5 or initial_noise_count > 2:
+                        break
+        except TimeoutExpiredError:
+            pass
+
+    def wait_for_idle(self) -> None:
+        """
+        Wait for 'idle' cluster state based on provided Resource
+        """
+        self._initial_wait()
+
+        sampler = TimeoutSampler(
+            wait_timeout=self.time_duration_seconds * 2,
+            sleep=5,
+            func=self.prometheus.query_sampler,
+            query=self.apiserver_requests_query,
+        )
+        sample = None
+        try:
+            for sample in sampler:
+                if sample and float(sample[0]["value"][1]) < self.idle_requests_value:
+                    return
+        except TimeoutExpiredError:
+            LOGGER.error(
+                f"Metric value: {sample} of {self.apiserver_requests_query!r} "
+                f"is not below minimum: {self.idle_requests_value}"
+            )
+            raise

pyproject.toml

@@ -8,8 +8,9 @@ addopts = [
   "--cov-config=pyproject.toml",
   "--cov-report=html",
   "--cov-report=term",
-  "--cov=ocp_scale_utilites.threaded.scale",
-  "--cov=ocp_scale_utilites.threaded.utils",
+  "--cov=ocp_scale_utilities.monitoring",
+  "--cov=ocp_scale_utilities.threaded.scale",
+  "--cov=ocp_scale_utilities.threaded.utils",
 ]
 
 [tool.coverage.run]
@@ -47,7 +48,7 @@ dev-dependencies = [ "ipdb>=0.13.13", "ipython>=8.12.3" ]
 [project]
 name = "openshift-python-scale-utilities"
 requires-python = ">=3.12.0"
-version = "0.1.1.1"
+version = "0.1.1.2"
 description = "OpenShift Python Scale Utilities"
 readme = "README.md"
 license = "Apache-2.0"
@@ -60,6 +61,7 @@ dependencies = [
   "pytest>=8.3.3",
   "timeout-sampler>=1.0.1",
   "openshift-python-wrapper~=11.0.10",
+  "openshift-python-utilities>=6.0.0",
 ]