Merge pull request #78 from IBM/SLAFix

SLA Changes

Author: baslerj

CHANGELOG.md

@@ -16,6 +16,20 @@ straightforward as possible.
 
 ### Fixed
 
+## [1.0.1] - 2021-08-27
+
+### Fixed
+
+* Reverts SLA-Endpoint changes to stop counting removed VM's.
+  * Pagesize code inside of the rest-request remains unchanged.
+
+### Changed
+
+* Changes SLA-Request count-field name and group name to a better matching candidate.
+  * Removes the need to cut of prefix via regex
+  * Removes associated regex code
+* Changes REST-Client query time measurement from using a timer to using the result-internal timer.
+
 ## [1.0.0] - 2021-08-26
 
 ### Major changes summary

python/influx/definitions.py

@@ -486,6 +486,7 @@ def add_table_definitions(cls, database: Database):
                 'vmCountBySLA':     Datatype.INT
             },
             tags=[
+                'slaId',
                 'slaName'
             ],
             retention_policy=cls._RP_DAYS_90(),

python/sppConnection/api_queries.py

@@ -4,9 +4,12 @@
     ApiQueries
 """
 import logging
-from typing import List, Dict, Any
+from typing import List, Dict, Any, Optional
 import json
+import urllib.parse
 
+from utils.execption_utils import ExceptionUtils
+from utils.spp_utils import SppUtils
 from sppConnection.rest_client import RequestType, RestClient
 
 
@@ -108,33 +111,67 @@ def get_all_vms(self) -> List[Dict[str, Any]]:
 
     def get_vms_per_sla(self) -> List[Dict[str, Any]]:
         """retrieves and calculates all vmware per SLA."""
-        endpoint = "/api/endeavour/catalog/recovery/hypervisorvm"
-        params = {
-            "action": "aggregate",
-            "pageSize": None
-        }
-        # other options: volume, vm, tag, tagcategory
-        post_data = {
-            "op":[
-                {
-                    "operation": "count",
-                    "fieldname": "protectionInfo.policyName",
-                    "outputname": "vmCountBySLA" # buggy request, does actually not change anything
-                }
-            ],
-            "group": [
-                "protectionInfo.policyName"
-            ]
-        }
 
-        return self.__rest_client.get_objects(
+        endpoint = "/ngp/slapolicy"
+        allow_list = ["name", "id"]
+        array_name = "slapolicies"
+
+        sla_policty_list = self.__rest_client.get_objects(
             endpoint=endpoint,
-            params=params,
-            request_type=RequestType.POST,
-            post_data=post_data,
-            add_time_stamp=True,
-            array_name="results")
+            allow_list=allow_list,
+            array_name=array_name,
+            add_time_stamp=False
+        )
+
+        result_list: List[Dict[str, Any]] = []
+        for sla_policty in sla_policty_list:
+            try:
+                sla_name: str = sla_policty["name"]
+            except KeyError as error:
+                ExceptionUtils.exception_info(error, extra_message="skipping one sla entry due missing name.")
+                continue
+            sla_id: Optional[str] = sla_policty.get("id", None)
+
+            result_dict: Dict[str, Any] = {}
+
+            ## hotadd:
+            sla_name = urllib.parse.quote_plus(sla_name)
+
+            endpoint = "/api/hypervisor/search"
+            params = {
+                "resourceType": "vm",
+                "from": "hlo",
+                "pageSize": 1,
+                "filter": json.dumps([
+                    {
+                        "property": "storageProfileName",
+                        "value": sla_name,
+                        "op": "="
+                    }
+                ])
+            }
+            # other options: volume, vm, tag, tagcategory
+            post_data = {
+                "name": "*",
+                "hypervisorType": "vmware",
+            }
+
+            (response_json, _) = self.__rest_client.query_url(
+                self.__rest_client.get_url(endpoint),
+                params,
+                RequestType.POST,
+                post_data)
+
+            result_dict["slaName"] = sla_name
+            result_dict["slaId"] = sla_id
+            result_dict["vmCountBySLA"] = response_json.get("total", None)
+
+            time_key, time = SppUtils.get_capture_timestamp_sec()
+            result_dict[time_key] = time
+
+            result_list.append(result_dict)
 
+        return result_list
 
     def get_sla_dump(self) -> List[Dict[str, Any]]:
         """retrieves all storage profiles."""

python/sppConnection/rest_client.py

@@ -357,7 +357,6 @@ def query_url(
 
             # send the query
             try:
-                start_time = time.perf_counter()
                 if(request_type == RequestType.GET):
                     response_query = get(
                         url=url, headers=self.__headers, verify=False,
@@ -368,8 +367,7 @@ def query_url(
                         url=url, headers=self.__headers, verify=False,
                         params=params, json=post_data, auth=auth,
                         timeout=(self.__initial_connection_timeout, self.__timeout))
-                end_time = time.perf_counter()
-                send_time = (end_time - start_time)
+                send_time = response_query.elapsed.total_seconds()
 
             except ReadTimeout as timeout_error:
 

python/sppmon.py

@@ -65,6 +65,7 @@
  08/22/2021 version 0.15   Added --fullLogs argument and reduced regular/loaded joblog query to SUMMARY-Only
  08/25/2021 version 0.15.1 Replaced SLA-Endpoint by so-far unknown endpoint, bringing it in line with other api-requests.
  08/27/2021 version 1.0.0  Release of SPPMon
+ 08/27/2021 version 1.0.1  Reverted parts of the SLA-Endpoint change
 """
 from __future__ import annotations
 
@@ -93,7 +94,7 @@
 from utils.spp_utils import SppUtils
 
 # Version:
-VERSION = "1.0.0  (2021/08/27)"
+VERSION = "1.0.1  (2021/08/27)"
 
 
 # ----------------------------------------------------------------------------

python/sppmonMethods/protection.py

@@ -6,7 +6,6 @@
 """
 import logging
 
-from re import compile, sub
 from typing import List, Dict, Any, Union, Optional
 
 from influx.influx_client import InfluxClient
@@ -55,25 +54,8 @@ def vms_per_sla(self) -> None:
         LOGGER.info("> calculating number of VMs per SLA")
         result = MethodUtils.query_something(
             name="VMs per SLA",
-            source_func=self.__api_queries.get_vms_per_sla,
-            rename_tuples=[
-                ("_id.protectionInfo.policyName", "slaName"),
-                ("count", "vmCountBySLA") # buggy request
-            ],
-            deactivate_verbose=True
+            source_func=self.__api_queries.get_vms_per_sla
         )
-
-        # the endpoint offers a prefix `vmware_`
-        # due to compability reasons the prefix is removed
-        # may be reintroduced if other types occur.
-        pattern = compile(r"^vmware_")
-        for sla in result:
-            # replaces the first occurence
-            sla['slaName'] = sub(pattern, "", sla['slaName'], 1)
-
-        if(self.__verbose):
-            MethodUtils.my_print(result)
-
         LOGGER.info(">> inserting number of VMs per SLA into DB")
         self.__influx_client.insert_dicts_to_buffer(
             table_name="slaStats", list_with_dicts=result)