Merge pull request #174 from Helene/analytics

Add analytics allowing inspect grafa-bridge performance

Author: Helene

source/analytics.py

@@ -0,0 +1,27 @@
+'''
+##############################################################################
+# Copyright 2024 IBM Corp.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##############################################################################
+
+Created on Feb 09, 2024
+
+@author: HWASSMAN
+'''
+
+global inspect
+inspect = False
+
+global urllib3_debug
+urllib3_debug = 0

source/collector.py

@@ -22,14 +22,15 @@
 
 import cherrypy
 import copy
+import analytics
 from queryHandler.Query import Query
 from messages import MSG
 from collections import defaultdict
 from typing import Optional, Any, List
 from threading import Thread
 from metadata import MetadataHandler
 from bridgeLogger import getBridgeLogger
-from utils import classattributes
+from utils import classattributes, cond_execution_time
 
 
 local_cache = set()
@@ -46,6 +47,7 @@ def __init__(self, columnInfo, dps, filtersMap):
 
         self.parse_tags(filtersMap)
 
+    @cond_execution_time(enabled=analytics.inspect)
     def parse_tags(self, filtersMap):
         tagsDict = defaultdict(set)
         logger = getBridgeLogger()
@@ -107,6 +109,9 @@ def __init__(self, name: str, desc: str, type: Optional[str] = None):
         self.desc = desc
         self.timeseries: list[TimeSeries] = []
 
+    def __format__(self, spec):
+        return f'{self.mname}_mTS'
+
     def str_descfmt(self, original_counters=False) -> [str]:
         """Format MetricTimeSeries description rows
             Output format:
@@ -267,6 +272,9 @@ def __init__(self, sensor: str, period: str, logger, request: QueryPolicy,
 
         self.prepare_static_metrics_data()
 
+    def __format__(self, spec):
+        return f'{self.sensor}_Collector'
+
     @property
     def md(self):
         return MetadataHandler()
@@ -370,13 +378,14 @@ def _collect(self):
                 self.metrics[columnInfo.keys[0].metric] = mt
         # self.logger.info(f'rows data {str(columnValues)}')
 
+    @cond_execution_time(enabled=analytics.inspect)
     def prepare_static_metrics_data(self):
         incl_metrics = list(self.request.metricsaggr.keys()
                             ) if self.request.metricsaggr else None
         self.setup_static_metrics_data(incl_metrics)
 
+    @cond_execution_time(enabled=analytics.inspect)
     def validate_query_filters(self):
-
         # check filterBy settings
         if self.request.filters:
 
@@ -424,6 +433,7 @@ def validate_query_filters(self):
                 raise cherrypy.HTTPError(
                     400, MSG['AttrNotValid'].format('filter'))
 
+    @cond_execution_time(enabled=analytics.inspect)
     def validate_group_tags(self):
         # check groupBy settings
         if self.request.grouptags:

source/opentsdb.py

@@ -22,10 +22,11 @@
 
 import cherrypy
 import re
+import analytics
 from messages import ERR, MSG
 from collections import defaultdict
 from collector import SensorCollector, QueryPolicy
-from utils import getTimeMultiplier
+from utils import getTimeMultiplier, execution_time, cond_execution_time
 from typing import List
 
 
@@ -49,6 +50,7 @@ def qh(self):
     def TOPO(self):
         return self.__md.metaData
 
+    @cond_execution_time(enabled=analytics.inspect)
     def format_response(self, data: dict, jreq: dict) -> List[dict]:
         respList = []
         metrics = set(data.values())
@@ -63,6 +65,7 @@ def format_response(self, data: dict, jreq: dict) -> List[dict]:
                 respList.append(res.to_dict(st.dps))
         return respList
 
+    @execution_time()
     def query(self, jreq: dict) -> List[dict]:
         resp = []
         collectors = []
@@ -107,6 +110,7 @@ def query(self, jreq: dict) -> List[dict]:
         cherrypy.response.headers['Access-Control-Allow-Origin'] = '*'
         return resp
 
+    @cond_execution_time(enabled=analytics.inspect)
     def build_collector(self, jreq: dict) -> SensorCollector:
 
         q = jreq.get('inputQuery')

source/prometheus.py

@@ -22,11 +22,12 @@
 
 import cherrypy
 import json
+import analytics
 from messages import MSG
 from typing import Optional
 from cherrypy.process.plugins import Monitor
 from collector import SensorCollector, QueryPolicy
-from utils import execution_time
+from utils import execution_time, cond_execution_time
 
 
 class PrometheusExporter(object):
@@ -56,6 +57,7 @@ def qh(self):
     def TOPO(self):
         return self.__md.metaData
 
+    @cond_execution_time(enabled=analytics.inspect)
     def format_response(self, data) -> [str]:
         resp = []
         for name, metric in data.items():
@@ -118,6 +120,7 @@ def initialize_cache_collectors(self):
                     frequency=collector.period,
                     name=thread_name).subscribe()
 
+    @cond_execution_time(enabled=analytics.inspect)
     def build_collector(self, sensor) -> SensorCollector:
 
         period = self.md.getSensorPeriod(sensor)
@@ -136,8 +139,8 @@ def build_collector(self, sensor) -> SensorCollector:
         request = QueryPolicy(**attrs)
         collector = SensorCollector(sensor, period, self.logger, request)
 
-        self.logger.trace(f'request instance {str(request.__dict__)}')
-        self.logger.trace(f'Created Collector instance {str(collector.__dict__)}')
+        # self.logger.trace(f'request instance {str(request.__dict__)}')
+        # self.logger.trace(f'Created Collector instance {str(collector.__dict__)}')
         return collector
 
     def GET(self, **params):

source/queryHandler/PerfmonRESTclient.py

@@ -20,6 +20,8 @@
 @author: HWASSMAN
 '''
 
+import analytics
+from utils import cond_execution_time
 # catch import failure on AIX since we will not be shipping our third-party libraries on AIX
 try:
     import requests
@@ -28,6 +30,11 @@
 except Exception:
     pass
 
+try:  # for Python 3
+    from http.client import HTTPConnection
+except ImportError:
+    from httplib import HTTPConnection
+HTTPConnection.debuglevel = analytics.urllib3_debug
 
 DEFAULT_HEADERS = {"Accept": "application/json",
                    "Content-type": "application/json"}
@@ -78,6 +85,7 @@ def __init__(self, logger, reqdata=None, session=None):
         self.requestData = reqdata
         self.logger = logger
 
+    @cond_execution_time(enabled=analytics.inspect)
     def doRequest(self):
         if self.requestData and isinstance(self.requestData, requests.Request):
             _prepRequest = self.session.prepare_request(self.requestData)

source/queryHandler/QueryHandler.py

@@ -24,10 +24,11 @@
 import operator
 import socket
 import time
+import analytics
 from collections import namedtuple, defaultdict
 from itertools import chain
 from typing import NamedTuple, Tuple
-from utils import execution_time
+from utils import cond_execution_time
 
 from .PerfmonRESTclient import perfHTTPrequestHelper, createRequestDataObj, getAuthHandler
 
@@ -433,7 +434,7 @@ def apiKeyData(self):
     def caCert(self):
         return self.__caCert
 
-    @execution_time()
+    @cond_execution_time(enabled=analytics.inspect)
     def getTopology(self, ignoreMetrics=False):
         '''
         Returns complete topology as a single JSON string
@@ -454,7 +455,6 @@ def getTopology(self, ignoreMetrics=False):
             self.logger.error(
                 'QueryHandler: getTopology response not valid json: {0} {1}'.format(res[:20], e))
 
-    @execution_time()
     def getAvailableMetrics(self):
         '''
         Returns output from topo -m
@@ -484,7 +484,7 @@ def deleteKeyFromTopology(self, keyStr, precheck=True):
             self.logger.error(
                 'QueryHandler: deleteKeysFromTopology response not valid json: {0} {1}'.format(response[:20], e))
 
-    @execution_time()
+    @cond_execution_time(enabled=analytics.inspect)
     def runQuery(self, query):
         '''
         runQuery: executes the given query based on the arguments.
@@ -509,7 +509,7 @@ def __do_RESTCall(self, endpoint, requestType='GET', params=None):
         Forward query request to the HTTPRequest client interface
         '''
 
-        self.logger.trace("__do_RESTcall invoke __ params: {} {} {}".format(endpoint, requestType, str(params)))
+        # self.logger.trace("__do_RESTcall invoke __ params: {} {} {}".format(endpoint, requestType, str(params)))
 
         try:
             _auth = getAuthHandler(*self.apiKeyData)
@@ -521,17 +521,17 @@ def __do_RESTCall(self, endpoint, requestType='GET', params=None):
             if _response.status_code == 200:
                 return _response.content.decode('utf-8', "strict")
             elif _response.status_code == 401:
-                self.logger.debug('Request headers:{}'.format(_response.request.headers))
-                self.logger.debug('Request url:{}'.format(_response.request.url))
+                self.logger.trace('Request headers:{}'.format(_response.request.headers))
+                self.logger.trace('Request url:{}'.format(_response.request.url))
                 msg = "Perfmon RESTcall error __ Server responded: {} {}".format(_response.status_code, _response.reason)
                 self.logger.details(msg)
                 raise PerfmonConnError("{} {}".format(_response.status_code, _response.reason))
             else:
                 msg = "Perfmon RESTcall error __ Server responded: {} {}".format(_response.status_code, _response.reason)
-                self.logger.debug(msg)
+                self.logger.trace(msg)
                 if _response.content:
                     contentMsg = _response.content.decode('utf-8', "strict")
-                    self.logger.details(f'Response content:{contentMsg}')
+                    self.logger.trace(f'Response content:{contentMsg}')
                 return None
         except TypeError as e:
             self.logger.exception(e)

source/utils.py

@@ -52,6 +52,37 @@ def wrapper(*args: Any, **kwargs: Any) -> T:
     return outer
 
 
+def cond_execution_time(enabled: bool = False, skip_attribute: bool = False) -> Callable[[Callable[..., T]], Callable[..., T]]:
+    """ Conditionally logs the name of the given function f with
+        passed parameter values and the time it takes to execute it.
+    """
+    def outer(f: Callable[..., T]) -> Callable[..., T]:
+        @wraps(f)
+        def wrapper(*args: Any, **kwargs: Any) -> T:
+            self = args[0]
+            if hasattr(self, 'logger'):
+                logger = self.logger
+            else:
+                from bridgeLogger import getBridgeLogger
+                logger = getBridgeLogger()
+            args_str = ', '.join(map(str, args[1:])) if len(args) > 1 else ''
+            kwargs_str = ', '.join(f'{k}={v}' for k, v in kwargs.items()) if len(kwargs) > 0 else ''
+            logger.trace(MSG['StartMethod'].format(f.__name__, ', '.join(filter(None, [args_str, kwargs_str]))))
+            t = time.time()
+            result = f(*args, **kwargs)
+            duration = time.time() - t
+            if not skip_attribute:
+                wrapper._execution_duration = duration  # type: ignore
+            logger.debug(MSG['RunMethod'].format(f.__name__, ', '.join(filter(None, [args_str, kwargs_str]))))
+            logger.debug(MSG['TimerInfo'].format(f.__name__, duration))
+            return result
+        return wrapper
+
+    def no_outer(f: Callable[..., T]) -> Callable[..., T]:
+        return f
+    return outer if enabled else no_outer
+
+
 def classattributes(default_attr: dict, more_allowed_attr: list):
     """ class __init__decorator
         Parses kwargs attributes, for optional arguments uses default values,