Merge pull request #317 from Helene/raw_counters

Improve rawCounters setting management by PrometheusExporter

Author: Helene

source/config.ini

@@ -13,8 +13,14 @@
 # Port number the bridge listening on for Prometheus server requests;
 # prometheus = 9250
 
-# Sensor counters deltas will be exported by default.
-# Set True if you want export original sensor counters.
+# By default, the original values from the platform will be displayed alongside 
+# the timestamps at which they were measured. This can slow down a ZIMON query
+# performance.
+# Set rawCounters to False to accelerate query execution time. Keep in mind that 
+# the exposed timestamps will represent the time at which the value was collected 
+# from the platform and added to a bucket.
+# Sensors including  'increasing counters' metric types will  still run in 
+# rawCounters mode.
 rawCounters = True
 
 ##################### API Protocol scheme #####################################

source/messages.py

@@ -45,6 +45,7 @@
        'MetricErr': 'Metric {0} cannot be found. Please check if the corresponding sensor is configured',
        'InconsistentParams': 'Received parameters {} inconsistent with request parameters {}',
        'SensorDisabled': 'Sensor for metric {} is disabled',
+       'SensorForceRawData': 'Sensor {} includes metrics type counter, which need to be collected as raw data',
        'EndpointNotSupported': 'Endpoint {} you try to access is not supported',
        'EndpointNotSupportedForPort': 'Endpoint {} is not supported for port {}',
        'NoData': 'Empty results received',  # Please check the pmcollector is properly configured and running.

source/metadata.py

@@ -185,6 +185,22 @@ def GET(self, **params):
             resp = self.SensorsConfig
             # resp = json.dumps(resp)
 
+        # /metadata/sensorsconfig
+        elif '/metadata/sensormetrics' == cherrypy.request.script_name:
+            resp = {}
+            sensors = []
+            # cherrypy.response.headers['Content-Type'] = 'application/json'
+            if params.get('sensor') is None:
+                sensors = self.metaData.sensorsSpec.keys()
+            else:
+                sensor = params.get('sensor')
+                self.logger.info(f"Received request for endpoint /metadata/sensormetrics: {sensor}")
+                sensors.append(sensor)
+            for sensor in sensors:
+                metricsData = self.metaData.getSensorMetricTypes(sensor)
+                resp[sensor] = metricsData
+            # resp = json.dumps(resp)
+
         del cherrypy.response.headers['Allow']
         cherrypy.response.headers['Access-Control-Allow-Origin'] = '*'
         # cherrypy.response.headers['Content-Type'] = 'application/json'

source/prometheus.py

@@ -133,6 +133,10 @@ def build_collector(self, sensor) -> SensorCollector:
         if self.raw_data:
             attrs = {'sensor': sensor, 'period': period,
                      'nsamples': period, 'rawData': True}
+        elif "counter" in self.TOPO.getSensorMetricTypes(sensor).values():
+            attrs = {'sensor': sensor, 'period': period,
+                     'nsamples': period, 'rawData': True}
+            self.logger.debug(MSG['SensorForceRawData'].format(sensor))
         else:
             attrs = {'sensor': sensor, 'period': period,
                      'nsamples': 1}

source/queryHandler/Topo.py

@@ -241,6 +241,14 @@ def getSensorMetricNames(self, searchSensor: str) -> list:
             return metrics.values()
         return []
 
+    def getSensorMetricTypes(self, searchSensor: str) -> Dict[str, str]:
+        sensorMetricsTypes = {}
+        metrics = self.__metricsDef.get(searchSensor, None)
+        if metrics:
+            typesDict = self.__metricsType
+            sensorMetricsTypes = {metric: typesDict[metric] for metric in metrics.values()}
+        return sensorMetricsTypes
+
     def getSensorsForMeasurementMetrics(self, searchMetrics):
         sensorsList = []
         for metric in searchMetrics:

source/zimonGrafanaIntf.py

@@ -290,6 +290,12 @@ def main(argv):
                          {'request.dispatch': cherrypy.dispatch.MethodDispatcher()}
                          }
                         )
+    # query for list zimon sensor metrics
+    cherrypy.tree.mount(mdHandler, '/metadata/sensormetrics',
+                        {'/':
+                         {'request.dispatch': cherrypy.dispatch.MethodDispatcher()}
+                         }
+                        )
 
     # register OpenTSDB API endpoints
     if args.get('port', None):

tests/test_topo_parser.py

@@ -22,3 +22,21 @@ def test_case01():
     sensorLabels = topo.getSensorLabels(searchSensor)
     assert len(sensorLabels) > 0
     assert "gpfs_disk_name" in sensorLabels
+
+
+@with_setup(my_setup)
+def test_case02():
+    searchSensor = 'GPFSDiskCap'
+    typesDict = topo.getSensorMetricTypes(searchSensor)
+    assert len(typesDict) > 0
+    assert 'counter' not in typesDict.values()
+    assert 'gpfs_disk_disksize' in typesDict.keys()
+    assert typesDict['gpfs_disk_disksize'] == 'quantity'
+
+
+@with_setup(my_setup)
+def test_case03():
+    searchSensor = 'GPFSNSDFS'
+    typesDict = topo.getSensorMetricTypes(searchSensor)
+    assert len(typesDict) > 0
+    assert 'counter' in typesDict.values()