Merge pull request #170 from Helene/code_improvements

Internal code improvements

Author: Helene

source/collector.py

@@ -32,7 +32,7 @@
 from utils import classattributes
 
 
-local_cache = []
+local_cache = set()
 
 
 class TimeSeries(object):
@@ -56,7 +56,7 @@ def parse_tags(self, filtersMap):
                 'Single ts identifiers', ', '.join(ident)))
             found = False
             for filtersDict in filtersMap:
-                if all((value in filtersDict.values()) for value in ident):
+                if set(filtersDict.values()) == set(ident):
                     logger.trace(MSG['ReceivAttrValues'].format(
                         'filtersKeys', ', '.join(filtersDict.keys())))
                     if len(self.columnInfo.keys) == 1:
@@ -68,17 +68,16 @@ def parse_tags(self, filtersMap):
                     break
             # detected zimon key, do we need refresh local TOPO?
             if not found:
-                already_reported = False
-                for cache_item in local_cache:
-                    if set(cache_item) == set(ident):
-                        logger.trace(MSG['NewKeyAlreadyReported'].format(ident))
-                        already_reported = True
-                        break
-                if not already_reported:
+                cache_size = len(local_cache)
+                local_cache.union(ident)
+                updated_size = len(local_cache)
+                if updated_size > cache_size:
                     logger.trace(MSG['NewKeyDetected'].format(ident))
                     local_cache.append(ident)
                     md = MetadataHandler()
                     Thread(name='AdHocMetaDataUpdate', target=md.update).start()
+                else:
+                    logger.trace(MSG['NewKeyAlreadyReported'].format(ident))
 
         for _key, _values in tagsDict.items():
             if len(_values) > 1:
@@ -357,10 +356,8 @@ def _collect(self):
             for value, columnInfo in zip(row.values, res.columnInfos):
                 columnValues[columnInfo][row.tstamp] = value
 
-        timeseries = []
         for columnInfo, dps in columnValues.items():
             ts = TimeSeries(columnInfo, dps, self.filtersMap)
-            timeseries.append(ts)
             if self.metrics.get(columnInfo.keys[0].metric) is not None:
                 self.logger.trace(MSG['MetricInResults'].format(
                     columnInfo.keys[0].metric))
@@ -370,7 +367,7 @@ def _collect(self):
                 self.logger.warning(MSG['MetricNotInResults'].format(
                     columnInfo.keys[0].metric))
                 mt = MetricTimeSeries(columnInfo.keys[0].metric, '')
-                mt.timeseries = timeseries
+                mt.timeseries.append(ts)
                 self.metrics[columnInfo.keys[0].metric] = mt
         # self.logger.info(f'rows data {str(columnValues)}')
 
@@ -432,16 +429,16 @@ def validate_group_tags(self):
 
         # check groupBy settings
         if self.request.grouptags:
-            filter_keys = self.md.metaData.getAllFilterKeysForSensor(
-                self.sensor)
+            filter_keys = set()
+            for filter in self.filtersMap:
+                filter_keys.update(filter.keys())
             if not filter_keys:
                 self.logger.error(MSG['GroupByErr'])
                 raise cherrypy.HTTPError(
-                    400, MSG['AttrNotValid'].format('filter'))
-            groupKeys = self.request.grouptags
-            if not all(key in filter_keys for key in groupKeys):
-                self.logger.error(MSG['AttrNotValid'].format('groupBy'))
+                    400, MSG['AttrNotValid'].format('groupBy key'))
+            if not (set(self.request.grouptags)).issubset(filter_keys):
+                self.logger.error(MSG['AttrNotValid'].format('groupBy key'))
                 self.logger.error(MSG['ReceivAttrValues'].format(
-                    'groupBy', ", ".join(filter_keys)))
+                    'groupBy keys', ", ".join(filter_keys)))
                 raise cherrypy.HTTPError(
-                    400, MSG['AttrNotValid'].format('filter'))
+                    400, MSG['AttrNotValid'].format('groupBy key'))

source/queryHandler/Topo.py

@@ -209,7 +209,7 @@ def getSensorForMetric(self, searchMetric):
         if (searchMetric.find("(") >= 0):
             searchMetric = searchMetric[searchMetric.find("(") + 1:-1]
         for sensor, metrics in self.__metricsDef.items():
-            if searchMetric in metrics.values():
+            if searchMetric in set(metrics.values()):
                 return sensor
         return None
 
@@ -245,7 +245,7 @@ def getAllFilterMapsForSensor(self, searchSensor):
         based on metadata topology returned from zimon "topo".
         '''
         filtersMaps = []
-        if searchSensor in self.allFiltersMaps.keys():
+        if searchSensor in set(self.allFiltersMaps.keys()):
             filtersMaps.extend(self.allFiltersMaps[searchSensor])
         return filtersMaps
 
@@ -282,13 +282,11 @@ def getAllFilterKeysForMetric(self, searchMetric):
         return keys
 
     def getAllFilterKeysForSensor(self, searchSensor):
-        keys = []
+        filter_keys = set()
         filtersMap = self.getAllFilterMapsForSensor(searchSensor)
-        for a in filtersMap:
-            keys.extend(list(a.keys()))
-        if len(keys) > 1:
-            return list(set(keys))
-        return keys
+        for filter in filtersMap:
+            filter_keys.update(filter.keys())
+        return list(filter_keys)
 
     def getAllFilterKeysForMeasurementsMetrics(self, searchMetrics):
         filterKeys = []