Skip to content

Commit 5d9dbb4

Browse files
HeleneHelene
committed
internal refactoring
move out the MetadataHandler in a separate module add decorator to measure method time execution Signed-off-by: hwassman <[email protected]>
1 parent 16e6b62 commit 5d9dbb4

File tree

6 files changed

+225
-145
lines changed

6 files changed

+225
-145
lines changed

‎source/messages.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,12 @@
4747
'BucketsizeChange': 'Based on requested downsample value: {} the bucketsize will be set: {}',
4848
'BucketsizeToPeriod': 'Bucketsize will be set to sensors period: {}',
4949
'ReceivedQuery': 'Received query request for query:{}, start:{}, end:{}',
50-
'RunQuery': 'Execute zimon query: {}',
5150
'AttrNotValid': 'Invalid attribute:{}',
5251
'AllowedAttrValues': 'For attribute {} applicable values:{}',
5352
'ReceivAttrValues': 'Received {}:{}',
54-
'TimerInfo': 'Processing {} took {} seconds',
53+
'StartMethod': 'Starting method: {} with args: {}.',
54+
'RunMethod': 'Executed method: {} with args: {}.',
55+
'TimerInfo': 'Processing {} took {:.3f} seconds.',
5556
'Query2port': 'For better bridge performance multithreaded port {} will be used',
5657
'CollectorConnInfo': 'Connection to the collector server established successfully',
5758
'BridgeVersionInfo': ' *** IBM Spectrum Scale bridge for Grafana - Version: {} ***',

‎source/metadata.py

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
'''
2+
##############################################################################
3+
# Copyright 2023 IBM Corp.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
##############################################################################
17+
18+
Created on Oct 25, 2023
19+
20+
@author: HWASSMAN
21+
'''
22+
23+
import cherrypy
24+
25+
from queryHandler.QueryHandler import QueryHandler2 as QueryHandler
26+
from queryHandler.Topo import Topo
27+
from queryHandler import SensorConfig
28+
from messages import MSG
29+
from metaclasses import Singleton
30+
from timeit import default_timer as timer
31+
from time import sleep
32+
33+
34+
local_cache = []
35+
36+
37+
class MetadataHandler(metaclass=Singleton):
38+
39+
def __init__(self, **kwargs):
40+
self.__qh = None
41+
self.__sensorsConf = None
42+
self.__metaData = None
43+
self.__metricsDesc = {}
44+
self.logger = kwargs['logger']
45+
self.server = kwargs['server']
46+
self.port = kwargs['port']
47+
self.apiKeyName = kwargs['apiKeyName']
48+
self.apiKeyValue = kwargs['apiKeyValue']
49+
self.caCertPath = kwargs.get('caCertPath', False)
50+
self.includeDiskData = kwargs.get('includeDiskData', False)
51+
self.sleepTime = kwargs.get('sleepTime', 60)
52+
53+
self.__initializeTables()
54+
self.__getSupportedMetrics()
55+
56+
@property
57+
def qh(self):
58+
if not self.__qh:
59+
self.__qh = QueryHandler(self.server, self.port, self.logger, self.apiKeyName, self.apiKeyValue, self.caCertPath)
60+
return self.__qh
61+
62+
@property
63+
def SensorsConfig(self):
64+
if not self.__sensorsConf or len(self.__sensorsConf) == 0:
65+
self.__sensorsConf = SensorConfig.readSensorsConfigFromMMSDRFS(self.logger)
66+
if not self.__sensorsConf:
67+
raise ValueError(MSG['NoSensorConfigData'])
68+
return self.__sensorsConf
69+
70+
@property
71+
def metaData(self):
72+
return self.__metaData
73+
74+
@property
75+
def metricsDesc(self):
76+
return self.__metricsDesc
77+
78+
def getSensorPeriod(self, metric):
79+
bucketSize = 0
80+
sensor = self.metaData.getSensorForMetric(metric)
81+
if not sensor:
82+
self.logger.error(MSG['MetricErr'].format(metric))
83+
raise cherrypy.HTTPError(404, MSG['MetricErr'].format(metric))
84+
elif sensor in ('GPFSPoolCap', 'GPFSInodeCap'):
85+
sensor = 'GPFSDiskCap'
86+
elif sensor in ('GPFSNSDFS', 'GPFSNSDPool'):
87+
sensor = 'GPFSNSDDisk'
88+
elif sensor == 'DomainStore':
89+
return 1
90+
91+
for sensorAttr in self.SensorsConfig:
92+
if sensorAttr['name'] == str('\"%s\"' % sensor):
93+
bucketSize = int(sensorAttr['period'])
94+
return bucketSize
95+
96+
def __getSupportedMetrics(self):
97+
"""retrieve all defined (enabled and disabled) metrics list by querying topo -m"""
98+
99+
metricSpec = {}
100+
101+
outp = self.qh.getAvailableMetrics()
102+
103+
if not outp or outp == "" or outp.startswith("Error:"):
104+
self.logger.warning(MSG['NoData'])
105+
return
106+
107+
for line in outp.split("\n"):
108+
if len(line) > 0:
109+
tokens = line.split(";")
110+
if tokens and len(tokens) > 2:
111+
name = tokens[0]
112+
desc = tokens[2] or "No description provided"
113+
metricSpec[name] = desc
114+
else:
115+
self.logger.warning(MSG['DataWrongFormat'].format(line))
116+
self.__metricsDesc = metricSpec
117+
118+
def __initializeTables(self):
119+
'''Read the topology from ZIMon and (re-)construct
120+
the tables for metrics, keys, key elements (tag keys)
121+
and key values (tag values)'''
122+
123+
self.__qh = QueryHandler(self.server, self.port, self.logger, self.apiKeyName, self.apiKeyValue)
124+
self.__sensorsConf = SensorConfig.readSensorsConfigFromMMSDRFS(self.logger)
125+
if not self.__sensorsConf:
126+
raise ValueError(MSG['NoSensorConfigData'])
127+
MAX_ATTEMPTS_COUNT = 3
128+
for attempt in range(1, MAX_ATTEMPTS_COUNT + 1):
129+
self.__metaData = Topo(self.qh.getTopology())
130+
if not (self.metaData and self.metaData.topo):
131+
if attempt > MAX_ATTEMPTS_COUNT:
132+
break
133+
# if no data returned because of the REST HTTP server is still starting, sleep and retry (max 3 times)
134+
self.logger.warning(MSG['NoDataStartNextAttempt'].format(attempt, MAX_ATTEMPTS_COUNT))
135+
sleep(self.sleepTime)
136+
else:
137+
foundItems = len(self.metaData.allParents) - 1
138+
sensors = self.metaData.sensorsSpec.keys()
139+
self.logger.info(MSG['MetaSuccess'])
140+
self.logger.details(MSG['ReceivAttrValues'].format('parents totally', foundItems))
141+
self.logger.debug(MSG['ReceivAttrValues'].format('parents', ", ".join(self.metaData.allParents)))
142+
self.logger.info(MSG['ReceivAttrValues'].format('sensors', ", ".join(sensors)))
143+
return
144+
raise ValueError(MSG['NoData'])
145+
146+
def update(self, refresh_all=False):
147+
'''Read the topology from ZIMon and update
148+
the tables for metrics, keys, key elements (tag keys)
149+
and key values (tag values)'''
150+
151+
if refresh_all:
152+
self.__sensorsConf = SensorConfig.readSensorsConfigFromMMSDRFS(self.logger)
153+
154+
tstart = timer()
155+
self.__metaData = Topo(self.qh.getTopology())
156+
tend = timer()
157+
if not (self.metaData and self.metaData.topo):
158+
self.logger.error(MSG['NoData']) # Please check the pmcollector is properly configured and running.
159+
raise cherrypy.HTTPError(404, MSG[404])
160+
self.logger.details(MSG['MetaSuccess'])
161+
self.logger.debug(MSG['ReceivAttrValues'].format('parents', ", ".join(self.metaData.allParents)))
162+
self.logger.debug(MSG['TimerInfo'].format('Metadata', str(tend - tstart)))
163+
return ({'msg': MSG['MetaSuccess']})

‎source/queryHandler/QueryHandler.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
from collections import namedtuple, defaultdict
2828
from itertools import chain
2929
from typing import NamedTuple, Tuple
30+
from utils import execution_time
3031

3132
from .PerfmonRESTclient import perfHTTPrequestHelper, createRequestDataObj, getAuthHandler
3233

@@ -432,6 +433,7 @@ def apiKeyData(self):
432433
def caCert(self):
433434
return self.__caCert
434435

436+
@execution_time()
435437
def getTopology(self, ignoreMetrics=False):
436438
'''
437439
Returns complete topology as a single JSON string
@@ -452,6 +454,7 @@ def getTopology(self, ignoreMetrics=False):
452454
self.logger.error(
453455
'QueryHandler: getTopology response not valid json: {0} {1}'.format(res[:20], e))
454456

457+
@execution_time()
455458
def getAvailableMetrics(self):
456459
'''
457460
Returns output from topo -m
@@ -481,6 +484,7 @@ def deleteKeyFromTopology(self, keyStr, precheck=True):
481484
self.logger.error(
482485
'QueryHandler: deleteKeysFromTopology response not valid json: {0} {1}'.format(response[:20], e))
483486

487+
@execution_time()
484488
def runQuery(self, query):
485489
'''
486490
runQuery: executes the given query based on the arguments.

‎source/queryHandler/SensorConfig.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,6 @@ def readSensorsConfig(logger=None, customFile=None):
108108

109109
def parseSensorsConfig(sensorsConfig, logger):
110110
""" Returns a list of dicts, describing definitions of sensors """
111-
logger.debug("invoke parseSensorsConfig")
112111
try:
113112
sensors = []
114113
sensorsStr = ""

‎source/utils.py

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
'''
2+
##############################################################################
3+
# Copyright 2023 IBM Corp.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
##############################################################################
17+
18+
Created on Oct 25, 2023
19+
20+
@author: HWASSMAN
21+
'''
22+
23+
import time
24+
from typing import Callable, TypeVar, Any
25+
from functools import wraps
26+
from messages import MSG
27+
28+
T = TypeVar('T')
29+
30+
31+
def execution_time(skip_attribute: bool = False) -> Callable[[Callable[..., T]], Callable[..., T]]:
32+
""" Logs the name of the given function f with passed parameter values
33+
and the time it takes to execute it.
34+
"""
35+
def outer(f: Callable[..., T]) -> Callable[..., T]:
36+
@wraps(f)
37+
def wrapper(*args: Any, **kwargs: Any) -> T:
38+
self = args[0]
39+
args_str = ', '.join(map(str, args[1:])) if len(args) > 1 else ''
40+
kwargs_str = ', '.join(f'{k}={v}' for k, v in kwargs.items()) if len(kwargs) > 0 else ''
41+
self.logger.trace(MSG['StartMethod'].format(f.__name__, ', '.join(filter(None, [args_str, kwargs_str]))))
42+
t = time.time()
43+
result = f(*args, **kwargs)
44+
duration = time.time() - t
45+
if not skip_attribute:
46+
wrapper._execution_duration = duration # type: ignore
47+
self.logger.trace(MSG['RunMethod'].format(f.__name__, ', '.join(filter(None, [args_str, kwargs_str]))))
48+
self.logger.trace(MSG['TimerInfo'].format(f.__name__, duration))
49+
return result
50+
return wrapper
51+
return outer

0 commit comments

Comments
 (0)