Merge pull request #31 from Helene/7.0.1a

merge changes fron v6.1.3

Author: Helene

docs/CHANGELOG.md

@@ -10,6 +10,17 @@ Expanded test module with unit tests for the apiKeyValue verification
 
 
 
+# Version 6.1.3 (05/21/2021)
+
+Added support for the 'includeDiskData' command line argument.
+- using the 'includeDiskData' option allows to query and fetch data from the archived (on disk data) for better precision. 
+
+Expanded test module with more unit tests
+
+Tested with Grafana 7.5.1 version
+
+
+
 # Version 6.1.2 (03/20/2021)
 
 Removed the "switching to the multi-threaded zimon port automatically" feature

docs/SUPPORT_MATRIX.md

@@ -5,6 +5,13 @@ The following matrix gives a quick overview of the supported software for the IB
 - IBM Spectrum Scale system must run 5.1.1 and above
 - Grafana 7.1.0 and above
 
+# Version 6.1.3 (05/21/2021)
+- Python 3.6
+- CherryPy 18.6.0
+- IBM Spectrum Scale system must run 5.1.0 and above or
+- IBM Spectrum Scale Container Native Storage Access(CNSA) devices having minReleaseLevel 5.1.0.1
+- Grafana 7.5.0 and above
+
 # Version 6.1.2 (03/20/2021)
 - Python 3.6
 - CherryPy 18.6.0

source/__version__.py

@@ -20,4 +20,4 @@
 @author: HWASSMAN
 '''
 
-__version__ = '7.0'
+__version__ = '7.0.1'

source/confParser.py

@@ -79,6 +79,11 @@ def merge_defaults_and_args(defaults, args):
     brConfig = dict(defaults)
     args = vars(args)
     brConfig.update({k: v for k, v in args.items() if v is not None and not (v == str(None))})
+    for k, v in brConfig.items():
+        if v == "no":
+            brConfig[k] = False
+        elif v == "yes":
+            brConfig[k] = True
     return brConfig
 
 
@@ -179,6 +184,7 @@ def parse_cmd_args(argv):
     parser.add_argument('-m', '--tlsCertFile', action="store", default=None, help='Name of TLS certificate file, f.e.: cert.pem (Required only for HTTPS port 8443)')
     parser.add_argument('-n', '--apiKeyName', action="store", default=None, help='Name of api key file (Default from config.ini: \'scale_grafana\')')
     parser.add_argument('-v', '--apiKeyValue', action=Password, nargs='?', dest='apiKeyValue', default=None, help='Enter your apiKey value:')
+    parser.add_argument('-d', '--includeDiskData', action="store", choices=["yes", "no"], default=None, help='Use or not the historical data from disk (Default from config.ini: "no")')
 
     args = parser.parse_args(argv)
     return args, ''

source/config.ini

@@ -31,7 +31,12 @@ apiKeyName = scale_grafana
 # The REST HTTPS API key value, f.e:
 # 	apiKeyValue = e40960c9-de0a-4c75-bc71-0bcae6db23b2
 
-#################################### Logging ##################################
+#################################### GPFS Server data query settings ###########
+[query]
+# Use or not the historical data from disk (default: no)
+includeDiskData = no
+
+#################################### Logging ###################################
 [logging]
 # Directory where the bridge can store logs
 logPath = /var/log/ibm_bridge_for_grafana

source/queryHandler/Query.py

@@ -49,11 +49,11 @@ class Query(object):
                   "operation", "protocol", "waiters_time_threshold", "export",
                   "nodegroup", "account", "filesystem", "tct_csap", "tct_operation", "cloud_nodeclass"])
 
-    def __init__(self, metrics=None, bucketsize=1, filters=None, groupby=None, includeDiskDate=False):
+    def __init__(self, metrics=None, bucketsize=1, filters=None, groupby=None, includeDiskData=False):
         '''
         Constructor, filters and groupby must be preformmated
         '''
-        self.includeDiskData = includeDiskDate     # disk or archived data (False or True)
+        self.includeDiskData = includeDiskData     # disk or archived data (False or True)
         self.bucket_size = bucketsize              # bucket size
 
         self.metrics = []             # list of string metrics

source/zimonGrafanaIntf.py

@@ -41,7 +41,7 @@
 
 class MetadataHandler():
 
-    def __init__(self, logger, server, port, apiKeyName, apiKeyValue):
+    def __init__(self, logger, server, port, apiKeyName, apiKeyValue, includeDiskData=False):
         self.__qh = None
         self.__sensorsConf = None
         self.__metaData = None
@@ -50,6 +50,7 @@ def __init__(self, logger, server, port, apiKeyName, apiKeyValue):
         self.port = port
         self.apiKeyName = apiKeyName
         self.apiKeyValue = apiKeyValue
+        self.includeDiskData = includeDiskData
 
         self.__initializeTables()
 
@@ -232,6 +233,10 @@ def __init__(self, logger, mdHandler):
     def qh(self):
         return self.__md.qh
 
+    @property
+    def md(self):
+        return self.__md
+
     @property
     def sensorsConf(self):
         return self.__md.SensorsConfig
@@ -299,7 +304,7 @@ def _validateQueryFilters(self, metricName, query):
 
     def _createZimonQuery(self, q, start, end):
         '''Creates zimon query string '''
-        query = Query()
+        query = Query(includeDiskData=self.md.includeDiskData)
         query.normalize_rates = False
         bucketSize = 1  # default
         inMetric = q.get('metric')
@@ -584,7 +589,7 @@ def main(argv):
         logger.info("%s", MSG['BridgeVersionInfo'].format(__version__))
         logger.details('zimonGrafanaItf invoked with parameters:\n %s', "\n".join("{}={}".format(k, v) for k, v in args.items()))
         # logger.details('zimonGrafanaItf invoked with parameters:\n %s', "\n".join("{}={}".format(k, type(v)) for k, v in args.items()))
-        mdHandler = MetadataHandler(logger, args.get('server'), args.get('serverPort'), args.get('apiKeyName'), args.get('apiKeyValue'))
+        mdHandler = MetadataHandler(logger, args.get('server'), args.get('serverPort'), args.get('apiKeyName'), args.get('apiKeyValue'), args.get('includeDiskData'))
     except (AttributeError, TypeError, ValueError) as e:
         logger.details('%s', MSG['IntError'].format(str(e)))
         logger.error(MSG['MetaError'])

tests/test_cli_parser.py

@@ -6,14 +6,15 @@
 
 
 def my_setup():
-    global a, b, c, d, e, f, g
+    global a, b, c, d, e, f, g, h
     a = ['-p', '8443', '-t', '/etc/my_tls']
     b = ['-a']
     c = ['-a', 'abc']
     d = ['-c', '10', '-v', '/opt/registry/certs']
     e = ['-c', '10', '-t', '/opt/registry/certs']
     f = ['-c', '10', '-s', '9.155.108.199', '-p', '8443', '-t', '/opt/registry/certs', '--tlsKeyFile', 'privkey.pem', '--tlsCertFile', 'cert.pem']
     g = ['-p', '4242', '-P', '9084']
+    h = ['-d', 'no']
 
 
 def test_case01():
@@ -76,3 +77,18 @@ def test_case07():
     assert len(result.keys()) > 0
     assert 'port' in result.keys()
     assert result.get('port') == 8443
+
+
+def test_case08():
+    args, msg = parse_cmd_args([])
+    result = vars(args)
+    assert 'includeDiskData' in result.keys()
+    assert result.get('includeDiskData') is None
+
+
+@with_setup(my_setup)
+def test_case09():
+    args, msg = parse_cmd_args(h)
+    result = vars(args)
+    assert 'includeDiskData' in result.keys()
+    assert result.get('includeDiskData') == 'no'

tests/test_configManager.py

@@ -71,3 +71,10 @@ def test_case10():
         assert int(result['port']) == 4242 and int(result['serverPort']) == 9084
     else:
         assert int(result['port']) == 4242 and int(result['serverPort']) == 9980
+
+
+def test_case11():
+    cm = ConfigManager()
+    result = cm.defaults
+    assert 'includeDiskData' in result.keys()
+    assert result['includeDiskData'] == 'no'

tests/test_params.py

@@ -3,11 +3,12 @@
 
 
 def my_setup():
-    global a, b, c, d, e, f, g
+    global a, b, c, d, e, f, g, m, n
     a = ConfigManager().defaults
     b, c = parse_cmd_args([])
     d, e = parse_cmd_args(['-p', '8443', '-t', '/etc/my_tls'])
     f, g = parse_cmd_args(['-p', '8443', '-t', None, '-k', 'None', '-m', "None"])
+    m, n = parse_cmd_args(['-d', 'yes'])
 
 
 @with_setup(my_setup)
@@ -43,3 +44,19 @@ def test_case04():
     assert 'tlsKeyFile' not in result.keys()
     assert 'tlsCertFile' not in result.keys()
     assert result.get('port') == 8443
+
+
+@with_setup(my_setup)
+def test_case05():
+    result = merge_defaults_and_args(a, f)
+    assert len(result.keys()) > 0
+    assert 'includeDiskData' in result.keys()
+    assert result.get('includeDiskData') == False
+
+
+@with_setup(my_setup)
+def test_case06():
+    result = merge_defaults_and_args(a, m)
+    assert len(result.keys()) > 0
+    assert 'includeDiskData' in result.keys()
+    assert result.get('includeDiskData') == True