Merge pull request #29 from Helene/v6.1.3

V6.1.3

Author: Helene

docs/CHANGELOG.md

@@ -1,3 +1,14 @@
+# Version 6.1.3 (05/21/2021)
+
+Added support for the 'includeDiskData' command line argument.
+- using the 'includeDiskData' option allows to query and fetch data from the archived (on disk data) for better precision. 
+
+Expanded test module with more unit tests
+
+Tested with Grafana 7.5.1 version
+
+
+
 # Version 6.1.2 (03/20/2021)
 
 Removed the "switching to the multi-threaded zimon port automatically" feature

docs/SUPPORT_MATRIX.md

@@ -1,5 +1,12 @@
 The following matrix gives a quick overview of the supported software for the IBM Spectrum Scale bridge for Grafana packages by version number:
 
+# Version 6.1.3 (05/21/2021)
+- Python 3.6
+- CherryPy 18.6.0
+- IBM Spectrum Scale system must run 5.1.0 and above or
+- IBM Spectrum Scale Container Native Storage Access(CNSA) devices having minReleaseLevel 5.1.0.1
+- Grafana 7.5.0 and above
+
 # Version 6.1.2 (03/20/2021)
 - Python 3.6
 - CherryPy 18.6.0

source/__version__.py

@@ -20,4 +20,4 @@
 @author: HWASSMAN
 '''
 
-__version__ = '6.1.2'
+__version__ = '6.1.3'

source/confParser.py

@@ -68,6 +68,11 @@ def merge_defaults_and_args(defaults, args):
     brConfig = dict(defaults)
     args = vars(args)
     brConfig.update({k: v for k, v in args.items() if v is not None and not (v == str(None))})
+    for k, v in brConfig.items():
+        if v == "no":
+            brConfig[k] = False
+        elif v == "yes":
+            brConfig[k] = True
     return brConfig
 
 
@@ -159,6 +164,7 @@ def parse_cmd_args(argv):
     parser.add_argument('-t', '--tlsKeyPath', action="store", default=None, help='Directory path of tls privkey.pem and cert.pem file location (Required only for HTTPS port 8443)')
     parser.add_argument('-k', '--tlsKeyFile', action="store", default=None, help='Name of TLS key file, f.e.: privkey.pem (Required only for HTTPS port 8443)')
     parser.add_argument('-m', '--tlsCertFile', action="store", default=None, help='Name of TLS certificate file, f.e.: cert.pem (Required only for HTTPS port 8443)')
+    parser.add_argument('-d', '--includeDiskData', action="store", choices=["yes", "no"], default=None, help='Use or not the historical data from disk (Default from config.ini: "no")')
 
     args = parser.parse_args(argv)
     return args, ''

source/config.ini

@@ -25,7 +25,12 @@ server = localhost
 # The http port to use
 serverPort = 9084
 
-#################################### Logging ##################################
+#################################### GPFS Server data query settings ###########
+[query]
+# Use or not the historical data from disk (default: no)
+includeDiskData = no
+
+#################################### Logging ###################################
 [logging]
 # Directory where the bridge can store logs
 logPath = /var/log/ibm_bridge_for_grafana

source/queryHandler/Query.py

@@ -49,11 +49,11 @@ class Query(object):
                   "operation", "protocol", "waiters_time_threshold", "export",
                   "nodegroup", "account", "filesystem", "tct_csap", "tct_operation", "cloud_nodeclass"])
 
-    def __init__(self, metrics=None, bucketsize=1, filters=None, groupby=None, includeDiskDate=False):
+    def __init__(self, metrics=None, bucketsize=1, filters=None, groupby=None, includeDiskData=False):
         '''
         Constructor, filters and groupby must be preformmated
         '''
-        self.includeDiskData = includeDiskDate     # disk or archived data (False or True)
+        self.includeDiskData = includeDiskData     # disk or archived data (False or True)
         self.bucket_size = bucketsize              # bucket size
 
         self.metrics = []             # list of string metrics

source/zimonGrafanaIntf.py

@@ -42,13 +42,14 @@
 
 class MetadataHandler():
 
-    def __init__(self, logger, server, port=9084):
+    def __init__(self, logger, server, port=9084, includeDiskData=False):
         self.__qh = None
         self.__sensorsConf = None
         self.__metaData = None
         self.logger = logger
         self.server = server
         self.port = port
+        self.includeDiskData = includeDiskData
 
         self.__initializeTables()
 
@@ -231,6 +232,10 @@ def __init__(self, logger, mdHandler):
     def qh(self):
         return self.__md.qh
 
+    @property
+    def md(self):
+        return self.__md
+
     @property
     def sensorsConf(self):
         return self.__md.SensorsConfig
@@ -298,7 +303,7 @@ def _validateQueryFilters(self, metricName, query):
 
     def _createZimonQuery(self, q, start, end):
         '''Creates zimon query string '''
-        query = Query()
+        query = Query(includeDiskData=self.md.includeDiskData)
         query.normalize_rates = False
         bucketSize = 1  # default
         inMetric = q.get('metric')
@@ -603,7 +608,7 @@ def main(argv):
         logger.info("%s", MSG['BridgeVersionInfo'].format(__version__))
         logger.details('zimonGrafanaItf invoked with parameters:\n %s', "\n".join("{}={}".format(k, v) for k, v in args.items()))
         validateCollectorConf(args, logger)
-        mdHandler = MetadataHandler(logger, args.get('server'), args.get('serverPort'))
+        mdHandler = MetadataHandler(logger, args.get('server'), args.get('serverPort'), args.get('includeDiskData'))
     except (AttributeError, ValueError, TypeError) as e:
         logger.details('%s', MSG['IntError'].format(str(e)))
         logger.error(MSG['MetaError'])

tests/test_cli_parser.py

@@ -4,14 +4,15 @@
 
 
 def my_setup():
-    global a, b, c, d, e, f, g
+    global a, b, c, d, e, f, g, h
     a = ['-p', '8443', '-t', '/etc/my_tls']
     b = ['-a']
     c = ['-a', 'abc']
     d = ['-c', '10', '-v', '/opt/registry/certs']
     e = ['-c', '10', '-t', '/opt/registry/certs']
     f = ['-c', '10', '-s', '9.155.108.199', '-p', '8443', '-t', '/opt/registry/certs', '--tlsKeyFile', 'privkey.pem', '--tlsCertFile', 'cert.pem']
     g = ['-p', '4242', '-P', '9084']
+    h = ['-d', 'no']
 
 
 def test_case01():
@@ -66,3 +67,17 @@ def test_case07():
     assert len(result.keys()) > 0
     assert 'port' in result.keys()
     assert result.get('port') == 8443
+
+def test_case08():
+    args, msg = parse_cmd_args([])
+    result = vars(args)
+    assert 'includeDiskData' in result.keys()
+    assert result.get('includeDiskData') == None
+
+@with_setup(my_setup)
+def test_case09():
+    args, msg = parse_cmd_args(h)
+    result = vars(args)
+    assert 'includeDiskData' in result.keys()
+    assert result.get('includeDiskData') == 'no'
+

tests/test_configManager.py

@@ -67,3 +67,10 @@ def test_case10():
     cm = ConfigManager()
     result = cm.defaults
     assert int(result['port']) == 4242 and int(result['serverPort']) == 9084
+
+
+def test_case11():
+    cm = ConfigManager()
+    result = cm.defaults
+    assert 'includeDiskData' in result.keys()
+    assert result['includeDiskData'] == 'no'

tests/test_params.py

@@ -3,11 +3,12 @@
 
 
 def my_setup():
-    global a, b, c, d, e, f, g
+    global a, b, c, d, e, f, g, m, n
     a = ConfigManager().defaults
     b, c = parse_cmd_args([])
     d, e = parse_cmd_args(['-p', '8443', '-t', '/etc/my_tls'])
     f, g = parse_cmd_args(['-p', '8443', '-t', None, '-k', 'None', '-m', "None"])
+    m, n = parse_cmd_args(['-d', 'yes'])
 
 
 @with_setup(my_setup)
@@ -43,3 +44,19 @@ def test_case04():
     assert 'tlsKeyFile' not in result.keys()
     assert 'tlsCertFile' not in result.keys()
     assert result.get('port') == 8443
+
+
+@with_setup(my_setup)
+def test_case05():
+    result = merge_defaults_and_args(a, f)
+    assert len(result.keys()) > 0
+    assert 'includeDiskData' in result.keys()
+    assert result.get('includeDiskData') == False
+
+
+@with_setup(my_setup)
+def test_case06():
+    result = merge_defaults_and_args(a, m)
+    assert len(result.keys()) > 0
+    assert 'includeDiskData' in result.keys()
+    assert result.get('includeDiskData') == True