Merge pull request #171 from Helene/api_port_mgm

Allow enable, disable and overwrite application ports

Author: Helene

source/confParser.py

@@ -58,7 +58,7 @@ def checkTLSsettings(args):
 
 def checkApplicationPort(args):
     if not args.get('port', None) and not args.get('prometheus', None):
-        return False, MSG['MissingParm']
+        return False, MSG['MissingPortParam']
     return True, ''
 
 
@@ -89,6 +89,8 @@ def getSettings(argv):
         return None, msg
     # check application port
     valid, msg = checkApplicationPort(settings)
+    if not valid:
+        return None, msg
     # check API key settings
     valid, msg = checkAPIsettings(settings)
     if not valid:
@@ -115,6 +117,8 @@ def merge_defaults_and_args(defaults, args):
             brConfig[k] = False
         elif v == "yes" or v == "True":
             brConfig[k] = True
+        elif isinstance(v, str) and v.isdigit():
+            brConfig[k] = int(v)
     return brConfig
 
 
@@ -196,19 +200,31 @@ def parse_cmd_args(argv):
                         help='Host name or ip address of the ZIMon collector (Default from config.ini: 127.0.0.1)')
     parser.add_argument('-P', '--serverPort', action="store", type=int, choices=[9980, 9981], default=None,
                         help='ZIMon collector port number (Default from config.ini: 9980)')
-    parser.add_argument('-l', '--logPath', action="store", default=None, help='location path of the log file (Default from config.ini: \'/var/log/ibm_bridge_for_grafana\')')
-    parser.add_argument('-f', '--logFile', action="store", default=None, help='Name of the log file (Default from config.ini: zserver.log). If no log file name specified \
+    parser.add_argument('-l', '--logPath', action="store", default=None,
+                        help='location path of the log file (Default from config.ini: \'/var/log/ibm_bridge_for_grafana\')')
+    parser.add_argument('-f', '--logFile', action="store", default=None,
+                        help='Name of the log file (Default from config.ini: zserver.log). If no log file name specified \
     all traces will be printed out directly on the command line')
     parser.add_argument('-c', '--logLevel', action="store", type=int, default=None,
                         help='log level. Available levels: 10 (DEBUG), 15 (MOREINFO), 20 (INFO), 30 (WARN), 40 (ERROR) (Default from config.ini: 15)')
-    parser.add_argument('-p', '--port', action="store", type=int, default=None, help='port number listening on HTTP(S) client connections (Default from config.ini: 4242)')
-    parser.add_argument('-r', '--protocol', action="store", choices=["http", "https"], default=None, help='Connection protocol HTTP/HTTPS (Default from config.ini: "http")')
-    parser.add_argument('-t', '--tlsKeyPath', action="store", default=None, help='Directory path of tls privkey.pem and cert.pem file location (Required only for HTTPS port 8443)')
-    parser.add_argument('-k', '--tlsKeyFile', action="store", default=None, help='Name of TLS key file, f.e.: privkey.pem (Required only for HTTPS port 8443)')
-    parser.add_argument('-m', '--tlsCertFile', action="store", default=None, help='Name of TLS certificate file, f.e.: cert.pem (Required only for HTTPS port 8443)')
-    parser.add_argument('-n', '--apiKeyName', action="store", default=None, help='Name of api key file (Default from config.ini: \'scale_grafana\')')
-    parser.add_argument('-v', '--apiKeyValue', action=Password, nargs='?', dest='apiKeyValue', default=None, help='Enter your apiKey value:')
-    parser.add_argument('-d', '--includeDiskData', action="store", choices=["yes", "no"], default=None, help='Use or not the historical data from disk (Default from config.ini: "no")')
+    parser.add_argument('-e', '--prometheus', action="store", default=None,
+                        help='port number listening on Prometheus HTTPS connections (Default from config.ini: 9250, if enabled)')
+    parser.add_argument('-p', '--port', action="store", default=None,
+                        help='port number listening on OpenTSDB API HTTP(S) connections (Default from config.ini: 4242, if enabled)')
+    parser.add_argument('-r', '--protocol', action="store", choices=["http", "https"], default=None,
+                        help='Connection protocol HTTP/HTTPS (Default from config.ini: "http")')
+    parser.add_argument('-t', '--tlsKeyPath', action="store", default=None,
+                        help='Directory path of tls privkey.pem and cert.pem file location (Required only for HTTPS ports 8443/9250)')
+    parser.add_argument('-k', '--tlsKeyFile', action="store", default=None,
+                        help='Name of TLS key file, f.e.: privkey.pem (Required only for HTTPS ports 8443/9250)')
+    parser.add_argument('-m', '--tlsCertFile', action="store", default=None,
+                        help='Name of TLS certificate file, f.e.: cert.pem (Required only for HTTPS ports 8443/9250)')
+    parser.add_argument('-n', '--apiKeyName', action="store", default=None,
+                        help='Name of api key file (Default from config.ini: \'scale_grafana\')')
+    parser.add_argument('-v', '--apiKeyValue', action=Password, nargs='?', dest='apiKeyValue', default=None,
+                        help='Enter your apiKey value:')
+    parser.add_argument('-d', '--includeDiskData', action="store", choices=["yes", "no"], default=None,
+                        help='Use or not the historical data from disk (Default from config.ini: "no")')
 
     args = parser.parse_args(argv)
     return args, ''

source/config.ini

@@ -6,7 +6,7 @@
 # 4242 - for HTTP connections 
 # 8443 - for HTTPS connections 
 # (Default: 4242)
-port = 4242
+# port = 4242
 
 # Protocol (http, https)
 protocol = http
@@ -18,7 +18,7 @@ protocol = http
 
 # Sensor counters deltas will be exported by default.
 # Set True if you want export original sensor counters.
-# rawCounters = False
+rawCounters = True
 
 
 #################################### API SSL OAuth ############################

source/messages.py

@@ -28,6 +28,7 @@
 MSG = {'IntError': 'Server internal error occurred. Reason: {}',
        'sysStart': 'Initial cherryPy server engine start have been invoked. Python version: {}, cherryPy version: {}.',
        'MissingParm': 'Missing mandatory parameters, quitting',
+       'MissingPortParam': 'Missing applications port configuration, quitting',
        'MissingSSLCert': 'Missing ssl configuration, quitting',
        'KeyPathError': 'KeyPath directory not found, quitting',
        'CertError': 'Missing certificates in the specified keyPath directory, quitting',
@@ -55,7 +56,7 @@
        'ReceivAttrValues': 'Received {}:{}',
        'StartMethod': 'Starting method: {} with args: {}.',
        'RunMethod': 'Executed method: {} with args: {}.',
-       'TimerInfo': 'Processing {} took {:.3f} seconds.',
+       'TimerInfo': 'Processing {} took {:.6f} seconds.',
        'Query2port': 'For better bridge performance multithreaded port {} will be used',
        'CollectorConnInfo': 'Connection to the collector server established successfully',
        'BridgeVersionInfo': ' *** IBM Storage Scale bridge for Grafana - Version: {} ***',
@@ -72,5 +73,6 @@
        'StartCustomThread': 'Started custom thread {}',
        'StopCustomThread': 'Stopped custom thread {}',
        'MetricInResults': 'Metric {} is in Collector.metrcs',
-       'MetricNotInResults': 'Metric {} is not in Collector.metrcs'
+       'MetricNotInResults': 'Metric {} is not in Collector.metrcs',
+       'ConnApplications': 'Registered applications: \n {}'
        }

source/zimonGrafanaIntf.py

@@ -183,6 +183,8 @@ def main(argv):
         print(msg)
         return
 
+    registered_apps = []
+
     # prepare the logger
     logger = configureLogging(args.get('logPath'), args.get('logFile', None),
                               args.get('logLevel'))
@@ -277,6 +279,8 @@ def main(argv):
                              }
                             )
 
+        registered_apps.append("OpenTSDB Api listening on Grafana queries")
+
     if args.get('prometheus', None):
         bind_prometheus_server(args)
         load_endpoints('prometheus_endpoints.json')
@@ -294,7 +298,7 @@ def main(argv):
                              }
                             )
         # query for list configured zimon sensors
-        cherrypy.tree.mount(api, '/sensorsconfig',
+        cherrypy.tree.mount(exporter, '/sensorsconfig',
                             {'/':
                              {'request.dispatch': cherrypy.dispatch.MethodDispatcher()}
                              }
@@ -312,6 +316,7 @@ def main(argv):
                                      {'request.dispatch': cherrypy.dispatch.MethodDispatcher()}
                                      }
                                     )
+        registered_apps.append("Prometheus Exporter Api listening on Prometheus requests")
 
     logger.info("%s", MSG['sysStart'].format(sys.version, cherrypy.__version__))
 
@@ -322,6 +327,7 @@ def main(argv):
         cherrypy.engine.subscribe('stop', watcher.stop_watch)
         cherrypy.engine.start()
         cherrypy.engine.log('test')
+        logger.info("%s", MSG['ConnApplications'].format(",\n ".join(registered_apps)))
         logger.info("server started")
         with open("/proc/{}/stat".format(os.getpid())) as f:
             data = f.read()
@@ -339,7 +345,7 @@ def main(argv):
         cherrypy.engine.stop()
         cherrypy.engine.exit()
 
-    api = None
+    api = exporter = None
 
     logger.warning("server stopped")
 

tests/test_cli_parser.py

@@ -14,7 +14,7 @@ def my_setup():
     e = ['-c', '10', '-t', '/opt/registry/certs']
     f = ['-c', '10', '-s', '9.155.108.199', '-p', '8443', '-t', '/opt/registry/certs', '--tlsKeyFile', 'privkey.pem', '--tlsCertFile', 'cert.pem']
     g = ['-p', '4242', '-P', '9084']
-    h = ['-d', 'no']
+    h = ['-p', '4242', '-e', '9250', '-d', 'no']
     k = ['-p', '4243', '-r', 'https']
 
 
@@ -35,8 +35,12 @@ def test_case03():
     args, msg = parse_cmd_args([])
     result = vars(args)
     elements = list(result.keys())
-    mandatoryItems = ['port', 'serverPort']
-    assert all(item in elements for item in mandatoryItems)
+    if version < "8.0":
+        mandatoryItems = ['port', 'serverPort']
+        assert all(item in elements for item in mandatoryItems)
+    else:
+        mandatoryItems = ['server', 'serverPort']
+        assert all(item in elements for item in mandatoryItems)
 
 
 @with_setup(my_setup)
@@ -73,17 +77,27 @@ def test_case06():
 
 @with_setup(my_setup)
 def test_case07():
-    args, msg = parse_cmd_args(a)
-    result = vars(args)
-    assert len(result.keys()) > 0
-    assert 'port' in result.keys()
-    assert 'protocol' in result.keys()
-    assert result.get('port') == 8443
-    assert result.get('protocol') is None
+    if version < "8.0":
+        args, msg = parse_cmd_args(a)
+        result = vars(args)
+        assert len(result.keys()) > 0
+        assert 'port' in result.keys()
+        assert 'protocol' in result.keys()
+        assert result.get('port') == 8443
+        assert result.get('protocol') is None
+    else:
+        args, msg = parse_cmd_args(a)
+        result = vars(args)
+        assert len(result.keys()) > 0
+        assert 'port' in result.keys()
+        assert 'protocol' in result.keys()
+        assert result.get('port') == '8443'
+        assert result.get('protocol') is None
 
 
+@with_setup(my_setup)
 def test_case08():
-    args, msg = parse_cmd_args([])
+    args, msg = parse_cmd_args(a)
     result = vars(args)
     assert 'includeDiskData' in result.keys()
     assert result.get('includeDiskData') is None

tests/test_configManager.py

@@ -27,7 +27,10 @@ def test_case04():
     assert len(connection) > 0
     assert isinstance(connection, dict)
     assert len(connection) > 0
-    assert 'port' in connection.keys()
+    if version < "8.0":
+        assert 'port' in connection.keys()
+    else:
+        assert 'port' not in connection.keys()
 
 
 def test_case05():
@@ -53,24 +56,31 @@ def test_case08():
     cm = ConfigManager()
     result = cm.defaults
     elements = list(result.keys())
-    mandatoryItems = ['port', 'serverPort']
-    assert all(item in elements for item in mandatoryItems)
+    if version < "8.0":
+        mandatoryItems = {'port', 'serverPort'}
+        assert mandatoryItems.issubset(set(elements))
+    else:
+        assert 'port' not in set(elements)
 
 
 def test_case09():
-    cm = ConfigManager()
-    result = cm.defaults
-    value = int(result['port'])
-    assert value == 4242
+    if version < "8.0":
+        cm = ConfigManager()
+        result = cm.defaults
+        value = int(result['port'])
+        assert value == 4242
 
 
 def test_case10():
     cm = ConfigManager()
     result = cm.defaults
     if version < "7.0":
         assert int(result['port']) == 4242 and int(result['serverPort']) == 9084
-    else:
+    elif version < "8.0":
         assert int(result['port']) == 4242 and int(result['serverPort']) == 9980
+    else:
+        assert result.get('port', None) is None
+        assert int(result['serverPort']) == 9980
 
 
 def test_case11():

tests/test_params.py

@@ -4,7 +4,7 @@
 
 
 def my_setup():
-    global a, b, c, d, e, f, g, m, n, o, p, y, x
+    global a, b, c, d, e, f, g, m, n, o, p, y, x, r, s
     a = ConfigManager().defaults
     y = ConfigManager().defaults.copy()
     y['apiKeyValue'] = '/tmp/mykey'
@@ -14,13 +14,18 @@ def my_setup():
     f, g = parse_cmd_args(['-p', '8443', '-t', None, '-k', 'None', '-m', "None"])
     m, n = parse_cmd_args(['-d', 'yes'])
     o, p = parse_cmd_args(['-v', 'e40960c9-de0a-4c75-bc71-0bcae6db23b2'])
+    r, s = parse_cmd_args(['-p', '4242', '-e', '9250'])
 
 
 @with_setup(my_setup)
 def test_case01():
     result = merge_defaults_and_args(a, b)
     assert len(result.keys()) > 0
-    assert 'port' in result.keys()
+    if version < "8.0":
+        assert 'port' in result.keys()
+    else:
+        assert 'port' not in result.keys()
+        assert 'prometheus' not in result.keys()
     assert 'serverPort' in result.keys()
     assert 'apiKeyValue' not in result.keys()
 
@@ -138,10 +143,25 @@ def test_case12():
 @with_setup(my_setup)
 def test_case13():
     x = a.copy()
-    del x['port']
+    if version < "8.0":
+        del x['port']
     result = merge_defaults_and_args(x, b)
     valid, msg = checkApplicationPort(result)
     assert len(result.keys()) > 0
     assert 'port' not in result.keys()
+    if version >= "8.0":
+        assert 'prometheus' not in result.keys()
     assert valid == False
     assert len(msg) > 0
+
+
+@with_setup(my_setup)
+def test_case14():
+    if version >= "8.0":
+        result = merge_defaults_and_args(a, r)
+        valid, msg = checkApplicationPort(result)
+        assert len(result.keys()) > 0
+        assert 'port' in result.keys()
+        assert 'prometheus' in result.keys()
+        assert valid == True
+        assert len(msg) == 0