Merge pull request #16 from IBM/v6.1

merge code changes for v6.1.1

Author: Helene

example_deployment_scripts/bridge_deployment/bridge-deployment.yaml

@@ -56,7 +56,11 @@ spec:
           - name: SERVER
             value: $(IBM_SPECTRUM_SCALE_PERF_QUERY_SERVICE_HOST)
           - name: TLSKEYPATH
-            value: /opt/registry/certs
+            value: /etc/bridge_ssl/certs
+          - name: TLSKEYFILE
+            value: "tls.key"
+          - name: TLSCERTFILE
+            value: "tls.crt"
           - name: PORT
             value: "8443"
           imagePullPolicy: Always
@@ -67,16 +71,17 @@ spec:
               containerPort: 8443
           volumeMounts:
             - name: logfiles
-              mountPath: /opt/IBM/bridge/logs
+              mountPath: /var/log/ibm_bridge_for_grafana/
             - name: config
               mountPath: /opt/IBM/zimon/
               readOnly: true
               optional: true
             - name: secret-volume
-              mountPath: /opt/registry/certs
+              mountPath: /etc/bridge_ssl/certs
+              type: DirectoryOrCreate
       initContainers:
       - name: init-scale-perf-query-service
-        image: busybox:1.28
+        image: busybox:latest
         command: ['sh', '-c', "until nslookup ibm-spectrum-scale-perf-query.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for ibm-spectrum-scale-perf-query service; sleep 2; done"]
       #######################################################################################################################
       #

source/Dockerfile

@@ -23,9 +23,16 @@ ARG HTTPPORT=4242
 ENV PORT=$HTTPPORT
 RUN echo "the HTTP/S port is set to $PORT" 
 
-ARG CERTPATH
+ARG CERTPATH=None
 ENV TLSKEYPATH=$CERTPATH
-RUN if [ -z "$TLSKEYPATH" ] && [ "$PORT" -eq 8443 ]; then echo "TLSKEYPATH FOR SSL CONNECTION NOT SET - ERROR"; exit 1; else echo "PASS"; fi
+
+ARG KEYFILE=None
+ENV TLSKEYFILE=$KEYFILE
+
+ARG CERTFILE=None
+ENV TLSCERTFILE=$CERTFILE
+
+RUN if [ -z "$TLSKEYPATH" ] || [ -z "$TLSCERTFILE" ] || [ -z "$TLSKEYFILE" ] && [ "$PORT" -eq 8443 ]; then echo "TLSKEYPATH FOR SSL CONNECTION NOT SET - ERROR"; exit 1; else echo "PASS"; fi
 RUN echo "the ssl certificates path is set to $TLSKEYPATH" 
 
 ARG PMCOLLECTORIP=0.0.0.0
@@ -35,7 +42,7 @@ RUN echo "the pmcollector server ip is set to $SERVER"
 
 WORKDIR /opt/IBM/bridge
 
-ARG DEFAULTLOGPATH='./logs/zserver.log'
+ARG DEFAULTLOGPATH='/var/log/ibm_bridge_for_grafanalogs/install.log'
 ENV LOGPATH=$DEFAULTLOGPATH
 RUN mkdir -p $(dirname $LOGPATH)
 RUN echo "the log will use $(dirname $LOGPATH)"
@@ -47,7 +54,7 @@ RUN echo "pmcollector_server: $SERVER" >> $LOGPATH
 RUN echo "ssl certificates location: $TLSKEYPATH" >> $LOGPATH
 RUN echo "HTTP/S port: $PORT" >> $LOGPATH
 
-CMD ["sh", "-c", "python3 zimonGrafanaIntf.py -c 10 -s $SERVER -p $PORT -t $TLSKEYPATH"]
+CMD ["sh", "-c", "python3 zimonGrafanaIntf.py -c 10 -s $SERVER -p $PORT -t $TLSKEYPATH --tlsKeyFile $TLSKEYFILE --tlsCertFile $TLSCERTFILE"]
 
 EXPOSE 4242 8443
 

source/__init__.py

@@ -0,0 +1,24 @@
+'''
+##############################################################################
+# Copyright 2019 IBM Corp.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##############################################################################
+
+Created on Apr 4, 2017
+
+@author: HWASSMAN
+'''
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))

source/confParser.py

@@ -24,22 +24,67 @@
 import os
 import logging.handlers
 from messages import MSG
+import configparser
 
 
-def findKeyFile(path):
-    for name in ["privkey.pem", "tls.key"]:
-        for root, dirs, files in os.walk(path):
-            if name in files:
-                return name
-    return None
+def checkFileExists(path, filename):
+    for root, dirs, files in os.walk(path):
+        if filename in files:
+            return True
+    return False
 
 
-def findCertFile(path):
-    for name in ["cert.pem", "tls.crt"]:
-        for root, dirs, files in os.walk(path):
-            if name in files:
-                return name
-    return None
+def checkTLSsettings(args):
+    if args.get('port') == 8443 and (not args.get('tlsKeyPath') or not args.get('tlsKeyFile') or not args.get('tlsCertFile')):
+        return False, MSG['MissingParm']
+    elif args.get('port') == 8443 and not os.path.exists(args.get('tlsKeyPath')):
+        return False, MSG['KeyPathError']
+    elif args.get('port') == 8443:
+        if (not checkFileExists(args.get('tlsKeyPath'), args.get('tlsCertFile'))) or (not checkFileExists(args.get('tlsKeyPath'), args.get('tlsKeyFile'))):
+            return False, MSG['CertError']
+    return True, ''
+
+
+def getSettings(argv):
+    settings = {}
+    msg = ''
+    args, msg = parse_cmd_args(argv)
+    defaults = parse_defaults_from_config_file()
+    if args and defaults:
+        settings = merge_defaults_and_args(defaults, args)
+    elif args:
+        settings = args
+    else:
+        return None, msg
+    # check TLS settings
+    valid, msg = checkTLSsettings(settings)
+    if valid:
+        return settings, ''
+    return None, msg
+
+
+def merge_defaults_and_args(defaults, args):
+    '''merge default config parameters with input parameters from the command line'''
+    brConfig = {}
+    brConfig = dict(defaults)
+    args = vars(args)
+    brConfig.update({k: v for k, v in args.items() if v is not None})
+    return brConfig
+
+
+def parse_defaults_from_config_file(fileName='config.ini'):
+    '''parse default parameters from a config file'''
+    defaults = {}
+    dirname, filename = os.path.split(os.path.abspath(__file__))
+    conf_file = os.path.join(dirname, fileName)
+    if os.path.isfile(conf_file):
+        config = configparser.ConfigParser()
+        config.optionxform = str
+        config.read(conf_file)
+        for sect in config.sections():
+            for name, value in config.items(sect):
+                defaults[name] = value
+    return defaults
 
 
 def parse_cmd_args(argv):
@@ -51,21 +96,13 @@ def parse_cmd_args(argv):
                         NOTE: Per default ZIMon does not accept queries from remote machines. \
                         To run the bridge from outside of the ZIMon collector, you need to modify ZIMon queryinterface settings (\'ZIMonCollector.cfg\')')
     parser.add_argument('-P', '--serverPort', action="store", type=int, default=9084, help='ZIMon collector port number (Default: 9084)')
-    parser.add_argument('-l', '--logFile', action="store", default="./logs/zserver.log", help='location of the log file (Default: ./logs/zserver.log')
+    parser.add_argument('-l', '--logPath', action="store", default="/var/log/ibm_bridge_for_grafana", help='location path of the log file (Default: /var/log/ibm_bridge_for_grafana')
+    parser.add_argument('-f', '--logFile', action="store", default="zserver.log", help='Name of the log file (Default: zserver.log')
     parser.add_argument('-c', '--logLevel', action="store", type=int, default=logging.INFO, help='log level 10 (DEBUG), 20 (INFO), 30 (WARN), 40 (ERROR) (Default: 20)')
     parser.add_argument('-p', '--port', action="store", type=int, choices=[4242, 8443], default=4242, help='port number listening on for HTTP(S) connections (Default: 4242)')
-    parser.add_argument('-t', '--tlsKeyPath', action="store", help='Directory path of tls privkey.pem and cert.pem file location (Required only for HTTPS port 8443)')
+    parser.add_argument('-t', '--tlsKeyPath', action="store", default=None, help='Directory path of tls privkey.pem and cert.pem file location (Required only for HTTPS port 8443)')
+    parser.add_argument('-k', '--tlsKeyFile', action="store", default=None, help='Name of TLS key file, f.e.: privkey.pem (Required only for HTTPS port 8443)')
+    parser.add_argument('-m', '--tlsCertFile', action="store", default=None, help='Name of TLS certificate file, f.e.: cert.pem (Required only for HTTPS port 8443)')
 
     args = parser.parse_args(argv)
-
-    if args.port == 8443 and not args.tlsKeyPath:
-        return None, MSG['MissingParm']
-    elif args.port == 8443 and not os.path.exists(args.tlsKeyPath):
-        return None, MSG['KeyPathError']
-    elif args.port == 8443:
-        certFile = findCertFile(args.tlsKeyPath)
-        keyFile = findKeyFile(args.tlsKeyPath)
-        if (not certFile) or (not keyFile):
-            return None, MSG['CertError']
-
     return args, ''

source/config.ini

@@ -0,0 +1,37 @@
+##################### Grafana Connection Defaults ############################
+[connection]
+# port number the bridge listening on for Grafana data requests
+# 4242 - for HTTP connections 
+# 8443 - for HTTPS connections 
+# (Default: 4242)
+port = 4242
+
+#################################### Grafana Generic OAuth ####################
+[tls]
+# Directory path of tls key and cert file location
+#tlsKeyPath = /etc/bridge_cert/
+
+#
+#tlsKeyFile = privkey.pem
+
+#
+#tlsCertFile = cert.pem
+
+#################################### GPFS Server ##############################
+[server]
+# The ip address to bind to, empty will bind to all interfaces
+server = localhost
+
+# The http port to use
+serverPort = 9084
+
+#################################### Logging ##################################
+[logging]
+# Directory where the bridge can store logs
+logPath = /var/log/ibm_bridge_for_grafana
+
+# log level 10 (DEBUG), 20 (INFO), 30 (WARN), 40 (ERROR) (Default: 20)
+logLevel = 20
+
+# Log file name (Default: zserver.log)
+logFile = zserver.log

source/messages.py

@@ -52,5 +52,6 @@
        'TimerInfo': 'Processing {} took {} seconds',
        'Query2port': 'For better bridge performance multithreaded port {} will be used',
        'CollectorConnInfo': 'Connection to the collector server established successfully',
-       'BridgeVersionInfo': ' *** IBM Spectrum Scale bridge for Grafana - Version: {} ***'
+       'BridgeVersionInfo': ' *** IBM Spectrum Scale bridge for Grafana - Version: {} ***',
+       'FileNotFound': 'The file {} not found.'
        }

source/zimonGrafanaIntf.py

@@ -34,7 +34,7 @@
 from queryHandler import SensorConfig
 from __version__ import __version__
 from messages import ERR, MSG
-from confParser import parse_cmd_args, findCertFile, findKeyFile
+from confParser import getSettings
 from collections import defaultdict
 from timeit import default_timer as timer
 
@@ -534,11 +534,11 @@ def processFormJSON(entity):
         cherrypy.serving.request.json = json.loads('{}')
 
 
-def configureLogging(logfile, loglevel):
+def configureLogging(logPath, logfile, loglevel):
     # create the logfile path if needed
-    path, folder = os.path.split(logfile)
-    if not os.path.exists(path):
-        os.makedirs(path)
+    if not os.path.exists(logPath):
+        os.makedirs(logPath)
+    logfile = os.path.join(logPath, logfile)
 
     # prepare the logger
     logger = logging.getLogger('zimonGrafanaIntf')
@@ -556,32 +556,32 @@ def configureLogging(logfile, loglevel):
 
 def validateCollectorConf(args, logger):
 
-    if not (args.server == 'localhost') and not (args.server == '127.0.0.1'):
+    if not (args.get('server') == 'localhost') and not (args.get('server') == '127.0.0.1'):
         try:
             s = socket.socket()
-            s.connect((args.server, args.serverPort))
+            s.connect((args.get('server'), args.get('serverPort')))
             print(MSG['CollectorConnInfo'])
         finally:
             s.close()
     else:
         # get queryport
         foundPorts = SensorConfig.getCollectorPorts(logger)
-        if foundPorts and str(args.serverPort) not in foundPorts:
+        if foundPorts and str(args.get('serverPort')) not in foundPorts:
             raise Exception("Invalid serverPort specified. Try with: %s" % str(foundPorts))
-        elif foundPorts[1] and not (args.serverPort == int(foundPorts[1])):
-            args.serverPort = int(foundPorts[1])
-            logger.info(MSG['Query2port'].format(args.serverPort))
+        elif foundPorts[1] and not (args.get('serverPort') == int(foundPorts[1])):
+            args['serverPort'] = int(foundPorts[1])
+            logger.info(MSG['Query2port'].format(args['serverPort']))
 
 
 def updateCherrypyConf(args):
 
-    path, folder = os.path.split(args.logFile)
+    path = args.get('logPath')
     if not os.path.exists(path):
         os.makedirs(path)
     accesslog = os.path.join(path, 'cherrypy_access.log')
     errorlog = os.path.join(path, 'cherrypy_error.log')
 
-    globalConfig = {'global': {'server.socket_port': args.port,
+    globalConfig = {'global': {'server.socket_port': args.get('port'),
                                'log.access_file': accesslog,
                                'log.error_file': errorlog}}
 
@@ -593,8 +593,8 @@ def updateCherrypyConf(args):
 
 
 def updateCherrypySslConf(args):
-    certPath = os.path.join(args.tlsKeyPath, findCertFile(args.tlsKeyPath))
-    keyPath = os.path.join(args.tlsKeyPath, findKeyFile(args.tlsKeyPath))
+    certPath = os.path.join(args.get('tlsKeyPath'), args.get('tlsCertFile'))
+    keyPath = os.path.join(args.get('tlsKeyPath'), args.get('tlsKeyFile'))
     sslConfig = {'global': {'server.ssl_module': 'builtin',
                             'server.ssl_certificate': certPath,
                             'server.ssl_private_key': keyPath}}
@@ -604,25 +604,26 @@ def updateCherrypySslConf(args):
 def main(argv):
 
     # parse input arguments
-    args, msg = parse_cmd_args(argv)
+    args, msg = getSettings(argv)
     if not args:
         print(msg)
+        return
 
     # prepare the logger
-    logger = configureLogging(args.logFile, args.logLevel)
-    logger.info('zimonGrafanaItf invoked with parameters:%s', str(args))
+    logger = configureLogging(args.get('logPath'), args.get('logFile'), args.get('logLevel'))
 
     # prepare cherrypy server configuration
     updateCherrypyConf(args)
-    if args.port == 8443:
+    if args.get('port') == 8443:
         updateCherrypySslConf(args)
 
     # prepare metadata
     try:
         print("\n" + MSG['BridgeVersionInfo'].format(__version__))
         logger.info("%s", MSG['BridgeVersionInfo'].format(__version__))
+        logger.info('zimonGrafanaItf invoked with parameters:\n %s', "\n".join("{}={}".format(k, v) for k, v in args.items()))
         validateCollectorConf(args, logger)
-        mdHandler = MetadataHandler(logger, args.server, args.serverPort)
+        mdHandler = MetadataHandler(logger, args.get('server'), args.get('serverPort'))
         print(MSG['MetaSuccess'])
         print(MSG['ReceivAttrValues'].format('sensors', "\n\n" + "\t".join(mdHandler.metaData.sensorsSpec.keys())))
     except (AttributeError, ValueError, TypeError) as e: