Skip to content

Commit 32a7faf

Browse files
HeleneHelene
authored
Merge pull request #21 from IBM/v6.1
merge code changes for v6.1.2
2 parents 6a5eef7 + 7702d98 commit 32a7faf

File tree

5 files changed

+154
-69
lines changed

5 files changed

+154
-69
lines changed

‎source/bridgeLogger.py

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
'''
2+
##############################################################################
3+
# Copyright 2019 IBM Corp.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
##############################################################################
17+
18+
Created on Mar 15, 2021
19+
20+
@author: HWASSMAN
21+
'''
22+
23+
import logging
24+
import logging.handlers
25+
import os
26+
27+
logging.TRACE = 5
28+
logging.addLevelName(logging.TRACE, 'TRACE')
29+
30+
logging.MOREINFO = 15
31+
logging.addLevelName(logging.MOREINFO, 'MOREINFO')
32+
33+
34+
class MyLogger(logging.getLoggerClass()):
35+
36+
def __init__(self, name, level=logging.NOTSET):
37+
super().__init__(name, level)
38+
39+
def trace(self, msg, *args, **kwargs):
40+
if self.isEnabledFor(logging.TRACE):
41+
self._log(logging.TRACE, msg, args, **kwargs)
42+
43+
def details(self, msg, *args, **kwargs):
44+
if self.isEnabledFor(logging.MOREINFO):
45+
self._log(logging.MOREINFO, msg, args, **kwargs)
46+
47+
48+
def configureLogging(logPath, logfile, loglevel=logging.INFO):
49+
50+
try:
51+
loglevel = logging._checkLevel(loglevel)
52+
except (ValueError, TypeError):
53+
loglevel = logging.INFO
54+
55+
# create the logfile path if needed
56+
if not os.path.exists(logPath):
57+
os.makedirs(logPath)
58+
logfile = os.path.join(logPath, logfile)
59+
60+
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)-8s - %(message)s')
61+
formatter1 = logging.Formatter('%(asctime)s - %(levelname)-8s - %(message)s', datefmt='%Y-%m-%d %H:%M')
62+
63+
# prepare the logger
64+
logging.setLoggerClass(MyLogger)
65+
logger = logging.getLogger(__name__)
66+
logger.setLevel(logging.TRACE)
67+
68+
strmhandler = logging.StreamHandler()
69+
strmhandler.setLevel(logging.INFO)
70+
strmhandler.setFormatter(formatter1)
71+
72+
rfhandler = logging.handlers.RotatingFileHandler(logfile, 'a', 1000000, 5) # 5 x 1M files
73+
rfhandler.setLevel(loglevel)
74+
rfhandler.setFormatter(formatter)
75+
76+
logger.addHandler(rfhandler)
77+
logger.addHandler(strmhandler)
78+
79+
logger.propagate = False # prevent propagation to default (console) logger
80+
return logger

‎source/confParser.py

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222

2323
import argparse
2424
import os
25-
import logging.handlers
2625
from messages import MSG
2726
import configparser
2827

@@ -95,10 +94,14 @@ def parse_cmd_args(argv):
9594
help='Host name or ip address of the ZIMon collector (Default: 127.0.0.1) \
9695
NOTE: Per default ZIMon does not accept queries from remote machines. \
9796
To run the bridge from outside of the ZIMon collector, you need to modify ZIMon queryinterface settings (\'ZIMonCollector.cfg\')')
98-
parser.add_argument('-P', '--serverPort', action="store", type=int, default=9084, help='ZIMon collector port number (Default: 9084)')
97+
parser.add_argument('-P', '--serverPort', action="store", type=int, choices=[9084, 9094], default=9084,
98+
help='ZIMon collector port number (Default: 9084) \
99+
NOTE: In some environments, for better bridge performance the usage of the multi-threaded port 9094 could be helpful.\
100+
In this case make sure the \'query2port = \"9094\"\' is enabled in the ZIMon queryinterface settings (\'ZIMonCollector.cfg\')')
99101
parser.add_argument('-l', '--logPath', action="store", default="/var/log/ibm_bridge_for_grafana", help='location path of the log file (Default: /var/log/ibm_bridge_for_grafana')
100102
parser.add_argument('-f', '--logFile', action="store", default="zserver.log", help='Name of the log file (Default: zserver.log')
101-
parser.add_argument('-c', '--logLevel', action="store", type=int, default=logging.INFO, help='log level 10 (DEBUG), 20 (INFO), 30 (WARN), 40 (ERROR) (Default: 20)')
103+
parser.add_argument('-c', '--logLevel', action="store", type=int, default=15,
104+
help='log level. Available levels: 10 (DEBUG), 15 (MOREINFO), 20 (INFO), 30 (WARN), 40 (ERROR) (Default: 15)')
102105
parser.add_argument('-p', '--port', action="store", type=int, choices=[4242, 8443], default=4242, help='port number listening on for HTTP(S) connections (Default: 4242)')
103106
parser.add_argument('-t', '--tlsKeyPath', action="store", default=None, help='Directory path of tls privkey.pem and cert.pem file location (Required only for HTTPS port 8443)')
104107
parser.add_argument('-k', '--tlsKeyFile', action="store", default=None, help='Name of TLS key file, f.e.: privkey.pem (Required only for HTTPS port 8443)')

‎source/queryHandler/QueryHandler.py

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -460,7 +460,7 @@ def __do_query(self, data):
460460
endstr = b'.\n'
461461
try:
462462
with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
463-
sock.settimeout(60)
463+
sock.settimeout(30)
464464
sock.connect((self.remote_ip, self.port))
465465
if sys.version >= '3':
466466
data = bytes(data, 'UTF-8')
@@ -482,13 +482,19 @@ def __do_query(self, data):
482482
chunks.append(chunk)
483483
except socket.timeout as e:
484484
self.logger.error(e)
485+
msg = None
486+
except OSError as e:
487+
self.logger.error(e)
488+
msg = None
485489
except Exception as e:
486490
self.logger.error(e)
487-
return None
488-
if msg.startswith('Error'):
489-
self.logger.error('QueryHandler: query returned no data: {0}'.format(msg))
490-
return None
491-
return msg
491+
msg = None
492+
finally:
493+
sock.close()
494+
if msg and msg.startswith('Error'):
495+
self.logger.error('QueryHandler: query returned no data: {0}'.format(msg))
496+
msg = None
497+
return msg
492498

493499
def getTopology(self, ignoreMetrics=False):
494500
'''

‎source/zimonGrafanaIntf.py

Lines changed: 38 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
from queryHandler import SensorConfig
3535
from __version__ import __version__
3636
from messages import ERR, MSG
37+
from bridgeLogger import configureLogging
3738
from confParser import getSettings
3839
from collections import defaultdict
3940
from timeit import default_timer as timer
@@ -82,10 +83,10 @@ def __initializeTables(self):
8283
foundItems = len(self.metaData.allParents) - 1
8384
sensors = self.metaData.sensorsSpec.keys()
8485
self.logger.info(MSG['MetaSuccess'])
85-
self.logger.info(MSG['ReceivAttrValues'].format('parents totally', foundItems))
86+
self.logger.details(MSG['ReceivAttrValues'].format('parents totally', foundItems))
8687
self.logger.debug(MSG['ReceivAttrValues'].format('parents', ", ".join(self.metaData.allParents)))
8788
self.logger.info(MSG['ReceivAttrValues'].format('sensors', ", ".join(sensors)))
88-
self.logger.info(MSG['TimerInfo'].format('Metadata', str(tend - tstart)))
89+
self.logger.details(MSG['TimerInfo'].format('Metadata', str(tend - tstart)))
8990

9091
def update(self):
9192
'''Read the topology from ZIMon and update
@@ -98,7 +99,7 @@ def update(self):
9899
if not (self.metaData and self.metaData.topo):
99100
self.logger.error(MSG['NoData']) # Please check the pmcollector is properly configured and running.
100101
raise cherrypy.HTTPError(404, MSG[404])
101-
self.logger.info(MSG['MetaSuccess'])
102+
self.logger.details(MSG['MetaSuccess'])
102103
self.logger.debug(MSG['ReceivAttrValues'].format('parents', ", ".join(self.metaData.allParents)))
103104
self.logger.debug(MSG['TimerInfo'].format('Metadata', str(tend - tstart)))
104105
return({'msg': MSG['MetaSuccess']})
@@ -252,14 +253,14 @@ def _getTimeMultiplier(self, timeunit):
252253
def _retrieveData(self, query, dsOp=None, dsInterval=None):
253254
'''Executes zimon query and returns results'''
254255

255-
self.logger.info(MSG['RunQuery'].format(query))
256+
self.logger.details(MSG['RunQuery'].format(query))
256257
tstart = timer()
257258
res = self.qh.runQuery(query)
258259
tend = timer()
259-
self.logger.info(MSG['TimerInfo'].format('runQuery: \"' + str(query) + '\"', str(tend - tstart)))
260+
self.logger.details(MSG['TimerInfo'].format('runQuery: \"' + str(query) + '\"', str(tend - tstart)))
260261
if res is None:
261262
return
262-
self.logger.info("res.rows length: {}".format(len(res.rows)))
263+
self.logger.details("res.rows length: {}".format(len(res.rows)))
263264
rows = res.rows
264265
if dsOp and dsInterval and len(res.rows) > 1:
265266
rows = res.downsampleResults(dsInterval, dsOp)
@@ -305,7 +306,7 @@ def _createZimonQuery(self, q, start, end):
305306
self.logger.error(MSG['MetricErr'].format(inMetric))
306307
raise cherrypy.HTTPError(404, MSG['MetricErr'].format(inMetric))
307308
else:
308-
self.logger.info(MSG['ReceivedQuery'].format(str(q), str(start), str(end)))
309+
self.logger.details(MSG['ReceivedQuery'].format(str(q), str(start), str(end)))
309310

310311
# add tagName or metric using the same method. There is no 'NOOP' option in openTSDB
311312
query.addMetric(inMetric, q.get('aggregator'))
@@ -342,13 +343,13 @@ def _createZimonQuery(self, q, start, end):
342343
dsBucketSize = self._calc_bucketSize(q.get('downsample'))
343344
if not dsOp and dsBucketSize > bucketSize:
344345
bucketSize = dsBucketSize
345-
self.logger.info(MSG['BucketsizeChange'].format(q.get('downsample'), bucketSize))
346+
self.logger.details(MSG['BucketsizeChange'].format(q.get('downsample'), bucketSize))
346347
elif dsBucketSize <= bucketSize:
347348
dsOp = dsInterval = None
348349
else:
349350
dsInterval = int(dsBucketSize / bucketSize)
350351
else:
351-
self.logger.info(MSG['BucketsizeToPeriod'].format(bucketSize))
352+
self.logger.details(MSG['BucketsizeToPeriod'].format(bucketSize))
352353

353354
query.setBucketSize(bucketSize)
354355

@@ -534,43 +535,23 @@ def processFormJSON(entity):
534535
cherrypy.serving.request.json = json.loads('{}')
535536

536537

537-
def configureLogging(logPath, logfile, loglevel):
538-
# create the logfile path if needed
539-
if not os.path.exists(logPath):
540-
os.makedirs(logPath)
541-
logfile = os.path.join(logPath, logfile)
542-
543-
# prepare the logger
544-
logger = logging.getLogger('zimonGrafanaIntf')
545-
rfhandler = logging.handlers.RotatingFileHandler(logfile, 'a', 1000000, 5) # 5 x 1M files
546-
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
547-
rfhandler.setFormatter(formatter)
548-
logger.addHandler(rfhandler)
549-
try:
550-
logger.setLevel(loglevel)
551-
except (ValueError, TypeError):
552-
logger.setLevel(logging.INFO)
553-
logger.propagate = False # prevent propagation to default (console) logger
554-
return logger
555-
556-
557538
def validateCollectorConf(args, logger):
558539

559-
if not (args.get('server') == 'localhost') and not (args.get('server') == '127.0.0.1'):
560-
try:
561-
s = socket.socket()
562-
s.connect((args.get('server'), args.get('serverPort')))
563-
print(MSG['CollectorConnInfo'])
564-
finally:
565-
s.close()
566-
else:
540+
# if not (args.get('server') == 'localhost') and not (args.get('server') == '127.0.0.1'):
541+
try:
542+
s = socket.socket()
543+
s.connect((args.get('server'), args.get('serverPort')))
544+
logger.info(MSG['CollectorConnInfo'])
545+
finally:
546+
s.close()
547+
# else:
567548
# get queryport
568-
foundPorts = SensorConfig.getCollectorPorts(logger)
569-
if foundPorts and str(args.get('serverPort')) not in foundPorts:
570-
raise Exception("Invalid serverPort specified. Try with: %s" % str(foundPorts))
571-
elif foundPorts[1] and not (args.get('serverPort') == int(foundPorts[1])):
572-
args['serverPort'] = int(foundPorts[1])
573-
logger.info(MSG['Query2port'].format(args['serverPort']))
549+
# foundPorts = SensorConfig.getCollectorPorts(logger)
550+
# if foundPorts and str(args.get('serverPort')) not in foundPorts:
551+
# raise Exception("Invalid serverPort specified. Try with: %s" % str(foundPorts))
552+
# elif foundPorts[1] and not (args.get('serverPort') == int(foundPorts[1])):
553+
# args['serverPort'] = int(foundPorts[1])
554+
# logger.details(MSG['Query2port'].format(args['serverPort']))
574555

575556

576557
def updateCherrypyConf(args):
@@ -619,24 +600,21 @@ def main(argv):
619600

620601
# prepare metadata
621602
try:
622-
print("\n" + MSG['BridgeVersionInfo'].format(__version__))
623603
logger.info("%s", MSG['BridgeVersionInfo'].format(__version__))
624-
logger.info('zimonGrafanaItf invoked with parameters:\n %s', "\n".join("{}={}".format(k, v) for k, v in args.items()))
604+
logger.details('zimonGrafanaItf invoked with parameters:\n %s', "\n".join("{}={}".format(k, v) for k, v in args.items()))
625605
validateCollectorConf(args, logger)
626606
mdHandler = MetadataHandler(logger, args.get('server'), args.get('serverPort'))
627-
print(MSG['MetaSuccess'])
628-
print(MSG['ReceivAttrValues'].format('sensors', "\n\n" + "\t".join(mdHandler.metaData.sensorsSpec.keys())))
629607
except (AttributeError, ValueError, TypeError) as e:
630-
logger.exception('%s', MSG['IntError'].format(str(e)))
631-
print(MSG['MetaError'])
608+
logger.details('%s', MSG['IntError'].format(str(e)))
609+
logger.error(MSG['MetaError'])
632610
return
633611
except (Exception, IOError) as e:
634-
logger.exception('%s', MSG['IntError'].format(str(e)))
635-
print(MSG['CollectorErr'])
612+
logger.details('%s', MSG['IntError'].format(str(e)))
613+
logger.errort(MSG['CollectorErr'])
636614
return
637615
except (OSError) as e:
638-
logger.exception('%s', MSG['IntError'].format(str(e)))
639-
print("ZiMon sensor configuration file not found")
616+
logger.details('%s', MSG['IntError'].format(str(e)))
617+
logger.error("ZiMon sensor configuration file not found")
640618
return
641619

642620
ph = PostHandler(logger, mdHandler)
@@ -681,25 +659,25 @@ def main(argv):
681659
}
682660
)
683661

684-
print(MSG['sysStart'].format(sys.version, cherrypy.__version__))
685662
logger.info("%s", MSG['sysStart'].format(sys.version, cherrypy.__version__))
686663

687664
try:
688665
cherrypy.engine.start()
689-
print("server started")
666+
logger.info("server started")
667+
logger.debug("Server started PID: {}".format(os.getpid()))
690668
cherrypy.engine.block()
691-
except TypeError:
692-
print("Server request could not be proceed. Reason:")
669+
except TypeError as e:
670+
logger.error("Server request could not be proceed. Reason: {}".format(e))
693671
raise cherrypy.HTTPError(500, ERR[500])
694-
except IOError:
695-
print("STOPPING: Server request could not be proceed. Reason:")
672+
except OSError as e:
673+
logger.error("STOPPING: Server request could not be proceed. Reason: {}".format(e))
696674
cherrypy.engine.stop()
697675
cherrypy.engine.exit()
698676

699677
ph = None
700678
gh = None
701679

702-
print("server stopped")
680+
logger.warn("server stopped")
703681

704682

705683
if __name__ == '__main__':

‎tests/test_logger.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import logging
2+
from source.bridgeLogger import configureLogging
3+
from nose.tools import assert_raises
4+
5+
6+
def test_case01():
7+
with assert_raises(TypeError):
8+
configureLogging()
9+
10+
11+
def test_case02():
12+
with assert_raises(TypeError):
13+
configureLogging('/tmp')
14+
15+
16+
def test_case03():
17+
result = configureLogging('/tmp', 'mylog', 'abc')
18+
assert isinstance(result, logging.Logger)

0 commit comments

Comments
 (0)