Skip to content

Commit 3a1e459

Browse files
HeleneHelene
authored
Merge pull request #218 from Helene/prometheus_conf_generator
Add plugin for generating Promtheus config file authomatically
2 parents ecb6cb1 + 0cede11 commit 3a1e459

File tree

3 files changed

+271
-1
lines changed

3 files changed

+271
-1
lines changed

‎source/confgenerator.py

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
'''
2+
##############################################################################
3+
# Copyright 2023 IBM Corp.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
##############################################################################
17+
18+
Created on Mai 30, 2024
19+
20+
@author: HWASSMAN
21+
'''
22+
23+
import cherrypy
24+
import json
25+
import socket
26+
import os
27+
try:
28+
# Optional dependency
29+
import yaml
30+
except ImportError as e:
31+
yaml = e
32+
from messages import MSG, ERR
33+
34+
35+
class PrometheusConfigGenerator(object):
36+
exposed = True
37+
38+
def __init__(self, logger, mdHandler, config_attr, endpoints):
39+
if isinstance(yaml, ImportError):
40+
raise yaml
41+
self.logger = logger
42+
self.__md = mdHandler
43+
self.endpoints = endpoints
44+
self.attr = config_attr
45+
46+
@property
47+
def md(self):
48+
return self.__md
49+
50+
@property
51+
def qh(self):
52+
return self.__md.qh
53+
54+
@property
55+
def TOPO(self):
56+
return self.__md.metaData
57+
58+
@staticmethod
59+
def host_ip():
60+
hostname = socket.getfqdn()
61+
local_ip = socket.gethostbyname_ex(hostname)[2][0]
62+
return local_ip
63+
64+
def generate_config(self):
65+
scrape_configs = []
66+
global_config = {"scrape_interval": "15s",
67+
"evaluation_interval": "15s",
68+
"query_log_file": "/var/log/prometheus/query.log"}
69+
alerting_config = {"alertmanagers": [{"static_configs": [{"targets": None}]}]}
70+
for endpoint, sensor in self.endpoints.items():
71+
period = self.md.getSensorPeriod(sensor)
72+
if period > 0:
73+
scrape_job = {}
74+
scrape_job["job_name"] = sensor
75+
scrape_job["scrape_interval"] = f"{period}s"
76+
scrape_job["honor_timestamps"] = True
77+
scrape_job["metrics_path"] = endpoint
78+
scrape_job["scheme"] = self.attr.get('protocol')
79+
if self.attr.get('tlsKeyPath'):
80+
certPath = os.path.join(self.attr.get('tlsKeyPath'),
81+
self.attr.get('tlsCertFile'))
82+
keyPath = os.path.join(self.attr.get('tlsKeyPath'),
83+
self.attr.get('tlsKeyFile'))
84+
tls = {"cert_file": certPath,
85+
"key_file": keyPath,
86+
"insecure_skip_verify": True}
87+
scrape_job["tls_config"] = tls
88+
if self.attr.get('enabled', False):
89+
basic_auth = {"username": self.attr.get('username'),
90+
"password": self.attr.get('password')}
91+
scrape_job["basic_auth"] = basic_auth
92+
targets = {"targets": [f"{self.host_ip()}:{self.attr.get('prometheus')}"]}
93+
scrape_job["static_configs"] = [targets]
94+
scrape_configs.append(scrape_job)
95+
prometheus_job = {}
96+
prometheus_job["job_name"] = "prometheus"
97+
prometheus_job["static_configs"] = [{"targets": ["localhost:9090"]}]
98+
scrape_configs.insert(0, prometheus_job)
99+
resp = {"global": global_config,
100+
"alerting": alerting_config,
101+
"rule_files": None,
102+
"scrape_configs": scrape_configs}
103+
yaml_string = yaml.dump(resp)
104+
return yaml_string
105+
106+
def GET(self, **params):
107+
'''Handle partial URLs such as /metrics_cpu
108+
TODO: add more explanation
109+
'''
110+
resp = []
111+
112+
conn = cherrypy.request.headers.get('Host').split(':')
113+
if int(conn[1]) != int(self.attr.get('prometheus')):
114+
raise cherrypy.HTTPError(400, ERR[400])
115+
116+
# generate prometheus.yml
117+
if '/prometheus.yml' == cherrypy.request.script_name:
118+
print(params)
119+
resp = self.generate_config()
120+
cherrypy.response.headers['Content-Type'] = 'text/plain'
121+
return resp
122+
123+
else:
124+
self.logger.error(MSG['EndpointNotSupported'].format(cherrypy.request.script_name))
125+
raise cherrypy.HTTPError(400,
126+
MSG['EndpointNotSupported'].format(
127+
cherrypy.request.script_name))
128+
129+
del cherrypy.response.headers['Allow']
130+
cherrypy.response.headers['Access-Control-Allow-Origin'] = '*'
131+
cherrypy.response.headers['Content-Type'] = 'application/json'
132+
resp = json.dumps(resp)
133+
return resp
134+
135+
def OPTIONS(self):
136+
# print('options_post')
137+
del cherrypy.response.headers['Allow']
138+
cherrypy.response.headers['Access-Control-Allow-Methods'] = 'GET, POST, NEW, OPTIONS'
139+
cherrypy.response.headers['Access-Control-Allow-Origin'] = '*'
140+
cherrypy.response.headers['Access-Control-Allow-Headers'] = 'Content-Type,Accept'
141+
cherrypy.response.headers['Access-Control-Max-Age'] = 604800

‎source/zimonGrafanaIntf.py

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
from messages import ERR, MSG
3636
from bridgeLogger import configureLogging, getBridgeLogger
3737
from confParser import getSettings
38+
from confgenerator import PrometheusConfigGenerator
3839
from metadata import MetadataHandler
3940
from opentsdb import OpenTsdbApi
4041
from prometheus import PrometheusExporter
@@ -258,6 +259,7 @@ def main(argv):
258259
logger.error("ZiMon sensor configuration file not found")
259260
return
260261

262+
# register MetaData Handler endpoints
261263
# query to force update of metadata (zimon)
262264
cherrypy.tree.mount(mdHandler, '/metadata/update',
263265
{'/':
@@ -271,6 +273,7 @@ def main(argv):
271273
}
272274
)
273275

276+
# register OpenTSDB API endpoints
274277
if args.get('port', None):
275278
bind_opentsdb_server(args)
276279
api = OpenTsdbApi(logger, mdHandler, args.get('port'))
@@ -308,6 +311,7 @@ def main(argv):
308311
)
309312
registered_apps.append("OpenTSDB Api listening on Grafana queries")
310313

314+
# register Prometheus Exporter API endpoints
311315
if args.get('prometheus', None):
312316
bind_prometheus_server(args)
313317
load_endpoints('prometheus_endpoints.json')
@@ -332,15 +336,38 @@ def main(argv):
332336
)
333337
registered_apps.append("Prometheus Exporter Api listening on Prometheus requests")
334338

339+
# register Prometheus config generator endpoints (only if PyYaml available)
340+
try:
341+
conf_generator = PrometheusConfigGenerator(logger,
342+
mdHandler,
343+
args,
344+
ENDPOINTS.get('prometheus', {}))
345+
cherrypy.tree.mount(conf_generator, '/prometheus.yml',
346+
{'/':
347+
{'request.dispatch': cherrypy.dispatch.MethodDispatcher()}
348+
}
349+
)
350+
registered_apps.append("Prometheus Config Generator Api")
351+
except ImportError:
352+
logger.warning("Prometheus Config Generator Api requires python PyYaml packages. Skip registering API.")
353+
354+
# register Profiler reporter endpoint
335355
profiler = Profiler(args.get('logPath'))
336356
# query for print out profiling report
337357
cherrypy.tree.mount(profiler, '/profiling',
338358
{'/':
339359
{'request.dispatch': cherrypy.dispatch.MethodDispatcher()}
340360
}
341361
)
362+
363+
# register cherrypy stats plugin endpoint
342364
if analytics.cherrypy_internal_stats:
343-
cherrypy.tree.mount(StatsPage(), '/cherrypy_internal_stats')
365+
cherrypy.tree.mount(StatsPage(), '/cherrypy_internal_stats',
366+
{'/':
367+
{'request.dispatch': cherrypy.dispatch.MethodDispatcher()}
368+
}
369+
)
370+
344371
logger.info("%s", MSG['sysStart'].format(sys.version, cherrypy.__version__))
345372

346373
try:

‎tests/test_confgenerator.py

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
import logging
2+
import os
3+
from unittest import mock
4+
from nose2.tools.decorators import with_setup
5+
from source.profiler import Profiler
6+
from source.metadata import MetadataHandler
7+
from source.confgenerator import PrometheusConfigGenerator
8+
9+
10+
def my_setup():
11+
global attr, args, endpoints, sensors_conf
12+
13+
attr = {'port': 4242, 'prometheus': 9250, 'rawCounters': True, 'protocol': 'http', 'enabled': True,
14+
'username': 'scale_admin', 'password': 'TXlWZXJ5U3Ryb25nUGFzc3cwcmQhCg==', 'server': 'localhost',
15+
'serverPort': 9980, 'retryDelay': 60, 'apiKeyName': 'scale_grafana',
16+
'apiKeyValue': 'c0a910e4-094a-46d8-b04d-c2f73a43fd17', 'caCertPath': False,
17+
'includeDiskData': False, 'logPath': '/var/log/ibm_bridge_for_grafana', 'logLevel': 10,
18+
'logFile': 'zserver.log'}
19+
args = {'server': 'localhost', 'port': 9980, 'retryDelay': 60,
20+
'apiKeyName': 'scale_grafana',
21+
'apiKeyValue': 'c0a910e4-094a-46d8-b04d-c2f73a43fd17'}
22+
23+
endpoints = {"/metrics_gpfs_disk": "GPFSDisk",
24+
"/metrics_gpfs_filesystem": "GPFSFilesystem",
25+
"//metrics_gpfs_fileset": "GPFSFileset",
26+
"/metrics_gpfs_pool": "GPFSPool"}
27+
28+
sensors_conf = [{'name': '"CPU"', 'period': '1'},
29+
{'name': '"Load"', 'period': '1'},
30+
{'name': '"Memory"', 'period': '1'},
31+
{'filter': '"netdev_name=veth.*|docker.*|flannel.*|cali.*|cbr.*"',
32+
'name': '"Network"', 'period': '1'},
33+
{'name': '"Netstat"', 'period': '10'},
34+
{'name': '"Diskstat"', 'period': '0'},
35+
{'filter': '"mountPoint=/.*/docker.*|/.*/kubelet.*"', 'name': '"DiskFree"', 'period': '600'},
36+
{'name': '"Infiniband"', 'period': '0'},
37+
{'name': '"GPFSDisk"', 'period': '0'},
38+
{'name': '"GPFSFilesystem"', 'period': '10'},
39+
{'name': '"GPFSNSDDisk"', 'period': '10', 'restrict': '"nsdNodes"'},
40+
{'name': '"GPFSPoolIO"', 'period': '0'}, {'name': '"GPFSVFSX"', 'period': '10'},
41+
{'name': '"GPFSIOC"', 'period': '0'}, {'name': '"GPFSVIO64"', 'period': '0'},
42+
{'name': '"GPFSPDDisk"', 'period': '10', 'restrict': '"nsdNodes"'},
43+
{'name': '"GPFSvFLUSH"', 'period': '0'}, {'name': '"GPFSNode"', 'period': '10'},
44+
{'name': '"GPFSNodeAPI"', 'period': '10'}, {'name': '"GPFSFilesystemAPI"', 'period': '10'},
45+
{'name': '"GPFSLROC"', 'period': '0'}, {'name': '"GPFSCHMS"', 'period': '0'},
46+
{'name': '"GPFSAFM"', 'period': '10', 'restrict': '"scale-16.vmlocal,scale-17.vmlocal"'},
47+
{'name': '"GPFSAFMFS"', 'period': '10', 'restrict': '"scale-16.vmlocal,scale-17.vmlocal"'},
48+
{'name': '"GPFSAFMFSET"', 'period': '10', 'restrict': '"scale-16.vmlocal,scale-17.vmlocal"'},
49+
{'name': '"GPFSRPCS"', 'period': '10'}, {'name': '"GPFSWaiters"', 'period': '10'},
50+
{'name': '"GPFSFilesetQuota"', 'period': '3600', 'restrict': '"@CLUSTER_PERF_SENSOR"'},
51+
{'name': '"GPFSFileset"', 'period': '300', 'restrict': '"@CLUSTER_PERF_SENSOR"'},
52+
{'name': '"GPFSPool"', 'period': '300', 'restrict': '"@CLUSTER_PERF_SENSOR"'},
53+
{'name': '"GPFSDiskCap"', 'period': '86400', 'restrict': '"@CLUSTER_PERF_SENSOR"'},
54+
{'name': '"GPFSEventProducer"', 'period': '0'}, {'name': '"GPFSMutex"', 'period': '0'},
55+
{'name': '"GPFSCondvar"', 'period': '0'}, {'name': '"TopProc"', 'period': '60'},
56+
{'name': '"GPFSQoS"', 'period': '0'}, {'name': '"GPFSFCM"', 'period': '0'},
57+
{'name': '"GPFSBufMgr"', 'period': '30'},
58+
{'name': '"NFSIO"', 'period': '10', 'restrict': '"cesNodes"', 'type': '"Generic"'},
59+
{'name': '"SMBStats"', 'period': '1', 'restrict': '"cesNodes"', 'type': '"Generic"'},
60+
{'name': '"SMBGlobalStats"', 'period': '1', 'restrict': '"cesNodes"', 'type': '"Generic"'},
61+
{'name': '"CTDBStats"', 'period': '1', 'restrict': '"cesNodes"', 'type': '"Generic"'},
62+
{'name': '"CTDBDBStats"', 'period': '1', 'restrict': '"cesNodes"', 'type': '"Generic"'}]
63+
64+
65+
@with_setup(my_setup)
66+
def test_case01():
67+
with mock.patch('source.metadata.MetadataHandler._MetadataHandler__initializeTables') as md_init:
68+
with mock.patch('source.metadata.MetadataHandler._MetadataHandler__getSupportedMetrics') as md_supp:
69+
with mock.patch('source.metadata.MetadataHandler.SensorsConfig', return_value=sensors_conf) as md_sensConf:
70+
with mock.patch('source.confgenerator.PrometheusConfigGenerator.host_ip', return_value='127.0.0.1'):
71+
logger = logging.getLogger(__name__)
72+
args['logger'] = logger
73+
md = MetadataHandler(**args)
74+
md.__initializeTables = md_init.return_value
75+
md.__getSupportedMetrics = md_supp.return_value
76+
md.SensorsConfig = md_sensConf.return_value
77+
conf_generator = PrometheusConfigGenerator(logger, md, attr, endpoints)
78+
resp = conf_generator.generate_config()
79+
assert isinstance(resp, str)
80+
assert len(resp) > 0
81+
82+
83+
@with_setup(my_setup)
84+
def test_case02():
85+
with mock.patch('source.metadata.MetadataHandler._MetadataHandler__initializeTables') as md_init:
86+
with mock.patch('source.metadata.MetadataHandler._MetadataHandler__getSupportedMetrics') as md_supp:
87+
with mock.patch('source.metadata.MetadataHandler.SensorsConfig', return_value=sensors_conf) as md_sensConf:
88+
with mock.patch('source.confgenerator.PrometheusConfigGenerator.host_ip', return_value='127.0.0.1'):
89+
logger = logging.getLogger(__name__)
90+
args['logger'] = logger
91+
md = MetadataHandler(**args)
92+
md.__initializeTables = md_init.return_value
93+
md.__getSupportedMetrics = md_supp.return_value
94+
md.SensorsConfig = md_sensConf.return_value
95+
conf_generator = PrometheusConfigGenerator(logger, md, attr, endpoints)
96+
profiler = Profiler()
97+
resp = profiler.run(conf_generator.generate_config)
98+
assert resp is not None
99+
assert os.path.exists(os.path.join(profiler.path, "profiling_generate_config.prof"))
100+
response = profiler.stats(os.path.join(profiler.path, "profiling_generate_config.prof"))
101+
assert response is not None
102+
print('\n'.join(response) + '\n')

0 commit comments

Comments
 (0)