Merge pull request #275 from Helene/prepae_8.0.3-prod

Publish v8.0.3 release

Author: Helene

Dockerfile

@@ -1,9 +1,23 @@
-ARG BASE=registry.access.redhat.com/ubi9/ubi:9.5-1732804088
-FROM $BASE
+ARG BUILD_ENV=prod
+ARG BASE=registry.access.redhat.com/ubi9/ubi:9.5-1736404036
+
+FROM $BASE as build_prod
+ONBUILD COPY ./requirements/requirements_ubi9.txt  /root/requirements_ubi9.txt
+
+FROM $BASE as build_test
+ONBUILD COPY ./requirements/requirements_ubi.in  /root/requirements_ubi.in
+
+FROM $BASE as build_custom
+ONBUILD COPY ./requirements/requirements.in  /root/requirements.in
+
+FROM build_${BUILD_ENV}
+
+ARG BUILD_ENV
+ARG BASE
 
 LABEL com.ibm.name="IBM Storage Scale bridge for Grafana"
 LABEL com.ibm.vendor="IBM"
-LABEL com.ibm.version="8.0.3-dev"
+LABEL com.ibm.version="8.0.3"
 LABEL com.ibm.url="https://github.com/IBM/ibm-spectrum-scale-bridge-for-grafana"
 LABEL com.ibm.description="This tool translates the IBM Storage Scale performance data collected internally \
 to the query requests acceptable by the Grafana integrated openTSDB plugin"
@@ -26,11 +40,9 @@ ENV GID=$GROUPID
 
 ARG HTTPPROTOCOL=http
 ENV PROTOCOL=$HTTPPROTOCOL
-RUN echo "the HTTP/S protocol is set to $PROTOCOL"
 
 ARG HTTPBASICAUTH=True
 ENV BASICAUTH=$HTTPBASICAUTH
-RUN echo "the HTTP/S basic authentication is set to $BASICAUTH"
 
 ARG AUTHUSER=None
 ENV BASICAUTHUSER=$AUTHUSER
@@ -40,15 +52,12 @@ ENV BASICAUTHPASSW=$AUTHPASSW
 
 ARG HTTPPORT=None
 ENV PORT=$HTTPPORT
-RUN echo "the OpentTSDB API HTTP/S port is set to $PORT"
 
 ARG PROMPORT=None
-ENV PROMETHEUS=$PROMPORT
-RUN echo "the Prometheus API HTTPS port is set to $PROMETHEUS" 
+ENV PROMETHEUS=$PROMPORT 
 
 ARG PERFMONPORT=9980
 ENV SERVERPORT=$PERFMONPORT
-RUN echo "the PERFMONPORT port is set to $SERVERPORT" 
 
 ARG CERTPATH='/etc/bridge_ssl/certs'
 ENV TLSKEYPATH=$CERTPATH
@@ -67,27 +76,33 @@ ENV APIKEYVALUE=$KEYVALUE
 
 ARG PMCOLLECTORIP=0.0.0.0
 ENV SERVER=$PMCOLLECTORIP
-RUN echo "the pmcollector server ip is set to $SERVER"
 
 ARG DEFAULTLOGPATH='/var/log/ibm_bridge_for_grafana'
 ENV LOGPATH=$DEFAULTLOGPATH
-RUN echo "the log will use $LOGPATH"
 
 ARG DEFAULTLOGLEVEL=15
 ENV LOGLEVEL=$DEFAULTLOGLEVEL
 
-COPY ./requirements/requirements_ubi9.txt  /root/requirements_ubi9.txt
-# COPY ./requirements/requirements_ubi.in  /root/requirements_ubi.in
-
-RUN yum install -y python39 python3-pip
-
-# RUN /usr/bin/python3 -m pip install pip-tools && \
-#     /usr/bin/python3 -m piptools compile /root/requirements_ubi.in  --output-file /root/requirements_ubi9.txt && \
-#     echo "Compiled python packages: $(cat /root/requirements_ubi9.txt)"
-
-RUN /usr/bin/python3 -m pip install -r /root/requirements_ubi9.txt && \
-    echo "Installed python version: $(/usr/bin/python3 -V)" && \
-    echo "Installed python packages: $(/usr/bin/python3 -m pip list)"
+RUN echo "the HTTP/S protocol is set to $PROTOCOL"  && \
+    echo "the HTTP/S basic authentication is set to $BASICAUTH"  && \
+    echo "the OpentTSDB API HTTP/S port is set to $PORT"  && \
+    echo "the Prometheus API HTTPS port is set to $PROMETHEUS"  && \
+    echo "the PERFMONPORT port is set to $SERVERPORT" && \
+    echo "the pmcollector server ip is set to $SERVER" && \
+    echo "the log will use $LOGPATH" 
+	
+RUN if [ $(expr "$BASE" : '.*python.*') -eq 0 ]; then \
+	yum install -y python39 python3-pip; \
+	if [ "$BUILD_ENV" = "build_test" ]; then \             
+	python3 -m pip install pip-tools && \
+	python3 -m piptools compile /root/requirements_ubi.in  --output-file /root/requirements_ubi9.txt && \
+	echo "Compiled python packages: $(cat /root/requirements_ubi9.txt)"; fi && \
+	python3 -m pip install -r /root/requirements_ubi9.txt && \
+        echo "Installed python version: $(python3 -V)" && \
+        echo "Installed python packages: $(python3 -m pip list)"; else \
+	echo "Already using python container as base image. No need to install it." && \ 
+	python3 -m pip install  -r /root/requirements.in && \
+	echo "Installed python packages: $(python3 -m pip list)"; fi
 
 USER root
 
@@ -101,8 +116,8 @@ COPY ./source/gpfsConfig/mmsdrfs* /var/mmfs/gen/
 COPY ./source/gpfsConfig/ZIMon* /opt/IBM/zimon/
 
 RUN if [ "${APIKEYVALUE:0:1}" = "/" ]; then ln -s $APIKEYVALUE /etc/perfmon-api-keys; echo "APIKEYVALUE is a PATH"; else echo "APIKEYVALUE not a PATH"; fi && \
-    if [ -z "$TLSKEYPATH" ] || [ -z "$TLSCERTFILE" ] || [ -z "$TLSKEYFILE" ] && [ "$PROTOCOL" = "https" ]; then echo "TLSKEYPATH FOR SSL CONNECTION NOT SET - ERROR"; exit 1; else echo "PASS"; fi
-RUN echo "the ssl certificates path is set to $TLSKEYPATH" 
+    if [ -z "$TLSKEYPATH" ] || [ -z "$TLSCERTFILE" ] || [ -z "$TLSKEYFILE" ] && [ "$PROTOCOL" = "https" ]; then echo "TLSKEYPATH FOR SSL CONNECTION NOT SET - ERROR"; exit 1; else echo "PASS"; fi && \
+    echo "the ssl certificates path is set to $TLSKEYPATH" 
 
 # Switch to the working directory
 WORKDIR /opt/IBM/bridge

docs/RELEASE_NOTES.md

@@ -1,3 +1,14 @@
+# Version 8.0.3 (01/20/2025)
+Added GPFSPoolCap, GPFSInodeCap and GPFSFCMDA sensors to the supported PrometheusExporter endpoints \
+Added HTTP Api REST endpoint for querying last metric sample (OpenTSDB plugin)
+Reworked Dockerfile allowing build grafana-bridge image from Redhat UBI9/Python3.9 \
+Changed the Dockerfile parent image to the registry.access.redhat.com/ubi9/ubi:9.5-1736404036 \ 
+
+Tested with Grafana version 11.0.0
+Tested with RedHat community-powered Grafana operator v.5
+
+
+
 # Version 8.0.2 (12/18/2024)
 Added GPFSNSDPool, GPFSNSDFS sensors to the supported PrometheusExporter endpoints
 Added LOGLEVEL to the Dockerfile editable command line arguments
@@ -41,6 +52,14 @@ Tested with RedHat community-powered Grafana operator v.5
 
 
 
+# Version 7.2.1 (01/15/2025)
+Changed the Dockerfile parent image to the registry.access.redhat.com/ubi9/ubi:9.5-1736404036 \
+
+Tested with Grafana version 11
+Tested with RedHat community-powered Grafana operator v.5
+
+
+
 # Version 7.2.0 (12/06/2024)
 Changed the Dockerfile parent image to the registry.access.redhat.com/ubi9/ubi:9.5-1732804088 \
 Speed up OpenTSDB /search/lookup REST Api endpoint response time \

docs/SUPPORT_MATRIX.md

@@ -1,4 +1,15 @@
 The following matrix gives a quick overview of the supported software for the IBM Storage Scale bridge for Grafana packages by version number:
+# Version 8.0.3 (01/20/2025)
+Classic Scale:
+ - Python 3.9
+ - CherryPy 18.10.0
+ - IBM Storage Scale system must run 5.2.2 and above
+ - Grafana 11.0.0 and above
+
+ Cloud native:
+ - IBM Storage Scale Container Native Storage Access(CNSA) devices having minReleaseLevel 5.2.3
+ - RedHat community-powered Grafana-Operator v5
+
 # Version 8.0.2 (12/18/2024)
 Classic Scale:
  - Python 3.9
@@ -7,7 +18,7 @@ Classic Scale:
  - Grafana 11.0.0 and above
 
  Cloud native:
- - IBM Storage Scale Container Native Storage Access(CNSA) devices having minReleaseLevel 5.2.0
+ - IBM Storage Scale Container Native Storage Access(CNSA) devices having minReleaseLevel 5.2.3
  - RedHat community-powered Grafana-Operator v5
 
 # Version 8.0.1 (12/10/2024)
@@ -18,7 +29,6 @@ Classic Scale:
  - Grafana 11.0.0 and above
 
  Cloud native:
- - IBM Storage Scale Container Native Storage Access(CNSA) devices having minReleaseLevel 5.2.0
  - RedHat community-powered Grafana-Operator v5
 
 # Version 8.0.0 (04/26/2024)
@@ -29,7 +39,17 @@ Classic Scale:
  - Grafana 10.2.3 and above
 
  Cloud native:
- - IBM Storage Scale Container Native Storage Access(CNSA) devices having minReleaseLevel 5.2.0
+ - RedHat community-powered Grafana-Operator v5
+
+# Version 7.2.1 (01/15/2025)
+Classic Scale:
+ - Python 3.9
+ - CherryPy 18.10.0
+ - IBM Storage Scale system must run 5.1.9 and above
+ - Grafana 11.0.0 and above
+ 
+ Cloud native:
+ - IBM Storage Scale Container Native Storage Access(CNSA) devices having minReleaseLevel 5.1.9.8,5.2.2.1
  - RedHat community-powered Grafana-Operator v5
 
 # Version 7.2.0 (12/06/2024)

requirements/requirements.in

@@ -0,0 +1,14 @@
+# This file is used for generating python requirements list "requirements.txt".
+# It includes the packages needed to be installed to run the IBM Storage Scale Performance Monitoring bridge for Grafana
+# on top of the Redhat ubi9/python-39 image.
+#
+# To update, run:
+#
+#  $ pip-compile requirements_ubi.in  --output-file requirements.txt
+#
+# setuptools
+cherrypy
+PyYAML
+# psutil
+# urllib3
+requests

requirements/requirements_ubi.in

@@ -1,6 +1,6 @@
 # This file is used for generating python requirements list "requirements_ubi9.txt".
 # It includes the packages needed to be installed to run the IBM Storage Scale Performance Monitoring bridge for Grafana
-# in an OpenShift production environment, on top of the redhat UBI8 image.
+# in an OpenShift production environment, on top of the Redhat UBI9 image.
 #
 # To update, run:
 #

source/__version__.py

@@ -20,4 +20,4 @@
 @author: HWASSMAN
 '''
 
-__version__ = '8.0.3-dev'
+__version__ = '8.0.3'

source/opentsdb.py

@@ -54,15 +54,29 @@ def TOPO(self):
     def format_response(self, data: dict, jreq: dict) -> List[dict]:
         respList = []
         metrics = set(data.values())
-        for metric in metrics:
-            for st in metric.timeseries:
-                res = SingleTimeSeriesResponse(jreq.get('inputQuery'),
-                                               jreq.get('showQuery'),
-                                               jreq.get('globalAnnotations'),
-                                               st.tags, st.aggregatedTags)
-                # self.logger.trace(f'OpenTSDB queryResponse for :
-                #                   {data.keys()[0]} with {len(st.dps)} datapoints')
-                respList.append(res.to_dict(st.dps))
+        if jreq.get('start') == 'last':
+            for metric in metrics:
+                for st in metric.timeseries:
+                    timestmp = ''
+                    val = 'null'
+                    if len(st.dps) > 0:
+                        timestmp = list(st.dps.keys())[0]
+                        val = st.dps[timestmp]
+                    res = LastSingleTimeSeriesResponse(jreq.get('inputQuery'),
+                                                       timestmp,
+                                                       val,
+                                                       st.tags)
+                    respList.append(res.to_dict())
+        else:
+            for metric in metrics:
+                for st in metric.timeseries:
+                    res = SingleTimeSeriesResponse(jreq.get('inputQuery'),
+                                                   jreq.get('showQuery'),
+                                                   jreq.get('globalAnnotations'),
+                                                   st.tags, st.aggregatedTags)
+                    # self.logger.trace(f'OpenTSDB queryResponse for :
+                    #                   {data.keys()[0]} with {len(st.dps)} datapoints')
+                    respList.append(res.to_dict(st.dps))
         return respList
 
     @execution_time()
@@ -115,29 +129,34 @@ def build_collector(self, jreq: dict) -> SensorCollector:
 
         q = jreq.get('inputQuery')
 
-        period = self.md.getSensorPeriodForMetric(q.get('metric'))
+        sensor = self.TOPO.getSensorForMetric(q.get('metric'))
+        period = self.md.getSensorPeriod(sensor)
         if period < 1:
             self.logger.error(MSG['SensorDisabled'].format(q.get('metric')))
             raise cherrypy.HTTPError(
                 400, MSG['SensorDisabled'].format(q.get('metric')))
 
-        sensor = self.TOPO.getSensorForMetric(q.get('metric'))
-
         args = {}
         args['metricsaggr'] = {q.get('metric'): q.get('aggregator')}
-        args['start'] = str(int(int(str(jreq.get('start'))) / 1000))
-        if jreq.get('end') is not None:
-            args['end'] = str(int(int(str(jreq.get('end'))) / 1000))
 
-        if q.get('downsample'):
-            args['dsOp'] = self._get_downsmpl_op(q.get('downsample'))
-            args['dsBucketSize'] = self._calc_bucket_size(q.get('downsample'))
+        if jreq.get('start') == 'last':
+            args['nsamples'] = 1
+            if q.get('tags'):
+                args['filters'] = q.get('tags')
+        else:
+            args['start'] = str(int(int(str(jreq.get('start'))) / 1000))
+            if jreq.get('end') is not None:
+                args['end'] = str(int(int(str(jreq.get('end'))) / 1000))
+
+            if q.get('downsample'):
+                args['dsOp'] = self._get_downsmpl_op(q.get('downsample'))
+                args['dsBucketSize'] = self._calc_bucket_size(q.get('downsample'))
 
-        if q.get('filters'):
-            filters, grouptags = self._parse_input_query_filters(
-                q.get('filters'))
-            args['filters'] = filters
-            args['grouptags'] = grouptags
+            if q.get('filters'):
+                filters, grouptags = self._parse_input_query_filters(
+                    q.get('filters'))
+                args['filters'] = filters
+                args['grouptags'] = grouptags
 
         args['rawData'] = q.get('explicitTags', False)
 
@@ -296,6 +315,42 @@ def GET(self, **params):
         elif 'lookup' in cherrypy.request.script_name:
             resp = self.lookup(params)
 
+        # /api/query/last
+        elif '/api/query/last' == cherrypy.request.script_name:
+            jreq = {}
+
+            if params.get('timeseries') is None:
+                self.logger.error(MSG['QueryError'].format('empty'))
+                raise cherrypy.HTTPError(400, ERR[400])
+
+            queries = []
+            timeseries = params.get('timeseries')
+            if not isinstance(timeseries, list):
+                timeseries = [timeseries]
+            for timeserie in timeseries:
+                try:
+                    metricDict = {}
+                    params_list = re.split(r'\{(.*)\}', timeserie.strip())
+                    if len(params_list[0]) == 0:
+                        break
+                    metricDict['metric'] = params_list[0]
+
+                    if len(params_list) > 1:
+                        attr = params_list[1]
+                        filterBy = dict(x.split('=') for x in attr.split(','))
+                        metricDict['tags'] = filterBy
+                    queries.append(metricDict)
+
+                except Exception as e:
+                    self.logger.exception(MSG['IntError'].format(str(e)))
+                    raise cherrypy.HTTPError(500, MSG[500])
+            if len(queries) == 0:
+                raise cherrypy.HTTPError(400, ERR[400])
+            jreq['start'] = 'last'
+            jreq['queries'] = queries
+
+            resp = self.query(jreq)
+
         elif 'aggregators' in cherrypy.request.script_name:
             resp = ["noop", "sum", "avg", "max", "min", "rate"]
 
@@ -336,7 +391,7 @@ def POST(self):
             raise cherrypy.HTTPError(400, ERR[400])
 
         # /api/query
-        if 'query' in cherrypy.request.script_name:
+        if '/api/query' == cherrypy.request.script_name:
 
             # read query request parameters
             jreq = cherrypy.request.json
@@ -399,3 +454,17 @@ def to_dict(self, dps: dict = None):
         if dps:
             res['dps'] = dps
         return res
+
+
+class LastSingleTimeSeriesResponse(object):
+
+    def __init__(self, inputQuery, timestmp, value, tags: dict = None):
+        self.metric = inputQuery.get('metric')
+        self.timestamp = timestmp
+        self.value = value
+        self.tags = tags or defaultdict(list)
+
+    def to_dict(self):
+        ''' Converts the LastSingleTimeSeriesResponse object to dict. '''
+        res = self.__dict__
+        return res

source/zimonGrafanaIntf.py

@@ -303,6 +303,12 @@ def main(argv):
                               }
                              }
                             )
+        # query metric last value
+        cherrypy.tree.mount(api, '/api/query/last',
+                            {'/':
+                             {'request.dispatch': cherrypy.dispatch.MethodDispatcher()}
+                             }
+                            )
         # query for metric name (openTSDB: zimon extension returns keys as well)
         cherrypy.tree.mount(api, '/api/suggest',
                             {'/':