Merge branch 'dev'

Author: cuprumtan

.github/workflows/mamonsu-tests-dev.yml

@@ -27,8 +27,8 @@ jobs:
     strategy:
       matrix:
         docker_os: ['centos:7']
-        zabbix_version: ['6.0.4']
-        pg_version: ['10', '11', '12', '13', '14']
+        zabbix_version: ['6.2.6']
+        pg_version: ['11', '12', '13', '14']
 
     steps:
       - name: Checkout mamonsu repo
@@ -58,15 +58,15 @@ jobs:
         run: bash ./github-actions-tests/zbx_install.sh --zbx-version="${{ matrix.zabbix_version }}"
       - name: Retrieve Zabbix address
         run: |
-          echo "::set-output name=zabbix_address::$(hostname -I | awk '{print $1}')"
+          echo "zabbix_address=$(hostname -I | awk '{print $1}')" >> $GITHUB_OUTPUT
         id: zabbix_address
       - name: Edit Zabbix address in agent.conf
         run: sed -i "s/\(address *= *\).*/\1 ${{ steps.zabbix_address.outputs.zabbix_address }}/" ${{ env.MAMONSU_PATH }}/github-actions-tests/sources/agent_3.5.2.conf
 
       - name: Copy test scripts to container
         run: docker exec $( echo "${{ matrix.docker_os }}" | sed 's/://' | sed 's/\.//' ) mkdir -p -m 755 /mamonsu/
       - run: docker cp . $( echo "${{ matrix.docker_os }}" | sed 's/://' | sed 's/\.//' ):/mamonsu
-      - name: Install prequisites
+      - name: Install prerequisites
         run: docker exec $( echo "${{ matrix.docker_os }}" | sed 's/://' | sed 's/\.//' ) bash /mamonsu/github-actions-tests/tests.sh --os="${{ matrix.docker_os }}" --pg-version="${{ matrix.pg_version }}"
 
       - name: Build and install mamonsu

.github/workflows/mamonsu-tests-master.yml

@@ -27,12 +27,12 @@ jobs:
     strategy:
       matrix:
         docker_os: ['ubuntu:20.04', 'centos:7']
-        zabbix_version: ['4.0.40', '5.0.23', '5.4.12', '6.0.4']
-        pg_version: ['10', '11', '12', '13', '14']
+        zabbix_version: ['4.0.44', '4.4.10', '5.0.30', '5.4.12', '6.0.12', '6.2.6']
+        pg_version: ['11', '12', '13', '14', '15']
         exclude:
-          # excludes PG 9.6 on CentOS 8
-          - docker_os: 'centos:8'
-            pg_version: '9.6'
+          # excludes PG 15 on CentOS
+          - docker_os: 'centos:7'
+            pg_version: '15'
 
     steps:
       - name: Checkout mamonsu repo
@@ -62,15 +62,15 @@ jobs:
         run: bash ./github-actions-tests/zbx_install.sh --zbx-version="${{ matrix.zabbix_version }}"
       - name: Retrieve Zabbix address
         run: |
-          echo "::set-output name=zabbix_address::$(hostname -I | awk '{print $1}')"
+          echo "zabbix_address=$(hostname -I | awk '{print $1}')" >> $GITHUB_OUTPUT
         id: zabbix_address
       - name: Edit Zabbix address in agent.conf
         run: sed -i "s/\(address *= *\).*/\1 ${{ steps.zabbix_address.outputs.zabbix_address }}/" ${{ env.MAMONSU_PATH }}/github-actions-tests/sources/agent_3.5.2.conf
 
       - name: Copy test scripts to container
         run: docker exec $( echo "${{ matrix.docker_os }}" | sed 's/://' | sed 's/\.//' ) mkdir -p -m 755 /mamonsu/
       - run: docker cp . $( echo "${{ matrix.docker_os }}" | sed 's/://' | sed 's/\.//' ):/mamonsu
-      - name: Install prequisites
+      - name: Install prerequisites
         run: docker exec $( echo "${{ matrix.docker_os }}" | sed 's/://' | sed 's/\.//' ) bash /mamonsu/github-actions-tests/tests.sh --os="${{ matrix.docker_os }}" --pg-version="${{ matrix.pg_version }}"
 
       - name: Build and install mamonsu

github-actions-tests/sources/metrics-linux-14.txt

@@ -87,13 +87,13 @@ pgsql.tuples[inserted]
 pgsql.tuples[returned]
 pgsql.tuples[updated]
 pgsql.uptime[]
-pgsql.wal.buffers_full
+pgsql.wal.buffers_full[]
 pgsql.wal.count[]
 pgsql.wal.fpi.count[]
 pgsql.wal.records.count[]
-pgsql.wal.sync_time
+pgsql.wal.sync_time[]
 pgsql.wal.write[]
-pgsql.wal.write_time
+pgsql.wal.write_time[]
 system.cpu[idle]
 system.cpu[iowait]
 system.cpu[irq]

github-actions-tests/tools/zabbix_cli.sh

@@ -99,7 +99,7 @@ mamonsu zabbix $OPTIONS host delete ${HOST_ID} | grep "hostids.*${HOST_ID}" || e
 
 # test 'mamonsu hostgroup'
 echo && echo "------> mamonsu zabbix hostgroup list"
-mamonsu zabbix $OPTIONS hostgroup list | grep Templates || exit 11
+mamonsu zabbix $OPTIONS hostgroup list | grep Linux || exit 11
 echo && echo "------> mamonsu zabbix hostgroup show"
 mamonsu zabbix $OPTIONS hostgroup show "Linux servers" | grep Linux || exit 11
 echo && echo "------> mamonsu zabbix hostgroup id"

mamonsu/lib/config.py

@@ -81,17 +81,19 @@ def __init__(self, cfg_file=None, plugin_directories=None):
                 sys.stderr.write('Config file is empty: {0}\n'.format(cfg_file))
                 sys.exit(1)
             if cfg_file is not None:
-                if platform.LINUX:
-                    config_status = int(repr(oct(os.stat(cfg_file).st_mode))[:-1][-3:])
-                    config_owner = pwd.getpwuid(os.stat(cfg_file).st_uid).pw_name
-                    if not (config_status == 600 and config_owner == "mamonsu"):
-                        logging.info(
-                            "Shut down because of incorrect config file {0} permissions. It must be r/w for mamonsu user only (600).".format(
-                                cfg_file))
-                        sys.stderr.write(
-                            "Please, check your config file {0} permissions. It must be r/w for mamonsu user only (600).\n".format(
-                                cfg_file))
-                        sys.exit(1)
+                # TODO: instead of hardcoded "mamonsu" user get real mamonsu owner user
+                #  and check if config file belongs to this user
+                # if platform.LINUX:
+                #     config_status = int(repr(oct(os.stat(cfg_file).st_mode))[:-1][-3:])
+                #     config_owner = pwd.getpwuid(os.stat(cfg_file).st_uid).pw_name
+                #     if not (config_status == 600 and config_owner == "mamonsu"):
+                #         logging.info(
+                #             "Shut down because of incorrect config file {0} permissions. It must be r/w for mamonsu user only (600).".format(
+                #                 cfg_file))
+                #         sys.stderr.write(
+                #             "Please, check your config file {0} permissions. It must be r/w for mamonsu user only (600).\n".format(
+                #                 cfg_file))
+                #         sys.exit(1)
                 self.config.read_file(open(cfg_file))
 
         plugins = self.fetch('plugins', 'directory', str)

mamonsu/lib/plugin.py

@@ -24,7 +24,7 @@ class Plugin(object):
     AgentPluginType = 'all'
 
     # PG version
-    VersionPG = '10'
+    VersionPG = '14'
 
     # Macros for run as agent type or as mamonsu
     Macros = {"mamonsu": "", "agent": "{$PG_CONNINFO},{$PG_PATH}"}

mamonsu/lib/runner.py

@@ -224,7 +224,7 @@ def is_any_equal(array):
 #  extract pg version from input
 def define_pg_version(version_args):
     if len(version_args) < 4:
-        if version_args == "14" or version_args == "11" or version_args == "12" or version_args == "13" or version_args == "10" \
+        if version_args == "15" or version_args == "14" or version_args == "11" or version_args == "12" or version_args == "13" or version_args == "10" \
                 or version_args == "9.6" or version_args == "9.5":
             version_number = version_args[0].split('.')
             for num in version_number:

mamonsu/lib/zbx_template.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 import re
+
 import mamonsu.lib.platform as platform
 from mamonsu.lib.const import Template
 from mamonsu.lib.plugin import Plugin
@@ -174,9 +175,9 @@ def xml(self, plg_type, plugins=None):
         template_data = {'template': self.Template, 'application': self.Application}
         template_data['items'] = self._get_all('items', plugins)
         template_data['discovery_rules'] = self._get_all('discovery_rules', plugins)
-        if Plugin.Type == 'agent':
-            template_data['macros'] = self.agent_macro()
         template_data['macros'] = self._get_all('macros', plugins)
+        if Plugin.Type == 'agent':
+            template_data['macros'] += self.agent_macro()
         template_data['screens'] = self.screen(plugins)
         template_data['triggers'] = self._get_all('triggers', plugins)
         template_data['graphs'] = self._get_all('graphs', plugins)
@@ -301,9 +302,9 @@ def screen(self, plugins=None, xml_key='screen'):
 
     def agent_macro(self, xml_key='macro'):
         result = ''
-        value = {'value': '-qAt -p 5433 -U postgres ', 'macro': "{$PG_CONNINFO}"}
+        value = {'value': '-qAt -p 5432 -U postgres ', 'macro': "{$PG_CONNINFO}"}
         result += '<{1}>{0}</{1}>'.format(self._format_args(self.macro_defaults, value), xml_key)
-        value = {'value': '/opt/pgpro/std-10/bin/psql', 'macro': "{$PG_PATH}"}
+        value = {'value': '/usr/bin/psql', 'macro': "{$PG_PATH}"}
         result += '<{1}>{0}</{1}>'.format(self._format_args(self.macro_defaults, value), xml_key)
         return result
 

mamonsu/plugins/pgsql/archive_command.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 
 from mamonsu.plugins.pgsql.plugin import PgsqlPlugin as Plugin
-from distutils.version import LooseVersion
 from .pool import Pooler
 from mamonsu.lib.zbx_template import ZbxTemplate
 import re
@@ -46,7 +45,7 @@ class ArchiveCommand(Plugin):
     ('x' || substring(pg_{1}_name(pg_current_{0}()) from 17 for 8))::bit(32)::int END AS current_wal_mod
     FROM pg_settings, pg_stat_archiver
     WHERE pg_settings.name = 'wal_segment_size')
-    greatest(coalesce(((segment_parts_count - last_wal_mod) + ((current_wal_div - last_wal_div - 1) * segment_parts_count) + current_wal_mod - 1) * segment_size, 0), 0)::bigint AS size_files
+    SELECT greatest(coalesce(((segment_parts_count - last_wal_mod) + ((current_wal_div - last_wal_div - 1) * segment_parts_count) + current_wal_mod - 1) * segment_size, 0), 0)::bigint AS size_files
     FROM values;
     """
 
@@ -59,10 +58,12 @@ class ArchiveCommand(Plugin):
     key = "pgsql.archive_command{0}"
     Items = [
         # key, desc, color, side, graph, delta, units
-        ("count_files_to_archive", "Files in archive_status Need to Archive Count", "006AAE", 0, 1, Plugin.DELTA.as_is, Plugin.UNITS.none),
+        ("count_files_to_archive", "Files in archive_status Need to Archive Count", "006AAE", 0, 1, Plugin.DELTA.as_is,
+         Plugin.UNITS.none),
         ("size_files_to_archive", "Files Need to Archive Size", "793F5D", 0, 0, Plugin.DELTA.as_is, Plugin.UNITS.bytes),
         ("archived_files", "Archived Files Count", "00CC00", 0, 1, Plugin.DELTA.simple_change, Plugin.UNITS.none),
-        ("failed_trying_to_archive", "Attempts to Archive Files Count", "FF5656", 0, 1, Plugin.DELTA.simple_change, Plugin.UNITS.none),
+        ("failed_trying_to_archive", "Attempts to Archive Files Count", "FF5656", 0, 1, Plugin.DELTA.simple_change,
+         Plugin.UNITS.none),
     ]
     old_archived_count = None
     old_failed_count = None
@@ -192,25 +193,27 @@ def triggers(self, template, dashboard=False):
         return template.trigger({
             "name": "PostgreSQL Archiver: count files need to archive on {HOSTNAME} more than 2",
             "expression": "{#TEMPLATE:" + self.right_type(self.key,
-                                                          self.Items[0][0]) + ".last()}>" + self.plugin_macros["archive_queue_files"][0][1]
+                                                          self.Items[0][0]) + ".last()}>" +
+                          self.plugin_macros["archive_queue_files"][0][1]
         })
 
     def keys_and_queries(self, template_zabbix):
         result = []
-        if LooseVersion(self.VersionPG) >= LooseVersion("10"):
+        if Pooler.server_version_greater("10"):
             result.append("{0}[*],$2 $1 -c \"{1}\"".format(self.key.format("." + self.Items[0][0]),
-                                                           self.query_agent_count_files.format("wal_lsn", "walfile")))
+                                                              self.query_agent_count_files.format("wal_lsn",
+                                                                                                  "walfile")))
             result.append("{0}[*],$2 $1 -c \"{1}\"".format(self.key.format("." + self.Items[1][0]),
-                                                           self.query_agent_size_files.format("wal_lsn", "walfile")))
+                                                              self.query_agent_size_files.format("wal_lsn", "walfile")))
         else:
             result.append("{0}[*],$2 $1 -c \"{1}\"".format(self.key.format("." + self.Items[0][0]),
-                                                           self.query_agent_count_files.format("xlog_location",
-                                                                                               "xlogfile")))
+                                                              self.query_agent_count_files.format("xlog_location",
+                                                                                                  "xlogfile")))
             result.append("{0}[*],$2 $1 -c \"{1}\"".format(self.key.format("." + self.Items[1][0]),
-                                                           self.query_agent_size_files.format("xlog_location",
-                                                                                              "xlogfile")))
+                                                              self.query_agent_size_files.format("xlog_location",
+                                                                                                 "xlogfile")))
         result.append("{0}[*],$2 $1 -c \"{1}\"".format(self.key.format("." + self.Items[2][0]),
-                                                       self.query_agent_archived_count))
+                                                          self.query_agent_archived_count))
         result.append("{0}[*],$2 $1 -c \"{1}\"".format(self.key.format("." + self.Items[3][0]),
-                                                       self.query_agent_failed_count))
+                                                          self.query_agent_failed_count))
         return template_zabbix.key_and_query(result)

mamonsu/plugins/pgsql/autovacuum.py

@@ -1,11 +1,10 @@
 # -*- coding: utf-8 -*-
-from distutils.version import LooseVersion
 from mamonsu.plugins.pgsql.plugin import PgsqlPlugin as Plugin
 from .pool import Pooler
 from mamonsu.lib.zbx_template import ZbxTemplate
 
+
 class Autovacuum(Plugin):
-    
     AgentPluginType = "pg"
     # TODO: unify keys and remove its direct mentioning in zbx.send() functions
     key_count = "pgsql.autovacuum.count{0}"
@@ -21,7 +20,8 @@ class Autovacuum(Plugin):
     def run(self, zbx):
         if Pooler.server_version_greater("10.0"):
             result_count = Pooler.run_sql_type("count_autovacuum", args=["backend_type = 'autovacuum worker'"])
-            result_utilization = Pooler.run_sql_type("autovacuum_utilization", args=["backend_type = 'autovacuum worker'"])
+            result_utilization = Pooler.run_sql_type("autovacuum_utilization",
+                                                     args=["backend_type = 'autovacuum worker'"])
         else:
             result_count = Pooler.run_sql_type("count_autovacuum", args=[
                 "query LIKE '%%autovacuum%%' AND state <> 'idle' AND pid <> pg_catalog.pg_backend_pid()"])
@@ -51,7 +51,7 @@ def items(self, template, dashboard=False):
                 "value_type": Plugin.VALUE_TYPE.numeric_float,
                 "units": Plugin.UNITS.percent,
                 "type": Plugin.TYPE.CALCULATED,
-                "params": "avg(pgsql.autovacuum.utilization[], 5m)",
+                "params": "avg(//{item}, 5m)".format(item=self.right_type(self.key_utilization)),
                 "delay": self.plugin_config("interval")
             }))
             result += (template.item({
@@ -60,7 +60,7 @@ def items(self, template, dashboard=False):
                 "value_type": Plugin.VALUE_TYPE.numeric_float,
                 "units": Plugin.UNITS.percent,
                 "type": Plugin.TYPE.CALCULATED,
-                "params": "avg(pgsql.autovacuum.utilization[], 15m)",
+                "params": "avg(//{item}, 15m)".format(item=self.right_type(self.key_utilization)),
                 "delay": self.plugin_config("interval")
             }))
             result += (template.item({
@@ -69,13 +69,13 @@ def items(self, template, dashboard=False):
                 "value_type": Plugin.VALUE_TYPE.numeric_float,
                 "units": Plugin.UNITS.percent,
                 "type": Plugin.TYPE.CALCULATED,
-                "params": "avg(pgsql.autovacuum.utilization[], 30m)",
+                "params": "avg(//{item}, 30m)".format(item=self.right_type(self.key_utilization)),
                 "delay": self.plugin_config("interval")
             }))
             return result
         else:
             return []
-    
+
     def graphs(self, template, dashboard=False):
         result = template.graph({
             "name": "PostgreSQL Autovacuum: Count of Autovacuum Workers",
@@ -97,16 +97,20 @@ def graphs(self, template, dashboard=False):
 
     def keys_and_queries(self, template_zabbix):
         result = []
-        if LooseVersion(self.VersionPG) >= LooseVersion("10"):
-            result.append("{0},$2 $1 -c \"{1}\"".format(self.key_count.format("[*]"),
-                                                        Pooler.SQL["count_autovacuum"][0].format("backend_type = 'autovacuum worker'")))
+        if Pooler.server_version_greater("10"):
+            # TODO: define another metric key because it duplicates native zabbix agents keys
+            # result.append("{0},$2 $1 -c \"{1}\"".format(self.key_count.format("[*]"),
+            #                                                Pooler.SQL["count_autovacuum"][0].format(
+            #                                                    "backend_type = 'autovacuum worker'")))
             result.append("{0},$2 $1 -c \"{1}\"".format(self.key_utilization.format("[*]"),
-                                                        Pooler.SQL["autovacuum_utilization"][0].format(
-                                                            "backend_type = 'autovacuum worker'")))
+                                                           Pooler.SQL["autovacuum_utilization"][0].format(
+                                                               "backend_type = 'autovacuum worker'")))
         else:
-            result.append("{0},$2 $1 -c \"{1}\"".format(self.key_count.format("[*]"),
-                                                        Pooler.SQL["count_autovacuum"][0].format("query LIKE '%%autovacuum%%' AND state <> 'idle' AND pid <> pg_catalog.pg_backend_pid()")))
+            # TODO: define another metric key because it duplicates native zabbix agents keys
+            # result.append("{0},$2 $1 -c \"{1}\"".format(self.key_count.format("[*]"),
+            #                                                Pooler.SQL["count_autovacuum"][0].format(
+            #                                                    "query LIKE '%%autovacuum%%' AND state <> 'idle' AND pid <> pg_catalog.pg_backend_pid()")))
             result.append("{0},$2 $1 -c \"{1}\"".format(self.key_utilization.format("[*]"),
-                                                        Pooler.SQL["autovacuum_utilization"][0].format(
-                                                            "query LIKE '%%autovacuum%%' AND state <> 'idle' AND pid <> pg_catalog.pg_backend_pid()")))
+                                                           Pooler.SQL["autovacuum_utilization"][0].format(
+                                                               "query LIKE '%%autovacuum%%' AND state <> 'idle' AND pid <> pg_catalog.pg_backend_pid()")))
         return template_zabbix.key_and_query(result)