feature: drafted new autovacuum utilization metrics

cuprumtan · cuprumtan · commit 5ee6c58c07a9 · 2022-08-16T11:56:06.000+03:00
(cherry picked from commit 3f937df)
diff --git a/Makefile.pkg b/Makefile.pkg
@@ -12,7 +12,7 @@ prepare_builddir: clean
 	chown -R root.root $(BUILDDIR)
 
 deb: prepare_builddir
-	cd $(BUILDDIR) && cp -a $(WORKDIR)/packaging/debian . && dpkg-buildpackage -b
+	cd $(BUILDDIR) && cp -a $(WORKDIR)/packaging/debian . && dpkg-buildpackage -b --no-sign
 	cp -av build/mamonsu*.deb .
 
 rpm: prepare_builddir $(BUILDDIR)/packaging/rpm/SPECS/$(SPEC)
diff --git a/github-actions-tests/sources/metrics-linux-10.txt b/github-actions-tests/sources/metrics-linux-10.txt
@@ -5,7 +5,8 @@ pgsql.archive_command[archived_files]
 pgsql.archive_command[count_files_to_archive]
 pgsql.archive_command[failed_trying_to_archive]
 pgsql.archive_command[size_files_to_archive]
-pgsql.autovacumm.count[]
+pgsql.autovacuum.count[]
+pgsql.autovacuum.utilization[]
 pgsql.bgwriter[buffers_alloc]
 pgsql.bgwriter[buffers_backend]
 pgsql.bgwriter[buffers_backend_fsync]
diff --git a/github-actions-tests/sources/metrics-linux-11.txt b/github-actions-tests/sources/metrics-linux-11.txt
@@ -5,7 +5,8 @@ pgsql.archive_command[archived_files]
 pgsql.archive_command[count_files_to_archive]
 pgsql.archive_command[failed_trying_to_archive]
 pgsql.archive_command[size_files_to_archive]
-pgsql.autovacumm.count[]
+pgsql.autovacuum.count[]
+pgsql.autovacuum.utilization[]
 pgsql.bgwriter[buffers_alloc]
 pgsql.bgwriter[buffers_backend]
 pgsql.bgwriter[buffers_backend_fsync]
diff --git a/github-actions-tests/sources/metrics-linux-12.txt b/github-actions-tests/sources/metrics-linux-12.txt
@@ -5,7 +5,8 @@ pgsql.archive_command[archived_files]
 pgsql.archive_command[count_files_to_archive]
 pgsql.archive_command[failed_trying_to_archive]
 pgsql.archive_command[size_files_to_archive]
-pgsql.autovacumm.count[]
+pgsql.autovacuum.count[]
+pgsql.autovacuum.utilization[]
 pgsql.bgwriter[buffers_alloc]
 pgsql.bgwriter[buffers_backend]
 pgsql.bgwriter[buffers_backend_fsync]
diff --git a/github-actions-tests/sources/metrics-linux-13.txt b/github-actions-tests/sources/metrics-linux-13.txt
@@ -5,7 +5,8 @@ pgsql.archive_command[archived_files]
 pgsql.archive_command[count_files_to_archive]
 pgsql.archive_command[failed_trying_to_archive]
 pgsql.archive_command[size_files_to_archive]
-pgsql.autovacumm.count[]
+pgsql.autovacuum.count[]
+pgsql.autovacuum.utilization[]
 pgsql.bgwriter[buffers_alloc]
 pgsql.bgwriter[buffers_backend]
 pgsql.bgwriter[buffers_backend_fsync]
diff --git a/github-actions-tests/sources/metrics-linux-14.txt b/github-actions-tests/sources/metrics-linux-14.txt
@@ -5,7 +5,8 @@ pgsql.archive_command[archived_files]
 pgsql.archive_command[count_files_to_archive]
 pgsql.archive_command[failed_trying_to_archive]
 pgsql.archive_command[size_files_to_archive]
-pgsql.autovacumm.count[]
+pgsql.autovacuum.count[]
+pgsql.autovacuum.utilization[]
 pgsql.bgwriter[buffers_alloc]
 pgsql.bgwriter[buffers_backend]
 pgsql.bgwriter[buffers_backend_fsync]
diff --git a/github-actions-tests/sources/metrics-linux-9.6.txt b/github-actions-tests/sources/metrics-linux-9.6.txt
@@ -5,7 +5,8 @@ pgsql.archive_command[archived_files]
 pgsql.archive_command[count_files_to_archive]
 pgsql.archive_command[failed_trying_to_archive]
 pgsql.archive_command[size_files_to_archive]
-pgsql.autovacumm.count[]
+pgsql.autovacuum.count[]
+pgsql.autovacuum.utilization[]
 pgsql.bgwriter[buffers_alloc]
 pgsql.bgwriter[buffers_backend]
 pgsql.bgwriter[buffers_backend_fsync]
diff --git a/github-actions-tests/sources/metrics-linux-full-list.txt b/github-actions-tests/sources/metrics-linux-full-list.txt
@@ -5,7 +5,8 @@ pgsql.archive_command[archived_files]
 pgsql.archive_command[count_files_to_archive]
 pgsql.archive_command[failed_trying_to_archive]
 pgsql.archive_command[size_files_to_archive]
-pgsql.autovacumm.count[]
+pgsql.autovacuum.count[]
+pgsql.autovacuum.utilization[]
 pgsql.bgwriter[buffers_alloc]
 pgsql.bgwriter[buffers_backend]
 pgsql.bgwriter[buffers_backend_fsync]
diff --git a/mamonsu/plugins/pgsql/autovacuum.py b/mamonsu/plugins/pgsql/autovacuum.py
@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+from distutils.version import LooseVersion
+from mamonsu.plugins.pgsql.plugin import PgsqlPlugin as Plugin
+from .pool import Pooler
+from mamonsu.lib.zbx_template import ZbxTemplate
+
+class Autovacuum(Plugin):
+    
+    AgentPluginType = "pg"
+    key = "pgsql.autovacuum.count{0}"
+
+    def run(self, zbx):
+        if Pooler.server_version_greater("10.0"):
+            result = Pooler.run_sql_type("count_autovacuum", args=["backend_type = 'autovacuum worker'"])
+        else:
+            result = Pooler.run_sql_type("count_autovacuum", args=[
+                "query LIKE '%%autovacuum%%' AND state <> 'idle' AND pid <> pg_catalog.pg_backend_pid()"])
+        zbx.send("pgsql.autovacuum.count[]", int(result[0][0]))
+    
+    def items(self, template, dashboard=False):
+        if not dashboard:
+            return template.item({
+                "name": "PostgreSQL Autovacuum: Count of Autovacuum Workers",
+                "key": self.right_type(self.key),
+                "delay": self.plugin_config("interval")
+            })
+        else:
+            return []
+    
+    def graphs(self, template, dashboard=False):
+        result = template.graph({
+            "name": "PostgreSQL Autovacuum: Count of Autovacuum Workers",
+            "items": [{
+                "key": self.right_type(self.key),
+                "color": "87C2B9",
+                "drawtype": 2
+            }]
+        })
+        if not dashboard:
+            return result
+        else:
+            return [{
+                "dashboard": {"name": "PostgreSQL Autovacuum: Count of Autovacuum Workers",
+                              "page": ZbxTemplate.dashboard_page_overview["name"],
+                              "size": ZbxTemplate.dashboard_widget_size_medium,
+                              "position": 5}
+            }]
+
+    def keys_and_queries(self, template_zabbix):
+        result = []
+        if LooseVersion(self.VersionPG) >= LooseVersion("10"):
+            result.append("{0},$2 $1 -c \"{1}\"".format(self.key.format("[*]"), 
+                                                        Pooler.SQL["count_autovacuum"][0].format("backend_type = 'autovacuum worker'")))
+        else:
+            result.append("{0},$2 $1 -c \"{1}\"".format(self.key.format("[*]"), 
+                                                        Pooler.SQL["count_autovacuum"][0].format("query LIKE '%%autovacuum%%' AND state <> 'idle' AND pid <> pg_catalog.pg_backend_pid()")))
+        return template_zabbix.key_and_query(result)
diff --git a/mamonsu/plugins/pgsql/databases.py b/mamonsu/plugins/pgsql/databases.py
@@ -4,6 +4,7 @@
 from mamonsu.plugins.pgsql.plugin import PgsqlPlugin as Plugin
 from .pool import Pooler
 from mamonsu.lib.zbx_template import ZbxTemplate
+from mamonsu.plugins.pgsql.autovacuum import Autovacuum
 
 
 class Databases(Plugin):
@@ -50,7 +51,6 @@ class Databases(Plugin):
     key_db_size = "pgsql.database.size{0}"
     key_db_age = "pgsql.database.max_age{0}"
     key_db_bloating_tables = "pgsql.database.bloating_tables{0}"
-    key_autovacumm = "pgsql.autovacumm.count{0}"
     key_invalid_indexes = "pgsql.database.invalid_indexes{0}"
 
     DEFAULT_CONFIG = {
@@ -81,42 +81,6 @@ def run(self, zbx):
         zbx.send("pgsql.database.discovery[]", zbx.json({"data": dbs}))
         del dbs, bloat_count, invalid_indexes_count
 
-        if Pooler.server_version_greater("10.0"):
-            result = Pooler.run_sql_type("count_autovacuum", args=["backend_type = 'autovacuum worker'"])
-        else:
-            result = Pooler.run_sql_type("count_autovacuum", args=[
-                "query LIKE '%%autovacuum%%' AND state <> 'idle' AND pid <> pg_catalog.pg_backend_pid()"])
-        zbx.send("pgsql.autovacumm.count[]", int(result[0][0]))
-
-    def items(self, template, dashboard=False):
-        if not dashboard:
-            return template.item({
-                "name": "PostgreSQL Autovacuum: Count of Autovacuum Workers",
-                "key": self.right_type(self.key_autovacumm),
-                "delay": self.plugin_config("interval")
-            })
-        else:
-            return []
-
-    def graphs(self, template, dashboard=False):
-        result = template.graph({
-            "name": "PostgreSQL Autovacuum: Count of Autovacuum Workers",
-            "items": [{
-                "key": self.right_type(self.key_autovacumm),
-                "color": "87C2B9",
-                "drawtype": 2
-            }]
-        })
-        if not dashboard:
-            return result
-        else:
-            return [{
-                "dashboard": {"name": "PostgreSQL Autovacuum: Count of Autovacuum Workers",
-                              "page": ZbxTemplate.dashboard_page_overview["name"],
-                              "size": ZbxTemplate.dashboard_widget_size_medium,
-                              "position": 5}
-            }]
-
     def discovery_rules(self, template, dashboard=False):
         rule = {
             "name": "PostgreSQL Databases Discovery",
@@ -166,7 +130,7 @@ def discovery_rules(self, template, dashboard=False):
                      "key": self.right_type(self.key_db_bloating_tables, var_discovery="{#DATABASE},"),
                      "drawtype": 2},
                     {"color": "793F5D",
-                     "key": self.right_type(self.key_autovacumm),
+                     "key": self.right_type(Autovacuum.key_count),
                      "yaxisside": 1,
                      "drawtype": 2}]
             },
@@ -177,7 +141,7 @@ def discovery_rules(self, template, dashboard=False):
                      "key": self.right_type(self.key_db_age, var_discovery="{#DATABASE},"),
                      "drawtype": 2},
                     {"color": "793F5D",
-                     "key": self.right_type(self.key_autovacumm),
+                     "key": self.right_type(Autovacuum.key_count),
                      "yaxisside": 1,
                      "drawtype": 2}]
             }]
@@ -197,12 +161,4 @@ def keys_and_queries(self, template_zabbix):
                                                               self.plugin_config("min_rows"))),
                   "{0},$3 $2 -d \"$1\" -c \"{1}\"".format(self.key_invalid_indexes.format("[*]"),
                                                           self.query_invalid_indexes)]
-        if LooseVersion(self.VersionPG) >= LooseVersion("10"):
-            result.append("{0},$2 $1 -c \"{1}\"".format(self.key_autovacumm.format("[*]"),
-                                                        Pooler.SQL["count_autovacuum"][0].format(
-                                                            "backend_type = 'autovacuum worker'")))
-        else:
-            result.append("{0},$2 $1 -c \"{1}\"".format(self.key_autovacumm.format("[*]"),
-                                                        Pooler.SQL["count_autovacuum"][0].format(
-                                                            "query LIKE '%%autovacuum%%' AND state <> 'idle' AND pid <> pg_catalog.pg_backend_pid()")))
         return template_zabbix.key_and_query(result)
diff --git a/mamonsu/plugins/pgsql/driver/pool.py b/mamonsu/plugins/pgsql/driver/pool.py
@@ -47,6 +47,25 @@ class Pool(object):
             SELECT mamonsu.count_autovacuum();
             """
         ),
+        "autovacuum_utilization": (
+            """
+            WITH count_tb AS (
+                    SELECT count(*)::float AS count
+                    FROM pg_catalog.pg_stat_activity
+                    WHERE {0}
+                    ),
+            settings_tb AS (
+                    SELECT setting::float
+                    FROM pg_catalog.pg_settings
+                    WHERE name = 'autovacuum_max_workers'
+            )
+            SELECT count_tb.count*100/settings_tb.setting
+            FROM count_tb, settings_tb;
+            """,
+            """
+            SELECT mamonsu.autovacuum_utilization();
+            """
+        ),
         "buffer_cache": (
             """
             SELECT sum(1) * (current_setting('block_size')::int8) AS size,
diff --git a/mamonsu/tools/bootstrap/sql.py b/mamonsu/tools/bootstrap/sql.py
@@ -81,6 +81,49 @@
 END
 $do$;
 
+DO
+$do$
+BEGIN
+   IF (SELECT setting::integer FROM pg_settings WHERE name = 'server_version_num') >= 100000 THEN
+      DROP FUNCTION IF EXISTS mamonsu.autovacuum_utilization();
+      CREATE OR REPLACE FUNCTION mamonsu.autovacuum_utilization()
+      RETURNS FLOAT AS $$
+         WITH count_tb AS (
+            SELECT count(*)::float AS count
+            FROM pg_catalog.pg_stat_activity
+            WHERE backend_type = 'autovacuum worker'
+         ),
+         settings_tb AS (
+            SELECT setting::float
+            FROM pg_catalog.pg_settings
+            WHERE name = 'autovacuum_max_workers'
+         )
+         SELECT count_tb.count*100/settings_tb.setting
+         FROM count_tb, settings_tb
+      $$ LANGUAGE SQL SECURITY DEFINER;
+   ELSE
+      DROP FUNCTION IF EXISTS mamonsu.count_autovacuum();
+      CREATE OR REPLACE FUNCTION mamonsu.count_autovacuum()
+      RETURNS FLOAT AS $$
+         WITH count_tb AS (
+            SELECT count(*)::float AS count
+            FROM pg_catalog.pg_stat_activity
+            WHERE query LIKE '%%autovacuum%%'
+            AND state <> 'idle'
+            AND pid <> pg_catalog.pg_backend_pid()
+         ),
+         settings_tb AS (
+            SELECT setting::float
+            FROM pg_catalog.pg_settings
+            WHERE name = 'autovacuum_max_workers'
+         )
+         SELECT count_tb.count*100/settings_tb.setting
+         FROM count_tb, settings_tb
+      $$ LANGUAGE SQL SECURITY DEFINER;
+   END IF;
+END
+$do$;
+
 DO
 $do$
 BEGIN
