Skip to content

Commit 5299263

Browse files
andrewpapsujkoandrewpapsujko
committed
feat: added metric that shows bytes held by non-active replication slot
and according trigger
1 parent 8c198ef commit 5299263

File tree

6 files changed

+172
-7
lines changed

6 files changed

+172
-7
lines changed

‎documentation/metrics.md‎

Lines changed: 62 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3691,7 +3691,8 @@ Default config:
36913691
### Replication
36923692

36933693
Default config:
3694-
        lag_more_than_in_sec = 300
3694+
        lag_more_than_in_sec = 300\
3695+
        critical_bytes_held_by_non_active_slot = 1073741824 bytes
36953696

36963697
### Items
36973698

@@ -3763,6 +3764,37 @@ Default config:
37633764

37643765
*Non-active Replication Slots* calculates as count of slots with `false` active status.
37653766

3767+
- **Bytes Held By Non-active Replication Slots**
3768+
3769+
Zabbix item:
3770+
<table>
3771+
<tr>
3772+
<th>Name</th>
3773+
<td>PostgreSQL Replication: Bytes held by non-active slot {#NON_ACTIVE_SLOT_NAME}</td>
3774+
</tr>
3775+
<tr>
3776+
<th>Key</th>
3777+
<td>pgsql.replication.non_active_slots_discovery[]</td>
3778+
</tr>
3779+
<tr>
3780+
<th>Type</th>
3781+
<td>Numeric (float)</td>
3782+
</tr>
3783+
<tr>
3784+
<th>Units</th>
3785+
<td>Bytes</td>
3786+
</tr>
3787+
<tr>
3788+
<th>Delta</th>
3789+
<td>As Is</td>
3790+
</tr>
3791+
<tr>
3792+
<th>Supported Version</th>
3793+
<td>10+</td>
3794+
</tr>
3795+
</table>
3796+
3797+
*Non-active Replication Slots* calculates as count of slots with `false` active status.
37663798

37673799
- **Streaming Replication Lag**
37683800

@@ -3861,12 +3893,40 @@ Default config:
38613893
</tr>
38623894
</table>
38633895

3896+
- **PostgreSQL Replication: Non-active Slots Discovery**
3897+
3898+
Items:
3899+
<table>
3900+
<tr>
3901+
<th>Name</th>
3902+
<td>PostgreSQL Replication: Bytes held by non-active slot {#NON_ACTIVE_SLOT_NAME}</td>
3903+
</tr>
3904+
<tr>
3905+
<th>Key</th>
3906+
<td>pgsql.replication.non_active_slots_discovery[]</td>
3907+
</tr>
3908+
<tr>
3909+
<th>Type</th>
3910+
<td>Numeric (float)</td>
3911+
</tr>
3912+
<tr>
3913+
<th>Units</th>
3914+
<td>Bytes</td>
3915+
</tr>
3916+
<tr>
3917+
<th>Delta</th>
3918+
<td>As Is</td>
3919+
</tr>
3920+
</table>
3921+
38643922
### Triggers
38653923

38663924
- **PostgreSQL Instance: server mode has been changed on {HOSTNAME} to {ITEM.LASTVALUE}**
38673925

38683926
- **PostgreSQL number of non-active replication slots on {HOSTNAME} (value={ITEM.LASTVALUE})**
3869-
3927+
Disabled by default
3928+
- **PostgreSQL Replication: bytes held by slot {#NON_ACTIVE_SLOT_NAME} is too high (value={ITEM.LASTVALUE})**
3929+
Triggers if *PostgreSQL Replication: Bytes held by non-active slot {#NON_ACTIVE_SLOT_NAME}* exceeds `critical_bytes_held_by_non_active_slot`.
38703930
- **PostgreSQL streaming lag too high on {HOSTNAME} (value={ITEM.LASTVALUE})**
38713931
Triggers if *PostgreSQL Replication: Streaming Replication Lag* exceeds `lag_more_than_in_sec`.
38723932

‎mamonsu/lib/default_config.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ def default_host():
3535
host = os.environ.get('PGHOST') or 'auto'
3636
if platform.FREEBSD:
3737
host = os.environ.get('PGHOST') or 'auto'
38+
if platform.DARWIN:
39+
host = os.environ.get('PGHOST') or 'auto'
3840
return host
3941

4042
@staticmethod

‎mamonsu/lib/platform.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,5 +3,6 @@
33
LINUX = (sys.platform == 'linux' or sys.platform == 'linux2')
44
WINDOWS = (sys.platform == 'win32' or sys.platform == 'win64')
55
FREEBSD = ('freebsd' in sys.platform)
6+
DARWIN = sys.platform == 'darwin'
67
UNIX = LINUX or FREEBSD
78
INTEGER_TYPES = int,

‎mamonsu/plugins/pgsql/driver/pool.py‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ class Pool(object):
8686
"""
8787
SELECT application_name,
8888
{0}
89-
coalesce((pg_{1}_{2}_diff(pg_current_{1}_{2}(), replay_lsn))::int, 0) AS total_lag
89+
coalesce((pg_{1}_{2}_diff(pg_current_{1}_{2}(), replay_{2}))::int, 0) AS total_lag
9090
FROM pg_stat_replication;
9191
""",
9292
"""
@@ -95,6 +95,30 @@ class Pool(object):
9595
total_lag
9696
FROM mamonsu.count_{1}_lag_lsn();
9797
"""
98+
),
99+
"wal_held_bytes_master": (
100+
"""
101+
SELECT slot_name,
102+
coalesce((pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn))::int, 0) AS wal_held_bytes
103+
FROM pg_replication_slots;
104+
""",
105+
"""
106+
SELECT slot_name,
107+
wal_held_bytes
108+
FROM mamonsu.bytes_held_by_inactive_slot_on_master();
109+
"""
110+
),
111+
"wal_held_bytes_replica": (
112+
"""
113+
SELECT slot_name,
114+
coalesce((pg_wal_lsn_diff(pg_last_wal_replay_lsn(), restart_lsn))::int, 0) AS wal_held_bytes
115+
FROM pg_replication_slots;
116+
""",
117+
"""
118+
SELECT slot_name,
119+
wal_held_bytes
120+
FROM mamonsu.bytes_held_by_inactive_slot_on_replica();
121+
"""
98122
)
99123
}
100124

‎mamonsu/plugins/pgsql/replication.py‎

Lines changed: 65 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ class Replication(Plugin):
1313
AgentPluginType = "pg"
1414
# key: (macro, value)
1515
plugin_macros = {
16-
"critical_lag_seconds": [("macro", "{$CRITICAL_LAG_SECONDS}"), ("value", 60 * 5)]
16+
"critical_lag_seconds": [("macro", "{$CRITICAL_LAG_SECONDS}"), ("value", 60 * 5)],
17+
"critical_bytes_held_by_none_active_slot": [("macro", "{$CRITICAL_BYTES_HELD_BY_NON_ACTIVE_SLOT}"), ("value", 1024 * 1024 * 1024)]
1718
}
1819

1920
# get time of replication lag
@@ -30,8 +31,15 @@ class Replication(Plugin):
3031
WHERE active = 'false';
3132
"""
3233

34+
query_bytes_held_by_non_active_slot = """
35+
SELECT slot_name, coalesce(pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn)::bigint, 0) AS wal_size_bytes
36+
FROM pg_replication_slots
37+
WHERE active = 'false';
38+
"""
39+
3340
# for discovery rule for name of each replica
3441
key_lsn_replication_discovery = "pgsql.replication.discovery{0}"
42+
key_replication_non_active_slots_discovery = "pgsql.replication.non_active_slots_discovery{0}"
3543
key_total_lag = "pgsql.replication.total_lag{0}"
3644
# for PG 10 and higher
3745
key_flush = "pgsql.replication.flush_lag{0}"
@@ -42,6 +50,7 @@ class Replication(Plugin):
4250

4351
key_replication = "pgsql.replication_lag{0}"
4452
key_non_active_slots = "pgsql.replication.non_active_slots{0}"
53+
key_non_active_slots_held_bytes = "pgsql.replication.non_active_slots_held_bytes{0}"
4554

4655
def run(self, zbx):
4756

@@ -79,6 +88,14 @@ def run(self, zbx):
7988
zbx.send("pgsql.replication.replay_lag[{0}]".format(info[0]), float(info[5]))
8089
zbx.send("pgsql.replication.discovery[]", zbx.json({"data": lags}))
8190
del lags
91+
bytes_held_by_non_active_slot = Pooler.run_sql_type("wal_held_bytes_master", args=[])
92+
if bytes_held_by_non_active_slot:
93+
discovery = []
94+
for info in bytes_held_by_non_active_slot:
95+
discovery.append({"{#NON_ACTIVE_SLOT_NAME}": info[0]})
96+
zbx.send("pgsql.replication.non_active_slots_held_bytes[{0}]".format(info[0]), int(info[1]))
97+
zbx.send("pgsql.replication.non_active_slots_discovery[]", zbx.json({"data": discovery}))
98+
del discovery
8299
elif Pooler.is_superuser() or Pooler.is_bootstraped():
83100
result_lags = Pooler.run_sql_type("wal_lag_lsn", args=[" ", "xlog", "location"])
84101
if result_lags:
@@ -90,7 +107,15 @@ def run(self, zbx):
90107
del lags
91108
else:
92109
self.disable_and_exit_if_not_superuser()
93-
110+
else:
111+
bytes_held_by_non_active_slot = Pooler.run_sql_type("wal_held_bytes_replica", args=[])
112+
if bytes_held_by_non_active_slot:
113+
discovery = []
114+
for info in bytes_held_by_non_active_slot:
115+
discovery.append({"{#NON_ACTIVE_SLOT_NAME}": info[0]})
116+
zbx.send("pgsql.replication.non_active_slots_held_bytes[{0}]".format(info[0]), int(info[1]))
117+
zbx.send("pgsql.replication.non_active_slots_discovery[]", zbx.json({"data": discovery}))
118+
del discovery
94119
non_active_slots = Pooler.query(self.query_non_active_slots)
95120
zbx.send(self.key_non_active_slots.format("[]"), int(non_active_slots[0][0]))
96121

@@ -132,7 +157,8 @@ def triggers(self, template, dashboard=False):
132157
}) + template.trigger({
133158
"name": "PostgreSQL Replication: number of non-active replication slots on {HOSTNAME} (value={ITEM.LASTVALUE})",
134159
"expression": "{#TEMPLATE:" + self.right_type(self.key_non_active_slots) + ".last()}&gt;" + str(
135-
NUMBER_NON_ACTIVE_SLOTS)
160+
NUMBER_NON_ACTIVE_SLOTS),
161+
"status": 1
136162
})
137163
return triggers
138164

@@ -198,7 +224,42 @@ def discovery_rules(self, template, dashboard=False):
198224
]
199225
}
200226
]
201-
return template.discovery_rule(rule=rule, conditions=conditions, items=items, graphs=graphs)
227+
active_slots_discovery_rule = template.discovery_rule(rule=rule, conditions=conditions, items=items, graphs=graphs)
228+
229+
rule = {
230+
"name": "PostgreSQL Replication: Non Active Slots Discovery",
231+
"key": self.key_replication_non_active_slots_discovery.format("[{0}]".format(self.Macros[self.Type]))
232+
}
233+
if Plugin.old_zabbix:
234+
conditions = []
235+
rule["filter"] = "{#NON_ACTIVE_SLOT_NAME}:.*"
236+
else:
237+
conditions = [{
238+
"condition": [
239+
{"macro": "{#NON_ACTIVE_SLOT_NAME}",
240+
"value": ".*",
241+
"operator": 8,
242+
"formulaid": "A"}
243+
]
244+
}]
245+
items = [
246+
{"key": self.right_type(self.key_non_active_slots_held_bytes, var_discovery="{#NON_ACTIVE_SLOT_NAME},"),
247+
"name": "PostgreSQL Replication: Bytes held by non-active slot {#NON_ACTIVE_SLOT_NAME}",
248+
"value_type": Plugin.VALUE_TYPE.numeric_float,
249+
"delay": self.plugin_config("interval"),
250+
"drawtype": 2}
251+
]
252+
graphs = []
253+
triggers = [
254+
{
255+
"name": "PostgreSQL Replication: bytes held by slot {#NON_ACTIVE_SLOT_NAME} is too high (value={ITEM.LASTVALUE})",
256+
"expression": "{#TEMPLATE:" + self.right_type(self.key_non_active_slots_held_bytes, var_discovery="{#NON_ACTIVE_SLOT_NAME},") + ".last()}&gt;" +
257+
self.plugin_macros["critical_bytes_held_by_none_active_slot"][0][1]
258+
}
259+
]
260+
non_active_slots_discovery_rule = template.discovery_rule(rule=rule, conditions=conditions, items=items, graphs=graphs, triggers=triggers)
261+
262+
return active_slots_discovery_rule + non_active_slots_discovery_rule
202263

203264
def keys_and_queries(self, template_zabbix):
204265
result = []

‎mamonsu/tools/bootstrap/sql.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,23 @@
236236
coalesce((pg_{7}_diff(pg_current_{7}(), replay_{9}))::bigint, 0) AS total_lag
237237
FROM pg_stat_replication
238238
$$ LANGUAGE SQL SECURITY DEFINER;
239+
240+
DROP FUNCTION IF EXISTS mamonsu.bytes_held_by_inactive_slot_on_master();
241+
CREATE OR REPLACE FUNCTION mamonsu.bytes_held_by_inactive_slot_on_master()
242+
RETURNS TABLE(slot_name TEXT, wal_held_bytes BIGINT) AS $$
243+
SELECT slot_name::TEXT, coalesce((pg_{7}_diff(pg_current_wal_lsn(), restart_lsn))::bigint, 0) AS wal_held_bytes
244+
FROM pg_replication_slots
245+
WHERE active = 'false'
246+
$$ LANGUAGE SQL SECURITY DEFINER;
247+
248+
DROP FUNCTION IF EXISTS mamonsu.bytes_held_by_inactive_slot_on_replica();
249+
CREATE OR REPLACE FUNCTION mamonsu.bytes_held_by_inactive_slot_on_replica()
250+
RETURNS TABLE(slot_name TEXT, wal_held_bytes BIGINT) AS $$
251+
SELECT slot_name::TEXT, coalesce((pg_{7}_diff(pg_last_wal_replay_lsn(), restart_lsn))::bigint, 0) AS wal_held_bytes
252+
FROM pg_replication_slots
253+
WHERE active = 'false'
254+
$$ LANGUAGE SQL SECURITY DEFINER;
255+
239256
"""
240257

241258
CreatePgBuffercacheFunctionsSQL = """

0 commit comments

Comments
 (0)