Skip to content

Commit 20637b6

Browse files
yungwineyungwine
committed
fix alerts timeouts for testnet
1 parent 2c1a238 commit 20637b6

File tree

1 file changed

+83
-77
lines changed

1 file changed

+83
-77
lines changed

‎modules/alert_bot.py‎

Lines changed: 83 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -21,82 +21,87 @@ class Alert:
2121
ELECTIONS_START_BEFORE = 8192
2222

2323

24-
ALERTS = {
25-
"low_wallet_balance": Alert(
26-
"low",
27-
"Validator wallet {wallet} balance is low: {balance} TON.",
28-
18*HOUR
29-
),
30-
"db_usage_80": Alert(
31-
"high",
32-
"""TON DB usage > 80%. Clean the TON database:
33-
https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming
34-
or (and) set node\'s archive ttl to lower value.""",
35-
24*HOUR
36-
),
37-
"db_usage_95": Alert(
38-
"critical",
39-
"""TON DB usage > 95%. Disk is almost full, clean the TON database immediately:
40-
https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming
41-
or (and) set node\'s archive ttl to lower value.""",
42-
6*HOUR
43-
),
44-
"low_efficiency": Alert(
45-
"high",
46-
"""Validator efficiency is low: {efficiency}%.""",
47-
VALIDATION_PERIOD // 3
48-
),
49-
"out_of_sync": Alert(
50-
"critical",
51-
"Node is out of sync on {sync} sec.",
52-
300
53-
),
54-
"service_down": Alert(
55-
"critical",
56-
"validator.service is down.",
57-
300
58-
),
59-
"adnl_connection_failed": Alert(
60-
"high",
61-
"ADNL connection to node failed",
62-
3*HOUR
63-
),
64-
"zero_block_created": Alert(
65-
"critical",
66-
"Validator has not created any blocks in the last {hours} hours.",
67-
VALIDATION_PERIOD // 3
68-
),
69-
"validator_slashed": Alert(
70-
"high",
71-
"Validator has been slashed in previous round for {amount} TON",
72-
FREEZE_PERIOD
73-
),
74-
"stake_not_accepted": Alert(
75-
"high",
76-
"Validator's stake has not been accepted",
77-
ELECTIONS_START_BEFORE
78-
),
79-
"stake_accepted": Alert(
80-
"info",
81-
"Validator's stake {stake} TON has been accepted",
82-
ELECTIONS_START_BEFORE
83-
),
84-
"stake_returned": Alert(
85-
"info",
86-
"Validator's stake {stake} TON has been returned on address {address}. The reward amount is {reward} TON.",
87-
60
88-
),
89-
"stake_not_returned": Alert(
90-
"high",
91-
"Validator's stake has not been returned on address {address}.",
92-
60
93-
),
94-
"voting": Alert(
95-
"high",
96-
"Found proposals with hashes `{hashes}` that have significant amount of votes, but current validator didn't vote for them. Please check @tonstatus for more details.",
97-
VALIDATION_PERIOD
98-
),
99-
}
24+
ALERTS = {}
25+
26+
27+
def init_alerts():
28+
global ALERTS
29+
ALERTS = {
30+
"low_wallet_balance": Alert(
31+
"low",
32+
"Validator wallet {wallet} balance is low: {balance} TON.",
33+
18 * HOUR
34+
),
35+
"db_usage_80": Alert(
36+
"high",
37+
"""TON DB usage > 80%. Clean the TON database:
38+
https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming
39+
or (and) set node\'s archive ttl to lower value.""",
40+
24 * HOUR
41+
),
42+
"db_usage_95": Alert(
43+
"critical",
44+
"""TON DB usage > 95%. Disk is almost full, clean the TON database immediately:
45+
https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming
46+
or (and) set node\'s archive ttl to lower value.""",
47+
6 * HOUR
48+
),
49+
"low_efficiency": Alert(
50+
"high",
51+
"""Validator efficiency is low: {efficiency}%.""",
52+
VALIDATION_PERIOD // 3
53+
),
54+
"out_of_sync": Alert(
55+
"critical",
56+
"Node is out of sync on {sync} sec.",
57+
300
58+
),
59+
"service_down": Alert(
60+
"critical",
61+
"validator.service is down.",
62+
300
63+
),
64+
"adnl_connection_failed": Alert(
65+
"high",
66+
"ADNL connection to node failed",
67+
3 * HOUR
68+
),
69+
"zero_block_created": Alert(
70+
"critical",
71+
"Validator has not created any blocks in the last {hours} hours.",
72+
VALIDATION_PERIOD // 3
73+
),
74+
"validator_slashed": Alert(
75+
"high",
76+
"Validator has been slashed in previous round for {amount} TON",
77+
FREEZE_PERIOD
78+
),
79+
"stake_not_accepted": Alert(
80+
"high",
81+
"Validator's stake has not been accepted",
82+
ELECTIONS_START_BEFORE
83+
),
84+
"stake_accepted": Alert(
85+
"info",
86+
"Validator's stake {stake} TON has been accepted",
87+
ELECTIONS_START_BEFORE
88+
),
89+
"stake_returned": Alert(
90+
"info",
91+
"Validator's stake {stake} TON has been returned on address {address}. The reward amount is {reward} TON.",
92+
60
93+
),
94+
"stake_not_returned": Alert(
95+
"high",
96+
"Validator's stake has not been returned on address {address}.",
97+
60
98+
),
99+
"voting": Alert(
100+
"high",
101+
"Found proposals with hashes `{hashes}` that have significant amount of votes, but current validator didn't vote for them. Please check @tonstatus for more details.",
102+
VALIDATION_PERIOD
103+
),
104+
}
100105

101106

102107
class AlertBotModule(MtcModule):
@@ -171,6 +176,7 @@ def init(self):
171176
self.hostname = get_hostname()
172177
self.ip = self.ton.get_validator_engine_ip()
173178
self.set_global_vars()
179+
init_alerts()
174180
self.inited = True
175181

176182
def get_alert_from_db(self, alert_name: str):
@@ -354,7 +360,7 @@ def check_voting(self):
354360
offers = self.ton.GetOffers()
355361
for offer in offers:
356362
if not offer['isPassed'] and offer['approvedPercent'] >= 50 and validator_index not in offer['votedValidators']:
357-
need_to_vote.append(offer['hash'])
363+
need_to_vote.append(offer['hash'])
358364
if need_to_vote:
359365
self.send_alert("voting", hashes=' '.join(need_to_vote))
360366

0 commit comments

Comments
 (0)