Skip to content

Commit 41e7846

Browse files
SanabriaRussoSanabriaRusso
committed
Merge pull request #16649 from MinaProtocol/pod-probes
1 parent ee5445a commit 41e7846

File tree

1 file changed

+119
-116
lines changed

1 file changed

+119
-116
lines changed

‎dockerfiles/scripts/healthcheck-utilities.sh‎

Lines changed: 119 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -1,147 +1,150 @@
1-
#
2-
# Determine whether a local daemon is SYNCed with its network
3-
#
1+
#!/usr/bin/env bash
42

3+
DAEMON_REST_PORT=${DAEMON_REST_PORT:=3085}
4+
5+
# Executes a GraphQL query and returns the result
6+
# Parameters:
7+
# $1 - The GraphQL query string
8+
function queryGraphQL() {
9+
local query="$1"
10+
curl --silent --show-error \
11+
--header "Content-Type:application/json" \
12+
-d"{ \"query\": \"query { $query } \" }" \
13+
"localhost:${DAEMON_REST_PORT}/graphql"
14+
}
15+
16+
# Retrieves a specific field from the daemon status
17+
# Parameters:
18+
# $1 - The field to retrieve from the daemon status
19+
function getDaemonStatus() {
20+
local field="$1"
21+
queryGraphQL "daemonStatus { $field }" | jq ".data.daemonStatus.$field"
22+
}
23+
24+
# Updates the sync status label of a Kubernetes pod
25+
# Parameters:
26+
# $1 - The app label of the pod to update
527
function updateSyncStatusLabel() {
6-
status=$(
7-
curl --silent --show-error --header "Content-Type:application/json" -d'{ "query": "query { syncStatus } " }' localhost:3085/graphql | \
8-
jq '.data.syncStatus'
9-
)
10-
str=$(echo ${status} | sed 's/"//g' )
11-
kubectl label --overwrite=true pod -l app=$1 syncStatus=${str}
28+
local status=$(queryGraphQL "syncStatus" | jq -r '.data.syncStatus')
29+
kubectl label --overwrite=true pod -l app=$1 syncStatus=${status}
1230
}
1331

32+
# Checks if the daemon is synced
33+
# Returns 0 if synced, 1 otherwise
1434
function isDaemonSynced() {
15-
status=$(
16-
curl --silent --show-error --header "Content-Type:application/json" -d'{ "query": "query { syncStatus } " }' localhost:3085/graphql | \
17-
jq '.data.syncStatus'
18-
)
19-
case ${status} in
20-
\"BOOTSTRAP\")
21-
;&
22-
\"CATCHUP\")
23-
;&
24-
\"CONNECTING\")
25-
;&
26-
\"SYNCED\")
35+
local status=$(queryGraphQL "syncStatus" | jq -r '.data.syncStatus')
36+
if [[ "$status" == "SYNCED" ]]; then
2737
return 0
28-
;;
29-
*)
30-
DAEMON_CONFIG="/root/daemon.json"
31-
if [ -f "$DAEMON_CONFIG" ]; then
32-
now=$(date +%s)
33-
timestamp=$(grep 'timestamp' ${DAEMON_CONFIG} | awk '{print $2}' | sed -e s/\"//g)
34-
timestamp_second=$(date -d ${timestamp} +%s)
35-
36-
[[ $now -le $timestamp_seconds ]] && return 0 # special case to claim synced before the genesis timestamp
37-
fi
38+
else
3839
echo "Daemon is out of sync with status: ${status}"
39-
4040
return 1
41-
esac
41+
fi
4242
}
4343

44-
#
45-
# Determine whether a local daemon has processed the highest observed block
46-
#
44+
# Checks if the current chain length matches the highest received block length
45+
# Returns 0 if matched, 1 otherwise
4746
function isChainlengthHighestReceived() {
48-
chainLength=$(
49-
curl --silent --show-error --header "Content-Type:application/json" -d'{ "query": "query { daemonStatus { blockchainLength } }" }' localhost:3085/graphql | \
50-
jq '.data.daemonStatus.blockchainLength'
51-
)
52-
53-
highestReceived=$(
54-
curl --silent --show-error --header "Content-Type:application/json" -d'{ "query": "query { daemonStatus { highestBlockLengthReceived } }" }' localhost:3085/graphql | \
55-
jq '.data.daemonStatus.highestBlockLengthReceived'
56-
)
57-
58-
[[ "${chainLength}" == "${highestReceived}" ]] && return 0 ||
59-
(echo "Daemon chain length[${chainLength}] is not at highest received[${highestReceived}]." && return 1)
47+
local chainLength=$(getDaemonStatus "blockchainLength")
48+
local highestReceived=$(getDaemonStatus "highestBlockLengthReceived")
49+
50+
if [[ "${chainLength}" == "${highestReceived}" ]]; then
51+
return 0
52+
else
53+
echo "Daemon chain length[${chainLength}] is not at highest received[${highestReceived}]."
54+
return 1
55+
fi
6056
}
6157

62-
#
63-
# Determine whether a local daemon has a peer count greater than some threshold
64-
#
58+
# Checks if the peer count is greater than a specified threshold
59+
# Parameters:
60+
# $1 - The minimum threshold for peer count (default: 2)
61+
# Returns 0 if peer count is above threshold, 1 otherwise
6562
function peerCountGreaterThan() {
66-
peerCountMinThreshold=${1:-2}
67-
peerCount=$(
68-
curl --silent --show-error --header "Content-Type:application/json" -d'{ "query": "query { daemonStatus { peers } }" }' localhost:3085/graphql | \
69-
jq '.data.daemonStatus.peers | length'
70-
)
71-
72-
[[ $peerCount -gt $peerCountMinThreshold ]] && return 0 ||
73-
(echo "Peer count[${peerCount}] is not greater than mininum threshold[${peerCountMinThreshold}]." && return 1)
63+
local peerCountMinThreshold=${1:-2}
64+
local peerCount=$(getDaemonStatus "peers" | jq 'length')
65+
66+
if [[ $peerCount -gt $peerCountMinThreshold ]]; then
67+
return 0
68+
else
69+
echo "Peer count[${peerCount}] is not greater than mininum threshold[${peerCountMinThreshold}]."
70+
return 1
71+
fi
7472
}
7573

76-
#
77-
# Determine whether a local daemon owns a wallet account and has allocated funds
78-
#
74+
# Checks if the daemon owns funds
75+
# Returns 0 if the daemon has at least one wallet with a positive balance, 1 otherwise
7976
function ownsFunds() {
80-
ownedWalletCount=$(
81-
curl --silent --show-error --header "Content-Type:application/json" -d'{ "query": "query { ownedWallets }" }' localhost:3085/graphql | \
82-
jq '.data.ownedWallets | length'
83-
)
84-
balanceTotal=$(
85-
curl --silent --show-error --header "Content-Type:application/json" -d'{ "query": "query { ownedWallets { balance { total } } }" }' localhost:3085/graphql | \
86-
jq '.data.ownedWallets[0].balance.total'
87-
)
88-
# remove leading and trailing quotes for integer interpretation
89-
balanceTotal=$(echo $balanceTotal | sed -e 's/^"//' -e 's/"$//')
90-
91-
[[ $ownedWalletCount -gt 0 ]] && [[ $balanceTotal -gt 0 ]] && return 0 ||
92-
(echo "Owned wallet count[${ownedWalletCount}] and/or balance total[${balanceTotal}] is insufficient." && return 1)
77+
local ownedWalletCount=$(queryGraphQL "ownedWallets" | jq '.data.ownedWallets | length')
78+
local balanceTotal=$(queryGraphQL "ownedWallets { balance { total } }" | jq -r '.data.ownedWallets[0].balance.total')
79+
80+
if [[ $ownedWalletCount -gt 0 ]] && [[ $balanceTotal -gt 0 ]]; then
81+
return 0
82+
else
83+
echo "Owned wallet count[${ownedWalletCount}] and/or balance total[${balanceTotal}] is insufficient."
84+
return 1
85+
fi
9386
}
9487

95-
#
96-
# Determine whether a local daemon process has sent sufficient user commands
97-
#
88+
# Checks if the number of sent user commands is greater than a specified threshold
89+
# Parameters:
90+
# $1 - The minimum threshold for sent user commands (default: 1)
91+
# Returns 0 if sent commands are above threshold, 1 otherwise
9892
function hasSentUserCommandsGreaterThan() {
99-
userCmdMinThreshold=${1:-1}
100-
userCmdSent=$(
101-
curl --silent --show-error --header "Content-Type:application/json" -d'{ "query": "query { daemonStatus { userCommandsSent } }" }' localhost:3085/graphql | \
102-
jq '.data.daemonStatus.userCommandsSent'
103-
)
104-
105-
[[ $userCmdSent -gt $userCmdMinThreshold ]] && return 0 ||
106-
(echo "User commands sent[${userCmdSent}] is not greater than mininum threshold[${userCmdMinThreshold}]." && return 1)
93+
local userCmdMinThreshold=${1:-1}
94+
local userCmdSent=$(getDaemonStatus "userCommandsSent")
95+
96+
if [[ $userCmdSent -gt $userCmdMinThreshold ]]; then
97+
return 0
98+
else
99+
echo "User commands sent[${userCmdSent}] is not greater than mininum threshold[${userCmdMinThreshold}]."
100+
return 1
101+
fi
107102
}
108103

109-
#
110-
# Determine whether a SNARK coordinator has an assigned SNARK-worker
111-
#
104+
# Checks if the daemon has a snark worker
105+
# Returns 0 if a snark worker is present, 1 otherwise
112106
function hasSnarkWorker() {
113-
snarkWorker=$(
114-
curl --silent --show-error --header "Content-Type:application/json" -d'{ "query": "query { daemonStatus { snarkWorker } }" }' localhost:3085/graphql | \
115-
jq '.data.daemonStatus.snarkWorker'
116-
)
117-
rc=$?
118-
119-
[[ $rc == 0 ]] && [[ -n "$snarkWorker" ]] && return 0 ||
120-
(echo "Snark worker error: ${rc} - $snarkWorker" && return 1)
107+
local snarkWorker=$(getDaemonStatus "snarkWorker")
108+
local rc=$?
109+
110+
if [[ $rc == 0 ]] && [[ -n "$snarkWorker" ]]; then
111+
return 0
112+
else
113+
echo "Snark worker error: ${rc} - $snarkWorker"
114+
return 1
115+
fi
121116
}
122117

123-
#
124-
# Determine whether an Archive node's highest observed block is in sync with its local Mina daemon
125-
#
118+
# Checks if the archive is synced with the local daemon
119+
# Parameters:
120+
# --db-host - The database host (default: localhost)
121+
# --db-port - The database port (default: 5432)
122+
# --db-user - The database user (default: postgres)
123+
# --db-password - The database password (default: foobar)
124+
# Returns 0 if the archive is synced, 1 otherwise
126125
function isArchiveSynced() {
127-
## "Usage: $0 [--db-host <host>] [--db-port <port>] [--db-user <user>] [--db-password <pass>]"
128-
129-
while [[ "$#" -gt 0 ]]; do case $1 in
126+
local host="localhost"
127+
local port="5432"
128+
local user="postgres"
129+
local password="foobar"
130+
while [[ "$#" -gt 0 ]]; do
131+
case $1 in
130132
--db-host) host="$2"; shift;;
131133
--db-port) port="$2"; shift;;
132134
--db-user) user="$2"; shift;;
133135
--db-password) password="$2"; shift;;
134136
*) echo "Unknown parameter passed: $1"; exit 1;;
135-
esac; shift; done
136-
137-
highestObserved=$(
138-
PGPASSWORD=${password:-foobar} psql -qtAX -h ${host:-localhost} -p ${port:-5432} -d archive -U ${user:-postgres} \
139-
-w -c "SELECT height FROM blocks ORDER BY height DESC LIMIT 1"
140-
)
141-
highestReceived=$(
142-
curl --silent --show-error --header "Content-Type:application/json" -d'{ "query": "query { daemonStatus { highestBlockLengthReceived } }" }' localhost:3085/graphql | \
143-
jq '.data.daemonStatus.highestBlockLengthReceived'
144-
)
145-
146-
[[ $highestObserved == $highestReceived ]] && return 0 || (echo "Archive[${highestObserved}] is out of sync with local daemon[${highestReceived}]" && return 1)
147-
}
137+
esac
138+
shift
139+
done
140+
141+
local highestObserved=$(PGPASSWORD=${password} psql -qtAX -h ${host} -p ${port} -d archive -U ${user} -w -c "SELECT height FROM blocks ORDER BY height DESC LIMIT 1")
142+
local highestReceived=$(getDaemonStatus "highestBlockLengthReceived")
143+
144+
if [[ $highestObserved == $highestReceived ]]; then
145+
return 0
146+
else
147+
echo "Archive[${highestObserved}] is out of sync with local daemon[${highestReceived}]"
148+
return 1
149+
fi
150+
}

0 commit comments

Comments
 (0)