Skip to content

Commit bbdcf30

Browse files
MarketenMarketen
committed
add healthchecks to chain & keyper
1 parent f90bc07 commit bbdcf30

File tree

3 files changed

+142
-11
lines changed

3 files changed

+142
-11
lines changed

‎shutter/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ ARG CHAIN_PORT
1616
ARG STAKER_SCRIPTS_VERSION
1717

1818
RUN apt-get update && \
19-
apt-get --yes --no-install-recommends install supervisor postgresql-client && \
19+
apt-get --yes --no-install-recommends install supervisor postgresql-client jq && \
2020
apt-get clean && \
2121
rm -rf /var/lib/apt/lists/*
2222

‎shutter/scripts/run_chain.sh‎

Lines changed: 71 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,79 @@
11
#!/bin/bash
22

3-
run_chain() {
3+
# To use staker scripts
4+
# shellcheck disable=SC1091
5+
. /etc/profile
6+
7+
SUPPORTED_NETWORKS="gnosis"
8+
9+
# Set environment variables before running the health checks.
10+
echo "[INFO | chain] Setting environment variables for beacon API and execution API..."
11+
export SHUTTER_BEACONAPIURL=$(get_beacon_api_url_from_global_env "$NETWORK" "$SUPPORTED_NETWORKS")
12+
export SHUTTER_GNOSIS_NODE_CONTRACTSURL=http://execution.gnosis.dncore.dappnode:8545
13+
14+
perform_node_healthcheck() {
15+
echo "[INFO | chain] Starting health check for beacon API and execution API..."
16+
17+
while true; do
18+
# Check the syncing status using JSON-RPC
19+
echo "[INFO | chain] Checking execution API syncing status at: $SHUTTER_GNOSIS_NODE_CONTRACTSURL/eth/v1/node/syncing"
20+
sync_status=$(curl -s -X POST "$SHUTTER_GNOSIS_NODE_CONTRACTSURL" \
21+
-H "Content-Type: application/json" \
22+
-d '{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":1}' | jq -r '.result')
23+
24+
if [ "$sync_status" != "false" ]; then
25+
echo "[WARN | chain] Execution API is syncing. Sync status: $sync_status. Retrying in 30 seconds..."
26+
sleep 30
27+
continue
28+
else
29+
echo "[INFO | chain] Execution API is not syncing."
30+
fi
31+
32+
# Check the syncing status using JSON-RPC
33+
echo "[INFO | chain] Checking execution API syncing status at: $SHUTTER_GNOSIS_NODE_CONTRACTSURL/eth/v1/node/syncing"
34+
35+
# Perform the request and capture the response
36+
response=$(curl -s -w "%{http_code}" -o /tmp/sync_response.json -X POST "$SHUTTER_GNOSIS_NODE_CONTRACTSURL" \
37+
-H "Content-Type: application/json" \
38+
-d '{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":1}')
39+
40+
# Extract HTTP status code and response body
41+
http_code="${response: -3}"
42+
response_body=$(cat /tmp/sync_response.json)
443

5-
echo "[INFO | chain] Starting chain..."
44+
# Check if the response body is empty or the HTTP status code is not 200
45+
if [ -z "$response_body" ] || [ "$http_code" -ne 200 ]; then
46+
echo "[ERROR | chain] No response or error from execution API. Not healthy"
47+
sleep 30
48+
continue
49+
fi
650

51+
# Parse the JSON response to check the syncing status
52+
syncing_status=$(echo "$response_body" | jq -r '.result.is_syncing')
53+
54+
# If syncing is true or an error occurs, treat it as not synced
55+
if [ "$syncing_status" = "true" ]; then
56+
echo "[WARN | chain] Execution API is syncing. Sync status: $syncing_status. Retrying in 30 seconds..."
57+
sleep 30
58+
continue
59+
else
60+
echo "[INFO | chain] Execution API is healthy."
61+
fi
62+
63+
# If we reach this point, all checks have passed.
64+
echo "[INFO | chain] All services are healthy. Exiting health check loop."
65+
break
66+
done
67+
}
68+
69+
run_chain() {
70+
echo "[INFO | chain] Starting chain process..."
71+
echo "[DEBUG | chain] Command: $SHUTTER_BIN chain --config \"$SHUTTER_CHAIN_CONFIG_FILE\""
772
$SHUTTER_BIN chain --config "$SHUTTER_CHAIN_CONFIG_FILE"
873
}
974

75+
# Run health checks first
76+
perform_node_healthcheck
77+
78+
# If everything is healthy, run the chain
1079
run_chain

‎shutter/scripts/run_keyper.sh‎

Lines changed: 70 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,88 @@
11
#!/bin/bash
22

3+
# To use staker scripts
4+
# shellcheck disable=SC1091
5+
. /etc/profile
6+
7+
SUPPORTED_NETWORKS="gnosis"
8+
9+
# Set environment variables before running the health checks.
10+
echo "[INFO | keyper] Setting environment variables for beacon API and execution API..."
11+
export SHUTTER_BEACONAPIURL=$(get_beacon_api_url_from_global_env "$NETWORK" "$SUPPORTED_NETWORKS")
12+
export SHUTTER_GNOSIS_NODE_CONTRACTSURL=http://execution.gnosis.dncore.dappnode:8545
13+
314
perform_chain_healthcheck() {
4-
echo "[INFO | keyper] Waiting for chain to be healthy..."
15+
echo "[INFO | keyper] Starting health check for chain, beacon API, and execution API..."
516

617
while true; do
7-
# Perform the health check
8-
if curl -sf http://localhost:26657/status >/dev/null; then
9-
echo "[INFO | keyper] Service is healthy. Exiting health check loop."
10-
break # Exit the loop if the service is healthy
18+
echo "[INFO | keyper] Checking chain health at: http://localhost:26657/status"
19+
if ! curl -sf http://localhost:26657/status >/dev/null; then
20+
echo "[WARN | keyper] Chain service is not healthy yet. Retrying in 30 seconds..."
21+
sleep 30
22+
continue
23+
else
24+
echo "[INFO | keyper] Chain service is healthy."
25+
fi
26+
27+
# Check the syncing status using JSON-RPC
28+
echo "[INFO | keyper] Checking execution API syncing status at: $SHUTTER_GNOSIS_NODE_CONTRACTSURL/eth/v1/node/syncing"
29+
sync_status=$(curl -s -X POST "$SHUTTER_GNOSIS_NODE_CONTRACTSURL" \
30+
-H "Content-Type: application/json" \
31+
-d '{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":1}' | jq -r '.result')
32+
33+
if [ "$sync_status" != "false" ]; then
34+
echo "[WARN | keyper] Execution API is syncing. Sync status: $sync_status. Retrying in 30 seconds..."
35+
sleep 30
36+
continue
37+
else
38+
echo "[INFO | keyper] Execution API is not syncing."
39+
fi
40+
41+
# Check the syncing status using JSON-RPC
42+
echo "[INFO | keyper] Checking execution API syncing status at: $SHUTTER_GNOSIS_NODE_CONTRACTSURL/eth/v1/node/syncing"
43+
44+
# Perform the request and capture the response
45+
response=$(curl -s -w "%{http_code}" -o /tmp/sync_response.json -X POST "$SHUTTER_GNOSIS_NODE_CONTRACTSURL" \
46+
-H "Content-Type: application/json" \
47+
-d '{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":1}')
48+
49+
# Extract HTTP status code and response body
50+
http_code="${response: -3}"
51+
response_body=$(cat /tmp/sync_response.json)
52+
53+
# Check if the response body is empty or the HTTP status code is not 200
54+
if [ -z "$response_body" ] || [ "$http_code" -ne 200 ]; then
55+
echo "[ERROR | keyper] No response or error from execution API. Not healthy"
56+
sleep 30
57+
continue
58+
fi
59+
60+
# Parse the JSON response to check the syncing status
61+
syncing_status=$(echo "$response_body" | jq -r '.result.is_syncing')
62+
63+
# If syncing is true or an error occurs, treat it as not synced
64+
if [ "$syncing_status" = "true" ]; then
65+
echo "[WARN | keyper] Execution API is syncing. Sync status: $syncing_status. Retrying in 30 seconds..."
66+
sleep 30
67+
continue
1168
else
12-
echo "[INFO | keyper] Service is not healthy yet. Retrying in 30 seconds..."
69+
echo "[INFO | keyper] Execution API is healthy."
1370
fi
1471

15-
# Wait for the next interval (30 seconds)
16-
sleep 30
72+
# If we reach this point, all checks have passed.
73+
echo "[INFO | keyper] All services are healthy. Exiting health check loop."
74+
break
1775
done
1876
}
1977

2078
run_keyper() {
79+
echo "[INFO | keyper] Starting gnosiskeyper..."
80+
echo "[DEBUG | keyper] Command: $SHUTTER_BIN gnosiskeyper --config \"$KEYPER_CONFIG_FILE\""
2181
$SHUTTER_BIN gnosiskeyper --config "$KEYPER_CONFIG_FILE"
2282
}
2383

84+
# Run health checks first
2485
perform_chain_healthcheck
2586

87+
# If everything is healthy, run keyper
2688
run_keyper

0 commit comments

Comments
 (0)