Skip to content

Commit 758981e

Browse files
june-huajune-hua
committed
Add monitoring to startup
1 parent 6bd0c0a commit 758981e

File tree

5 files changed

+111
-22
lines changed

5 files changed

+111
-22
lines changed

‎startupscript/butane/004-git-clone-devcontainer.sh‎

Lines changed: 2 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,7 @@ if [[ $# -lt 1 ]]; then
1717
usage
1818
fi
1919

20-
21-
# Map the server to appropriate service path
22-
function get_service_url() {
23-
case "$1" in
24-
"dev-stable") echo "https://workbench-dev.verily.com/api/$2" ;;
25-
"dev-unstable") echo "https://workbench-dev-unstable.verily.com/api/$2" ;;
26-
"test") echo "https://workbench-test.verily.com/api/$2" ;;
27-
"prod") echo "https://workbench.verily.com/api/$2" ;;
28-
*) return 1 ;;
29-
esac
30-
}
31-
readonly -f get_service_url
32-
20+
source /home/core/service-utils.sh
3321
source /home/core/metadata-utils.sh
3422

3523
# To accommodate the use of SSH URLs for public Git repositories, set the following Git configuration:
@@ -58,12 +46,7 @@ api_url="${api_url%.git}"
5846
private_status=$(curl --retry 5 -s "${api_url}" | jq -r ".status")
5947
if [[ "${PRIVATE_DEVCONTAINER_ENABLED}" == "TRUE" && "${private_status}" == 404 ]]; then
6048
# Get ECM service URL
61-
SERVER="$(get_metadata_value "terra-cli-server" "")"
62-
readonly SERVER
63-
if [[ -z "${SERVER}" ]]; then
64-
SERVER="prod"
65-
fi
66-
if ! ECM_SERVICE_URL="$(get_service_url "${SERVER}" "ecm")"; then
49+
if ! ECM_SERVICE_URL="$(get_service_url "ecm")"; then
6750
exit 1
6851
fi
6952

‎startupscript/butane/monitoring-utils.sh‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/bin/bash
2+
# monitoring-utils.sh defines helper functions for notifying WSM of VM startup states.
3+
4+
source /home/core/service-utils.sh
5+
6+
WORKSPACE_ID="$(get_metadata_value "terra-workspace-id" "")"
7+
RESOURCE_ID="$(get_metadata_value "terra-resource-id" "")"
8+
readonly WORKSPACE_ID RESOURCE_ID
9+
10+
# Get WSM endpoint URL
11+
if ! WSM_SERVICE_URL="$(get_service_url "wsm")"; then
12+
exit 1
13+
fi
14+
LOG_URL="${WSM_SERVICE_URL}/api/workspaces/${WORKSPACE_ID}/resource/${RESOURCE_ID}/instance-state"
15+
16+
function record_devcontainer_end() {
17+
if [[ $# -lt 2 || ("$2" != "0" && "$2" != "1") ]]; then
18+
echo "usage: record_devcontainer_end <success/fail - 1/0>"
19+
exit 1
20+
fi
21+
SUCCESS="$1"
22+
payload=$(cat <<EOF
23+
{
24+
"state": "DEVCONTAINER_END",
25+
"success": ${SUCCESS}
26+
}
27+
EOF
28+
)
29+
response=$(curl -s -X POST "${LOG_URL}" \
30+
-H "Authorization: Bearer $(/home/core/wb.sh auth print-access-token)" \
31+
-H "Content-Type: application/json" \
32+
-d "${payload}" \
33+
-w "\n%{http_code}")
34+
http_code=$(echo "$response" | tail -n1)
35+
response_body=$(echo "$response" | head -n -1)
36+
if [[ $http_code -ne 200 ]]; then
37+
echo "Failed to record VM state. HTTP ${http_code}: ${response_body}" >&2
38+
return 1
39+
fi
40+
echo "VM state recorded successfully: ${response_body}"
41+
}
42+
43+
function record_devcontainer_start() {
44+
payload=$(cat <<EOF
45+
{
46+
"state": "DEVCONTAINER_START",
47+
"success": 1
48+
}
49+
EOF
50+
)
51+
response=$(curl -s -X POST "${LOG_URL}" \
52+
-H "Authorization: Bearer $(/home/core/wb.sh auth print-access-token)" \
53+
-H "Content-Type: application/json" \
54+
-d "${payload}" \
55+
-w "\n%{http_code}")
56+
http_code=$(echo "$response" | tail -n1)
57+
response_body=$(echo "$response" | head -n -1)
58+
if [[ $http_code -ne 200 ]]; then
59+
echo "Failed to record VM state. HTTP ${http_code}: ${response_body}" >&2
60+
return 1
61+
fi
62+
echo "VM state recorded successfully: ${response_body}"
63+
}

‎startupscript/butane/pre-devcontainer.sh‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,16 @@
11
#!/bin/bash
22

3-
# pre-devcontainer.sh creates a file used by the devcontainer service to keep
4-
# track of the number of service failures.
3+
# pre-devcontainer.sh creates a file used by the devcontainer service for monitoring
4+
# and to keep track of the number of service failures.
55

6-
touch /tmp/devcontainer-failure-count
6+
touch /tmp/devcontainer-failure-count
7+
8+
MONITORING_UTILS_FILE="/home/core/monitoring-utils.sh"
9+
FIRST_BOOT_START_FILE="/home/core/first-boot-start"
10+
if [[ -f "${MONITORING_UTILS_FILE}" && ! -f "${FIRST_BOOT_START_FILE}" ]]; then
11+
# First boot file does not exist
12+
# Record startup begin for monitoring
13+
source "${MONITORING_UTILS_FILE}"
14+
record_devcontainer_start "${CLOUD_PLATFORM}"
15+
fi
16+
touch "${FIRST_BOOT_START_FILE}"

‎startupscript/butane/probe-proxy-readiness.sh‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,21 @@ if docker ps -q --filter "name=proxy-agent" | grep -q . \
1515
&& docker ps -q --filter "name=application-server" | grep -q .; then
1616
echo "Proxy is ready."
1717
status="$(get_guest_attribute "startup_script/status" "")"
18+
success=0
1819
if [[ "${status}" != "ERROR" ]]; then
1920
set_metadata "startup_script/status" "COMPLETE"
21+
success=1
2022
fi
23+
24+
FIRST_BOOT_END_FILE="/home/core/first-boot-end"
25+
MONITORING_UTILS_FILE="/home/core/monitoring-utils.sh"
26+
if [[ -f "${MONITORING_UTILS_FILE}" && ! -f "${FIRST_BOOT_END_FILE}" ]]; then
27+
# first boot file does not exist
28+
# record startup end for monitoring
29+
source "${MONITORING_UTILS_FILE}"
30+
record_devcontainer_end "${success}"
31+
fi
32+
touch "${FIRST_BOOT_END_FILE}"
2133
else
2234
echo "proxy-agent or application-server is not started"
2335
exit 1

‎startupscript/butane/service-utils.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/bash
2+
# service-utils defines helper functions for determining service path.
3+
4+
# Map the server to appropriate service path
5+
function get_service_url() {
6+
SERVER="$(get_metadata_value "terra-cli-server" "")"
7+
if [[ -z "${SERVER}" ]]; then
8+
SERVER="dev-stable"
9+
fi
10+
readonly SERVER
11+
12+
13+
case "${SERVER}" in
14+
"dev-stable") echo "https://workbench-dev.verily.com/api/${SERVER}" ;;
15+
"dev-unstable") echo "https://workbench-dev-unstable.verily.com/api/${SERVER}" ;;
16+
"test") echo "https://workbench-test.verily.com/api/${SERVER}" ;;
17+
"prod") echo "https://workbench.verily.com/api/${SERVER}" ;;
18+
*) return 1 ;;
19+
esac
20+
}
21+
readonly -f get_service_url

0 commit comments

Comments
 (0)