Skip to content

Commit 31d88a6

Browse files
stanbrubstanbrub
committed
Preserver the terraform mess
1 parent 19c77bd commit 31d88a6

File tree

6 files changed

+177
-52
lines changed

6 files changed

+177
-52
lines changed

‎.github/resources/adhoc-server-deploy.json‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"hostname": "my-server",
3+
"description": "Ephemeral adhoc benchmark server",
4+
"os": "ubuntu/noble",
5+
"type": "s2.c2.small",
6+
"location": "CHI",
7+
"installDefaultSshKeys": true,
8+
"publicIpAddresses": [
9+
{
10+
"type": "PRIMARY"
11+
}
12+
],
13+
"cloudInit": "",
14+
"tags": [
15+
{
16+
"name": "ephemeral",
17+
"value": "true"
18+
},
19+
{
20+
"name": "created",
21+
"value": "2026-01-23T22:00:00Z"
22+
}
23+
]
24+
}
25+

‎.github/resources/adhoc-server-deploy.tf‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
terraform {
2+
required_providers {
3+
pnap = {
4+
source = "phoenixnap/pnap"
5+
version = "=0.29.0"
6+
}
7+
}
8+
}
9+
10+
provider "pnap" {
11+
client_id = var.client_id
12+
client_secret = var.client_secret
13+
}
14+
15+
variable "client_id" {}
16+
variable "client_secret" {}
17+
variable "hostname" {}
18+
variable "plan" {}
19+
20+
resource "pnap_server" "adhoc_server" {
21+
hostname = var.hostname
22+
description = "Ephemeral adhoc benchmark server"
23+
os = "ubuntu/noble"
24+
type = var.plan
25+
location = "CHI"
26+
install_default_ssh_keys = true
27+
28+
network_configuration {
29+
public_network_configuration {}
30+
private_network_configuration {}
31+
ip_blocks_configuration {}
32+
}
33+
34+
esxi {
35+
datastore_configuration {}
36+
}
37+
38+
netris_softgate {}
39+
40+
cloud_init {
41+
user_data = file("${path.module}/adhoc-server-init.yml")
42+
}
43+
44+
tags {
45+
tag_assignment {
46+
name = "ephemeral"
47+
value = "true"
48+
}
49+
}
50+
51+
tags {
52+
tag_assignment {
53+
name = "created"
54+
value = timestamp()
55+
}
56+
}
57+
}
58+
59+
output "public_ip" {
60+
value = tolist(pnap_server.adhoc_server.public_ip_addresses)[0]
61+
}
62+
63+
output "server_id" {
64+
value = pnap_server.adhoc_server.id
65+
}
66+

‎.github/resources/adhoc-server-init.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#cloud-config
2+
users:
3+
- name: benchmark
4+
sudo: ALL=(ALL) NOPASSWD:ALL
5+
groups: sudo
6+
shell: /bin/bash
7+

‎.github/resources/terraform.tfstate‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"version":4,"terraform_version":"1.11.4","serial":1,"lineage":"bc6741f5-e40c-48f7-58f3-bece59ba01cc","outputs":{},"resources":[],"check_results":null}

‎.github/scripts/adhoc.sh‎

Lines changed: 73 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,14 @@
33
set -o errexit
44
set -o pipefail
55

6-
# Copyright (c) 2023-2024 Deephaven Data Labs and Patent Pending
6+
# Copyright (c) 2023-2026 Deephaven Data Labs and Patent Pending
77

88
# Provides what is needed to set up an adhoc benchmark run, including bare metal and labels
99
# ex. adhoc.sh make-labels "where" "0.36.0" "user123:branch-name-123"
10-
# ex. adhoc.sh metal-deploy api-key project-id c3.small.x86 server-name "2 days"
11-
# ex. adhoc.sh metal-delete api-key device-id service-name
10+
# ex. adhoc.sh scale-nums 10 5
11+
# ex. adhoc.sh deploy-metal api-key project-id s2.c2.small server-name
12+
# ex. adhoc.sh delete-metal api-key device-id service-name
13+
# ex. adhoc.sh purge-metal api-key project-id
1214

1315
if [[ $# < 2 ]]; then
1416
echo "$0: Missing action or its arguments"
@@ -21,13 +23,6 @@ OUTPUT_NAME=adhoc-${ACTION}.out
2123

2224
rm -f ${OUTPUT_NAME}; touch ${OUTPUT_NAME}
2325

24-
# Get metal device info including ip address
25-
getDeviceInfo() {
26-
curl --no-progress-meter --max-time 10 -X GET -H "X-Auth-Token: $1" \
27-
"https://api.equinix.com/metal/v1/devices/$2?include=ip_addresses,state&exclude=root_password,ssh_keys" \
28-
| jq | tee get-device-response.json | jq -r "$3"
29-
}
30-
3126
# Get the label part of an image/branch name
3227
# ex. edge@sha256:15ab331629805076cdf5ed6666186c6b578298ab493a980779338d153214640e
3328
# ex. user123:1111-my-pull-request
@@ -48,69 +43,66 @@ if [[ ${ACTION} == "make-labels" ]]; then
4843
IMAGE1=$3
4944
IMAGE2=$4
5045
echo "Making Labels: ${PREFIX}"
51-
46+
5247
LABEL1=$(getSetLabel ${PREFIX} ${IMAGE1})
5348
LABEL2=$(getSetLabel ${PREFIX} ${IMAGE2})
54-
49+
5550
echo "PREFIX=${PREFIX}" | tee -a ${OUTPUT_NAME}
5651
echo "SET_LABEL_1=${LABEL1}" | tee -a ${OUTPUT_NAME}
5752
echo "SET_LABEL_2=${LABEL2}" | tee -a ${OUTPUT_NAME}
5853
fi
5954

60-
# Format some number used for scaling the tests
55+
# Format some numbers used for scaling the tests
6156
if [[ ${ACTION} == "scale-nums" ]]; then
6257
INPUT_ROW_COUNT=$2
6358
INPUT_ITERATIONS=$3
6459
echo "Scaling Numbers"
65-
60+
6661
TEST_ROW_COUNT=$((${INPUT_ROW_COUNT} * 1000000))
6762
TEST_ITERATIONS=${INPUT_ITERATIONS}
6863
if [ $((${INPUT_ITERATIONS} % 2)) == 0 ]; then
6964
TEST_ITERATIONS=$((${INPUT_ITERATIONS} + 1))
7065
fi
71-
66+
7267
echo "INPUT_ROW_COUNT=${INPUT_ROW_COUNT}" | tee -a ${OUTPUT_NAME}
7368
echo "INPUT_ITERATIONS=${INPUT_ITERATIONS}" | tee -a ${OUTPUT_NAME}
7469
echo "TEST_ROW_COUNT=${TEST_ROW_COUNT}" | tee -a ${OUTPUT_NAME}
7570
echo "TEST_ITERATIONS=${TEST_ITERATIONS}" | tee -a ${OUTPUT_NAME}
7671
fi
7772

78-
# Deploy a bare metal server using the Equinix ReST API
73+
# Deploy a bare metal server using tofu
7974
if [[ ${ACTION} == "deploy-metal" ]]; then
8075
API_KEY=$2
8176
PROJECT_ID=$3
8277
PLAN=$4
8378
ACTOR=$(echo "adhoc-$5-"$(${SCRIPT_DIR}/base.sh $(date +%s%03N) 36) | tr '[:upper:]' '[:lower:]')
84-
EXPIRE_WHEN=$6
8579
echo "Deploying Server: ${ACTOR}"
86-
8780
BEGIN_SECS=$(date +%s)
88-
DEVICE_ID=$(curl --fail-with-body -X POST \
89-
-H "Content-Type: application/json" -H "X-Auth-Token: ${API_KEY}" \
90-
"https://api.equinix.com/metal/v1/projects/${PROJECT_ID}/devices?exclude=plan,ssh_keys,provisioning_events,network_ports,operating_system" \
91-
-d '{
92-
"metro": "sv",
93-
"plan": "'${PLAN}'",
94-
"operating_system": "ubuntu_22_04",
95-
"hostname": "'${ACTOR}'",
96-
"termination_time": "'$(date --iso-8601=seconds -d "+${EXPIRE_WHEN}")'"
97-
}' | jq | tee create-device-response.json | jq -r '.id')
98-
99-
IP_ADDRESS="null"
100-
for i in $(seq 100); do
101-
echo -n "$i) Device Status: "
102-
STATE=$(getDeviceInfo ${API_KEY} ${DEVICE_ID} ".state")
103-
echo "${STATE}"
104-
if [[ "${STATE}" == "active" ]]; then break; fi
105-
sleep 6
81+
82+
export TF_VAR_client_id="${PROJECT_ID}"
83+
export TF_VAR_client_secret="${API_KEY}"
84+
export TF_VAR_hostname="${ACTOR}"
85+
export TF_VAR_plan="${PLAN}"
86+
87+
pushd ${SCRIPT_DIR}/../resources
88+
tofu init -input=false
89+
tofu apply -auto-approve -input=false
90+
IP_ADDRESS=$(tofu output -raw public_ip)
91+
DEVICE_ID=$(tofu output -raw server_id)
92+
popd
93+
94+
STATUS=0
95+
for i in {1..30}; do
96+
if ssh -o StrictHostKeyChecking=no benchmark@"${IP_ADDRESS}" "echo ok" 2>/dev/null; then
97+
STATUS=1
98+
break
99+
fi
100+
sleep 10
106101
done
107-
102+
108103
DURATION=$(($(date +%s) - ${BEGIN_SECS}))
109-
110-
IP_ADDRESS=$(getDeviceInfo ${API_KEY} ${DEVICE_ID} ".ip_addresses[0].address")
111-
STATE=$(getDeviceInfo ${API_KEY} ${DEVICE_ID} ".state")
112-
if [[ "${IP_ADDRESS}" == "null" ]] || [[ "${STATE}" != "active" ]]; then
113-
echo "Failed to provision device after ${DURATION} seconds"
104+
if [[ ${STATUS} -eq 0 ]]; then
105+
echo "Failed to provision device ${ACTOR} after ${DURATION} seconds"
114106
exit 1
115107
fi
116108

@@ -119,23 +111,57 @@ if [[ ${ACTION} == "deploy-metal" ]]; then
119111
echo "DEVICE_NAME=${ACTOR}" | tee -a ${OUTPUT_NAME}
120112
echo "DEVICE_ID=${DEVICE_ID}" | tee -a ${OUTPUT_NAME}
121113
echo "DEVICE_ADDR=${IP_ADDRESS}" | tee -a ${OUTPUT_NAME}
122-
echo "DEVICE_EXPIRE=${EXPIRE_WHEN}" | tee -a ${OUTPUT_NAME}
123114
fi
124115

125-
# Delete a bare metal server using the Equlinix ReST API
116+
# Delete a bare metal server. Expects that tofu state exists from a previous tofu create
126117
if [[ ${ACTION} == "delete-metal" ]]; then
127118
API_KEY=$2
128119
DEVICE_ID=$3
129120
DEVICE_NAME=$4
130121

131-
curl --no-progress-meter --max-time 10 --fail-with-body -X DELETE -H "X-Auth-Token: ${API_KEY}" \
132-
"https://api.equinix.com/metal/v1/devices/${DEVICE_ID}" \
133-
| jq | tee delete-device-response.json
122+
pushd ${SCRIPT_DIR}/../resources
123+
tofu destroy -auto-approve
124+
popd
134125

135126
echo "ACTION=${ACTION}" | tee -a ${OUTPUT_NAME}
136127
echo "DEVICE_NAME=${DEVICE_NAME}" | tee -a ${OUTPUT_NAME}
137128
echo "DEVICE_ID=${DEVICE_ID}" | tee -a ${OUTPUT_NAME}
138129
fi
139130

131+
# Purge all ephemeral metal that's past its expiration date
132+
if [[ ${ACTION} == "purge-metal" ]]; then
133+
API_KEY=$2
134+
PROJECT_ID=$3
135+
EXPIRATION_HOURS=24
136+
137+
echo "Starting Ephemeral Server Cleanup"
138+
echo "Max Hours to Expiration: ${EXPIRATION_HOURS}"
139+
echo "Requesting OAuth2 Token"
140+
TOKEN=$(curl -s -X POST -d "grant_type=client_credentials" -d "client_id=${PROJECT_ID}" -d "client_secret=${API_KEY}" \
141+
https://auth.phoenixnap.com/auth/realms/BMC/protocol/openid-connect/token | jq -r '.access_token')
142+
143+
if [[ -z "$TOKEN" || "$TOKEN" == "null" ]]; then
144+
echo "Failed to obtain OAuth2 Token"
145+
exit 1
146+
fi
147+
echo "OAuth2 Token Acquired"
148+
149+
CUTOFF=$(date -u -d "$TTL_HOURS hours ago" +"%Y-%m-%dT%H:%M:%SZ")
150+
echo "Fetching Ephemeral Servers"
151+
SERVERS=$(curl -s -H "Authorization: Bearer $TOKEN" "https://api.phoenixnap.com/bmc/v1/servers?tag=ephemeral")
152+
COUNT=$(echo "$servers" | jq 'length')
153+
echo "Found ${COUNT} Ephemeral Servers."
154+
155+
echo "${SERVERS}" | jq -c '.[]' | while read server; do
156+
id=$(echo "$server" | jq -r '.id')
157+
hostname=$(echo "$server" | jq -r '.hostname')
158+
created=$(echo "$server" | jq -r '.creationDate')
159+
if [[ "$created" < "${CUTOFF}" ]]; then
160+
echo "Deleting Server: $hostname ($id)"
161+
curl -s -X DELETE -H "Authorization: Bearer $TOKEN" "https://api.phoenixnap.com/bmc/v1/servers/$id" > /dev/null
162+
fi
163+
done
140164

165+
echo "ACTION=${ACTION}" | tee -a ${OUTPUT_NAME}
166+
fi
141167

‎.github/workflows/adhoc-auto-remote-benchmarks.yml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2023-2024 Deephaven Data Labs and Patent Pending
1+
# Copyright (c) 2023-2026 Deephaven Data Labs and Patent Pending
22

33
# Provision a server and run benchmarks on that system, cleaning up after
44
# - Deploys a new server on the fly
@@ -65,10 +65,9 @@ jobs:
6565
test_iterations: ${{ steps.scale-nums.outputs.TEST_ITERATIONS }}
6666
env:
6767
SD: .github/scripts
68-
METAL_EXPIRE: "2 days"
6968
METAL_API_KEY: ${{ secrets.BENCHMARK_METAL_AUTH_TOKEN }}
7069
METAL_PROJECT_ID: ${{ secrets.BENCHMARK_METAL_PROJECT_ID }}
71-
METAL_PLAN: "c3.small.x86"
70+
METAL_PLAN: "s2.c2.small"
7271
METAL_ACTOR: "${{ github.actor }}"
7372

7473
steps:
@@ -84,11 +83,12 @@ jobs:
8483
run: |
8584
${SD}/adhoc.sh scale-nums ${{ inputs.scale_row_count }} ${{ inputs.test_iterations }}
8685
cat adhoc-scale-nums.out >> "$GITHUB_OUTPUT"
87-
86+
87+
- uses: hashicorp/setup-terraform@v3
8888
- name: Deploy Bare Metal
8989
id: deploy-metal
9090
run: |
91-
${SD}/adhoc.sh deploy-metal "${METAL_API_KEY}" "${METAL_PROJECT_ID}" "${METAL_PLAN}" "${METAL_ACTOR}" "${METAL_EXPIRE}"
91+
${SD}/adhoc.sh deploy-metal "${METAL_API_KEY}" "${METAL_PROJECT_ID}" "${METAL_PLAN}" "${METAL_ACTOR}"
9292
cat adhoc-deploy-metal.out >> "$GITHUB_OUTPUT"
9393
9494
- name: Archive Setup Logs

0 commit comments

Comments
 (0)