From b4d870d3ccf1054900ef018c7e89fd0728de99de Mon Sep 17 00:00:00 2001 From: bertiethorpe Date: Tue, 22 Apr 2025 10:33:40 +0000 Subject: [PATCH 1/3] fix nightly cleanup for duplicate server names --- .github/workflows/nightly-cleanup.yml | 44 ++++++++++++++------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/.github/workflows/nightly-cleanup.yml b/.github/workflows/nightly-cleanup.yml index 0f7156fad..d842c50bd 100644 --- a/.github/workflows/nightly-cleanup.yml +++ b/.github/workflows/nightly-cleanup.yml @@ -67,31 +67,33 @@ jobs: for cluster_prefix in ${ci_clusters} do echo "Processing cluster: $cluster_prefix" + # Get all servers with the matching name for control node - CONTROL_SERVERS=$(openstack server list --name ${cluster_prefix}-control --format json) - SERVER_COUNT=$(echo "$CONTROL_SERVERS" | jq length) + CONTROL_SERVERS=$(openstack server list --name ${cluster_prefix}-control --long --format json) + + # Get unique server names to avoid duplicate cleanup + UNIQUE_NAMES=$(echo "$CONTROL_SERVERS" | jq -r '.[].Name' | sort | uniq) + for name in $UNIQUE_NAMES; do + echo "Checking control node: $name" + + # Get the first matching server ID by name + server=$(echo "$CONTROL_SERVERS" | jq -r '.[] | select(.Name=="'"$name"'") | .ID' | head -n1) + + # Make sure server still exists (wasn't deleted earlier) + if ! openstack server show "$server" &>/dev/null; then + echo "Server $server no longer exists, skipping $name." + continue + fi + + # Get tags for the server + TAGS=$(openstack server show "$server" --column tags --format value) - if [[ $SERVER_COUNT -gt 1 ]]; then - echo "Multiple servers found for control node '${cluster_prefix}-control'. Checking tags for each..." - - for server in $(echo "$CONTROL_SERVERS" | jq -r '.[].ID'); do - # Get tags for each control node - TAGS=$(openstack server show "$server" --column tags --format value) - - if [[ $TAGS =~ "keep" ]]; then - echo "Skipping ${cluster_prefix} (server ${server}) - control instance is tagged as keep" - else - ./dev/delete-cluster.py ${cluster_prefix} --force - fi - done - else - # If only one server, extract its tags and proceed - TAGS=$(echo "$CONTROL_SERVERS" | jq -r '.[0].Tags') if [[ $TAGS =~ "keep" ]]; then - echo "Skipping ${cluster_prefix} - control instance is tagged as keep" + echo "Skipping $cluster_prefix - control instance is tagged as keep" else - ./dev/delete-cluster.py ${cluster_prefix} --force + echo "Deleting cluster $cluster_prefix (server $server)..." + ./dev/delete-cluster.py $cluster_prefix --force fi - fi + done done shell: bash From 8de7d0127afe4ca140db7ff9d56c395e3308d0f2 Mon Sep 17 00:00:00 2001 From: bertiethorpe Date: Tue, 22 Apr 2025 13:23:31 +0000 Subject: [PATCH 2/3] remove check for 'keep' tags in nightly cleanup --- .github/workflows/nightly-cleanup.yml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/workflows/nightly-cleanup.yml b/.github/workflows/nightly-cleanup.yml index d842c50bd..766424cf5 100644 --- a/.github/workflows/nightly-cleanup.yml +++ b/.github/workflows/nightly-cleanup.yml @@ -56,7 +56,7 @@ jobs: fi shell: bash - - name: Delete clusters if control node not tagged with keep + - name: Delete clusters run: | . venv/bin/activate if [[ -z ${ci_clusters} ]]; then @@ -69,12 +69,12 @@ jobs: echo "Processing cluster: $cluster_prefix" # Get all servers with the matching name for control node - CONTROL_SERVERS=$(openstack server list --name ${cluster_prefix}-control --long --format json) + CONTROL_SERVERS=$(openstack server list --name ${cluster_prefix}-control --format json) # Get unique server names to avoid duplicate cleanup UNIQUE_NAMES=$(echo "$CONTROL_SERVERS" | jq -r '.[].Name' | sort | uniq) for name in $UNIQUE_NAMES; do - echo "Checking control node: $name" + echo "Deleting cluster with control node: $name" # Get the first matching server ID by name server=$(echo "$CONTROL_SERVERS" | jq -r '.[] | select(.Name=="'"$name"'") | .ID' | head -n1) @@ -85,15 +85,8 @@ jobs: continue fi - # Get tags for the server - TAGS=$(openstack server show "$server" --column tags --format value) - - if [[ $TAGS =~ "keep" ]]; then - echo "Skipping $cluster_prefix - control instance is tagged as keep" - else - echo "Deleting cluster $cluster_prefix (server $server)..." - ./dev/delete-cluster.py $cluster_prefix --force - fi + echo "Deleting cluster $cluster_prefix (server $server)..." + ./dev/delete-cluster.py $cluster_prefix --force done done shell: bash From 8da423bdd21e9c3d7bf29d3f6fd634084e4ec0b6 Mon Sep 17 00:00:00 2001 From: bertiethorpe Date: Tue, 22 Apr 2025 13:37:04 +0000 Subject: [PATCH 3/3] simplify cluster deletion --- .github/workflows/nightly-cleanup.yml | 32 ++++----------------------- 1 file changed, 4 insertions(+), 28 deletions(-) diff --git a/.github/workflows/nightly-cleanup.yml b/.github/workflows/nightly-cleanup.yml index 766424cf5..cc1d14c1e 100644 --- a/.github/workflows/nightly-cleanup.yml +++ b/.github/workflows/nightly-cleanup.yml @@ -56,37 +56,13 @@ jobs: fi shell: bash - - name: Delete clusters + - name: Delete CI clusters run: | . venv/bin/activate - if [[ -z ${ci_clusters} ]]; then + if [[ -z "${ci_clusters}" ]]; then echo "No clusters to delete." exit 0 fi - - for cluster_prefix in ${ci_clusters} - do - echo "Processing cluster: $cluster_prefix" - - # Get all servers with the matching name for control node - CONTROL_SERVERS=$(openstack server list --name ${cluster_prefix}-control --format json) - - # Get unique server names to avoid duplicate cleanup - UNIQUE_NAMES=$(echo "$CONTROL_SERVERS" | jq -r '.[].Name' | sort | uniq) - for name in $UNIQUE_NAMES; do - echo "Deleting cluster with control node: $name" - - # Get the first matching server ID by name - server=$(echo "$CONTROL_SERVERS" | jq -r '.[] | select(.Name=="'"$name"'") | .ID' | head -n1) - - # Make sure server still exists (wasn't deleted earlier) - if ! openstack server show "$server" &>/dev/null; then - echo "Server $server no longer exists, skipping $name." - continue - fi - - echo "Deleting cluster $cluster_prefix (server $server)..." - ./dev/delete-cluster.py $cluster_prefix --force - done - done + echo "Deleting clusters: ${ci_clusters}" + ./dev/delete-cluster.py ${ci_clusters} --force shell: bash