Merge branch 'main' into feature/k3s-monitoring

Author: wtripp180901

.github/workflows/nightly-cleanup.yml

@@ -1,17 +1,8 @@
 name: Cleanup CI clusters
 on:
   workflow_dispatch:
-      inputs:
-        ci_cloud:
-          description: 'Select the CI_CLOUD'
-          required: true
-          type: choice
-          options:
-            - LEAFCLOUD
-            - SMS
-            - ARCUS
   schedule:
-    - cron: '0 20 * * *'  # Run at 8PM - image sync runs at midnight
+    - cron: '0 21 * * *'  # Run at 9PM - image sync runs at midnight
 
 jobs:
   ci_cleanup:
@@ -52,20 +43,35 @@ jobs:
       - name: Find CI clusters
         run: |
           . venv/bin/activate
-          CI_CLUSTERS=$(openstack server list | grep --only-matching 'slurmci-RL.-[0-9]\+'  | sort | uniq)
-          echo "ci_clusters=${CI_CLUSTERS}" >> GITHUB_ENV
+          CI_CLUSTERS=$(openstack server list | grep --only-matching 'slurmci-RL.-[0-9]\+'  | sort | uniq || true)
+          echo "DEBUG: Raw CI clusters: $CI_CLUSTERS"
+      
+          if [[ -z "$CI_CLUSTERS" ]]; then
+            echo "No matching CI clusters found."
+          else
+            # Flatten multiline value so can be passed as env var
+            CI_CLUSTERS_FORMATTED=$(echo "$CI_CLUSTERS" | tr '\n' ' ' | sed 's/ $//')
+            echo "DEBUG: Formatted CI clusters: $CI_CLUSTERS_FORMATTED"
+            echo "ci_clusters=$CI_CLUSTERS_FORMATTED" >> $GITHUB_ENV
+          fi
         shell: bash
 
       - name: Delete clusters if control node not tagged with keep
         run: |
           . venv/bin/activate
-          for cluster_prefix in ${CI_CLUSTERS}
+          if [[ -z ${ci_clusters} ]]; then
+            echo "No clusters to delete."
+            exit 0
+          fi
+          
+          for cluster_prefix in ${ci_clusters}
           do
+            echo "Processing cluster: $cluster_prefix"
             TAGS=$(openstack server show ${cluster_prefix}-control --column tags --format value)
             if [[ $TAGS =~ "keep" ]]; then
               echo "Skipping ${cluster_prefix} - control instance is tagged as keep"
             else
-              yes | ./dev/delete-cluster.py ${cluster_prefix}
+              ./dev/delete-cluster.py ${cluster_prefix} --force
             fi
           done
         shell: bash

ansible/adhoc/rebuild.yml

@@ -16,3 +16,6 @@
     - command: "openstack server rebuild {{ instance_id | default(inventory_hostname) }}{% if rebuild_image is defined %} --image {{ rebuild_image }}{% endif %}"
       delegate_to: localhost
     - wait_for_connection:
+        delay: 60
+        timeout: 600
+

ansible/extras.yml

@@ -40,6 +40,7 @@
 - name: Install k9s
   become: yes
   hosts: k9s
+  tags: k9s
   tasks:
   - import_role:
       name: k9s

ansible/roles/cluster_infra/templates/resources.tf.j2

@@ -471,7 +471,7 @@ resource "openstack_compute_instance_v2" "control" {
     {%- endif %}
     bootcmd:
     %{for volume in [openstack_blockstorage_volume_v3.state, {% if not cluster_home_manila_share | bool %} openstack_blockstorage_volume_v3.home {% endif %}]}
-    - BLKDEV=$(readlink -f $(ls /dev/disk/by-id/*${substr(volume.id, 0, 20)}* | head -n1 )); blkid -o value -s TYPE $BLKDEV ||  mke2fs -t ext4 -L ${lower(split(" ", volume.description)[0])} $BLKDEV
+    - BLKDEV=$(readlink -f $(ls /dev/disk/by-id/*${replace(substr(volume.id, 0, 20), "-", "*")}* | head -n1 )); blkid -o value -s TYPE $BLKDEV ||  mke2fs -t ext4 -L ${lower(split(" ", volume.description)[0])} $BLKDEV
     %{endfor}
     mounts:
         - [LABEL=state, {{ appliances_state_dir }}, auto]

ansible/roles/k9s/tasks/main.yml

@@ -3,7 +3,7 @@
   - name: Check if k9s is installed
     ansible.builtin.stat:
       path: "/usr/bin/k9s"
-    register: result
+    register: _k9s_stat_result
 
   - name: Install k9s and clean up temporary files
     block:
@@ -14,7 +14,7 @@
         owner: root
         group: root
         mode: "744"
-      when: not result.stat.exists
+      when: not _k9s_stat_result.stat.exists
 
     - name: Download k9s
       ansible.builtin.get_url:
@@ -41,4 +41,4 @@
       ansible.builtin.file:
         path: /tmp/k9s
         state: absent
-    when: not result.stat.exists
+    when: not _k9s_stat_result.stat.exists

dev/delete-cluster.py

@@ -4,18 +4,18 @@
 Delete infrastructure for a cluster without using Terraform. Useful for CI clusters.
 
 Usage:
-    delete-cluster.py PREFIX
+    delete-cluster.py PREFIX [--force]
 
 Where PREFIX is the string at the start of the resource's names.
-It will list matching resources and prompt to confirm deletion.
+If --force is provided, it will delete all resources without confirmation.
 """
 
-import sys, json, subprocess, pprint
+import sys, json, subprocess
 
 
 CLUSTER_RESOURCES = ['server', 'port', 'volume']
 
-def delete_cluster(cluster_prefix):
+def delete_cluster(cluster_prefix, force=False):
     to_delete = {}
     for resource_type in CLUSTER_RESOURCES:
         to_delete[resource_type] = []
@@ -29,7 +29,8 @@ def delete_cluster(cluster_prefix):
             except:
                 print(resource_type, item)
                 raise
-    if input('Delete these (y/n)?:') == 'y':
+    
+    if force or input('Delete these (y/n)?:') == 'y':
         for resource_type in CLUSTER_RESOURCES:
             items = [v['ID'] for v in to_delete[resource_type]]
             if items:
@@ -40,7 +41,10 @@ def delete_cluster(cluster_prefix):
         print('Cancelled - no resources deleted')
 
 if __name__ == '__main__':
-    if len(sys.argv) != 2:
+    if len(sys.argv) < 2 or len(sys.argv) > 3:
         print('ERROR: Incorrect argument(s).\n' + __doc__)
         exit(1)
-    delete_cluster(sys.argv[1])
+    force_flag = '--force' in sys.argv
+    cluster_prefix = sys.argv[1]
+    delete_cluster(cluster_prefix, force_flag)
+