Merge branch 'main' into feat/compute-script

Author: bertiethorpe

.github/workflows/nightly-cleanup.yml

@@ -1,17 +1,8 @@
 name: Cleanup CI clusters
 on:
   workflow_dispatch:
-      inputs:
-        ci_cloud:
-          description: 'Select the CI_CLOUD'
-          required: true
-          type: choice
-          options:
-            - LEAFCLOUD
-            - SMS
-            - ARCUS
   schedule:
-    - cron: '0 20 * * *'  # Run at 8PM - image sync runs at midnight
+    - cron: '0 21 * * *'  # Run at 9PM - image sync runs at midnight
 
 jobs:
   ci_cleanup:
@@ -52,20 +43,35 @@ jobs:
       - name: Find CI clusters
         run: |
           . venv/bin/activate
-          CI_CLUSTERS=$(openstack server list | grep --only-matching 'slurmci-RL.-[0-9]\+'  | sort | uniq)
-          echo "ci_clusters=${CI_CLUSTERS}" >> GITHUB_ENV
+          CI_CLUSTERS=$(openstack server list | grep --only-matching 'slurmci-RL.-[0-9]\+'  | sort | uniq || true)
+          echo "DEBUG: Raw CI clusters: $CI_CLUSTERS"
+      
+          if [[ -z "$CI_CLUSTERS" ]]; then
+            echo "No matching CI clusters found."
+          else
+            # Flatten multiline value so can be passed as env var
+            CI_CLUSTERS_FORMATTED=$(echo "$CI_CLUSTERS" | tr '\n' ' ' | sed 's/ $//')
+            echo "DEBUG: Formatted CI clusters: $CI_CLUSTERS_FORMATTED"
+            echo "ci_clusters=$CI_CLUSTERS_FORMATTED" >> $GITHUB_ENV
+          fi
         shell: bash
 
       - name: Delete clusters if control node not tagged with keep
         run: |
           . venv/bin/activate
-          for cluster_prefix in ${CI_CLUSTERS}
+          if [[ -z ${ci_clusters} ]]; then
+            echo "No clusters to delete."
+            exit 0
+          fi
+          
+          for cluster_prefix in ${ci_clusters}
           do
+            echo "Processing cluster: $cluster_prefix"
             TAGS=$(openstack server show ${cluster_prefix}-control --column tags --format value)
             if [[ $TAGS =~ "keep" ]]; then
               echo "Skipping ${cluster_prefix} - control instance is tagged as keep"
             else
-              yes | ./dev/delete-cluster.py ${cluster_prefix}
+              ./dev/delete-cluster.py ${cluster_prefix} --force
             fi
           done
         shell: bash

ansible/.gitignore

@@ -60,5 +60,9 @@ roles/*
 !roles/tuned/**
 !roles/compute_init/
 !roles/compute_init/**
+!roles/k3s/
+!roles/k3s/**
+!roles/k9s/
+!roles/k9s/**
 !roles/lustre/
 !roles/lustre/**

ansible/bootstrap.yml

@@ -259,3 +259,11 @@
   tasks:
     - include_role:
         name: azimuth_cloud.image_utils.linux_ansible_init
+
+- hosts: k3s
+  become: yes
+  tags: k3s
+  tasks:
+    - ansible.builtin.include_role:
+        name: k3s
+        tasks_from: install.yml

ansible/cleanup.yml

@@ -38,7 +38,7 @@
 
 - name: Cleanup /tmp
   command : rm -rf /tmp/*
-
+    
 - name: Get package facts
   package_facts:
 

ansible/extras.yml

@@ -43,4 +43,12 @@
   become: yes
   tasks:
     - import_role:
-        name: compute_init
+        name: compute_init
+
+- name: Install k9s
+  become: yes
+  hosts: k9s
+  tags: k9s
+  tasks:
+  - import_role:
+      name: k9s

ansible/roles/cluster_infra/templates/resources.tf.j2

@@ -7,6 +7,19 @@ data "openstack_identity_auth_scope_v3" "scope" {
   name = "{{ cluster_name }}"
 }
 
+####
+#### Data resources
+####
+
+resource "terraform_data" "k3s_token" {
+    input = "{{ k3s_token }}"
+    lifecycle {
+        ignore_changes = [
+            input, # makes it a write-once value (set via Ansible)
+        ]
+    }   
+}
+
 #####
 ##### Security groups for the cluster
 #####
@@ -386,6 +399,8 @@ resource "openstack_compute_instance_v2" "login" {
         ansible_init_coll_{{ loop.index0 }}_source = "{{ collection.source }}"
       {% endif %}
     {% endfor %} 
+    k3s_server = openstack_compute_instance_v2.control.network[0].fixed_ip_v4
+    k3s_token = "{{ k3s_token }}"
   }
 }
 
@@ -400,6 +415,7 @@ resource "openstack_compute_instance_v2" "control" {
 
   network {
     port = openstack_networking_port_v2.control.id
+    access_network = true
   }
 
   {% if cluster_storage_network is defined %}
@@ -455,7 +471,7 @@ resource "openstack_compute_instance_v2" "control" {
     {%- endif %}
     bootcmd:
     %{for volume in [openstack_blockstorage_volume_v3.state, {% if not cluster_home_manila_share | bool %} openstack_blockstorage_volume_v3.home {% endif %}]}
-    - BLKDEV=$(readlink -f $(ls /dev/disk/by-id/*${substr(volume.id, 0, 20)}* | head -n1 )); blkid -o value -s TYPE $BLKDEV ||  mke2fs -t ext4 -L ${lower(split(" ", volume.description)[0])} $BLKDEV
+    - BLKDEV=$(readlink -f $(ls /dev/disk/by-id/*${replace(substr(volume.id, 0, 20), "-", "*")}* | head -n1 )); blkid -o value -s TYPE $BLKDEV ||  mke2fs -t ext4 -L ${lower(split(" ", volume.description)[0])} $BLKDEV
     %{endfor}
     mounts:
         - [LABEL=state, {{ appliances_state_dir }}, auto]
@@ -479,6 +495,7 @@ resource "openstack_compute_instance_v2" "control" {
         ansible_init_coll_{{ loop.index0 }}_source = "{{ collection.source }}"
       {% endif %}
     {% endfor %} 
+    k3s_token = "{{ k3s_token }}"
   }
 }
 
@@ -548,6 +565,8 @@ resource "openstack_compute_instance_v2" "{{ partition.name }}" {
         ansible_init_coll_{{ loop.index0 }}_source = "{{ collection.source }}"
       {% endif %}
     {% endfor %} 
+    k3s_server = openstack_compute_instance_v2.control.network[0].fixed_ip_v4
+    k3s_token = "{{ k3s_token }}"
   }
 }
 

ansible/roles/k3s/README.md

@@ -0,0 +1,16 @@
+k3s
+=====
+
+Installs k3s agent and server services on nodes and an ansible-init playbook to activate them. The service that each node will activate on init is determined by OpenStack metadata. Also includes Helm install. Currently only supports a single k3s-server
+(i.e one control node). Install based on the [official k3s ansible role](https://github.com/k3s-io/k3s-ansible).
+
+
+Requirements
+------------
+
+`azimuth_cloud.image_utils.linux_ansible_init` must have been run previously on targeted nodes during image build.
+
+Role Variables
+--------------
+
+- `k3s_version`: Optional str. K3s version to install, see [official releases](https://github.com/k3s-io/k3s/releases/).

ansible/roles/k3s/defaults/main.yml

@@ -0,0 +1,5 @@
+# Warning: changes to these variables won't be reflected in the cluster/image if k3s is already installed
+k3s_version: "v1.31.0+k3s1"
+k3s_selinux_release: v1.6.latest.1
+k3s_selinux_rpm_version: 1.6-1
+k3s_helm_version: v3.11.0

ansible/roles/k3s/files/start_k3s.yml

@@ -0,0 +1,36 @@
+- hosts: localhost
+  become: true
+  vars:
+    os_metadata: "{{ lookup('url', 'http://169.254.169.254/openstack/latest/meta_data.json') | from_json }}"
+    k3s_token: "{{ os_metadata.meta.k3s_token }}"
+    k3s_server_name: "{{ os_metadata.meta.k3s_server }}"
+    service_name: "{{ 'k3s-agent' if k3s_server_name is defined else 'k3s' }}"
+  tasks:
+    - name: Ensure password directory exists
+      ansible.builtin.file: 
+        path: "/etc/rancher/node"
+        state: directory
+        
+    - name: Set agent node password as token # uses token to keep password consistent between reimages
+      ansible.builtin.copy:
+        dest: /etc/rancher/node/password
+        content: "{{ k3s_token }}"
+      
+    - name: Add the token for joining the cluster to the environment
+      no_log: true # avoid logging the server token
+      ansible.builtin.lineinfile:
+        path: "/etc/systemd/system/{{ service_name }}.service.env"
+        line: "K3S_TOKEN={{ k3s_token }}"
+
+    - name: Add server url to agents
+      ansible.builtin.lineinfile:
+        path: "/etc/systemd/system/{{ service_name }}.service.env"
+        line: "K3S_URL=https://{{ k3s_server_name }}:6443"
+      when: k3s_server_name is defined
+
+    - name: Start k3s service
+      ansible.builtin.systemd:
+        name: "{{ service_name }}"
+        daemon_reload: true
+        state: started
+        enabled: true

ansible/roles/k3s/tasks/install.yml

@@ -0,0 +1,78 @@
+---
+
+- name: Check for existing k3s installation
+  stat:
+    path: /var/lib/rancher/k3s
+  register: stat_result
+
+- name: Perform air-gapped installation of k3s
+  # Using air-gapped install so containers are pre-installed to avoid rate-limiting from registries on cluster startup
+  when: not stat_result.stat.exists
+  block:
+
+  - name: Download k3s binary
+    ansible.builtin.get_url:
+      url: "https://github.com/k3s-io/k3s/releases/download/{{ k3s_version | urlencode }}/k3s"
+      dest: /usr/bin/k3s
+      owner: root
+      group: root
+      mode: "0755"
+
+  - name: Install k3s SELinux policy package
+    yum:
+      name: "https://github.com/k3s-io/k3s-selinux/releases/download/{{ k3s_selinux_release }}/k3s-selinux-{{ k3s_selinux_rpm_version }}.el{{ ansible_distribution_major_version }}.noarch.rpm"
+      disable_gpg_check: true
+
+  - name: Create image directory
+    ansible.builtin.file: 
+      path: "/var/lib/rancher/k3s/agent/images"
+      state: directory
+
+  - name: Install k3s' internal images
+    ansible.builtin.get_url:
+      url: "https://github.com/k3s-io/k3s/releases/download/{{ k3s_version | urlencode }}/k3s-airgap-images-amd64.tar.zst"
+      dest: /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar.zst
+
+  - name: Download k3s install script
+    ansible.builtin.get_url:
+      url: https://get.k3s.io/
+      timeout: 120
+      dest: /usr/bin/k3s-install.sh
+      owner: root
+      group: root
+      mode: "0755"
+
+  - name: Install k3s
+    ansible.builtin.shell:
+      cmd: /usr/bin/k3s-install.sh
+    environment:
+      INSTALL_K3S_VERSION: "{{ k3s_version }}"
+      INSTALL_K3S_EXEC: "{{ item }}"
+      INSTALL_K3S_SKIP_START: "true"
+      INSTALL_K3S_SKIP_ENABLE: "true"
+      INSTALL_K3S_BIN_DIR: "/usr/bin"
+      INSTALL_K3S_SKIP_DOWNLOAD: "true"
+    changed_when: true
+    loop:
+      - server --disable=traefik
+      - agent
+
+- name: Install helm
+  unarchive:
+    src: "https://get.helm.sh/helm-{{ k3s_helm_version }}-linux-amd64.tar.gz"
+    dest: /usr/bin
+    extra_opts: "--strip-components=1"
+    owner: root
+    group: root
+    mode: 0755
+    remote_src: true
+
+- name: Add k3s kubeconfig as environment variable
+  ansible.builtin.lineinfile:
+    path: /etc/environment
+    line: "KUBECONFIG=/etc/rancher/k3s/k3s.yaml"
+
+- name: Install ansible-init playbook for k3s agent or server activation
+  copy:
+    src: start_k3s.yml
+    dest: /etc/ansible-init/playbooks/0-start-k3s.yml