merge

Author: wtripp180901

.github/workflows/fatimage.yml

@@ -117,4 +117,4 @@ jobs:
           path: |
             ./image-id.txt
             ./image-name.txt
-          overwrite: true
+          overwrite: true

README.md

@@ -55,21 +55,28 @@ You will also need to install [OpenTofu](https://opentofu.org/docs/intro/install
 
 ### Create a new environment
 
-Use the `cookiecutter` template to create a new environment to hold your configuration. In the repository root run:
+Run the following from the repository root to activate the venv:
 
     . venv/bin/activate
+
+Use the `cookiecutter` template to create a new environment to hold your configuration:
+
     cd environments
     cookiecutter skeleton
 
 and follow the prompts to complete the environment name and description.
 
 **NB:** In subsequent sections this new environment is refered to as `$ENV`.
 
-Now generate secrets for this environment:
+Activate the new environment:
+
+    . environments/$ENV/activate
+
+And generate secrets for it:
 
     ansible-playbook ansible/adhoc/generate-passwords.yml
 
-### Define infrastructure configuration
+### Define and deploy infrastructure
 
 Create an OpenTofu variables file to define the required infrastructure, e.g.:
 
@@ -91,20 +98,28 @@ Create an OpenTofu variables file to define the required infrastructure, e.g.:
         }
     }
 
-Variables marked `*` refer to OpenStack resources which must already exist. The above is a minimal configuration - for all variables
-and descriptions see `environments/$ENV/terraform/terraform.tfvars`.
+Variables marked `*` refer to OpenStack resources which must already exist. The above is a minimal configuration - for all variables and descriptions see `environments/$ENV/terraform/terraform.tfvars`.
+
+To deploy this infrastructure, ensure the venv and the environment are [activated](#create-a-new-environment) and run:
 
-### Deploy appliance
+    export OS_CLOUD=openstack
+    cd environments/$ENV/terraform/
+    tofu apply
+
+and follow the prompts. Note the OS_CLOUD environment variable assumes that OpenStack credentials are defined using a [clouds.yaml](https://docs.openstack.org/python-openstackclient/latest/configuration/index.html#clouds-yaml) file in a default location with the default cloud name of `openstack`.
+
+### Configure appliance
+
+To configure the appliance, ensure the venv and the environment are [activated](#create-a-new-environment) and run:
 
     ansible-playbook ansible/site.yml
 
-You can now log in to the cluster using:
+Once it completes you can log in to the cluster using:
 
     ssh rocky@$login_ip
 
 where the IP of the login node is given in `environments/$ENV/inventory/hosts.yml`
 
-
 ## Overview of directory structure
 
 - `environments/`: See [docs/environments.md](docs/environments.md).

ansible/.gitignore

@@ -64,3 +64,5 @@ roles/*
 !roles/k9s/**
 !roles/kube_prometheus_stack
 !roles/kube_prometheus_stack/**
+!roles/lustre/
+!roles/lustre/**

ansible/cleanup.yml

@@ -38,11 +38,6 @@
 
 - name: Cleanup /tmp
   command : rm -rf /tmp/*
-
-- name: Delete ansible-init sentinel file created if ansible-init has run during build
-  ansible.builtin.file:
-    path: /var/lib/ansible-init.done
-    state: absent
 
 - name: Get package facts
   package_facts:

ansible/fatimage.yml

@@ -25,7 +25,7 @@
 
 - hosts: builder
   become: yes
-  gather_facts: no
+  gather_facts: yes
   tasks:
     # - import_playbook: iam.yml
     - name: Install FreeIPA client
@@ -44,6 +44,11 @@
         name: stackhpc.os-manila-mount
         tasks_from: install.yml
       when: "'manila' in group_names"
+    - name: Install Lustre packages
+      include_role:
+        name: lustre
+        tasks_from: install.yml
+      when: "'lustre' in group_names"
 
 - import_playbook: extras.yml
 
@@ -57,6 +62,7 @@
         name: mysql
         tasks_from: install.yml
       when: "'mysql' in group_names"
+
     - name: OpenHPC
       import_role:
         name: stackhpc.openhpc
@@ -83,18 +89,21 @@
       import_role:
         name: openondemand
         tasks_from: vnc_compute.yml
+
       when: "'openondemand_desktop' in group_names"
+
     - name: Open Ondemand jupyter node
       import_role:
         name: openondemand
         tasks_from: jupyter_compute.yml
-      when: "'openondemand' in group_names"
+      when: "'openondemand_jupyter' in group_names"
 
     # - import_playbook: monitoring.yml:
     - import_role:
         name: opensearch
         tasks_from: install.yml
       when: "'opensearch' in group_names"
+
     # slurm_stats - nothing to do
     - import_role:
         name: filebeat

ansible/filesystems.yml

@@ -24,3 +24,13 @@
   tasks:
     - include_role:
         name: stackhpc.os-manila-mount
+
+- name: Setup Lustre clients
+  hosts: lustre
+  become: true
+  tags: lustre
+  tasks:
+    - include_role:
+        name: lustre
+        # NB install is ONLY run in builder
+        tasks_from: configure.yml

ansible/roles/cluster_infra/templates/resources.tf.j2

@@ -15,7 +15,7 @@ resource "terraform_data" "k3s_token" {
     input = "{{ k3s_token }}"
     lifecycle {
         ignore_changes = [
-            input,
+            input, # makes it a write-once value (set via Ansible)
         ]
     }   
 }

ansible/roles/k3s/README.md

@@ -8,7 +8,7 @@ Installs k3s agent and server services on nodes and an ansible-init playbook to
 Requirements
 ------------
 
-`azimuth_cloud.image_utils.linux_ansible_init` must have been run previously on targeted nodes
+`azimuth_cloud.image_utils.linux_ansible_init` must have been run previously on targeted nodes during image build.
 
 Role Variables
 --------------

ansible/roles/k3s/defaults/main.yml

@@ -2,4 +2,4 @@
 k3s_version: "v1.31.0+k3s1"
 k3s_selinux_release: v1.6.latest.1
 k3s_selinux_rpm_version: 1.6-1
-rocky_version: "{{ ansible_distribution_major_version }}"
+k3s_helm_version: v3.11.0

ansible/roles/k3s/tasks/main.yml

@@ -5,7 +5,8 @@
     path: /var/lib/rancher/k3s
   register: stat_result
 
-- name: Download and air-gapped installation of k3s
+- name: Perform air-gapped installation of k3s
+  # Using air-gapped install so containers are pre-installed to avoid rate-limiting from registries on cluster startup
   when: not stat_result.stat.exists
   block:
 
@@ -19,7 +20,7 @@
 
   - name: Install k3s SELinux policy package
     yum:
-      name: "https://github.com/k3s-io/k3s-selinux/releases/download/{{ k3s_selinux_release }}/k3s-selinux-{{ k3s_selinux_rpm_version }}.el{{ rocky_version }}.noarch.rpm"
+      name: "https://github.com/k3s-io/k3s-selinux/releases/download/{{ k3s_selinux_release }}/k3s-selinux-{{ k3s_selinux_rpm_version }}.el{{ ansible_distribution_major_version }}.noarch.rpm"
       disable_gpg_check: true
 
   - name: Create image directory
@@ -58,7 +59,7 @@
 
 - name: Install helm
   unarchive:
-    src: https://get.helm.sh/helm-v3.11.0-linux-amd64.tar.gz
+    src: "https://get.helm.sh/helm-{{ k3s_helm_version }}-linux-amd64.tar.gz"
     dest: /usr/bin
     extra_opts: "--strip-components=1"
     owner: root