Merge branch 'main' into feature/image-patch

Author: sjpb

group_vars/openstack.yml

@@ -17,4 +17,8 @@ terraform_project_path: "{{ playbook_dir }}/terraform"
 terraform_state: "{{ cluster_state | default('present') }}"
 cluster_ssh_user: rocky
 
-state_volume_size: 150 # GB
+# Set the size of the state volume to metrics_db_maximum_size + 10
+state_volume_size: "{{ metrics_db_maximum_size + 10 }}"
+
+state_volume_device_path: "{{ cluster_state_volume_device_path | default('/dev/vdb') }}"
+home_volume_device_path: "{{ cluster_home_volume_device_path | default('/dev/vdc') }}"

group_vars/prometheus.yml

@@ -6,3 +6,6 @@ openondemand_address: "{{ hostvars[groups['openondemand'].0].api_address if 'ope
 # Override group_var set in ansible-slurm-appliance all group - unless
 # OOD is being deployed then there won't be an OOD group
 prometheus_scrape_configs: "{{ prometheus_scrape_configs_default + (openondemand_scrape_configs if ( 'openondemand' in groups ) else [] ) }}"
+
+# Set Prometheus storage retention size
+prometheus_storage_retention_size: "{{ metrics_db_maximum_size }}GB"

roles/cluster_infra/templates/resources.tf.j2

@@ -81,7 +81,7 @@ resource "openstack_blockstorage_volume_v3" "state" {
 resource "openstack_blockstorage_volume_v3" "home" {
     name = "{{ cluster_name }}-home"
     description = "Home for control node"
-    size = "{{ home_volme_size }}"
+    size = "{{ home_volume_size }}"
 }
 
 
@@ -162,11 +162,11 @@ resource "openstack_compute_instance_v2" "control" {
     fs_setup:
       - label: state
         filesystem: ext4
-        device: /dev/vdb
+        device: {{ state_volume_device_path }}
         partition: auto
       - label: home
         filesystem: ext4
-        device: /dev/vdc
+        device: {{ home_volume_device_path }}
         partition: auto
     mounts:
         - [LABEL=state, /var/lib/state]

roles/persist_openhpc_secrets/tasks/main.yml

@@ -2,27 +2,34 @@
 
 - name: Check if OpenHPC secrets exist in persistent storage
   stat:
-    path: "{{ appliances_state_dir }}/openhpc_secrets.fact"
+    path: "{{ appliances_state_dir }}/ansible/facts.d/openhpc_secrets.fact"
   register: openhpc_secrets_stat
 
-- name: Ensure Ansible facts directory exists
+- name: Ensure Ansible facts directories exist
   file:
-    path: /etc/ansible/facts.d
+    path: "{{ item }}"
     state: directory
-    recurse: yes
-
-- name: Write OpenHPC secrets to persistent storage
+    owner: root
+    mode: 0600
+  loop:
+    - "{{ appliances_state_dir }}/ansible.facts.d"
+    - "/etc/ansible/facts.d"
+    
+- name: Write OpenHPC secrets
   template:
     src: openhpc_secrets.fact
-    dest: "{{ appliances_state_dir }}/openhpc_secrets.fact"
-  when: "not openhpc_secrets_stat.stat.exists" # required as templated ones are random
+    dest: "{{ appliances_state_dir }}/ansible.facts.d/openhpc_secrets.fact"
+    owner: root
+    mode: 0600
+  when: "not openhpc_secrets_stat.stat.exists"
 
-- name: Make OpenHPC secrets available to local facts
-  copy:
-    src: "{{ appliances_state_dir }}/openhpc_secrets.fact"
+- name: Symlink persistent facts to facts_path
+  file:
+    state: link
+    src: "{{ appliances_state_dir }}/ansible.facts.d/openhpc_secrets.fact"
     dest: /etc/ansible/facts.d/openhpc_secrets.fact
-    remote_src: yes
-
-- name: Re-read facts after adding custom fact
+    owner: root
+    
+- name: Read facts
   ansible.builtin.setup:
     filter: ansible_local

ui-meta/slurm-infra.yml

@@ -27,17 +27,31 @@ parameters:
     immutable: true
     options:
       min_ram: 2048
-      min_disk: 10
+      min_disk: 20
 
-  - name: home_volme_size
-    label: Home volume size
-    description: The size in GB of the volume to use for home directories
+  - name: home_volume_size
+    label: Home volume size (GB)
+    description: The size of the cloud volume to use for home directories
     kind: integer
     immutable: true
     options:
-      min: 1
+      min: 10
     default: 100
 
+  - name: metrics_db_maximum_size
+    label: Metrics database size (GB)
+    description: |
+      The oldest metrics records in the [Prometheus](https://prometheus.io/) database will be 
+      discarded to ensure that the database does not grow larger than this size.
+      
+      **A cloud volume of this size +10GB will be created to hold and persist the metrics 
+      database and important Slurm files.**
+    kind: integer
+    immutable: true
+    options:
+      min: 10
+    default: 10
+
   - name: cluster_run_validation
     label: Post-configuration validation
     description: >-