Allow Prometheus database size limit to be set in UI

Author: m-bull

group_vars/openstack.yml

@@ -17,5 +17,8 @@ terraform_project_path: "{{ playbook_dir }}/terraform"
 terraform_state: "{{ cluster_state | default('present') }}"
 cluster_ssh_user: rocky
 
+# Set the size of the state volume to metrics_db_maximum_size + 10
+state_volume_size: "{{ metrics_db_maximum_size + 10 }}"
+
 state_volume_device_path: "{{ cluster_state_volume_device_path | default('/dev/vdb') }}"
 home_volume_device_path: "{{ cluster_home_volume_device_path | default('/dev/vdc') }}"

group_vars/prometheus.yml

@@ -7,5 +7,5 @@ openondemand_address: "{{ hostvars[groups['openondemand'].0].api_address if 'ope
 # OOD is being deployed then there won't be an OOD group
 prometheus_scrape_configs: "{{ prometheus_scrape_configs_default + (openondemand_scrape_configs if ( 'openondemand' in groups ) else [] ) }}"
 
-# Fix Prometheus storage retention size to 80% of state volume size
-prometheus_storage_retention_size: "{{ ( 0.8 * ( cluster_state_volume_size | int ) ) | int ~ 'GB' }}"
+# Set Prometheus storage retention size
+prometheus_storage_retention_size: "{{ metrics_db_maximum_size }}GB"

roles/cluster_infra/templates/resources.tf.j2

@@ -75,7 +75,7 @@ resource "openstack_networking_secgroup_rule_v2" "secgroup_slurm_login_rule_ingr
 resource "openstack_blockstorage_volume_v3" "state" {
     name = "{{ cluster_name }}-state"
     description = "State for control node"
-    size = "{{ cluster_state_volume_size }}"
+    size = "{{ state_volume_size }}"
 }
 
 resource "openstack_blockstorage_volume_v3" "home" {

ui-meta/slurm-infra.yml

@@ -30,14 +30,28 @@ parameters:
       min_disk: 20
 
   - name: home_volume_size
-    label: Home volume size
-    description: The size in GB of the volume to use for home directories
+    label: Home volume size (GB)
+    description: The size of the cloud volume to use for home directories
     kind: integer
     immutable: true
     options:
       min: 10
     default: 100
 
+  - name: metrics_db_maximum_size
+    label: Metrics database size (GB)
+    description: |
+      The oldest metrics records in the [Prometheus](https://prometheus.io/) database will be 
+      discarded to ensure that the database does not grow larger than this size.
+      
+      **A cloud volume of this size +10GB will be created to hold and persist the metrics 
+      database and important Slurm files.**
+    kind: integer
+    immutable: true
+    options:
+      min: 10
+    default: 10
+
   - name: cluster_run_validation
     label: Post-configuration validation
     description: >-