Support latest ansible-slurm-appliance

* Support Rocky 8.5
* Remove support for Kibana
* Remove ad-hoc Grafana install
* Support updated HPCtests
* Use latest stackhpc.terraform-infra

Author: m-bull

README.md

@@ -1,4 +1,5 @@
 # caas-slurm-appliance
 
 This repository adapts the [StackHPC Slurm Appliance](https://github.com/stackhpc/ansible-slurm-appliance)
-for use within the Cluster-as-a-Service system of the [Cloud Portal](https://github.com/stackhpc/jasmin-cloud).
+for use within the Cluster-as-a-Service system of the [Azimuth Cloud Portal](https://github.com/stackhpc/azimuth).
+

ansible.cfg

@@ -6,6 +6,7 @@ host_key_checking = False
 remote_tmp = /tmp
 # Enable our custom vars plugin that parses variables from the current working directory
 vars_plugins_enabled = host_group_vars,cwd_host_group_vars
+roles_path = roles:vendor/stackhpc/ansible-slurm-appliance/ansible/roles
 
 [ssh_connection]
 ssh_args = -o ControlMaster=auto -o ControlPersist=240s -o PreferredAuthentications=publickey -o UserKnownHostsFile=/dev/null
@@ -14,10 +15,3 @@ pipelining = True
 # This ensures that if the proxy connection is interrupted, rendering the other hosts
 # unreachable, the connection is retried instead of failing the entire play
 retries = 10
-
-[tags]
-# Temporarily, the grafana_install tag is skipped and we install Grafana manually
-# https://github.com/grafana/grafana/issues/36935
-# TODO: REMOVE THIS
-# By default, we also skip the HPL tests
-skip = grafana_install,hpl-solo,hpl-all

group_vars/cluster.yml

@@ -2,12 +2,12 @@
 update_enable: "{{ cluster_upgrade_system_packages | default('false') | bool }}"
 
 # Read the secrets from the Ansible local facts on the control host
-secrets_openhpc_grafana_admin_password: "{{ hostvars[groups['control'][0]].ansible_local.openhpc_secrets.grafana_admin_password }}"
-secrets_openhpc_elasticsearch_admin_password: "{{ hostvars[groups['control'][0]].ansible_local.openhpc_secrets.elasticsearch_admin_password }}"
-secrets_openhpc_elasticsearch_kibana_password: "{{ hostvars[groups['control'][0]].ansible_local.openhpc_secrets.elasticsearch_kibana_password }}"
-secrets_openhpc_mysql_root_password: "{{ hostvars[groups['control'][0]].ansible_local.openhpc_secrets.mysql_root_password }}"
-secrets_openhpc_mysql_slurm_password: "{{ hostvars[groups['control'][0]].ansible_local.openhpc_secrets.mysql_slurm_password }}"
-secrets_openhpc_mungekey: "{{ hostvars[groups['control'][0]].ansible_local.openhpc_secrets.mungekey }}"
+vault_grafana_admin_password: "{{ hostvars[groups['control'][0]].ansible_local.openhpc_secrets.vault_grafana_admin_password }}"
+vault_elasticsearch_admin_password: "{{ hostvars[groups['control'][0]].ansible_local.openhpc_secrets.vault_elasticsearch_admin_password }}"
+vault_elasticsearch_kibana_password: "{{ hostvars[groups['control'][0]].ansible_local.openhpc_secrets.vault_elasticsearch_kibana_password }}"
+vault_mysql_root_password: "{{ hostvars[groups['control'][0]].ansible_local.openhpc_secrets.vault_mysql_root_password }}"
+vault_mysql_slurm_password: "{{ hostvars[groups['control'][0]].ansible_local.openhpc_secrets.vault_mysql_slurm_password }}"
+vault_openhpc_mungekey: "{{ hostvars[groups['control'][0]].ansible_local.openhpc_secrets.vault_openhpc_mungekey }}"
 
 # Override this to cope with the case where the podman group just doesn't exist
 appliances_local_users_podman_enable: "{{ groups.get('podman', []) | length > 0 }}"

group_vars/openstack.yml

@@ -1,6 +1,12 @@
 # The default Terraform state key for backends that support it
 terraform_state_key: "cluster/{{ cluster_id }}/tfstate"
 
+terraform_backend_config_defaults:
+  consul:
+    path: "{{ terraform_state_key }}"
+    gzip: "true"
+  local: {}
+
 #####
 ## WARNING
 ##
@@ -24,16 +30,15 @@ cluster_groups_required:
 
 # These are the additional groups required for monitoring (see everything layout)
 cluster_groups_monitoring:
-  podman:        [opendistro, kibana, filebeat]
+  podman:        [opendistro, filebeat]
   prometheus:    [control]
   grafana:       [control]
   alertmanager:  [control]
   node_exporter: [cluster]
   opendistro:    [control]
-  kibana:        [control]
   slurm_stats:   [control]
   filebeat:      [slurm_stats]
 
 # Additional groups for running the cluster validation
 cluster_groups_validation:
-  openhpc_tests: [cluster]
+  hpctests: [login]

requirements.yml

@@ -1,26 +1,36 @@
 ---
 roles:
   - src: stackhpc.nfs
-  - src: stackhpc.openhpc
-  - src: cloudalchemy.node_exporter
+  - name: stackhpc.openhpc
+    src: https://github.com/stackhpc/ansible-role-openhpc
+    type: git
+    version: v0.10.0
+  #- src: cloudalchemy.node_exporter
+  - src: https://github.com/stackhpc/ansible-node-exporter.git
+    name: cloudalchemy.node_exporter
+    version: support-rhel-clones # Fixes for Rocky
   - src: cloudalchemy.blackbox-exporter
-  - src: cloudalchemy.prometheus
+  #- src: cloudalchemy.prometheus
+  - src: https://github.com/cloudalchemy/ansible-prometheus.git
+    name: cloudalchemy.prometheus
+    version: 0795bdb1a5a424044dc7e8200129f61e53140523 # Fixes for Rocky
   - src: cloudalchemy.alertmanager
   - src: cloudalchemy.grafana
   - src: geerlingguy.mysql
   - src: jriguera.configdrive
   - name: stackhpc.terraform-infra
     src: https://github.com/stackhpc/ansible-role-terraform-infra
     type: git
-    version: b750fd38be4eedf111eff0ed8833295efe3b6422
+    version: 4d9d67b5a1866edf6988a1f7e9e64868df8f65ae
 
 collections:
   - name: ansible.posix
+  - name: ansible.netcommon
   - name: community.general
   - name: community.grafana
   - name: community.mysql
   - name: containers.podman
   - name: openstack.cloud
   - name: https://github.com/stackhpc/ansible_collection_slurm_openstack_tools
     type: git
-    version: main
+    version: v0.1.0

roles/persist_openhpc_secrets/tasks/main.yml

@@ -17,4 +17,8 @@
       template:
         src: openhpc_secrets.fact
         dest: /etc/ansible/facts.d/openhpc_secrets.fact
+
+    - name: Re-read facts after adding custom fact
+      ansible.builtin.setup:
+        filter: ansible_local
   when: "not openhpc_secrets_stat.stat.exists"

roles/persist_openhpc_secrets/templates/openhpc_secrets.fact

@@ -1,8 +1,8 @@
 {
-  "grafana_admin_password": "{{ lookup('password', '/dev/null') }}",
-  "elasticsearch_admin_password": "{{ lookup('password', '/dev/null') }}",
-  "elasticsearch_kibana_password": "{{ lookup('password', '/dev/null') }}",
-  "mysql_root_password": "{{ lookup('password', '/dev/null') }}",
-  "mysql_slurm_password": "{{ lookup('password', '/dev/null') }}",
-  "mungekey": "{{ lookup('pipe', 'dd if=/dev/urandom bs=1 count=1024 2>/dev/null | base64') | regex_replace('\s+', '') }}"
+  "vault_grafana_admin_password": "{{ lookup('password', '/dev/null') }}",
+  "vault_elasticsearch_admin_password": "{{ lookup('password', '/dev/null') }}",
+  "vault_elasticsearch_kibana_password": "{{ lookup('password', '/dev/null') }}",
+  "vault_mysql_root_password": "{{ lookup('password', '/dev/null') }}",
+  "vault_mysql_slurm_password": "{{ lookup('password', '/dev/null') }}",
+  "vault_openhpc_mungekey": "{{ lookup('pipe', 'dd if=/dev/urandom bs=1 count=1024 2>/dev/null | base64') | regex_replace('\s+', '') }}"
 }

slurm-infra.yml

@@ -14,9 +14,14 @@
   roles:
     - role: stackhpc.terraform-infra
       vars:
+        # Image cloud user, use extraVars.__ALL__.cluster_user in values.yaml
+        # to change if required
+        cluster_ssh_user: "{{ cluster_user | default('rocky') }}"
         # Variables controlling the Terraform provisioning
         terraform_project_path: "{{ playbook_dir }}/terraform"
         terraform_state: "{{ cluster_state | default('present') }}"
+        terraform_backend_type: "{{ ( 'CONSUL_HTTP_ADDR' in ansible_env ) | ternary('consul', 'local') }}"
+        terraform_backend_config: "{{ terraform_backend_config_defaults[terraform_backend_type] }}"
         terraform_variables:
           cluster_name: "{{ cluster_name }}"
           cluster_network: "{{ cluster_network }}"
@@ -77,7 +82,11 @@
 # Configure the hosts as a Slurm cluster
 # Use the playbooks invidually rather than the site playbook as it avoids the
 # need to define the environment variables referencing an environment
+
+# validate.yml asserts presence of a control group which doesn't exist when 
+# destroying infra, so only validate when we're not destroying
 - import_playbook: vendor/stackhpc/ansible-slurm-appliance/ansible/validate.yml
+  when: cluster_state is not defined or (cluster_state is defined and cluster_state != "absent")
 
 # The first task in the bootstrap playbook causes the home directory of the centos user to be moved
 # on the first run
@@ -100,24 +109,7 @@
       meta: reset_connection
 
 - import_playbook: vendor/stackhpc/ansible-slurm-appliance/ansible/bootstrap.yml
-
 - import_playbook: vendor/stackhpc/ansible-slurm-appliance/ansible/filesystems.yml
 - import_playbook: vendor/stackhpc/ansible-slurm-appliance/ansible/slurm.yml
-
-# Temporarily, the grafana_install tag is skipped (see ansible.cfg) and we install Grafana manually here
-# https://github.com/grafana/grafana/issues/36935
-# TODO: REMOVE THIS
-- name: Install Grafana
-  hosts: grafana
-  tags: grafana
-  gather_facts: no
-  become: yes
-  tasks:
-    - name: Install Grafana from RPM
-      yum:
-        name: https://dl.grafana.com/oss/release/grafana-8.0.6-1.x86_64.rpm
-        disable_gpg_check: yes
-        state: present
-
 - import_playbook: vendor/stackhpc/ansible-slurm-appliance/ansible/monitoring.yml
-- import_playbook: vendor/stackhpc/ansible-slurm-appliance/ansible/adhoc/test.yml
+- import_playbook: vendor/stackhpc/ansible-slurm-appliance/ansible/adhoc/hpctests.yml