Merge pull request #10 from stackhpc/update/2022-9-29

Updated slurm app with fixed monitoring and home, state volumes

Author: m-bull

group_vars/cluster.yml

@@ -1,11 +1,6 @@
-# Convert the variable supplied by the portal into the one expected by the Slurm appliance
-update_enable: "{{ cluster_upgrade_system_packages | default('false') | bool }}"
-# The update logs are written on the Ansible controller
-# In CaaS, the Ansible controller is an ephemeral AWX pod, so all that matters is that
-# this is a location that is writable by the container user
-update_log_path: "{{ playbook_dir }}/.tmp/logs/{{ inventory_hostname }}-updates.log"
-# Same for the hpctests output directory
-hpctests_outdir: "{{ playbook_dir }}/.tmp/hpctests"
+# Account for the fact we are running outside of the expected environment system:
+# NB: this only works for playbooks in ansible/*, not in ansible/adhoc!
+appliances_repository_root: "{{ playbook_dir }}/../"
 
 # Read the secrets from the Ansible local facts on the control host
 vault_azimuth_user_password: "{{ hostvars[groups['control'][0]].ansible_local.openhpc_secrets.vault_azimuth_user_password }}"

group_vars/control.yml

@@ -0,0 +1,2 @@
+# Define path for persistent state on control node volume:
+appliances_state_dir: /var/lib/state

group_vars/filebeat.yml

@@ -0,0 +1,6 @@
+# Skip plotting pingpong as matplotlib not in runner environment
+hpctests_pingpong_plot: false
+
+# In CaaS, the Ansible controller is an ephemeral AWX pod, so all that matters is that
+# this is a location that is writable by the container user
+hpctests_outdir: "{{ playbook_dir }}/.tmp/hpctests"

group_vars/hpctests.yml

@@ -16,3 +16,8 @@ terraform_project_path: "{{ playbook_dir }}/terraform"
 
 terraform_state: "{{ cluster_state | default('present') }}"
 cluster_ssh_user: rocky
+
+state_volume_size: 150 # GB
+
+state_volume_device_path: "{{ cluster_state_volume_device_path | default('/dev/vdb') }}"
+home_volume_device_path: "{{ cluster_home_volume_device_path | default('/dev/vdc') }}"

group_vars/opendistro.yml

@@ -3,10 +3,10 @@ roles:
   - src: stackhpc.nfs
     version: v21.2.1
   - src: https://github.com/stackhpc/ansible-role-openhpc.git
-    version: v0.12.0
+    version: v0.16.0
     name: stackhpc.openhpc
   - src: https://github.com/stackhpc/ansible-node-exporter.git
-    version: support-rhel-clones
+    version: feature/no-install
     name: cloudalchemy.node_exporter
   - src: cloudalchemy.blackbox-exporter
     version: 1.0.0
@@ -15,27 +15,30 @@ roles:
     name: cloudalchemy.prometheus
   - src: cloudalchemy.alertmanager
     version: 0.19.1
-  - src: cloudalchemy.grafana
-    version: 0.18.0
-  - src: geerlingguy.mysql
-    version: 3.3.2
+  - src: https://github.com/stackhpc/ansible-grafana.git
+    name: cloudalchemy.grafana
+    version: service-state
   - src: jriguera.configdrive
+  # No versions available
   - src: https://github.com/OSC/ood-ansible.git
     name: osc.ood
     version: v2.0.5
 
 collections:
+  - name: containers.podman
+  - name: community.grafana
+  - name: https://github.com/stackhpc/ansible_collection_slurm_openstack_tools
+    type: git
+    version: v0.2.0
   - name: ansible.posix
   - name: ansible.netcommon
   - name: community.general
     version: 4.5.0  # https://github.com/ansible-collections/community.general/pull/4281
-  - name: community.grafana
+
   - name: community.mysql
-  - name: containers.podman
+
   - name: openstack.cloud
   - name: https://github.com/stackhpc/ansible-collection-terraform
     type: git
     version: ae1dc46a9d266bcdc6e79a6e290edbb080596f7f
-  - name: https://github.com/stackhpc/ansible_collection_slurm_openstack_tools
-    type: git
-    version: v0.1.0
+    

group_vars/openstack.yml

@@ -18,6 +18,12 @@ cluster_groups_required:
   mysql:         [control]
   update:        [cluster]
   basic_users:   [cluster]
+  fail2ban:      [login]
+  firewalld:     [fail2ban]
+  # ignore these for the moment:
+  #etc_hosts:     []
+  # cloud_init:   [etc_hosts]
+  systemd:        [opendistro, grafana, control, prometheus]
 
 # These are the additional groups required for monitoring (see everything layout)
 cluster_groups_monitoring:

requirements.yml

@@ -69,6 +69,22 @@ resource "openstack_networking_secgroup_rule_v2" "secgroup_slurm_login_rule_ingr
   security_group_id = "${openstack_networking_secgroup_v2.secgroup_slurm_login.id}"
 }
 
+#####
+##### Volumes
+#####
+resource "openstack_blockstorage_volume_v3" "state" {
+    name = "{{ cluster_name }}-state"
+    description = "State for control node"
+    size = "{{ state_volume_size }}"
+}
+
+resource "openstack_blockstorage_volume_v3" "home" {
+    name = "{{ cluster_name }}-home"
+    description = "Home for control node"
+    size = "{{ home_volme_size }}"
+}
+
+
 #####
 ##### Cluster nodes
 #####
@@ -111,12 +127,50 @@ resource "openstack_compute_instance_v2" "control" {
     name = "{{ cluster_network }}"
   }
   security_groups = ["${openstack_networking_secgroup_v2.secgroup_slurm_cluster.name}"]
-  # Use cloud-init to inject the SSH keys
+
+  # root device:
+  block_device {
+      uuid = "{{ cluster_image }}"
+      source_type  = "image"
+      destination_type = "local"
+      boot_index = 0
+      delete_on_termination = true
+  }
+
+  # state volume:
+  block_device {
+      destination_type = "volume"
+      source_type  = "volume"
+      boot_index = -1
+      uuid = openstack_blockstorage_volume_v3.state.id
+  }
+
+  # home volume:
+  block_device {
+      destination_type = "volume"
+      source_type  = "volume"
+      boot_index = -1
+      uuid = openstack_blockstorage_volume_v3.home.id
+  }
+
+  # Use cloud-init to a) inject SSH keys b) configure volumes
   user_data = <<-EOF
     #cloud-config
     ssh_authorized_keys:
       - {{ cluster_deploy_ssh_public_key }}
       - {{ cluster_user_ssh_public_key }}
+    fs_setup:
+      - label: state
+        filesystem: ext4
+        device: {{ state_volume_device_path }}
+        partition: auto
+      - label: home
+        filesystem: ext4
+        device: {{ home_volume_device_path }}
+        partition: auto
+    mounts:
+        - [LABEL=state, /var/lib/state]
+        - [LABEL=home, /exports/home, auto, "x-systemd.required-by=nfs-server.service,x-systemd.before=nfs-server.service"]
   EOF
 }
 

roles/cluster_infra/defaults/main.yml

@@ -40,7 +40,7 @@
   gather_facts: false
   tasks:
     - name: Set up Ansible user
-      user: "{{ appliances_local_users_ansible_user }}"
+      user: "{{ (appliances_local_users_default | selectattr('user.name', 'eq', appliances_local_users_ansible_user_name))[0]['user'] }}"
       become_method: "sudo"
       # Need to change working directory otherwise we try to switch back to non-existent directory.
       become_flags: '-i'