Merge pull request #11 from stackhpc/feature/image-patch

Make patch image-based

Author: sjpb

group_vars/cluster.yml

@@ -17,3 +17,5 @@ appliances_local_users_podman_enable: "{{ groups.get('podman', []) | length > 0
 # The server name for Open OnDemand depends on whether Zenith is enabled or not
 openondemand_servername_default: "{{ hostvars[groups['openstack'][0]].cluster_floating_ip_address | replace('.', '-') ~ '.sslip.io' }}"
 openondemand_servername: "{{ zenith_fqdn_ood | default(openondemand_servername_default) }}"
+
+appliances_state_dir: /var/lib/state

group_vars/control.yml

@@ -1,2 +1,17 @@
 # Use the IP address instead
-nfs_server_default: "{{ hostvars[groups['control'] | first ].ansible_default_ipv4.address }}"
+nfs_server: "{{ hostvars[groups['control'] | first ].ansible_default_ipv4.address }}"
+
+nfs_configurations:
+  - comment: Export /exports/home from Slurm control node as /home
+    nfs_enable:
+        server:  "{{ inventory_hostname in groups['control'] }}"
+        clients: "{{ inventory_hostname in groups['cluster'] and inventory_hostname not in groups['control'] }}"
+    nfs_export: "/exports/home" # assumes skeleton TF is being used
+    nfs_client_mnt_point: "/home"
+  - comment: Export /var/lib/state from Slurm control node to OOD
+    nfs_enable:
+        server:  "{{ inventory_hostname in groups['control'] }}"
+        clients: "{{ inventory_hostname in groups['openondemand'] }}"
+    nfs_export: "{{ appliances_state_dir }}"
+    nfs_client_mnt_point: "{{ appliances_state_dir }}"
+    nfs_client_mnt_options: "x-systemd.required-by=zenith-ood.service,x-systemd.before=zenith-ood.service"

group_vars/nfs.yml

@@ -73,5 +73,3 @@ _opeonondemand_unset_auth: '    RequestHeader unset Authorization'
 
 # Fix grafana proxying for basic auth if anonymous grafana access enabled:
 openondemand_node_proxy_directives: "{{ _opeonondemand_unset_auth if (openondemand_auth == 'basic_pam' and 'openondemand_host_regex' and 'grafana' in groups and hostvars[groups['grafana'][0]]._grafana_auth_is_anonymous) else '' }}"
-
-

group_vars/openondemand.yml

@@ -40,5 +40,5 @@ collections:
   - name: openstack.cloud
   - name: https://github.com/stackhpc/ansible-collection-terraform
     type: git
-    version: ae1dc46a9d266bcdc6e79a6e290edbb080596f7f
+    version: 75fb75132bbc77e3e78a05ba674458131da2b1dd
 

requirements.yml

@@ -1,3 +1,9 @@
+- debug:
+    msg: |
+      terraform_backend_type: {{ terraform_backend_type }}
+      terraform_state: {{ terraform_state }}
+      cluster_upgrade_system_packages: {{ cluster_upgrade_system_packages | default('undefined') }}
+
 # We need to convert the floating IP id to an address for Terraform
 - name: Look up floating IP
   include_role:
@@ -10,11 +16,42 @@
   set_fact:
     cluster_floating_ip_address: "{{ os_floating_ip_info.floating_ip_address }}"
 
+- name: Install Terraform binary
+  include_role:
+    name: stackhpc.terraform.install
+
 - name: Make Terraform project directory
   file:
     path: "{{ terraform_project_path }}"
     state: directory
 
+- name: Write backend configuration
+  copy:
+    content: |
+      terraform {
+        backend "{{ terraform_backend_type }}" { }
+      }
+    dest: "{{ terraform_project_path }}/backend.tf"
+
+# Patching in this appliance is implemented as a switch to a new base image
+# So unless explicitly patching, we want to use the same image as last time
+# To do this, we query the previous Terraform state before updating
+- block:
+    - name: Get previous Terraform state
+      stackhpc.terraform.terraform_output:
+        binary_path: "{{ terraform_binary_path }}"
+        project_path: "{{ terraform_project_path }}"
+        backend_config: "{{ terraform_backend_config }}"
+      register: cluster_infra_terraform_output
+
+    - name: Extract image from Terraform state
+      set_fact:
+        cluster_previous_image: "{{ cluster_infra_terraform_output.outputs.cluster_image.value }}"
+      when: '"cluster_image" in cluster_infra_terraform_output.outputs'
+  when:
+    - terraform_state == "present"
+    - cluster_upgrade_system_packages is not defined or not cluster_upgrade_system_packages
+
 - name: Template Terraform files into project directory
   template:
     src: "{{ item }}.j2"
@@ -24,10 +61,6 @@
     - providers.tf
     - resources.tf
 
-- name: Install Terraform binary
-  include_role:
-    name: stackhpc.terraform.install
-
 - name: Provision infrastructure
   include_role:
     name: stackhpc.terraform.infra

roles/cluster_infra/tasks/main.yml

@@ -36,3 +36,8 @@ output "cluster_nodes" {
     ]
   )
 }
+
+output "cluster_image" {
+  description = "The id of the image used to build the cluster nodes"
+  value       = "{{ cluster_previous_image | default(cluster_image) }}"
+}

roles/cluster_infra/templates/outputs.tf.j2

@@ -91,7 +91,7 @@ resource "openstack_blockstorage_volume_v3" "home" {
 
 resource "openstack_compute_instance_v2" "login" {
   name      = "{{ cluster_name }}-login-0"
-  image_id  = "{{ cluster_image }}"
+  image_id  = "{{ cluster_previous_image | default(cluster_image) }}"
   {% if login_flavor_name is defined %}
   flavor_name = "{{ login_flavor_name }}"
   {% else %}
@@ -116,7 +116,7 @@ resource "openstack_compute_instance_v2" "login" {
 
 resource "openstack_compute_instance_v2" "control" {
   name      = "{{ cluster_name }}-control-0"
-  image_id  = "{{ cluster_image }}"
+  image_id  = "{{ cluster_previous_image | default(cluster_image) }}"
   {% if control_flavor_name is defined %}
   flavor_name = "{{ control_flavor_name }}"
   {% else %}
@@ -169,7 +169,7 @@ resource "openstack_compute_instance_v2" "control" {
         device: /dev/{{ block_device_prefix }}c
         partition: auto
     mounts:
-        - [LABEL=state, /var/lib/state]
+        - [LABEL=state, /var/lib/state, auto, "x-systemd.required-by=nfs-server.service,x-systemd.before=nfs-server.service"]
         - [LABEL=home, /exports/home, auto, "x-systemd.required-by=nfs-server.service,x-systemd.before=nfs-server.service"]
   EOF
 }
@@ -178,7 +178,7 @@ resource "openstack_compute_instance_v2" "compute" {
   count = {{ compute_count }}
 
   name      = "{{ cluster_name }}-compute-${count.index}"
-  image_id  = "{{ cluster_image }}"
+  image_id  = "{{ cluster_previous_image | default(cluster_image) }}"
   flavor_id = "{{ compute_flavor }}"
 
   network {

roles/cluster_infra/templates/resources.tf.j2

@@ -0,0 +1,33 @@
+---
+
+- name: Ensure hostkeys directory exists on persistent storage
+  file:
+    path: "{{ appliances_state_dir }}/hostkeys/{{ inventory_hostname }}"
+    state: directory
+    owner: root
+    group: root
+    mode: 0600
+
+- name: Copy hostkeys from persistent storage
+  # won't fail if no keys are in persistent storage
+  copy:
+    src: "{{ appliances_state_dir }}/hostkeys/{{ inventory_hostname }}/"
+    dest: /etc/ssh/
+    remote_src: true
+
+- name: Find hostkeys
+  find:
+    path: /etc/ssh/
+    patterns: ssh_host_*_key*
+  register: _find_ssh_keys
+
+- name: Persist hostkeys
+  copy:
+    dest: "{{ appliances_state_dir }}/hostkeys/{{ inventory_hostname }}/"
+    src: "{{ item }}"
+    remote_src: true
+    mode: preserve
+  loop: "{{ _find_ssh_keys.files | map(attribute='path') }}"
+
+- meta: reset_connection
+

roles/persist_hostkeys/tasks/main.yml

@@ -1,6 +1,6 @@
 ---
 
-- name: Check if OpenHPC secrets exist
+- name: Check if OpenHPC secrets exist in persistent storage
   stat:
     path: "{{ appliances_state_dir }}/ansible.facts.d/openhpc_secrets.fact"
   register: openhpc_secrets_stat
@@ -32,4 +32,4 @@
 
 - name: Read facts
   ansible.builtin.setup:
-    filter: ansible_local
+    filter: ansible_local