Merge branch 'main' into feat/isolated-env-2

Author: sjpb

ansible/roles/nhc/templates/nhc.conf.j2

@@ -4,7 +4,9 @@
 * || HOSTNAME="$HOSTNAME_S"
 
 ## Filesystem checks
-{% for mount in ansible_mounts %}
+{% for mount in ansible_mounts | rejectattr('mount', 'eq', '/efi') %}
+{# /efi is mounted both directly and via systemd1 autofs, which NHC can't cope with #}
+{# use `awk '{print $5 " " $10 " " $4 " " $9}' /proc/self/mountinfo | sort -k1` to check that is the only case #}
 {% set mount_mode = 'rw' if 'rw' in mount.options.split(',') else 'ro' %}
 {{ ansible_fqdn }} || check_fs_mount_{{ mount_mode }} -t "{{ mount.fstype }}" -s "{{ mount.device }}" -f "{{ mount.mount }}"
 {% endfor %}

ansible/site.yml

@@ -27,7 +27,6 @@
 - import_playbook: slurm.yml
 - import_playbook: portal.yml
 - import_playbook: monitoring.yml
-- import_playbook: final.yml
 
 - name: Run post.yml hook
   vars:
@@ -37,4 +36,6 @@
   import_playbook: "{{ hook_path if hook_path | exists else 'noop.yml' }}"
   when: hook_path | exists
 
+- import_playbook: final.yml
+
 ...

environments/skeleton/{{cookiecutter.environment}}/tofu/additional.tf

@@ -0,0 +1,68 @@
+module "additional" {
+  source = "./node_group"
+
+  for_each = var.additional_nodegroups
+
+  # must be set for group:
+  nodes = each.value.nodes
+  flavor = each.value.flavor
+
+  # always taken from top-level value:
+  cluster_name = var.cluster_name
+  cluster_domain_suffix = var.cluster_domain_suffix
+  key_pair = var.key_pair
+  environment_root = var.environment_root
+  
+  # can be set for group, defaults to top-level value:
+  image_id = lookup(each.value, "image_id", var.cluster_image_id)
+  vnic_types = lookup(each.value, "vnic_types", var.vnic_types)
+  volume_backed_instances = lookup(each.value, "volume_backed_instances", var.volume_backed_instances)
+  root_volume_size = lookup(each.value, "root_volume_size", var.root_volume_size)
+  root_volume_type = lookup(each.value, "root_volume_type", var.root_volume_type)
+  gateway_ip = lookup(each.value, "gateway_ip", var.gateway_ip)
+  nodename_template = lookup(each.value, "nodename_template", var.cluster_nodename_template)
+  
+  # optionally set for group:
+  networks = concat(var.cluster_networks, lookup(each.value, "extra_networks", []))
+  # here null means "use module var default"
+  extra_volumes = lookup(each.value, "extra_volumes", null)
+  fip_addresses = lookup(each.value, "fip_addresses", null)
+  fip_network = lookup(each.value, "fip_network", null)
+  match_ironic_node = lookup(each.value, "match_ironic_node", null)
+  availability_zone = lookup(each.value, "availability_zone", null)
+  ip_addresses = lookup(each.value, "ip_addresses", null)
+  security_group_ids = lookup(each.value, "security_group_ids", [for o in data.openstack_networking_secgroup_v2.nonlogin: o.id])
+
+  # can't be set for additional nodes
+  compute_init_enable = []
+  ignore_image_changes = false
+
+  # computed
+  # not using openstack_compute_instance_v2.control.access_ip_v4 to avoid
+  # updates to node metadata on deletion/recreation of the control node:
+  control_address = openstack_networking_port_v2.control[var.cluster_networks[0].network].all_fixed_ips[0]
+  baremetal_nodes = data.external.baremetal_nodes.result
+
+  # input dict validation:
+  group_name = each.key
+  group_keys = keys(each.value)
+  allowed_keys = [
+    "nodes",
+    "flavor",
+    "image_id",
+    "extra_networks",
+    "vnic_types",
+    "volume_backed_instances",
+    "root_volume_size",
+    "root_volume_type",
+    "extra_volumes",
+    "fip_addresses",
+    "fip_network",
+    "match_ironic_node",
+    "availability_zone",
+    "ip_addresses",
+    "gateway_ip",
+    "nodename_template",
+    "security_group_ids",
+  ]
+}

environments/skeleton/{{cookiecutter.environment}}/tofu/control.tf

@@ -4,7 +4,7 @@ locals {
     [for v in data.openstack_blockstorage_volume_v3.state: v],
     [for v in data.openstack_blockstorage_volume_v3.home: v]
   )
-  nodename = templatestring(
+  control_fqdn = templatestring(
     var.cluster_nodename_template,
     {
       node = "control",
@@ -38,7 +38,7 @@ resource "openstack_networking_port_v2" "control" {
 
 resource "openstack_compute_instance_v2" "control" {
 
-  name = split(".", local.nodename)[0]
+  name = split(".", local.control_fqdn)[0]
   image_id = var.cluster_image_id
   flavor_name = var.control_node_flavor
   key_pair = var.key_pair
@@ -80,7 +80,7 @@ resource "openstack_compute_instance_v2" "control" {
 
   user_data = <<-EOF
     #cloud-config
-    fqdn: ${local.nodename}
+    fqdn: ${local.control_fqdn}
     
     bootcmd:
       %{for volume in local.control_volumes}

environments/skeleton/{{cookiecutter.environment}}/tofu/inventory.tf

@@ -2,10 +2,12 @@ resource "local_file" "hosts" {
   content  = templatefile("${path.module}/inventory.tpl",
                           {
                             "cluster_name": var.cluster_name,
-                            "cluster_domain_suffix": var.cluster_domain_suffix,
+                            "cluster_domain_suffix": var.cluster_domain_suffix
                             "control": openstack_compute_instance_v2.control
+                            "control_fqdn": local.control_fqdn
                             "login_groups": module.login
                             "compute_groups": module.compute
+                            "additional_groups": module.additional
                             "state_dir": var.state_dir
                             "cluster_home_volume": var.home_volume_provisioning != "none"
                           },

environments/skeleton/{{cookiecutter.environment}}/tofu/inventory.tpl

@@ -11,18 +11,21 @@ control:
             ansible_host: ${control.access_ip_v4}
             instance_id: ${control.id}
             networks: ${jsonencode({for n in control.network: n.name => {"fixed_ip_v4": n.fixed_ip_v4, "fixed_ip_v6": n.fixed_ip_v6}})}
+            node_fqdn: ${control_fqdn}
     vars:
         appliances_state_dir: ${state_dir} # NB needs to be set on group not host otherwise it is ignored in packer build!
 
+# --- login nodes ---
 %{ for group_name in keys(login_groups) ~}
 ${cluster_name}_${group_name}:
     hosts:
-%{ for node in login_groups[group_name]["compute_instances"] ~}
+%{ for nodename, node in login_groups[group_name]["compute_instances"] ~}
         ${ node.name }:
             ansible_host: ${node.access_ip_v4}
             instance_id: ${ node.id }
             image_id: ${ node.image_id }
             networks: ${jsonencode({for n in node.network: n.name => {"fixed_ip_v4": n.fixed_ip_v4, "fixed_ip_v6": n.fixed_ip_v6}})}
+            node_fqdn: ${login_groups[group_name]["fqdns"][nodename]}
 %{ endfor ~}
 %{ endfor ~}
 
@@ -32,22 +35,51 @@ login:
         ${cluster_name}_${group_name}:
 %{ endfor ~}
 
+# --- compute nodes ---
 %{ for group_name in keys(compute_groups) ~}
 ${cluster_name}_${group_name}:
     hosts:
-%{ for node in compute_groups[group_name]["compute_instances"] ~}
+%{ for nodename, node in compute_groups[group_name]["compute_instances"] ~}
         ${ node.name }:
             ansible_host: ${node.access_ip_v4}
             instance_id: ${ node.id }
             networks: ${jsonencode({for n in node.network: n.name => {"fixed_ip_v4": n.fixed_ip_v4, "fixed_ip_v6": n.fixed_ip_v6}})}
+            node_fqdn: ${compute_groups[group_name]["fqdns"][nodename]}
 %{ endfor ~}
     vars:
         # NB: this is the target image, not necessarily what is provisioned
         image_id: ${compute_groups[group_name]["image_id"]}
+
+${group_name}:
+    children:
+        ${cluster_name}_${group_name}:
+
 %{ endfor ~}
 
 compute:
     children:
 %{ for group_name in keys(compute_groups) ~}
         ${cluster_name}_${group_name}:
 %{ endfor ~}
+
+# --- additional nodes ---
+%{ for group_name in keys(additional_groups) ~}
+${cluster_name}_${group_name}:
+    hosts:
+%{ for nodename, node in additional_groups[group_name]["compute_instances"] ~}
+        ${ node.name }:
+            ansible_host: ${node.access_ip_v4}
+            instance_id: ${ node.id }
+            networks: ${jsonencode({for n in node.network: n.name => {"fixed_ip_v4": n.fixed_ip_v4, "fixed_ip_v6": n.fixed_ip_v6}})}
+            node_fqdn: ${additional_groups[group_name]["fqdns"][nodename]}
+%{ endfor ~}
+${group_name}:
+    children:
+        ${cluster_name}_${group_name}:
+
+%{ endfor ~}
+additional:
+    children:
+%{ for group_name in keys(additional_groups) ~}
+        ${cluster_name}_${group_name}:
+%{ endfor ~}

environments/skeleton/{{cookiecutter.environment}}/tofu/login.tf

@@ -22,7 +22,7 @@ module "login" {
   gateway_ip = lookup(each.value, "gateway_ip", var.gateway_ip)
   nodename_template = lookup(each.value, "nodename_template", var.cluster_nodename_template)
 
-  # optionally set for group
+  # optionally set for group:
   networks = concat(var.cluster_networks, lookup(each.value, "extra_networks", []))
   # here null means "use module var default"
   extra_volumes = lookup(each.value, "extra_volumes", null)

environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/nodes.tf

@@ -13,8 +13,8 @@ locals {
   # Workaround for lifecycle meta-argument only taking static values
   compute_instances = var.ignore_image_changes ? openstack_compute_instance_v2.compute_fixed_image : openstack_compute_instance_v2.compute
 
-  # Define nodenames here to avoid repetition
-  nodenames = {
+  # Define fully qualified nodenames here to avoid repetition
+  fqdns = {
     for n in var.nodes: n => templatestring(
       var.nodename_template,
       {
@@ -74,7 +74,7 @@ resource "openstack_compute_instance_v2" "compute_fixed_image" {
 
   for_each = var.ignore_image_changes ? toset(var.nodes) : []
 
-  name = split(".", local.nodenames[each.key])[0]
+  name = split(".", local.fqdns[each.key])[0]
   image_id = var.image_id
   flavor_name = var.flavor
   key_pair = var.key_pair
@@ -112,7 +112,7 @@ resource "openstack_compute_instance_v2" "compute_fixed_image" {
 
   user_data = <<-EOF
     #cloud-config
-    fqdn: ${local.nodenames[each.key]}
+    fqdn: ${local.fqdns[each.key]}
   EOF
 
   availability_zone = var.match_ironic_node ? "${var.availability_zone}::${var.baremetal_nodes[each.key]}" : null
@@ -129,7 +129,7 @@ resource "openstack_compute_instance_v2" "compute" {
 
   for_each = var.ignore_image_changes ? [] : toset(var.nodes)
 
-  name = split(".", local.nodenames[each.key])[0]
+  name = split(".", local.fqdns[each.key])[0]
   image_id = var.image_id
   flavor_name = var.flavor
   key_pair = var.key_pair
@@ -167,7 +167,7 @@ resource "openstack_compute_instance_v2" "compute" {
 
   user_data = <<-EOF
     #cloud-config
-    fqdn: ${local.nodenames[each.key]}
+    fqdn: ${local.fqdns[each.key]}
   EOF
 
   availability_zone = var.match_ironic_node ? "${var.availability_zone}::${var.baremetal_nodes[each.key]}" : null
@@ -183,9 +183,13 @@ resource "openstack_networking_floatingip_associate_v2" "fip" {
 }
 
 output "compute_instances" {
-    value = local.compute_instances
+  value = local.compute_instances
 }
 
 output "image_id" {
   value = var.image_id
 }
+
+output "fqdns" {
+  value = local.fqdns
+}

environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/variables.tf

@@ -72,7 +72,8 @@ variable "extra_volumes" {
 }
 
 variable "security_group_ids" {
-    type = list
+    type = list(string)
+    nullable = false
 }
 
 variable "control_address" {

environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf

@@ -125,11 +125,38 @@ variable "compute" {
             availability_zone: Name of availability zone - ignored unless match_ironic_node is true (default: "nova")
             gateway_ip: Address to add default route via
             nodename_template: Overrides variable cluster_nodename_template
+
+        Nodes are added to the following inventory groups:
+        - $group_name
+        - $cluster_name + '_' + $group_name - this is used for the stackhpc.openhpc role
+        - 'compute'
     EOF
 
     type = any # can't do any better; TF type constraints can't cope with heterogeneous inner mappings
 }
 
+variable "additional_nodegroups" {
+    default = {}
+    description = <<-EOF
+        Mapping defining homogenous groups of nodes for arbitrary purposes.
+        These nodes are not in the compute or login inventory groups so they
+        will not run slurmd.
+
+        Keys are names of groups.
+        Values are a mapping as for the "login" variable, with the addition of
+        the optional entry:
+        
+            security_group_ids: List of strings giving IDs of security groups
+                                to apply. If not specified the groups from the
+                                variable nonlogin_security_groups are applied.
+
+        Nodes are added to the following inventory groups:
+        - $group_name
+        - $cluster_name + '_' + $group_name
+        - 'additional'
+    EOF
+}
+
 variable "environment_root" {
     type = string
     description = "Path to environment root, automatically set by activate script"