From 42249267ae7e5c5261ae89a102cdd7c3b3005781 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 4 Apr 2025 10:43:04 +0200 Subject: [PATCH 01/14] Support fixed IP addresses for control node The variable control_ip_address was documented but not implemented. Since we support multiple networks, change it to control_ip_addresses and implement it. Closes #642. --- docs/production.md | 6 +++--- .../skeleton/{{cookiecutter.environment}}/tofu/control.tf | 3 ++- .../skeleton/{{cookiecutter.environment}}/tofu/variables.tf | 6 ++++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/production.md b/docs/production.md index 06e7d374e..3917b8fcb 100644 --- a/docs/production.md +++ b/docs/production.md @@ -141,13 +141,13 @@ and referenced from the `site` and `production` environments, e.g.: resource "openstack_networking_port_v2" "control" { ... fixed_ip { - subnet_id = data.openstack_networking_subnet_v2.cluster_subnet.id - ip_address = var.control_ip_address + subnet_id = data.openstack_networking_subnet_v2.cluster_subnet.id + ip_address = (var.control_ip_addresses != {})? var.control_ip_addresses[each.key]: null } } ``` - Note the variable `control_ip_address` is new. + Note the variable `control_ip_addresses` is new. Using fixed IPs will require either using admin credentials or policy changes. diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/control.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/control.tf index 412a0176c..58bbf14bf 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/control.tf +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/control.tf @@ -24,7 +24,8 @@ resource "openstack_networking_port_v2" "control" { admin_state_up = "true" fixed_ip { - subnet_id = data.openstack_networking_subnet_v2.cluster_subnet[each.key].id + subnet_id = data.openstack_networking_subnet_v2.cluster_subnet[each.key].id + ip_address = (var.control_ip_addresses != {})? var.control_ip_addresses[each.key]: null } no_security_groups = lookup(each.value, "no_security_groups", false) diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf index eec5d6848..3518329ba 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf @@ -24,6 +24,12 @@ variable "key_pair" { description = "Name of an existing keypair in OpenStack" } +variable "control_ip_addresses" { + type = map(string) + description = "Mapping of networks to fixed IP addresses used by control node" + default = {} +} + variable "control_node_flavor" { type = string description = "Flavor name for control node" From d5972b381847a06de7348fee24fc92eb1f232352 Mon Sep 17 00:00:00 2001 From: Steve Brasier <33413598+sjpb@users.noreply.github.com> Date: Thu, 29 May 2025 15:22:29 +0100 Subject: [PATCH 02/14] Update environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf --- .../skeleton/{{cookiecutter.environment}}/tofu/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf index 3518329ba..1d38541f1 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf @@ -26,7 +26,7 @@ variable "key_pair" { variable "control_ip_addresses" { type = map(string) - description = "Mapping of networks to fixed IP addresses used by control node" + description = "Mapping of fixed IP addresses for control node, keyed by network name" default = {} } From 9c3bb9695e84e9c06c96f36db5c4e849759fc1f7 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Thu, 29 May 2025 14:26:31 +0000 Subject: [PATCH 03/14] tf style tweaks --- .../skeleton/{{cookiecutter.environment}}/tofu/control.tf | 2 +- .../skeleton/{{cookiecutter.environment}}/tofu/variables.tf | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/control.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/control.tf index 58bbf14bf..cc3516883 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/control.tf +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/control.tf @@ -25,7 +25,7 @@ resource "openstack_networking_port_v2" "control" { fixed_ip { subnet_id = data.openstack_networking_subnet_v2.cluster_subnet[each.key].id - ip_address = (var.control_ip_addresses != {})? var.control_ip_addresses[each.key]: null + ip_address = var.control_ip_addresses != {} ? var.control_ip_addresses[each.key] : null } no_security_groups = lookup(each.value, "no_security_groups", false) diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf index 1d38541f1..09702b789 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf @@ -26,7 +26,10 @@ variable "key_pair" { variable "control_ip_addresses" { type = map(string) - description = "Mapping of fixed IP addresses for control node, keyed by network name" + description = <<-EOT + Mapping of fixed IP addresses for control node, keyed by network name. + The default means the cloud will select an address. + EOT default = {} } From d3acbe200fd5a959229072ef1dd89db23cc63fe9 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Thu, 29 May 2025 14:35:04 +0000 Subject: [PATCH 04/14] update production docs for control_ip_addresses --- docs/production.md | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/docs/production.md b/docs/production.md index 3917b8fcb..0c554562f 100644 --- a/docs/production.md +++ b/docs/production.md @@ -134,22 +134,9 @@ and referenced from the `site` and `production` environments, e.g.: - Consider whether having (read-only) access to Grafana without login is OK. If not, remove `grafana_auth_anonymous` in `environments/$ENV/inventory/group_vars/all/grafana.yml` -- Modify `environments/site/tofu/nodes.tf` to provide fixed IPs for at least - the control node, and (if not using FIPs) the login node(s): - - ``` - resource "openstack_networking_port_v2" "control" { - ... - fixed_ip { - subnet_id = data.openstack_networking_subnet_v2.cluster_subnet.id - ip_address = (var.control_ip_addresses != {})? var.control_ip_addresses[each.key]: null - } - } - ``` - - Note the variable `control_ip_addresses` is new. - - Using fixed IPs will require either using admin credentials or policy changes. +- If fixed IP(s) are required for the control node, set the OpenTofu variable + `control_ip_addresses`. This will require either using admin credentials or + policy changes. - If floating IPs are required for login nodes, modify the OpenTofu configurations appropriately. From bfa66abe99451156382fd0beca584f8fb81b717c Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Thu, 29 May 2025 18:49:03 +0000 Subject: [PATCH 05/14] fix control IP address logic --- .../skeleton/{{cookiecutter.environment}}/tofu/control.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/control.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/control.tf index cc3516883..7e2e51470 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/control.tf +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/control.tf @@ -25,7 +25,7 @@ resource "openstack_networking_port_v2" "control" { fixed_ip { subnet_id = data.openstack_networking_subnet_v2.cluster_subnet[each.key].id - ip_address = var.control_ip_addresses != {} ? var.control_ip_addresses[each.key] : null + ip_address = lookup(var.control_ip_addresses, each.key, null) } no_security_groups = lookup(each.value, "no_security_groups", false) From 9b5ed88b28fb2a7c5bc5277cd0c471740e00c3c1 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Thu, 29 May 2025 18:57:24 +0000 Subject: [PATCH 06/14] add validation for control_ip_addresses --- .../skeleton/{{cookiecutter.environment}}/tofu/variables.tf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf index 09702b789..2eb3fb407 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf @@ -31,6 +31,11 @@ variable "control_ip_addresses" { The default means the cloud will select an address. EOT default = {} + validation { + # check all keys are network names in cluster_networks + condition = length(setsubtract(keys(var.control_ip_addresses), [for n in var.cluster_networks: n.network])) == 0 + error_message = "keys in var.control_ip_addresses must match network names in var.cluster_networks" + } } variable "control_node_flavor" { From c97615f9c6167ced02f5abc29049db1a75e571b8 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Wed, 11 Jun 2025 08:41:42 +0000 Subject: [PATCH 07/14] remove fixed IPs from production docs - not standard process --- docs/production.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/production.md b/docs/production.md index 0c554562f..e52e9d180 100644 --- a/docs/production.md +++ b/docs/production.md @@ -134,10 +134,6 @@ and referenced from the `site` and `production` environments, e.g.: - Consider whether having (read-only) access to Grafana without login is OK. If not, remove `grafana_auth_anonymous` in `environments/$ENV/inventory/group_vars/all/grafana.yml` -- If fixed IP(s) are required for the control node, set the OpenTofu variable - `control_ip_addresses`. This will require either using admin credentials or - policy changes. - - If floating IPs are required for login nodes, modify the OpenTofu configurations appropriately. From 1afc5eaaf0945414d7e4409e50f19d46623e6273 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Wed, 11 Jun 2025 10:46:17 +0000 Subject: [PATCH 08/14] support ip_addresses fo all nodes --- .../tofu/compute.tf | 2 + .../tofu/login.tf | 2 + .../tofu/node_group/nodes.tf | 16 ++-- .../tofu/node_group/variables.tf | 21 +++++ .../tofu/variables.tf | 84 +++++++++++-------- 5 files changed, 83 insertions(+), 42 deletions(-) diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/compute.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/compute.tf index d1f2275b6..4e6186e35 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/compute.tf +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/compute.tf @@ -30,6 +30,7 @@ module "compute" { ignore_image_changes = lookup(each.value, "ignore_image_changes", null) match_ironic_node = lookup(each.value, "match_ironic_node", null) availability_zone = lookup(each.value, "availability_zone", null) + ip_addresses = lookup(each.value, "ip_addresses", null) # computed # not using openstack_compute_instance_v2.control.access_ip_v4 to avoid @@ -55,6 +56,7 @@ module "compute" { "extra_volumes", "match_ironic_node", "availability_zone", + "ip_addresses", "gateway_ip", "nodename_template", ] diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/login.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/login.tf index 7836d0db4..301822ef0 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/login.tf +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/login.tf @@ -30,6 +30,7 @@ module "login" { fip_network = lookup(each.value, "fip_network", null) match_ironic_node = lookup(each.value, "match_ironic_node", null) availability_zone = lookup(each.value, "availability_zone", null) + ip_addresses = lookup(each.value, "ip_addresses", null) # can't be set for login compute_init_enable = [] @@ -59,6 +60,7 @@ module "login" { "fip_network", "match_ironic_node", "availability_zone", + "ip_addresses", "gateway_ip", "nodename_template", ] diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/nodes.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/nodes.tf index cf79e6a52..5e8449381 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/nodes.tf +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/nodes.tf @@ -47,22 +47,26 @@ resource "openstack_compute_volume_attach_v2" "compute" { resource "openstack_networking_port_v2" "compute" { for_each = {for item in setproduct(var.nodes, var.networks): - "${item[0]}-${item[1].network}" => item[1] + "${item[0]}-${item[1].network}" => { + node_idx = index(var.nodes, item[0]) + net = item[1] + } } name = "${var.cluster_name}-${each.key}" - network_id = data.openstack_networking_network_v2.network[each.value.network].id + network_id = data.openstack_networking_network_v2.network[each.value.net.network].id admin_state_up = "true" fixed_ip { - subnet_id = data.openstack_networking_subnet_v2.subnet[each.value.network].id + subnet_id = data.openstack_networking_subnet_v2.subnet[each.value.net.network].id + ip_address = try(var.ip_addresses[each.value.net.network][each.value.node_idx], null) } - no_security_groups = lookup(each.value, "no_security_groups", false) - security_group_ids = lookup(each.value, "no_security_groups", false) ? [] : var.security_group_ids + no_security_groups = lookup(each.value.net, "no_security_groups", false) + security_group_ids = lookup(each.value.net, "no_security_groups", false) ? [] : var.security_group_ids binding { - vnic_type = lookup(var.vnic_types, each.value.network, "normal") + vnic_type = lookup(var.vnic_types, each.value.net.network, "normal") } } diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/variables.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/variables.tf index 33e047b1a..72c3f004d 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/variables.tf +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/variables.tf @@ -119,6 +119,27 @@ variable "fip_network" { nullable = false } +variable "ip_addresses" { + type = map(list(string)) + description = <<-EOT + Mapping of list of fixed IP addresses for nodes, keyed by network name, + in same order as nodes parameter. For any networks not specified here + the cloud will select addresses. + + NB: Changing IP addresses after deployment may hit terraform provider bugs. + EOT + default = {} + nullable = false + validation { + condition = length(setsubtract(keys(var.ip_addresses), var.networks[*].network)) == 0 + error_message = "Keys in ip_addresses for nodegroup \"${var.group_name}\" must match network names in var.cluster_networks" + } + validation { + condition = alltrue([for v in values(var.ip_addresses): length(v) == length(var.nodes)]) + error_message = "Values in ip_addresses for nodegroup \"${var.group_name}\" must be a list of the same length as var.nodes" + } +} + variable "match_ironic_node" { type = bool description = "Whether to launch instances on the Ironic node of the same name as each cluster node" diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf index 2eb3fb407..c17db6584 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf @@ -28,13 +28,15 @@ variable "control_ip_addresses" { type = map(string) description = <<-EOT Mapping of fixed IP addresses for control node, keyed by network name. - The default means the cloud will select an address. + For any networks not specified here the cloud will select an address. + + NB: Changing IP addresses after deployment may hit terraform provider bugs. EOT default = {} validation { # check all keys are network names in cluster_networks - condition = length(setsubtract(keys(var.control_ip_addresses), [for n in var.cluster_networks: n.network])) == 0 - error_message = "keys in var.control_ip_addresses must match network names in var.cluster_networks" + condition = length(setsubtract(keys(var.control_ip_addresses), var.cluster_networks[*].network)) == 0 + error_message = "Keys in var.control_ip_addresses must match network names in var.cluster_networks" } } @@ -44,39 +46,45 @@ variable "control_node_flavor" { } variable "login" { - type = any - description = <<-EOF - Mapping defining homogenous groups of login nodes. Multiple groups may - be useful for e.g. separating nodes for ssh and Open Ondemand usage, or - to define login nodes with different capabilities such as high-memory. + default = {} + description = <<-EOF + Mapping defining homogenous groups of login nodes. Multiple groups may + be useful for e.g. separating nodes for ssh and Open Ondemand usage, or + to define login nodes with different capabilities such as high-memory. - Keys are names of groups. - Values are a mapping as follows: + Keys are names of groups. + Values are a mapping as follows: + + Required: + nodes: List of node names + flavor: String flavor name + Optional: + image_id: Overrides variable cluster_image_id + extra_networks: List of mappings in same format as cluster_networks + vnic_types: Overrides variable vnic_types + volume_backed_instances: Overrides variable volume_backed_instances + root_volume_size: Overrides variable root_volume_size + extra_volumes: Mapping defining additional volumes to create and attach + Keys are unique volume name. + Values are a mapping with: + size: Size of volume in GB + **NB**: The order in /dev is not guaranteed to match the mapping + fip_addresses: List of addresses of floating IPs to associate with + nodes, in the same order as nodes parameter. The + floating IPs must already be allocated to the project. + fip_network: Name of network containing ports to attach FIPs to. Only + required if multiple networks are defined. + ip_addresses: Mapping of list of fixed IP addresses for nodes, keyed + by network name, in same order as nodes parameter. + For any networks not specified here the cloud will + select addresses. + match_ironic_node: Set true to launch instances on the Ironic node of the same name as each cluster node + availability_zone: Name of availability zone - ignored unless match_ironic_node is true (default: "nova") + gateway_ip: Address to add default route via + nodename_template: Overrides variable cluster_nodename_template + EOF - Required: - nodes: List of node names - flavor: String flavor name - Optional: - image_id: Overrides variable cluster_image_id - extra_networks: List of mappings in same format as cluster_networks - vnic_types: Overrides variable vnic_types - volume_backed_instances: Overrides variable volume_backed_instances - root_volume_size: Overrides variable root_volume_size - extra_volumes: Mapping defining additional volumes to create and attach - Keys are unique volume name. - Values are a mapping with: - size: Size of volume in GB - **NB**: The order in /dev is not guaranteed to match the mapping - fip_addresses: List of addresses of floating IPs to associate with nodes, - in the same order as nodes parameter. The floating IPs - must already be allocated to the project. - fip_network: Name of network containing ports to attach FIPs to. Only - required if multiple networks are defined. - match_ironic_node: Set true to launch instances on the Ironic node of the same name as each cluster node - availability_zone: Name of availability zone - ignored unless match_ironic_node is true (default: "nova") - gateway_ip: Address to add default route via - nodename_template: Overrides variable cluster_nodename_template - EOF + type = any } variable "cluster_image_id" { @@ -85,7 +93,7 @@ variable "cluster_image_id" { } variable "compute" { - + default = {} description = <<-EOF Mapping defining homogenous groups of compute nodes. Groups are used in Slurm partition definitions. @@ -109,12 +117,16 @@ variable "compute" { Values are a mapping with: size: Size of volume in GB **NB**: The order in /dev is not guaranteed to match the mapping + ip_addresses: Mapping of list of fixed IP addresses for nodes, keyed + by network name, in same order as nodes parameter. + For any networks not specified here the cloud will + select addresses. match_ironic_node: Set true to launch instances on the Ironic node of the same name as each cluster node availability_zone: Name of availability zone - ignored unless match_ironic_node is true (default: "nova") gateway_ip: Address to add default route via nodename_template: Overrides variable cluster_nodename_template EOF - default = {} + type = any # can't do any better; TF type constraints can't cope with heterogeneous inner mappings } From e1b171de3e775885c0e22a5eae989c6be976c5af Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Wed, 11 Jun 2025 10:48:20 +0000 Subject: [PATCH 09/14] make stackhpc tofu format consistent --- environments/.stackhpc/tofu/main.tf | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/environments/.stackhpc/tofu/main.tf b/environments/.stackhpc/tofu/main.tf index 8d78401bf..ad1549164 100644 --- a/environments/.stackhpc/tofu/main.tf +++ b/environments/.stackhpc/tofu/main.tf @@ -69,23 +69,23 @@ module "cluster" { control_node_flavor = var.control_node_flavor login = { - login: { - nodes: ["login-0"] - flavor: var.other_node_flavor + login = { + nodes = ["login-0"] + flavor = var.other_node_flavor } } compute = { - standard: { # NB: can't call this default! - nodes: ["compute-0", "compute-1"] - flavor: var.other_node_flavor - compute_init_enable: ["compute", "chrony", "etc_hosts", "nfs", "basic_users", "eessi", "tuned", "cacerts"] - ignore_image_changes: true + standard = { # NB: can't call this default! + nodes = ["compute-0", "compute-1"] + flavor = var.other_node_flavor + compute_init_enable = ["compute", "chrony", "etc_hosts", "nfs", "basic_users", "eessi", "tuned", "cacerts"] + ignore_image_changes = true } # Normally-empty partition for testing: - extra: { - nodes: [] - #nodes: ["extra-0", "extra-1"] - flavor: var.other_node_flavor + extra = { + nodes = [] + #nodes = ["extra-0", "extra-1"] + flavor = var.other_node_flavor } } From 25af907e99659d41901ecd8817d452f90d6d927f Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Wed, 11 Jun 2025 14:27:19 +0000 Subject: [PATCH 10/14] add support for additional_nodegroups --- .../tofu/inventory.tf | 1 + .../tofu/inventory.tpl | 28 +++++++++++++++++++ .../tofu/login.tf | 2 +- .../tofu/variables.tf | 22 +++++++++++++++ 4 files changed, 52 insertions(+), 1 deletion(-) diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/inventory.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/inventory.tf index 81ac46d6c..00fe3396d 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/inventory.tf +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/inventory.tf @@ -6,6 +6,7 @@ resource "local_file" "hosts" { "control": openstack_compute_instance_v2.control "login_groups": module.login "compute_groups": module.compute + "additional_groups": module.additional "state_dir": var.state_dir "cluster_home_volume": var.home_volume_provisioning != "none" }, diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/inventory.tpl b/environments/skeleton/{{cookiecutter.environment}}/tofu/inventory.tpl index bdc0b52ea..72d97b49d 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/inventory.tpl +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/inventory.tpl @@ -14,6 +14,7 @@ control: vars: appliances_state_dir: ${state_dir} # NB needs to be set on group not host otherwise it is ignored in packer build! +# --- login nodes --- %{ for group_name in keys(login_groups) ~} ${cluster_name}_${group_name}: hosts: @@ -32,6 +33,7 @@ login: ${cluster_name}_${group_name}: %{ endfor ~} +# --- compute nodes --- %{ for group_name in keys(compute_groups) ~} ${cluster_name}_${group_name}: hosts: @@ -44,6 +46,11 @@ ${cluster_name}_${group_name}: vars: # NB: this is the target image, not necessarily what is provisioned image_id: ${compute_groups[group_name]["image_id"]} + +${group_name}: + children: + ${cluster_name}_${group_name}: + %{ endfor ~} compute: @@ -51,3 +58,24 @@ compute: %{ for group_name in keys(compute_groups) ~} ${cluster_name}_${group_name}: %{ endfor ~} + +# --- additional nodes --- +%{ for group_name in keys(additional_groups) ~} +${cluster_name}_${group_name}: + hosts: +%{ for node in additional_groups[group_name]["compute_instances"] ~} + ${ node.name }: + ansible_host: ${node.access_ip_v4} + instance_id: ${ node.id } + networks: ${jsonencode({for n in node.network: n.name => {"fixed_ip_v4": n.fixed_ip_v4, "fixed_ip_v6": n.fixed_ip_v6}})} +%{ endfor ~} +${group_name}: + children: + ${cluster_name}_${group_name}: + +%{ endfor ~} +additional: + children: +%{ for group_name in keys(additional_groups) ~} + ${cluster_name}_${group_name}: +%{ endfor ~} diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/login.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/login.tf index 301822ef0..b8abe4ea3 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/login.tf +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/login.tf @@ -22,7 +22,7 @@ module "login" { gateway_ip = lookup(each.value, "gateway_ip", var.gateway_ip) nodename_template = lookup(each.value, "nodename_template", var.cluster_nodename_template) - # optionally set for group + # optionally set for group: networks = concat(var.cluster_networks, lookup(each.value, "extra_networks", [])) # here null means "use module var default" extra_volumes = lookup(each.value, "extra_volumes", null) diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf index c17db6584..4b7117960 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf @@ -125,11 +125,33 @@ variable "compute" { availability_zone: Name of availability zone - ignored unless match_ironic_node is true (default: "nova") gateway_ip: Address to add default route via nodename_template: Overrides variable cluster_nodename_template + + Nodes are added to the following inventory groups: + - $group_name + - $cluster_name + '_' + $group_name - this is used for the stackhpc.openhpc role + - 'compute' EOF type = any # can't do any better; TF type constraints can't cope with heterogeneous inner mappings } +variable "additional_nodegroups" { + default = {} + description = <<-EOF + Mapping defining homogenous groups of nodes for arbitrary purposes. + These nodes are not in the compute or login inventory groups so they + will not run slurmd. + + Keys are names of groups. + Values are a mapping as for the "login" variable. + + Nodes are added to the following inventory groups: + - $group_name + - $cluster_name + '_' + $group_name + - 'additional' + EOF +} + variable "environment_root" { type = string description = "Path to environment root, automatically set by activate script" From 54809f0b47dc2cb103d787b9a5d05ad8775da925 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Fri, 13 Jun 2025 11:02:21 +0000 Subject: [PATCH 11/14] add missing additional nodegroup file --- .../tofu/additional_nodegroups.tf | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 environments/skeleton/{{cookiecutter.environment}}/tofu/additional_nodegroups.tf diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/additional_nodegroups.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/additional_nodegroups.tf new file mode 100644 index 000000000..790cacf03 --- /dev/null +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/additional_nodegroups.tf @@ -0,0 +1,67 @@ +module "additional" { + source = "./node_group" + + for_each = var.additional_nodegroups + + # must be set for group: + nodes = each.value.nodes + flavor = each.value.flavor + + # always taken from top-level value: + cluster_name = var.cluster_name + cluster_domain_suffix = var.cluster_domain_suffix + key_pair = var.key_pair + environment_root = var.environment_root + + # can be set for group, defaults to top-level value: + image_id = lookup(each.value, "image_id", var.cluster_image_id) + vnic_types = lookup(each.value, "vnic_types", var.vnic_types) + volume_backed_instances = lookup(each.value, "volume_backed_instances", var.volume_backed_instances) + root_volume_size = lookup(each.value, "root_volume_size", var.root_volume_size) + root_volume_type = lookup(each.value, "root_volume_type", var.root_volume_type) + gateway_ip = lookup(each.value, "gateway_ip", var.gateway_ip) + nodename_template = lookup(each.value, "nodename_template", var.cluster_nodename_template) + + # optionally set for group: + networks = concat(var.cluster_networks, lookup(each.value, "extra_networks", [])) + # here null means "use module var default" + extra_volumes = lookup(each.value, "extra_volumes", null) + fip_addresses = lookup(each.value, "fip_addresses", null) + fip_network = lookup(each.value, "fip_network", null) + match_ironic_node = lookup(each.value, "match_ironic_node", null) + availability_zone = lookup(each.value, "availability_zone", null) + ip_addresses = lookup(each.value, "ip_addresses", null) + + # can't be set for additional nodes + compute_init_enable = [] + ignore_image_changes = false + + # computed + # not using openstack_compute_instance_v2.control.access_ip_v4 to avoid + # updates to node metadata on deletion/recreation of the control node: + control_address = openstack_networking_port_v2.control[var.cluster_networks[0].network].all_fixed_ips[0] + security_group_ids = [for o in data.openstack_networking_secgroup_v2.nonlogin: o.id] + baremetal_nodes = data.external.baremetal_nodes.result + + # input dict validation: + group_name = each.key + group_keys = keys(each.value) + allowed_keys = [ + "nodes", + "flavor", + "image_id", + "extra_networks", + "vnic_types", + "volume_backed_instances", + "root_volume_size", + "root_volume_type", + "extra_volumes", + "fip_addresses", + "fip_network", + "match_ironic_node", + "availability_zone", + "ip_addresses", + "gateway_ip", + "nodename_template", + ] +} From 998246dcb70317d23b3ab04ac4c1c0377e7e2ce5 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Fri, 13 Jun 2025 14:41:05 +0000 Subject: [PATCH 12/14] rename additional tf file for consistency --- .../tofu/{additional_nodegroups.tf => additional.tf} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename environments/skeleton/{{cookiecutter.environment}}/tofu/{additional_nodegroups.tf => additional.tf} (100%) diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/additional_nodegroups.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/additional.tf similarity index 100% rename from environments/skeleton/{{cookiecutter.environment}}/tofu/additional_nodegroups.tf rename to environments/skeleton/{{cookiecutter.environment}}/tofu/additional.tf From de861777ed32c49151314d476cfdf9794b814235 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Fri, 13 Jun 2025 14:54:26 +0000 Subject: [PATCH 13/14] support changing security groups for additional nodes --- .../{{cookiecutter.environment}}/tofu/additional.tf | 3 ++- .../tofu/node_group/variables.tf | 3 ++- .../{{cookiecutter.environment}}/tofu/variables.tf | 7 ++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/additional.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/additional.tf index 790cacf03..1f7c48ce5 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/additional.tf +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/additional.tf @@ -31,6 +31,7 @@ module "additional" { match_ironic_node = lookup(each.value, "match_ironic_node", null) availability_zone = lookup(each.value, "availability_zone", null) ip_addresses = lookup(each.value, "ip_addresses", null) + security_group_ids = lookup(each.value, "security_group_ids", [for o in data.openstack_networking_secgroup_v2.nonlogin: o.id]) # can't be set for additional nodes compute_init_enable = [] @@ -40,7 +41,6 @@ module "additional" { # not using openstack_compute_instance_v2.control.access_ip_v4 to avoid # updates to node metadata on deletion/recreation of the control node: control_address = openstack_networking_port_v2.control[var.cluster_networks[0].network].all_fixed_ips[0] - security_group_ids = [for o in data.openstack_networking_secgroup_v2.nonlogin: o.id] baremetal_nodes = data.external.baremetal_nodes.result # input dict validation: @@ -63,5 +63,6 @@ module "additional" { "ip_addresses", "gateway_ip", "nodename_template", + "security_group_ids", ] } diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/variables.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/variables.tf index 72c3f004d..44c862fe5 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/variables.tf +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/variables.tf @@ -72,7 +72,8 @@ variable "extra_volumes" { } variable "security_group_ids" { - type = list + type = list(string) + nullable = false } variable "control_address" { diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf index 4b7117960..592119c39 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf @@ -143,7 +143,12 @@ variable "additional_nodegroups" { will not run slurmd. Keys are names of groups. - Values are a mapping as for the "login" variable. + Values are a mapping as for the "login" variable, with the addition of + the optional entry: + + security_group_ids: List of strings giving IDs of security groups + to apply. If not specified the groups from the + variable nonlogin_security_groups are applied. Nodes are added to the following inventory groups: - $group_name From 4efb1a1d79053c2b386834fa0f8e469b86abb121 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Wed, 25 Jun 2025 11:31:24 +0000 Subject: [PATCH 14/14] add node_fqdn from PR#702 to additional_nodes --- .../skeleton/{{cookiecutter.environment}}/tofu/inventory.tpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/inventory.tpl b/environments/skeleton/{{cookiecutter.environment}}/tofu/inventory.tpl index 39738f243..6c11b3259 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/tofu/inventory.tpl +++ b/environments/skeleton/{{cookiecutter.environment}}/tofu/inventory.tpl @@ -66,11 +66,12 @@ compute: %{ for group_name in keys(additional_groups) ~} ${cluster_name}_${group_name}: hosts: -%{ for node in additional_groups[group_name]["compute_instances"] ~} +%{ for nodename, node in additional_groups[group_name]["compute_instances"] ~} ${ node.name }: ansible_host: ${node.access_ip_v4} instance_id: ${ node.id } networks: ${jsonencode({for n in node.network: n.name => {"fixed_ip_v4": n.fixed_ip_v4, "fixed_ip_v6": n.fixed_ip_v6}})} + node_fqdn: ${additional_groups[group_name]["fqdns"][nodename]} %{ endfor ~} ${group_name}: children: