make state volume provisioning optional

Author: sjpb

docs/production.md

@@ -76,52 +76,50 @@ and referenced from the `site` and `production` environments, e.g.:
   instances) it may be necessary to configure or proxy `chronyd` via an
   environment hook.
 
-- By default, the cookiecutter-provided OpenTofu configuration provisions a
-  volume named "$cluster_name-home" and attaches it to the control node. This
-  is used to persist the NFS-shared home directories when the control node is
-  re-built. In production environments it is undesirable to have this volume
-  deleted if the cluster is deleted. Therefore the volume should be manually
+- By default, the cookiecutter-provided OpenTofu configuration provisions two
+  volumes and attaches them to the control node:
+    - "$cluster_name-home" for NFS-shared home directories
+    - "$cluster_name-state" for monitoring and Slurm data
+  The volumes mean this data is persisted when the control node is rebuilt.
+  However if the cluster is destroyed with `tofu destroy` then the volumes will
+  also be deleted. This is undesirable for production environments and usually
+  also for staging environments. Therefore the volumes should be manually
   created, e.g. via the CLI:
 
-      openstack volume create --size 100 mycluster-home # size in GB
+      openstack volume create --size 200 mycluster-home # size in GB
+      openstack volume create --size 100 mycluster-state
 
-  and OpenTofu configured to use that volume rather than managing one itself
-  by setting
+  and OpenTofu configured to use those volumes instead of managing them itself
+  by setting:
 
       home_volume_provisioning = "attach"
+      state_volume_provisioning = "attach"
 
-  either for a specific environment in the cluster module block in
+  either for a specific environment within the cluster module block in
   `environments/$ENV/tofu/main.tf`, or as the site default by changing the
   default in `environments/site/tofu/variables.tf`.
   
-  For a development environment allowing OpenTofu to manage the volume using
-  the default value
-  
-      home_volume_provisioning = "manage"
-  
-  is usually appropriate, as it allows for multiple clusters to be created with
-  this environment.
+  For a development environment allowing OpenTofu to manage the volumes using
+  the default value of `"manage"` for those varibles is usually appropriate, as
+  it allows for multiple clusters to be created with this environment.
   
   If no home volume at all is required because the home directories are provided
-  by a parallel filesystem (e.g. manila) use
+  by a parallel filesystem (e.g. manila) set
 
       home_volume_provisioning = "none"
 
   In this case the NFS share for home directories is automatically disabled.
 
-- By default, the cookiecutter-provided OpenTofu configuration provisions a
-  volume named "$cluster_name-state" and attaches it to the control node. This
-  is used to persist monitoring and Slurm data when the control node is re-built.
-  In production and staging environments it is undesirable to have this volume
-  deleted if the cluster is deleted. Therefore the volume should be manually
-  created, e.g. via the CLI:
+  **NB:** To apply "attach" options to existing clusters, first remove the
+    volume(s) from the tofu state, e.g.:
 
-      openstack volume create --size 100 mycluster-state # size in GB
-  
-  and the `openstack_blockstorage_volume_v3.state` resource changed to a [data
-  resource](https://opentofu.org/docs/language/data-sources/). This ensures
-  that even if the cluster is deleted via `tofu destroy`, the volume will persist.
+      tofu state list # find the volume(s)
+      tofu state rm 'module.cluster.openstack_blockstorage_volume_v3.state[0]'
   
+  This leaves the volume itself intact, but means OpenTofu "forgets" it. Then
+  set the "attach" options and run `tofu apply` again - this should show there
+  are no changes planned.
+
 - Enable `etc_hosts` templating:
 
     ```yaml

environments/skeleton/{{cookiecutter.environment}}/tofu/control.tf

@@ -1,7 +1,7 @@
 locals {
   control_volumes = concat(
-    [openstack_blockstorage_volume_v3.state],
-    # convert map to list:
+    # convert maps to list:
+    [for v in data.openstack_blockstorage_volume_v3.state: v],
     [for v in data.openstack_blockstorage_volume_v3.home: v]
   )
   nodename = templatestring(

environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf

@@ -125,6 +125,28 @@ variable "state_volume_type" {
     default = null
 }
 
+variable "state_volume_provisioning" {
+    type = string
+    default = "manage"
+    description = <<-EOT
+        How to manage the state volume. Valid values are:
+            "manage": (Default) OpenTofu will create a volume "$cluster_name-state"
+                      and delete it when the cluster is destroyed. A volume
+                      with this name must not already exist. Use for demo and
+                      dev environments.
+            "attach": A single volume named "$cluster_name-state" must already
+                      exist. It is not managed by OpenTofu so e.g. is left
+                      intact if the cluster is destroyed. Use for production
+                      environments.
+        EOT
+    validation {
+      condition = contains(["manage", "attach"], var.state_volume_provisioning)
+      error_message = <<-EOT
+        home_volume_provisioning must be "manage" or "attach"
+    EOT
+    }
+}
+
 variable "home_volume_size" {
     type = number
     description = "Size of state volume on control node, in GB."
@@ -161,7 +183,9 @@ variable "home_volume_provisioning" {
         EOT
     validation {
       condition = contains(["manage", "attach", "none"], var.home_volume_provisioning)
-      error_message = "home_volume_provisioning must be one of manage, attach or none"
+      error_message = <<-EOT
+        home_volume_provisioning must be one of "manage", "attach" or "none"
+    EOT
     }
 }
 

environments/skeleton/{{cookiecutter.environment}}/tofu/volumes.tf

@@ -1,14 +1,39 @@
 resource "openstack_blockstorage_volume_v3" "state" {
+
+    # NB: Changes to this resource's "address" i.e. (label or for_each key)
+    # may lose state data for existing clusters using this volume
+
+    count = var.state_volume_provisioning == "manage" ? 1 : 0
+
     name = "${var.cluster_name}-state" # last word used to label filesystem
     description = "State for control node"
     size = var.state_volume_size
     volume_type = var.state_volume_type
 }
 
+data "openstack_blockstorage_volume_v3" "state" {
+
+/*  We use a data resource whether or not TF is managing the volume, so the
+    logic is all in one place. But that means this needs a dependency on the
+    actual resource to avoid a race.
+
+    Because there may be no volume, this has to use for_each.
+*/
+
+    for_each = toset(
+        (var.state_volume_provisioning == "manage") ?
+            [for v in openstack_blockstorage_volume_v3.state: v.name] :
+                ["${var.cluster_name}-state"]
+    )
+
+    name = each.key
+
+}
+
 resource "openstack_blockstorage_volume_v3" "home" {
 
     # NB: Changes to this resource's "address" i.e. (label or for_each key)
-    # will lose user data for existing clusters using this volume
+    # may lose user data for existing clusters using this volume
 
     count = var.home_volume_provisioning == "manage" ? 1 : 0
 
@@ -20,12 +45,7 @@ resource "openstack_blockstorage_volume_v3" "home" {
 
 data "openstack_blockstorage_volume_v3" "home" {
 
-/*  We use a data resource whether or not TF is managing the volume, so the
-    logic is all in one place. But that means this needs a dependency on the
-    actual resource to avoid a race.
-    
-    Because there may be no volume, this has to use for_each.
-*/
+/*  Comments as for the state volume. */
 
     for_each = toset(
         (var.home_volume_provisioning == "manage") ?