Perform Slurm database upgrade if necessary (#670)

* bump OpenHPC snapshots to v3.1.1 (slurm 24.11.5) and v2.9.1 (slurm 23.11.11) for CVE-2025-43904

* bump CI image

* extend timeout for slurmdbd startup to cope with major version upgrade on startup

* configure openhpc for slurmdbd backup/update

* support mysql tasks in openhpc role

* remove slurmdbd startup timeout increase - got borked during merge from main

* mysql package now installed separately in role from openhpc_packages

* bump CI image to get mysql client installed

* delete snapshot when cleaning up in CI

* bump openhpc role to commit

* bump openhpc role to release

Author: sjpb

.github/workflows/stackhpc.yml

@@ -230,6 +230,16 @@ jobs:
         env:
           DEMO_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
+      - name: Delete possible volume snapshot from slurm upgrade
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate  
+          if [ -n "$SNAPSHOT" ]
+          then
+              echo Deleting $SNAPSHOT
+              openstack volume snapshot delete $SNAPSHOT
+          fi
+
       - name: Delete infrastructure
         run: |
           . venv/bin/activate

environments/.stackhpc/tofu/cluster_image.auto.tfvars.json

@@ -1,6 +1,6 @@
 {
     "cluster_image": {
-        "RL8": "openhpc-RL8-250513-1045-ca44f898",
-        "RL9": "openhpc-RL9-250513-1046-ca44f898"
+        "RL8": "openhpc-RL8-250514-1502-5a923b2c",
+        "RL9": "openhpc-RL9-250514-1502-5a923b2c"
     }
 }

environments/common/inventory/group_vars/all/openhpc.yml

@@ -58,7 +58,12 @@ ohpc_default_extra_repos:
   "9": []
   "8": []
 
-# systemd.service.unit.TimeoutStartSec to wait for slurmdbd startup
-# Set long enought to avoid problems with a major version upgrade
-# Currently implemented in environments/common/inventory/group_vars/all/systemd.yml
-openhpc_slurmdbd_timeout_start_sec: '45 minutes'
+# configure slurm database pre-upgrade backups:
+openhpc_slurm_accounting_storage_service: mysql
+openhpc_slurm_accounting_storage_backup_cmd: >-
+  openstack volume snapshot create
+  --volume {{ openhpc_cluster_name }}-state
+  --force
+  {{ openhpc_cluster_name }}-state-{{ ansible_date_time.iso8601_basic_short }}
+openhpc_slurm_accounting_storage_backup_host: localhost
+openhpc_slurm_accounting_storage_backup_become: false

environments/common/inventory/group_vars/all/systemd.yml

@@ -14,13 +14,7 @@ systemd_dropins:
     content: "{{ _systemd_requiresmount_statedir }}"
   slurmdbd:
     group: openhpc
-    content: |
-      {{ _systemd_requiresmount_statedir }}
-      
-      [Service]
-      # Allow slurmdbd to complete major version upgrades
-      TimeoutStartSec={{ openhpc_slurmdbd_timeout_start_sec }}
-
+    content: "{{ _systemd_requiresmount_statedir }}"
   slurmctld:
     group: openhpc
     content: "{{ _systemd_requiresmount_statedir }}"

requirements.yml

@@ -4,7 +4,7 @@ roles:
     version: v25.3.2
     name: stackhpc.nfs
   - src: https://github.com/stackhpc/ansible-role-openhpc.git
-    version: v0.28.0
+    version: v0.30.0
     name: stackhpc.openhpc
   - src: https://github.com/stackhpc/ansible-node-exporter.git
     version: stackhpc