Merge branch 'master' into feat/no-ohpc

Author: sjpb

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ jobs:
     # as the container command is flaky.
     # This job builds an image using the upstream rockylinux:9.3 image which ensures
     # that the image used for the molecule workflow is always updated.
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         working-directory: molecule/images
@@ -36,7 +36,7 @@ jobs:
 
   molecule:
     name: Molecule
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     needs: build
     strategy:
       fail-fast: false
@@ -51,17 +51,17 @@ jobs:
           - test2
           - test3
           - test4
-          - test5
           - test6
-          - test7
           - test8
           - test9
           - test10
           - test11
           - test12
           - test13
-          - test14
-        exclude: []
+        exclude:
+          # mariadb package provides /usr/bin/mysql on RL8 which doesn't work with geerlingguy/mysql role
+          - scenario: test4
+            image: 'rockylinux:8.9'
 
     steps:
       - name: Check out the codebase.
@@ -107,7 +107,7 @@ jobs:
 
   checks:
     name: Checks
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - name: Check out the codebase.
         uses: actions/checkout@v3

README.md

@@ -4,30 +4,57 @@ openhpc_slurm_service_started: "{{ openhpc_slurm_service_enabled }}"
 openhpc_slurm_service:
 openhpc_slurm_control_host: "{{ inventory_hostname }}"
 #openhpc_slurm_control_host_address:
-openhpc_slurm_partitions: []
+openhpc_partitions: "{{ openhpc_nodegroups }}"
+openhpc_nodegroups: []
 openhpc_cluster_name:
 openhpc_packages:
   - slurm-libpmi-ohpc
 openhpc_resume_timeout: 300
 openhpc_retry_delay: 10
 openhpc_job_maxtime: '60-0' # quote this to avoid ansible converting some formats to seconds, which is interpreted as minutes by Slurm
-openhpc_config: "{{ openhpc_extra_config | default({}) }}"
+openhpc_default_config:
+  # This only defines values which are not Slurm defaults
+  SlurmctldHost: "{{ openhpc_slurm_control_host }}{% if openhpc_slurm_control_host_address is defined %}({{ openhpc_slurm_control_host_address }}){% endif %}"
+  ProctrackType: proctrack/linuxproc # TODO: really want cgroup but needs cgroup.conf and workaround for CI
+  SlurmdSpoolDir: /var/spool/slurm # NB: not OpenHPC default!
+  SlurmUser: slurm
+  StateSaveLocation: "{{ openhpc_state_save_location }}"
+  SlurmctldTimeout: 300
+  SchedulerType: sched/backfill
+  SelectType: select/cons_tres
+  SelectTypeParameters: CR_Core
+  PriorityWeightPartition: 1000
+  PreemptType: preempt/partition_prio
+  PreemptMode: SUSPEND,GANG
+  AccountingStoragePass: "{{ openhpc_slurm_accounting_storage_pass | default('omit') }}"
+  AccountingStorageHost: "{{ openhpc_slurm_accounting_storage_host }}"
+  AccountingStoragePort: "{{ openhpc_slurm_accounting_storage_port }}"
+  AccountingStorageType: "{{ openhpc_slurm_accounting_storage_type }}"
+  AccountingStorageUser: "{{ openhpc_slurm_accounting_storage_user }}"
+  JobCompLoc: "{{ openhpc_slurm_job_comp_loc }}"
+  JobCompType: "{{ openhpc_slurm_job_comp_type }}"
+  JobAcctGatherFrequency: "{{ openhpc_slurm_job_acct_gather_frequency }}"
+  JobAcctGatherType: "{{ openhpc_slurm_job_acct_gather_type }}"
+  SlurmctldSyslogDebug: info
+  SlurmdSyslogDebug: info
+  PropagateResourceLimitsExcept: MEMLOCK
+  Epilog: /etc/slurm/slurm.epilog.clean
+  ReturnToService: 2
+openhpc_cgroup_default_config:
+  ConstrainCores: "yes"
+  ConstrainDevices: "yes"
+  ConstrainRAMSpace: "yes"
+  ConstrainSwapSpace: "yes"
+
+openhpc_config: {}
+openhpc_cgroup_config: {}
 openhpc_gres_template: gres.conf.j2
 openhpc_cgroup_template: cgroup.conf.j2
-openhpc_slurm_configless: "{{ 'enable_configless' in openhpc_config.get('SlurmctldParameters', []) }}"
 
 openhpc_state_save_location: /var/spool/slurm
 openhpc_slurmd_spool_dir: /var/spool/slurm
 openhpc_slurm_conf_path: /etc/slurm/slurm.conf
 openhpc_slurm_conf_template: slurm.conf.j2
-openhpc_config_files:
-  - template:
-      dest: "{{ openhpc_slurm_conf_path | dirname }}/gres.conf"
-      src: "{{ openhpc_gres_template }}"
-      mode: "0600"
-      owner: slurm
-      group: slurm
-    enable: control
 
 # Accounting
 openhpc_slurm_accounting_storage_host: "{{ openhpc_slurmdbd_host }}"
@@ -113,12 +140,16 @@ ohpc_default_extra_repos:
       gpgcheck: true
       gpgkey: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-8"
 
-# Concatenate extra repo definitions here
-ohpc_extra_repos: "{{ ohpc_default_extra_repos[ansible_distribution_major_version] + openhpc_extra_repos }}"
-
-openhpc_munge_key:
+openhpc_munge_key_b64:
 openhpc_login_only_nodes: ''
 openhpc_module_system_install: true # only works for install-ohpc.yml/main.yml
 
 # Auto detection
 openhpc_ram_multiplier: 0.95
+
+# Database upgrade
+openhpc_slurm_accounting_storage_service: ''
+openhpc_slurm_accounting_storage_backup_cmd: ''
+openhpc_slurm_accounting_storage_backup_host: "{{ openhpc_slurm_accounting_storage_host }}"
+openhpc_slurm_accounting_storage_backup_become: true
+openhpc_slurm_accounting_storage_client_package: mysql

defaults/main.yml

@@ -0,0 +1,86 @@
+{  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "definitions": {
+    "gres": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "conf": {
+            "type": "string",
+            "minLength": 1
+          },
+          "file": {
+            "type": "string",
+            "minLength": 1
+          }
+        },
+        "required": [
+          "conf"
+        ]
+      }
+    }
+  },
+  "properties": {
+    "name": {
+      "type": "string",
+      "minLength": 1
+    },
+    "ram_mb": {
+      "type": "number",
+    },
+    "ram_multiplier": {
+      "type": "number",
+    },
+    "features": {
+    	"type": "array",
+        "items": {
+        	"type": "string"
+        }
+    },
+    "node_params": {
+    	"type": "object",
+    },            
+    "gres_autodetect": {
+      "type": "string",
+      "minLength": 1
+    },
+    "gres": {
+      "$ref": "#/definitions/gres"
+    }
+  },
+  "required": [
+    "name"
+  ],
+  "if": {
+    "properties": {
+      "gres_autodetect": {
+        "const": "off"
+      }
+    }
+  },
+  "then": {
+    "properties": {
+      "gres": {
+        "items": {
+          "required": [
+            "file"
+          ]
+        }
+      }
+    }
+  },
+  "else": {
+    "properties": {
+      "gres": {
+        "items": {
+          "not": {
+            "required": [
+              "file"
+            ]
+          }
+        }
+      }
+    }
+  }
+}

files/nodegroup.schema

@@ -22,16 +22,6 @@
 # Pattern to match a hostname with numerical ending
 pattern = re.compile("^(.*\D(?=\d))(\d+)$")
 
-def _get_hostvar(context, var_name, inventory_hostname=None):
-    if inventory_hostname is None:
-        namespace = context
-    else:
-        if inventory_hostname not in context['hostvars']:
-            raise errors.AnsibleFilterError(
-                "Inventory hostname '%s' not in hostvars" % inventory_hostname)
-        namespace = context["hostvars"][inventory_hostname]
-    return namespace.get(var_name)
-
 def hostlist_expression(hosts):
     """ Group hostnames using Slurm's hostlist expression format.
 
@@ -91,11 +81,46 @@ def dict2parameters(d):
     parts = ['%s=%s' % (k, v) for k, v in d.items()]
     return ' '.join(parts)
 
+def config2dict(lines):
+    """ Convert a sequence of output lines from `scontrol show config` to a dict.
+        
+        As per man page uppercase keys are derived parameters, mixed case are from
+        from config files.
+
+        The following case-insensitive conversions of values are carried out:
+        - '(null)' and 'n/a' are converted to None.
+        - yes and no are converted to True and False respectively
+        
+        Except for these, values are always strings.
+    """
+    cfg = {}
+    for line in lines:
+        if '=' not in line: # ditch blank/info lines
+            continue
+        else:
+            parts = [x.strip() for x in line.split('=', maxsplit=1)] # maxplit handles '=' in values
+            if len(parts) != 2:
+                raise errors.AnsibleFilterError(f'line {line} cannot be split into key=value')
+            k, v = parts
+            small_v = v.lower()
+            if small_v == '(null)':
+                v = None
+            elif small_v == 'n/a':
+                v = None
+            elif small_v == 'no':
+                v = False
+            elif small_v == 'yes':
+                v = True
+            cfg[k] = v
+    return cfg
+
+
 class FilterModule(object):
 
     def filters(self):
         return {
             'hostlist_expression': hostlist_expression,
             'error': error,
             'dict2parameters': dict2parameters,
+            'config2dict': config2dict,
         }

filter_plugins/slurm_conf.py

@@ -1,10 +1,4 @@
 ---
-# NOTE: We need this running before slurmdbd
-- name: Restart Munge service
-  service:
-    name: "munge"
-    state: restarted
-  when: openhpc_slurm_service_started | bool
 
 # NOTE: we need this running before slurmctld start
 - name: Issue slurmdbd restart command
@@ -58,7 +52,15 @@
   service:
     name: "slurmd"
     state: restarted
+  retries: 5
+  register: slurmd_restart
+  until: slurmd_restart is success
+  delay: 30
   when:
     - openhpc_slurm_service_started | bool
     - openhpc_enable.batch | default(false) | bool
     # 2nd condition required as notification happens on controller, which isn't necessarily a compute note
+
+- name: Reload facts
+  ansible.builtin.setup:
+    filter: ansible_local

handlers/main.yml

@@ -6,23 +6,23 @@ Test options in "Other" column flow down through table unless changed.
 
 Test   | # Partitions | Groups in partitions?   | Other
 ---    | ---          | ---                     | ---
-test1  | 1            | N                       | 2x compute node, sequential names (default test), config on all nodes
+test1  | 1            | N                       | 2x compute node, sequential names (default test)
 test1b | 1            | N                       | 1x compute node
 test1c | 1            | N                       | 2x compute nodes, nonsequential names
 test2  | 2            | N                       | 4x compute node, sequential names
-test3  | 1            | Y                       | -
+test3  | 1            | Y                       | 4x compute nodes in 2x groups, single partition
 test4  | 1            | N                       | 2x compute node, accounting enabled
-test5  | 1            | N                       | As for #1 but configless
-test6  | 1            | N                       | 0x compute nodes, configless
-test7  | 1            | N                       | 1x compute node, no login node so specified munge key, configless (checks image build should work)
-test8  | 1            | N                       | 2x compute node, 2x login-only nodes, configless
+test5  | -            | -                       | [removed, now always configless]
+test6  | 1            | N                       | 0x compute nodes
+test7  | 1            | N                       | [removed, image build should just run install.yml task, this is not expected to work]
+test8  | 1            | N                       | 2x compute node, 2x login-only nodes
 test9  | 1            | N                       | As test8 but uses `--limit=testohpc-control,testohpc-compute-0` and checks login nodes still end up in slurm.conf
-test10 | 1            | N                       | As for #5 but then tries to add an additional node
-test11 | 1            | N                       | As for #5 but then deletes a node (actually changes the partition due to molecule/ansible limitations)
-test12 | 1            | N                       | As for #5 but enabling job completion and testing `sacct -c`
-test13 | 1            | N                       | As for #5 but tests `openhpc_config` variable.
-test14 | 1            | N                       | As for #5 but also tests `extra_nodes` via State=DOWN nodes.
-
+test10 | 1            | N                       | As for #1 but then tries to add an additional node
+test11 | 1            | N                       | As for #1 but then deletes a node (actually changes the partition due to molecule/ansible limitations)
+test12 | 1            | N                       | As for #1 but enabling job completion and testing `sacct -c`
+test13 | 1            | N                       | As for #1 but tests `openhpc_config` variable.
+test14 | -            | -                       | [removed, extra_nodes removed]
+test15 | 1            | Y                       | As for #1 but also tests partitions with different name but with the same NodeName.
 
 # Local Installation & Running
 

molecule/README.md

@@ -7,7 +7,7 @@
       batch: "{{ inventory_hostname in groups['testohpc_compute'] }}"
       runtime: true
     openhpc_slurm_control_host: "{{ groups['testohpc_login'] | first }}"
-    openhpc_slurm_partitions:
+    openhpc_nodegroups:
       - name: "compute"
     openhpc_cluster_name: testohpc
   tasks:

molecule/test1/converge.yml

@@ -9,8 +9,8 @@ platforms:
       - testohpc_login
     command: /sbin/init
     tmpfs:
-      - /run
-      - /tmp
+      /run: rw
+      /tmp: rw
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:ro
     network: net1
@@ -21,8 +21,8 @@ platforms:
       - testohpc_compute
     command: /sbin/init
     tmpfs:
-      - /run
-      - /tmp
+      /run: rw
+      /tmp: rw
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:ro
     network: net1
@@ -33,8 +33,8 @@ platforms:
       - testohpc_compute
     command: /sbin/init
     tmpfs:
-      - /run
-      - /tmp
+      /run: rw
+      /tmp: rw
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:ro
     network: net1