add nhc

sjpb · sjpb · commit 14629043bb67 · 2025-04-23T09:54:08.000Z
diff --git a/ansible/.gitignore b/ansible/.gitignore
@@ -90,3 +90,5 @@ roles/*
 !roles/gateway/**
 !roles/alertmanager/
 !roles/alertmanager/**
+!roles/nhc/
+!roles/nhc/**
diff --git a/ansible/bootstrap.yml b/ansible/bootstrap.yml
@@ -134,6 +134,7 @@
 
 - hosts: dnf_repos
   become: yes
+  tags: dnf_repos
   tasks:
   - name: Check that creds won't be leaked to users
     ansible.builtin.assert:
diff --git a/ansible/extras.yml b/ansible/extras.yml
@@ -58,17 +58,6 @@
     - import_role:
         name: persist_hostkeys
 
-
-- name: Setup NFS export for compute node configuration
-  hosts: compute_init:!builder
-  # NB: has to be after eeesi and os-manila-mount
-  tags: compute_init
-  become: yes
-  tasks:
-    - include_role:
-        name: compute_init
-        tasks_from: export.yml
-
 - name: Install k9s
   become: yes
   hosts: k9s
@@ -85,3 +74,21 @@
   - name: Install additional packages
     dnf:
       name: "{{ appliances_extra_packages }}"
+
+- hosts: nhc
+  become: yes
+  tags: nhc
+  tasks:
+    - name: Configure node health checks
+      import_role:
+        name: nhc
+
+- name: Setup NFS export for compute_init
+  hosts: compute_init:!builder
+  # NB: done last so other orles can prepare configuration etc
+  tags: compute_init
+  become: yes
+  tasks:
+    - include_role:
+        name: compute_init
+        tasks_from: export.yml
diff --git a/ansible/roles/compute_init/README.md b/ansible/roles/compute_init/README.md
@@ -84,6 +84,7 @@ it also requires an image build with the role name added to the
 | slurm.yml                | openhpc [10]            | All slurmd functionality        | No                  |
 | slurm.yml                | (set memory limits)     | Fully supported                 | No                  |
 | slurm.yml                | (block ssh)             | Fully supported                 | No                  |
+| slurm.yml                | nhc                     | Fully supported                 | No                  |
 | portal.yml               | (openondemand server)   | Not relevant for compute nodes  | n/a                 |
 | portal.yml               | (openondemand vnc desktop)  | None required - use image build | No              |
 | portal.yml               | (openondemand jupyter server) | None required - use image build | No            |
diff --git a/ansible/roles/compute_init/files/compute-init.yml b/ansible/roles/compute_init/files/compute-init.yml
@@ -19,6 +19,7 @@
     enable_basic_users: "{{ os_metadata.meta.basic_users | default(false) | bool }}"
     enable_eessi: "{{ os_metadata.meta.eessi | default(false) | bool }}"
     enable_chrony: "{{ os_metadata.meta.chrony | default(false) | bool }}"
+    enable_nhc: "{{ os_metadata.meta.nhc | default(false) | bool }}"
 
     # TODO: "= role defaults" - could be moved to a vars_file: on play with similar precedence effects
     resolv_conf_nameservers: []
@@ -350,6 +351,11 @@
         enabled: true
         state: started
 
+    - name: Provide NHC configuration
+      ansible.builtin.include_role:
+        name: nhc
+        tasks_from: import.yml
+      when: enable_nhc
 
     - name: Ensure node is resumed
       # TODO: consider if this is always safe for all job states?
diff --git a/ansible/roles/compute_init/tasks/export.yml b/ansible/roles/compute_init/tasks/export.yml
@@ -98,3 +98,9 @@
     name: sshd
     tasks_from: export.yml
   when: "'sshd' in group_names"
+
+- name: Retrieve generated NHC config
+  import_role:
+    name: nhc
+    tasks_from: export.yml
+  when: "'nhc' in group_names"
diff --git a/ansible/roles/compute_init/tasks/install.yml b/ansible/roles/compute_init/tasks/install.yml
@@ -49,6 +49,8 @@
       dest: roles/
     - src: ../../lustre
       dest: roles/
+    - src: ../../nhc
+      dest: roles/
 
 - name: Add filter_plugins to ansible.cfg
   lineinfile:
diff --git a/ansible/roles/nhc/README.md b/ansible/roles/nhc/README.md
@@ -0,0 +1,107 @@
+# Node Health Checks (nhc)
+
+Deploys and configures the LBNL [Node Health Check](https://github.com/mej/nhc)
+(NHC) which will put nodes in `DOWN` state if they fail periodic checks on
+various aspects.
+
+Due to the integration with Slurm this is tightly linked to the configuration
+for the [stackhpc.openhpc](../stackhpc.openhpc/README.md) role.
+
+## Enabling
+
+By [default](../../../environments/common/inventory/group_vars/all/openhpc.yml)
+the required `nhc-ohpc` packages are installed in all images.
+
+To enable node health checks, ensure the `nhc` group contains the `compute` group:
+
+```yaml
+# environments/site/inventory/groups:
+[nhc:children]
+# Hosts to configure for node health checks
+compute
+```
+
+This will:
+1. Add NHC-related configuration to the `slurm.conf` Slurm configuration file.
+The default configuration is defined in `openhpc_config_nhc`
+(see [environments/common/inventory/group_vars/all/openhpc.yml](../../../environments/common/inventory/group_vars/all/openhpc.yml)).
+It will run healthchecks on all `IDLE` nodes which are not `DRAINED` or `NOT_RESPONDING`
+every 300 seconds. See [slurm.conf parameters](https://slurm.schedmd.com/slurm.conf.html)
+`HealthCheckInterval`, `HealthCheckNodeState`, `HealthCheckProgram`. These may
+be overriden if required by redefining `openhpc_config_nhc` in e.g.
+`environments/site/inventory/group_vars/nhc/yml`.
+
+2. Define a default configuration for health checks for each compute node
+individually using [nhc-genconf](https://github.com/mej/nhc?tab=readme-ov-file#config-file-auto-generation)
+The generated checks include:
+    - Filesystem mounts
+    - Filesystem space
+    - CPU info
+    - Memory and swap
+    - Network interfaces
+    - Various processes
+
+    See `/etc/nhc/nhc.conf` on a compute node for the full configuration.
+
+The automatically generated checks may be modified or disabled using the
+`nhc_replacements` role variable described below.
+
+If a node healthcheck run fails, Slurm will mark the node `DOWN`. With the
+default [alerting configuration](../../../docs/alerting.md) this will trigger
+an alert.
+
+## Updating Health Checks
+
+The above approach assumes that when the `site.yml` playbook is run all nodes
+are functioning correctly. Therefore if changes are made to aspects covered by
+the healthchecks (see above) without re-running this playbook, use the following
+to update the autogenerated health checks:
+
+  ```shell
+  ansible-playbook ansible/extras.yml --tags nhc
+  ```
+
+## Role Variables
+
+- `nhc_replacements`: Optional, default empty list. A list of mappings
+  defining replacements in the autogenerated health checks. Items must have
+  keys `regexp` and `replace` which are as for [ansible.builtin.replace](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/replace_module.html).
+  Note that the NHC [configuration line format](https://github.com/mej/nhc?tab=readme-ov-file#configuration-file-syntax) is:
+
+      TARGET || CHECK
+
+  where for autogenerated checks `TARGET` is the hostname. So a regex like:
+
+      '^(\s+\S+\s+)\|\|(\s+.*)$'
+
+  captures the `TARGET` and `||` separator as `\1` and the actual check as `\2`.
+  Hence the following item would comment-out checks on a particular interface
+  on all nodes:
+
+      - regexp:  '^(\s+\S+\s+\|\|\s+)(check_hw_eth eth0)$'
+        replace: '#\1\2'
+
+  See documentation for `ansible.builtin.replace` for more information. This is
+  an example only - for this actual case removing the line entirely with
+  `replace: ''` might be better. Using https://regex101.com/ (in Python
+  mode) or similar may be useful during development.
+
+- `nhc_replacements_default`: Optional. As above, but by default includes a
+  mapping to remote the autogenerated timestamp line from the check configuration
+  file for idempotency.
+
+## Structure
+
+This role contains 3x task files, which run at different times:
+- `main.yml`: Runs from `site.yml` -> `slurm.yml`. Generates health check
+  configuration.
+- `export.yml`: Runs from `site.yml` -> `extras.yml` via role `compute_init`
+  tasks `export.yml`. Copies the generated health check configuration to the
+  control node NFS share for compute-init. 
+- `import.yml`: Runs on boot via `compute_init/files/compute-init.yml` and
+  copies the node's generated health check configuration from the control node
+  NFS share to local disk.
+
+Note that the `stackhpc.openhpc` role:
+- Installs the required package
+- Configures slurm
diff --git a/ansible/roles/nhc/defaults/main.yml b/ansible/roles/nhc/defaults/main.yml
@@ -0,0 +1,6 @@
+nhc_replacements_default:
+  # note this matches a multiline string
+  - regexp: '# This file was automatically generated by nhc-genconf\n#.*'
+    replace: '# This file was automatically generated by nhc-genconf\n# (timestamp removed for idempotency)'
+
+nhc_replacements: []
diff --git a/ansible/roles/nhc/tasks/export.yml b/ansible/roles/nhc/tasks/export.yml
@@ -0,0 +1,5 @@
+- name: Fetch generated NHC config to control node storage
+  ansible.builtin.fetch:
+    src: /etc/nhc/nhc.conf
+    flat: true
+    dest: "/exports/cluster/hostconfig{{ inventory_hostname }}/nhc.conf"
diff --git a/ansible/roles/nhc/tasks/import.yml b/ansible/roles/nhc/tasks/import.yml
@@ -0,0 +1,8 @@
+- name: Copy stored NHC configuration to active location
+  ansible.builtin.copy:
+    remote_src: true
+    src: "/exports/cluster share/hostconfig/{{ ansible_host }}"
+    dest: /etc/nhc/nhc.conf
+    owner: root
+    group: root
+    mode: u=rw,go= # HealthCheckProgram is run by root
diff --git a/ansible/roles/nhc/tasks/main.yml b/ansible/roles/nhc/tasks/main.yml
@@ -0,0 +1,25 @@
+- name: Create temporary file for autogenerated NHC configuration
+  ansible.builtin.tempfile:
+  register: nhc_tempfile
+
+- name: Run NHC autoconfiguration
+  ansible.builtin.command:
+    cmd: "nhc-genconf -c {{ nhc_tempfile.path }}"
+  register: _nhc_genconf
+  failed_when: _nhc_genconf.rc > 1 # https://github.com/mej/nhc/issues/158
+
+- name: Apply modifications to autogenerated NHC configuration
+  ansible.builtin.replace:
+    path: "{{ nhc_tempfile.path }}"
+    regexp: "{{ item.regexp }}"
+    replace: "{{ item.replace }}"
+  loop: "{{ nhc_replacements_default + nhc_replacements }}"
+
+- name: Copy modified NHC configuration to active location
+  ansible.builtin.copy:
+    remote_src: true
+    src: "{{ nhc_tempfile.path }}"
+    dest: /etc/nhc/nhc.conf
+    owner: root
+    group: root
+    mode: u=rw,go= # HealthCheckProgram is run by root
diff --git a/ansible/slurm.yml b/ansible/slurm.yml
@@ -63,3 +63,13 @@
     - include_role:
         name: stackhpc.openhpc
         tasks_from: "{{ 'runtime.yml' if appliances_mode == 'configure' else 'main.yml' }}"
+
+- name: Setup Node Health Checks
+  # Has to be done here as it requires openhpc repos etc for installation
+  hosts: nhc
+  become: yes
+  tags:
+    - nhc
+  tasks:
+    - include_role:
+        name: nhc
diff --git a/docs/production.md b/docs/production.md
@@ -151,3 +151,5 @@ and referenced from the `site` and `production` environments, e.g.:
   is properly tuned, again, demonstrated through testing.
 
 - Enable alertmanager if Slack is available - see [docs/alerting.md](./alerting.md).
+
+- Enable node health checks - see [ansible/roles/nhc/README.md](../ansible/roles/nhc/README.md).
diff --git a/environments/.stackhpc/tofu/main.tf b/environments/.stackhpc/tofu/main.tf
@@ -78,7 +78,16 @@ module "cluster" {
         standard: { # NB: can't call this default!
             nodes: ["compute-0", "compute-1"]
             flavor: var.other_node_flavor
-            compute_init_enable: ["compute", "chrony", "etc_hosts", "nfs", "basic_users", "eessi", "tuned", "cacerts"]
+            compute_init_enable: [
+                "compute",
+                "chrony",
+                "etc_hosts",
+                "nfs",
+                "basic_users",
+                "eessi",
+                "tuned",
+                "cacerts"
+            ]
             ignore_image_changes: true
         }
         # Normally-empty partition for testing:
diff --git a/environments/common/inventory/group_vars/all/openhpc.yml b/environments/common/inventory/group_vars/all/openhpc.yml
@@ -24,26 +24,51 @@ openhpc_packages_default:
   - slurm-libpmi-ohpc # to allow intel mpi to work properly
   - ohpc-gnu12-openmpi4-perf-tools # for hpctests
   - openblas-gnu12-ohpc # for hpctests (HPL)
+  - nhc-ohpc # node health checks
   # EPEL packages:
   - apptainer
   - podman-compose
 openhpc_packages_extra: []
 openhpc_packages: "{{ openhpc_packages_default + openhpc_packages_extra }}"
 openhpc_munge_key: "{{ vault_openhpc_mungekey | b64decode }}"
 openhpc_login_only_nodes: login
-openhpc_config_default:
+openhpc_state_save_location: "{{ appliances_state_dir + '/slurmctld' if appliances_state_dir is defined else '/var/spool' }}"
+
+# additional site/environment-specific slurm.conf parameters:
+# NB: override in environments/site/inventory/group_vars/all/openhpc.yml, not here:
+openhpc_config_extra: {}
+
+# default additional slurm.conf parameters for the appliance:
+openhpc_config_appliance_default:
   SlurmctldParameters:
     - enable_configless
   TaskPlugin: task/cgroup,task/affinity
   ReturnToService: 2 # is stackhpc.openhpc default, but templating bug means it is needed here too
+
+# default additional slurm.conf parameters when "rebuild" enabled:
 openhpc_config_rebuild:
   RebootProgram: /opt/slurm-tools/bin/slurm-openstack-rebuild
   SlurmctldParameters:
     - reboot_from_controller
   ResumeTimeout: 300
-openhpc_config_extra: {}
-openhpc_config: "{{ openhpc_config_default | combine(openhpc_config_rebuild if groups['rebuild'] | length > 0 else {}, openhpc_config_extra, list_merge='append') }}"
-openhpc_state_save_location: "{{ appliances_state_dir + '/slurmctld' if appliances_state_dir is defined else '/var/spool' }}"
+
+# default additional slurm.conf parameters when "nhc" enabled:
+openhpc_config_nhc:
+  HealthCheckProgram: /usr/sbin/nhc
+  HealthCheckInterval: 300
+  HealthCheckNodeState: NONDRAINED_IDLE
+
+# indirection to allow automatic construction of slurm.conf parameters:
+openhpc_config_groups:
+  - enabled: "{{ groups['rebuild'] | length > 0 }}"
+    config: "{{ openhpc_config_rebuild }}"
+  - enabled: "{{ groups['nhc'] | length > 0 }}"
+    config: "{{ openhpc_config_nhc }}"
+  - enabled: true
+    config: "{{ openhpc_config_extra }}"
+
+# constructed slurm.conf parameters for stackpc.openhpc role var:
+openhpc_config: "{{ openhpc_config_default | combine(openhpc_config_groups | selectattr('enabled') | map(attribute='config'), list_merge='append') }}"
 
 openhpc_install_type: ohpc # 'ohpc' or 'generic', see https://github.com/stackhpc/ansible-slurm-appliance/pull/326
 
diff --git a/environments/common/inventory/groups b/environments/common/inventory/groups
@@ -182,3 +182,6 @@ extra_packages
 
 [gateway]
 # Add builder to this group to install gateway ansible-init playbook into image
+
+[nhc]
+# Hosts to configure for node health checks - either entire 'compute' group or empty
diff --git a/environments/common/layouts/everything b/environments/common/layouts/everything
@@ -125,3 +125,7 @@ builder
 [gateway:children]
 # Add builder to this group to install gateway ansible-init playbook into image
 builder
+
+[nhc:children]
+# Hosts to configure for node health checks
+compute
