Merge branch 'main' into feat/k3s-bootstrap

wtripp180901 · wtripp180901 · commit 7bb9920a73c4 · 2025-02-21T10:12:51.000Z
diff --git a/.github/workflows/stackhpc.yml b/.github/workflows/stackhpc.yml
@@ -182,9 +182,8 @@ jobs:
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
-          ansible-playbook -v --limit compute ansible/adhoc/rebuild.yml
-          ansible-playbook -v ansible/ci/check_slurm.yml
           ansible-playbook -v ansible/adhoc/reboot_via_slurm.yml
+          ansible-playbook -v ansible/ci/check_slurm.yml
 
       - name: Check sacct state survived reimage
         run: |
diff --git a/ansible/bootstrap.yml b/ansible/bootstrap.yml
@@ -52,6 +52,13 @@
     - import_role:
         name: proxy
 
+- hosts: chrony
+  tags: chrony
+  become: yes
+  tasks:
+    - import_role:
+        name: mrlesmithjr.chrony
+
 - hosts: cluster
   gather_facts: false
   become: yes
diff --git a/ansible/roles/compute_init/README.md b/ansible/roles/compute_init/README.md
@@ -3,10 +3,13 @@
 Experimental functionality to allow compute nodes to rejoin the cluster after
 a reboot without running the `ansible/site.yml` playbook.
 
+**CAUTION:** The approach used here of exporting cluster secrets over NFS
+is considered to be a security risk due to the potential for cluster users to
+mount the share on a user-controlled machine by tunnelling through a login
+node. This feature should not be enabled on production clusters at this time.
+
 To enable this:
-1. Add the `compute` group (or a subset) into the `compute_init` group. This is
-   the default when using cookiecutter to create an environment, via the
-   "everything" template.
+1. Add the `compute` group (or a subset) into the `compute_init` group.
 2. Build an image which includes the `compute_init` group. This is the case
    for StackHPC-built release images.
 3. Enable the required functionalities during boot, by setting the
@@ -40,6 +43,7 @@ it also requires an image build with the role name added to the
 | bootstrap.yml            | (wait for ansible-init) | Not relevant during boot        | n/a                 |
 | bootstrap.yml            | resolv_conf             | Fully supported                 | No                  |
 | bootstrap.yml            | etc_hosts               | Fully supported                 | No                  |
+| bootstrap.yml            | chrony                  | Fully supported                 | No                  |
 | bootstrap.yml            | proxy                   | None at present                 | No                  |
 | bootstrap.yml            | (/etc permissions)      | None required - use image build | No                  |
 | bootstrap.yml            | (ssh /home fix)         | None required - use image build | No                  |
diff --git a/ansible/roles/compute_init/files/compute-init.yml b/ansible/roles/compute_init/files/compute-init.yml
@@ -17,6 +17,7 @@
     enable_manila: "{{ os_metadata.meta.manila | default(false) | bool }}"
     enable_basic_users: "{{ os_metadata.meta.basic_users | default(false) | bool }}"
     enable_eessi: "{{ os_metadata.meta.eessi | default(false) | bool }}"
+    enable_chrony: "{{ os_metadata.meta.chrony | default(false) | bool }}"
 
     # TODO: "= role defaults" - could be moved to a vars_file: on play with similar precedence effects
     resolv_conf_nameservers: []
@@ -100,6 +101,11 @@
 
       # TODO: should /mnt/cluster now be UNMOUNTED to avoid future hang-ups?
 
+    - name: Run chrony role
+      ansible.builtin.include_role:
+        name: mrlesmithjr.chrony
+      when: enable_chrony | bool
+
     - name: Configure resolve.conf
       block:
         - name: Set nameservers in /etc/resolv.conf
diff --git a/ansible/roles/compute_init/tasks/install.yml b/ansible/roles/compute_init/tasks/install.yml
@@ -43,6 +43,8 @@
       dest: tasks/tuned.yml
     - src: ../../stackhpc.nfs/tasks/nfs-clients.yml
       dest: tasks/nfs-clients.yml
+    - src: ../../mrlesmithjr.chrony
+      dest: roles/
 
 - name: Add filter_plugins to ansible.cfg
   lineinfile:
diff --git a/ansible/roles/hpctests/README.md b/ansible/roles/hpctests/README.md
@@ -29,9 +29,22 @@ Role Variables
 - `hpctests_ucx_net_devices`: Optional. Control which network device/interface to use, e.g. `mlx5_1:0`. The default of `all` (as per UCX) may not be appropriate for multi-rail nodes with different bandwidths on each device. See [here](https://openucx.readthedocs.io/en/master/faq.html#what-is-the-default-behavior-in-a-multi-rail-environment) and [here](https://github.com/openucx/ucx/wiki/UCX-environment-parameters#setting-the-devices-to-use). Alternatively a mapping of partition name (as `hpctests_partition`) to device/interface can be used. For partitions not defined in the mapping the default of `all` is used.
 - `hpctests_outdir`: Optional. Directory to use for test output on local host. Defaults to `$HOME/hpctests` (for local user).
 - `hpctests_hpl_NB`: Optional, default 192. The HPL block size "NB" - for Intel CPUs see [here](https://software.intel.com/content/www/us/en/develop/documentation/onemkl-linux-developer-guide/top/intel-oneapi-math-kernel-library-benchmarks/intel-distribution-for-linpack-benchmark/configuring-parameters.html).
-- `hpctests_hpl_mem_frac`: Optional, default 0.8. The HPL problem size "N" will be selected to target using this fraction of each node's memory.
+- `hpctests_hpl_mem_frac`: Optional, default 0.3. The HPL problem size "N" will
+   be selected to target using this fraction of each node's memory -
+   **CAUTION: see note below**.
 - `hpctests_hpl_arch`: Optional, default 'linux64'. Arbitrary architecture name for HPL build. HPL is compiled on the first compute node of those selected (see `hpctests_nodes`), so this can be used to create different builds for different types of compute node.
 
+
+---
+**CAUTION**
+
+> The default of `hpctests_hpl_mem_frac=0.3` will not significantly load nodes.
+Values up to ~0.8 may be appropriate for a stress test but ensure cloud
+operators are aware in case this overloads e.g. power supplies or cooling.
+Values > 0.8 require longer runtimes and increase the risk of out-of-memory
+errors without normally significantly increasing the stress on the node.
+---
+
 The following variables should not generally be changed:
 - `hpctests_pre_cmd`: Optional. Command(s) to include in sbatch templates before module load commands.
 - `hpctests_pingmatrix_modules`: Optional. List of modules to load for pingmatrix test. Defaults are suitable for OpenHPC 2.x cluster using the required packages.
diff --git a/ansible/roles/hpctests/defaults/main.yml b/ansible/roles/hpctests/defaults/main.yml
@@ -9,7 +9,7 @@ hpctests_outdir: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/hpctests"
 hpctests_ucx_net_devices: all
 hpctests_hpl_version: "2.3"
 hpctests_hpl_NB: 192
-hpctests_hpl_mem_frac: 0.8
+hpctests_hpl_mem_frac: 0.3
 hpctests_hpl_arch: linux64
 #hpctests_nodes:
 #hpctests_partition:
diff --git a/dev/setup-env.sh b/dev/setup-env.sh
@@ -2,28 +2,30 @@
 
 set -euo pipefail
 
-if [[ -f /etc/os-release ]]; then
-    . /etc/os-release
-    OS=$ID
-    OS_VERSION=$VERSION_ID
-else
-    exit 1
-fi
+PYTHON_VERSION=${PYTHON_VERSION:-}
 
-MAJOR_VERSION=$(echo $OS_VERSION | cut -d. -f1)
+if [[ "$PYTHON_VERSION" == "" ]]; then
+    if [[ -f /etc/os-release ]]; then
+        . /etc/os-release
+        OS=$ID
+        OS_VERSION=$VERSION_ID
+    else
+        exit 1
+    fi
 
-PYTHON_VERSION=""
+    MAJOR_VERSION=$(echo $OS_VERSION | cut -d. -f1)
 
-if [[ "$OS" == "ubuntu" && "$MAJOR_VERSION" == "22" ]]; then
-    PYTHON_VERSION="/usr/bin/python3.10"
-elif [[ "$OS" == "rocky" && "$MAJOR_VERSION" == "8" ]]; then
-    # python3.9+ doesn't have selinux bindings
-    PYTHON_VERSION="/usr/bin/python3.8" # use `sudo yum install python38` on Rocky Linux 8 to install this
-elif [[ "$OS" == "rocky" && "$MAJOR_VERSION" == "9" ]]; then
-    PYTHON_VERSION="/usr/bin/python3.9"
-else
-    echo "Unsupported OS version: $OS $MAJOR_VERSION"
-    exit 1
+    if [[ "$OS" == "ubuntu" && "$MAJOR_VERSION" == "22" ]]; then
+        PYTHON_VERSION="/usr/bin/python3.10"
+    elif [[ "$OS" == "rocky" && "$MAJOR_VERSION" == "8" ]]; then
+        # python3.9+ doesn't have selinux bindings
+        PYTHON_VERSION="/usr/bin/python3.8" # use `sudo yum install python38` on Rocky Linux 8 to install this
+    elif [[ "$OS" == "rocky" && "$MAJOR_VERSION" == "9" ]]; then
+        PYTHON_VERSION="/usr/bin/python3.9"
+    else
+        echo "Unsupported OS version: $OS $MAJOR_VERSION"
+        exit 1
+    fi
 fi
 
 if [[ ! -d "venv" ]]; then
diff --git a/docs/chrony.md b/docs/chrony.md
@@ -0,0 +1,21 @@
+# Chrony configuration
+
+Use variables from the [mrlesmithjr.chrony](https://github.com/mrlesmithjr/ansible-chrony) role.
+
+For example in: `environments/<environment>/inventory/group_vars/all/chrony`:
+
+```
+---
+chrony_ntp_servers:
+  - server: ntp-0.example.org
+    options:
+      - option: iburst
+      - option: minpoll
+        val: 8
+  - server: ntp-1.example.org
+    options:
+      - option: iburst
+      - option: minpoll
+        val: 8
+
+```
diff --git a/docs/production.md b/docs/production.md
@@ -127,3 +127,6 @@ and referenced from the `site` and `production` environments, e.g.:
 
 - Note [PR 473](https://github.com/stackhpc/ansible-slurm-appliance/pull/473)
   may help identify any site-specific configuration. 
+
+- See the [hpctests docs](../ansible/roles/hpctests/README.md) for advice on
+  raising `hpctests_hpl_mem_frac` during tests.
diff --git a/environments/.stackhpc/inventory/extra_groups b/environments/.stackhpc/inventory/extra_groups
@@ -24,6 +24,9 @@ cluster
 login
 compute
 
+[chrony:children]
+cluster
+
 [tuned:children]
 # Install tuned into fat image
 builder
@@ -41,3 +44,6 @@ control
 
 [cacerts:children]
 cluster
+
+[compute_init:children]
+compute
diff --git a/environments/.stackhpc/tofu/main.tf b/environments/.stackhpc/tofu/main.tf
@@ -77,7 +77,7 @@ module "cluster" {
         standard: { # NB: can't call this default!
             nodes: ["compute-0", "compute-1"]
             flavor: var.other_node_flavor
-            compute_init_enable: ["compute", "etc_hosts", "nfs", "basic_users", "eessi", "tuned", "cacerts"]
+            compute_init_enable: ["compute", "chrony", "etc_hosts", "nfs", "basic_users", "eessi", "tuned", "cacerts"]
             ignore_image_changes: true
         }
         # Example of how to add another partition:
diff --git a/environments/common/inventory/groups b/environments/common/inventory/groups
@@ -176,3 +176,6 @@ extra_packages
 
 [cacerts]
 # Hosts to configure CA certificates and trusts on
+
+[chrony]
+# Hosts where crony configuration is applied. See docs/chrony.md for more details.
diff --git a/environments/common/layouts/everything b/environments/common/layouts/everything
@@ -93,9 +93,8 @@ cluster
 [sshd]
 # Hosts where the OpenSSH server daemon should be configured
 
-[compute_init:children]
+[compute_init]
 # EXPERIMENTAL: Compute hosts to enable joining cluster on boot on
-compute
 
 [k3s_server:children]
 # Hosts to run k3s server (should only be single node i.e control node)
@@ -119,3 +118,6 @@ builder
 
 [cacerts]
 # Hosts to configure CA certificates and trusts on
+
+[chrony]
+# Hosts where crony configuration is applied. See docs/chrony.md for more details.
diff --git a/requirements.yml b/requirements.yml
@@ -22,6 +22,8 @@ roles:
   - src: https://github.com/stackhpc/ansible-role-os-manila-mount.git
     name: stackhpc.os-manila-mount
     version: v25.1.1
+  - src: mrlesmithjr.chrony
+    version: v0.1.4
 
 collections:
   - name: containers.podman
