Merge branch 'main' into feat/lustre-compute-init

bertiethorpe · web-flow · commit f3aa9a30101c · 2025-02-25T11:37:51.000Z
diff --git a/.github/workflows/stackhpc.yml b/.github/workflows/stackhpc.yml
@@ -182,9 +182,8 @@ jobs:
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
-          ansible-playbook -v --limit compute ansible/adhoc/rebuild.yml
-          ansible-playbook -v ansible/ci/check_slurm.yml
           ansible-playbook -v ansible/adhoc/reboot_via_slurm.yml
+          ansible-playbook -v ansible/ci/check_slurm.yml
 
       - name: Check sacct state survived reimage
         run: |
diff --git a/ansible/bootstrap.yml b/ansible/bootstrap.yml
@@ -52,6 +52,13 @@
     - import_role:
         name: proxy
 
+- hosts: chrony
+  tags: chrony
+  become: yes
+  tasks:
+    - import_role:
+        name: mrlesmithjr.chrony
+
 - hosts: cluster
   gather_facts: false
   become: yes
diff --git a/ansible/roles/basic_users/README.md b/ansible/roles/basic_users/README.md
@@ -2,16 +2,19 @@
 basic_users
 ===========
 
-Setup users on cluster nodes using `/etc/passwd` and manipulating `$HOME`, i.e. without requiring LDAP etc. Features:
+Setup users on cluster nodes using `/etc/passwd` and manipulating `$HOME`, i.e.
+without requiring LDAP etc. Features:
 - UID/GID is consistent across cluster (and explicitly defined).
 - SSH key generated and propagated to all nodes to allow login between cluster nodes.
 - An "external" SSH key can be added to allow login from elsewhere.
-- Login to the control node is prevented.
+- Login to the control node is prevented (by default)
 - When deleting users, systemd user sessions are terminated first.
 
 Requirements
 ------------
-- $HOME (for normal users, i.e. not `centos`) is assumed to be on a shared filesystem.
+- `$HOME` (for normal users, i.e. not `rocky`) is assumed to be on a shared
+  filesystem. Actions affecting that shared filesystem are run on a single host,
+  see `basic_users_manage_homedir` below.
 
 Role Variables
 --------------
@@ -22,9 +25,15 @@ Role Variables
   - `shell` if *not* set will be `/sbin/nologin` on the `control` node and the default shell on other users. Explicitly setting this defines the shell for all nodes.
   - An additional key `public_key` may optionally be specified to define a key to log into the cluster.
   - An additional key `sudo` may optionally be specified giving a string (possibly multiline) defining sudo rules to be templated.
+  - `ssh_key_type` defaults to `ed25519` instead of the `ansible.builtin.user` default of `rsa`.
   - Any other keys may present for other purposes (i.e. not used by this role).
 - `basic_users_groups`: Optional, default empty list. A list of mappings defining information for each group. Mapping keys/values are passed through as parameters to [ansible.builtin.group](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/group_module.html) and default values are as given there.
 - `basic_users_override_sssd`: Optional bool, default false. Whether to disable `sssd` when ensuring users/groups exist with this role. Permits creating local users/groups even if they clash with users provided via sssd (e.g. from LDAP). Ignored if host is not in group `sssd` as well. Note with this option active `sssd` will be stopped and restarted each time this role is run.
+- `basic_users_manage_homedir`: Optional bool, must be true on a single host to
+  determine which host runs tasks affecting the shared filesystem. The default
+  is to use the first play host which is not the control node, because the
+  default NFS configuration does not have the shared `/home` directory mounted
+  on the control node.
 
 Dependencies
 ------------
diff --git a/ansible/roles/basic_users/defaults/main.yml b/ansible/roles/basic_users/defaults/main.yml
@@ -1,9 +1,10 @@
-basic_users_manage_homedir: "{{ (ansible_hostname == (ansible_play_hosts | first)) }}"
+basic_users_manage_homedir: "{{ ansible_hostname == (ansible_play_hosts | difference(groups['control']) | first) }}"
 basic_users_userdefaults:
   state: present
   create_home: "{{ basic_users_manage_homedir }}"
   generate_ssh_key:  "{{ basic_users_manage_homedir }}"
   ssh_key_comment: "{{ item.name }}"
+  ssh_key_type: ed25519
   shell: "{{'/sbin/nologin' if 'control' in group_names else omit }}"
 basic_users_users: []
 basic_users_groups: []
diff --git a/ansible/roles/basic_users/tasks/main.yml b/ansible/roles/basic_users/tasks/main.yml
@@ -46,21 +46,21 @@
     - item.state | default('present') == 'present'
     - item.public_key is defined
     - basic_users_manage_homedir
-  run_once: true
 
 - name: Write generated public key as authorized for SSH access
+  # this only runs on the basic_users_manage_homedir so has registered var
+  # from that host too
   authorized_key:
     user: "{{ item.name }}"
     state: present
     manage_dir: no
     key: "{{ item.ssh_public_key }}"
-  loop: "{{ hostvars[ansible_play_hosts | first].basic_users_info.results }}"
+  loop: "{{ basic_users_info.results }}"
   loop_control:
     label: "{{ item.name }}"
   when:
   - item.ssh_public_key is defined
   - basic_users_manage_homedir
-  run_once: true
 
 - name: Write sudo rules
   blockinfile:
diff --git a/ansible/roles/compute_init/README.md b/ansible/roles/compute_init/README.md
@@ -3,10 +3,13 @@
 Experimental functionality to allow compute nodes to rejoin the cluster after
 a reboot without running the `ansible/site.yml` playbook.
 
+**CAUTION:** The approach used here of exporting cluster secrets over NFS
+is considered to be a security risk due to the potential for cluster users to
+mount the share on a user-controlled machine by tunnelling through a login
+node. This feature should not be enabled on production clusters at this time.
+
 To enable this:
-1. Add the `compute` group (or a subset) into the `compute_init` group. This is
-   the default when using cookiecutter to create an environment, via the
-   "everything" template.
+1. Add the `compute` group (or a subset) into the `compute_init` group.
 2. Build an image which includes the `compute_init` group. This is the case
    for StackHPC-built release images.
 3. Enable the required functionalities during boot, by setting the
@@ -40,6 +43,7 @@ it also requires an image build with the role name added to the
 | bootstrap.yml            | (wait for ansible-init) | Not relevant during boot        | n/a                 |
 | bootstrap.yml            | resolv_conf             | Fully supported                 | No                  |
 | bootstrap.yml            | etc_hosts               | Fully supported                 | No                  |
+| bootstrap.yml            | chrony                  | Fully supported                 | No                  |
 | bootstrap.yml            | proxy                   | None at present                 | No                  |
 | bootstrap.yml            | (/etc permissions)      | None required - use image build | No                  |
 | bootstrap.yml            | (ssh /home fix)         | None required - use image build | No                  |
diff --git a/ansible/roles/compute_init/files/compute-init.yml b/ansible/roles/compute_init/files/compute-init.yml
@@ -18,6 +18,7 @@
     enable_lustre: "{{ os_metadata.meta.lustre | default(false) | bool }}"
     enable_basic_users: "{{ os_metadata.meta.basic_users | default(false) | bool }}"
     enable_eessi: "{{ os_metadata.meta.eessi | default(false) | bool }}"
+    enable_chrony: "{{ os_metadata.meta.chrony | default(false) | bool }}"
 
     # TODO: "= role defaults" - could be moved to a vars_file: on play with similar precedence effects
     resolv_conf_nameservers: []
@@ -101,6 +102,11 @@
 
       # TODO: should /mnt/cluster now be UNMOUNTED to avoid future hang-ups?
 
+    - name: Run chrony role
+      ansible.builtin.include_role:
+        name: mrlesmithjr.chrony
+      when: enable_chrony | bool
+
     - name: Configure resolve.conf
       block:
         - name: Set nameservers in /etc/resolv.conf
diff --git a/ansible/roles/compute_init/tasks/install.yml b/ansible/roles/compute_init/tasks/install.yml
@@ -45,6 +45,8 @@
       dest: tasks/nfs-clients.yml
     - src: ../../lustre
       dest: roles/
+    - src: ../../mrlesmithjr.chrony
+      dest: roles/
 
 - name: Add filter_plugins to ansible.cfg
   lineinfile:
diff --git a/ansible/roles/hpctests/README.md b/ansible/roles/hpctests/README.md
@@ -29,9 +29,22 @@ Role Variables
 - `hpctests_ucx_net_devices`: Optional. Control which network device/interface to use, e.g. `mlx5_1:0`. The default of `all` (as per UCX) may not be appropriate for multi-rail nodes with different bandwidths on each device. See [here](https://openucx.readthedocs.io/en/master/faq.html#what-is-the-default-behavior-in-a-multi-rail-environment) and [here](https://github.com/openucx/ucx/wiki/UCX-environment-parameters#setting-the-devices-to-use). Alternatively a mapping of partition name (as `hpctests_partition`) to device/interface can be used. For partitions not defined in the mapping the default of `all` is used.
 - `hpctests_outdir`: Optional. Directory to use for test output on local host. Defaults to `$HOME/hpctests` (for local user).
 - `hpctests_hpl_NB`: Optional, default 192. The HPL block size "NB" - for Intel CPUs see [here](https://software.intel.com/content/www/us/en/develop/documentation/onemkl-linux-developer-guide/top/intel-oneapi-math-kernel-library-benchmarks/intel-distribution-for-linpack-benchmark/configuring-parameters.html).
-- `hpctests_hpl_mem_frac`: Optional, default 0.8. The HPL problem size "N" will be selected to target using this fraction of each node's memory.
+- `hpctests_hpl_mem_frac`: Optional, default 0.3. The HPL problem size "N" will
+   be selected to target using this fraction of each node's memory -
+   **CAUTION: see note below**.
 - `hpctests_hpl_arch`: Optional, default 'linux64'. Arbitrary architecture name for HPL build. HPL is compiled on the first compute node of those selected (see `hpctests_nodes`), so this can be used to create different builds for different types of compute node.
 
+
+---
+**CAUTION**
+
+> The default of `hpctests_hpl_mem_frac=0.3` will not significantly load nodes.
+Values up to ~0.8 may be appropriate for a stress test but ensure cloud
+operators are aware in case this overloads e.g. power supplies or cooling.
+Values > 0.8 require longer runtimes and increase the risk of out-of-memory
+errors without normally significantly increasing the stress on the node.
+---
+
 The following variables should not generally be changed:
 - `hpctests_pre_cmd`: Optional. Command(s) to include in sbatch templates before module load commands.
 - `hpctests_pingmatrix_modules`: Optional. List of modules to load for pingmatrix test. Defaults are suitable for OpenHPC 2.x cluster using the required packages.
diff --git a/ansible/roles/hpctests/defaults/main.yml b/ansible/roles/hpctests/defaults/main.yml
@@ -9,7 +9,7 @@ hpctests_outdir: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/hpctests"
 hpctests_ucx_net_devices: all
 hpctests_hpl_version: "2.3"
 hpctests_hpl_NB: 192
-hpctests_hpl_mem_frac: 0.8
+hpctests_hpl_mem_frac: 0.3
 hpctests_hpl_arch: linux64
 #hpctests_nodes:
 #hpctests_partition:
diff --git a/ansible/roles/proxy/README.md b/ansible/roles/proxy/README.md
@@ -6,6 +6,6 @@ Define http/s proxy configuration.
 
 - `proxy_http_proxy`: Required. Address of http proxy. E.g. "http://10.1.0.28:3128" for a Squid proxy on default port.
 - `proxy_https_proxy`: Optional. Address of https proxy. Default is `{{ proxy_http_proxy }}`.
-- `proxy_no_proxy`: Optional. Comma-separated list of addresses not to proxy. Default is to concatenate `inventory_hostname` (for hostnames) and `ansible_host` (for host IPs) for all Ansible hosts.
+- `proxy_no_proxy_extra`: Optional. List of additional addresses not to proxy. Will be combined with default list which includes `inventory_hostname` (for hostnames) and `ansible_host` (for host IPs) for all Ansible hosts.
 - `proxy_dnf`: Optional bool. Whether to configure yum/dnf proxying through `proxy_http_proxy`. Default `true`.
 - `proxy_systemd`: Optional bool. Whether to give processes started by systemd the above http, https and no_proxy configuration. **NB** Running services will need restarting if this is changed. Default `true`.
diff --git a/ansible/roles/proxy/defaults/main.yml b/ansible/roles/proxy/defaults/main.yml
@@ -1,5 +1,7 @@
-# proxy_http_proxy: 
+# proxy_http_proxy:
 proxy_https_proxy: "{{ proxy_http_proxy }}"
-proxy_no_proxy: "{{ (['localhost', '127.0.0.1'] + groups['all'] + hostvars.values() | map(attribute='ansible_host')) | sort | join(',') }}"
+proxy_no_proxy_defaults: "{{ ['localhost', '127.0.0.1'] + groups['all'] + hostvars.values() | map(attribute='ansible_host') }}"
+proxy_no_proxy_extras: []
+proxy_no_proxy: "{{ (proxy_no_proxy_defaults + proxy_no_proxy_extras) | sort | join(',') }}"
 proxy_dnf: true
 proxy_systemd: true
diff --git a/dev/ansible-ssh b/dev/ansible-ssh
@@ -11,7 +11,7 @@ import os
 from collections import defaultdict
 
 def _optional_arg(prototype, *values):
-    # returns empty string if any of the the values are falsey
+    # returns empty string if any of the values are falsey
     filtered = [value for value in values if value]
     return prototype.format(*values) if len(values) == len(filtered) else ""
 
diff --git a/dev/setup-env.sh b/dev/setup-env.sh
@@ -2,28 +2,30 @@
 
 set -euo pipefail
 
-if [[ -f /etc/os-release ]]; then
-    . /etc/os-release
-    OS=$ID
-    OS_VERSION=$VERSION_ID
-else
-    exit 1
-fi
+PYTHON_VERSION=${PYTHON_VERSION:-}
 
-MAJOR_VERSION=$(echo $OS_VERSION | cut -d. -f1)
+if [[ "$PYTHON_VERSION" == "" ]]; then
+    if [[ -f /etc/os-release ]]; then
+        . /etc/os-release
+        OS=$ID
+        OS_VERSION=$VERSION_ID
+    else
+        exit 1
+    fi
 
-PYTHON_VERSION=""
+    MAJOR_VERSION=$(echo $OS_VERSION | cut -d. -f1)
 
-if [[ "$OS" == "ubuntu" && "$MAJOR_VERSION" == "22" ]]; then
-    PYTHON_VERSION="/usr/bin/python3.10"
-elif [[ "$OS" == "rocky" && "$MAJOR_VERSION" == "8" ]]; then
-    # python3.9+ doesn't have selinux bindings
-    PYTHON_VERSION="/usr/bin/python3.8" # use `sudo yum install python38` on Rocky Linux 8 to install this
-elif [[ "$OS" == "rocky" && "$MAJOR_VERSION" == "9" ]]; then
-    PYTHON_VERSION="/usr/bin/python3.9"
-else
-    echo "Unsupported OS version: $OS $MAJOR_VERSION"
-    exit 1
+    if [[ "$OS" == "ubuntu" && "$MAJOR_VERSION" == "22" ]]; then
+        PYTHON_VERSION="/usr/bin/python3.10"
+    elif [[ "$OS" == "rocky" && "$MAJOR_VERSION" == "8" ]]; then
+        # python3.9+ doesn't have selinux bindings
+        PYTHON_VERSION="/usr/bin/python3.8" # use `sudo yum install python38` on Rocky Linux 8 to install this
+    elif [[ "$OS" == "rocky" && "$MAJOR_VERSION" == "9" ]]; then
+        PYTHON_VERSION="/usr/bin/python3.9"
+    else
+        echo "Unsupported OS version: $OS $MAJOR_VERSION"
+        exit 1
+    fi
 fi
 
 if [[ ! -d "venv" ]]; then
diff --git a/docs/chrony.md b/docs/chrony.md
@@ -0,0 +1,21 @@
+# Chrony configuration
+
+Use variables from the [mrlesmithjr.chrony](https://github.com/mrlesmithjr/ansible-chrony) role.
+
+For example in: `environments/<environment>/inventory/group_vars/all/chrony`:
+
+```
+---
+chrony_ntp_servers:
+  - server: ntp-0.example.org
+    options:
+      - option: iburst
+      - option: minpoll
+        val: 8
+  - server: ntp-1.example.org
+    options:
+      - option: iburst
+      - option: minpoll
+        val: 8
+
+```
diff --git a/docs/openondemand.md b/docs/openondemand.md
@@ -30,7 +30,7 @@ The above functionality is configured by running the `ansible/portal.yml` playbo
 
 See the [ansible/roles/openondemand/README.md](../ansible/roles/openondemand/README.md) for more details on the variables described below.
 
-The following variables have been given default values to allow Open OnDemand to work in a newly created environment without additional configuration, but generally should be overridden in `environment/site/inventory/group_vars/all/` with site-specific values:
+The following variables have been given default values to allow Open OnDemand to work in a newly created environment without additional configuration, but generally should be overridden in `environments/site/inventory/group_vars/all/` with site-specific values:
 - `openondemand_servername` - this must be defined for both `openondemand` and `grafana` hosts (when Grafana is enabled). Default is `ansible_host` (i.e. the IP address) of the first host in the `openondemand` group.
 - `openondemand_auth` and any corresponding options. Defaults to `basic_pam`.
 - `openondemand_desktop_partition` and `openondemand_jupyter_partition` if the corresponding inventory groups are defined. Defaults to the first compute group defined in the `compute` OpenTofu variable in `environments/$ENV/tofu`.
diff --git a/docs/production.md b/docs/production.md
@@ -67,7 +67,7 @@ and referenced from the `site` and `production` environments, e.g.:
 - Vault-encrypt secrets. Running the `generate-passwords.yml` playbook creates
   a secrets file at `environments/$ENV/inventory/group_vars/all/secrets.yml`.
   To ensure staging environments are a good model for production this should
-  generally be moved into the `site` environment. It should be be encrypted
+  generally be moved into the `site` environment. It should be encrypted
   using [Ansible vault](https://docs.ansible.com/ansible/latest/user_guide/vault.html)
   and then committed to the repository.
 
@@ -96,7 +96,7 @@ and referenced from the `site` and `production` environments, e.g.:
     cluster
     ```
 
-- Configure Open OnDemand - see [specific documentation](openondemand.README.md).
+- Configure Open OnDemand - see [specific documentation](openondemand.md).
 
 - Remove the `demo_user` user from `environments/$ENV/inventory/group_vars/all/basic_users.yml`
 
@@ -127,3 +127,6 @@ and referenced from the `site` and `production` environments, e.g.:
 
 - Note [PR 473](https://github.com/stackhpc/ansible-slurm-appliance/pull/473)
   may help identify any site-specific configuration. 
+
+- See the [hpctests docs](../ansible/roles/hpctests/README.md) for advice on
+  raising `hpctests_hpl_mem_frac` during tests.
diff --git a/environments/.stackhpc/inventory/extra_groups b/environments/.stackhpc/inventory/extra_groups
@@ -24,6 +24,9 @@ cluster
 login
 compute
 
+[chrony:children]
+cluster
+
 [tuned:children]
 # Install tuned into fat image
 builder
@@ -41,3 +44,6 @@ control
 
 [cacerts:children]
 cluster
+
+[compute_init:children]
+compute
diff --git a/environments/.stackhpc/tofu/main.tf b/environments/.stackhpc/tofu/main.tf
@@ -80,7 +80,7 @@ module "cluster" {
         standard: { # NB: can't call this default!
             nodes: ["compute-0", "compute-1"]
             flavor: var.other_node_flavor
-            compute_init_enable: ["compute", "etc_hosts", "nfs", "basic_users", "eessi", "tuned", "cacerts"]
+            compute_init_enable: ["compute", "chrony", "etc_hosts", "nfs", "basic_users", "eessi", "tuned", "cacerts"]
             ignore_image_changes: true
         }
         # Example of how to add another partition:
diff --git a/environments/README.md b/environments/README.md
@@ -94,7 +94,7 @@ varibles are set e.g role variables for the `stackhpc.nfs` role can be found in
 
 ## Parent pointers
 
-As the environments form a chain, a symlink pointing to the parent can be be created.
+As the environments form a chain, a symlink pointing to the parent can be created.
 
     `ln -s ../common/ parent`
 
diff --git a/environments/common/inventory/groups b/environments/common/inventory/groups
@@ -168,3 +168,6 @@ extra_packages
 
 [cacerts]
 # Hosts to configure CA certificates and trusts on
+
+[chrony]
+# Hosts where crony configuration is applied. See docs/chrony.md for more details.
diff --git a/environments/common/layouts/everything b/environments/common/layouts/everything
@@ -93,9 +93,8 @@ cluster
 [sshd]
 # Hosts where the OpenSSH server daemon should be configured
 
-[compute_init:children]
+[compute_init]
 # EXPERIMENTAL: Compute hosts to enable joining cluster on boot on
-compute
 
 [k3s:children]
 # Hosts to run k3s server/agent
@@ -114,3 +113,6 @@ builder
 
 [cacerts]
 # Hosts to configure CA certificates and trusts on
+
+[chrony]
+# Hosts where crony configuration is applied. See docs/chrony.md for more details.
diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/control.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/control.tf
@@ -14,7 +14,8 @@ resource "openstack_networking_port_v2" "control" {
     subnet_id = data.openstack_networking_subnet_v2.cluster_subnet[each.key].id
   }
 
-  security_group_ids = [for o in data.openstack_networking_secgroup_v2.nonlogin: o.id]
+  port_security_enabled = lookup(each.value, "port_security_enabled", true)
+  security_group_ids = lookup(each.value, "port_security_enabled", true) ? [for o in data.openstack_networking_secgroup_v2.nonlogin: o.id] : []
 
   binding {
     vnic_type = lookup(var.vnic_types, each.key, "normal")
diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/nodes.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/nodes.tf
diff --git a/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf b/environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf
diff --git a/requirements.yml b/requirements.yml
