Merge branch 'feat/slurm-rebuild' into docs/compute-init-roles-v2

Author: bertiethorpe

.github/workflows/stackhpc.yml

@@ -178,12 +178,13 @@ jobs:
           ansible-playbook -v ansible/site.yml
           ansible-playbook -v ansible/ci/check_slurm.yml
 
-      - name: Test reimage of compute nodes and compute-init (via rebuild adhoc)
+      - name: Test compute node reimage, compute-init, and reboot
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
           ansible-playbook -v --limit compute ansible/adhoc/rebuild.yml
           ansible-playbook -v ansible/ci/check_slurm.yml
+          ansible-playbook -v ansible/adhoc/reboot_via_slurm.yml
 
       - name: Check sacct state survived reimage
         run: |

ansible/adhoc/reboot_via_slurm.yml

@@ -0,0 +1,23 @@
+# Reboot compute nodes via slurm. The nodes will be rebuilt if image in hostvars is different to the active one in OpenStack.
+# Example:
+#   ansible-playbook -v ansible/adhoc/reboot_via_slurm.yml
+
+- hosts: login
+  become: yes
+  gather_facts: no
+  tasks:
+    - name: Submit a Slurm job to reboot compute nodes
+      ansible.builtin.shell: |
+        set -e
+        srun --reboot -N 2 uptime
+      become_user: root
+      register: slurm_result
+      failed_when: slurm_result.rc != 0
+
+    - name: Fetch Slurm logs if reboot fails
+      ansible.builtin.shell: |
+        journalctl -u slurmctld --since "10 minutes ago" | tail -n 50
+      become_user: root
+      register: slurm_logs
+      when: slurm_result.rc != 0
+      delegate_to: "{{ groups['control'] | first }}"

ansible/roles/compute_init/tasks/export.yml

@@ -2,9 +2,9 @@
   file:
     path: /exports/cluster
     state: directory
-    owner: root
+    owner: slurm
     group: root
-    mode: u=rwX,go=
+    mode: u=rX,g=rwX,o=
   run_once: true
   delegate_to: "{{ groups['control'] | first }}"
 
@@ -23,21 +23,27 @@
   file:
     path: /exports/cluster/hostvars/{{ inventory_hostname }}/
     state: directory
-    mode: u=rwX,go=
-    # TODO: owner,mode,etc
+    owner: slurm
+    group: root
+    mode: u=rX,g=rwX,o=
   delegate_to: "{{ groups['control'] | first }}"
 
 - name: Template out hostvars
   template:
     src: hostvars.yml.j2
     dest: /exports/cluster/hostvars/{{ inventory_hostname }}/hostvars.yml
-    mode: u=rw,go=
+    owner: slurm
+    group: root
+    mode: u=r,g=rw,o=
   delegate_to: "{{ groups['control'] | first }}"
 
 - name: Copy manila share info to /exports/cluster
   copy:
     content: "{{ os_manila_mount_share_info_var | to_nice_yaml }}"
     dest: /exports/cluster/manila_share_info.yml
+    owner: root
+    group: root
+    mode: u=rw,g=r
   run_once: true
   delegate_to: "{{ groups['control'] | first }}"
   when: os_manila_mount_share_info is defined

ansible/roles/k3s/files/start_k3s.yml

@@ -5,6 +5,7 @@
     k3s_token: "{{ os_metadata.meta.k3s_token }}"
     k3s_server_name: "{{ os_metadata.meta.control_address }}"
     service_name: "{{ 'k3s-agent' if k3s_server_name is defined else 'k3s' }}"
+    access_ip: "{{ os_metadata.meta.access_ip }}"
   tasks:
     - name: Ensure password directory exists
       ansible.builtin.file: 
@@ -22,6 +23,13 @@
         path: "/etc/systemd/system/{{ service_name }}.service.env"
         line: "K3S_TOKEN={{ k3s_token }}"
 
+    - name: Add the node IP to the environment
+      # NB this isn't natively setable via envvars, have to modify
+      # INSTALL_K3S_EXEC to support it
+      ansible.builtin.lineinfile:
+        path: "/etc/systemd/system/{{ service_name }}.service.env"
+        line: "K3S_NODE_IP={{ access_ip }}"
+
     - name: Add server url to agents
       ansible.builtin.lineinfile:
         path: "/etc/systemd/system/{{ service_name }}.service.env"

ansible/roles/k3s/tasks/install.yml

@@ -47,7 +47,7 @@
       cmd: /usr/bin/k3s-install.sh
     environment:
       INSTALL_K3S_VERSION: "{{ k3s_version }}"
-      INSTALL_K3S_EXEC: "{{ item }}"
+      INSTALL_K3S_EXEC: "{{ item }} --node-ip=${K3S_NODE_IP}"
       INSTALL_K3S_SKIP_START: "true"
       INSTALL_K3S_SKIP_ENABLE: "true"
       INSTALL_K3S_BIN_DIR: "/usr/bin"

ansible/roles/rebuild/tasks/main.yml

@@ -4,17 +4,17 @@
   file:
     path: /etc/openstack
     state: directory
-    owner: root
+    owner: slurm
     group: root
-    mode: '0400'
+    mode: u=rX,g=rwX
 
 - name: Copy out clouds.yaml
   copy:
     src: "{{ openhpc_rebuild_clouds }}"
     dest: /etc/openstack/clouds.yaml
-    owner: root
+    owner: slurm
     group: root
-    mode: '0400'
+    mode: u=r,g=rw
 
 - name: Setup slurm tools
   include_role:

ansible/roles/slurm_tools/tasks/main.yml

@@ -27,7 +27,7 @@
   module_defaults:
     ansible.builtin.pip:
       virtualenv: /opt/slurm-tools
-      virtualenv_command: python3 -m venv
+      virtualenv_command: "{{ 'python3.9 -m venv' if ansible_distribution_major_version == '8' else 'python3 -m venv' }}"
       state: latest
   become: true
   become_user: "{{ pytools_user }}"

ansible/slurm.yml

@@ -10,10 +10,11 @@
         name:  mysql
 
 - name: Setup slurm-driven rebuild
-  hosts: rebuild
+  hosts: rebuild:!builder
   become: yes
   tags:
     - rebuild
+    - openhpc
   tasks:
     - import_role:
         name: rebuild

docs/networks.md

@@ -0,0 +1,102 @@
+# Networking
+
+The default OpenTofu configurations in the appliance do not provision networks,
+subnets or associated infrastructure such as routers. The requirements are that:
+1. At least one network exists.
+2. The first network defined spans all nodes, referred to as the "access network".
+3. Only one subnet per network is attached to nodes.
+4. At least one network on each node provides outbound internet access (either
+directly, or via a proxy).
+
+Futhermore, it is recommended that the deploy host has an interface on the
+access network. While it is possible to e.g. use a floating IP on a login node
+as an SSH proxy to access the other nodes, this can create problems in recovering
+the cluster if the login node is unavailable and can make Ansible problems harder
+to debug.
+
+This page describes supported configurations and how to implement them using
+the OpenTofu variables. These will normally be set in
+`environments/site/tofu/terraform.tfvars` for the site base environment. If they
+need to be overriden for specific environments, this can be done via an OpenTofu
+module as discussed [here](./production.md).
+
+Note that if an OpenStack subnet has a gateway IP defined then nodes with ports
+attached to that subnet will get a default route set via that gateway.
+
+## Single network
+This is the simplest possible configuration. A single network and subnet is
+used for all nodes. The subnet provides outbound internet access via the default
+route defined by the subnet gateway (often an OpenStack router to an external
+network).
+
+```terraform
+cluster_networks = [
+  {
+    network = "netA"
+    subnet = "subnetA"
+  }
+]
+...
+```
+
+## Multiple homogenous networks
+This is similar to the above, except each node has multiple networks. The first
+network, "netA" is the access network. Note that only one subnet must have a
+gateway defined, else default routes via both subnets will be present causing
+routing problems. It also shows the second network (netB) using direct-type
+vNICs for RDMA.
+
+```terraform
+cluster_networks = [
+  {
+    network = "netA"
+    subnet = "subnetA"
+  },
+  {
+    network = "netB"
+    subnet = "subnetB"
+  },
+]
+
+vnic_types = {
+    netB = "direct"
+}
+...
+```
+
+
+## Additional networks on some nodes
+
+This example shows how to modify variables for specific node groups. In this
+case a baremetal node group has a second network attached. As above, only a
+single subnet can have a gateway IP.
+
+```terraform
+cluster_networks = [
+  {
+    network = "netA"
+    subnet = "subnetA"
+  }
+]
+
+compute = {
+  general = {
+    nodes = ["general-0", "general-1"]
+  }
+  baremetal = {
+    nodes = ["baremetal-0", "baremetal-1"]
+    extra_networks = [
+      {
+        network = "netB"
+        subnet = "subnetB"
+      }
+    ]
+    vnic_types = {
+        netA = "baremetal"
+        netB = "baremetal"
+    ...
+    }
+  }
+}
+...
+```

environments/.stackhpc/tofu/LEAFCLOUD.tfvars

@@ -1,5 +1,9 @@
-cluster_net = "slurmapp-ci"
-cluster_subnet = "slurmapp-ci"
+cluster_networks = [
+    {
+        network = "slurmapp-ci"
+        subnet = "slurmapp-ci"
+    }
+]
 control_node_flavor = "ec1.medium" # small ran out of memory, medium gets down to ~100Mi mem free on deployment
 other_node_flavor = "en1.xsmall"
 state_volume_type = "unencrypted"