Merge branch 'main' into update/timestamps

Author: bertiethorpe

.github/workflows/stackhpc.yml

@@ -43,8 +43,18 @@ jobs:
       TF_VAR_cluster_name: slurmci-${{ matrix.os_version }}-${{ github.run_number }}
       CI_CLOUD: ${{ vars.CI_CLOUD }} # default from repo settings
       TF_VAR_os_version: ${{ matrix.os_version }}
+      STACKHPC_TF_DIR: environments/.stackhpc/tofu
     steps:
-      - uses: actions/checkout@v2
+
+      - name: Find the latest release
+        run: |
+          echo LATEST_RELEASE_TAG=$(curl -s https://api.github.com/repos/stackhpc/ansible-slurm-appliance/releases/latest | jq -r .tag_name) >> "$GITHUB_ENV"
+      
+      - name: Checkout latest release
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.LATEST_RELEASE_TAG }}
+          fetch-depth: 0
 
       - name: Override CI_CLOUD if PR label is present
         if: ${{ github.event_name == 'pull_request' }}
@@ -60,9 +70,10 @@ jobs:
             fi
           done
 
-      - name: Record settings for CI cloud
+      - name: Record debug info
         run: |
-          echo CI_CLOUD: ${{ env.CI_CLOUD }}
+          echo LATEST_RELEASE_TAG: $LATEST_RELEASE_TAG
+          echo CI_CLOUD: $CI_CLOUD
 
       - name: Setup ssh
         run: |
@@ -76,7 +87,7 @@ jobs:
         run: cat environments/.stackhpc/bastion_fingerprints >> ~/.ssh/known_hosts
         shell: bash
 
-      - name: Install ansible etc
+      - name: Install ansible, pip and galaxy requirements
         run: dev/setup-env.sh
 
       - name: Install OpenTofu
@@ -86,7 +97,7 @@ jobs:
 
       - name: Initialise tofu
         run: tofu init
-        working-directory: ${{ github.workspace }}/environments/.stackhpc/tofu
+        working-directory: ${{ env.STACKHPC_TF_DIR }}
 
       - name: Write clouds.yaml
         run: |
@@ -103,42 +114,90 @@ jobs:
         env:
           DEMO_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
-      - name: Provision nodes using fat image
+      - name: Provision nodes using latest release image
         id: provision_servers
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
-          cd $APPLIANCES_ENVIRONMENT_ROOT/tofu
+          cd $STACKHPC_TF_DIR
           tofu apply -auto-approve -var-file="${{ env.CI_CLOUD }}.tfvars"
 
       - name: Delete infrastructure if provisioning failed
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
-          cd $APPLIANCES_ENVIRONMENT_ROOT/tofu
+          cd $STACKHPC_TF_DIR
           tofu destroy -auto-approve -var-file="${{ env.CI_CLOUD }}.tfvars"
         if: failure() && steps.provision_servers.outcome == 'failure'
 
-      - name: Configure cluster
+      - name: Configure cluster at latest release
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
           ansible all -m wait_for_connection
           ansible-playbook -v ansible/site.yml
           ansible-playbook -v ansible/ci/check_slurm.yml
 
-      - name: Run MPI-based tests
+      - name: Run MPI-based tests at latest release
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
-          ansible-playbook -vv ansible/adhoc/hpctests.yml
+          ansible-playbook -vv ansible/adhoc/hpctests.yml --tags pingpong
 
       # - name: Run EESSI tests
       #   run: |
       #     . venv/bin/activate
       #     . environments/.stackhpc/activate
       #     ansible-playbook -vv ansible/ci/check_eessi.yml
 
+      - name: Checkout current branch
+        run: git checkout ${{ github.head_ref || github.ref_name }}
+
+      - name: Update ansible, pip and galaxy requirements
+        run: dev/setup-env.sh
+
+      - name: Reimage login and control nodes to image in current branch
+        id: reimage_non_compute
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+          cd $STACKHPC_TF_DIR
+          tofu init
+          tofu apply -auto-approve -var-file="${{ env.CI_CLOUD }}.tfvars"
+          
+      - name: Configure cluster using current branch
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+          ansible all -m wait_for_connection
+          ansible-playbook -v ansible/site.yml
+          ansible-playbook -v ansible/ci/check_slurm.yml
+
+      - name: Reimage compute nodes to image in current branch using slurm - tests compute-init
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+          ansible-playbook -v ansible/adhoc/reboot_via_slurm.yml
+          ansible-playbook -v ansible/ci/check_slurm.yml
+
+      - name: Check sacct state survived reimage to current branch
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+          ansible-playbook -vv ansible/ci/check_sacct_hpctests.yml
+
+      - name: Check MPI-based tests are shown in Grafana
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+          ansible-playbook -vv ansible/ci/check_grafana.yml
+
+      - name: Run MPI-based tests again in current branch
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+          ansible-playbook -vv ansible/adhoc/hpctests.yml
+
       - name: Confirm Open Ondemand is up (via SOCKS proxy)
         run: |
           . venv/bin/activate
@@ -170,43 +229,10 @@ jobs:
         env:
           DEMO_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
-      - name: Test reimage of login and control nodes (via rebuild adhoc)
-        run: |
-          . venv/bin/activate
-          . environments/.stackhpc/activate
-          ansible-playbook -v --limit control,login ansible/adhoc/rebuild.yml
-          ansible-playbook -v ansible/site.yml
-          ansible-playbook -v ansible/ci/check_slurm.yml
-
-      - name: Test compute node reboot and compute-init
-        run: |
-          . venv/bin/activate
-          . environments/.stackhpc/activate
-          ansible-playbook -v ansible/adhoc/reboot_via_slurm.yml
-          ansible-playbook -v ansible/ci/check_slurm.yml
-
-      - name: Check sacct state survived reimage
-        run: |
-          . venv/bin/activate
-          . environments/.stackhpc/activate
-          ansible-playbook -vv ansible/ci/check_sacct_hpctests.yml
-
-      - name: Check MPI-based tests are shown in Grafana
-        run: |
-          . venv/bin/activate
-          . environments/.stackhpc/activate
-          ansible-playbook -vv ansible/ci/check_grafana.yml
-
       - name: Delete infrastructure
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
-          cd $APPLIANCES_ENVIRONMENT_ROOT/tofu
-          tofu destroy -auto-approve -var-file="${{ env.CI_CLOUD }}.tfvars"
+          cd $STACKHPC_TF_DIR
+          tofu destroy -auto-approve -var-file="${{ env.CI_CLOUD }}.tfvars" || echo "tofu failed in $STACKHPC_TF_DIR"
         if: ${{ success() || cancelled() }}
-
-      # - name: Delete images
-      #   run: |
-      #     . venv/bin/activate
-      #     . environments/.stackhpc/activate
-      #     ansible-playbook -vv ansible/ci/delete_images.yml

README.md

@@ -31,7 +31,7 @@ It requires an OpenStack cloud, and an Ansible "deploy host" with access to that
 
 Before starting ensure that:
 - You have root access on the deploy host.
-- You can create instances from the [latest Slurm appliance image](https://github.com/stackhpc/ansible-slurm-appliance/releases), which already contains the required packages. This is built and tested in StackHPC's CI. Although you can use a Rocky Linux 9 GenericCloud instead, it is not recommended.
+- You can create instances from the [latest Slurm appliance image](https://github.com/stackhpc/ansible-slurm-appliance/releases), which already contains the required packages. This is built and tested in StackHPC's CI.
 - You have an SSH keypair defined in OpenStack, with the private key available on the deploy host.
 - Created instances have access to internet (note proxies can be setup through the appliance if necessary).
 - Created instances have accurate/synchronised time (for VM instances this is usually provided by the hypervisor; if not or for bare metal instances it may be necessary to configure a time service via the appliance).
@@ -49,6 +49,7 @@ These instructions assume the deployment host is running Rocky Linux 8:
     sudo yum install -y git python38
     git clone https://github.com/stackhpc/ansible-slurm-appliance
     cd ansible-slurm-appliance
+    git checkout ${latest-release-tag}
     ./dev/setup-env.sh
 
 You will also need to install [OpenTofu](https://opentofu.org/docs/intro/install/rpm/).

ansible/.gitignore

@@ -84,5 +84,7 @@ roles/*
 !roles/pytools/**
 !roles/rebuild/
 !roles/rebuild/**
+!roles/slurm_tools/
+!roles/slurm_tools/**
 !roles/gateway/
 !roles/gateway/**

ansible/adhoc/cudatests.yml

@@ -1,6 +1,6 @@
 - hosts: cuda
   become: yes
-  gather_facts: no
+  gather_facts: yes
   tags: cuda_samples
   tasks:
     - import_role:

ansible/ci/check_grafana.yml

@@ -23,4 +23,4 @@
       delay: 5
       vars:
         _found_jobs: "{{ _slurm_stats_jobs.docs | map(attribute='JobName', default='(json error in slurmstats data)') }}"
-        _expected_jobs: ['hpl-solo.sh', 'pingpong.sh', 'pingmatrix.sh']
+        _expected_jobs: ['pingpong.sh']

ansible/ci/check_sacct_hpctests.yml

@@ -5,10 +5,6 @@
     sacct_stdout_expected: |- # based on CI running hpctests as the first job
       JobID,JobName,State
       1,pingpong.sh,COMPLETED
-      2,pingmatrix.sh,COMPLETED
-      3,hpl-build-linux64.sh,COMPLETED
-      4_0,hpl-solo.sh,COMPLETED
-      4_1,hpl-solo.sh,COMPLETED
   tasks:
     - name: Get info for ended jobs
       shell:

ansible/roles/basic_users/README.md

@@ -11,44 +11,52 @@ without requiring LDAP etc. Features:
 - Login to the control node is prevented (by default).
 - When deleting users, systemd user sessions are terminated first.
 
-> [!IMPORTANT] This role assumes that `$HOME` for users managed by this role
-(e.g. not `rocky` and other system users) is on a shared filesystem. The export
-of this shared filesystem may be root squashed if its server is in the
-`basic_user` group - see configuration examples below.
+> [!IMPORTANT] The defaults for this role assumes that `$HOME` for users
+managed by this role (e.g. not `rocky` and other system users) is on a shared
+filesystem. The export of this shared filesystem may be root squashed if its
+server is in the `basic_user` group - see configuration examples below.
 
 Role Variables
 --------------
 
-- `basic_users_users`: Optional, default empty list. A list of mappings defining information for each user. In general, mapping keys/values are passed through as parameters to [ansible.builtin.user](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/user_module.html) and default values are as given there. However:
-  - `create_home` and `generate_ssh_key`: Normally set automatically. Can be
-    set `false` if necessary to disable home directory creation/cluster ssh
-    key creation. Should not be set `true` to avoid trying to modify home
-    directories from multiple nodes simultaneously.
+- `basic_users_homedir_server`: Optional inventory hostname in the `basic_users`
+  group defining the host to use to create home directories. If the home
+  directory export is root squashed, this host *must* be the home directory
+  server. Default is the `control` node which is appropriate for the default
+  appliance configuration. Not relevant if `create_home` is false for all users.
+- `basic_users_homedir_server_path`: Optional path prefix for home directories on
+   the `basic_users_homedir_server`, i.e. on the "server side". Default is
+   `/exports/home` which is appropriate for the default appliance configuration.
+- `basic_users_homedir_client`: Optional inventory hostname in the `basic_users`
+  group defining the host to use to create ssh keys etc in home directories.
+  This should be a host mounting the home directories. Default is the first
+  node in the `login` group which is appropriate for the default appliance
+  configuration.
+- `basic_users_users`: Optional, default empty list. A list of mappings defining
+   information for each user. In general, mapping keys/values are passed through
+   as parameters to [ansible.builtin.user](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/user_module.html)
+   and default values are as given there, with the following differences:
+  - `generate_ssh_key`: Default is `true`, and the generated key is added to
+     the user's authorized keys.
   - `ssh_key_comment`: Default is user name.
   - `home`: Set automatically based on the user name and
-    `basic_users_homedir_host_path`. Can be overriden if required for e.g.
-     users with non-standard home directory paths.
+    `basic_users_homedir_server_path`. Can be overriden for users with
+     non-standard home directory paths.
   - `uid`: Should be set, so that the UID/GID is consistent across the cluster
     (which Slurm requires).
   - `shell`: If *not* set will be `/sbin/nologin` on the `control` node to
      prevent users logging in to this node, and the default shell on other
      nodes. Explicitly setting this defines the shell for all nodes and if the
      shared home directories are mounted on the control node will allow the
      user to log in to the control node.
-  - An additional key `public_key` may optionally be specified to define a key to log into the cluster.
-  - An additional key `sudo` may optionally be specified giving a string (possibly multiline) defining sudo rules to be templated.
-  - `ssh_key_type` defaults to `ed25519` instead of the `ansible.builtin.user` default of `rsa`.
+  - `public_key`: Optional, define a key to log into the cluster with.
+  - `sudo`: Optional, a (possibly multiline) string defining sudo rules for the
+     user.
+  - `ssh_key_type` defaults to `ed25519` instead of the `ansible.builtin.user`
+     default of `rsa`.
   - Any other keys may present for other purposes (i.e. not used by this role).
 - `basic_users_groups`: Optional, default empty list. A list of mappings defining information for each group. Mapping keys/values are passed through as parameters to [ansible.builtin.group](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/group_module.html) and default values are as given there.
 - `basic_users_override_sssd`: Optional bool, default false. Whether to disable `sssd` when ensuring users/groups exist with this role. Permits creating local users/groups even if they clash with users provided via sssd (e.g. from LDAP). Ignored if host is not in group `sssd` as well. Note with this option active `sssd` will be stopped and restarted each time this role is run.
-- `basic_users_homedir_host`: Optional inventory hostname defining the host
-  to use to create home directories. If the home directory export is root
-  squashed, this host *must* be the home directory server. Default is the
-  `control` node which is appropriate for the default appliance configuration.
-  Not relevant if `create_home` is false for all users.
-- `basic_users_homedir_host_path`: Optional path prefix for home directories on
-   the `basic_users_homedir_host`, i.e. on the "server side". Default is
-   `/exports/home` which is appropriate for the default appliance configuration.
 
 Dependencies
 ------------

ansible/roles/basic_users/defaults/main.yml

@@ -1,6 +1,6 @@
-basic_users_homedir_host: "{{ groups['control'] | first }}" # no way, generally, to find the nfs_server
-basic_users_homedir_host_path: /exports/home
-# _basic_users_manage_homedir: "{{ ansible_hostname == basic_users_homedir_host }}"
+basic_users_homedir_server: "{{ groups['control'] | first }}" # no way, generally, to find the nfs_server
+basic_users_homedir_server_path: /exports/home
+basic_users_homedir_client: "{{ groups['login'] | first }}"  
 basic_users_userdefaults:
   state: present # need this here so don't have to add default() everywhere
   generate_ssh_key:  true