Merge branch 'main' into temp/try-image-build-without-volume

bertiethorpe · web-flow · commit d6b45133149c · 2025-04-01T14:34:22.000+01:00
diff --git a/.github/workflows/stackhpc.yml b/.github/workflows/stackhpc.yml
@@ -43,8 +43,18 @@ jobs:
       TF_VAR_cluster_name: slurmci-${{ matrix.os_version }}-${{ github.run_number }}
       CI_CLOUD: ${{ vars.CI_CLOUD }} # default from repo settings
       TF_VAR_os_version: ${{ matrix.os_version }}
+      STACKHPC_TF_DIR: environments/.stackhpc/tofu
     steps:
-      - uses: actions/checkout@v2
+
+      - name: Find the latest release
+        run: |
+          echo LATEST_RELEASE_TAG=$(curl -s https://api.github.com/repos/stackhpc/ansible-slurm-appliance/releases/latest | jq -r .tag_name) >> "$GITHUB_ENV"
+      
+      - name: Checkout latest release
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.LATEST_RELEASE_TAG }}
+          fetch-depth: 0
 
       - name: Override CI_CLOUD if PR label is present
         if: ${{ github.event_name == 'pull_request' }}
@@ -60,9 +70,10 @@ jobs:
             fi
           done
 
-      - name: Record settings for CI cloud
+      - name: Record debug info
         run: |
-          echo CI_CLOUD: ${{ env.CI_CLOUD }}
+          echo LATEST_RELEASE_TAG: $LATEST_RELEASE_TAG
+          echo CI_CLOUD: $CI_CLOUD
 
       - name: Setup ssh
         run: |
@@ -76,7 +87,7 @@ jobs:
         run: cat environments/.stackhpc/bastion_fingerprints >> ~/.ssh/known_hosts
         shell: bash
 
-      - name: Install ansible etc
+      - name: Install ansible, pip and galaxy requirements
         run: dev/setup-env.sh
 
       - name: Install OpenTofu
@@ -86,7 +97,7 @@ jobs:
 
       - name: Initialise tofu
         run: tofu init
-        working-directory: ${{ github.workspace }}/environments/.stackhpc/tofu
+        working-directory: ${{ env.STACKHPC_TF_DIR }}
 
       - name: Write clouds.yaml
         run: |
@@ -103,42 +114,90 @@ jobs:
         env:
           DEMO_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
-      - name: Provision nodes using fat image
+      - name: Provision nodes using latest release image
         id: provision_servers
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
-          cd $APPLIANCES_ENVIRONMENT_ROOT/tofu
+          cd $STACKHPC_TF_DIR
           tofu apply -auto-approve -var-file="${{ env.CI_CLOUD }}.tfvars"
 
       - name: Delete infrastructure if provisioning failed
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
-          cd $APPLIANCES_ENVIRONMENT_ROOT/tofu
+          cd $STACKHPC_TF_DIR
           tofu destroy -auto-approve -var-file="${{ env.CI_CLOUD }}.tfvars"
         if: failure() && steps.provision_servers.outcome == 'failure'
 
-      - name: Configure cluster
+      - name: Configure cluster at latest release
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
           ansible all -m wait_for_connection
           ansible-playbook -v ansible/site.yml
           ansible-playbook -v ansible/ci/check_slurm.yml
 
-      - name: Run MPI-based tests
+      - name: Run MPI-based tests at latest release
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
-          ansible-playbook -vv ansible/adhoc/hpctests.yml
+          ansible-playbook -vv ansible/adhoc/hpctests.yml --tags pingpong
 
       # - name: Run EESSI tests
       #   run: |
       #     . venv/bin/activate
       #     . environments/.stackhpc/activate
       #     ansible-playbook -vv ansible/ci/check_eessi.yml
 
+      - name: Checkout current branch
+        run: git checkout ${{ github.head_ref || github.ref_name }}
+
+      - name: Update ansible, pip and galaxy requirements
+        run: dev/setup-env.sh
+
+      - name: Reimage login and control nodes to image in current branch
+        id: reimage_non_compute
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+          cd $STACKHPC_TF_DIR
+          tofu init
+          tofu apply -auto-approve -var-file="${{ env.CI_CLOUD }}.tfvars"
+          
+      - name: Configure cluster using current branch
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+          ansible all -m wait_for_connection
+          ansible-playbook -v ansible/site.yml
+          ansible-playbook -v ansible/ci/check_slurm.yml
+
+      - name: Reimage compute nodes to image in current branch using slurm - tests compute-init
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+          ansible-playbook -v ansible/adhoc/reboot_via_slurm.yml
+          ansible-playbook -v ansible/ci/check_slurm.yml
+
+      - name: Check sacct state survived reimage to current branch
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+          ansible-playbook -vv ansible/ci/check_sacct_hpctests.yml
+
+      - name: Check MPI-based tests are shown in Grafana
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+          ansible-playbook -vv ansible/ci/check_grafana.yml
+
+      - name: Run MPI-based tests again in current branch
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+          ansible-playbook -vv ansible/adhoc/hpctests.yml
+
       - name: Confirm Open Ondemand is up (via SOCKS proxy)
         run: |
           . venv/bin/activate
@@ -170,43 +229,10 @@ jobs:
         env:
           DEMO_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
-      - name: Test reimage of login and control nodes (via rebuild adhoc)
-        run: |
-          . venv/bin/activate
-          . environments/.stackhpc/activate
-          ansible-playbook -v --limit control,login ansible/adhoc/rebuild.yml
-          ansible-playbook -v ansible/site.yml
-          ansible-playbook -v ansible/ci/check_slurm.yml
-
-      - name: Test compute node reboot and compute-init
-        run: |
-          . venv/bin/activate
-          . environments/.stackhpc/activate
-          ansible-playbook -v ansible/adhoc/reboot_via_slurm.yml
-          ansible-playbook -v ansible/ci/check_slurm.yml
-
-      - name: Check sacct state survived reimage
-        run: |
-          . venv/bin/activate
-          . environments/.stackhpc/activate
-          ansible-playbook -vv ansible/ci/check_sacct_hpctests.yml
-
-      - name: Check MPI-based tests are shown in Grafana
-        run: |
-          . venv/bin/activate
-          . environments/.stackhpc/activate
-          ansible-playbook -vv ansible/ci/check_grafana.yml
-
       - name: Delete infrastructure
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
-          cd $APPLIANCES_ENVIRONMENT_ROOT/tofu
-          tofu destroy -auto-approve -var-file="${{ env.CI_CLOUD }}.tfvars"
+          cd $STACKHPC_TF_DIR
+          tofu destroy -auto-approve -var-file="${{ env.CI_CLOUD }}.tfvars" || echo "tofu failed in $STACKHPC_TF_DIR"
         if: ${{ success() || cancelled() }}
-
-      # - name: Delete images
-      #   run: |
-      #     . venv/bin/activate
-      #     . environments/.stackhpc/activate
-      #     ansible-playbook -vv ansible/ci/delete_images.yml
diff --git a/ansible/.gitignore b/ansible/.gitignore
@@ -84,5 +84,7 @@ roles/*
 !roles/pytools/**
 !roles/rebuild/
 !roles/rebuild/**
+!roles/slurm_tools/
+!roles/slurm_tools/**
 !roles/gateway/
 !roles/gateway/**
diff --git a/ansible/ci/check_grafana.yml b/ansible/ci/check_grafana.yml
@@ -23,4 +23,4 @@
       delay: 5
       vars:
         _found_jobs: "{{ _slurm_stats_jobs.docs | map(attribute='JobName', default='(json error in slurmstats data)') }}"
-        _expected_jobs: ['hpl-solo.sh', 'pingpong.sh', 'pingmatrix.sh']
+        _expected_jobs: ['pingpong.sh']
diff --git a/ansible/ci/check_sacct_hpctests.yml b/ansible/ci/check_sacct_hpctests.yml
@@ -5,10 +5,6 @@
     sacct_stdout_expected: |- # based on CI running hpctests as the first job
       JobID,JobName,State
       1,pingpong.sh,COMPLETED
-      2,pingmatrix.sh,COMPLETED
-      3,hpl-build-linux64.sh,COMPLETED
-      4_0,hpl-solo.sh,COMPLETED
-      4_1,hpl-solo.sh,COMPLETED
   tasks:
     - name: Get info for ended jobs
       shell:
diff --git a/ansible/ci/update_timestamps.yml b/ansible/ci/update_timestamps.yml
@@ -0,0 +1,16 @@
+- hosts: localhost
+  tasks:
+    - name: Get latest timestamps from sources
+      latest_timestamps:
+        repos_dict: "{{ appliances_pulp_repos }}"
+        content_url: "https://ark.stackhpc.com/pulp/content"
+      register: _result
+
+    - name: Overwrite repo timestamps with latest
+      ansible.builtin.copy:
+        dest: "{{ appliances_repository_root }}/environments/common/inventory/group_vars/all/timestamps.yml"
+        content: "{{ repo_template | to_nice_yaml(indent=2) }}"
+        backup: true
+      vars:
+        repo_template:
+          appliances_pulp_repos: "{{ _result.timestamps }}"
diff --git a/ansible/library/latest_timestamps.py b/ansible/library/latest_timestamps.py
@@ -0,0 +1,76 @@
+__metaclass__ = type
+
+DOCUMENTATION = r'''
+---
+module: latest_timestamps
+short_description: Gets the latest set of snapshots from Pulp
+version_added: "1.0.0"
+description: Gets the latest set of snapshots from given source URLs and returns dictionary to replace 'appliances_repo_timestamps' with
+author:
+    - William Tripp
+    - Steve Brasier
+'''
+
+EXAMPLES = r'''
+- name: Get latest timestamps
+  latest_timestamps:
+    repos_dict: "{{ appliances_repo_timestamp_sources }}"
+    content_url: "https://ark.stackhpc.com/pulp/content"
+  register: result
+'''
+
+RETURN = r'''
+latest_dict:
+    description: Dictionary with updated timestamps
+    type: dict
+    returned: always
+changed_timestamps:
+    description: List of repos that have updated timestamps
+    type: str[]
+    returned: always
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+import requests
+from bs4 import BeautifulSoup
+
+def run_module():
+    module_args = dict(
+        repos_dict=dict(type='dict', required=True),
+        content_url=dict(type='str', required=True)
+    )
+
+    result = dict(
+        changed=False,
+        original_message='',
+        message=''
+    )
+
+    module = AnsibleModule(
+        argument_spec=module_args,
+        supports_check_mode=True
+    )
+
+    timestamps = dict(module.params['repos_dict'])
+    for repo in timestamps:
+        for version in timestamps[repo]:
+
+            html_txt = requests.get(
+                    url= module.params['content_url'] + '/' + timestamps[repo][version]['path']
+                ).text
+            timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all() # getting raw list of timestamps from html
+            timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # stripping xml tags
+            latest_timestamp = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed
+            timestamps[repo][version]['timestamp'] = latest_timestamp
+            
+    result['timestamps'] = dict(sorted(timestamps.items()))
+
+    module.exit_json(**result)
+
+
+def main():
+    run_module()
+
+
+if __name__ == '__main__':
+    main()
diff --git a/ansible/roles/compute_init/files/compute-init.yml b/ansible/roles/compute_init/files/compute-init.yml
@@ -61,7 +61,13 @@
         owner: slurm
         group: root
         mode: u=rX,g=rwX,o=
-    
+
+    - name: Wait for NFS to reachable (checks host network up)
+      ansible.builtin.wait_for:
+        port: 2049
+        host: '{{ server_node_ip }}'
+        timeout: 120
+
     - name: Mount /mnt/cluster
       mount:
         path: /mnt/cluster
@@ -70,8 +76,6 @@
         opts: ro,sync
         state: mounted
       register: _mount_mnt_cluster
-      ignore_errors: true
-      # TODO: add some retries here?
 
     - block:
         - name: Report skipping initialization if cannot mount nfs
@@ -193,7 +197,7 @@
         tasks_from: nfs-clients.yml
       when:
         - enable_nfs
-        - nfs_enable.clients | default(item.nfs_enable.clients) | bool
+        - nfs_enable.clients | bool or ('nfs_enable' in item and item.nfs_enable.clients | bool)
       loop: "{{ nfs_configurations }}"
 
     - name: Manila mounts
diff --git a/ansible/roles/slurm_tools/defaults/main.yml b/ansible/roles/slurm_tools/defaults/main.yml
@@ -1,4 +1,4 @@
 ---
 pytools_editable: false
-pytools_gitref: master
+pytools_gitref: v2.0
 pytools_user: root
diff --git a/environments/.stackhpc/ansible.cfg b/environments/.stackhpc/ansible.cfg
@@ -10,6 +10,7 @@ inventory = ../common/inventory,inventory
 collections_path = ../../ansible/collections
 roles_path = ../../ansible/roles
 filter_plugins = ../../ansible/filter_plugins
+library = ../../ansible/library
 
 [ssh_connection]
 ssh_args = -o ServerAliveInterval=10 -o ControlMaster=auto -o ControlPath=~/.ssh/%r@%h-%p -o ControlPersist=240s -o PreferredAuthentications=publickey -o UserKnownHostsFile=/dev/null
diff --git a/environments/.stackhpc/tofu/cluster_image.auto.tfvars.json b/environments/.stackhpc/tofu/cluster_image.auto.tfvars.json
@@ -1,6 +1,6 @@
 {
     "cluster_image": {
-        "RL8": "openhpc-RL8-250319-1045-69713f23",
-        "RL9": "openhpc-RL9-250319-1045-69713f23"
+        "RL8": "openhpc-RL8-250401-1100-9a3cffdb",
+        "RL9": "openhpc-RL9-250401-1100-9a3cffdb"
     }
 }
diff --git a/environments/common/inventory/group_vars/all/defaults.yml b/environments/common/inventory/group_vars/all/defaults.yml
diff --git a/environments/common/inventory/group_vars/all/nfs.yml b/environments/common/inventory/group_vars/all/nfs.yml
diff --git a/environments/common/inventory/group_vars/all/timestamps.yml b/environments/common/inventory/group_vars/all/timestamps.yml
diff --git a/requirements.txt b/requirements.txt

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"cluster_image": {`
`3`		`- "RL8": "openhpc-RL8-250319-1045-69713f23",`
`4`		`- "RL9": "openhpc-RL9-250319-1045-69713f23"`
	`3`	`+ "RL8": "openhpc-RL8-250401-1100-9a3cffdb",`
	`4`	`+ "RL9": "openhpc-RL9-250401-1100-9a3cffdb"`
`5`	`5`	`}`
`6`	`6`	`}`