Merge branch 'main' into cuda

Author: sjpb

.github/CODEOWNERS

@@ -1 +1 @@
-./.github/CODEOWNERS
+* @stackhpc/batch

.github/workflows/fatimage.yml

@@ -48,14 +48,14 @@ jobs:
           . venv/bin/activate
           . environments/.stackhpc/activate
           cd packer/
-          packer init
+          packer init .
           PACKER_LOG=1 packer build -only openstack.openhpc -on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
 
       - name: Get created image name from manifest
         id: manifest
         run: |
           . venv/bin/activate
-          IMAGE_ID=$(jq --raw-output '.builds[-1].artifact_id' packer-manifest.json)
+          IMAGE_ID=$(jq --raw-output '.builds[-1].artifact_id' packer/packer-manifest.json)
           while ! openstack image show -f value -c name $IMAGE_ID; do
             sleep 30
           done

.github/workflows/stackhpc.yml

@@ -66,24 +66,13 @@ jobs:
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           terraform apply -auto-approve
 
-      - name: Get server provisioning failure messages
-        id: provision_failure
-        run: |
-          . venv/bin/activate
-          . environments/.stackhpc/activate
-          cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
-          TF_FAIL_MSGS="$(../../skeleton/\{\{cookiecutter.environment\}\}/terraform/getfaults.py  $PWD)"
-          echo TF failure messages: $TF_FAIL_MSGS
-          echo "::set-output name=messages::${TF_FAIL_MSGS}"
-        if: always() && steps.provision_servers.outcome == 'failure'
-        
-      - name: Delete infrastructure if failed due to lack of hosts
+      - name: Delete infrastructure if provisioning failed
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           terraform destroy -auto-approve
-        if: ${{ always() && steps.provision_servers.outcome == 'failure' && contains(steps.provision_failure.messages, 'not enough hosts available') }}
+        if: failure() && steps.provision_servers.outcome == 'failure'
 
       - name: Configure cluster
         run: |
@@ -99,6 +88,12 @@ jobs:
           . environments/.stackhpc/activate
           ansible-playbook -vv ansible/adhoc/hpctests.yml
 
+      - name: Run EESSI tests
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+          ansible-playbook -vv ansible/ci/check_eessi.yml
+
       - name: Confirm Open Ondemand is up (via SOCKS proxy)
         run: |
           . venv/bin/activate

ansible/.gitignore

@@ -36,3 +36,9 @@ roles/*
 !roles/systemd/**
 !roles/cuda/
 !roles/cuda/**
+!roles/freeipa/
+!roles/freeipa/**
+!roles/proxy/
+!roles/proxy/**
+!roles/resolv_conf/
+!roles/resolv_conf/**

ansible/bootstrap.yml

@@ -13,6 +13,14 @@
           to update these variable names. ** NB: The actual secrets will not be changed.**
       when: "'secrets_openhpc_' in (hostvars[inventory_hostname] | join)"
 
+- hosts: resolv_conf
+  become: yes
+  gather_facts: false
+  tags: resolv_conf
+  tasks:
+    - import_role:
+        name: resolv_conf
+
 - hosts: etc_hosts
   gather_facts: false
   tags: etc_hosts
@@ -21,6 +29,14 @@
     - import_role:
         name: etc_hosts
 
+- hosts: proxy
+  gather_facts: false
+  tags: proxy
+  become: yes
+  tasks:
+    - import_role:
+        name: proxy
+
 - hosts: cluster
   gather_facts: false
   tasks:
@@ -96,6 +112,16 @@
         tasks_from: config.yml
       tags: config
 
+- name: Setup EESSI
+  hosts: eessi
+  tags: eessi
+  become: true
+  gather_facts: false
+  tasks:
+    - name: Install and configure EESSI
+      import_role:
+        name: eessi
+
 - hosts: update
   gather_facts: false
   become: yes

ansible/ci/check_eessi.yml

@@ -0,0 +1,34 @@
+---
+- name: Run EESSI test job
+  hosts: login[0]
+  vars:
+    eessi_test_rootdir: /home/eessi_test
+  tasks:
+    - name: Create test root directory
+      file:
+        path: "{{ eessi_test_rootdir }}"
+        state: directory
+        owner: "{{ ansible_user }}"
+        group: "{{ ansible_user }}"
+      become: true
+      
+    - name: Clone eessi-demo repo
+      ansible.builtin.git:
+        repo: "https://github.com/eessi/eessi-demo.git"
+        dest: "{{ eessi_test_rootdir }}/eessi-demo"
+
+    - name: Run test job
+      ansible.builtin.shell:
+        cmd: |
+          source /cvmfs/pilot.eessi-hpc.org/latest/init/bash
+          srun ./run.sh
+        chdir: "{{ eessi_test_rootdir }}/eessi-demo/TensorFlow"
+        executable: /bin/bash
+      register: job_output
+
+    - name: Fail if job output contains error
+      fail:
+        # Note: Job prints live progress bar to terminal, so use regex filter to remove this from stdout
+        msg: "Test job using EESSI modules failed. Job output was: {{ job_output.stdout | regex_replace('\b', '') }}"        
+      when: '"Epoch 5/5" not in job_output.stdout'
+      

ansible/ci/check_sacct_hpctests.yml

@@ -2,7 +2,7 @@
   gather_facts: false
   become: true
   vars:
-    sacct_stdout_expected: |- # based on CI running hpctests as the first job - NB note no trailing newline
+    sacct_stdout_expected: |- # based on CI running hpctests as the first job
       JobID,JobName,State
       1,pingpong.sh,COMPLETED
       2,pingmatrix.sh,COMPLETED
@@ -18,10 +18,10 @@
       register: sacct
     - name: Check info for ended jobs
       assert:
-        that: sacct.stdout == sacct_stdout_expected
+        that: sacct_stdout_expected in sacct.stdout
         fail_msg: |
           Expected:
           --{{ sacct_stdout_expected }}--
           Got:
           --{{ sacct.stdout }}--
-        success_msg: sacct shows hpctests jobs as first and only jobs
+        success_msg: sacct shows hpctests jobs as first jobs in list

ansible/cleanup.yml

@@ -5,17 +5,28 @@
 - name: Remove dnf caches
   command: dnf clean all
 
+# If image build happens on a Neutron subnet with property dns_namservers defined, then cloud-init
+# disables NetworkManager's control of /etc/resolv.conf and appends nameservers itself.
+# We don't want network configuration during instance boot to depend on the configuration
+# of the network the builder was on, so we reset these aspects.
 - name: Delete /etc/resolv.conf
-  # required as if cloud-init (rather than network manager) controls this on next boot it won't be entirely overrwritten
   file:
     path: /etc/resolv.conf
     state: absent
+  when: "'resolv_conf' not in group_names" # if its been overriden, deleting it is the wrong thing to do
 
-- name: Delete any injected ssh config for rocky
+- name: Reenable NetworkManager control of resolv.conf
+  # NB: This *doesn't* delete the 90-dns-none.conf file created by the resolv_conf role
+  # as if nameservers are explicitly being set by that role we don't want to allow NM
+  # to override it again.
   file:
-    path: /home/rocky/.ssh/
+    path: /etc/NetworkManager/conf.d/99-cloud-init.conf
+    state: absent
+
+- name: Delete any injected ssh config for ansible_user
+  file:
+    path: "/home/{{ ansible_user }}/.ssh/"
     state: absent
 
 - name: Run cloud-init cleanup
   command: cloud-init clean --logs --seed
-

ansible/fatimage.yml

@@ -1,7 +1,21 @@
 # Builder version of site.yml just installing binaries
 
+- name: Run pre.yml hook
+  vars:
+    appliances_environment_root: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}"
+    hook_path: "{{ appliances_environment_root }}/hooks/pre.yml"
+  import_playbook: "{{ hook_path if hook_path | exists else 'noop.yml' }}"
+  when: hook_path | exists
+
 - import_playbook: bootstrap.yml
 
+- name: Run post-bootstrap.yml hook
+  vars:
+    appliances_environment_root: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}"
+    hook_path: "{{ appliances_environment_root }}/hooks/post-bootstrap.yml"
+  import_playbook: "{{ hook_path if hook_path | exists else 'noop.yml' }}"
+  when: hook_path | exists
+
 - hosts: builder
   become: yes
   gather_facts: no
@@ -100,6 +114,7 @@
     - name: unpack prometheus binaries
       become: false
       unarchive:
+        remote_src: yes
         src: "/tmp/prometheus-{{ prometheus_version }}.linux-{{ go_arch }}.tar.gz"
         dest: "/tmp"
         creates: "/tmp/prometheus-{{ prometheus_version }}.linux-{{ go_arch }}/prometheus"
@@ -124,6 +139,18 @@
 
     # - import_playbook: iam.yml - nothing to do
 
+- name: Run post.yml hook
+  vars:
+    appliances_environment_root: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}"
+    hook_path: "{{ appliances_environment_root }}/hooks/post.yml"
+  import_playbook: "{{ hook_path if hook_path | exists else 'noop.yml' }}"
+  when: hook_path | exists
+
+- hosts: builder
+  become: yes
+  gather_facts: no
+  tasks:
+    # - meta: end_here
     - name: Cleanup image
       import_tasks: cleanup.yml
 

ansible/roles/eessi/README.md

@@ -0,0 +1,34 @@
+EESSI
+=====
+
+Configure the EESSI pilot respository for use on given hosts.
+
+Requirements
+------------
+
+None.
+
+Role Variables
+--------------
+
+- `cvmfs_quota_limit_mb`: Optional int. Maximum size of local package cache on each node in MB.
+- `cvmfs_config_overrides`: Optional dict. Set of key-value pairs for additional CernVM-FS settings see [official docs](https://cvmfs.readthedocs.io/en/stable/cpt-configure.html) for list of options. Each dict key should correspond to a valid config variable (e.g. `CVMFS_HTTP_PROXY`) and the corresponding dict value will be set as the variable value (e.g. `https://my-proxy.com`). These configuration parameters will be written to the `/etc/cvmfs/default.local` config file on each host in the form `KEY=VALUE`.
+
+Dependencies
+------------
+
+None.
+
+Example Playbook
+----------------
+
+```yaml
+- name: Setup EESSI
+  hosts: eessi
+  tags: eessi
+  become: true
+  tasks:
+    - name: Install and configure EESSI
+      import_role:
+        name: eessi
+```