Merge branch 'main' into sms-deployment

Author: bertiethorpe

ansible/bootstrap.yml

@@ -1,5 +1,20 @@
 ---
 
+- hosts: cluster
+  gather_facts: false
+  become: yes
+  tasks:
+    - name: Check if ansible-init is installed
+      stat:
+        path: /etc/systemd/system/ansible-init.service
+      register: _stat_ansible_init_unitfile
+    
+    - name: Wait for ansible-init to finish
+      wait_for:
+        path: /var/lib/ansible-init.done
+        timeout: "{{ ansible_init_wait }}" # seconds
+      when: _stat_ansible_init_unitfile.stat.exists
+
 - hosts: localhost
   gather_facts: false
   become: false
@@ -235,3 +250,11 @@
   tasks:
     - include_role:
         name: ofed
+
+- hosts: ansible_init
+  gather_facts: yes
+  become: yes
+  tags: linux_ansible_init
+  tasks:
+    - include_role:
+        name: azimuth_cloud.image_utils.linux_ansible_init

ansible/fatimage.yml

@@ -68,11 +68,13 @@
         tasks_from: install-package.yml
         vars_from: "Rocky/{{ ansible_distribution_major_version }}.yml"
     # # FUTURE: install-apps.yml - this is git clones
+
     - name: Open Ondemand server (apps)
       include_role:
         name: osc.ood
         tasks_from: install-apps.yml
         vars_from: "Rocky/{{ ansible_distribution_major_version }}.yml"
+
     - name: Open Ondemand remote desktop
       import_role:
         name: openondemand

ansible/roles/cluster_infra/defaults/main.yml

@@ -0,0 +1,2 @@
+ansible_init_collections: []
+ansible_init_playbooks: []

ansible/roles/cluster_infra/templates/resources.tf.j2

@@ -370,6 +370,23 @@ resource "openstack_compute_instance_v2" "login" {
       - "${openstack_compute_keypair_v2.cluster_keypair.public_key}"
     {%- endif %}
   EOF
+
+  metadata = {
+    {% for playbook in ansible_init_playbooks %}
+      ansible_init_pb_{{ loop.index0 }}_name = "{{ playbook.name }}"
+      {% if playbook.stage is defined %}
+        ansible_init_pb_{{ loop.index0 }}_stage = "{{ playbook.stage }}"
+      {% endif %}
+    {% endfor %}
+    {% for collection in ansible_init_collections %}
+      ansible_init_coll_{{ loop.index0 }}_name = "{{ collection.name }}"
+      ansible_init_coll_{{ loop.index0 }}_type = "{{ collection.type }}"
+      ansible_init_coll_{{ loop.index0 }}_version = "{{ collection.version }}"
+      {% if collection.source is defined %}
+        ansible_init_coll_{{ loop.index0 }}_source = "{{ collection.source }}"
+      {% endif %}
+    {% endfor %} 
+  }
 }
 
 resource "openstack_compute_instance_v2" "control" {
@@ -446,6 +463,23 @@ resource "openstack_compute_instance_v2" "control" {
         - [LABEL=home, /exports/home, auto]
         {% endif %}
   EOF
+
+  metadata = {
+    {% for playbook in ansible_init_playbooks %}
+      ansible_init_pb_{{ loop.index0 }}_name = "{{ playbook.name }}"
+      {% if playbook.stage is defined %}
+        ansible_init_pb_{{ loop.index0 }}_stage = "{{ playbook.stage }}"
+      {% endif %}
+    {% endfor %}
+    {% for collection in ansible_init_collections %}
+      ansible_init_coll_{{ loop.index0 }}_name = "{{ collection.name }}"
+      ansible_init_coll_{{ loop.index0 }}_type = "{{ collection.type }}"
+      ansible_init_coll_{{ loop.index0 }}_version = "{{ collection.version }}"
+      {% if collection.source is defined %}
+        ansible_init_coll_{{ loop.index0 }}_source = "{{ collection.source }}"
+      {% endif %}
+    {% endfor %} 
+  }
 }
 
 {% for partition in openhpc_slurm_partitions %}
@@ -498,6 +532,23 @@ resource "openstack_compute_instance_v2" "{{ partition.name }}" {
       - "${openstack_compute_keypair_v2.cluster_keypair.public_key}"
     {%- endif %}
   EOF
+
+  metadata = {
+    {% for playbook in ansible_init_playbooks %}
+      ansible_init_pb_{{ loop.index0 }}_name = "{{ playbook.name }}"
+      {% if playbook.stage is defined %}
+        ansible_init_pb_{{ loop.index0 }}_stage = "{{ playbook.stage }}"
+      {% endif %}
+    {% endfor %}
+    {% for collection in ansible_init_collections %}
+      ansible_init_coll_{{ loop.index0 }}_name = "{{ collection.name }}"
+      ansible_init_coll_{{ loop.index0 }}_type = "{{ collection.type }}"
+      ansible_init_coll_{{ loop.index0 }}_version = "{{ collection.version }}"
+      {% if collection.source is defined %}
+        ansible_init_coll_{{ loop.index0 }}_source = "{{ collection.source }}"
+      {% endif %}
+    {% endfor %} 
+  }
 }
 
 {% endfor %}

ansible/roles/hpctests/templates/pingmatrix.sh.j2

@@ -16,4 +16,8 @@ echo UCX_NET_DEVICES: $UCX_NET_DEVICES
 module load {{ hpctests_pingmatrix_modules | join(' ' ) }}
 
 mpicc -o nxnlatbw mpi_nxnlatbw.c
-mpirun nxnlatbw
+
+# mpirun flags force using UCX TCP transports, overriding higher
+# priority of OpenMPI btl/openib component, which is also using RDMA
+# https://wiki.stackhpc.com/s/985dae84-7bd8-4924-94b7-9629a7827100 
+mpirun -mca pml_ucx_tls any -mca pml_ucx_devices any nxnlatbw

ansible/roles/hpctests/templates/pingpong.sh.j2

@@ -16,4 +16,8 @@ echo UCX_NET_DEVICES: $UCX_NET_DEVICES
 module load {{ hpctests_pingpong_modules | join(' ' ) }}
 
 #srun --mpi=pmi2 IMB-MPI1 pingpong # doesn't work in ohpc v2.1
-mpirun IMB-MPI1 pingpong
+
+# mpirun flags force using UCX TCP transports, overriding higher 
+# priority of OpenMPI btl/openib component, which is also using RDMA 
+# https://wiki.stackhpc.com/s/985dae84-7bd8-4924-94b7-9629a7827100
+mpirun -mca pml_ucx_tls any -mca pml_ucx_devices any IMB-MPI1 pingpong

environments/common/inventory/group_vars/all/ansible_init.yml

@@ -0,0 +1 @@
+ansible_init_wait: 1200 # seconds

environments/common/inventory/groups

@@ -132,3 +132,6 @@ freeipa_client
 
 [tuned]
 # Hosts to run TuneD configuration
+
+[ansible_init]
+# Hosts to run linux-anisble-init

environments/common/layouts/everything

@@ -78,3 +78,7 @@ openhpc
 
 [tuned:children]
 # Hosts to run TuneD configuration
+
+[ansible_init:children]
+# Hosts to run ansible-init
+cluster

requirements.txt

@@ -1,6 +1,6 @@
 ansible==6.0.0
 openstacksdk
-python-openstackclient
+python-openstackclient==6.6.1 # v7.0.0 has a bug re. rebuild
 python-manilaclient
 jmespath
 passlib[bcrypt]==1.7.4