Merge branch 'main' into fix/templating2

Author: sjpb

.github/workflows/stackhpc.yml

@@ -14,7 +14,7 @@ jobs:
         cloud:
           - "arcus"   # Arcus OpenStack in rcp-cloud-portal-demo project, with RoCE
       fail-fast: false # as want clouds to continue independently
-    concurrency: ${{ matrix.cloud }}
+    concurrency: ${{ github.ref }} # to branch/PR
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
@@ -52,30 +52,52 @@ jobs:
           smslabs_CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }}
           arcus_CLOUDS_YAML: ${{ secrets.ARCUS_CLOUDS_YAML }}
 
-      - name: Provision infrastructure
-        id: provision
+      - name: Provision ports, inventory and other infrastructure apart from nodes
+        id: provision_ports
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
-          terraform apply -auto-approve
+          TF_VAR_create_nodes=false terraform apply -auto-approve
         env:
           OS_CLOUD: openstack
           TF_VAR_cluster_name: ci${{ github.run_id }}
 
+      - name: Setup environment-specific inventory/terraform inputs
+        run: |
+          . venv/bin/activate
+          . environments/${{ matrix.cloud }}/activate
+          ansible-playbook ansible/adhoc/generate-passwords.yml
+          echo vault_testuser_password: "$TESTUSER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/all/test_user.yml
+          ansible-playbook ansible/adhoc/template-cloud-init.yml
+        env:
+          ANSIBLE_FORCE_COLOR: True
+          TESTUSER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+
+      - name: Provision servers
+        id: provision_servers
+        run: |
+          . venv/bin/activate
+          . environments/${{ matrix.cloud }}/activate
+          cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
+          terraform apply -auto-approve
+        env:
+          OS_CLOUD: openstack
+          TF_VAR_cluster_name: ci${{ github.run_id }}
+
       - name: Get server provisioning failure messages
         id: provision_failure
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           TF_FAIL_MSGS="$(../../skeleton/\{\{cookiecutter.environment\}\}/terraform/getfaults.py  $PWD)"
-          echo $TF_FAIL_MSGS
+          echo TF failure messages: $TF_FAIL_MSGS
           echo "::set-output name=messages::${TF_FAIL_MSGS}"
         env:
           OS_CLOUD: openstack
           TF_VAR_cluster_name: ci${{ github.run_id }}
-        if: always() && steps.provision.outcome == 'failure'
+        if: always() && steps.provision_servers.outcome == 'failure'
 
       - name: Delete infrastructure if failed due to lack of hosts
         run: |
@@ -86,20 +108,17 @@ jobs:
         env:
           OS_CLOUD: openstack
           TF_VAR_cluster_name: ci${{ github.run_id }}
-        if: ${{ always() && steps.provision.outcome == 'failure' && contains('not enough hosts available', steps.provision_failure.messages) }}
+        if: ${{ always() && steps.provision_servers.outcome == 'failure' && contains(steps.provision_failure.messages, 'not enough hosts available') }}
 
       - name: Directly configure cluster
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           ansible all -m wait_for_connection
-          ansible-playbook ansible/adhoc/generate-passwords.yml
-          echo test_user_password: "$TEST_USER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/basic_users/defaults.yml
           ansible-playbook -vv ansible/site.yml
         env:
           OS_CLOUD: openstack
           ANSIBLE_FORCE_COLOR: True
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
       - name: Run MPI-based tests
         run: |
@@ -135,23 +154,24 @@ jobs:
             --server-response \
             --no-check-certificate \
             --http-user=testuser \
-            --http-password=${TEST_USER_PASSWORD} https://${openondemand_servername} \
+            --http-password=${TESTUSER_PASSWORD} https://${openondemand_servername} \
             2>&1)
           (echo $statuscode | grep "200 OK") || (echo $statuscode  && exit 1)
         env:
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+          TESTUSER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
       - name: Build packer images
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
-          echo test_user_password: "$TEST_USER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/basic_users/defaults.yml
+          ansible-playbook ansible/adhoc/generate-passwords.yml
+          echo vault_testuser_password: "$TESTUSER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/all/test_user.yml
           cd packer/
           PACKER_LOG=1 packer build -on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
         env:
           OS_CLOUD: openstack
           ANSIBLE_FORCE_COLOR: True
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+          TESTUSER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
       - name: Test reimage of nodes
         run: |

ansible/.gitignore

@@ -26,6 +26,10 @@ roles/*
 !roles/slurm_exporter/**
 !roles/firewalld/
 !roles/firewalld/**
+!roles/etc_hosts/
+!roles/etc_hosts/**
+!roles/cloud_init/
+!roles/cloud_init/**
 !roles/mysql/
 !roles/mysql/**
 !roles/systemd/

ansible/adhoc/hpctests.yml

@@ -4,7 +4,7 @@
 
 ---
 
-- hosts: hpctests[0] # TODO: might want to make which node is used selectable?
+- hosts: login[0] # TODO: might want to make which node is used selectable?
   become: false
   gather_facts: false
   tasks:

ansible/adhoc/template-cloud-init.yml

@@ -0,0 +1,9 @@
+- hosts: cloud_init
+  become: no
+  gather_facts: no
+  tasks:
+    - name: Template out cloud-init userdata
+      import_role:
+        name: cloud_init
+        tasks_from: template.yml
+      delegate_to: localhost

ansible/bootstrap.yml

@@ -13,6 +13,19 @@
           to update these variable names. ** NB: The actual secrets will not be changed.**
       when: "'secrets_openhpc_' in (hostvars[inventory_hostname] | join)"
 
+- hosts: etc_hosts
+  gather_facts: false
+  tags: etc_hosts
+  become: yes
+  tasks:
+    - name: Template /etc/hosts
+      copy:
+        content: "{{ etc_hosts_template }}"
+        dest: /etc/hosts
+        owner: root
+        group: root
+        mode: u=rw,og=r
+
 - hosts: cluster
   gather_facts: false
   tasks:
@@ -74,6 +87,20 @@
     - import_role:
         name: fail2ban
 
+- name: Setup podman
+  hosts: podman
+  tags: podman
+  tasks:
+    - import_role:
+        name: podman
+        tasks_from: prereqs.yml
+      tags: prereqs
+
+    - import_role:
+        name: podman
+        tasks_from: config.yml
+      tags: config
+
 - hosts: update
   gather_facts: false
   become: yes

ansible/ci/test_reimage.yml

@@ -45,10 +45,10 @@
   gather_facts: no
   tags: reimage_compute
   tasks:
-    # TODO: This is specific to smslabs/arcus environment config - could generalise to all compute nodes
+    # TODO: This is specific to arcus environment config - could generalise to all compute nodes
     - name: Request compute node rebuild via Slurm
       shell:
-        cmd: scontrol reboot ASAP nextstate=RESUME reason='rebuild image:{{ compute_build.artifact_id }}' {{ openhpc_cluster_name }}-compute-[0-1]
+        cmd: scontrol reboot ASAP nextstate=RESUME reason='rebuild image:{{ compute_build.artifact_id }}' {{ openhpc_cluster_name }}-compute-[0-3]
       become: yes
 
     - name: Check compute node rebuild completed

ansible/monitoring.yml

@@ -1,20 +1,6 @@
 # ---
 # # NOTE: Requires slurmdbd
 
-- name: Setup podman
-  hosts: podman
-  tags: podman
-  tasks:
-    - import_role:
-        name: podman
-        tasks_from: prereqs.yml
-      tags: prereqs
-
-    - import_role:
-        name: podman
-        tasks_from: config.yml
-      tags: config
-
 - name: Setup elasticsearch
   hosts: opendistro
   tags: opendistro

ansible/roles/cloud_init/README.md

@@ -0,0 +1,31 @@
+# cloud_init
+
+Create cloud init userdata for instance groups.
+
+# Requirements
+Image and cloud environment supporting cloud-init.
+
+# Role Variables
+
+- `cloud_init_output_path`: Required. Path to output userdata files to.
+- `cloud_init_userdata_templates`: Optional list. Each element is a dict with keys/values as follows:
+    - `module`: Required str. Name of cloud_init [module](https://cloudinit.readthedocs.io/en/latest/topics/modules.html)
+    - `group`: Optional str. Name of inventory group to which this config applies - if no group is specified then it applies to all groups. This allows defining `cloud_init_userdata_templates` for group `all`.
+    - `template`: Jinja template for cloud_init module [configuration](https://cloudinit.readthedocs.io/en/latest/topics/modules.html).
+
+  Elements may repeat `module`; the resulting userdata cloud-config file will will contain configuration from all applicable (by group) elements for that module.
+  
+  Note that the appliance [constructs](../../../environments/common/inventory/group_vars/all/cloud_init.yml) `cloud_init_userdata_templates` from `cloud_init_userdata_templates_default` and `cloud_init_userdata_templates_extra` to 
+  allow easier customisation in specific environments.
+
+# Dependencies
+None.
+
+# Example Playbook
+See `ansible/adhoc/rebuild.yml`.
+
+# License
+Apache 2.0
+
+# Author Information
+[email protected]

ansible/roles/cloud_init/defaults/main.yml

@@ -0,0 +1,2 @@
+#cloud_init_output_path: 
+cloud_init_userdata_templates: []

ansible/roles/cloud_init/tasks/template.yml

@@ -0,0 +1,5 @@
+
+- name: Template out cloud-init userdata
+  ansible.builtin.template:
+    src: userdata.yml.j2
+    dest: "{{ cloud_init_output_path }}/{{ inventory_hostname }}.userdata.yml"