Merge branch 'main' into refactor/ci

sjpb · sjpb · commit 7551559efac2 · 2022-09-29T08:49:55.000Z
diff --git a/.github/workflows/stackhpc.yml b/.github/workflows/stackhpc.yml
@@ -54,44 +54,57 @@ jobs:
         env:
           arcus_CLOUDS_YAML: ${{ secrets.ARCUS_CLOUDS_YAML }}
       
-      - name: Provision infrastructure
-        id: provision
+      - name: Provision ports, inventory and other infrastructure apart from nodes
+        run: |
+          . venv/bin/activate
+          . environments/${{ matrix.cloud }}/activate
+          cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
+          TF_VAR_create_nodes=false terraform apply -auto-approve
+      
+      - name: Setup environment-specific inventory/terraform inputs
+        run: |
+          . venv/bin/activate
+          . environments/${{ matrix.cloud }}/activate
+          ansible-playbook ansible/adhoc/generate-passwords.yml
+          echo vault_testuser_password: "$TESTUSER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/all/test_user.yml
+          ansible-playbook ansible/adhoc/template-cloud-init.yml
+        env:
+          TESTUSER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+
+      - name: Provision servers
+        id: provision_servers
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           terraform apply -auto-approve
-        
+
       - name: Get server provisioning failure messages
         id: provision_failure
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           TF_FAIL_MSGS="$(../../skeleton/\{\{cookiecutter.environment\}\}/terraform/getfaults.py  $PWD)"
-          echo $TF_FAIL_MSGS
+          echo TF failure messages: $TF_FAIL_MSGS
           echo "::set-output name=messages::${TF_FAIL_MSGS}"
-        if: always() && steps.provision.outcome == 'failure'
+        if: always() && steps.provision_servers.outcome == 'failure'
         
       - name: Delete infrastructure if failed due to lack of hosts
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           terraform destroy -auto-approve
-        if: ${{ always() && steps.provision.outcome == 'failure' && contains('not enough hosts available', steps.provision_failure.messages) }}
+        if: ${{ always() && steps.provision_servers.outcome == 'failure' && contains(steps.provision_failure.messages, 'not enough hosts available') }}
 
       - name: Directly configure cluster
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           ansible all -m wait_for_connection
-          ansible-playbook ansible/adhoc/generate-passwords.yml
-          echo vault_testuser_password: "$TEST_USER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/basic_users/defaults.yml
           ansible-playbook -v ansible/site.yml
           ansible-playbook -v ansible/ci/check_slurm.yml
-        env:
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
       - name: Run MPI-based tests
         run: |
@@ -124,23 +137,20 @@ jobs:
             --server-response \
             --no-check-certificate \
             --http-user=testuser \
-            --http-password=${TEST_USER_PASSWORD} https://${openondemand_servername} \
+            --http-password=${TESTUSER_PASSWORD} https://${openondemand_servername} \
             2>&1)
           (echo $statuscode | grep "200 OK") || (echo $statuscode  && exit 1)
         env:
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+          TESTUSER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
       - name: Build packer images
         id: packer_build
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
-          echo vault_testuser_password: "$TEST_USER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/basic_users/defaults.yml
           cd packer/
           PACKER_LOG=1 packer build -on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
           ../dev/output_manifest.py packer-manifest.json # Sets NEW_{COMPUTE,CONTROL,LOGIN}_IMAGE_ID outputs
-        env:
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
       - name: Test reimage of login nodes (via rebuild adhoc)
         run: |
diff --git a/ansible/.gitignore b/ansible/.gitignore
@@ -26,6 +26,10 @@ roles/*
 !roles/slurm_exporter/**
 !roles/firewalld/
 !roles/firewalld/**
+!roles/etc_hosts/
+!roles/etc_hosts/**
+!roles/cloud_init/
+!roles/cloud_init/**
 !roles/mysql/
 !roles/mysql/**
 !roles/systemd/
diff --git a/ansible/adhoc/template-cloud-init.yml b/ansible/adhoc/template-cloud-init.yml
@@ -0,0 +1,9 @@
+- hosts: cloud_init
+  become: no
+  gather_facts: no
+  tasks:
+    - name: Template out cloud-init userdata
+      import_role:
+        name: cloud_init
+        tasks_from: template.yml
+      delegate_to: localhost
diff --git a/ansible/bootstrap.yml b/ansible/bootstrap.yml
@@ -13,6 +13,19 @@
           to update these variable names. ** NB: The actual secrets will not be changed.**
       when: "'secrets_openhpc_' in (hostvars[inventory_hostname] | join)"
 
+- hosts: etc_hosts
+  gather_facts: false
+  tags: etc_hosts
+  become: yes
+  tasks:
+    - name: Template /etc/hosts
+      copy:
+        content: "{{ etc_hosts_template }}"
+        dest: /etc/hosts
+        owner: root
+        group: root
+        mode: u=rw,og=r
+
 - hosts: cluster
   gather_facts: false
   tasks:
diff --git a/ansible/roles/cloud_init/README.md b/ansible/roles/cloud_init/README.md
@@ -0,0 +1,31 @@
+# cloud_init
+
+Create cloud init userdata for instance groups.
+
+# Requirements
+Image and cloud environment supporting cloud-init.
+
+# Role Variables
+
+- `cloud_init_output_path`: Required. Path to output userdata files to.
+- `cloud_init_userdata_templates`: Optional list. Each element is a dict with keys/values as follows:
+    - `module`: Required str. Name of cloud_init [module](https://cloudinit.readthedocs.io/en/latest/topics/modules.html)
+    - `group`: Optional str. Name of inventory group to which this config applies - if no group is specified then it applies to all groups. This allows defining `cloud_init_userdata_templates` for group `all`.
+    - `template`: Jinja template for cloud_init module [configuration](https://cloudinit.readthedocs.io/en/latest/topics/modules.html).
+
+  Elements may repeat `module`; the resulting userdata cloud-config file will will contain configuration from all applicable (by group) elements for that module.
+  
+  Note that the appliance [constructs](../../../environments/common/inventory/group_vars/all/cloud_init.yml) `cloud_init_userdata_templates` from `cloud_init_userdata_templates_default` and `cloud_init_userdata_templates_extra` to 
+  allow easier customisation in specific environments.
+
+# Dependencies
+None.
+
+# Example Playbook
+See `ansible/adhoc/rebuild.yml`.
+
+# License
+Apache 2.0
+
+# Author Information
+steveb@stackhpc.com
diff --git a/ansible/roles/cloud_init/defaults/main.yml b/ansible/roles/cloud_init/defaults/main.yml
@@ -0,0 +1,2 @@
+#cloud_init_output_path: 
+cloud_init_userdata_templates: []
diff --git a/ansible/roles/cloud_init/tasks/template.yml b/ansible/roles/cloud_init/tasks/template.yml
@@ -0,0 +1,5 @@
+
+- name: Template out cloud-init userdata
+  ansible.builtin.template:
+    src: userdata.yml.j2
+    dest: "{{ cloud_init_output_path }}/{{ inventory_hostname }}.userdata.yml"
diff --git a/ansible/roles/cloud_init/templates/userdata.yml.j2 b/ansible/roles/cloud_init/templates/userdata.yml.j2
@@ -0,0 +1,13 @@
+#cloud-config
+disable_ec2_metadata: true
+
+{% for module, tmpls in cloud_init_userdata_templates | groupby(attribute='module') %}
+{% for tmpl in tmpls %}
+{% if not 'group' in tmpl or tmpl.group in group_names %}
+{% if loop.first %}
+{{ module }}:
+{% endif %}
+{{ tmpl.template }}
+{% endif %}
+{% endfor %}
+{% endfor %}
diff --git a/ansible/roles/etc_hosts/README.md b/ansible/roles/etc_hosts/README.md
@@ -0,0 +1,5 @@
+# etc_hosts
+
+This role provides documentation only.
+
+Hosts in the `etc_hosts` groups get `/etc/hosts` created via `cloud-init`. The generated file defines all hosts in this group using `ansible_host` as the IP address and `inventory_hostname` as the canonical hostname. This may need overriding for multi-homed hosts. See `environments/common/inventory/group_vars/all/cloud_init.yml` for configuration.
diff --git a/environments/arcus/.gitignore b/environments/arcus/.gitignore
@@ -1,3 +1,8 @@
 partitions.yml
 secrets.yml
 hosts
+terraform.tfvars
+.terraform.lock.hcl
+logs/
+hpctests/
+inventory/group_vars/all/test_user.yml
diff --git a/environments/arcus/cloud_init/.gitkeep b/environments/arcus/cloud_init/.gitkeep
diff --git a/environments/arcus/hooks/pre.yml b/environments/arcus/hooks/pre.yml
@@ -1,17 +1,3 @@
-- hosts: all # NB: As this doesn't exclude `builder` /etc/hosts is baked into images, which should let `openstack server rebuild` work
-  become: true
-  tags: etc_hosts
-  tasks:
-    - name: Create /etc/hosts for all nodes as DNS doesn't work
-      blockinfile:
-        path: /etc/hosts
-        create: yes
-        state: present
-        block: |
-          {% for hostname in groups['all'] %}
-          {{ hostvars[hostname]['ansible_host'] }} {{ hostname }}
-          {% endfor %}
-
 - hosts: control
   become: yes
   gather_facts: false
@@ -28,4 +14,4 @@
       loop:
         - "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/inventory/hosts"
         - "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/inventory/group_vars/all/secrets.yml"
-        - "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/inventory/group_vars/basic_users/defaults.yml"
+        - "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/inventory/group_vars/all/test_user.yml"
diff --git a/environments/arcus/inventory/extra_groups b/environments/arcus/inventory/extra_groups
@@ -4,3 +4,6 @@ cluster
 [rebuild:children]
 control
 compute
+
+[etc_hosts:children]
+cluster
diff --git a/environments/arcus/inventory/group_vars/all/basic_users.yml b/environments/arcus/inventory/group_vars/all/basic_users.yml
@@ -0,0 +1,7 @@
+# has to be defined on 'all' group so localhost can template out for cloud-init
+testuser_password: "{{ lookup('env', 'TESTUSER_PASSWORD') | default(vault_testuser_password, true) }}"
+
+basic_users_users:
+  - name: testuser # can't use rocky as $HOME isn't shared!
+    password: "{{ testuser_password | password_hash('sha512', 65534 | random(seed=inventory_hostname) | string) }}" # idempotent
+    uid: 1005
diff --git a/environments/arcus/inventory/group_vars/all/cloud_init.yml b/environments/arcus/inventory/group_vars/all/cloud_init.yml
@@ -0,0 +1 @@
+../../../../skeleton/{{cookiecutter.environment}}/inventory/group_vars/all/cloud_init.yml
diff --git a/environments/arcus/inventory/group_vars/basic_users/overrides.yml b/environments/arcus/inventory/group_vars/basic_users/overrides.yml
@@ -1,4 +1,4 @@
-test_user_password: "{{ lookup('env', 'TEST_USER_PASSWORD') | default(vault_testuser_password, true) }}" # CI uses env, debug can set vault_testuser_password
+test_user_password: "{{ lookup('env', 'TESTUSER_PASSWORD') | default(vault_testuser_password, true) }}" # CI uses env, debug can set vault_testuser_password
 
 basic_users_users:
   - name: testuser # can't use rocky as $HOME isn't shared!
diff --git a/environments/arcus/terraform/main.tf b/environments/arcus/terraform/main.tf
@@ -8,6 +8,12 @@ variable "cluster_name" {
     description = "Name for cluster, used as prefix for resources - set by environment var in CI"
 }
 
+variable "create_nodes" {
+    description = "Whether to create nodes (servers) or just ports and other infra"
+    type = bool # can't use bool as want to pass from command-line
+    default = true
+}
+
 module "cluster" {
     source = "../../skeleton/{{cookiecutter.environment}}/terraform/"
 
@@ -42,6 +48,7 @@ module "cluster" {
         compute-2: "extra"
         compute-3: "extra"
     }
+    create_nodes = var.create_nodes
     
     environment_root = var.environment_root
 }
diff --git a/environments/common/inventory/group_vars/all/cloud_init.yml b/environments/common/inventory/group_vars/all/cloud_init.yml
@@ -0,0 +1,27 @@
+cloud_init_output_path: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/cloud_init"
+
+etc_hosts_template: |
+  127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
+  ::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
+
+  {% for hostname in groups['etc_hosts'] | sort -%}
+  {{ hostvars[hostname]['ansible_host'] }} {{ hostname }}
+  {% endfor -%}
+
+cloud_init_userdata_etchosts:
+  module: write_files
+  group: etc_hosts
+  template: |-
+    # etc_hosts group:
+      - path: /etc/hosts
+        encoding: text/plain
+        permissions: "0644"
+        owner: root:root
+        content: |
+          {{ etc_hosts_template | indent(width=6, first=false) }}
+
+cloud_init_userdata_templates_default: # provides indirection so environments could copy parts of default without having to rewrite the entire thing
+  - "{{ cloud_init_userdata_etchosts }}"
+
+cloud_init_userdata_templates_extra: []
+cloud_init_userdata_templates: "{{ cloud_init_userdata_templates_default + cloud_init_userdata_templates_extra }}"
diff --git a/environments/common/inventory/groups b/environments/common/inventory/groups
@@ -87,6 +87,13 @@ fail2ban
 [openondemand_jupyter]
 # Subset of compute to run a Jupyter Notebook servers on via Open Ondemand
 
+[etc_hosts]
+# Hosts to manage /etc/hosts e.g. if no internal DNS. See ansible/roles/etc_hosts/README.md
+
+[cloud_init:children]
+# Hosts to template out cloud_init data for
+etc_hosts
+
 [systemd:children]
 # Hosts to make systemd unit adjustments on
 opendistro
diff --git a/environments/common/layouts/everything b/environments/common/layouts/everything
@@ -51,3 +51,6 @@ compute
 [openondemand_jupyter:children]
 # Subset of compute to run a Jupyter Notebook servers on via Open Ondemand
 compute
+
+[etc_hosts]
+# Hosts to manage /etc/hosts e.g. if no internal DNS. See ansible/roles/etc_hosts/README.md
diff --git a/environments/skeleton/{{cookiecutter.environment}}/cloud_init/.gitkeep b/environments/skeleton/{{cookiecutter.environment}}/cloud_init/.gitkeep
diff --git a/environments/skeleton/{{cookiecutter.environment}}/inventory/group_vars/all/cloud_init.yml b/environments/skeleton/{{cookiecutter.environment}}/inventory/group_vars/all/cloud_init.yml
@@ -0,0 +1,22 @@
+# This provides cloud_init templates to manage and mount the Terraform-provided
+# volumes on the control node for `appliances_state_dir` and $HOME.
+# Note volumes MUST be defined using block_device{} inside openstack_compute_instance_v2,
+# NOT attached after boot, else the device ordering is liable to change e.g. on reboots.
+
+cloud_init_userdata_templates_extra:
+  - module: fs_setup
+    group: control
+    template: |
+        - label: state
+          filesystem: ext4
+          device: /dev/vdb
+          partition: auto
+        - label: home
+          filesystem: ext4
+          device: /dev/vdc
+          partition: auto
+  - module: mounts
+    group: control
+    template: |
+        - [LABEL=state, /var/lib/state]
+        - [LABEL=home, /exports/home, auto, "x-systemd.required-by=nfs-server.service,x-systemd.before=nfs-server.service"]
diff --git a/environments/skeleton/{{cookiecutter.environment}}/terraform/inventory.tf b/environments/skeleton/{{cookiecutter.environment}}/terraform/inventory.tf
@@ -1,11 +1,11 @@
 resource "local_file" "hosts" {
   content  = templatefile("${path.module}/inventory.tpl",
                           {
-                            "cluster_name": var.cluster_name
-                            "control": openstack_compute_instance_v2.control,
-                            "state_dir": var.state_dir
-                            "logins": openstack_compute_instance_v2.login,
-                            "computes": openstack_compute_instance_v2.compute,
+                            "cluster_name": var.cluster_name,
+                            "control": openstack_networking_port_v2.control,
+                            "state_dir": var.state_dir,
+                            "logins": openstack_networking_port_v2.login,
+                            "computes": openstack_networking_port_v2.compute,
                             "compute_types": var.compute_types,
                             "compute_nodes": var.compute_nodes,
                             "subnet": data.openstack_networking_subnet_v2.cluster_subnet,
diff --git a/environments/skeleton/{{cookiecutter.environment}}/terraform/inventory.tpl b/environments/skeleton/{{cookiecutter.environment}}/terraform/inventory.tpl
@@ -3,20 +3,20 @@ ansible_user=rocky
 openhpc_cluster_name=${cluster_name}
 
 [control]
-${control.name} ansible_host=${[for n in control.network: n.fixed_ip_v4 if n.access_network][0]} server_networks='${jsonencode({for net in control.network: net.name => [ net.fixed_ip_v4 ] })}'
+${control.name} ansible_host=${control.all_fixed_ips[0]}
 
 [control:vars]
 # NB needs to be set on group not host otherwise it is ignored in packer build!
 appliances_state_dir=${state_dir}
 
 [login]
 %{ for login in logins ~}
-${login.name} ansible_host=${[for n in login.network: n.fixed_ip_v4 if n.access_network][0]} server_networks='${jsonencode({for net in login.network: net.name => [ net.fixed_ip_v4 ] })}'
+${login.name} ansible_host=${login.all_fixed_ips[0]}
 %{ endfor ~}
 
 [compute]
 %{ for compute in computes ~}
-${compute.name} ansible_host=${[for n in compute.network: n.fixed_ip_v4 if n.access_network][0]} server_networks='${jsonencode({for net in compute.network: net.name => [ net.fixed_ip_v4 ] })}'
+${compute.name} ansible_host=${compute.all_fixed_ips[0]}
 %{ endfor ~}
 
 # Define groups for slurm parititions:
diff --git a/environments/skeleton/{{cookiecutter.environment}}/terraform/nodes.tf b/environments/skeleton/{{cookiecutter.environment}}/terraform/nodes.tf
diff --git a/environments/skeleton/{{cookiecutter.environment}}/terraform/variables.tf b/environments/skeleton/{{cookiecutter.environment}}/terraform/variables.tf

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+#cloud_init_output_path:`
	`2`	`+cloud_init_userdata_templates: []`