Use workflow images in aio ci

This change begins using stackhpc-built images in AIO CI jobs.

The disk size of the AIO runners is minimised, and an LVM
configuration is now included in the AIO environment.

Minor improvements to the build process and AIO configuration are
included to allow the CI tests to pass.

Author: Alex-Welsh

.github/workflows/stackhpc-all-in-one.yml

@@ -27,14 +27,14 @@ on:
         description: Neutron ML2 plugin
         type: string
         required: true
-      vm_image:
-        description: Image for the all-in-one VM
+      vm_image_override:
+        description: Full name of an image to use instead of the default
         type: string
-        default: bb8c0a34-533f-42fb-a49b-3461e677f3f6
+        default: ""
       vm_interface:
         description: Default network interface name
         type: string
-        default: eth0
+        default: ens3
       vm_flavor:
         description: Flavor for the all-in-one VM
         type: string
@@ -76,6 +76,21 @@ jobs:
         with:
            submodules: true
 
+      - name: Output image tag
+        id: image_tag
+        run: |
+          echo ::set-output name=image_tag::$(grep stackhpc_${{ inputs.os_distribution }}_${{ inputs.os_release }}_overcloud_host_image_version etc/kayobe/environments/ci-aio/stackhpc-ci.yml | awk '{print $2}')
+
+      # Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag
+      - name: Output image name
+        id: image_name
+        run: |
+          if [ -z "${{ inputs.vm_image_override }}" ]; then
+            echo ::set-output name=image_name::overcloud-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ steps.image_tag.outputs.image_tag }}
+          else
+            echo ::set-output name=image_name::${{ inputs.vm_image_override }}
+          fi
+
       - name: Install terraform
         uses: hashicorp/setup-terraform@v2
 
@@ -110,7 +125,7 @@ jobs:
         env:
           SSH_USERNAME: "${{ inputs.ssh_username }}"
           VM_NAME: "skc-ci-aio-${{ inputs.neutron_plugin }}-${{ github.run_id }}"
-          VM_IMAGE: ${{ inputs.vm_image }}
+          VM_IMAGE: ${{ steps.image_name.outputs.image_name }}
           VM_FLAVOR: ${{ inputs.vm_flavor }}
           VM_NETWORK: ${{ inputs.vm_network }}
           VM_SUBNET: ${{ inputs.vm_subnet }}
@@ -191,6 +206,16 @@ jobs:
         run: |
           sudo docker image pull $KAYOBE_IMAGE
 
+      - name: Run growroot
+        run: |
+          sudo -E docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/growroot.yml'
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+
       - name: Host configure
         run: |
           sudo -E docker run -t --rm \
@@ -243,6 +268,13 @@ jobs:
         run: |
           test $(wc -l < tempest-artifacts/failed-tests) -lt 1
 
+      - name: Upload logs artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: kolla-logs-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}
+          path: logs.zip
+        if: failure()
+
       - name: Destroy
         run: terraform destroy -auto-approve
         working-directory: ${{ github.workspace }}/terraform/aio

.github/workflows/stackhpc-pull-request.yml

@@ -77,8 +77,6 @@ jobs:
       neutron_plugin: ovs
       # NOTE: The current SMS lab Rocky8 image has moved ahead of our release
       # train snapshots, causing failures installing some packages.
-      vm_image: Rocky8-2022-11-08
-      vm_interface: ens3
       OS_CLOUD: sms-lab-release
     secrets: inherit
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
@@ -95,14 +93,12 @@ jobs:
       neutron_plugin: ovn
       # NOTE: The current SMS lab Rocky8 image has moved ahead of our release
       # train snapshots, causing failures installing some packages.
-      vm_image: Rocky8-2022-11-08
-      vm_interface: ens3
       OS_CLOUD: sms-lab-release
     secrets: inherit
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
 
-  all-in-one-ubuntu-ovs:
-    name: aio (Ubuntu OVS)
+  all-in-one-ubuntu-focal-ovs:
+    name: aio (Ubuntu Focal OVS)
     needs:
       - build-kayobe-image
     uses: ./.github/workflows/stackhpc-all-in-one.yml
@@ -112,13 +108,12 @@ jobs:
       os_release: focal
       ssh_username: ubuntu
       neutron_plugin: ovs
-      vm_image: Ubuntu-20.04
       OS_CLOUD: sms-lab-release
     secrets: inherit
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
 
-  all-in-one-ubuntu-ovn:
-    name: aio (Ubuntu OVN)
+  all-in-one-ubuntu-focal-ovn:
+    name: aio (Ubuntu Focal OVN)
     needs:
       - build-kayobe-image
     uses: ./.github/workflows/stackhpc-all-in-one.yml
@@ -128,7 +123,36 @@ jobs:
       os_release: focal
       ssh_username: ubuntu
       neutron_plugin: ovn
-      vm_image: Ubuntu-20.04
+      OS_CLOUD: sms-lab-release
+    secrets: inherit
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+
+  all-in-one-ubuntu-jammy-ovs:
+    name: aio (Ubuntu Jammy OVS)
+    needs:
+      - build-kayobe-image
+    uses: ./.github/workflows/stackhpc-all-in-one.yml
+    with:
+      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+      os_distribution: ubuntu
+      os_release: jammy
+      ssh_username: ubuntu
+      neutron_plugin: ovs
+      OS_CLOUD: sms-lab-release
+    secrets: inherit
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+
+  all-in-one-ubuntu-jammy-ovn:
+    name: aio (Ubuntu Jammy OVN)
+    needs:
+      - build-kayobe-image
+    uses: ./.github/workflows/stackhpc-all-in-one.yml
+    with:
+      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+      os_distribution: ubuntu
+      os_release: jammy
+      ssh_username: ubuntu
+      neutron_plugin: ovn
       OS_CLOUD: sms-lab-release
     secrets: inherit
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
@@ -144,8 +168,6 @@ jobs:
       os_release: "9"
       ssh_username: cloud-user
       neutron_plugin: ovs
-      vm_image: Rocky9
-      vm_interface: ens3
       OS_CLOUD: sms-lab-release
     secrets: inherit
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
@@ -161,8 +183,6 @@ jobs:
       os_release: "9"
       ssh_username: cloud-user
       neutron_plugin: ovn
-      vm_image: Rocky9
-      vm_interface: ens3
       OS_CLOUD: sms-lab-release
     secrets: inherit
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'

etc/kayobe/environments/ci-aio/controllers.yml

@@ -5,3 +5,8 @@
 # User with which to access the controllers via SSH during bootstrap, in order
 # to setup the Kayobe user account. Default is {{ os_distribution }}.
 controller_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cloud-user' }}"
+
+# Controller lvm configuration. See intentory/group_vars/controllers/lvm.yml
+# for the exact configuration.
+controller_lvm_groups:
+  - "{{ stackhpc_lvm_group_rootvg }}"

etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/lvm.yml

@@ -0,0 +1,34 @@
+---
+# List of extra LVs to include in the rootvg VG.
+stackhpc_lvm_group_rootvg_lvs_extra:
+  - "{{ stackhpc_lvm_lv_docker }}"
+
+###############################################################################
+# StackHPC LVM Logical Volume (LV) configuration.
+
+# StackHPC LVM lv_swap LV size.
+stackhpc_lvm_lv_swap_size: 500m
+
+# StackHPC LVM lv_root LV size.
+stackhpc_lvm_lv_root_size: 2g
+
+# StackHPC LVM lv_tmp LV size.
+stackhpc_lvm_lv_tmp_size: 2g
+
+# StackHPC LVM lv_var LV size.
+stackhpc_lvm_lv_var_size: 2g
+
+# StackHPC LVM lv_var_tmp LV size.
+stackhpc_lvm_lv_var_tmp_size: 2g
+
+# StackHPC LVM lv_log LV size.
+stackhpc_lvm_lv_log_size: 2g
+
+# StackHPC LVM lv_audit LV size.
+stackhpc_lvm_lv_audit_size: 250m
+
+# StackHPC LVM lv_home LV size.
+stackhpc_lvm_lv_home_size: 2g
+
+# StackHPC LVM lv_docker LV size.
+stackhpc_lvm_lv_docker_size: 95%FREE

etc/kayobe/environments/ci-aio/stackhpc-ci.yml

@@ -80,3 +80,10 @@ stackhpc_docker_registry_password: !vault |
   38333133393730633666613965653364316162353337313330346164303631313731646461363461
   3963323635373866630a633533376339363734626664333765313665623662613764363038383735
   38646138376438643533376161376634653439386230353365316239613430363338
+
+# Overcloud host image tags
+stackhpc_centos_8-stream_overcloud_host_image_version: "yoga-20230525T095243"
+stackhpc_rocky_8_overcloud_host_image_version: "yoga-20230524T084837"
+stackhpc_rocky_9_overcloud_host_image_version: "yoga-20230515T145140"
+stackhpc_ubuntu_focal_overcloud_host_image_version: "yoga-20230609T120720"
+stackhpc_ubuntu_jammy_overcloud_host_image_version: "yoga-20230609T120720"

etc/kayobe/stackhpc-overcloud-dib.yml

@@ -57,14 +57,21 @@ stackhpc_overcloud_dib_env_vars:
   # FIXME: Support templating repo files.
   # DIB_YUM_MINIMAL_BOOTSTRAP_REPOS: /path/to/dir/containing/dib-mirror-*.repo
   YUM: dnf
+  # Workaround for stack user home ownership bug
+  DIB_IMAGE_CACHE: "/tmp/yum"
 
 # StackHPC overcloud DIB image packages.
 stackhpc_overcloud_dib_packages:
   - "logrotate"
   - "net-tools"
   - "vim"
+  - "git"
+  - "less"
   - "{% if os_distribution == 'ubuntu' %}netbase{% endif %}"
   - "{% if os_distribution == 'ubuntu' %}iputils-ping{% endif %}"
+  - "{% if os_distribution == 'ubuntu' %}curl{% endif %}"
+  - "{% if os_distribution == 'ubuntu' %}apt-utils{% endif %}"
+  - "{% if os_distribution == 'centos' %}openssh-clients{% endif %}"
 
 # StackHPC overcloud DIB image block device configuration.
 # This image layout conforms to the CIS partition benchmarks.

etc/kayobe/stackhpc-overcloud-host-images.yml

@@ -0,0 +1,28 @@
+---
+##############################
+# Overcloud host image sources
+
+# WIP - currently unused
+# CentOS stream 8 overcloud host image source
+stackhpc_centos_8-stream_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/{{ openstack_release }}/centos/8-stream/{{ stackhpc_centos_overcloud_host_image_version }}/overcloud-centos-8-stream.qcow2"
+stackhpc_centos_8-stream_overcloud_host_image_version: "yoga-20230525T095243"
+
+# Rocky 8 overcloud host image source
+stackhpc_rocky_8_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/{{ openstack_release }}/rocky/8/{{ stackhpc_rocky_overcloud_host_image_version }}/overcloud-rocky-linux-8.qcow2"
+stackhpc_rocky_8_overcloud_host_image_version: "yoga-20230524T084837"
+
+# Rocky 9 overcloud host image source
+stackhpc_rocky_9_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/{{ openstack_release }}/rocky/9/{{ stackhpc_rocky_overcloud_host_image_version }}/overcloud-rocky-linux-9.qcow2"
+stackhpc_rocky_9_overcloud_host_image_version: "yoga-20230515T145140"
+
+# Ubuntu focal overcloud host image source
+stackhpc_ubuntu_focal_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/{{ openstack_release }}/ubuntu/focal/{{ stackhpc_ubuntu_overcloud_host_image_version }}/overcloud-ubuntu-focal.qcow2"
+stackhpc_ubuntu_focal_overcloud_host_image_version: "yoga-20230609T120720"
+
+# Ubuntu jammy overcloud host image source
+stackhpc_ubuntu_jammy_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/{{ openstack_release }}/ubuntu/jammy/{{ stackhpc_ubuntu_overcloud_host_image_version }}/overcloud-ubuntu-jammy.qcow2"
+stackhpc_ubuntu_jammy_overcloud_host_image_version: "yoga-20230609T120720"
+
+# Whether to download overcloud host images from Ark. Default is true.
+# WIP - not yet implemented. Waiting on upstream bifrost changes
+# stackhpc_download_overcloud_images: false

terraform/aio/vm.tf

@@ -59,7 +59,7 @@ resource "openstack_compute_instance_v2" "kayobe-aio" {
   block_device {
     uuid                  = local.image_is_uuid ? var.aio_vm_image: data.openstack_images_image_v2.image[0].id
     source_type           = "image"
-    volume_size           = 100
+    volume_size           = 35
     boot_index            = 0
     destination_type      = "volume"
     delete_on_termination = true
@@ -73,8 +73,12 @@ resource "null_resource" "kayobe-aio" {
       host = openstack_compute_instance_v2.kayobe-aio.access_ip_v4
       user = var.ssh_username
       private_key = file("id_rsa")
+      script_path = "/home/${var.ssh_username}/start.sh"
     }
 
-    inline = ["echo 'connected!'"]
+    inline = [
+      "#!/bin/sh",
+      "echo 'connected!'"
+      ]
   }
 }