Merge pull request #515 from stackhpc/aio-images

Use workflow images in aio ci

Author: Alex-Welsh

.github/workflows/stackhpc-all-in-one.yml

@@ -27,14 +27,14 @@ on:
         description: Neutron ML2 plugin
         type: string
         required: true
-      vm_image:
-        description: Image for the all-in-one VM
+      vm_image_override:
+        description: Full name of an image to use instead of the default
         type: string
-        default: bb8c0a34-533f-42fb-a49b-3461e677f3f6
+        default: ""
       vm_interface:
         description: Default network interface name
         type: string
-        default: eth0
+        default: ens3
       vm_flavor:
         description: Flavor for the all-in-one VM
         type: string
@@ -76,6 +76,21 @@ jobs:
         with:
            submodules: true
 
+      - name: Output image tag
+        id: image_tag
+        run: |
+          echo image_tag=$(grep stackhpc_${{ inputs.os_distribution }}_${{ inputs.os_release }}_overcloud_host_image_version etc/kayobe/environments/ci-aio/stackhpc-ci.yml | awk '{print $2}') >> $GITHUB_OUTPUT
+
+      # Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag
+      - name: Output image name
+        id: image_name
+        run: |
+          if [ -z "${{ inputs.vm_image_override }}" ]; then
+            echo image_name=overcloud-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ steps.image_tag.outputs.image_tag }} >> $GITHUB_OUTPUT
+          else
+            echo image_name=${{ inputs.vm_image_override }} >> $GITHUB_OUTPUT
+          fi
+
       - name: Install terraform
         uses: hashicorp/setup-terraform@v2
 
@@ -110,7 +125,7 @@ jobs:
         env:
           SSH_USERNAME: "${{ inputs.ssh_username }}"
           VM_NAME: "skc-ci-aio-${{ inputs.neutron_plugin }}-${{ github.run_id }}"
-          VM_IMAGE: ${{ inputs.vm_image }}
+          VM_IMAGE: ${{ steps.image_name.outputs.image_name }}
           VM_FLAVOR: ${{ inputs.vm_flavor }}
           VM_NETWORK: ${{ inputs.vm_network }}
           VM_SUBNET: ${{ inputs.vm_subnet }}
@@ -191,6 +206,16 @@ jobs:
         run: |
           sudo docker image pull $KAYOBE_IMAGE
 
+      - name: Run growroot
+        run: |
+          sudo -E docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/growroot.yml'
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+
       - name: Host configure
         run: |
           sudo -E docker run -t --rm \

.github/workflows/stackhpc-pull-request.yml

@@ -75,10 +75,6 @@ jobs:
       os_distribution: rocky
       os_release: "8"
       neutron_plugin: ovs
-      # NOTE: The current SMS lab Rocky8 image has moved ahead of our release
-      # train snapshots, causing failures installing some packages.
-      vm_image: Rocky8-2022-11-08
-      vm_interface: ens3
       OS_CLOUD: sms-lab-release
     secrets: inherit
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
@@ -93,10 +89,6 @@ jobs:
       os_distribution: rocky
       os_release: "8"
       neutron_plugin: ovn
-      # NOTE: The current SMS lab Rocky8 image has moved ahead of our release
-      # train snapshots, causing failures installing some packages.
-      vm_image: Rocky8-2022-11-08
-      vm_interface: ens3
       OS_CLOUD: sms-lab-release
     secrets: inherit
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
@@ -112,7 +104,6 @@ jobs:
       os_release: focal
       ssh_username: ubuntu
       neutron_plugin: ovs
-      vm_image: Ubuntu-20.04
       OS_CLOUD: sms-lab-release
     secrets: inherit
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
@@ -128,7 +119,36 @@ jobs:
       os_release: focal
       ssh_username: ubuntu
       neutron_plugin: ovn
-      vm_image: Ubuntu-20.04
+      OS_CLOUD: sms-lab-release
+    secrets: inherit
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+
+  all-in-one-ubuntu-jammy-ovs:
+    name: aio (Ubuntu Jammy OVS)
+    needs:
+      - build-kayobe-image
+    uses: ./.github/workflows/stackhpc-all-in-one.yml
+    with:
+      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+      os_distribution: ubuntu
+      os_release: jammy
+      ssh_username: ubuntu
+      neutron_plugin: ovs
+      OS_CLOUD: sms-lab-release
+    secrets: inherit
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+
+  all-in-one-ubuntu-jammy-ovn:
+    name: aio (Ubuntu Jammy OVN)
+    needs:
+      - build-kayobe-image
+    uses: ./.github/workflows/stackhpc-all-in-one.yml
+    with:
+      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+      os_distribution: ubuntu
+      os_release: jammy
+      ssh_username: ubuntu
+      neutron_plugin: ovn
       OS_CLOUD: sms-lab-release
     secrets: inherit
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
@@ -144,8 +164,6 @@ jobs:
       os_release: "9"
       ssh_username: cloud-user
       neutron_plugin: ovs
-      vm_image: Rocky9
-      vm_interface: ens3
       OS_CLOUD: sms-lab-release
     secrets: inherit
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
@@ -161,8 +179,6 @@ jobs:
       os_release: "9"
       ssh_username: cloud-user
       neutron_plugin: ovn
-      vm_image: Rocky9
-      vm_interface: ens3
       OS_CLOUD: sms-lab-release
     secrets: inherit
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'

etc/kayobe/environments/ci-aio/controllers.yml

@@ -5,3 +5,8 @@
 # User with which to access the controllers via SSH during bootstrap, in order
 # to setup the Kayobe user account. Default is {{ os_distribution }}.
 controller_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cloud-user' }}"
+
+# Controller lvm configuration. See intentory/group_vars/controllers/lvm.yml
+# for the exact configuration.
+controller_lvm_groups:
+  - "{{ stackhpc_lvm_group_rootvg }}"

etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/lvm.yml

@@ -0,0 +1,34 @@
+---
+# List of extra LVs to include in the rootvg VG.
+stackhpc_lvm_group_rootvg_lvs_extra:
+  - "{{ stackhpc_lvm_lv_docker }}"
+
+###############################################################################
+# StackHPC LVM Logical Volume (LV) configuration.
+
+# StackHPC LVM lv_swap LV size.
+stackhpc_lvm_lv_swap_size: 500m
+
+# StackHPC LVM lv_root LV size.
+stackhpc_lvm_lv_root_size: 2g
+
+# StackHPC LVM lv_tmp LV size.
+stackhpc_lvm_lv_tmp_size: 2g
+
+# StackHPC LVM lv_var LV size.
+stackhpc_lvm_lv_var_size: 2g
+
+# StackHPC LVM lv_var_tmp LV size.
+stackhpc_lvm_lv_var_tmp_size: 2g
+
+# StackHPC LVM lv_log LV size.
+stackhpc_lvm_lv_log_size: 2g
+
+# StackHPC LVM lv_audit LV size.
+stackhpc_lvm_lv_audit_size: 250m
+
+# StackHPC LVM lv_home LV size.
+stackhpc_lvm_lv_home_size: 2g
+
+# StackHPC LVM lv_docker LV size.
+stackhpc_lvm_lv_docker_size: 95%FREE

etc/kayobe/environments/ci-aio/stackhpc-ci.yml

@@ -80,3 +80,10 @@ stackhpc_docker_registry_password: !vault |
   38333133393730633666613965653364316162353337313330346164303631313731646461363461
   3963323635373866630a633533376339363734626664333765313665623662613764363038383735
   38646138376438643533376161376634653439386230353365316239613430363338
+
+# Overcloud host image tags
+stackhpc_centos_8-stream_overcloud_host_image_version: "yoga-20230525T095243"
+stackhpc_rocky_8_overcloud_host_image_version: "yoga-20230629T135322"
+stackhpc_rocky_9_overcloud_host_image_version: "yoga-20230515T145140"
+stackhpc_ubuntu_focal_overcloud_host_image_version: "yoga-20230609T120720"
+stackhpc_ubuntu_jammy_overcloud_host_image_version: "yoga-20230609T120720"

etc/kayobe/stackhpc-overcloud-dib.yml

@@ -57,14 +57,21 @@ stackhpc_overcloud_dib_env_vars:
   # FIXME: Support templating repo files.
   # DIB_YUM_MINIMAL_BOOTSTRAP_REPOS: /path/to/dir/containing/dib-mirror-*.repo
   YUM: dnf
+  # Workaround for stack user home ownership bug
+  DIB_IMAGE_CACHE: "/tmp/yum"
 
 # StackHPC overcloud DIB image packages.
 stackhpc_overcloud_dib_packages:
   - "logrotate"
   - "net-tools"
   - "vim"
+  - "git"
+  - "less"
   - "{% if os_distribution == 'ubuntu' %}netbase{% endif %}"
   - "{% if os_distribution == 'ubuntu' %}iputils-ping{% endif %}"
+  - "{% if os_distribution == 'ubuntu' %}curl{% endif %}"
+  - "{% if os_distribution == 'ubuntu' %}apt-utils{% endif %}"
+  - "{% if os_distribution == 'centos' %}openssh-clients{% endif %}"
 
 # StackHPC overcloud DIB image block device configuration.
 # This image layout conforms to the CIS partition benchmarks.

terraform/aio/vm.tf

@@ -59,7 +59,7 @@ resource "openstack_compute_instance_v2" "kayobe-aio" {
   block_device {
     uuid                  = local.image_is_uuid ? var.aio_vm_image: data.openstack_images_image_v2.image[0].id
     source_type           = "image"
-    volume_size           = 100
+    volume_size           = 35
     boot_index            = 0
     destination_type      = "volume"
     delete_on_termination = true
@@ -73,8 +73,14 @@ resource "null_resource" "kayobe-aio" {
       host = openstack_compute_instance_v2.kayobe-aio.access_ip_v4
       user = var.ssh_username
       private_key = file("id_rsa")
+      # Terraform will run the start script from /tmp by default. For the
+      # current images, /tmp is noexec, so the path must be changed
+      script_path = "/home/${var.ssh_username}/start.sh"
     }
 
-    inline = ["echo 'connected!'"]
+    inline = [
+      "#!/bin/sh",
+      "echo 'connected!'"
+      ]
   }
 }