Add periodic builds for overcloud image builds.

Author: MaxBed4d

.ansible-lint-ignore

@@ -5,3 +5,5 @@ etc/kayobe/ansible/vault-generate-internal-tls.yml fqcn[action-core]
 etc/kayobe/ansible/vault-generate-test-external-tls.yml fqcn[action-core]
 etc/kayobe/ansible/rabbitmq-reset.yml command-instead-of-module
 etc/kayobe/ansible/ubuntu-upgrade.yml syntax-check[missing-file]
+etc/kayobe/ansible/check-kayobe-version.yml command-instead-of-module
+etc/kayobe/ansible/check-kolla-ansible-version.yml command-instead-of-module

.automation

@@ -2,6 +2,14 @@
 name: Build Amphora image
 on:
   workflow_dispatch:
+    inputs:
+      runner_env:
+        description: Which cloud to run on?
+        type: choice
+        default: SMS Lab
+        options:
+          - SMS Lab
+          - Leafcloud
     secrets:
       KAYOBE_VAULT_PASSWORD:
         required: true
@@ -17,10 +25,17 @@ env:
   KAYOBE_ENVIRONMENT: ci-builder
   KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 jobs:
+  runner-selection:
+    uses: ./.github/workflows/runner-selector.yml
+    with:
+      runner_env: ${{ inputs.runner_env }}
   amphora-image-build:
     name: Build Amphora image
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
-    runs-on: arc-skc-host-image-builder-runner
+    environment: ${{ inputs.runner_env }}
+    runs-on: ${{ needs.runner-selection.outputs.runner_name_image_build }}
+    needs:
+      - runner-selection
     permissions: {}
     steps:
 
@@ -90,10 +105,10 @@ jobs:
           aio_vm_name = "skc-amphora-image-builder"
           # Must be an Ubuntu Jammy host to successfully build all images
           # This MUST NOT be an LVM image. It can cause confusing conficts with the built image.
-          aio_vm_image = "Ubuntu-22.04"
-          aio_vm_flavor = "en1.medium"
-          aio_vm_network = "stackhpc-ci"
-          aio_vm_subnet = "stackhpc-ci"
+          aio_vm_image = "${{ vars.HOST_IMAGE_BUILD_IMAGE }}"
+          aio_vm_flavor = "${{ vars.HOST_IMAGE_BUILD_FLAVOR }}"
+          aio_vm_network = "${{ vars.HOST_IMAGE_BUILD_NETWORK }}"
+          aio_vm_subnet = "${{ vars.HOST_IMAGE_BUILD_SUBNET }}"
           aio_vm_interface = "ens3"
           EOF
         working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
@@ -122,7 +137,7 @@ jobs:
           exit 1
         working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
         env:
-          OS_CLOUD: "openstack"
+          OS_CLOUD: ${{ vars.OS_CLOUD }}
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
 
@@ -250,7 +265,7 @@ jobs:
         run: terraform destroy -auto-approve
         working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
         env:
-          OS_CLOUD: openstack
+          OS_CLOUD: ${{ vars.OS_CLOUD }}
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
         if: always()

.github/workflows/amphora-image-build.yml

@@ -7,8 +7,8 @@ on:
         description: Build Rocky Linux 9
         type: boolean
         default: true
-      ubuntu-jammy:
-        description: Build Ubuntu 22.04 Jammy
+      ubuntu-noble:
+        description: Build Ubuntu 24.04 Noble
         type: boolean
         default: true
       runner_env:
@@ -212,20 +212,20 @@ jobs:
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
-      - name: Build a Ubuntu 22.04 Jammy IPA image
-        id: build_ubuntu_jammy_ipa
+      - name: Build a Ubuntu 24.04 Noble IPA image
+        id: build_ubuntu_noble_ipa
         continue-on-error: true
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe overcloud deployment image build --force-rebuild \
           -e os_distribution="ubuntu" \
-          -e os_release="jammy" \
+          -e os_release="noble" \
           -e ipa_ci_builder_distribution="ubuntu" \
-          -e ipa_ci_builder_release="jammy"
+          -e ipa_ci_builder_release="noble"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.ubuntu-jammy
+        if: inputs.ubuntu-noble
 
       - name: Show last error logs
         continue-on-error: true
@@ -235,9 +235,9 @@ jobs:
           kayobe seed host command run --command "tail -200 /opt/kayobe/images/ipa/ipa.stdout" --show-output
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: steps.build_ubuntu_jammy_ipa.outcome == 'failure'
+        if: steps.build_ubuntu_noble_ipa.outcome == 'failure'
 
-      - name: Upload Ubuntu 22.04 Jammy IPA kernel image to Ark
+      - name: Upload Ubuntu 24.04 Noble IPA kernel image to Ark
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
@@ -247,14 +247,13 @@ jobs:
           -e artifact_type=ipa-images \
           -e artifact_tag=${{ steps.ipa_image_tag.outputs.ipa_image_tag }} \
           -e os_distribution="ubuntu" \
-          -e os_release="jammy" \
-          -e file_regex='*.kernel' \
-          -e upload_checksum=true
+          -e os_release="noble" \
+          -e file_regex='*.kernel'
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy_ipa.outcome == 'success'
+        if: inputs.ubuntu-noble && steps.build_ubuntu_noble_ipa.outcome == 'success'
 
-      - name: Upload Ubuntu 22.04 Jammy IPA ramdisk image to Ark
+      - name: Upload Ubuntu 24.04 Noble IPA ramdisk image to Ark
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
@@ -264,12 +263,11 @@ jobs:
           -e artifact_type=ipa-images \
           -e artifact_tag=${{ steps.ipa_image_tag.outputs.ipa_image_tag }} \
           -e os_distribution="ubuntu" \
-          -e os_release="jammy" \
-          -e file_regex='*.initramfs' \
-          -e upload_checksum=true
+          -e os_release="noble" \
+          -e file_regex='*.initramfs'
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy_ipa.outcome == 'success'
+        if: inputs.ubuntu-noble && steps.build_ubuntu_noble_ipa.outcome == 'success'
 
       - name: Build a Rocky 9 IPA image
         id: build_rocky_9_ipa
@@ -307,8 +305,7 @@ jobs:
           -e artifact_tag=${{ steps.ipa_image_tag.outputs.ipa_image_tag }} \
           -e os_distribution="rocky" \
           -e os_release="9" \
-          -e file_regex='*.kernel' \
-          -e upload_checksum=true
+          -e file_regex='*.kernel'
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.rocky9 && steps.build_rocky_9_ipa.outcome == 'success'
@@ -324,8 +321,7 @@ jobs:
           -e artifact_tag=${{ steps.ipa_image_tag.outputs.ipa_image_tag }} \
           -e os_distribution="rocky" \
           -e os_release="9" \
-          -e file_regex='*.initramfs' \
-          -e upload_checksum=true
+          -e file_regex='*.initramfs'
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.rocky9 && steps.build_rocky_9_ipa.outcome == 'success'
@@ -348,7 +344,7 @@ jobs:
           echo "Builds failed. See workflow artifacts for details." &&
           exit 1
         if: steps.build_rocky_9_ipa.outcome == 'failure' ||
-            steps.build_ubuntu_jammy_ipa.outcome == 'failure'
+            steps.build_ubuntu_noble_ipa.outcome == 'failure'
 
       - name: Destroy
         run: terraform destroy -auto-approve

.github/workflows/ipa-image-build.yml

@@ -7,8 +7,8 @@ on:
         description: Promote Rocky Linux 9
         type: boolean
         default: true
-      ubuntu-jammy:
-        description: Promote Ubuntu 22.04 Jammy
+      ubuntu-noble:
+        description: Promote Ubuntu 24.04 Noble
         type: boolean
         default: true
       image_tag:
@@ -21,11 +21,11 @@ jobs:
   ipa-image-promote:
     name: Promote IPA image
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - name: Validate inputs
         run: |
-          if [[ ${{ inputs.rocky9 }} == 'false' && ${{ inputs.ubuntu-jammy }} == 'false' ]]; then
+          if [[ ${{ inputs.rocky9 }} == 'false' && ${{ inputs.ubuntu-noble }} == 'false' ]]; then
             echo "At least one distribution must be selected"
             exit 1
           fi
@@ -85,16 +85,16 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.rocky9
 
-      - name: Promote Ubuntu Jammy 22.04 IPA image artifact
+      - name: Promote Ubuntu Noble 24.04 IPA image artifact
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe playbook run \
           src/kayobe-config/etc/kayobe/ansible/pulp-artifact-promote.yml \
           -e artifact_type="ipa-images" \
           -e os_distribution='ubuntu' \
-          -e os_release='jammy'
+          -e os_release='noble'
         env:
           ARTIFACT_TAG: ${{ inputs.image_tag }}
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.ubuntu-jammy
+        if: inputs.ubuntu-noble

.github/workflows/ipa-image-promote.yml

@@ -17,7 +17,6 @@ class OSRelease:
 @dataclass
 class OpenStackRelease:
     version: str
-    previous_version: str
     os_releases: t.List[OSRelease]
 
 
@@ -31,12 +30,15 @@ class Scenario:
 
 ROCKY_9 = OSRelease("rocky", "9", "cloud-user")
 UBUNTU_JAMMY = OSRelease("ubuntu", "jammy", "ubuntu")
+UBUNTU_NOBLE = OSRelease("ubuntu", "noble", "ubuntu")
 # NOTE(upgrade): Add supported releases here.
 OPENSTACK_RELEASES = [
-    OpenStackRelease("2024.1", "2023.1", [ROCKY_9, UBUNTU_JAMMY]),
-    OpenStackRelease("2023.1", "zed", [ROCKY_9, UBUNTU_JAMMY]),
+    OpenStackRelease("2023.1", [ROCKY_9, UBUNTU_JAMMY]),
+    OpenStackRelease("2024.1", [ROCKY_9, UBUNTU_JAMMY]),
+    OpenStackRelease("2025.1", [ROCKY_9, UBUNTU_NOBLE]),
 ]
 NEUTRON_PLUGINS = ["ovs", "ovn"]
+VERSION_HIERARCHY = ["zed", "2023.1", "2024.1", "2025.1"]
 
 
 def main() -> None:
@@ -50,13 +52,20 @@ def random_scenario() -> Scenario:
     openstack_release = random.choice(OPENSTACK_RELEASES)
     os_release = random.choice(openstack_release.os_releases)
     neutron_plugin = random.choice(NEUTRON_PLUGINS)
-    upgrade = 'major' if random.random() > 0.6 else 'none'
+    upgrade = "major" if random.random() > 0.6 else "none"
     return Scenario(openstack_release, os_release, neutron_plugin, upgrade)
 
 
 def generate_inputs(scenario: Scenario) -> t.Dict[str, str]:
     branch = get_branch(scenario.openstack_release.version)
-    previous_branch = get_branch(scenario.openstack_release.previous_version)
+    previous_branch = get_branch(
+        VERSION_HIERARCHY[
+            VERSION_HIERARCHY.index(scenario.openstack_release.version) - 1
+        ]
+    )
+    terraform_kayobe_multinode_version = get_tkm_version(
+        scenario.openstack_release.version
+    )
     inputs = {
         "os_distribution": scenario.os_release.distribution,
         "os_release": scenario.os_release.release,
@@ -65,6 +74,7 @@ def generate_inputs(scenario: Scenario) -> t.Dict[str, str]:
         "upgrade": scenario.upgrade,
         "stackhpc_kayobe_config_version": branch,
         "stackhpc_kayobe_config_previous_version": previous_branch,
+        "terraform_kayobe_multinode_version": terraform_kayobe_multinode_version,
     }
     return inputs
 
@@ -73,6 +83,13 @@ def get_branch(version: str) -> str:
     return f"stackhpc/{version}"
 
 
+def get_tkm_version(version: str) -> str:
+    if version in ["zed", "2023.1"]:
+        return "ea61ea1730e179e05e8f0e58b759267664c555e7"
+    else:
+        return "main"
+
+
 def write_output(name: str, value: str) -> None:
     print(f"{name}={value}")
 

.github/workflows/multinode-inputs.py

@@ -0,0 +1,31 @@
+---
+name: Periodic overcloud host images build
+on:
+  schedule:
+    # Runs every Monday at 3:30 AM ("30 3 * * 1").
+    - cron: "30 3 * * 1"
+
+jobs:
+  x86-overcloud-host-image-build:
+    name: Build x86_64 overcloud host images
+    uses: ./.github/workflows/overcloud-host-image-build.yml
+    with:
+      runner_env: "SMS Lab"
+      cpu-platform-architecture: "x86_64"
+      rocky9: true
+      ubuntu-noble: true
+      ubuntu-jammy: true
+    secrets: inherit
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+
+  aarch64-overcloud-host-image-build:
+    name: Build aarch64 overcloud host images
+    uses: ./.github/workflows/overcloud-host-image-build.yml
+    with:
+      runner_env: "SMS Lab"
+      cpu-platform-architecture: "aarch64"
+      rocky9: true
+      ubuntu-noble: true
+      ubuntu-jammy: true
+    secrets: inherit
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'