add cross-arch aarch64 container image builds

Author: bbezak

.github/workflows/stackhpc-container-image-build.yml

@@ -99,13 +99,14 @@ jobs:
         run: |
           output="{'distro': ["
           if [[ ${{ inputs.rocky-linux-9 }} == 'true' ]]; then
-              output+="{'name': 'rocky', 'release': 9},"
+              output+="{'name': 'rocky', 'release': 9, 'arch': 'amd64'},"
+              output+="{'name': 'rocky', 'release': 9, 'arch': 'aarch64'},"
           fi
           if [[ ${{ inputs.ubuntu-jammy }} == 'true' ]]; then
-              output+="{'name': 'ubuntu', 'release': 'jammy'},"
+              output+="{'name': 'ubuntu', 'release': 'jammy', 'arch': 'amd64'},"
           fi
           if [[ ${{ inputs.ubuntu-noble }} == 'true' ]]; then
-              output+="{'name': 'ubuntu', 'release': 'noble'},"
+              output+="{'name': 'ubuntu', 'release': 'noble', 'arch': 'amd64'},"
           fi
           # remove trailing comma
           output="${output%,}"
@@ -130,6 +131,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.generate-tag.outputs.matrix) }}
+      # Exclude ubuntu aarch64 builds for now
+      exclude:
+        - distro: ubuntu
+          arch: aarch64
     needs:
       - generate-tag
       - runner-selection
@@ -211,9 +216,16 @@ jobs:
         continue-on-error: true
         run: |
           args="${{ inputs.regexes }}"
+          if [[ "${{ matrix.arch }}" == 'aarch64' ]]; then
+            args="$args -e kolla_base_arch=${{ matrix.arch }}"
+          fi
           args="$args -e kolla_base_distro=${{ matrix.distro.name }}"
           args="$args -e kolla_base_distro_version=${{ matrix.distro.release }}"
-          args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}"
+          if [[ "${{ matrix.distro.name }}" == 'rocky' ]]; then
+            args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}-${{ matrix.arch }}"
+          else
+            args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}"
+          fi
           args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
@@ -226,6 +238,10 @@ jobs:
         run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-overcloud.log
         if: inputs.overcloud
 
+      - name: Copy build configs to output directory
+        run: sudo cp -rnL /opt/kayobe/etc/kolla/* image-build-logs/
+        if: inputs.overcloud
+
       - name: Build kolla seed images
         id: build_seed_images
         continue-on-error: true
@@ -239,14 +255,14 @@ jobs:
           kayobe seed container image build $args
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.seed
+        if: inputs.seed && matrix.arch == 'amd64'
 
       - name: Copy seed container image build logs to output directory
         run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-seed.log
-        if: inputs.seed
+        if: inputs.seed && matrix.arch == 'amd64'
 
       - name: Get built container images
-        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}" > ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images
+        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" > ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images
 
       - name: Fail if no images have been built
         run: if [ $(wc -l < ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images) -le 1 ]; then exit 1; fi
@@ -307,7 +323,7 @@ jobs:
       - name: Upload output artifact
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.distro.name }}-${{ matrix.distro.release }}-logs
+          name: ${{ matrix.distro.name }}-${{ matrix.distro.release }}-${{ matrix.arch }}-logs
           path: image-build-logs
           retention-days: 7
         if: ${{ !cancelled() }}

etc/kayobe/environments/ci-builder/stackhpc-ci.yml

@@ -50,7 +50,7 @@ stackhpc_repo_mirror_password: !vault |
 
 # Build against released Pulp repository versions.
 stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}"
-stackhpc_repo_rhel9_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version }}"
+stackhpc_repo_rhel9_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_version }}"
 stackhpc_repo_rhel9_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rhel9_rabbitmq_server_version }}"
 stackhpc_repo_ubuntu_jammy_version: "{{ stackhpc_pulp_repo_ubuntu_jammy_version }}"
 stackhpc_repo_ubuntu_jammy_security_version: "{{ stackhpc_pulp_repo_ubuntu_jammy_security_version }}"
@@ -61,15 +61,15 @@ stackhpc_repo_ubuntu_noble_version: "{{ stackhpc_pulp_repo_ubuntu_noble_version
 stackhpc_repo_ubuntu_noble_security_version: "{{ stackhpc_pulp_repo_ubuntu_noble_security_version }}"
 stackhpc_repo_docker_ce_ubuntu_noble_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_noble_version }}"
 stackhpc_repo_ceph_reef_debian_version: "{{ stackhpc_pulp_repo_ceph_reef_debian_version }}"
-stackhpc_repo_centos_stream_9_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_version }}"
-stackhpc_repo_centos_stream_9_openstack_caracal_version: "{{ stackhpc_pulp_repo_centos_stream_9_openstack_caracal_version }}"
-stackhpc_repo_centos_stream_9_opstools_version: "{{ stackhpc_pulp_repo_centos_stream_9_opstools_version }}"
-stackhpc_repo_centos_stream_9_storage_ceph_reef_version: "{{ stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_version }}"
-stackhpc_repo_centos_stream_9_docker_version: "{{ stackhpc_pulp_repo_centos_stream_9_docker_version }}"
-stackhpc_repo_rhel_9_treasuredata_5_version: "{{ stackhpc_pulp_repo_rhel_9_treasuredata_5_version }}"
-stackhpc_repo_rhel_9_mariadb_10_11_version: "{{ stackhpc_pulp_repo_rhel_9_mariadb_10_11_version }}"
-stackhpc_repo_rhel_9_influxdb_version: "{{ stackhpc_pulp_repo_rhel_9_influxdb_version }}"
-stackhpc_repo_epel_9_version: "{{ stackhpc_pulp_repo_epel_9_version }}"
+stackhpc_repo_centos_stream_9_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_nfv_openvswitch_version }}"
+stackhpc_repo_centos_stream_9_openstack_caracal_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_openstack_caracal_version }}"
+stackhpc_repo_centos_stream_9_opstools_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_opstools_version }}"
+stackhpc_repo_centos_stream_9_storage_ceph_reef_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_storage_ceph_reef_version }}"
+stackhpc_repo_centos_stream_9_docker_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_docker_version }}"
+stackhpc_repo_rhel_9_treasuredata_5_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_treasuredata_5_version }}"
+stackhpc_repo_rhel_9_mariadb_10_11_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_mariadb_10_11_version }}"
+stackhpc_repo_rhel_9_influxdb_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_influxdb_version }}"
+stackhpc_repo_epel_9_version: "{{ stackhpc_pulp_repo_multiarch_epel_9_version }}"
 stackhpc_repo_opensearch_2_x_version: "{{ stackhpc_pulp_repo_opensearch_2_x_version }}"
 stackhpc_repo_opensearch_dashboards_2_x_version: "{{ stackhpc_pulp_repo_opensearch_dashboards_2_x_version }}"
 ## Use derived vars from etc/kayobe/pulp.yml to switch between
@@ -79,7 +79,7 @@ stackhpc_repo_rocky_9_appstream_version: "{{ stackhpc_pulp_repo_rocky_9_appstrea
 stackhpc_repo_rocky_9_extras_version: "{{ stackhpc_pulp_repo_rocky_9_extras_version }}"
 stackhpc_repo_rocky_9_crb_version: "{{ stackhpc_pulp_repo_rocky_9_crb_version }}"
 stackhpc_repo_rocky_9_highavailability_version: "{{ stackhpc_pulp_repo_rocky_9_highavailability_version }}"
-stackhpc_repo_rocky_9_sig_security_common_version: "{{ stackhpc_pulp_repo_rocky_9_sig_security_common_version }}"
+stackhpc_repo_rocky_9_sig_security_common_version: "{{ stackhpc_pulp_repo_rocky_9_sig_security_common_aarch64_version if kolla_base_arch == 'aarch64' else stackhpc_pulp_repo_rocky_9_sig_security_common_version }}"
 stackhpc_repo_rhel9_doca_version: "{{ stackhpc_pulp_repo_rhel9_doca_version }}"
 
 # Rocky-and-CI-specific Pulp urls

etc/kayobe/kolla.yml

@@ -222,8 +222,8 @@ stackhpc_rocky_9_third_party_repos:
     file: "CentOS-Ceph-Reef.repo"
     tag: "centos-ceph-reef"
   - url: "{{ stackhpc_repo_rhel9_rabbitmq_erlang_url }}"
-    file: "rabbitmq_rabbitmq-erlang.repo"
-    tag: "rabbitmq_rabbitmq-erlang"
+    file: "{{ 'copr-rabbitmq-erlang.repo' if kolla_base_arch == 'aarch64' else 'rabbitmq_rabbitmq-erlang.repo' }}"
+    tag: "{{ 'openstack-kolla-copr-rabbitmq-kolla' if kolla_base_arch == 'aarch64' else 'rabbitmq_rabbitmq-erlang' }}"
   - url: "{{ stackhpc_repo_rhel9_rabbitmq_server_url }}"
     file: "rabbitmq_rabbitmq-server.repo"
     tag: "rabbitmq_rabbitmq-server"
@@ -287,13 +287,18 @@ kolla_build_blocks:
         mkdir -p /etc/yum.repos.d.backup && \
         tar -czf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d . && \
     {% endif %}
+        echo "===== Before sed modifications I =====" && \
+        cat /etc/yum.repos.d/*.repo && \
     {% for repo in stackhpc_yum_repos %}
         sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} && \
     {% endif %}
     {% endfor %}
+        && \
+        echo "===== After sed modifications I =====" && \
+        cat /etc/yum.repos.d/*.repo
     {% else %}
     RUN \
         rm /etc/apt/sources.list && \
@@ -321,13 +326,18 @@ kolla_build_blocks:
         tar -xzf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d && \
         tar -czf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d . && \
     {% endif %}
+        echo "===== Before sed modifications II =====" && \
+        cat /etc/yum.repos.d/*.repo && \
     {% for repo in base_centos_repo_overrides_post_yum_list %}
         sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[ \t]*\(https.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} &&{% endif %} \
     {% endfor %}
+        && \
+        echo "===== After sed modifications II =====" && \
+        cat /etc/yum.repos.d/*.repo \
     {% endif %}
   # With the UCA keyring installed we can now add all repos.
   base_ubuntu_package_sources_list: |

etc/kayobe/pulp-repo-versions.yml

@@ -2,25 +2,36 @@
 # This file is autogenerated by Ansible using the following workflow:
 # https://github.com/stackhpc/stackhpc-release-train/actions/workflows/package-update-kayobe.yml
 stackhpc_pulp_repo_centos_stream_9_docker_version: 20241210T000909
+stackhpc_pulp_repo_centos_stream_9_docker_aarch64_version: 20241210T012225
 stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_version: 20250528T022338
+stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_aarch64_version: 20250528T015409
 stackhpc_pulp_repo_centos_stream_9_openstack_caracal_version: 20241212T022636
+stackhpc_pulp_repo_centos_stream_9_openstack_caracal_aarch64_version: 20241212T020643
 stackhpc_pulp_repo_centos_stream_9_opstools_version: 20231213T031318
+stackhpc_pulp_repo_centos_stream_9_opstools_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_version: 20240923T233036
+stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_ceph_reef_debian_version: 20240925T152022
 stackhpc_pulp_repo_docker_ce_ubuntu_jammy_version: 20241218T154614
 stackhpc_pulp_repo_docker_ce_ubuntu_noble_version: 20250401T001425
 stackhpc_pulp_repo_elrepo_9_version: 20241129T235743
+stackhpc_pulp_repo_elrepo_9_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_epel_9_version: 20241216T235733
+stackhpc_pulp_repo_epel_9_aarch64_version: 20241217T012754
 stackhpc_pulp_repo_grafana_version: 20241216T002739
 stackhpc_pulp_repo_opensearch_2_x_version: 20241106T010702
 stackhpc_pulp_repo_opensearch_dashboards_2_x_version: 20241106T010702
 stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version: 20241217T002152
+stackhpc_pulp_repo_rhel9_rabbitmq_erlang_aarch64_version: 20241213T015928
 stackhpc_pulp_repo_rhel9_rabbitmq_server_version: 20241217T002152
 stackhpc_pulp_repo_rhel_9_influxdb_version: 20241217T002152
+stackhpc_pulp_repo_rhel_9_influxdb_aarch64_version: 20241217T012754
 stackhpc_pulp_repo_rhel_9_mariadb_10_11_version: 20241102T004913
+stackhpc_pulp_repo_rhel_9_mariadb_10_11_aarch64_version: 20241102T015940
 stackhpc_pulp_repo_rhel_9_rabbitmq_erlang_version: 20240711T091318
 stackhpc_pulp_repo_rhel_9_rabbitmq_server_version: 20240711T091318
 stackhpc_pulp_repo_rhel_9_treasuredata_5_version: 20241115T002028
+stackhpc_pulp_repo_rhel_9_treasuredata_5_aarch64_version: 20241115T010217
 stackhpc_pulp_repo_rocky_9_1_appstream_version: 20231207T013715
 stackhpc_pulp_repo_rocky_9_1_baseos_version: 20231206T014015
 stackhpc_pulp_repo_rocky_9_1_crb_version: 20231211T120328
@@ -39,17 +50,23 @@ stackhpc_pulp_repo_rocky_9_3_highavailability_version: 20240510T001129
 stackhpc_pulp_repo_rocky_9_4_appstream_version: 20240816T002610
 stackhpc_pulp_repo_rocky_9_4_baseos_version: 20240816T002610
 stackhpc_pulp_repo_rocky_9_4_crb_version: 20240816T002610
+stackhpc_pulp_repo_rocky_9_5_crb_aarch64_version: 20241217T012754
 stackhpc_pulp_repo_rocky_9_4_extras_version: 20240816T002610
 stackhpc_pulp_repo_rocky_9_4_highavailability_version: 20240816T002610
 stackhpc_pulp_repo_rocky_9_5_appstream_version: 20241217T005008
+stackhpc_pulp_repo_rocky_9_5_appstream_aarch64_version: 20241217T012754
 stackhpc_pulp_repo_rocky_9_5_baseos_version: 20241216T013503
+stackhpc_pulp_repo_rocky_9_5_baseos_aarch64_version: 20241216T011954
 stackhpc_pulp_repo_rocky_9_5_crb_version: 20241217T005008
 stackhpc_pulp_repo_rocky_9_5_extras_version: 20241216T004230
+stackhpc_pulp_repo_rocky_9_5_extras_aarch64_version: 20241216T011954
 stackhpc_pulp_repo_rocky_9_5_highavailability_version: 20241202T003154
+stackhpc_pulp_repo_rocky_9_5_highavailability_aarch64_version: 20241203T010516
 stackhpc_pulp_repo_rocky_9_sig_security_common_version: 20241127T003858
-stackhpc_pulp_repo_ubuntu_cloud_archive_version: 20250416T042645
-stackhpc_pulp_repo_ubuntu_jammy_security_version: 20250417T070229
-stackhpc_pulp_repo_ubuntu_jammy_version: 20250417T070229
+stackhpc_pulp_repo_rocky_9_sig_security_common_aarch64_version: 20241127T011228
+stackhpc_pulp_repo_ubuntu_cloud_archive_version: 20241217T045049
+stackhpc_pulp_repo_ubuntu_jammy_security_version: 20241217T071258
+stackhpc_pulp_repo_ubuntu_jammy_version: 20241217T071258
 stackhpc_pulp_repo_ubuntu_noble_security_version: 20250417T033501
 stackhpc_pulp_repo_ubuntu_noble_version: 20250417T033501
 stackhpc_pulp_repo_rhel_9_4_doca_version: 20241211T153620

etc/kayobe/pulp.yml

@@ -246,11 +246,11 @@ stackhpc_pulp_sync_rocky_9: "{{ os_distribution == 'rocky' }}"
 stackhpc_pulp_repo_rocky_9_minor_version: 5
 # Rocky 9 Snapshot versions. The defaults use the appropriate version from
 # pulp-repo-versions.yml for the selected minor release.
-stackhpc_pulp_repo_rocky_9_appstream_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_%s_appstream_version' % stackhpc_pulp_repo_rocky_9_minor_version) }}"
-stackhpc_pulp_repo_rocky_9_baseos_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_%s_baseos_version' % stackhpc_pulp_repo_rocky_9_minor_version) }}"
-stackhpc_pulp_repo_rocky_9_extras_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_%s_extras_version' % stackhpc_pulp_repo_rocky_9_minor_version) }}"
-stackhpc_pulp_repo_rocky_9_crb_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_%s_crb_version' % stackhpc_pulp_repo_rocky_9_minor_version) }}"
-stackhpc_pulp_repo_rocky_9_highavailability_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_%s_highavailability_version' % stackhpc_pulp_repo_rocky_9_minor_version) }}"
+stackhpc_pulp_repo_rocky_9_baseos_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_baseos' ~ ('_aarch64' if kolla_base_arch == 'aarch64' else '') ~ '_version') }}"
+stackhpc_pulp_repo_rocky_9_appstream_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_appstream' ~ ('_aarch64' if kolla_base_arch == 'aarch64' else '') ~ '_version') }}"
+stackhpc_pulp_repo_rocky_9_extras_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_extras' ~ ('_aarch64' if kolla_base_arch == 'aarch64' else '') ~ '_version') }}"
+stackhpc_pulp_repo_rocky_9_crb_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_crb' ~ ('_aarch64' if kolla_base_arch == 'aarch64' else '') ~ '_version') }}"
+stackhpc_pulp_repo_rocky_9_highavailability_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_highavailability' ~ ('_aarch64' if kolla_base_arch == 'aarch64' else '') ~ '_version') }}"
 
 # Whether to sync packages common to all RHEL 9 derivatives.
 stackhpc_pulp_sync_el_9: "{{ stackhpc_pulp_sync_rocky_9 | bool }}"