Merge branch 'stackhpc/2024.1' into cross-arch-builds-2024.1

bbezak · bbezak · commit ff0ae1fd7995 · 2025-07-15T18:03:15.000+02:00
Change-Id: I3c0fc7adebe274920592ab461c3e803c0f78a27c
diff --git a/.github/workflows/package-build-ofed.yml b/.github/workflows/package-build-ofed.yml
@@ -28,6 +28,11 @@ jobs:
     runs-on: arc-skc-host-image-builder-runner
     permissions: {}
     steps:
+      - name: Generate OFED tag
+        id: ofed_tag
+        run: |
+          echo "ofed_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT
+
       - name: Install Package
         uses: ConorMacBride/install-package@main
         with:
@@ -42,32 +47,14 @@ jobs:
         with:
           path: src/kayobe-config
 
-      - name: Determine OpenStack release
-        id: openstack_release
-        run: |
-          BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
-          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
-
-      - name: Generate OFED tag
-        id: ofed_tag
-        run: |
-          echo "ofed_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT
-
-      - name: Clone StackHPC Kayobe repository
-        uses: actions/checkout@v4
-        with:
-          repository: stackhpc/kayobe
-          ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
-          path: src/kayobe
-
       - name: Install Kayobe
         run: |
           mkdir -p venvs &&
           pushd venvs &&
           python3 -m venv kayobe &&
           source kayobe/bin/activate &&
           pip install -U pip &&
-          pip install ../src/kayobe
+          pip install -r ../src/kayobe-config/requirements.txt
 
       - name: Install terraform
         uses: hashicorp/setup-terraform@v2
diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
@@ -139,23 +139,14 @@ jobs:
       - generate-tag
       - runner-selection
     steps:
-      - name: Purge workspace (Arm runner only)
-        if: runner.arch == 'ARM64'
+      - name: Purge workspace
         run: sudo rm -rf "$GITHUB_WORKSPACE"/*
 
       - name: Install package dependencies
         run: |
           sudo apt update
           sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv curl jq wget
 
-      - name: Install gh
-        run: |
-          sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
-          sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
-          echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
-          sudo apt update
-          sudo apt install gh -y
-
       - name: Checkout
         uses: actions/checkout@v4
         with:
@@ -232,7 +223,7 @@ jobs:
             args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}"
           fi
           args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
-          args="$args -e kolla_build_log_path=/var/log/kolla-build-${{ steps.write-kolla-tag.outputs.kolla-tag }}.log"
+          args="$args -e kolla_build_log_path=$GITHUB_WORKSPACE/image-build-logs/kolla-build-overcloud.log"
           args="$args -e base_path=$GITHUB_WORKSPACE/opt/kayobe"
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
@@ -241,12 +232,6 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.overcloud
 
-      - name: Copy overcloud container image build logs to output directory
-        run: |
-          sudo mv /var/log/kolla-build-${{ steps.write-kolla-tag.outputs.kolla-tag }}.log \
-                  image-build-logs/kolla-build-overcloud.log
-        if: inputs.overcloud
-
       - name: Copy build configs to output directory
         run: sudo cp -rnL "$GITHUB_WORKSPACE/opt/kayobe/etc/kolla/"* image-build-logs/
         if: inputs.overcloud
@@ -377,92 +362,31 @@ jobs:
         run: |
           sudo apt update
           sudo apt install -y git unzip python3-wheel python3-pip python3-venv curl jq wget openssh-server openssh-client
+
       - name: Install gh
         run: |
           sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
           sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
           echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
           sudo apt update
           sudo apt install gh -y
-      - name: Checkout Kayobe Config
-        uses: actions/checkout@v4
-        with:
-          path: src/kayobe-config
-
-      - name: Install Kayobe
-        run: |
-          mkdir -p venvs &&
-          pushd venvs &&
-          python3 -m venv kayobe &&
-          source kayobe/bin/activate &&
-          pip install -U pip &&
-          pip install -r ../src/kayobe-config/requirements.txt
-      # Required for Pulp auth proxy deployment and Docker registry login.
-      # Normally installed during host configure.
-      - name: Install Docker Python SDK
-        run: |
-          sudo pip install docker
-      - name: Configure localhost as a seed
-        run: |
-          cat > src/kayobe-config/etc/kayobe/environments/ci-builder/inventory/hosts << EOF
-          # A 'seed' host used for building images.
-          # Use localhost for container image builds.
-          [seed]
-          localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3
-          EOF
-      # See etc/kayobe/ansible/roles/pulp_auth_proxy/README.md for details.
-      # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the
-      # runner and dind containers.
-      - name: Deploy an authenticating package repository mirror proxy
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
       - name: Download artifacts
         uses: actions/download-artifact@v4
 
       - name: Combine pushed images lists
         run: |
           find . -name 'push-attempt-images.txt' -exec cat {} + > all-pushed-images.txt
-      - name: Log in to Docker registry
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
-      - name: Create and Push Docker Manifests
-        run: |
-          set -ex
-          mkdir -p logs
-          images=$(cat all-pushed-images.txt | sort | uniq)
-          # Filter out Ubuntu and Rocky Bifrost images
-          manifest_images=$(echo "$images" | grep -E '.*-(amd64|aarch64)$' | sed -E 's/-(amd64|aarch64)$//' | sort | uniq)
-          if [ -z "$manifest_images" ]; then
-            echo "No Rocky overcloud images found. Skipping manifest creation." | tee -a logs/manifest-creation.log
-            exit 0
-          fi
-          for base_image in $manifest_images; do
-            arch_images=""
-            for arch in amd64 aarch64; do
-              arch_image="${base_image}-${arch}"
-              # Check if the image exists in the registry
-              if docker manifest inspect "$arch_image" > /dev/null 2>&1; then
-                arch_images="$arch_images $arch_image"
-              fi
-            done
-            if [ -n "$arch_images" ]; then
-              echo "Creating manifest for $base_image with images:$arch_images" | tee -a logs/manifest-creation.log
-              docker manifest create "$base_image" $arch_images | tee -a logs/manifest-creation.log
-              docker manifest push "$base_image" | tee -a logs/manifest-creation.log
-            else
-              echo "No images found for $base_image, skipping." | tee -a logs/manifest-creation.log
-            fi
-          done
+      - name: Log in to container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ark.stackhpc.com
+          username: ${{ secrets.RLS_TRAIN_CI_ARK_REGISTRY_USER }}
+          password: ${{ secrets.RLS_TRAIN_CI_ARK_REGISTRY_PASS }}
+
+      - name: Create and push Docker manifests
+        run: tools/multiarch-manifests.sh
 
       - name: Upload manifest logs
         uses: actions/upload-artifact@v4
diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml
@@ -4,6 +4,6 @@
 # where the key is the OS distro and the value is the tag to deploy.
 kolla_image_tags:
   openstack:
-    rocky-9: 2024.1-rocky-9-20250711T161715
-    ubuntu-jammy: 2024.1-ubuntu-jammy-20250711T161715
-    ubuntu-noble: 2024.1-ubuntu-noble-20250711T161715
+    rocky-9: 2024.1-rocky-9-20250715T083226
+    ubuntu-jammy: 2024.1-ubuntu-jammy-20250715T083226
+    ubuntu-noble: 2024.1-ubuntu-noble-20250715T083226
diff --git a/etc/kayobe/ofed.yml b/etc/kayobe/ofed.yml
@@ -3,18 +3,22 @@
 
 ###############################################################################
 # DOCA host version
-stackhpc_pulp_doca_version: 2.9.1
+stackhpc_pulp_doca_version: "{{ '2.9.3' if stackhpc_pulp_repo_rocky_9_minor_version == '6' else '2.9.1' }}"
 
 ###############################################################################
 # Pulp configuration for DOCA OFED
 
 # Whether to sync OFED repositories into the local Pulp service
 stackhpc_pulp_sync_ofed: "{{ groups['mlnx'] | length > 0 }}"
 
+# DOCA Snapshot lookup vars
+doca_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_doca_version | replace('.', '_') }}_rhel9_{{ stackhpc_pulp_repo_rocky_9_minor_version }}_version"
+doca_modules_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_doca_version | replace('.', '_') }}_rhel9_{{ stackhpc_pulp_repo_rocky_9_minor_version }}_modules_version"
+
 # DOCA Snapshot versions. The defaults use the appropriate version from
 # pulp-repo-versions.yml
-stackhpc_pulp_repo_rhel9_doca_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rhel_9_{{ stackhpc_pulp_repo_rocky_9_minor_version }}_doca_version') }}"
-stackhpc_pulp_repo_rhel9_doca_modules_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rhel_9_{{ stackhpc_pulp_repo_rocky_9_minor_version }}_doca_modules_version') }}"
+stackhpc_pulp_repo_rhel9_doca_version: "{{ lookup('vars', doca_version_lookup_var) }}"
+stackhpc_pulp_repo_rhel9_doca_modules_version: "{{ lookup('vars', doca_modules_version_lookup_var) }}"
 
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml
@@ -79,11 +79,9 @@ stackhpc_pulp_repo_ubuntu_jammy_security_version: 20250417T070229
 stackhpc_pulp_repo_ubuntu_jammy_version: 20250417T070229
 stackhpc_pulp_repo_ubuntu_noble_security_version: 20250417T033501
 stackhpc_pulp_repo_ubuntu_noble_version: 20250417T033501
-stackhpc_pulp_repo_rhel_9_4_doca_version: 20241211T153620
-stackhpc_pulp_repo_rhel_9_4_doca_modules_version: 20241213T112245
-stackhpc_pulp_repo_rhel_9_5_doca_version: 20241211T171301
-stackhpc_pulp_repo_rhel_9_5_doca_modules_version: 20250115T150314
-##### NOTE: Dummy variables, currently no RL9.6 DOCA
-stackhpc_pulp_repo_rhel_9_6_doca_modules_version: 00000000T000000
-stackhpc_pulp_repo_rhel_9_6_doca_version: 00000000T000000
-######
+stackhpc_pulp_repo_doca_2_9_1_rhel9_4_version: 20241211T153620
+stackhpc_pulp_repo_doca_2_9_1_rhel9_4_modules_version: 20241213T112245
+stackhpc_pulp_repo_doca_2_9_1_rhel9_5_version: 20241211T171301
+stackhpc_pulp_repo_doca_2_9_1_rhel9_5_modules_version: 20250115T150314
+stackhpc_pulp_repo_doca_2_9_3_rhel9_6_version: 20250703T135021
+stackhpc_pulp_repo_doca_2_9_3_rhel9_6_modules_version: 20250714T141841
diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml
@@ -243,7 +243,7 @@ stackhpc_pulp_distribution_deb_production: >-
 # Whether to sync Rocky Linux 9 packages.
 stackhpc_pulp_sync_rocky_9: "{{ os_distribution == 'rocky' }}"
 # Rocky 9 minor version number. Supported values: 1, 2, 3, 4, 5, 6. Default is 6
-stackhpc_pulp_repo_rocky_9_minor_version: 6
+stackhpc_pulp_repo_rocky_9_minor_version: '6'
 # Rocky 9 Snapshot versions. The defaults use the appropriate version from
 # pulp-repo-versions.yml for the selected minor release.
 
diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml
@@ -165,11 +165,11 @@ stackhpc_repo_elrepo_9_version: "{{ stackhpc_repo_distribution }}"
 
 # Kolla source repository.
 stackhpc_kolla_source_url: "https://github.com/stackhpc/kolla"
-stackhpc_kolla_source_version: stackhpc/18.6.0.13
+stackhpc_kolla_source_version: stackhpc/18.7.0.2
 
 # Kolla Ansible source repository.
 stackhpc_kolla_ansible_source_url: "https://github.com/stackhpc/kolla-ansible"
-stackhpc_kolla_ansible_source_version: stackhpc/18.6.0.15
+stackhpc_kolla_ansible_source_version: stackhpc/18.7.0.1
 
 ###############################################################################
 # Container image registry
diff --git a/releasenotes/notes/doca-2-9-3-238838fb78e0c7d9.yaml b/releasenotes/notes/doca-2-9-3-238838fb78e0c7d9.yaml
@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Added support for DOCA OFED on Rocky Linux 9.6 at version ``2.9.3``. The
+    package versions for Rocky 9.4 and 9.5 remain unchanged, using ``2.9.1``.
diff --git a/tools/multiarch-manifests.sh b/tools/multiarch-manifests.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Script used by the Image build workflow to build & push multiarch manifests.
+
+set -ex
+
+mkdir -p logs
+images=$(cat all-pushed-images.txt | sort | uniq)
+
+# Filter out Ubuntu and Rocky Bifrost images
+manifest_images=$(echo "$images" \
+  | grep -E '.*-(amd64|aarch64)$' \
+  | sed -E 's/-(amd64|aarch64)$//' \
+  | sort | uniq)
+
+if [ -z "$manifest_images" ]; then
+  echo "No Rocky overcloud images found. Skipping manifest creation." | tee -a logs/manifest-creation.log
+  exit 0
+fi
+
+for base_image in $manifest_images; do
+  arch_images=""
+  for arch in amd64 aarch64; do
+    arch_image="${base_image}-${arch}"
+    # Check if the image exists in the registry
+    if docker manifest inspect "$arch_image" > /dev/null 2>&1; then
+      arch_images="$arch_images $arch_image"
+    fi
+  done
+  if [ -n "$arch_images" ]; then
+    echo "Creating manifest for $base_image with images:$arch_images" | tee -a logs/manifest-creation.log
+    docker manifest create "$base_image" $arch_images | tee -a logs/manifest-creation.log
+    docker manifest push "$base_image" | tee -a logs/manifest-creation.log
+  else
+    echo "No images found for $base_image, skipping." | tee -a logs/manifest-creation.log
+  fi
+done
