Automated Kolla dependencies updates

Author: assumptionsandg

.github/workflows/package-build-ofed.yml

@@ -1,254 +1,72 @@
----
-name: Build OFED packages
+name: Update dependencies
+
 on:
+  # Allow manual executions
   workflow_dispatch:
-    inputs:
-      rocky9:
-        description: Build Rocky Linux 9
-        type: boolean
-        default: true
-    secrets:
-      KAYOBE_VAULT_PASSWORD:
-        required: true
-      CLOUDS_YAML:
-        required: true
-      OS_APPLICATION_CREDENTIAL_ID:
-        required: true
-      OS_APPLICATION_CREDENTIAL_SECRET:
-        required: true
 
-env:
-  ANSIBLE_FORCE_COLOR: True
-  KAYOBE_ENVIRONMENT: ci-builder
-  KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 jobs:
-  overcloud-ofed-packages:
-    name: Build OFED packages
-    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
-    runs-on: arc-skc-host-image-builder-runner
-    permissions: {}
+  propose_github_release_updates:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - key: kolla
+            repo_path: ${{ github.workspace }}/src/kolla
+            path: ${{ github.workspace }}/src/kayobe-config/etc/kayobe/stackhpc.yml
+            repository: stackhpc/kolla
+            version_jsonpath: stackhpc_kolla_source_version
+
+          - key: kolla-ansible
+            repo_path: ${{ github.workspace }}/src/kolla-ansible
+            path: ${{ github.workspace }}/src/kayobe-config/etc/kayobe/stackhpc.yml
+            repository: stackhpc/kolla-ansible
+            version_jsonpath: stackhpc_kolla_ansible_source_version
+
+          - key: kayobe
+            repo_path: ${{ github.workspace }}/src/kayobe
+            path: ${{ github.workspace }}/src/kayobe-config/requirements.txt
+            repository: stackhpc/kayobe
+    permissions:
+      contents: write
+      pull-requests: write
+    name: ${{ matrix.key }}
     steps:
-      - name: Install Package
-        uses: ConorMacBride/install-package@main
-        with:
-          apt: git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq
-
-      - name: Start the SSH service
-        run: |
-          sudo /etc/init.d/ssh start
-
       - name: Checkout
         uses: actions/checkout@v4
-        with:
-          path: src/kayobe-config
+          path: ${{ github.workspace }}/src/kayobe-config
 
       - name: Determine OpenStack release
         id: openstack_release
         run: |
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
           echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
 
-      - name: Clone StackHPC Kayobe repository
+      - name: Checkout the dependency repo
         uses: actions/checkout@v4
         with:
-          repository: stackhpc/kayobe
-          ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
-          path: src/kayobe
-
-      - name: Install Kayobe
-        run: |
-          mkdir -p venvs &&
-          pushd venvs &&
-          python3 -m venv kayobe &&
-          source kayobe/bin/activate &&
-          pip install -U pip &&
-          pip install ../src/kayobe
-
-      - name: Install terraform
-        uses: hashicorp/setup-terraform@v2
-
-      - name: Initialise terraform
-        run: terraform init
-        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
-
-      - name: Generate SSH keypair
-        run: ssh-keygen -f id_rsa -N ''
-        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
-
-      - name: Generate clouds.yaml
-        run: |
-          cat << EOF > clouds.yaml
-          ${{ secrets.CLOUDS_YAML }}
-          EOF
-        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
-
-      - name: Output image tag
-        id: image_tag
-        run: |
-          echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
-
-      # Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag
-      - name: Output image name
-        id: image_name
-        run: |
-          echo image_name=overcloud-rocky-9-${{ steps.image_tag.outputs.image_tag }} >> $GITHUB_OUTPUT
-
-      - name: Generate terraform.tfvars
-        run: |
-          cat << EOF > terraform.tfvars
-          ssh_public_key = "id_rsa.pub"
-          ssh_username = "cloud-user"
-          aio_vm_name = "skc-ofed-builder"
-          aio_vm_image = "${{ env.VM_IMAGE }}"
-          aio_vm_flavor = "en1.medium"
-          aio_vm_network = "stackhpc-ci"
-          aio_vm_subnet = "stackhpc-ci"
-          aio_vm_interface = "ens3"
-          EOF
-        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
-        env:
-          VM_IMAGE: ${{ steps.image_name.outputs.image_name }}
-
-      - name: Terraform Plan
-        run: terraform plan
-        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
-        env:
-          OS_CLOUD: "openstack"
-          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
-          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
-
-      - name: Terraform Apply
-        run: |
-          for attempt in $(seq 5); do
-              if terraform apply -auto-approve; then
-                  echo "Created infrastructure on attempt $attempt"
-                  exit 0
-              fi
-              echo "Failed to create infrastructure on attempt $attempt"
-              sleep 10
-              terraform destroy -auto-approve
-              sleep 60
-          done
-          echo "Failed to create infrastructure after $attempt attempts"
-          exit 1
-        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
-        env:
-          OS_CLOUD: "openstack"
-          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
-          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
-
-      - name: Get Terraform outputs
-        id: tf_outputs
-        run: |
-          terraform output -json
-        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
-
-      - name: Write Terraform outputs
-        run: |
-          cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml
-          ${{ steps.tf_outputs.outputs.stdout }}
-          EOF
+          repository: ${{ matrix.repository }}
+          ref: stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
+          path: ${{ matrix.repo_path }}
 
-      - name: Write Terraform network config
+      - name: Get latest tag
+        id: latest_tag
         run: |
-          cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-network-allocation.yml
-          ---
-          aio_ips:
-            builder: "{{ access_ip_v4.value }}"
-          EOF
+          TAG=$(git describe --tags --abbrev=0 --match stackhpc/\*)
+          echo latest_tag=${TAG} >> $GITHUB_OUTPUT
+        working-directory: ${{ matrix.repo_path }}
 
-      - name: Write Terraform network interface config
-        run: |
-          mkdir -p src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed
-          rm -f src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces
-          cat << EOF > src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces
-          admin_interface: "{{ access_interface.value }}"
-          aio_interface: "{{ access_interface.value }}"
-          EOF
-
-      - name: Manage SSH keys
-        run: |
-          mkdir -p ~/.ssh
-          touch ~/.ssh/authorized_keys
-          cat src/kayobe-config/terraform/aio/id_rsa.pub >> ~/.ssh/authorized_keys
-          cp src/kayobe-config/terraform/aio/id_rsa* ~/.ssh/
-
-      - name: Bootstrap the control host
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe control host bootstrap
-
-      - name: Run growroot playbook
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/growroot.yml
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-
-      - name: Configure the seed host (Builder VM)
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host configure --skip-tags network,docker
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-
-      - name: Run a distro-sync
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host command run --become --command "dnf distro-sync --refresh"
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-
-      - name: Reset BLS entries on the seed host
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/reset-bls-entries.yml \
-          -e "reset_bls_host=ofed-builder"
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-
-      - name: Disable noexec in /var/tmp
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host command run --become --command "sed -i 's/noexec,//g' /etc/fstab"
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-
-      - name: Reboot to apply the kernel update
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/reboot.yml
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-
-      - name: Run OFED builder playbook
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/build-ofed-rocky.yml
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+      - name: Update dependency key (kolla)
+        uses: azimuth-cloud/github-actions/config-update@master
+        with:
+          path: ${{ matrix.path }}
+          updates: |
+            ${{ matrix.version_jsonpath }}=${{ steps.latest_tag.outputs.latest_tag }}
+        if: contains(matrix.key, 'kolla')
 
-      - name: Run OFED upload playbook
+      - name: Update dependency key (kayobe)
         run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-
-      - name: Destroy
-        run: terraform destroy -auto-approve
-        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
+          REPLACE=$(sed -i "s/@stackhpc\/.*$/$TAG/g" $REQUIREMENTS)
+        if: contains(matrix.key, 'kayobe')
         env:
-          OS_CLOUD: openstack
-          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
-          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
-        if: always()
+          TAG: ${{ steps.latest_tag.outputs.latest_tag }}
+          REQUIREMENTS: ${{ matrix.path }}

.github/workflows/update-dependencies.yml

@@ -0,0 +1,92 @@
+name: Update dependencies
+
+on:
+  # Allow manual executions
+  workflow_dispatch:
+  # Run nightly
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  propose_github_release_updates:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - key: kolla
+            repo_path: ${{ github.workspace }}/src/kolla
+            path: ${{ github.workspace }}/src/kayobe-config/etc/kayobe/stackhpc.yml
+            repository: stackhpc/kolla
+            version_jsonpath: stackhpc_kolla_source_version
+
+          - key: kolla-ansible
+            repo_path: ${{ github.workspace }}/src/kolla-ansible
+            path: ${{ github.workspace }}/src/kayobe-config/etc/kayobe/stackhpc.yml
+            repository: stackhpc/kolla-ansible
+            version_jsonpath: stackhpc_kolla_ansible_source_version
+
+          - key: kayobe
+            repo_path: ${{ github.workspace }}/src/kayobe
+            path: ${{ github.workspace }}/src/kayobe-config/requirements.txt
+            repository: stackhpc/kayobe
+    permissions:
+      contents: write
+      pull-requests: write
+    name: ${{ matrix.key }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+          path: ${{ github.workspace }}/src/kayobe-config
+
+      - name: Determine OpenStack release
+        id: openstack_release
+        run: |
+          BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
+          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
+
+      - name: Checkout the dependency repo
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ matrix.repository }}
+          ref: stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
+          path: ${{ matrix.repo_path }}
+
+      - name: Get latest tag
+        id: latest_tag
+        run: |
+          TAG=$(git describe --tags --abbrev=0 --match stackhpc/\*)
+          echo latest_tag=${TAG} >> $GITHUB_OUTPUT
+        working-directory: ${{ matrix.repo_path }}
+
+      - name: Update dependency key (kolla)
+        uses: azimuth-cloud/github-actions/config-update@master
+        with:
+          path: ${{ matrix.path }}
+          updates: |
+            ${{ matrix.version_jsonpath }}=${{ steps.latest_tag.outputs.latest_tag }}
+        if: contains(matrix.key, 'kolla')
+
+      - name: Update dependency key (kayobe)
+        run: |
+          REPLACE=$(sed -i "s/@stackhpc\/.*$/$TAG/g" $REQUIREMENTS)
+        if: contains(matrix.key, 'kayobe')
+        env:
+          TAG: ${{ steps.latest_tag.outputs.latest_tag }}
+          REQUIREMENTS: ${{ matrix.path }}
+
+      - name: Propose changes via PR if required
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }}
+          path: ${{ github.workspace }}/src/kayobe-config
+          commit-message: >-
+            Bump ${{ matrix.key }} to ${{ steps.next.outputs.version }}
+          branch: update-dependency/${{ matrix.key }}
+          delete-branch: true
+          title: >-
+            Bump ${{ matrix.key }} to ${{ steps.next.outputs.version }}
+          body: >
+            This PR was created automatically to update
+            ${{ matrix.key }} to ${{ steps.next.outputs.version }}.
+          labels: |
+            stackhpc-ci