Merge branch 'stackhpc/2024.1' into bump-ubuntu-jammy

Author: Alex-Welsh

.automation

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -euE
+set -o pipefail
+
+PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+KAYOBE_AUTOMATION_DIR="$(realpath "${PARENT}/../../.automation")"
+
+function main {
+	if [ "${PULP_DO_CONTAINER_SYNC:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-container-sync.yml' -e stackhpc_pulp_images_kolla_filter="${PULP_KOLLA_FILTER:-}"
+	fi
+	if [ "${PULP_DO_CONTAINER_PUBLISH:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-container-publish.yml' -e stackhpc_pulp_images_kolla_filter="${PULP_KOLLA_FILTER:-}"
+	fi
+	if [ "${PULP_DO_REPO_SYNC:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-repo-sync.yml'
+	fi
+	if [ "${PULP_DO_REPO_PUBLISH:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-repo-publish.yml'
+	fi
+	if [ "${PULP_DO_REPO_PROMOTE:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-repo-promote-production.yml'
+	fi
+}
+
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+	main
+fi

.automation.conf/run-books/pulp-sync-content.sh

@@ -32,6 +32,13 @@ jobs:
     runs-on: arc-skc-host-image-builder-runner
     permissions: {}
     steps:
+      - name: Validate inputs
+        run: |
+          if [[ ${{ inputs.rocky9 }} == 'false' && ${{ inputs.ubuntu-jammy }} == 'false' ]]; then
+            echo "At least one distribution must be selected"
+            exit 1
+          fi
+
       - name: Install Package
         uses: ConorMacBride/install-package@main
         with:

.github/workflows/overcloud-host-image-build.yml

@@ -23,6 +23,13 @@ jobs:
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     runs-on: ubuntu-22.04
     steps:
+      - name: Validate inputs
+        run: |
+          if [[ ${{ inputs.rocky9 }} == 'false' && ${{ inputs.ubuntu-jammy }} == 'false' ]]; then
+            echo "At least one distribution must be selected"
+            exit 1
+          fi
+
       - uses: actions/checkout@v4
         with:
           path: src/kayobe-config

.github/workflows/overcloud-host-image-promote.yml

@@ -34,6 +34,13 @@ jobs:
     runs-on: arc-skc-host-image-builder-runner
     permissions: {}
     steps:
+      - name: Validate inputs
+        run: |
+          if [[ ${{ inputs.rocky9 }} == 'false' && ${{ inputs.ubuntu-jammy }} == 'false' ]]; then
+            echo "At least one distribution must be selected"
+            exit 1
+          fi
+
       - name: Install package dependencies
         run: |
           sudo apt update

.github/workflows/overcloud-host-image-upload.yml

@@ -167,7 +167,7 @@ jobs:
           VM_NETWORK: ${{ inputs.vm_network }}
           VM_SUBNET: ${{ inputs.vm_subnet }}
           VM_INTERFACE: ${{ inputs.vm_interface }}
-          VM_VOLUME_SIZE: ${{ inputs.upgrade && '55' || '40' }}
+          VM_VOLUME_SIZE: ${{ inputs.upgrade && '65' || '50' }}
           VM_TAGS: '["skc-ci-aio", "PR=${{ github.event.number }}"]'
 
       - name: Terraform Plan
@@ -214,22 +214,12 @@ jobs:
       - name: Write Terraform network config
         run: |
           cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-networks.yml
-
-          admin_oc_net_name: admin
-          admin_cidr: "{{ access_cidr.value }}"
-          admin_allocation_pool_start: 0.0.0.0
-          admin_allocation_pool_end: 0.0.0.0
-          admin_gateway: "{{ access_gw.value }}"
-          admin_bootproto: dhcp
-          admin_ips:
+          admin_oc_net_name: ethernet
+          ethernet_cidr: "{{ access_cidr.value }}"
+          ethernet_allocation_pool_start: 0.0.0.0
+          ethernet_allocation_pool_end: 0.0.0.0
+          ethernet_ips:
             controller0: "{{ access_ip_v4.value }}"
-          admin_zone: admin
-          EOF
-
-      - name: Write Terraform network interface config
-        run: |
-          cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/controllers/tf-network-interfaces
-          admin_interface: "{{ access_interface.value }}"
           EOF
 
       - name: Write all-in-one scenario config

.github/workflows/stackhpc-all-in-one.yml

@@ -9,12 +9,12 @@ on:
         required: false
         default: ""
       overcloud:
-        description: Build overcloud images?
+        description: Build container images for overcloud services?
         type: boolean
         required: false
         default: true
       seed:
-        description: Build seed images?
+        description: Build container images for seed services?
         type: boolean
         required: false
         default: false
@@ -52,6 +52,17 @@ jobs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
       openstack_release: ${{ steps.openstack_release.outputs.openstack_release }}
     steps:
+      - name: Validate inputs
+        run: |
+          if [[ ${{ inputs.rocky-linux-9 }} == 'false' && ${{ inputs.ubuntu-jammy }} == 'false' ]]; then
+            echo "At least one distribution must be selected"
+            exit 1
+          fi
+          if [[ ${{ inputs.overcloud }} == 'false' && ${{ inputs.seed }} == 'false' ]]; then
+            echo "At least one of overcloud or seed must be selected"
+            exit 1
+          fi
+
       - name: Checkout
         uses: actions/checkout@v4
 

.github/workflows/stackhpc-container-image-build.yml

@@ -35,8 +35,15 @@ is not enabled by default. To enable it, set the following in
       kolla_enable_manila: true
       kolla_enable_manila_backend_cephfs_native: true
 
-And re-run ``kayobe overcloud service deploy`` if you are working on an existing
-deployment.
+If you are working on an existing deployment, you need to do the following first.
+
+1. Create CephFS pools: ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-pools.yml``
+2. Create cephx key for Manila: ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-keys.yml``
+3. Run Manila related Ceph commands: ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-commands-post.yml``
+4. Gather Ceph configuration and keyring for Manila: ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-gather-keys.yml``
+5. Configure Storage network on Seed node: ``kayobe seed host configure -t network,ip-allocation,snat``
+
+Then, run ``kayobe overcloud service deploy`` to deploy Manila.
 
 To test it, you will need two virtual machines. Cirros does not support the Ceph
 kernel client, so you will need to use a different image. Any regular Linux
@@ -108,35 +115,35 @@ Then create a share type and share:
 
 .. code-block:: bash
 
-      manila type-create cephfs-type false --is_public true
-      manila type-key cephfs-type set vendor_name=Ceph storage_protocol=CEPHFS
-      manila create --name test-share --share-type cephfs-type CephFS 2
+      openstack share type create cephfs-type false --public true
+      openstack share type set cephfs-type --extra-specs vendor_name=Ceph, storage_protocol=CEPHFS
+      openstack share create --name test-share --share-type cephfs-type --public true CephFS 2
 
 Wait until the share is available:
 
 .. code-block:: bash
 
-      manila list
+      openstack share list
 
 Then allow access to the shares to two users:
 
 .. code-block:: bash
 
-      manila access-allow test-share cephx alice
-      manila access-allow test-share cephx bob
+      openstack share access create test-share cephx alice
+      openstack share access create test-share cephx bob
 
 Show the access list to make sure the state of both entries is ``active`` and
 take note of the access keys:
 
 .. code-block:: bash
 
-      manila access-list test-share
+      openstack share access list test-share
 
 And take note of the path to the share:
 
 .. code-block:: bash
 
-      manila share-export-location-list test-share
+      openstack share export location list test-share
 
 SSH into the first instance, create a directory for the share, and mount it:
 

doc/source/contributor/environments/ci-multinode.rst

@@ -124,16 +124,18 @@ configuration.
 Known issues
 ============
 
-* OVN breaks on Rocky 9 deployments where hostnames are FQDNs.
-  Before upgrading, you must make sure no compute or controller nodes have any
-  ``.`` characters in their hostnames. Run the command below to check:
-
-  .. code-block:: bash
-
-     kayobe overcloud host command run --command "grep -v \'\.\' /etc/hostname" --show-output
-
-  There is currently no known fix for this issue aside from reprovisioning. A
-  patch will be developed soon.
+* Due to an incorrect default value NGS will attempt to use v3alpha for the api
+  path when communicating with etcd3. This isn't possible as in Caracal etcd is
+  running a newer version that has dropped support for v3alpha. You can work
+  around this in custom config, see the SMS PR for an example:
+  https://github.com/stackhpc/smslab-kayobe-config/pull/354
+
+* Due to a `security-related change in the GRUB package on Rocky Linux 9
+  <https://access.redhat.com/security/cve/CVE-2023-4001>`__, the operating
+  system can become unbootable (boot will stop at a ``grub>`` prompt). Remove
+  the ``--root-dev-only`` option from ``/boot/efi/EFI/rocky/grub.cfg`` after
+  applying package updates. This will happen automatically as a post hook when
+  running the ``kayobe overcloud host package update`` command.
 
 Security baseline
 =================
@@ -187,10 +189,15 @@ to 3.12, then to 3.13 on Antelope before the Caracal upgrade. This upgrade
 should not cause an API outage (though it should still be considered "at
 risk").
 
+Some errors have been observed in testing when the upgrades are perfomed
+back-to-back. A 200s delay eliminates this issue. On particularly large or slow
+deployments, consider increasing this timeout.
+
 .. code-block:: bash
 
    kayobe overcloud service configuration generate --node-config-dir /tmp/ignore -kt none
    kayobe kolla ansible run "rabbitmq-upgrade 3.12"
+   sleep 200
    kayobe kolla ansible run "rabbitmq-upgrade 3.13"
 
 RabbitMQ quorum queues
@@ -863,6 +870,15 @@ To update all eligible packages, use ``*``, escaping if necessary:
 
    kayobe overcloud host package update --packages "*" --limit <host>
 
+.. note::
+
+   Due to a `security-related change in the GRUB package on Rocky Linux 9
+   <https://access.redhat.com/security/cve/CVE-2023-4001>`__, the operating
+   system can become unbootable (boot will stop at a ``grub>`` prompt). Remove
+   the ``--root-dev-only`` option from ``/boot/efi/EFI/rocky/grub.cfg`` after
+   applying package updates. This will happen automatically as a post hook when
+   running the ``kayobe overcloud host package update`` command.
+
 If the kernel has been upgraded, reboot the host or batch of hosts to pick up
 the change:
 

doc/source/operations/upgrading-openstack.rst

@@ -16,6 +16,7 @@
         name:
           - git+https://github.com/stackhpc/ADVise
         state: latest
+        virtualenv_command: "python3 -m venv"
 
     - name: Create data directory
       file: