Merge branch 'stackhpc/2023.1' into bump-conntrack-table-size

Author: Alex-Welsh

.automation

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -euE
+set -o pipefail
+
+PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+KAYOBE_AUTOMATION_DIR="$(realpath "${PARENT}/../../.automation")"
+
+function main {
+	if [ "${PULP_DO_CONTAINER_SYNC:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-container-sync.yml' -e stackhpc_pulp_images_kolla_filter="${PULP_KOLLA_FILTER:-}"
+	fi
+	if [ "${PULP_DO_CONTAINER_PUBLISH:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-container-publish.yml' -e stackhpc_pulp_images_kolla_filter="${PULP_KOLLA_FILTER:-}"
+	fi
+	if [ "${PULP_DO_REPO_SYNC:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-repo-sync.yml'
+	fi
+	if [ "${PULP_DO_REPO_PUBLISH:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-repo-publish.yml'
+	fi
+	if [ "${PULP_DO_REPO_PROMOTE:-}" = true ]; then
+		${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-repo-promote-production.yml'
+	fi
+}
+
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+	main
+fi

.automation.conf/run-books/pulp-sync-content.sh

@@ -26,7 +26,7 @@ class Scenario:
     openstack_release: OpenStackRelease
     os_release: OSRelease
     neutron_plugin: str
-    upgrade: bool
+    upgrade: str
 
 
 ROCKY_9 = OSRelease("rocky", "9", "cloud-user")
@@ -50,7 +50,7 @@ def random_scenario() -> Scenario:
     openstack_release = random.choice(OPENSTACK_RELEASES)
     os_release = random.choice(openstack_release.os_releases)
     neutron_plugin = random.choice(NEUTRON_PLUGINS)
-    upgrade = random.random() > 0.6
+    upgrade = 'major' if random.random() > 0.6 else 'none'
     return Scenario(openstack_release, os_release, neutron_plugin, upgrade)
 
 
@@ -62,7 +62,7 @@ def generate_inputs(scenario: Scenario) -> t.Dict[str, str]:
         "os_release": scenario.os_release.release,
         "ssh_username": scenario.os_release.ssh_username,
         "neutron_plugin": scenario.neutron_plugin,
-        "upgrade": str(scenario.upgrade).lower(),
+        "upgrade": scenario.upgrade,
         "stackhpc_kayobe_config_version": branch,
         "stackhpc_kayobe_config_previous_version": previous_branch,
     }

.github/workflows/multinode-inputs.py

@@ -87,7 +87,7 @@ jobs:
       - name: Install OpenStack client
         run: |
           source venvs/kayobe/bin/activate &&
-          pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }}
+          pip install python-openstackclient -c https://raw.githubusercontent.com/stackhpc/requirements/refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}/upper-constraints.txt
 
       - name: Output Rocky Linux 9 image tag
         id: rocky_9_image_tag

.github/workflows/overcloud-host-image-upload.yml

@@ -167,7 +167,7 @@ jobs:
           VM_NETWORK: ${{ inputs.vm_network }}
           VM_SUBNET: ${{ inputs.vm_subnet }}
           VM_INTERFACE: ${{ inputs.vm_interface }}
-          VM_VOLUME_SIZE: ${{ inputs.upgrade && '55' || '40' }}
+          VM_VOLUME_SIZE: ${{ inputs.upgrade && '65' || '50' }}
           VM_TAGS: '["skc-ci-aio", "PR=${{ github.event.number }}"]'
 
       - name: Terraform Plan
@@ -214,22 +214,12 @@ jobs:
       - name: Write Terraform network config
         run: |
           cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-networks.yml
-
-          admin_oc_net_name: admin
-          admin_cidr: "{{ access_cidr.value }}"
-          admin_allocation_pool_start: 0.0.0.0
-          admin_allocation_pool_end: 0.0.0.0
-          admin_gateway: "{{ access_gw.value }}"
-          admin_bootproto: dhcp
-          admin_ips:
+          admin_oc_net_name: ethernet
+          ethernet_cidr: "{{ access_cidr.value }}"
+          ethernet_allocation_pool_start: 0.0.0.0
+          ethernet_allocation_pool_end: 0.0.0.0
+          ethernet_ips:
             controller0: "{{ access_ip_v4.value }}"
-          admin_zone: admin
-          EOF
-
-      - name: Write Terraform network interface config
-        run: |
-          cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/controllers/tf-network-interfaces
-          admin_interface: "{{ access_interface.value }}"
           EOF
 
       - name: Write all-in-one scenario config

.github/workflows/stackhpc-all-in-one.yml

@@ -34,7 +34,7 @@ jobs:
 
       - name: Install OpenStack client
         run: |
-          pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }}
+          pip install python-openstackclient -c https://raw.githubusercontent.com/stackhpc/requirements/refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}/upper-constraints.txt
 
       - name: Clean up aio instances over 3 hours old
         run: |

.github/workflows/stackhpc-ci-cleanup.yml

@@ -35,14 +35,14 @@ jobs:
     name: Multinode periodic
     needs:
       - generate-inputs
-    uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.2.0
+    uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.4.0
     with:
-      multinode_name: mn-prdc-${{ github.run_id }}
+      multinode_name: mn-periodic-${{ github.run_id }}
       os_distribution: ${{ needs.generate-inputs.outputs.os_distribution }}
       os_release: ${{ needs.generate-inputs.outputs.os_release }}
       ssh_username: ${{ needs.generate-inputs.outputs.ssh_username }}
       neutron_plugin: ${{ needs.generate-inputs.outputs.neutron_plugin }}
-      upgrade: ${{ needs.generate-inputs.outputs.upgrade == 'true' }}
+      upgrade: ${{ needs.generate-inputs.outputs.upgrade }}
       stackhpc_kayobe_config_version: ${{ needs.generate-inputs.outputs.stackhpc_kayobe_config_version }}
       stackhpc_kayobe_config_previous_version: ${{ needs.generate-inputs.outputs.stackhpc_kayobe_config_previous_version }}
       enable_slack_alert: true

.github/workflows/stackhpc-multinode-periodic.yml

@@ -27,8 +27,12 @@ name: Multinode
           - ovs
       upgrade:
         description: Whether to perform an upgrade
-        type: boolean
-        default: false
+        default: none
+        type: choice
+        options:
+          - none
+          - minor
+          - major
       break_on:
         description: When to break execution for manual interaction
         type: choice
@@ -52,7 +56,7 @@ name: Multinode
 jobs:
   multinode:
     name: Multinode
-    uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.2.0
+    uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.4.0
     with:
       multinode_name: ${{ inputs.multinode_name }}
       os_distribution: ${{ inputs.os_distribution }}
@@ -66,6 +70,6 @@ jobs:
       ssh_key: ${{ inputs.ssh_key }}
       stackhpc_kayobe_config_version: ${{ github.ref_name }}
       # NOTE(upgrade): Reference the PREVIOUS release here.
-      stackhpc_kayobe_config_previous_version: stackhpc/zed
+      stackhpc_kayobe_config_previous_version: ${{ inputs.upgrade == 'major' && 'stackhpc/zed' || 'stackhpc/2023.1' }}
       terraform_kayobe_multinode_version: ${{ inputs.terraform_kayobe_multinode_version }}
     secrets: inherit

.github/workflows/stackhpc-multinode.yml

@@ -1,6 +1,8 @@
-====
-Ceph
-====
+.. _cephadm-kayobe:
+
+================
+Cephadm & Kayobe
+================
 
 This section describes how to use the Cephadm integration included in StackHPC
 Kayobe configuration to deploy Ceph.

doc/source/configuration/cephadm.rst

@@ -57,26 +57,26 @@ Runner Deployment
     Ideally an Infra VM could be used here or failing that the control host.
     Wherever it is deployed the host will need access to the :code:`admin_network`, :code:`public_network` and the :code:`pulp registry` on the seed.
 
-2. Edit the environment's :code:`${KAYOBE_CONFIG_PATH}/environments/${KAYOBE_ENVIRONMENT}/inventory/groups` to add the predefined :code:`github-runners` group to :code:`infra-vms`
+2. Edit the environment's :code:`$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/inventory/groups` to add the predefined :code:`github-runners` group to :code:`infra-vms`
 
 .. code-block:: ini
 
     [infra-vms:children]
     github-runners
 
-3. Edit the environment's :code:`${KAYOBE_CONFIG_PATH}/environments/${KAYOBE_ENVIRONMENT}/inventory/hosts` to define the host(s) that will host the runners.
+3. Edit the environment's :code:`$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/inventory/hosts` to define the host(s) that will host the runners.
 
 .. code-block:: ini
 
     [github-runners]
     prod-runner-01
 
-4. Provide all the relevant Kayobe :code:`group_vars` for :code:`github-runners` under :code:`${KAYOBE_CONFIG_PATH}/environments/${KAYOBE_ENVIRONMENT}/inventory/group_vars/github-runners`
+4. Provide all the relevant Kayobe :code:`group_vars` for :code:`github-runners` under :code:`$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/github-runners`
     * `infra-vms` ensuring all required `infra_vm_extra_network_interfaces` are defined
     * `network-interfaces`
     * `python-interpreter.yml` ensuring that `ansible_python_interpreter: /usr/bin/python3` has been set
 
-5. Edit the ``${KAYOBE_CONFIG_PATH}/inventory/group_vars/github-runners/runners.yml`` file which will contain the variables required to deploy a series of runners.
+5. Edit the ``$KAYOBE_CONFIG_PATH/inventory/group_vars/github-runners/runners.yml`` file which will contain the variables required to deploy a series of runners.
    Below is a core set of variables that will require consideration and modification for successful deployment of the runners.
    The number of runners deployed can be configured by removing and extending the dict :code:`github-runners`.
    As for how many runners present three is suitable number as this would prevent situations where long running jobs could halt progress other tasks whilst waiting for a free runner.
@@ -120,7 +120,7 @@ Runner Deployment
 
 7. If the host is an actual Infra VM then please refer to upstream `Infrastructure VMs <https://docs.openstack.org/kayobe/latest/configuration/reference/infra-vms.html>`__ documentation for additional configuration and steps.
 
-8. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/deploy-github-runner.yml`
+8. Run :code:`kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/deploy-github-runner.yml`
 
 9. Check runners have registered properly by visiting the repository's :code:`Action` tab -> :code:`Runners` -> :code:`Self-hosted runners`
 
@@ -130,9 +130,9 @@ Runner Deployment
 Workflow Deployment
 -------------------
 
-1. Edit :code:`${KAYOBE_CONFIG_PATH}/inventory/group_vars/github-writer/writer.yml` in the base configuration making the appropriate changes to your deployments specific needs. See documentation for `stackhpc.kayobe_workflows.github <https://github.com/stackhpc/ansible-collection-kayobe-workflows/tree/main/roles/github>`__.
+1. Edit :code:`$KAYOBE_CONFIG_PATH/inventory/group_vars/github-writer/writer.yml` in the base configuration making the appropriate changes to your deployments specific needs. See documentation for `stackhpc.kayobe_workflows.github <https://github.com/stackhpc/ansible-collection-kayobe-workflows/tree/main/roles/github>`__.
 
-2. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/write-github-workflows.yml`
+2. Run :code:`kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/write-github-workflows.yml`
 
 3. Add all required secrets and variables to repository either via the GitHub UI or GitHub CLI (may require repository owner)