2024.1: 2023.1 merge

.github/workflows/multinode-inputs.py

@@ -0,0 +1,80 @@
+# Generate inputs for the reusable multinode.yml workflow.
+# The test scenario is randomly selected.
+# The inputs are printed to stdout in GitHub step output key=value format.
+
+from dataclasses import dataclass
+import random
+import typing as t
+
+
+@dataclass
+class OSRelease:
+    distribution: str
+    release: str
+    ssh_username: str
+
+
+@dataclass
+class OpenStackRelease:
+    version: str
+    previous_version: str
+    os_releases: t.List[OSRelease]
+
+
+@dataclass
+class Scenario:
+    openstack_release: OpenStackRelease
+    os_release: OSRelease
+    neutron_plugin: str
+    upgrade: bool
+
+
+ROCKY_9 = OSRelease("rocky", "9", "cloud-user")
+UBUNTU_JAMMY = OSRelease("ubuntu", "jammy", "ubuntu")
+# NOTE(upgrade): Add supported releases here.
+OPENSTACK_RELEASES = [
+    OpenStackRelease("2023.1", "zed", [ROCKY_9, UBUNTU_JAMMY])
+]
+NEUTRON_PLUGINS = ["ovs", "ovn"]
+
+
+def main() -> None:
+    scenario = random_scenario()
+    inputs = generate_inputs(scenario)
+    for name, value in inputs.items():
+        write_output(name, value)
+
+
+def random_scenario() -> Scenario:
+    openstack_release = random.choice(OPENSTACK_RELEASES)
+    os_release = random.choice(openstack_release.os_releases)
+    neutron_plugin = random.choice(NEUTRON_PLUGINS)
+    upgrade = random.random() > 0.6
+    return Scenario(openstack_release, os_release, neutron_plugin, upgrade)
+
+
+def generate_inputs(scenario: Scenario) -> t.Dict[str, str]:
+    branch = get_branch(scenario.openstack_release.version)
+    previous_branch = get_branch(scenario.openstack_release.previous_version)
+    inputs = {
+        "os_distribution": scenario.os_release.distribution,
+        "os_release": scenario.os_release.release,
+        "ssh_username": scenario.os_release.ssh_username,
+        "neutron_plugin": scenario.neutron_plugin,
+        "upgrade": str(scenario.upgrade).lower(),
+        "stackhpc_kayobe_config_version": branch,
+        "stackhpc_kayobe_config_previous_version": previous_branch,
+    }
+    return inputs
+
+
+def get_branch(version: str) -> str:
+    return f"stackhpc/{version}"
+
+
+def write_output(name: str, value: str) -> None:
+    print(f"{name}={value}")
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/overcloud-host-image-build.yml

@@ -191,7 +191,7 @@ jobs:
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe seed host command run \
-          --command "sudo dnf config-manager --set-enabled crb && sudo dnf -y install epel-release && sudo dnf -y install zstd debootstrap kpartx cloud-init" --show-output
+          --command "sudo dnf config-manager --set-enabled crb && sudo dnf -y install epel-release && sudo dnf -y install cloud-init debootstrap git kpartx zstd" --show-output
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 

.github/workflows/stackhpc-multinode-periodic.yml

@@ -0,0 +1,50 @@
+---
+# This workflow provides a periodic deploy of a multi-node test cluster.
+# The test scenario is randomly selected.
+
+name: Multinode periodic
+'on':
+  schedule:
+    # Runs nightly at 2:42 AM.
+    - cron: "42 2 * * *"
+jobs:
+  generate-inputs:
+    name: Generate inputs
+    runs-on: ubuntu-latest
+    outputs:
+      os_distribution: ${{ steps.generate-inputs.outputs.os_distribution }}
+      os_release: ${{ steps.generate-inputs.outputs.os_release }}
+      ssh_username: ${{ steps.generate-inputs.outputs.ssh_username }}
+      neutron_plugin: ${{ steps.generate-inputs.outputs.neutron_plugin }}
+      upgrade: ${{ steps.generate-inputs.outputs.upgrade }}
+      stackhpc_kayobe_config_version: ${{ steps.generate-inputs.outputs.stackhpc_kayobe_config_version }}
+      stackhpc_kayobe_config_previous_version: ${{ steps.generate-inputs.outputs.stackhpc_kayobe_config_previous_version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Generate inputs for multinode workflow
+        id: generate-inputs
+        run: |
+          python3 .github/workflows/multinode-inputs.py >> $GITHUB_OUTPUT
+
+      - name: Display generated inputs
+        run: |
+          echo '${{ toJSON(steps.generate-inputs.outputs) }}'
+  multinode:
+    name: Multinode periodic
+    needs:
+      - generate-inputs
+    uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/[email protected]
+    with:
+      multinode_name: mn-prdc-${{ github.run_id }}
+      os_distribution: ${{ needs.generate-inputs.outputs.os_distribution }}
+      os_release: ${{ needs.generate-inputs.outputs.os_release }}
+      ssh_username: ${{ needs.generate-inputs.outputs.ssh_username }}
+      neutron_plugin: ${{ needs.generate-inputs.outputs.neutron_plugin }}
+      upgrade: ${{ needs.generate-inputs.outputs.upgrade == 'true' }}
+      stackhpc_kayobe_config_version: ${{ needs.generate-inputs.outputs.stackhpc_kayobe_config_version }}
+      stackhpc_kayobe_config_previous_version: ${{ needs.generate-inputs.outputs.stackhpc_kayobe_config_previous_version }}
+      enable_slack_alert: true
+    secrets: inherit
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'

.github/workflows/stackhpc-multinode.yml

@@ -52,7 +52,7 @@ name: Multinode
 jobs:
   multinode:
     name: Multinode
-    uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/[email protected].0
+    uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/[email protected].1
     with:
       multinode_name: ${{ inputs.multinode_name }}
       os_distribution: ${{ inputs.os_distribution }}

.pre-commit-config.yaml

@@ -0,0 +1,11 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/sirwart/ripsecrets
+    rev: v0.1.7
+    hooks:
+      - id: ripsecrets

doc/source/contributor/index.rst

@@ -11,3 +11,4 @@ This guide is for contributors of the StackHPC Kayobe configuration project.
    release-notes
    environments/index
    package-updates
+   pre-commit

doc/source/contributor/pre-commit.rst

@@ -0,0 +1,47 @@
+================
+Pre-commit Hooks
+================
+
+StackHPC Kayobe configuration carries support for
+`pre-commit hooks <https://pre-commit.com/>`_ which simplify the use of git
+hooks enabling the identification and repairing of broken or poor code
+before committing.
+These hooks are designed to make working within SKC easier and less error prone.
+
+Currently the following hooks are provided:
+
+- ``check-yaml``: perform basic yaml syntax linting
+- ``end-of-file-fixer``: identify and automatically fix missing newline
+- ``trailing-whitespace``: identify and automatically fix excessive white space
+- ``ripsecrets``: identify and prevent secrets from being committed to the branch
+
+.. warning::
+   The hook ``ripsecrets`` is capable of preventing the accidental leaking of secrets
+   such as those found within `secrets.yml` or `passwords.yml`.
+   However if the secret is contained within a file on it's own and lacks a certain level
+   of entropy then the secret will not be identified as such as and maybe leaked as a result.
+
+Installation of `pre-commit` hooks is handled via the `install-pre-commit-hooks` playbook
+found within the Ansible directory.
+Either run the playbook manually or add the playbook as a hook within Kayobe config such as
+within `control-host-bootstrap/post.d`.
+Once done you should find `pre-commit` is available within the `kayobe` virtualenv.
+
+To run the playbook using the following command
+
+- ``kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/install-pre-commit-hooks.yml``
+
+Whereas to run the playbook when control host bootstrap runs ensure it registered as symlink using the following command
+
+- ``mkdir -p ${KAYOBE_CONFIG_PATH}/hooks/control-host-bootstrap/post.d``
+- ``ln -s ${KAYOBE_CONFIG_PATH}/ansible/install-pre-commit-hooks.yml ${KAYOBE_CONFIG_PATH}/hooks/control-host-bootstrap/post.d/install-pre-commit-hooks.yml``
+
+All that remains is the installation of the hooks themselves which can be accomplished either by
+running `pre-commit run` or using `git commit` when you have changes that need to be committed.
+This will trigger a brief installation process of the hooks which may take a few minutes.
+This a one time process and will not be required again unless new hooks are added or existing ones are updated.
+
+.. note::
+   Currently if you run ``pre-commit run --all-files`` it will make a series of changes to
+   release notes that lack new lines as well configuration files that ``check-yaml`` does not
+   approve of.

etc/kayobe/ansible/install-pre-commit-hooks.yml

@@ -0,0 +1,21 @@
+---
+- name: Install pre-commit hooks
+  hosts: localhost
+  gather_facts: false
+  vars:
+    pre_commit_version: 3.5.0
+  tasks:
+    - name: Install pre-commit hooks
+      block:
+        - name: Install pre-commit hooks into kayobe virtual env
+          ansible.builtin.pip:
+            name: pre-commit
+            version: "{{ pre_commit_version  }}"
+            virtualenv: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') | default(omit, true) }}"
+          register: pip_install
+
+        - name: Register pre-commit hooks with git
+          ansible.builtin.command:
+            cmd: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') | default(lookup('ansible.builtin.env', 'HOME') ~ '/.local', true) }}/bin/pre-commit install"
+          args:
+            chdir: "{{ playbook_dir | dirname | dirname | dirname }}"

etc/kayobe/environments/ci-multinode/kolla/globals.yml

@@ -1,3 +1,4 @@
+# yamllint disable-file
 ---
 ############################################################################
 # TLS
@@ -60,3 +61,10 @@ designate_ns_record:
 designate_backend: "bind9"
 designate_recursion: "yes"
 designate_forwarders_addresses: "1.1.1.1; 8.8.8.8"
+
+############################################################################
+# RabbitMQ
+
+# Ensure Rabbit is deployed with HA rather than quorum queues (to test migrations)
+om_enable_rabbitmq_high_availability: true
+om_enable_rabbitmq_quorum_queues: false

etc/kayobe/kolla.yml

@@ -147,14 +147,6 @@ kolla_sources:
     type: git
     location: https://github.com/stackhpc/networking-generic-switch.git
     reference: stackhpc/{{ openstack_release }}
-  octavia-api-plugin-ovn-octavia-provider:
-    type: git
-    location: https://github.com/stackhpc/ovn-octavia-provider.git
-    reference: stackhpc/{{ openstack_release }}
-  octavia-driver-agent-plugin-ovn-octavia-provider:
-    type: git
-    location: https://github.com/stackhpc/ovn-octavia-provider.git
-    reference: stackhpc/{{ openstack_release }}
 
 ###############################################################################
 # Kolla image build configuration.

releasenotes/notes/add-pre-commit-hooks-07ce3b82bbe1d7a3.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Add playbook to install pre-commit hooks and register them with git.
+    The hooks currently configured to be installed will check yaml syntax,
+    fix new line at end of file and remove excess whitespace. This is
+    currently opt-in which can be achieved by running `install-pre-commit-hooks`
+    playbook.