Skip to content

Commit d0b4fa8

Browse files
Alex-WelshAlex-Welsh
committed
Merge remote-tracking branch 'origin/stackhpc/2024.1' into caracal-image-build
2 parents 965d5df + 3a1c2a3 commit d0b4fa8

File tree

14 files changed

+271
-73
lines changed

14 files changed

+271
-73
lines changed

‎.github/workflows/multinode-inputs.py‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# Generate inputs for the reusable multinode.yml workflow.
2+
# The test scenario is randomly selected.
3+
# The inputs are printed to stdout in GitHub step output key=value format.
4+
5+
from dataclasses import dataclass
6+
import random
7+
import typing as t
8+
9+
10+
@dataclass
11+
class OSRelease:
12+
distribution: str
13+
release: str
14+
ssh_username: str
15+
16+
17+
@dataclass
18+
class OpenStackRelease:
19+
version: str
20+
previous_version: str
21+
os_releases: t.List[OSRelease]
22+
23+
24+
@dataclass
25+
class Scenario:
26+
openstack_release: OpenStackRelease
27+
os_release: OSRelease
28+
neutron_plugin: str
29+
upgrade: bool
30+
31+
32+
ROCKY_9 = OSRelease("rocky", "9", "cloud-user")
33+
UBUNTU_JAMMY = OSRelease("ubuntu", "jammy", "ubuntu")
34+
# NOTE(upgrade): Add supported releases here.
35+
OPENSTACK_RELEASES = [
36+
OpenStackRelease("2023.1", "zed", [ROCKY_9, UBUNTU_JAMMY])
37+
]
38+
NEUTRON_PLUGINS = ["ovs", "ovn"]
39+
40+
41+
def main() -> None:
42+
scenario = random_scenario()
43+
inputs = generate_inputs(scenario)
44+
for name, value in inputs.items():
45+
write_output(name, value)
46+
47+
48+
def random_scenario() -> Scenario:
49+
openstack_release = random.choice(OPENSTACK_RELEASES)
50+
os_release = random.choice(openstack_release.os_releases)
51+
neutron_plugin = random.choice(NEUTRON_PLUGINS)
52+
upgrade = random.random() > 0.6
53+
return Scenario(openstack_release, os_release, neutron_plugin, upgrade)
54+
55+
56+
def generate_inputs(scenario: Scenario) -> t.Dict[str, str]:
57+
branch = get_branch(scenario.openstack_release.version)
58+
previous_branch = get_branch(scenario.openstack_release.previous_version)
59+
inputs = {
60+
"os_distribution": scenario.os_release.distribution,
61+
"os_release": scenario.os_release.release,
62+
"ssh_username": scenario.os_release.ssh_username,
63+
"neutron_plugin": scenario.neutron_plugin,
64+
"upgrade": str(scenario.upgrade).lower(),
65+
"stackhpc_kayobe_config_version": branch,
66+
"stackhpc_kayobe_config_previous_version": previous_branch,
67+
}
68+
return inputs
69+
70+
71+
def get_branch(version: str) -> str:
72+
return f"stackhpc/{version}"
73+
74+
75+
def write_output(name: str, value: str) -> None:
76+
print(f"{name}={value}")
77+
78+
79+
if __name__ == "__main__":
80+
main()

‎.github/workflows/overcloud-host-image-build.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ jobs:
191191
source venvs/kayobe/bin/activate &&
192192
source src/kayobe-config/kayobe-env --environment ci-builder &&
193193
kayobe seed host command run \
194-
--command "sudo dnf config-manager --set-enabled crb && sudo dnf -y install epel-release && sudo dnf -y install zstd debootstrap kpartx cloud-init" --show-output
194+
--command "sudo dnf config-manager --set-enabled crb && sudo dnf -y install epel-release && sudo dnf -y install cloud-init debootstrap git kpartx zstd" --show-output
195195
env:
196196
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
197197

‎.github/workflows/stackhpc-multinode-periodic.yml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
---
2+
# This workflow provides a periodic deploy of a multi-node test cluster.
3+
# The test scenario is randomly selected.
4+
5+
name: Multinode periodic
6+
'on':
7+
schedule:
8+
# Runs nightly at 2:42 AM.
9+
- cron: "42 2 * * *"
10+
jobs:
11+
generate-inputs:
12+
name: Generate inputs
13+
runs-on: ubuntu-latest
14+
outputs:
15+
os_distribution: ${{ steps.generate-inputs.outputs.os_distribution }}
16+
os_release: ${{ steps.generate-inputs.outputs.os_release }}
17+
ssh_username: ${{ steps.generate-inputs.outputs.ssh_username }}
18+
neutron_plugin: ${{ steps.generate-inputs.outputs.neutron_plugin }}
19+
upgrade: ${{ steps.generate-inputs.outputs.upgrade }}
20+
stackhpc_kayobe_config_version: ${{ steps.generate-inputs.outputs.stackhpc_kayobe_config_version }}
21+
stackhpc_kayobe_config_previous_version: ${{ steps.generate-inputs.outputs.stackhpc_kayobe_config_previous_version }}
22+
steps:
23+
- name: Checkout
24+
uses: actions/checkout@v4
25+
26+
- name: Generate inputs for multinode workflow
27+
id: generate-inputs
28+
run: |
29+
python3 .github/workflows/multinode-inputs.py >> $GITHUB_OUTPUT
30+
31+
- name: Display generated inputs
32+
run: |
33+
echo '${{ toJSON(steps.generate-inputs.outputs) }}'
34+
multinode:
35+
name: Multinode periodic
36+
needs:
37+
- generate-inputs
38+
uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/[email protected]
39+
with:
40+
multinode_name: mn-prdc-${{ github.run_id }}
41+
os_distribution: ${{ needs.generate-inputs.outputs.os_distribution }}
42+
os_release: ${{ needs.generate-inputs.outputs.os_release }}
43+
ssh_username: ${{ needs.generate-inputs.outputs.ssh_username }}
44+
neutron_plugin: ${{ needs.generate-inputs.outputs.neutron_plugin }}
45+
upgrade: ${{ needs.generate-inputs.outputs.upgrade == 'true' }}
46+
stackhpc_kayobe_config_version: ${{ needs.generate-inputs.outputs.stackhpc_kayobe_config_version }}
47+
stackhpc_kayobe_config_previous_version: ${{ needs.generate-inputs.outputs.stackhpc_kayobe_config_previous_version }}
48+
enable_slack_alert: true
49+
secrets: inherit
50+
if: github.repository == 'stackhpc/stackhpc-kayobe-config'

‎.github/workflows/stackhpc-multinode.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ name: Multinode
5252
jobs:
5353
multinode:
5454
name: Multinode
55-
uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/[email protected].0
55+
uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/[email protected].1
5656
with:
5757
multinode_name: ${{ inputs.multinode_name }}
5858
os_distribution: ${{ inputs.os_distribution }}

‎.pre-commit-config.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
repos:
2+
- repo: https://github.com/pre-commit/pre-commit-hooks
3+
rev: v4.4.0
4+
hooks:
5+
- id: check-yaml
6+
- id: end-of-file-fixer
7+
- id: trailing-whitespace
8+
- repo: https://github.com/sirwart/ripsecrets
9+
rev: v0.1.7
10+
hooks:
11+
- id: ripsecrets

‎doc/source/contributor/index.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,4 @@ This guide is for contributors of the StackHPC Kayobe configuration project.
1111
release-notes
1212
environments/index
1313
package-updates
14+
pre-commit

‎doc/source/contributor/pre-commit.rst‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
================
2+
Pre-commit Hooks
3+
================
4+
5+
StackHPC Kayobe configuration carries support for
6+
`pre-commit hooks <https://pre-commit.com/>`_ which simplify the use of git
7+
hooks enabling the identification and repairing of broken or poor code
8+
before committing.
9+
These hooks are designed to make working within SKC easier and less error prone.
10+
11+
Currently the following hooks are provided:
12+
13+
- ``check-yaml``: perform basic yaml syntax linting
14+
- ``end-of-file-fixer``: identify and automatically fix missing newline
15+
- ``trailing-whitespace``: identify and automatically fix excessive white space
16+
- ``ripsecrets``: identify and prevent secrets from being committed to the branch
17+
18+
.. warning::
19+
The hook ``ripsecrets`` is capable of preventing the accidental leaking of secrets
20+
such as those found within `secrets.yml` or `passwords.yml`.
21+
However if the secret is contained within a file on it's own and lacks a certain level
22+
of entropy then the secret will not be identified as such as and maybe leaked as a result.
23+
24+
Installation of `pre-commit` hooks is handled via the `install-pre-commit-hooks` playbook
25+
found within the Ansible directory.
26+
Either run the playbook manually or add the playbook as a hook within Kayobe config such as
27+
within `control-host-bootstrap/post.d`.
28+
Once done you should find `pre-commit` is available within the `kayobe` virtualenv.
29+
30+
To run the playbook using the following command
31+
32+
- ``kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/install-pre-commit-hooks.yml``
33+
34+
Whereas to run the playbook when control host bootstrap runs ensure it registered as symlink using the following command
35+
36+
- ``mkdir -p ${KAYOBE_CONFIG_PATH}/hooks/control-host-bootstrap/post.d``
37+
- ``ln -s ${KAYOBE_CONFIG_PATH}/ansible/install-pre-commit-hooks.yml ${KAYOBE_CONFIG_PATH}/hooks/control-host-bootstrap/post.d/install-pre-commit-hooks.yml``
38+
39+
All that remains is the installation of the hooks themselves which can be accomplished either by
40+
running `pre-commit run` or using `git commit` when you have changes that need to be committed.
41+
This will trigger a brief installation process of the hooks which may take a few minutes.
42+
This a one time process and will not be required again unless new hooks are added or existing ones are updated.
43+
44+
.. note::
45+
Currently if you run ``pre-commit run --all-files`` it will make a series of changes to
46+
release notes that lack new lines as well configuration files that ``check-yaml`` does not
47+
approve of.

‎doc/source/operations/upgrading-openstack.rst‎

Lines changed: 49 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -35,64 +35,6 @@ Notable changes in the |current_release| Release
3535
There are many changes in the OpenStack |current_release| release described in
3636
the release notes for each project. Here are some notable ones.
3737

38-
RabbitMQ SLURP upgrade
39-
----------------------
40-
41-
Because this is a SLURP upgrade, RabbitMQ must be upgraded manually from 3.11,
42-
to 3.12, then to 3.13 on Antelope before the Caracal upgrade. This upgrade
43-
should not cause an API outage (though it should still be considered "at
44-
risk").
45-
46-
There are two prerequisites:
47-
48-
1. Kolla-Ansible should be upgraded to the latest version:
49-
50-
.. code-block:: bash
51-
52-
cd $KOLLA_SOURCE_PATH
53-
git fetch && git pull
54-
$KOLLA_VENV_PATH/bin/pip install .
55-
56-
2. The RabbitMQ container image tag must be equal to or newer than
57-
``20240823T101942``. Check the timestamps in
58-
``etc/kayobe/kolla-image-tags.yml``.
59-
60-
Once complete, upgrade RabbitMQ:
61-
62-
.. code-block:: bash
63-
64-
kayobe overcloud service configuration generate --node-config-dir /tmp/ignore -kt none
65-
kayobe kolla ansibe run "rabbitmq-upgrade 3.12"
66-
kayobe kolla ansibe run "rabbitmq-upgrade 3.13"
67-
68-
RabbitMQ quorum queues
69-
----------------------
70-
71-
In Caracal, quorum queues are enabled by default for RabbitMQ. This is
72-
different to Antelope which used HA queues. Before upgrading to Caracal, it is
73-
strongly recommended that you migrate from HA to quorum queues. The migration
74-
is automated using a script.
75-
76-
.. warning::
77-
This migration will stop all services using RabbitMQ and cause an
78-
extended API outage while queues are migrated. It should only be
79-
performed in a pre-agreed maintenance window.
80-
81-
Set the following variables in your kolla globals file (i.e.
82-
``$KAYOBE_CONFIG_PATH/kolla/globals.yml`` or
83-
``$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/globals.yml``):
84-
85-
.. code-block:: yaml
86-
87-
om_enable_rabbitmq_high_availability: false
88-
om_enable_rabbitmq_quorum_queues: true
89-
90-
Then execute the migration script:
91-
92-
.. code-block:: bash
93-
94-
$KAYOBE_CONFIG_PATH/../../tools/rabbitmq-quorum-migration.sh
95-
9638
Heat disabled by default
9739
------------------------
9840

@@ -115,8 +57,8 @@ TODO: guide for disabling Heat
11557
Designate sink disabled by default
11658
----------------------------------
11759

118-
Designate sink is optional designate service which listens for event
119-
Notifications, primarily from Nova and Neutron. It is disabled by default (when
60+
Designate sink is an optional Designate service which listens for event
61+
notifications, primarily from Nova and Neutron. It is disabled by default (when
12062
designate is enabled) in Caracal. It is not required for Designate to function.
12163

12264
If you still wish to use it, you should set the flag manually:
@@ -211,6 +153,53 @@ suggestions:
211153
* Update the deployment to use the latest |previous_release| images and
212154
configuration.
213155

156+
RabbitMQ SLURP upgrade
157+
----------------------
158+
159+
.. note::
160+
The upgrade is reliant on recent changes. Make sure you have updated to
161+
the latest version of kolla ansible and deployed the latest kolla containers
162+
before proceeding.
163+
164+
Because this is a SLURP upgrade, RabbitMQ must be upgraded manually from 3.11,
165+
to 3.12, then to 3.13 on Antelope before the Caracal upgrade. This upgrade
166+
should not cause an API outage (though it should still be considered "at
167+
risk").
168+
169+
.. code-block:: bash
170+
171+
kayobe overcloud service configuration generate --node-config-dir /tmp/ignore -kt none
172+
kayobe kolla ansible run "rabbitmq-upgrade 3.12"
173+
kayobe kolla ansible run "rabbitmq-upgrade 3.13"
174+
175+
RabbitMQ quorum queues
176+
----------------------
177+
178+
In Caracal, quorum queues are enabled by default for RabbitMQ. This is
179+
different to Antelope which used HA queues. Before upgrading to Caracal, it is
180+
strongly recommended that you migrate from HA to quorum queues. The migration
181+
is automated using a script.
182+
183+
.. warning::
184+
This migration will stop all services using RabbitMQ and cause an
185+
extended API outage while queues are migrated. It should only be
186+
performed in a pre-agreed maintenance window.
187+
188+
Set the following variables in your kolla globals file (i.e.
189+
``$KAYOBE_CONFIG_PATH/kolla/globals.yml`` or
190+
``$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/globals.yml``):
191+
192+
.. code-block:: yaml
193+
194+
om_enable_rabbitmq_high_availability: false
195+
om_enable_rabbitmq_quorum_queues: true
196+
197+
Then execute the migration script:
198+
199+
.. code-block:: bash
200+
201+
$KAYOBE_CONFIG_PATH/../../tools/rabbitmq-quorum-migration.sh
202+
214203
Preparation
215204
===========
216205

‎etc/kayobe/ansible/install-pre-commit-hooks.yml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
---
2+
- name: Install pre-commit hooks
3+
hosts: localhost
4+
gather_facts: false
5+
vars:
6+
pre_commit_version: 3.5.0
7+
tasks:
8+
- name: Install pre-commit hooks
9+
block:
10+
- name: Install pre-commit hooks into kayobe virtual env
11+
ansible.builtin.pip:
12+
name: pre-commit
13+
version: "{{ pre_commit_version }}"
14+
virtualenv: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') | default(omit, true) }}"
15+
register: pip_install
16+
17+
- name: Register pre-commit hooks with git
18+
ansible.builtin.command:
19+
cmd: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') | default(lookup('ansible.builtin.env', 'HOME') ~ '/.local', true) }}/bin/pre-commit install"
20+
args:
21+
chdir: "{{ playbook_dir | dirname | dirname | dirname }}"

‎etc/kayobe/environments/ci-aio/stackhpc-ci.yml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,6 @@
55
# Docker namespace to use for Kolla images. Default is 'kolla'.
66
kolla_docker_namespace: stackhpc-dev
77

8-
# Disable some services to reduce memory footprint.
9-
kolla_enable_heat: false
10-
118
###############################################################################
129
# StackHPC configuration.
1310

0 commit comments

Comments
 (0)