Merge pull request #785 from stackhpc/2023.1-zed-merge

2023.1: zed merge

Author: markgoddard

.github/workflows/stackhpc-promote.yml

@@ -3,7 +3,7 @@ name: Promote package repositories
 on:
   push:
     branches:
-      # NOTE(mgoddard): Reference only the current release branch here.
+      # NOTE(upgrade): Reference only the current release branch here.
       - stackhpc/2023.1
 jobs:
   promote:

doc/source/operations/octavia.rst

@@ -38,3 +38,55 @@ when building new images.
 
 To rollback an image update, simply delete the old image. The next newest image with
 a tag matching ``amp_image_tag`` will be selected.
+
+Manually deleting broken load balancers
+=======================================
+
+Sometimes, a load balancer will get stuck in a broken state of ``PENDING_CREATE`` or ``PENDING_UPDATE``.
+When in this state, the load balancer cannot be deleted; you will see the error ``Invalid state PENDING_CREATE of loadbalancer resource``.
+To delete a load balancer in this state, you will need to manually update its provisioning status in the database.
+
+Find the database password:
+
+.. code-block:: console
+
+  ansible-vault view --vault-password-file <path-to-vault-pw> $KOLLA_CONFIG_PATH/passwords.yml
+
+  # Search for database_password with:
+  /^database
+
+Access the database from a controller:
+
+.. code-block:: console
+
+  docker exec -it mariadb bash
+  mysql -u root -p  octavia
+  # Enter the database password when promted.
+
+List the load balancers to find the ID of the broken one(s):
+
+.. code-block:: console
+
+  SELECT * FROM load_balancer;
+
+Set the provisioning status to ERROR for any broken load balancer:
+
+.. code-block:: console
+
+  UPDATE load_balancer SET provisioning_status='ERROR' WHERE id='<id>';
+
+Delete the load balancer from the OpenStack CLI, cascading if any stray
+Amphorae are hanging around:
+
+.. code-block:: console
+
+  openstack loadbalancer delete <id> --cascade
+
+
+Sometimes, Amphora may also fail to delete if they are stuck in state
+``BOOTING``. These can be resolved entirely from the OpenStack CLI:
+
+.. code-block:: console
+
+  openstack loadbalancer amphora configure <amphora-id>
+  openstack loadbalancer amphora delete <amphora-id>

doc/source/operations/upgrading.rst

@@ -71,6 +71,31 @@ to::
 
     ark.stackhpc.com/stackhpc/etcd:zed-rocky-9-20230821T155947
 
+OVN enabled by default
+----------------------
+
+OVN is now enabled by default in StackHPC Kayobe Configuration.  This change
+was made to align with our standard deployment configuration.
+
+There is currently not a tested migration path from OVS to OVN on a running
+system. If you are using a Neutron plugin other than ML2/OVN, set
+``kolla_enable_ovn`` to ``false`` in ``etc/kayobe/kolla.yml``.
+
+For new deployments using OVN, see
+:kolla-ansible-doc:`reference/networking/neutron.html#ovn-ml2-ovn`.
+
+Known issues
+============
+
+* Rebuilds of servers with volumes are broken if there are any Nova compute
+  services running an older release, including any that are down. Old compute
+  services should be removed using ``openstack compute service delete``, then
+  remaining compute services restarted. See `LP#2040264
+  <https://bugs.launchpad.net/nova/+bug/2040264>`__.
+
+* The OVN sync repair tool removes metadata ports, breaking OVN load balancers.
+  See `LP#2038091 <https://bugs.launchpad.net/neutron/+bug/2038091>`__.
+
 Security baseline
 =================
 

etc/kayobe/ansible/growroot.yml

@@ -77,7 +77,7 @@
           vars:
             pv: "{{ pvs.stdout | from_json }}"
             disk_tmp: "{{ pv.report[0].pv[0].pv_name[:-1] }}"
-            disk: "{{ disk_tmp[:-1] if disk_tmp[-1] == 'p' else disk_tmp }}"
+            disk: "{{ disk_tmp[:-1] if disk_tmp[-1] == 'p' and disk_tmp[:4] == 'nvme' else disk_tmp }}"
             part_num: "{{ pv.report[0].pv[0].pv_name[-1] }}"
           become: true
           failed_when: "growpart.rc != 0 and 'NOCHANGE' not in growpart.stdout"

etc/kayobe/cephadm.yml

@@ -12,7 +12,7 @@ cephadm_ceph_release: "quincy"
 cephadm_image: "{{ stackhpc_docker_registry if stackhpc_sync_ceph_images | bool else 'quay.io' }}/ceph/ceph:{{ cephadm_image_tag }}"
 
 # Ceph container image tag.
-cephadm_image_tag: "v17.2.6"
+cephadm_image_tag: "v17.2.7"
 
 # Ceph custom repo workaround for Ubuntu Jammy as there are no official ceph repos for jammy.
 cephadm_custom_repos: "{{ ansible_facts['distribution_release'] == 'jammy' }}"
@@ -92,7 +92,7 @@ cephadm_commands_post: "{{ cephadm_commands_post_default + cephadm_commands_post
 cephadm_commands_pre_default: []
 cephadm_commands_pre_extra: []
 
-cephadm_commands_post_default: "{{ ['mgr module enable prometheus'] if kolla_enable_prometheus_ceph_mgr_exporter | bool else [] }}"
+cephadm_commands_post_default: "{{ ['mgr module enable prometheus'] if kolla_enable_prometheus_ceph_mgr_exporter | default(False) | bool else [] }}"
 cephadm_commands_post_extra: []
 
 ###############################################################################

etc/kayobe/inventory/group_vars/all/vault

@@ -10,13 +10,13 @@ seed_hashicorp_registry_password: "{{ stackhpc_docker_registry_password if stack
 seed_consul_docker_image: "{{ stackhpc_docker_registry ~ '/' if stackhpc_sync_hashicorp_images | bool else '' }}hashicorp/consul"
 
 # Seed Consul container image tag.
-seed_consul_docker_tag: "1.16.1"
+seed_consul_docker_tag: "1.16.3"
 
 # Seed Vault container image.
 seed_vault_docker_image: "{{ stackhpc_docker_registry ~ '/' if stackhpc_sync_hashicorp_images | bool else '' }}hashicorp/vault"
 
 # Seed Vault container image tag.
-seed_vault_docker_tag: "1.14.1"
+seed_vault_docker_tag: "1.14.6"
 
 # Seed Vault PKI Role name
 seed_vault_pki_role_name: "ServerCert"

etc/kayobe/kolla.yml

@@ -106,12 +106,9 @@ kolla_tag: "{{ openstack_release }}-{{ kolla_base_distro }}-{{ kolla_base_distro
 #     type: git
 #     location: https://github.com/openstack/ironic
 #     reference: master
-# NOTE (Alex-Welsh): The reference for many of these entries could be
-# 'stackhpc/{{ openstack_release }}' which would keep the branch up to date
-# for the current release. This is nice in theory but in practice, the stackhpc
-# forks change with every release and the elements in this list change with
-# them. Explicitly using /2021.3 makes it more intuitive to find and edit these
-# entries.
+# NOTE(upgrade): These sources should be checked with each release. StackHPC
+# branches are only required when we have custom backports. For a new release,
+# we may have caught up with upstream.
 kolla_sources:
   bifrost-base-additions-stackhpc-inspector-plugins:
     # Install our custom inspector plugins.
@@ -121,11 +118,11 @@ kolla_sources:
   cloudkitty-base:
     type: git
     location: https://github.com/stackhpc/cloudkitty.git
-    reference: stackhpc/2023.1
+    reference: stackhpc/{{ openstack_release }}
   horizon-plugin-cloudkitty-dashboard:
     type: git
     location: https://github.com/stackhpc/cloudkitty-dashboard.git
-    reference: stackhpc/2023.1
+    reference: stackhpc/{{ openstack_release }}
   ironic-inspector-additions-stackhpc-inspector-plugins:
     # Install our custom inspector plugins.
     type: git
@@ -135,11 +132,15 @@ kolla_sources:
   magnum-base:
     type: git
     location: https://github.com/stackhpc/magnum.git
-    reference: stackhpc/2023.1
+    reference: stackhpc/{{ openstack_release }}
+  neutron-base:
+    type: git
+    location: https://github.com/stackhpc/neutron.git
+    reference: stackhpc/{{ openstack_release }}
   neutron-base-plugin-networking-generic-switch:
     type: git
     location: https://github.com/stackhpc/networking-generic-switch.git
-    reference: stackhpc/2023.1
+    reference: stackhpc/{{ openstack_release }}
 
 ###############################################################################
 # Kolla image build configuration.
@@ -284,18 +285,22 @@ kolla_build_blocks:
     {% if stackhpc_kolla_clean_up_repo_mirrors | bool %}
     {% if kolla_base_distro == 'rocky' %}
     RUN \
-        tar -xzf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d && \
-        if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/yum.repos.d; then \
-           echo "Found repository mirror in Yum repositories"; \
-           exit 1; \
+        if [ -f /etc/yum.repos.d.backup/repos.tar.gz ]; then \
+          tar -xzf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d && \
+          rm -rf /etc/yum.repos.d.backup/; \
         fi && \
-        rm -rf /etc/yum.repos.d.backup/
+        if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/yum.repos.d; then \
+          echo "Found repository mirror in Yum repositories"; \
+          exit 1; \
+        fi
     {% else %}
     RUN \
-        mv /etc/apt/sources.list.backup /etc/apt/sources.list && \
+        if [ -f /etc/apt/sources.list.backup ]; then \
+          mv /etc/apt/sources.list.backup /etc/apt/sources.list; \
+        fi && \
         if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/apt/sources.list; then \
-           echo "Found repository mirror in APT repositories"; \
-           exit 1; \
+          echo "Found repository mirror in APT repositories"; \
+          exit 1; \
         fi
     {% endif %}
     {% endif %}

etc/kayobe/kolla/globals.yml

@@ -12,9 +12,17 @@ kayobe_image_tags:
   bifrost:
     rocky: 2023.1-rocky-9-20231013T151957
     ubuntu: 2023.1-ubuntu-jammy-20231013T151957
+  cloudkitty:
+    rocky: 2023.1-rocky-9-20231115T110235
+    ubuntu: 2023.1-ubuntu-jammy-20231115T110235
+  neutron:
+    rocky: 2023.1-rocky-9-20231115T110249
+    ubuntu: 2023.1-ubuntu-jammy-20231115T110249
 
 openstack_tag: "{% raw %}{{ kayobe_image_tags['openstack'][kolla_base_distro] }}{% endraw %}"
 bifrost_tag: "{% raw %}{{ kayobe_image_tags['bifrost'][kolla_base_distro] }}{% endraw %}"
+cloudkitty_tag: "{% raw %}{{ kayobe_image_tags['cloudkitty'][kolla_base_distro] }}{% endraw %}"
+neutron_tag: "{% raw %}{{ kayobe_image_tags['neutron'][kolla_base_distro] }}{% endraw %}"
 
 om_enable_rabbitmq_high_availability: true
 
@@ -40,6 +48,7 @@ prometheus_ceph_mgr_exporter_endpoints:
 # Use inventory hostnames as labels
 prometheus_instance_label: "{% raw %}{{ ansible_facts.hostname }}{% endraw %}"
 
-#############################################################################
-
-neutron_ovn_distributed_fip: true
+# Make openstack-exporter use Nova API version 2.1 to keep metrics the same as
+# in Yoga. This is required to include a valid value for the flavor_id label on
+# openstack_nova_server_status metrics.
+prometheus_openstack_exporter_compute_api_version: "2.1"

releasenotes/notes/bump-ceph-images-f9d4721aa1d37729.yaml

@@ -0,0 +1,4 @@
+---
+upgrade:
+  - |
+    Updates default Ceph images to v17.2.7 for Quincy.

releasenotes/notes/bump-hashicorp-images-7ef66e1328983bbe.yaml

@@ -0,0 +1,4 @@
+---
+upgrade:
+  - |
+    Updates Consul to 1.16.3 and Vault to 1.14.6.