Merge branch 'stackhpc/2023.1' into feature/2023.1/friendly-interface-names

Author: tomclark0

.automation

@@ -32,4 +32,4 @@ max_microversion = 3.70
 build_timeout = 600
 
 [dashboard]
-dashboard_url = http://192.168.39.2
+dashboard_url = https://192.168.39.2

.automation.conf/tempest/tempest-ci-multinode.overrides.conf

@@ -6,6 +6,18 @@ This document describes how to deploy Hashicorp Vault for
 internal PKI purposes using the
 `StackHPC Hashicorp collection <https://galaxy.ansible.com/stackhpc/hashicorp>`_
 
+Vault may be used as a Certificate Authority to generate certificates for:
+
+* OpenStack internal API
+* OpenStack backend APIs
+* RabbitMQ
+
+TLS support is described in the :kolla-ansible-doc:`Kolla Ansible documentation
+<admin/tls.html>` and the :kayobe-doc:`Kayobe documentation
+<configuration/reference/kolla-ansible.html#tls-encryption-of-apis>`.
+
+Vault may also be used as the secret store for Barbican.
+
 Background
 ==========
 
@@ -137,6 +149,30 @@ Setup Vault HA on the overcloud hosts
 Certificates generation
 =======================
 
+Create the external TLS certificates (testing only)
+---------------------------------------------------
+
+Typically external API TLS certificates should be generated by a organisation's trusted internal or third-party CA.
+For test and development purposes it is possible to use Vault as a CA for the external API.
+
+1. Run the playbook
+
+   .. code-block::
+
+      kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml
+
+2. Use ansible-vault to encrypt the PEM bundle in $KAYOBE_CONFIG_PATH/kolla/certificates/haproxy.pem. Commit the PEM bundle to the kayobe configuration.
+
+   .. code-block::
+
+      ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/kolla/certificates/haproxy.pem
+
+   Or if environments are being used
+
+   .. code-block::
+
+      ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/haproxy.pem
+
 Create the internal TLS certificates
 ------------------------------------
 
@@ -189,14 +225,23 @@ Certificates deployment
 Enable the required TLS variables in kayobe and kolla
 -----------------------------------------------------
 
-1. Set the following in kayobe-config/etc/kayobe/kolla.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla.yml
+1. If using Vault as a CA for the external API, set the following in kayobe-config/etc/kayobe/kolla.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla.yml
+
+   .. code-block::
+
+      # Whether TLS is enabled for the external API endpoints. Default is 'no'.
+      kolla_enable_tls_external: yes
+
+   See :ref:`tempest-cacert` for information on adding CA certificates to the trust store when running Tempest.
+
+2. Set the following in kayobe-config/etc/kayobe/kolla.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla.yml
 
    .. code-block::
 
       # Whether TLS is enabled for the internal API endpoints. Default is 'no'.
       kolla_enable_tls_internal: yes
 
-2. Set the following in etc/kayobe/kolla/globals.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/globals.yml
+3. Set the following in etc/kayobe/kolla/globals.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/globals.yml
 
    .. code-block::
 
@@ -213,7 +258,7 @@ Enable the required TLS variables in kayobe and kolla
       # If using RabbitMQ TLS:
       rabbitmq_enable_tls: "yes"
 
-3. Deploy backend and internal TLS
+4. Deploy OpenStack
 
    .. warning::
 

doc/source/configuration/vault.rst

@@ -251,6 +251,25 @@ ever contain one host. The seed is usually used as the tempest runner however
 it is also common to use the Ansible control host or an infrastructure VM. The
 main requirement of the host is that it can reach the OpenStack API.
 
+.. _tempest-cacert:
+
+Tempest CA certificate
+----------------------
+
+If your public OpenStack API uses TLS with a Certificate Authority (CA) that is
+not trusted by the Python CA trust store, it may be necessary to add a CA
+certificate to the trust store in the container that runs Tempest. This can be
+done by defining a ``tempest_cacert`` Ansible variable to a path containing the
+CA certificate. You may wish to use ``kayobe_config_path`` or
+``kayobe_env_config_path`` to be agnostic to the path where kayobe-config is
+mounted within the container. For example:
+
+.. code-block:: yaml
+   :caption: ``etc/kayobe/tempest.yml``
+
+   # Add the Vault CA certificate to the rally container when running tempest.
+   tempest_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt"
+
 Running Tempest
 ===============
 

doc/source/operations/tempest.rst

@@ -0,0 +1,44 @@
+---
+# When OVS HW offloading is enabled - typically in conjunction with VF-LAG and ASAP^2
+# the DMESG log reports frequent errors on the internal OVS Bridge interface:
+# "tc mirred to Houston: device bond0-ovs is down".
+# This interface is down by default. The errors are mitigated by bringing the interface up.
+# For further context, see:
+# https://bugs.launchpad.net/charm-neutron-openvswitch/+bug/1899364
+# https://patchwork.kernel.org/project/netdevbpf/patch/c2ef23da1d9a4eb62f4e7b7c4540f9bafb553c15.1658420239.git.dcaratti@redhat.com/
+# To deploy this playbook, use the following commands:
+# kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/fix-houston.yml
+# Enable with Kayobe Hooks by running:
+# mkdir -p ${KAYOBE_CONFIG_PATH}/hooks/overcloud-service-deploy/post.d
+# cd ${KAYOBE_CONFIG_PATH}/hooks/overcloud-service-deploy/post.d
+# ln -s ../../../ansible/fix-houston.yml 90-fix-houston.yml
+
+- name: Create Systemd Unit to workaround 'tc mirred to Houston' error
+  hosts: network,compute
+  become: yes
+
+  tasks:
+    - name: Include kolla-ansible host vars
+      include_vars: "{{ kolla_config_path }}/inventory/overcloud/host_vars/{{ inventory_hostname }}"
+
+    - name: Create systemd service for -ovs network interface
+      template:
+        src: fix-houston-interface.service.j2
+        dest: "/etc/systemd/system/fix-houston-{{ item }}.service"
+      loop: "{{ neutron_bridge_name.split(',') }}"
+      vars:
+        interface_name: "{{ item }}"
+      when: neutron_bridge_name | length > 0
+      notify: reload systemd
+
+    - name: Enable and start systemd service for -ovs network interface
+      systemd:
+        name: "fix-houston-{{ item }}"
+        enabled: yes
+        state: started
+      when: neutron_bridge_name | length > 0
+      loop: "{{ neutron_bridge_name.split(',') }}"
+
+  handlers:
+    - name: reload systemd
+      command: systemctl daemon-reload

etc/kayobe/ansible/fix-houston.yml

@@ -19,6 +19,9 @@
         name: "{{ repository_name }}_{{ promotion_tag }}"
         base_path: "{{ base_path }}/{{ promotion_tag }}"
       register: distribution_details
+      until: distribution_details is success
+      retries: 3
+      delay: 5
 
     - name: Fail if the image does not exist
       fail:
@@ -34,6 +37,10 @@
         base_path: "{{ base_path }}/{{ promotion_tag }}"
         content_guard: release
         state: present
+      register: content_guard_result
+      until: content_guard_result is success
+      retries: 3
+      delay: 5
 
     - name: Print version tag and os
       debug:

etc/kayobe/ansible/pulp-host-image-promote.yml

@@ -25,6 +25,10 @@
         password: "{{ remote_pulp_password }}"
         file: "{{ found_files.files[0].path }}"
         state: present
+      register: upload_result
+      until: upload_result is success
+      retries: 3
+      delay: 60
 
     - name: Get sha256 hash
       ansible.builtin.stat:
@@ -40,6 +44,10 @@
         sha256: "{{ file_stats.stat.checksum }}"
         relative_path: "{{ found_files.files[0].path | basename }}"
         state: present
+      register: file_content_result
+      until: file_content_result is success
+      retries: 3
+      delay: 5
 
     - name: Ensure file repo exists
       pulp.squeezer.file_repository:
@@ -48,6 +56,10 @@
         password: "{{ remote_pulp_password }}"
         name: "{{ repository_name }}"
         state: present
+      register: file_repo_result
+      until: file_repo_result is success
+      retries: 3
+      delay: 5
 
     - name: Add content to file repo
       pulp.squeezer.file_repository_content:
@@ -58,6 +70,10 @@
         present_content:
           - relative_path: "{{ found_files.files[0].path | basename }}"
             sha256: "{{ file_stats.stat.checksum }}"
+      register: file_repo_content_result
+      until: file_repo_content_result is success
+      retries: 3
+      delay: 5
 
     - name: Create a new publication to point to this version
       pulp.squeezer.file_publication:
@@ -67,6 +83,9 @@
         repository: "{{ repository_name }}"
         state: present
       register: publication_details
+      until: publication_details is success
+      retries: 3
+      delay: 5
 
     - name: Update distribution for latest version
       pulp.squeezer.file_distribution:
@@ -79,6 +98,9 @@
         content_guard: development
         state: present
       register: latest_distribution_details
+      until: latest_distribution_details is success
+      retries: 3
+      delay: 5
 
     - name: Create distribution for given version
       pulp.squeezer.file_distribution:
@@ -91,6 +113,10 @@
         content_guard: development
         state: present
       when: latest_distribution_details.changed
+      register: distribution_result
+      until: distribution_result is success
+      retries: 3
+      delay: 5
 
     - name: Update new images file with versioned path
       lineinfile:

etc/kayobe/ansible/pulp-host-image-upload.yml

@@ -0,0 +1,20 @@
+[Unit]
+# This service addresses a specific issue when OVS HW offloading is enabled
+# typically in conjunction with VF-LAG and ASAP^2
+# the DMESG log reports frequent errors on the internal OVS Bridge interface:
+# "tc mirred to Houston: device bond0-ovs is down".
+# This interface is down by default. The errors are mitigated by bringing the interface up.
+# For further context, see:
+# https://bugs.launchpad.net/charm-neutron-openvswitch/+bug/1899364
+# https://patchwork.kernel.org/project/netdevbpf/patch/c2ef23da1d9a4eb62f4e7b7c4540f9bafb553c15.1658420239.git.dcaratti@redhat.com/
+Description=Bring up {{ interface_name }} interface
+After=kolla-openvswitch_vswitchd-container.service
+
+[Service]
+Type=oneshot
+ExecStartPre=/usr/bin/timeout 60s /bin/bash -c 'until ip link show {{ interface_name }}; do sleep 1; done'
+ExecStart=/sbin/ip link set {{ interface_name }} up
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target

etc/kayobe/ansible/templates/fix-houston-interface.service.j2

@@ -0,0 +1,55 @@
+---
+- name: Generate external API certificate (for testing only)
+  hosts: controllers
+  run_once: true
+  vars:
+    vault_api_addr: "https://{{ kolla_internal_fqdn }}:8200"
+    # NOTE: Using the same CA as internal TLS.
+    vault_intermediate_ca_name: "OS-TLS-INT"
+  tasks:
+    - name: Include Vault keys
+      include_vars:
+        file: "{{ kayobe_env_config_path }}/vault/overcloud-vault-keys.json"
+        name: vault_keys
+
+    - name: Issue a certificate for external TLS
+      hashivault_pki_cert_issue:
+        url: "{{ vault_api_addr }}"
+        ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}"
+        token: "{{ vault_keys.root_token }}"
+        mount_point: "{{ vault_intermediate_ca_name }}"
+        role: "{{ overcloud_vault_pki_external_tls_role_name }}"
+        common_name: "{% if kolla_external_fqdn != kolla_external_vip_address %}{{ kolla_external_fqdn }}{% endif %}"
+        extra_params:
+          ip_sans: "{{ kolla_external_vip_address }}"
+      register: external_cert
+
+    - name: Ensure certificates directory exists
+      file:
+        path: "{{ kayobe_env_config_path }}/kolla/certificates"
+        state: directory
+      delegate_to: localhost
+
+    - name: Ensure CA certificates directory exists
+      file:
+        path: "{{ kayobe_env_config_path }}/kolla/certificates/ca"
+        state: directory
+      delegate_to: localhost
+
+    - name: Copy external API PEM bundle
+      no_log: true
+      copy:
+        dest: "{{ kayobe_env_config_path }}/kolla/certificates/haproxy.pem"
+        content: |
+          {{ external_cert.data.certificate }}
+          {{ external_cert.data.issuing_ca }}
+          {{ external_cert.data.private_key }}
+        mode: 0600
+      delegate_to: localhost
+
+    - name: Copy root CA
+      copy:
+        src: "{{ kayobe_env_config_path }}/vault/OS-TLS-ROOT.pem"
+        dest: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt"
+        mode: 0600
+      delegate_to: localhost

etc/kayobe/ansible/vault-generate-test-external-tls.yml

@@ -8,12 +8,16 @@ kolla_enable_designate: true
 kolla_enable_redis: true
 kolla_enable_barbican: true
 
-# The multinode environment supports Backend and internal TLS , but it must be
-# enabled in the correct order. See
+# The multinode environment supports backend, external and internal TLS , but
+# it must be enabled in the correct order. See
 # https://stackhpc-kayobe-config.readthedocs.io/en/stackhpc-yoga/configuration/vault.html
 # for details.
+# kolla_enable_tls_external: true
 # kolla_enable_tls_internal: true
 
+kolla_public_openrc_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if os_distribution in ['centos', 'rocky'] else '/etc/ssl/certs/ca-certificates.crt' }}"
+kolla_admin_openrc_cacert: "{{ kolla_public_openrc_cacert }}"
+
 # The multinode environment supports Manila but it is not enabled by default.
 # kolla_enable_manila: true
 # kolla_enable_manila_backend_cephfs_native: true