Merge pull request #1032 from stackhpc/2023.1-multinode-public-tls

2023.1: Enable TLS for public API in multinode envs, update docker-rally

Author: markgoddard

.automation

@@ -32,4 +32,4 @@ max_microversion = 3.70
 build_timeout = 600
 
 [dashboard]
-dashboard_url = http://192.168.39.2
+dashboard_url = https://192.168.39.2

.automation.conf/tempest/tempest-ci-multinode.overrides.conf

@@ -149,6 +149,30 @@ Setup Vault HA on the overcloud hosts
 Certificates generation
 =======================
 
+Create the external TLS certificates (testing only)
+---------------------------------------------------
+
+Typically external API TLS certificates should be generated by a organisation's trusted internal or third-party CA.
+For test and development purposes it is possible to use Vault as a CA for the external API.
+
+1. Run the playbook
+
+   .. code-block::
+
+      kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml
+
+2. Use ansible-vault to encrypt the PEM bundle in $KAYOBE_CONFIG_PATH/kolla/certificates/haproxy.pem. Commit the PEM bundle to the kayobe configuration.
+
+   .. code-block::
+
+      ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/kolla/certificates/haproxy.pem
+
+   Or if environments are being used
+
+   .. code-block::
+
+      ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/haproxy.pem
+
 Create the internal TLS certificates
 ------------------------------------
 
@@ -201,14 +225,23 @@ Certificates deployment
 Enable the required TLS variables in kayobe and kolla
 -----------------------------------------------------
 
-1. Set the following in kayobe-config/etc/kayobe/kolla.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla.yml
+1. If using Vault as a CA for the external API, set the following in kayobe-config/etc/kayobe/kolla.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla.yml
+
+   .. code-block::
+
+      # Whether TLS is enabled for the external API endpoints. Default is 'no'.
+      kolla_enable_tls_external: yes
+
+   See :ref:`tempest-cacert` for information on adding CA certificates to the trust store when running Tempest.
+
+2. Set the following in kayobe-config/etc/kayobe/kolla.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla.yml
 
    .. code-block::
 
       # Whether TLS is enabled for the internal API endpoints. Default is 'no'.
       kolla_enable_tls_internal: yes
 
-2. Set the following in etc/kayobe/kolla/globals.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/globals.yml
+3. Set the following in etc/kayobe/kolla/globals.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/globals.yml
 
    .. code-block::
 
@@ -225,7 +258,7 @@ Enable the required TLS variables in kayobe and kolla
       # If using RabbitMQ TLS:
       rabbitmq_enable_tls: "yes"
 
-3. Deploy backend and internal TLS
+4. Deploy OpenStack
 
    .. warning::
 

doc/source/configuration/vault.rst

@@ -251,6 +251,25 @@ ever contain one host. The seed is usually used as the tempest runner however
 it is also common to use the Ansible control host or an infrastructure VM. The
 main requirement of the host is that it can reach the OpenStack API.
 
+.. _tempest-cacert:
+
+Tempest CA certificate
+----------------------
+
+If your public OpenStack API uses TLS with a Certificate Authority (CA) that is
+not trusted by the Python CA trust store, it may be necessary to add a CA
+certificate to the trust store in the container that runs Tempest. This can be
+done by defining a ``tempest_cacert`` Ansible variable to a path containing the
+CA certificate. You may wish to use ``kayobe_config_path`` or
+``kayobe_env_config_path`` to be agnostic to the path where kayobe-config is
+mounted within the container. For example:
+
+.. code-block:: yaml
+   :caption: ``etc/kayobe/tempest.yml``
+
+   # Add the Vault CA certificate to the rally container when running tempest.
+   tempest_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt"
+
 Running Tempest
 ===============
 

doc/source/operations/tempest.rst

@@ -0,0 +1,55 @@
+---
+- name: Generate external API certificate (for testing only)
+  hosts: controllers
+  run_once: true
+  vars:
+    vault_api_addr: "https://{{ kolla_internal_fqdn }}:8200"
+    # NOTE: Using the same CA as internal TLS.
+    vault_intermediate_ca_name: "OS-TLS-INT"
+  tasks:
+    - name: Include Vault keys
+      include_vars:
+        file: "{{ kayobe_env_config_path }}/vault/overcloud-vault-keys.json"
+        name: vault_keys
+
+    - name: Issue a certificate for external TLS
+      hashivault_pki_cert_issue:
+        url: "{{ vault_api_addr }}"
+        ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}"
+        token: "{{ vault_keys.root_token }}"
+        mount_point: "{{ vault_intermediate_ca_name }}"
+        role: "{{ overcloud_vault_pki_external_tls_role_name }}"
+        common_name: "{% if kolla_external_fqdn != kolla_external_vip_address %}{{ kolla_external_fqdn }}{% endif %}"
+        extra_params:
+          ip_sans: "{{ kolla_external_vip_address }}"
+      register: external_cert
+
+    - name: Ensure certificates directory exists
+      file:
+        path: "{{ kayobe_env_config_path }}/kolla/certificates"
+        state: directory
+      delegate_to: localhost
+
+    - name: Ensure CA certificates directory exists
+      file:
+        path: "{{ kayobe_env_config_path }}/kolla/certificates/ca"
+        state: directory
+      delegate_to: localhost
+
+    - name: Copy external API PEM bundle
+      no_log: true
+      copy:
+        dest: "{{ kayobe_env_config_path }}/kolla/certificates/haproxy.pem"
+        content: |
+          {{ external_cert.data.certificate }}
+          {{ external_cert.data.issuing_ca }}
+          {{ external_cert.data.private_key }}
+        mode: 0600
+      delegate_to: localhost
+
+    - name: Copy root CA
+      copy:
+        src: "{{ kayobe_env_config_path }}/vault/OS-TLS-ROOT.pem"
+        dest: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt"
+        mode: 0600
+      delegate_to: localhost

etc/kayobe/ansible/vault-generate-test-external-tls.yml

@@ -8,12 +8,16 @@ kolla_enable_designate: true
 kolla_enable_redis: true
 kolla_enable_barbican: true
 
-# The multinode environment supports Backend and internal TLS , but it must be
-# enabled in the correct order. See
+# The multinode environment supports backend, external and internal TLS , but
+# it must be enabled in the correct order. See
 # https://stackhpc-kayobe-config.readthedocs.io/en/stackhpc-yoga/configuration/vault.html
 # for details.
+# kolla_enable_tls_external: true
 # kolla_enable_tls_internal: true
 
+kolla_public_openrc_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if os_distribution in ['centos', 'rocky'] else '/etc/ssl/certs/ca-certificates.crt' }}"
+kolla_admin_openrc_cacert: "{{ kolla_public_openrc_cacert }}"
+
 # The multinode environment supports Manila but it is not enabled by default.
 # kolla_enable_manila: true
 # kolla_enable_manila_backend_cephfs_native: true

etc/kayobe/environments/ci-multinode/kolla.yml

@@ -0,0 +1,3 @@
+---
+# Add the Vault CA certificate to the rally container when running tempest.
+tempest_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt"

etc/kayobe/environments/ci-multinode/tempest.yml

@@ -64,6 +64,9 @@ overcloud_vault_pki_internal_tls_role_name: "{{ overcloud_vault_pki_default_role
 # Overcloud Vault PKI Backend TLS Role name
 overcloud_vault_pki_backend_tls_role_name: "{{ overcloud_vault_pki_default_role_name }}"
 
+# Overcloud Vault PKI External TLS Role name (for testing only)
+overcloud_vault_pki_external_tls_role_name: "{{ overcloud_vault_pki_default_role_name }}"
+
 # Overcloud Vault PKI Roles definition
 overcloud_vault_pki_roles:
   - name: "{{ overcloud_vault_pki_default_role_name }}"

etc/kayobe/inventory/group_vars/all/vault

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Supports adding CA certificates to the Tempest container trust store.