Merge pull request #1694 from stackhpc/openbao-ha-raft

Add support for OpenBao with HA Raft

Author: bbezak

doc/source/configuration/monitoring.rst

@@ -169,12 +169,18 @@ for the exporter.
 If you are deploying in a cloud with internal TLS, you may be required
 to provide a CA certificate for the OpenStack Capacity exporter if your
 certificate is not signed by a trusted CA. For example, to use a CA certificate
-named ``vault.crt`` that is also added to the Kolla containers:
+named ``vault.crt`` or ``openbao.crt`` that is also added to the Kolla containers:
 
 .. code-block:: yaml
 
     stackhpc_os_capacity_openstack_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt"
 
+or
+
+.. code-block:: yaml
+
+    stackhpc_os_capacity_openstack_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/openbao.crt"
+
 Alternatively, to disable certificate verification for the OpenStack Capacity
 exporter:
 

etc/kayobe/ansible/openbao-deploy-overcloud.yml

@@ -21,7 +21,12 @@
   gather_facts: true
   hosts: controllers
   vars:
-    openbao_bind_address: "{{ internal_net_name | net_ip }}"
+    openbao_bind_addr: "{{ internal_net_name | net_ip }}"
+    # This is the IP address of the first controller and therefore the leader within
+    # OpenBao. This could be replaced with the VIP address of the internal network if
+    # HAProxy has been configured to load balance the OpenBao API.
+    openbao_raft_leaders:
+      - "{{ internal_net_name | net_ip(inventory_hostname=groups['controllers'][0]) }}"
   tasks:
     - name: Set a fact about the virtualenv on the remote system
       ansible.builtin.set_fact:
@@ -46,7 +51,7 @@
 
     - name: Template out TLS key and cert
       ansible.builtin.copy:
-        # Within the OpenBao container these uids & gids map to the vault user
+        # Within the OpenBao container these uids & gids map to the openbao user
         src: "{{ kayobe_env_config_path }}/openbao/{{ item }}"
         dest: /opt/kayobe/openbao/{{ item }}
         owner: 100
@@ -55,6 +60,7 @@
       loop:
         - "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.crt"
         - "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.key"
+        - "OS-TLS-INT.crt"
       become: true
 
     - name: Apply OpenBao role
@@ -71,6 +77,7 @@
         openbao_docker_tag: "{{ overcloud_openbao_docker_tag }}"
         openbao_tls_cert: "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.crt"
         openbao_tls_key: "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.key"
+        openbao_tls_ca: "OS-TLS-INT.crt"
         copy_self_signed_ca: true
         openbao_api_addr: https://{{ internal_net_name | net_ip }}:8200
         openbao_write_keys_file: true
@@ -91,6 +98,28 @@
         vault_unseal_keys: "{{ openbao_keys.keys_base64 }}"
       environment:
         https_proxy: ""
+      run_once: true
+
+      # As the first instance is now unsealed the other instances will now need some
+      # time to connect before we can proceed.
+    - name: Wait for OpenBao Raft peers to connect
+      ansible.builtin.wait_for:
+        timeout: 30
+      delegate_to: localhost
+
+      # Raft peers take few seconds before they report an unsealed state therefore
+      # we must wait.
+    - name: Unseal OpenBao
+      ansible.builtin.import_role:
+        name: stackhpc.hashicorp.vault_unseal
+      vars:
+        vault_api_addr: https://{{ internal_net_name | net_ip }}:8200
+        vault_unseal_token: "{{ openbao_keys.root_token }}"
+        vault_unseal_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}"
+        vault_unseal_keys: "{{ openbao_keys.keys_base64 }}"
+        vault_unseal_timeout: 10
+      environment:
+        https_proxy: ""
 
 - name: Configure PKI
   any_errors_fatal: true

etc/kayobe/ansible/openbao-deploy-seed.yml

@@ -4,8 +4,8 @@
   gather_facts: true
   hosts: seed
   vars:
-    openbao_bind_address: "{{ ansible_facts['lo'].ipv4.address }}"
-    openbao_api_addr: "http://{{ openbao_bind_address }}:8200"
+    openbao_bind_addr: "{{ ansible_facts['lo'].ipv4.address }}"
+    openbao_api_addr: "http://{{ openbao_bind_addr }}:8200"
   tasks:
     - name: Set a fact about the virtualenv on the remote system
       ansible.builtin.set_fact:

etc/kayobe/ansible/requirements.yml

@@ -9,7 +9,7 @@ collections:
   - name: stackhpc.pulp
     version: 0.5.5
   - name: stackhpc.hashicorp
-    version: 2.6.1
+    version: 2.7.1
   - name: stackhpc.kayobe_workflows
     version: 1.1.0
 roles:

etc/kayobe/environments/ci-multinode/stackhpc-monitoring.yml

@@ -1,3 +1,3 @@
 ---
 # Path to a CA certificate file to trust in the OpenStack Capacity exporter.
-stackhpc_os_capacity_openstack_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt"
+stackhpc_os_capacity_openstack_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/openbao.crt"

etc/kayobe/environments/ci-multinode/tempest.yml

@@ -3,4 +3,4 @@
 rally_no_sensitive_log: false
 
 # Add the Vault CA certificate to the rally container when running tempest.
-tempest_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt"
+tempest_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/openbao.crt"

etc/kayobe/inventory/group_vars/all/openbao.yml

@@ -77,3 +77,6 @@ seed_openbao_pki_certificate_subject:
     role: "{{ seed_openbao_pki_role_name }}"
     extra_params:
       ip_sans: "{% for host in groups['controllers'] %}{{ internal_net_name | net_ip(host) }}{% if not loop.last %},{% endif %}{% endfor %},{{ kolla_internal_vip_address }}"
+
+# Enable OpenBao UI
+openbao_enable_ui: true

releasenotes/notes/add-openbao-raft-ha-e8d78ffe68913512.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Add support for highly available Raft when using OpenBao on
+    overcloud hosts.