From e31772e50dc06bdf9ca70a6a28cc08420d3f0e0d Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 13 May 2025 18:29:21 +0100 Subject: [PATCH 1/3] feat: add support for `Raft` HA in `OpenBao` Add support for deploying `OpenBao` with Raft storage backend configured in a high available manner. --- roles/openbao/README.md | 3 ++- roles/openbao/defaults/main.yml | 43 ++++++++++++++++++--------------- tests/test_openbao.yml | 3 ++- 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/roles/openbao/README.md b/roles/openbao/README.md index 01f07dc..994f531 100644 --- a/roles/openbao/README.md +++ b/roles/openbao/README.md @@ -23,7 +23,7 @@ Role variables * `openbao_cluster_name`: OpenBao cluster name (e.g. "prod_cluster") * `openbao_config_dir`: Directory into which to bind mount OpenBao configuration * Optional - * `openbao_bind_address`: Which IP address should OpenBao bind to (default: "127.0.0.1") + * `openbao_bind_addr`: Which IP address should OpenBao bind to (default: "127.0.0.1") * `openbao_api_addr`: OpenBao [API addr](https://openbao.org/docs/configuration/#high-availability-parameters) - Full URL including protocol and port (default: "http://127.0.0.1:8200") * `openbao_init_addr`: OpenBao init addr (used only for initialisation purposes) - full URL including protocol and port (default: "http://127.0.0.1:8200") * `openbao_docker_name`: Docker - under which name to run the OpenBao image (default: "bao") @@ -38,6 +38,7 @@ Role variables * `openbao_write_keys_file`: Whether to write the root token and unseal keys to a file. Default `false` * `openbao_write_keys_file_host`: Host on which to write root token and unseal keys. Default `localhost` * `openbao_write_keys_file_path`: Path of file to write root token and unseal keys. Default `bao-keys.json` + * `openbao_raft_leaders`: List of IPs belonging to Raft leaders. Expected that the first and only entry is the IP address of the first OpenBao instance as this would be initialised whereas as the others will not. Root and unseal keys -------------------- diff --git a/roles/openbao/defaults/main.yml b/roles/openbao/defaults/main.yml index 869167e..b63f948 100644 --- a/roles/openbao/defaults/main.yml +++ b/roles/openbao/defaults/main.yml @@ -7,27 +7,34 @@ openbao_docker_name: "openbao" openbao_docker_image: "openbao/openbao" openbao_docker_tag: "latest" +openbao_config_dir: "" + openbao_cluster_name: "" -openbao_protocol: "{{ 'https' if openbao_tls_key and openbao_tls_cert else 'http' }}" -# Allow openbao_vip_url and openbao_vip_address for backwards compatibility. -openbao_vip_address: "{{ openbao_vip_url | default(openbao_bind_address) }}" -openbao_api_addr: "{{ openbao_protocol ~ '://' ~ openbao_vip_address ~ ':8200' }}" -openbao_bind_address: "127.0.0.1" -openbao_init_addr: "http://127.0.0.1:8200" + openbao_tls_key: "" openbao_tls_cert: "" -openbao_config_dir: "" +openbao_protocol: "{{ 'https' if openbao_tls_key and openbao_tls_cert else 'http' }}" + +openbao_api_addr: "{{ openbao_bind_addr ~ ':' ~ openbao_api_port }}" +openbao_bind_addr: "127.0.0.1" +openbao_init_addr: "{{ openbao_api_addr }}" +openbao_cluster_addr: "{{ openbao_bind_addr ~ ':' ~ openbao_cluster_port }}" + +openbao_api_port: 8200 +openbao_cluster_port: 8201 + +openbao_raft_leaders: [] openbao_config: > { "cluster_name": "{{ openbao_cluster_name }}", "ui": false, "api_addr": "{{ openbao_api_addr }}", - "cluster_addr": "http://127.0.0.1:8201", + "cluster_addr": "{{ openbao_protocol }}://{{ openbao_cluster_addr }}", "listener": [{ "tcp": { - "address": "{{ openbao_bind_address }}:8200", + "address": "{{ openbao_bind_addr }}:{{ openbao_api_port }}", {% if openbao_tls_key and openbao_tls_cert %} "tls_min_version": "tls12", "tls_key_file": "/openbao/config/{{ openbao_tls_key }}", @@ -35,19 +42,17 @@ openbao_config: > {% else %} "tls_disable": "true" {% endif %} - }{% if openbao_bind_address != '127.0.0.1' %}, - }, - { - "tcp": { - "address": "127.0.0.1:8200", - "tls_disable": "true" } - {% endif %} }], "storage": { - "raft": { - "node_id": "raft_{{ ansible_facts.nodename }}", - "path": "/openbao/file" + "raft": { + "node_id": "raft_{{ ansible_facts.nodename }}", + "path": "/openbao/file", + {% if openbao_raft_leaders | length > 0 %} + "retry_join": { + "leader_api_addr": "{{ openbao_protocol }}://{{ openbao_raft_leaders | first }}:{{ openbao_api_port }}" + } + {% endif %} } }, "telemetry": { diff --git a/tests/test_openbao.yml b/tests/test_openbao.yml index b92fb99..af9eae9 100644 --- a/tests/test_openbao.yml +++ b/tests/test_openbao.yml @@ -5,7 +5,8 @@ vars: openbao_config_dir: "/etc/openbao" openbao_log_keys: true - openbao_api_addr: "{{ 'http' ~ '://' ~ '127.0.0.1' ~ ':8200' }}" + openbao_bind_addr: "127.0.0.1" + openbao_api_addr: "{{ 'http' ~ '://' ~ openbao_bind_addr ~ ':8200' }}" openbao_set_keys_fact: true openbao_write_keys_file: true tasks: From 8fcc54f35fc76868dfc7f1231b377e7c3ae160be Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 13 May 2025 19:06:35 +0100 Subject: [PATCH 2/3] feat: add test for `OpenBao` high availability --- .ansible-lint | 1 + .github/workflows/pull_request.yml | 1 + roles/openbao/defaults/main.yml | 2 +- tests/inventory | 5 + tests/test_openbao_ha.yml | 179 +++++++++++++++++++++++++++++ 5 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 tests/test_openbao_ha.yml diff --git a/.ansible-lint b/.ansible-lint index 73f5de1..1e3d5ec 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -8,3 +8,4 @@ skip_list: - meta-no-info warn_list: - yaml[line-length] + - run-once[task] diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 1f6a374..2afd358 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -24,6 +24,7 @@ jobs: ansible_version: "2.18" type: - openbao + - openbao_ha - vault steps: - name: Github Checkout 🛎 diff --git a/roles/openbao/defaults/main.yml b/roles/openbao/defaults/main.yml index b63f948..1c0a944 100644 --- a/roles/openbao/defaults/main.yml +++ b/roles/openbao/defaults/main.yml @@ -46,7 +46,7 @@ openbao_config: > }], "storage": { "raft": { - "node_id": "raft_{{ ansible_facts.nodename }}", + "node_id": "raft_{{ inventory_hostname }}", "path": "/openbao/file", {% if openbao_raft_leaders | length > 0 %} "retry_join": { diff --git a/tests/inventory b/tests/inventory index 15748a7..33753ba 100644 --- a/tests/inventory +++ b/tests/inventory @@ -3,3 +3,8 @@ localhost ansible_connection=local [openbao] localhost ansible_connection=local + +[openbao_ha] +raft_01 ansible_connection=local openbao_bind_addr=127.0.0.1 openbao_docker_name=bao_01 openbao_config_dir=/etc/bao_01 +raft_02 ansible_connection=local openbao_bind_addr=127.0.0.2 openbao_docker_name=bao_02 openbao_config_dir=/etc/bao_02 +raft_03 ansible_connection=local openbao_bind_addr=127.0.0.3 openbao_docker_name=bao_03 openbao_config_dir=/etc/bao_03 diff --git a/tests/test_openbao_ha.yml b/tests/test_openbao_ha.yml new file mode 100644 index 0000000..23261d0 --- /dev/null +++ b/tests/test_openbao_ha.yml @@ -0,0 +1,179 @@ +--- +- name: Deploy HA OpenBao + gather_facts: true + hosts: openbao_ha + vars: + openbao_log_keys: true + openbao_api_addr: "{{ 'http' ~ '://' ~ openbao_bind_addr ~ ':8200' }}" + openbao_set_keys_fact: true + openbao_write_keys_file: true + openbao_raft_leaders: + - "127.0.0.1" + _openbao_default_volumes: + - "{{ openbao_config_dir }}/config:/openbao/config" + - "{{ openbao_config_dir }}/openbao_file:/openbao/file" + - "{{ openbao_config_dir }}/openbao_logs:/openbao/logs" + tasks: + - name: Debug + ansible.builtin.debug: + var: openbao_api_addr + + - name: Ensure /etc/openbao exists + ansible.builtin.file: + path: /etc/openbao + state: directory + mode: "0700" + become: true + + - name: Include openbao role + ansible.builtin.include_role: + name: openbao + + - name: Include openbao role (idemoptence test) + ansible.builtin.include_role: + name: openbao + + # As this test is evaluating OpenBao configured for high availability backed + # by `Raft` we must first ensure that the primary or leader instance is unsealed + # before attempting to unseal the other members. + - name: Unseal vault + ansible.builtin.include_role: + name: vault_unseal + vars: + vault_api_addr: "{{ openbao_api_addr }}" + vault_unseal_keys: "{{ openbao_keys.keys_base64 }}" + run_once: true + + # As the first instance is now unsealed the other instances will now need some + # time to connect before we can proceed. + - name: Wait for OpenBao Raft peers to connect + ansible.builtin.wait_for: + timeout: 30 + delegate_to: localhost + + # Raft peers take few seconds before they report an unsealed state therefore + # we must wait. + - name: Unseal vault + ansible.builtin.include_role: + name: vault_unseal + vars: + vault_api_addr: "{{ openbao_api_addr }}" + vault_unseal_keys: "{{ openbao_keys.keys_base64 }}" + vault_unseal_timeout: 10 + +- name: Deploy HA OpenBao + gather_facts: true + hosts: openbao_ha + run_once: true + vars: + openbao_log_keys: true + openbao_api_addr: "{{ 'http' ~ '://' ~ openbao_bind_addr ~ ':8200' }}" + openbao_set_keys_fact: true + openbao_write_keys_file: true + tasks: + - name: Include OpenBao keys + ansible.builtin.include_vars: + file: "bao-keys.json" + name: openbao_keys + + - name: Configure PKI - create root/intermediate and generate certificates + vars: + vault_pki_certificate_subject: + - role: 'ServerCert' + common_name: "OS-CERT-TEST" + extra_params: + ttl: "8760h" + ip_sans: "127.0.0.1" + alt_names: "example.com" + exclude_cn_from_sans: true + vault_pki_certificates_directory: "/tmp/" + vault_pki_generate_certificates: true + vault_pki_intermediate_ca_name: "OS-TLS-INT" + vault_pki_intermediate_create: true + vault_pki_intermediate_roles: + - name: "ServerCert" + config: + max_ttl: 8760h + ttl: 8760h + allow_any_name: true + allow_ip_sans: true + require_cn: false + server_flag: true + key_type: rsa + key_bits: 4096 + country: ["UK"] + locality: ["Bristol"] + organization: ["StackHPC"] + ou: ["HPC"] + vault_pki_root_ca_name: "OS-TLS-ROOT" + vault_pki_root_create: true + vault_pki_write_certificate_files: true + vault_pki_write_int_ca_to_file: true + vault_pki_write_pem_bundle: false + vault_pki_write_root_ca_to_file: true + vault_api_addr: "{{ openbao_api_addr }}" + vault_token: "{{ openbao_keys.root_token }}" + block: + - name: Configure PKI - create root/intermediate and generate certificates + ansible.builtin.include_role: + name: vault_pki + + - name: Configure PKI - create root/intermediate and generate certificates (idempotence test) + ansible.builtin.include_role: + name: vault_pki + + - name: Configure PKI - generate certificate pem bundle + vars: + vault_pki_certificate_subject: + - role: 'ServerCert' + common_name: "OS-CERT-TEST2" + extra_params: + ttl: "8760h" + ip_sans: "192.168.38.72" + exclude_cn_from_sans: true + vault_pki_certificates_directory: "/tmp/" + vault_pki_generate_certificates: true + vault_pki_intermediate_ca_name: "OS-TLS-INT" + vault_pki_intermediate_create: false + vault_pki_root_ca_name: "OS-TLS-ROOT" + vault_pki_root_create: false + vault_pki_write_certificate_files: true + vault_pki_write_pem_bundle: true + vault_api_addr: "{{ openbao_api_addr }}" + vault_token: "{{ openbao_keys.root_token }}" + block: + - name: Configure PKI - generate certificate pem bundle + ansible.builtin.include_role: + name: vault_pki + + - name: Configure PKI - generate certificate pem bundle (idempotence test) + ansible.builtin.include_role: + name: vault_pki + + - name: Validate if certificates exist + ansible.builtin.stat: + path: "/tmp/{{ item }}" + register: stat_result + failed_when: not stat_result.stat.exists + loop: + - OS-CERT-TEST.crt + - OS-CERT-TEST2.pem + + - name: Concatenate CAs + ansible.builtin.shell: | + cat /tmp/OS-TLS-ROOT.pem /tmp/OS-TLS-INT.crt > /tmp/CA-CHAIN.pem + args: + executable: /bin/bash + become: true + changed_when: true + + - name: Verify certificate chain + ansible.builtin.command: | + openssl verify -CAfile /tmp/CA-CHAIN.pem + /tmp/{{ item }} + register: verify_result + failed_when: verify_result.rc != 0 + loop: + - OS-CERT-TEST.crt + - OS-CERT-TEST2.pem + changed_when: false From 0b3a591a3f7a8ad34c52dcecbea55f5ec10632f4 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 14 May 2025 12:13:39 +0100 Subject: [PATCH 3/3] fix: `idempotence` typo in tests --- tests/test_openbao.yml | 2 +- tests/test_openbao_ha.yml | 2 +- tests/test_vault.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_openbao.yml b/tests/test_openbao.yml index af9eae9..813018b 100644 --- a/tests/test_openbao.yml +++ b/tests/test_openbao.yml @@ -25,7 +25,7 @@ ansible.builtin.include_role: name: openbao - - name: Include openbao role (idemoptence test) + - name: Include openbao role (idempotence test) ansible.builtin.include_role: name: openbao diff --git a/tests/test_openbao_ha.yml b/tests/test_openbao_ha.yml index 23261d0..008e295 100644 --- a/tests/test_openbao_ha.yml +++ b/tests/test_openbao_ha.yml @@ -29,7 +29,7 @@ ansible.builtin.include_role: name: openbao - - name: Include openbao role (idemoptence test) + - name: Include openbao role (idempotence test) ansible.builtin.include_role: name: openbao diff --git a/tests/test_vault.yml b/tests/test_vault.yml index a25c66d..c40b2b7 100644 --- a/tests/test_vault.yml +++ b/tests/test_vault.yml @@ -20,7 +20,7 @@ include_role: name: vault - - name: Include vault role (idemoptence test) + - name: Include vault role (idempotence test) include_role: name: vault