Merge pull request #528 from stackhpc/octavia-certs-expiry-monitoring

Add support for checking Octavia cert expiration

Author: MoteHue

ansible/roles/octavia-certificates/defaults/main.yml

@@ -43,3 +43,6 @@ octavia_certs_client_req_organizational_unit: "{{ octavia_certs_organizational_u
 # NOTE(yoctozepto): This should ideally be per controller, i.e. controller
 # generates its key&CSR and this CA signs it.
 octavia_certs_client_req_common_name: client.example.org
+
+# Used with command `kolla-ansible octavia-certificates --check-expiry <days>`.
+octavia_certs_check_expiry: "no"

ansible/roles/octavia-certificates/tasks/check_expiry.yml

@@ -0,0 +1,24 @@
+---
+- name: Gather information on certificates
+  community.crypto.x509_certificate_info:
+    path: "{{ node_custom_config }}/octavia/{{ item }}"
+    valid_at:
+      point_1: "+{{ octavia_certs_expiry_limit | int }}d"
+  register: cert_info
+  delegate_to: localhost
+  with_items:
+    - "server_ca.cert.pem"
+    - "client_ca.cert.pem"
+    - "client.cert-and-key.pem"
+
+- name: Check whether certificates are valid within {{ octavia_certs_expiry_limit }} days
+  assert:
+    that:
+      - item.valid_at.point_1
+    fail_msg: "{{ item.item }} will expire within {{ octavia_certs_expiry_limit }} days, on {{ item.not_after }}"
+    success_msg: "{{ item.item }} will not expire within {{ octavia_certs_expiry_limit }} days. It expires on {{ item.not_after }}"
+    quiet: True
+  loop: "{{ cert_info.results }}"
+  loop_control:
+    label: "{{ item.item }}"
+  delegate_to: localhost

ansible/roles/octavia-certificates/tasks/main.yml

@@ -7,38 +7,45 @@
 # Kolla Ansible prepares and controls the Client CA certificate and key.
 # Client CA is used to generate certificates for Octavia controllers.
 
-- name: Ensure server_ca and client_ca directories exist
-  file:
-    path: "{{ octavia_certs_work_dir }}/{{ item }}"
-    state: "directory"
-    mode: 0770
-  loop:
-    - server_ca
-    - client_ca
-
-- name: Copy openssl.cnf
-  copy:
-    src: "{{ octavia_certs_openssl_cnf_path }}"
-    dest: "{{ octavia_certs_work_dir }}/openssl.cnf"
-
-- import_tasks: server_ca.yml
-
-- import_tasks: client_ca.yml
-
-- import_tasks: client_cert.yml
-
-- name: Ensure {{ node_custom_config }}/octavia directory exists
-  file:
-    path: "{{ node_custom_config }}/octavia"
-    state: "directory"
-    mode: 0770
-
-- name: Copy the to-be-deployed keys and certs to {{ node_custom_config }}/octavia
-  copy:
-    src: "{{ octavia_certs_work_dir }}/{{ item.src }}"
-    dest: "{{ node_custom_config }}/octavia/{{ item.dest }}"
-  with_items:
-    - { src: "server_ca/server_ca.cert.pem", dest: "server_ca.cert.pem" }
-    - { src: "server_ca/server_ca.key.pem", dest: "server_ca.key.pem" }
-    - { src: "client_ca/client_ca.cert.pem", dest: "client_ca.cert.pem" }
-    - { src: "client_ca/client.cert-and-key.pem", dest: "client.cert-and-key.pem" }
+- name: Check if any certificates are going to expire
+  include_tasks: check_expiry.yml
+  when: octavia_certs_check_expiry | bool
+
+- block:
+    - name: Ensure server_ca and client_ca directories exist
+      file:
+        path: "{{ octavia_certs_work_dir }}/{{ item }}"
+        state: "directory"
+        mode: 0770
+      loop:
+        - server_ca
+        - client_ca
+
+    - name: Copy openssl.cnf
+      copy:
+        src: "{{ octavia_certs_openssl_cnf_path }}"
+        dest: "{{ octavia_certs_work_dir }}/openssl.cnf"
+
+    - import_tasks: server_ca.yml
+
+    - import_tasks: client_ca.yml
+
+    - import_tasks: client_cert.yml
+
+    - name: Ensure {{ node_custom_config }}/octavia directory exists
+      file:
+        path: "{{ node_custom_config }}/octavia"
+        state: "directory"
+        mode: 0770
+
+    - name: Copy the to-be-deployed keys and certs to {{ node_custom_config }}/octavia
+      copy:
+        src: "{{ octavia_certs_work_dir }}/{{ item.src }}"
+        dest: "{{ node_custom_config }}/octavia/{{ item.dest }}"
+      with_items:
+        - { src: "server_ca/server_ca.cert.pem", dest: "server_ca.cert.pem" }
+        - { src: "server_ca/server_ca.key.pem", dest: "server_ca.key.pem" }
+        - { src: "client_ca/client_ca.cert.pem", dest: "client_ca.cert.pem" }
+        - { src: "client_ca/client.cert-and-key.pem", dest: "client.cert-and-key.pem" }
+
+  when: not octavia_certs_check_expiry | bool

doc/source/reference/networking/octavia.rst

@@ -75,6 +75,16 @@ used to encrypt the CA key:
 
 .. _octavia-network:
 
+Monitoring certificate expiry
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can use the following command to check if any of the certificates will
+expire within a given number of days:
+
+.. code-block:: console
+
+   kolla-ansible octavia-certificates --check-expiry <days>
+
 Networking
 ----------
 

releasenotes/notes/octavia-check-certificate-expiry-9a80a68cf31cbba4.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    The flag ``--check-expiry`` has been added to the ``octavia-certificates``
+    command. ``kolla-ansible octavia-certificates --check-expiry <days>`` will
+    check if the Octavia certificates are set to expire within a given number
+    of days.

tests/run.yml

@@ -518,6 +518,8 @@
             executable: /bin/bash
             chdir: "{{ kolla_ansible_src_dir }}"
           when: scenario == "octavia"
+          environment:
+            KOLLA_ANSIBLE_VENV_PATH: "{{ kolla_ansible_venv_path }}"
 
         - name: Run test-masakari.sh script
           script:

tests/test-octavia.sh

@@ -8,6 +8,12 @@ set -o errexit
 # Enable unbuffered output for Ansible in Jenkins.
 export PYTHONUNBUFFERED=1
 
+function check_certificate_expiry {
+    RAW_INVENTORY=/etc/kolla/inventory
+    source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
+    kolla-ansible octavia-certificates --check-expiry 7
+    deactivate
+}
 
 function register_amphora_image {
     amphora_url=https://tarballs.opendev.org/openstack/octavia/test-images/test-only-amphora-x64-haproxy-ubuntu-focal.qcow2
@@ -79,6 +85,9 @@ function test_octavia {
 }
 
 function test_octavia_logged {
+    # Check if any certs expire within a week.
+    check_certificate_expiry
+
     . /etc/kolla/admin-openrc.sh
     . ~/openstackclient-venv/bin/activate
     test_octavia

tools/kolla-ansible

@@ -198,6 +198,7 @@ Commands:
     stop                 Stop Kolla containers
     certificates         Generate self-signed certificate for TLS *For Development Only*
     octavia-certificates Generate certificates for octavia deployment
+                             --check-expiry <days> to check if certificates expire within that many days
     upgrade              Upgrades existing OpenStack Environment
     upgrade-bifrost      Upgrades an existing bifrost container
     genconfig            Generate configuration files for enabled OpenStack services
@@ -265,7 +266,7 @@ function version {
 check_environment_coherence
 
 SHORT_OPTS="hi:p:t:k:e:CD:v"
-LONG_OPTS="help,version,inventory:,playbook:,skip-tags:,tags:,key:,extra:,check,diff,verbose,configdir:,passwords:,limit:,forks:,vault-id:,ask-vault-pass,vault-password-file:,yes-i-really-really-mean-it,include-images,include-dev:,full,incremental"
+LONG_OPTS="help,version,inventory:,playbook:,skip-tags:,tags:,key:,extra:,check,diff,verbose,configdir:,passwords:,limit:,forks:,vault-id:,ask-vault-pass,vault-password-file:,yes-i-really-really-mean-it,include-images,include-dev:,full,incremental,check-expiry:"
 
 RAW_ARGS="$*"
 ARGS=$(getopt -o "${SHORT_OPTS}" -l "${LONG_OPTS}" --name "$0" -- "$@") || { usage >&2; exit 2; }
@@ -283,6 +284,7 @@ DANGER_CONFIRM=
 INCLUDE_IMAGES=
 INCLUDE_DEV=
 BACKUP_TYPE="full"
+OCTAVIA_CERTS_EXPIRY=
 # Serial is not recommended and disabled by default. Users can enable it by
 # configuring ANSIBLE_SERIAL variable.
 ANSIBLE_SERIAL=${ANSIBLE_SERIAL:-0}
@@ -400,6 +402,11 @@ while [ "$#" -gt 0 ]; do
             shift 1
             ;;
 
+    (--check-expiry)
+            OCTAVIA_CERTS_EXPIRY="$2"
+            shift 2
+            ;;
+
     (--version)
             version
             exit 0
@@ -534,6 +541,9 @@ EOF
 (octavia-certificates)
         ACTION="Generate octavia Certificates"
         PLAYBOOK="${BASEDIR}/ansible/octavia-certificates.yml"
+        if [[ ! -z "${OCTAVIA_CERTS_EXPIRY}" ]]; then
+            EXTRA_OPTS="$EXTRA_OPTS -e octavia_certs_check_expiry=yes -e octavia_certs_expiry_limit=${OCTAVIA_CERTS_EXPIRY}"
+        fi
         ;;
 (genconfig)
         ACTION="Generate configuration files for enabled OpenStack services"