Merge pull request #819 from wireapp/move-repmgr-secret-to-k8s

Move secret configuration for postgresql cluster to a seperate playbook

Author: sghosh23

ansible/inventory/offline/99-static

@@ -77,15 +77,13 @@
 # cassandra_network_interface = enp1s0
 # setting either cassandra backup directive to 'True' below requires a valid s3 bucket name as well
 # also, enabling backups will install `awscli` via pip, which requires an internet connection
-# cassandra_backup_enabled = False 
+# cassandra_backup_enabled = False
 # cassandra_incremental_backup_enabled = False
 # cassandra_backup_s3_bucket = <bucketname>
 
 [postgresql:vars]
 postgresql_network_interface = enp1s0
-wire_dbname = wire-server
-wire_user = wire-server
-wire_namespace = default  # Kubernetes namespace for secret storage
+
 
 [elasticsearch:vars]
 # elasticsearch_network_interface = enp1s0

ansible/inventory/offline/group_vars/postgresql/postgresql.yml

@@ -3,10 +3,25 @@ postgresql_version: 17
 postgresql_data_dir: /var/lib/postgresql/{{ postgresql_version }}/main
 postgresql_conf_dir: /etc/postgresql/{{ postgresql_version }}/main
 
+# wire-server database configuration
+wire_dbname: wire-server
+wire_user: wire-server
+wire_namespace: default  # Kubernetes namespace for secret storage
+
 # repmgr HA configuration
 repmgr_user: repmgr
-repmgr_password: "securepassword"
 repmgr_database: repmgr
+# Note: repmgr_password is NOT defined here - it's dynamically set by postgresql-secrets.yml
+#       The password is fetched from K8s secret or auto-generated during deployment.
+#       To manually pre-create: kubectl create secret generic repmgr-postgresql-secret -n <namespace> --from-literal=password=<your-password>
+
+# Kubernetes Secret configuration for repmgr
+repmgr_secret_name: "repmgr-postgresql-secret"
+repmgr_namespace: "{{ wire_namespace | default('default') }}"
+
+# Kubernetes Secret configuration for wire-server PostgreSQL user
+# This is managed alongside repmgr credentials in postgresql-secrets.yml
+wire_pg_secret_name: "wire-postgresql-external-secret"
 
 # Node configuration for repmgr
 repmgr_node_config:

ansible/postgresql-deploy.yml

@@ -1,41 +1,60 @@
+# ===================================================================
+# PostgreSQL Deployment Pipeline
+# ===================================================================
+# This playbook orchestrates PostgreSQL cluster deployment.
+# Playbooks run in strict order due to dependencies.
+#
+# Dependency Chain:
+#   secrets → primary → replica → verify → wire-setup → monitoring
+#   (cleanup/install are independent but should run first)
+#
+# Tag Strategy:
+#   - Use --skip-tags to exclude specific steps
+#   - Do NOT use --tags to run individual steps (will break dependencies)
+#   - Recommended: Run full playbook or use skip-tags
+#
+# Usage Examples:
+#   Full deployment(Cleans up the previous state if any, sets up HA pg cluster, deploy the guard against split-brain issues and sets up wire-server DB):    ansible-playbook -i <inventory_file> postgresql-deploy.yml
+#   Skip cleanup(cleans up the previous repmgr state and re-initializes the database as new state):       ansible-playbook -i <inventory_file> postgresql-deploy.yml --skip-tags cleanup
+#   Skip monitoring(In general this step should not be skipped as it sets up the guard against split-brain issues):    ansible-playbook -i <inventory_file> postgresql-deploy.yml --skip-tags monitoring
+#   Skip wire-setup (sets up the wire-server database and user):    ansible-playbook -i <inventory_file> postgresql-deploy.yml --skip-tags wire-setup
+#
+#
+# To run from a specific point, comment out earlier playbooks.
+# ===================================================================
+
 - name: Clean previous deployment state
   import_playbook: postgresql-playbooks/clean_existing_setup.yml
-  tags:
-    - postgresql
-    - cleanup
+  tags: [postgresql, cleanup]
 
 - name: Install PostgreSQL packages
   import_playbook: postgresql-playbooks/postgresql-install.yml
-  tags:
-    - postgresql
-    - install
+  tags: [postgresql, install]
+
+- name: Setup PostgreSQL passwords from Kubernetes Secrets
+  import_playbook: postgresql-playbooks/postgresql-secrets.yml
+  tags: [postgresql, secrets]
+  # Sets facts: repmgr_password, wire_pass (required by all following playbooks)
 
 - name: Deploy PostgreSQL primary node
   import_playbook: postgresql-playbooks/postgresql-deploy-primary.yml
-  tags:
-    - postgresql
-    - primary
+  tags: [postgresql, primary]
+  # Requires: repmgr_password
 
 - name: Deploy PostgreSQL replica nodes
   import_playbook: postgresql-playbooks/postgresql-deploy-replica.yml
-  tags:
-    - postgresql
-    - replica
+  tags: [postgresql, replica]
+  # Requires: repmgr_password
 
 - name: Verify PostgreSQL deployment
   import_playbook: postgresql-playbooks/postgresql-verify-HA.yml
-  tags:
-    - postgresql
-    - verify
+  tags: [postgresql, verify]
 
 - name: Setup wire-server postgresql database and user
   import_playbook: postgresql-playbooks/postgresql-wire-setup.yml
-  tags:
-    - postgresql
-    - wire-setup
+  tags: [postgresql, wire-setup]
+  # Requires: wire_pass
 
 - name: Deploy cluster monitoring
   import_playbook: postgresql-playbooks/postgresql-monitoring.yml
-  tags:
-    - postgresql
-    - monitoring
+  tags: [postgresql, monitoring]

ansible/postgresql-playbooks/postgresql-deploy-primary.yml

@@ -9,6 +9,33 @@
     replica_node2: "{{ hostvars[(groups.get('postgresql_ro', []) | last) | default('postgresql3')]['ansible_default_ipv4']['address'] | default(hostvars[(groups.get('postgresql_ro', []) | last) | default('postgresql3')]['ansible_host'] | default((groups.get('postgresql_ro', []) | last) | default('postgresql3'))) }}"
     pg_service_name: "postgresql@{{ postgresql_version }}-main.service"
   tasks:
+    # ===== PREREQUISITE VALIDATION =====
+    - name: Validate required secrets are available
+      ansible.builtin.assert:
+        that:
+          - repmgr_password is defined
+          - repmgr_password | length > 0
+        fail_msg: |
+          ❌ PREREQUISITE FAILED: repmgr_password is not available!
+
+          This playbook requires the repmgr password to be set as an Ansible fact.
+
+          Solution:
+            Run the complete deployment pipeline:
+              ansible-playbook postgresql-deploy.yml
+
+            OR run with the postgresql tag to include secrets:
+              ansible-playbook postgresql-deploy.yml --tags postgresql
+
+            OR run secrets playbook first:
+              ansible-playbook postgresql-playbooks/postgresql-secrets.yml
+              ansible-playbook postgresql-playbooks/postgresql-deploy-primary.yml
+
+          The postgresql-secrets.yml playbook fetches/creates passwords from Kubernetes
+          and sets them as Ansible facts for use by deployment playbooks.
+        success_msg: "✅ Prerequisites validated: repmgr_password is available"
+      run_once: true
+
     - name: Ensure repmgr scripts directory exists
       ansible.builtin.file:
         path: /opt/repmgr/scripts
@@ -103,11 +130,18 @@
           when: repmgr_user_check.stdout.strip() == "0"
           register: create_repmgr_user
 
+        - name: Update repmgr user password (if user already exists)
+          ansible.builtin.shell: |
+            sudo -u postgres psql -c "ALTER USER {{ repmgr_user }} WITH PASSWORD '{{ repmgr_password }}';"
+          when: repmgr_user_check.stdout.strip() != "0"
+          register: update_repmgr_password
+          no_log: true
+
         - name: Display user creation result
           ansible.builtin.debug:
             msg: |
-              repmgr user status: {{ 'CREATED' if repmgr_user_check.stdout.strip() == "0" else 'ALREADY EXISTS' }}
-          when: create_repmgr_user is defined
+              repmgr user status: {{ 'CREATED' if repmgr_user_check.stdout.strip() == "0" else 'ALREADY EXISTS (password updated)' }}
+          when: create_repmgr_user is defined or update_repmgr_password is defined
 
         - name: Check if repmgr database exists
           ansible.builtin.shell: |
@@ -201,13 +235,33 @@
             PGPASSWORD: "{{ repmgr_password }}"
           register: repmgr_connection_test
 
-        - name: Start repmgrd service
+        - name: Determine if repmgrd needs restart
+          ansible.builtin.set_fact:
+            repmgrd_needs_restart: >-
+              {{
+                (update_repmgr_password is defined and update_repmgr_password.changed) or
+                primary_conf_result.changed
+              }}
+
+        - name: Restart repmgrd if password or config changed
+          ansible.builtin.systemd:
+            name: "repmgrd@{{ postgresql_version }}-main"
+            state: restarted
+            enabled: yes
+            daemon_reload: yes
+          when:
+            - repmgr_connection_test is succeeded
+            - repmgrd_needs_restart
+
+        - name: Start repmgrd service (if not restarted)
           ansible.builtin.systemd:
             name: "repmgrd@{{ postgresql_version }}-main"
             state: started
             enabled: yes
             daemon_reload: yes
-          when: repmgr_connection_test is succeeded
+          when:
+            - repmgr_connection_test is succeeded
+            - not repmgrd_needs_restart
 
         - name: Verify repmgrd is running
           ansible.builtin.systemd:

ansible/postgresql-playbooks/postgresql-deploy-replica.yml

@@ -9,6 +9,33 @@
     current_replica: "{{ ansible_default_ipv4.address | default(ansible_host) }}"
     pg_service_name: "postgresql@{{ postgresql_version }}-main.service"
   tasks:
+    # ===== PREREQUISITE VALIDATION =====
+    - name: Validate required secrets are available
+      ansible.builtin.assert:
+        that:
+          - repmgr_password is defined
+          - repmgr_password | length > 0
+        fail_msg: |
+          ❌ PREREQUISITE FAILED: repmgr_password is not available!
+
+          This playbook requires the repmgr password to be set as an Ansible fact.
+
+          Solution:
+            Run the complete deployment pipeline:
+              ansible-playbook postgresql-deploy.yml
+
+            OR run with the postgresql tag to include secrets:
+              ansible-playbook postgresql-deploy.yml --tags postgresql
+
+            OR run secrets playbook first:
+              ansible-playbook postgresql-playbooks/postgresql-secrets.yml
+              ansible-playbook postgresql-playbooks/postgresql-deploy-replica.yml
+
+          The postgresql-secrets.yml playbook fetches/creates passwords from Kubernetes
+          and sets them as Ansible facts for use by deployment playbooks.
+        success_msg: "✅ Prerequisites validated: repmgr_password is available"
+      run_once: true
+
     # ===== INITIAL STATUS CHECK =====
     - name: Check replica configuration status
       block:
@@ -57,6 +84,7 @@
         group: postgres
         mode: "{{ item.mode }}"
         backup: yes
+      register: replica_conf_result
       loop:
         - src: ../templates/postgresql/pg_hba.conf.j2
           dest: "{{ postgresql_conf_dir }}/pg_hba.conf"
@@ -276,12 +304,28 @@
               sudo -u postgres repmgr node status
           when: (verify_replica_reg.query_result[0].cnt | int) != 1
 
-        - name: Start repmgrd service
+        - name: Determine if repmgrd needs restart
+          ansible.builtin.set_fact:
+            repmgrd_needs_restart: >-
+              {{
+                replica_conf_result.changed
+              }}
+
+        - name: Restart repmgrd if config changed
+          ansible.builtin.systemd:
+            name: "repmgrd@{{ postgresql_version }}-main"
+            state: restarted
+            enabled: yes
+            daemon_reload: yes
+          when: repmgrd_needs_restart
+
+        - name: Start repmgrd service (if not restarted)
           ansible.builtin.systemd:
             name: "repmgrd@{{ postgresql_version }}-main"
             state: started
             enabled: yes
             daemon_reload: yes
+          when: not repmgrd_needs_restart
 
         - name: Verify repmgrd is running
           ansible.builtin.systemd: