From cb7d360639c44d21d8abdf7e3b32d6b254712d33 Mon Sep 17 00:00:00 2001 From: Roman Chudov Date: Sat, 3 May 2025 05:51:00 +0300 Subject: [PATCH 1/7] Ansible roles, keepalived added. --- Makefile | 8 - group_vars/promoters.yml | 13 ++ haproxy.yml | 35 ---- inventory.ini.example | 4 +- pg_observe.yml | 136 ------------ playbook.yml | 198 ++---------------- roles/etcd/tasks/main.yml | 94 +++++++++ .../etcd/templates}/etcd.conf.yml.j2 | 0 .../etcd/templates}/etcd.service.j2 | 0 roles/haproxy/tasks/main.yml | 18 ++ .../haproxy/templates}/haproxy.cfg.j2 | 4 +- roles/keepalived/defaults/main.yml | 4 + roles/keepalived/files/check_haproxy.sh | 12 ++ roles/keepalived/files/chk_patroni_leader.sh | 41 ++++ roles/keepalived/handlers/main.yml | 8 + roles/keepalived/tasks/main.yml | 131 ++++++++++++ roles/keepalived/templates/keepalived.conf.j2 | 68 ++++++ roles/metrics/tasks/main.yml | 119 +++++++++++ .../templates}/node_exporter.service.j2 | 0 .../templates}/postgres_exporter.service.j2 | 0 .../templates}/postgres_exporter.yaml.j2 | 0 roles/patroni/tasks/main.yml | 91 ++++++++ .../patroni/templates}/patroni.service.j2 | 0 .../patroni/templates}/patroni.yml.j2 | 3 - roles/pgbouncer/defaults/main.yml | 9 + roles/pgbouncer/tasks/main.yml | 92 ++++++++ roles/pgbouncer/templates/pgbouncer.ini.j2 | 15 ++ .../pgbouncer/templates/pgbouncer.service.j2 | 17 ++ roles/pgbouncer/templates/userlist.txt.j2 | 1 + 29 files changed, 752 insertions(+), 369 deletions(-) delete mode 100644 haproxy.yml delete mode 100644 pg_observe.yml create mode 100644 roles/etcd/tasks/main.yml rename {templates => roles/etcd/templates}/etcd.conf.yml.j2 (100%) rename {templates => roles/etcd/templates}/etcd.service.j2 (100%) create mode 100644 roles/haproxy/tasks/main.yml rename {templates => roles/haproxy/templates}/haproxy.cfg.j2 (82%) create mode 100644 roles/keepalived/defaults/main.yml create mode 100644 roles/keepalived/files/check_haproxy.sh create mode 100644 roles/keepalived/files/chk_patroni_leader.sh create mode 100644 roles/keepalived/handlers/main.yml create mode 100644 roles/keepalived/tasks/main.yml create mode 100644 roles/keepalived/templates/keepalived.conf.j2 create mode 100644 roles/metrics/tasks/main.yml rename {templates => roles/metrics/templates}/node_exporter.service.j2 (100%) rename {templates => roles/metrics/templates}/postgres_exporter.service.j2 (100%) rename {templates => roles/metrics/templates}/postgres_exporter.yaml.j2 (100%) create mode 100644 roles/patroni/tasks/main.yml rename {templates => roles/patroni/templates}/patroni.service.j2 (100%) rename {templates => roles/patroni/templates}/patroni.yml.j2 (95%) create mode 100644 roles/pgbouncer/defaults/main.yml create mode 100644 roles/pgbouncer/tasks/main.yml create mode 100644 roles/pgbouncer/templates/pgbouncer.ini.j2 create mode 100644 roles/pgbouncer/templates/pgbouncer.service.j2 create mode 100644 roles/pgbouncer/templates/userlist.txt.j2 diff --git a/Makefile b/Makefile index 948e18a..fa4be04 100644 --- a/Makefile +++ b/Makefile @@ -11,14 +11,6 @@ up: init up-with-clean-etcd: init ansible-playbook -i inventory.ini playbook.yml -e "clean_etcd=true" -.PHONY: haproxy -haproxy: init - ansible-playbook -i inventory.ini haproxy.yml - -.PHONY: pg-observe -pg-observe: init - ansible-playbook -i inventory.ini pg_observe.yml - .PHONY: prometheus-grafana prometheus-grafana: init ansible-playbook -i inventory.ini prometheus_grafana.yml diff --git a/group_vars/promoters.yml b/group_vars/promoters.yml index c2ff259..6a8cdfd 100644 --- a/group_vars/promoters.yml +++ b/group_vars/promoters.yml @@ -16,11 +16,14 @@ patroni_retry_timeout: 10 patroni_max_lag: 1048576 patroni_use_pg_rewind: true patroni_use_slots: true +patroni_config: "/etc/patroni.yml" # etcd settings etcd_client_port: 2379 # PostgreSQL settigns +pg_version: 16 +data_dir: "/var/lib/postgresql/{{ pg_version }}/main" postgresql_version: 16 postgresql_encoding: UTF8 postgresql_port: 5432 @@ -49,6 +52,16 @@ postgresql_custom_parameters: # Additional pg_hba.conf rules (optional) #postgresql_additional_pg_hba: +# PgBouncer params +pgbouncer_listen_port: 6432 + +# HAProxy params +haproxy_config: "/etc/haproxy/haproxy.cfg" + +# Cluster variables +cluster_vip_1: "192.168.64.100" +vip_interface: "{{ ansible_default_ipv4.interface }}" # interface name (ex. "ens32") + # Prometheus settings prometheus_retention: "30d" prometheus_storage_path: "/var/lib/prometheus" diff --git a/haproxy.yml b/haproxy.yml deleted file mode 100644 index 602ee0a..0000000 --- a/haproxy.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -- name: Install and configure HAProxy - hosts: haproxy - become: true - vars: - haproxy_config: "/etc/haproxy/haproxy.cfg" - postgres_port: 5432 - patroni_port: 8008 - - tasks: - - name: Install HAProxy - ansible.builtin.apt: - name: haproxy - state: present - update_cache: true - - - name: Create HAProxy configuration - ansible.builtin.template: - src: templates/haproxy.cfg.j2 - dest: "{{ haproxy_config }}" - mode: '0644' - notify: Restart HAProxy - - - name: Enable and start HAProxy - ansible.builtin.systemd: - name: haproxy - state: started - enabled: true - daemon_reload: true - - handlers: - - name: Restart HAProxy - ansible.builtin.systemd: - name: haproxy - state: restarted diff --git a/inventory.ini.example b/inventory.ini.example index 8a7c3fb..218293d 100644 --- a/inventory.ini.example +++ b/inventory.ini.example @@ -3,8 +3,8 @@ patroni1 ansible_host=192.168.64.2 node_id=1 patroni2 ansible_host=192.168.64.3 node_id=2 patroni3 ansible_host=192.168.64.4 node_id=3 -[haproxy] -haproxy1 ansible_host=192.168.64.5 +[keepalived_hosts] +vip_host ansible_host=192.168.64.5 [prometheus_grafana] prometheus_grafana1 ansible_host=192.168.64.7 diff --git a/pg_observe.yml b/pg_observe.yml deleted file mode 100644 index 1820bb2..0000000 --- a/pg_observe.yml +++ /dev/null @@ -1,136 +0,0 @@ -#################################### -## SETUP OBSERVABILITY ## -#################################### ---- -- name: Install observability for PostgreSQL - hosts: promoters - become: true - tasks: - - name: Create node_exporter system user - ansible.builtin.user: - name: node_exporter - system: true - shell: /sbin/nologin - create_home: false - - - name: Download and install node_exporter - block: - - name: Download node_exporter - ansible.builtin.get_url: - url: "https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_version }}/\ - node_exporter-{{ node_exporter_version }}.{{ arch }}.tar.gz" - dest: /tmp/node_exporter.tar.gz - mode: '0644' - - - name: Extract node_exporter - ansible.builtin.unarchive: - src: /tmp/node_exporter.tar.gz - dest: /tmp - remote_src: true - - - name: Copy node_exporter binary - ansible.builtin.copy: - src: "/tmp/node_exporter-{{ node_exporter_version }}.{{ arch }}/node_exporter" - dest: "/usr/local/bin/node_exporter" - mode: '0755' - remote_src: true - - - name: Download and install postgres_exporter - block: - - name: Download postgres_exporter - ansible.builtin.get_url: - url: "https://github.com/prometheus-community/postgres_exporter/releases/download/\ - v{{ postgres_exporter_version }}/postgres_exporter-{{ postgres_exporter_version }}.{{ arch }}.tar.gz" - dest: /tmp/postgres_exporter.tar.gz - mode: '0644' - - - name: Extract postgres_exporter - ansible.builtin.unarchive: - src: /tmp/postgres_exporter.tar.gz - dest: /tmp - remote_src: true - - - name: Copy postgres_exporter binary - ansible.builtin.copy: - src: "/tmp/postgres_exporter-{{ postgres_exporter_version }}.{{ arch }}/postgres_exporter" - dest: "/usr/local/bin/postgres_exporter" - mode: '0755' - remote_src: true - - - name: Create systemd unit for node_exporter - ansible.builtin.template: - src: templates/node_exporter.service.j2 - dest: /etc/systemd/system/node_exporter.service - mode: '0644' - - - name: Ensure postgres_exporter config directory exists - ansible.builtin.file: - path: /etc/postgres_exporter - state: directory - mode: '0755' - owner: postgres - group: postgres - - - name: Create postgres_exporter config - ansible.builtin.template: - src: templates/postgres_exporter.yaml.j2 - dest: /etc/postgres_exporter/postgres_exporter.yaml - mode: '0640' - owner: postgres - group: postgres - - - name: Create systemd unit for postgres_exporter - ansible.builtin.template: - src: templates/postgres_exporter.service.j2 - dest: /etc/systemd/system/postgres_exporter.service - mode: '0644' - - - name: Start and enable exporters - ansible.builtin.systemd: - name: "{{ item }}" - state: started - enabled: true - daemon_reload: true - with_items: - - node_exporter - - postgres_exporter - - - name: Allow Prometheus metrics ports in firewall - community.general.ufw: - rule: allow - port: "{{ item }}" - proto: tcp - with_items: - - "{{ node_exporter_port }}" # node_exporter - - "{{ postgres_exporter_port }}" # postgres_exporter - - - name: Check node_exporter metrics endpoint - ansible.builtin.uri: - url: "http://localhost:{{ node_exporter_port }}/metrics" - status_code: 200 - return_content: true - register: node_exporter_metrics - - - name: Validate node_exporter metrics - ansible.builtin.debug: - msg: "Node exporter metrics collected successfully: {{ node_exporter_metrics.content | regex_search('node_cpu_seconds_total') is not none }}" - - - name: Check postgres_exporter metrics endpoint - ansible.builtin.uri: - url: "http://localhost:{{ postgres_exporter_port }}/metrics" - status_code: 200 - return_content: true - register: postgres_exporter_metrics - - - name: Validate postgres_exporter metrics - ansible.builtin.debug: - msg: "Postgres exporter metrics collected successfully: {{ postgres_exporter_metrics.content | regex_search('pg_database_size') is not none }}" - - handlers: - - name: Restart services - ansible.builtin.systemd: - name: "{{ item }}" - state: restarted - with_items: - - node_exporter - - postgres_exporter diff --git a/playbook.yml b/playbook.yml index 021e012..3b6572c 100644 --- a/playbook.yml +++ b/playbook.yml @@ -1,187 +1,19 @@ --- -- name: Install Patroni cluster +- name: Deploy Patroni cluster hosts: promoters become: true + roles: +# - etcd +# - patroni + - metrics + - pgbouncer + - haproxy + - keepalived + +- name: Deploy Keepalived VIP + hosts: keepalived_vip + become: true vars: - pg_version: 16 - data_dir: "/var/lib/postgresql/{{ pg_version }}/main" - patroni_config: "/etc/patroni.yml" - - tasks: - - name: Install dependencies - ansible.builtin.apt: - name: - - python3-pip - - python3-psycopg2 - - etcd - - wget - - gnupg - update_cache: true - - - name: Adding GPG-key PostgreSQL - ansible.builtin.get_url: - url: https://www.postgresql.org/media/keys/ACCC4CF8.asc - dest: /etc/apt/trusted.gpg.d/postgresql.asc - mode: '0644' - - - name: Adding apt repository PostgreSQL - ansible.builtin.apt_repository: - repo: "deb http://apt.postgresql.org/pub/repos/apt {{ ansible_lsb.codename }}-pgdg main" - state: present - filename: "pgdg" - - - name: Update apt - ansible.builtin.apt: - update_cache: true - - - name: Install PostgreSQL {{ pg_version }} - ansible.builtin.apt: - name: postgresql-{{ pg_version }} - state: present - - - name: Install patroni - ansible.builtin.pip: - name: patroni[etcd] - - - name: Stop PostgreSQL - ansible.builtin.service: - name: postgresql - state: stopped - enabled: false - - - name: Create config Patroni - ansible.builtin.template: - src: patroni.yml.j2 - dest: /etc/patroni.yml - mode: '0644' - - - name: Set owner of data directory PostgreSQL - ansible.builtin.file: - path: "{{ data_dir }}" - state: directory - owner: postgres - group: postgres - mode: '0700' - recurse: true - - - name: Clean previous state of etcd - ansible.builtin.file: - path: /var/lib/etcd - state: absent - when: clean_etcd is defined and clean_etcd - - - name: Create directory for etcd data - ansible.builtin.file: - path: /var/lib/etcd - state: directory - owner: etcd - group: etcd - mode: '0700' - - - name: Stop etcd if running - ansible.builtin.systemd: - name: etcd - state: stopped - failed_when: false - - - name: Generate etcd config - ansible.builtin.template: - src: templates/etcd.conf.yml.j2 - dest: /etc/etcd.yml - mode: '0644' - - - name: Install systemd unit for etcd - ansible.builtin.template: - src: templates/etcd.service.j2 - dest: /etc/systemd/system/etcd.service - mode: '0644' - - - name: Restart etcd - ansible.builtin.systemd: - name: etcd - daemon_reload: true - enabled: true - state: started - - - name: Start etcd on first node - when: inventory_hostname == "patroni1" - block: - - name: Start etcd on first node - ansible.builtin.systemd: - name: etcd - daemon_reload: true - enabled: true - state: started - - - name: Wait for first etcd node will start - ansible.builtin.wait_for: - host: "{{ ansible_host }}" - port: 2379 - timeout: 30 - - - name: Wait for the first node to be ready - when: inventory_hostname != "patroni1" - ansible.builtin.wait_for: - host: "{{ hostvars['patroni1']['ansible_host'] }}" - port: 2379 - timeout: 30 - - - name: Launching other etcd nodes - when: inventory_hostname != "patroni1" - ansible.builtin.systemd: - name: etcd - daemon_reload: true - enabled: true - state: started - - - name: Wait for cluster formation - ansible.builtin.shell: | - timeout 30 bash -c 'until etcdctl --endpoints=http://{{ ansible_host }}:2379 cluster-health; do sleep 5; done' - register: cluster_health - until: cluster_health.rc == 0 - retries: 10 - delay: 5 - # noqa no-changed-when - - - name: Check cluster state - ansible.builtin.shell: | - etcdctl --endpoints=http://{{ ansible_host }}:2379 member list - register: etcd_members - changed_when: false - - - name: Create patroni logs directory - ansible.builtin.file: - path: /var/log/patroni - state: directory - mode: '0755' - - - name: Create systemd unit for Patroni - ansible.builtin.template: - src: templates/patroni.service.j2 - dest: /etc/systemd/system/patroni.service - mode: '0644' - - - name: Create config for patroni logrotate - ansible.builtin.copy: - dest: /etc/logrotate.d/patroni - mode: '0644' - content: | - /var/log/patroni/patroni.log { - daily - rotate 7 - compress - delaycompress - missingok - notifempty - create 0640 postgres postgres - } - - - name: Reload systemd - ansible.builtin.systemd: - daemon_reload: true - - - name: Enable and start Patroni systemd unit - ansible.builtin.systemd: - name: patroni - state: started - enabled: true + add_balancer: true + roles: + - keepalived diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml new file mode 100644 index 0000000..bd03aa4 --- /dev/null +++ b/roles/etcd/tasks/main.yml @@ -0,0 +1,94 @@ +- name: Install etcd via apt + ansible.builtin.apt: + name: + - etcd + update_cache: true + +- name: Update apt + ansible.builtin.apt: + update_cache: true + +- name: Clean previous state of etcd + ansible.builtin.file: + path: /var/lib/etcd + state: absent + when: clean_etcd is defined and clean_etcd + +- name: Create directory for etcd data + ansible.builtin.file: + path: /var/lib/etcd + state: directory + owner: etcd + group: etcd + mode: '0700' + +- name: Stop etcd if running + ansible.builtin.systemd: + name: etcd + state: stopped + failed_when: false + +- name: Generate etcd config + ansible.builtin.template: + src: etcd.conf.yml.j2 + dest: /etc/etcd.yml + mode: '0644' + +- name: Install systemd unit for etcd + ansible.builtin.template: + src: etcd.service.j2 + dest: /etc/systemd/system/etcd.service + mode: '0644' + +- name: Restart etcd + ansible.builtin.systemd: + name: etcd + daemon_reload: true + enabled: true + state: started + +- name: Start etcd on first node + when: inventory_hostname == "patroni1" + block: + - name: Start etcd on first node + ansible.builtin.systemd: + name: etcd + daemon_reload: true + enabled: true + state: started + +- name: Wait for first etcd node will start + ansible.builtin.wait_for: + host: "{{ ansible_host }}" + port: 2379 + timeout: 30 + +- name: Wait for the first node to be ready + when: inventory_hostname != "patroni1" + ansible.builtin.wait_for: + host: "{{ hostvars['patroni1']['ansible_host'] }}" + port: 2379 + timeout: 30 + +- name: Launching other etcd nodes + when: inventory_hostname != "patroni1" + ansible.builtin.systemd: + name: etcd + daemon_reload: true + enabled: true + state: started + +- name: Wait for cluster formation + ansible.builtin.shell: | + timeout 30 bash -c 'until etcdctl --endpoints=http://{{ ansible_host }}:2379 cluster-health; do sleep 5; done' + register: cluster_health + until: cluster_health.rc == 0 + retries: 10 + delay: 5 + # noqa no-changed-when + +- name: Check cluster state + ansible.builtin.shell: | + etcdctl --endpoints=http://{{ ansible_host }}:2379 member list + register: etcd_members + changed_when: false diff --git a/templates/etcd.conf.yml.j2 b/roles/etcd/templates/etcd.conf.yml.j2 similarity index 100% rename from templates/etcd.conf.yml.j2 rename to roles/etcd/templates/etcd.conf.yml.j2 diff --git a/templates/etcd.service.j2 b/roles/etcd/templates/etcd.service.j2 similarity index 100% rename from templates/etcd.service.j2 rename to roles/etcd/templates/etcd.service.j2 diff --git a/roles/haproxy/tasks/main.yml b/roles/haproxy/tasks/main.yml new file mode 100644 index 0000000..16a980b --- /dev/null +++ b/roles/haproxy/tasks/main.yml @@ -0,0 +1,18 @@ +- name: Install HAProxy + ansible.builtin.apt: + name: haproxy + state: present + update_cache: true + +- name: Create HAProxy configuration + ansible.builtin.template: + src: haproxy.cfg.j2 + dest: "{{ haproxy_config }}" + mode: '0644' + +- name: Enable and start HAProxy + ansible.builtin.systemd: + name: haproxy + state: restarted + enabled: true + daemon_reload: true diff --git a/templates/haproxy.cfg.j2 b/roles/haproxy/templates/haproxy.cfg.j2 similarity index 82% rename from templates/haproxy.cfg.j2 rename to roles/haproxy/templates/haproxy.cfg.j2 index 57bee31..d03adf6 100644 --- a/templates/haproxy.cfg.j2 +++ b/roles/haproxy/templates/haproxy.cfg.j2 @@ -41,7 +41,7 @@ backend patroni_master_backend http-check expect status 200 default-server inter 3s fall 3 rise 2 {% for host in groups['promoters'] %} - server {{ host }} {{ hostvars[host]['ansible_host'] }}:{{ postgres_port }} check port {{ patroni_port }} + server {{ host }} {{ hostvars[host]['ansible_host'] }}:{{ pgbouncer_listen_port }} check port {{ patroni_api_port }} {% endfor %} backend patroni_replicas_backend @@ -51,5 +51,5 @@ backend patroni_replicas_backend http-check expect status 200 default-server inter 3s fall 3 rise 2 {% for host in groups['promoters'] %} - server {{ host }} {{ hostvars[host]['ansible_host'] }}:{{ postgres_port }} check port {{ patroni_port }} + server {{ host }} {{ hostvars[host]['ansible_host'] }}:{{ pgbouncer_listen_port }} check port {{ patroni_api_port }} {% endfor %} diff --git a/roles/keepalived/defaults/main.yml b/roles/keepalived/defaults/main.yml new file mode 100644 index 0000000..4488949 --- /dev/null +++ b/roles/keepalived/defaults/main.yml @@ -0,0 +1,4 @@ +vip_interface: enp0s1 +virtual_ip: 192.168.64.100 +keepalived_auth_pass: mypassword +haproxy_stats_port: 7000 diff --git a/roles/keepalived/files/check_haproxy.sh b/roles/keepalived/files/check_haproxy.sh new file mode 100644 index 0000000..eae41e1 --- /dev/null +++ b/roles/keepalived/files/check_haproxy.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +hosts=({% for host in groups['promoters'] %}{{ hostvars[host]['ansible_host'] }} {% endfor %}) + +for host in "${hosts[@]}"; do + curl -sf http://$host:{{ haproxy_stats_port }}/ > /dev/null 2>&1 + if [ $? -eq 0 ]; then + exit 0 + fi +done + +exit 1 diff --git a/roles/keepalived/files/chk_patroni_leader.sh b/roles/keepalived/files/chk_patroni_leader.sh new file mode 100644 index 0000000..535ec77 --- /dev/null +++ b/roles/keepalived/files/chk_patroni_leader.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +if [ ! -f /etc/patroni.yml ]; then + echo "Configuration file /etc/patroni.yml not found!" + exit 1 +fi + +JQ_PATH=$(command -v jq) +if [[ -z "$JQ_PATH" || ! -x "$JQ_PATH" ]]; then + echo "jq is not installed or not executable!" + exit 1 +fi + +if [[ ! -x /usr/local/bin/patronictl ]]; then + echo "patronictl not found or not executable!" + exit 1 +fi + +NODENAME=$(grep -E "^name:" /etc/patroni.yml | cut -d: -f2 | tr -d '[:blank:]') + +if [ -z "$NODENAME" ]; then + echo "Nodename is blank!" + exit 1 +fi + +PATRONICTL_OUT=$(/usr/local/bin/patronictl -c /etc/patroni.yml list --format json 2>/dev/null) + +if [ -z "$PATRONICTL_OUT" ]; then + echo "No patronictl output or command failed!" + exit 1 +fi + +LEADER=$(echo "$PATRONICTL_OUT" | jq --raw-output ".[] | select((.Role == \"Leader\") and (.State == \"running\")) | .Member") + +if [ "$NODENAME" == "$LEADER" ]; then + echo "Is leader!" + exit 0 +else + echo "Is not leader!" + exit 1 +fi \ No newline at end of file diff --git a/roles/keepalived/handlers/main.yml b/roles/keepalived/handlers/main.yml new file mode 100644 index 0000000..d9612bd --- /dev/null +++ b/roles/keepalived/handlers/main.yml @@ -0,0 +1,8 @@ +--- +- name: Restart keepalived + ansible.builtin.systemd_service: + daemon_reload: true + name: keepalived + enabled: true + state: restarted + listen: "Restart keepalived" diff --git a/roles/keepalived/tasks/main.yml b/roles/keepalived/tasks/main.yml new file mode 100644 index 0000000..712c8c1 --- /dev/null +++ b/roles/keepalived/tasks/main.yml @@ -0,0 +1,131 @@ +#--- +#- name: Install keepalived +# apt: +# name: keepalived +# state: present +# update_cache: true +# +#- name: Configure keepalived +# template: +# src: keepalived.conf.j2 +# dest: /etc/keepalived/keepalived.conf +# mode: '0644' +# owner: root +# group: root +# notify: restart keepalived +# +#- name: Make script executable +# file: +# path: /usr/local/bin/check_haproxy.sh +# mode: '0755' +# state: file +# +#- name: Ensure keepalived is enabled and started +# systemd: +# name: keepalived +# state: started +# enabled: true + +--- + +- name: Install jq + ansible.builtin.apt: + name: jq + state: present + +- name: Install keepalived + ansible.builtin.apt: + name: keepalived + state: present + allow_downgrade: true + register: package_status + until: package_status is success + delay: 5 + retries: 3 + environment: "{{ proxy_env | default({}) }}" + +- name: Create directory /opt/etc/keepalived + ansible.builtin.file: + path: /opt/etc/keepalived + state: directory + mode: '0755' + owner: root + group: root + +- name: Create directory /opt/keepalived/scripts + ansible.builtin.file: + path: /opt/keepalived/scripts + state: directory + mode: '0755' + owner: root + group: root + +- name: Create vrrp_script + ansible.builtin.copy: + content: | + #!/bin/bash + /bin/kill -0 `cat /run/haproxy/haproxy.pid` + dest: /opt/keepalived/scripts/haproxy_check.sh + owner: root + group: root + mode: "0700" + notify: "Restart keepalived" + +- name: Generate keepalived conf file + when: add_balancer is not defined or not add_balancer|bool + ansible.builtin.template: + src: templates/keepalived.conf.j2 + dest: /etc/keepalived/keepalived.conf + mode: "0644" + notify: "Restart keepalived" + +- name: Generate check scripts for keepalived + become: true + with_items: + - f: files/chk_patroni_leader.sh + d: /opt/keepalived/scripts + ansible.builtin.template: + src: '{{ item.f }}' + dest: '{{ item.d }}' + mode: '0755' + owner: root + group: root + +- name: Block for tasks with balancer + when: add_balancer is defined and add_balancer|bool + block: # for add_balancer.yml + - name: "Fetch keepalived.conf conf file from {{ groups.promoters[0] }}" + run_once: true + ansible.builtin.fetch: + src: /etc/keepalived/keepalived.conf + dest: files/keepalived.conf + validate_checksum: true + flat: true + delegate_to: "{{ groups.promoters[0] }}" + + - name: Copy keepalived.conf conf file to replica + ansible.builtin.copy: + src: files/keepalived.conf + dest: /etc/keepalived/keepalived.conf + mode: "0644" + notify: "Restart keepalived" + + - name: Remove keepalived.conf file from localhost + become: false + run_once: true + ansible.builtin.file: + path: files/keepalived.conf + state: absent + delegate_to: localhost + + - name: Prepare keepalived.conf conf file (replace "interface") + ansible.builtin.lineinfile: + path: /etc/keepalived/keepalived.conf + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + backrefs: true + loop: + - { regexp: '^.*interface', line: ' interface {{ vip_interface }}' } + loop_control: + label: "{{ item.line }}" + notify: "Restart keepalived" diff --git a/roles/keepalived/templates/keepalived.conf.j2 b/roles/keepalived/templates/keepalived.conf.j2 new file mode 100644 index 0000000..aad1d59 --- /dev/null +++ b/roles/keepalived/templates/keepalived.conf.j2 @@ -0,0 +1,68 @@ +#vrrp_instance VI_1 { +# state MASTER +# interface {{ vip_interface }} # Сетевой интерфейс (например, eth0) +# virtual_router_id 51 # Идентификатор VRRP (уникальный номер) +# priority 100 # Приоритет (для экземпляра на этом отдельном узле) +# advert_int 1 +# +# authentication { +# auth_type PASS +# auth_pass {{ keepalived_auth_pass }} +# } +# +# virtual_ipaddress { +# {{ virtual_ip }} +# } +# +# track_script { +# chk_patroni_or_haproxy +# } +#} +# +#vrrp_script chk_patroni_or_haproxy { +# script "/usr/local/bin/check_haproxy.sh" +# interval 2 +# weight 2 +#} + +global_defs { + router_id ocp_vrrp + enable_script_security + script_user root +} + +vrrp_script chk_patroni_leader { + script "timeout 3 /opt/keepalived/scripts/chk_patroni_leader.sh" + interval 5 + fall 1 + rise 1 +} + +vrrp_script haproxy_check { + script "/opt/keepalived/scripts/haproxy_check.sh" + interval 2 + weight 2 +} + +vrrp_instance pgcluster_1 { + interface {{ vip_interface }} + state BACKUP + priority {{ ansible_default_ipv4.address.split('.').3 }} + virtual_router_id 10 + authentication { + auth_type PASS + auth_pass password + } + virtual_ipaddress { + {{ cluster_vip_1 }} + } + unicast_src_ip {{ ansible_host }} + unicast_peer { + {% for host in groups['promoters'] %} + {{ hostvars[host]['ansible_default_ipv4'].address }} + {% endfor %} + } + track_script { + chk_patroni_leader + } +} diff --git a/roles/metrics/tasks/main.yml b/roles/metrics/tasks/main.yml new file mode 100644 index 0000000..4426bff --- /dev/null +++ b/roles/metrics/tasks/main.yml @@ -0,0 +1,119 @@ +- name: Create node_exporter system user + ansible.builtin.user: + name: node_exporter + system: true + shell: /sbin/nologin + create_home: false + +- name: Download and install node_exporter + block: + - name: Download node_exporter + ansible.builtin.get_url: + url: "https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_version }}/\ + node_exporter-{{ node_exporter_version }}.{{ arch }}.tar.gz" + dest: /tmp/node_exporter.tar.gz + mode: '0644' + + - name: Extract node_exporter + ansible.builtin.unarchive: + src: /tmp/node_exporter.tar.gz + dest: /tmp + remote_src: true + + - name: Copy node_exporter binary + ansible.builtin.copy: + src: "/tmp/node_exporter-{{ node_exporter_version }}.{{ arch }}/node_exporter" + dest: "/usr/local/bin/node_exporter" + mode: '0755' + remote_src: true + +- name: Download and install postgres_exporter + block: + - name: Download postgres_exporter + ansible.builtin.get_url: + url: "https://github.com/prometheus-community/postgres_exporter/releases/download/\ + v{{ postgres_exporter_version }}/postgres_exporter-{{ postgres_exporter_version }}.{{ arch }}.tar.gz" + dest: /tmp/postgres_exporter.tar.gz + mode: '0644' + + - name: Extract postgres_exporter + ansible.builtin.unarchive: + src: /tmp/postgres_exporter.tar.gz + dest: /tmp + remote_src: true + + - name: Copy postgres_exporter binary + ansible.builtin.copy: + src: "/tmp/postgres_exporter-{{ postgres_exporter_version }}.{{ arch }}/postgres_exporter" + dest: "/usr/local/bin/postgres_exporter" + mode: '0755' + remote_src: true + +- name: Create systemd unit for node_exporter + ansible.builtin.template: + src: node_exporter.service.j2 + dest: /etc/systemd/system/node_exporter.service + mode: '0644' + +- name: Ensure postgres_exporter config directory exists + ansible.builtin.file: + path: /etc/postgres_exporter + state: directory + mode: '0755' + owner: postgres + group: postgres + +- name: Create postgres_exporter config + ansible.builtin.template: + src: postgres_exporter.yaml.j2 + dest: /etc/postgres_exporter/postgres_exporter.yaml + mode: '0640' + owner: postgres + group: postgres + +- name: Create systemd unit for postgres_exporter + ansible.builtin.template: + src: postgres_exporter.service.j2 + dest: /etc/systemd/system/postgres_exporter.service + mode: '0644' + +- name: Start and enable exporters + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + daemon_reload: true + with_items: + - node_exporter + - postgres_exporter + +- name: Allow Prometheus metrics ports in firewall + community.general.ufw: + rule: allow + port: "{{ item }}" + proto: tcp + with_items: + - "{{ node_exporter_port }}" # node_exporter + - "{{ postgres_exporter_port }}" # postgres_exporter + +- name: Check node_exporter metrics endpoint + ansible.builtin.uri: + url: "http://localhost:{{ node_exporter_port }}/metrics" + status_code: 200 + return_content: true + register: node_exporter_metrics + +- name: Validate node_exporter metrics + ansible.builtin.debug: + msg: "Node exporter metrics collected successfully: {{ node_exporter_metrics.content | regex_search('node_cpu_seconds_total') is not none }}" + +- name: Check postgres_exporter metrics endpoint + ansible.builtin.uri: + url: "http://localhost:{{ postgres_exporter_port }}/metrics" + status_code: 200 + return_content: true + register: postgres_exporter_metrics + +- name: Validate postgres_exporter metrics + ansible.builtin.debug: + msg: "Postgres exporter metrics collected successfully: {{ postgres_exporter_metrics.content | regex_search('pg_database_size') is not none }}" diff --git a/templates/node_exporter.service.j2 b/roles/metrics/templates/node_exporter.service.j2 similarity index 100% rename from templates/node_exporter.service.j2 rename to roles/metrics/templates/node_exporter.service.j2 diff --git a/templates/postgres_exporter.service.j2 b/roles/metrics/templates/postgres_exporter.service.j2 similarity index 100% rename from templates/postgres_exporter.service.j2 rename to roles/metrics/templates/postgres_exporter.service.j2 diff --git a/templates/postgres_exporter.yaml.j2 b/roles/metrics/templates/postgres_exporter.yaml.j2 similarity index 100% rename from templates/postgres_exporter.yaml.j2 rename to roles/metrics/templates/postgres_exporter.yaml.j2 diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml new file mode 100644 index 0000000..90832df --- /dev/null +++ b/roles/patroni/tasks/main.yml @@ -0,0 +1,91 @@ +- name: Install dependencies + ansible.builtin.apt: + name: + - python3-pip + - python3-psycopg2 + - wget + - gnupg + update_cache: true + +- name: Adding GPG-key PostgreSQL + ansible.builtin.get_url: + url: https://www.postgresql.org/media/keys/ACCC4CF8.asc + dest: /etc/apt/trusted.gpg.d/postgresql.asc + mode: '0644' + +- name: Adding apt repository PostgreSQL + ansible.builtin.apt_repository: + repo: "deb http://apt.postgresql.org/pub/repos/apt {{ ansible_lsb.codename }}-pgdg main" + state: present + filename: "pgdg" + +- name: Update apt + ansible.builtin.apt: + update_cache: true + +- name: Install PostgreSQL {{ pg_version }} + ansible.builtin.apt: + name: postgresql-{{ pg_version }} + state: present + +- name: Install patroni + ansible.builtin.pip: + name: patroni[etcd] + +- name: Stop PostgreSQL + ansible.builtin.service: + name: postgresql + state: stopped + enabled: false + +- name: Create config Patroni + ansible.builtin.template: + src: patroni.yml.j2 + dest: /etc/patroni.yml + mode: '0644' + +- name: Set owner of data directory PostgreSQL + ansible.builtin.file: + path: "{{ data_dir }}" + state: directory + owner: postgres + group: postgres + mode: '0700' + recurse: true + +- name: Create patroni logs directory + ansible.builtin.file: + path: /var/log/patroni + state: directory + mode: '0755' + +- name: Create systemd unit for Patroni + ansible.builtin.template: + src: patroni.service.j2 + dest: /etc/systemd/system/patroni.service + mode: '0644' + +- name: Create config for patroni logrotate + ansible.builtin.copy: + dest: /etc/logrotate.d/patroni + mode: '0644' + content: | + /var/log/patroni/patroni.log { + daily + rotate 7 + compress + delaycompress + missingok + notifempty + create 0640 postgres postgres + } + +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true + +- name: Enable and start Patroni systemd unit + ansible.builtin.systemd: + name: patroni + state: restarted + enabled: true diff --git a/templates/patroni.service.j2 b/roles/patroni/templates/patroni.service.j2 similarity index 100% rename from templates/patroni.service.j2 rename to roles/patroni/templates/patroni.service.j2 diff --git a/templates/patroni.yml.j2 b/roles/patroni/templates/patroni.yml.j2 similarity index 95% rename from templates/patroni.yml.j2 rename to roles/patroni/templates/patroni.yml.j2 index 752d041..deeecef 100644 --- a/templates/patroni.yml.j2 +++ b/roles/patroni/templates/patroni.yml.j2 @@ -61,9 +61,6 @@ postgresql: - host all {{ postgresql_superuser_name }} 0.0.0.0/32 trust - host all {{ postgresql_superuser_name }} {{ ansible_host }}/32 trust - host replication {{ postgresql_replication_user }} 127.0.0.1/32 {{ postgresql_replication_auth_method }} -{% for host in groups['haproxy'] %} - - host all all {{ hostvars[host]['ansible_host'] }}/32 md5 -{% endfor %} {% for host in groups['promoters'] %} {% if host != inventory_hostname %} - host replication {{ postgresql_replication_user }} {{ hostvars[host]['ansible_host'] }}/32 {{ postgresql_replication_auth_method }} diff --git a/roles/pgbouncer/defaults/main.yml b/roles/pgbouncer/defaults/main.yml new file mode 100644 index 0000000..8be13d0 --- /dev/null +++ b/roles/pgbouncer/defaults/main.yml @@ -0,0 +1,9 @@ +pgbouncer_listen_port: 6432 +pgbouncer_listen_addr: 0.0.0.0 +pgbouncer_auth_user: postgres +pgbouncer_auth_pass: password +pgbouncer_pool_mode: transaction +pgbouncer_max_client_conn: 100 +pgbouncer_default_pool_size: 20 +postgresql_host: 127.0.0.1 +postgresql_port: 5432 diff --git a/roles/pgbouncer/tasks/main.yml b/roles/pgbouncer/tasks/main.yml new file mode 100644 index 0000000..1e50772 --- /dev/null +++ b/roles/pgbouncer/tasks/main.yml @@ -0,0 +1,92 @@ +- name: Install pgbouncer via apt + apt: + name: pgbouncer + state: present + update_cache: true + +- name: Ensure pgbouncer group exists + group: + name: pgbouncer + state: present + +- name: Ensure pgbouncer user exists + user: + name: pgbouncer + group: pgbouncer + home: /var/lib/pgbouncer + shell: /usr/sbin/nologin + system: true + state: present + +- name: Ensure /var/run/pgbouncer directory exists + file: + path: /var/run/pgbouncer + state: directory + owner: pgbouncer + group: pgbouncer + mode: '0755' + +- name: Ensure log directory exists + file: + path: /var/log/pgbouncer + state: directory + owner: pgbouncer + group: pgbouncer + mode: '0755' + +- name: Copy pgbouncer.ini + template: + src: pgbouncer.ini.j2 + dest: /etc/pgbouncer/pgbouncer.ini + mode: '0644' + +- name: Ensure correct ownership of /etc/pgbouncer/userlist.txt + file: + path: /etc/pgbouncer/userlist.txt + state: file + owner: pgbouncer + group: pgbouncer + mode: '0600' + +- name: Create systemd service for pgbouncer + template: + src: pgbouncer.service.j2 + dest: /etc/systemd/system/pgbouncer.service + mode: '0644' + +- name: Create empty pgbouncer.log if it does not exist + file: + path: /var/log/pgbouncer/pgbouncer.log + state: touch + owner: pgbouncer + group: pgbouncer + mode: '0640' + +- name: Ensure logrotate config for pgbouncer exists + copy: + dest: /etc/logrotate.d/pgbouncer + content: | + /var/log/pgbouncer/pgbouncer.log { + daily + rotate 7 + compress + delaycompress + missingok + notifempty + create 0640 pgbouncer pgbouncer + postrotate + systemctl reload pgbouncer > /dev/null 2>/dev/null || true + endscript + } + owner: root + group: root + mode: '0644' + +- name: Reload systemd to read updated service file + command: systemctl daemon-reload + +- name: Enable and start pgbouncer service + systemd: + name: pgbouncer + state: started + enabled: true diff --git a/roles/pgbouncer/templates/pgbouncer.ini.j2 b/roles/pgbouncer/templates/pgbouncer.ini.j2 new file mode 100644 index 0000000..534bbe7 --- /dev/null +++ b/roles/pgbouncer/templates/pgbouncer.ini.j2 @@ -0,0 +1,15 @@ +[databases] +* = host={{ postgresql_host }} port={{ postgresql_port }} + +[pgbouncer] +listen_addr = {{ pgbouncer_listen_addr }} +listen_port = {{ pgbouncer_listen_port }} +auth_type = md5 +auth_file = /etc/pgbouncer/userlist.txt +logfile = /var/log/pgbouncer/pgbouncer.log +pidfile = /var/run/pgbouncer/pgbouncer.pid +admin_users = {{ pgbouncer_auth_user }} +pool_mode = {{ pgbouncer_pool_mode }} +server_reset_query = DISCARD ALL +max_client_conn = {{ pgbouncer_max_client_conn }} +default_pool_size = {{ pgbouncer_default_pool_size }} diff --git a/roles/pgbouncer/templates/pgbouncer.service.j2 b/roles/pgbouncer/templates/pgbouncer.service.j2 new file mode 100644 index 0000000..9972f0d --- /dev/null +++ b/roles/pgbouncer/templates/pgbouncer.service.j2 @@ -0,0 +1,17 @@ +[Unit] +Description=PgBouncer +After=network.target + +[Service] +Type=simple +ExecStart=/usr/sbin/pgbouncer /etc/pgbouncer/pgbouncer.ini +ExecReload=/bin/kill -HUP $MAINPID +Restart=always +User=pgbouncer +Group=pgbouncer +PIDFile=/var/run/pgbouncer/pgbouncer.pid +StandardOutput=append:/var/log/pgbouncer/pgbouncer.log +StandardError=append:/var/log/pgbouncer/pgbouncer.log + +[Install] +WantedBy=multi-user.target diff --git a/roles/pgbouncer/templates/userlist.txt.j2 b/roles/pgbouncer/templates/userlist.txt.j2 new file mode 100644 index 0000000..95ef611 --- /dev/null +++ b/roles/pgbouncer/templates/userlist.txt.j2 @@ -0,0 +1 @@ +"{{ pgbouncer_auth_user }}" "{{ pgbouncer_auth_pass }}" From 43e6d46387c5beef79185dc72d02fd777ce1d854 Mon Sep 17 00:00:00 2001 From: Roman Chudov Date: Sat, 3 May 2025 07:10:30 +0300 Subject: [PATCH 2/7] fixes. --- Makefile | 7 +- playbook.yml | 12 +-- roles/haproxy/tasks/main.yml | 2 +- roles/keepalived/files/check_haproxy.sh | 12 --- roles/keepalived/tasks/main.yml | 84 ++++--------------- roles/keepalived/templates/keepalived.conf.j2 | 27 ------ roles/patroni/tasks/main.yml | 4 +- 7 files changed, 22 insertions(+), 126 deletions(-) delete mode 100644 roles/keepalived/files/check_haproxy.sh diff --git a/Makefile b/Makefile index fa4be04..3cd65f0 100644 --- a/Makefile +++ b/Makefile @@ -18,15 +18,14 @@ prometheus-grafana: init .PHONY: haproxy.check.master haproxy.check.master: init @PGPASSWORD=`grep postgresql_superuser_password group_vars/promoters.yml | awk '{print $$2}'` && \ - ANSIBLE_HOST=`grep haproxy1 inventory.ini | awk '{print $$2}' | sed 's/ansible_host=//'` && \ - PGPASSWORD=$$PGPASSWORD psql -h $$ANSIBLE_HOST -p 5000 -U postgres -c "SELECT pg_is_in_recovery()" | grep -q 'f' && \ + ANSIBLE_HOST='192.168.64.100' && \ + PGPASSWORD=$$PGPASSWORD psql -h 192.168.64.100 -p 5000 -U postgres -c "SELECT pg_is_in_recovery()" | grep -q 'f' && \ echo "HAProxy master is OK" || echo "HAProxy master check failed" .PHONY: haproxy.check.slave haproxy.check.slave: init @PGPASSWORD=`grep postgresql_superuser_password group_vars/promoters.yml | awk '{print $$2}'` && \ - ANSIBLE_HOST=`grep haproxy1 inventory.ini | awk '{print $$2}' | sed 's/ansible_host=//'` && \ - PGPASSWORD=$$PGPASSWORD psql -h $$ANSIBLE_HOST -p 5001 -U postgres -c "SELECT pg_is_in_recovery()" | grep -q 't' && \ + PGPASSWORD=$$PGPASSWORD psql -h 192.168.64.100 -p 5001 -U postgres -c "SELECT pg_is_in_recovery()" | grep -q 't' && \ echo "HAProxy slave is OK" || echo "HAProxy slave check failed" .PHONY: haproxy.check diff --git a/playbook.yml b/playbook.yml index 3b6572c..361b85b 100644 --- a/playbook.yml +++ b/playbook.yml @@ -3,17 +3,9 @@ hosts: promoters become: true roles: -# - etcd -# - patroni + - etcd + - patroni - metrics - pgbouncer - haproxy - keepalived - -- name: Deploy Keepalived VIP - hosts: keepalived_vip - become: true - vars: - add_balancer: true - roles: - - keepalived diff --git a/roles/haproxy/tasks/main.yml b/roles/haproxy/tasks/main.yml index 16a980b..40c32d2 100644 --- a/roles/haproxy/tasks/main.yml +++ b/roles/haproxy/tasks/main.yml @@ -1,4 +1,4 @@ -- name: Install HAProxy +- name: Install HAProxy via apt ansible.builtin.apt: name: haproxy state: present diff --git a/roles/keepalived/files/check_haproxy.sh b/roles/keepalived/files/check_haproxy.sh deleted file mode 100644 index eae41e1..0000000 --- a/roles/keepalived/files/check_haproxy.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -hosts=({% for host in groups['promoters'] %}{{ hostvars[host]['ansible_host'] }} {% endfor %}) - -for host in "${hosts[@]}"; do - curl -sf http://$host:{{ haproxy_stats_port }}/ > /dev/null 2>&1 - if [ $? -eq 0 ]; then - exit 0 - fi -done - -exit 1 diff --git a/roles/keepalived/tasks/main.yml b/roles/keepalived/tasks/main.yml index 712c8c1..18a8421 100644 --- a/roles/keepalived/tasks/main.yml +++ b/roles/keepalived/tasks/main.yml @@ -1,39 +1,10 @@ -#--- -#- name: Install keepalived -# apt: -# name: keepalived -# state: present -# update_cache: true -# -#- name: Configure keepalived -# template: -# src: keepalived.conf.j2 -# dest: /etc/keepalived/keepalived.conf -# mode: '0644' -# owner: root -# group: root -# notify: restart keepalived -# -#- name: Make script executable -# file: -# path: /usr/local/bin/check_haproxy.sh -# mode: '0755' -# state: file -# -#- name: Ensure keepalived is enabled and started -# systemd: -# name: keepalived -# state: started -# enabled: true - --- - -- name: Install jq +- name: Install jq via apt ansible.builtin.apt: name: jq state: present -- name: Install keepalived +- name: Install keepalived via apt ansible.builtin.apt: name: keepalived state: present @@ -64,13 +35,18 @@ ansible.builtin.copy: content: | #!/bin/bash - /bin/kill -0 `cat /run/haproxy/haproxy.pid` + /bin/kill -0 `cat /var/run/haproxy.pid` dest: /opt/keepalived/scripts/haproxy_check.sh owner: root group: root mode: "0700" notify: "Restart keepalived" +- name: Add execution rights to haproxy_check.sh + ansible.builtin.file: + path: /opt/keepalived/scripts/haproxy_check.sh + mode: 'u+x' + - name: Generate keepalived conf file when: add_balancer is not defined or not add_balancer|bool ansible.builtin.template: @@ -91,41 +67,9 @@ owner: root group: root -- name: Block for tasks with balancer - when: add_balancer is defined and add_balancer|bool - block: # for add_balancer.yml - - name: "Fetch keepalived.conf conf file from {{ groups.promoters[0] }}" - run_once: true - ansible.builtin.fetch: - src: /etc/keepalived/keepalived.conf - dest: files/keepalived.conf - validate_checksum: true - flat: true - delegate_to: "{{ groups.promoters[0] }}" - - - name: Copy keepalived.conf conf file to replica - ansible.builtin.copy: - src: files/keepalived.conf - dest: /etc/keepalived/keepalived.conf - mode: "0644" - notify: "Restart keepalived" - - - name: Remove keepalived.conf file from localhost - become: false - run_once: true - ansible.builtin.file: - path: files/keepalived.conf - state: absent - delegate_to: localhost - - - name: Prepare keepalived.conf conf file (replace "interface") - ansible.builtin.lineinfile: - path: /etc/keepalived/keepalived.conf - regexp: "{{ item.regexp }}" - line: "{{ item.line }}" - backrefs: true - loop: - - { regexp: '^.*interface', line: ' interface {{ vip_interface }}' } - loop_control: - label: "{{ item.line }}" - notify: "Restart keepalived" +- name: Enable and start keepalived + ansible.builtin.systemd: + name: keepalived + state: restarted + enabled: true + daemon_reload: true diff --git a/roles/keepalived/templates/keepalived.conf.j2 b/roles/keepalived/templates/keepalived.conf.j2 index aad1d59..dcecbf8 100644 --- a/roles/keepalived/templates/keepalived.conf.j2 +++ b/roles/keepalived/templates/keepalived.conf.j2 @@ -1,30 +1,3 @@ -#vrrp_instance VI_1 { -# state MASTER -# interface {{ vip_interface }} # Сетевой интерфейс (например, eth0) -# virtual_router_id 51 # Идентификатор VRRP (уникальный номер) -# priority 100 # Приоритет (для экземпляра на этом отдельном узле) -# advert_int 1 -# -# authentication { -# auth_type PASS -# auth_pass {{ keepalived_auth_pass }} -# } -# -# virtual_ipaddress { -# {{ virtual_ip }} -# } -# -# track_script { -# chk_patroni_or_haproxy -# } -#} -# -#vrrp_script chk_patroni_or_haproxy { -# script "/usr/local/bin/check_haproxy.sh" -# interval 2 -# weight 2 -#} - global_defs { router_id ocp_vrrp enable_script_security diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml index 90832df..d97b3ef 100644 --- a/roles/patroni/tasks/main.yml +++ b/roles/patroni/tasks/main.yml @@ -1,4 +1,4 @@ -- name: Install dependencies +- name: Install dependencies via apt ansible.builtin.apt: name: - python3-pip @@ -23,7 +23,7 @@ ansible.builtin.apt: update_cache: true -- name: Install PostgreSQL {{ pg_version }} +- name: Install PostgreSQL {{ pg_version }} via apt ansible.builtin.apt: name: postgresql-{{ pg_version }} state: present From 63ee9264c01e3f7ead2b28ed6a011f84e454f7a5 Mon Sep 17 00:00:00 2001 From: Roman Chudov Date: Sat, 3 May 2025 07:13:51 +0300 Subject: [PATCH 3/7] keepalived: master state for node 1. --- roles/keepalived/templates/keepalived.conf.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/keepalived/templates/keepalived.conf.j2 b/roles/keepalived/templates/keepalived.conf.j2 index dcecbf8..0d92e9c 100644 --- a/roles/keepalived/templates/keepalived.conf.j2 +++ b/roles/keepalived/templates/keepalived.conf.j2 @@ -19,7 +19,7 @@ vrrp_script haproxy_check { vrrp_instance pgcluster_1 { interface {{ vip_interface }} - state BACKUP + state {% if hostvars[inventory_hostname].node_id == "1" %}MASTER{% else %}BACKUP{% endif %} priority {{ ansible_default_ipv4.address.split('.').3 }} virtual_router_id 10 authentication { @@ -27,7 +27,7 @@ vrrp_instance pgcluster_1 { auth_pass password } virtual_ipaddress { - {{ cluster_vip_1 }} + {{ cluster_vip_1 }} } unicast_src_ip {{ ansible_host }} unicast_peer { From 21d79eed8108c92e33586245e0d257455705fe93 Mon Sep 17 00:00:00 2001 From: Roman Chudov Date: Sat, 3 May 2025 07:14:32 +0300 Subject: [PATCH 4/7] fix. --- inventory.ini.example | 3 --- 1 file changed, 3 deletions(-) diff --git a/inventory.ini.example b/inventory.ini.example index 218293d..1b27767 100644 --- a/inventory.ini.example +++ b/inventory.ini.example @@ -3,8 +3,5 @@ patroni1 ansible_host=192.168.64.2 node_id=1 patroni2 ansible_host=192.168.64.3 node_id=2 patroni3 ansible_host=192.168.64.4 node_id=3 -[keepalived_hosts] -vip_host ansible_host=192.168.64.5 - [prometheus_grafana] prometheus_grafana1 ansible_host=192.168.64.7 From 52c538923c8c00a6e679c16b85cd0a7c1f0bf07c Mon Sep 17 00:00:00 2001 From: Roman Chudov Date: Sat, 3 May 2025 07:19:41 +0300 Subject: [PATCH 5/7] fix. --- roles/keepalived/templates/keepalived.conf.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/keepalived/templates/keepalived.conf.j2 b/roles/keepalived/templates/keepalived.conf.j2 index 0d92e9c..972a372 100644 --- a/roles/keepalived/templates/keepalived.conf.j2 +++ b/roles/keepalived/templates/keepalived.conf.j2 @@ -19,7 +19,8 @@ vrrp_script haproxy_check { vrrp_instance pgcluster_1 { interface {{ vip_interface }} - state {% if hostvars[inventory_hostname].node_id == "1" %}MASTER{% else %}BACKUP{% endif %} + state {% if inventory_hostname == "patroni1" %}MASTER{% else %}BACKUP{% endif %} + priority {{ ansible_default_ipv4.address.split('.').3 }} virtual_router_id 10 authentication { From d4258c10d238b972a9f05e053b1facf5020948c1 Mon Sep 17 00:00:00 2001 From: Roman Chudov Date: Sat, 3 May 2025 07:33:56 +0300 Subject: [PATCH 6/7] linter fixes. --- Makefile | 5 +--- group_vars/promoters.yml | 3 +-- roles/keepalived/defaults/main.yml | 4 +-- roles/keepalived/templates/keepalived.conf.j2 | 2 +- roles/patroni/tasks/main.yml | 2 +- roles/pgbouncer/defaults/main.yml | 4 +-- roles/pgbouncer/tasks/main.yml | 26 +++++++++---------- roles/pgbouncer/templates/pgbouncer.ini.j2 | 2 +- 8 files changed, 20 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index 3cd65f0..03268b7 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,4 @@ check-metrics: check-metrics.node-exporter check-metrics.postgres-exporter .PHONY: lint lint: - ansible-lint playbook.yml - ansible-lint haproxy.yml - ansible-lint pg_observe.yml - ansible-lint prometheus_grafana.yml + find . -name "*.yml" -exec ansible-lint {} \; diff --git a/group_vars/promoters.yml b/group_vars/promoters.yml index 6a8cdfd..34761b7 100644 --- a/group_vars/promoters.yml +++ b/group_vars/promoters.yml @@ -50,7 +50,7 @@ postgresql_custom_parameters: wal_keep_segments: 32 # Additional pg_hba.conf rules (optional) -#postgresql_additional_pg_hba: +# postgresql_additional_pg_hba: # PgBouncer params pgbouncer_listen_port: 6432 @@ -60,7 +60,6 @@ haproxy_config: "/etc/haproxy/haproxy.cfg" # Cluster variables cluster_vip_1: "192.168.64.100" -vip_interface: "{{ ansible_default_ipv4.interface }}" # interface name (ex. "ens32") # Prometheus settings prometheus_retention: "30d" diff --git a/roles/keepalived/defaults/main.yml b/roles/keepalived/defaults/main.yml index 4488949..dd60efc 100644 --- a/roles/keepalived/defaults/main.yml +++ b/roles/keepalived/defaults/main.yml @@ -1,4 +1,2 @@ -vip_interface: enp0s1 -virtual_ip: 192.168.64.100 +keepalived_vip_interface: enp0s1 keepalived_auth_pass: mypassword -haproxy_stats_port: 7000 diff --git a/roles/keepalived/templates/keepalived.conf.j2 b/roles/keepalived/templates/keepalived.conf.j2 index 972a372..86d3801 100644 --- a/roles/keepalived/templates/keepalived.conf.j2 +++ b/roles/keepalived/templates/keepalived.conf.j2 @@ -18,7 +18,7 @@ vrrp_script haproxy_check { } vrrp_instance pgcluster_1 { - interface {{ vip_interface }} + interface {{ keepalived_vip_interface }} state {% if inventory_hostname == "patroni1" %}MASTER{% else %}BACKUP{% endif %} priority {{ ansible_default_ipv4.address.split('.').3 }} diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml index d97b3ef..3e7bfc9 100644 --- a/roles/patroni/tasks/main.yml +++ b/roles/patroni/tasks/main.yml @@ -23,7 +23,7 @@ ansible.builtin.apt: update_cache: true -- name: Install PostgreSQL {{ pg_version }} via apt +- name: Install PostgreSQL {{ pg_version }} ansible.builtin.apt: name: postgresql-{{ pg_version }} state: present diff --git a/roles/pgbouncer/defaults/main.yml b/roles/pgbouncer/defaults/main.yml index 8be13d0..9c1c7f7 100644 --- a/roles/pgbouncer/defaults/main.yml +++ b/roles/pgbouncer/defaults/main.yml @@ -5,5 +5,5 @@ pgbouncer_auth_pass: password pgbouncer_pool_mode: transaction pgbouncer_max_client_conn: 100 pgbouncer_default_pool_size: 20 -postgresql_host: 127.0.0.1 -postgresql_port: 5432 +pgbouncer_postgresql_host: 127.0.0.1 +pgbouncer_postgresql_port: 5432 diff --git a/roles/pgbouncer/tasks/main.yml b/roles/pgbouncer/tasks/main.yml index 1e50772..5c2ae17 100644 --- a/roles/pgbouncer/tasks/main.yml +++ b/roles/pgbouncer/tasks/main.yml @@ -1,16 +1,16 @@ - name: Install pgbouncer via apt - apt: + ansible.builtin.apt: name: pgbouncer state: present update_cache: true - name: Ensure pgbouncer group exists - group: + ansible.builtin.group: name: pgbouncer state: present - name: Ensure pgbouncer user exists - user: + ansible.builtin.user: name: pgbouncer group: pgbouncer home: /var/lib/pgbouncer @@ -19,7 +19,7 @@ state: present - name: Ensure /var/run/pgbouncer directory exists - file: + ansible.builtin.file: path: /var/run/pgbouncer state: directory owner: pgbouncer @@ -27,7 +27,7 @@ mode: '0755' - name: Ensure log directory exists - file: + ansible.builtin.file: path: /var/log/pgbouncer state: directory owner: pgbouncer @@ -35,13 +35,13 @@ mode: '0755' - name: Copy pgbouncer.ini - template: + ansible.builtin.template: src: pgbouncer.ini.j2 dest: /etc/pgbouncer/pgbouncer.ini mode: '0644' - name: Ensure correct ownership of /etc/pgbouncer/userlist.txt - file: + ansible.builtin.file: path: /etc/pgbouncer/userlist.txt state: file owner: pgbouncer @@ -49,13 +49,13 @@ mode: '0600' - name: Create systemd service for pgbouncer - template: + ansible.builtin.template: src: pgbouncer.service.j2 dest: /etc/systemd/system/pgbouncer.service mode: '0644' - name: Create empty pgbouncer.log if it does not exist - file: + ansible.builtin.file: path: /var/log/pgbouncer/pgbouncer.log state: touch owner: pgbouncer @@ -63,7 +63,7 @@ mode: '0640' - name: Ensure logrotate config for pgbouncer exists - copy: + ansible.builtin.copy: dest: /etc/logrotate.d/pgbouncer content: | /var/log/pgbouncer/pgbouncer.log { @@ -82,11 +82,9 @@ group: root mode: '0644' -- name: Reload systemd to read updated service file - command: systemctl daemon-reload - - name: Enable and start pgbouncer service - systemd: + ansible.builtin.systemd: name: pgbouncer state: started enabled: true + daemon_reload: true diff --git a/roles/pgbouncer/templates/pgbouncer.ini.j2 b/roles/pgbouncer/templates/pgbouncer.ini.j2 index 534bbe7..c153fd1 100644 --- a/roles/pgbouncer/templates/pgbouncer.ini.j2 +++ b/roles/pgbouncer/templates/pgbouncer.ini.j2 @@ -1,5 +1,5 @@ [databases] -* = host={{ postgresql_host }} port={{ postgresql_port }} +* = host={{ pgbouncer_postgresql_host }} port={{ pgbouncer_postgresql_port }} [pgbouncer] listen_addr = {{ pgbouncer_listen_addr }} From 94b77ab41e291aae07a6be7b2cb65983bdbdee74 Mon Sep 17 00:00:00 2001 From: Roman Chudov Date: Sat, 3 May 2025 07:52:42 +0300 Subject: [PATCH 7/7] README.md updated. --- README.md | 76 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 44 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 3891888..7c7047d 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,18 @@ # ansible-patroni -Ansible playbook for deploy PostgreSQL Patroni cluster +Ansible playbook for deploying PostgreSQL Patroni cluster ⚠️ **WARNING: This configuration is for testing/development purposes only. DO NOT use in production without proper security hardening!** ## Overview -This Ansible playbook automates the deployment of a PostgreSQL high-availability cluster using Patroni and etcd. The setup includes a 3-node cluster configuration with automatic failover capabilities and monitoring. +This Ansible playbook automates the deployment of a PostgreSQL high-availability cluster using Patroni and etcd. The setup includes the following components for connection management and high availability: +- `Keepalived` creates a Virtual IP (VIP) for failover. +- `HAProxy` acts as a proxy layer for distributing requests. +- `PgBouncer` operates as a connection pooler. + +The connection path: +``` +Keepalived VIP -> HAProxy -> PgBouncer -> PostgreSQL +``` ## Prerequisites - Ubuntu/Debian-based system @@ -15,31 +23,28 @@ This Ansible playbook automates the deployment of a PostgreSQL high-availability - PostgreSQL 16 - Patroni - etcd (for distributed configuration) +- PgBouncer (for connection pooling) +- Keepalived (for VIP) +- HAProxy (for load balancing) - Python 3 and pip - Node Exporter (system metrics) - Postgres Exporter (PostgreSQL metrics) ## Quick Start 1. Clone this repository: -2. Create config files by command `make init` -3. Update inventory.ini with your servers: +2. Create config files with the command `make init` +3. Update `inventory.ini` with your servers: ```ini [promoters] patroni1 ansible_host=192.168.64.2 node_id=1 patroni2 ansible_host=192.168.64.3 node_id=2 patroni3 ansible_host=192.168.64.4 node_id=3 ``` -4. Update ansible.cfg with your user (root by default) +4. Update `ansible.cfg` with your user (root by default). 5. Run the playbooks: ```shell # Deploy Patroni cluster make up - -# Deploy HAProxy -make haproxy - -# Deploy monitoring -make pg-observe ``` ## Default Configuration @@ -52,9 +57,20 @@ make pg-observe - HAProxy PostgreSQL (master): 5000 - HAProxy PostgreSQL (replica): 5001 - HAProxy Statistics: 7000 +- PgBouncer: 6432 - Node Exporter: 9100 - Postgres Exporter: 9187 +### Keepalived Configuration +- VIP: Used for automatic failover between HAProxy instances. +- Configured to determine the `MASTER` node based on its `node_id` in the cluster. + +### PgBouncer Configuration +- Connection pooling for PostgreSQL to reduce the overhead of establishing frequent connections. +- Default port: `6432`. +- Authentication method: Userlist file. +- Configurations stored in `/etc/pgbouncer/pgbouncer.ini`. + ### PostgreSQL Settings - Version: 16 - Encoding: UTF8 @@ -90,27 +106,6 @@ make pg-observe - User: postgres - Config Path: /etc/postgres_exporter/postgres_exporter.yaml -## File Structure -``` -. -├── ansible.cfg # Ansible configuration -├── inventory.ini # Server inventory -├── playbook.yml # Main playbook -├── haproxy.yml # HAProxy playbook -├── pg_observe.yml # Monitoring playbook -└── group_vars/ -│ └── promoters.yml # Variables and settings -└── templates/ -│ ├── etcd.conf.yml.j2 # etcd configuration -│ ├── patroni.yml.j2 # Patroni configuration -│ ├── haproxy.cfg.j2 # HAProxy configuration -│ ├── postgres_exporter.yaml.j2 # Postgres exporter config -│ ├── etcd.service.j2 # etcd systemd service -│ ├── patroni.service.j2 # Patroni systemd service -│ ├── node_exporter.service.j2 # Node exporter systemd service -│ └── postgres_exporter.service.j2 # Postgres exporter systemd service -``` - ## Important Security Notes This deployment includes several configurations that are NOT suitable for production: - Basic default passwords @@ -123,9 +118,11 @@ This deployment includes several configurations that are NOT suitable for produc ## Logging - Patroni logs: /var/log/patroni/patroni.log - etcd logs: /var/log/etcd.log +- PgBouncer logs: /var/log/pgbouncer/pgbouncer.log - Log rotation is configured for Patroni logs (7 days retention) - Node Exporter logs: journalctl -u node_exporter - Postgres Exporter logs: journalctl -u postgres_exporter +- Keepalived logs: journalctl -u keepalived ## Service Management ``` bash @@ -139,6 +136,21 @@ systemctl status etcd systemctl start etcd systemctl stop etcd +# HAProxy service +systemctl status haproxy +systemctl start haproxy +systemctl stop haproxy + +# PgBouncer service +systemctl status pgbouncer +systemctl start pgbouncer +systemctl stop pgbouncer + +# Keepalived service +systemctl status keepalived +systemctl start keepalived +systemctl stop keepalived + # Monitoring services systemctl status node_exporter systemctl status postgres_exporter