From 2579b8495006d50bf14ab59c33dd94d070f77e30 Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 7 Aug 2024 15:25:07 +0800 Subject: [PATCH 1/4] chore: init /var/log/wal-g/wal-verification-results --- ebssurrogate/scripts/surrogate-bootstrap-nix.sh | 2 ++ ebssurrogate/scripts/surrogate-bootstrap.sh | 2 ++ scripts/91-log_cleanup.sh | 3 +++ 3 files changed, 7 insertions(+) diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh index 5bb021d96..717c99217 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -258,7 +258,9 @@ function clean_system { # Setup wal-g logs mkdir /mnt/var/log/wal-g + touch /mnt/var/log/wal-g/{backup-push.log,backup-fetch.log,wal-push.log,wal-fetch.log,pitr.log} + mkdir /mnt/var/log/wal-g/wal-verification-results #Creatre Sysstat directory for SAR mkdir /mnt/var/log/sysstat diff --git a/ebssurrogate/scripts/surrogate-bootstrap.sh b/ebssurrogate/scripts/surrogate-bootstrap.sh index 54eb98fb5..bdba2d767 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap.sh @@ -258,7 +258,9 @@ function clean_system { # Setup wal-g logs mkdir /mnt/var/log/wal-g + touch /mnt/var/log/wal-g/{backup-push.log,backup-fetch.log,wal-push.log,wal-fetch.log,pitr.log} + mkdir /mnt/var/log/wal-g/wal-verification-results #Creatre Sysstat directory for SAR mkdir /mnt/var/log/sysstat diff --git a/scripts/91-log_cleanup.sh b/scripts/91-log_cleanup.sh index 24073afcb..592e92117 100644 --- a/scripts/91-log_cleanup.sh +++ b/scripts/91-log_cleanup.sh @@ -17,8 +17,11 @@ mkdir /var/log/postgresql chown postgres:postgres /var/log/postgresql mkdir /var/log/wal-g + cd /var/log/wal-g touch backup-push.log backup-fetch.log wal-push.log wal-fetch.log pitr.log +mkdir /var/log/wal-g/wal-verification-results + chown -R postgres:postgres /var/log/wal-g chmod -R 0300 /var/log/wal-g From 9be3942618273932b11faeb8267f527fe4648769 Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 7 Aug 2024 15:25:26 +0800 Subject: [PATCH 2/4] chore: implement logrotate for /var/log/wal-g/wal-verification-results --- ansible/files/logrotate_config/logrotate-wal-verify.conf | 6 ++++++ ansible/tasks/finalize-ami.yml | 1 + 2 files changed, 7 insertions(+) create mode 100644 ansible/files/logrotate_config/logrotate-wal-verify.conf diff --git a/ansible/files/logrotate_config/logrotate-wal-verify.conf b/ansible/files/logrotate_config/logrotate-wal-verify.conf new file mode 100644 index 000000000..f04bb5866 --- /dev/null +++ b/ansible/files/logrotate_config/logrotate-wal-verify.conf @@ -0,0 +1,6 @@ +/var/log/wal-g/wal-verification-results/*.json { + weekly + rotate 0 + nocreate + missingok +} diff --git a/ansible/tasks/finalize-ami.yml b/ansible/tasks/finalize-ami.yml index 7f0de3ac8..db6aea6fd 100644 --- a/ansible/tasks/finalize-ami.yml +++ b/ansible/tasks/finalize-ami.yml @@ -51,6 +51,7 @@ - { file: "logrotate-postgres-csv.conf" } - { file: "logrotate-postgres.conf" } - { file: "logrotate-walg.conf" } + - { file: "logrotate-wal-verify.conf" } - { file: "logrotate-postgres-auth.conf" } - name: Ensure default Postgres logrotate config is removed From 9271a85dba50bb326b60c55aa550432d3049da17 Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 7 Aug 2024 15:25:53 +0800 Subject: [PATCH 3/4] chore: init service for async WAL verification --- ansible/files/verify-wal.service.j2 | 12 ++++++++++++ ansible/tasks/internal/admin-api.yml | 5 +++++ 2 files changed, 17 insertions(+) create mode 100644 ansible/files/verify-wal.service.j2 diff --git a/ansible/files/verify-wal.service.j2 b/ansible/files/verify-wal.service.j2 new file mode 100644 index 000000000..164dab433 --- /dev/null +++ b/ansible/files/verify-wal.service.j2 @@ -0,0 +1,12 @@ +[Unit] +Description=Async WAL verification + +[Service] +Type=simple +User=adminapi +ExecStart=/usr/bin/admin-mgr wal-verify-timeline --save-output-to-file true +Restart=no +OOMScoreAdjust=-1000 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/tasks/internal/admin-api.yml b/ansible/tasks/internal/admin-api.yml index cea0109fd..bdd88f9d0 100644 --- a/ansible/tasks/internal/admin-api.yml +++ b/ansible/tasks/internal/admin-api.yml @@ -79,6 +79,11 @@ src: files/adminapi.service.j2 dest: /etc/systemd/system/adminapi.service +- name: adminapi - create service file for verify wal process + template: + src: files/verify-wal.service.j2 + dest: /etc/systemd/system/verify-wal.service + - name: UFW - Allow connections to adminapi ports ufw: rule: allow From 087dd03310528d6a779a1f2732ca878152b8f8a0 Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 7 Aug 2024 15:26:11 +0800 Subject: [PATCH 4/4] chore: ensure admin api has sufficient permissions for WAL verification service --- ansible/files/adminapi.sudoers.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/files/adminapi.sudoers.conf b/ansible/files/adminapi.sudoers.conf index eada0a94b..7e97d3f00 100644 --- a/ansible/files/adminapi.sudoers.conf +++ b/ansible/files/adminapi.sudoers.conf @@ -17,6 +17,8 @@ Cmnd_Alias PGBOUNCER = /bin/systemctl start pgbouncer.service, /bin/systemctl st %adminapi ALL= NOPASSWD: /usr/bin/systemctl restart postgresql.service %adminapi ALL= NOPASSWD: /usr/bin/systemctl show -p NRestarts postgresql.service %adminapi ALL= NOPASSWD: /usr/bin/systemctl restart adminapi.service +%adminapi ALL= NOPASSWD: /usr/bin/systemctl is-active verify-wal.service +%adminapi ALL= NOPASSWD: /usr/bin/systemctl start verify-wal.service %adminapi ALL= NOPASSWD: /bin/systemctl daemon-reload %adminapi ALL= NOPASSWD: /bin/systemctl restart services.slice %adminapi ALL= NOPASSWD: /usr/sbin/nft -f /etc/nftables/supabase_managed.conf