[GEN-10044] chore: updates to allow for async WAL verification

ansible/files/adminapi.sudoers.conf

@@ -17,6 +17,8 @@ Cmnd_Alias PGBOUNCER = /bin/systemctl start pgbouncer.service, /bin/systemctl st
 %adminapi ALL= NOPASSWD: /usr/bin/systemctl restart postgresql.service
 %adminapi ALL= NOPASSWD: /usr/bin/systemctl show -p NRestarts postgresql.service
 %adminapi ALL= NOPASSWD: /usr/bin/systemctl restart adminapi.service
+%adminapi ALL= NOPASSWD: /usr/bin/systemctl is-active verify-wal.service
+%adminapi ALL= NOPASSWD: /usr/bin/systemctl start verify-wal.service
 %adminapi ALL= NOPASSWD: /bin/systemctl daemon-reload
 %adminapi ALL= NOPASSWD: /bin/systemctl restart services.slice
 %adminapi ALL= NOPASSWD: /usr/sbin/nft -f /etc/nftables/supabase_managed.conf

ansible/files/logrotate_config/logrotate-wal-verify.conf

@@ -0,0 +1,6 @@
+/var/log/wal-g/wal-verification-results/*.json {
+    weekly
+    rotate 0  
+    nocreate
+    missingok
+}

ansible/files/verify-wal.service.j2

@@ -0,0 +1,12 @@
+[Unit]
+Description=Async WAL verification
+
+[Service]
+Type=simple
+User=adminapi
+ExecStart=/usr/bin/admin-mgr wal-verify-timeline --save-output-to-file true
+Restart=no
+OOMScoreAdjust=-1000
+
+[Install]
+WantedBy=multi-user.target

ansible/tasks/finalize-ami.yml

@@ -51,6 +51,7 @@
     - { file: "logrotate-postgres-csv.conf" }
     - { file: "logrotate-postgres.conf" }
     - { file: "logrotate-walg.conf" }
+    - { file: "logrotate-wal-verify.conf" }
     - { file: "logrotate-postgres-auth.conf" }
 
 - name: Ensure default Postgres logrotate config is removed

ansible/tasks/internal/admin-api.yml

@@ -79,6 +79,11 @@
     src: files/adminapi.service.j2
     dest: /etc/systemd/system/adminapi.service
 
+- name: adminapi - create service file for verify wal process
+  template:
+    src: files/verify-wal.service.j2
+    dest: /etc/systemd/system/verify-wal.service
+
 - name: UFW - Allow connections to adminapi ports
   ufw:
     rule: allow

ebssurrogate/scripts/surrogate-bootstrap-nix.sh

@@ -258,7 +258,9 @@ function clean_system {
 
 	# Setup wal-g logs
 	mkdir /mnt/var/log/wal-g
+
 	touch /mnt/var/log/wal-g/{backup-push.log,backup-fetch.log,wal-push.log,wal-fetch.log,pitr.log}
+	mkdir /mnt/var/log/wal-g/wal-verification-results
 
 	#Creatre Sysstat directory for SAR
 	mkdir /mnt/var/log/sysstat

ebssurrogate/scripts/surrogate-bootstrap.sh

@@ -258,7 +258,9 @@ function clean_system {
 
 	# Setup wal-g logs
 	mkdir /mnt/var/log/wal-g
+
 	touch /mnt/var/log/wal-g/{backup-push.log,backup-fetch.log,wal-push.log,wal-fetch.log,pitr.log}
+	mkdir /mnt/var/log/wal-g/wal-verification-results
 
 	#Creatre Sysstat directory for SAR
 	mkdir /mnt/var/log/sysstat

scripts/91-log_cleanup.sh

@@ -17,8 +17,11 @@ mkdir /var/log/postgresql
 chown postgres:postgres /var/log/postgresql
 
 mkdir /var/log/wal-g
+
 cd /var/log/wal-g
 touch backup-push.log backup-fetch.log wal-push.log wal-fetch.log pitr.log
+mkdir /var/log/wal-g/wal-verification-results
+
 chown -R postgres:postgres /var/log/wal-g
 chmod -R 0300 /var/log/wal-g