[DPE-9356] Update PostgreSQL mount paths to /var/lib/pg with 16/main subdirectory

docs/explanation/juju.md

@@ -26,7 +26,7 @@ In the context of this documentation, the pertinent changes are as follows:
 |`run`|`exec`|
 |`run-action --wait`|`run`|
 
-See the [Juju 3.0 release notes](https://documentation.ubuntu.com/juju/3.6/reference/juju/juju-roadmap-and-releases/#juju-3-0-0-22-oct-2022) for the comprehensive list of changes.
+See the [Juju 3.0 release notes](https://documentation.ubuntu.com/juju/3.6/releasenotes/unsupported/juju_3.x.x/#juju-3-0) for the comprehensive list of changes.
 
 Example substitutions:
 

docs/explanation/logs.md

@@ -78,7 +78,7 @@ The naming convention of the Pgbackrest logs is `<model name>.patroni-<postgresq
 2023-10-11 13:07:45.146 P00   INFO: expire command end: completed successfully (353ms)
 root@pg-0:/# cat /var/log/pgbackrest/discourse.patroni-pg-backup.log 
 -------------------PROCESS START-------------------
-2023-10-11 13:06:29.857 P00   INFO: backup command begin 2.47: --no-backup-standby --exec-id=843-b0d896e1 --log-level-console=debug --pg1-path=/var/lib/postgresql/data/pgdata --pg1-user=backup --repo1-path=/postgresql-test --repo1-retention-full=9999999 --repo1-s3-bucket=dragop-test-bucket --repo1-s3-endpoint=https://s3.eu-central-1.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=eu-central-1 --repo1-s3-uri-style=host --repo1-type=s3 --stanza=discourse.patroni-pg --start-fast --type=full
+2023-10-11 13:06:29.857 P00   INFO: backup command begin 2.47: --no-backup-standby --exec-id=843-b0d896e1 --log-level-console=debug --pg1-path=/var/lib/pg/data/16/main --pg1-user=backup --repo1-path=/postgresql-test --repo1-retention-full=9999999 --repo1-s3-bucket=dragop-test-bucket --repo1-s3-endpoint=https://s3.eu-central-1.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=eu-central-1 --repo1-s3-uri-style=host --repo1-type=s3 --stanza=discourse.patroni-pg --start-fast --type=full
 2023-10-11 13:06:30.869 P00   INFO: execute non-exclusive backup start: backup begins after the requested immediate checkpoint completes
 2023-10-11 13:06:31.671 P00   INFO: backup start archive = 000000010000000000000004, lsn = 0/4000060
 2023-10-11 13:06:31.671 P00   INFO: check archive for prior segment 000000010000000000000003

docs/how-to/data-migration/migrate-data-via-pg-dump.md

@@ -21,7 +21,7 @@ Always test migration in a safe environment before performing it in production!
 
 -  **[Your application is compatible](/explanation/legacy-charm) with Charmed PostgreSQL K8s** 
 - A client machine with access to the deployed legacy charm
-- `juju v.2.9` or later  (check [Juju 3.0 Release Notes](https://documentation.ubuntu.com/juju/3.6/reference/juju/juju-roadmap-and-releases/#juju-3-0-0-22-oct-2022) for more information about key differences)
+- `juju v.2.9` or later  (check [Juju 3.0 Release Notes](https://documentation.ubuntu.com/juju/3.6/releasenotes/unsupported/juju_3.x.x/#juju-3-0) for more information about key differences)
 - Enough storage in the cluster to support backup/restore of the databases.
 
 ## Obtain existing database credentials

docs/reference/troubleshooting.md

@@ -119,9 +119,9 @@ postgresql         enabled   active    today at 12:29 UTC
 root@postgresql-k8s-0:/# ps auxww
 USER         PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
 root           1  0.0  0.0 718264 10916 ?        Ssl  12:29   0:00 /charm/bin/pebble run --create-dirs --hold --http :38813 --verbose
-postgres      14  0.1  0.1 565020 39412 ?        Sl   12:29   0:01 python3 /usr/bin/patroni /var/lib/postgresql/data/patroni.yml
+postgres      14  0.1  0.1 565020 39412 ?        Sl   12:29   0:01 python3 /usr/bin/patroni /var/lib/pg/data/patroni.yml
 postgres      30  0.0  0.0 1082704 9076 ?        Sl   12:30   0:00 /usr/bin/prometheus-postgres-exporter
-postgres      48  0.0  0.0 215488 28912 ?        S    12:30   0:00 /usr/lib/postgresql/14/bin/postgres -D /var/lib/postgresql/data/pgdata --config-file=/var/lib/postgresql/data/pgdata/postgresql.conf --listen_addresses=0.0.0.0 --port=5432 --cluster_name=patroni-postgresql-k8s --wal_level=logical --hot_standby=on --max_connections=100 --max_wal_senders=10 --max_prepared_transactions=0 --max_locks_per_transaction=64 --track_commit_timestamp=off --max_replication_slots=10 --max_worker_processes=8 --wal_log_hints=on
+postgres      48  0.0  0.0 215488 28912 ?        S    12:30   0:00 /usr/lib/postgresql/16/bin/postgres -D /var/lib/pg/data/16/main --config-file=/var/lib/pg/data/16/main/postgresql.conf --listen_addresses=0.0.0.0 --port=5432 --cluster_name=patroni-postgresql-k8s --wal_level=logical --hot_standby=on --max_connections=100 --max_wal_senders=10 --max_prepared_transactions=0 --max_locks_per_transaction=64 --track_commit_timestamp=off --max_replication_slots=10 --max_worker_processes=8 --wal_log_hints=on
 postgres      50  0.0  0.0  70080  7488 ?        Ss   12:30   0:00 postgres: patroni-postgresql-k8s: logger 
 postgres      52  0.0  0.0 215592  9136 ?        Ss   12:30   0:00 postgres: patroni-postgresql-k8s: checkpointer 
 postgres      53  0.0  0.0 215604  9632 ?        Ss   12:30   0:00 postgres: patroni-postgresql-k8s: background writer 

metadata.yaml

@@ -29,19 +29,19 @@ containers:
     resource: postgresql-image
     mounts:
       - storage: archive
-        location: /var/lib/postgresql/archive
+        location: /var/lib/pg/archive
       - storage: data
-        location: /var/lib/postgresql/data
+        location: /var/lib/pg/data
       - storage: logs
-        location: /var/lib/postgresql/logs
+        location: /var/lib/pg/logs
       - storage: temp
-        location: /var/lib/postgresql/temp
+        location: /var/lib/pg/temp
 
 resources:
   postgresql-image:
     type: oci-image
     description: OCI image for PostgreSQL
-    upstream-source: ghcr.io/canonical/charmed-postgresql@sha256:fcb648db2c418403fbf5f58f04892c4677f113526097ca7314cbd9f527bb1baf  # renovate: oci-image tag: 16.11-24.04_edge
+    upstream-source: ghcr.io/canonical/charmed-postgresql@sha256:840100acce8597fe94f1ba5a4d8599a50d4acc9b351d9ec146d9ebb6099e5aa9  # renovate: oci-image tag: 16.11-24.04_edge
 
 peers:
   database-peers:
@@ -109,19 +109,19 @@ storage:
   archive:
     type: filesystem
     description: Storage mount used for holding local backups (before typically sending them to remote object storage) when relevant/needed.
-    location: /var/lib/postgresql/archive
+    location: /var/lib/pg/archive
   data:
     type: filesystem
     description: Storage mount used for storing all tables, indexes, and so on (except those from temporary tablespaces).
-    location: /var/lib/postgresql/data
+    location: /var/lib/pg/data
   logs:
     type: filesystem
     description: Storage mount used for storing all the logs that are part of the transactional commit path (WAL files).
-    location: /var/lib/postgresql/logs
+    location: /var/lib/pg/logs
   temp:
     type: filesystem
     description: Storage mount used for storing temporary tablespaces (where typically sort operations happen).
-    location: /var/lib/postgresql/temp
+    location: /var/lib/pg/temp
 
 assumes:
   - k8s-api

scripts/authorisation_rules_observer.py

@@ -19,7 +19,7 @@
 PATRONI_CLUSTER_STATUS_ENDPOINT = "cluster"
 PATRONI_CONFIG_STATUS_ENDPOINT = "config"
 TLS_CA_BUNDLE_FILE = "peer_ca_bundle.pem"
-PATRONI_CONF_FILE_PATH = "/var/lib/postgresql/data/patroni.yml"
+PATRONI_CONF_FILE_PATH = "/var/lib/pg/data/patroni.yml"
 
 # File path for the spawned cluster topology observer process to write logs.
 LOG_FILE_PATH = "/var/log/authorisation_rules_observer.log"

src/backups.py

@@ -31,9 +31,13 @@
 from tenacity import RetryError, Retrying, stop_after_attempt, wait_fixed
 
 from constants import (
+    ARCHIVE_PATH,
     BACKUP_TYPE_OVERRIDES,
     BACKUP_USER,
+    LOGS_STORAGE_PATH,
     PGBACKREST_LOGROTATE_FILE,
+    POSTGRESQL_DATA_PATH,
+    TEMP_STORAGE_PATH,
     WORKLOAD_OS_GROUP,
     WORKLOAD_OS_USER,
 )
@@ -195,7 +199,7 @@ def can_use_s3_repository(self) -> tuple[bool, str]:
 
             system_identifier_from_instance, error = self._execute_command([
                 f"/usr/lib/postgresql/{self.charm._patroni.rock_postgresql_version.split('.')[0]}/bin/pg_controldata",
-                "/var/lib/postgresql/data/pgdata",
+                POSTGRESQL_DATA_PATH,
             ])
             if error != "":
                 raise Exception(error)
@@ -272,15 +276,24 @@ def _create_bucket_if_not_exists(self) -> None:
 
     def _empty_data_files(self) -> None:
         """Empty the PostgreSQL data directory in preparation of backup restore."""
-        try:
-            self.container.exec(["rm", "-r", "/var/lib/postgresql/data/pgdata"]).wait_output()
-        except ExecError as e:
-            # If previous PITR restore was unsuccessful, there is no such directory.
-            if "No such file or directory" not in str(e.stderr):
-                logger.exception(
-                    "Failed to empty data directory in prep for backup restore", exc_info=e
-                )
-                raise
+        # Clear all storage directories, not just data. The logs directory must be cleared
+        # so that when new replicas join after restore, pg_basebackup can use the --waldir
+        # option (which requires an empty directory).
+        for path in [
+            ARCHIVE_PATH,
+            self.charm._actual_pgdata_path,
+            LOGS_STORAGE_PATH,
+            TEMP_STORAGE_PATH,
+        ]:
+            try:
+                self.container.exec(["find", path, "-mindepth", "1", "-delete"]).wait_output()
+            except ExecError as e:
+                # If previous PITR restore was unsuccessful, there may be no such directory.
+                if "No such file or directory" not in str(e.stderr):
+                    logger.exception(
+                        f"Failed to empty {path} in prep for backup restore", exc_info=e
+                    )
+                    raise
 
     def _change_connectivity_to_database(self, connectivity: bool) -> None:
         """Enable or disable the connectivity to the database."""
@@ -1212,6 +1225,7 @@ def _render_pgbackrest_conf_file(self) -> bool:
             secret_key=s3_parameters["secret-key"],
             stanza=self.stanza_name,
             storage_path=self.charm._storage_path,
+            pgdata_path=POSTGRESQL_DATA_PATH,
             user=BACKUP_USER,
             retention_full=s3_parameters["delete-older-than-days"],
             process_max=max(cpu_count - 2, 1),
@@ -1234,7 +1248,9 @@ def _render_pgbackrest_conf_file(self) -> bool:
                 "/home/postgres/rotate_logs.py",
                 f.read(),
             )
-        self.container.start(self.charm.rotate_logs_service)
+        services = self.container.pebble.get_services(names=[self.charm.rotate_logs_service])
+        if services:
+            self.container.start(self.charm.rotate_logs_service)
 
         return True