Move to spilo-18/spilo-cdp-18 (#1191)

- Support PG18
- Patroni 4.1.0
- bg_mon from the original repo

Not included for PG18:
- pglogical-ticker
- pgl-ddl-deploy

---------

Co-authored-by: Ida Novindasari <idanovinda@gmail.com>

Author: hughcapet

delivery.yaml

@@ -3,7 +3,8 @@ allow_concurrent_steps: true
 
 build_env: &BUILD_ENV
   BASE_IMAGE: container-registry.zalando.net/library/ubuntu-22.04
-  PGVERSION: 17
+  PGVERSION: 18
+  PGOLDVERSIONS: "16 17"
   MULTI_ARCH_REGISTRY: container-registry-test.zalando.net/acid
 
 pipeline:
@@ -32,7 +33,7 @@ pipeline:
       docker buildx build --platform "linux/amd64,linux/arm64" \
                           --build-arg PGVERSION="$PGVERSION" \
                           --build-arg BASE_IMAGE="$BASE_IMAGE" \
-                          --build-arg PGOLDVERSIONS="14 15 16" \
+                          --build-arg PGOLDVERSIONS="$PGOLDVERSIONS" \
                           -t "$ECR_TEST_IMAGE" \
                           --push .
 
@@ -61,7 +62,7 @@ pipeline:
       docker buildx build --platform "linux/amd64,linux/arm64" \
                           --build-arg PGVERSION="$PGVERSION" \
                           --build-arg BASE_IMAGE="$BASE_IMAGE" \
-                          --build-arg PGOLDVERSIONS="14 15 16" \
+                          --build-arg PGOLDVERSIONS="$PGOLDVERSIONS" \
                           -t "$ECR_TEST_IMAGE" \
                           --push .
       cdp-promote-image "$ECR_TEST_IMAGE"
@@ -92,7 +93,7 @@ pipeline:
       docker buildx build --platform "linux/amd64,linux/arm64" \
                           --build-arg PGVERSION="$PGVERSION" \
                           --build-arg BASE_IMAGE="$BASE_IMAGE" \
-                          --build-arg PGOLDVERSIONS="14 15 16" \
+                          --build-arg PGOLDVERSIONS="$PGOLDVERSIONS" \
                           -t "$ECR_TEST_IMAGE" \
                           --push .
       cdp-promote-image "$ECR_TEST_IMAGE"

postgres-appliance/Dockerfile

@@ -1,5 +1,5 @@
 ARG BASE_IMAGE=ubuntu:22.04
-ARG PGVERSION=17
+ARG PGVERSION=18
 ARG DEMO=false
 ARG COMPRESS=false
 ARG ADDITIONAL_LOCALES=
@@ -46,7 +46,7 @@ ARG PGVERSION
 ARG TIMESCALEDB_APACHE_ONLY=true
 ARG TIMESCALEDB_TOOLKIT=true
 ARG COMPRESS
-ARG PGOLDVERSIONS="13 14 15 16"
+ARG PGOLDVERSIONS="14 15 16 17"
 ARG WITH_PERL=false
 
 ARG DEB_PG_SUPPORTED_VERSIONS="$PGOLDVERSIONS $PGVERSION"
@@ -69,7 +69,7 @@ COPY --from=dependencies-builder /builddeps/wal-g /usr/local/bin/
 COPY build_scripts/patroni.sh build_scripts/compress_build.sh /builddeps/
 
 # Install patroni
-ENV PATRONIVERSION=4.0.6
+ENV PATRONIVERSION=4.1.0
 
 WORKDIR /
 

postgres-appliance/bootstrap/maybe_pg_upgrade.py

@@ -40,7 +40,7 @@ def perform_pitr(postgresql, cluster_version, bin_version, config):
     except Exception:
         logs = tail_postgres_logs()
         # Spilo has no other locales except en_EN.UTF-8, therefore we are safe here.
-        if int(cluster_version) >= 13 and 'recovery ended before configured recovery target was reached' in logs:
+        if 'recovery ended before configured recovery target was reached' in logs:
             # Starting from version 13 Postgres stopped promoting when recovery target wasn't reached.
             # In order to improve the user experience we reset all possible recovery targets and retry.
             recovery_conf = config[config['method']].get('recovery_conf', {})
@@ -103,7 +103,7 @@ def main():
     except Exception as e:
         logger.error('Failed to update extensions: %r', e)
 
-    upgrade.analyze()
+    upgrade.analyze(bin_version)
 
 
 def call_maybe_pg_upgrade():

postgres-appliance/build_scripts/base.sh

@@ -51,7 +51,7 @@ if [ "$WITH_PERL" != "true" ]; then
     equivs-build perl
 fi
 
-curl -sL "https://github.com/zalando-pg/bg_mon/archive/$BG_MON_COMMIT.tar.gz" | tar xz
+curl -sL "https://github.com/CyberDem0n/bg_mon/archive/$BG_MON_COMMIT.tar.gz" | tar xz
 curl -sL "https://github.com/zalando-pg/pg_auth_mon/archive/$PG_AUTH_MON_COMMIT.tar.gz" | tar xz
 curl -sL "https://github.com/zubkov-andrei/pg_profile/archive/$PG_PROFILE.tar.gz" | tar xz
 
@@ -83,10 +83,8 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do
                 "postgresql-${version}-pgaudit"
                 "postgresql-${version}-pldebugger"
                 "postgresql-${version}-pglogical"
-                "postgresql-${version}-pglogical-ticker"
                 "postgresql-${version}-plpgsql-check"
                 "postgresql-${version}-pg-checksums"
-                "postgresql-${version}-pgl-ddl-deploy"
                 "postgresql-${version}-pgq-node"
                 "postgresql-${version}-postgis-${POSTGIS_VERSION%.*}"
                 "postgresql-${version}-postgis-${POSTGIS_VERSION%.*}-scripts"
@@ -95,10 +93,12 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do
                 "postgresql-${version}-decoderbufs"
                 "postgresql-${version}-pllua"
                 "postgresql-${version}-pgvector"
-                "postgresql-${version}-roaringbitmap")
+                "postgresql-${version}-roaringbitmap"
+                "postgresql-${version}-pgfaceting")
 
-        if [ "$version" -ge 14 ]; then
-            EXTRAS+=("postgresql-${version}-pgfaceting")
+        if [ "$version" != "18" ]; then
+            EXTRAS+=("postgresql-${version}-pgl-ddl-deploy"
+                    "postgresql-${version}-pglogical-ticker")
         fi
 
         if [ "$WITH_PERL" = "true" ]; then

postgres-appliance/major_upgrade/inplace_upgrade.py

@@ -451,7 +451,7 @@ def restore_custom_statistics_target(self):
                         except Exception:
                             logger.error("Failed to execute '%s'", query)
 
-    def reanalyze(self):
+    def custom_stats_target_reanalyze(self):
         from patroni.postgresql.connection import get_connection_cursor
 
         if not self._statistics:
@@ -470,12 +470,15 @@ def reanalyze(self):
                     except Exception:
                         logger.error("Failed to execute '%s'", query)
 
+    def full_reanalyze(self):
+        self.postgresql.analyze(self.desired_version)
+
     def analyze(self):
         try:
             self.reset_custom_statistics_target()
         except Exception as e:
             logger.error('Failed to reset custom statistics targets: %r', e)
-        self.postgresql.analyze(True)
+        self.postgresql.analyze(self.desired_version, in_stages=True)
         try:
             self.restore_custom_statistics_target()
         except Exception as e:
@@ -634,7 +637,10 @@ def do_upgrade(self):
 
         analyze_thread.join()
 
-        self.reanalyze()
+        if int(self.desired_version) < 18:
+            self.custom_stats_target_reanalyze()
+        else:
+            self.full_reanalyze()
 
         logger.info('Total upgrade time (with analyze): %s', time.time() - downtime_start)
         self.postgresql.bootstrap.call_post_bootstrap(self.config['bootstrap'])

postgres-appliance/major_upgrade/pg_upgrade.py

@@ -207,8 +207,9 @@ def prepare_new_pgdata(self, version):
         locale = self.query("SELECT datcollate FROM pg_database WHERE datname='template1';")[0][0]
         encoding = self.query('SHOW server_encoding')[0][0]
         initdb_config = [{'locale': locale}, {'encoding': encoding}]
-        if self.query("SELECT current_setting('data_checksums')::bool")[0][0]:
-            initdb_config.append('data-checksums')
+        checksums_enabled = self.query("SELECT current_setting('data_checksums')::bool")[0][0]
+        if checksums_enabled == (int(version) < 18):
+            initdb_config.append('data-checksums' if checksums_enabled else 'no-data-checksums')
 
         logger.info('initdb config: %s', initdb_config)
 
@@ -268,9 +269,12 @@ def do_upgrade(self):
         return self.pg_upgrade() and self.restore_shared_preload_libraries()\
                  and self.switch_pgdata() and self.cleanup_old_pgdata()
 
-    def analyze(self, in_stages=False):
-        vacuumdb_args = ['--analyze-in-stages'] if in_stages else []
-        logger.info('Rebuilding statistics (vacuumdb%s)', (' ' + vacuumdb_args[0] if in_stages else ''))
+    def analyze(self, version, in_stages=False):
+        vacuumdb_args = []
+        if in_stages:
+            vacuumdb_args = ['--analyze-in-stages'] if int(version) < 18 else ['--analyze-in-stages',
+                                                                               '--missing-stats-only']
+        logger.info('Rebuilding statistics (vacuumdb%s)', (' ' + ' '.join(vacuumdb_args) if in_stages else ''))
         if 'username' in self.config.superuser:
             vacuumdb_args += ['-U', self.config.superuser['username']]
         vacuumdb_args += ['-Z', '-j']

postgres-appliance/scripts/post_init.sh

@@ -139,16 +139,12 @@ CREATE TABLE IF NOT EXISTS public.postgres_log (
     query_pos integer,
     location text,
     application_name text,
+    backend_type text,
+    leader_pid integer,
+    query_id bigint,
     CONSTRAINT postgres_log_check CHECK (false) NO INHERIT
 );
 GRANT SELECT ON public.postgres_log TO admin;"
-if [ "$PGVER" -ge 13 ]; then
-    echo "ALTER TABLE public.postgres_log ADD COLUMN IF NOT EXISTS backend_type text;"
-fi
-if [ "$PGVER" -ge 14 ]; then
-    echo "ALTER TABLE public.postgres_log ADD COLUMN IF NOT EXISTS leader_pid integer;"
-    echo "ALTER TABLE public.postgres_log ADD COLUMN IF NOT EXISTS query_id bigint;"
-fi
 
 # Sunday could be 0 or 7 depending on the format, we just create both
 LOG_SHIP_HOURLY=$(echo "SELECT text(current_setting('log_rotation_age') = '1h')" | psql -tAX -d postgres 2> /dev/null | tail -n 1)

postgres-appliance/scripts/spilo_commons.py

@@ -12,13 +12,13 @@
 
 # (min_version, max_version, shared_preload_libraries, extwlist.extensions)
 extensions = {
-    'timescaledb':    (9.6, 17, True,  True),
-    'pg_cron':        (9.5, 17, True,  False),
-    'pg_stat_kcache': (9.4, 17, True,  False),
-    'pg_partman':     (9.4, 17, False, True)
+    'timescaledb':    (9.6, 18, True,  True),
+    'pg_cron':        (9.5, 18, True,  False),
+    'pg_stat_kcache': (9.4, 18, True,  False),
+    'pg_partman':     (9.4, 18, False, True)
 }
 if os.environ.get('ENABLE_PG_MON') == 'true':
-    extensions['pg_mon'] = (11,  17, True,  False)
+    extensions['pg_mon'] = (11,  18, True,  False)
 
 
 def adjust_extensions(old, version, extwlist=False):

postgres-appliance/tests/docker-compose.yml

@@ -50,7 +50,7 @@ services:
               postgresql:
                 parameters:
                   shared_buffers: 32MB
-            PGVERSION: '13'
+            PGVERSION: '14'
             # Just to test upgrade with clone. Without CLONE_SCOPE they don't work
             CLONE_WAL_S3_BUCKET: *bucket
             CLONE_AWS_ACCESS_KEY_ID: *access_key