Merge branch 'master' into release_2_5

kulaginm · kulaginm · commit ba28354de7f6 · 2021-08-03T22:24:33.000+03:00
diff --git a/.travis.yml b/.travis.yml
@@ -26,6 +26,7 @@ notifications:
 
 # Default MODE is basic, i.e. all tests with PG_PROBACKUP_TEST_BASIC=ON
 env:
+    - PG_VERSION=14 PG_BRANCH=REL_14_STABLE
     - PG_VERSION=13 PG_BRANCH=REL_13_STABLE
     - PG_VERSION=12 PG_BRANCH=REL_12_STABLE
     - PG_VERSION=11 PG_BRANCH=REL_11_STABLE
@@ -42,9 +43,11 @@ env:
 #    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=replica
 #    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=retention
 #    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=restore
+    - PG_VERSION=15 PG_BRANCH=master
 
-#jobs:
-#    allow_failures:
+jobs:
+    allow_failures:
+        - if: env(PG_BRANCH) = master
 #        - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore)
 
 # Only run CI for master branch commits to limit our travis usage
diff --git a/src/stream.c b/src/stream.c
@@ -184,7 +184,12 @@ StreamLog(void *arg)
 #endif
 
 
-#if PG_VERSION_NUM >= 110000
+#if PG_VERSION_NUM >= 150000
+	/* Create temp repslot */
+	if (temp_slot)
+		CreateReplicationSlot(stream_arg->conn, replication_slot,
+			NULL, temp_slot, true, true, false, false);
+#elif PG_VERSION_NUM >= 110000
 	/* Create temp repslot */
 	if (temp_slot)
 		CreateReplicationSlot(stream_arg->conn, replication_slot,
diff --git a/tests/Readme.md b/tests/Readme.md
@@ -30,6 +30,11 @@ Specify path to pg_probackup binary file. By default tests use <Path to Git repo
 Remote backup depends on key authentication to local machine via ssh as current user.
  export PGPROBACKUP_SSH_REMOTE=ON
 
+Run tests that are relied on advanced debugging features. For this mode, pg_probackup should be compiled without optimizations. For example:
+CFLAGS="-O0" ./configure --prefix=/path/to/prefix --enable-debug --enable-cassert --enable-depend --enable-tap-tests
+
+ export PGPROBACKUP_GDB=ON
+
 Run suit of basic simple tests:
  export PG_PROBACKUP_TEST_BASIC=ON
 
diff --git a/tests/__init__.py b/tests/__init__.py
@@ -25,7 +25,8 @@ def load_tests(loader, tests, pattern):
     suite.addTests(loader.loadTestsFromModule(archive))
     suite.addTests(loader.loadTestsFromModule(backup))
     suite.addTests(loader.loadTestsFromModule(catchup))
-    suite.addTests(loader.loadTestsFromModule(compatibility))
+    if 'PGPROBACKUPBIN_OLD' in os.environ and os.environ['PGPROBACKUPBIN_OLD']:
+        suite.addTests(loader.loadTestsFromModule(compatibility))
     suite.addTests(loader.loadTestsFromModule(checkdb))
     suite.addTests(loader.loadTestsFromModule(config))
 #    suite.addTests(loader.loadTestsFromModule(cfs_backup))
diff --git a/tests/archive.py b/tests/archive.py
@@ -1557,8 +1557,8 @@ def test_archive_catalog_1(self):
         double segment - compressed and not
         """
         if not self.archive_compress:
-            return self.fail(
-                'You need to enable ARCHIVE_COMPRESSION for this test to run')
+            self.skipTest('You need to enable ARCHIVE_COMPRESSION '
+                          'for this test to run')
 
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
@@ -1612,8 +1612,8 @@ def test_archive_catalog_2(self):
         double segment - compressed and not
         """
         if not self.archive_compress:
-            return self.fail(
-                'You need to enable ARCHIVE_COMPRESSION for this test to run')
+            self.skipTest('You need to enable ARCHIVE_COMPRESSION '
+                          'for this test to run')
 
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
@@ -1669,6 +1669,9 @@ def test_archive_options(self):
         check that '--archive-host', '--archive-user', '--archiver-port'
         and '--restore-command' are working as expected.
         """
+        if not self.remote:
+            self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE"
+                          " for run this test")
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         node = self.make_simple_node(
diff --git a/tests/backup.py b/tests/backup.py
@@ -2350,45 +2350,47 @@ def test_backup_with_less_privileges_role(self):
 
         replica.slow_start(replica=True)
 
+        # Archive backups from replica in this test are disabled,
+        # because WAL archiving on replica in idle DB in PostgreSQL is broken:
+        # replica will not archive the previous WAL until it receives new records in the next WAL file,
+        # this "lazy" archiving can be seen in src/backend/replication/walreceiver.c:XLogWalRcvWrite()
+        # (see !XLByteInSeg checking and XLogArchiveNotify() calling).
+        #
         # self.switch_wal_segment(node)
-        # self.switch_wal_segment(node)
-
-        self.backup_node(
-            backup_dir, 'replica', replica,
-            datname='backupdb', options=['-U', 'backup'])
+        #self.backup_node(
+        #    backup_dir, 'replica', replica,
+        #    datname='backupdb', options=['-U', 'backup'])
 
         # stream full backup from replica
         self.backup_node(
             backup_dir, 'replica', replica,
             datname='backupdb', options=['--stream', '-U', 'backup'])
 
-#        self.switch_wal_segment(node)
-
         # PAGE backup from replica
-        self.switch_wal_segment(node)
-        self.backup_node(
-            backup_dir, 'replica', replica, backup_type='page',
-            datname='backupdb', options=['-U', 'backup', '--archive-timeout=30s'])
+        #self.switch_wal_segment(node)
+        #self.backup_node(
+        #    backup_dir, 'replica', replica, backup_type='page',
+        #    datname='backupdb', options=['-U', 'backup', '--archive-timeout=30s'])
 
         self.backup_node(
             backup_dir, 'replica', replica, backup_type='page',
             datname='backupdb', options=['--stream', '-U', 'backup'])
 
         # DELTA backup from replica
-        self.switch_wal_segment(node)
-        self.backup_node(
-            backup_dir, 'replica', replica, backup_type='delta',
-            datname='backupdb', options=['-U', 'backup'])
+        #self.switch_wal_segment(node)
+        #self.backup_node(
+        #    backup_dir, 'replica', replica, backup_type='delta',
+        #    datname='backupdb', options=['-U', 'backup'])
         self.backup_node(
             backup_dir, 'replica', replica, backup_type='delta',
             datname='backupdb', options=['--stream', '-U', 'backup'])
 
         # PTRACK backup from replica
         if self.ptrack:
-            self.switch_wal_segment(node)
-            self.backup_node(
-                backup_dir, 'replica', replica, backup_type='ptrack',
-                datname='backupdb', options=['-U', 'backup'])
+            #self.switch_wal_segment(node)
+            #self.backup_node(
+            #    backup_dir, 'replica', replica, backup_type='ptrack',
+            #    datname='backupdb', options=['-U', 'backup'])
             self.backup_node(
                 backup_dir, 'replica', replica, backup_type='ptrack',
                 datname='backupdb', options=['--stream', '-U', 'backup'])
diff --git a/tests/checkdb.py b/tests/checkdb.py
@@ -450,6 +450,11 @@ def test_checkdb_block_validation_sanity(self):
     # @unittest.skip("skip")
     def test_checkdb_sigint_handling(self):
         """"""
+        if not self.gdb:
+            self.skipTest(
+                "Specify PGPROBACKUP_GDB and build without "
+                "optimizations for run this test"
+            )
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         node = self.make_simple_node(
diff --git a/tests/delta.py b/tests/delta.py
@@ -472,6 +472,11 @@ def test_delta_vacuum_full(self):
         make node, make full and delta stream backups,
         restore them and check data correctness
         """
+        if not self.gdb:
+            self.skipTest(
+                "Specify PGPROBACKUP_GDB and build without "
+                "optimizations for run this test"
+            )
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         node = self.make_simple_node(
@@ -508,7 +513,7 @@ def test_delta_vacuum_full(self):
         process.start()
 
         while not gdb.stopped_in_breakpoint:
-            sleep(1)
+            time.sleep(1)
 
         gdb.continue_execution_until_break(20)
 
diff --git a/tests/external.py b/tests/external.py
@@ -373,6 +373,9 @@ def test_external_backward_compatibility(self):
         restore delta backup, check that incremental chain
         restored correctly
         """
+        if not self.probackup_old_path:
+            self.skipTest("You must specify PGPROBACKUPBIN_OLD"
+                          " for run this test")
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         node = self.make_simple_node(
@@ -474,6 +477,9 @@ def test_external_backward_compatibility_merge_1(self):
         take delta backup with new binary and 2 external directories
         merge delta backup ajd restore it
         """
+        if not self.probackup_old_path:
+            self.skipTest("You must specify PGPROBACKUPBIN_OLD"
+                          " for run this test")
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         node = self.make_simple_node(
@@ -566,6 +572,9 @@ def test_external_backward_compatibility_merge_2(self):
         take delta backup with new binary and 2 external directories
         merge delta backup and restore it
         """
+        if not self.probackup_old_path:
+            self.skipTest("You must specify PGPROBACKUPBIN_OLD"
+                          " for run this test")
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         node = self.make_simple_node(
@@ -687,6 +696,9 @@ def test_external_backward_compatibility_merge_2(self):
     # @unittest.skip("skip")
     def test_external_merge(self):
         """"""
+        if not self.probackup_old_path:
+            self.skipTest("You must specify PGPROBACKUPBIN_OLD"
+                          " for run this test")
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         node = self.make_simple_node(
diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py
@@ -184,15 +184,15 @@ def __init__(self, *args, **kwargs):
         self.test_env['LC_MESSAGES'] = 'C'
         self.test_env['LC_TIME'] = 'C'
 
-        self.paranoia = False
-        if 'PG_PROBACKUP_PARANOIA' in self.test_env:
-            if self.test_env['PG_PROBACKUP_PARANOIA'] == 'ON':
-                self.paranoia = True
-
-        self.archive_compress = False
-        if 'ARCHIVE_COMPRESSION' in self.test_env:
-            if self.test_env['ARCHIVE_COMPRESSION'] == 'ON':
-                self.archive_compress = True
+        self.gdb = 'PGPROBACKUP_GDB' in os.environ and \
+              os.environ['PGPROBACKUP_GDB'] == 'ON'
+
+        self.paranoia = 'PG_PROBACKUP_PARANOIA' in self.test_env and \
+            self.test_env['PG_PROBACKUP_PARANOIA'] == 'ON'
+
+        self.archive_compress = 'ARCHIVE_COMPRESSION' in self.test_env and \
+            self.test_env['ARCHIVE_COMPRESSION'] == 'ON'
+
         try:
             testgres.configure_testgres(
                 cache_initdb=False,
diff --git a/tests/pgpro2068.py b/tests/pgpro2068.py
@@ -18,6 +18,11 @@ def test_minrecpoint_on_replica(self):
         """
         https://jira.postgrespro.ru/browse/PGPRO-2068
         """
+        if not self.gdb:
+            self.skipTest(
+                "Specify PGPROBACKUP_GDB and build without "
+                "optimizations for run this test"
+            )
         fname = self.id().split('.')[3]
         node = self.make_simple_node(
             base_dir=os.path.join(module_name, fname, 'node'),
diff --git a/tests/replica.py b/tests/replica.py
@@ -625,6 +625,11 @@ def test_replica_promote(self):
     def test_replica_stop_lsn_null_offset(self):
         """
         """
+        if not self.gdb:
+            self.skipTest(
+                "Specify PGPROBACKUP_GDB and build without "
+                "optimizations for run this test"
+            )
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         master = self.make_simple_node(
@@ -707,6 +712,11 @@ def test_replica_stop_lsn_null_offset(self):
     def test_replica_stop_lsn_null_offset_next_record(self):
         """
         """
+        if not self.gdb:
+            self.skipTest(
+                "Specify PGPROBACKUP_GDB and build without "
+                "optimizations for run this test"
+            )
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         master = self.make_simple_node(
@@ -976,6 +986,11 @@ def test_replica_toast(self):
         make archive master, take full and page archive backups from master,
         set replica, make archive backup from replica
         """
+        if not self.gdb:
+            self.skipTest(
+                "Specify PGPROBACKUP_GDB and build without "
+                "optimizations for run this test"
+            )
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         master = self.make_simple_node(
@@ -1076,6 +1091,11 @@ def test_replica_toast(self):
     def test_start_stop_lsn_in_the_same_segno(self):
         """
         """
+        if not self.gdb:
+            self.skipTest(
+                "Specify PGPROBACKUP_GDB and build without "
+                "optimizations for run this test"
+            )
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         master = self.make_simple_node(
diff --git a/tests/restore.py b/tests/restore.py
@@ -189,7 +189,7 @@ def test_restore_to_time(self):
         node = self.make_simple_node(
             base_dir=os.path.join(module_name, fname, 'node'),
             initdb_params=['--data-checksums'],
-            pg_options={'TimeZone': 'Europe/Moscow'})
+            pg_options={'TimeZone': 'GMT'})
 
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
@@ -202,7 +202,9 @@ def test_restore_to_time(self):
 
         backup_id = self.backup_node(backup_dir, 'node', node)
 
-        target_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        target_time = node.execute(
+            "postgres", "SELECT to_char(now(), 'YYYY-MM-DD HH24:MI:SS+00')"
+        )[0][0]
         pgbench = node.pgbench(
             stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
         pgbench.wait()
@@ -2843,6 +2845,9 @@ def test_partial_restore_backward_compatibility_1(self):
         """
         old binary should be of version < 2.2.0
         """
+        if not self.probackup_old_path:
+            self.skipTest("You must specify PGPROBACKUPBIN_OLD"
+                          " for run this test")
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         node = self.make_simple_node(
@@ -2946,6 +2951,9 @@ def test_partial_restore_backward_compatibility_merge(self):
         """
         old binary should be of version < 2.2.0
         """
+        if not self.probackup_old_path:
+            self.skipTest("You must specify PGPROBACKUPBIN_OLD"
+                          " for run this test")
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         node = self.make_simple_node(
@@ -3621,7 +3629,9 @@ def test_pg_12_probackup_recovery_conf_compatibility(self):
 
         pg_probackup version must be 12 or greater
         """
-
+        if not self.probackup_old_path:
+            self.skipTest("You must specify PGPROBACKUPBIN_OLD"
+                          " for run this test")
         if self.pg_config_version < self.version_to_num('12.0'):
            return unittest.skip('You need PostgreSQL >= 12 for this test')
 
diff --git a/tests/show.py b/tests/show.py
diff --git a/tests/snapfs.py b/tests/snapfs.py
diff --git a/tests/validate.py b/tests/validate.py
diff --git a/travis/run_tests.sh b/travis/run_tests.sh
