tests: minor fixes

Author: gsmolk

tests/archive.py

@@ -5,6 +5,7 @@
 import subprocess
 from sys import exit
 from time import sleep
+from shutil import copyfile
 
 
 module_name = 'archive'
@@ -39,8 +40,7 @@ def test_pgpro434_1(self):
 
         result = node.safe_psql("postgres", "SELECT * FROM t_heap")
         self.backup_node(
-            backup_dir, 'node', node,
-            options=["--log-level-file=verbose"])
+            backup_dir, 'node', node)
         node.cleanup()
 
         self.restore_node(
@@ -53,8 +53,7 @@ def test_pgpro434_1(self):
 
         # Make backup
         self.backup_node(
-            backup_dir, 'node', node,
-            options=["--log-level-file=verbose"])
+            backup_dir, 'node', node)
         node.cleanup()
 
         # Restore Database
@@ -253,7 +252,6 @@ def test_pgpro434_3(self):
                 backup_dir, 'node', node,
                 options=[
                     "--archive-timeout=60",
-                    "--log-level-file=verbose",
                     "--stream"]
                     )
             # we should die here because exception is what we expect to happen
@@ -402,7 +400,7 @@ def test_arhive_push_file_exists_overwrite(self):
         self.del_test_dir(module_name, fname)
 
     # @unittest.expectedFailure
-    # @unittest.skip("skip")
+    @unittest.skip("skip")
     def test_replica_archive(self):
         """
         make node without archiving, take stream backup and
@@ -417,7 +415,7 @@ def test_replica_archive(self):
             initdb_params=['--data-checksums'],
             pg_options={
                 'max_wal_senders': '2',
-                'checkpoint_timeout': '30s',
+                'archive_timeout': '10s',
                 'max_wal_size': '1GB'}
             )
         self.init_pb(backup_dir)
@@ -433,7 +431,7 @@ def test_replica_archive(self):
             "postgres",
             "create table t_heap as select i as id, md5(i::text) as text, "
             "md5(repeat(i::text,10))::tsvector as tsvector "
-            "from generate_series(0,256) i")
+            "from generate_series(0,2560) i")
 
         self.backup_node(backup_dir, 'master', master, options=['--stream'])
         before = master.safe_psql("postgres", "SELECT * FROM t_heap")
@@ -459,17 +457,16 @@ def test_replica_archive(self):
             "md5(repeat(i::text,10))::tsvector as tsvector "
             "from generate_series(256,512) i")
         before = master.safe_psql("postgres", "SELECT * FROM t_heap")
-        # ADD INSTANCE 'REPLICA'
-
-        sleep(1)
 
         backup_id = self.backup_node(
             backup_dir, 'replica', replica,
             options=[
                 '--archive-timeout=30',
                 '--master-host=localhost',
                 '--master-db=postgres',
-                '--master-port={0}'.format(master.port)])
+                '--master-port={0}'.format(master.port),
+                '--stream'])
+
         self.validate_pb(backup_dir, 'replica')
         self.assertEqual(
             'OK', self.show_pb(backup_dir, 'replica', backup_id)['status'])
@@ -493,16 +490,28 @@ def test_replica_archive(self):
             "postgres",
             "insert into t_heap as select i as id, md5(i::text) as text, "
             "md5(repeat(i::text,10))::tsvector as tsvector "
-            "from generate_series(512,768) i")
+            "from generate_series(512,20680) i")
+
         before = master.safe_psql("postgres", "SELECT * FROM t_heap")
+
+        master.safe_psql(
+            "postgres",
+            "CHECKPOINT")
+
+#        copyfile(
+#            os.path.join(backup_dir, 'wal/master/000000010000000000000002'),
+#            os.path.join(backup_dir, 'wal/replica/000000010000000000000002'))
+
         backup_id = self.backup_node(
             backup_dir, 'replica',
             replica, backup_type='page',
             options=[
-                '--archive-timeout=30', '--log-level-file=verbose',
-                '--master-host=localhost', '--master-db=postgres',
-                '--master-port={0}'.format(master.port)]
-                )
+                '--archive-timeout=30',
+                '--master-db=postgres',
+                '--master-host=localhost',
+                '--master-port={0}'.format(master.port),
+                '--stream'])
+
         self.validate_pb(backup_dir, 'replica')
         self.assertEqual(
             'OK', self.show_pb(backup_dir, 'replica', backup_id)['status'])
@@ -511,8 +520,10 @@ def test_replica_archive(self):
         node.cleanup()
         self.restore_node(
             backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id)
+
         node.append_conf(
             'postgresql.auto.conf', 'port = {0}'.format(node.port))
+
         node.slow_start()
         # CHECK DATA CORRECTNESS
         after = node.safe_psql("postgres", "SELECT * FROM t_heap")
@@ -537,7 +548,7 @@ def test_master_and_replica_parallel_archiving(self):
             set_replication=True,
             initdb_params=['--data-checksums'],
             pg_options={
-                'checkpoint_timeout': '30s'}
+                'archive_timeout': '10s'}
             )
         replica = self.make_simple_node(
             base_dir="{0}/{1}/replica".format(module_name, fname))
@@ -568,7 +579,7 @@ def test_master_and_replica_parallel_archiving(self):
         pgdata_replica = self.pgdata_content(replica.data_dir)
         self.compare_pgdata(pgdata_master, pgdata_replica)
 
-        self.set_replica(master, replica, synchronous=True)
+        self.set_replica(master, replica)
         # ADD INSTANCE REPLICA
         self.add_instance(backup_dir, 'replica', replica)
         # SET ARCHIVING FOR REPLICA
@@ -579,16 +590,26 @@ def test_master_and_replica_parallel_archiving(self):
         after = replica.safe_psql("postgres", "SELECT * FROM t_heap")
         self.assertEqual(before, after)
 
+        master.psql(
+            "postgres",
+            "insert into t_heap select i as id, md5(i::text) as text, "
+            "md5(repeat(i::text,10))::tsvector as tsvector "
+            "from generate_series(0, 60000) i")
+
         # TAKE FULL ARCHIVE BACKUP FROM REPLICA
+        copyfile(
+             os.path.join(backup_dir, 'wal/master/000000010000000000000001'),
+             os.path.join(backup_dir, 'wal/replica/000000010000000000000001'))
+
         backup_id = self.backup_node(
             backup_dir, 'replica', replica,
             options=[
-                '--archive-timeout=20',
-                '--log-level-file=verbose',
+                '--archive-timeout=30',
                 '--master-host=localhost',
                 '--master-db=postgres',
-                '--master-port={0}'.format(master.port)]
-                )
+                '--master-port={0}'.format(master.port),
+                '--stream'])
+
         self.validate_pb(backup_dir, 'replica')
         self.assertEqual(
             'OK', self.show_pb(backup_dir, 'replica', backup_id)['status'])
@@ -618,7 +639,8 @@ def test_master_and_replica_concurrent_archiving(self):
             set_replication=True,
             initdb_params=['--data-checksums'],
             pg_options={
-                'checkpoint_timeout': '30s'}
+                'checkpoint_timeout': '30s',
+                'archive_timeout': '10s'}
             )
         replica = self.make_simple_node(
             base_dir="{0}/{1}/replica".format(module_name, fname))

tests/backup_test.py

@@ -328,7 +328,7 @@ def test_page_corruption_heal_via_ptrack_1(self):
 
         self.backup_node(
             backup_dir, 'node', node, backup_type="full",
-            options=["-j", "4", "--stream", '--log-level-file=verbose'])
+            options=["-j", "4", "--stream", "--log-level-file=verbose"])
 
         # open log file and check
         with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f:

tests/compatibility.py

@@ -94,8 +94,7 @@ def test_backward_compatibility_page(self):
         pgbench.stdout.close()
 
         self.backup_node(
-            backup_dir, 'node', node, backup_type='page',
-            options=['--log-level-file=verbose'])
+            backup_dir, 'node', node, backup_type='page')
 
         if self.paranoia:
             pgdata = self.pgdata_content(node.data_dir)
@@ -195,8 +194,7 @@ def test_backward_compatibility_delta(self):
         pgbench.stdout.close()
 
         self.backup_node(
-            backup_dir, 'node', node, backup_type='delta',
-            options=['--log-level-file=verbose'])
+            backup_dir, 'node', node, backup_type='delta')
 
         if self.paranoia:
             pgdata = self.pgdata_content(node.data_dir)
@@ -296,8 +294,7 @@ def test_backward_compatibility_ptrack(self):
         pgbench.stdout.close()
 
         self.backup_node(
-            backup_dir, 'node', node, backup_type='delta',
-            options=['--log-level-file=verbose'])
+            backup_dir, 'node', node, backup_type='delta')
 
         if self.paranoia:
             pgdata = self.pgdata_content(node.data_dir)

tests/compression.py

@@ -55,9 +55,7 @@ def test_compression_stream_zlib(self):
         page_backup_id = self.backup_node(
             backup_dir, 'node', node, backup_type='page',
             options=[
-                '--stream', '--compress-algorithm=zlib',
-                '--log-level-console=verbose',
-                '--log-level-file=verbose'])
+                '--stream', '--compress-algorithm=zlib'])
 
         # PTRACK BACKUP
         node.safe_psql(
@@ -535,8 +533,7 @@ def test_uncompressable_pages(self):
             backup_dir, 'node', node,
             backup_type='full',
             options=[
-                '--compress',
-                '--log-level-file=verbose'])
+                '--compress'])
 
         node.cleanup()
 
@@ -547,8 +544,7 @@ def test_uncompressable_pages(self):
             backup_dir, 'node', node,
             backup_type='full',
             options=[
-                '--compress',
-                '--log-level-file=verbose'])
+                '--compress'])
 
         # Clean after yourself
         # self.del_test_dir(module_name, fname)

tests/delta.py

@@ -80,13 +80,7 @@ def test_delta_vacuum_truncate_1(self):
         pgdata = self.pgdata_content(node.data_dir)
 
         self.restore_node(
-            backup_dir,
-            'node',
-            node_restored,
-            options=[
-                "-j", "1",
-                "--log-level-file=verbose"
-            ]
+            backup_dir, 'node', node_restored
         )
 
         # Physical comparison
@@ -176,8 +170,6 @@ def test_delta_vacuum_truncate_2(self):
             'node',
             node_restored,
             options=[
-                "-j", "1",
-                "--log-level-file=verbose",
                 "-T", "{0}={1}".format(
                     old_tablespace, new_tablespace)]
         )
@@ -251,13 +243,7 @@ def test_delta_vacuum_truncate_3(self):
         pgdata = self.pgdata_content(node.data_dir)
 
         self.restore_node(
-            backup_dir,
-            'node',
-            node_restored,
-            options=[
-                "-j", "1",
-                "--log-level-file=verbose"
-            ]
+            backup_dir, 'node', node_restored
         )
 
         # Physical comparison
@@ -683,7 +669,7 @@ def test_create_db(self):
             node_restored,
             backup_id=backup_id,
             options=[
-                "-j", "4", "--log-level-file=verbose",
+                "-j", "4",
                 "--immediate",
                 "--recovery-target-action=promote"])
 
@@ -717,7 +703,7 @@ def test_create_db(self):
             node_restored,
             backup_id=backup_id,
             options=[
-                "-j", "4", "--log-level-file=verbose",
+                "-j", "4",
                 "--immediate",
                 "--recovery-target-action=promote"]
         )
@@ -815,7 +801,7 @@ def test_exists_in_previous_backup(self):
         backup_id = self.backup_node(
             backup_dir, 'node', node,
             backup_type='delta',
-            options=["--stream", "--log-level-file=verbose"]
+            options=["--stream"]
         )
 #        if self.paranoia:
 #            pgdata_delta = self.pgdata_content(
@@ -844,7 +830,7 @@ def test_exists_in_previous_backup(self):
             node_restored,
             backup_id=backup_id,
             options=[
-                "-j", "4", "--log-level-file=verbose",
+                "-j", "4",
                 "--immediate",
                 "--recovery-target-action=promote"])
 
@@ -1135,7 +1121,7 @@ def test_delta_delete(self):
         self.del_test_dir(module_name, fname)
 
     # @unittest.skip("skip")
-    def test_page_corruption_heal_via_ptrack_1(self):
+    def test_delta_corruption_heal_via_ptrack_1(self):
         """make node, corrupt some page, check that backup failed"""
         fname = self.id().split('.')[3]
         node = self.make_simple_node(
@@ -1174,8 +1160,10 @@ def test_page_corruption_heal_via_ptrack_1(self):
                 f.close
 
         self.backup_node(
-            backup_dir, 'node', node, backup_type="delta",
-            options=["-j", "4", "--stream", "--log-level-file=verbose"])
+            backup_dir, 'node', node,
+            backup_type="delta",
+            options=["-j", "4", "--stream", '--log-level-file=verbose'])
+
 
         # open log file and check
         with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f:

tests/exclude.py

@@ -143,7 +143,7 @@ def test_exclude_unlogged_tables_1(self):
 
         self.backup_node(
             backup_dir, 'node', node, backup_type='ptrack',
-            options=['--stream', '--log-level-file=verbose']
+            options=['--stream']
         )
 
         pgdata = self.pgdata_content(node.data_dir)

tests/false_positive.py

@@ -143,7 +143,7 @@ def test_ptrack_concurrent_get_and_clear_1(self):
         self.backup_node(backup_dir, 'node', node, options=['--stream'])
         gdb = self.backup_node(
             backup_dir, 'node', node, backup_type='ptrack',
-            options=['--stream', '--log-level-file=verbose'],
+            options=['--stream'],
             gdb=True
         )
 
@@ -227,7 +227,7 @@ def test_ptrack_concurrent_get_and_clear_2(self):
         self.backup_node(backup_dir, 'node', node, options=['--stream'])
         gdb = self.backup_node(
             backup_dir, 'node', node, backup_type='ptrack',
-            options=['--stream', '--log-level-file=verbose'],
+            options=['--stream'],
             gdb=True
         )