Merge remote-tracking branch 'origin/testgres1'

Author: gsmolk

tests/auth_test.py

@@ -8,7 +8,7 @@
 import signal
 
 from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
-from testgres import ClusterException as StartNodeException
+from testgres import StartNodeException
 
 module_name = 'auth_test'
 skip_test = False
@@ -123,7 +123,7 @@ def test_right_password(self):
             self.fail(e.value)
 
     def test_right_password_and_wrong_pgpass(self):
-        """ Test case: PGPB_AUTH05 - correct password and incorrect .pgpass"""
+        """ Test case: PGPB_AUTH05 - correct password and incorrect .pgpass (-W)"""
         line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'wrong_password'])
         create_pgpass(self.pgpass_file, line)
         try:
@@ -197,7 +197,7 @@ def test_pgpassword_and_wrong_pgpass(self):
 
 def run_pb_with_auth(cmd, password=None, kill=False):
     try:
-        with spawn(" ".join(cmd), timeout=10) as probackup:
+        with spawn(" ".join(cmd), encoding='utf-8', timeout=10) as probackup:
             result = probackup.expect("Password for user .*:", 5)
             if kill:
                 probackup.kill(signal.SIGINT)

tests/helpers/ptrack_helpers.py

@@ -11,7 +11,7 @@
 import select
 import psycopg2
 from time import sleep
-
+import re
 
 idx_ptrack = {
     't_heap': {
@@ -156,6 +156,14 @@ def __init__(self, *args, **kwargs):
         if 'ARCHIVE_COMPRESSION' in self.test_env:
             if self.test_env['ARCHIVE_COMPRESSION'] == 'ON':
                 self.archive_compress = True
+        try:
+            testgres.configure_testgres(
+                cache_initdb=False,
+                cached_initdb_dir=False,
+                cache_pg_config=False,
+                node_cleanup_full=False)
+        except:
+            pass
 
         self.helpers_path = os.path.dirname(os.path.realpath(__file__))
         self.dir_path = os.path.abspath(
@@ -193,11 +201,16 @@ def make_simple_node(
 
         real_base_dir = os.path.join(self.tmp_path, base_dir)
         shutil.rmtree(real_base_dir, ignore_errors=True)
+        os.makedirs(real_base_dir)
 
         node = testgres.get_new_node('test', base_dir=real_base_dir)
-        node.init(initdb_params=initdb_params)
+        node.should_rm_dirs = True
+        node.init(
+           initdb_params=initdb_params, allow_streaming=set_replication)
+        print(node.data_dir)
 
         # Sane default parameters, not a shit with fsync = off from testgres
+        node.append_conf("postgresql.auto.conf", "max_connections = 100")
         node.append_conf("postgresql.auto.conf", "shared_buffers = 10MB")
         node.append_conf("postgresql.auto.conf", "fsync = on")
         node.append_conf("postgresql.auto.conf", "wal_level = minimal")
@@ -217,8 +230,13 @@ def make_simple_node(
 
         # Allow replication in pg_hba.conf
         if set_replication:
-            node.set_replication_conf()
-            node.append_conf("postgresql.auto.conf", "max_wal_senders = 10")
+            node.append_conf(
+                "pg_hba.conf",
+                "local replication all trust\n")
+            node.append_conf(
+                "postgresql.auto.conf",
+                "max_wal_senders = 10")
+
         return node
 
     def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False):
@@ -235,7 +253,8 @@ def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False
             )
 
         if not tblspc_path:
-            tblspc_path = os.path.join(node.base_dir, '{0}'.format(tblspc_name))
+            tblspc_path = os.path.join(
+                node.base_dir, '{0}'.format(tblspc_name))
         cmd = "CREATE TABLESPACE {0} LOCATION '{1}'".format(
             tblspc_name, tblspc_path)
         if cfs:
@@ -272,11 +291,11 @@ def get_md5_per_page_for_fork(self, file, size_in_pages):
 
         size = size_in_pages
         for segment_number in range(nsegments):
-            if size-131072 > 0:
+            if size - 131072 > 0:
                 pages_per_segment[segment_number] = 131072
             else:
                 pages_per_segment[segment_number] = size
-            size = size-131072
+            size = size - 131072
 
         for segment_number in range(nsegments):
             offset = 0
@@ -369,6 +388,7 @@ def check_ptrack_sanity(self, idx_dict):
                                 idx_dict['ptrack'][PageNum])
                         )
                 continue
+
             # Ok, all pages in new_pages that do not have
             # corresponding page in old_pages are been dealt with.
             # We can now safely proceed to comparing old and new pages
@@ -394,7 +414,7 @@ def check_ptrack_sanity(self, idx_dict):
                     if PageNum == 0 and idx_dict['type'] == 'spgist':
                         if self.verbose:
                             print(
-                                'SPGIST is a special snowflake, so don`t'
+                                'SPGIST is a special snowflake, so don`t '
                                 'fret about losing ptrack for blknum 0'
                             )
                         continue
@@ -721,9 +741,9 @@ def set_archiving(
             archive_mode = 'on'
 
         node.append_conf(
-                "postgresql.auto.conf",
-                "wal_level = archive"
-                )
+            "postgresql.auto.conf",
+            "wal_level = archive"
+        )
         node.append_conf(
                 "postgresql.auto.conf",
                 "archive_mode = {0}".format(archive_mode)
@@ -817,7 +837,15 @@ def get_username(self):
         return pwd.getpwuid(os.getuid())[0]
 
     def version_to_num(self, version):
-        return testgres.version_to_num(version)
+        if not version:
+            return 0
+        parts = version.split(".")
+        while len(parts) < 3:
+            parts.append("0")
+        num = 0
+        for part in parts:
+            num = num * 100 + int(re.sub("[^\d]", "", part))
+        return num
 
     def switch_wal_segment(self, node):
         """ Execute pg_switch_wal/xlog() in given node"""
@@ -829,7 +857,11 @@ def switch_wal_segment(self, node):
             node.safe_psql("postgres", "select pg_switch_xlog()")
 
     def get_version(self, node):
-        return testgres.get_config()["VERSION_NUM"]
+        return self.version_to_num(
+            testgres.get_pg_config()["VERSION"].split(" ")[1])
+
+    def get_bin_path(self, binary):
+        return testgres.get_bin_path(binary)
 
     def del_test_dir(self, module_name, fname):
         """ Del testdir and optimistically try to del module dir"""
@@ -960,7 +992,11 @@ def compare_pgdata(self, original_pgdata, restored_pgdata):
                                         )
                         for page in restored_pgdata['files'][file]['md5_per_page']:
                             if page not in original_pgdata['files'][file]['md5_per_page']:
-                                error_message += '\n Extra page {0}\n File: {1}\n'.format(page, os.path.join(restored_pgdata['pgdata'], file))
+                                error_message += '\n Extra page {0}\n '
+                                'File: {1}\n'.format(
+                                    page,
+                                    os.path.join(
+                                        restored_pgdata['pgdata'], file))
 
             else:
                 error_message += (

tests/ptrack.py

@@ -3,7 +3,8 @@
 from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
 from datetime import datetime, timedelta
 import subprocess
-from testgres import ClusterException
+from testgres import ClusterTestgresException as ClusterException
+from testgres import QueryException
 import shutil
 import sys
 import time
@@ -1233,7 +1234,7 @@ def test_create_db(self):
                 "\n Output: {0} \n CMD: {1}".format(
                     repr(self.output), self.cmd)
             )
-        except ClusterException as e:
+        except QueryException as e:
             self.assertTrue(
                 'FATAL:  database "db1" does not exist' in e.message,
                 '\n Unexpected Error Message: {0}\n CMD: {1}'.format(

tests/ptrack_recovery.py

@@ -40,7 +40,7 @@ def test_ptrack_recovery(self):
 
         if self.verbose:
             print('Killing postmaster. Losing Ptrack changes')
-        node.pg_ctl('stop', {'-m': 'immediate', '-D': '{0}'.format(node.data_dir)})    
+        node.stop(['-m', 'immediate', '-D', node.data_dir])
         if not node.status():
             node.start()
         else:

tests/replica.py

@@ -39,7 +39,7 @@ def test_replica_stream_ptrack_backup(self):
         self.set_replica(master, replica)
 
         # Check data correctness on replica
-        replica.start({"-t": "600"})
+        replica.start(["-t", "600"])
         after = replica.safe_psql("postgres", "SELECT * FROM t_heap")
         self.assertEqual(before, after)
 
@@ -120,7 +120,7 @@ def test_replica_archive_page_backup(self):
         # Settings for Replica
         self.set_replica(master, replica)
         self.set_archiving(backup_dir, 'replica', replica, replica=True)
-        replica.start({"-t": "600"})
+        replica.start(["-t", "600"])
 
         # Check data correctness on replica
         after = replica.safe_psql("postgres", "SELECT * FROM t_heap")

tests/restore_test.py

@@ -20,6 +20,7 @@ def test_restore_full_to_latest(self):
             initdb_params=['--data-checksums'],
             pg_options={'wal_level': 'replica'}
             )
+
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
@@ -126,7 +127,7 @@ def test_restore_to_specific_timeline(self):
             self.restore_node(backup_dir, 'node', node, options=["-j", "4"]),
             '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd))
 
-        node.start(params={'-t':'10'})
+        node.start(params=['-t','10'])
         while node.safe_psql("postgres", "select pg_is_in_recovery()") == 't\n':
             time.sleep(1)