PGPRO-427: added test_page_backup_with_corrupted_wal_segment

Author: gsmolk

tests/page.py

@@ -722,6 +722,103 @@ def test_page_backup_with_lost_wal_segment(self):
             self.show_pb(backup_dir, 'node')[1]['status'],
             'Backup {0} should have STATUS "ERROR"')
 
+        # Multi-thread PAGE backup
+        try:
+            self.backup_node(
+                backup_dir, 'node', node,
+                backup_type='page',
+                options=["-j", "4", '--log-level-file=verbose'])
+            self.assertEqual(
+                1, 0,
+                "Expecting Error because of wal segment disappearance.\n "
+                "Output: {0} \n CMD: {1}".format(
+                    self.output, self.cmd))
+        except ProbackupException as e:
+            self.assertTrue(
+                'INFO: Wait for LSN' in e.message and
+                'in archived WAL segment' in e.message and
+                'WARNING: could not read WAL record at' in e.message and
+                'ERROR: WAL segment "{0}" is absent\n'.format(
+                    file) in e.message,
+                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
+                    repr(e.message), self.cmd))
+
+        self.assertEqual(
+            'ERROR',
+            self.show_pb(backup_dir, 'node')[2]['status'],
+            'Backup {0} should have STATUS "ERROR"')
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)
+
+    # @unittest.skip("skip")
+    def test_page_backup_with_corrupted_wal_segment(self):
+        """
+        make node with archiving
+        make archive backup, then generate some wals with pgbench,
+        corrupt latest archived wal segment
+        run page backup, expecting error because of missing wal segment
+        make sure that backup status is 'ERROR'
+        """
+        fname = self.id().split('.')[3]
+        node = self.make_simple_node(
+            base_dir="{0}/{1}/node".format(module_name, fname),
+            initdb_params=['--data-checksums'],
+            pg_options={'wal_level': 'replica'}
+            )
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+        node.start()
+
+        self.backup_node(backup_dir, 'node', node)
+
+        # make some wals
+        node.pgbench_init(scale=3)
+
+        # delete last wal segment
+        wals_dir = os.path.join(backup_dir, 'wal', 'node')
+        wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(
+            wals_dir, f)) and not f.endswith('.backup')]
+        wals = map(str, wals)
+        file = os.path.join(wals_dir, max(wals))
+#        file = os.path.join(wals_dir, '000000010000000000000004')
+        print(file)
+        with open(file, "rb+", 0) as f:
+            f.seek(42)
+            f.write(b"blah")
+            f.flush()
+            f.close
+
+        if self.archive_compress:
+            file = file[:-3]
+
+        # Single-thread PAGE backup
+        try:
+            self.backup_node(
+                backup_dir, 'node', node,
+                backup_type='page', options=['--log-level-file=verbose'])
+            self.assertEqual(
+                1, 0,
+                "Expecting Error because of wal segment disappearance.\n "
+                "Output: {0} \n CMD: {1}".format(
+                    self.output, self.cmd))
+        except ProbackupException as e:
+            self.assertTrue(
+                'INFO: Wait for LSN' in e.message and
+                'in archived WAL segment' in e.message and
+                'WARNING: could not read WAL record at' in e.message and
+                'ERROR: WAL segment "{0}" is absent\n'.format(
+                    file) in e.message,
+                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
+                    repr(e.message), self.cmd))
+
+        self.assertEqual(
+            'ERROR',
+            self.show_pb(backup_dir, 'node')[1]['status'],
+            'Backup {0} should have STATUS "ERROR"')
+
         # Multi-thread PAGE backup
         try:
             self.backup_node(