Skip to content

Commit 5a41524

Browse files
gsmolkgsmolk
committed
tests: improve test coverage of partial incremental restore case
1 parent 6782370 commit 5a41524

File tree

3 files changed

+352
-4
lines changed

3 files changed

+352
-4
lines changed

‎tests/compatibility.py‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -359,7 +359,10 @@ def test_backward_compatibility_ptrack(self):
359359
node_restored.cleanup()
360360
self.restore_node(
361361
backup_dir, 'node', node_restored,
362-
options=["-j", "4", "--recovery-target-action=promote"])
362+
options=[
363+
"-j", "4",
364+
"--recovery-target=latest",
365+
"--recovery-target-action=promote"])
363366

364367
if self.paranoia:
365368
pgdata_restored = self.pgdata_content(node_restored.data_dir)
@@ -384,7 +387,10 @@ def test_backward_compatibility_ptrack(self):
384387

385388
self.restore_node(
386389
backup_dir, 'node', node_restored,
387-
options=["-j", "4", "--recovery-target-action=promote"])
390+
options=[
391+
"-j", "4",
392+
"--recovery-target=latest",
393+
"--recovery-target-action=promote"])
388394

389395
if self.paranoia:
390396
pgdata_restored = self.pgdata_content(node_restored.data_dir)
@@ -593,7 +599,7 @@ def test_backward_compatibility_merge(self):
593599

594600
self.merge_backup(backup_dir, "node", backup_id)
595601

596-
print(self.show_pb(backup_dir, as_text=True, as_json=False))
602+
self.show_pb(backup_dir, as_text=True, as_json=False)
597603

598604
# restore OLD FULL with new binary
599605
node_restored = self.make_simple_node(

‎tests/external.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -743,8 +743,11 @@ def test_external_merge(self):
743743
pgdata = self.pgdata_content(
744744
node.base_dir, exclude_dirs=['logs'])
745745

746+
print(self.show_pb(backup_dir, 'node', as_json=False, as_text=True))
747+
746748
# Merge
747-
self.merge_backup(backup_dir, 'node', backup_id=backup_id)
749+
print(self.merge_backup(backup_dir, 'node', backup_id=backup_id,
750+
options=['--log-level-file=VERBOSE']))
748751

749752
# RESTORE
750753
node.cleanup()

‎tests/incr_restore.py‎

Lines changed: 339 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1647,5 +1647,344 @@ def test_incr_restore_zero_size_file_lsn(self):
16471647
# Clean after yourself
16481648
self.del_test_dir(module_name, fname)
16491649

1650+
def test_incremental_partial_restore_exclude_checksum(self):
1651+
""""""
1652+
fname = self.id().split('.')[3]
1653+
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
1654+
node = self.make_simple_node(
1655+
base_dir=os.path.join(module_name, fname, 'node'),
1656+
initdb_params=['--data-checksums'])
1657+
1658+
self.init_pb(backup_dir)
1659+
self.add_instance(backup_dir, 'node', node)
1660+
self.set_archiving(backup_dir, 'node', node)
1661+
node.slow_start()
1662+
1663+
for i in range(1, 10, 1):
1664+
node.safe_psql(
1665+
'postgres',
1666+
'CREATE database db{0}'.format(i))
1667+
1668+
db_list_raw = node.safe_psql(
1669+
'postgres',
1670+
'SELECT to_json(a) '
1671+
'FROM (SELECT oid, datname FROM pg_database) a').rstrip()
1672+
1673+
db_list_splitted = db_list_raw.splitlines()
1674+
1675+
db_list = {}
1676+
for line in db_list_splitted:
1677+
line = json.loads(line)
1678+
db_list[line['datname']] = line['oid']
1679+
1680+
node.pgbench_init(scale=20)
1681+
1682+
# FULL backup
1683+
self.backup_node(backup_dir, 'node', node)
1684+
pgdata = self.pgdata_content(node.data_dir)
1685+
1686+
pgbench = node.pgbench(options=['-T', '10', '-c', '1'])
1687+
pgbench.wait()
1688+
1689+
# PAGE backup
1690+
backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page')
1691+
1692+
# restore FULL backup into second node2
1693+
node1 = self.make_simple_node(
1694+
base_dir=os.path.join(module_name, fname, 'node1'))
1695+
node1.cleanup()
1696+
1697+
node2 = self.make_simple_node(
1698+
base_dir=os.path.join(module_name, fname, 'node2'))
1699+
node2.cleanup()
1700+
1701+
# restore some data into node2
1702+
self.restore_node(backup_dir, 'node', node2)
1703+
1704+
# partial restore backup into node1
1705+
self.restore_node(
1706+
backup_dir, 'node',
1707+
node1, options=[
1708+
"--db-exclude=db1",
1709+
"--db-exclude=db5"])
1710+
1711+
pgdata1 = self.pgdata_content(node1.data_dir)
1712+
1713+
# partial incremental restore backup into node2
1714+
print(self.restore_node(
1715+
backup_dir, 'node',
1716+
node2, options=[
1717+
"--db-exclude=db1",
1718+
"--db-exclude=db5",
1719+
"-I", "checksum"]))
1720+
1721+
pgdata2 = self.pgdata_content(node2.data_dir)
1722+
1723+
self.compare_pgdata(pgdata1, pgdata2)
1724+
1725+
self.set_auto_conf(node2, {'port': node2.port})
1726+
1727+
node2.slow_start()
1728+
1729+
node2.safe_psql(
1730+
'postgres',
1731+
'select 1')
1732+
1733+
try:
1734+
node2.safe_psql(
1735+
'db1',
1736+
'select 1')
1737+
except QueryException as e:
1738+
self.assertIn('FATAL', e.message)
1739+
1740+
try:
1741+
node2.safe_psql(
1742+
'db5',
1743+
'select 1')
1744+
except QueryException as e:
1745+
self.assertIn('FATAL', e.message)
1746+
1747+
with open(node2.pg_log_file, 'r') as f:
1748+
output = f.read()
1749+
1750+
self.assertNotIn('PANIC', output)
1751+
1752+
# Clean after yourself
1753+
self.del_test_dir(module_name, fname, [node, node2])
1754+
1755+
def test_incremental_partial_restore_exclude_lsn(self):
1756+
""""""
1757+
fname = self.id().split('.')[3]
1758+
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
1759+
node = self.make_simple_node(
1760+
base_dir=os.path.join(module_name, fname, 'node'),
1761+
initdb_params=['--data-checksums'])
1762+
1763+
self.init_pb(backup_dir)
1764+
self.add_instance(backup_dir, 'node', node)
1765+
self.set_archiving(backup_dir, 'node', node)
1766+
node.slow_start()
1767+
1768+
for i in range(1, 10, 1):
1769+
node.safe_psql(
1770+
'postgres',
1771+
'CREATE database db{0}'.format(i))
1772+
1773+
db_list_raw = node.safe_psql(
1774+
'postgres',
1775+
'SELECT to_json(a) '
1776+
'FROM (SELECT oid, datname FROM pg_database) a').rstrip()
1777+
1778+
db_list_splitted = db_list_raw.splitlines()
1779+
1780+
db_list = {}
1781+
for line in db_list_splitted:
1782+
line = json.loads(line)
1783+
db_list[line['datname']] = line['oid']
1784+
1785+
node.pgbench_init(scale=20)
1786+
1787+
# FULL backup
1788+
self.backup_node(backup_dir, 'node', node)
1789+
pgdata = self.pgdata_content(node.data_dir)
1790+
1791+
pgbench = node.pgbench(options=['-T', '10', '-c', '1'])
1792+
pgbench.wait()
1793+
1794+
# PAGE backup
1795+
backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page')
1796+
1797+
node.stop()
1798+
1799+
# restore FULL backup into second node2
1800+
node1 = self.make_simple_node(
1801+
base_dir=os.path.join(module_name, fname, 'node1'))
1802+
node1.cleanup()
1803+
1804+
node2 = self.make_simple_node(
1805+
base_dir=os.path.join(module_name, fname, 'node2'))
1806+
node2.cleanup()
1807+
1808+
# restore some data into node2
1809+
self.restore_node(backup_dir, 'node', node2)
1810+
1811+
# partial restore backup into node1
1812+
self.restore_node(
1813+
backup_dir, 'node',
1814+
node1, options=[
1815+
"--db-exclude=db1",
1816+
"--db-exclude=db5"])
1817+
1818+
pgdata1 = self.pgdata_content(node1.data_dir)
1819+
1820+
# partial incremental restore backup into node2
1821+
node2.port = node.port
1822+
node2.slow_start()
1823+
node2.stop()
1824+
print(self.restore_node(
1825+
backup_dir, 'node',
1826+
node2, options=[
1827+
"--db-exclude=db1",
1828+
"--db-exclude=db5",
1829+
"-I", "lsn"]))
1830+
1831+
pgdata2 = self.pgdata_content(node2.data_dir)
1832+
1833+
self.compare_pgdata(pgdata1, pgdata2)
1834+
1835+
self.set_auto_conf(node2, {'port': node2.port})
1836+
1837+
node2.slow_start()
1838+
1839+
node2.safe_psql(
1840+
'postgres',
1841+
'select 1')
1842+
1843+
try:
1844+
node2.safe_psql(
1845+
'db1',
1846+
'select 1')
1847+
except QueryException as e:
1848+
self.assertIn('FATAL', e.message)
1849+
1850+
try:
1851+
node2.safe_psql(
1852+
'db5',
1853+
'select 1')
1854+
except QueryException as e:
1855+
self.assertIn('FATAL', e.message)
1856+
1857+
with open(node2.pg_log_file, 'r') as f:
1858+
output = f.read()
1859+
1860+
self.assertNotIn('PANIC', output)
1861+
1862+
# Clean after yourself
1863+
self.del_test_dir(module_name, fname, [node2])
1864+
1865+
def test_incremental_partial_restore_exclude_tablespace_checksum(self):
1866+
""""""
1867+
fname = self.id().split('.')[3]
1868+
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
1869+
node = self.make_simple_node(
1870+
base_dir=os.path.join(module_name, fname, 'node'),
1871+
initdb_params=['--data-checksums'])
1872+
1873+
self.init_pb(backup_dir)
1874+
self.add_instance(backup_dir, 'node', node)
1875+
self.set_archiving(backup_dir, 'node', node)
1876+
node.slow_start()
1877+
1878+
# cat_version = node.get_control_data()["Catalog version number"]
1879+
# version_specific_dir = 'PG_' + node.major_version_str + '_' + cat_version
1880+
1881+
# PG_10_201707211
1882+
# pg_tblspc/33172/PG_9.5_201510051/16386/
1883+
1884+
self.create_tblspace_in_node(node, 'somedata')
1885+
1886+
node_tablespace = self.get_tblspace_path(node, 'somedata')
1887+
1888+
tbl_oid = node.safe_psql(
1889+
'postgres',
1890+
"SELECT oid "
1891+
"FROM pg_tablespace "
1892+
"WHERE spcname = 'somedata'").rstrip()
1893+
1894+
for i in range(1, 10, 1):
1895+
node.safe_psql(
1896+
'postgres',
1897+
'CREATE database db{0} tablespace somedata'.format(i))
1898+
1899+
db_list_raw = node.safe_psql(
1900+
'postgres',
1901+
'SELECT to_json(a) '
1902+
'FROM (SELECT oid, datname FROM pg_database) a').rstrip()
1903+
1904+
db_list_splitted = db_list_raw.splitlines()
1905+
1906+
db_list = {}
1907+
for line in db_list_splitted:
1908+
line = json.loads(line)
1909+
db_list[line['datname']] = line['oid']
1910+
1911+
# FULL backup
1912+
backup_id = self.backup_node(backup_dir, 'node', node)
1913+
1914+
# node1
1915+
node1 = self.make_simple_node(
1916+
base_dir=os.path.join(module_name, fname, 'node1'))
1917+
node1.cleanup()
1918+
node1_tablespace = self.get_tblspace_path(node1, 'somedata')
1919+
1920+
# node2
1921+
node2 = self.make_simple_node(
1922+
base_dir=os.path.join(module_name, fname, 'node2'))
1923+
node2.cleanup()
1924+
node2_tablespace = self.get_tblspace_path(node2, 'somedata')
1925+
1926+
# in node2 restore full backup
1927+
self.restore_node(
1928+
backup_dir, 'node',
1929+
node2, options=[
1930+
"-T", "{0}={1}".format(
1931+
node_tablespace, node2_tablespace)])
1932+
1933+
# partial restore into node1
1934+
self.restore_node(
1935+
backup_dir, 'node',
1936+
node1, options=[
1937+
"--db-exclude=db1",
1938+
"--db-exclude=db5",
1939+
"-T", "{0}={1}".format(
1940+
node_tablespace, node1_tablespace)])
1941+
1942+
# with open(os.path.join(node1_tablespace, "hello"), "w") as f:
1943+
# f.close()
1944+
pgdata1 = self.pgdata_content(node1.data_dir)
1945+
1946+
# partial incremental restore into node2
1947+
self.restore_node(
1948+
backup_dir, 'node',
1949+
node2, options=[
1950+
"-I", "checksum",
1951+
"--db-exclude=db1",
1952+
"--db-exclude=db5",
1953+
"-T", "{0}={1}".format(
1954+
node_tablespace, node2_tablespace)])
1955+
pgdata2 = self.pgdata_content(node2.data_dir)
1956+
1957+
self.compare_pgdata(pgdata1, pgdata2)
1958+
1959+
1960+
self.set_auto_conf(node2, {'port': node2.port})
1961+
node2.slow_start()
1962+
1963+
node2.safe_psql(
1964+
'postgres',
1965+
'select 1')
1966+
1967+
try:
1968+
node2.safe_psql(
1969+
'db1',
1970+
'select 1')
1971+
except QueryException as e:
1972+
self.assertIn('FATAL', e.message)
1973+
1974+
try:
1975+
node2.safe_psql(
1976+
'db5',
1977+
'select 1')
1978+
except QueryException as e:
1979+
self.assertIn('FATAL', e.message)
1980+
1981+
with open(node2.pg_log_file, 'r') as f:
1982+
output = f.read()
1983+
1984+
self.assertNotIn('PANIC', output)
1985+
1986+
# Clean after yourself
1987+
self.del_test_dir(module_name, fname, [node2])
1988+
16501989
# check that MinRecPoint and BackupStartLsn are correctly used in case of --incrementa-lsn
16511990
# incremental restore + partial restore.

0 commit comments

Comments
 (0)