Skip to content

Commit 884e8b0

Browse files
dlepikhovadlepikhova
authored
[pbckp-128] dry-run option for catchup (#477)
* Added dry-run option for catchup. Run catchup without affect on the files and WAL
1 parent 7be2e73 commit 884e8b0

File tree

5 files changed

+220
-32
lines changed

5 files changed

+220
-32
lines changed

‎src/catchup.c‎

Lines changed: 52 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
*
33
* catchup.c: sync DB cluster
44
*
5-
* Copyright (c) 2021, Postgres Professional
5+
* Copyright (c) 2022, Postgres Professional
66
*
77
*-------------------------------------------------------------------------
88
*/
@@ -507,16 +507,20 @@ catchup_multithreaded_copy(int num_threads,
507507
/* Run threads */
508508
thread_interrupted = false;
509509
threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads);
510-
for (i = 0; i < num_threads; i++)
510+
if (!dry_run)
511511
{
512-
elog(VERBOSE, "Start thread num: %i", i);
513-
pthread_create(&threads[i], NULL, &catchup_thread_runner, &(threads_args[i]));
512+
for (i = 0; i < num_threads; i++)
513+
{
514+
elog(VERBOSE, "Start thread num: %i", i);
515+
pthread_create(&threads[i], NULL, &catchup_thread_runner, &(threads_args[i]));
516+
}
514517
}
515518

516519
/* Wait threads */
517520
for (i = 0; i < num_threads; i++)
518521
{
519-
pthread_join(threads[i], NULL);
522+
if (!dry_run)
523+
pthread_join(threads[i], NULL);
520524
all_threads_successful &= threads_args[i].completed;
521525
transfered_bytes_result += threads_args[i].transfered_bytes;
522526
}
@@ -706,9 +710,14 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
706710

707711
/* Start stream replication */
708712
join_path_components(dest_xlog_path, dest_pgdata, PG_XLOG_DIR);
709-
fio_mkdir(dest_xlog_path, DIR_PERMISSION, FIO_LOCAL_HOST);
710-
start_WAL_streaming(source_conn, dest_xlog_path, &instance_config.conn_opt,
711-
current.start_lsn, current.tli, false);
713+
if (!dry_run)
714+
{
715+
fio_mkdir(dest_xlog_path, DIR_PERMISSION, FIO_LOCAL_HOST);
716+
start_WAL_streaming(source_conn, dest_xlog_path, &instance_config.conn_opt,
717+
current.start_lsn, current.tli, false);
718+
}
719+
else
720+
elog(INFO, "WAL streaming skipping with --dry-run option");
712721

713722
source_filelist = parray_new();
714723

@@ -779,9 +788,9 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
779788

780789
/* Build the page map from ptrack information */
781790
make_pagemap_from_ptrack_2(source_filelist, source_conn,
782-
source_node_info.ptrack_schema,
783-
source_node_info.ptrack_version_num,
784-
dest_redo.lsn);
791+
source_node_info.ptrack_schema,
792+
source_node_info.ptrack_version_num,
793+
dest_redo.lsn);
785794
time(&end_time);
786795
elog(INFO, "Pagemap successfully extracted, time elapsed: %.0f sec",
787796
difftime(end_time, start_time));
@@ -820,9 +829,9 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
820829
char dirpath[MAXPGPATH];
821830

822831
join_path_components(dirpath, dest_pgdata, file->rel_path);
823-
824832
elog(VERBOSE, "Create directory '%s'", dirpath);
825-
fio_mkdir(dirpath, DIR_PERMISSION, FIO_LOCAL_HOST);
833+
if (!dry_run)
834+
fio_mkdir(dirpath, DIR_PERMISSION, FIO_LOCAL_HOST);
826835
}
827836
else
828837
{
@@ -853,15 +862,18 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
853862
elog(VERBOSE, "Create directory \"%s\" and symbolic link \"%s\"",
854863
linked_path, to_path);
855864

856-
/* create tablespace directory */
857-
if (fio_mkdir(linked_path, file->mode, FIO_LOCAL_HOST) != 0)
858-
elog(ERROR, "Could not create tablespace directory \"%s\": %s",
859-
linked_path, strerror(errno));
860-
861-
/* create link to linked_path */
862-
if (fio_symlink(linked_path, to_path, true, FIO_LOCAL_HOST) < 0)
863-
elog(ERROR, "Could not create symbolic link \"%s\" -> \"%s\": %s",
864-
linked_path, to_path, strerror(errno));
865+
if (!dry_run)
866+
{
867+
/* create tablespace directory */
868+
if (fio_mkdir(linked_path, file->mode, FIO_LOCAL_HOST) != 0)
869+
elog(ERROR, "Could not create tablespace directory \"%s\": %s",
870+
linked_path, strerror(errno));
871+
872+
/* create link to linked_path */
873+
if (fio_symlink(linked_path, to_path, true, FIO_LOCAL_HOST) < 0)
874+
elog(ERROR, "Could not create symbolic link \"%s\" -> \"%s\": %s",
875+
linked_path, to_path, strerror(errno));
876+
}
865877
}
866878
}
867879

@@ -930,7 +942,10 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
930942
char fullpath[MAXPGPATH];
931943

932944
join_path_components(fullpath, dest_pgdata, file->rel_path);
933-
fio_delete(file->mode, fullpath, FIO_LOCAL_HOST);
945+
if (!dry_run)
946+
{
947+
fio_delete(file->mode, fullpath, FIO_LOCAL_HOST);
948+
}
934949
elog(VERBOSE, "Deleted file \"%s\"", fullpath);
935950

936951
/* shrink dest pgdata list */
@@ -961,7 +976,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
961976
catchup_isok = transfered_datafiles_bytes != -1;
962977

963978
/* at last copy control file */
964-
if (catchup_isok)
979+
if (catchup_isok && !dry_run)
965980
{
966981
char from_fullpath[MAXPGPATH];
967982
char to_fullpath[MAXPGPATH];
@@ -972,7 +987,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
972987
transfered_datafiles_bytes += source_pg_control_file->size;
973988
}
974989

975-
if (!catchup_isok)
990+
if (!catchup_isok && !dry_run)
976991
{
977992
char pretty_time[20];
978993
char pretty_transfered_data_bytes[20];
@@ -1010,14 +1025,18 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
10101025
pg_free(stop_backup_query_text);
10111026
}
10121027

1013-
wait_wal_and_calculate_stop_lsn(dest_xlog_path, stop_backup_result.lsn, &current);
1028+
if (!dry_run)
1029+
wait_wal_and_calculate_stop_lsn(dest_xlog_path, stop_backup_result.lsn, &current);
10141030

10151031
#if PG_VERSION_NUM >= 90600
10161032
/* Write backup_label */
10171033
Assert(stop_backup_result.backup_label_content != NULL);
1018-
pg_stop_backup_write_file_helper(dest_pgdata, PG_BACKUP_LABEL_FILE, "backup label",
1019-
stop_backup_result.backup_label_content, stop_backup_result.backup_label_content_len,
1020-
NULL);
1034+
if (!dry_run)
1035+
{
1036+
pg_stop_backup_write_file_helper(dest_pgdata, PG_BACKUP_LABEL_FILE, "backup label",
1037+
stop_backup_result.backup_label_content, stop_backup_result.backup_label_content_len,
1038+
NULL);
1039+
}
10211040
free(stop_backup_result.backup_label_content);
10221041
stop_backup_result.backup_label_content = NULL;
10231042
stop_backup_result.backup_label_content_len = 0;
@@ -1040,6 +1059,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
10401059
#endif
10411060

10421061
/* wait for end of wal streaming and calculate wal size transfered */
1062+
if (!dry_run)
10431063
{
10441064
parray *wal_files_list = NULL;
10451065
wal_files_list = parray_new();
@@ -1091,17 +1111,17 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
10911111
}
10921112

10931113
/* Sync all copied files unless '--no-sync' flag is used */
1094-
if (sync_dest_files)
1114+
if (sync_dest_files && !dry_run)
10951115
catchup_sync_destination_files(dest_pgdata, FIO_LOCAL_HOST, source_filelist, source_pg_control_file);
10961116
else
10971117
elog(WARNING, "Files are not synced to disk");
10981118

10991119
/* Cleanup */
1100-
if (dest_filelist)
1120+
if (dest_filelist && !dry_run)
11011121
{
11021122
parray_walk(dest_filelist, pgFileFree);
1103-
parray_free(dest_filelist);
11041123
}
1124+
parray_free(dest_filelist);
11051125
parray_walk(source_filelist, pgFileFree);
11061126
parray_free(source_filelist);
11071127
pgFileFree(source_pg_control_file);

‎src/help.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,7 @@ help_pg_probackup(void)
261261
printf(_(" [--remote-proto] [--remote-host]\n"));
262262
printf(_(" [--remote-port] [--remote-path] [--remote-user]\n"));
263263
printf(_(" [--ssh-options]\n"));
264+
printf(_(" [--dry-run]\n"));
264265
printf(_(" [--help]\n"));
265266

266267
if ((PROGRAM_URL || PROGRAM_EMAIL))
@@ -1047,6 +1048,7 @@ help_catchup(void)
10471048
printf(_(" [--remote-proto] [--remote-host]\n"));
10481049
printf(_(" [--remote-port] [--remote-path] [--remote-user]\n"));
10491050
printf(_(" [--ssh-options]\n"));
1051+
printf(_(" [--dry-run]\n"));
10501052
printf(_(" [--help]\n\n"));
10511053

10521054
printf(_(" -b, --backup-mode=catchup-mode catchup mode=FULL|DELTA|PTRACK\n"));
@@ -1081,4 +1083,6 @@ help_catchup(void)
10811083
printf(_(" --remote-user=username user name for ssh connection (default: current user)\n"));
10821084
printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n"));
10831085
printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n"));
1086+
1087+
printf(_(" --dry-run perform a trial run without any changes\n\n"));
10841088
}

‎tests/catchup.py‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1455,3 +1455,157 @@ def test_config_exclusion(self):
14551455
dst_pg.stop()
14561456
#self.assertEqual(1, 0, 'Stop test')
14571457
self.del_test_dir(module_name, self.fname)
1458+
1459+
#########################################
1460+
# --dry-run
1461+
#########################################
1462+
def test_dry_run_catchup_full(self):
1463+
"""
1464+
Test dry-run option for full catchup
1465+
"""
1466+
# preparation 1: source
1467+
src_pg = self.make_simple_node(
1468+
base_dir = os.path.join(module_name, self.fname, 'src'),
1469+
set_replication = True
1470+
)
1471+
src_pg.slow_start()
1472+
1473+
# preparation 2: make clean shutdowned lagging behind replica
1474+
dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst'))
1475+
1476+
src_pg.pgbench_init(scale = 10)
1477+
pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum'])
1478+
pgbench.wait()
1479+
1480+
# save the condition before dry-run
1481+
content_before = self.pgdata_content(dst_pg.data_dir)
1482+
1483+
# do full catchup
1484+
self.catchup_node(
1485+
backup_mode = 'FULL',
1486+
source_pgdata = src_pg.data_dir,
1487+
destination_node = dst_pg,
1488+
options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--dry-run']
1489+
)
1490+
1491+
# compare data dirs before and after catchup
1492+
self.compare_pgdata(
1493+
content_before,
1494+
self.pgdata_content(dst_pg.data_dir)
1495+
)
1496+
1497+
# Cleanup
1498+
src_pg.stop()
1499+
self.del_test_dir(module_name, self.fname)
1500+
1501+
def test_dry_run_catchup_ptrack(self):
1502+
"""
1503+
Test dry-run option for catchup in incremental ptrack mode
1504+
"""
1505+
if not self.ptrack:
1506+
return unittest.skip('Skipped because ptrack support is disabled')
1507+
1508+
# preparation 1: source
1509+
src_pg = self.make_simple_node(
1510+
base_dir = os.path.join(module_name, self.fname, 'src'),
1511+
set_replication = True,
1512+
ptrack_enable = True,
1513+
initdb_params = ['--data-checksums']
1514+
)
1515+
src_pg.slow_start()
1516+
src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack")
1517+
1518+
src_pg.pgbench_init(scale = 10)
1519+
pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum'])
1520+
pgbench.wait()
1521+
1522+
# preparation 2: make clean shutdowned lagging behind replica
1523+
dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst'))
1524+
self.catchup_node(
1525+
backup_mode = 'FULL',
1526+
source_pgdata = src_pg.data_dir,
1527+
destination_node = dst_pg,
1528+
options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream']
1529+
)
1530+
self.set_replica(src_pg, dst_pg)
1531+
dst_options = {}
1532+
dst_options['port'] = str(dst_pg.port)
1533+
self.set_auto_conf(dst_pg, dst_options)
1534+
dst_pg.slow_start(replica = True)
1535+
dst_pg.stop()
1536+
1537+
# save the condition before dry-run
1538+
content_before = self.pgdata_content(dst_pg.data_dir)
1539+
1540+
# do incremental catchup
1541+
self.catchup_node(
1542+
backup_mode = 'PTRACK',
1543+
source_pgdata = src_pg.data_dir,
1544+
destination_node = dst_pg,
1545+
options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--dry-run']
1546+
)
1547+
1548+
# compare data dirs before and after cathup
1549+
self.compare_pgdata(
1550+
content_before,
1551+
self.pgdata_content(dst_pg.data_dir)
1552+
)
1553+
1554+
# Cleanup
1555+
src_pg.stop()
1556+
self.del_test_dir(module_name, self.fname)
1557+
1558+
def test_dry_run_catchup_delta(self):
1559+
"""
1560+
Test dry-run option for catchup in incremental delta mode
1561+
"""
1562+
1563+
# preparation 1: source
1564+
src_pg = self.make_simple_node(
1565+
base_dir = os.path.join(module_name, self.fname, 'src'),
1566+
set_replication = True,
1567+
initdb_params = ['--data-checksums'],
1568+
pg_options = { 'wal_log_hints': 'on' }
1569+
)
1570+
src_pg.slow_start()
1571+
1572+
src_pg.pgbench_init(scale = 10)
1573+
pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum'])
1574+
pgbench.wait()
1575+
1576+
# preparation 2: make clean shutdowned lagging behind replica
1577+
dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst'))
1578+
self.catchup_node(
1579+
backup_mode = 'FULL',
1580+
source_pgdata = src_pg.data_dir,
1581+
destination_node = dst_pg,
1582+
options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream']
1583+
)
1584+
self.set_replica(src_pg, dst_pg)
1585+
dst_options = {}
1586+
dst_options['port'] = str(dst_pg.port)
1587+
self.set_auto_conf(dst_pg, dst_options)
1588+
dst_pg.slow_start(replica = True)
1589+
dst_pg.stop()
1590+
1591+
# save the condition before dry-run
1592+
content_before = self.pgdata_content(dst_pg.data_dir)
1593+
1594+
# do delta catchup
1595+
self.catchup_node(
1596+
backup_mode = 'DELTA',
1597+
source_pgdata = src_pg.data_dir,
1598+
destination_node = dst_pg,
1599+
options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream', "--dry-run"]
1600+
)
1601+
1602+
# compare data dirs before and after cathup
1603+
self.compare_pgdata(
1604+
content_before,
1605+
self.pgdata_content(dst_pg.data_dir)
1606+
)
1607+
1608+
# Cleanup
1609+
src_pg.stop()
1610+
self.del_test_dir(module_name, self.fname)
1611+

‎tests/expected/option_help.out‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
178178
[--remote-proto] [--remote-host]
179179
[--remote-port] [--remote-path] [--remote-user]
180180
[--ssh-options]
181+
[--dry-run]
181182
[--help]
182183

183184
Read the website for details <https://github.com/postgrespro/pg_probackup>.

‎travis/run_tests.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,11 +100,20 @@ source pyenv/bin/activate
100100
pip3 install testgres
101101

102102
echo "############### Testing:"
103+
echo PG_PROBACKUP_PARANOIA=${PG_PROBACKUP_PARANOIA}
104+
echo ARCHIVE_COMPRESSION=${ARCHIVE_COMPRESSION}
105+
echo PGPROBACKUPBIN_OLD=${PGPROBACKUPBIN_OLD}
106+
echo PGPROBACKUPBIN=${PGPROBACKUPBIN}
107+
echo PGPROBACKUP_SSH_REMOTE=${PGPROBACKUP_SSH_REMOTE}
108+
echo PGPROBACKUP_GDB=${PGPROBACKUP_GDB}
109+
echo PG_PROBACKUP_PTRACK=${PG_PROBACKUP_PTRACK}
103110
if [ "$MODE" = "basic" ]; then
104111
export PG_PROBACKUP_TEST_BASIC=ON
112+
echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC}
105113
python3 -m unittest -v tests
106114
python3 -m unittest -v tests.init
107115
else
116+
echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC}
108117
python3 -m unittest -v tests.$MODE
109118
fi
110119

0 commit comments

Comments
 (0)