catchup functionality expansion (#419)

[PR #419] New command 'catchup' for synchronizing stale standby with master by transfering only changes blocks, or creating standby from scratch.

Co-authored-by: Elena Indrupskaya <[email protected]>
Co-authored-by: Grigory Smolkin <[email protected]>

Author: 3 people

.travis.yml

@@ -26,24 +26,26 @@ notifications:
 
 # Default MODE is basic, i.e. all tests with PG_PROBACKUP_TEST_BASIC=ON
 env:
-    - PG_VERSION=14 PG_BRANCH=REL_14_STABLE
-    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE
-    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE
-    - PG_VERSION=11 PG_BRANCH=REL_11_STABLE
+    - PG_VERSION=15 PG_BRANCH=master PTRACK_PATCH_PG_VERSION=13
+    - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_VERSION=13
+    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13
+    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_VERSION=12
+    - PG_VERSION=11 PG_BRANCH=REL_11_STABLE PTRACK_PATCH_PG_VERSION=11
     - PG_VERSION=10 PG_BRANCH=REL_10_STABLE
     - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE
     - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE
-#    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=archive
-#    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=backup
-#    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=compression
-#    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=delta
-#    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=locking
-#    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=merge
-#    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=page
-#    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=replica
-#    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=retention
-#    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=restore
-    - PG_VERSION=15 PG_BRANCH=master
+#    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=off MODE=archive
+#    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 MODE=backup
+#    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 MODE=catchup
+#    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=off MODE=compression
+#    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=off MODE=delta
+#    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=off MODE=locking
+#    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 MODE=merge
+#    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=off MODE=page
+#    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 MODE=ptrack
+#    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 MODE=replica
+#    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=off MODE=retention
+#    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_VERSION=13 MODE=restore
 
 jobs:
     allow_failures:
@@ -54,3 +56,4 @@ jobs:
 #branches:
 #    only:
 #    - master
+

doc/pgprobackup.xml

@@ -263,7 +263,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
 		fio_mkdir(stream_xlog_path, DIR_PERMISSION, FIO_BACKUP_HOST);
 
 		start_WAL_streaming(backup_conn, stream_xlog_path, &instance_config.conn_opt,
-							current.start_lsn, current.tli);
+							current.start_lsn, current.tli, true);
 
 		/* Make sure that WAL streaming is working
 		 * PAGE backup in stream mode is waited twice, first for
@@ -2051,8 +2051,6 @@ backup_files(void *arg)
 							 instance_config.compress_alg,
 							 instance_config.compress_level,
 							 arguments->nodeInfo->checksum_version,
-							 arguments->nodeInfo->ptrack_version_num,
-							 arguments->nodeInfo->ptrack_schema,
 							 arguments->hdr_map, false);
 		}
 		else
@@ -2350,7 +2348,7 @@ calculate_datasize_of_filelist(parray *filelist)
 	{
 		pgFile	   *file = (pgFile *) parray_get(filelist, i);
 
-		if (file->external_dir_num != 0)
+		if (file->external_dir_num != 0 || file->excluded)
 			continue;
 
 		if (S_ISDIR(file->mode))