Merge branch 'pgpro-1457'

Author: gsmolk

README.md

@@ -27,6 +27,7 @@ Using `pg_probackup`, you can take full or incremental backups:
 * `Full` backups contain all the data files required to restore the database cluster from scratch.
 * `Incremental` backups only store the data that has changed since the previous backup. It allows to decrease the backup size and speed up backup operations. `pg_probackup` supports the following modes of incremental backups:
   * `PAGE` backup. In this mode, `pg_probackup` scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups contain only the pages that were mentioned in WAL records. This requires all the WAL files since the previous backup to be present in the WAL archive. If the size of these files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space.
+  * `DELTA` backup. In this mode, `pg_probackup` read all data files in PGDATA directory and only those pages, that where changed since previous backup, are copied. Continuous archiving is not necessary for it to operate. Also this mode impose I/O stress equal to `Full` backup and takes the same amount of time.
   * `PTRACK` backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special `PTRACK` bitmap for this relation. As one page requires just one bit in the `PTRACK` fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly.
 
 Regardless of the chosen backup type, all backups taken with `pg_probackup` support the following archiving strategies:

src/backup.c

@@ -479,7 +479,8 @@ do_backup_instance(void)
 	 * backup on current timeline exists and get its filelist.
 	 */
 	if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||
-		current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
+		current.backup_mode == BACKUP_MODE_DIFF_PTRACK ||
+		current.backup_mode == BACKUP_MODE_DIFF_DELTA)
 	{
 		parray	   *backup_list;
 		/* get list of backups already taken */
@@ -495,7 +496,8 @@ do_backup_instance(void)
 
 		pgBackupGetPath(prev_backup, prev_backup_filelist_path, lengthof(prev_backup_filelist_path),
 						DATABASE_FILE_LIST);
-		prev_backup_filelist = dir_read_file_list(pgdata, prev_backup_filelist_path);
+		/* Files of previous backup needed by DELTA backup */
+		prev_backup_filelist = dir_read_file_list(NULL, prev_backup_filelist_path);
 
 		/* If lsn is not NULL, only pages with higher lsn will be copied. */
 		prev_backup_start_lsn = prev_backup->start_lsn;
@@ -1945,6 +1947,25 @@ backup_files(void *arg)
 
 		if (S_ISREG(buf.st_mode))
 		{
+			/* Check that file exist in previous backup */
+			if (current.backup_mode == BACKUP_MODE_DIFF_DELTA)
+			{
+				int			p;
+				char	   *relative;
+				int n_prev_backup_files_list = parray_num(arguments->prev_backup_filelist);
+				relative = GetRelativePath(file->path, arguments->from_root);
+				for (p = 0; p < n_prev_backup_files_list; p++)
+				{
+					pgFile *prev_file = (pgFile *) parray_get(arguments->prev_backup_filelist, p);
+					if (strcmp(relative, prev_file->path) == 0)
+					{
+						/* File exists in previous backup */
+						file->exists_in_prev = true;
+						elog(INFO, "File exists at the time of previous backup %s", relative);
+						break;
+					}
+				}
+			}
 			/* copy the file into backup */
 			if (file->is_datafile && !file->is_cfs)
 			{

src/catalog.c

@@ -21,7 +21,7 @@
 #include <time.h>
 #include <unistd.h>
 
-static const char *backupModes[] = {"", "PAGE", "PTRACK", "FULL"};
+static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"};
 static pgBackup *readBackupControlFile(const char *path);
 
 static bool exit_hook_registered = false;
@@ -593,6 +593,8 @@ parse_backup_mode(const char *value)
 		return BACKUP_MODE_DIFF_PAGE;
 	else if (len > 0 && pg_strncasecmp("ptrack", v, len) == 0)
 		return BACKUP_MODE_DIFF_PTRACK;
+	else if (len > 0 && pg_strncasecmp("delta", v, len) == 0)
+		return BACKUP_MODE_DIFF_DELTA;
 
 	/* Backup mode is invalid, so leave with an error */
 	elog(ERROR, "invalid backup-mode \"%s\"", value);
@@ -610,6 +612,8 @@ deparse_backup_mode(BackupMode mode)
 			return "page";
 		case BACKUP_MODE_DIFF_PTRACK:
 			return "ptrack";
+		case BACKUP_MODE_DIFF_DELTA:
+			return "delta";
 		case BACKUP_MODE_INVALID:
 			return "invalid";
 	}

src/data.c

@@ -132,11 +132,10 @@ parse_page(Page page, XLogRecPtr *lsn)
  */
 static int
 read_page_from_file(pgFile *file, BlockNumber blknum,
-					FILE *in, Page page)
+					FILE *in, Page page, XLogRecPtr *page_lsn)
 {
 	off_t				offset = blknum*BLCKSZ;
 	size_t				read_len = 0;
-	XLogRecPtr			page_lsn;
 
 	/* read the block */
 	if (fseek(in, offset, SEEK_SET) != 0)
@@ -167,7 +166,7 @@ read_page_from_file(pgFile *file, BlockNumber blknum,
 	 * If after several attempts page header is still invalid, throw an error.
 	 * The same idea is applied to checksum verification.
 	 */
-	if (!parse_page(page, &page_lsn))
+	if (!parse_page(page, page_lsn))
 	{
 		int i;
 		/* Check if the page is zeroed. */
@@ -232,6 +231,7 @@ backup_data_page(backup_files_args *arguments,
 	BackupPageHeader	header;
 	Page 				page = malloc(BLCKSZ);
 	Page 				compressed_page = NULL;
+	XLogRecPtr			page_lsn = 0;
 	size_t				write_buffer_size;
 	char				write_buffer[BLCKSZ+sizeof(header)];
 
@@ -252,7 +252,7 @@ backup_data_page(backup_files_args *arguments,
 		while(!page_is_valid && try_again)
 		{
 			int result = read_page_from_file(file, blknum,
-											 in, page);
+											 in, page, &page_lsn);
 
 			try_again--;
 			if (result == 0)
@@ -316,6 +316,25 @@ backup_data_page(backup_files_args *arguments,
 			if (is_checksum_enabled)
 				((PageHeader) page)->pd_checksum = pg_checksum_page(page, absolute_blknum);
 		}
+		/* get lsn from page, provided by pg_ptrack_get_block() */
+		if (backup_mode == BACKUP_MODE_DIFF_DELTA &&
+			file->exists_in_prev &&
+			header.compressed_size != PageIsTruncated &&
+			!parse_page(page, &page_lsn))
+				elog(ERROR, "Cannot parse page after pg_ptrack_get_block. "
+								"Possible risk of a memory corruption");
+
+	}
+
+	if (backup_mode == BACKUP_MODE_DIFF_DELTA &&
+		file->exists_in_prev &&
+		header.compressed_size != PageIsTruncated &&
+		page_lsn < prev_backup_start_lsn)
+	{
+		elog(VERBOSE, "Skipping blknum: %u in file: %s", blknum, file->path);
+		(*n_skipped)++;
+		free(page);
+		return;
 	}
 
 	if (header.compressed_size != PageIsTruncated)
@@ -478,6 +497,8 @@ backup_data_file(backup_files_args* arguments,
 							 &n_blocks_skipped, backup_mode);
 			n_blocks_read++;
 		}
+		if (backup_mode == BACKUP_MODE_DIFF_DELTA)
+			file->n_blocks = n_blocks_read;
 	}
 	/* If page map is not empty we scan only changed blocks, */
 	else
@@ -541,17 +562,22 @@ restore_data_file(const char *from_root,
 				  pgBackup *backup)
 {
 	char				to_path[MAXPGPATH];
-	FILE			   *in;
-	FILE			   *out;
+	FILE			   *in = NULL;
+	FILE			   *out = NULL;
 	BackupPageHeader	header;
 	BlockNumber			blknum;
+	size_t              file_size;
 
-	/* open backup mode file for read */
-	in = fopen(file->path, "r");
-	if (in == NULL)
+	/* BYTES_INVALID allowed only in case of restoring file from DELTA backup */
+	if (file->write_size != BYTES_INVALID)
 	{
-		elog(ERROR, "cannot open backup file \"%s\": %s", file->path,
-			 strerror(errno));
+		/* open backup mode file for read */
+		in = fopen(file->path, "r");
+		if (in == NULL)
+		{
+			elog(ERROR, "cannot open backup file \"%s\": %s", file->path,
+				 strerror(errno));
+		}
 	}
 
 	/*
@@ -577,6 +603,10 @@ restore_data_file(const char *from_root,
 		DataPage	compressed_page; /* used as read buffer */
 		DataPage	page;
 
+		/* File didn`t changed. Nothig to copy */
+		if (file->write_size == BYTES_INVALID)
+			break;
+
 		/* read BackupPageHeader */
 		read_len = fread(&header, 1, sizeof(header), in);
 		if (read_len != sizeof(header))
@@ -652,12 +682,38 @@ restore_data_file(const char *from_root,
 		}
 	}
 
+    /*
+     * DELTA backup have no knowledge about truncated blocks as PAGE or PTRACK do
+     * But during DELTA backup we read every file in PGDATA and thus DELTA backup
+     * knows exact size of every file at the time of backup.
+     * So when restoring file from DELTA backup we, knowning it`s size at
+     * a time of a backup, can truncate file to this size.
+     */
+
+	if (backup->backup_mode == BACKUP_MODE_DIFF_DELTA)
+	{
+		/* get file current size */
+		fseek(out, 0, SEEK_END);
+		file_size = ftell(out);
+		if (file_size > file->n_blocks * BLCKSZ)
+		{
+			/*
+			 * Truncate file to this length.
+			 */
+			if (ftruncate(fileno(out), file->n_blocks * BLCKSZ) != 0)
+				elog(ERROR, "cannot truncate \"%s\": %s",
+					 file->path, strerror(errno));
+			elog(INFO, "Delta truncate file %s to block %u", file->path, file->n_blocks);
+		}
+	}
+
 	/* update file permission */
 	if (chmod(to_path, file->mode) == -1)
 	{
 		int errno_tmp = errno;
 
-		fclose(in);
+		if (in)
+			fclose(in);
 		fclose(out);
 		elog(ERROR, "cannot change mode of \"%s\": %s", to_path,
 			 strerror(errno_tmp));
@@ -667,7 +723,8 @@ restore_data_file(const char *from_root,
 		fsync(fileno(out)) != 0 ||
 		fclose(out))
 		elog(ERROR, "cannot write \"%s\": %s", to_path, strerror(errno));
-	fclose(in);
+	if (in)
+		fclose(in);
 }
 
 /*

src/dir.c

@@ -166,6 +166,8 @@ pgFileInit(const char *path)
 	file->path = pgut_malloc(strlen(path) + 1);
 	strcpy(file->path, path);		/* enough buffer size guaranteed */
 	file->is_cfs = false;
+	file->exists_in_prev = false;	/* can change only in Incremental backup. */
+	file->n_blocks = -1;			/* can change only in DELTA backup. Number of blocks readed during backup */
 	file->compress_alg = NOT_DEFINED_COMPRESS;
 	return file;
 }
@@ -721,6 +723,9 @@ print_file_list(FILE *out, const parray *files, const char *root)
 		if (S_ISLNK(file->mode))
 			fprintf(out, ",\"linked\":\"%s\"", file->linked);
 
+		if (file->n_blocks != -1)
+			fprintf(out, ",\"n_blocks\":\"%i\"", file->n_blocks);
+
 		fprintf(out, "}\n");
 	}
 }
@@ -888,7 +893,8 @@ dir_read_file_list(const char *root, const char *file_txt)
 					is_datafile,
 					is_cfs,
 					crc,
-					segno;
+					segno,
+					n_blocks;
 		pgFile	   *file;
 
 		get_control_value(buf, "path", path, NULL, true);
@@ -902,6 +908,7 @@ dir_read_file_list(const char *root, const char *file_txt)
 		get_control_value(buf, "linked", linked, NULL, false);
 		get_control_value(buf, "segno", NULL, &segno, false);
 		get_control_value(buf, "compress_alg", compress_alg_string, NULL, false);
+		get_control_value(buf, "n_blocks", NULL, &n_blocks, false);
 
 		if (root)
 			join_path_components(filepath, root, path);
@@ -919,6 +926,7 @@ dir_read_file_list(const char *root, const char *file_txt)
 		if (linked[0])
 			file->linked = pgut_strdup(linked);
 		file->segno = (int) segno;
+		file->n_blocks = (int) n_blocks;
 
 		parray_append(files, file);
 	}

src/help.c

@@ -190,7 +190,7 @@ help_backup(void)
 	printf(_("                 [--replica-timeout=timeout]\n\n"));
 
 	printf(_("  -B, --backup-path=backup-path    location of the backup storage area\n"));
-	printf(_("  -b, --backup-mode=backup-mode    backup mode=FULL|PAGE|PTRACK\n"));
+	printf(_("  -b, --backup-mode=backup-mode    backup mode=FULL|PAGE|DELTA|PTRACK\n"));
 	printf(_("      --instance=instance_name     name of the instance\n"));
 	printf(_("  -C, --smooth-checkpoint          do smooth checkpoint before backup\n"));
 	printf(_("      --stream                     stream the transaction log and include it in the backup\n"));

src/pg_probackup.h

@@ -97,8 +97,10 @@ typedef struct pgFile
 	Oid		relOid;			/* relOid extracted from path, if applicable */
 	char	*forkName;		/* forkName extracted from path, if applicable */
 	int		segno;			/* Segment number for ptrack */
+	int		n_blocks;		/* size of the file in blocks, readed during DELTA backup */
 	bool	is_cfs;			/* Flag to distinguish files compressed by CFS*/
 	bool	is_database;
+	bool	exists_in_prev;	/* Mark files, both data and regular, that exists in previous backup */
 	CompressAlg compress_alg; /* compression algorithm applied to the file */
 	volatile uint32 lock;	/* lock for synchronization of parallel threads  */
 	datapagemap_t pagemap;	/* bitmap of pages updated since previous backup */
@@ -126,7 +128,8 @@ typedef enum BackupMode
 {
 	BACKUP_MODE_INVALID = 0,
 	BACKUP_MODE_DIFF_PAGE,		/* incremental page backup */
-	BACKUP_MODE_DIFF_PTRACK,	/* incremental page backup with ptrack system*/
+	BACKUP_MODE_DIFF_PTRACK,	/* incremental page backup with ptrack system */
+	BACKUP_MODE_DIFF_DELTA,		/* incremental page backup with lsn comparison */
 	BACKUP_MODE_FULL			/* full backup */
 } BackupMode;
 

src/restore.c

@@ -378,17 +378,6 @@ restore_backup(pgBackup *backup)
 	pgBackupGetPath(backup, database_path, lengthof(database_path), DATABASE_DIR);
 	pgBackupGetPath(backup, list_path, lengthof(list_path), DATABASE_FILE_LIST);
 	files = dir_read_file_list(database_path, list_path);
-	for (i = parray_num(files) - 1; i >= 0; i--)
-	{
-		pgFile *file = (pgFile *) parray_get(files, i);
-
-		/*
-		 * Remove files which haven't changed since previous backup
-		 * and was not backed up
-		 */
-		if (file->write_size == BYTES_INVALID)
-			pgFileFree(parray_remove(files, i));
-	}
 
 	/* setup threads */
 	for (i = 0; i < parray_num(files); i++)
@@ -716,24 +705,31 @@ restore_files(void *arg)
 				 i + 1, (unsigned long) parray_num(arguments->files), rel_path);
 
 
-		/* Directories was created before */
-		if (S_ISDIR(file->mode))
+		/*
+		 * For PAGE and PTRACK backups skip files which haven't changed
+		 * since previous backup and thus were not backed up.
+		 * We cannot do the same when restoring DELTA backup because we need information
+		 * about every file to correctly truncate them.
+		 */
+		if (file->write_size == BYTES_INVALID &&
+			(arguments->backup->backup_mode == BACKUP_MODE_DIFF_PAGE
+			|| arguments->backup->backup_mode == BACKUP_MODE_DIFF_PTRACK))
 		{
-			elog(LOG, "directory, skip");
+			elog(VERBOSE, "The file didn`t changed. Skip restore: %s", file->path);
 			continue;
 		}
 
-		/* not backed up */
-		if (file->write_size == BYTES_INVALID)
+		/* Directories was created before */
+		if (S_ISDIR(file->mode))
 		{
-			elog(LOG, "not backed up, skip");
+			elog(VERBOSE, "directory, skip");
 			continue;
 		}
 
 		/* Do not restore tablespace_map file */
 		if (path_is_prefix_of_path(PG_TABLESPACE_MAP_FILE, rel_path))
 		{
-			elog(LOG, "skip tablespace_map");
+			elog(VERBOSE, "skip tablespace_map");
 			continue;
 		}
 
@@ -750,8 +746,9 @@ restore_files(void *arg)
 			copy_file(from_root, pgdata, file);
 
 		/* print size of restored file */
-		elog(LOG, "Restored file %s : %lu bytes",
-			 file->path, (unsigned long) file->write_size);
+		if (file->write_size != BYTES_INVALID)
+			elog(LOG, "Restored file %s : %lu bytes",
+				file->path, (unsigned long) file->write_size);
 	}
 }
 

src/validate.c

@@ -51,7 +51,8 @@ pgBackupValidate(pgBackup *backup)
 
 	if (backup->backup_mode != BACKUP_MODE_FULL &&
 		backup->backup_mode != BACKUP_MODE_DIFF_PAGE &&
-		backup->backup_mode != BACKUP_MODE_DIFF_PTRACK)
+		backup->backup_mode != BACKUP_MODE_DIFF_PTRACK &&
+		backup->backup_mode != BACKUP_MODE_DIFF_DELTA)
 		elog(INFO, "Invalid backup_mode of backup %s", base36enc(backup->start_time));
 
 	pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR);