Merge branch 'master' into stable

Author: za-arthur

Makefile

@@ -1,7 +1,7 @@
 PROGRAM = pg_probackup
 OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \
 	src/delete.o src/dir.o src/fetch.o src/help.o src/init.o \
-	src/pg_probackup.o src/restore.o src/show.o src/status.o \
+	src/pg_probackup.o src/restore.o src/show.o \
 	src/util.o src/validate.o src/datapagemap.o src/parsexlog.o \
 	src/xlogreader.o src/streamutil.o src/receivelog.o \
 	src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o \
@@ -38,7 +38,7 @@ EXTRA_CLEAN += src/walmethods.c src/walmethods.h
 INCLUDES += src/walmethods.h
 endif
 
-PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc
+PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_srcdir)/$(subdir)/src
 override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS)
 PG_LIBS = $(libpq_pgport) ${PTHREAD_CFLAGS}
 

gen_probackup_project.pl

@@ -145,7 +145,6 @@ sub build_pgprobackup
 		'pg_probackup.c',
 		'restore.c',
 		'show.c',
-		'status.c',
 		'util.c',
 		'validate.c'
 		);

src/archive.c

@@ -7,10 +7,10 @@
  *
  *-------------------------------------------------------------------------
  */
+
 #include "pg_probackup.h"
 
 #include <unistd.h>
-#include <sys/stat.h>
 
 /*
  * pg_probackup specific archive command for archive backups

src/backup.c

@@ -10,25 +10,28 @@
 
 #include "pg_probackup.h"
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/time.h>
-#include <unistd.h>
-#include <dirent.h>
-#include <time.h>
-
+#if PG_VERSION_NUM < 110000
 #include "catalog/catalog.h"
+#endif
 #include "catalog/pg_tablespace.h"
-#include "datapagemap.h"
-#include "libpq/pqsignal.h"
 #include "pgtar.h"
 #include "receivelog.h"
-#include "storage/bufpage.h"
 #include "streamutil.h"
+
+#include <sys/stat.h>
+#include <unistd.h>
+
 #include "utils/thread.h"
 
+#define PG_STOP_BACKUP_TIMEOUT 300
+
+/*
+ * Macro needed to parse ptrack.
+ * NOTE Keep those values syncronised with definitions in ptrack.h
+ */
+#define PTRACK_BITS_PER_HEAPBLOCK 1
+#define HEAPBLOCKS_PER_BYTE (BITS_PER_BYTE / PTRACK_BITS_PER_HEAPBLOCK)
+
 static int	standby_message_timeout = 10 * 1000;	/* 10 sec = default */
 static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr;
 static XLogRecPtr stop_stream_lsn = InvalidXLogRecPtr;
@@ -531,7 +534,7 @@ do_backup_instance(void)
 		prev_backup_start_lsn = prev_backup->start_lsn;
 		current.parent_backup = prev_backup->start_time;
 
-		pgBackupWriteBackupControlFile(&current);
+		write_backup(&current);
 	}
 
 	/*
@@ -906,11 +909,13 @@ do_backup(time_t start_time)
 	/* Start backup. Update backup status. */
 	current.status = BACKUP_STATUS_RUNNING;
 	current.start_time = start_time;
+	StrNCpy(current.program_version, PROGRAM_VERSION,
+			sizeof(current.program_version));
 
 	/* Create backup directory and BACKUP_CONTROL_FILE */
 	if (pgBackupCreateDir(&current))
 		elog(ERROR, "cannot create backup directory");
-	pgBackupWriteBackupControlFile(&current);
+	write_backup(&current);
 
 	elog(LOG, "Backup destination is initialized");
 
@@ -932,7 +937,7 @@ do_backup(time_t start_time)
 	/* Backup is done. Update backup status */
 	current.end_time = time(NULL);
 	current.status = BACKUP_STATUS_DONE;
-	pgBackupWriteBackupControlFile(&current);
+	write_backup(&current);
 
 	//elog(LOG, "Backup completed. Total bytes : " INT64_FORMAT "",
 	//		current.data_bytes);
@@ -2010,7 +2015,7 @@ backup_cleanup(bool fatal, void *userdata)
 			 base36enc(current.start_time));
 		current.end_time = time(NULL);
 		current.status = BACKUP_STATUS_ERROR;
-		pgBackupWriteBackupControlFile(&current);
+		write_backup(&current);
 	}
 
 	/*
@@ -2096,12 +2101,13 @@ backup_files(void *arg)
 
 		if (S_ISREG(buf.st_mode))
 		{
+			pgFile	  **prev_file;
+
 			/* Check that file exist in previous backup */
 			if (current.backup_mode != BACKUP_MODE_FULL)
 			{
 				char	   *relative;
 				pgFile		key;
-				pgFile	  **prev_file;
 
 				relative = GetRelativePath(file->path, arguments->from_root);
 				key.path = relative;
@@ -2131,20 +2137,27 @@ backup_files(void *arg)
 					continue;
 				}
 			}
-			/* TODO:
-			 * Check if file exists in previous backup
-			 * If exists:
-			 *   if mtime > start_backup_time of parent backup,
-			 *    copy file to backup
-			 *   if mtime < start_backup_time
-			 *    calculate crc, compare crc to old file
-			 *      if crc is the same -> skip file
-			 */
-			else if (!copy_file(arguments->from_root, arguments->to_root, file))
+			else 
 			{
-				file->write_size = BYTES_INVALID;
-				elog(VERBOSE, "File \"%s\" was not copied to backup", file->path);
-				continue;
+				bool skip = false;
+
+				/* If non-data file has not changed since last backup... */
+				if (file->exists_in_prev &&
+					buf.st_mtime < current.parent_backup)
+				{
+					calc_file_checksum(file);
+					/* ...and checksum is the same... */
+					if (EQ_CRC32C(file->crc, (*prev_file)->crc))
+						skip = true; /* ...skip copying file. */
+				}
+				if (skip ||
+					!copy_file(arguments->from_root, arguments->to_root, file))
+				{
+					file->write_size = BYTES_INVALID;
+					elog(VERBOSE, "File \"%s\" was not copied to backup",
+						 file->path);
+					continue;
+				}
 			}
 
 			elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes",

src/catalog.c

@@ -11,13 +11,8 @@
 #include "pg_probackup.h"
 
 #include <dirent.h>
-#include <fcntl.h>
 #include <signal.h>
-#include <sys/file.h>
 #include <sys/stat.h>
-#include <sys/types.h>
-#include <stdlib.h>
-#include <time.h>
 #include <unistd.h>
 
 static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"};
@@ -221,6 +216,25 @@ read_backup(time_t timestamp)
 	return readBackupControlFile(conf_path);
 }
 
+/*
+ * Save the backup status into BACKUP_CONTROL_FILE.
+ *
+ * We need to reread the backup using its ID and save it changing only its
+ * status.
+ */
+void
+write_backup_status(pgBackup *backup)
+{
+	pgBackup   *tmp;
+
+	tmp = read_backup(backup->start_time);
+
+	tmp->status = backup->status;
+	write_backup(tmp);
+
+	pgBackupFree(tmp);
+}
+
 /*
  * Get backup_mode in string representation.
  */
@@ -431,7 +445,8 @@ pgBackupWriteControl(FILE *out, pgBackup *backup)
 	fprintf(out, "block-size = %u\n", backup->block_size);
 	fprintf(out, "xlog-block-size = %u\n", backup->wal_block_size);
 	fprintf(out, "checksum-version = %u\n", backup->checksum_version);
-	fprintf(out, "program-version = %s\n", PROGRAM_VERSION);
+	if (backup->program_version[0] != '\0')
+		fprintf(out, "program-version = %s\n", backup->program_version);
 	if (backup->server_version[0] != '\0')
 		fprintf(out, "server-version = %s\n", backup->server_version);
 
@@ -481,17 +496,19 @@ pgBackupWriteControl(FILE *out, pgBackup *backup)
 		fprintf(out, "primary_conninfo = '%s'\n", backup->primary_conninfo);
 }
 
-/* create BACKUP_CONTROL_FILE */
+/*
+ * Save the backup content into BACKUP_CONTROL_FILE.
+ */
 void
-pgBackupWriteBackupControlFile(pgBackup *backup)
+write_backup(pgBackup *backup)
 {
 	FILE   *fp = NULL;
-	char	ini_path[MAXPGPATH];
+	char	conf_path[MAXPGPATH];
 
-	pgBackupGetPath(backup, ini_path, lengthof(ini_path), BACKUP_CONTROL_FILE);
-	fp = fopen(ini_path, "wt");
+	pgBackupGetPath(backup, conf_path, lengthof(conf_path), BACKUP_CONTROL_FILE);
+	fp = fopen(conf_path, "wt");
 	if (fp == NULL)
-		elog(ERROR, "cannot open configuration file \"%s\": %s", ini_path,
+		elog(ERROR, "Cannot open configuration file \"%s\": %s", conf_path,
 			strerror(errno));
 
 	pgBackupWriteControl(fp, backup);

src/configure.c

@@ -8,9 +8,6 @@
  */
 
 #include "pg_probackup.h"
-#include "utils/logger.h"
-
-#include "pqexpbuffer.h"
 
 #include "utils/json.h"