Merge branch 'master' into stable

Author: alubennikova

src/backup.c

@@ -867,8 +867,8 @@ do_backup(void)
 	current.status = BACKUP_STATUS_DONE;
 	pgBackupWriteBackupControlFile(&current);
 
-	elog(LOG, "Backup completed. Total bytes : " INT64_FORMAT "",
-		 current.data_bytes);
+	//elog(LOG, "Backup completed. Total bytes : " INT64_FORMAT "",
+	//		current.data_bytes);
 
 	pgBackupValidate(&current);
 
@@ -910,7 +910,8 @@ check_server_version(void)
 			 server_version % 100, "9.6");
 
 	/* Do exclusive backup only for PostgreSQL 9.5 */
-	exclusive_backup = server_version < 90600;
+	exclusive_backup = server_version < 90600 ||
+		current.backup_mode == BACKUP_MODE_DIFF_PTRACK;
 
 	/* Save server_version to use it in future */
 	res = pgut_execute(backup_conn, "show server_version", 0, NULL);
@@ -1401,13 +1402,19 @@ wait_wal_lsn(XLogRecPtr lsn, bool wait_prev_segment)
 		timeout = archive_timeout;
 	}
 
+	if (wait_prev_segment)
+		elog(LOG, "Looking for segment: %s", wal_segment);
+	else
+		elog(LOG, "Looking for LSN: %X/%X in segment: %s", (uint32) (lsn >> 32), (uint32) lsn, wal_segment);
+
 	/* Wait until target LSN is archived or streamed */
 	while (true)
 	{
 		bool		file_exists = fileExists(wal_segment_full_path);
 
 		if (file_exists)
 		{
+			elog(LOG, "Found segment: %s", wal_segment);
 			/* Do not check LSN for previous WAL segment */
 			if (wait_prev_segment)
 				return;
@@ -1418,7 +1425,10 @@ wait_wal_lsn(XLogRecPtr lsn, bool wait_prev_segment)
 			if ((stream_wal && wal_contains_lsn(wal_dir, lsn, tli)) ||
 				(!stream_wal && wal_contains_lsn(arclog_path, lsn, tli)))
 				/* Target LSN was found */
+			{
+				elog(LOG, "Found LSN: %X/%X", (uint32) (lsn >> 32), (uint32) lsn);
 				return;
+			}
 		}
 
 		sleep(1);
@@ -1653,6 +1663,8 @@ pg_stop_backup(pgBackup *backup)
 		}
 		if (!res)
 			elog(ERROR, "pg_stop backup() failed");
+		else
+			elog(INFO, "pg_stop backup() successfully executed");
 	}
 
 	backup_in_progress = false;
@@ -1990,13 +2002,15 @@ backup_files(void *arg)
 /*
  * Extract information about files in backup_list parsing their names:
  * - remove temp tables from the list
+ * - remove unlogged tables from the list (leave the _init fork)
  * - set flags for database directories
  * - set flags for datafiles
  */
 static void
 parse_backup_filelist_filenames(parray *files, const char *root)
 {
 	size_t		i;
+	Oid unlogged_file_reloid = 0;
 
 	for (i = 0; i < parray_num(files); i++)
 	{
@@ -2157,13 +2171,46 @@ parse_backup_filelist_filenames(parray *files, const char *root)
 					/* auxiliary fork of the relfile */
 					sscanf(filename, "%u_%s", &(file->relOid), file->forkName);
 					elog(VERBOSE, "relOid %u, forkName %s, filepath %s", file->relOid, file->forkName, relative);
-					if (strcmp(file->forkName, "ptrack") == 0)
+
+					/* handle unlogged relations */
+					if (strcmp(file->forkName, "init") == 0)
+					{
+						/*
+						 * Do not backup files of unlogged relations.
+						 * scan filelist backward and exclude these files.
+						 */
+						int unlogged_file_num = i-1;
+						pgFile	   *unlogged_file = (pgFile *) parray_get(files, unlogged_file_num);
+
+						unlogged_file_reloid = file->relOid;
+
+						while (unlogged_file_num >= 0 &&
+							   (unlogged_file_reloid != 0) &&
+							   (unlogged_file->relOid == unlogged_file_reloid))
+						{
+							unlogged_file->size = 0;
+							pgFileFree(unlogged_file);
+							parray_remove(files, unlogged_file_num);
+							unlogged_file_num--;
+							i--;
+							unlogged_file = (pgFile *) parray_get(files, unlogged_file_num);
+						}
+					}
+					else if (strcmp(file->forkName, "ptrack") == 0)
 					{
 						/* Do not backup ptrack files */
 						pgFileFree(file);
 						parray_remove(files, i);
 						i--;
 					}
+					else if ((unlogged_file_reloid != 0) &&
+							 (file->relOid == unlogged_file_reloid))
+					{
+						/* Do not backup forks of unlogged relations */
+						pgFileFree(file);
+						parray_remove(files, i);
+						i--;
+					}
 					continue;
 				}
 
@@ -2201,6 +2248,15 @@ parse_backup_filelist_filenames(parray *files, const char *root)
 					 */
 					elog(VERBOSE, "relOid %u, segno %d, suffix %s, filepath %s", file->relOid, file->segno, suffix, relative);
 				}
+
+				if ((unlogged_file_reloid != 0) &&
+					(file->relOid == unlogged_file_reloid))
+				{
+					/* Do not backup segments of unlogged files */
+					pgFileFree(file);
+					parray_remove(files, i);
+					i--;
+				}
 			}
 		}
 	}

src/data.c

@@ -140,7 +140,7 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn,
 	/*
 	 * Read the page and verify its header and checksum.
 	 * Under high write load it's possible that we've read partly
-	 * flushed page, so try several times befor throwing an error.
+	 * flushed page, so try several times before throwing an error.
 	 */
 	while(try_checksum--)
 	{
@@ -152,6 +152,20 @@ backup_data_page(pgFile *file, XLogRecPtr prev_backup_start_lsn,
 
 		if (read_len != BLCKSZ)
 		{
+			if (read_len == 0)
+			{
+				elog(LOG, "File %s, block %u, file was truncated",
+					 file->path, blknum);
+				return;
+			}
+			else if (try_checksum)
+			{
+				elog(LOG, "File: %s, block %u, expected block size %lu, but read %d, try again",
+					 file->path, blknum, read_len, BLCKSZ);
+				usleep(100);
+				continue;
+			}
+
 			elog(ERROR, "File: %s, invalid block size of block %u : %lu",
 				 file->path, blknum, read_len);
 		}
@@ -494,7 +508,7 @@ restore_data_file(const char *from_root,
 		if (header.block < blknum)
 			elog(ERROR, "backup is broken at block %u", blknum);
 
-		elog(VERBOSE, "file %s, header compressed size %d", file->path, header.compressed_size);
+		//elog(VERBOSE, "file %s, header compressed size %d", file->path, header.compressed_size);
 		Assert(header.compressed_size <= BLCKSZ);
 
 		read_len = fread(compressed_page.data, 1,

src/pg_probackup.c

@@ -17,7 +17,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
-const char *PROGRAM_VERSION	= "2.0.11";
+const char *PROGRAM_VERSION	= "2.0.12";
 const char *PROGRAM_URL		= "https://github.com/postgrespro/pg_probackup";
 const char *PROGRAM_EMAIL	= "https://github.com/postgrespro/pg_probackup/issues";
 

src/restore.c

@@ -131,6 +131,18 @@ do_restore_or_validate(time_t target_backup_id,
 		if (target_backup_id && current_backup->start_time > target_backup_id)
 			continue;
 
+		/*
+		 * [PGPRO-1164] If BACKUP_ID is not provided for restore command,
+		 *  we must find the first valid(!) backup.
+		 */
+
+		if (is_restore && target_backup_id == 0 && current_backup->status != BACKUP_STATUS_OK)
+		{
+			elog(WARNING, "Skipping backup %s, because it has non-valid status: %s",
+				base36enc(current_backup->start_time), status2str(current_backup->status));
+			continue;
+		}
+
 		/*
 		 * We found target backup. Check its status and
 		 * ensure that it satisfies recovery target.

src/utils/pgut.c

@@ -47,6 +47,7 @@ static PGcancel *volatile cancel_conn = NULL;
 /* Interrupted by SIGINT (Ctrl+C) ? */
 bool		interrupted = false;
 bool		in_cleanup = false;
+bool		in_password = false;
 
 static bool parse_pair(const char buffer[], char key[], char value[]);
 
@@ -1080,7 +1081,8 @@ parse_pair(const char buffer[], char key[], char value[])
 static void
 prompt_for_password(const char *username)
 {
-	pqsignal(SIGINT, oldhandler);
+	in_password = true;
+
 	if (password)
 	{
 		free(password);
@@ -1107,7 +1109,8 @@ prompt_for_password(const char *username)
 		password = simple_prompt(message, 100, false);
 	}
 #endif
-	init_cancel_handler();
+
+	in_password = false;
 }
 
 PGconn *
@@ -1238,6 +1241,8 @@ pgut_connect_replication_extended(const char *pghost, const char *pgport,
 		{
 			PQfinish(tmpconn);
 			prompt_for_password(username);
+			keywords[i] = "password";
+			values[i] = password;
 			continue;
 		}
 
@@ -1525,6 +1530,15 @@ on_interrupt(void)
 	/* Set interruped flag */
 	interrupted = true;
 
+	/* User promts password, call on_cleanup() byhand */
+	if (in_password)
+	{
+		on_cleanup();
+
+		pqsignal(SIGINT, oldhandler);
+		kill(0, SIGINT);
+	}
+
 	/* Send QueryCancel if we are processing a database query */
 	if (!in_cleanup && cancel_conn != NULL &&
 		PQcancel(cancel_conn, errbuf, sizeof(errbuf)))

src/utils/pgut.h

@@ -105,6 +105,7 @@ extern bool			prompt_password;
 
 extern bool			interrupted;
 extern bool			in_cleanup;
+extern bool			in_password;	/* User prompts password */
 
 extern int pgut_getopt(int argc, char **argv, pgut_option options[]);
 extern void pgut_readopt(const char *path, pgut_option options[], int elevel);

src/validate.c

@@ -51,12 +51,12 @@ pgBackupValidate(pgBackup *backup)
 		return;
 	}
 
-	elog(LOG, "Validate backup %s", backup_id_string);
+	elog(INFO, "Validating backup %s", backup_id_string);
 
 	if (backup->backup_mode != BACKUP_MODE_FULL &&
 		backup->backup_mode != BACKUP_MODE_DIFF_PAGE &&
 		backup->backup_mode != BACKUP_MODE_DIFF_PTRACK)
-		elog(LOG, "Invalid backup_mode of backup %s", backup_id_string);
+		elog(INFO, "Invalid backup_mode of backup %s", backup_id_string);
 
 	pgBackupGetPath(backup, base_path, lengthof(base_path), DATABASE_DIR);
 	pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST);
@@ -99,7 +99,7 @@ pgBackupValidate(pgBackup *backup)
 	if (corrupted)
 		elog(WARNING, "Backup %s is corrupted", backup_id_string);
 	else
-		elog(LOG, "Backup %s is valid", backup_id_string);
+		elog(INFO, "Backup %s is valid", backup_id_string);
 	free(backup_id_string);
 }
 

tests/expected/option_version.out

@@ -1 +1 @@
-pg_probackup 2.0.11
+pg_probackup 2.0.12