Merge remote-tracking branch 'origin/master' into issue_188

Author: kuzmindb

README.md

@@ -1,4 +1,5 @@
 [![Build Status](https://travis-ci.com/postgrespro/pg_probackup.svg?branch=master)](https://travis-ci.com/postgrespro/pg_probackup)
+[![GitHub release](https://img.shields.io/github/v/release/postgrespro/pg_probackup?include_prereleases)](https://github.com/postgrespro/pg_probackup/releases/latest)
 
 # pg_probackup
 
@@ -53,15 +54,15 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp
 
 ## Current release
 
-[2.3.2](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.2)
+[2.3.4](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.4)
 
 ## Documentation
 
 Documentation can be found at [github](https://postgrespro.github.io/pg_probackup) and [Postgres Professional documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup)
 
 ## Installation and Setup
 ### Windows Installation
-Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.3.1).
+Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.3.3).
 
 ### Linux Installation
 #### pg_probackup for vanilla PostgreSQL

doc/pgprobackup.xml

@@ -740,6 +740,16 @@ ALTER ROLE backup WITH REPLICATION;
       DELTA, and PTRACK backups in the
       <link linkend="pbk-stream-mode">STREAM</link> WAL mode.
     </para>
+    <note>
+      <para>
+        If you are planning to rely on
+        <ulink url="https://postgrespro.com/docs/postgresql/current/libpq-pgpass.html">.pgpass</ulink>
+        for authentication when running backup in STREAM mode,
+        then .pgpass must contain credentials for <literal>replication</literal> database,
+        used to establish connection via replication protocol. Example:
+        pghost:5432:replication:backup_user:my_strong_password
+      </para>
+    </note>
   </refsect2>
   <refsect2 id="pbk-setting-up-continuous-wal-archiving">
     <title>Setting up Continuous WAL Archiving</title>
@@ -1158,20 +1168,20 @@ CREATE EXTENSION ptrack;
       </listitem>
       <listitem>
       <para>
-       To enable tracking page updates, set <varname>ptrack_map_size</varname>
+       To enable tracking page updates, set <varname>ptrack.map_size</varname>
        parameter to a positive integer and restart the server.
       </para>
       <para>
        For optimal performance, it is recommended to set
-       <varname>ptrack_map_size</varname> to
+       <varname>ptrack.map_size</varname> to
        <literal><replaceable>N</replaceable> / 1024</literal>, where
        <replaceable>N</replaceable> is the size of the
        <productname>PostgreSQL</productname> cluster, in MB. If you set this
        parameter to a lower value, PTRACK is more likely to map several blocks
        together, which leads to false-positive results when tracking changed
        blocks and increases the incremental backup size as unchanged blocks
        can also be copied into the incremental backup.
-       Setting <varname>ptrack_map_size</varname> to a higher value
+       Setting <varname>ptrack.map_size</varname> to a higher value
        does not affect PTRACK operation. The maximum allowed value is 1024.
       </para>
      </listitem>
@@ -1191,11 +1201,11 @@ GRANT EXECUTE ON FUNCTION pg_ptrack_get_block(oid, oid, oid, bigint) TO backup;
 
    <note>
     <para>
-     If you change the <varname>ptrack_map_size</varname> parameter value,
+     If you change the <varname>ptrack.map_size</varname> parameter value,
      the previously created PTRACK map file is cleared,
      and tracking newly changed blocks starts from scratch. Thus, you have
      to retake a full backup before taking incremental PTRACK backups after
-     changing <varname>ptrack_map_size</varname>.
+     changing <varname>ptrack.map_size</varname>.
     </para>
    </note>
 
@@ -4045,10 +4055,10 @@ pg_probackup archive-push -B <replaceable>backup_dir</replaceable> --instance <r
         corruption.
       </para>
       <para>
-        To speed up archiving, you can specify the <option>-j</option> option
-        to run <command>archive-push</command> on multiple threads.
-        If you provide the <option>--batch-size</option> option, WAL files
-        will be copied in batches of the specified size.
+        To speed up archiving, you can specify the <option>--batch-size</option> option
+        to copy WAL segments in batches of the specified size.
+        If <option>--batch-size</option> option is used, then you can also specify
+        the <option>-j</option> option to copy the batch of WAL segments on multiple threads.
       </para>
       <para>
         WAL segments copied to the archive are synced to disk unless
@@ -4086,10 +4096,10 @@ pg_probackup archive-get -B <replaceable>backup_dir</replaceable> --instance <re
       </para>
 
       <para>
-        To speed up recovery, you can specify the <option>-j</option> option
-        to run <command>archive-get</command> on multiple threads.
-        If you provide the <option>--batch-size</option> option, WAL segments
-        will be copied in batches of the specified size.
+        To speed up recovery, you can specify the <option>--batch-size</option> option
+        to copy WAL segments in batches of the specified size.
+        If <option>--batch-size</option> option is used, then you can also specify
+        the <option>-j</option> option to copy the batch of WAL segments on multiple threads.
       </para>
 
       <para>
@@ -4714,7 +4724,7 @@ pg_probackup archive-get -B <replaceable>backup_dir</replaceable> --instance <re
 <term><option>--password</option></term>
       <listitem>
       <para>
-        Forces a password prompt.
+        Forces a password prompt. (Deprecated)
       </para>
       </listitem>
       </varlistentry>

src/archive.c

@@ -944,7 +944,7 @@ setup_push_filelist(const char *archive_status_dir, const char *first_file,
 
 	/* get list of files from archive_status */
 	status_files = parray_new();
-	dir_list_file(status_files, archive_status_dir, false, false, false, 0, FIO_DB_HOST);
+	dir_list_file(status_files, archive_status_dir, false, false, false, true, 0, FIO_DB_HOST);
 	parray_qsort(status_files, pgFileComparePath);
 
 	for (i = 0; i < parray_num(status_files); i++)

src/backup.c

@@ -192,7 +192,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 		{
 			/* try to setup multi-timeline backup chain */
 			elog(WARNING, "Valid backup on current timeline %u is not found, "
-						"try to look up on previous timelines",
+						"trying to look up on previous timelines",
 						current.tli);
 
 			tli_list = catalog_get_timelines(&instance_config);
@@ -333,7 +333,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 
 	/* list files with the logical path. omit $PGDATA */
 	dir_list_file(backup_files_list, instance_config.pgdata,
-				  true, true, false, 0, FIO_DB_HOST);
+				  true, true, false, true, 0, FIO_DB_HOST);
 
 	/*
 	 * Get database_map (name to oid) for use in partial restore feature.
@@ -350,7 +350,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 			/* External dirs numeration starts with 1.
 			 * 0 value is not external dir */
 			dir_list_file(backup_files_list, parray_get(external_dirs, i),
-						  false, true, false, i+1, FIO_DB_HOST);
+						  false, true, false, true, i+1, FIO_DB_HOST);
 
 	/* close ssh session in main thread */
 	fio_disconnect();
@@ -401,10 +401,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 
 	if (current.backup_mode != BACKUP_MODE_FULL)
 	{
-		elog(LOG, "current_tli:%X", current.tli);
-		elog(LOG, "prev_backup->start_lsn: %X/%X",
+		elog(LOG, "Current tli: %X", current.tli);
+		elog(LOG, "Parent start_lsn: %X/%X",
 			 (uint32) (prev_backup->start_lsn >> 32), (uint32) (prev_backup->start_lsn));
-		elog(LOG, "current.start_lsn: %X/%X",
+		elog(LOG, "start_lsn: %X/%X",
 			 (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn));
 	}
 
@@ -436,10 +436,11 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 			/*
 			 * Build the page map from ptrack information.
 			 */
-			if (nodeInfo->ptrack_version_num == 20)
+			if (nodeInfo->ptrack_version_num >= 20)
 				make_pagemap_from_ptrack_2(backup_files_list, backup_conn,
-											nodeInfo->ptrack_schema,
-											prev_backup_start_lsn);
+										   nodeInfo->ptrack_schema,
+										   nodeInfo->ptrack_version_num,
+										   prev_backup_start_lsn);
 			else if (nodeInfo->ptrack_version_num == 15 ||
 					 nodeInfo->ptrack_version_num == 16 ||
 					 nodeInfo->ptrack_version_num == 17)
@@ -582,9 +583,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 	/* Notify end of backup */
 	pg_stop_backup(&current, pg_startbackup_conn, nodeInfo);
 
-	elog(LOG, "current.stop_lsn: %X/%X",
-		 (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn));
-
 	/* In case of backup from replica >= 9.6 we must fix minRecPoint,
 	 * First we must find pg_control in backup_files_list.
 	 */
@@ -626,7 +624,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 		/* Scan backup PG_XLOG_DIR */
 		xlog_files_list = parray_new();
 		join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR);
-		dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, 0,
+		dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, true, 0,
 					  FIO_BACKUP_HOST);
 
 		/* TODO: Drop streamed WAL segments greater than stop_lsn */
@@ -884,15 +882,10 @@ do_backup(time_t start_time, bool no_validate,
 #endif
 
 	get_ptrack_version(backup_conn, &nodeInfo);
-//	elog(WARNING, "ptrack_version_num %d", ptrack_version_num);
+	//	elog(WARNING, "ptrack_version_num %d", ptrack_version_num);
 
 	if (nodeInfo.ptrack_version_num > 0)
-	{
-		if (nodeInfo.ptrack_version_num >= 20)
-			nodeInfo.is_ptrack_enable = pg_ptrack_enable2(backup_conn);
-		else
-			nodeInfo.is_ptrack_enable = pg_ptrack_enable(backup_conn);
-	}
+		nodeInfo.is_ptrack_enable = pg_ptrack_enable(backup_conn, nodeInfo.ptrack_version_num);
 
 	if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
 	{
@@ -1746,65 +1739,66 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
 		/* Calculate LSN */
 		stop_backup_lsn_tmp = ((uint64) lsn_hi) << 32 | lsn_lo;
 
+		/* It is ok for replica to return invalid STOP LSN
+		 * UPD: Apparently it is ok even for a master.
+		 */
 		if (!XRecOffIsValid(stop_backup_lsn_tmp))
 		{
-			/* It is ok for replica to return STOP LSN with NullXRecOff
-			 * UPD: Apparently it is ok even for master.
-			 */
-			if (XRecOffIsNull(stop_backup_lsn_tmp))
-			{
-				char	   *xlog_path,
-							stream_xlog_path[MAXPGPATH];
-				XLogSegNo	segno = 0;
-				XLogRecPtr	lsn_tmp = InvalidXLogRecPtr;
+			char	   *xlog_path,
+						stream_xlog_path[MAXPGPATH];
+			XLogSegNo	segno = 0;
+			XLogRecPtr	lsn_tmp = InvalidXLogRecPtr;
 
-				/*
-				 * Even though the value is invalid, it's expected postgres behaviour
-				 * and we're trying to fix it below.
-				 */
-				elog(LOG, "Null offset in stop_backup_lsn value %X/%X, trying to fix",
-					 (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp));
+			/*
+			 * Even though the value is invalid, it's expected postgres behaviour
+			 * and we're trying to fix it below.
+			 */
+			elog(LOG, "Invalid offset in stop_lsn value %X/%X, trying to fix",
+				 (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp));
 
-				/*
-				 * Note: even with gdb it is very hard to produce automated tests for
-				 * contrecord + NullXRecOff, so emulate it for manual testing.
-				 */
-				//stop_backup_lsn_tmp = stop_backup_lsn_tmp - XLOG_SEG_SIZE;
-				//elog(WARNING, "New Invalid stop_backup_lsn value %X/%X",
-				//	 (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp));
+			/*
+			 * Note: even with gdb it is very hard to produce automated tests for
+			 * contrecord + invalid LSN, so emulate it for manual testing.
+			 */
+			//stop_backup_lsn_tmp = stop_backup_lsn_tmp - XLOG_SEG_SIZE;
+			//elog(WARNING, "New Invalid stop_backup_lsn value %X/%X",
+			//	 (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp));
 
-				if (stream_wal)
-				{
-					pgBackupGetPath2(backup, stream_xlog_path,
-									 lengthof(stream_xlog_path),
-									 DATABASE_DIR, PG_XLOG_DIR);
-					xlog_path = stream_xlog_path;
-				}
-				else
-					xlog_path = arclog_path;
+			if (stream_wal)
+			{
+				pgBackupGetPath2(backup, stream_xlog_path,
+								 lengthof(stream_xlog_path),
+								 DATABASE_DIR, PG_XLOG_DIR);
+				xlog_path = stream_xlog_path;
+			}
+			else
+				xlog_path = arclog_path;
 
-				GetXLogSegNo(stop_backup_lsn_tmp, segno, instance_config.xlog_seg_size);
+			GetXLogSegNo(stop_backup_lsn_tmp, segno, instance_config.xlog_seg_size);
 
-				/*
-				 * Note, that there is no guarantee that corresponding WAL file even exists.
-				 * Replica may return LSN from future and keep staying in present.
-				 * Or it can return LSN with NullXRecOff.
-				 *
-				 * That's bad, since we want to get real LSN to save it in backup label file
-				 * and to use it in WAL validation.
-				 *
-				 * So we try to do the following:
-				 * 1. Wait 'archive_timeout' seconds for segment containing stop_lsn and
-				 *	  look for the first valid record in it.
-				 * 	  It solves the problem of occasional invalid XRecOff on write-busy system.
-				 * 2. Failing that, look for record in previous segment with endpoint
-				 *	  equal or greater than stop_lsn. It may(!) solve the problem of NullXRecOff
-				 *	  on write-idle system. If that fails too, error out.
-				 */
+			/*
+			 * Note, that there is no guarantee that corresponding WAL file even exists.
+			 * Replica may return LSN from future and keep staying in present.
+			 * Or it can return invalid LSN.
+			 *
+			 * That's bad, since we want to get real LSN to save it in backup label file
+			 * and to use it in WAL validation.
+			 *
+			 * So we try to do the following:
+			 * 1. Wait 'archive_timeout' seconds for segment containing stop_lsn and
+			 *	  look for the first valid record in it.
+			 * 	  It solves the problem of occasional invalid LSN on write-busy system.
+			 * 2. Failing that, look for record in previous segment with endpoint
+			 *	  equal or greater than stop_lsn. It may(!) solve the problem of invalid LSN
+			 *	  on write-idle system. If that fails too, error out.
+			 */
 
+			/* stop_lsn is pointing to a 0 byte of xlog segment */
+			if (stop_backup_lsn_tmp % instance_config.xlog_seg_size == 0)
+			{
 				/* Wait for segment with current stop_lsn, it is ok for it to never arrive */
 				wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli,
-							false, true, WARNING, stream_wal);
+							 false, true, WARNING, stream_wal);
 
 				/* Get the first record in segment with current stop_lsn */
 				lsn_tmp = get_first_record_lsn(xlog_path, segno, backup->tli,
@@ -1840,17 +1834,39 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
 									(uint32) (stop_backup_lsn_tmp >> 32),
 									(uint32) (stop_backup_lsn_tmp));
 				}
+			}
+			/* stop lsn is aligned to xlog block size, just find next lsn */
+			else if (stop_backup_lsn_tmp % XLOG_BLCKSZ == 0)
+			{
+				/* Wait for segment with current stop_lsn */
+				wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli,
+							 false, true, ERROR, stream_wal);
+
+				/* Get the next closest record in segment with current stop_lsn */
+				lsn_tmp = get_next_record_lsn(xlog_path, segno, backup->tli,
+										       instance_config.xlog_seg_size,
+										       instance_config.archive_timeout,
+										       stop_backup_lsn_tmp);
 
-				/* Setting stop_backup_lsn will set stop point for streaming */
-				stop_backup_lsn = lsn_tmp;
-				stop_lsn_exists = true;
+				/* sanity */
+				if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp))
+					elog(ERROR, "Failed to get WAL record next to %X/%X",
+								(uint32) (stop_backup_lsn_tmp >> 32),
+								(uint32) (stop_backup_lsn_tmp));
 			}
 			/* PostgreSQL returned something very illegal as STOP_LSN, error out */
 			else
 				elog(ERROR, "Invalid stop_backup_lsn value %X/%X",
 					 (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp));
+
+			/* Setting stop_backup_lsn will set stop point for streaming */
+			stop_backup_lsn = lsn_tmp;
+			stop_lsn_exists = true;
 		}
 
+		elog(LOG, "stop_lsn: %X/%X",
+			(uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn));
+
 		/* Write backup_label and tablespace_map */
 		if (!exclusive_backup)
 		{

src/catalog.c

@@ -872,7 +872,7 @@ catalog_get_timelines(InstanceConfig *instance)
 
 	/* read all xlog files that belong to this archive */
 	sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name);
-	dir_list_file(xlog_files_list, arclog_path, false, false, false, 0, FIO_BACKUP_HOST);
+	dir_list_file(xlog_files_list, arclog_path, false, false, false, true, 0, FIO_BACKUP_HOST);
 	parray_qsort(xlog_files_list, pgFileComparePath);
 
 	timelineinfos = parray_new();

src/checkdb.c

@@ -208,7 +208,7 @@ do_block_validation(char *pgdata, uint32 checksum_version)
 
 	/* list files with the logical path. omit $PGDATA */
 	dir_list_file(files_list, pgdata,
-				  true, true, false, 0, FIO_DB_HOST);
+				  true, true, false, true, 0, FIO_DB_HOST);
 
 	/*
 	 * Sort pathname ascending.