Port to PostgreSQL 11

Author: za-arthur

Makefile

@@ -32,7 +32,7 @@ else
 srchome=$(top_srcdir)
 endif
 
-ifeq ($(MAJORVERSION),10)
+ifneq (,$(filter 10 11 12,$(MAJORVERSION)))
 OBJS += src/walmethods.o
 EXTRA_CLEAN += src/walmethods.c src/walmethods.h
 INCLUDES += src/walmethods.h
@@ -64,7 +64,7 @@ src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h
 	rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@
 
 
-ifeq ($(MAJORVERSION),10)
+ifneq (,$(filter 10 11 12,$(MAJORVERSION)))
 src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c
 	rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.c $@
 src/walmethods.h: $(top_srcdir)/src/bin/pg_basebackup/walmethods.h

src/backup.c

@@ -650,8 +650,9 @@ do_backup_instance(void)
 		 * reading WAL segments present in archives up to the point
 		 * where this backup has started.
 		 */
-		extractPageMap(arclog_path, prev_backup->start_lsn, current.tli,
-					   current.start_lsn, backup_files_list);
+		extractPageMap(arclog_path, current.tli, xlog_seg_size,
+					   prev_backup->start_lsn, current.start_lsn,
+					   backup_files_list);
 	}
 	else if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
 	{
@@ -827,6 +828,11 @@ do_backup(time_t start_time)
 
 	current.primary_conninfo = pgut_get_conninfo_string(backup_conn);
 
+#if PG_VERSION_NUM >= 110000
+	if (!RetrieveWalSegSize(backup_conn))
+		elog(ERROR, "Failed to retreive wal_segment_size");
+#endif
+
 	current.compress_alg = compress_alg;
 	current.compress_level = compress_level;
 
@@ -918,8 +924,9 @@ do_backup(time_t start_time)
 	/* compute size of wal files of this backup stored in the archive */
 	if (!current.stream)
 	{
-		current.wal_bytes = XLOG_SEG_SIZE *
-							(current.stop_lsn/XLogSegSize - current.start_lsn/XLogSegSize + 1);
+		current.wal_bytes = xlog_seg_size *
+			(current.stop_lsn / xlog_seg_size -
+			 current.start_lsn / xlog_seg_size + 1);
 	}
 
 	/* Backup is done. Update backup status */
@@ -1467,10 +1474,10 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment)
 	tli = get_current_timeline(false);
 
 	/* Compute the name of the WAL file containig requested LSN */
-	XLByteToSeg(lsn, targetSegNo);
+	GetXLogSegNo(lsn, targetSegNo, xlog_seg_size);
 	if (wait_prev_segment)
 		targetSegNo--;
-	XLogFileName(wal_segment, tli, targetSegNo);
+	GetXLogFileName(wal_segment, tli, targetSegNo, xlog_seg_size);
 
 	/*
 	 * In pg_start_backup we wait for 'lsn' in 'pg_wal' directory iff it is
@@ -1536,7 +1543,7 @@ wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, bool wait_prev_segment)
 			/*
 			 * A WAL segment found. Check LSN on it.
 			 */
-			if (wal_contains_lsn(wal_segment_dir, lsn, tli))
+			if (wal_contains_lsn(wal_segment_dir, lsn, tli, xlog_seg_size))
 				/* Target LSN was found */
 			{
 				elog(LOG, "Found LSN: %X/%X", (uint32) (lsn >> 32), (uint32) lsn);
@@ -1946,7 +1953,7 @@ pg_stop_backup(pgBackup *backup)
 
 		elog(LOG, "Getting the Recovery Time from WAL");
 
-		if (!read_recovery_info(xlog_path, backup->tli,
+		if (!read_recovery_info(xlog_path, backup->tli, xlog_seg_size,
 								backup->start_lsn, backup->stop_lsn,
 								&backup->recovery_time, &backup->recovery_xid))
 		{
@@ -2553,7 +2560,7 @@ StreamLog(void *arg)
 	/*
 	 * Always start streaming at the beginning of a segment
 	 */
-	startpos -= startpos % XLOG_SEG_SIZE;
+	startpos -= startpos % xlog_seg_size;
 
 	/* Initialize timeout */
 	stream_stop_timeout = 0;

src/catalog.c

@@ -572,23 +572,23 @@ readBackupControlFile(const char *path)
 	pgBackupInit(backup);
 	if (access(path, F_OK) != 0)
 	{
-		elog(WARNING, "control file \"%s\" doesn't exist", path);
+		elog(WARNING, "Control file \"%s\" doesn't exist", path);
 		pgBackupFree(backup);
 		return NULL;
 	}
 
-	parsed_options = pgut_readopt(path, options, WARNING);
+	parsed_options = pgut_readopt(path, options, WARNING, true);
 
 	if (parsed_options == 0)
 	{
-		elog(WARNING, "control file \"%s\" is empty", path);
+		elog(WARNING, "Control file \"%s\" is empty", path);
 		pgBackupFree(backup);
 		return NULL;
 	}
 
 	if (backup->start_time == 0)
 	{
-		elog(WARNING, "invalid ID/start-time, control file \"%s\" is corrupted", path);
+		elog(WARNING, "Invalid ID/start-time, control file \"%s\" is corrupted", path);
 		pgBackupFree(backup);
 		return NULL;
 	}
@@ -607,7 +607,7 @@ readBackupControlFile(const char *path)
 		if (sscanf(start_lsn, "%X/%X", &xlogid, &xrecoff) == 2)
 			backup->start_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff;
 		else
-			elog(WARNING, "invalid START_LSN \"%s\"", start_lsn);
+			elog(WARNING, "Invalid START_LSN \"%s\"", start_lsn);
 		free(start_lsn);
 	}
 
@@ -619,7 +619,7 @@ readBackupControlFile(const char *path)
 		if (sscanf(stop_lsn, "%X/%X", &xlogid, &xrecoff) == 2)
 			backup->stop_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff;
 		else
-			elog(WARNING, "invalid STOP_LSN \"%s\"", stop_lsn);
+			elog(WARNING, "Invalid STOP_LSN \"%s\"", stop_lsn);
 		free(stop_lsn);
 	}
 
@@ -644,7 +644,7 @@ readBackupControlFile(const char *path)
 		else if (strcmp(status, "CORRUPT") == 0)
 			backup->status = BACKUP_STATUS_CORRUPT;
 		else
-			elog(WARNING, "invalid STATUS \"%s\"", status);
+			elog(WARNING, "Invalid STATUS \"%s\"", status);
 		free(status);
 	}
 

src/configure.c

@@ -102,6 +102,13 @@ void
 pgBackupConfigInit(pgBackupConfig *config)
 {
 	config->system_identifier = 0;
+
+#if PG_VERSION_NUM >= 110000
+	config->xlog_seg_size = 0;
+#else
+	config->xlog_seg_size = XLOG_SEG_SIZE;
+#endif
+
 	config->pgdata = NULL;
 	config->pgdatabase = NULL;
 	config->pghost = NULL;
@@ -140,6 +147,9 @@ writeBackupCatalogConfig(FILE *out, pgBackupConfig *config)
 	fprintf(out, "#Backup instance info\n");
 	fprintf(out, "PGDATA = %s\n", config->pgdata);
 	fprintf(out, "system-identifier = " UINT64_FORMAT "\n", config->system_identifier);
+#if PG_VERSION_NUM >= 110000
+	fprintf(out, "xlog-seg-size = %u\n", config->xlog_seg_size);
+#endif
 
 	fprintf(out, "#Connection parameters:\n");
 	if (config->pgdatabase)
@@ -253,6 +263,9 @@ readBackupCatalogConfigFile(void)
 		{ 'u', 0, "replica-timeout",		&(config->replica_timeout),		SOURCE_CMDLINE,	SOURCE_DEFAULT,	OPTION_UNIT_MS },
 		/* other options */
 		{ 'U', 0, "system-identifier",		&(config->system_identifier),	SOURCE_FILE_STRICT },
+#if PG_VERSION_NUM >= 110000
+		{'u', 0, "xlog-seg-size",			&config->xlog_seg_size,			SOURCE_FILE_STRICT},
+#endif
 		/* archive options */
 		{ 'u', 0, "archive-timeout",		&(config->archive_timeout),		SOURCE_CMDLINE,	SOURCE_DEFAULT,	OPTION_UNIT_MS },
 		{0}
@@ -263,11 +276,44 @@ readBackupCatalogConfigFile(void)
 	join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE);
 
 	pgBackupConfigInit(config);
-	pgut_readopt(path, options, ERROR);
+	pgut_readopt(path, options, ERROR, true);
+
+#if PG_VERSION_NUM >= 110000
+	if (!IsValidWalSegSize(config->xlog_seg_size))
+		elog(ERROR, "Invalid WAL segment size %u", config->xlog_seg_size);
+#endif
 
 	return config;
 }
 
+/*
+ * Read xlog-seg-size from BACKUP_CATALOG_CONF_FILE.
+ */
+uint32
+get_config_xlog_seg_size(void)
+{
+#if PG_VERSION_NUM >= 110000
+	char		path[MAXPGPATH];
+	uint32		seg_size;
+	pgut_option options[] =
+	{
+		{'u', 0, "xlog-seg-size", &seg_size, SOURCE_FILE_STRICT},
+		{0}
+	};
+
+	join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE);
+	pgut_readopt(path, options, ERROR, false);
+
+	if (!IsValidWalSegSize(seg_size))
+		elog(ERROR, "Invalid WAL segment size %u", seg_size);
+
+	return seg_size;
+
+#else
+	return (uint32) XLOG_SEG_SIZE;
+#endif
+}
+
 static void
 opt_log_level_console(pgut_option *opt, const char *arg)
 {
@@ -349,6 +395,11 @@ show_configure_json(pgBackupConfig *config)
 	json_add_key(buf, "system-identifier", json_level, true);
 	appendPQExpBuffer(buf, UINT64_FORMAT, config->system_identifier);
 
+#if PG_VERSION_NUM >= 110000
+	json_add_key(buf, "xlog-seg-size", json_level, true);
+	appendPQExpBuffer(buf, "%u", config->xlog_seg_size);
+#endif
+
 	/* Connection parameters */
 	if (config->pgdatabase)
 		json_add_value(buf, "pgdatabase", config->pgdatabase, json_level, true);

src/delete.c

@@ -15,16 +15,17 @@
 #include <unistd.h>
 
 static int pgBackupDeleteFiles(pgBackup *backup);
-static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli);
+static void delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli,
+							uint32 xlog_seg_size);
 
 int
 do_delete(time_t backup_id)
 {
 	int			i;
 	parray	   *backup_list,
 			   *delete_list;
+	pgBackup   *target_backup = NULL;
 	time_t		parent_id = 0;
-	bool		backup_found = false;
 	XLogRecPtr	oldest_lsn = InvalidXLogRecPtr;
 	TimeLineID	oldest_tli = 0;
 
@@ -56,9 +57,9 @@ do_delete(time_t backup_id)
 
 				/* Save backup id to retreive increment backups */
 				parent_id = backup->start_time;
-				backup_found = true;
+				target_backup = backup;
 			}
-			else if (backup_found)
+			else if (target_backup)
 			{
 				if (backup->backup_mode != BACKUP_MODE_FULL &&
 					backup->parent_backup == parent_id)
@@ -93,6 +94,8 @@ do_delete(time_t backup_id)
 	/* Clean WAL segments */
 	if (delete_wal)
 	{
+		Assert(target_backup);
+
 		/* Find oldest LSN, used by backups */
 		for (i = (int) parray_num(backup_list) - 1; i >= 0; i--)
 		{
@@ -106,7 +109,7 @@ do_delete(time_t backup_id)
 			}
 		}
 
-		delete_walfiles(oldest_lsn, oldest_tli);
+		delete_walfiles(oldest_lsn, oldest_tli, xlog_seg_size);
 	}
 
 	/* cleanup */
@@ -225,7 +228,7 @@ do_retention_purge(void)
 	/* Purge WAL files */
 	if (delete_wal)
 	{
-		delete_walfiles(oldest_lsn, oldest_tli);
+		delete_walfiles(oldest_lsn, oldest_tli, xlog_seg_size);
 	}
 
 	/* Cleanup */
@@ -313,7 +316,8 @@ pgBackupDeleteFiles(pgBackup *backup)
  *    oldest_lsn.
  */
 static void
-delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli)
+delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli,
+				uint32 xlog_seg_size)
 {
 	XLogSegNo   targetSegNo;
 	char		oldestSegmentNeeded[MAXFNAMELEN];
@@ -329,8 +333,9 @@ delete_walfiles(XLogRecPtr oldest_lsn, TimeLineID oldest_tli)
 
 	if (!XLogRecPtrIsInvalid(oldest_lsn))
 	{
-		XLByteToSeg(oldest_lsn, targetSegNo);
-		XLogFileName(oldestSegmentNeeded, oldest_tli, targetSegNo);
+		GetXLogSegNo(oldest_lsn, targetSegNo, xlog_seg_size);
+		GetXLogFileName(oldestSegmentNeeded, oldest_tli, targetSegNo,
+						xlog_seg_size);
 
 		elog(LOG, "removing WAL segments older than %s", oldestSegmentNeeded);
 	}
@@ -436,7 +441,7 @@ do_delete_instance(void)
 	parray_free(backup_list);
 
 	/* Delete all wal files. */
-	delete_walfiles(InvalidXLogRecPtr, 0);
+	delete_walfiles(InvalidXLogRecPtr, 0, xlog_seg_size);
 
 	/* Delete backup instance config file */
 	join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE);

src/init.c

@@ -54,7 +54,7 @@ do_add_instance(void)
 {
 	char		path[MAXPGPATH];
 	char		arclog_path_dir[MAXPGPATH];
-    struct stat st;
+	struct stat st;
 	pgBackupConfig *config = pgut_new(pgBackupConfig);
 
 	/* PGDATA is always required */
@@ -64,6 +64,8 @@ do_add_instance(void)
 
 	/* Read system_identifier from PGDATA */
 	system_identifier = get_system_identifier(pgdata);
+	/* Starting from PostgreSQL 11 read WAL segment size from PGDATA */
+	xlog_seg_size = get_xlog_seg_size(pgdata);
 
 	/* Ensure that all root directories already exist */
 	if (access(backup_path, F_OK) != 0)
@@ -97,6 +99,7 @@ do_add_instance(void)
 	 */
 	pgBackupConfigInit(config);
 	config->system_identifier = system_identifier;
+	config->xlog_seg_size = xlog_seg_size;
 	config->pgdata = pgdata;
 	writeBackupCatalogConfigFile(config);
 

src/merge.c

@@ -320,9 +320,10 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup)
 			to_backup->data_bytes += file->write_size;
 	}
 	/* compute size of wal files of this backup stored in the archive */
-	if (!current.stream)
-		to_backup->wal_bytes = XLOG_SEG_SIZE *
-			(to_backup->stop_lsn / XLogSegSize - to_backup->start_lsn / XLogSegSize + 1);
+	if (!to_backup->stream)
+		to_backup->wal_bytes = xlog_seg_size *
+			(to_backup->stop_lsn / xlog_seg_size -
+			 to_backup->start_lsn / xlog_seg_size + 1);
 	else
 		to_backup->wal_bytes = BYTES_INVALID;