Merge branch 'master' into pgpro-1918

Author: gsmolk

README.md

@@ -39,7 +39,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp
 `pg_probackup` currently has the following limitations:
 * Creating backups from a remote server is currently not supported.
 * The server from which the backup was taken and the restored server must be compatible by the [block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-BLOCK-SIZE) and [wal_block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number.
-* Microsoft Windows operating system is not supported.
+* Microsoft Windows operating system support is in beta stage.
 * Configuration files outside of PostgreSQL data directory are not included into the backup and should be backed up separately.
 
 ## Installation and Setup
@@ -71,13 +71,28 @@ yum install pg_probackup-{11,10,9.6,9.5}
 yumdownloader --source pg_probackup-{11,10,9.6,9.5}
 ```
 
+Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup.html#pg-probackup-install-and-setup).
+
+## Building from source
+### Linux
+
 To compile `pg_probackup`, you must have a PostgreSQL installation and raw source tree. To install `pg_probackup`, execute this in the module's directory:
 
 ```shell
 make USE_PGXS=1 PG_CONFIG=<path_to_pg_config> top_srcdir=<path_to_PostgreSQL_source_tree>
 ```
+### Windows
 
-Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup.html#pg-probackup-install-and-setup).
+Currently pg_probackup can be build using only MSVC 2013.
+Build PostgreSQL using [pgwininstall](https://github.com/postgrespro/pgwininstall) or [PostgreSQL instruction](https://www.postgresql.org/docs/10/install-windows-full.html) with MSVC 2013.
+If zlib support is needed, src/tools/msvc/config.pl must contain path to directory with compiled zlib. [Example](https://gist.githubusercontent.com/gsmol/80989f976ce9584824ae3b1bfb00bd87/raw/240032950d4ac4801a79625dd00c8f5d4ed1180c/gistfile1.txt)
+
+```shell
+CALL "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall" amd64
+SET PATH=%PATH%;C:\Perl64\bin
+SET PATH=%PATH%;C:\msys64\usr\bin
+gen_probackup_project.pl C:\path_to_postgresql_source_tree
+```
 
 ## Documentation
 

src/backup.c

@@ -2213,9 +2213,9 @@ backup_files(void *arg)
 		struct stat	buf;
 		pgFile	   *file = (pgFile *) parray_get(arguments->files_list, i);
 
-		elog(VERBOSE, "Copying file:  \"%s\" ", file->path);
 		if (!pg_atomic_test_set_flag(&file->lock))
 			continue;
+		elog(VERBOSE, "Copying file:  \"%s\" ", file->path);
 
 		/* check for interrupt */
 		if (interrupted)

src/delete.c

@@ -294,15 +294,7 @@ delete_backup_files(pgBackup *backup)
 			elog(INFO, "Progress: (%zd/%zd). Process file \"%s\"",
 				 i + 1, num_files, file->path);
 
-		if (remove(file->path))
-		{
-			if (errno == ENOENT)
-				elog(VERBOSE, "File \"%s\" is absent", file->path);
-			else
-				elog(ERROR, "Cannot remove \"%s\": %s", file->path,
-					 strerror(errno));
-			return;
-		}
+		pgFileDelete(file);
 	}
 
 	parray_walk(files, pgFileFree);

src/dir.c

@@ -118,7 +118,7 @@ typedef struct TablespaceCreatedList
 
 static int BlackListCompare(const void *str1, const void *str2);
 
-static bool dir_check_file(const char *root, pgFile *file);
+static char dir_check_file(const char *root, pgFile *file);
 static void dir_list_file_internal(parray *files, const char *root,
 								   pgFile *parent, bool exclude,
 								   bool omit_symlink, parray *black_list);
@@ -415,6 +415,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
 
 		while (fgets(buf, lengthof(buf), black_list_file) != NULL)
 		{
+			black_item[0] = '\0';
 			join_path_components(black_item, instance_config.pgdata, buf);
 
 			if (black_item[strlen(black_item) - 1] == '\n')
@@ -423,7 +424,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
 			if (black_item[0] == '#' || black_item[0] == '\0')
 				continue;
 
-			parray_append(black_list, black_item);
+			parray_append(black_list, pgut_strdup(black_item));
 		}
 
 		fclose(black_list_file);
@@ -446,8 +447,18 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
 
 	if (!add_root)
 		pgFileFree(file);
+
+	if (black_list)
+	{
+		parray_walk(black_list, pfree);
+		parray_free(black_list);
+	}
 }
 
+#define CHECK_FALSE				0
+#define CHECK_TRUE				1
+#define CHECK_EXCLUDE_FALSE		2
+
 /*
  * Check file or directory.
  *
@@ -456,16 +467,21 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
  * Skip files:
  * - skip temp tables files
  * - skip unlogged tables files
+ * Skip recursive tablespace content
  * Set flags for:
  * - database directories
  * - datafiles
  */
-static bool
+static char
 dir_check_file(const char *root, pgFile *file)
 {
 	const char *rel_path;
 	int			i;
 	int			sscanf_res;
+	bool		in_tablespace = false;
+
+	rel_path = GetRelativePath(file->path, root);
+	in_tablespace = path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path);
 
 	/* Check if we need to exclude file by name */
 	if (S_ISREG(file->mode))
@@ -478,7 +494,7 @@ dir_check_file(const char *root, pgFile *file)
 				{
 					/* Skip */
 					elog(VERBOSE, "Excluding file: %s", file->name);
-					return false;
+					return CHECK_FALSE;
 				}
 		}
 
@@ -487,14 +503,14 @@ dir_check_file(const char *root, pgFile *file)
 			{
 				/* Skip */
 				elog(VERBOSE, "Excluding file: %s", file->name);
-				return false;
+				return CHECK_FALSE;
 			}
 	}
 	/*
 	 * If the directory name is in the exclude list, do not list the
 	 * contents.
 	 */
-	else if (S_ISDIR(file->mode))
+	else if (S_ISDIR(file->mode) && !in_tablespace)
 	{
 		/*
 		 * If the item in the exclude list starts with '/', compare to
@@ -510,20 +526,18 @@ dir_check_file(const char *root, pgFile *file)
 				{
 					elog(VERBOSE, "Excluding directory content: %s",
 						 file->name);
-					return false;
+					return CHECK_EXCLUDE_FALSE;
 				}
 			}
 			else if (strcmp(file->name, pgdata_exclude_dir[i]) == 0)
 			{
 				elog(VERBOSE, "Excluding directory content: %s",
 					 file->name);
-				return false;
+				return CHECK_EXCLUDE_FALSE;
 			}
 		}
 	}
 
-	rel_path = GetRelativePath(file->path, root);
-
 	/*
 	 * Do not copy tablespaces twice. It may happen if the tablespace is located
 	 * inside the PGDATA.
@@ -539,14 +553,33 @@ dir_check_file(const char *root, pgFile *file)
 		 * pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY
 		 */
 		if (!path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path))
-			return false;
+			return CHECK_FALSE;
 		sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%s",
 							&tblspcOid, tmp_rel_path);
 		if (sscanf_res == 0)
-			return false;
+			return CHECK_FALSE;
 	}
 
-	if (path_is_prefix_of_path("global", rel_path))
+	if (in_tablespace)
+	{
+		char		tmp_rel_path[MAXPGPATH];
+
+		sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%[^/]/%u/",
+							&(file->tblspcOid), tmp_rel_path,
+							&(file->dbOid));
+
+		/*
+		 * We should skip other files and directories rather than
+		 * TABLESPACE_VERSION_DIRECTORY, if this is recursive tablespace.
+		 */
+		if (sscanf_res == 2 && strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) != 0)
+			return CHECK_FALSE;
+
+		if (sscanf_res == 3 && S_ISDIR(file->mode) &&
+			strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) == 0)
+			file->is_database = true;
+	}
+	else if (path_is_prefix_of_path("global", rel_path))
 	{
 		file->tblspcOid = GLOBALTABLESPACE_OID;
 
@@ -562,22 +595,10 @@ dir_check_file(const char *root, pgFile *file)
 		if (S_ISDIR(file->mode) && strcmp(file->name, "base") != 0)
 			file->is_database = true;
 	}
-	else if (path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path))
-	{
-		char		tmp_rel_path[MAXPGPATH];
-
-		sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%[^/]/%u/",
-							&(file->tblspcOid), tmp_rel_path,
-							&(file->dbOid));
-
-		if (sscanf_res == 3 && S_ISDIR(file->mode) &&
-			strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) == 0)
-			file->is_database = true;
-	}
 
 	/* Do not backup ptrack_init files */
 	if (S_ISREG(file->mode) && strcmp(file->name, "ptrack_init") == 0)
-		return false;
+		return CHECK_FALSE;
 
 	/*
 	 * Check files located inside database directories including directory
@@ -587,10 +608,10 @@ dir_check_file(const char *root, pgFile *file)
 		file->name && file->name[0])
 	{
 		if (strcmp(file->name, "pg_internal.init") == 0)
-			return false;
+			return CHECK_FALSE;
 		/* Do not backup temp files */
 		else if (file->name[0] == 't' && isdigit(file->name[1]))
-			return false;
+			return CHECK_FALSE;
 		else if (isdigit(file->name[0]))
 		{
 			char	   *fork_name;
@@ -605,14 +626,14 @@ dir_check_file(const char *root, pgFile *file)
 
 				/* Do not backup ptrack files */
 				if (strcmp(file->forkName, "ptrack") == 0)
-					return false;
+					return CHECK_FALSE;
 			}
 			else
 			{
 				len = strlen(file->name);
 				/* reloid.cfm */
 				if (len > 3 && strcmp(file->name + len - 3, "cfm") == 0)
-					return true;
+					return CHECK_TRUE;
 
 				sscanf_res = sscanf(file->name, "%u.%d.%s", &(file->relOid),
 									&(file->segno), suffix);
@@ -624,7 +645,7 @@ dir_check_file(const char *root, pgFile *file)
 		}
 	}
 
-	return true;
+	return CHECK_TRUE;
 }
 
 /*
@@ -659,6 +680,7 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent,
 	{
 		pgFile	   *file;
 		char		child[MAXPGPATH];
+		char		check_res;
 
 		join_path_components(child, parent->path, dent->d_name);
 
@@ -694,21 +716,24 @@ dir_list_file_internal(parray *files, const char *root, pgFile *parent,
 			continue;
 		}
 
-		/* We add the directory anyway */
-		if (S_ISDIR(file->mode))
-			parray_append(files, file);
-
-		if (exclude && !dir_check_file(root, file))
+		if (exclude)
 		{
-			if (S_ISREG(file->mode))
+			check_res = dir_check_file(root, file);
+			if (check_res == CHECK_FALSE)
+			{
+				/* Skip */
 				pgFileFree(file);
-			/* Skip */
-			continue;
+				continue;
+			}
+			else if (check_res == CHECK_EXCLUDE_FALSE)
+			{
+				/* We add the directory itself which content was excluded */
+				parray_append(files, file);
+				continue;
+			}
 		}
 
-		/* At least add the file */
-		if (S_ISREG(file->mode))
-			parray_append(files, file);
+		parray_append(files, file);
 
 		/*
 		 * If the entry is a directory call dir_list_file_internal()
@@ -876,6 +901,7 @@ opt_tablespace_map(ConfigOption *opt, const char *arg)
 	char	   *dst_ptr;
 	const char *arg_ptr;
 
+	memset(cell, 0, sizeof(TablespaceListCell));
 	dst_ptr = dst = cell->old_dir;
 	for (arg_ptr = arg; *arg_ptr; arg_ptr++)
 	{
@@ -1219,11 +1245,7 @@ print_file_list(FILE *out, const parray *files, const char *root)
 		if (file->is_datafile)
 			fprintf(out, ",\"segno\":\"%d\"", file->segno);
 
-#ifndef WIN32
-		if (S_ISLNK(file->mode))
-#else
-		if (pgwin32_is_junction(file->path))
-#endif
+		if (file->linked)
 			fprintf(out, ",\"linked\":\"%s\"", file->linked);
 
 		if (file->n_blocks != BLOCKNUM_INVALID)