[Issue #217] delete of backup with hidden files works correctly now

Author: gsmolk

src/archive.c

@@ -944,7 +944,7 @@ setup_push_filelist(const char *archive_status_dir, const char *first_file,
 
 	/* get list of files from archive_status */
 	status_files = parray_new();
-	dir_list_file(status_files, archive_status_dir, false, false, false, 0, FIO_DB_HOST);
+	dir_list_file(status_files, archive_status_dir, false, false, false, true, 0, FIO_DB_HOST);
 	parray_qsort(status_files, pgFileComparePath);
 
 	for (i = 0; i < parray_num(status_files); i++)

src/backup.c

@@ -333,7 +333,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 
 	/* list files with the logical path. omit $PGDATA */
 	dir_list_file(backup_files_list, instance_config.pgdata,
-				  true, true, false, 0, FIO_DB_HOST);
+				  true, true, false, true, 0, FIO_DB_HOST);
 
 	/*
 	 * Get database_map (name to oid) for use in partial restore feature.
@@ -350,7 +350,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 			/* External dirs numeration starts with 1.
 			 * 0 value is not external dir */
 			dir_list_file(backup_files_list, parray_get(external_dirs, i),
-						  false, true, false, i+1, FIO_DB_HOST);
+						  false, true, false, true, i+1, FIO_DB_HOST);
 
 	/* close ssh session in main thread */
 	fio_disconnect();
@@ -627,7 +627,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 		/* Scan backup PG_XLOG_DIR */
 		xlog_files_list = parray_new();
 		join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR);
-		dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, 0,
+		dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, true, 0,
 					  FIO_BACKUP_HOST);
 
 		/* TODO: Drop streamed WAL segments greater than stop_lsn */

src/catalog.c

@@ -872,7 +872,7 @@ catalog_get_timelines(InstanceConfig *instance)
 
 	/* read all xlog files that belong to this archive */
 	sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name);
-	dir_list_file(xlog_files_list, arclog_path, false, false, false, 0, FIO_BACKUP_HOST);
+	dir_list_file(xlog_files_list, arclog_path, false, false, false, true, 0, FIO_BACKUP_HOST);
 	parray_qsort(xlog_files_list, pgFileComparePath);
 
 	timelineinfos = parray_new();

src/checkdb.c

@@ -208,7 +208,7 @@ do_block_validation(char *pgdata, uint32 checksum_version)
 
 	/* list files with the logical path. omit $PGDATA */
 	dir_list_file(files_list, pgdata,
-				  true, true, false, 0, FIO_DB_HOST);
+				  true, true, false, true, 0, FIO_DB_HOST);
 
 	/*
 	 * Sort pathname ascending.

src/delete.c

@@ -750,7 +750,7 @@ delete_backup_files(pgBackup *backup)
 
 	/* list files to be deleted */
 	files = parray_new();
-	dir_list_file(files, backup->root_dir, false, true, true, 0, FIO_BACKUP_HOST);
+	dir_list_file(files, backup->root_dir, false, true, true, false, 0, FIO_BACKUP_HOST);
 
 	/* delete leaf node first */
 	parray_qsort(files, pgFileComparePathDesc);
@@ -984,7 +984,7 @@ do_delete_instance(void)
 
 	/* Delete all wal files. */
 	xlog_files_list = parray_new();
-	dir_list_file(xlog_files_list, arclog_path, false, false, false, 0, FIO_BACKUP_HOST);
+	dir_list_file(xlog_files_list, arclog_path, false, false, false, false, 0, FIO_BACKUP_HOST);
 
 	for (i = 0; i < parray_num(xlog_files_list); i++)
 	{

src/dir.c

@@ -125,7 +125,7 @@ static int pgCompareString(const void *str1, const void *str2);
 
 static char dir_check_file(pgFile *file);
 static void dir_list_file_internal(parray *files, pgFile *parent, bool exclude,
-								   bool follow_symlink,
+								   bool follow_symlink, bool skip_hidden,
 								   int external_dir_num, fio_location location);
 static void opt_path_map(ConfigOption *opt, const char *arg,
 						 TablespaceList *list, const char *type);
@@ -575,7 +575,7 @@ db_map_entry_free(void *entry)
  */
 void
 dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink,
-			  bool add_root, int external_dir_num, fio_location location)
+			  bool add_root, bool skip_hidden, int external_dir_num, fio_location location)
 {
 	pgFile	   *file;
 
@@ -601,7 +601,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink
 	if (add_root)
 		parray_append(files, file);
 
-	dir_list_file_internal(files, file, exclude, follow_symlink,
+	dir_list_file_internal(files, file, exclude, follow_symlink, skip_hidden,
 						   external_dir_num, location);
 
 	if (!add_root)
@@ -821,7 +821,7 @@ dir_check_file(pgFile *file)
  */
 static void
 dir_list_file_internal(parray *files, pgFile *parent, bool exclude,
-					   bool follow_symlink,
+					   bool follow_symlink, bool skip_hidden,
 					   int external_dir_num, fio_location location)
 {
 	DIR			  *dir;
@@ -868,7 +868,7 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude,
 		}
 
 		/* skip hidden files and directories */
-		if (file->name[0] == '.')
+		if (skip_hidden && file->name[0] == '.')
 		{
 			elog(WARNING, "Skip hidden file: '%s'", file->path);
 			pgFileFree(file);
@@ -911,7 +911,7 @@ dir_list_file_internal(parray *files, pgFile *parent, bool exclude,
 		 */
 		if (S_ISDIR(file->mode))
 			dir_list_file_internal(files, file, exclude, follow_symlink,
-								   external_dir_num, location);
+								   skip_hidden, external_dir_num, location);
 	}
 
 	if (errno && errno != ENOENT)

src/merge.c

@@ -1055,7 +1055,7 @@ remove_dir_with_files(const char *path)
 	int			i;
 	char 		full_path[MAXPGPATH];
 
-	dir_list_file(files, path, true, true, true, 0, FIO_LOCAL_HOST);
+	dir_list_file(files, path, true, true, true, false, 0, FIO_LOCAL_HOST);
 	parray_qsort(files, pgFileComparePathDesc);
 	for (i = 0; i < parray_num(files); i++)
 	{

src/pg_probackup.h

@@ -845,7 +845,7 @@ extern const char* deparse_compress_alg(int alg);
 
 /* in dir.c */
 extern void dir_list_file(parray *files, const char *root, bool exclude,
-						  bool follow_symlink, bool add_root,
+						  bool follow_symlink, bool add_root, bool skip_hidden,
 						  int external_dir_num, fio_location location);
 
 extern void create_data_directories(parray *dest_files,