Merge branch 'REL_2_6_rework_stat' into REL_2_6

Author: funny-falcon

src/archive.c

@@ -388,9 +388,9 @@ push_file_internal(const char *wal_file_name, const char *pg_xlog_dir,
     char to_fullpath[MAXPGPATH];
     char to_fullpath_part[MAXPGPATH];
 /* partial handling */
-    struct stat st;
+    pio_stat_t st;
     int partial_try_count = 0;
-    ssize_t partial_file_size = 0;
+    int64_t partial_file_size = 0;
     bool partial_is_stale = true;
     size_t len;
     err_i err = $noerr();
@@ -466,11 +466,11 @@ push_file_internal(const char *wal_file_name, const char *pg_xlog_dir,
             elog(LOG,
                  "Temp WAL file already exists, waiting on it %u seconds: \"%s\"",
                  archive_timeout, to_fullpath_part);
-            partial_file_size = st.st_size;
+            partial_file_size = st.pst_size;
         }
 
         /* file size is changing */
-        if (st.st_size != partial_file_size)
+        if (st.pst_size != partial_file_size)
             partial_is_stale = false;
 
         sleep(1);

src/backup.c

@@ -391,7 +391,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
 		pgFile	   *file = (pgFile *) parray_get(backup_files_list, i);
 
 		/* if the entry was a directory, create it in the backup */
-		if (S_ISDIR(file->mode))
+		if (file->kind == PIO_KIND_DIRECTORY)
 		{
 			char		dirpath[MAXPGPATH];
 
@@ -568,7 +568,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
 			pgFile *file = (pgFile *) parray_get(backup_files_list, i);
 
 			/* TODO: sync directory ? */
-			if (S_ISDIR(file->mode))
+			if (file->kind == PIO_KIND_DIRECTORY)
 				continue;
 
 			if (file->write_size <= 0)
@@ -1836,7 +1836,7 @@ pg_stop_backup_write_file_helper(const char *path, const char *filename, const c
 		file = pgFileNew(full_filename, filename, true, 0,
 						 FIO_BACKUP_HOST);
 
-		if (S_ISREG(file->mode))
+		if (file->kind == PIO_KIND_REGULAR)
 		{
 			file->crc = pgFileGetCRC32C(full_filename, false);
 
@@ -1990,7 +1990,7 @@ backup_files(void *arg)
 		pgFile	*prev_file = NULL;
 
 		/* We have already copied all directories */
-		if (S_ISDIR(file->mode))
+		if (file->kind == PIO_KIND_DIRECTORY)
 			continue;
 
 		if (arguments->thread_num == 1)
@@ -2045,9 +2045,9 @@ backup_files(void *arg)
 		}
 
 		/* Encountered some strange beast */
-		if (!S_ISREG(file->mode))
-			elog(WARNING, "Unexpected type %d of file \"%s\", skipping",
-							file->mode, from_fullpath);
+		if (file->kind != PIO_KIND_REGULAR)
+			elog(WARNING, "Unexpected type %s of file \"%s\", skipping",
+				 pio_file_kind2str(file->kind, from_fullpath), from_fullpath);
 
 		/* Check that file exist in previous backup */
 		if (current.backup_mode != BACKUP_MODE_FULL)
@@ -2120,7 +2120,7 @@ parse_filelist_filenames(parray *files, const char *root)
 		pgFile	   *file = (pgFile *) parray_get(files, i);
 		int 		sscanf_result;
 
-		if (S_ISREG(file->mode) &&
+		if (file->kind == PIO_KIND_REGULAR &&
 			path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path))
 		{
 			/*
@@ -2147,7 +2147,7 @@ parse_filelist_filenames(parray *files, const char *root)
 			}
 		}
 
-		if (S_ISREG(file->mode) && file->tblspcOid != 0 &&
+		if (file->kind == PIO_KIND_REGULAR && file->tblspcOid != 0 &&
 			file->name && file->name[0])
 		{
 			if (file->forkName == init)
@@ -2217,7 +2217,7 @@ set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i)
 
 		if (strstr(prev_file->rel_path, cfs_tblspc_path) != NULL)
 		{
-			if (S_ISREG(prev_file->mode) && prev_file->is_datafile)
+			if (prev_file->kind == PIO_KIND_REGULAR && prev_file->is_datafile)
 			{
 				elog(LOG, "Setting 'is_cfs' on file %s, name %s",
 					prev_file->rel_path, prev_file->name);
@@ -2374,7 +2374,7 @@ calculate_datasize_of_filelist(parray *filelist)
 		if (file->external_dir_num != 0 || file->excluded)
 			continue;
 
-		if (S_ISDIR(file->mode))
+		if (file->kind == PIO_KIND_DIRECTORY)
 		{
 			// TODO is a dir always 4K?
 			bytes += 4096;

src/catalog.c

@@ -847,12 +847,22 @@ pgBackupGetBackupMode(pgBackup *backup, bool show_color)
 static bool
 IsDir(const char *dirpath, const char *entry, fio_location location)
 {
+	FOBJ_FUNC_ARP();
 	char		path[MAXPGPATH];
-	struct stat	st;
+	pio_stat_t	st;
+	err_i 		err;
 
 	join_path_components(path, dirpath, entry);
 
-	return fio_stat(location, path, &st, false) == 0 && S_ISDIR(st.st_mode);
+	st = $i(pioStat, pioDriveForLocation(location),
+			.path = path, .follow_symlink = false, .err = &err);
+	if ($haserr(err))
+	{
+		ft_logerr(FT_WARNING, $errmsg(err), "IsDir");
+		return false;
+	}
+
+	return st.pst_kind == PIO_KIND_DIRECTORY;
 }
 
 /*
@@ -1074,6 +1084,7 @@ get_backup_filelist(pgBackup *backup, bool strict)
 		char		path[MAXPGPATH];
 		char		linked[MAXPGPATH];
 		char		compress_alg_string[MAXPGPATH];
+		char 		kind[16];
 		int64		write_size,
 					uncompressed_size,
 					mode,		/* bit length of mode_t depends on platforms */
@@ -1115,6 +1126,11 @@ get_backup_filelist(pgBackup *backup, bool strict)
 		/*
 		 * Optional fields
 		 */
+		if (get_control_value_str(buf, "kind", kind, sizeof(kind), false))
+			file->kind = pio_str2file_kind(kind, path);
+		else /* fallback to mode for old backups */
+			file->kind = pio_statmode2file_kind(file->mode, path);
+
 		if (get_control_value_str(buf, "linked", linked, sizeof(linked), false) && linked[0])
 		{
 			file->linked = pgut_strdup(linked);
@@ -1146,7 +1162,7 @@ get_backup_filelist(pgBackup *backup, bool strict)
 		if (!file->is_datafile || file->is_cfs)
 			file->size = file->uncompressed_size;
 
-		if (file->external_dir_num == 0 && S_ISREG(file->mode))
+		if (file->external_dir_num == 0 && file->kind == PIO_KIND_REGULAR)
 		{
 			bool is_datafile = file->is_datafile;
 			set_forkname(file);
@@ -2564,14 +2580,14 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
 		if (file->write_size == FILE_NOT_FOUND)
 			continue;
 
-		if (S_ISDIR(file->mode))
+		if (file->kind == PIO_KIND_DIRECTORY)
 		{
 			backup_size_on_disk += 4096;
 			uncompressed_size_on_disk += 4096;
 		}
 
 		/* Count the amount of the data actually copied */
-		if (S_ISREG(file->mode) && file->write_size > 0)
+		if (file->kind == PIO_KIND_REGULAR && file->write_size > 0)
 		{
 			/*
 			 * Size of WAL files in 'pg_wal' is counted separately
@@ -2587,11 +2603,13 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
 		}
 
 		len = sprintf(line, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", "
-					 "\"mode\":\"%u\", \"is_datafile\":\"%u\", "
+					 "\"kind\":\"%s\", \"mode\":\"%u\", \"is_datafile\":\"%u\", "
 					 "\"is_cfs\":\"%u\", \"crc\":\"%u\", "
 					 "\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\", "
 					 "\"dbOid\":\"%u\"",
-					file->rel_path, file->write_size, file->mode,
+					file->rel_path, file->write_size,
+					pio_file_kind2str(file->kind, file->rel_path),
+					file->mode,
 					file->is_datafile ? 1 : 0,
 					file->is_cfs ? 1 : 0,
 					file->crc,

src/catchup.c

@@ -378,7 +378,7 @@ catchup_thread_runner(void *arg)
 		pgFile	*dest_file = NULL;
 
 		/* We have already copied all directories */
-		if (S_ISDIR(file->mode))
+		if (file->kind == PIO_KIND_DIRECTORY)
 			continue;
 
 		if (file->excluded)
@@ -400,9 +400,9 @@ catchup_thread_runner(void *arg)
 		join_path_components(to_fullpath, arguments->to_root, file->rel_path);
 
 		/* Encountered some strange beast */
-		if (!S_ISREG(file->mode))
-			elog(WARNING, "Unexpected type %d of file \"%s\", skipping",
-							file->mode, from_fullpath);
+		if (file->kind != PIO_KIND_REGULAR)
+			elog(WARNING, "Unexpected kind %s of file \"%s\", skipping",
+				 pio_file_kind2str(file->kind, from_fullpath), from_fullpath);
 
 		/* Check that file exist in dest pgdata */
 		if (arguments->backup_mode != BACKUP_MODE_FULL)
@@ -439,7 +439,7 @@ catchup_thread_runner(void *arg)
 
 		if (file->write_size == BYTES_INVALID)
 		{
-			elog(LOG, "Skipping the unchanged file: \"%s\", read %zu bytes", from_fullpath, file->read_size);
+			elog(LOG, "Skipping the unchanged file: \"%s\", read %lld bytes", from_fullpath, (long long)file->read_size);
 			continue;
 		}
 
@@ -546,7 +546,7 @@ catchup_sync_destination_files(const char* pgdata_path, fio_location location, p
 		 * - but PG itself is not relying on fs, its durable_sync
 		 *   includes directory sync
 		 */
-		if (S_ISDIR(file->mode) || file->excluded)
+		if (file->kind == PIO_KIND_DIRECTORY || file->excluded)
 			continue;
 
 		Assert(file->external_dir_num == 0);
@@ -630,8 +630,8 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
 
 	/* for fancy reporting */
 	time_t		start_time, end_time;
-	ssize_t		transfered_datafiles_bytes = 0;
-	ssize_t		transfered_walfiles_bytes = 0;
+	int64_t		transfered_datafiles_bytes = 0;
+	int64_t		transfered_walfiles_bytes = 0;
 	char		pretty_source_bytes[20];
 	err_i		err = $noerr();
 
@@ -807,7 +807,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
 		pgFile	   *file = (pgFile *) parray_get(source_filelist, i);
 		char parent_dir[MAXPGPATH];
 
-		if (!S_ISDIR(file->mode) || file->excluded)
+		if (file->kind != PIO_KIND_DIRECTORY || file->excluded)
 			continue;
 
 		/*

src/checkdb.c

@@ -147,7 +147,7 @@ check_files(void *arg)
 			elog(ERROR, "interrupted during checkdb");
 
 		/* No need to check directories */
-		if (S_ISDIR(file->mode))
+		if (file->kind == PIO_KIND_DIRECTORY)
 			continue;
 
 		if (!pg_atomic_test_set_flag(&file->lock))
@@ -161,7 +161,7 @@ check_files(void *arg)
 			elog(INFO, "Progress: (%d/%d). Process file \"%s\"",
 				 i + 1, n_files_list, from_fullpath);
 
-		if (S_ISREG(file->mode))
+		if (file->kind == PIO_KIND_REGULAR)
 		{
 			/* check only uncompressed by cfs datafiles */
 			if (file->is_datafile && !file->is_cfs)

src/data.c

@@ -507,7 +507,8 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat
 	 * NOTE This is a normal situation, if the file size has changed
 	 * since the moment we computed it.
 	 */
-	file->n_blocks = file->size/BLCKSZ;
+	file->n_blocks = (typeof(file->n_blocks))(file->size/BLCKSZ);
+	Assert((int64_t)file->n_blocks * BLCKSZ == file->size);
 
 	/*
 	 * Skip unchanged file only if it exists in previous backup.
@@ -611,12 +612,15 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat
 			elog(ERROR, "Cannot read file \"%s\"", from_fullpath);
 	}
 
-	file->read_size = rc * BLCKSZ;
+	file->read_size = (int64_t)rc * BLCKSZ;
 
 	/* refresh n_blocks for FULL and DELTA */
 	if (backup_mode == BACKUP_MODE_FULL ||
 	    backup_mode == BACKUP_MODE_DIFF_DELTA)
-		file->n_blocks = file->read_size / BLCKSZ;
+	{
+		file->n_blocks = (typeof(file->n_blocks))(file->read_size / BLCKSZ);
+		Assert((int64_t)file->n_blocks * BLCKSZ == file->read_size);
+	}
 
 	/* Determine that file didn`t changed in case of incremental backup */
 	if (backup_mode != BACKUP_MODE_FULL &&
@@ -650,7 +654,7 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat
 void
 catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath,
 				  XLogRecPtr sync_lsn, BackupMode backup_mode,
-				  uint32 checksum_version, size_t prev_size)
+				  uint32 checksum_version, int64_t prev_size)
 {
 	int         rc;
 	bool        use_pagemap;
@@ -760,7 +764,7 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa
 			elog(ERROR, "Cannot read file \"%s\"", from_fullpath);
 	}
 
-	file->read_size = rc * BLCKSZ;
+	file->read_size = (int64_t)rc * BLCKSZ;
 
 	/* Determine that file didn`t changed in case of incremental catchup */
 	if (backup_mode != BACKUP_MODE_FULL &&
@@ -1595,7 +1599,8 @@ check_data_file(ConnectionArgs *arguments, pgFile *file,
 	 * NOTE This is a normal situation, if the file size has changed
 	 * since the moment we computed it.
 	 */
-	nblocks = file->size/BLCKSZ;
+	nblocks = (typeof(nblocks))(file->size/BLCKSZ);
+	Assert((int64_t)nblocks * BLCKSZ == file->size);
 
 	for (blknum = 0; blknum < nblocks; blknum++)
 	{
@@ -2275,7 +2280,7 @@ copy_pages(const char *to_fullpath, const char *from_fullpath,
 		elog(ERROR, "Cannot seek to end of file position in destination file \"%s\": %s",
 			 to_fullpath, strerror(errno));
 	{
-		long pos = ftell(out);
+		int64_t pos = ftell(out);
 
 		if (pos < 0)
 			elog(ERROR, "Cannot get position in destination file \"%s\": %s",

src/delete.c

@@ -794,8 +794,8 @@ delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timeli
 	char 		first_to_del_str[MAXFNAMELEN];
 	char 		oldest_to_keep_str[MAXFNAMELEN];
 	int			i;
-	size_t		wal_size_logical = 0;
-	size_t		wal_size_actual = 0;
+	int64_t		wal_size_logical = 0;
+	int64_t		wal_size_actual = 0;
 	char		wal_pretty_size[20];
 	bool		purge_all = false;
 
@@ -837,7 +837,7 @@ delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timeli
 	/* sanity */
 	if (OldestToKeepSegNo > FirstToDeleteSegNo)
 	{
-		wal_size_logical = (OldestToKeepSegNo - FirstToDeleteSegNo) * xlog_seg_size;
+		wal_size_logical = (int64_t)(OldestToKeepSegNo - FirstToDeleteSegNo) * xlog_seg_size;
 
 		/* In case of 'purge all' scenario OldestToKeepSegNo will be deleted too */
 		if (purge_all)