Merge branch 'master' into try-merge-2.6

Author: funny-falcon

src/catalog.c

@@ -1075,6 +1075,7 @@ get_backup_filelist(pgBackup *backup, bool strict)
 		char		linked[MAXPGPATH];
 		char		compress_alg_string[MAXPGPATH];
 		int64		write_size,
+					uncompressed_size,
 					mode,		/* bit length of mode_t depends on platforms */
 					is_datafile,
 					is_cfs,
@@ -1138,8 +1139,30 @@ get_backup_filelist(pgBackup *backup, bool strict)
 		if (get_control_value_int64(buf, "hdr_size", &hdr_size, false))
 			file->hdr_size = (int) hdr_size;
 
-		if (file->external_dir_num == 0)
+		if (get_control_value_int64(buf, "full_size", &uncompressed_size, false))
+			file->uncompressed_size = uncompressed_size;
+		else
+			file->uncompressed_size = write_size;
+		if (!file->is_datafile || file->is_cfs)
+			file->size = file->uncompressed_size;
+
+		if (file->external_dir_num == 0 && S_ISREG(file->mode))
+		{
+			bool is_datafile = file->is_datafile;
 			set_forkname(file);
+			if (is_datafile != file->is_datafile)
+			{
+				if (is_datafile)
+					elog(WARNING, "File '%s' was stored as datafile, but looks like it is not",
+						 file->rel_path);
+				else
+					elog(WARNING, "File '%s' was stored as non-datafile, but looks like it is",
+						 file->rel_path);
+				/* Lets fail in tests */
+				Assert(file->is_datafile == file->is_datafile);
+				file->is_datafile = is_datafile;
+			}
+		}
 
 		parray_append(files, file);
 	}
@@ -2567,6 +2590,11 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
 					file->external_dir_num,
 					file->dbOid);
 
+		if (file->uncompressed_size != 0 &&
+				file->uncompressed_size != file->write_size)
+			len += sprintf(line+len, ",\"full_size\":\"" INT64_FORMAT "\"",
+						   file->uncompressed_size);
+
 		if (file->is_datafile)
 			len += sprintf(line+len, ",\"segno\":\"%d\"", file->segno);
 

src/data.c

@@ -801,13 +801,17 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
 	 * and its mtime is less than parent backup start time ... */
 	if ((pg_strcasecmp(file->name, RELMAPPER_FILENAME) != 0) &&
 		(prev_file && file->exists_in_prev &&
+		 file->size == prev_file->size &&
 		 file->mtime <= parent_backup_time))
 	{
 		/*
 		 * file could be deleted under our feets.
 		 * But then backup_non_data_file_internal will handle it safely
 		 */
-		file->crc = fio_get_crc32(FIO_DB_HOST, from_fullpath, false, true);
+		if (file->forkName != cfm)
+			file->crc = fio_get_crc32(FIO_DB_HOST, from_fullpath, false, true);
+		else
+			file->crc = fio_get_crc32_truncated(FIO_DB_HOST, from_fullpath, true);
 
 		/* ...and checksum is the same... */
 		if (EQ_CRC32C(file->crc, prev_file->crc))
@@ -1332,8 +1336,12 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
 	if (already_exists)
 	{
 		/* compare checksums of already existing file and backup file */
-		pg_crc32 file_crc = fio_get_crc32(FIO_DB_HOST, to_fullpath, false,
-										  false);
+		pg_crc32 file_crc;
+		if (tmp_file->forkName == cfm &&
+			    tmp_file->uncompressed_size > tmp_file->write_size)
+			file_crc = fio_get_crc32_truncated(FIO_DB_HOST, to_fullpath, false);
+		else
+			file_crc = fio_get_crc32(FIO_DB_HOST, to_fullpath, false, false);
 
 		if (file_crc == tmp_file->crc)
 		{
@@ -1390,10 +1398,12 @@ backup_non_data_file_internal(const char *from_fullpath,
 							const char *to_fullpath, pgFile *file,
 							bool missing_ok)
 {
-	FILE	*in = NULL;
 	FILE	*out = NULL;
-	ssize_t  read_len = 0;
-	char	*buf = NULL;
+	char	*errmsg = NULL;
+	int 	rc;
+	bool	cut_zero_tail;
+
+	cut_zero_tail = file->forkName == cfm;
 
 	INIT_CRC32C(file->crc);
 
@@ -1415,107 +1425,44 @@ backup_non_data_file_internal(const char *from_fullpath,
 
 	/* backup remote file  */
 	if (fio_is_remote(FIO_DB_HOST))
-	{
-		char *errmsg = NULL;
-		int rc = fio_send_file(from_fullpath, to_fullpath, out, file, &errmsg);
+		rc = fio_send_file(from_fullpath, out, cut_zero_tail, file, &errmsg);
+	else
+		rc = fio_send_file_local(from_fullpath, out, cut_zero_tail, file, &errmsg);
 
-		/* handle errors */
-		if (rc == FILE_MISSING)
-		{
-			/* maybe deleted, it's not error in case of backup */
-			if (missing_ok)
-			{
-				elog(LOG, "File \"%s\" is not found", from_fullpath);
-				file->write_size = FILE_NOT_FOUND;
-				goto cleanup;
-			}
-			else
-				elog(ERROR, "File \"%s\" is not found", from_fullpath);
-		}
-		else if (rc == WRITE_FAILED)
-			elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno));
-		else if (rc != SEND_OK)
+	/* handle errors */
+	if (rc == FILE_MISSING)
+	{
+		/* maybe deleted, it's not error in case of backup */
+		if (missing_ok)
 		{
-			if (errmsg)
-				elog(ERROR, "%s", errmsg);
-			else
-				elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath);
+			elog(LOG, "File \"%s\" is not found", from_fullpath);
+			file->write_size = FILE_NOT_FOUND;
+			goto cleanup;
 		}
-
-		pg_free(errmsg);
+		else
+			elog(ERROR, "File \"%s\" is not found", from_fullpath);
 	}
-	/* backup local file */
-	else
+	else if (rc == WRITE_FAILED)
+		elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno));
+	else if (rc != SEND_OK)
 	{
-		/* open source file for read */
-		in = fopen(from_fullpath, PG_BINARY_R);
-		if (in == NULL)
-		{
-			/* maybe deleted, it's not error in case of backup */
-			if (errno == ENOENT)
-			{
-				if (missing_ok)
-				{
-					elog(LOG, "File \"%s\" is not found", from_fullpath);
-					file->write_size = FILE_NOT_FOUND;
-					goto cleanup;
-				}
-				else
-					elog(ERROR, "File \"%s\" is not found", from_fullpath);
-			}
-
-			elog(ERROR, "Cannot open file \"%s\": %s", from_fullpath,
-				 strerror(errno));
-		}
-
-		/* disable stdio buffering for local input/output files to avoid triple buffering */
-		setvbuf(in, NULL, _IONBF, BUFSIZ);
-		setvbuf(out, NULL, _IONBF, BUFSIZ);
-
-		/* allocate 64kB buffer */
-		buf = pgut_malloc(CHUNK_SIZE);
-
-		/* copy content and calc CRC */
-		for (;;)
-		{
-			read_len = fread(buf, 1, CHUNK_SIZE, in);
-
-			if (ferror(in))
-				elog(ERROR, "Cannot read from file \"%s\": %s",
-					 from_fullpath, strerror(errno));
-
-			if (read_len > 0)
-			{
-				if (fwrite(buf, 1, read_len, out) != read_len)
-					elog(ERROR, "Cannot write to file \"%s\": %s", to_fullpath,
-						 strerror(errno));
-
-				/* update CRC */
-				COMP_CRC32C(file->crc, buf, read_len);
-				file->read_size += read_len;
-			}
-
-			if (feof(in))
-				break;
-		}
+		if (errmsg)
+			elog(ERROR, "%s", errmsg);
+		else
+			elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath);
 	}
 
-	file->write_size = (int64) file->read_size;
-
-	if (file->write_size > 0)
-		file->uncompressed_size = file->write_size;
+	file->uncompressed_size = file->read_size;
 
 cleanup:
+	if (errmsg != NULL)
+		pg_free(errmsg);
+
 	/* finish CRC calculation and store into pgFile */
 	FIN_CRC32C(file->crc);
 
-	if (in && fclose(in))
-		elog(ERROR, "Cannot close the file \"%s\": %s", from_fullpath, strerror(errno));
-
 	if (out && fclose(out))
 		elog(ERROR, "Cannot close the file \"%s\": %s", to_fullpath, strerror(errno));
-
-	pg_free(buf);
 }
 
 /*