PGPRO-2096: For pre 2.0.22 validate using CRC-32C

Author: za-arthur

gen_probackup_project.pl

@@ -157,6 +157,7 @@ sub build_pgprobackup
 		'thread.c'
 		);
 	$probackup->AddFile('src/backend/access/transam/xlogreader.c');
+	$probackup->AddFile('src/backend/utils/hash/pg_crc.c');
 	$probackup->AddFiles(
 		'src/bin/pg_basebackup', 
 		'receivelog.c',

msvs/template.pg_probackup.vcxproj

@@ -165,6 +165,7 @@
   </ItemDefinitionGroup>
   <ItemGroup>
     <ClCompile Include="@PGSRC@\backend\access\transam\xlogreader.c" />
+    <ClCompile Include="@PGSRC@\backend\utils\hash\pg_crc.c" />
     <ClCompile Include="@PGSRC@\bin\pg_basebackup\receivelog.c" />
     <ClCompile Include="@PGSRC@\bin\pg_basebackup\streamutil.c" />
     <ClCompile Include="@PGSRC@\bin\pg_basebackup\walmethods.c" />

msvs/template.pg_probackup96.vcxproj

@@ -165,6 +165,7 @@
   </ItemDefinitionGroup>
   <ItemGroup>
     <ClCompile Include="@PGSRC@\backend\access\transam\xlogreader.c" />
+    <ClCompile Include="@PGSRC@\backend\utils\hash\pg_crc.c" />
     <ClCompile Include="@PGSRC@\bin\pg_basebackup\receivelog.c" />
     <ClCompile Include="@PGSRC@\bin\pg_basebackup\streamutil.c" />
     <ClCompile Include="@PGSRC@\bin\pg_rewind\datapagemap.c" />

msvs/template.pg_probackup_2.vcxproj

@@ -160,6 +160,7 @@
 <!--  @PGROOT@\lib;@ADDLIBS@ @PGSRC@  @ADDINCLUDE@ -->
   <ItemGroup>
     <ClCompile Include="@PGSRC@\backend\access\transam\xlogreader.c" />
+    <ClCompile Include="@PGSRC@\backend\utils\hash\pg_crc.c" />
     <ClCompile Include="@PGSRC@\bin\pg_basebackup\receivelog.c" />
     <ClCompile Include="@PGSRC@\bin\pg_basebackup\streamutil.c" />
     <ClCompile Include="@PGSRC@\bin\pg_basebackup\walmethods.c" />

src/dir.c

@@ -259,35 +259,57 @@ pgFileDelete(pgFile *file)
 }
 
 pg_crc32
-pgFileGetCRC(const char *file_path)
+pgFileGetCRC(const char *file_path, bool use_crc32c)
 {
 	FILE	   *fp;
 	pg_crc32	crc = 0;
 	char		buf[1024];
 	size_t		len;
 	int			errno_tmp;
 
+#define INIT_FILE_CRC32(crc) \
+do { \
+	if (use_crc32c) \
+		INIT_CRC32C(crc); \
+	else \
+		INIT_TRADITIONAL_CRC32(crc); \
+} while (0)
+#define COMP_FILE_CRC32(crc, data, len) \
+do { \
+	if (use_crc32c) \
+		COMP_CRC32C((crc), (data), (len)); \
+	else \
+		COMP_TRADITIONAL_CRC32(crc, data, len); \
+} while (0)
+#define FIN_FILE_CRC32(crc) \
+do { \
+	if (use_crc32c) \
+		FIN_CRC32C(crc); \
+	else \
+		FIN_TRADITIONAL_CRC32(crc); \
+} while (0)
+
 	/* open file in binary read mode */
 	fp = fopen(file_path, PG_BINARY_R);
 	if (fp == NULL)
 		elog(ERROR, "cannot open file \"%s\": %s",
 			file_path, strerror(errno));
 
 	/* calc CRC of backup file */
-	INIT_TRADITIONAL_CRC32(crc);
+	INIT_FILE_CRC32(crc);
 	while ((len = fread(buf, 1, sizeof(buf), fp)) == sizeof(buf))
 	{
 		if (interrupted)
 			elog(ERROR, "interrupted during CRC calculation");
-		COMP_TRADITIONAL_CRC32(crc, buf, len);
+		COMP_FILE_CRC32(crc, buf, len);
 	}
 	errno_tmp = errno;
 	if (!feof(fp))
 		elog(WARNING, "cannot read \"%s\": %s", file_path,
 			strerror(errno_tmp));
 	if (len > 0)
-		COMP_TRADITIONAL_CRC32(crc, buf, len);
-	FIN_TRADITIONAL_CRC32(crc);
+		COMP_FILE_CRC32(crc, buf, len);
+	FIN_FILE_CRC32(crc);
 
 	fclose(fp);
 

src/merge.c

@@ -503,7 +503,7 @@ merge_files(void *arg)
 				 * do that.
 				 */
 				file->write_size = pgFileSize(to_path_tmp);
-				file->crc = pgFileGetCRC(to_path_tmp);
+				file->crc = pgFileGetCRC(to_path_tmp, false);
 			}
 			pgFileDelete(file);
 		}

src/pg_probackup.h

@@ -503,7 +503,7 @@ extern pgFile *pgFileNew(const char *path, bool omit_symlink);
 extern pgFile *pgFileInit(const char *path);
 extern void pgFileDelete(pgFile *file);
 extern void pgFileFree(void *file);
-extern pg_crc32 pgFileGetCRC(const char *file_path);
+extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c);
 extern int pgFileComparePath(const void *f1, const void *f2);
 extern int pgFileComparePathDesc(const void *f1, const void *f2);
 extern int pgFileCompareLinked(const void *f1, const void *f2);
@@ -564,7 +564,8 @@ extern void remove_not_digit(char *buf, size_t len, const char *str);
 extern const char *base36enc(long unsigned int value);
 extern char *base36enc_dup(long unsigned int value);
 extern long unsigned int base36dec(const char *text);
-extern int parse_server_version(char *server_version_str);
+extern uint32 parse_server_version(const char *server_version_str);
+extern uint32 parse_program_version(const char *program_version);
 
 #ifdef WIN32
 #ifdef _DEBUG

src/util.c

@@ -280,12 +280,14 @@ time2iso(char *buf, size_t len, time_t time)
 	}
 }
 
-/* Parse string representation of the server version */
-int
-parse_server_version(char *server_version_str)
+/*
+ * Parse string representation of the server version.
+ */
+uint32
+parse_server_version(const char *server_version_str)
 {
 	int			nfields;
-	int			result = 0;
+	uint32		result = 0;
 	int			major_version = 0;
 	int			minor_version = 0;
 
@@ -304,7 +306,31 @@ parse_server_version(char *server_version_str)
 		result = major_version * 10000;
 	}
 	else
-		elog(ERROR, "Unknown server version format");
+		elog(ERROR, "Unknown server version format %s", server_version_str);
+
+	return result;
+}
+
+/*
+ * Parse string representation of the program version.
+ */
+uint32
+parse_program_version(const char *program_version)
+{
+	int			nfields;
+	int			major = 0,
+				minor = 0,
+				micro = 0;
+	uint32		result = 0;
+
+	if (program_version == NULL || program_version[0] == '\0')
+		return 0;
+
+	nfields = sscanf(program_version, "%d.%d.%d", &major, &minor, &micro);
+	if (nfields == 3)
+		result = major * 10000 + minor * 100 + micro;
+	else
+		elog(ERROR, "Unknown program version format %s", program_version);
 
 	return result;
 }

src/validate.c

@@ -19,6 +19,8 @@ static void *pgBackupValidateFiles(void *arg);
 static void do_validate_instance(void);
 
 static bool corrupted_backup_found = false;
+/* Program version of a current backup */
+static uint32 validate_backup_version = 0;
 
 typedef struct
 {
@@ -90,6 +92,11 @@ pgBackupValidate(pgBackup *backup)
 		pg_atomic_clear_flag(&file->lock);
 	}
 
+	/*
+	 * We use program version to calculate checksum in pgBackupValidateFiles()
+	 */
+	validate_backup_version = parse_program_version(backup->program_version);
+
 	/* init thread args with own file lists */
 	threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads);
 	threads_args = (validate_files_arg *)
@@ -205,7 +212,18 @@ pgBackupValidateFiles(void *arg)
 			break;
 		}
 
-		crc = pgFileGetCRC(file->path);
+		/*
+		 * Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we
+		 * use CRC-32.
+		 *
+		 * pg_control stores its content and checksum of the content, calculated
+		 * using CRC-32C. If we calculate checksum of the whole pg_control using
+		 * CRC-32C we get same checksum constantly. It might be because of the
+		 * CRC-32C algorithm.
+		 * To avoid this problem we need to use different algorithm, CRC-32 in
+		 * this case.
+		 */
+		crc = pgFileGetCRC(file->path, validate_backup_version <= 20021);
 		if (crc != file->crc)
 		{
 			elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",