Skip to content

Commit 693bffe

Browse files
funny-falconfunny-falcon
committed
[PBCKP-146] fix cfm truncated crc calculation in delta/page backup
- On backup we should compare truncated crc with previous version. - copying remote file didn't honor "don't truncate first 64 bytes" rule. - crc calculation didn't honoer "don't truncate first 64 bytes" rule.
1 parent 8570825 commit 693bffe

File tree

4 files changed

+48
-13
lines changed

4 files changed

+48
-13
lines changed

‎src/data.c‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -806,7 +806,10 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
806806
* file could be deleted under our feets.
807807
* But then backup_non_data_file_internal will handle it safely
808808
*/
809-
file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true);
809+
if (file->forkName != cfm)
810+
file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true);
811+
else
812+
file->crc = fio_get_crc32_truncated(from_fullpath, FIO_DB_HOST, true);
810813

811814
/* ...and checksum is the same... */
812815
if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc))
@@ -1334,7 +1337,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
13341337
pg_crc32 file_crc;
13351338
if (tmp_file->forkName == cfm &&
13361339
tmp_file->uncompressed_size > tmp_file->write_size)
1337-
file_crc = fio_get_crc32_truncated(to_fullpath, FIO_DB_HOST);
1340+
file_crc = fio_get_crc32_truncated(to_fullpath, FIO_DB_HOST, false);
13381341
else
13391342
file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false);
13401343

‎src/pg_probackup.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1082,7 +1082,7 @@ extern void fio_pgFileDelete(pgFile *file, const char *full_path);
10821082
extern void pgFileFree(void *file);
10831083

10841084
extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok);
1085-
extern pg_crc32 pgFileGetCRCTruncated(const char *file_path, bool use_crc32c);
1085+
extern pg_crc32 pgFileGetCRCTruncated(const char *file_path, bool use_crc32c, bool missing_ok);
10861086
extern pg_crc32 pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok);
10871087

10881088
extern int pgFileMapComparePath(const void *f1, const void *f2);

‎src/utils/file.c‎

Lines changed: 40 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1377,8 +1377,6 @@ fio_get_crc32_ex(const char *file_path, fio_location location,
13771377
{
13781378
if (decompress && truncated)
13791379
elog(ERROR, "Could not calculate CRC for compressed truncated file");
1380-
if (missing_ok && truncated)
1381-
elog(ERROR, "CRC calculation for missing truncated file is forbidden");
13821380

13831381
if (fio_is_remote(location))
13841382
{
@@ -1408,7 +1406,7 @@ fio_get_crc32_ex(const char *file_path, fio_location location,
14081406
if (decompress)
14091407
return pgFileGetCRCgz(file_path, true, missing_ok);
14101408
else if (truncated)
1411-
return pgFileGetCRCTruncated(file_path, true);
1409+
return pgFileGetCRCTruncated(file_path, true, missing_ok);
14121410
else
14131411
return pgFileGetCRC(file_path, true, missing_ok);
14141412
}
@@ -1422,9 +1420,10 @@ fio_get_crc32(const char *file_path, fio_location location,
14221420
}
14231421

14241422
pg_crc32
1425-
fio_get_crc32_truncated(const char *file_path, fio_location location)
1423+
fio_get_crc32_truncated(const char *file_path, fio_location location,
1424+
bool missing_ok)
14261425
{
1427-
return fio_get_crc32_ex(file_path, location, false, false, true);
1426+
return fio_get_crc32_ex(file_path, location, false, missing_ok, true);
14281427
}
14291428

14301429
/* Remove file */
@@ -3003,6 +3002,7 @@ fio_send_file_impl(int out, char const* path)
30033002
fio_header hdr;
30043003
char *buf = pgut_malloc(CHUNK_SIZE);
30053004
size_t read_len = 0;
3005+
int64_t read_size = 0;
30063006
char *errormsg = NULL;
30073007

30083008
/* open source file for read */
@@ -3066,7 +3066,19 @@ fio_send_file_impl(int out, char const* path)
30663066
if (read_len > 0)
30673067
{
30683068
/* send chunk */
3069-
size_t non_zero_len = find_zero_tail(buf, read_len);
3069+
int64_t non_zero_len = find_zero_tail(buf, read_len);
3070+
/*
3071+
* It is dirty trick to silence warnings in CFS GC process:
3072+
* backup at least cfs header size bytes.
3073+
*/
3074+
if (read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY &&
3075+
read_size + read_len > 0)
3076+
{
3077+
non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY,
3078+
read_size + read_len);
3079+
non_zero_len -= read_size;
3080+
}
3081+
30703082
if (non_zero_len > 0)
30713083
{
30723084
hdr.cop = FIO_PAGE;
@@ -3082,6 +3094,8 @@ fio_send_file_impl(int out, char const* path)
30823094
hdr.arg = read_len - non_zero_len;
30833095
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
30843096
}
3097+
3098+
read_size += read_len;
30853099
}
30863100

30873101
if (feof(fp))
@@ -3166,7 +3180,7 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok)
31663180
* Read the local file to compute CRC for it extened to real_size.
31673181
*/
31683182
pg_crc32
3169-
pgFileGetCRCTruncated(const char *file_path, bool use_crc32c)
3183+
pgFileGetCRCTruncated(const char *file_path, bool use_crc32c, bool missing_ok)
31703184
{
31713185
FILE *fp;
31723186
char *buf;
@@ -3180,6 +3194,15 @@ pgFileGetCRCTruncated(const char *file_path, bool use_crc32c)
31803194
fp = fopen(file_path, PG_BINARY_R);
31813195
if (fp == NULL)
31823196
{
3197+
if (errno == ENOENT)
3198+
{
3199+
if (missing_ok)
3200+
{
3201+
FIN_FILE_CRC32(use_crc32c, st.crc);
3202+
return st.crc;
3203+
}
3204+
}
3205+
31833206
elog(ERROR, "Cannot open file \"%s\": %s",
31843207
file_path, strerror(errno));
31853208
}
@@ -3200,6 +3223,14 @@ pgFileGetCRCTruncated(const char *file_path, bool use_crc32c)
32003223
elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno));
32013224

32023225
non_zero_len = find_zero_tail(buf, len);
3226+
/* same trick as in fio_send_file */
3227+
if (st.read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY &&
3228+
st.read_size + len > 0)
3229+
{
3230+
non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY,
3231+
st.read_size + len);
3232+
non_zero_len -= st.read_size;
3233+
}
32033234
if (non_zero_len)
32043235
{
32053236
fio_send_file_crc(&st, buf, non_zero_len);
@@ -3894,12 +3925,12 @@ fio_communicate(int in, int out)
38943925
break;
38953926
case FIO_GET_CRC32:
38963927
Assert((hdr.arg & GET_CRC32_TRUNCATED) == 0 ||
3897-
(hdr.arg & GET_CRC32_TRUNCATED) == GET_CRC32_TRUNCATED);
3928+
(hdr.arg & (GET_CRC32_TRUNCATED|GET_CRC32_DECOMPRESS)) == GET_CRC32_TRUNCATED);
38983929
/* calculate crc32 for a file */
38993930
if ((hdr.arg & GET_CRC32_DECOMPRESS))
39003931
crc = pgFileGetCRCgz(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0);
39013932
else if ((hdr.arg & GET_CRC32_TRUNCATED))
3902-
crc = pgFileGetCRCTruncated(buf, true);
3933+
crc = pgFileGetCRCTruncated(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0);
39033934
else
39043935
crc = pgFileGetCRC(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0);
39053936
IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc));

‎src/utils/file.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,8 @@ extern void fio_disconnect(void);
123123
extern int fio_sync(char const* path, fio_location location);
124124
extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location,
125125
bool decompress, bool missing_ok);
126-
extern pg_crc32 fio_get_crc32_truncated(const char *file_path, fio_location location);
126+
extern pg_crc32 fio_get_crc32_truncated(const char *file_path, fio_location location,
127+
bool missing_ok);
127128

128129
extern int fio_rename(char const* old_path, char const* new_path, fio_location location);
129130
extern int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location);

0 commit comments

Comments
 (0)