Skip to content

Commit f0e4b42

Browse files
za-arthurza-arthur
committed
Merge branch 'master' into stable
2 parents 8b45e97 + 9804b79 commit f0e4b42

18 files changed

+146
-104
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
/tests/helpers/*pyc
3434

3535
# Extra files
36+
/src/pg_crc.c
3637
/src/datapagemap.c
3738
/src/datapagemap.h
3839
/src/logging.h

‎Makefile

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,23 @@
11
PROGRAM = pg_probackup
2-
OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \
3-
src/delete.o src/dir.o src/fetch.o src/help.o src/init.o \
4-
src/pg_probackup.o src/restore.o src/show.o \
5-
src/util.o src/validate.o src/datapagemap.o src/parsexlog.o \
6-
src/xlogreader.o src/streamutil.o src/receivelog.o \
7-
src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o \
8-
src/utils/json.o src/utils/thread.o src/merge.o
92

10-
EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \
11-
src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h
3+
# utils
4+
OBJS = src/utils/json.o src/utils/logger.o src/utils/parray.o \
5+
src/utils/pgut.o src/utils/thread.o
126

13-
INCLUDES = src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h
7+
OBJS += src/archive.o src/backup.o src/catalog.o src/configure.o src/data.o \
8+
src/delete.o src/dir.o src/fetch.o src/help.o src/init.o src/merge.o \
9+
src/parsexlog.o src/pg_probackup.o src/restore.o src/show.o src/util.o \
10+
src/validate.o
11+
12+
# borrowed files
13+
OBJS += src/pg_crc.o src/datapagemap.o src/receivelog.o src/streamutil.o \
14+
src/xlogreader.o
15+
16+
EXTRA_CLEAN = src/pg_crc.c src/datapagemap.c src/datapagemap.h src/logging.h \
17+
src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h \
18+
src/xlogreader.c
19+
20+
INCLUDES = src/datapagemap.h src/logging.h src/streamutil.h src/receivelog.h
1421

1522
ifdef USE_PGXS
1623
PG_CONFIG = pg_config
@@ -46,14 +53,14 @@ all: checksrcdir $(INCLUDES);
4653

4754
$(PROGRAM): $(OBJS)
4855

49-
src/xlogreader.c: $(top_srcdir)/src/backend/access/transam/xlogreader.c
50-
rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@
5156
src/datapagemap.c: $(top_srcdir)/src/bin/pg_rewind/datapagemap.c
5257
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.c $@
5358
src/datapagemap.h: $(top_srcdir)/src/bin/pg_rewind/datapagemap.h
5459
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.h $@
5560
src/logging.h: $(top_srcdir)/src/bin/pg_rewind/logging.h
5661
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/logging.h $@
62+
src/pg_crc.c: $(top_srcdir)/src/backend/utils/hash/pg_crc.c
63+
rm -f $@ && $(LN_S) $(srchome)/src/backend/utils/hash/pg_crc.c $@
5764
src/receivelog.c: $(top_srcdir)/src/bin/pg_basebackup/receivelog.c
5865
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/receivelog.c $@
5966
src/receivelog.h: $(top_srcdir)/src/bin/pg_basebackup/receivelog.h
@@ -62,6 +69,8 @@ src/streamutil.c: $(top_srcdir)/src/bin/pg_basebackup/streamutil.c
6269
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.c $@
6370
src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h
6471
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@
72+
src/xlogreader.c: $(top_srcdir)/src/backend/access/transam/xlogreader.c
73+
rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@
6574

6675

6776
ifeq (,$(filter 9.5 9.6,$(MAJORVERSION)))

‎gen_probackup_project.pl

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ BEGIN
77
if (@ARGV==1)
88
{
99
$pgsrc = shift @ARGV;
10-
if($pgsrc == "--help"){
10+
if($pgsrc eq "--help"){
1111
print STDERR "Usage $0 pg-source-dir \n";
1212
print STDERR "Like this: \n";
1313
print STDERR "$0 C:/PgProject/postgresql.10dev/postgrespro \n";
@@ -157,6 +157,7 @@ sub build_pgprobackup
157157
'thread.c'
158158
);
159159
$probackup->AddFile('src/backend/access/transam/xlogreader.c');
160+
$probackup->AddFile('src/backend/utils/hash/pg_crc.c');
160161
$probackup->AddFiles(
161162
'src/bin/pg_basebackup',
162163
'receivelog.c',

‎msvs/template.pg_probackup.vcxproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,7 @@
165165
</ItemDefinitionGroup>
166166
<ItemGroup>
167167
<ClCompile Include="@PGSRC@\backend\access\transam\xlogreader.c" />
168+
<ClCompile Include="@PGSRC@\backend\utils\hash\pg_crc.c" />
168169
<ClCompile Include="@PGSRC@\bin\pg_basebackup\receivelog.c" />
169170
<ClCompile Include="@PGSRC@\bin\pg_basebackup\streamutil.c" />
170171
<ClCompile Include="@PGSRC@\bin\pg_basebackup\walmethods.c" />

‎msvs/template.pg_probackup96.vcxproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,7 @@
165165
</ItemDefinitionGroup>
166166
<ItemGroup>
167167
<ClCompile Include="@PGSRC@\backend\access\transam\xlogreader.c" />
168+
<ClCompile Include="@PGSRC@\backend\utils\hash\pg_crc.c" />
168169
<ClCompile Include="@PGSRC@\bin\pg_basebackup\receivelog.c" />
169170
<ClCompile Include="@PGSRC@\bin\pg_basebackup\streamutil.c" />
170171
<ClCompile Include="@PGSRC@\bin\pg_rewind\datapagemap.c" />

‎msvs/template.pg_probackup_2.vcxproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@
160160
<!-- @PGROOT@\lib;@ADDLIBS@ @PGSRC@ @ADDINCLUDE@ -->
161161
<ItemGroup>
162162
<ClCompile Include="@PGSRC@\backend\access\transam\xlogreader.c" />
163+
<ClCompile Include="@PGSRC@\backend\utils\hash\pg_crc.c" />
163164
<ClCompile Include="@PGSRC@\bin\pg_basebackup\receivelog.c" />
164165
<ClCompile Include="@PGSRC@\bin\pg_basebackup\streamutil.c" />
165166
<ClCompile Include="@PGSRC@\bin\pg_basebackup\walmethods.c" />

‎src/backup.c

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,7 @@ remote_copy_file(PGconn *conn, pgFile* file)
306306
to_path, strerror(errno_tmp));
307307
}
308308

309-
INIT_CRC32C(file->crc);
309+
INIT_TRADITIONAL_CRC32(file->crc);
310310

311311
/* read from stream and write to backup file */
312312
while (1)
@@ -332,14 +332,14 @@ remote_copy_file(PGconn *conn, pgFile* file)
332332
{
333333
write_buffer_size = Min(row_length, sizeof(buf));
334334
memcpy(buf, copybuf, write_buffer_size);
335-
COMP_CRC32C(file->crc, buf, write_buffer_size);
335+
COMP_TRADITIONAL_CRC32(file->crc, buf, write_buffer_size);
336336

337337
/* TODO calc checksum*/
338338
if (fwrite(buf, 1, write_buffer_size, out) != write_buffer_size)
339339
{
340340
errno_tmp = errno;
341341
/* oops */
342-
FIN_CRC32C(file->crc);
342+
FIN_TRADITIONAL_CRC32(file->crc);
343343
fclose(out);
344344
PQfinish(conn);
345345
elog(ERROR, "cannot write to \"%s\": %s", to_path,
@@ -363,7 +363,7 @@ remote_copy_file(PGconn *conn, pgFile* file)
363363
}
364364

365365
file->write_size = (int64) file->read_size;
366-
FIN_CRC32C(file->crc);
366+
FIN_TRADITIONAL_CRC32(file->crc);
367367

368368
fclose(out);
369369
}
@@ -2137,7 +2137,7 @@ backup_files(void *arg)
21372137
continue;
21382138
}
21392139
}
2140-
else
2140+
else
21412141
{
21422142
bool skip = false;
21432143

@@ -2147,7 +2147,7 @@ backup_files(void *arg)
21472147
{
21482148
calc_file_checksum(file);
21492149
/* ...and checksum is the same... */
2150-
if (EQ_CRC32C(file->crc, (*prev_file)->crc))
2150+
if (EQ_TRADITIONAL_CRC32(file->crc, (*prev_file)->crc))
21512151
skip = true; /* ...skip copying file. */
21522152
}
21532153
if (skip ||

‎src/data.c

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -416,7 +416,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum,
416416
blknum, header.compressed_size, write_buffer_size); */
417417

418418
/* Update CRC */
419-
COMP_CRC32C(*crc, write_buffer, write_buffer_size);
419+
COMP_TRADITIONAL_CRC32(*crc, write_buffer, write_buffer_size);
420420

421421
/* write data page */
422422
if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size)
@@ -476,13 +476,13 @@ backup_data_file(backup_files_arg* arguments,
476476
/* reset size summary */
477477
file->read_size = 0;
478478
file->write_size = 0;
479-
INIT_CRC32C(file->crc);
479+
INIT_TRADITIONAL_CRC32(file->crc);
480480

481481
/* open backup mode file for read */
482482
in = fopen(file->path, PG_BINARY_R);
483483
if (in == NULL)
484484
{
485-
FIN_CRC32C(file->crc);
485+
FIN_TRADITIONAL_CRC32(file->crc);
486486

487487
/*
488488
* If file is not found, this is not en error.
@@ -587,7 +587,7 @@ backup_data_file(backup_files_arg* arguments,
587587
to_path, strerror(errno));
588588
fclose(in);
589589

590-
FIN_CRC32C(file->crc);
590+
FIN_TRADITIONAL_CRC32(file->crc);
591591

592592
/*
593593
* If we have pagemap then file in the backup can't be a zero size.
@@ -839,7 +839,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
839839
struct stat st;
840840
pg_crc32 crc;
841841

842-
INIT_CRC32C(crc);
842+
INIT_TRADITIONAL_CRC32(crc);
843843

844844
/* reset size summary */
845845
file->read_size = 0;
@@ -849,7 +849,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
849849
in = fopen(file->path, PG_BINARY_R);
850850
if (in == NULL)
851851
{
852-
FIN_CRC32C(crc);
852+
FIN_TRADITIONAL_CRC32(crc);
853853
file->crc = crc;
854854

855855
/* maybe deleted, it's not error */
@@ -898,7 +898,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
898898
strerror(errno_tmp));
899899
}
900900
/* update CRC */
901-
COMP_CRC32C(crc, buf, read_len);
901+
COMP_TRADITIONAL_CRC32(crc, buf, read_len);
902902

903903
file->read_size += read_len;
904904
}
@@ -925,14 +925,14 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
925925
strerror(errno_tmp));
926926
}
927927
/* update CRC */
928-
COMP_CRC32C(crc, buf, read_len);
928+
COMP_TRADITIONAL_CRC32(crc, buf, read_len);
929929

930930
file->read_size += read_len;
931931
}
932932

933933
file->write_size = (int64) file->read_size;
934934
/* finish CRC calculation and store into pgFile */
935-
FIN_CRC32C(crc);
935+
FIN_TRADITIONAL_CRC32(crc);
936936
file->crc = crc;
937937

938938
/* update file permission */
@@ -1350,7 +1350,7 @@ calc_file_checksum(pgFile *file)
13501350
pg_crc32 crc;
13511351

13521352
Assert(S_ISREG(file->mode));
1353-
INIT_CRC32C(crc);
1353+
INIT_TRADITIONAL_CRC32(crc);
13541354

13551355
/* reset size summary */
13561356
file->read_size = 0;
@@ -1360,7 +1360,7 @@ calc_file_checksum(pgFile *file)
13601360
in = fopen(file->path, PG_BINARY_R);
13611361
if (in == NULL)
13621362
{
1363-
FIN_CRC32C(crc);
1363+
FIN_TRADITIONAL_CRC32(crc);
13641364
file->crc = crc;
13651365

13661366
/* maybe deleted, it's not error */
@@ -1387,7 +1387,7 @@ calc_file_checksum(pgFile *file)
13871387
break;
13881388

13891389
/* update CRC */
1390-
COMP_CRC32C(crc, buf, read_len);
1390+
COMP_TRADITIONAL_CRC32(crc, buf, read_len);
13911391

13921392
file->write_size += read_len;
13931393
file->read_size += read_len;
@@ -1402,7 +1402,7 @@ calc_file_checksum(pgFile *file)
14021402
}
14031403

14041404
/* finish CRC calculation and store into pgFile */
1405-
FIN_CRC32C(crc);
1405+
FIN_TRADITIONAL_CRC32(crc);
14061406
file->crc = crc;
14071407

14081408
fclose(in);
@@ -1501,8 +1501,8 @@ validate_one_page(Page page, pgFile *file,
15011501
lsn = PageXLogRecPtrGet(phdr->pd_lsn);
15021502

15031503
if (lsn > stop_lsn)
1504-
elog(WARNING, "File: %s, block %u, checksum is not enabled."
1505-
"page is from future: pageLSN %X/%X stopLSN %X/%X",
1504+
elog(WARNING, "File: %s, block %u, checksum is not enabled. "
1505+
"Page is from future: pageLSN %X/%X stopLSN %X/%X",
15061506
file->path, blknum, (uint32) (lsn >> 32), (uint32) lsn,
15071507
(uint32) (stop_lsn >> 32), (uint32) stop_lsn);
15081508
else
@@ -1515,8 +1515,8 @@ validate_one_page(Page page, pgFile *file,
15151515
lsn = PageXLogRecPtrGet(phdr->pd_lsn);
15161516

15171517
if (lsn > stop_lsn)
1518-
elog(WARNING, "File: %s, block %u, checksum is correct."
1519-
"page is from future: pageLSN %X/%X stopLSN %X/%X",
1518+
elog(WARNING, "File: %s, block %u, checksum is correct. "
1519+
"Page is from future: pageLSN %X/%X stopLSN %X/%X",
15201520
file->path, blknum, (uint32) (lsn >> 32), (uint32) lsn,
15211521
(uint32) (stop_lsn >> 32), (uint32) stop_lsn);
15221522
else

‎src/dir.c

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -259,35 +259,57 @@ pgFileDelete(pgFile *file)
259259
}
260260

261261
pg_crc32
262-
pgFileGetCRC(const char *file_path)
262+
pgFileGetCRC(const char *file_path, bool use_crc32c)
263263
{
264264
FILE *fp;
265265
pg_crc32 crc = 0;
266266
char buf[1024];
267267
size_t len;
268268
int errno_tmp;
269269

270+
#define INIT_FILE_CRC32(crc) \
271+
do { \
272+
if (use_crc32c) \
273+
INIT_CRC32C(crc); \
274+
else \
275+
INIT_TRADITIONAL_CRC32(crc); \
276+
} while (0)
277+
#define COMP_FILE_CRC32(crc, data, len) \
278+
do { \
279+
if (use_crc32c) \
280+
COMP_CRC32C((crc), (data), (len)); \
281+
else \
282+
COMP_TRADITIONAL_CRC32(crc, data, len); \
283+
} while (0)
284+
#define FIN_FILE_CRC32(crc) \
285+
do { \
286+
if (use_crc32c) \
287+
FIN_CRC32C(crc); \
288+
else \
289+
FIN_TRADITIONAL_CRC32(crc); \
290+
} while (0)
291+
270292
/* open file in binary read mode */
271293
fp = fopen(file_path, PG_BINARY_R);
272294
if (fp == NULL)
273295
elog(ERROR, "cannot open file \"%s\": %s",
274296
file_path, strerror(errno));
275297

276298
/* calc CRC of backup file */
277-
INIT_CRC32C(crc);
299+
INIT_FILE_CRC32(crc);
278300
while ((len = fread(buf, 1, sizeof(buf), fp)) == sizeof(buf))
279301
{
280302
if (interrupted)
281303
elog(ERROR, "interrupted during CRC calculation");
282-
COMP_CRC32C(crc, buf, len);
304+
COMP_FILE_CRC32(crc, buf, len);
283305
}
284306
errno_tmp = errno;
285307
if (!feof(fp))
286308
elog(WARNING, "cannot read \"%s\": %s", file_path,
287309
strerror(errno_tmp));
288310
if (len > 0)
289-
COMP_CRC32C(crc, buf, len);
290-
FIN_CRC32C(crc);
311+
COMP_FILE_CRC32(crc, buf, len);
312+
FIN_FILE_CRC32(crc);
291313

292314
fclose(fp);
293315

‎src/merge.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup)
326326
to_backup->wal_bytes = BYTES_INVALID;
327327

328328
pgBackupWriteFileList(to_backup, files, from_database_path);
329-
write_backup_status(to_backup);
329+
write_backup(to_backup);
330330

331331
/* Cleanup */
332332
pfree(threads_args);
@@ -503,7 +503,7 @@ merge_files(void *arg)
503503
* do that.
504504
*/
505505
file->write_size = pgFileSize(to_path_tmp);
506-
file->crc = pgFileGetCRC(to_path_tmp);
506+
file->crc = pgFileGetCRC(to_path_tmp, false);
507507
}
508508
pgFileDelete(file);
509509
}

0 commit comments

Comments
 (0)