Skip to content

Commit 7329256

Browse files
gsmolkgsmolk
committed
[Issue #324] "--no-validate" and "--no-sync" flags for merge and delete commands
1 parent 88c1312 commit 7329256

File tree

8 files changed

+61
-62
lines changed

8 files changed

+61
-62
lines changed

‎src/backup.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -887,7 +887,7 @@ do_backup(pgSetBackupParams *set_backup_params,
887887
* which are expired according to retention policies
888888
*/
889889
if (delete_expired || merge_expired || delete_wal)
890-
do_retention();
890+
do_retention(no_validate, no_sync);
891891

892892
return 0;
893893
}

‎src/delete.c

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ static void delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tli,
1919
static void do_retention_internal(parray *backup_list, parray *to_keep_list,
2020
parray *to_purge_list);
2121
static void do_retention_merge(parray *backup_list, parray *to_keep_list,
22-
parray *to_purge_list);
22+
parray *to_purge_list, bool no_validate, bool no_sync);
2323
static void do_retention_purge(parray *to_keep_list, parray *to_purge_list);
2424
static void do_retention_wal(bool dry_run);
2525

@@ -123,7 +123,7 @@ do_delete(time_t backup_id)
123123
* which FULL backup should be keeped for redundancy obligation(only valid do),
124124
* but if invalid backup is not guarded by retention - it is removed
125125
*/
126-
void do_retention(void)
126+
void do_retention(bool no_validate, bool no_sync)
127127
{
128128
parray *backup_list = NULL;
129129
parray *to_keep_list = parray_new();
@@ -172,7 +172,7 @@ void do_retention(void)
172172
do_retention_internal(backup_list, to_keep_list, to_purge_list);
173173

174174
if (merge_expired && !dry_run && !backup_list_is_empty)
175-
do_retention_merge(backup_list, to_keep_list, to_purge_list);
175+
do_retention_merge(backup_list, to_keep_list, to_purge_list, no_validate, no_sync);
176176

177177
if (delete_expired && !dry_run && !backup_list_is_empty)
178178
do_retention_purge(to_keep_list, to_purge_list);
@@ -424,7 +424,8 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
424424

425425
/* Merge partially expired incremental chains */
426426
static void
427-
do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_list)
427+
do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_list,
428+
bool no_validate, bool no_sync)
428429
{
429430
int i;
430431
int j;
@@ -543,7 +544,7 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l
543544
*/
544545

545546
keep_backup = parray_get(merge_list, 0);
546-
merge_chain(merge_list, full_backup, keep_backup);
547+
merge_chain(merge_list, full_backup, keep_backup, no_validate, no_sync);
547548
backup_merged = true;
548549

549550
for (j = parray_num(merge_list) - 2; j >= 0; j--)
@@ -554,8 +555,8 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l
554555
parray_rm(to_purge_list, tmp_backup, pgBackupCompareId);
555556
parray_set(to_keep_list, i, NULL);
556557
}
557-
558-
pgBackupValidate(full_backup, NULL);
558+
if (!no_validate)
559+
pgBackupValidate(full_backup, NULL);
559560
if (full_backup->status == BACKUP_STATUS_CORRUPT)
560561
elog(ERROR, "Merging of backup %s failed", base36enc(full_backup->start_time));
561562

‎src/help.c

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -197,11 +197,12 @@ help_pg_probackup(void)
197197
printf(_(" [--wal-depth=wal-depth]\n"));
198198
printf(_(" [-i backup-id | --delete-expired | --merge-expired | --status=backup_status]\n"));
199199
printf(_(" [--delete-wal]\n"));
200-
printf(_(" [--dry-run]\n"));
200+
printf(_(" [--dry-run] [--no-validate] [--no-sync]\n"));
201201
printf(_(" [--help]\n"));
202202

203203
printf(_("\n %s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
204204
printf(_(" -i backup-id [--progress] [-j num-threads]\n"));
205+
printf(_(" [--no-validate] [--no-sync]\n"));
205206
printf(_(" [--help]\n"));
206207

207208
printf(_("\n %s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME);
@@ -631,13 +632,16 @@ help_delete(void)
631632
printf(_(" [-j num-threads] [--progress]\n"));
632633
printf(_(" [--retention-redundancy=retention-redundancy]\n"));
633634
printf(_(" [--retention-window=retention-window]\n"));
634-
printf(_(" [--wal-depth=wal-depth]\n\n"));
635+
printf(_(" [--wal-depth=wal-depth]\n"));
636+
printf(_(" [--no-validate] [--no-sync]\n\n"));
635637

636638
printf(_(" -B, --backup-path=backup-path location of the backup storage area\n"));
637639
printf(_(" --instance=instance_name name of the instance\n"));
638640
printf(_(" -i, --backup-id=backup-id backup to delete\n"));
639641
printf(_(" -j, --threads=NUM number of parallel threads\n"));
640642
printf(_(" --progress show progress\n"));
643+
printf(_(" --no-validate disable validation during retention merge\n"));
644+
printf(_(" --no-sync do not sync merged files to disk\n"));
641645

642646
printf(_("\n Retention options:\n"));
643647
printf(_(" --delete-expired delete backups expired according to current\n"));
@@ -681,6 +685,7 @@ help_merge(void)
681685
{
682686
printf(_("\n%s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
683687
printf(_(" -i backup-id [-j num-threads] [--progress]\n"));
688+
printf(_(" [--no-validate] [--no-sync]\n"));
684689
printf(_(" [--log-level-console=log-level-console]\n"));
685690
printf(_(" [--log-level-file=log-level-file]\n"));
686691
printf(_(" [--log-filename=log-filename]\n"));
@@ -695,6 +700,8 @@ help_merge(void)
695700

696701
printf(_(" -j, --threads=NUM number of parallel threads\n"));
697702
printf(_(" --progress show progress\n"));
703+
printf(_(" --no-validate disable validation during retention merge\n"));
704+
printf(_(" --no-sync do not sync merged files to disk\n"));
698705

699706
printf(_("\n Logging options:\n"));
700707
printf(_(" --log-level-console=log-level-console\n"));

‎src/merge.c

Lines changed: 37 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ typedef struct
3030
bool program_version_match;
3131
bool use_bitmap;
3232
bool is_retry;
33+
bool no_sync;
3334

3435
/*
3536
* Return value from the thread.
@@ -50,13 +51,13 @@ static void
5051
merge_data_file(parray *parent_chain, pgBackup *full_backup,
5152
pgBackup *dest_backup, pgFile *dest_file,
5253
pgFile *tmp_file, const char *to_root, bool use_bitmap,
53-
bool is_retry);
54+
bool is_retry, bool no_sync);
5455

5556
static void
5657
merge_non_data_file(parray *parent_chain, pgBackup *full_backup,
5758
pgBackup *dest_backup, pgFile *dest_file,
5859
pgFile *tmp_file, const char *full_database_dir,
59-
const char *full_external_prefix);
60+
const char *full_external_prefix, bool no_sync);
6061

6162
static bool is_forward_compatible(parray *parent_chain);
6263

@@ -68,7 +69,7 @@ static bool is_forward_compatible(parray *parent_chain);
6869
* - Remove unnecessary files, which doesn't exist in the target backup anymore
6970
*/
7071
void
71-
do_merge(time_t backup_id)
72+
do_merge(time_t backup_id, bool no_validate, bool no_sync)
7273
{
7374
parray *backups;
7475
parray *merge_list = parray_new();
@@ -405,9 +406,10 @@ do_merge(time_t backup_id)
405406
catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0, true, true);
406407

407408
/* do actual merge */
408-
merge_chain(merge_list, full_backup, dest_backup);
409+
merge_chain(merge_list, full_backup, dest_backup, no_validate, no_sync);
409410

410-
pgBackupValidate(full_backup, NULL);
411+
if (!no_validate)
412+
pgBackupValidate(full_backup, NULL);
411413
if (full_backup->status == BACKUP_STATUS_CORRUPT)
412414
elog(ERROR, "Merging of backup %s failed", base36enc(backup_id));
413415

@@ -434,7 +436,8 @@ do_merge(time_t backup_id)
434436
* that chain is ok.
435437
*/
436438
void
437-
merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
439+
merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup,
440+
bool no_validate, bool no_sync)
438441
{
439442
int i;
440443
char *dest_backup_id;
@@ -554,25 +557,28 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
554557
* with sole exception of FULL backup. If it has MERGING status
555558
* then it isn't valid backup until merging is finished.
556559
*/
557-
elog(INFO, "Validate parent chain for backup %s",
558-
base36enc(dest_backup->start_time));
559-
560-
for (i = parray_num(parent_chain) - 1; i >= 0; i--)
560+
if (!no_validate)
561561
{
562-
pgBackup *backup = (pgBackup *) parray_get(parent_chain, i);
562+
elog(INFO, "Validate parent chain for backup %s",
563+
base36enc(dest_backup->start_time));
563564

564-
/* FULL backup is not to be validated if its status is MERGING */
565-
if (backup->backup_mode == BACKUP_MODE_FULL &&
566-
backup->status == BACKUP_STATUS_MERGING)
565+
for (i = parray_num(parent_chain) - 1; i >= 0; i--)
567566
{
568-
continue;
569-
}
567+
pgBackup *backup = (pgBackup *) parray_get(parent_chain, i);
570568

571-
pgBackupValidate(backup, NULL);
569+
/* FULL backup is not to be validated if its status is MERGING */
570+
if (backup->backup_mode == BACKUP_MODE_FULL &&
571+
backup->status == BACKUP_STATUS_MERGING)
572+
{
573+
continue;
574+
}
572575

573-
if (backup->status != BACKUP_STATUS_OK)
574-
elog(ERROR, "Backup %s has status %s, merge is aborted",
575-
base36enc(backup->start_time), status2str(backup->status));
576+
pgBackupValidate(backup, NULL);
577+
578+
if (backup->status != BACKUP_STATUS_OK)
579+
elog(ERROR, "Backup %s has status %s, merge is aborted",
580+
base36enc(backup->start_time), status2str(backup->status));
581+
}
576582
}
577583

578584
/*
@@ -665,6 +671,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
665671
arg->program_version_match = program_version_match;
666672
arg->use_bitmap = use_bitmap;
667673
arg->is_retry = is_retry;
674+
arg->no_sync = no_sync;
668675
/* By default there are some error */
669676
arg->ret = 1;
670677

@@ -1102,14 +1109,16 @@ merge_files(void *arg)
11021109
dest_file, tmp_file,
11031110
arguments->full_database_dir,
11041111
arguments->use_bitmap,
1105-
arguments->is_retry);
1112+
arguments->is_retry,
1113+
arguments->no_sync);
11061114
else
11071115
merge_non_data_file(arguments->parent_chain,
11081116
arguments->full_backup,
11091117
arguments->dest_backup,
11101118
dest_file, tmp_file,
11111119
arguments->full_database_dir,
1112-
arguments->full_external_prefix);
1120+
arguments->full_external_prefix,
1121+
arguments->no_sync);
11131122

11141123
done:
11151124
parray_append(arguments->merge_filelist, tmp_file);
@@ -1202,7 +1211,8 @@ reorder_external_dirs(pgBackup *to_backup, parray *to_external,
12021211
void
12031212
merge_data_file(parray *parent_chain, pgBackup *full_backup,
12041213
pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file,
1205-
const char *full_database_dir, bool use_bitmap, bool is_retry)
1214+
const char *full_database_dir, bool use_bitmap, bool is_retry,
1215+
bool no_sync)
12061216
{
12071217
FILE *out = NULL;
12081218
char *buffer = pgut_malloc(STDIO_BUFSIZE);
@@ -1273,7 +1283,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup,
12731283
return;
12741284

12751285
/* sync second temp file to disk */
1276-
if (fio_sync(to_fullpath_tmp2, FIO_BACKUP_HOST) != 0)
1286+
if (!no_sync && fio_sync(to_fullpath_tmp2, FIO_BACKUP_HOST) != 0)
12771287
elog(ERROR, "Cannot sync merge temp file \"%s\": %s",
12781288
to_fullpath_tmp2, strerror(errno));
12791289

@@ -1294,7 +1304,8 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup,
12941304
void
12951305
merge_non_data_file(parray *parent_chain, pgBackup *full_backup,
12961306
pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file,
1297-
const char *full_database_dir, const char *to_external_prefix)
1307+
const char *full_database_dir, const char *to_external_prefix,
1308+
bool no_sync)
12981309
{
12991310
int i;
13001311
char to_fullpath[MAXPGPATH];
@@ -1378,7 +1389,7 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup,
13781389
to_fullpath_tmp, BACKUP_MODE_FULL, 0, false);
13791390

13801391
/* sync temp file to disk */
1381-
if (fio_sync(to_fullpath_tmp, FIO_BACKUP_HOST) != 0)
1392+
if (!no_sync && fio_sync(to_fullpath_tmp, FIO_BACKUP_HOST) != 0)
13821393
elog(ERROR, "Cannot sync merge temp file \"%s\": %s",
13831394
to_fullpath_tmp, strerror(errno));
13841395

‎src/pg_probackup.c

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -128,10 +128,6 @@ bool compress_shortcut = false;
128128
/* other options */
129129
char *instance_name;
130130

131-
/* TODO: quick hack */
132-
bool merge_no_validate = false;
133-
bool merge_no_sync = false;
134-
135131
/* archive push options */
136132
int batch_size = 1;
137133
static char *wal_file_path;
@@ -834,8 +830,6 @@ main(int argc, char *argv[])
834830
case SHOW_CMD:
835831
return do_show(instance_name, current.backup_id, show_archive);
836832
case DELETE_CMD:
837-
merge_no_validate = no_validate;
838-
merge_no_sync = no_sync;
839833

840834
if (delete_expired && backup_id_string)
841835
elog(ERROR, "You cannot specify --delete-expired and (-i, --backup-id) options together");
@@ -851,15 +845,13 @@ main(int argc, char *argv[])
851845
if (delete_status)
852846
do_delete_status(&instance_config, delete_status);
853847
else
854-
do_retention();
848+
do_retention(no_validate, no_sync);
855849
}
856850
else
857851
do_delete(current.backup_id);
858852
break;
859853
case MERGE_CMD:
860-
merge_no_validate = no_validate;
861-
merge_no_sync = no_sync;
862-
do_merge(current.backup_id);
854+
do_merge(current.backup_id, no_validate, no_sync);
863855
break;
864856
case SHOW_CONFIG_CMD:
865857
do_show_config();

‎src/pg_probackup.h

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -786,10 +786,6 @@ extern bool compress_shortcut;
786786
/* other options */
787787
extern char *instance_name;
788788

789-
/* temp merge options */
790-
extern bool merge_no_validate;
791-
extern bool merge_no_sync;
792-
793789
/* show options */
794790
extern ShowFormat show_format;
795791

@@ -843,10 +839,10 @@ extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetT
843839
extern bool tliIsPartOfHistory(const parray *timelines, TimeLineID tli);
844840

845841
/* in merge.c */
846-
extern void do_merge(time_t backup_id);
842+
extern void do_merge(time_t backup_id, bool no_validate, bool no_sync);
847843
extern void merge_backups(pgBackup *backup, pgBackup *next_backup);
848-
extern void merge_chain(parray *parent_chain,
849-
pgBackup *full_backup, pgBackup *dest_backup);
844+
extern void merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup,
845+
bool no_validate, bool no_sync);
850846

851847
extern parray *read_database_map(pgBackup *backup);
852848

@@ -873,7 +869,7 @@ extern int do_show(const char *instance_name, time_t requested_backup_id, bool s
873869
/* in delete.c */
874870
extern void do_delete(time_t backup_id);
875871
extern void delete_backup_files(pgBackup *backup);
876-
extern void do_retention(void);
872+
extern void do_retention(bool no_validate, bool no_sync);
877873
extern int do_delete_instance(void);
878874
extern void do_delete_status(InstanceConfig *instance_config, const char *status);
879875

‎src/utils/file.c

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1169,9 +1169,6 @@ int fio_sync(char const* path, fio_location location)
11691169
{
11701170
int fd;
11711171

1172-
if (merge_no_sync)
1173-
return 0;
1174-
11751172
fd = open(path, O_WRONLY | PG_BINARY, FILE_PERMISSIONS);
11761173
if (fd < 0)
11771174
return -1;

‎src/validate.c

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -129,9 +129,6 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
129129
// dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list,
130130
// params->partial_restore_type);
131131

132-
if (merge_no_validate)
133-
goto skip_validation;
134-
135132
/* setup threads */
136133
for (i = 0; i < parray_num(files); i++)
137134
{
@@ -183,8 +180,6 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
183180
pfree(threads);
184181
pfree(threads_args);
185182

186-
skip_validation:
187-
188183
/* cleanup */
189184
parray_walk(files, pgFileFree);
190185
parray_free(files);

0 commit comments

Comments
 (0)