Skip to content

Commit b606d38

Browse files
portanteportante
committed
Parallelize archive directory checks; compare file lists
In an attempt to speed up the verification process we have parallelized the checks of the archive (primary) and the backup directories. In addition, instead of comparing the output of the md5sum commands, we compare the content of the combined md5sum files (which are kept in sorted order by making sure the list of controllers is sorted before processing, and leveraging "ls"'s sorted output.
1 parent a56fa2b commit b606d38

File tree

5 files changed

+50
-49
lines changed

5 files changed

+50
-49
lines changed

‎server/pbench/bin/gold/test-9.2.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,5 +32,5 @@
3232
+++ test-report-status.log file contents
3333
/var/tmp/pbench-test-server/test-report-status.log:pbench-verify-backup-tarballs.run-1900-01-01T00:00:00-UTC(unit-test)
3434
/var/tmp/pbench-test-server/test-report-status.log:* Files that exist only in primary directory - extra files in this list are probably OK: they just have not been backed up yet.
35-
/var/tmp/pbench-test-server/test-report-status.log:/var/tmp/pbench-test-server/pbench/archive/fs-version-001/controller/pbench-user-benchmark_38_2016-05-18_19:36:32.tar.xz: OK
35+
/var/tmp/pbench-test-server/test-report-status.log:4fe570aea2be3dab96b328da5a8cfcc9 ./controller/pbench-user-benchmark_38_2016-05-18_19:36:32.tar.xz
3636
--- test-report-status.log file contents

‎server/pbench/bin/gold/test-9.3.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,6 @@
3131
--- test-execution.log file contents
3232
+++ test-report-status.log file contents
3333
/var/tmp/pbench-test-server/test-report-status.log:pbench-verify-backup-tarballs.run-1900-01-01T00:00:00-UTC(unit-test)
34-
/var/tmp/pbench-test-server/test-report-status.log:* Files that exist only in backup directory - this should not happen.
35-
/var/tmp/pbench-test-server/test-report-status.log:/var/tmp/pbench-test-server/pbench/archive.backup/controller/pbench-user-benchmark_38_2016-05-18_19:36:32.tar.xz: OK
34+
/var/tmp/pbench-test-server/test-report-status.log:* Files that exist only in backup directory - this should only happen if a backup or primary tar ball becomes corrupted.
35+
/var/tmp/pbench-test-server/test-report-status.log:4fe570aea2be3dab96b328da5a8cfcc9 ./controller/pbench-user-benchmark_38_2016-05-18_19:36:32.tar.xz
3636
--- test-report-status.log file contents

‎server/pbench/bin/gold/test-9.4.txt‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,19 +26,19 @@
2626
--- pbench tree state
2727
+++ pbench log file contents
2828
/var/tmp/pbench-test-server/pbench/logs/pbench-verify-backup-tarballs/pbench-verify-backup-tarballs.log:start-1900-01-01T00:00:00-UTC
29-
/var/tmp/pbench-test-server/pbench/logs/pbench-verify-backup-tarballs/pbench-verify-backup-tarballs.log:md5sum: WARNING: 1 computed checksum did NOT match
3029
/var/tmp/pbench-test-server/pbench/logs/pbench-verify-backup-tarballs/pbench-verify-backup-tarballs.log:end-1900-01-01T00:00:00-UTC
3130
--- pbench log file contents
3231
+++ test-execution.log file contents
3332
/var/tmp/pbench-test-server/test-execution.log:/var/tmp/pbench-test-server/opt/pbench-server/unittest-scripts/pbench-report-status --name pbench-verify-backup-tarballs --timestamp run-1900-01-01T00:00:00-UTC --type status /var/tmp/pbench-test-server/pbench/tmp/pbench-verify-backup-tarballs/index_mail_contents
3433
--- test-execution.log file contents
3534
+++ test-report-status.log file contents
3635
/var/tmp/pbench-test-server/test-report-status.log:pbench-verify-backup-tarballs.run-1900-01-01T00:00:00-UTC(unit-test)
37-
/var/tmp/pbench-test-server/test-report-status.log:* In /var/tmp/pbench-test-server/pbench/archive/fs-version-001: The calculated MD5 of the following entries failed to match the stored MD5
36+
/var/tmp/pbench-test-server/test-report-status.log:* In /var/tmp/pbench-test-server/pbench/archive/fs-version-001: the calculated MD5 of the following entries failed to match the stored MD5
37+
/var/tmp/pbench-test-server/test-report-status.log:md5sum: WARNING: 1 computed checksum did NOT match
3838
/var/tmp/pbench-test-server/test-report-status.log:/var/tmp/pbench-test-server/pbench/archive/fs-version-001/controller/pbench-user-benchmark_38_2016-05-18_19:36:32.tar.xz: FAILED
3939
/var/tmp/pbench-test-server/test-report-status.log:
4040
/var/tmp/pbench-test-server/test-report-status.log:* Files that exist only in primary directory - extra files in this list are probably OK: they just have not been backed up yet.
41-
/var/tmp/pbench-test-server/test-report-status.log:/var/tmp/pbench-test-server/pbench/archive/fs-version-001/controller/pbench-user-benchmark_38_2016-05-18_19:36:32.tar.xz: FAILED
42-
/var/tmp/pbench-test-server/test-report-status.log:* Files that exist only in backup directory - this should not happen.
43-
/var/tmp/pbench-test-server/test-report-status.log:/var/tmp/pbench-test-server/pbench/archive.backup/controller/pbench-user-benchmark_38_2016-05-18_19:36:32.tar.xz: OK
41+
/var/tmp/pbench-test-server/test-report-status.log:5fe570aea2be3dab96b328da5a8cfcc9 ./controller/pbench-user-benchmark_38_2016-05-18_19:36:32.tar.xz
42+
/var/tmp/pbench-test-server/test-report-status.log:* Files that exist only in backup directory - this should only happen if a backup or primary tar ball becomes corrupted.
43+
/var/tmp/pbench-test-server/test-report-status.log:4fe570aea2be3dab96b328da5a8cfcc9 ./controller/pbench-user-benchmark_38_2016-05-18_19:36:32.tar.xz
4444
--- test-report-status.log file contents

‎server/pbench/bin/gold/test-9.5.txt‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,19 +26,19 @@
2626
--- pbench tree state
2727
+++ pbench log file contents
2828
/var/tmp/pbench-test-server/pbench/logs/pbench-verify-backup-tarballs/pbench-verify-backup-tarballs.log:start-1900-01-01T00:00:00-UTC
29-
/var/tmp/pbench-test-server/pbench/logs/pbench-verify-backup-tarballs/pbench-verify-backup-tarballs.log:md5sum: WARNING: 1 computed checksum did NOT match
3029
/var/tmp/pbench-test-server/pbench/logs/pbench-verify-backup-tarballs/pbench-verify-backup-tarballs.log:end-1900-01-01T00:00:00-UTC
3130
--- pbench log file contents
3231
+++ test-execution.log file contents
3332
/var/tmp/pbench-test-server/test-execution.log:/var/tmp/pbench-test-server/opt/pbench-server/unittest-scripts/pbench-report-status --name pbench-verify-backup-tarballs --timestamp run-1900-01-01T00:00:00-UTC --type status /var/tmp/pbench-test-server/pbench/tmp/pbench-verify-backup-tarballs/index_mail_contents
3433
--- test-execution.log file contents
3534
+++ test-report-status.log file contents
3635
/var/tmp/pbench-test-server/test-report-status.log:pbench-verify-backup-tarballs.run-1900-01-01T00:00:00-UTC(unit-test)
37-
/var/tmp/pbench-test-server/test-report-status.log:* In /var/tmp/pbench-test-server/pbench/archive.backup: The calculated MD5 of the following entries failed to match the stored MD5
36+
/var/tmp/pbench-test-server/test-report-status.log:* In /var/tmp/pbench-test-server/pbench/archive.backup: the calculated MD5 of the following entries failed to match the stored MD5
37+
/var/tmp/pbench-test-server/test-report-status.log:md5sum: WARNING: 1 computed checksum did NOT match
3838
/var/tmp/pbench-test-server/test-report-status.log:/var/tmp/pbench-test-server/pbench/archive.backup/controller/pbench-user-benchmark_38_2016-05-18_19:36:32.tar.xz: FAILED
3939
/var/tmp/pbench-test-server/test-report-status.log:
4040
/var/tmp/pbench-test-server/test-report-status.log:* Files that exist only in primary directory - extra files in this list are probably OK: they just have not been backed up yet.
41-
/var/tmp/pbench-test-server/test-report-status.log:/var/tmp/pbench-test-server/pbench/archive/fs-version-001/controller/pbench-user-benchmark_38_2016-05-18_19:36:32.tar.xz: OK
42-
/var/tmp/pbench-test-server/test-report-status.log:* Files that exist only in backup directory - this should not happen.
43-
/var/tmp/pbench-test-server/test-report-status.log:/var/tmp/pbench-test-server/pbench/archive.backup/controller/pbench-user-benchmark_38_2016-05-18_19:36:32.tar.xz: FAILED
41+
/var/tmp/pbench-test-server/test-report-status.log:4fe570aea2be3dab96b328da5a8cfcc9 ./controller/pbench-user-benchmark_38_2016-05-18_19:36:32.tar.xz
42+
/var/tmp/pbench-test-server/test-report-status.log:* Files that exist only in backup directory - this should only happen if a backup or primary tar ball becomes corrupted.
43+
/var/tmp/pbench-test-server/test-report-status.log:3fe570aea2be3dab96b328da5a8cfcc9 ./controller/pbench-user-benchmark_38_2016-05-18_19:36:32.tar.xz
4444
--- test-report-status.log file contents

‎server/pbench/bin/pbench-verify-backup-tarballs‎

Lines changed: 37 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -46,38 +46,37 @@ backup=$BDIR
4646
controllers=$TMP/$PROG/controllers.$$
4747
files=$TMP/$PROG/files.$$
4848
allmd5s=$TMP/$PROG/allmd5s.$$
49-
listp=$TMP/$PROG/primary.$$
50-
listb=$TMP/$PROG/backup.$$
51-
tmp=$TMP/$PROG/tmp.$$
52-
ponly=$TMP/$PROG/primary.only
53-
bonly=$TMP/$PROG/backup.only
49+
list=$TMP/$PROG/list.$$
50+
out=$TMP/$PROG/out.$$
51+
only=$TMP/$PROG/only.$$
5452
report=$TMP/$PROG/report.$$
5553
index_content=$TMP/$PROG/index_mail_contents
5654

5755
# make sure the directory exists
5856
mkdir -p $TMP/$PROG
5957

60-
trap "rm -f $controllers $files $allmd5s $listp $listp.failed $listb $listb.failed $tmp $ponly $bonly $report $index_content; rmdir $TMP/$PROG" EXIT INT QUIT
58+
trap "rm -f $controllers.[pb] $files.[pb] $allmd5s.[pb] $list.[pb] $out.[pb] $list.[pb].failed $only.[pb] $report $index_content; rmdir $TMP/$PROG" EXIT INT QUIT
6159

6260
# First argument is the exit code to use when a directory does not exist.
6361
function checkmd5 {
64-
find . -maxdepth 1 -type d | grep -v '^\.$' > $controllers
62+
find . -maxdepth 1 -type d | grep -v '^\.$' | sort > $controllers.$1
6563
while read d ;do
66-
pushd $d > /dev/null || exit $1
67-
ls | grep md5 | grep -v DUPLICATE__NAME > $files
68-
> $allmd5s
64+
pushd $d > /dev/null || exit $2
65+
ls | grep md5 | grep -v DUPLICATE__NAME > $files.$1
66+
> $allmd5s.$1
6967
while read f ;do
7068
if [ -s $f ]; then
71-
cat $f >> $allmd5s
69+
cat $f >> $allmd5s.$1
7270
else
7371
echo "FAIL: Empty .md5 file: $d/$f"
7472
fi
75-
done < $files
76-
if [ -s $allmd5s ] ;then
77-
md5sum --check --warn $allmd5s | sed 's;^;'$d/';'
73+
done < $files.$1
74+
if [ -s $allmd5s.$1 ] ;then
75+
cat $allmd5s.$1 | sed -r 's;(^[0-9a-f]+) (.+);\1 '$d/'\2;' >> $list.$1
76+
md5sum --check --warn $allmd5s.$1 | sed 's;^;'$d/';'
7877
fi
7978
popd >/dev/null
80-
done < $controllers
79+
done < $controllers.$1
8180
}
8281

8382
log_init $(basename $0) $LOGSDIR/$(basename $0)
@@ -88,38 +87,40 @@ echo "start-$(timestamp)"
8887
> $index_content
8988

9089
# primary archive
91-
> $listp
90+
> $out.p
9291
if cd $primary ;then
93-
checkmd5 5 | sort > $listp
92+
checkmd5 p 5 > $out.p 2>&1 &
9493
fi
9594

9695
# backup location
97-
> $listb
96+
> $out.b
9897
if cd $backup ;then
99-
checkmd5 6 | sort > $listb
98+
checkmd5 b 6 > $out.b 2>&1 &
10099
fi
101100

101+
wait
102+
102103
let ret=0
103104

104105
# Construct the report file
105106
> $report
106107

107-
grep FAIL $listp > $listp.failed
108-
if [ -s $listp.failed ] ;then
109-
(echo "* In $primary: The calculated MD5 of the following entries failed to match the stored MD5"
110-
cat $listp.failed | sed 's;^\.;'$primary';'; echo) >> $report
111-
elif [ -s $listp -a ! -s $listp.failed ] ;then
108+
grep -E "(^md5sum: |FAIL)" $out.p > $list.p.failed
109+
if [ -s $list.p.failed ] ;then
110+
(echo "* In $primary: the calculated MD5 of the following entries failed to match the stored MD5"
111+
cat $list.p.failed | sed 's;^\.;'$primary';'; echo) >> $report
112+
elif [ -s $out.p -a ! -s $list.p.failed ] ;then
112113
:
113114
else
114115
echo "Primary list is empty - is $primary mounted?" >> $report
115116
ret=7
116117
fi
117118

118-
grep FAIL $listb > $listb.failed
119-
if [ -s $listb.failed ] ;then
120-
(echo "* In $backup: The calculated MD5 of the following entries failed to match the stored MD5"
121-
cat $listb.failed | sed 's;^\.;'$backup';'; echo) >> $report
122-
elif [ -s $listb -a ! -s $listb.failed ] ;then
119+
grep -E "(^md5sum: |FAIL)" $out.b > $list.b.failed
120+
if [ -s $list.b.failed ] ;then
121+
(echo "* In $backup: the calculated MD5 of the following entries failed to match the stored MD5"
122+
cat $list.b.failed | sed 's;^\.;'$backup';'; echo) >> $report
123+
elif [ -s $out.b -a ! -s $list.b.failed ] ;then
123124
:
124125
else
125126
echo "Backup list is empty - is $backup mounted?" >> $report
@@ -133,17 +134,17 @@ fi
133134
# will be empty.
134135

135136
if [[ $ret == 0 ]] ;then
136-
comm -13 $listp $listb > $bonly
137-
comm -23 $listp $listb > $ponly
137+
comm -23 --nocheck-order $list.p $list.b > $only.p
138+
comm -13 --nocheck-order $list.p $list.b > $only.b
138139

139-
if [ -s $ponly ] ;then
140+
if [ -s $only.p ] ;then
140141
(echo "* Files that exist only in primary directory - extra files in this list are probably OK: they just have not been backed up yet.";
141-
cat $ponly | sed 's;^\.;'$primary';') >> $report
142+
cat $only.p | sed 's;^\.;'$primary';') >> $report
142143
fi
143144

144-
if [ -s $bonly ] ;then
145-
(echo "* Files that exist only in backup directory - this should not happen.";
146-
cat $bonly | sed 's;^\.;'$backup';') >> $report
145+
if [ -s $only.b ] ;then
146+
(echo "* Files that exist only in backup directory - this should only happen if a backup or primary tar ball becomes corrupted.";
147+
cat $only.b | sed 's;^\.;'$backup';') >> $report
147148
fi
148149
fi
149150

0 commit comments

Comments
 (0)