Skip to content

Commit 1e7d6ff

Browse files
portanteportante
committed
Stabilize test output for unpack tar balls
The new unpack tar balls algorithm no longer sorts by size, but by modification time from newest to oldest. We update the code comments to reflect the current behavior (which was using "change time") and fix the code to actually use "modification time" (the `%Tk` `find` printf format).
1 parent 285c98c commit 1e7d6ff

File tree

5 files changed

+50
-26
lines changed

5 files changed

+50
-26
lines changed

‎server/bin/gold/test-16.txt‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -287,13 +287,13 @@ run-1970-01-01T00:00:00-UTC: symlink target for /var/tmp/pbench-test-server/test
287287
----- pbench-unpack-tarballs-small/pbench-unpack-tarballs-small.error
288288
+++++ pbench-unpack-tarballs-small/pbench-unpack-tarballs-small.log
289289
run-1970-01-01T00:00:00-UTC
290-
ln -s /var/tmp/pbench-test-server/test-16/pbench/public_html/incoming/controller02/benchmark-result-small_1900-01-01T00:00:00 /var/tmp/pbench-test-server/test-16/pbench/public_html/results/controller02/prefix02/benchmark-result-small_1900-01-01T00:00:00
291-
run-1970-01-01T00:00:00-UTC: controller02/benchmark-result-small_1900-01-01T00:00:00: success - elapsed time (secs): 0 - size (bytes): 848
292-
ln -s /var/tmp/pbench-test-server/test-16/pbench/public_html/incoming/controller00/benchmark-result-large_1900-01-01T00:00:00 /var/tmp/pbench-test-server/test-16/pbench/public_html/results/controller00/benchmark-result-large_1900-01-01T00:00:00
293-
run-1970-01-01T00:00:00-UTC: controller00/benchmark-result-large_1900-01-01T00:00:00: success - elapsed time (secs): 0 - size (bytes): 5920
294290
ln -s /var/tmp/pbench-test-server/test-16/pbench/public_html/incoming/controller01/benchmark-result-medium_1900-01-01T00:00:00 /var/tmp/pbench-test-server/test-16/pbench/public_html/results/controller01/prefix01/benchmark-result-medium_1900-01-01T00:00:00
295291
ln -s /var/tmp/pbench-test-server/test-16/pbench/public_html/incoming/controller01/benchmark-result-medium_1900-01-01T00:00:00 /var/tmp/pbench-test-server/test-16/pbench/public_html/users/user01/controller01/prefix01/benchmark-result-medium_1900-01-01T00:00:00
296292
run-1970-01-01T00:00:00-UTC: controller01/benchmark-result-medium_1900-01-01T00:00:00: success - elapsed time (secs): 0 - size (bytes): 3180
293+
ln -s /var/tmp/pbench-test-server/test-16/pbench/public_html/incoming/controller00/benchmark-result-large_1900-01-01T00:00:00 /var/tmp/pbench-test-server/test-16/pbench/public_html/results/controller00/benchmark-result-large_1900-01-01T00:00:00
294+
run-1970-01-01T00:00:00-UTC: controller00/benchmark-result-large_1900-01-01T00:00:00: success - elapsed time (secs): 0 - size (bytes): 5920
295+
ln -s /var/tmp/pbench-test-server/test-16/pbench/public_html/incoming/controller02/benchmark-result-small_1900-01-01T00:00:00 /var/tmp/pbench-test-server/test-16/pbench/public_html/results/controller02/prefix02/benchmark-result-small_1900-01-01T00:00:00
296+
run-1970-01-01T00:00:00-UTC: controller02/benchmark-result-small_1900-01-01T00:00:00: success - elapsed time (secs): 0 - size (bytes): 848
297297
run-1970-01-01T00:00:00-UTC: Processed 3 tarballs
298298
1970-01-01T00:00:00.000000 DEBUG pbench-unpack-tarballs-small.indexer update_templates -- done templates (start ts: 1970-01-01T00:00:00-UTC, end ts: 1970-01-01T00:00:00-UTC, duration: 0.00s, successes: 1, retries: 0)
299299
1970-01-01T00:00:00.000000 DEBUG pbench-unpack-tarballs-small.report post_status -- posted status (start ts: 1970-01-01T00:00:00-UTC, end ts: 1970-01-01T00:00:00-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0)

‎server/bin/gold/test-17.txt‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -288,13 +288,13 @@ run-1970-01-01T00:00:00-UTC: symlink target for /var/tmp/pbench-test-server/test
288288
----- pbench-unpack-tarballs-small/pbench-unpack-tarballs-small.error
289289
+++++ pbench-unpack-tarballs-small/pbench-unpack-tarballs-small.log
290290
run-1970-01-01T00:00:00-UTC
291-
ln -s /var/tmp/pbench-test-server/test-17/pbench/public_html/incoming/controller02/benchmark-result-small_1900-01-01T00:00:00 /var/tmp/pbench-test-server/test-17/pbench/public_html/results/controller02/prefix02/benchmark-result-small_1900-01-01T00:00:00
292-
run-1970-01-01T00:00:00-UTC: controller02/benchmark-result-small_1900-01-01T00:00:00: success - elapsed time (secs): 0 - size (bytes): 848
293-
ln -s /var/tmp/pbench-test-server/test-17/pbench/public_html/incoming/controller00/benchmark-result-large_1900-01-01T00:00:00 /var/tmp/pbench-test-server/test-17/pbench/public_html/results/controller00/benchmark-result-large_1900-01-01T00:00:00
294-
run-1970-01-01T00:00:00-UTC: controller00/benchmark-result-large_1900-01-01T00:00:00: success - elapsed time (secs): 0 - size (bytes): 5920
295291
ln -s /var/tmp/pbench-test-server/test-17/pbench/public_html/incoming/controller01/benchmark-result-medium_1900-01-01T00:00:00 /var/tmp/pbench-test-server/test-17/pbench/public_html/results/controller01/prefix01/benchmark-result-medium_1900-01-01T00:00:00
296292
ln -s /var/tmp/pbench-test-server/test-17/pbench/public_html/incoming/controller01/benchmark-result-medium_1900-01-01T00:00:00 /var/tmp/pbench-test-server/test-17/pbench/public_html/users/user01/controller01/prefix01/benchmark-result-medium_1900-01-01T00:00:00
297293
run-1970-01-01T00:00:00-UTC: controller01/benchmark-result-medium_1900-01-01T00:00:00: success - elapsed time (secs): 0 - size (bytes): 3180
294+
ln -s /var/tmp/pbench-test-server/test-17/pbench/public_html/incoming/controller00/benchmark-result-large_1900-01-01T00:00:00 /var/tmp/pbench-test-server/test-17/pbench/public_html/results/controller00/benchmark-result-large_1900-01-01T00:00:00
295+
run-1970-01-01T00:00:00-UTC: controller00/benchmark-result-large_1900-01-01T00:00:00: success - elapsed time (secs): 0 - size (bytes): 5920
296+
ln -s /var/tmp/pbench-test-server/test-17/pbench/public_html/incoming/controller02/benchmark-result-small_1900-01-01T00:00:00 /var/tmp/pbench-test-server/test-17/pbench/public_html/results/controller02/prefix02/benchmark-result-small_1900-01-01T00:00:00
297+
run-1970-01-01T00:00:00-UTC: controller02/benchmark-result-small_1900-01-01T00:00:00: success - elapsed time (secs): 0 - size (bytes): 848
298298
run-1970-01-01T00:00:00-UTC: Processed 3 tarballs
299299
1970-01-01T00:00:00.000000 DEBUG pbench-unpack-tarballs-small.indexer update_templates -- done templates (start ts: 1970-01-01T00:00:00-UTC, end ts: 1970-01-01T00:00:00-UTC, duration: 0.00s, successes: 1, retries: 0)
300300
1970-01-01T00:00:00.000000 DEBUG pbench-unpack-tarballs-small.report post_status -- posted status (start ts: 1970-01-01T00:00:00-UTC, end ts: 1970-01-01T00:00:00-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0)

‎server/bin/gold/test-5.2.txt‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1015,20 +1015,20 @@ run-1970-01-01T00:00:00-UTC: Total 3 files processed, with 0 md5 failures and 0
10151015
----- pbench-unpack-tarballs-small/pbench-unpack-tarballs-small.error
10161016
+++++ pbench-unpack-tarballs-small/pbench-unpack-tarballs-small.log
10171017
run-1970-01-01T00:00:00-UTC
1018-
ln -s /var/tmp/pbench-test-server/test-5.2/pbench/public_html/incoming/ONE::controllerC/tarball-simple0-prefix_YYYY-MM-DDTHH-mm-ss /var/tmp/pbench-test-server/test-5.2/pbench/public_html/results/ONE::controllerC/prefix0/prefix1/tarball-simple0-prefix_YYYY-MM-DDTHH-mm-ss
1019-
run-1970-01-01T00:00:00-UTC: ONE::controllerC/tarball-simple0-prefix_YYYY-MM-DDTHH-mm-ss: success - elapsed time (secs): 0 - size (bytes): 228
1020-
ln -s /var/tmp/pbench-test-server/test-5.2/pbench/public_html/incoming/ONE::controllerB/tarball-simple2_YYYY-MM-DDTHH-mm-ss /var/tmp/pbench-test-server/test-5.2/pbench/public_html/results/ONE::controllerB/tarball-simple2_YYYY-MM-DDTHH-mm-ss
1021-
run-1970-01-01T00:00:00-UTC: ONE::controllerB/tarball-simple2_YYYY-MM-DDTHH-mm-ss: success - elapsed time (secs): 0 - size (bytes): 220
1022-
ln -s /var/tmp/pbench-test-server/test-5.2/pbench/public_html/incoming/ONE::controllerA/tarball-simple1_YYYY-MM-DDTHH-mm-ss /var/tmp/pbench-test-server/test-5.2/pbench/public_html/results/ONE::controllerA/tarball-simple1_YYYY-MM-DDTHH-mm-ss
1023-
run-1970-01-01T00:00:00-UTC: ONE::controllerA/tarball-simple1_YYYY-MM-DDTHH-mm-ss: success - elapsed time (secs): 0 - size (bytes): 224
1024-
ln -s /var/tmp/pbench-test-server/test-5.2/pbench/public_html/incoming/controller-g-normal/tarball-normal_YYYY.MM.DDTHH.MM.SS /var/tmp/pbench-test-server/test-5.2/pbench/public_html/results/controller-g-normal/tarball-normal_YYYY.MM.DDTHH.MM.SS
1025-
run-1970-01-01T00:00:00-UTC: controller-g-normal/tarball-normal_YYYY.MM.DDTHH.MM.SS: success - elapsed time (secs): 0 - size (bytes): 216
1018+
ln -s /var/tmp/pbench-test-server/test-5.2/pbench/public_html/incoming/controller-b-with-prefixes/tarball-0_YYYY.MM.DDTHH.MM.SS /var/tmp/pbench-test-server/test-5.2/pbench/public_html/results/controller-b-with-prefixes/tarball-0_YYYY.MM.DDTHH.MM.SS
1019+
run-1970-01-01T00:00:00-UTC: controller-b-with-prefixes/tarball-0_YYYY.MM.DDTHH.MM.SS: success - elapsed time (secs): 0 - size (bytes): 212
10261020
ln -s /var/tmp/pbench-test-server/test-5.2/pbench/public_html/incoming/controller-b-with-prefixes/tarball-w-prefix-dot_YYYY.MM.DDTHH.MM.SS /var/tmp/pbench-test-server/test-5.2/pbench/public_html/results/controller-b-with-prefixes/tarball-w-prefix-dot_YYYY.MM.DDTHH.MM.SS
10271021
run-1970-01-01T00:00:00-UTC: controller-b-with-prefixes/tarball-w-prefix-dot_YYYY.MM.DDTHH.MM.SS: success - elapsed time (secs): 0 - size (bytes): 224
10281022
ln -s /var/tmp/pbench-test-server/test-5.2/pbench/public_html/incoming/controller-b-with-prefixes/tarball-w-dot-prefix_YYYY.MM.DDTHH.MM.SS /var/tmp/pbench-test-server/test-5.2/pbench/public_html/results/controller-b-with-prefixes/tarball-w-dot-prefix_YYYY.MM.DDTHH.MM.SS
10291023
run-1970-01-01T00:00:00-UTC: controller-b-with-prefixes/tarball-w-dot-prefix_YYYY.MM.DDTHH.MM.SS: success - elapsed time (secs): 0 - size (bytes): 224
1030-
ln -s /var/tmp/pbench-test-server/test-5.2/pbench/public_html/incoming/controller-b-with-prefixes/tarball-0_YYYY.MM.DDTHH.MM.SS /var/tmp/pbench-test-server/test-5.2/pbench/public_html/results/controller-b-with-prefixes/tarball-0_YYYY.MM.DDTHH.MM.SS
1031-
run-1970-01-01T00:00:00-UTC: controller-b-with-prefixes/tarball-0_YYYY.MM.DDTHH.MM.SS: success - elapsed time (secs): 0 - size (bytes): 212
1024+
ln -s /var/tmp/pbench-test-server/test-5.2/pbench/public_html/incoming/controller-g-normal/tarball-normal_YYYY.MM.DDTHH.MM.SS /var/tmp/pbench-test-server/test-5.2/pbench/public_html/results/controller-g-normal/tarball-normal_YYYY.MM.DDTHH.MM.SS
1025+
run-1970-01-01T00:00:00-UTC: controller-g-normal/tarball-normal_YYYY.MM.DDTHH.MM.SS: success - elapsed time (secs): 0 - size (bytes): 216
1026+
ln -s /var/tmp/pbench-test-server/test-5.2/pbench/public_html/incoming/ONE::controllerA/tarball-simple1_YYYY-MM-DDTHH-mm-ss /var/tmp/pbench-test-server/test-5.2/pbench/public_html/results/ONE::controllerA/tarball-simple1_YYYY-MM-DDTHH-mm-ss
1027+
run-1970-01-01T00:00:00-UTC: ONE::controllerA/tarball-simple1_YYYY-MM-DDTHH-mm-ss: success - elapsed time (secs): 0 - size (bytes): 224
1028+
ln -s /var/tmp/pbench-test-server/test-5.2/pbench/public_html/incoming/ONE::controllerC/tarball-simple0-prefix_YYYY-MM-DDTHH-mm-ss /var/tmp/pbench-test-server/test-5.2/pbench/public_html/results/ONE::controllerC/prefix0/prefix1/tarball-simple0-prefix_YYYY-MM-DDTHH-mm-ss
1029+
run-1970-01-01T00:00:00-UTC: ONE::controllerC/tarball-simple0-prefix_YYYY-MM-DDTHH-mm-ss: success - elapsed time (secs): 0 - size (bytes): 228
1030+
ln -s /var/tmp/pbench-test-server/test-5.2/pbench/public_html/incoming/ONE::controllerB/tarball-simple2_YYYY-MM-DDTHH-mm-ss /var/tmp/pbench-test-server/test-5.2/pbench/public_html/results/ONE::controllerB/tarball-simple2_YYYY-MM-DDTHH-mm-ss
1031+
run-1970-01-01T00:00:00-UTC: ONE::controllerB/tarball-simple2_YYYY-MM-DDTHH-mm-ss: success - elapsed time (secs): 0 - size (bytes): 220
10321032
run-1970-01-01T00:00:00-UTC: Processed 7 tarballs
10331033
1970-01-01T00:00:00.000000 DEBUG pbench-unpack-tarballs-small.indexer update_templates -- done templates (start ts: 1970-01-01T00:00:00-UTC, end ts: 1970-01-01T00:00:00-UTC, duration: 0.00s, successes: 1, retries: 0)
10341034
1970-01-01T00:00:00.000000 DEBUG pbench-unpack-tarballs-small.report post_status -- posted status (start ts: 1970-01-01T00:00:00-UTC, end ts: 1970-01-01T00:00:00-UTC, duration: 0.00s, successes: 1, duplicates: 0, failures: 0, retries: 0)

‎server/bin/pbench-unpack-tarballs.sh‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -98,27 +98,28 @@ mail_content=${tmp}/mail_content.log
9898
index_content=${tmp}/index_mail_contents
9999
> ${index_content}
100100

101-
# get the list of files we'll be operating on - sort them by size
101+
# The list of files we'll be operating on, when complete, sorted newest to
102+
# oldest by last modification time.
102103
list=${tmp}/${PROG}.list
103104

104105
function gen_work_list() {
105106
SECONDS=0
106107
# Find all the links in all the ${ARCHIVE}/<controller>/${linksrc}
107-
# directories, emitting a list of their full paths with the size in bytes
108-
# of the file the link points to, and then sort them so that we process
109-
# the smallest tar balls first.
108+
# directories, emitting a list of their modification times, size in bytes,
109+
# and full paths of the file the link points to, and then sort them so
110+
# that we process the tar balls from newest to oldest.
110111
rm -f ${list}
111112
> ${list}.unsorted
112113
# First we find all the ${linksrc} directories
113114
for linksrc_dir in $(find ${ARCHIVE}/ -maxdepth 2 -type d -name ${linksrc}); do
114115
# Find all the links in a given ${linksrc} directory that are links to
115116
# actual files (bad links are not emitted!). For now, if it's a
116117
# duplicate name, just punt and avoid producing an error.
117-
find -L ${linksrc_dir} -type f -name '*.tar.xz' ! -name 'DUPLICATE__NAME*' ${lb_arg} ${ub_arg} -printf "%CY-%Cm-%CdT%CT %s %p\n" 2>/dev/null >> ${list}.unsorted
118+
find -L ${linksrc_dir} -type f -name '*.tar.xz' ! -name 'DUPLICATE__NAME*' ${lb_arg} ${ub_arg} -printf "%TY-%Tm-%TdT%TT %s %p\n" 2>/dev/null >> ${list}.unsorted
118119
if [[ ${lowerbound} == 0 ]]; then
119120
# Find all the links in the same ${linksrc} directory that don't
120121
# link to anything so that we can count them as errors below.
121-
find -L $linksrc_dir -type l -name '*.tar.xz' ! -name 'DUPLICATE__NAME*' -printf "%CY-%Cm-%CdT%CT %s %p\n" 2>/dev/null >> ${list}.unsorted
122+
find -L $linksrc_dir -type l -name '*.tar.xz' ! -name 'DUPLICATE__NAME*' -printf "%TY-%Tm-%TdT%TT %s %p\n" 2>/dev/null >> ${list}.unsorted
122123
fi
123124
done
124125
sort -k 1 -r ${list}.unsorted > ${list}
@@ -134,7 +135,7 @@ function move_symlink() {
134135
local linksrc="${3}"
135136
local linkdest="${4}"
136137
mv ${ARCHIVE}/${hostname}/${linksrc}/${resultname}.tar.xz ${ARCHIVE}/${hostname}/${linkdest}/${resultname}.tar.xz
137-
status=${?}
138+
local status=${?}
138139
if [[ ${status} -ne 0 ]]; then
139140
log_error "${TS}: Cannot move symlink ${ARCHIVE}/${hostname}/${resultname}.tar.xz from ${linksrc} to ${linkdest}: code ${status}" "${mail_content}"
140141
fi
@@ -143,6 +144,7 @@ function move_symlink() {
143144

144145
function do_work() {
145146
SECONDS=0
147+
local status=0
146148
local max_seconds=${1}
147149
while read date size result; do
148150
ntotal=${ntotal}+1

‎server/bin/state/test-5.2.setup‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎server/bin/state/test-5.2.setup‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/bin/bash
2+
3+
mkdir pbench-local/archive.backup || exit ${?}
4+
mkdir -p pbench-local/s3-backup/testbucket || exit ${?}
5+
6+
# Make sure all the tar balls are ordered by modification time so we
7+
# have consistent and expected results from the unit tests.
8+
9+
function _dotouch() {
10+
# CCYYMMDDhhmm.ss
11+
touch -t 201901011242.${1} ${2}
12+
return ${?}
13+
}
14+
15+
_dotouch 01 pbench-satellite/archive/fs-version-001/controllerB/tarball-simple2_YYYY-MM-DDTHH-mm-ss.tar.xz || exit ${?}
16+
_dotouch 02 pbench-satellite/archive/fs-version-001/controllerC/tarball-simple0-prefix_YYYY-MM-DDTHH-mm-ss.tar.xz || exit ${?}
17+
_dotouch 03 pbench-satellite/archive/fs-version-001/controllerA/tarball-simple1_YYYY-MM-DDTHH-mm-ss.tar.xz || exit ${?}
18+
_dotouch 04 pbench-local/pbench-move-results-receive/fs-version-002/controller-g-normal/tarball-normal_YYYY.MM.DDTHH.MM.SS.tar.xz || exit ${?}
19+
_dotouch 05 pbench-local/pbench-move-results-receive/fs-version-002/controller-b-with-prefixes/tarball-w-dot-prefix_YYYY.MM.DDTHH.MM.SS.tar.xz || exit ${?}
20+
_dotouch 06 pbench-local/pbench-move-results-receive/fs-version-002/controller-b-with-prefixes/tarball-w-prefix-dot_YYYY.MM.DDTHH.MM.SS.tar.xz || exit ${?}
21+
_dotouch 07 pbench-local/pbench-move-results-receive/fs-version-002/controller-b-with-prefixes/tarball-0_YYYY.MM.DDTHH.MM.SS.tar.xz || exit ${?}
22+
23+
exit ${?}

0 commit comments

Comments
 (0)