Skip to content

Commit 020a845

Browse files
tenstormaviportante
tenstormavi
authored and
portante
committed
Rewrite backup tarballs to support S3
This commit will do following things: 1. pbench-backup-tarballs.py: Add a new python script to backup result to s3 This script will be invoked by the pbench-backup-tarballs. After invoking it will check the TO-BACKUP state directory for new results and figure out which are the results that need to be backed up to s3. 2. pbench-verify-backup-tarballs.py: Add a new python script for results verification This script will be invoked by the pbench-verify-backup-tarballs. The task will be verifying the results in archive, backup and s3 bucket. 3. Add new configuration section for s3 4. Add new module for handling s3 configuration information 5. Fix Unit tests and dependencies
1 parent 1bfb91c commit 020a845

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

73 files changed

+3174
-685
lines changed

‎.travis.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ install:
2020
- sudo apt-get install python3.6 --force-yes -y
2121
- sudo wget -O /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py
2222
- sudo python3.6 /tmp/get-pip.py
23-
- sudo pip install 'configtools<0.4.0' elasticsearch
23+
- sudo pip install 'configtools<0.4.0' elasticsearch sh boto3
2424
- sudo ln -sf python3.6 /usr/bin/python3
2525
- sudo apt-key adv --fetch-keys http://dl.yarnpkg.com/debian/pubkey.gpg
2626
- sudo echo "deb http://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list

‎server/bin/gold/test-1.txt‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,8 @@ pbench-clean-up-dangling-results-links: Bad ARCHIVE=/var/tmp/pbench-test-server/
2828
pbench-edit-prefixes: Bad ARCHIVE=/var/tmp/pbench-test-server/test-1/pbench/archive/fs-version-001
2929
--- Finished pbench-edit-prefixes (status=1)
3030
+++ Running pbench-backup-tarballs
31-
--- Finished pbench-backup-tarballs (status=1)
31+
--- Finished pbench-backup-tarballs (status=3)
3232
+++ Running pbench-verify-backup-tarballs
33-
pbench-verify-backup-tarballs: Bad archive: /var/tmp/pbench-test-server/test-1/pbench/archive/fs-version-001
3433
--- Finished pbench-verify-backup-tarballs (status=1)
3534
+++ Running pbench-satellite-cleanup
3635
pbench-satellite-cleanup: Bad ARCHIVE=/var/tmp/pbench-test-server/test-1/pbench-satellite/archive/fs-version-001
@@ -105,10 +104,11 @@ drwxrwxr-x - public_html/users
105104
+++ pbench-local tree state (/var/tmp/pbench-test-server/test-1/pbench-local)
106105
drwxrwxr-x - logs
107106
drwxrwxr-x - logs/pbench-backup-tarballs
108-
-rw-rw-r-- 148 logs/pbench-backup-tarballs/pbench-backup-tarballs.error
109-
-rw-rw-r-- 30 logs/pbench-backup-tarballs/pbench-backup-tarballs.log
107+
-rw-rw-r-- 280 logs/pbench-backup-tarballs/pbench-backup-tarballs.log
110108
drwxrwxr-x - logs/pbench-index
111109
-rw-rw-r-- 158 logs/pbench-index/pbench-index.log
110+
drwxrwxr-x - logs/pbench-verify-backup-tarballs
111+
-rw-rw-r-- 241 logs/pbench-verify-backup-tarballs/pbench-verify-backup-tarballs.log
112112
drwxrwxr-x - pbench-move-results-receive
113113
drwxrwxr-x - pbench-move-results-receive/fs-version-001
114114
drwxrwxr-x - pbench-move-results-receive/fs-version-002
@@ -163,15 +163,15 @@ drwxrwxr-x - tmp
163163
--- pbench-satellite-local tree state
164164
+++ pbench log file contents
165165
++++ pbench-local/logs
166-
+++++ pbench-backup-tarballs/pbench-backup-tarballs.error
167-
pbench-backup-tarballs: The ARCHIVE directory does not resolve to a real location, /var/tmp/pbench-test-server/test-1/pbench/archive/fs-version-001
168-
----- pbench-backup-tarballs/pbench-backup-tarballs.error
169166
+++++ pbench-backup-tarballs/pbench-backup-tarballs.log
170-
start-1900-01-01T00:00:00-UTC
167+
1970-01-01T00:00:00.000000 ERROR pbench-backup-tarballs.pbench-backup-tarballs sanity_check -- The ARCHIVE directory /var/tmp/pbench-test-server/test-1/pbench/archive/fs-version-001, does not resolve /var/tmp/pbench-test-server/test-1/pbench/archive/fs-version-001 to a directory
171168
----- pbench-backup-tarballs/pbench-backup-tarballs.log
172169
+++++ pbench-index/pbench-index.log
173170
1970-01-01T00:00:00.000000 ERROR pbench-index.pbench-index main -- pbench-index: Bad ARCHIVE=/var/tmp/pbench-test-server/test-1/pbench/archive/fs-version-001
174171
----- pbench-index/pbench-index.log
172+
+++++ pbench-verify-backup-tarballs/pbench-verify-backup-tarballs.log
173+
1970-01-01T00:00:00.000000 ERROR pbench-verify-backup-tarballs.pbench-verify-backup-tarballs main -- The setting for ARCHIVE in the config file is /var/tmp/pbench-test-server/test-1/pbench/archive/fs-version-001, but that is not a directory
174+
----- pbench-verify-backup-tarballs/pbench-verify-backup-tarballs.log
175175
---- pbench-local/logs
176176
++++ pbench-satellite-local/logs
177177
---- pbench-satellite-local/logs

‎server/bin/gold/test-11.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ lrwxrwxrwx 70 users -> /var/tmp/pbench-test-server/test-11/pbench-satell
6161
drwxrwxr-x - archive
6262
drwxrwxr-x - archive/fs-version-001
6363
drwxrwxr-x - archive/fs-version-001/ONE::controller
64+
drwxrwxr-x - archive/fs-version-001/ONE::controller/BACKED-UP
6465
drwxrwxr-x - archive/fs-version-001/ONE::controller/BAD-MD5
6566
drwxrwxr-x - archive/fs-version-001/ONE::controller/COPIED-SOS
6667
drwxrwxr-x - archive/fs-version-001/ONE::controller/INDEXED
@@ -72,6 +73,7 @@ drwxrwxr-x - archive/fs-version-001/ONE::controller/SATELLITE-MD5-PASSE
7273
lrwxrwxrwx 113 archive/fs-version-001/ONE::controller/SATELLITE-MD5-PASSED/fio__2016-08-16_22:03:11.tar.xz -> /var/tmp/pbench-test-server/test-11/pbench/archive/fs-version-001/ONE::controller/fio__2016-08-16_22:03:11.tar.xz
7374
drwxrwxr-x - archive/fs-version-001/ONE::controller/SYNCED
7475
drwxrwxr-x - archive/fs-version-001/ONE::controller/TO-BACKUP
76+
drwxrwxr-x - archive/fs-version-001/ONE::controller/TO-BACKUP-LARGE
7577
drwxrwxr-x - archive/fs-version-001/ONE::controller/TO-COPY-SOS
7678
drwxrwxr-x - archive/fs-version-001/ONE::controller/TO-DELETE
7779
drwxrwxr-x - archive/fs-version-001/ONE::controller/TO-INDEX

‎server/bin/gold/test-16.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ lrwxrwxrwx 70 users -> /var/tmp/pbench-test-server/test-16/pbench-satell
7878
drwxrwxr-x - archive
7979
drwxrwxr-x - archive/fs-version-001
8080
drwxrwxr-x - archive/fs-version-001/controller00
81+
drwxrwxr-x - archive/fs-version-001/controller00/BACKED-UP
8182
drwxrwxr-x - archive/fs-version-001/controller00/BAD-MD5
8283
drwxrwxr-x - archive/fs-version-001/controller00/COPIED-SOS
8384
drwxrwxr-x - archive/fs-version-001/controller00/INDEXED
@@ -88,6 +89,7 @@ drwxrwxr-x - archive/fs-version-001/controller00/SATELLITE-MD5-FAILED
8889
drwxrwxr-x - archive/fs-version-001/controller00/SATELLITE-MD5-PASSED
8990
drwxrwxr-x - archive/fs-version-001/controller00/SYNCED
9091
drwxrwxr-x - archive/fs-version-001/controller00/TO-BACKUP
92+
drwxrwxr-x - archive/fs-version-001/controller00/TO-BACKUP-LARGE
9193
drwxrwxr-x - archive/fs-version-001/controller00/TO-COPY-SOS
9294
drwxrwxr-x - archive/fs-version-001/controller00/TO-DELETE
9395
drwxrwxr-x - archive/fs-version-001/controller00/TO-INDEX
@@ -101,6 +103,7 @@ drwxrwxr-x - archive/fs-version-001/controller00/WONT-INDEX
101103
-rw-rw-r-- 5920 archive/fs-version-001/controller00/benchmark-result-large_1900-01-01T00:00:00.tar.xz
102104
-rw-rw-r-- 84 archive/fs-version-001/controller00/benchmark-result-large_1900-01-01T00:00:00.tar.xz.md5
103105
drwxrwxr-x - archive/fs-version-001/controller01
106+
drwxrwxr-x - archive/fs-version-001/controller01/BACKED-UP
104107
drwxrwxr-x - archive/fs-version-001/controller01/BAD-MD5
105108
drwxrwxr-x - archive/fs-version-001/controller01/COPIED-SOS
106109
-rw-rw-r-- 3120 archive/fs-version-001/controller01/DUPLICATE__NAME.1.benchmark-result-medium_1900-01-01T00:00:00.tar.xz
@@ -113,6 +116,7 @@ drwxrwxr-x - archive/fs-version-001/controller01/SATELLITE-MD5-FAILED
113116
drwxrwxr-x - archive/fs-version-001/controller01/SATELLITE-MD5-PASSED
114117
drwxrwxr-x - archive/fs-version-001/controller01/SYNCED
115118
drwxrwxr-x - archive/fs-version-001/controller01/TO-BACKUP
119+
drwxrwxr-x - archive/fs-version-001/controller01/TO-BACKUP-LARGE
116120
drwxrwxr-x - archive/fs-version-001/controller01/TO-COPY-SOS
117121
drwxrwxr-x - archive/fs-version-001/controller01/TO-DELETE
118122
drwxrwxr-x - archive/fs-version-001/controller01/TO-INDEX
@@ -128,6 +132,7 @@ drwxrwxr-x - archive/fs-version-001/controller01/WONT-INDEX
128132
drwxrwxr-x - archive/fs-version-001/controller02
129133
drwxrwxr-x - archive/fs-version-001/controller02/.prefix
130134
-rw-rw-r-- 9 archive/fs-version-001/controller02/.prefix/benchmark-result-small_1900-01-01T00:00:00.prefix
135+
drwxrwxr-x - archive/fs-version-001/controller02/BACKED-UP
131136
drwxrwxr-x - archive/fs-version-001/controller02/BAD-MD5
132137
drwxrwxr-x - archive/fs-version-001/controller02/COPIED-SOS
133138
drwxrwxr-x - archive/fs-version-001/controller02/INDEXED
@@ -138,6 +143,7 @@ drwxrwxr-x - archive/fs-version-001/controller02/SATELLITE-MD5-FAILED
138143
drwxrwxr-x - archive/fs-version-001/controller02/SATELLITE-MD5-PASSED
139144
drwxrwxr-x - archive/fs-version-001/controller02/SYNCED
140145
drwxrwxr-x - archive/fs-version-001/controller02/TO-BACKUP
146+
drwxrwxr-x - archive/fs-version-001/controller02/TO-BACKUP-LARGE
141147
drwxrwxr-x - archive/fs-version-001/controller02/TO-COPY-SOS
142148
drwxrwxr-x - archive/fs-version-001/controller02/TO-DELETE
143149
drwxrwxr-x - archive/fs-version-001/controller02/TO-INDEX

‎server/bin/gold/test-17.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ lrwxrwxrwx 70 users -> /var/tmp/pbench-test-server/test-17/pbench-satell
7878
drwxrwxr-x - archive
7979
drwxrwxr-x - archive/fs-version-001
8080
drwxrwxr-x - archive/fs-version-001/controller00
81+
drwxrwxr-x - archive/fs-version-001/controller00/BACKED-UP
8182
drwxrwxr-x - archive/fs-version-001/controller00/BAD-MD5
8283
drwxrwxr-x - archive/fs-version-001/controller00/COPIED-SOS
8384
drwxrwxr-x - archive/fs-version-001/controller00/INDEXED
@@ -87,6 +88,7 @@ drwxrwxr-x - archive/fs-version-001/controller00/SATELLITE-MD5-FAILED
8788
drwxrwxr-x - archive/fs-version-001/controller00/SATELLITE-MD5-PASSED
8889
drwxrwxr-x - archive/fs-version-001/controller00/SYNCED
8990
drwxrwxr-x - archive/fs-version-001/controller00/TO-BACKUP
91+
drwxrwxr-x - archive/fs-version-001/controller00/TO-BACKUP-LARGE
9092
drwxrwxr-x - archive/fs-version-001/controller00/TO-COPY-SOS
9193
drwxrwxr-x - archive/fs-version-001/controller00/TO-DELETE
9294
drwxrwxr-x - archive/fs-version-001/controller00/TO-INDEX
@@ -101,6 +103,7 @@ drwxrwxr-x - archive/fs-version-001/controller00/WONT-INDEX
101103
-rw-rw-r-- 5920 archive/fs-version-001/controller00/benchmark-result-large_1900-01-01T00:00:00.tar.xz
102104
-rw-rw-r-- 84 archive/fs-version-001/controller00/benchmark-result-large_1900-01-01T00:00:00.tar.xz.md5
103105
drwxrwxr-x - archive/fs-version-001/controller01
106+
drwxrwxr-x - archive/fs-version-001/controller01/BACKED-UP
104107
drwxrwxr-x - archive/fs-version-001/controller01/BAD-MD5
105108
drwxrwxr-x - archive/fs-version-001/controller01/COPIED-SOS
106109
-rw-rw-r-- 3120 archive/fs-version-001/controller01/DUPLICATE__NAME.1.benchmark-result-medium_1900-01-01T00:00:00.tar.xz
@@ -112,6 +115,7 @@ drwxrwxr-x - archive/fs-version-001/controller01/SATELLITE-MD5-FAILED
112115
drwxrwxr-x - archive/fs-version-001/controller01/SATELLITE-MD5-PASSED
113116
drwxrwxr-x - archive/fs-version-001/controller01/SYNCED
114117
drwxrwxr-x - archive/fs-version-001/controller01/TO-BACKUP
118+
drwxrwxr-x - archive/fs-version-001/controller01/TO-BACKUP-LARGE
115119
drwxrwxr-x - archive/fs-version-001/controller01/TO-COPY-SOS
116120
drwxrwxr-x - archive/fs-version-001/controller01/TO-DELETE
117121
drwxrwxr-x - archive/fs-version-001/controller01/TO-INDEX
@@ -128,6 +132,7 @@ drwxrwxr-x - archive/fs-version-001/controller01/WONT-INDEX
128132
drwxrwxr-x - archive/fs-version-001/controller02
129133
drwxrwxr-x - archive/fs-version-001/controller02/.prefix
130134
-rw-rw-r-- 9 archive/fs-version-001/controller02/.prefix/benchmark-result-small_1900-01-01T00:00:00.prefix
135+
drwxrwxr-x - archive/fs-version-001/controller02/BACKED-UP
131136
drwxrwxr-x - archive/fs-version-001/controller02/BAD-MD5
132137
drwxrwxr-x - archive/fs-version-001/controller02/COPIED-SOS
133138
drwxrwxr-x - archive/fs-version-001/controller02/INDEXED
@@ -137,6 +142,7 @@ drwxrwxr-x - archive/fs-version-001/controller02/SATELLITE-MD5-FAILED
137142
drwxrwxr-x - archive/fs-version-001/controller02/SATELLITE-MD5-PASSED
138143
drwxrwxr-x - archive/fs-version-001/controller02/SYNCED
139144
drwxrwxr-x - archive/fs-version-001/controller02/TO-BACKUP
145+
drwxrwxr-x - archive/fs-version-001/controller02/TO-BACKUP-LARGE
140146
drwxrwxr-x - archive/fs-version-001/controller02/TO-COPY-SOS
141147
drwxrwxr-x - archive/fs-version-001/controller02/TO-DELETE
142148
drwxrwxr-x - archive/fs-version-001/controller02/TO-INDEX

‎server/bin/gold/test-19.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ lrwxrwxrwx 70 users -> /var/tmp/pbench-test-server/test-19/pbench-satell
7878
drwxrwxr-x - archive
7979
drwxrwxr-x - archive/fs-version-001
8080
drwxrwxr-x - archive/fs-version-001/controller00
81+
drwxrwxr-x - archive/fs-version-001/controller00/BACKED-UP
8182
drwxrwxr-x - archive/fs-version-001/controller00/BAD-MD5
8283
drwxrwxr-x - archive/fs-version-001/controller00/COPIED-SOS
8384
drwxrwxr-x - archive/fs-version-001/controller00/INDEXED
@@ -88,6 +89,7 @@ drwxrwxr-x - archive/fs-version-001/controller00/SATELLITE-MD5-FAILED
8889
drwxrwxr-x - archive/fs-version-001/controller00/SATELLITE-MD5-PASSED
8990
drwxrwxr-x - archive/fs-version-001/controller00/SYNCED
9091
drwxrwxr-x - archive/fs-version-001/controller00/TO-BACKUP
92+
drwxrwxr-x - archive/fs-version-001/controller00/TO-BACKUP-LARGE
9193
drwxrwxr-x - archive/fs-version-001/controller00/TO-COPY-SOS
9294
drwxrwxr-x - archive/fs-version-001/controller00/TO-DELETE
9395
drwxrwxr-x - archive/fs-version-001/controller00/TO-INDEX
@@ -100,6 +102,7 @@ drwxrwxr-x - archive/fs-version-001/controller00/WONT-INDEX
100102
-rw-rw-r-- 5920 archive/fs-version-001/controller00/benchmark-result-large_1900-01-01T00:00:00.tar.xz
101103
-rw-rw-r-- 84 archive/fs-version-001/controller00/benchmark-result-large_1900-01-01T00:00:00.tar.xz.md5
102104
drwxrwxr-x - archive/fs-version-001/controller01
105+
drwxrwxr-x - archive/fs-version-001/controller01/BACKED-UP
103106
drwxrwxr-x - archive/fs-version-001/controller01/BAD-MD5
104107
drwxrwxr-x - archive/fs-version-001/controller01/COPIED-SOS
105108
-rw-rw-r-- 3120 archive/fs-version-001/controller01/DUPLICATE__NAME.1.benchmark-result-medium_1900-01-01T00:00:00.tar.xz
@@ -112,6 +115,7 @@ drwxrwxr-x - archive/fs-version-001/controller01/SATELLITE-MD5-FAILED
112115
drwxrwxr-x - archive/fs-version-001/controller01/SATELLITE-MD5-PASSED
113116
drwxrwxr-x - archive/fs-version-001/controller01/SYNCED
114117
drwxrwxr-x - archive/fs-version-001/controller01/TO-BACKUP
118+
drwxrwxr-x - archive/fs-version-001/controller01/TO-BACKUP-LARGE
115119
drwxrwxr-x - archive/fs-version-001/controller01/TO-COPY-SOS
116120
drwxrwxr-x - archive/fs-version-001/controller01/TO-DELETE
117121
drwxrwxr-x - archive/fs-version-001/controller01/TO-INDEX
@@ -127,6 +131,7 @@ drwxrwxr-x - archive/fs-version-001/controller01/WONT-INDEX
127131
drwxrwxr-x - archive/fs-version-001/controller02
128132
drwxrwxr-x - archive/fs-version-001/controller02/.prefix
129133
-rw-rw-r-- 9 archive/fs-version-001/controller02/.prefix/benchmark-result-small_1900-01-01T00:00:00.prefix
134+
drwxrwxr-x - archive/fs-version-001/controller02/BACKED-UP
130135
drwxrwxr-x - archive/fs-version-001/controller02/BAD-MD5
131136
drwxrwxr-x - archive/fs-version-001/controller02/COPIED-SOS
132137
drwxrwxr-x - archive/fs-version-001/controller02/INDEXED
@@ -136,6 +141,7 @@ drwxrwxr-x - archive/fs-version-001/controller02/SATELLITE-MD5-FAILED
136141
drwxrwxr-x - archive/fs-version-001/controller02/SATELLITE-MD5-PASSED
137142
drwxrwxr-x - archive/fs-version-001/controller02/SYNCED
138143
drwxrwxr-x - archive/fs-version-001/controller02/TO-BACKUP
144+
drwxrwxr-x - archive/fs-version-001/controller02/TO-BACKUP-LARGE
139145
drwxrwxr-x - archive/fs-version-001/controller02/TO-COPY-SOS
140146
drwxrwxr-x - archive/fs-version-001/controller02/TO-DELETE
141147
drwxrwxr-x - archive/fs-version-001/controller02/TO-INDEX

0 commit comments

Comments
 (0)