Skip to content

Commit f446d8b

Browse files
committed
Add deploy script.
Create example config files for deployment and indexing. Delete hardwired crontabs: the deploy script takes care of that. Fix scripts for bash -x invocation.
1 parent afac248 commit f446d8b

19 files changed

+249
-202
lines changed

bgtasks/Makefile

Lines changed: 0 additions & 34 deletions
This file was deleted.

bgtasks/config/deploy.conf.example

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# this file describes the elements of deploying the pbench background tools
2+
[DEFAULT]
3+
4+
default-user=pbench
5+
default-deploy-dir=/home/pbench
6+
default-script-dir = %(default-deploy-dir)s/bin
7+
default-lib-dir = %(default-deploy-dir)s/lib
8+
default-crontab-dir = %(default-deploy-dir)s/crontab
9+
default-lock-dir = %(default-deploy-dir)s/locks
10+
deploy-script-dir = %(default-script-dir)s
11+
deploy-lib-dir = %(default-lib-dir)s
12+
deploy-crontab-dir=%(default-crontab-dir)s
13+
deploy-lock-dir = %(default-lock-dir)s
14+
15+
[deploy]
16+
roles = pbench-results pbench-backup inactive
17+
user=%(default-user)s
18+
# where to find files locally
19+
script-dir = pbench/bin
20+
lib-dir = pbench/lib
21+
# generally needed scripts
22+
scripts = pbench-base.sh
23+
24+
###########################################################################
25+
# hosts
26+
[pbench-results]
27+
host = pbench.results.example.com
28+
mailfrom = %(default-user)s@%(host)s
29+
tasks = pbench-unpack-tarballs, pbench-copy-sosreports, pbench-edit-prefixes, pbench-index,
30+
pbench-clean-up-dangling-results
31+
32+
[pbench-backup]
33+
host = pbench-backup.example.com
34+
mailfrom = %(default-user)s@%(host)s
35+
tasks = pbench-backup-tarballs
36+
37+
[inactive]
38+
host = non-participant.example.com
39+
mailfrom = %(default-user)s@%(host)s
40+
tasks =
41+
42+
###########################################################################
43+
# tasks
44+
[pbench-backup-tarballs]
45+
scripts = pbench-backup-tarballs
46+
crontab = 41 4 * * * flock -n %(deploy-lock-dir)s/pbench-backup-tarballs.lock %(deploy-script-dir)s/pbench-backup-tarballs /pbench /srv/vos/pbench/archive.backup
47+
48+
[pbench-unpack-tarballs]
49+
scripts = pbench-unpack-tarballs
50+
crontab = * * * * * flock -n %(deploy-lock-dir)s/pbench-unpack-tarballs.lock %(deploy-script-dir)s/pbench-unpack-tarballs /pbench
51+
52+
[pbench-copy-sosreports]
53+
scripts = pbench-copy-sosreports
54+
crontab = 41 * * * * flock -n %(deploy-lock-dir)s/pbench-copy-sosreports.lock %(deploy-script-dir)s/pbench-copy-sosreports /pbench
55+
56+
[pbench-edit-prefixes]
57+
scripts = pbench-edit-prefixes
58+
crontab = * * * * * flock -n %(deploy-lock-dir)s/pbench-edit-prefixes.lock %(deploy-script-dir)s/pbench-edit-prefixes /pbench
59+
60+
[pbench-index]
61+
scripts = pbench-index, index-pbench
62+
libs = config vos mappings
63+
crontab = 26 * * * * flock -n %(deploy-lock-dir)s/pbench-index.lock %(deploy-script-dir)s/pbench-index /pbench
64+
65+
[pbench-clean-up-dangling-results]
66+
scripts = pbench-clean-up-dangling-results
67+
crontab = 51 3 * * * flock -n %(deploy-lock-dir)s/pbench-clean-up-dangling-results-links.lock %(deploy-script-dir)s/pbench-clean-up-dangling-results-links /pbench

bgtasks/deploy

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
#! /bin/bash
2+
# -*- mode: shell-script -*-
3+
4+
# deploy script for pbench background tasks
5+
# depends on ./config/deploy.conf
6+
# Dependcency: configtools
7+
8+
while getopts dh x ;do
9+
case $x in
10+
d)
11+
debug=1
12+
;;
13+
h)
14+
help=1
15+
;;
16+
esac
17+
done
18+
19+
if [[ $help == 1 ]] ;then
20+
echo "Usage: deploy [-dh]"
21+
exit 0
22+
fi
23+
24+
if [[ $debug == 1 ]] ;then
25+
rsync="echo /usr/bin/rsync"
26+
ssh="echo /usr/bin/ssh"
27+
else
28+
rsync=/usr/bin/rsync
29+
ssh=/usr/bin/ssh
30+
fi
31+
32+
# override for debugging, comment out for production
33+
34+
35+
opts=$SHELLOPTS
36+
case $opts in
37+
*xtrace*)
38+
dir=$(dirname $(which $0))
39+
PROG=$(basename $(which $0))
40+
;;
41+
*)
42+
dir=$(dirname $0)
43+
PROG=$(basename $0)
44+
;;
45+
esac
46+
47+
export CONFIG=$dir/config/deploy.conf
48+
49+
if [ ! -f $CONFIG ] ;then
50+
echo "$prog: required config file $CONFIG does not exist" > /dev/stdout
51+
exit 1
52+
fi
53+
54+
PATH=/opt/configtools/bin:$PATH
55+
56+
if which getconf.py > /dev/null 2>&1 ;then
57+
:
58+
else
59+
echo "The configtools package must be installed."
60+
exit 2
61+
fi
62+
63+
TMP=/tmp/deploy-bgtasks.$$
64+
mkdir -p $TMP
65+
trap "rm -rf $TMP" EXIT INT QUIT
66+
67+
function do_scripts {
68+
role=$1
69+
user=$2
70+
host=$3
71+
shift 3
72+
scripts=$*
73+
74+
scriptdir=$dir/$(getconf.py script-dir deploy)
75+
files=""
76+
for script in $scripts ;do
77+
if [ -f $scriptdir/$script ] ;then
78+
files="$files $scriptdir/$script"
79+
fi
80+
done
81+
$rsync -avz $files $user@$host:$(getconf.py deploy-script-dir $role)
82+
}
83+
84+
# refactor
85+
function do_libs {
86+
role=$1
87+
user=$2
88+
host=$3
89+
shift 3
90+
libs=$*
91+
92+
libdir=$dir/$(getconf.py lib-dir deploy)
93+
files=""
94+
for lib in $libs ;do
95+
if [ -e $libdir/$lib ] ;then
96+
files="$files $libdir/$lib"
97+
fi
98+
done
99+
$rsync -avz $files $user@$host:$(getconf.py deploy-lib-dir $host deploy)
100+
}
101+
102+
function do_crontab {
103+
role=$1
104+
user=$2
105+
host=$3
106+
crontabs=$4
107+
108+
# make sure the directory exists
109+
$ssh $user@$host mkdir -p $(getconf.py deploy-crontab-dir $role deploy)
110+
# copy the file
111+
$rsync -avz $crontabs $user@$host:$(getconf.py deploy-crontab-dir $host deploy)/crontab
112+
# refresh cron
113+
$ssh $user@$host crontab $(getconf.py deploy-crontab-dir $role deploy)/crontab
114+
}
115+
116+
function deploy_host {
117+
role=$1
118+
host=$(getconf.py host $role)
119+
120+
user=$(getconf.py user $role deploy)
121+
tasks=$(getconf.py -l tasks $role)
122+
123+
hostscripts=""
124+
if [ ! -z "$tasks" ] ;then
125+
# general scripts
126+
hostscripts=$(getconf.py -l scripts deploy)
127+
hostscripts="$hostscripts $(getconf.py -l scripts $role)"
128+
fi
129+
# scripts and libs are concatenated and passed as arguments
130+
scripts="$hostscripts"
131+
libs=""
132+
# Don't pass crontab entries as arguments: newlines are mangled.
133+
# Use a file instead.
134+
crontabs=$TMP/crontab.$host
135+
# make sure the file starts out empty.
136+
> $crontabs
137+
for task in $tasks ;do
138+
# avoid introducing unnecessary spaces: the -z below won't work.
139+
if [ -z "$scripts" ] ;then
140+
scripts="$(getconf.py -l scripts $task)"
141+
else
142+
scripts="$scripts $(getconf.py -l scripts $task)"
143+
fi
144+
145+
if [ -z "$libs" ] ;then
146+
libs="$(getconf.py -l libs $task)"
147+
else
148+
libs="$libs $(getconf.py -l libs $task)"
149+
fi
150+
151+
getconf.py -l crontab $task >> $crontabs
152+
done
153+
154+
if [ ! -z "$scripts" ] ;then
155+
do_scripts $role $user $host $scripts
156+
fi
157+
if [ ! -z "$libs" ] ;then
158+
do_libs $role $user $host $libs
159+
fi
160+
if [ -s "$crontabs" ] ;then
161+
do_crontab $role $user $host "$crontabs"
162+
fi
163+
}
164+
165+
roles=$(getconf.py -l roles deploy)
166+
for role in $roles ;do
167+
deploy_host $role
168+
done
169+
170+
exit 0

bgtasks/pbench/bin/gold/test-1.txt

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,6 @@ pbench-index: Bad TMP=/var/tmp/pbench-test-bgtasks/pbench/tmp
1313
+++ Running pbench-clean-up-dangling-results-links
1414
pbench-clean-up-dangling-results-links: Bad TMP=/var/tmp/pbench-test-bgtasks/pbench/tmp
1515
--- Finished pbench-clean-up-dangling-results-links (status=1}
16-
+++ Running pbench-create-results-links
17-
pbench-create-results-links: Bad TMP=/var/tmp/pbench-test-bgtasks/pbench/tmp
18-
--- Finished pbench-create-results-links (status=1}
1916
+++ Running pbench-edit-prefixes
2017
pbench-edit-prefixes: Bad TMP=/var/tmp/pbench-test-bgtasks/pbench/tmp
2118
--- Finished pbench-edit-prefixes (status=1}

bgtasks/pbench/bin/gold/test-2.txt

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,6 @@ pbench-index: Bad ARCHIVE=/var/tmp/pbench-test-bgtasks/pbench/archive/fs-version
1010
--- Finished pbench-index (status=1}
1111
+++ Running pbench-clean-up-dangling-results-links
1212
--- Finished pbench-clean-up-dangling-results-links (status=1}
13-
+++ Running pbench-create-results-links
14-
--- Finished pbench-create-results-links (status=1}
1513
+++ Running pbench-edit-prefixes
1614
--- Finished pbench-edit-prefixes (status=1}
1715
+++ pbench tree state
@@ -23,9 +21,6 @@ pbench-index: Bad ARCHIVE=/var/tmp/pbench-test-bgtasks/pbench/archive/fs-version
2321
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-copy-sosreports
2422
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-copy-sosreports/pbench-copy-sosreports.error
2523
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-copy-sosreports/pbench-copy-sosreports.log
26-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links
27-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links/pbench-create-results-links.error
28-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links/pbench-create-results-links.log
2924
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-edit-prefixes
3025
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-edit-prefixes/pbench-edit-prefixes.error
3126
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-edit-prefixes/pbench-edit-prefixes.log
@@ -37,8 +32,6 @@ pbench-index: Bad ARCHIVE=/var/tmp/pbench-test-bgtasks/pbench/archive/fs-version
3732
+++ pbench log file contents
3833
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-clean-up-dangling-results-links/pbench-clean-up-dangling-results-links.log:pbench-clean-up-dangling-results-links: Bad ARCHIVE=/var/tmp/pbench-test-bgtasks/pbench/archive/fs-version-001
3934
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-copy-sosreports/pbench-copy-sosreports.log:pbench-copy-sosreports: Bad ARCHIVE=/var/tmp/pbench-test-bgtasks/pbench/archive/fs-version-001
40-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links/pbench-create-results-links.log:run-1900-01-01T00:00:00-UTC
41-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links/pbench-create-results-links.log:pbench-create-results-links: BAD ARCHIVE=/var/tmp/pbench-test-bgtasks/pbench/archive/fs-version-001
4235
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-edit-prefixes/pbench-edit-prefixes.log:pbench-edit-prefixes: Bad ARCHIVE=/var/tmp/pbench-test-bgtasks/pbench/archive/fs-version-001
4336
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-unpack-tarballs/pbench-unpack-tarballs.log:pbench-unpack-tarballs: Bad ARCHIVE=/var/tmp/pbench-test-bgtasks/pbench/archive/fs-version-001
4437
--- pbench log file contents

bgtasks/pbench/bin/gold/test-3.txt

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,6 @@ pbench-backup-tarballs: Missing target backup directory argument
99
--- Finished pbench-index (status=0}
1010
+++ Running pbench-clean-up-dangling-results-links
1111
--- Finished pbench-clean-up-dangling-results-links (status=1}
12-
+++ Running pbench-create-results-links
13-
--- Finished pbench-create-results-links (status=1}
1412
+++ Running pbench-edit-prefixes
1513
--- Finished pbench-edit-prefixes (status=1}
1614
+++ pbench tree state
@@ -24,9 +22,6 @@ pbench-backup-tarballs: Missing target backup directory argument
2422
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-copy-sosreports
2523
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-copy-sosreports/pbench-copy-sosreports.error
2624
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-copy-sosreports/pbench-copy-sosreports.log
27-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links
28-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links/pbench-create-results-links.error
29-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links/pbench-create-results-links.log
3025
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-edit-prefixes
3126
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-edit-prefixes/pbench-edit-prefixes.error
3227
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-edit-prefixes/pbench-edit-prefixes.log
@@ -41,8 +36,6 @@ pbench-backup-tarballs: Missing target backup directory argument
4136
+++ pbench log file contents
4237
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-clean-up-dangling-results-links/pbench-clean-up-dangling-results-links.log:pbench-clean-up-dangling-results-links: Bad RESULTS=/var/tmp/pbench-test-bgtasks/pbench/public_html/results
4338
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-copy-sosreports/pbench-copy-sosreports.log:pbench-copy-sosreports: Bad INCOMING=/var/tmp/pbench-test-bgtasks/pbench/public_html/incoming
44-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links/pbench-create-results-links.log:run-1900-01-01T00:00:00-UTC
45-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links/pbench-create-results-links.log:pbench-create-results-links: BAD INCOMING=/var/tmp/pbench-test-bgtasks/pbench/public_html/incoming
4639
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-edit-prefixes/pbench-edit-prefixes.log:pbench-edit-prefixes: Bad INCOMING=/var/tmp/pbench-test-bgtasks/pbench/public_html/incoming
4740
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-index/pbench-index.log:run-1900-01-01T00:00:00-UTC: starting at 1900-01-01T00:00:00-UTC
4841
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-index/pbench-index.log:run-1900-01-01T00:00:00-UTC: ending at 1900-01-01T00:00:00-UTC, indexed 0 (skipped 0) results, 0 errors

bgtasks/pbench/bin/gold/test-4.txt

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,6 @@ pbench-backup-tarballs: Missing target backup directory argument
99
--- Finished pbench-index (status=0}
1010
+++ Running pbench-clean-up-dangling-results-links
1111
--- Finished pbench-clean-up-dangling-results-links (status=1}
12-
+++ Running pbench-create-results-links
13-
--- Finished pbench-create-results-links (status=0}
1412
+++ Running pbench-edit-prefixes
1513
--- Finished pbench-edit-prefixes (status=1}
1614
+++ pbench tree state
@@ -24,9 +22,6 @@ pbench-backup-tarballs: Missing target backup directory argument
2422
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-copy-sosreports
2523
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-copy-sosreports/pbench-copy-sosreports.error
2624
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-copy-sosreports/pbench-copy-sosreports.log
27-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links
28-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links/pbench-create-results-links.error
29-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links/pbench-create-results-links.log
3025
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-edit-prefixes
3126
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-edit-prefixes/pbench-edit-prefixes.error
3227
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-edit-prefixes/pbench-edit-prefixes.log
@@ -44,8 +39,6 @@ pbench-backup-tarballs: Missing target backup directory argument
4439
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-clean-up-dangling-results-links/pbench-clean-up-dangling-results-links.log:pbench-clean-up-dangling-results-links: Bad RESULTS=/var/tmp/pbench-test-bgtasks/pbench/public_html/results
4540
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-copy-sosreports/pbench-copy-sosreports.log:run-1900-01-01T00:00:00-UTC: starting at 1900-01-01T00:00:00-UTC
4641
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-copy-sosreports/pbench-copy-sosreports.log:run-1900-01-01T00:00:00-UTC: ending at 1900-01-01T00:00:00-UTC, processed 0 sosreports for 0 results directories with 0 errors
47-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links/pbench-create-results-links.log:run-1900-01-01T00:00:00-UTC
48-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links/pbench-create-results-links.log:0 links made, 0 errors encountered
4942
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-edit-prefixes/pbench-edit-prefixes.log:pbench-edit-prefixes: Bad RESULTS=/var/tmp/pbench-test-bgtasks/pbench/public_html/results
5043
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-index/pbench-index.log:run-1900-01-01T00:00:00-UTC: starting at 1900-01-01T00:00:00-UTC
5144
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-index/pbench-index.log:run-1900-01-01T00:00:00-UTC: ending at 1900-01-01T00:00:00-UTC, indexed 0 (skipped 0) results, 0 errors

bgtasks/pbench/bin/gold/test-5.txt

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,6 @@ pbench-backup-tarballs: Missing target backup directory argument
99
--- Finished pbench-index (status=0}
1010
+++ Running pbench-clean-up-dangling-results-links
1111
--- Finished pbench-clean-up-dangling-results-links (status=0}
12-
+++ Running pbench-create-results-links
13-
--- Finished pbench-create-results-links (status=0}
1412
+++ Running pbench-edit-prefixes
1513
--- Finished pbench-edit-prefixes (status=0}
1614
+++ pbench tree state
@@ -24,9 +22,6 @@ pbench-backup-tarballs: Missing target backup directory argument
2422
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-copy-sosreports
2523
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-copy-sosreports/pbench-copy-sosreports.error
2624
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-copy-sosreports/pbench-copy-sosreports.log
27-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links
28-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links/pbench-create-results-links.error
29-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links/pbench-create-results-links.log
3025
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-edit-prefixes
3126
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-edit-prefixes/pbench-edit-prefixes.error
3227
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-edit-prefixes/pbench-edit-prefixes.log
@@ -44,8 +39,6 @@ pbench-backup-tarballs: Missing target backup directory argument
4439
+++ pbench log file contents
4540
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-copy-sosreports/pbench-copy-sosreports.log:run-1900-01-01T00:00:00-UTC: starting at 1900-01-01T00:00:00-UTC
4641
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-copy-sosreports/pbench-copy-sosreports.log:run-1900-01-01T00:00:00-UTC: ending at 1900-01-01T00:00:00-UTC, processed 0 sosreports for 0 results directories with 0 errors
47-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links/pbench-create-results-links.log:run-1900-01-01T00:00:00-UTC
48-
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-create-results-links/pbench-create-results-links.log:0 links made, 0 errors encountered
4942
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-edit-prefixes/pbench-edit-prefixes.log:run-1900-01-01T00:00:00-UTC
5043
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-edit-prefixes/pbench-edit-prefixes.log:run-1900-01-01T00:00:00-UTC: Processed 0 edit-prefix requests
5144
/var/tmp/pbench-test-bgtasks/pbench/logs/pbench-index/pbench-index.log:run-1900-01-01T00:00:00-UTC: starting at 1900-01-01T00:00:00-UTC

0 commit comments

Comments
 (0)