Skip to content

Commit 8aa0f27

Browse files
ndokoschaitanyaenr
ndokos
authored and
chaitanyaenr
committed
pbench-report-status: fix it and make it findable for execution.
Fixes for pbench-report-status: - get rid of timezone in timestamp - ES did not like it. - use dots instead of dashes in the index name. Modify example config files: split into pbench-server-default.cfg and pbench-server.cfg. Add script-dir to config file. pbench-base.sh: add script-dir to PATH, so that all the scripts can execute pbench-report-status. pbench-index: when setting subj, use double quotes so the variables inside the strings will be expanded.
1 parent 32e0ea0 commit 8aa0f27

File tree

5 files changed

+133
-125
lines changed

5 files changed

+133
-125
lines changed

‎server/pbench/bin/pbench-base.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,16 @@ TOP=$(getconf.py pbench-top-dir pbench-files)
3333
BDIR=$(getconf.py pbench-backup-dir pbench-files)
3434
export LOGSDIR=$(getconf.py pbench-logs-dir pbench-files)
3535

36+
if [[ -z "$_PBENCH_SERVER_TEST" ]]; then
37+
# the real thing
38+
BINDIR=$(getconf.py script-dir pbench-server)
39+
else
40+
# running unit tests
41+
BINDIR=.
42+
fi
43+
# need this to find pbench-report-status
44+
PATH=$BINDIR:$PATH
45+
3646
ARCHIVE=${TOP}/archive/fs-version-001
3747
INOTIFY_STATE_DIR=${ARCHIVE}/inotify_state
3848
INCOMING=${TOP}/public_html/incoming

‎server/pbench/bin/pbench-index‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -206,13 +206,13 @@ if [[ $nidx -gt 0 || $nskip -gt 0 || $nerrs -gt 0 ]]; then
206206
# That solved the problem of embedded parens - in the meantime, we got rid of the
207207
# parens, so it has become moot.
208208
if [[ $nerrs > 0 && $nskip > 0 ]] ;then
209-
subj='$PROG.$TS - Indexed $nidx results, skipped $nskip results, w/ $nerrs errors'
209+
subj="$PROG.$TS - Indexed $nidx results, skipped $nskip results, w/ $nerrs errors"
210210
elif [[ $nskip > 0 ]] ;then
211-
subj='$PROG.$TS - Indexed $nidx results, skipped $nskip results'
211+
subj="$PROG.$TS - Indexed $nidx results, skipped $nskip results"
212212
elif [[ $nerrs > 0 ]] ;then
213-
subj='$PROG.$TS($PBENCH_ENV) - Indexed $nidx results, w/ $nerrs errors'
213+
subj="$PROG.$TS($PBENCH_ENV) - Indexed $nidx results, w/ $nerrs errors"
214214
else
215-
subj='$PROG.$TS($PBENCH_ENV) - Indexed $nidx results'
215+
subj="$PROG.$TS($PBENCH_ENV) - Indexed $nidx results"
216216
fi
217217
if [[ "$_PBENCH_SERVER_TEST" != 1 ]] ;then
218218
# send mail if not unit testing

‎server/pbench/bin/pbench-report-status‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ while true; do
2121
;;
2222
-t|--timestamp)
2323
timestamp="${2#run-}"
24+
timestamp="${timestamp%-UTC}"
2425
shift 2
2526
;;
2627
-T|--type)
@@ -72,7 +73,7 @@ CONFIG=/opt/pbench-server/lib/config/pbench-index.cfg
7273

7374
server=$(getconf.py server Server)
7475
datestamp=${timestamp%-*}
75-
index=$(getconf.py index_prefix Settings)-"$name"-"$datestamp"
76+
index=$(getconf.py index_prefix Settings)."$name"."$datestamp"
7677
md5=$(md5sum $file | cut -d' ' -f1)
7778

7879
# Translate newlines to spaces and translate tildes which are used as

‎server/pbench/lib/config/pbench-server-default.cfg.example‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
# Do not change anything in this file: anything in here can be overridden
2+
# by including the section/option with the correct value in pbench-server.cfg.
3+
4+
# This file describes the elements of deploying the pbench background tools.
5+
[DEFAULT]
6+
mailto = [email protected]
7+
version = 001
8+
9+
default-host = perf.example.com
10+
default-user = pbench
11+
default-group = pbench
12+
13+
default-deploy-dir = /opt/pbench-server
14+
default-script-dir = %(default-deploy-dir)s/bin
15+
default-lib-dir = %(default-deploy-dir)s/lib
16+
default-crontab-dir = %(default-lib-dir)s/crontab
17+
default-lock-dir = %(default-lib-dir)s/locks
18+
19+
deploy-script-dir = %(default-script-dir)s
20+
deploy-lib-dir = %(default-lib-dir)s
21+
deploy-crontab-dir=%(default-crontab-dir)s
22+
deploy-lock-dir = %(default-lock-dir)s
23+
24+
## See pbench-server-setup documentation for filesystem setup
25+
deploy-pbench-dir = /pbench
26+
deploy-archive-dir = %(deploy-pbench-dir)s/archive/fs-version-%(version)s
27+
deploy-unpack-dir = %(deploy-pbench-dir)s/public_html/incoming
28+
deploy-pbench-local-dir = /pbench-local
29+
deploy-pbench-logs-dir = %(deploy-pbench-local-dir)s/logs
30+
deploy-pbench-backup-dir = %(deploy-pbench-local-dir)s/archive.backup
31+
deploy-pbench-tmp-dir = %(deploy-pbench-local-dir)s/tmp
32+
33+
[pbench-files]
34+
pbench-top-dir = %(deploy-pbench-dir)s
35+
pbench-backup-dir = %(deploy-pbench-backup-dir)s
36+
pbench-logs-dir = %(deploy-pbench-logs-dir)s
37+
pbench-tmp-dir = %(deploy-pbench-tmp-dir)s
38+
pbench-unpack-dir = %(deploy-unpack-dir)s
39+
40+
###########################################################################
41+
## deployment section
42+
###########################################################################
43+
[pbench-server]
44+
user=%(default-user)s
45+
group=%(default-group)s
46+
install-dir = %(default-deploy-dir)s
47+
script-dir = %(deploy-script-dir)s
48+
crontabdir = %(deploy-crontab-dir)s
49+
roles = pbench-results pbench-backup
50+
# when a tarball comes in to this server, what processing should it undergo?
51+
# production-like servers do at least the first two and possibly more of these:
52+
#dispatch-states = TO-UNPACK, TO-INDEX, TO-COPY-SOS, TO-BACKUP
53+
54+
# satellite servers do these:
55+
#dispatch-states = TO-UNPACK, TO-SYNC
56+
57+
###########################################################################
58+
# we need to install some stuff in the apache document root
59+
# so we either get it directly or look in the config file.
60+
# N.B. Different distros use different config files.
61+
# The following works on Fedora, RHEL, CentOS.
62+
[apache]
63+
# configfile = /etc/httpd/conf/httpd.conf
64+
documentroot = g/var/www/html
65+
66+
# this *has* to agree with the setting in the pbench-agent config file
67+
[results]
68+
webserver = pbench.example.com
69+
host_info_url = http://%(webserver)s/pbench-results-host-info.versioned/pbench-results-host-info.URL%(version)s
70+
71+
###########################################################################
72+
# crontab roles
73+
[pbench-results]
74+
host = %(default-host)s
75+
mailfrom = %(default-user)s@%(host)s
76+
tasks = pbench-dispatch, pbench-unpack-tarballs, pbench-copy-sosreports, pbench-edit-prefixes, pbench-index
77+
78+
[pbench-backup]
79+
host = %(default-host)s
80+
mailfrom = %(default-user)s@%(host)s
81+
tasks = pbench-backup-tarballs, pbench-verify-backup-tarballs
82+
83+
###########################################################################
84+
# crontab tasks
85+
[pbench-dispatch]
86+
crontab = * * * * * flock -n %(deploy-lock-dir)s/pbench-dispatch.lock %(deploy-script-dir)s/pbench-dispatch
87+
88+
[pbench-backup-tarballs]
89+
crontab = 41 4 * * * flock -n %(deploy-lock-dir)s/pbench-backup-tarballs.lock %(deploy-script-dir)s/pbench-backup-tarballs %(deploy-pbench-backup-dir)s
90+
91+
[pbench-verify-backup-tarballs]
92+
crontab = 59 4 * * Sunday flock -n %(deploy-lock-dir)s/pbench-verify-backup-tarballs.lock %(deploy-script-dir)s/pbench-verify-backup-tarballs %(deploy-archive-dir)s %(deploy-pbench-backup-dir)s
93+
94+
[pbench-unpack-tarballs]
95+
crontab = * * * * * flock -n %(deploy-lock-dir)s/pbench-unpack-tarballs.lock %(deploy-script-dir)s/pbench-unpack-tarballs %(deploy-unpack-dir)s
96+
97+
[pbench-move-unpacked]
98+
njobs = 1
99+
crontab = * * * * * flock -n %(deploy-lock-dir)s/pbench-move-unpacked.lock %(deploy-script-dir)s/pbench-move-unpacked %(deploy-unpack-dir)s
100+
101+
[pbench-copy-sosreports]
102+
crontab = 41 * * * * flock -n %(deploy-lock-dir)s/pbench-copy-sosreports.lock %(deploy-script-dir)s/pbench-copy-sosreports
103+
104+
[pbench-edit-prefixes]
105+
crontab = * * * * * flock -n %(deploy-lock-dir)s/pbench-edit-prefixes.lock %(deploy-script-dir)s/pbench-edit-prefixes
106+
107+
[pbench-index]
108+
crontab = * * * * * flock -n %(deploy-lock-dir)s/pbench-index.lock %(deploy-script-dir)s/pbench-index

‎server/pbench/lib/config/pbench-server.cfg.example‎

Lines changed: 9 additions & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -1,134 +1,23 @@
1-
# this file describes the elements of deploying the pbench background tools
1+
# This file describes the elements of deploying the pbench background tools.
22
[DEFAULT]
3-
# CHANGE ME!
3+
# cvalues here override those in pbench-server-default.cfg.
4+
# An installation will want to override at least the following:
45
mailto = [email protected]
5-
version = 001
6-
7-
# CHANGE ME!
86
default-host = pbench.example.com
9-
default-user = pbench
10-
default-group = pbench
11-
default-deploy-dir = /opt/pbench-server
12-
13-
default-script-dir = %(default-deploy-dir)s/bin
14-
default-lib-dir = %(default-deploy-dir)s/lib
15-
default-crontab-dir = %(default-lib-dir)s/crontab
16-
default-lock-dir = %(default-lib-dir)s/locks
17-
18-
deploy-script-dir = %(default-script-dir)s
19-
deploy-lib-dir = %(default-lib-dir)s
20-
deploy-crontab-dir=%(default-crontab-dir)s
21-
deploy-lock-dir = %(default-lock-dir)s
227

23-
## See pbench server setup documentation for filesystem setup
24-
# TOP and ARCHIVE are on pbench volume
25-
deploy-pbench-dir = /pbench
26-
deploy-archive-dir = %(deploy-pbench-dir)s/archive/fs-version-%(version)s
27-
# CHANGE ME!
28-
# LOGSDIR is local - XXX TBD
29-
deploy-pbench-logs-dir = /srv/tmp/logs
30-
# CHANGE ME!
31-
# BDIR may be remote
8+
# Backup dir may be remote
329
deploy-backup-host = pbench-backup.example.com
3310
deploy-pbench-backup-dir = %(default-user)s@%(deploy-backup-host)s:/pbench-local/archive.backup
34-
# CHANGE ME!
35-
# TMP is local
36-
deploy-pbench-tmp-dir = /srv/tmp
37-
38-
[pbench-files]
39-
pbench-top-dir = %(deploy-pbench-dir)s
40-
pbench-backup-dir = %(deploy-pbench-backup-dir)s
41-
pbench-logs-dir = %(deploy-pbench-logs-dir)s
42-
pbench-tmp-dir = %(deploy-pbench-tmp-dir)s
43-
44-
###########################################################################
45-
## runtime section
46-
###########################################################################
47-
[sosreport]
48-
user = %(default-user)s
49-
# CHANGE ME!
50-
host = pbench-sosreports.example.com
51-
dir = /path/to/sosreport/dir
52-
# CHANGE ME!
53-
[email protected]
5411

5512
###########################################################################
5613
## deployment section
5714
###########################################################################
5815
[pbench-server]
59-
user = %(default-user)s
60-
group = %(default-group)s
16+
# we now index status into ES, but in some cases we might still want to use mail.
6117
use-mail-for-status = yes
62-
install-dir = %(default-deploy-dir)s
63-
# CHANGE ME!
6418
roles = pbench-results, pbench-backup
65-
# roles = pbench-results, pbench-backup, pbench-rsync-satellites
66-
67-
# we need to install some stuff in the apache document root
68-
# so we either get it directly or look in the config file.
69-
# N.B. Different distros use different config files.
70-
# The following works on Fedora, RHEL, CentOS.
71-
[apache]
72-
# configfile = /etc/httpd/conf/httpd.conf
73-
documentroot = /var/www/html
74-
75-
# this *has* to agree with the setting in the pbench-agent config file
76-
[results]
77-
78-
# CHANGE ME!
79-
webserver = pbench.example.com
80-
host_info_url = http://%(webserver)s/pbench-results-host-info.versioned/pbench-results-host-info.URL%(version)s
81-
82-
###########################################################################
83-
# crontab roles
84-
[pbench-results]
85-
host = %(default-host)s
86-
mailfrom = %(default-user)s@%(host)s
87-
# CHANGE ME!
88-
tasks = pbench-unpack-tarballs, pbench-copy-sosreports, pbench-edit-prefixes, pbench-index
89-
90-
[pbench-backup]
91-
host = %(default-host)s
92-
mailfrom = %(default-user)s@%(host)s
93-
tasks = pbench-backup-tarballs, pbench-verify-backup-tarballs
94-
95-
[pbench-rsync-satellites]
96-
# tasks = pbench-rsync-satellite, pbench-age-out-satellite
97-
tasks = pbench-rsync-satellite
98-
satellites = EC2
99-
100-
# the crontab making script will loop over satellites, creating rsync and age-out entries for each.
101-
# satellites
102-
[EC2]
103-
prefix = EC2
104-
satellite-host = host.ec2.example.com
105-
satellite-archive = /path/to/pbench/archive/fs-version-001
106-
107-
###########################################################################
108-
# crontab tasks
109-
[pbench-backup-tarballs]
110-
crontab = 41 4 * * * flock -n %(deploy-lock-dir)s/pbench-backup-tarballs.lock %(deploy-script-dir)s/pbench-backup-tarballs
111-
112-
[pbench-verify-backup-tarballs]
113-
crontab = 59 4 * * Sunday flock -n %(deploy-lock-dir)s/pbench-verify-backup-tarballs.lock %(deploy-script-dir)s/pbench-verify-backup-tarballs %(deploy-pbench-backup-dir)s
114-
115-
[pbench-unpack-tarballs]
116-
crontab = * * * * * flock -n %(deploy-lock-dir)s/pbench-unpack-tarballs.lock %(deploy-script-dir)s/pbench-unpack-tarballs
117-
118-
[pbench-copy-sosreports]
119-
crontab = 41 * * * * flock -n %(deploy-lock-dir)s/pbench-copy-sosreports.lock %(deploy-script-dir)s/pbench-copy-sosreports
120-
121-
[pbench-edit-prefixes]
122-
crontab = * * * * * flock -n %(deploy-lock-dir)s/pbench-edit-prefixes.lock %(deploy-script-dir)s/pbench-edit-prefixes
123-
124-
[pbench-index]
125-
crontab = 26 * * * * flock -n %(deploy-lock-dir)s/pbench-index.lock %(deploy-script-dir)s/pbench-index
126-
127-
[pbench-clean-up-dangling-results]
128-
crontab = 51 3 * * * flock -n %(deploy-lock-dir)s/pbench-clean-up-dangling-results-links.lock %(deploy-script-dir)s/pbench-clean-up-dangling-results-links
129-
130-
[pbench-rsync-satellite]
131-
crontab = 51 23 * * * flock -n %(deploy-lock-dir)s/pbench-rsync-satellite.lock.$PREFIX %(deploy-script-dir)s/pbench-rsync-satellite $PREFIX $SATELLITE_HOST $SATELLITE_ARCHIVE
13219

133-
[pbench-age-out-satellite]
134-
crontab = 51 1 * * 0 flock -n %(deploy-lock-dir)s/pbench-age-out-satellite.lock.$PREFIX %(deploy-script-dir)s/pbench-age-out-satellite $PREFIX $SATELLITE_HOST $SATELLITE_ARCHIVE
20+
# the rest will come from the default config file.
21+
[config]
22+
paths = /opt/pbench-server/lib/config
23+
files = pbench-server-default.cfg

0 commit comments

Comments
 (0)