Add server maint role, drop tool indexing

We add the notion of a maintenance role for the audit service, and
drop the tool data indexing by default.

We add commented out `pbench-server-backup` and `Indexing` sections
to the default server configuration file with a comment that they
are deliberately omitted from the defaults.  This avoids making any
code changes to accommodate always having a default.

We drop the default "vos" user for sosreports and use the default user.
This let's us remove a few unnecessary config files and their symlinks
from the unit tests.

We also take this opportunity to make sure all the configuration
files in the tree a properly sync'd with each other.

Author: portante

server/ansible/Inventory/pbench-server.hosts.example

@@ -18,6 +18,4 @@
 pbench_config_url = http://pbench.example.com/server/config/{{ cenv }}
 
 # list of config files to fetch
-pbench_config_files = '["pbench-server.cfg", "pbench-server-backup.cfg"]'
-
-
+pbench_config_files = '["pbench-server.cfg"]'

server/bin/gold/test-24.txt

@@ -7,6 +7,7 @@
 _PBENCH_SERVER_CONFIG=/var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/config/pbench-server.cfg
 [email protected]
 [email protected]
+1 3 * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-audit-server.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-audit-server
 * * * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-prep-002.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-server-prep-shim-002
 * * * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-prep-003.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-server-prep-shim-003
 * * * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-dispatch.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-dispatch
@@ -15,10 +16,8 @@ [email protected]
 * * * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-unpack-tarballs-large.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-unpack-tarballs large
 * * * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-unpack-tarballs-huge.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-unpack-tarballs huge
 1 2 * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-cull-unpacked-tarballs.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-cull-unpacked-tarballs
-1 3 * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-audit-server.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-audit-server
 23 * * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-copy-sosreports.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-copy-sosreports
 * * * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-index.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-index
-* * * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-index-tool-data.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-index --tool-data
 53 4 * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-backup-tarballs.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-backup-tarballs
 53 5 * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-verify-backup-tarballs.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-verify-backup-tarballs
 * * * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-sync-satellite-ONE.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-sync-satellite satellite-one
@@ -27,6 +26,7 @@ [email protected]
 _PBENCH_SERVER_CONFIG=/var/tmp/pbench-test-server/test-24/opt/pbench-server-satellite/lib/config/pbench-server.cfg
 [email protected]
 [email protected]
+1 3 * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server-satellite/lib/locks/pbench-audit-server.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server-satellite/bin/pbench-audit-server
 * * * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server-satellite/lib/locks/pbench-prep-002.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server-satellite/bin/pbench-server-prep-shim-002
 * * * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server-satellite/lib/locks/pbench-dispatch.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server-satellite/bin/pbench-dispatch
 * * * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server-satellite/lib/locks/pbench-unpack-tarballs.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server-satellite/bin/pbench-unpack-tarballs

server/bin/gold/test-8.txt

@@ -7,17 +7,16 @@
 _PBENCH_SERVER_CONFIG=/var/tmp/pbench-test-server/test-8/opt/pbench-server/lib/config/pbench-server.cfg
 [email protected]
 [email protected]
+1 3 * * *  flock -n /var/tmp/pbench-test-server/test-8/opt/pbench-server/lib/locks/pbench-audit-server.lock /var/tmp/pbench-test-server/test-8/opt/pbench-server/bin/pbench-audit-server
 * * * * *  flock -n /var/tmp/pbench-test-server/test-8/opt/pbench-server/lib/locks/pbench-prep-002.lock /var/tmp/pbench-test-server/test-8/opt/pbench-server/bin/pbench-server-prep-shim-002
 * * * * *  flock -n /var/tmp/pbench-test-server/test-8/opt/pbench-server/lib/locks/pbench-dispatch.lock /var/tmp/pbench-test-server/test-8/opt/pbench-server/bin/pbench-dispatch
 * * * * *  flock -n /var/tmp/pbench-test-server/test-8/opt/pbench-server/lib/locks/pbench-unpack-tarballs-small.lock /var/tmp/pbench-test-server/test-8/opt/pbench-server/bin/pbench-unpack-tarballs small
 * * * * *  flock -n /var/tmp/pbench-test-server/test-8/opt/pbench-server/lib/locks/pbench-unpack-tarballs-medium.lock /var/tmp/pbench-test-server/test-8/opt/pbench-server/bin/pbench-unpack-tarballs medium
 * * * * *  flock -n /var/tmp/pbench-test-server/test-8/opt/pbench-server/lib/locks/pbench-unpack-tarballs-large.lock /var/tmp/pbench-test-server/test-8/opt/pbench-server/bin/pbench-unpack-tarballs large
 * * * * *  flock -n /var/tmp/pbench-test-server/test-8/opt/pbench-server/lib/locks/pbench-unpack-tarballs-huge.lock /var/tmp/pbench-test-server/test-8/opt/pbench-server/bin/pbench-unpack-tarballs huge
 1 2 * * *  flock -n /var/tmp/pbench-test-server/test-8/opt/pbench-server/lib/locks/pbench-cull-unpacked-tarballs.lock /var/tmp/pbench-test-server/test-8/opt/pbench-server/bin/pbench-cull-unpacked-tarballs
-1 3 * * *  flock -n /var/tmp/pbench-test-server/test-8/opt/pbench-server/lib/locks/pbench-audit-server.lock /var/tmp/pbench-test-server/test-8/opt/pbench-server/bin/pbench-audit-server
 23 * * * *  flock -n /var/tmp/pbench-test-server/test-8/opt/pbench-server/lib/locks/pbench-copy-sosreports.lock /var/tmp/pbench-test-server/test-8/opt/pbench-server/bin/pbench-copy-sosreports
 * * * * *  flock -n /var/tmp/pbench-test-server/test-8/opt/pbench-server/lib/locks/pbench-index.lock /var/tmp/pbench-test-server/test-8/opt/pbench-server/bin/pbench-index
-* * * * *  flock -n /var/tmp/pbench-test-server/test-8/opt/pbench-server/lib/locks/pbench-index-tool-data.lock /var/tmp/pbench-test-server/test-8/opt/pbench-server/bin/pbench-index --tool-data
 53 4 * * *  flock -n /var/tmp/pbench-test-server/test-8/opt/pbench-server/lib/locks/pbench-backup-tarballs.lock /var/tmp/pbench-test-server/test-8/opt/pbench-server/bin/pbench-backup-tarballs
 53 5 * * *  flock -n /var/tmp/pbench-test-server/test-8/opt/pbench-server/lib/locks/pbench-verify-backup-tarballs.lock /var/tmp/pbench-test-server/test-8/opt/pbench-server/bin/pbench-verify-backup-tarballs
 * * * * *  flock -n /var/tmp/pbench-test-server/test-8/opt/pbench-server/lib/locks/pbench-sync-satellite-ONE.lock /var/tmp/pbench-test-server/test-8/opt/pbench-server/bin/pbench-sync-satellite satellite-one
@@ -26,6 +25,7 @@ [email protected]
 _PBENCH_SERVER_CONFIG=/var/tmp/pbench-test-server/test-8/opt/pbench-server-satellite/lib/config/pbench-server.cfg
 [email protected]
 [email protected]
+1 3 * * *  flock -n /var/tmp/pbench-test-server/test-8/opt/pbench-server-satellite/lib/locks/pbench-audit-server.lock /var/tmp/pbench-test-server/test-8/opt/pbench-server-satellite/bin/pbench-audit-server
 * * * * *  flock -n /var/tmp/pbench-test-server/test-8/opt/pbench-server-satellite/lib/locks/pbench-prep-002.lock /var/tmp/pbench-test-server/test-8/opt/pbench-server-satellite/bin/pbench-server-prep-shim-002
 * * * * *  flock -n /var/tmp/pbench-test-server/test-8/opt/pbench-server-satellite/lib/locks/pbench-dispatch.lock /var/tmp/pbench-test-server/test-8/opt/pbench-server-satellite/bin/pbench-dispatch
 * * * * *  flock -n /var/tmp/pbench-test-server/test-8/opt/pbench-server-satellite/lib/locks/pbench-unpack-tarballs.lock /var/tmp/pbench-test-server/test-8/opt/pbench-server-satellite/bin/pbench-unpack-tarballs

server/bin/state/config-satellite/pbench-server.cfg

@@ -1,23 +1,24 @@
-install-dir = %(unittest-dir)s/opt/pbench-server-satellite
 default-host = pbench-satellite.example.com
+install-dir = %(unittest-dir)s/opt/pbench-server-satellite
 
 ###########################################################################
 ## Deployment section
 ###########################################################################
 [pbench-server]
-commit_id = unit-test
-environment = unit-test
-admin-email = [email protected]
 pbench-top-dir = %(unittest-dir)s/pbench-satellite
 pbench-local-dir = %(unittest-dir)s/pbench-satellite-local
-# A satellite just unpacks and syncs
+# An example satellite might override the default roles to drop the backup role.
+roles = pbench-maintenance, pbench-prep, pbench-results
+# An example satellite that just unpacks and syncs
 dispatch-states = TO-UNPACK, TO-SYNC
-# A satellite just needs to prep and process results, no backup necessary
-roles = pbench-prep, pbench-results
 # This has to be set to the same value as what the mock'd "hostname"
 # command returns.
 realhost = pbench.example.com
+# Unit test environment configuration values
 debug_unittest = True
+commit_id = unit-test
+environment = unit-test
+admin-email = [email protected]
 
 [apache]
 documentroot = %(unittest-dir)s/var-www-html-satellite
@@ -27,12 +28,11 @@ logger_type = file
 
 ###########################################################################
 # crontab roles
-#
-# For a satellite server we typically don't need to index results or handle
-# sosreports, so we override the tasks to drop those two.
 [pbench-results]
-# The results role on a satellite does not need to copy sosreports or index,
-# but we do need to make sure we run sync-cleanup.
+# The results role on our example satellite does not need to copy sosreports,
+# index data, or cull unpacked tar balls, but we do need to make sure we run
+# sync-cleanup.  As a satellite, we also only need at most one process
+# unpacking tar balls.
 tasks = pbench-dispatch, pbench-unpack-tarballs, pbench-sync-cleanup
 
 ###########################################################################

server/bin/state/config/pbench-server.cfg

@@ -4,15 +4,21 @@ install-dir = %(unittest-dir)s/opt/pbench-server
 ## Deployment section
 ###########################################################################
 [pbench-server]
-commit_id = unit-test
-environment = unit-test
-admin-email = [email protected]
 pbench-top-dir = %(unittest-dir)s/pbench
 pbench-local-dir = %(unittest-dir)s/pbench-local
 pbench-backup-dir = %(pbench-local-dir)s/archive.backup
 # Add role for sync'ing with satellites
-roles = pbench-prep, pbench-results, pbench-backup, pbench-sync-satellites
+roles = pbench-maintenance, pbench-prep, pbench-results, pbench-backup, pbench-sync-satellites
 debug_unittest = True
+commit_id = unit-test
+environment = unit-test
+admin-email = [email protected]
+
+[pbench-server-backup]
+endpoint_url = %(unittest-dir)s/pbench-local/s3-backup
+access_key_id = ACCESS_KEY_ID
+secret_access_key = SECRET_ACCESS_KEY
+bucket_name = testbucket
 
 [Indexing]
 server = elasticsearch.example.com:9280
@@ -27,11 +33,6 @@ logger_type = file
 
 ###########################################################################
 # crontab roles
-[pbench-server-backup]
-endpoint_url = %(unittest-dir)s/pbench-local/s3-backup
-bucket_name = testbucket
-access_key_id = ACCESS_KEY_ID
-secret_access_key = SECRET_ACCESS_KEY
 
 # The definition of the crontab role for sync'ing satellite pbench servers.
 [pbench-sync-satellites]

server/bin/state/test-17.config

@@ -4,15 +4,21 @@ install-dir = %(unittest-dir)s/opt/pbench-server
 ## Deployment section
 ###########################################################################
 [pbench-server]
-commit_id = unit-test
-environment = unit-test
-admin-email = [email protected]
 pbench-top-dir = %(unittest-dir)s/pbench
 pbench-local-dir = %(unittest-dir)s/pbench-local
 pbench-backup-dir = %(pbench-local-dir)s/archive.backup
 # Add role for sync'ing with satellites
-roles = pbench-prep, pbench-results, pbench-backup, pbench-sync-satellites
+roles = pbench-maintenance, pbench-prep, pbench-results, pbench-backup, pbench-sync-satellites
 debug_unittest = True
+commit_id = unit-test
+environment = unit-test
+admin-email = [email protected]
+
+[pbench-server-backup]
+endpoint_url = %(unittest-dir)s/pbench-local/s3-backup
+access_key_id = ACCESS_KEY_ID
+secret_access_key = SECRET_ACCESS_KEY
+bucket_name = testbucket
 
 [Indexing]
 server = elasticsearch.example.com:9280
@@ -26,6 +32,7 @@ logger_type = file
 
 ###########################################################################
 # crontab roles
+
 # The definition of the crontab role for sync'ing satellite pbench servers.
 [pbench-sync-satellites]
 host = %(default-host)s

server/bin/state/test-20.config

@@ -3,14 +3,6 @@ install-dir = %(unittest-dir)s/opt/pbench-server
 [pbench-server]
 debug_ref_datetime = 1970-02-14T00:00:00.000000
 
-###########################################################################
-## runtime section
-###########################################################################
-[sosreport]
-user = %(default-user)s
-host = pbench-sosreports.example.com
-dir = /path/to/sosreport/dir
-
 ###########################################################################
 # The rest will come from the global state config file and the default config file.
 [config]