Add new "cull unpacked tar balls" service

We now run a culling service once a day to find all unpacked tar ball
directories whose age is beyond the configured maximum.  We also have
changed the unpacked tar balls service to only unpack tar balls less
than the configured maximum age.

One can keep an already unpacked tar ball by creating a `.__pbench_keep__`
file in the top level of the unpacked tar ball directory.

A report is generated much like the following:

```
Culled 3 unpacked tar ball directories (0 errors) in 0.00 secs

Actions Taken:
  - controller-no-prefixes/tarball_culled_1970.01.01T00.00.00 (0 errors, 0.00 secs)
      $ rm results/controller-no-prefixes/tarball_culled_1970.01.01T00.00.00  # succ
      $ mv incoming/controller-no-prefixes/tarball_culled_1970.01.01T00.00.00 incoming/controller-no-prefixes/.delete.tarball_culled_1970.01.01T00.00.00  # succ
      $ rmtree incoming/controller-no-prefixes/.delete.tarball_culled_1970.01.01T00.00.00  # succ
  - controller-prefixes/tarball_culled-w-prefix_1970.01.01T00.00.00 (0 errors, 0.00 secs)
      $ rm results/controller-prefixes/pre0/pre1/pre2/tarball_culled-w-prefix_1970.01.01T00.00.00  # succ
      $ mv incoming/controller-prefixes/tarball_culled-w-prefix_1970.01.01T00.00.00 incoming/controller-prefixes/.delete.tarball_culled-w-prefix_1970.01.01T00.00.00  # succ
      $ rmtree incoming/controller-prefixes/.delete.tarball_culled-w-prefix_1970.01.01T00.00.00  # succ
  - controller-prefixes/tarball_culled-w-userA_1970.01.01T00.00.00 (0 errors, 0.00 secs)
      $ rm results/controller-prefixes/path0/path1/tarball_culled-w-userA_1970.01.01T00.00.00  # succ
      $ rm users/userA/controller-prefixes/path0/path1/tarball_culled-w-userA_1970.01.01T00.00.00  # succ
      $ mv incoming/controller-prefixes/tarball_culled-w-userA_1970.01.01T00.00.00 incoming/controller-prefixes/.delete.tarball_culled-w-userA_1970.01.01T00.00.00  # succ
      $ rmtree incoming/controller-prefixes/.delete.tarball_culled-w-userA_1970.01.01T00.00.00  # succ
```

Author: portante

server/bin/gold/test-0.1.txt

@@ -22,6 +22,9 @@ pbench-index-tool-data: Bad LOGSDIR=/var/tmp/pbench-test-server/test-0.1/pbench-
 +++ Running pbench-clean-up-dangling-results-links
 pbench-clean-up-dangling-results-links: Bad LOGSDIR=/var/tmp/pbench-test-server/test-0.1/pbench-local/logs (config file /var/tmp/pbench-test-server/test-0.1/opt/pbench-server/lib/config/pbench-server.cfg)
 --- Finished pbench-clean-up-dangling-results-links (status=1)
++++ Running pbench-cull-unpacked-tarballs
+pbench-cull-unpacked-tarballs: Bad LOGSDIR=/var/tmp/pbench-test-server/test-0.1/pbench-local/logs
+--- Finished pbench-cull-unpacked-tarballs (status=2)
 +++ Running pbench-backup-tarballs
 pbench-backup-tarballs: Bad LOGSDIR=/var/tmp/pbench-test-server/test-0.1/pbench-local/logs
 --- Finished pbench-backup-tarballs (status=1)

server/bin/gold/test-0.2.txt

@@ -22,6 +22,9 @@ pbench-index-tool-data: Bad TMP=/var/tmp/pbench-test-server/test-0.2/pbench-loca
 +++ Running pbench-clean-up-dangling-results-links
 pbench-clean-up-dangling-results-links: Bad TMP=/var/tmp/pbench-test-server/test-0.2/pbench-local/tmp (config file /var/tmp/pbench-test-server/test-0.2/opt/pbench-server/lib/config/pbench-server.cfg)
 --- Finished pbench-clean-up-dangling-results-links (status=1)
++++ Running pbench-cull-unpacked-tarballs
+pbench-cull-unpacked-tarballs: Bad TMP=/var/tmp/pbench-test-server/test-0.2/pbench-local/tmp
+--- Finished pbench-cull-unpacked-tarballs (status=2)
 +++ Running pbench-backup-tarballs
 pbench-backup-tarballs: Bad TMP=/var/tmp/pbench-test-server/test-0.2/pbench-local/tmp
 --- Finished pbench-backup-tarballs (status=1)

server/bin/gold/test-0.txt

@@ -22,6 +22,9 @@ pbench-index-tool-data: Bad TOP=/var/tmp/pbench-test-server/test-0/pbench
 +++ Running pbench-clean-up-dangling-results-links
 pbench-clean-up-dangling-results-links: Bad TOP=/var/tmp/pbench-test-server/test-0/pbench (config file /var/tmp/pbench-test-server/test-0/opt/pbench-server/lib/config/pbench-server.cfg)
 --- Finished pbench-clean-up-dangling-results-links (status=1)
++++ Running pbench-cull-unpacked-tarballs
+pbench-cull-unpacked-tarballs: Bad TOP=/var/tmp/pbench-test-server/test-0/pbench
+--- Finished pbench-cull-unpacked-tarballs (status=2)
 +++ Running pbench-backup-tarballs
 pbench-backup-tarballs: Bad TOP=/var/tmp/pbench-test-server/test-0/pbench
 --- Finished pbench-backup-tarballs (status=1)

server/bin/gold/test-1.txt

@@ -20,6 +20,8 @@ pbench-copy-sosreports: Bad ARCHIVE=/var/tmp/pbench-test-server/test-1/pbench/ar
 +++ Running pbench-clean-up-dangling-results-links
 pbench-clean-up-dangling-results-links: Bad ARCHIVE=/var/tmp/pbench-test-server/test-1/pbench/archive/fs-version-001
 --- Finished pbench-clean-up-dangling-results-links (status=1)
++++ Running pbench-cull-unpacked-tarballs
+--- Finished pbench-cull-unpacked-tarballs (status=4)
 +++ Running pbench-backup-tarballs
 --- Finished pbench-backup-tarballs (status=3)
 +++ Running pbench-verify-backup-tarballs
@@ -88,6 +90,8 @@ drwxrwxr-x          - public_html/users
 drwxrwxr-x          - logs
 drwxrwxr-x          - logs/pbench-backup-tarballs
 -rw-rw-r--        280 logs/pbench-backup-tarballs/pbench-backup-tarballs.log
+drwxrwxr-x          - logs/pbench-cull-unpacked-tarballs
+-rw-rw-r--        226 logs/pbench-cull-unpacked-tarballs/pbench-cull-unpacked-tarballs.log
 drwxrwxr-x          - logs/pbench-index
 drwxrwxr-x          - logs/pbench-index-tool-data
 -rw-rw-r--        178 logs/pbench-index-tool-data/pbench-index-tool-data.log
@@ -143,6 +147,9 @@ drwxrwxr-x          - tmp
 +++++ pbench-backup-tarballs/pbench-backup-tarballs.log
 1970-01-01T00:00:42.000000 ERROR pbench-backup-tarballs.pbench-backup-tarballs sanity_check -- The ARCHIVE directory /var/tmp/pbench-test-server/test-1/pbench/archive/fs-version-001, does not resolve /var/tmp/pbench-test-server/test-1/pbench/archive/fs-version-001 to a directory
 ----- pbench-backup-tarballs/pbench-backup-tarballs.log
++++++ pbench-cull-unpacked-tarballs/pbench-cull-unpacked-tarballs.log
+1970-01-01T00:00:42.000000 ERROR pbench-cull-unpacked-tarballs.pbench-cull-unpacked-tarballs main -- The configured ARCHIVE directory, /var/tmp/pbench-test-server/test-1/pbench/archive/fs-version-001, is not a valid directory
+----- pbench-cull-unpacked-tarballs/pbench-cull-unpacked-tarballs.log
 +++++ pbench-index-tool-data/pbench-index-tool-data.log
 1970-01-01T00:00:42.000000 ERROR pbench-index-tool-data.pbench-index main -- pbench-index-tool-data: Bad ARCHIVE=/var/tmp/pbench-test-server/test-1/pbench/archive/fs-version-001
 ----- pbench-index-tool-data/pbench-index-tool-data.log

server/bin/gold/test-2.txt

@@ -100,6 +100,8 @@ pbench-copy-sosreports: Bad INCOMING=/var/tmp/pbench-test-server/test-2/pbench/p
 --- Finished pbench-index (status=3)
 +++ Running pbench-clean-up-dangling-results-links
 --- Finished pbench-clean-up-dangling-results-links (status=0)
++++ Running pbench-cull-unpacked-tarballs
+--- Finished pbench-cull-unpacked-tarballs (status=4)
 +++ Running pbench-backup-tarballs
 Template:  pbench-unittests.v3.server-reports
 Index:  pbench-unittests.v3.server-reports.1970-01 1
@@ -228,6 +230,8 @@ drwxrwxr-x          - logs/pbench-backup-tarballs
 drwxrwxr-x          - logs/pbench-clean-up-dangling-results-links
 -rw-rw-r--          0 logs/pbench-clean-up-dangling-results-links/pbench-clean-up-dangling-results-links.error
 -rw-rw-r--          0 logs/pbench-clean-up-dangling-results-links/pbench-clean-up-dangling-results-links.log
+drwxrwxr-x          - logs/pbench-cull-unpacked-tarballs
+-rw-rw-r--        225 logs/pbench-cull-unpacked-tarballs/pbench-cull-unpacked-tarballs.log
 drwxrwxr-x          - logs/pbench-dispatch
 -rw-rw-r--          0 logs/pbench-dispatch/pbench-dispatch.error
 -rw-rw-r--        576 logs/pbench-dispatch/pbench-dispatch.log
@@ -307,6 +311,9 @@ drwxrwxr-x          - tmp
 ----- pbench-clean-up-dangling-results-links/pbench-clean-up-dangling-results-links.error
 +++++ pbench-clean-up-dangling-results-links/pbench-clean-up-dangling-results-links.log
 ----- pbench-clean-up-dangling-results-links/pbench-clean-up-dangling-results-links.log
++++++ pbench-cull-unpacked-tarballs/pbench-cull-unpacked-tarballs.log
+1970-01-01T00:00:42.000000 ERROR pbench-cull-unpacked-tarballs.pbench-cull-unpacked-tarballs main -- The configured INCOMING directory, /var/tmp/pbench-test-server/test-2/pbench/public_html/incoming, is not a valid directory
+----- pbench-cull-unpacked-tarballs/pbench-cull-unpacked-tarballs.log
 +++++ pbench-dispatch/pbench-dispatch.error
 ----- pbench-dispatch/pbench-dispatch.error
 +++++ pbench-dispatch/pbench-dispatch.log

server/bin/gold/test-24.txt

@@ -14,6 +14,7 @@ [email protected]
 * * * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-unpack-tarballs-medium.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-unpack-tarballs medium
 * * * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-unpack-tarballs-large.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-unpack-tarballs large
 * * * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-unpack-tarballs-huge.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-unpack-tarballs huge
+1 2 * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-cull-unpacked-tarballs.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-cull-unpacked-tarballs
 1 3 * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-audit-server.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-audit-server
 23 * * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-copy-sosreports.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-copy-sosreports
 * * * * *  flock -n /var/tmp/pbench-test-server/test-24/opt/pbench-server/lib/locks/pbench-index.lock /var/tmp/pbench-test-server/test-24/opt/pbench-server/bin/pbench-index