An assortment of Pbench Ops fixes and fun (distributed-system-analysis#3612)

* An assortment of Pbench Ops fixes and fun

This fixes several issues observed during ops review:

1. The `/api/v1/endpoints` API fails if the server is shut down
2. `tar` unpack errors can result in enormous `stderr` output, which is
   captured in the `Audit` log; truncate it to 5Mb
3. Change the `pbench-audit` utility to use `dateutil.parser` instead of
   `click.DateTime()` so we can include fractional seconds and timezone.

During the time when we broke PostgreSQL, we failed to create metadata for a
number of datasets that were allowed to upload. (Whether we should allow this
vs failing the upload is a separate issue.) We have want to repair the
excessively large `Audit` attributes records. So I took a stab at some
wondrous and magical SQL queries and hackery to begin a new `pbench-repair`
utility. Right now, it repairs long audit attributes "intelligently" by
trimming individual JSON key values; and it add metadata to datasets which
lack critical values. Currently, this includes `server.tarball-path` (which
we need to enable TOC and visualization), `dataset.metalog` (capturing the
tarball `metadata.log` file), and `server.benchmark` for visualization.

There are other `server` namespace values (including expiration time) that
could be repaired: I decided not to worry about that as we're not doing
expiration anyway. (Though I might add it over the weekend, since it shouldn't
be hard.) And there are probably other things we might want to repair in the
future using this framework.

I tested this in a `runlocal` container, using `psql` to "break" datasets and
repair them. I hacked the local `repair.py` with a low "max error" limit to
force truncation of audit attributes:

```
pbench-repair --detail --errors --verify --progress 10
(22:52:08) Repairing audit
|| 60:FAILURE upload fio_rw_2018.02.01T22.40.57 [message] truncated (107) to 105
|| 116:SUCCESS apikey None [key] truncated (197) to 105
22 audit records had attributes too long
2 records were fixed
(22:52:08) Repairing metadata
|| fio_rw_2018.02.01T22.40.57 has no server.tarball-path: setting /srv/pbench/archive/fs-version-001/dhcp31-45.perf.lab.eng.bos.redhat.com/08516cc7448035be2cc502f0517783fa/fio_rw_2018.02.01T22.40.57.tar.xz
|| fio_rw_2018.02.01T22.40.57 has no metalog: setting from metadata.log
|| fio_rw_2018.02.01T22.40.57 has no server.benchmark: setting 'fio'
|| pbench-user-benchmark_example-vmstat_2018.10.24T14.38.18 has no server.tarball-path: setting /srv/pbench/archive/fs-version-001/ansible-host/45f0e2af41977b89e07bae4303dc9972/pbench-user-benchmark_example-vmstat_2018.10.24T14.38.18.tar.xz
|| pbench-user-benchmark_example-vmstat_2018.10.24T14.38.18 has no metalog: setting from metadata.log
|| pbench-user-benchmark_example-vmstat_2018.10.24T14.38.18 has no server.benchmark: setting 'pbench-user-benchmark'
2 server.tarball-path repairs, 0 failures
2 dataset.metalog repairs, 0 failures
2 server.benchmark repairs
```

Author: dbutenhof

lib/pbench/cli/server/__init__.py

@@ -1,7 +1,7 @@
 import datetime
 from threading import Thread
 import time
-from typing import Any, Optional
+from typing import Any, Optional, Union
 
 import click
 from click import Context, Parameter, ParamType
@@ -11,6 +11,27 @@
 from pbench.server.database import init_db
 
 
+class DateParser(ParamType):
+    """The DateParser type converts date strings into `datetime` objects.
+
+    This is a variant of click's built-in DateTime parser, but uses the
+    more flexible dateutil.parser
+    """
+
+    name = "dateparser"
+
+    def convert(
+        self, value: Any, param: Optional[Parameter], ctx: Optional[Context]
+    ) -> Any:
+        if isinstance(value, datetime.datetime):
+            return value
+
+        try:
+            return parser.parse(value)
+        except Exception as e:
+            self.fail(f"{value!r} cannot be converted to a datetime: {str(e)!r}")
+
+
 class Detail:
     """Encapsulate generation of additional diagnostics"""
 
@@ -63,29 +84,32 @@ def warning(self, message: str):
 class Verify:
     """Encapsulate -v status messages."""
 
-    def __init__(self, verify: bool):
+    def __init__(self, verify: Union[bool, int]):
         """Initialize the object.
 
         Args:
             verify: True to write status messages.
         """
-        self.verify = verify
+        if isinstance(verify, int):
+            self.verify = verify
+        else:
+            self.verify = 1 if verify else 0
 
     def __bool__(self) -> bool:
         """Report whether verification is enabled.
 
         Returns:
             True if verification is enabled.
         """
-        return self.verify
+        return bool(self.verify)
 
-    def status(self, message: str):
+    def status(self, message: str, level: int = 1):
         """Write a message if verification is enabled.
 
         Args:
             message: status string
         """
-        if self.verify:
+        if self.verify >= level:
             ts = datetime.datetime.now().astimezone()
             click.secho(f"({ts:%H:%M:%S}) {message}", fg="green", err=True)
 
@@ -138,27 +162,6 @@ def watcher(self):
             )
 
 
-class DateParser(ParamType):
-    """The DateParser type converts date strings into `datetime` objects.
-
-    This is a variant of click's built-in DateTime parser, but uses the
-    more flexible dateutil.parser
-    """
-
-    name = "dateparser"
-
-    def convert(
-        self, value: Any, param: Optional[Parameter], ctx: Optional[Context]
-    ) -> Any:
-        if isinstance(value, datetime.datetime):
-            return value
-
-        try:
-            return parser.parse(value)
-        except Exception as e:
-            self.fail(f"{value!r} cannot be converted to a datetime: {str(e)!r}")
-
-
 def config_setup(context: object) -> PbenchServerConfig:
     config = PbenchServerConfig.create(context.config)
     # We're going to need the DB to track dataset state, so setup DB access.

lib/pbench/cli/server/audit.py

@@ -5,7 +5,7 @@
 import click
 
 from pbench.cli import pass_cli_context
-from pbench.cli.server import config_setup, Verify
+from pbench.cli.server import config_setup, DateParser, Verify
 from pbench.cli.server.options import common_options
 from pbench.server import BadConfig, OperationCode
 from pbench.server.database.database import Database
@@ -110,6 +110,7 @@ def auditor(kwargs) -> Iterator[str]:
 
 @click.command(name="pbench-audit")
 @pass_cli_context
+@click.option("--id", type=int, help="Select by audit event ID")
 @click.option(
     "--ids",
     default=False,
@@ -130,6 +131,7 @@ def auditor(kwargs) -> Iterator[str]:
 @click.option("--object-id", type=str, help="Select by object ID")
 @click.option("--object-name", type=str, help="Select by object name")
 @click.option("--page", default=False, is_flag=True, help="Paginate the output")
+@click.option("--root-id", type=int, help="Select by audit event root ID")
 @click.option("--user-id", type=str, help="Select by user ID")
 @click.option("--user-name", type=str, help="Select by username")
 @click.option(
@@ -142,12 +144,12 @@ def auditor(kwargs) -> Iterator[str]:
 )
 @click.option(
     "--since",
-    type=click.DateTime(),
+    type=DateParser(),
     help="Select entries on or after specified date/time",
 )
 @click.option(
     "--until",
-    type=click.DateTime(),
+    type=DateParser(),
     help="Select entries on or before specified date/time",
 )
 @click.option(