A "re-imagining" of the old pbench-reindex (distributed-system-analysis#3614)

* A "re-imagining" of the old `pbench-reindex`

This supports deleting a datasets's indexed data and triggering re-indexing.
It also provides a mechanism to globally enable or disable indexing.

In the short term, this provides a workaround for our sharding issues, which
are preventing us from generating useful indexing anyway, by turning it off
entirely for new datasets. We can also delete indexed data (and the SQL
index maps) for existing datasets and, on demand, trigger re-indexing for
selected datasets.

Author: dbutenhof

lib/pbench/cli/server/__init__.py

@@ -1,8 +1,11 @@
 import datetime
 from threading import Thread
 import time
+from typing import Any, Optional
 
 import click
+from click import Context, Parameter, ParamType
+from dateutil import parser
 
 from pbench.server import PbenchServerConfig
 from pbench.server.database import init_db
@@ -30,13 +33,13 @@ def __bool__(self) -> bool:
         return self.detail
 
     def error(self, message: str):
-        """Write a message if details are enabled.
+        """Write an error message if error details are enabled.
 
         Args:
             message: Detail string
         """
         if self.errors:
-            click.secho(f"|| {message}", fg="red")
+            click.secho(f"|E| {message}", fg="red", err=True)
 
     def message(self, message: str):
         """Write a message if details are enabled.
@@ -45,7 +48,16 @@ def message(self, message: str):
             message: Detail string
         """
         if self.detail:
-            click.echo(f"|| {message}")
+            click.echo(f"|I| {message}")
+
+    def warning(self, message: str):
+        """Write a warning message if error details are enabled.
+
+        Args:
+            message: Detail string
+        """
+        if self.errors:
+            click.secho(f"|W| {message}", fg="blue", err=True)
 
 
 class Verify:
@@ -75,7 +87,7 @@ def status(self, message: str):
         """
         if self.verify:
             ts = datetime.datetime.now().astimezone()
-            click.secho(f"({ts:%H:%M:%S}) {message}", fg="green")
+            click.secho(f"({ts:%H:%M:%S}) {message}", fg="green", err=True)
 
 
 class Watch:
@@ -120,10 +132,33 @@ def watcher(self):
             hours, remainder = divmod(delta, 3600)
             minutes, seconds = divmod(remainder, 60)
             click.secho(
-                f"[{hours:02d}:{minutes:02d}:{seconds:02d}] {self.status}", fg="cyan"
+                f"[{hours:02d}:{minutes:02d}:{seconds:02d}] {self.status}",
+                fg="cyan",
+                err=True,
             )
 
 
+class DateParser(ParamType):
+    """The DateParser type converts date strings into `datetime` objects.
+
+    This is a variant of click's built-in DateTime parser, but uses the
+    more flexible dateutil.parser
+    """
+
+    name = "dateparser"
+
+    def convert(
+        self, value: Any, param: Optional[Parameter], ctx: Optional[Context]
+    ) -> Any:
+        if isinstance(value, datetime.datetime):
+            return value
+
+        try:
+            return parser.parse(value)
+        except Exception as e:
+            self.fail(f"{value!r} cannot be converted to a datetime: {str(e)!r}")
+
+
 def config_setup(context: object) -> PbenchServerConfig:
     config = PbenchServerConfig.create(context.config)
     # We're going to need the DB to track dataset state, so setup DB access.