Search: Unlock JSON and YAML export formats

Author: amotl

CHANGES.rst

@@ -13,6 +13,7 @@ in progress
 - History: Add ``id`` and ``uid`` dashboard attributes to report
 - History: Unlock YAML export format
 - History: Add new options ``--head``, ``--tail``, and ``--reverse``
+- Search: Unlock JSON and YAML export formats
 
 2023-03-05 0.14.1
 =================

grafana_wtf/commands.py

@@ -10,7 +10,7 @@
 
 from grafana_wtf import __appname__, __version__
 from grafana_wtf.core import GrafanaWtf
-from grafana_wtf.report.data import output_results
+from grafana_wtf.report.data import output_results, DataSearchReport
 from grafana_wtf.report.textual import TextualSearchReport
 from grafana_wtf.report.tabular import TabularSearchReport, get_table_format, TabularEditHistoryReport
 from grafana_wtf.util import (
@@ -40,7 +40,7 @@ def run():
       --grafana-token=<grafana-token>   Grafana API Key token
       --select-dashboard=<uuid>         Restrict operation to dashboard by UID.
                                         Can be a list of comma-separated dashboard UIDs.
-      --format=<format>                 Output format. [default: json]
+      --format=<format>                 Output format. One of textual, tabular, json, yaml.
       --cache-ttl=<cache-ttl>           Time-to-live for the request cache in seconds. [default: 300]
       --drop-cache                      Drop cache before requesting resources
       --concurrency=<concurrency>       Run multiple requests in parallel. [default: 5]
@@ -176,6 +176,13 @@ def run():
         else:
             raise
 
+    # Compute default output format.
+    if not options.format:
+        if options.find or options.replace:
+            options.format = "textual"
+        else:
+            options.format = "json"
+
     # Sanity checks
     if grafana_url is None:
         raise DocoptExit(
@@ -195,6 +202,7 @@ def run():
     output_format = options["format"]
 
     if options.find or options.replace:
+
         if options.select_dashboard:
             # Restrict scan to list of dashboards.
             dashboard_uids = read_list(options.select_dashboard)
@@ -206,11 +214,13 @@ def run():
 
         result = engine.search(options.search_expression or None)
 
-        if output_format.startswith("tabular"):
+        if output_format.startswith("tab"):
             table_format = get_table_format(output_format)
             generator = partial(TabularSearchReport, tblfmt=table_format)
-        else:
+        elif output_format.startswith("text"):
             generator = TextualSearchReport
+        else:
+            generator = partial(DataSearchReport, format=output_format)
 
         report = generator(grafana_url, verbose=options.verbose)
         report.display(options.search_expression, result)
@@ -220,6 +230,7 @@ def run():
         engine.clear_cache()
 
     if options.log:
+
         entries = engine.log(dashboard_uid=options.dashboard_uid)
         entries = sorted(entries, key=itemgetter("datetime"))
 
@@ -237,14 +248,15 @@ def run():
         if options.reverse:
             entries = list(reversed(entries))
 
-        if output_format.startswith("tabular"):
+        if output_format.startswith("tab"):
             report = TabularEditHistoryReport(data=entries)
             output = report.render(output_format)
             print(output)
         else:
             output_results(output_format, entries)
 
     if options.explore and options.datasources:
+
         results = engine.explore_datasources()
 
         unused_count = len(results["unused"])

grafana_wtf/report/data.py

@@ -1,10 +1,21 @@
 import json
+import logging
+from collections import OrderedDict
 from typing import List
 
+from grafana_wtf.report.tabular import TabularSearchReport
 from grafana_wtf.util import yaml_dump
 
+log = logging.getLogger(__name__)
+
 
 def output_results(output_format: str, results: List):
+    output = serialize_results(output_format, results)
+    print(output)
+
+
+def serialize_results(output_format: str, results: List):
+
     if output_format == "json":
         output = json.dumps(results, indent=4)
 
@@ -14,4 +25,25 @@ def output_results(output_format: str, results: List):
     else:
         raise ValueError(f'Unknown output format "{output_format}"')
 
-    print(output)
+    return output
+
+
+class DataSearchReport(TabularSearchReport):
+    def __init__(self, grafana_url, verbose=False, format=None):
+        self.grafana_url = grafana_url
+        self.verbose = verbose
+        self.format = format
+
+    def display(self, expression, result):
+        expression = expression or "*"
+        log.info(f"Searching for expression '{expression}' at Grafana instance {self.grafana_url}")
+
+        output = OrderedDict(
+            meta=OrderedDict(
+                grafana=self.grafana_url,
+                expression=expression,
+            ),
+            datasources=self.get_output_items("Datasource", result.datasources, self.compute_url_datasource),
+            dashboards=self.get_output_items("Dashboard", result.dashboards, self.compute_url_dashboard),
+        )
+        output_results(self.format, output)

grafana_wtf/report/tabular.py

@@ -25,6 +25,10 @@ def __init__(self, grafana_url, tblfmt="psql", verbose=False):
         super().__init__(grafana_url, verbose=verbose)
 
     def output_items(self, label, items, url_callback):
+        items_rows = self.get_output_items(label, items, url_callback)
+        print(tabulate(items_rows, headers="keys", tablefmt=self.format))
+
+    def get_output_items(self, label, items, url_callback):
         items_rows = [
             {
                 "Type": label,
@@ -33,7 +37,7 @@ def output_items(self, label, items, url_callback):
             }
             for item in items
         ]
-        print(tabulate(items_rows, headers="keys", tablefmt=self.format))
+        return items_rows
 
     def get_bibdata_dict(self, item, **kwargs):
         # Sanity checks.

grafana_wtf/report/textual.py

@@ -21,13 +21,14 @@ def __init__(self, grafana_url, verbose=False):
     def display(self, expression, result):
         expression = expression or "*"
         print('Searching for expression "{}" at Grafana instance {}'.format(_m(expression), self.grafana_url))
-        self.output_items(_s("Data Sources"), result.datasources, self.compute_url_datasource)
-        self.output_items(_s("Dashboards"), result.dashboards, self.compute_url_dashboard)
+        self.output_items("Data Sources", result.datasources, self.compute_url_datasource)
+        self.output_items("Dashboards", result.dashboards, self.compute_url_dashboard)
 
     def output_items(self, label, items, url_callback):
         # Output section name (data source vs. dashboard).
+        hits = len(items)
         print("=" * 42)
-        print("{label}: {hits} hits.".format(hits=_m(len(items)), label=label))
+        print(f"{_s(label)}: {_m(hits)} hits.")
         print("=" * 42)
         print()
 

tests/test_commands.py

@@ -143,6 +143,34 @@ def test_find_tabular_dashboard_success(ldi_resources, capsys):
     assert output_table_normalized == reference_table
 
 
+def test_find_format_json(ldi_resources, capsys):
+    # Only provision specific dashboard(s).
+    ldi_resources(dashboards=["tests/grafana/dashboards/ldi-v27.json", "tests/grafana/dashboards/ldi-v33.json"])
+
+    # Run command and capture output.
+    set_command("find ldi_readings --format=json")
+    grafana_wtf.commands.run()
+    captured = capsys.readouterr()
+
+    # Verify output.
+    data = json.loads(captured.out)
+    assert len(data) == 3
+
+
+def test_find_format_yaml(ldi_resources, capsys):
+    # Only provision specific dashboard(s).
+    ldi_resources(dashboards=["tests/grafana/dashboards/ldi-v27.json", "tests/grafana/dashboards/ldi-v33.json"])
+
+    # Run command and capture output.
+    set_command("find ldi_readings --format=yaml")
+    grafana_wtf.commands.run()
+    captured = capsys.readouterr()
+
+    # Verify output.
+    data = yaml.safe_load(captured.out)
+    assert len(data) == 3
+
+
 def test_replace_dashboard_success(ldi_resources, capsys):
     # Only provision specific dashboard(s).
     ldi_resources(dashboards=["tests/grafana/dashboards/ldi-v27.json", "tests/grafana/dashboards/ldi-v33.json"])