History: Add SQL querying capabilities

Author: amotl

CHANGES.rst

@@ -14,6 +14,7 @@ in progress
 - History: Unlock YAML export format
 - History: Add new options ``--head``, ``--tail``, and ``--reverse``
 - Search: Unlock JSON and YAML export formats
+- History: Add SQL querying capabilities
 
 2023-03-05 0.14.1
 =================

grafana_wtf/commands.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# (c) 2019-2021 Andreas Motl <[email protected]>
+# (c) 2019-2023 Andreas Motl <[email protected]>
 # License: GNU Affero General Public License, Version 3
 import logging
 import os
@@ -15,6 +15,7 @@
 from grafana_wtf.report.tabular import TabularSearchReport, get_table_format, TabularEditHistoryReport
 from grafana_wtf.util import (
     configure_http_logging,
+    filter_with_sql,
     normalize_options,
     read_list,
     setup_logging,
@@ -31,7 +32,7 @@ def run():
       grafana-wtf [options] explore dashboards
       grafana-wtf [options] find [<search-expression>]
       grafana-wtf [options] replace <search-expression> <replacement> [--dry-run]
-      grafana-wtf [options] log [<dashboard_uid>] [--number=<count>] [--head=<count>] [--tail=<count>] [--reverse]
+      grafana-wtf [options] log [<dashboard_uid>] [--number=<count>] [--head=<count>] [--tail=<count>] [--reverse] [--sql=<sql>]
       grafana-wtf --version
       grafana-wtf (-h | --help)
 
@@ -129,6 +130,22 @@ def run():
       # Output full history table in Markdown format
       grafana-wtf log --format=tabular:pipe
 
+      # Display dashboards with only a single edit, in JSON format.
+      grafana-wtf log --sql="
+        SELECT uid, url, COUNT(version) as number_of_edits
+        FROM dashboard_versions
+        GROUP BY uid, url
+        HAVING number_of_edits=1
+      "
+
+      # Display dashboards with only a single edit, in YAML format, `url` attribute only.
+      grafana-wtf log --format=yaml --sql="
+        SELECT url
+        FROM dashboard_versions
+        GROUP BY uid, url
+        HAVING COUNT(version)=1
+      "
+
     Cache control:
 
       # Use infinite cache expiration time, essentially caching forever.
@@ -231,8 +248,23 @@ def run():
 
     if options.log:
 
+        # Sanity checks.
+        if output_format.startswith("tab") and options.sql:
+            raise DocoptExit(
+                f"Options --format={output_format} and --sql can not be used together, only data output is supported."
+            )
+
         entries = engine.log(dashboard_uid=options.dashboard_uid)
-        entries = sorted(entries, key=itemgetter("datetime"))
+
+        if options.sql is not None:
+            log.info(f"Filtering result with SQL expression: {options.sql}")
+            entries = filter_with_sql(
+                data=entries,
+                view_name="dashboard_versions",
+                expression=options.sql,
+            )
+        else:
+            entries = sorted(entries, key=itemgetter("datetime"))
 
         if options.number is not None:
             limit = int(options.number)

grafana_wtf/util.py

@@ -5,6 +5,7 @@
 import json
 import logging
 import sys
+import typing as t
 from collections import OrderedDict
 
 import yaml
@@ -171,3 +172,34 @@ def to_list(value):
     if not isinstance(value, list):
         value = [value]
     return value
+
+
+trecord = t.List[t.Dict[str, str]]
+
+
+def filter_with_sql(data: trecord, view_name: str, expression: str) -> trecord:
+    """
+    Filter data in "records" shape by SQL expression, using pandas and DuckDB.
+
+    - https://duckdb.org/
+    - https://pandas.pydata.org/
+
+    Example::
+
+        SELECT uid, url, COUNT(version) as number_of_edits
+        FROM dashboard_versions
+        GROUP BY uid, url
+        HAVING number_of_edits=1
+
+    :param data: Data in "record" shape, aka. list of dictionaries
+    :param expression: SQL expression
+    :param view_name: View name the data is registered at, when querying per SQL.
+    :return:
+    """
+    import pandas as pd
+    import duckdb
+    df = pd.DataFrame.from_records(data)
+    duckdb.register(view_name, df)
+    results = duckdb.sql(expression)
+    entries = results.to_df().to_dict(orient="records")
+    return entries

setup.py

@@ -13,6 +13,9 @@
     "docopt>=0.6.2,<0.7",
     "munch>=2.5.0,<3",
     "tqdm>=4.60.0,<5",
+    # Filtering
+    "pandas<1.6",
+    "duckdb<0.8",
     # Grafana
     "requests>=2.23.0,<3",
     "grafana-client>=2.1.0,<4",

tests/test_commands.py

@@ -317,6 +317,29 @@ def test_log_yaml_success(ldi_resources, capsys, caplog):
     assert len(data) == 3
 
 
+def test_log_filter_sql(ldi_resources, capsys, caplog):
+    # Only provision specific dashboard(s).
+    ldi_resources(dashboards=["tests/grafana/dashboards/ldi-v27.json", "tests/grafana/dashboards/ldi-v33.json"])
+
+    # Run command and capture output.
+    set_command("""log --format=yaml --sql='
+        SELECT url
+        FROM dashboard_versions
+        GROUP BY uid, url
+        HAVING COUNT(version)=1
+    '
+    """)
+    with caplog.at_level(logging.DEBUG):
+        grafana_wtf.commands.run()
+    captured = capsys.readouterr()
+
+    assert captured.out.strip().split("\n") == [
+        "- url: http://localhost:33333/d/ioUrPwQiz/luftdaten-info-generic-trend-v27",
+        "- url: http://localhost:33333/d/jpVsQxRja/luftdaten-info-generic-trend-v33",
+        "- url: http://localhost:33333/dashboards/f/testdrive/testdrive",
+    ]
+
+
 def test_explore_datasources_used(create_datasource, create_dashboard, capsys, caplog):
     # Create two data sources and a dashboard which uses them.
     ds_foo = create_datasource(name="foo")