Add explore dashboards --data-details option

This extends the output by many more details about data inquiry/queries.

Author: amotl

CHANGES.rst

@@ -10,6 +10,8 @@ in progress
 - CI: Update to Grafana 8.5.27, 9.5.8, and 10.1.1
 - Grafana 9.3: Work around delete folder operation returning empty body
 - Grafana 9.5: Use standard UUIDs instead of short UIDs
+- Add ``explore dashboards --data-details`` option, to extend the output
+  by many more details about data inquiry / queries. Thanks, @meyerder.
 
 2023-07-30 0.15.2
 =================

README.rst

@@ -154,6 +154,12 @@ How to find dashboards which use non-existing data sources?
     # Display only dashboards which have missing data sources, along with their names.
     grafana-wtf explore dashboards --format=json | jq '.[] | select( .datasources_missing ) | .dashboard + {ds_missing: .datasources_missing[] | [.name]}'
 
+How to list all queries used in all dashboards?
+::
+
+    grafana-wtf explore dashboards --data-details --format=json | \
+        jq -r '.[].details | values[] | .[].query // "null"'
+
 
 Searching for strings
 =====================

grafana_wtf/commands.py

@@ -29,7 +29,7 @@ def run():
     Usage:
       grafana-wtf [options] info
       grafana-wtf [options] explore datasources
-      grafana-wtf [options] explore dashboards
+      grafana-wtf [options] explore dashboards [--data-details]
       grafana-wtf [options] find [<search-expression>]
       grafana-wtf [options] replace <search-expression> <replacement> [--dry-run]
       grafana-wtf [options] log [<dashboard_uid>] [--number=<count>] [--head=<count>] [--tail=<count>] [--reverse] [--sql=<sql>]
@@ -91,6 +91,12 @@ def run():
       # Display all dashboards using data sources with a specific type. Here: InfluxDB.
       grafana-wtf explore dashboards --format=json | jq 'select( .[] | .datasources | .[].type=="influxdb" )'
 
+      # Display dashboards and many more details about where data source queries are happening.
+      # Specifically, within "panels/targets", "annotations", and "templating" slots.
+      grafana-wtf explore dashboards --data-details --format=json
+
+      # Display all database queries within dashboards.
+      grafana-wtf explore dashboards --data-details --format=json | jq -r '.[].details | values[] | .[].query // "null"'
 
     Find dashboards and data sources:
 
@@ -298,7 +304,7 @@ def run():
         output_results(output_format, results)
 
     if options.explore and options.dashboards:
-        results = engine.explore_dashboards()
+        results = engine.explore_dashboards(with_data_details=options.data_details)
         output_results(output_format, results)
 
     if options.info:

grafana_wtf/core.py

@@ -447,7 +447,7 @@ def explore_datasources(self):
 
         return response
 
-    def explore_dashboards(self):
+    def explore_dashboards(self, with_data_details: bool = False):
         # Prepare indexes, mapping dashboards by uid, datasources by name
         # as well as dashboards to datasources and vice versa.
         ix = Indexer(engine=self)
@@ -481,8 +481,8 @@ def explore_dashboards(self):
                 dashboard=dashboard, datasources=datasources_existing, grafana_url=self.grafana_url
             )
 
-            # Format results in a more compact form, using only a subset of all the attributes.
-            result = item.format_compact()
+            # Format results, using only a subset of all the attributes.
+            result = item.format(with_data_details=with_data_details)
 
             # Add information about missing data sources.
             if datasources_missing:

grafana_wtf/model.py

@@ -91,21 +91,99 @@ class DashboardExplorationItem:
     datasources: List[Munch]
     grafana_url: str
 
-    def format_compact(self):
+    def format(self, with_data_details: bool = False):
+        """
+        Generate a representation from selected information.
+
+        - dashboard
+        - datasources
+        - details
+            - panels/targets
+            - annotations
+            - templating
+        """
         dbshort = OrderedDict(
             title=self.dashboard.dashboard.title,
             uid=self.dashboard.dashboard.uid,
             path=self.dashboard.meta.url,
             url=urljoin(self.grafana_url, self.dashboard.meta.url),
         )
-        item = OrderedDict(dashboard=dbshort)
+
+        dsshort = []
         for datasource in self.datasources:
-            item.setdefault("datasources", [])
-            dsshort = OrderedDict(
+            item = OrderedDict(
                 uid=datasource.get("uid"),
                 name=datasource.name,
                 type=datasource.type,
                 url=datasource.url,
             )
-            item["datasources"].append(dsshort)
-        return item
+            dsshort.append(item)
+
+        data = Munch(dashboard=dbshort, datasources=dsshort)
+        if with_data_details:
+            data.details = self.collect_data_details()
+        return data
+
+    def collect_data_details(self):
+        """
+        Collect details concerned about data from dashboard information.
+        """
+
+        dbdetails = DashboardDetails(dashboard=self.dashboard)
+
+        ds_panels = self.collect_data_nodes(dbdetails.panels)
+        ds_annotations = self.collect_data_nodes(dbdetails.annotations)
+        ds_templating = self.collect_data_nodes(dbdetails.templating)
+
+        targets = []
+        for panel in ds_panels:
+            panel_item = self._format_panel_compact(panel)
+            for target in panel.targets:
+                target["_panel"] = panel_item
+                targets.append(target)
+
+        response = OrderedDict(targets=targets, annotations=ds_annotations, templating=ds_templating)
+
+        return response
+
+    @staticmethod
+    def collect_data_nodes(element):
+        """
+        Select all element nodes which have a "datasource" attribute.
+        """
+        element = element or []
+        items = []
+
+        def add(item):
+            if item is not None and item not in items:
+                items.append(item)
+
+        for node in element:
+            if "datasource" in node and node["datasource"]:
+                add(node)
+
+        return items
+
+    @staticmethod
+    def _format_panel_compact(panel):
+        """
+        Return a compact representation of panel information.
+        """
+        attributes = ["id", "title", "type", "datasource"]
+        data = OrderedDict()
+        for attribute in attributes:
+            data[attribute] = panel.get(attribute)
+        return data
+
+    @staticmethod
+    def _format_data_node_compact(item: Dict) -> Dict:
+        """
+        Return a compact representation of an element concerned about data.
+        """
+        data = OrderedDict()
+        data["datasource"] = item.get("datasource")
+        data["type"] = item.get("type")
+        for key, value in item.items():
+            if "query" in key.lower():
+                data[key] = value
+        return data

tests/test_commands.py

@@ -1,5 +1,6 @@
 import warnings
 
+from munch import munchify
 from packaging import version
 
 warnings.filterwarnings("ignore", category=DeprecationWarning, module=".*docopt.*")
@@ -463,6 +464,36 @@ def test_explore_dashboards_grafana7up(grafana_version, ldi_resources, capsys, c
     assert dashboard["datasources_missing"][0]["type"] is None
 
 
+def test_explore_dashboards_data_details(ldi_resources, capsys, caplog):
+    """
+    Explore more details of dashboards, wrt. to data and queries.
+    """
+
+    # Only provision specific dashboard.
+    ldi_resources(dashboards=["tests/grafana/dashboards/ldi-v33.json"])
+
+    # Compute exploration.
+    set_command("explore dashboards --data-details", "--format=yaml")
+
+    # Run command and capture YAML output.
+    with caplog.at_level(logging.DEBUG):
+        grafana_wtf.commands.run()
+    captured = capsys.readouterr()
+    data = yaml.safe_load(captured.out)
+
+    # Proof the output is correct.
+    assert len(data) == 1
+    dashboard = munchify(data[0])
+    assert dashboard.details.targets[0]._panel.id == 18
+    assert dashboard.details.targets[0]._panel.type == "graph"
+    assert dashboard.details.targets[0]._panel.datasource.type == "influxdb"
+    assert dashboard.details.targets[0]._panel.datasource.uid == "PDF2762CDFF14A314"
+    assert dashboard.details.targets[0].fields == [{'func': 'mean', 'name': 'P1'}]
+    assert dashboard.details.templating[0].query == \
+           "SELECT osm_country_code AS __value, country_and_countrycode AS __text " \
+           "FROM ldi_network ORDER BY osm_country_code"
+
+
 def test_explore_dashboards_empty_annotations(grafana_version, create_datasource, create_dashboard, capsys, caplog):
     # Create a dashboard with an anomalous value in the "annotations" slot.
     dashboard = mkdashboard(title="foo")
@@ -488,7 +519,7 @@ def test_explore_dashboards_empty_annotations(grafana_version, create_datasource
         assert len(dashboard["dashboard"]["uid"]) == 36
     else:
         assert len(dashboard["dashboard"]["uid"]) == 9
-    assert "datasources" not in dashboard
+    assert dashboard["datasources"] == []
     assert "datasources_missing" not in dashboard