Add --queries-only option to explore dashboards subcommand

Author: amotl

CHANGES.rst

@@ -10,6 +10,7 @@ in progress
 - Fix wrong ``jq`` commands in documentation. Thanks, @rahulnandan.
 - Fix collecting data information from dashboards w/o ``targets`` slots
   in panels
+- Add ``--queries-only`` option to ``explore dashboards`` subcommand
 
 2023-10-03 0.16.0
 =================

README.rst

@@ -206,8 +206,8 @@ How to find dashboards using specific data sources?
 How to list all queries used in all dashboards?
 ::
 
-    grafana-wtf explore dashboards --data-details --format=json | \
-        jq -r '.[].details | values[] | .[].query // "null"'
+    grafana-wtf explore dashboards --data-details --queries-only --format=json | \
+        jq '.[].details | values[] | .[] | .expr,.jql,.query,.rawSql | select( . != null and . != "" )'
 
 
 Searching for strings

grafana_wtf/commands.py

@@ -33,7 +33,7 @@ def run():
     Usage:
       grafana-wtf [options] info
       grafana-wtf [options] explore datasources
-      grafana-wtf [options] explore dashboards [--data-details]
+      grafana-wtf [options] explore dashboards [--data-details] [--queries-only]
       grafana-wtf [options] find [<search-expression>]
       grafana-wtf [options] replace <search-expression> <replacement> [--dry-run]
       grafana-wtf [options] log [<dashboard_uid>] [--number=<count>] [--head=<count>] [--tail=<count>] [--reverse] [--sql=<sql>]
@@ -98,11 +98,11 @@ def run():
       grafana-wtf explore dashboards --format=json | jq '.[] | select(.datasources | .[].type=="influxdb")'
 
       # Display dashboards and many more details about where data source queries are happening.
-      # Specifically, within "panels/targets", "annotations", and "templating" slots.
+      # Specifically, within "panels", "annotations", and "templating" slots.
       grafana-wtf explore dashboards --data-details --format=json
 
       # Display all database queries within dashboards.
-      grafana-wtf explore dashboards --data-details --format=json | jq -r '.[].details | values[] | .[].query // "null"'
+      grafana-wtf explore dashboards --data-details --queries-only --format=json | jq '.[].details | values[] | .[] | .expr,.jql,.query,.rawSql | select( . != null and . != "" )'
 
     Find dashboards and data sources:
 
@@ -315,7 +315,7 @@ def run():
         output_results(output_format, results)
 
     if options.explore and options.dashboards:
-        results = engine.explore_dashboards(with_data_details=options.data_details)
+        results = engine.explore_dashboards(with_data_details=options.data_details, queries_only=options.queries_only)
         output_results(output_format, results)
 
     if options.info:

grafana_wtf/core.py

@@ -447,7 +447,7 @@ def explore_datasources(self):
 
         return response
 
-    def explore_dashboards(self, with_data_details: bool = False):
+    def explore_dashboards(self, with_data_details: bool = False, queries_only: bool = False):
         # Prepare indexes, mapping dashboards by uid, datasources by name
         # as well as dashboards to datasources and vice versa.
         ix = Indexer(engine=self)
@@ -482,7 +482,9 @@ def explore_dashboards(self, with_data_details: bool = False):
             )
 
             # Format results, using only a subset of all the attributes.
-            result = item.format(with_data_details=with_data_details)
+            result = item.format(with_data_details=with_data_details, queries_only=queries_only)
+            if result is None:
+                continue
 
             # Add information about missing data sources.
             if datasources_missing:

grafana_wtf/model.py

@@ -44,6 +44,118 @@ def templating(self) -> List:
         return self.dashboard.dashboard.get("templating", {}).get("list", [])
 
 
+@dataclasses.dataclass
+class DashboardDataDetails:
+    """
+    Manage details concerned about "data"-relevant information,
+    a subset of the complete bunch of dashboard information.
+    """
+
+    panels: List[Dict]
+    annotations: List[Dict]
+    templating: List[Dict]
+
+    def to_munch(self):
+        return Munch(panels=self.panels, annotations=self.annotations, templating=self.templating)
+
+    @classmethod
+    def from_dashboard_details(cls, dbdetails: DashboardDetails):
+        ds_panels = cls.collect_data_nodes(dbdetails.panels)
+        ds_annotations = cls.collect_data_nodes(dbdetails.annotations)
+        ds_templating = cls.collect_data_nodes(dbdetails.templating)
+
+        # Inline panel information into data details.
+        targets = []
+        for panel in ds_panels:
+            panel_item = cls._format_panel_compact(panel)
+            if "targets" in panel:
+                for target in panel.targets:
+                    target["_panel"] = panel_item
+                    targets.append(target)
+
+        return cls(panels=targets, annotations=ds_annotations, templating=ds_templating)
+
+    @staticmethod
+    def collect_data_nodes(element):
+        """
+        Select all element nodes which have a "datasource" attribute.
+        """
+        element = element or []
+        items = []
+
+        def add(item):
+            if item is not None and item not in items:
+                items.append(item)
+
+        for node in element:
+            if "datasource" in node and node["datasource"]:
+                add(node)
+
+        return items
+
+    @staticmethod
+    def _format_panel_compact(panel):
+        """
+        Return a compact representation of panel information.
+        """
+        attributes = ["id", "title", "type", "datasource"]
+        data = OrderedDict()
+        for attribute in attributes:
+            data[attribute] = panel.get(attribute)
+        return data
+
+    @staticmethod
+    def _format_data_node_compact(item: Dict) -> Dict:
+        """
+        Return a compact representation of an element concerned about data.
+        """
+        data = OrderedDict()
+        data["datasource"] = item.get("datasource")
+        data["type"] = item.get("type")
+        for key, value in item.items():
+            if "query" in key.lower():
+                data[key] = value
+        return data
+
+    def queries_only(self):
+        """
+        Return a representation of data details information, only where query expressions are present.
+        """
+        # All attributes containing query-likes.
+        attributes_query_likes = ["expr", "jql", "query", "rawSql", "target"]
+
+        attributes = [
+            # Carry over datasource and panel references for informational purposes.
+            "datasource",
+            "_panel",
+        ] + attributes_query_likes
+
+        def transform(section):
+            new_items = []
+            for item in section:
+                new_item = OrderedDict()
+                for key, value in item.items():
+                    if key in attributes:
+                        new_item[key] = value
+                if any(attr in attributes_query_likes for attr in new_item.keys()):
+                    # Filter annotations without any value.
+                    if "target" in new_item and isinstance(new_item["target"], dict):
+                        if new_item["target"].get("type") == "dashboard":
+                            continue
+                    # Unnest items with nested "query" slot.
+                    for slot in ["query", "target"]:
+                        if slot in new_item and isinstance(new_item[slot], dict) and "query" in new_item[slot]:
+                            new_item["query"] = new_item[slot]["query"]
+                    new_items.append(new_item)
+            return new_items
+
+        return DashboardDataDetails(
+            panels=transform(self.panels),
+            annotations=transform(self.annotations),
+            templating=transform(self.templating),
+        )
+
+
 @dataclasses.dataclass
 class DatasourceItem:
     uid: Optional[str] = None
@@ -94,7 +206,7 @@ class DashboardExplorationItem:
     datasources: List[Munch]
     grafana_url: str
 
-    def format(self, with_data_details: bool = False):
+    def format(self, with_data_details: bool = False, queries_only: bool = False):
         """
         Generate a representation from selected information.
 
@@ -124,70 +236,19 @@ def format(self, with_data_details: bool = False):
 
         data = Munch(dashboard=dbshort, datasources=dsshort)
         if with_data_details:
-            data.details = self.collect_data_details()
+            details = self.collect_data_details(queries_only=queries_only)
+            if not details.panels and not details.annotations and not details.templating:
+                return None
+            data.details = details.to_munch()
         return data
 
-    def collect_data_details(self):
+    def collect_data_details(self, queries_only: bool = False):
         """
         Collect details concerned about data from dashboard information.
         """
 
         dbdetails = DashboardDetails(dashboard=self.dashboard)
-
-        ds_panels = self.collect_data_nodes(dbdetails.panels)
-        ds_annotations = self.collect_data_nodes(dbdetails.annotations)
-        ds_templating = self.collect_data_nodes(dbdetails.templating)
-
-        targets = []
-        for panel in ds_panels:
-            panel_item = self._format_panel_compact(panel)
-            if "targets" in panel:
-                for target in panel.targets:
-                    target["_panel"] = panel_item
-                    targets.append(target)
-
-        response = OrderedDict(targets=targets, annotations=ds_annotations, templating=ds_templating)
-
-        return response
-
-    @staticmethod
-    def collect_data_nodes(element):
-        """
-        Select all element nodes which have a "datasource" attribute.
-        """
-        element = element or []
-        items = []
-
-        def add(item):
-            if item is not None and item not in items:
-                items.append(item)
-
-        for node in element:
-            if "datasource" in node and node["datasource"]:
-                add(node)
-
-        return items
-
-    @staticmethod
-    def _format_panel_compact(panel):
-        """
-        Return a compact representation of panel information.
-        """
-        attributes = ["id", "title", "type", "datasource"]
-        data = OrderedDict()
-        for attribute in attributes:
-            data[attribute] = panel.get(attribute)
-        return data
-
-    @staticmethod
-    def _format_data_node_compact(item: Dict) -> Dict:
-        """
-        Return a compact representation of an element concerned about data.
-        """
-        data = OrderedDict()
-        data["datasource"] = item.get("datasource")
-        data["type"] = item.get("type")
-        for key, value in item.items():
-            if "query" in key.lower():
-                data[key] = value
-        return data
+        dbdatadetails = DashboardDataDetails.from_dashboard_details(dbdetails)
+        if queries_only:
+            dbdatadetails = dbdatadetails.queries_only()
+        return dbdatadetails

tests/test_commands.py

@@ -487,11 +487,11 @@ def test_explore_dashboards_data_details(ldi_resources, capsys, caplog):
     # Proof the output is correct.
     assert len(data) == 1
     dashboard = munchify(data[0])
-    assert dashboard.details.targets[0]._panel.id == 18
-    assert dashboard.details.targets[0]._panel.type == "graph"
-    assert dashboard.details.targets[0]._panel.datasource.type == "influxdb"
-    assert dashboard.details.targets[0]._panel.datasource.uid == "PDF2762CDFF14A314"
-    assert dashboard.details.targets[0].fields == [{"func": "mean", "name": "P1"}]
+    assert dashboard.details.panels[0]._panel.id == 18
+    assert dashboard.details.panels[0]._panel.type == "graph"
+    assert dashboard.details.panels[0]._panel.datasource.type == "influxdb"
+    assert dashboard.details.panels[0]._panel.datasource.uid == "PDF2762CDFF14A314"
+    assert dashboard.details.panels[0].fields == [{"func": "mean", "name": "P1"}]
     assert (
         dashboard.details.templating[0].query
         == "SELECT osm_country_code AS __value, country_and_countrycode AS __text "