API Compare Dashboard Report

lib/superset/dashboard/compare.rb

@@ -36,44 +36,50 @@ def second_dashboard
       end
 
       def list_datasets
-        puts "\n ====== DASHBOARD DATASETS ====== "
+        puts "\n >>>>>>>>>>>>>>>>>>>>>>>>> DASHBOARD DATASETS <<<<<<<<<<<<<<<<<<<<<<<<<<<< "
         Superset::Dashboard::Datasets::List.new(dashboard_id: first_dashboard_id).list
+        puts "\n"
         Superset::Dashboard::Datasets::List.new(dashboard_id: second_dashboard_id).list
       end
 
       def list_charts
-        puts "\n ====== DASHBOARD CHARTS ====== "
+        puts "\n >>>>>>>>>>>>>>>>>>>>>>>>> DASHBOARD CHARTS <<<<<<<<<<<<<<<<<<<<<<<<<<<< "
         Superset::Dashboard::Charts::List.new(first_dashboard_id).list
-        puts ''
+        puts "\n"
         Superset::Dashboard::Charts::List.new(second_dashboard_id).list
       end
 
       def list_native_filters
-        puts "\n ====== DASHBOARD NATIVE FILTERS ====== "
+        puts "\n >>>>>>>>>>>>>>>>>>>>>>>>> DASHBOARD NATIVE FILTERS <<<<<<<<<<<<<<<<<<<<<<<<<<<< "
         list_native_filters_for(first_dashboard)
-        puts ''
+        puts "\n"
         list_native_filters_for(second_dashboard)
       end
 
       def list_cross_filters
-        puts "\n ====== DASHBOARD CROSS FILTERS ====== "
+        puts "\n >>>>>>>>>>>>>>>>>>>>>>>>> DASHBOARD CROSS FILTERS <<<<<<<<<<<<<<<<<<<<<<<<<<<< "
         list_cross_filters_for(first_dashboard)
-        puts ''
         list_cross_filters_for(second_dashboard)
       end
 
       def native_filter_configuration(dashboard_result)
         rows = []
         JSON.parse(dashboard_result['json_metadata'])['native_filter_configuration'].each do |filter|
-          filter['targets'].each {|t| rows << [ t['column']['name'], t['datasetId'] ] }
+          filter['targets'].each do |t|
+            if t['column']
+              rows << [ filter['name'], t['column']['name'], t['datasetId'] ]
+            else
+              rows << [ filter['name'], '>NO DATASET LINKED<', t['datasetId'] ] # some filters don't have a dataset linked, ie date filter
+            end
+          end
         end
         rows
       end
 
       def list_native_filters_for(dashboard_result)
         puts Terminal::Table.new(
           title: [dashboard_result['id'], dashboard_result['dashboard_title']].join(' - '),
-          headings: ['Filter Name', 'Dataset Id'],
+          headings: ['Filter Name', 'Dataset Column', 'Dataset Id'],
           rows: native_filter_configuration(dashboard_result)
         )
       end

lib/superset/dashboard/datasets/list.rb

@@ -9,8 +9,8 @@ module Datasets
       class List < Superset::Request
         attr_reader :id, :include_filter_datasets # id - dashboard id
 
-        def self.call(id)
-          self.new(id).list
+        def self.call(dashboard_id: id)
+          self.new(dashboard_id: id).list
         end
 
         def initialize(dashboard_id:, include_filter_datasets: false)
@@ -48,6 +48,28 @@ def datasets_details
           chart_datasets + filter_datasets(filter_dataset_ids_not_used_in_charts)
         end
 
+        def rows
+          datasets_details.map do |d|
+            [
+              d[:id],
+              d[:datasource_name],
+              d[:database][:id],
+              d[:database][:name],
+              d[:database][:backend],
+              d[:schema],
+              d[:filter_only]
+            ]
+          end
+        end
+
+        def title
+          @title ||= [id, dashboard.title].join(' ')
+        end
+
+        def dashboard
+          @dashboard ||= Superset::Dashboard::Get.new(id)
+        end
+
         private
 
         def filter_dataset_ids
@@ -73,30 +95,7 @@ def route
         def list_attributes
           ['id', 'datasource_name', 'database_id', 'database_name', 'database_backend', 'schema', 'filter_only'].map(&:to_sym)
         end
-
-        def rows
-          datasets_details.map do |d|
-            [
-              d[:id],
-              d[:datasource_name],
-              d[:database][:id],
-              d[:database][:name],
-              d[:database][:backend],
-              d[:schema],
-              d[:filter_only]
-            ]
-          end
-        end
-
-        # when displaying a list of datasets, show dashboard title as well
-        def title
-          @title ||= [id, dashboard.title].join(' ')
-        end
-
-        def dashboard
-          @dashboard ||= Superset::Dashboard::Get.new(id)
-        end
       end
     end
   end
-end
+end

lib/superset/dashboard/list.rb

@@ -87,6 +87,10 @@ def ids_not_in_filters
         ids_not_in.map {|id| "(col:id,opr:neq,value:'#{id}')"}.join(',')
       end
 
+      def order_by
+        ",order_column:changed_on,order_direction:desc"
+      end
+
       def list_attributes
         [:id, :dashboard_title, :status, :url]
       end

lib/superset/dashboard/list_all.rb

@@ -0,0 +1,62 @@
+# Superset has a 100 result limit for requests
+# This is a wrapper for Superset::Dashboard::List to recursively list all dashboards 
+
+# TODO - would be very handy to create a parent class for this
+#        to then be able to use the same pattern for other ::List classes
+
+module Superset
+  module Dashboard
+    class ListAll
+      include Display
+
+      def initialize(**kwargs)
+        kwargs.each do |key, value|
+          instance_variable_set("@#{key}", value)
+          self.class.attr_reader key
+        end
+      end
+
+      def constructor_args
+        instance_variables.each_with_object({}) do |var, hash|
+          hash[var.to_s.delete('@').to_sym] = instance_variable_get(var)
+        end
+      end
+
+      def perform
+        page_num = 0
+        boards = []
+        boards << next_group = Dashboard::List.new(page_num: page_num, **constructor_args).result
+        while !next_group.empty?
+          boards << next_group = Dashboard::List.new(page_num: page_num += 1, **constructor_args).result
+        end
+        @result = boards.flatten
+      end
+
+      def result
+        @result ||= []
+      end
+
+      def rows
+        result.map do |d|
+          list_attributes.map do |la|
+            la == :url ? "#{superset_host}#{d[la]}" : d[la]
+          end
+        end
+      end
+
+      def ids
+        result.map { |d| d[:id] }
+      end
+
+      private
+
+      def list_attributes
+        [:id, :dashboard_title, :status, :url]
+      end
+
+      def superset_host
+        ENV['SUPERSET_HOST']
+      end
+    end
+  end
+end

lib/superset/dataset/get.rb

@@ -38,7 +38,7 @@ def database_id
       end
 
       def sql
-        ['sql']
+        result['sql']
       end
 
       private

lib/superset/display.rb

@@ -18,6 +18,10 @@ def rows
       end
     end
 
+    def rows_hash
+      rows.map { |value| list_attributes.zip(value).to_h }
+    end
+
     def title
       self.class.to_s
     end

lib/superset/request.rb

@@ -33,7 +33,7 @@ def superset_host
     end
 
     def query_params
-      [filters, pagination].join
+      [filters, pagination, order_by].join
     end
 
     private
@@ -54,6 +54,13 @@ def pagination
       "page:#{page_num},page_size:#{PAGE_SIZE}"
     end
 
+    def order_by
+      # order options are to not be consistant across all objects
+      # eg changed_on is NOT available on all objects .. requires customization in each ::List class
+      #
+      # Example only: ",order_column:changed_on,order_direction:desc"
+    end
+
     def filters
       ""
     end
@@ -63,3 +70,4 @@ def logger
     end
   end
 end
+

lib/superset/services/dashboard_report.rb

@@ -0,0 +1,138 @@
+# Creates a log report on a set of dashboards
+# providing count of charts, datasets, and databases used in each dashboard
+# as well as optional data sovereignty information
+
+# Data Sovereignty in this context requires that all datasets used in a dashboard are from one database schema only.
+# Primarily used to identify potential issues with embedded dashboards where data sovereignty is a concern.
+
+# Usage:
+# Superset::Services::DashboardReport.new(dashboard_ids: [1,2,3]).perform
+
+module Superset
+  module Services
+    class DashboardReport
+
+      attr_reader :dashboard_ids, :report_on_data_sovereignty_only
+
+      def initialize(dashboard_ids: [], report_on_data_sovereignty_only: true)
+        @dashboard_ids = dashboard_ids
+        @report_on_data_sovereignty_only = report_on_data_sovereignty_only
+      end
+
+      def perform
+        create_dashboard_report
+        load_data_sovereignty_issues
+
+        report_on_data_sovereignty_only ? display_data_sovereignty_report : @report
+      end
+
+      private
+
+      def display_data_sovereignty_report
+        # filter by dashboards where 
+        # 1. A filter dataset is not part of the dashboard datasets (might be ok for some cases, ie a dummy dataset listing dates only)
+        # 2. There is more than one distinct dataset schema (never ok for embedded dashboards where the expected schema num is only one)
+
+        puts "Data Sovereignty Report"
+        puts "-----------------------"
+        puts "Possible Invalid Dashboard Datasets: #{@data_sovereignty_issues.count}"
+        @data_sovereignty_issues
+      end
+
+      # possible data sovereignty issues
+      def load_data_sovereignty_issues
+        @data_sovereignty_issues ||= begin
+          @report.map do |dashboard|
+            reasons = []
+            chart_dataset_ids = dashboard[:datasets][:chart_datasets].map{|d| d[:id]}
+
+            # add WARNING msg if any filters datasets are not part of the chart datasets
+            unknown_datasets = dashboard[:filters][:filter_dataset_ids] - chart_dataset_ids
+            if unknown_datasets.any?
+              reasons << "WARNING: One or more filter datasets is not included in chart datasets for " \
+                          "filter dataset ids: #{unknown_datasets.join(', ')}."
+              reasons << "DETAILS: #{unknown_dataset_details(unknown_datasets)}"
+            end
+
+            # add ERROR msg if multiple chart dataset schemas are found, ie all datasets should be sourced from the same db schema
+            chart_dataset_schemas = dashboard[:datasets][:chart_datasets].map{|d| d[:schema]}.uniq
+            if chart_dataset_schemas.count > 1
+              reasons << "ERROR: Multiple distinct chart dataset schemas found. Expected 1. Found #{chart_dataset_schemas.count}. " \
+                          "schema names: #{chart_dataset_schemas.join(', ') }"
+            end
+
+            { reasons: reasons, dashboard: dashboard } if reasons.any?
+          end.compact
+        end
+      end
+
+      def unknown_dataset_details(unknown_datasets)
+        unknown_datasets.map do |dataset_id|
+          d = Superset::Dataset::Get.new(dataset_id)
+          d.result
+          { id: d.id, name: d.title }
+        rescue Happi::Error::NotFound => e
+          { id: dataset_id, name: '>>>> ERROR: DATASET DOES NOT EXIST <<<<' }
+        end
+      end
+
+      def create_dashboard_report
+        @report ||= begin
+          dashboard_ids.map do |dashboard_id|
+            dashboard = dashboard_result(dashboard_id)
+            {
+              dashboard_id: dashboard_id,
+              dashboard_title: dashboard['dashboard_title'],
+              dashboard_url: dashboard['url'],
+              dashboard_tags: dashboard_tags(dashboard),
+              filters: filter_details(dashboard),
+              charts: chart_count(dashboard),
+              datasets: dataset_details(dashboard_id),
+            }
+          end
+        end
+      end
+
+      def filter_details(dashboard)
+        { 
+          filter_count: filter_count(dashboard),
+          filter_dataset_ids: filter_datasets(dashboard)
+        }
+      end
+
+      def filter_count(dashboard)
+        dashboard['json_metadata']['native_filter_configuration']&.count || 0
+      end
+
+      def filter_datasets(dashboard)
+        dashboard['json_metadata']['native_filter_configuration'].map do |filter|
+          filter['targets'].map{|d| d['datasetId']} if filter['type'] == 'NATIVE_FILTER'
+        end.flatten.compact.uniq
+      end
+
+      def chart_count(dashboard)
+        dashboard['json_metadata']['chart_configuration'].count
+      end
+
+      def dataset_details(dashboard_id)
+        datasets = Superset::Dashboard::Datasets::List.new(dashboard_id: dashboard_id, include_filter_datasets: true).rows_hash
+        { 
+          dataset_count: datasets.count,
+          chart_datasets: datasets
+        }
+      end
+
+      def dashboard_tags(dashboard)
+        dashboard['tags'].map{|t| t['name']}.join('|')
+      end
+
+      def dashboard_result(dashboard_id)
+       # convert json_metadata within result to a hash
+       board = Superset::Dashboard::Get.new(dashboard_id)
+       board.result['json_metadata'] = JSON.parse(board.result['json_metadata'])
+       board.result['url'] = board.url # add full url to the dashboard result
+       board.result
+      end
+    end
+  end
+end