Merge pull request #41 from rdytech/NEP-18619_adjust_dashboard_dataset_endpoint_to_include_filter_dataset

[NEP-18619]: Adjust the endpoint for Dashboard::Datasets::List to optionally include filter datasets

Author: soundarya-mv

CHANGELOG.md

@@ -1,5 +1,13 @@
 ## Change Log
 
+## 0.2.3 - 2024-11-15
+
+* modifies the `Superset::Dashboard::Datasets::List.new(id).dataset_details` and `Superset::Dashboard::Datasets::List.new(id).list` to optionally include filter datasets as well to duplicate those during the dashboard duplication process. It also add a new column "Filter only" in the result which shows if a dataset is used only on filters
+* This also adds an additional check in source dataset duplication that if any dataset already exists with the new name in the target schema we can use the existing dataset itself for the new dashboard also.
+* Exporting a Dashboard will also include the Zip file in the repo backup (destination_path) folder.
+* Exporting a Dashboard and copying the files to the backup folder will also clear out any previous backup files that no longer exist in the current zip export.
+* Adds a put endpoint for chart and datasets which is needed for the bulk update of set of charts/datasets
+
 ## 0.2.2 - 2024-10-10
 
 * add ImportDashboardAcrossEnvironments class for transfering between superset environments

Gemfile.lock

@@ -14,7 +14,7 @@ GIT
 PATH
   remote: .
   specs:
-    superset (0.2.2)
+    superset (0.2.3)
       dotenv (~> 2.7)
       enumerate_it (~> 1.7.0)
       faraday (~> 1.0)

doc/usage.md

@@ -10,7 +10,9 @@ Superset::Database::GetSchemas.new(1).list # get schemas for database 1
 
 Superset::Dashboard::List.call
 Superset::Dashboard::List.new(title_contains: 'Sales').list
-Superset::Dashboard::Datasets::List.new(10).list  # get all datasets for dashboard 10
+Superset::Dashboard::Datasets::List.new(dashboard_id: 10).list  # get all datasets for dashboard 10
+
+Superset::Dashboard::Datasets::List.new(dashboard_id: 10, include_filter_datasets: true).list  # get all datasets for dashboard 10 including the ones used only in dashboard filters
 Superset::Dashboard::WarmUpCache.new(dashboard_id: 10).perform
 
 Superset::Dataset::List.call

lib/superset/dashboard/bulk_delete_cascade.rb

@@ -31,7 +31,7 @@ def perform
       private
 
       def delete_datasets(dashboard_id)
-        datasets_to_delete = Superset::Dashboard::Datasets::List.new(dashboard_id).datasets_details.map{|d| d[:id] }
+        datasets_to_delete = Superset::Dashboard::Datasets::List.new(dashboard_id: dashboard_id).datasets_details.map{|d| d[:id] }
         Superset::Dataset::BulkDelete.new(dataset_ids: datasets_to_delete).perform if datasets_to_delete.any?
       end
 

lib/superset/dashboard/compare.rb

@@ -37,8 +37,8 @@ def second_dashboard
 
       def list_datasets
         puts "\n ====== DASHBOARD DATASETS ====== "
-        Superset::Dashboard::Datasets::List.new(first_dashboard_id).list
-        Superset::Dashboard::Datasets::List.new(second_dashboard_id).list
+        Superset::Dashboard::Datasets::List.new(dashboard_id: first_dashboard_id).list
+        Superset::Dashboard::Datasets::List.new(dashboard_id: second_dashboard_id).list
       end
 
       def list_charts

lib/superset/dashboard/datasets/list.rb

@@ -7,14 +7,15 @@ module Superset
   module Dashboard
     module Datasets
       class List < Superset::Request
-        attr_reader :id  # dashboard id
+        attr_reader :id, :include_filter_datasets # id - dashboard id
 
         def self.call(id)
           self.new(id).list
         end
 
-        def initialize(id)
-          @id = id
+        def initialize(dashboard_id:, include_filter_datasets: false)
+          @id = dashboard_id
+          @include_filter_datasets = include_filter_datasets
         end
 
         def perform
@@ -36,37 +37,64 @@ def schemas
         end
 
         def datasets_details
-          result.map do |details|
+          chart_datasets = result.map do |details|
             details.slice('id', 'datasource_name', 'schema', 'sql').merge('database' => details['database'].slice('id', 'name', 'backend')).with_indifferent_access
           end
+          return chart_datasets unless include_filter_datasets
+          chart_dataset_ids = chart_datasets.map{|d| d['id'] }
+          filter_dataset_ids_not_used_in_charts = filter_dataset_ids - chart_dataset_ids
+          return chart_datasets if filter_dataset_ids_not_used_in_charts.empty?
+          # returning chart and filter datasets
+          chart_datasets + filter_datasets(filter_dataset_ids_not_used_in_charts)
         end
 
         private
 
+        def filter_dataset_ids
+          @filter_dataset_ids ||= dashboard.filter_configuration.map { |c| c['targets'] }.flatten.compact.map { |c| c['datasetId'] }.flatten.compact.uniq
+        end
+
+        def filter_datasets(filter_dataset_ids_not_used_in_charts)
+          filter_dataset_ids_not_used_in_charts.map do |filter_dataset_id|
+            filter_dataset = Superset::Dataset::Get.new(filter_dataset_id).result
+            database_info = {
+              'id' => filter_dataset['database']['id'],
+              'name' => filter_dataset['database']['database_name'],
+              'backend' => filter_dataset['database']['backend']
+            }
+            filter_dataset.slice('id', 'datasource_name', 'schema', 'sql').merge('database' => database_info, 'filter_only': true).with_indifferent_access
+          end
+        end
+
         def route
           "dashboard/#{id}/datasets"
         end
 
         def list_attributes
-          ['id', 'datasource_name', 'database_id', 'database_name', 'database_backend', 'schema'].map(&:to_sym)
+          ['id', 'datasource_name', 'database_id', 'database_name', 'database_backend', 'schema', 'filter_only'].map(&:to_sym)
         end
 
         def rows
-          result.map do |d|
+          datasets_details.map do |d|
             [
               d[:id],
               d[:datasource_name],
               d[:database][:id],
               d[:database][:name],
               d[:database][:backend],
-              d[:schema]
+              d[:schema],
+              d[:filter_only]
             ]
           end
         end
 
         # when displaying a list of datasets, show dashboard title as well
         def title
-          @title ||= [id, Superset::Dashboard::Get.new(id).title].join(' ')
+          @title ||= [id, dashboard.title].join(' ')
+        end
+
+        def dashboard
+          @dashboard ||= Superset::Dashboard::Get.new(id)
         end
       end
     end

lib/superset/dashboard/get.rb

@@ -25,6 +25,10 @@ def title
       def json_metadata
         JSON.parse(result['json_metadata'])
       end
+      
+      def filter_configuration
+        json_metadata['native_filter_configuration'] || []
+      end
 
       def positions
         JSON.parse(result['position_json'])

lib/superset/dashboard/warm_up_cache.rb

@@ -35,7 +35,7 @@ def validate_dashboard_id
       end
 
       def fetch_dataset_details(dashboard_id)
-        Superset::Dashboard::Datasets::List.new(dashboard_id).datasets_details.map { |dataset| dataset['database'].slice('name').merge(dataset.slice('datasource_name'))}
+        Superset::Dashboard::Datasets::List.new(dashboard_id: dashboard_id).datasets_details.map { |dataset| dataset['database'].slice('name').merge(dataset.slice('datasource_name'))}
       end
     end
   end

lib/superset/services/duplicate_dashboard.rb

@@ -93,13 +93,18 @@ def duplicate_source_dashboard_datasets
           # duplicate the dataset, renaming to use of suffix as the target_schema
           # reason: there is a bug(or feature) in the SS API where a dataset name must be uniq when duplicating.  
           # (note however renaming in the GUI to a dup name works fine)
-          new_dataset_id = Superset::Dataset::Duplicate.new(source_dataset_id: dataset[:id], new_dataset_name: "#{dataset[:datasource_name]}-#{target_schema}").perform
-
+          new_dataset_name = "#{dataset[:datasource_name]}-#{target_schema}"
+          existing_datasets = Superset::Dataset::List.new(title_equals: new_dataset_name, schema_equals: target_schema).result
+          if existing_datasets.any?
+            logger.info "Dataset #{existing_datasets[0]["table_name"]} already exists. Reusing it"
+            new_dataset_id = existing_datasets[0]["id"] # assuming that we do not name multiple datasets with same name in a single schema
+          else
+            new_dataset_id = Superset::Dataset::Duplicate.new(source_dataset_id: dataset[:id], new_dataset_name: new_dataset_name).perform
+            # update the new dataset with the target schema and target database
+            Superset::Dataset::UpdateSchema.new(source_dataset_id: new_dataset_id, target_database_id: target_database_id, target_schema: target_schema).perform
+          end
           # keep track of the previous dataset and the matching new dataset_id
           dataset_duplication_tracker <<  { source_dataset_id: dataset[:id], new_dataset_id: new_dataset_id }
-
-          # update the new dataset with the target schema and target database
-          Superset::Dataset::UpdateSchema.new(source_dataset_id: new_dataset_id, target_database_id: target_database_id, target_schema: target_schema).perform
         end
       end
 
@@ -179,7 +184,7 @@ def new_dashboard
 
       # retrieve the datasets that will be duplicated
       def source_dashboard_datasets
-        @source_dashboard_datasets ||= Superset::Dashboard::Datasets::List.new(source_dashboard_id).datasets_details
+        @source_dashboard_datasets ||= Superset::Dashboard::Datasets::List.new(dashboard_id: source_dashboard_id, include_filter_datasets: true).datasets_details
       rescue => e
         raise "Unable to retrieve datasets for source dashboard #{source_dashboard_id}: #{e.message}"
       end
@@ -199,7 +204,7 @@ def validate_params
         raise ValidationError, "The source dashboard datasets are required to point to one schema only. Actual schema list is #{source_dashboard_schemas.join(',')}" if source_dashboard_has_more_than_one_schema?
         raise ValidationError, "One or more source dashboard filters point to a different schema than the dashboard charts. Identified Unpermittied Filter Dataset Ids are #{unpermitted_filter_dataset_ids.to_s}" if unpermitted_filter_dataset_ids.any?
 
-        # new dataset validations
+        # new dataset validations - Need to be commented for EU dashboard duplication as we are using the existing datasets for the new dashboard
         raise ValidationError, "DATASET NAME CONFLICT: The Target Schema #{target_schema} already has existing datasets named: #{target_schema_matching_dataset_names.join(',')}" unless target_schema_matching_dataset_names.empty?
         validate_source_dashboard_datasets_sql_does_not_hard_code_schema
 
@@ -241,7 +246,7 @@ def source_dashboard_dataset_names
       # here we will need to decide if we want to use the existing dataset or not see NEP-????
       # for now we will exit with an error if we find any existing datasets of the same name
       def target_schema_matching_dataset_names
-        source_dashboard_dataset_names.map do |source_dataset_name|
+        @target_schema_matching_dataset_names ||= source_dashboard_dataset_names.map do |source_dataset_name|
           existing_names = Superset::Dataset::List.new(title_contains: source_dataset_name, schema_equals: target_schema).result.map{|t|t['table_name']}.uniq # contains match to cover with suffix as well
           unless existing_names.flatten.empty?
             logger.error "  HALTING PROCESS: Schema #{target_schema} already has Dataset called #{existing_names}"
@@ -255,11 +260,7 @@ def source_dashboard_dataset_ids
       end
 
       def source_dashboard_filter_dataset_ids
-        filters_configuration = JSON.parse(source_dashboard.result['json_metadata'])['native_filter_configuration'] || []
-        return Array.new unless filters_configuration && filters_configuration.any?
-
-        # pull only the filters dataset ids from the dashboard
-        filters_configuration.map { |c| c['targets'] }.flatten.compact.map { |c| c['datasetId'] }.flatten.compact
+        @filter_dataset_ids ||= source_dashboard.filter_configuration.map { |c| c['targets'] }.flatten.compact.map { |c| c['datasetId'] }.flatten.compact.uniq
       end
 
       # Primary Assumption is that all charts datasets on the source dashboard are pointing to the same database schema

lib/superset/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Superset
-  VERSION = "0.2.2"
+  VERSION = "0.2.3"
 end