[NEP-18619]: Adjust the endpoint for Dashboard::Datasets::List to optionally include filter datasets

Author: soundarya-mv

lib/superset/dashboard/datasets/list.rb

@@ -7,14 +7,15 @@ module Superset
   module Dashboard
     module Datasets
       class List < Superset::Request
-        attr_reader :id  # dashboard id
+        attr_reader :id, :include_filter_datasets # dashboard id
 
         def self.call(id)
           self.new(id).list
         end
 
-        def initialize(id)
+        def initialize(id, include_filter_datasets = false)
           @id = id
+          @include_filter_datasets = include_filter_datasets
         end
 
         def perform
@@ -36,13 +37,31 @@ def schemas
         end
 
         def datasets_details
-          result.map do |details|
+          chart_datasets = result.map do |details|
             details.slice('id', 'datasource_name', 'schema', 'sql').merge('database' => details['database'].slice('id', 'name', 'backend')).with_indifferent_access
           end
+          return chart_datasets unless include_filter_datasets
+          chart_dataset_ids = chart_datasets.map{|d| d['id'] }
+          filter_dataset_ids_not_used_in_charts = filter_dataset_ids - chart_dataset_ids
+          return chart_datasets if filter_dataset_ids_not_used_in_charts.empty?
+          filter_datasets = filter_dataset_ids_not_used_in_charts.map do |filter_dataset_id|
+            filter_dataset = Superset::Dataset::Get.new(filter_dataset_id).result
+            database_info = {
+              'id' => filter_dataset['database']['id'],
+              'name' => filter_dataset['database']['database_name'],
+              'backend' => filter_dataset['database']['backend']
+            }
+            filter_dataset.slice('id', 'datasource_name', 'schema', 'sql').merge('database' => database_info).with_indifferent_access
+          end
+          dashboard_datasets = chart_datasets + filter_datasets
         end
 
         private
 
+        def filter_dataset_ids
+          @filter_dataset_ids ||= Superset::Dashboard::Filters::List.new(id).perform
+        end
+
         def route
           "dashboard/#{id}/datasets"
         end

lib/superset/dashboard/filters/list.rb

@@ -0,0 +1,27 @@
+module Superset
+  module Dashboard
+    module Filters
+      class List < Superset::Request
+        attr_reader :id
+
+        def initialize(id)
+          @id = id
+        end
+
+        def perform
+          filters_configuration = JSON.parse(dashboard.result['json_metadata'])['native_filter_configuration'] || []
+          return Array.new unless filters_configuration && filters_configuration.any?
+  
+          # pull only the filters dataset ids from the dashboard
+          filters_configuration.map { |c| c['targets'] }.flatten.compact.map { |c| c['datasetId'] }.flatten.compact
+        end
+
+        private
+
+        def dashboard
+          dashboard ||= Superset::Dashboard::Get.new(id)
+        end
+      end
+    end
+  end
+end

lib/superset/services/duplicate_dashboard.rb

@@ -93,7 +93,13 @@ def duplicate_source_dashboard_datasets
           # duplicate the dataset, renaming to use of suffix as the target_schema
           # reason: there is a bug(or feature) in the SS API where a dataset name must be uniq when duplicating.  
           # (note however renaming in the GUI to a dup name works fine)
-          new_dataset_id = Superset::Dataset::Duplicate.new(source_dataset_id: dataset[:id], new_dataset_name: "#{dataset[:datasource_name]}-#{target_schema}").perform
+          new_dataset_name = "#{dataset[:datasource_name]}-#{target_schema}"
+          existing_datasets = Dataset::List.new(title_equals: new_dataset_name, schema_equals: source_dataset.schema).result
+          if existing_dataset.any?
+            new_dataset_id = existing_datasets[0]["id"] # assuming that we do not name multiple datasets with same name in a single schema
+          else
+            new_dataset_id = Superset::Dataset::Duplicate.new(source_dataset_id: dataset[:id], new_dataset_name: new_dataset_name).perform
+          end
 
           # keep track of the previous dataset and the matching new dataset_id
           dataset_duplication_tracker <<  { source_dataset_id: dataset[:id], new_dataset_id: new_dataset_id }
@@ -179,7 +185,7 @@ def new_dashboard
 
       # retrieve the datasets that will be duplicated
       def source_dashboard_datasets
-        @source_dashboard_datasets ||= Superset::Dashboard::Datasets::List.new(source_dashboard_id).datasets_details
+        @source_dashboard_datasets ||= Superset::Dashboard::Datasets::List.new(source_dashboard_id, true).datasets_details
       rescue => e
         raise "Unable to retrieve datasets for source dashboard #{source_dashboard_id}: #{e.message}"
       end
@@ -255,11 +261,7 @@ def source_dashboard_dataset_ids
       end
 
       def source_dashboard_filter_dataset_ids
-        filters_configuration = JSON.parse(source_dashboard.result['json_metadata'])['native_filter_configuration'] || []
-        return Array.new unless filters_configuration && filters_configuration.any?
-
-        # pull only the filters dataset ids from the dashboard
-        filters_configuration.map { |c| c['targets'] }.flatten.compact.map { |c| c['datasetId'] }.flatten.compact
+        @filter_dataset_ids ||= Superset::Dashboard::Filters::List.new(id).perform
       end
 
       # Primary Assumption is that all charts datasets on the source dashboard are pointing to the same database schema