adjust import across env class and add spec

Author: jbat

lib/superset/services/import_dashboard_across_environment.rb

@@ -1,19 +1,18 @@
 =begin
-Superset::Services::ImportDashboardAcrossEnvironments
-
 This service is used to duplicate a dashboard from one environment to another.
-It will not create any database connections from an imported dashboard zip, therefore the target database yml configuration
-must already exist in the target superset environment.
+It will not create any database connections from an imported dashboard zip, therefore the target_database_yaml_file configuration
+must already exist as a database connection in the target superset environment.
 
 Currently handles only 1 Database yaml file in the zip file. ( ie only 1 common database connection per dashboards datasets )
 
-Requirements: 
- - target_database_yaml_file
- - target_database_schema
- - dashboard_export_zip
+Required Attributes: 
+ - target_database_yaml_file - location of the target database yaml config file
+ - target_database_schema - the schema name to be used in the target database
+ - dashboard_export_zip - location of the source dashboard export zip file to tranferred to a new superset Env
 
 Usage:
 Assuming you have exported a dashboard from the source environment and have the zip file, and have exported the target database yaml file
+
 Superset::Services::ImportDashboardAcrossEnvironments.new(
  target_database_yaml_file: '/tmp/database.yaml',
  target_database_schema: 'insert_schema_here',
@@ -22,15 +21,13 @@
 
 =end
 
-#require 'superset/file_utilities'
+require 'superset/file_utilities'
 require 'yaml'
 
 module Superset
   module Services
     class ImportDashboardAcrossEnvironments
-#      include FileUtilities
-
-      attr_reader :target_database_yaml_file, :target_database_schema, :dashboard_export_zip
+      include FileUtilities
 
       def initialize(target_database_yaml_file:, target_database_schema: ,dashboard_export_zip:)
         @target_database_yaml_file   = target_database_yaml_file
@@ -49,12 +46,14 @@ def perform
         create_new_dashboard_zip
       end
 
-      private
-
       def dashboard_config
         @dashboard_config ||= Superset::Services::DashboardLoader.new(dashboard_export_zip: dashboard_export_zip).perform
       end
 
+      private
+
+      attr_reader :target_database_yaml_file, :target_database_schema, :dashboard_export_zip
+
       def remove_source_database_config
         return if dashboard_config[:databases].blank?
         previous_database_name = dashboard_config[:databases]&.first[:content][:database_name]
@@ -79,13 +78,15 @@ def update_dataset_configs
         dashboard_config[:datasets].each do |dataset|
           dataset[:content][:database_uuid] = dashboard_config[:databases].first[:content][:uuid]
           dataset[:content][:schema]        = target_database_schema
-          File.open(dataset[:filename], 'w') { |f| f.write dataset[:content].to_yaml }
+          stringified_content = deep_transform_keys_to_strings(dataset[:content])
+          File.open(dataset[:filename], 'w') { |f| f.write stringified_content.to_yaml }
         end
       end
 
       def create_new_dashboard_zip
         Zip::File.open(new_zip_file, Zip::File::CREATE) do |zipfile|
           Dir[File.join(dashboard_export_root_path, '**', '**')].each do |file|
+#            puts "File Sub:" + file.sub(dashboard_export_root_path + '/', File.basename(dashboard_export_root_path) + '/' ).to_s
             zipfile.add(file.sub(dashboard_export_root_path + '/', File.basename(dashboard_export_root_path) + '/' ), file) if File.file?(file)
           end
         end
@@ -120,10 +121,25 @@ def previous_database_name
 
       def validate_params
         raise "Dashboard Export Zip file does not exist" unless File.exist?(dashboard_export_zip)
+        raise "Dashboard Export Zip file is not a zip file" unless File.extname(dashboard_export_zip) == '.zip'
         raise "Target Database YAML file does not exist" unless File.exist?(target_database_yaml_file)
-        raise "Only 1 Database YAML file is allowed in the zip file" if dashboard_config[:databases].size > 1
+        raise "Currently this class handles boards with single Database configs only. Multiple Database configs exist in zip file." if dashboard_config[:databases].size > 1
         raise "Target Database Schema cannot be blank" if target_database_schema.blank?
       end
+
+      # Method to recursively transform keys to strings
+      def deep_transform_keys_to_strings(value)
+        case value
+        when Hash
+          value.each_with_object({}) do |(k, v), result|
+            result[k.to_s] = deep_transform_keys_to_strings(v)
+          end
+        when Array
+          value.map { |v| deep_transform_keys_to_strings(v) }
+        else
+          value
+        end
+      end
     end
   end
 end

spec/fixtures/dashboard_export_with_multiple_databases.zip

@@ -0,0 +1,17 @@
+database_name: prod-examples
+sqlalchemy_uri: postgresql+psycopg2://superset:XXXXXXXXXX@the-prod-db-host:5432/superset
+cache_timeout: null
+expose_in_sqllab: true
+allow_run_async: true
+allow_ctas: true
+allow_cvas: true
+allow_dml: true
+allow_csv_upload: true
+extra:
+  metadata_params: {}
+  engine_params: {}
+  metadata_cache_timeout: {}
+  allows_virtual_table_explore: true
+  schemas_allowed_for_csv_upload: null
+uuid: cc21999b-649f-451f-816a-85b5977b9582
+version: 1.0.0

spec/fixtures/database-prod-examples.yaml

@@ -0,0 +1,98 @@
+require 'superset/services/import_dashboard_across_environment'
+
+RSpec.describe Superset::Services::ImportDashboardAcrossEnvironments do
+  let(:target_database_yaml_file) { 'spec/fixtures/database-prod-examples.yaml' }
+  let(:target_database_schema) { 'public' }
+  let(:dashboard_export_zip) { 'spec/fixtures/dashboard_18_export_20240322.zip' }
+  let(:service) { described_class.new(
+    target_database_yaml_file: target_database_yaml_file, 
+    target_database_schema:    target_database_schema, 
+    dashboard_export_zip:      dashboard_export_zip) }
+
+  def list_zip_contents(zip_file_path)
+    Zip::File.open(zip_file_path) do |zip_file|
+      zip_file.map(&:name)
+    end
+  end
+
+  describe '#perform' do
+    let(:result_zip) { service.perform }
+    let(:test_unziped_path) { '/tmp/test_import_dashboard_across_env/' }
+
+    before do
+      # unziping the resulting zip to test_unziped_path for inspection
+      service.unzip_file(result_zip, test_unziped_path)
+    end
+
+    context 'returns a zip file' do
+      specify 'that is of file type zip' do
+        expect(File.extname(result_zip)).to eq('.zip')
+      end
+
+      specify 'with the same number of files and directories as the source zip' do
+        expect(list_zip_contents(result_zip).count).to eq(list_zip_contents(dashboard_export_zip).count)
+      end
+
+      specify 'with exactly 1 database yaml matching the contents of the target_database_yaml included' do
+        result_zip_database_yamls =  Dir.glob(File.join(test_unziped_path, '**', 'databases', '*.yaml'))
+        expect(result_zip_database_yamls.count).to eq(1)
+        expect(File.basename(result_zip_database_yamls.first)).to eq(File.basename(target_database_yaml_file))
+        expect(YAML.load_file(result_zip_database_yamls.first)).to eq(YAML.load_file(target_database_yaml_file))
+      end
+
+      specify 'with dataset yamls updated to include the new target database uuid' do
+        result_zip_dataset_yamls =  Dir.glob(File.join(test_unziped_path, '**', 'datasets', '**', '*.yaml'))
+        expect(result_zip_dataset_yamls.count).to eq(1)
+        expect(YAML.load_file(result_zip_dataset_yamls.first)[:database_uuid]).to eq(YAML.load_file(target_database_yaml_file)[:uuid])
+      end
+    end
+  end
+
+  describe '#validate_params' do
+    context 'when all parameters are valid' do
+      it 'does not raise any errors' do
+        expect { service.send(:validate_params) }.not_to raise_error
+      end
+    end
+
+    context 'when dashboard_export_zip does not exist' do
+      let(:dashboard_export_zip) { 'non-existant-filename.zip' }
+
+      it 'raises an error' do
+        expect { service.send(:validate_params) }.to raise_error(RuntimeError, "Dashboard Export Zip file does not exist")
+      end
+    end
+
+    context 'when dashboard_export_zip is not a zip file' do
+      let(:dashboard_export_zip) { 'spec/fixtures/database-prod-examples.yaml' }
+
+      it 'raises an error' do
+        expect { service.send(:validate_params) }.to raise_error(RuntimeError, "Dashboard Export Zip file is not a zip file")
+      end
+    end
+
+    context 'when target_database_yaml_file does not exist' do
+      let(:target_database_yaml_file) { 'non-existant-filename.zip' }
+
+      it 'raises an error' do
+        expect { service.send(:validate_params) }.to raise_error(RuntimeError, "Target Database YAML file does not exist")
+      end
+    end
+
+    context 'when multiple database configs exist in zip file' do
+      let(:dashboard_export_zip) { 'spec/fixtures/dashboard_export_with_multiple_databases.zip' }
+
+      it 'raises an error' do
+        expect { service.send(:validate_params) }.to raise_error(RuntimeError, "Currently this class handles boards with single Database configs only. Multiple Database configs exist in zip file.")
+      end
+    end
+
+    context 'when target_database_schema is blank' do
+      let(:target_database_schema) { '' }
+
+      it 'raises an error' do
+        expect { service.send(:validate_params) }.to raise_error(RuntimeError, "Target Database Schema cannot be blank")
+      end
+    end
+  end
+end