diff --git a/doc/migrating_dashboards_across_environments.md b/doc/migrating_dashboards_across_environments.md index 8fb70db..72c1f64 100644 --- a/doc/migrating_dashboards_across_environments.md +++ b/doc/migrating_dashboards_across_environments.md @@ -20,8 +20,8 @@ Assuming your API env for ruby is setup for your target superset environment. new_import_zip = Superset::Services::ImportDashboardAcrossEnvironments.new( dashboard_export_zip: 'path_to/dashboard_101_export_20241010.zip', - target_database_yaml_file: 'path_to/env2_db_config.yaml', - target_database_schema: 'acme', + target_database_yaml_file: 'path_to/env2_db_config.yaml', + target_database_schema: 'acme', ).perform # now import the adjusted zip to the target superset env @@ -142,7 +142,8 @@ With the condition that the database in staging and production are structurally 3. Get a copy of the production database YAML configuration file 4. In the exported dashboard files, replace the staging database YAML with the production YAML 5. In the dataset YAML files, replace all instances of the previously noted staging database UUID with the new production UUID -6. Zip the files and import them to the production environment +6. In the dataset YAML files, remove the attribute catalog. On import the default database target catalog will be applied +7. Zip the files and import them to the production environment The process above assumes that whoever is migrating the dashboard has a copy of the target database YAML files so that in steps 3 and 4 we can then replace the staging database YAML with the production one. @@ -170,4 +171,4 @@ It will attempt to update the same dashboard as the UUID for the dashboard has n Some helpful references relating to cross-environment workflows: - [Managing Content Across Workspaces](https://docs.preset.io/docs/managing-content-across-workspaces) -- [Superset Slack AI Explanation](https://apache-superset.slack.com/archives/C072KSLBTC1/p1722382347022689) \ No newline at end of file +- [Superset Slack AI Explanation](https://apache-superset.slack.com/archives/C072KSLBTC1/p1722382347022689) diff --git a/lib/superset/dataset/update_schema.rb b/lib/superset/dataset/update_schema.rb index 10092e3..3577a50 100644 --- a/lib/superset/dataset/update_schema.rb +++ b/lib/superset/dataset/update_schema.rb @@ -2,13 +2,14 @@ module Superset module Dataset class UpdateSchema < Superset::Request - attr_reader :source_dataset_id, :target_database_id, :target_schema, :remove_copy_suffix + attr_reader :source_dataset_id, :target_database_id, :target_schema, :remove_copy_suffix, :catalog - def initialize(source_dataset_id: , target_database_id: , target_schema: , remove_copy_suffix: false) + def initialize(source_dataset_id: , target_database_id: , target_schema: , remove_copy_suffix: false, catalog: nil ) @source_dataset_id = source_dataset_id @target_database_id = target_database_id @target_schema = target_schema @remove_copy_suffix = remove_copy_suffix + @catalog = catalog end def perform @@ -34,9 +35,10 @@ def response def params_updated @params_updated ||= begin new_params = source_dataset.slice(*acceptable_attributes).with_indifferent_access - + # primary database and schema changes new_params.merge!("database_id": target_database_id) # add the target database id + new_params['catalog'] = catalog # if unknown, will default to nil, superset will then update to the actual schemas catalog value upon saving new_params['schema'] = target_schema new_params['owners'] = new_params['owners'].map {|o| o['id'] } # expects an array of user ids new_params['table_name'] = new_params['table_name'].gsub(/ \(COPY\)/, '') if remove_copy_suffix diff --git a/lib/superset/services/duplicate_dashboard.rb b/lib/superset/services/duplicate_dashboard.rb index 03e2260..d7f67bd 100644 --- a/lib/superset/services/duplicate_dashboard.rb +++ b/lib/superset/services/duplicate_dashboard.rb @@ -91,7 +91,7 @@ def dataset_duplication_tracker def duplicate_source_dashboard_datasets source_dashboard_datasets.each do |dataset| # duplicate the dataset, renaming to use of suffix as the target_schema - # reason: there is a bug(or feature) in the SS API where a dataset name must be uniq when duplicating. + # reason: there is a bug(or feature) in the SS API where a dataset name must be uniq when duplicating. # (note however renaming in the GUI to a dup name works fine) new_dataset_name = "#{dataset[:datasource_name]}-#{target_schema}" existing_datasets = Superset::Dataset::List.new(title_equals: new_dataset_name, schema_equals: target_schema).result diff --git a/lib/superset/services/import_dashboard_across_environment.rb b/lib/superset/services/import_dashboard_across_environment.rb index e997c1b..3c9fde1 100644 --- a/lib/superset/services/import_dashboard_across_environment.rb +++ b/lib/superset/services/import_dashboard_across_environment.rb @@ -4,8 +4,10 @@ must already exist as a database connection in the target superset environment. Currently handles only 1 Database yaml file in the zip file. ( ie only 1 common database connection per dashboards datasets ) +Targeted towards Dashboards for an individual Clients Database Data only. +Most often used in EXTERNAL facing embedded client dashboards. -Required Attributes: +Required Attributes: - target_database_yaml_file - location of the target database yaml config file - target_database_schema - the schema name to be used in the target database - dashboard_export_zip - location of the source dashboard export zip file to tranferred to a new superset Env @@ -78,6 +80,10 @@ def update_dataset_configs dashboard_config[:datasets].each do |dataset| dataset[:content][:database_uuid] = dashboard_config[:databases].first[:content][:uuid] dataset[:content][:schema] = target_database_schema + + # by clearing out the source's catalog, this automatically allows superset to set to the default catalog on the target + dataset[:content][:catalog] = nil + stringified_content = deep_transform_keys_to_strings(dataset[:content]) File.open(dataset[:filename], 'w') { |f| f.write stringified_content.to_yaml } end @@ -103,7 +109,7 @@ def new_database_yaml_file_path def dashboard_export_root_path # locate the unziped dashboard_export_* directory as named by superset app, eg dashboard_export_20240821T001536 - @dashboard_export_root_path ||= begin + @dashboard_export_root_path ||= begin pattern = File.join(dashboard_config[:tmp_uniq_dashboard_path], 'dashboard_export_*') Dir.glob(pattern).first end diff --git a/spec/superset/dataset/update_schema_spec.rb b/spec/superset/dataset/update_schema_spec.rb index 9e439ae..ae4efa2 100644 --- a/spec/superset/dataset/update_schema_spec.rb +++ b/spec/superset/dataset/update_schema_spec.rb @@ -2,16 +2,19 @@ RSpec.describe Superset::Dataset::UpdateSchema do subject { described_class.new( - source_dataset_id: source_dataset_id, - target_database_id: target_database_id, + source_dataset_id: source_dataset_id, + target_database_id: target_database_id, target_schema: target_schema, - remove_copy_suffix: remove_copy_suffix) } + remove_copy_suffix: remove_copy_suffix, + catalog: catalog + ) } let(:source_dataset_id) { 226 } let(:source_schema) { 'schema_one' } let(:target_database_id) { 6 } let(:target_schema) { 'schema_three' } let(:remove_copy_suffix) { false } + let(:catalog) { nil } let(:source_dataset) do { @@ -87,7 +90,7 @@ context 'with invalid params' do context 'source_dataset_id is empty' do let(:source_dataset_id) { nil } - + specify do expect { subject.perform }.to raise_error(RuntimeError, "Error: source_dataset_id integer is required") end @@ -95,12 +98,12 @@ context 'target_database_id is empty' do let(:target_database_id) { nil } - + specify do expect { subject.perform }.to raise_error(RuntimeError, "Error: target_database_id integer is required") end end - + context 'target_schema is empty' do let(:target_schema) { nil } @@ -129,9 +132,10 @@ end describe '#params_updated' do - context 'with remove_copy_suffix true' do + context 'with remove_copy_suffix false' do specify 'set the new target schema and target database correctly' do expect(subject.params_updated['schema']).to eq(target_schema) + expect(subject.params_updated['catalog']).to eq(nil) expect(subject.params_updated['database_id']).to eq(target_database_id) expect(subject.params_updated['table_name']).to eq('JR SP Service Counts (COPY)') # unchanged if remove_copy_suffix is false end @@ -144,5 +148,13 @@ expect(subject.params_updated['table_name']).to eq('JR SP Service Counts') # removed (COPY) suffix end end + + context 'with catalog set to blah' do + let(:catalog) { 'blah' } + + specify do + expect(subject.params_updated['catalog']).to eq('blah') + end + end end end