Skip to content

Commit bec1874

Browse files
committed
adds specs for gsub
1 parent 0369d66 commit bec1874

File tree

2 files changed

+162
-61
lines changed

2 files changed

+162
-61
lines changed

lib/superset/services/duplicate_dashboard.rb

Lines changed: 49 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,14 @@ def perform
3131
# Duplicate these Datasets to the new target schema and target database
3232
duplicate_source_dashboard_datasets
3333

34-
# Update the Charts on the New Dashboard with the New Datasets
34+
# Update the Charts on the New Dashboard with the New Datasets and update the Dashboard json_metadata for the charts
3535
update_charts_with_new_datasets
3636

3737
# Duplicate filters to the new target schema and target database
3838
duplicate_source_dashboard_filters
3939

40+
update_source_dashboard_json_metadata
41+
4042
created_embedded_config
4143

4244
add_tags_to_new_dashboard
@@ -77,71 +79,74 @@ def dataset_duplication_tracker
7779
@dataset_duplication_tracker ||= []
7880
end
7981

82+
def duplicate_source_dashboard_datasets
83+
source_dashboard_datasets.each do |dataset|
84+
# duplicate the dataset, renaming to use of suffix as the target_schema
85+
# reason: there is a bug(or feature) in the SS API where a dataset name must be uniq when duplicating.
86+
# (note however renaming in the GUI to a dup name works fine)
87+
new_dataset_id = Superset::Dataset::Duplicate.new(source_dataset_id: dataset[:id], new_dataset_name: "#{dataset[:datasource_name]}-#{target_schema}").perform
88+
89+
# keep track of the previous dataset and the matching new dataset_id
90+
dataset_duplication_tracker << { source_dataset_id: dataset[:id], new_dataset_id: new_dataset_id }
91+
92+
# update the new dataset with the target schema and target database
93+
Superset::Dataset::UpdateSchema.new(source_dataset_id: new_dataset_id, target_database_id: target_database_id, target_schema: target_schema).perform
94+
end
95+
end
96+
8097
def update_charts_with_new_datasets
81-
logger.info "Updating Charts with New Datasets ..."
98+
logger.info "Updating Charts to point to New Datasets and updating Dashboard json_metadata ..."
99+
# note dashboard json_metadata currently still points to the old chart ids and is updated here
100+
101+
new_dashboard_json_metadata_json_string = new_dashboard_json_metadata_configuration.to_json # need to convert to string for gsub
82102
# get all chart ids for the new dashboard
83103
new_charts_list = Superset::Dashboard::Charts::List.new(new_dashboard.id).result
84-
chart_ids_list = new_charts_list&.map { |r| r['id'] }&.compact
104+
new_chart_ids_list = new_charts_list&.map { |r| r['id'] }&.compact
105+
# get all chart details for the source dashboard
85106
original_charts = Superset::Dashboard::Charts::List.new(source_dashboard_id).result.map { |r| [r['slice_name'], r['id']] }.to_h
86107
new_charts = new_charts_list.map { |r| [r['id'], r['slice_name']] }.to_h
87-
json_metadata = new_dashboard.result['json_metadata']
88-
return unless chart_ids_list.any?
108+
return unless new_chart_ids_list.any?
89109

90110
# for each chart, update the charts current dataset_id with the new dataset_id
91-
chart_ids_list.each do |chart_id|
111+
new_chart_ids_list.each do |new_chart_id|
92112

93-
# get the CURRENT dataset_id for the chart
94-
current_chart_dataset_id = Superset::Chart::Get.new(chart_id).datasource_id
113+
# get the CURRENT dataset_id for the new chart
114+
current_chart_dataset_id = Superset::Chart::Get.new(new_chart_id).datasource_id
95115

96-
# find the new dataset_id for the chart, based on the current_chart_dataset_id
116+
# find the new dataset_id for the new chart, based on the current_chart_dataset_id
97117
new_dataset_id = dataset_duplication_tracker.find { |dataset| dataset[:source_dataset_id] == current_chart_dataset_id }&.fetch(:new_dataset_id, nil)
98118

99-
# update the chart to target the new dataset_id and to the reference the new target_dashboard_id
100-
Superset::Chart::UpdateDataset.new(chart_id: chart_id, target_dataset_id: new_dataset_id, target_dashboard_id: new_dashboard.id).perform
101-
logger.info " Update Chart #{chart_id} to new dataset_id #{new_dataset_id}"
102-
103-
# update json metadata
104-
original_chart_id = original_charts[new_charts[chart_id]]
119+
# update the new chart to target the new dataset_id and to the reference the new target_dashboard_id
120+
Superset::Chart::UpdateDataset.new(chart_id: new_chart_id, target_dataset_id: new_dataset_id, target_dashboard_id: new_dashboard.id).perform
121+
logger.info " Update Chart #{new_chart_id} to new dataset_id #{new_dataset_id}"
105122

106-
json_metadata.gsub!(original_chart_id.to_s, chart_id.to_s)
107-
# TODO: this gsub could be flawed, as it is a string replace.
108-
# ie replacing 20 would also replace 200, 201, 202 etc which could cause disastorous results .. see NEP-17737
123+
# update json metadata swaping the old chart_id with the new chart_id
124+
original_chart_id = original_charts[new_charts[new_chart_id]]
125+
regex_with_numeric_boundaries = Regexp.new("\\b#{original_chart_id.to_s}\\b")
126+
new_dashboard_json_metadata_json_string.gsub!(regex_with_numeric_boundaries, new_chart_id.to_s)
109127
end
110128

111-
# TODO: bring in the filter json update here as well .. to save on another API call, see duplicate_source_dashboard_filters
112-
Superset::Dashboard::Put.new(target_dashboard_id: new_dashboard.id, params: { "json_metadata" => json_metadata }).perform
113-
logger.info " Updated new Dashboard json_metadata charts with new dataset ids"
114-
end
115-
116-
def duplicate_source_dashboard_datasets
117-
source_dashboard_datasets.each do |dataset|
118-
# duplicate the dataset, renaming to use of suffix as the target_schema
119-
# reason: there is a bug(or feature) in the SS API where a dataset name must be uniq when duplicating.
120-
# (note however renaming in the GUI to a dup name works fine)
121-
new_dataset_id = Superset::Dataset::Duplicate.new(source_dataset_id: dataset[:id], new_dataset_name: "#{dataset[:datasource_name]}-#{target_schema}").perform
122-
123-
# keep track of the previous dataset and the matching new dataset_id
124-
dataset_duplication_tracker << { source_dataset_id: dataset[:id], new_dataset_id: new_dataset_id }
125-
126-
# update the new dataset with the target schema and target database
127-
Superset::Dataset::UpdateSchema.new(source_dataset_id: new_dataset_id, target_database_id: target_database_id, target_schema: target_schema).perform
128-
end
129+
# convert back to hash .. and store in the new_dashboard_json_metadata_configuration
130+
@new_dashboard_json_metadata_configuration = JSON.parse(new_dashboard_json_metadata_json_string)
129131
end
130132

131133
def duplicate_source_dashboard_filters
132134
return unless source_dashboard_filter_dataset_ids.length.positive?
133135

134136
logger.info "Updating Filters to point to new dataset targets ..."
135-
json_metadata = new_dashboard.json_metadata
136-
configuration = json_metadata['native_filter_configuration']&.map do |filter_config|
137+
configuration = new_dashboard_json_metadata_configuration['native_filter_configuration']&.map do |filter_config|
137138
targets = filter_config['targets']
138139
target_filter_dataset_id = dataset_duplication_tracker.find { |d| d[:source_dataset_id] == targets.first["datasetId"] }&.fetch(:new_dataset_id, nil)
139140
filter_config['targets'] = [targets.first.merge({ "datasetId"=> target_filter_dataset_id })]
140141
filter_config
141142
end
142143

143-
json_metadata['native_filter_configuration'] = configuration || []
144-
Superset::Dashboard::Put.new(target_dashboard_id: new_dashboard.id, params: { "json_metadata" => json_metadata.to_json }).perform
144+
@new_dashboard_json_metadata_configuration['native_filter_configuration'] = configuration || []
145+
end
146+
147+
def update_source_dashboard_json_metadata
148+
logger.info " Updated new Dashboard json_metadata charts with new dataset ids"
149+
Superset::Dashboard::Put.new(target_dashboard_id: new_dashboard.id, params: { "json_metadata" => @new_dashboard_json_metadata_configuration.to_json }).perform
145150
end
146151

147152
def new_dashboard
@@ -159,6 +164,10 @@ def new_dashboard
159164
raise "Dashboard::Copy error: #{e.message}"
160165
end
161166

167+
def new_dashboard_json_metadata_configuration
168+
@new_dashboard_json_metadata_configuration ||= new_dashboard.json_metadata
169+
end
170+
162171
# retrieve the datasets that will be duplicated
163172
def source_dashboard_datasets
164173
@source_dashboard_datasets ||= Superset::Dashboard::Datasets::List.new(source_dashboard_id).datasets_details

spec/superset/services/duplicate_dashboard_spec.rb

Lines changed: 113 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
tags: tags ) }
1111

1212
let(:source_dashboard_id) { 1 }
13-
let(:source_dashboard) { double('source_dashboard', id: source_dashboard_id, url: "http://superset-host.com/superset/dashboard/#{source_dashboard_id}", json_metadata: json_metadata) }
13+
let(:source_dashboard) { double('source_dashboard', id: source_dashboard_id, url: "http://superset-host.com/superset/dashboard/#{source_dashboard_id}", json_metadata: json_metadata_initial_settings) }
1414

1515
let(:target_schema) { 'schema_two' }
1616
let(:target_database_id) { 6 }
@@ -23,22 +23,74 @@
2323
{id: source_dataset_1, datasource_name: "Dataset 1", schema: "schema_one", database: {id: 9, name: "db_9", backend: "postgresql"}, sql: 'SELECT * FROM table1'},
2424
{id: source_dataset_2, datasource_name: "Dataset 2", schema: "schema_one", database: {id: 9, name: "db_9", backend: "postgresql"}, sql: 'SELECT * FROM table1'}
2525
]}
26-
let(:json_metadata) { { 'native_filter_configuration' => [{ 'targets' => [{ 'datasetId' => 101 }]}]} }
27-
let(:new_json_metadata) { { 'native_filter_configuration' => [{ 'targets' => [{ 'datasetId' => 201 }]}]} }
26+
let(:source_chart_1) { 1001 }
27+
let(:source_chart_2) { 1002 }
28+
2829
let(:source_dashboard_filter_dataset_ids) { [source_dataset_1, source_dataset_2] }
2930
let(:source_dashboard_dataset_ids) {[source_dataset_1, source_dataset_2 ]}
3031
let(:dataset_duplication_tracker) { [{ source_dataset_id: 101, new_dataset_id: 201 }] }
3132

3233
let(:new_dashboard_id) { 2 }
33-
let(:new_dashboard) { double('new_dashboard', id: new_dashboard_id, url: "http://superset-host.com/superset/dashboard/#{new_dashboard_id}", json_metadata: json_metadata) }
34+
let(:new_dashboard) do
35+
OpenStruct.new(
36+
id: new_dashboard_id,
37+
url: "http://superset-host.com/superset/dashboard/#{new_dashboard_id}",
38+
result: { 'json_metadata' => json_metadata_initial_settings.to_json },
39+
json_metadata: json_metadata_initial_settings ) # mock the new_dashboard_json_metadata method
40+
end
41+
# { double('new_dashboard', id: new_dashboard_id, url: "http://superset-host.com/superset/dashboard/#{new_dashboard_id}", json_metadata: json_metadata_initial_settings) }
3442

3543
let(:new_dataset_1) { 201 }
3644
let(:new_dataset_2) { 202 }
3745

38-
let(:new_chart_1) { 3001 }
39-
let(:new_chart_2) { 3002 }
46+
let(:new_chart_1) { 2001 }
47+
let(:new_chart_2) { 2002 }
4048
let(:existing_target_datasets_list) {[]}
4149

50+
# initial json metadata settings will be copied to the new dashboard and requires updating
51+
let(:json_metadata_initial_settings) do
52+
{
53+
"chart_configuration"=>
54+
{ "#{source_chart_1}"=>{"id"=>source_chart_1, "crossFilters"=>{"scope"=>"global", "chartsInScope"=>[source_chart_2]}},
55+
"#{source_chart_2}"=>{"id"=>source_chart_2, "crossFilters"=>{"scope"=>"global", "chartsInScope"=>[source_chart_1]}}},
56+
"global_chart_configuration"=>{"scope"=>{"rootPath"=>["ROOT_ID"], "excluded"=>[]}, "chartsInScope"=>[source_chart_1, source_chart_2]},
57+
"native_filter_configuration"=>
58+
[
59+
{"id"=>"NATIVE_FILTER-k-UxewZyI",
60+
"name"=>"JobTitleLimit5",
61+
"targets"=>[{"datasetId"=>source_dataset_1, "column"=>{"name"=>"job_title"}}],
62+
"chartsInScope"=>[source_chart_1, source_chart_2]},
63+
{"id"=>"NATIVE_FILTER-eoi3FEQ1C",
64+
"name"=>"Count",
65+
"filterType"=>"filter_select",
66+
"targets"=>[{"datasetId"=>source_dataset_2, "column"=>{"name"=>"count"}}],
67+
"chartsInScope"=>[source_chart_1, source_chart_2]}
68+
]
69+
}
70+
end
71+
72+
# expected json metadata settings after the new dashboard is created and json is updated
73+
let(:json_metadata_updated_settings) do
74+
{
75+
"chart_configuration"=>
76+
{ "#{new_chart_1}"=>{"id"=>new_chart_1, "crossFilters"=>{"scope"=>"global", "chartsInScope"=>[new_chart_2]}},
77+
"#{new_chart_2}"=>{"id"=>new_chart_2, "crossFilters"=>{"scope"=>"global", "chartsInScope"=>[new_chart_1]}}},
78+
"global_chart_configuration"=>{"scope"=>{"rootPath"=>["ROOT_ID"], "excluded"=>[]}, "chartsInScope"=>[new_chart_1, new_chart_2]},
79+
"native_filter_configuration"=>
80+
[
81+
{"id"=>"NATIVE_FILTER-k-UxewZyI",
82+
"name"=>"JobTitleLimit5",
83+
"targets"=>[{"datasetId"=>new_dataset_1, "column"=>{"name"=>"job_title"}}],
84+
"chartsInScope"=>[new_chart_1, new_chart_2]},
85+
{"id"=>"NATIVE_FILTER-eoi3FEQ1C",
86+
"name"=>"Count",
87+
"filterType"=>"filter_select",
88+
"targets"=>[{"datasetId"=>new_dataset_2, "column"=>{"name"=>"count"}}],
89+
"chartsInScope"=>[new_chart_1, new_chart_2]}
90+
]
91+
}
92+
end
93+
4294
before do
4395
allow(subject).to receive(:superset_host).and_return('http://superset-host.com')
4496
allow(subject).to receive(:target_database_available_schemas).and_return(target_database_available_schemas)
@@ -49,7 +101,6 @@
49101
allow(subject).to receive(:source_dashboard_filter_dataset_ids).and_return(source_dashboard_filter_dataset_ids)
50102
allow(subject).to receive(:source_dashboard_dataset_ids).and_return(source_dashboard_dataset_ids)
51103
allow(subject).to receive(:dataset_duplication_tracker).and_return(dataset_duplication_tracker)
52-
allow(new_dashboard).to receive(:result).and_return({ 'json_metadata' => json_metadata.to_json })
53104
end
54105

55106
describe '#perform' do
@@ -64,29 +115,70 @@
64115
expect(Superset::Dataset::UpdateSchema).to receive(:new).with(source_dataset_id: new_dataset_2, target_database_id: target_database_id, target_schema: target_schema).and_return(double(perform: new_dataset_2))
65116

66117
# getting the list of charts for the source dashboard
67-
allow(Superset::Dashboard::Charts::List).to receive(:new).with(source_dashboard_id).and_return(double(result: [{ 'slice_name' => "test", "id" => 3001}, { 'slice_name' => "test", "id" => 3002}], chart_ids: [new_chart_1, new_chart_2]))
68-
allow(Superset::Dashboard::Charts::List).to receive(:new).with(new_dashboard_id).and_return(double(result: [{ 'slice_name' => "test", "id" => 3001}, { 'slice_name' => "test", "id" => 3002}]))
118+
allow(Superset::Dashboard::Charts::List).to receive(:new).with(source_dashboard_id).and_return(double(result: [{ 'slice_name' => "chart 1", "id" => source_chart_1}, { 'slice_name' => "chart 2", "id" => source_chart_2}])) # , chart_ids: [source_chart_1, source_chart_2]
119+
allow(Superset::Dashboard::Charts::List).to receive(:new).with(new_dashboard_id).and_return(double(result: [{ 'slice_name' => "chart 1", "id" => new_chart_1}, { 'slice_name' => "chart 2", "id" => new_chart_2}]))
69120

70121
# getting the current dataset_id for the new charts .. still pointing to the old datasets
71-
expect(Superset::Chart::Get).to receive(:new).with(3001).and_return(double(datasource_id: source_dataset_1))
72-
expect(Superset::Chart::Get).to receive(:new).with(3002).and_return(double(datasource_id: source_dataset_2))
122+
expect(Superset::Chart::Get).to receive(:new).with(new_chart_1).and_return(double(datasource_id: source_dataset_1))
123+
expect(Superset::Chart::Get).to receive(:new).with(new_chart_2).and_return(double(datasource_id: source_dataset_2))
73124

74125
# updating the new charts to point to the new datasets
75126
expect(Superset::Chart::UpdateDataset).to receive(:new).with(chart_id: new_chart_1, target_dataset_id: new_dataset_1, target_dashboard_id: new_dashboard_id).and_return(double(perform: true))
76127
expect(Superset::Chart::UpdateDataset).to receive(:new).with(chart_id: new_chart_2, target_dataset_id: new_dataset_2, target_dashboard_id: new_dashboard_id).and_return(double(perform: true))
77128

78-
# get json metadata
79-
#expect(Superset::Dashboard::Get).to receive(:new).with(new_dashboard_id).and_return(double(json_metadata: json_metadata))
80-
#expect(Superset::Dashboard::Get).to receive(:new).with(source_dashboard_id).and_return(double(url: 'test.com', result: { 'json_metadata' => json_metadata.to_json }))
81-
82-
# update dashboard json metadata chart datasets -- TODO add the chart config expected change here
83-
expect(Superset::Dashboard::Put).to receive(:new).once.with(target_dashboard_id: new_dashboard_id, params: { 'json_metadata' => json_metadata.to_json }).and_return(double(perform: true))
84-
# update dashboard json metadata filter datasets
85-
expect(Superset::Dashboard::Put).to receive(:new).once.with(target_dashboard_id: new_dashboard_id, params: { 'json_metadata' => new_json_metadata.to_json }).and_return(double(perform: true))
129+
# update dashboard json metadata chart datasets
130+
expect(Superset::Dashboard::Put).to receive(:new).once.with(target_dashboard_id: new_dashboard_id, params: { 'json_metadata' => json_metadata_updated_settings.to_json }).and_return(double(perform: true))
86131
end
87132

88-
context 'returns the new dashboard details' do
89-
specify { expect(subject.perform).to eq( { new_dashboard_id: 2, new_dashboard_url: "http://superset-host.com/superset/dashboard/2" }) }
133+
context 'completes duplicate process' do
134+
context 'and returns the new dashboard details' do
135+
specify do
136+
expect(subject.perform).to eq( { new_dashboard_id: 2, new_dashboard_url: "http://superset-host.com/superset/dashboard/2" })
137+
end
138+
end
139+
140+
context 'and updates the json_metadata as expected' do
141+
context 'with stardard json metadata ids' do
142+
specify do
143+
expect(subject.new_dashboard_json_metadata_configuration).to eq(json_metadata_initial_settings)
144+
subject.perform
145+
expect(subject.new_dashboard_json_metadata_configuration).to eq(json_metadata_updated_settings)
146+
end
147+
end
148+
149+
context 'with non stardard json metadata ids to confirm gsub' do
150+
let(:source_chart_1) { 11 }
151+
let(:source_chart_2) { 1111 }
152+
let(:json_metadata_initial_settings) do
153+
{
154+
"chart_configuration"=>
155+
{ "#{source_chart_1}"=>{"id"=>source_chart_1, "crossFilters"=>{"scope"=>"global", "chartsInScope"=>[source_chart_2]}} ,
156+
"#{source_chart_2}"=>{"id"=>source_chart_2, "crossFilters"=>{"scope"=>"global", "chartsInScope"=>[source_chart_1]}} } ,
157+
"global_chart_configuration"=>{"scope"=>{"rootPath"=>["ROOT_ID"], "excluded"=>[]}, "chartsInScope"=>[source_chart_1, source_chart_2]},
158+
"native_filter_configuration"=>[]
159+
}
160+
end
161+
162+
let(:new_chart_1) { 222 }
163+
let(:new_chart_2) { 22222 }
164+
let!(:json_metadata_updated_settings) do
165+
{
166+
"chart_configuration"=>
167+
{ "#{new_chart_1}"=>{"id"=>new_chart_1, "crossFilters"=>{"scope"=>"global", "chartsInScope"=>[new_chart_2]}} ,
168+
"#{new_chart_2}"=>{"id"=>new_chart_2, "crossFilters"=>{"scope"=>"global", "chartsInScope"=>[new_chart_1]}} } ,
169+
"global_chart_configuration"=>{"scope"=>{"rootPath"=>["ROOT_ID"], "excluded"=>[]}, "chartsInScope"=>[new_chart_1, new_chart_2]},
170+
"native_filter_configuration"=>[]
171+
}
172+
end
173+
174+
specify do
175+
expect(subject.new_dashboard_json_metadata_configuration).to eq(json_metadata_initial_settings)
176+
subject.perform
177+
expect(subject.new_dashboard_json_metadata_configuration).to eq(json_metadata_updated_settings)
178+
end
179+
end
180+
end
181+
90182
end
91183

92184
context 'and embedded domains' do

0 commit comments

Comments
 (0)