Merge pull request #36 from rdytech/NEP-18450-database-export-dashboard-import

NEP-18450 add db export, dashboard import

Author: jbat

CHANGELOG.md

@@ -1,4 +1,10 @@
 ## Change Log
+
+## 0.2.1 - 2024-09-17
+
+* add Superset::Database::Export class for exporting database configurations
+* add Superset::Dashboard::Import class for importing a dashboards
+
 ## 0.2.0 - 2024-08-19
 
 * Adding RLS filter clause to the 'api/v1/security/guest_token/' API params in guest_token.rb - https://github.com/rdytech/superset-client/pull/31

Gemfile.lock

@@ -14,7 +14,7 @@ GIT
 PATH
   remote: .
   specs:
-    superset (0.2.0)
+    superset (0.2.1)
       dotenv (~> 2.7)
       enumerate_it (~> 1.7.0)
       faraday (~> 1.0)

doc/duplicate_dashboards.md

@@ -206,9 +206,6 @@ Putting it simply, the current thinking is to delete all the replica dashboards
 
 ### Bringing the Duplicate Dashboard process into Superset core
 
-(WIP) The goal would be to have the DuplicateDashboard process as a part of the core superset codebase.
-
-To that end this Superset Improvement Proposal (SIP) .. is a starting point.
-
-{add SIP request here}
+An ideal direction would be to have the DuplicateDashboard process as a part of the core superset codebase.
 
+A Superset discussion thread has been started in  [Duplicating Dashboards into a new database or schema](https://github.com/apache/superset/discussions/29899)

lib/superset/client.rb

@@ -44,7 +44,13 @@ def connection
       @connection ||= Faraday.new(superset_host) do |f|
         f.authorization :Bearer, access_token
         f.use FaradayMiddleware::ParseJson, content_type: 'application/json'
-        f.request :json
+
+        if self.config.use_json
+          f.request :json
+        else
+          f.request :multipart
+          f.request :url_encoded
+        end
 
         f.adapter :net_http
       end

lib/superset/dashboard/export.rb

@@ -13,7 +13,7 @@ module Dashboard
     class Export < Request
       include FileUtilities
 
-      TMP_SUPERSET_DASHBOARD_PATH = '/tmp/superset_dashboards'
+      TMP_SUPERSET_DASHBOARD_PATH = '/tmp/superset_dashboard_exports'
 
       attr_reader :dashboard_id, :destination_path
 
@@ -95,4 +95,4 @@ def datestamp
       end
     end
   end
-end
+end

lib/superset/dashboard/get.rb

@@ -4,6 +4,7 @@ class Get < Superset::Request
 
       attr_reader :id
 
+      # note .. this endpoint also accepts a dashboards uuid as the identifier
       def initialize(id)
         @id = id
       end

lib/superset/dashboard/import.rb

@@ -0,0 +1,64 @@
+# Import the provided Dashboard zip file
+# In the context of this API import process, assumption is that the database.yaml file details will match
+# an existing database in the Target Superset Environment.
+
+# Scenario 1: Export from Env1 -- Import to Env1 into the SAME Environment
+#             Will result in updating/over writing the dashboard with the contents of the zip file 
+
+# Scenario 2: Export from Env1 -- Import to Env2 into a DIFFERENT Environment
+#             Assumption is that the database.yaml will match a database configuration in the target env. 
+#             Initial import will result in creating a new dashboard with the contents of the zip file. 
+#             Subsequent imports will result in updating/over writing the previous imported dashboard with the contents of the zip file.
+
+# the overwrite flag will determine if the dashboard will be updated or created new
+# overwrite: false .. will result in an error if a dashboard with the same UUID already exists
+
+# Usage
+# Superset::Dashboard::Import.new(source_zip_file: '/tmp/dashboard.zip').perform
+#
+
+module Superset
+  module Dashboard
+    class Import < Request
+
+      attr_reader :source_zip_file, :overwrite
+
+      def initialize(source_zip_file: , overwrite: true)
+        @source_zip_file = source_zip_file
+        @overwrite = overwrite
+      end
+
+      def perform
+        validate_params
+
+        response
+      end
+
+      def response
+        @response ||= client(use_json: false).post(
+          route,
+          payload
+        )
+      end
+
+      private
+
+      def validate_params
+        raise ArgumentError, 'source_zip_file is required' if source_zip_file.nil?
+        raise ArgumentError, 'source_zip_file does not exist' unless File.exist?(source_zip_file)
+        raise ArgumentError, 'overwrite must be a boolean' unless [true, false].include?(overwrite)
+      end
+
+      def payload
+        {
+          formData: Faraday::UploadIO.new(source_zip_file, 'application/zip'),
+          overwrite: overwrite.to_s
+        }
+      end
+
+      def route
+        "dashboard/import/"
+      end
+    end
+  end
+end

lib/superset/database/export.rb

@@ -0,0 +1,119 @@
+# Will export the Database zip file to /tmp/superset_database_exports with zip filename adjusted to include the database_id
+# Example zipfile: dashboard_#{database_id}_export_#{datestamp}.zip
+#
+# File will then be unziped and all files copied into the destination_path with the database_id as a subfolder
+# Optianally remove the dataset yaml files from the export
+#
+# Usage
+# Superset::Database::Export.new(database_id: 1, destination_path: '/tmp/superset_database_exports/').perform
+
+# Superset::Database::Export.new(database_id: 1, destination_path: '/tmp/superset_database_exports/', remove_dataset_yamls: true).perform
+#
+
+require 'superset/file_utilities'
+
+module Superset
+  module Database
+    class Export < Request
+      include FileUtilities
+
+      TMP_SUPERSET_DATABASE_PATH = '/tmp/superset_database_exports'.freeze
+
+      attr_reader :database_id, :destination_path, :remove_dataset_yamls
+
+      def initialize(database_id: , destination_path: , remove_dataset_yamls: true)
+        @database_id = database_id
+        @destination_path = destination_path.chomp('/')
+        @remove_dataset_yamls = remove_dataset_yamls
+      end
+
+      def perform
+        create_tmp_dir
+        save_exported_zip_file
+        unzip_files
+        copy_export_files_to_destination_path
+
+        Dir.glob("#{destination_path_with_db_id}/databases/*")
+      end
+
+      def response
+        @response ||= client.call(
+          :get,
+          client.url(route),
+          client.param_check(params)
+        )
+      end
+
+      def exported_zip_path
+        @exported_zip_path ||= "#{tmp_uniq_database_path}/database_#{database_id}_export_#{datestamp}.zip"
+      end
+
+      private
+
+      def params
+        # The Swagger API interface indicates this endpoint should take an array of integers
+        # however this does not work within the Swagger interface or when testing the API 
+        # Investigating the Superset GUI with Dev Tools shows that the format below is used
+
+        { "q": "!(#{database_id})" }
+      end
+
+      def save_exported_zip_file
+        File.open(exported_zip_path, 'wb') { |fp| fp.write(response.body) }
+      end
+
+      def unzip_files
+        @extracted_files = unzip_file(exported_zip_path, tmp_uniq_database_path)
+        remove_dataset_yaml_files if remove_dataset_yamls
+      end
+
+      def download_folder
+        File.dirname(extracted_files[0])
+      end
+
+      def destination_path_with_db_id
+        @destination_path_with_db_id ||= File.join(destination_path, database_id.to_s)
+      end
+
+      def copy_export_files_to_destination_path
+        FileUtils.mkdir_p(destination_path_with_db_id) unless File.directory?(destination_path_with_db_id)
+
+        Dir.glob("#{download_folder}/*").each do |item|
+          FileUtils.cp_r(item, destination_path_with_db_id)
+        end
+      end
+
+      def remove_dataset_yaml_files
+        datasets_directories = Dir.glob( File.join(tmp_uniq_database_path, '/*/datasets') )
+
+        datasets_directories.each do |directory|
+          FileUtils.rm_rf(directory) if Dir.exist?(directory)
+        end
+      end
+
+      def create_tmp_dir
+        FileUtils.mkdir_p(tmp_uniq_database_path) unless File.directory?(tmp_uniq_database_path)
+      end
+
+      def tmp_uniq_database_path
+        @tmp_uniq_database_path ||= File.join(TMP_SUPERSET_DATABASE_PATH, uuid)
+      end
+
+      def uuid
+        SecureRandom.uuid
+      end
+
+      def extracted_files
+        @extracted_files ||= []
+      end
+
+      def route
+        "database/export/"
+      end
+
+      def datestamp
+        @datestamp ||= Time.now.strftime('%Y%m%d')
+      end
+    end
+  end
+end

lib/superset/dataset/get.rb

@@ -29,17 +29,6 @@ def title
         result['name']
       end
 
-      private
-
-      def route
-        "dataset/#{id}"
-      end
-
-      def display_headers
-        %w[title schema database_name, database_id]
-      end
-  
-
       def database_name
         result['database']['database_name']
       end
@@ -51,6 +40,16 @@ def database_id
       def sql
         ['sql']
       end
+
+      private
+
+      def route
+        "dataset/#{id}"
+      end
+
+      def display_headers
+        %w[title schema database_name, database_id]
+      end
     end
   end
 end

lib/superset/file_utilities.rb

@@ -9,11 +9,12 @@ def unzip_file(zip_file, destination)
           entry_path = File.join(destination, entry.name)
           entries << entry_path
           FileUtils.mkdir_p(File.dirname(entry_path))
-          zip.extract(entry, entry_path)
+
+          zip.extract(entry, entry_path) unless File.exist?(entry_path)
         end
       end
-      puts entries
-      entries # return array of extracted files
+
+      entries
     end
   end
 end