Exporter for Databricks SQL objects (#1199)

* initial work for SQLA exporter
* working exporter for queries & sql endpoints
* export of visualizations & dashboards
* visualizations need to be re-implemented to play nicely with widgets

Author: alexott

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## 0.5.4
 
 * Completely removed custom client-side validation in `databricks_service_principal` ([#1193](https://github.com/databrickslabs/terraform-provider-databricks/issues/1193)).
+* Added export functionality for Databricks SQL objects - endpoints, queries, dashboards, widgets, visualizations ([#1199](https://github.com/databrickslabs/terraform-provider-databricks/pull/1199)).
 
 Updated dependency versions:
 

exporter/context.go

@@ -47,23 +47,25 @@ import (
 */
 
 type importContext struct {
-	Module      string
-	Context     context.Context
-	Client      *common.DatabricksClient
-	State       stateApproximation
-	Importables map[string]importable
-	Resources   map[string]*schema.Resource
-	Scope       importedResources
-	Files       map[string]*hclwrite.File
-	Directory   string
-	importing   map[string]bool
-	nameFixes   []regexFix
-	hclFixes    []regexFix
-	allUsers    []scim.User
-	allGroups   []scim.Group
-	mountMap    map[string]mount
-	variables   map[string]string
-	testEmits   map[string]bool
+	Module            string
+	Context           context.Context
+	Client            *common.DatabricksClient
+	State             stateApproximation
+	Importables       map[string]importable
+	Resources         map[string]*schema.Resource
+	Scope             importedResources
+	Files             map[string]*hclwrite.File
+	Directory         string
+	importing         map[string]bool
+	nameFixes         []regexFix
+	hclFixes          []regexFix
+	allUsers          []scim.User
+	allGroups         []scim.Group
+	mountMap          map[string]mount
+	variables         map[string]string
+	testEmits         map[string]bool
+	sqlDatasources    map[string]string
+	sqlVisualizations map[string]string
 
 	debug               bool
 	mounts              bool

exporter/exporter_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/databrickslabs/terraform-provider-databricks/repos"
 	"github.com/databrickslabs/terraform-provider-databricks/scim"
 	"github.com/databrickslabs/terraform-provider-databricks/secrets"
+	"github.com/databrickslabs/terraform-provider-databricks/sql"
 	"github.com/databrickslabs/terraform-provider-databricks/workspace"
 	"github.com/hashicorp/hcl/v2/hclwrite"
 
@@ -224,6 +225,27 @@ var emptyWorkspace = qa.HTTPFixture{
 	Response: workspace.ObjectList{},
 }
 
+var emptySqlEndpoints = qa.HTTPFixture{
+	Method:       "GET",
+	Resource:     "/api/2.0/sql/endpoints",
+	Response:     map[string]interface{}{},
+	ReuseRequest: true,
+}
+
+var emptySqlDashboards = qa.HTTPFixture{
+	Method:       "GET",
+	Resource:     "/api/2.0/preview/sql/dashboards",
+	Response:     map[string]interface{}{},
+	ReuseRequest: true,
+}
+
+var emptySqlQueries = qa.HTTPFixture{
+	Method:       "GET",
+	Resource:     "/api/2.0/preview/sql/queries",
+	Response:     map[string]interface{}{},
+	ReuseRequest: true,
+}
+
 func TestImportingUsersGroupsSecretScopes(t *testing.T) {
 	qa.HTTPFixturesApply(t,
 		[]qa.HTTPFixture{
@@ -232,6 +254,9 @@ func TestImportingUsersGroupsSecretScopes(t *testing.T) {
 			emptyGitCredentials,
 			emptyWorkspace,
 			emptyIpAccessLIst,
+			emptySqlDashboards,
+			emptySqlEndpoints,
+			emptySqlQueries,
 			{
 				Method:   "GET",
 				Resource: "/api/2.0/preview/scim/v2/Groups?",
@@ -393,6 +418,9 @@ func TestImportingNoResourcesError(t *testing.T) {
 			emptyGitCredentials,
 			emptyIpAccessLIst,
 			emptyWorkspace,
+			emptySqlEndpoints,
+			emptySqlQueries,
+			emptySqlDashboards,
 			{
 				Method:       "GET",
 				Resource:     "/api/2.0/global-init-scripts",
@@ -1068,3 +1096,89 @@ func TestImportingIPAccessLists(t *testing.T) {
 			assert.NoError(t, err)
 		})
 }
+
+func TestImportingSqlObjects(t *testing.T) {
+	qa.HTTPFixturesApply(t,
+		[]qa.HTTPFixture{
+			meAdminFixture,
+			emptyRepos,
+			emptyIpAccessLIst,
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/global-init-scripts",
+				Response: map[string]interface{}{},
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/sql/endpoints",
+				Response: getJSONObject("test-data/get-sql-endpoints.json"),
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/sql/endpoints/f562046bc1272886",
+				Response: getJSONObject("test-data/get-sql-endpoint.json"),
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/preview/sql/data_sources",
+				Response: []sql.DataSource{
+					{
+						ID:         "147164a6-8316-4a9d-beff-f57261801374",
+						EndpointID: "f562046bc1272886",
+					},
+				},
+				ReuseRequest: true,
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/permissions/sql/endpoints/f562046bc1272886",
+				Response: getJSONObject("test-data/get-sql-endpoint-permissions.json"),
+			},
+			{
+				Method:       "GET",
+				Resource:     "/api/2.0/preview/sql/dashboards",
+				Response:     getJSONObject("test-data/get-sql-dashboards.json"),
+				ReuseRequest: true,
+			},
+			{
+				Method:       "GET",
+				Resource:     "/api/2.0/preview/sql/dashboards/9cb0c8f5-6262-4a1f-a741-2181de76028f",
+				Response:     getJSONObject("test-data/get-sql-dashboard.json"),
+				ReuseRequest: true,
+			},
+			{
+				Method:       "GET",
+				Resource:     "/api/2.0/preview/sql/queries",
+				Response:     getJSONObject("test-data/get-sql-queries.json"),
+				ReuseRequest: true,
+			},
+			{
+				Method:       "GET",
+				Resource:     "/api/2.0/preview/sql/queries/16c4f969-eea0-4aad-8f82-03d79b078dcc",
+				Response:     getJSONObject("test-data/get-sql-query.json"),
+				ReuseRequest: true,
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/preview/sql/permissions/queries/16c4f969-eea0-4aad-8f82-03d79b078dcc",
+				Response: getJSONObject("test-data/get-sql-query-permissions.json"),
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/preview/sql/permissions/dashboards/9cb0c8f5-6262-4a1f-a741-2181de76028f",
+				Response: getJSONObject("test-data/get-sql-dashboard-permissions.json"),
+			},
+		},
+		func(ctx context.Context, client *common.DatabricksClient) {
+			tmpDir := fmt.Sprintf("/tmp/tf-%s", qa.RandomName())
+			defer os.RemoveAll(tmpDir)
+
+			ic := newImportContext(client)
+			ic.Directory = tmpDir
+			ic.listing = "sql,access"
+			ic.services = "sql,access"
+
+			err := ic.Run()
+			assert.NoError(t, err)
+		})
+}