Resource for global configuration of SQL Endpoints (#899)

alexott · web-flow · commit f16672070ceb · 2021-11-06T22:56:51.000+01:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,11 @@
 # Version changelog
 
+## 0.3.11
+
+* Added `databricks_sql_global_config` resource to provide global configuration for SQL Endpoints ([#855](https://github.com/databrickslabs/terraform-provider-databricks/issues/855))
+* Added `databricks_mount` resource to mount arbitrary cloud storage ([#497](https://github.com/databrickslabs/terraform-provider-databricks/issues/497))
+* Improved implementation of `databricks_repo` by creating the parent folder structure ([#895](https://github.com/databrickslabs/terraform-provider-databricks/pull/895))
+
 ## 0.3.10
 
 * Added `private_access_level` and `allowed_vpc_endpoint_ids` to `databricks_mws_private_access_settings` resource, which is also now updatable ([#867](https://github.com/databrickslabs/terraform-provider-databricks/issues/867)).
diff --git a/README.md b/README.md
@@ -50,6 +50,7 @@
 | [databricks_spark_version](docs/data-sources/spark_version.md) data
 | [databricks_sql_dashboard](docs/resources/sql_dashboard.md)
 | [databricks_sql_endpoint](docs/resources/sql_endpoint.md)
+| [databricks_sql_global_config](docs/resources/sql_global_config.md)
 | [databricks_sql_permissions](docs/resources/sql_permissions.md)
 | [databricks_sql_query](docs/resources/sql_query.md)
 | [databricks_sql_visualization](docs/resources/sql_visualization.md)
diff --git a/common/client.go b/common/client.go
@@ -413,7 +413,7 @@ func (c *DatabricksClient) configureHTTPCLient() {
 
 // IsAzure returns true if client is configured for Azure Databricks - either by using AAD auth or with host+token combination
 func (c *DatabricksClient) IsAzure() bool {
-	return c.resourceID() != "" || strings.Contains(c.Host, ".azuredatabricks.net") || c.AzureUseMSI
+	return c.resourceID() != "" || c.AzureClientID != "" || c.AzureUseMSI || strings.Contains(c.Host, ".azuredatabricks.net")
 }
 
 // IsAws returns true if client is configured for AWS
diff --git a/common/client_test.go b/common/client_test.go
@@ -182,3 +182,21 @@ func TestDatabricksClient_AuthenticateAzure(t *testing.T) {
 		"behavior.html more info. Environment variables used: ARM_CLIENT_SECRET, ARM_CLIENT_ID. "+
 		"Please check https://registry.terraform.io/providers/databrickslabs/databricks/latest/docs#authentication for details")
 }
+
+func TestDatabricksIsGcp(t *testing.T) {
+	dc, err := configureAndAuthenticate(&DatabricksClient{
+		Host:  "https://demo.gcp.databricks.com/",
+		Token: "dapi123",
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, true, dc.IsGcp())
+}
+
+func TestIsAzure_Error(t *testing.T) {
+	dc := &DatabricksClient{
+		Token:      "connfigured",
+		ConfigFile: "testdata/.databrickscfg",
+		Profile:    "notoken",
+	}
+	assert.Equal(t, false, dc.IsAzure())
+}
diff --git a/docs/index.md b/docs/index.md
@@ -47,6 +47,7 @@ Databricks SQL
 * Create [databricks_sql_endpoint](resources/sql_endpoint.md) controlled by [databricks_permissions](resources/permissions.md).
 * Manage [queries](resources/sql_query.md) and their [visualizations](resources/sql_visualization.md).
 * Manage [dashboards](resources/sql_dashboard.md) and their [widgets](resources/sql_widget.md).
+* Provide [global configuration for all SQL Endpoints](docs/resources/sql_global_config.md)
 
 ## Example Usage
 
diff --git a/docs/resources.png b/docs/resources.png
diff --git a/docs/resources/sql_dashboard.md b/docs/resources/sql_dashboard.md
@@ -3,7 +3,7 @@ subcategory: "Databricks SQL"
 ---
 # databricks_sql_dashboard Resource
 
--> **Public Preview** This feature is in [Public Preview](https://docs.databricks.com/release-notes/release-types.html). Contact your Databricks representative to request access.
+-> **Public Preview** This feature is in [Public Preview](https://docs.databricks.com/release-notes/release-types.html).
 
 To manage [SQLA resources](https://docs.databricks.com/sql/get-started/concepts.html) you must have `allow_sql_analytics_access` on your [databricks_group](group.md#allow_sql_analytics_access) or [databricks_user](user.md#allow_sql_analytics_access).
 
diff --git a/docs/resources/sql_endpoint.md b/docs/resources/sql_endpoint.md
@@ -3,7 +3,7 @@ subcategory: "Databricks SQL"
 ---
 # databricks_sql_endpoint Resource
 
--> **Public Preview** This feature is in [Public Preview](https://docs.databricks.com/release-notes/release-types.html). Contact your Databricks representative to request access.
+-> **Public Preview** This feature is in [Public Preview](https://docs.databricks.com/release-notes/release-types.html).
 
 To create [SQL endpoints](https://docs.databricks.com/sql/get-started/concepts.html) you must have `allow_sql_analytics_access` on your [databricks_group](group.md#allow_sql_analytics_access) or [databricks_user](user.md#allow_sql_analytics_access).
 
diff --git a/docs/resources/sql_global_config.md b/docs/resources/sql_global_config.md
@@ -0,0 +1,36 @@
+---
+subcategory: "Databricks SQL"
+---
+# databricks_sql_global_config Resource
+
+-> **Public Preview** This feature is in [Public Preview](https://docs.databricks.com/release-notes/release-types.html).
+
+This resource configures the security policy, instance profile (AWS only), and data access properties for all SQL endpoints of workspace. *Please note that changing parameters of this resources will restart all running SQL endpoints.*  To use this resource you need to be an administrator.
+
+## Example usage
+
+```hcl
+resource "databricks_sql_global_config" "this" {
+  security_policy = "DATA_ACCESS_CONTROL"
+  instance_profile_arn = "arn:...."
+  data_access_config = {
+    "spark.sql.session.timeZone": "UTC"
+  }
+}
+```
+
+## Argument Reference
+
+The following arguments are supported (see [documentation](https://docs.databricks.com/sql/api/sql-endpoints.html#global-edit) for more details):
+
+* `security_policy` (Optional, String) - The policy for controlling access to datasets. Default value: `DATA_ACCESS_CONTROL`, consult documentation for list of possible values
+* `data_access_config` (Optional, Map) - data access configuration for SQL Endpoints, such as configuration for an external Hive metastore, Hadoop Filesystem configuration, etc.  Please note that the list of supported configuration properties is limited, so refer to the [documentation](https://docs.databricks.com/sql/admin/data-access-configuration.html#supported-properties) for a full list.  Apply will fail if you're specifying not permitted configuration.
+* `instance_profile_arn` (Optional, String) - Instance profile used to access storage from SQL endpoints. Please note that this parameter is only for AWS, and will generate an error if used on other clouds.
+
+## Import
+
+You can import a `databricks_sql_global_config` resource with command like the following (you need to use `global` as ID):
+
+```bash
+$ terraform import databricks_sql_global_config.this global
+```
diff --git a/docs/resources/sql_query.md b/docs/resources/sql_query.md
@@ -3,7 +3,7 @@ subcategory: "Databricks SQL"
 ---
 # databricks_sql_query Resource
 
--> **Public Preview** This feature is in [Public Preview](https://docs.databricks.com/release-notes/release-types.html). Contact your Databricks representative to request access.
+-> **Public Preview** This feature is in [Public Preview](https://docs.databricks.com/release-notes/release-types.html).
 
 To manage [SQLA resources](https://docs.databricks.com/sql/get-started/concepts.html) you must have `allow_sql_analytics_access` on your [databricks_group](group.md#allow_sql_analytics_access) or [databricks_user](user.md#allow_sql_analytics_access).
 
diff --git a/docs/resources/sql_visualization.md b/docs/resources/sql_visualization.md
@@ -3,7 +3,7 @@ subcategory: "Databricks SQL"
 ---
 # databricks_sql_visualization Resource
 
--> **Public Preview** This feature is in [Public Preview](https://docs.databricks.com/release-notes/release-types.html). Contact your Databricks representative to request access.
+-> **Public Preview** This feature is in [Public Preview](https://docs.databricks.com/release-notes/release-types.html).
 
 To manage [SQLA resources](https://docs.databricks.com/sql/get-started/concepts.html) you must have `allow_sql_analytics_access` on your [databricks_group](group.md#allow_sql_analytics_access) or [databricks_user](user.md#allow_sql_analytics_access).
 
@@ -42,4 +42,4 @@ resource "databricks_sql_visualization" "q1v1" {
     }
   )
 }
-```
+```
diff --git a/docs/resources/sql_widget.md b/docs/resources/sql_widget.md
@@ -3,7 +3,7 @@ subcategory: "Databricks SQL"
 ---
 # databricks_sql_widget Resource
 
--> **Public Preview** This feature is in [Public Preview](https://docs.databricks.com/release-notes/release-types.html). Contact your Databricks representative to request access.
+-> **Public Preview** This feature is in [Public Preview](https://docs.databricks.com/release-notes/release-types.html).
 
 To manage [SQLA resources](https://docs.databricks.com/sql/get-started/concepts.html) you must have `allow_sql_analytics_access` on your [databricks_group](group.md#allow_sql_analytics_access) or [databricks_user](user.md#allow_sql_analytics_access).
 
diff --git a/identity/resource_group_test.go b/identity/resource_group_test.go
@@ -279,8 +279,8 @@ func TestResourceGroupUpdate(t *testing.T) {
 		allow_sql_analytics_access = true
 		`,
 		RequiresNew: true,
-		Update: true,
-		ID:     "abc",
+		Update:      true,
+		ID:          "abc",
 	}.Apply(t)
 	assert.NoError(t, err, err)
 	assert.Equal(t, "abc", d.Id(), "Id should be the same as in reading")
@@ -308,9 +308,9 @@ func TestResourceGroupUpdate_Error(t *testing.T) {
 			"display_name":               "Data Ninjas",
 			"allow_instance_pool_create": true,
 		},
-		Update: true,
+		Update:      true,
 		RequiresNew: true,
-		ID:     "abc",
+		ID:          "abc",
 	}.ExpectError(t, "Internal error happened")
 }
 
diff --git a/provider/provider.go b/provider/provider.go
@@ -81,6 +81,7 @@ func DatabricksProvider() *schema.Provider {
 
 			"databricks_sql_dashboard":     sqlanalytics.ResourceDashboard(),
 			"databricks_sql_endpoint":      sqlanalytics.ResourceSQLEndpoint(),
+			"databricks_sql_global_config": sqlanalytics.ResourceSQLGlobalConfig(),
 			"databricks_sql_query":         sqlanalytics.ResourceQuery(),
 			"databricks_sql_visualization": sqlanalytics.ResourceVisualization(),
 			"databricks_sql_widget":        sqlanalytics.ResourceWidget(),
diff --git a/sqlanalytics/acceptance/sql_endpoint_test.go b/sqlanalytics/acceptance/sql_endpoint_test.go
@@ -11,7 +11,7 @@ func TestPreviewAccSQLEndpoint(t *testing.T) {
 		{
 			Template: `resource "databricks_sql_endpoint" "this" {
 				name = "tf-{var.RANDOM}"
-				cluster_size = "Small"
+				cluster_size = "XX-Small"
 				max_num_clusters = 1
 			}`,
 		},
diff --git a/sqlanalytics/acceptance/sql_global_config_test.go b/sqlanalytics/acceptance/sql_global_config_test.go
@@ -0,0 +1,19 @@
+package acceptance
+
+import (
+	"testing"
+
+	"github.com/databrickslabs/terraform-provider-databricks/internal/acceptance"
+)
+
+func TestPreviewAccSQLGlobalConfig(t *testing.T) {
+	acceptance.Test(t, []acceptance.Step{
+		{
+			Template: `resource "databricks_sql_global_config" "this" {
+				data_access_config = {
+					"spark.sql.session.timeZone": "UTC"
+				}  
+			}`,
+		},
+	})
+}
diff --git a/sqlanalytics/resource_sql_global_config.go b/sqlanalytics/resource_sql_global_config.go
@@ -0,0 +1,123 @@
+package sqlanalytics
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databrickslabs/terraform-provider-databricks/common"
+
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+)
+
+type ConfPair struct {
+	Key   string `json:"key"`
+	Value string `json:"value"`
+}
+
+// GlobalConfig ...
+type GlobalConfig struct {
+	SecurityPolicy          string            `json:"security_policy,omitempty" tf:"default:DATA_ACCESS_CONTROL"`
+	DataAccessConfig        map[string]string `json:"data_access_config,omitempty"`
+	InstanceProfileARN      string            `json:"instance_profile_arn,omitempty"`
+	EnableServerlessCompute bool              `json:"enable_serverless_compute,omitempty" tf:"default:false"`
+}
+
+// GlobalConfigForRead ...
+type GlobalConfigForRead struct {
+	SecurityPolicy          string     `json:"security_policy"`
+	DataAccessConfig        []ConfPair `json:"data_access_config"`
+	InstanceProfileARN      string     `json:"instance_profile_arn,omitempty"`
+	EnableServerlessCompute bool       `json:"enable_serverless_compute,omitempty"`
+}
+
+// NewSqlGlobalConfigAPI ...
+func NewSqlGlobalConfigAPI(ctx context.Context, m interface{}) globalConfigAPI {
+	return globalConfigAPI{m.(*common.DatabricksClient), ctx}
+}
+
+// sAPI ...
+type globalConfigAPI struct {
+	client  *common.DatabricksClient
+	context context.Context
+}
+
+// Set ...
+func (a globalConfigAPI) Set(gc GlobalConfig) error {
+	data := map[string]interface{}{
+		"security_policy":           gc.SecurityPolicy,
+		"enable_serverless_compute": gc.EnableServerlessCompute,
+	}
+	if a.client.Host == "" {
+		err := a.client.Authenticate(a.context)
+		if err != nil {
+			return err
+		}
+	}
+	if a.client.IsAws() {
+		data["instance_profile_arn"] = gc.InstanceProfileARN
+	} else if gc.InstanceProfileARN != "" {
+		return fmt.Errorf("can't use instance_profile_arn outside of AWS")
+	}
+	cfg := make([]ConfPair, 0, len(gc.DataAccessConfig))
+	for k, v := range gc.DataAccessConfig {
+		cfg = append(cfg, ConfPair{Key: k, Value: v})
+	}
+	data["data_access_config"] = cfg
+
+	return a.client.Put(a.context, "/sql/config/endpoints", data)
+}
+
+func (a globalConfigAPI) Get() (GlobalConfig, error) {
+	gc := GlobalConfig{}
+	gcr := GlobalConfigForRead{}
+	if err := a.client.Get(a.context, "/sql/config/endpoints", nil, &gcr); err != nil {
+		return gc, err
+	}
+	gc.InstanceProfileARN = gcr.InstanceProfileARN
+	gc.SecurityPolicy = gcr.SecurityPolicy
+	gc.EnableServerlessCompute = gcr.EnableServerlessCompute
+	gc.DataAccessConfig = make(map[string]string, len(gcr.DataAccessConfig))
+	for _, v := range gcr.DataAccessConfig {
+		gc.DataAccessConfig[v.Key] = v.Value
+	}
+
+	return gc, nil
+}
+
+// ResourceSQLGlobalConfig ...
+func ResourceSQLGlobalConfig() *schema.Resource {
+	s := common.StructToSchema(GlobalConfig{}, func(
+		m map[string]*schema.Schema) map[string]*schema.Schema {
+		m["instance_profile_arn"].Default = ""
+		return m
+	})
+
+	set_func := func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
+		var gc GlobalConfig
+		if err := common.DataToStructPointer(d, s, &gc); err != nil {
+			return err
+		}
+		if err := NewSqlGlobalConfigAPI(ctx, c).Set(gc); err != nil {
+			return err
+		}
+		d.SetId("global")
+		return nil
+	}
+
+	return common.Resource{
+		Create: set_func,
+		Read: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
+			gc, err := NewSqlGlobalConfigAPI(ctx, c).Get()
+			if err != nil {
+				return err
+			}
+			err = common.StructToData(gc, s, d)
+			return err
+		},
+		Update: set_func,
+		Delete: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
+			return NewSqlGlobalConfigAPI(ctx, c).Set(GlobalConfig{SecurityPolicy: "DATA_ACCESS_CONTROL"})
+		},
+		Schema: s,
+	}.ToResource()
+}
diff --git a/sqlanalytics/resource_sql_global_config_test.go b/sqlanalytics/resource_sql_global_config_test.go

Original file line number	Diff line number	Diff line change
`@@ -413,7 +413,7 @@ func (c *DatabricksClient) configureHTTPCLient() {`
`413`	`413`
`414`	`414`	`// IsAzure returns true if client is configured for Azure Databricks - either by using AAD auth or with host+token combination`
`415`	`415`	`func (c *DatabricksClient) IsAzure() bool {`
`416`		`- return c.resourceID() != "" \|\| strings.Contains(c.Host, ".azuredatabricks.net") \|\| c.AzureUseMSI`
	`416`	`+ return c.resourceID() != "" \|\| c.AzureClientID != "" \|\| c.AzureUseMSI \|\| strings.Contains(c.Host, ".azuredatabricks.net")`
`417`	`417`	`}`
`418`	`418`
`419`	`419`	`// IsAws returns true if client is configured for AWS`