Adding new experimental resources (#1021)

Added `databricks_external_location` resource

Author: nfx

catalog/resource_external_location.go

@@ -0,0 +1,85 @@
+package catalog
+
+import (
+	"context"
+	"net/url"
+
+	"github.com/databrickslabs/terraform-provider-databricks/common"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+)
+
+type ExternalLocationsAPI struct {
+	client  *common.DatabricksClient
+	context context.Context
+}
+
+func NewExternalLocationsAPI(ctx context.Context, m interface{}) ExternalLocationsAPI {
+	return ExternalLocationsAPI{m.(*common.DatabricksClient), ctx}
+}
+
+type ExternalLocationInfo struct {
+	Name           string `json:"name" tf:"force_new"`
+	URL            string `json:"url"`
+	CredentialName string `json:"credential_name"`
+	Comment        string `json:"comment,omitempty"`
+	Owner          string `json:"owner,omitempty" tf:"computed"`
+	MetastoreID    string `json:"metastore_id,omitempty" tf:"computed"`
+}
+
+func (a ExternalLocationsAPI) create(el *ExternalLocationInfo) error {
+	return a.client.Post(a.context, "/unity-catalog/external-locations", el, &el)
+}
+
+func (a ExternalLocationsAPI) get(name string) (el ExternalLocationInfo, err error) {
+	err = a.client.Get(a.context, "/unity-catalog/external-locations/"+url.PathEscape(name), nil, &el)
+	return
+}
+
+func (a ExternalLocationsAPI) update(name string, el ExternalLocationInfo) error {
+	return a.client.Patch(a.context, "/unity-catalog/external-locations/"+url.PathEscape(name), el)
+}
+
+func (a ExternalLocationsAPI) delete(name string) error {
+	return a.client.Delete(a.context, "/unity-catalog/external-locations/"+url.PathEscape(name), nil)
+}
+
+func ResourceExternalLocation() *schema.Resource {
+	s := common.StructToSchema(ExternalLocationInfo{},
+		func(m map[string]*schema.Schema) map[string]*schema.Schema {
+			return m
+		})
+	return common.Resource{
+		Schema: s,
+		Create: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
+			var el ExternalLocationInfo
+			common.DataToStructPointer(d, s, &el)
+			err := NewExternalLocationsAPI(ctx, c).create(&el)
+			if err != nil {
+				return err
+			}
+			d.SetId(el.Name)
+			return nil
+		},
+		Read: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
+			el, err := NewExternalLocationsAPI(ctx, c).get(d.Id())
+			if err != nil {
+				return err
+			}
+			return common.StructToData(el, s, d)
+		},
+		Update: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
+			var el ExternalLocationInfo
+			common.DataToStructPointer(d, s, &el)
+			return NewExternalLocationsAPI(ctx, c).update(d.Id(), ExternalLocationInfo{
+				Name:  d.Id(),
+				URL: el.URL,
+				CredentialName: el.CredentialName,
+				Comment: el.Comment,
+				Owner: el.Owner,
+			})
+		},
+		Delete: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
+			return NewExternalLocationsAPI(ctx, c).delete(d.Id())
+		},
+	}.ToResource()
+}

catalog/resource_external_location_test.go

@@ -0,0 +1,44 @@
+package catalog
+
+import (
+	"testing"
+
+	"github.com/databrickslabs/terraform-provider-databricks/qa"
+)
+
+func TestExternalLocationCornerCases(t *testing.T) {
+	qa.ResourceCornerCases(t, ResourceExternalLocation())
+}
+
+func TestCreateExternalLocation(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "POST",
+				Resource: "/api/2.0/unity-catalog/external-locations",
+				ExpectedRequest: ExternalLocationInfo{
+					Name:           "abc",
+					URL:            "s3://foo/bar",
+					CredentialName: "bcd",
+					Comment:        "def",
+				},
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/unity-catalog/external-locations/abc",				
+				Response: ExternalLocationInfo {
+					Owner: "efg",
+					MetastoreID: "fgh",
+				},
+			},
+		},
+		Resource: ResourceExternalLocation(),
+		Create:   true,
+		HCL: `
+		name = "abc"
+		url = "s3://foo/bar"
+		credential_name = "bcd"
+		comment = "def"
+		`,
+	}.ApplyNoError(t)
+}

catalog/resource_grants.go

@@ -129,6 +129,18 @@ var mapping = securableMapping{
 		"USAGE":          true,
 		"ALL PRIVILEGES": true,
 	},
+	"storage_credential": {
+		"CREATE TABLE":    true,
+		"READ FILES":      true,
+		"WRITE FILES":     true,
+		"ALL PRIVILEGES":  true,
+	},
+	"external_location": {
+		"CREATE TABLE":   true,
+		"READ FILES":     true,
+		"WRITE FILES":    true,
+		"ALL PRIVILEGES": true,
+	},
 }
 
 func setToStrings(set *schema.Set) (ss []string) {
@@ -213,6 +225,10 @@ func ResourceGrants() *schema.Resource {
 	return common.Resource{
 		Schema: s,
 		CustomizeDiff: func(ctx context.Context, d *schema.ResourceDiff, c interface{}) error {
+			if d.Id() == "" {
+				// unfortunately we cannot do validation before dependent resources exist with tfsdkv2
+				return nil
+			}
 			var grants PermissionsList
 			common.DiffToStructPointer(d, s, &grants)
 			return mapping.validate(d, grants)

catalog/resource_grants_test.go

@@ -211,6 +211,52 @@ func TestGrantCreate(t *testing.T) {
 	}.ApplyNoError(t)
 }
 
+func TestGrantUpdate(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "PATCH",
+				Resource: "/api/2.0/unity-catalog/permissions/table/foo.bar.baz",
+				ExpectedRequest: PermissionsDiff{
+					Changes: []PermissionsChange{
+						{
+							Principal: "me",
+							Add:       []string{"MODIFY", "SELECT"},
+						},
+					},
+				},
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/unity-catalog/permissions/table/foo.bar.baz",
+
+				Response: PermissionsList{
+					Assignments: []PrivilegeAssignment{
+						{
+							Principal:  "me",
+							Privileges: []string{"MODIFY", "SELECT"},
+						},
+					},
+				},
+			},
+		},
+		Resource: ResourceGrants(),
+		Update:   true,
+		ID: "table/foo.bar.baz",
+		InstanceState: map[string]string{
+			"table": "foo.bar.baz",
+		},
+		HCL: `
+		table = "foo.bar.baz"
+
+		grant {
+			principal = "me"
+			privileges = ["MODIFY", "SELECT"]
+		}
+		`,
+	}.ApplyNoError(t)
+}
+
 func TestGrantReadMalformedId(t *testing.T) {
 	qa.ResourceFixture{
 		Resource: ResourceGrants(),

provider/provider.go

@@ -60,6 +60,7 @@ func DatabricksProvider() *schema.Provider {
 			"databricks_cluster_policy":              policies.ResourceClusterPolicy(),
 			"databricks_dbfs_file":                   storage.ResourceDBFSFile(),
 			"databricks_directory":                   workspace.ResourceDirectory(),
+			"databricks_external_location":           catalog.ResourceExternalLocation(),
 			"databricks_global_init_script":          workspace.ResourceGlobalInitScript(),
 			"databricks_grants":                      catalog.ResourceGrants(),
 			"databricks_group":                       scim.ResourceGroup(),

qa/testing.go

@@ -224,7 +224,7 @@ func (f ResourceFixture) Apply(t *testing.T) (*schema.ResourceData, error) {
 		return resourceData, fmt.Errorf("resource is not expected to be removed")
 	}
 	newState := resourceData.State()
-	diff, err = schemaMap.Diff(ctx, newState, resourceConfig, nil, client, true)
+	diff, err = schemaMap.Diff(ctx, newState, resourceConfig, f.Resource.CustomizeDiff, client, true)
 	if err != nil {
 		return nil, err
 	}

scripts/nightly/awsit.tf

@@ -96,7 +96,7 @@ resource "databricks_mws_storage_configurations" "this" {
   storage_configuration_name = "${local.prefix}-cx-terraform-it"
 }
 
-output "aws_bucket" {
+output "aws_root_bucket" {
   value = aws_s3_bucket.root_storage_bucket.bucket
 }
 
@@ -277,4 +277,17 @@ resource "azurerm_container_group" "aws" {
       protocol = "TCP"
     }
   }
+}
+
+output "aws_workspace_url" {
+  value = databricks_mws_workspaces.this.workspace_url
+}
+
+output "aws_workspace_id" {
+  value = databricks_mws_workspaces.this.workspace_id
+}
+
+output "aws_workspace_pat" {
+  value = databricks_mws_workspaces.this.token[0].token_value
+  sensitive = true
 }

scripts/nightly/main.tf

@@ -6,6 +6,10 @@ locals {
   tags       = azurerm_resource_group.this.tags
 }
 
+output "prefix" {
+  value = local.prefix
+}
+
 output "tags" {
   value = local.tags
 }