Add resources for configuring Serverless network connectivity (#3402)

* add `databricks_mws_network_connectivity_config`

* add `mws_ncc_binding` resource

* add `mws_ncc_private_endpoint_rule`

* add doc

* fix tests

* add provider

* add integration tests

* force new

* fix test

* feedback

* feedback

* Update docs/resources/mws_ncc_private_endpoint_rule.md

Co-authored-by: vickyzhu-db <[email protected]>

* feedback

* correct storage account

* wrong id

* more computed

* feedback

* new integration test helpers

* GA on Azure

---------

Co-authored-by: vickyzhu-db <[email protected]>

Author: nkvuong

access/resource_permission_assignment.go

@@ -3,7 +3,6 @@ package access
 import (
 	"context"
 	"fmt"
-	"strconv"
 
 	"github.com/databricks/databricks-sdk-go/apierr"
 	"github.com/databricks/terraform-provider-databricks/common"
@@ -65,14 +64,6 @@ func (a PermissionAssignmentAPI) List() (list PermissionAssignmentList, err erro
 	return
 }
 
-func mustInt64(s string) int64 {
-	n, err := strconv.ParseInt(s, 10, 64)
-	if err != nil {
-		panic(err)
-	}
-	return n
-}
-
 // ResourcePermissionAssignment performs of users to a workspace
 // from a workspace context, though it requires additional set
 // data resource for "workspace account scim", whicl will be added later.
@@ -101,7 +92,7 @@ func ResourcePermissionAssignment() common.Resource {
 				return err
 			}
 			data := entity{
-				PrincipalId: mustInt64(d.Id()),
+				PrincipalId: common.MustInt64(d.Id()),
 			}
 			permissions, err := list.ForPrincipal(data.PrincipalId)
 			if err != nil {

common/util.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"log"
 	"regexp"
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
@@ -34,3 +35,11 @@ func SuppressDiffWhitespaceChange(k, old, new string, d *schema.ResourceData) bo
 	log.Printf("[DEBUG] Suppressing diff for %v: old=%#v new=%#v", k, old, new)
 	return strings.TrimSpace(old) == strings.TrimSpace(new)
 }
+
+func MustInt64(s string) int64 {
+	n, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		panic(err)
+	}
+	return n
+}

common/util_test.go

@@ -30,3 +30,7 @@ func TestSuppressDiffWhitespaceChange(t *testing.T) {
 	assert.True(t, SuppressDiffWhitespaceChange("k", "value", "  value  ", nil))
 	assert.False(t, SuppressDiffWhitespaceChange("k", "value", "new_value", nil))
 }
+
+func TestMustInt64(t *testing.T) {
+	assert.Equal(t, int64(123), MustInt64("123"))
+}

docs/resources/mws_ncc_binding.md

@@ -0,0 +1,45 @@
+---
+subcategory: "Deployment"
+---
+# databricks_mws_ncc_binding Resource
+
+-> **Note** Initialize provider with `alias = "account"`, `host = "https://accounts.azuredatabricks.net"` and use `provider = databricks.account` for all `databricks_mws_*` resources.
+
+-> **Public Preview** This feature is available for AWS & Azure only, and is in [Public Preview](https://docs.databricks.com/release-notes/release-types.html) in AWS.
+
+Allows you to attach a [Network Connectivity Config](mws_network_connectivity_config) object to a [databricks_mws_workspaces](mws_workspaces.md) resource to create a [Databricks Workspace that leverages serverless network connectivity configs](https://learn.microsoft.com/en-us/azure/databricks/sql/admin/serverless-firewall).
+
+The NCC and workspace must be in the same region.
+
+## Example Usage
+
+```hcl
+variable "region" {}
+variable "prefix" {}
+
+resource "databricks_mws_network_connectivity_config" "ncc" {
+  provider = databricks.account
+  name     = "Network Connectivity Config for ${var.prefix}"
+  region   = var.region
+}
+
+resource "databricks_mws_ncc_binding" "ncc_binding" {
+  provider                       = databricks.account
+  network_connectivity_config_id = databricks_mws_network_connectivity_config.ncc.network_connectivity_config_id
+  workspace_id                   = var.databricks_workspace_id
+}
+```
+
+## Argument Reference
+
+The following arguments are available:
+
+* `network_connectivity_config_id` - Canonical unique identifier of Network Connectivity Config in Databricks Account.
+* `workspace_id` - Identifier of the workspace to attach the NCC to. Change forces creation of a new resource.
+
+## Related Resources
+
+The following resources are used in the context:
+
+* [databricks_mws_workspaces](mws_workspaces.md) to set up Databricks workspaces.
+* [databricks_mws_network_connectivity_config](mws_network_connectivity_config.md) to create Network Connectivity Config objects.

docs/resources/mws_ncc_private_endpoint_rule.md

@@ -0,0 +1,70 @@
+---
+subcategory: "Deployment"
+---
+# databricks_mws_ncc_private_endpoint_rule Resource
+
+-> **Note** Initialize provider with `alias = "account"`, `host = "https://accounts.azuredatabricks.net"` and use `provider = databricks.account` for all `databricks_mws_*` resources.
+
+-> **Note** This feature is only available in Azure.
+
+Allows you to create a private endpoint in a [Network Connectivity Config](mws_network_connectivity_config.md) that can be used to [configure private connectivity from serverless compute](https://learn.microsoft.com/en-us/azure/databricks/security/network/serverless-network-security/serverless-private-link).
+
+## Example Usage
+
+```hcl
+variable "region" {}
+variable "prefix" {}
+
+resource "databricks_mws_network_connectivity_config" "ncc" {
+  provider = databricks.account
+  name     = "Network Connectivity Config for ${var.prefix}"
+  region   = var.region
+}
+
+resource "databricks_mws_ncc_private_endpoint_rule" "storage" {
+  provider                       = databricks.account
+  network_connectivity_config_id = databricks_mws_network_connectivity_config.ncc.network_connectivity_config_id
+  resource_id                    = "/subscriptions/653bb673-1234-abcd-a90b-d064d5d53ca4/resourcegroups/example-resource-group/providers/Microsoft.Storage/storageAccounts/examplesa"
+  group_id                       = "blob"
+}
+```
+
+## Argument Reference
+
+The following arguments are available:
+
+* `network_connectivity_config_id` - Canonical unique identifier of Network Connectivity Config in Databricks Account. Change forces creation of a new resource.
+* `resource_id` - The Azure resource ID of the target resource. Change forces creation of a new resource.
+* `group_id` - The sub-resource type (group ID) of the target resource. Must be one of `blob`, `dfs`, `sqlServer` or `mysqlServer`. Note that to connect to workspace root storage (root DBFS), you need two endpoints, one for blob and one for dfs. Change forces creation of a new resource.
+
+## Attribute Reference
+
+In addition to all arguments above, the following attributes are exported:
+
+* `rule_id`- the ID of a private endpoint rule.
+* `endpoint_name` - The name of the Azure private endpoint resource, e.g. "databricks-088781b3-77fa-4132-b429-1af0d91bc593-pe-3cb31234"
+* `connection_state` - The current status of this private endpoint. The private endpoint rules are effective only if the connection state is ESTABLISHED. Remember that you must approve new endpoints on your resources in the Azure portal before they take effect.
+The possible values are:
+  * `PENDING`: The endpoint has been created and pending approval.
+  * `ESTABLISHED`: The endpoint has been approved and is ready to be used in your serverless compute resources.
+  * `REJECTED`: Connection was rejected by the private link resource owner.
+  * `DISCONNECTED`: Connection was removed by the private link resource owner, the private endpoint becomes informative and should be deleted for clean-up.
+* `deactivated` - Whether this private endpoint is deactivated.
+* `deactivated_at` - Time in epoch milliseconds when this object was deactivated.
+* `creation_time` - Time in epoch milliseconds when this object was created.
+* `updated_time` - Time in epoch milliseconds when this object was updated.
+
+## Import
+
+This resource can be imported by Databricks account ID and Network Connectivity Config ID.
+
+```sh
+terraform import databricks_mws_ncc_private_endpoint_rule.rule <network_connectivity_config_id>/<rule_id>
+```
+
+## Related Resources
+
+The following resources are used in the context:
+
+* [databricks_mws_network_connectivity_config](mws_network_connectivity_config.md) to create Network Connectivity Config objects.
+* [databricks_mws_ncc_binding](mws_ncc_binding.md) to attach an NCC to a workspace.

docs/resources/mws_network_connectivity_config.md

@@ -0,0 +1,59 @@
+---
+subcategory: "Deployment"
+---
+# databricks_mws_network_connectivity_config Resource
+
+-> **Note** Initialize provider with `alias = "account"`, `host = "https://accounts.azuredatabricks.net"` and use `provider = databricks.account` for all `databricks_mws_*` resources.
+
+-> **Public Preview** This feature is available for AWS & Azure only, and is in [Public Preview](https://docs.databricks.com/release-notes/release-types.html) in AWS.
+
+Allows you to create a [Network Connectivity Config] that can be used as part of a [databricks_mws_workspaces](mws_workspaces.md) resource to create a [Databricks Workspace that leverages serverless network connectivity configs](https://learn.microsoft.com/en-us/azure/databricks/security/network/serverless-network-security/serverless-firewall).
+
+## Example Usage
+
+```hcl
+variable "region" {}
+variable "prefix" {}
+
+resource "databricks_mws_network_connectivity_config" "ncc" {
+  provider = databricks.account
+  name     = "Network Connectivity Config for ${var.prefix}"
+  region   = var.region
+}
+
+resource "databricks_mws_ncc_binding" "ncc_binding" {
+  provider                       = databricks.account
+  network_connectivity_config_id = databricks_mws_network_connectivity_config.ncc.network_connectivity_config_id
+  workspace_id                   = var.databricks_workspace_id
+}
+```
+
+## Argument Reference
+
+The following arguments are available:
+
+* `name` - Name of Network Connectivity Config in Databricks Account. Change forces creation of a new resource.
+* `region` - Region of the Network Connectivity Config. NCCs can only be referenced by your workspaces in the same region. Change forces creation of a new resource.
+
+## Attribute Reference
+
+In addition to all arguments above, the following attributes are exported:
+
+* `network_connectivity_config_id` - Canonical unique identifier of Network Connectivity Config in Databricks Account
+* `default_rules.azure_service_endpoint_rule` - This provides a list of subnets. These subnets need to be allowed in your Azure resources in order for Databricks to access. See `default_rules.azure_service_endpoint_rule.target_services` for the supported Azure services.
+
+## Import
+
+This resource can be imported by Databricks account ID and Network Connectivity Config ID.
+
+```sh
+terraform import databricks_mws_network_connectivity_config.ncc <account_id>/<network_connectivity_config_id>
+```
+
+## Related Resources
+
+The following resources are used in the context:
+
+* [databricks_mws_workspaces](mws_workspaces.md) to set up Databricks workspaces.
+* [databricks_mws_ncc_binding](mws_ncc_binding.md) to attach an NCC to a workspace.
+* [databricks_mws_ncc_private_endpoint_rule](mws_ncc_private_endpoint_rule.md) to create a private endpoint rule.

internal/acceptance/mws_network_connectivity_config_test.go

@@ -0,0 +1,56 @@
+package acceptance
+
+import (
+	"testing"
+)
+
+func TestMwsAccNetworkConnectivityConfig(t *testing.T) {
+	if isAzure(t) {
+		accountLevel(t, step{
+			Template: `
+			resource "databricks_mws_network_connectivity_config" "this" {
+				name = "tf-{var.RANDOM}"
+				region = "eastus2"
+			}
+
+			resource "databricks_mws_ncc_private_endpoint_rule" "this" {
+				network_connectivity_config_id = databricks_mws_network_connectivity_config.this.network_connectivity_config_id
+				resource_id = "/subscriptions/2a5a4578-9ca9-47e2-ba46-f6ee6cc731f2/resourceGroups/deco-prod-azure-eastus2-rg/providers/Microsoft.Storage/storageAccounts/decotestprodunity"
+				group_id = "blob"
+			}
+			`,
+		}, step{
+			Template: `
+			resource "databricks_mws_network_connectivity_config" "this" {
+				name = "tf-{var.RANDOM}"
+				region = "eastus2"
+			}
+
+			resource "databricks_mws_ncc_private_endpoint_rule" "this" {
+				network_connectivity_config_id = databricks_mws_network_connectivity_config.this.network_connectivity_config_id
+				resource_id = "/subscriptions/2a5a4578-9ca9-47e2-ba46-f6ee6cc731f2/resourceGroups/deco-prod-azure-eastus2-rg/providers/Microsoft.Storage/storageAccounts/decotestprodunity"
+				group_id = "blob"
+			}
+			`,
+		})
+	}
+	if isAws(t) {
+		accountLevel(t, step{
+			Template: `
+			resource "databricks_mws_network_connectivity_config" "this" {
+				account_id = "{env.DATABRICKS_ACCOUNT_ID}"
+				name = "tf-{var.RANDOM}"
+				region = "{env.AWS_REGION}"
+			}
+			`,
+		}, step{
+			Template: `
+			resource "databricks_mws_network_connectivity_config" "this" {
+				account_id = "{env.DATABRICKS_ACCOUNT_ID}"
+				name = "tf-{var.RANDOM}"
+				region = "{env.AWS_REGION}"
+			}
+			`,
+		})
+	}
+}

mws/resource_mws_ncc_binding.go

@@ -0,0 +1,54 @@
+package mws
+
+import (
+	"context"
+	"log"
+
+	"github.com/databricks/databricks-sdk-go/service/provisioning"
+	"github.com/databricks/terraform-provider-databricks/common"
+
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+)
+
+func ResourceMwsNccBinding() common.Resource {
+	type binding struct {
+		WorkspaceId int64  `json:"workspace_id" tf:"force_new"`
+		NccId       string `json:"network_connectivity_config_id"`
+	}
+	s := common.StructToSchema(binding{}, common.NoCustomize)
+	p := common.NewPairSeparatedID("workspace_id", "network_connectivity_config_id", "/")
+	createOrUpdate := func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
+		acc, err := c.AccountClient()
+		if err != nil {
+			return err
+		}
+		wait, err := acc.Workspaces.Update(ctx, provisioning.UpdateWorkspaceRequest{
+			NetworkConnectivityConfigId: d.Get("network_connectivity_config_id").(string),
+			WorkspaceId:                 int64(d.Get("workspace_id").(int)),
+		})
+		if err != nil {
+			return err
+		}
+		_, err = wait.Get()
+		if err != nil {
+			return err
+		}
+		p.Pack(d)
+		return nil
+	}
+	return common.Resource{
+		Schema: s,
+		Create: createOrUpdate,
+		Read: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
+			if d.IsNewResource() {
+				log.Print("[WARN] Importing NCC binding is not supported, skipping...")
+			}
+			return nil
+		},
+		Update: createOrUpdate,
+		Delete: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
+			log.Printf("[WARN] Cannot remove network connectivity config binding, only update is supported.")
+			return nil
+		},
+	}
+}