[Exporter] Correctly handle DB-managed UC objects (#4323)

## Changes
<!-- Summary of your changes that are easy to understand -->

Databricks has started to introduce UC objects that are created and
managed by Databricks, i.e., storage credentials and external locations,
so we can't create resources to manage them.

This PR converts such managed UC objects into data sources so we can
attach permissions to them and handle references. The change of
ownership is still not implemented - filed
#4321
to handle it.

Other changes include:

- refactoring of creation of new resource data objects
- fixing a problem with the codegen of data sources, so we don't need a
dedicated `Body` implementation.

## Tests
<!-- 
How is this tested? Please see the checklist below and also describe any
other relevant tests
-->

- [x] `make test` run locally
- [ ] relevant change in `docs/` folder
- [ ] covered with integration tests in `internal/acceptance`
- [ ] using Go SDK
- [ ] using TF Plugin Framework

Author: alexott

exporter/codegen.go

@@ -662,8 +662,13 @@ func (ic *importContext) processSingleResource(resourcesChan resourceChannel,
 				log.Printf("[ERROR] error calling ir.Body for %v: %s", r, err.Error())
 			}
 		} else {
-			resourceBlock := body.AppendNewBlock("resource", []string{r.Resource, r.Name})
-			err = ic.dataToHcl(ir, []string{}, ic.Resources[r.Resource], r, resourceBlock.Body())
+			blockType := "resource"
+			if r.Mode == "data" {
+				blockType = r.Mode
+			}
+			resourceBlock := body.AppendNewBlock(blockType, []string{r.Resource, r.Name})
+			err = ic.dataToHcl(ic.Importables[r.Resource],
+				[]string{}, ic.Resources[r.Resource], r, resourceBlock.Body())
 			if err != nil {
 				log.Printf("[ERROR] error generating body for %v: %s", r, err.Error())
 			}

exporter/importables.go

@@ -89,7 +89,8 @@ var (
 		"storage_credential": {`CREATE_EXTERNAL_LOCATION`, `CREATE_EXTERNAL_TABLE`},
 		"foreign_connection": {`CREATE_FOREIGN_CATALOG`},
 	}
-	ParentDirectoryExtraKey = "parent_directory"
+	ParentDirectoryExtraKey   = "parent_directory"
+	dbManagedExternalLocation = "__databricks_managed_storage_location"
 )
 
 func generateMountBody(ic *importContext, body *hclwrite.Body, r *resource) error {
@@ -899,9 +900,8 @@ var resourcesMap map[string]importable = map[string]importable{
 			{Path: "libraries.jar", Resource: "databricks_repo", Match: "workspace_path",
 				MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
 		},
-		// TODO: special formatting required, where JSON is written line by line
-		// so that we're able to do the references
-		Body: resourceOrDataBlockBody,
+		// TODO: implement a custom Body that will write with special formatting, where
+		// JSON is written line by line so that we're able to do the references
 	},
 	"databricks_group": {
 		Service:        "groups",
@@ -1051,7 +1051,6 @@ var resourcesMap map[string]importable = map[string]importable{
 
 			return nil
 		},
-		Body: resourceOrDataBlockBody,
 	},
 	"databricks_group_member": {
 		Service:        "groups",
@@ -2291,7 +2290,6 @@ var resourcesMap map[string]importable = map[string]importable{
 			}
 			return nil
 		},
-		Body: resourceOrDataBlockBody,
 		Depends: []reference{
 			{Path: "path", Resource: "databricks_user", Match: "home"},
 			{Path: "path", Resource: "databricks_service_principal", Match: "home"},
@@ -2964,6 +2962,14 @@ var resourcesMap map[string]importable = map[string]importable{
 		WorkspaceLevel: true,
 		Service:        "uc-storage-credentials",
 		Import: func(ic *importContext, r *resource) error {
+			if r.ID == "__databricks_managed_storage_credential" {
+				// it's created by default and can't be imported
+				// TODO: add check for "securable_kind":"STORAGE_CREDENTIAL_DB_AWS_IAM" when we get it in the credential
+				r.Mode = "data"
+				data := tfcatalog.ResourceStorageCredential().ToResource().TestResourceData()
+				obj := tfcatalog.StorageCredentialInfo{Name: r.ID}
+				r.Data = ic.generateNewData(data, "databricks_storage_credential", r.ID, obj)
+			}
 			ic.emitUCGrantsWithOwner("storage_credential/"+r.ID, r)
 			if r.Data != nil {
 				isolationMode := r.Data.Get("isolation_mode").(string)
@@ -3036,6 +3042,14 @@ var resourcesMap map[string]importable = map[string]importable{
 		WorkspaceLevel: true,
 		Service:        "uc-external-locations",
 		Import: func(ic *importContext, r *resource) error {
+			if r.ID == dbManagedExternalLocation {
+				// it's created by default and can't be imported
+				// TODO: add check for "securable_kind":"EXTERNAL_LOCATION_DB_STORAGE" when we get it in the credential
+				r.Mode = "data"
+				data := tfcatalog.ResourceExternalLocation().ToResource().TestResourceData()
+				obj := tfcatalog.ExternalLocationInfo{Name: r.ID}
+				r.Data = ic.generateNewData(data, "databricks_external_location", r.ID, obj)
+			}
 			ic.emitUCGrantsWithOwner("external_location/"+r.ID, r)
 			credentialName := r.Data.Get("credential_name").(string)
 			ic.Emit(&resource{
@@ -3067,7 +3081,15 @@ var resourcesMap map[string]importable = map[string]importable{
 			}
 			return nil
 		},
-		ShouldOmitField: shouldOmitWithIsolationMode,
+		ShouldOmitField: func(ic *importContext, pathString string, as *schema.Schema, d *schema.ResourceData) bool {
+			if (pathString == "url" || pathString == "credential_name") && d.Get("name").(string) == dbManagedExternalLocation {
+				return true
+			}
+			if pathString == "isolation_mode" {
+				return d.Get(pathString).(string) != "ISOLATION_MODE_ISOLATED"
+			}
+			return shouldOmitForUnityCatalog(ic, pathString, as, d)
+		},
 		// This external location is automatically created when metastore is created with the `storage_root`
 		Ignore: func(ic *importContext, r *resource) bool {
 			return r.ID == "metastore_default_location"

exporter/util.go

@@ -19,7 +19,6 @@ import (
 
 	"github.com/databricks/databricks-sdk-go/service/catalog"
 
-	"github.com/hashicorp/hcl/v2/hclwrite"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
 )
@@ -310,7 +309,7 @@ func (ic *importContext) saveFileIn(dir, name string, content []byte) (string, e
 	return relativeName, nil
 }
 
-func defaultShouldOmitFieldFunc(ic *importContext, pathString string, as *schema.Schema, d *schema.ResourceData) bool {
+func defaultShouldOmitFieldFunc(_ *importContext, pathString string, as *schema.Schema, d *schema.ResourceData) bool {
 	if as.Computed {
 		return true
 	} else if as.Default != nil && d.Get(pathString) == as.Default {
@@ -320,14 +319,16 @@ func defaultShouldOmitFieldFunc(ic *importContext, pathString string, as *schema
 	return false
 }
 
-func resourceOrDataBlockBody(ic *importContext, body *hclwrite.Body, r *resource) error {
-	blockType := "resource"
-	if r.Mode == "data" {
-		blockType = r.Mode
+func (ic *importContext) generateNewData(data *schema.ResourceData, resourceType, rID string, obj any) *schema.ResourceData {
+	data.MarkNewResource()
+	data.SetId(rID)
+	scm := ic.Resources[resourceType].Schema
+	err := common.StructToData(obj, scm, data)
+	if err != nil {
+		log.Printf("[ERROR] can't convert %s object to data: %v. obj=%v", resourceType, err, obj)
+		return nil
 	}
-	resourceBlock := body.AppendNewBlock(blockType, []string{r.Resource, r.Name})
-	return ic.dataToHcl(ic.Importables[r.Resource],
-		[]string{}, ic.Resources[r.Resource], r, resourceBlock.Body())
+	return data
 }
 
 func generateUniqueID(v string) string {

exporter/util_workspace.go

@@ -9,7 +9,6 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/databricks/terraform-provider-databricks/common"
 	"github.com/databricks/terraform-provider-databricks/workspace"
 
 	"golang.org/x/exp/slices"
@@ -246,14 +245,7 @@ func (ic *importContext) maybeEmitWorkspaceObject(resourceType, path string, obj
 				data = workspace.ResourceDirectory().ToResource().TestResourceData()
 			}
 			if data != nil {
-				scm := ic.Resources[resourceType].Schema
-				data.MarkNewResource()
-				data.SetId(path)
-				err := common.StructToData(obj, scm, data)
-				if err != nil {
-					log.Printf("[ERROR] can't convert %s object to data: %v. obj=%v", resourceType, err, obj)
-					data = nil
-				}
+				data = ic.generateNewData(data, resourceType, path, obj)
 			}
 		}
 		ic.Emit(&resource{