Exporter: add support for generation of dependencies between resources (#3377)

* Exporter: add support for generation of dependencies between resources

Now we can correct dependencies between some UC objects and their parent's grants in case if the owner of
these objects is different than the current user and grants should be adjusted to provide
a possibility to create sub-resources.

This fixes #3057

* Add code coverage

Author: alexott

exporter/context.go

@@ -1672,6 +1672,57 @@ func (ic *importContext) dataToHcl(i importable, path []string,
 			return fmt.Errorf("unsupported schema type: %v", path)
 		}
 	}
+	// Generate `depends_on` only for top-level resource because `dataToHcl` is called recursively
+	if len(path) == 0 && len(res.DependsOn) > 0 {
+		notIgnoredResources := []*resource{}
+		for _, dr := range res.DependsOn {
+			dr := dr
+			if dr.Data == nil {
+				tdr := ic.Scope.FindById(dr.Resource, dr.ID)
+				if tdr == nil {
+					log.Printf("[WARN] can't find resource %s in scope", dr)
+					continue
+				}
+				dr = tdr
+			}
+			if ic.Importables[dr.Resource].Ignore == nil || !ic.Importables[dr.Resource].Ignore(ic, dr) {
+				found := false
+				for _, v := range notIgnoredResources {
+					if v.ID == dr.ID && v.Resource == dr.Resource {
+						found = true
+						break
+					}
+				}
+				if !found {
+					notIgnoredResources = append(notIgnoredResources, dr)
+				}
+			}
+		}
+		if len(notIgnoredResources) > 0 {
+			toks := hclwrite.Tokens{}
+			toks = append(toks, &hclwrite.Token{
+				Type:  hclsyntax.TokenOBrack,
+				Bytes: []byte{'['},
+			})
+			for i, dr := range notIgnoredResources {
+				if i > 0 {
+					toks = append(toks, &hclwrite.Token{
+						Type:  hclsyntax.TokenComma,
+						Bytes: []byte{','},
+					})
+				}
+				toks = append(toks, hclwrite.TokensForTraversal(hcl.Traversal{
+					hcl.TraverseRoot{Name: dr.Resource},
+					hcl.TraverseAttr{Name: ic.ResourceName(dr)},
+				})...)
+			}
+			toks = append(toks, &hclwrite.Token{
+				Type:  hclsyntax.TokenCBrack,
+				Bytes: []byte{']'},
+			})
+			body.SetAttributeRaw("depends_on", toks)
+		}
+	}
 	return nil
 }
 

exporter/context_test.go

@@ -10,7 +10,9 @@ import (
 	"github.com/databricks/terraform-provider-databricks/provider"
 	"github.com/databricks/terraform-provider-databricks/qa"
 	"github.com/databricks/terraform-provider-databricks/workspace"
+	"github.com/hashicorp/hcl/v2/hclwrite"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -452,3 +454,45 @@ func TestExtractResourceIdFromImportBlockString(t *testing.T) {
 	  }`)
 	assert.Equal(t, "", id)
 }
+
+func TestGenerateDependsOn(t *testing.T) {
+	ic := importContextForTest()
+	ic.incremental = true
+
+	f := hclwrite.NewEmptyFile()
+	body := f.Body()
+	dr := &resource{
+		Resource: "databricks_catalog",
+		ID:       "test",
+		Data: ic.Resources["databricks_catalog"].Data(
+			&terraform.InstanceState{
+				ID: "test",
+				Attributes: map[string]string{
+					"name": "test",
+				},
+			}),
+	}
+	ic.Scope.Append(dr)
+	r := &resource{
+		Resource: "databricks_schema",
+		Name:     "test.schema",
+		Data: ic.Resources["databricks_schema"].Data(
+			&terraform.InstanceState{
+				ID: "test",
+				Attributes: map[string]string{
+					"catalog_name": "test",
+					"name":         "schema",
+				},
+			}),
+		DependsOn: []*resource{dr,
+			{Resource: dr.Resource, ID: dr.ID},
+			{Resource: dr.Resource, ID: "unknown"},
+		},
+	}
+	r.AddDependsOn(&resource{Resource: dr.Resource, ID: "test2", Data: dr.Data})
+	resourceBlock := body.AppendNewBlock("resource", []string{r.Resource, r.Name})
+	err := ic.dataToHcl(ic.Importables[r.Resource], []string{}, ic.Resources[r.Resource], r, resourceBlock.Body())
+	require.NoError(t, err)
+	formatted := hclwrite.Format(f.Bytes())
+	assert.Contains(t, string(formatted), "depends_on   = [databricks_catalog.test, databricks_catalog.test2]")
+}

exporter/importables.go

@@ -2390,7 +2390,11 @@ var resourcesMap map[string]importable = map[string]importable{
 			common.DataToStructPointer(r.Data, s, &cat)
 
 			// Emit: UC Connection, List schemas, Catalog grants, ...
-			ic.emitUCGrantsWithOwner("catalog/"+cat.Name, r)
+			owner, catalogGrantsResource := ic.emitUCGrantsWithOwner("catalog/"+cat.Name, r)
+			dependsOn := []*resource{}
+			if owner != "" && owner != ic.meUserName {
+				dependsOn = append(dependsOn, catalogGrantsResource)
+			}
 			// TODO: emit owner?  Should we do this? Because it's a account-level identity... Create a separate function for that...
 			if cat.ConnectionName != "" {
 				ic.Emit(&resource{
@@ -2409,8 +2413,9 @@ var resourcesMap map[string]importable = map[string]importable{
 						continue
 					}
 					ic.EmitIfUpdatedAfterMillis(&resource{
-						Resource: "databricks_schema",
-						ID:       schema.FullName,
+						Resource:  "databricks_schema",
+						ID:        schema.FullName,
+						DependsOn: dependsOn,
 					}, schema.UpdatedAt, fmt.Sprintf("schema '%s'", schema.FullName))
 				}
 			}
@@ -2470,14 +2475,20 @@ var resourcesMap map[string]importable = map[string]importable{
 			schemaFullName := r.ID
 			catalogName := r.Data.Get("catalog_name").(string)
 			schemaName := r.Data.Get("name").(string)
-			ic.emitUCGrantsWithOwner("schema/"+schemaFullName, r)
+			owner, schemaGrantResource := ic.emitUCGrantsWithOwner("schema/"+schemaFullName, r)
+			dependsOn := []*resource{}
+			if owner != "" && owner != ic.meUserName {
+				dependsOn = append(dependsOn, schemaGrantResource)
+			}
+			// TODO: think if we need to emit upstream dependencies in case if we're going bottom-up
 			ic.Emit(&resource{
 				Resource: "databricks_catalog",
 				ID:       catalogName,
 			})
+			// r.AddDependsOn(&resource{Resource: "databricks_grants", ID: "catalog/" + catalogName})
+
+			// TODO: somehow add depends on catalog's grant...
 			// TODO: emit owner? See comment in catalog resource
-			// TODO: list tables
-			// list registered models
 			models, err := ic.workspaceClient.RegisteredModels.ListAll(ic.Context,
 				catalog.ListRegisteredModelsRequest{
 					CatalogName: catalogName,
@@ -2488,8 +2499,9 @@ var resourcesMap map[string]importable = map[string]importable{
 			}
 			for _, model := range models {
 				ic.EmitIfUpdatedAfterMillis(&resource{
-					Resource: "databricks_registered_model",
-					ID:       model.FullName,
+					Resource:  "databricks_registered_model",
+					ID:        model.FullName,
+					DependsOn: dependsOn,
 				}, model.UpdatedAt, fmt.Sprintf("registered model '%s'", model.FullName))
 			}
 			// list volumes
@@ -2503,11 +2515,12 @@ var resourcesMap map[string]importable = map[string]importable{
 			}
 			for _, volume := range volumes {
 				ic.EmitIfUpdatedAfterMillis(&resource{
-					Resource: "databricks_volume",
-					ID:       volume.FullName,
+					Resource:  "databricks_volume",
+					ID:        volume.FullName,
+					DependsOn: dependsOn,
 				}, volume.UpdatedAt, fmt.Sprintf("volume '%s'", volume.FullName))
 			}
-
+			// list tables
 			tables, err := ic.workspaceClient.Tables.ListAll(ic.Context, catalog.ListTablesRequest{
 				CatalogName: catalogName,
 				SchemaName:  schemaName,
@@ -2519,13 +2532,17 @@ var resourcesMap map[string]importable = map[string]importable{
 				switch table.TableType {
 				case "MANAGED", "EXTERNAL", "VIEW":
 					ic.EmitIfUpdatedAfterMillis(&resource{
-						Resource: "databricks_sql_table",
-						ID:       table.FullName,
+						Resource:  "databricks_sql_table",
+						ID:        table.FullName,
+						DependsOn: dependsOn,
 					}, table.UpdatedAt, fmt.Sprintf("table '%s'", table.FullName))
 				default:
 					log.Printf("[DEBUG] Skipping table %s of type %s", table.FullName, table.TableType)
 				}
 			}
+			// TODO: list VectorSearch indexes
+
+			// TODO: list online tables
 
 			return nil
 		},
@@ -2543,15 +2560,12 @@ var resourcesMap map[string]importable = map[string]importable{
 			volumeFullName := r.ID
 			ic.emitUCGrantsWithOwner("volume/"+volumeFullName, r)
 
-			catalogName := r.Data.Get("catalog_name").(string)
+			schemaFullName := r.Data.Get("catalog_name").(string) + "." + r.Data.Get("schema_name").(string)
 			ic.Emit(&resource{
 				Resource: "databricks_schema",
-				ID:       catalogName + "." + r.Data.Get("schema_name").(string),
-			})
-			ic.Emit(&resource{
-				Resource: "databricks_catalog",
-				ID:       catalogName,
+				ID:       schemaFullName,
 			})
+			// r.AddDependsOn(&resource{Resource: "databricks_grants", ID: "schema/" + schemaFullName})
 			// TODO: emit owner? See comment in catalog resource
 			return nil
 		},
@@ -2576,15 +2590,12 @@ var resourcesMap map[string]importable = map[string]importable{
 		Import: func(ic *importContext, r *resource) error {
 			tableFullName := r.ID
 			ic.emitUCGrantsWithOwner("table/"+tableFullName, r)
-			catalogName := r.Data.Get("catalog_name").(string)
+			schemaFullName := r.Data.Get("catalog_name").(string) + "." + r.Data.Get("schema_name").(string)
 			ic.Emit(&resource{
 				Resource: "databricks_schema",
-				ID:       catalogName + "." + r.Data.Get("schema_name").(string),
-			})
-			ic.Emit(&resource{
-				Resource: "databricks_catalog",
-				ID:       catalogName,
+				ID:       schemaFullName,
 			})
+			// r.AddDependsOn(&resource{Resource: "databricks_grants", ID: "schema/" + schemaFullName})
 			// TODO: emit owner? See comment in catalog resource
 			return nil
 		},
@@ -2702,10 +2713,12 @@ var resourcesMap map[string]importable = map[string]importable{
 		Service:        "uc-external-locations",
 		Import: func(ic *importContext, r *resource) error {
 			ic.emitUCGrantsWithOwner("external_location/"+r.ID, r)
+			credentialName := r.Data.Get("credential_name").(string)
 			ic.Emit(&resource{
 				Resource: "databricks_storage_credential",
-				ID:       r.Data.Get("credential_name").(string),
+				ID:       credentialName,
 			})
+			// r.AddDependsOn(&resource{Resource: "databricks_grants", ID: "storage_credential/" + credentialName})
 			return nil
 		},
 		List: func(ic *importContext) error {
@@ -2862,15 +2875,12 @@ var resourcesMap map[string]importable = map[string]importable{
 		Import: func(ic *importContext, r *resource) error {
 			modelFullName := r.ID
 			ic.emitUCGrantsWithOwner("model/"+modelFullName, r)
-			catalogName := r.Data.Get("catalog_name").(string)
+			schemaFullName := r.Data.Get("catalog_name").(string) + "." + r.Data.Get("schema_name").(string)
 			ic.Emit(&resource{
 				Resource: "databricks_schema",
-				ID:       catalogName + "." + r.Data.Get("catalog_name").(string),
-			})
-			ic.Emit(&resource{
-				Resource: "databricks_catalog",
-				ID:       catalogName,
+				ID:       schemaFullName,
 			})
+			// r.AddDependsOn(&resource{Resource: "databricks_grants", ID: "schema/" + schemaFullName})
 			// TODO: emit owner? See comment in catalog resource
 			return nil
 		},
@@ -2985,6 +2995,7 @@ var resourcesMap map[string]importable = map[string]importable{
 					Resource: "databricks_volume",
 					ID:       volumeId,
 				})
+				// r.AddDependsOn(&resource{Resource: "databricks_grants", ID: "volume/" + volumeId})
 			}
 
 			// download & store file

exporter/importables_test.go

@@ -1974,10 +1974,9 @@ func TestVolumes(t *testing.T) {
 		Data: d,
 	})
 	assert.NoError(t, err)
-	require.Equal(t, 3, len(ic.testEmits))
+	require.Equal(t, 2, len(ic.testEmits))
 	assert.True(t, ic.testEmits["databricks_grants[<unknown>] (id: volume/vtest)"])
 	assert.True(t, ic.testEmits["databricks_schema[<unknown>] (id: ctest.stest)"])
-	assert.True(t, ic.testEmits["databricks_catalog[<unknown>] (id: ctest)"])
 
 	//
 	shouldOmitFunc := resourcesMap["databricks_volume"].ShouldOmitField
@@ -2004,10 +2003,9 @@ func TestSqlTables(t *testing.T) {
 		Data: d,
 	})
 	assert.NoError(t, err)
-	require.Equal(t, 3, len(ic.testEmits))
+	require.Equal(t, 2, len(ic.testEmits))
 	assert.True(t, ic.testEmits["databricks_grants[<unknown>] (id: table/ttest)"])
 	assert.True(t, ic.testEmits["databricks_schema[<unknown>] (id: ctest.stest)"])
-	assert.True(t, ic.testEmits["databricks_catalog[<unknown>] (id: ctest)"])
 
 	//
 	shouldOmitFunc := resourcesMap["databricks_sql_table"].ShouldOmitField
@@ -2035,10 +2033,9 @@ func TestRegisteredModels(t *testing.T) {
 		Data: d,
 	})
 	assert.NoError(t, err)
-	require.Equal(t, 3, len(ic.testEmits))
+	require.Equal(t, 2, len(ic.testEmits))
 	assert.True(t, ic.testEmits["databricks_grants[<unknown>] (id: model/mtest)"])
-	assert.True(t, ic.testEmits["databricks_schema[<unknown>] (id: ctest.ctest)"])
-	assert.True(t, ic.testEmits["databricks_catalog[<unknown>] (id: ctest)"])
+	assert.True(t, ic.testEmits["databricks_schema[<unknown>] (id: ctest.stest)"])
 
 	//
 	shouldOmitFunc := resourcesMap["databricks_registered_model"].ShouldOmitField

exporter/model.go

@@ -238,6 +238,8 @@ type resource struct {
 	Data *schema.ResourceData
 	// Arbitrary data to be used by importable
 	ExtraData map[string]any
+	// References to dependencies - it could be fully resolved resource, with Data, etc., or it could be just resource type + ID
+	DependsOn []*resource
 }
 
 func (r *resource) AddExtraData(key string, value any) {
@@ -247,6 +249,10 @@ func (r *resource) AddExtraData(key string, value any) {
 	r.ExtraData[key] = value
 }
 
+func (r *resource) AddDependsOn(dep *resource) {
+	r.DependsOn = append(r.DependsOn, dep)
+}
+
 func (r *resource) GetExtraData(key string) (any, bool) {
 	if r.ExtraData == nil {
 		return nil, false
@@ -397,3 +403,15 @@ func (a *importedResources) Sorted() []*resource {
 	sort.Sort(c)
 	return c
 }
+
+func (a *importedResources) FindById(resourceType, id string) *resource {
+	defer a.mutex.RLocker().Unlock()
+	a.mutex.RLocker().Lock()
+	for _, r := range a.resources {
+		if r.Resource == resourceType && r.ID == id {
+			return r
+		}
+	}
+
+	return nil
+}

exporter/util.go

@@ -1338,17 +1338,21 @@ func generateIgnoreObjectWithoutName(resourceType string) func(ic *importContext
 	}
 }
 
-func (ic *importContext) emitUCGrantsWithOwner(id string, parentResource *resource) {
+func (ic *importContext) emitUCGrantsWithOwner(id string, parentResource *resource) (string, *resource) {
 	gr := &resource{
 		Resource: "databricks_grants",
 		ID:       id,
 	}
+	var owner string
 	if parentResource.Data != nil {
-		if owner, ok := parentResource.Data.GetOk("owner"); ok {
-			gr.AddExtraData("owner", owner)
+		ownerRaw, ok := parentResource.Data.GetOk("owner")
+		if ok {
+			gr.AddExtraData("owner", ownerRaw)
+			owner = ownerRaw.(string)
 		}
 	}
 	ic.Emit(gr)
+	return owner, gr
 }
 
 func (ic *importContext) addTfVar(name, value string) {