Exporter: don't emit all UC objects when handling dependencies (#3556)

Right now, if we discover any UC dependency, we emit upstream objects as well, such as
schemas, catalogs, etc., but in their `Import` operations we're doing all nested objects
unnecessarily - so emit one init script from a UC Volume may lead to emitting of the whole
UC Catalog with all schemas/tables/volumes...

With this change, we list all nested objects (schemas in catalog, tables/models/volumes in
a schema) only if these services are explicitly specified in the `-listing` option.

Fixes #3555

Author: alexott

docs/guides/experimental-exporter.md

@@ -87,13 +87,13 @@ Services are just logical groups of resources used for filtering and organizatio
 * `uc-external-locations` - **listing** exports [databricks_external_location](../resources/external_location.md) resource.
 * `uc-grants` -  [databricks_grants](../resources/grants.md). *Please note that during export the list of grants is expanded to include the identity that does the export! This is done to allow to create objects in case when catalogs/schemas have different owners than current identity.*.
 * `uc-metastores` - **listing** [databricks_metastore](../resources/metastore.md) and [databricks_metastore_assignment](../resource/metastore_assignment.md) (only on account-level).  *Please note that when using workspace-level configuration, only metastores from the workspace's region are listed!*
-* `uc-models` - [databricks_registered_model](../resources/registered_model.md)
-* `uc-schemas` -  [databricks_schema](../resources/schema.md)
+* `uc-models` - **listing** (*we can't list directly, only via dependencies to top-level object*) [databricks_registered_model](../resources/registered_model.md)
+* `uc-schemas` - **listing** (*we can't list directly, only via dependencies to top-level object*) [databricks_schema](../resources/schema.md)
 * `uc-shares` - **listing** [databricks_share](../resources/share.md) and [databricks_recipient](../resources/recipient.md)
 * `uc-storage-credentials` - **listing** exports [databricks_storage_credential](../resources/storage_credential.md) resources on workspace or account level.
 * `uc-system-schemas` - **listing** exports [databricks_system_schema](../resources/system_schema.md) resources for the UC metastore of the current workspace.
-* `uc-tables` - [databricks_sql_table](../resources/sql_table.md) resource.
-* `uc-volumes` -  [databricks_volume](../resources/volume.md)
+* `uc-tables` - **listing** (*we can't list directly, only via dependencies to top-level object*) [databricks_sql_table](../resources/sql_table.md) resource.
+* `uc-volumes` - **listing** (*we can't list directly, only via dependencies to top-level object*) [databricks_volume](../resources/volume.md)
 * `users` - [databricks_user](../resources/user.md) and [databricks_service_principal](../resources/service_principal.md) are written to their own file, simply because of their amount. If you use SCIM provisioning, migrating workspaces is the only use case for importing `users` service.
 * `workspace` - **listing** [databricks_workspace_conf](../resources/workspace_conf.md) and [databricks_global_init_script](../resources/global_init_script.md)
 

exporter/context.go

@@ -1416,6 +1416,11 @@ func (ic *importContext) isServiceEnabled(service string) bool {
 	return exists
 }
 
+func (ic *importContext) isServiceInListing(service string) bool {
+	_, exists := ic.listing[service]
+	return exists
+}
+
 func (ic *importContext) EmitIfUpdatedAfterMillis(r *resource, modifiedAt int64, message string) {
 	updatedSinceMs := ic.getUpdatedSinceMs()
 	if ic.incremental && modifiedAt < updatedSinceMs {

exporter/importables.go

@@ -2341,20 +2341,22 @@ var resourcesMap map[string]importable = map[string]importable{
 				})
 			} else if cat.ShareName == "" {
 				// TODO: We need to be careful here if we add more catalog types... Really we need to have CatalogType in resource
-				schemas, err := ic.workspaceClient.Schemas.ListAll(ic.Context, catalog.ListSchemasRequest{CatalogName: r.ID})
-				if err != nil {
-					return err
-				}
-				ignoredSchemas := []string{"information_schema"}
-				for _, schema := range schemas {
-					if schema.CatalogType != "MANAGED_CATALOG" || slices.Contains(ignoredSchemas, schema.Name) {
-						continue
+				if ic.isServiceInListing("uc-schemas") {
+					schemas, err := ic.workspaceClient.Schemas.ListAll(ic.Context, catalog.ListSchemasRequest{CatalogName: r.ID})
+					if err != nil {
+						return err
+					}
+					ignoredSchemas := []string{"information_schema"}
+					for _, schema := range schemas {
+						if schema.CatalogType != "MANAGED_CATALOG" || slices.Contains(ignoredSchemas, schema.Name) {
+							continue
+						}
+						ic.EmitIfUpdatedAfterMillis(&resource{
+							Resource:  "databricks_schema",
+							ID:        schema.FullName,
+							DependsOn: dependsOn,
+						}, schema.UpdatedAt, fmt.Sprintf("schema '%s'", schema.FullName))
 					}
-					ic.EmitIfUpdatedAfterMillis(&resource{
-						Resource:  "databricks_schema",
-						ID:        schema.FullName,
-						DependsOn: dependsOn,
-					}, schema.UpdatedAt, fmt.Sprintf("schema '%s'", schema.FullName))
 				}
 			}
 			if cat.IsolationMode == "ISOLATED" {
@@ -2427,55 +2429,61 @@ var resourcesMap map[string]importable = map[string]importable{
 
 			// TODO: somehow add depends on catalog's grant...
 			// TODO: emit owner? See comment in catalog resource
-			models, err := ic.workspaceClient.RegisteredModels.ListAll(ic.Context,
-				catalog.ListRegisteredModelsRequest{
-					CatalogName: catalogName,
-					SchemaName:  schemaName,
-				})
-			if err != nil { // TODO: should we continue?
-				return err
+			if ic.isServiceInListing("uc-models") {
+				models, err := ic.workspaceClient.RegisteredModels.ListAll(ic.Context,
+					catalog.ListRegisteredModelsRequest{
+						CatalogName: catalogName,
+						SchemaName:  schemaName,
+					})
+				if err != nil { // TODO: should we continue?
+					return err
+				}
+				for _, model := range models {
+					ic.EmitIfUpdatedAfterMillis(&resource{
+						Resource:  "databricks_registered_model",
+						ID:        model.FullName,
+						DependsOn: dependsOn,
+					}, model.UpdatedAt, fmt.Sprintf("registered model '%s'", model.FullName))
+				}
 			}
-			for _, model := range models {
-				ic.EmitIfUpdatedAfterMillis(&resource{
-					Resource:  "databricks_registered_model",
-					ID:        model.FullName,
-					DependsOn: dependsOn,
-				}, model.UpdatedAt, fmt.Sprintf("registered model '%s'", model.FullName))
-			}
-			// list volumes
-			volumes, err := ic.workspaceClient.Volumes.ListAll(ic.Context,
-				catalog.ListVolumesRequest{
+			if ic.isServiceInListing("uc-volumes") {
+				// list volumes
+				volumes, err := ic.workspaceClient.Volumes.ListAll(ic.Context,
+					catalog.ListVolumesRequest{
+						CatalogName: catalogName,
+						SchemaName:  schemaName,
+					})
+				if err != nil {
+					return err
+				}
+				for _, volume := range volumes {
+					ic.EmitIfUpdatedAfterMillis(&resource{
+						Resource:  "databricks_volume",
+						ID:        volume.FullName,
+						DependsOn: dependsOn,
+					}, volume.UpdatedAt, fmt.Sprintf("volume '%s'", volume.FullName))
+				}
+			}
+			if ic.isServiceInListing("uc-tables") {
+				// list tables
+				tables, err := ic.workspaceClient.Tables.ListAll(ic.Context, catalog.ListTablesRequest{
 					CatalogName: catalogName,
 					SchemaName:  schemaName,
 				})
-			if err != nil {
-				return err
-			}
-			for _, volume := range volumes {
-				ic.EmitIfUpdatedAfterMillis(&resource{
-					Resource:  "databricks_volume",
-					ID:        volume.FullName,
-					DependsOn: dependsOn,
-				}, volume.UpdatedAt, fmt.Sprintf("volume '%s'", volume.FullName))
-			}
-			// list tables
-			tables, err := ic.workspaceClient.Tables.ListAll(ic.Context, catalog.ListTablesRequest{
-				CatalogName: catalogName,
-				SchemaName:  schemaName,
-			})
-			if err != nil {
-				return err
-			}
-			for _, table := range tables {
-				switch table.TableType {
-				case "MANAGED", "EXTERNAL", "VIEW":
-					ic.EmitIfUpdatedAfterMillis(&resource{
-						Resource:  "databricks_sql_table",
-						ID:        table.FullName,
-						DependsOn: dependsOn,
-					}, table.UpdatedAt, fmt.Sprintf("table '%s'", table.FullName))
-				default:
-					log.Printf("[DEBUG] Skipping table %s of type %s", table.FullName, table.TableType)
+				if err != nil {
+					return err
+				}
+				for _, table := range tables {
+					switch table.TableType {
+					case "MANAGED", "EXTERNAL", "VIEW":
+						ic.EmitIfUpdatedAfterMillis(&resource{
+							Resource:  "databricks_sql_table",
+							ID:        table.FullName,
+							DependsOn: dependsOn,
+						}, table.UpdatedAt, fmt.Sprintf("table '%s'", table.FullName))
+					default:
+						log.Printf("[DEBUG] Skipping table %s of type %s", table.FullName, table.TableType)
+					}
 				}
 			}
 			// TODO: list VectorSearch indexes

exporter/importables_test.go

@@ -1641,6 +1641,7 @@ func TestImportManagedCatalog(t *testing.T) {
 	}, func(ctx context.Context, client *common.DatabricksClient) {
 		ic := importContextForTestWithClient(ctx, client)
 		ic.enableServices("uc-catalogs,uc-grants,uc-schemas")
+		ic.enableListing("uc-schemas")
 		ic.currentMetastore = currentMetastoreResponse
 		d := tfcatalog.ResourceCatalog().ToResource().TestResourceData()
 		d.SetId("ctest")
@@ -1697,6 +1698,7 @@ func TestImportIsolatedManagedCatalog(t *testing.T) {
 	}, func(ctx context.Context, client *common.DatabricksClient) {
 		ic := importContextForTestWithClient(ctx, client)
 		ic.enableServices("uc-catalogs,uc-grants,uc-schemas")
+		ic.enableListing("uc-schemas,uc-volumes,uc-models,uc-tables")
 		ic.currentMetastore = currentMetastoreResponse
 		d := tfcatalog.ResourceCatalog().ToResource().TestResourceData()
 		d.SetId("ctest")
@@ -1760,6 +1762,7 @@ func TestImportSchema(t *testing.T) {
 	}, func(ctx context.Context, client *common.DatabricksClient) {
 		ic := importContextForTestWithClient(ctx, client)
 		ic.enableServices("uc-catalogs,uc-grants,uc-schemas,uc-volumes,uc-models,uc-tables")
+		ic.enableListing("uc-schemas,uc-volumes,uc-models,uc-tables")
 		ic.currentMetastore = currentMetastoreResponse
 		d := tfcatalog.ResourceSchema().ToResource().TestResourceData()
 		d.SetId("ctest.stest")