Exporter: track removed objects during the Emit phase (#3554)

* Exporter: track removed objects during the `Emit` phase

In the current implementation when TF resource data has `.id` set to empty string this
means the following:

* object doesn't exist on backend - common situation for jobs and DLT pipelines when
  dependencies are deleted.
* Grants or Permissions are empty, so we don't need to generate a code for that.

Right now, all such objects are tracked and reported individually via `Ignore` function,
but not all objects have it.  Also, for big exports, not necessary objects are polluting
the internal state without any useful effect.

The current PR checks if `.id` is empty during the `Emit` phase, adds them to the list of
ignored objects (except permissions & grants), and don't propagate them to the internal
state.

* Add more tests

Author: alexott

exporter/importables.go

@@ -13,6 +13,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/databricks/databricks-sdk-go/apierr"
 	"github.com/databricks/databricks-sdk-go/service/catalog"
 	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/iam"
@@ -234,11 +235,6 @@ var resourcesMap map[string]importable = map[string]importable{
 				"inst_pool_"+ic.Importables["databricks_instance_pool"].Name(ic, r.Data))
 			return nil
 		},
-		Ignore: func(ic *importContext, r *resource) bool {
-			isIgnored := r.Data.Get("instance_pool_name") == ""
-			ic.addIgnoredResource(fmt.Sprintf("databricks_instance_pool. id=%s", r.ID))
-			return isIgnored
-		},
 	},
 	"databricks_instance_profile": {
 		Service: "access",
@@ -1511,6 +1507,9 @@ var resourcesMap map[string]importable = map[string]importable{
 			notebooksAPI := workspace.NewNotebooksAPI(ic.Context, ic.Client)
 			contentB64, err := notebooksAPI.Export(r.ID, ic.notebooksFormat)
 			if err != nil {
+				if apierr.IsMissing(err) {
+					ic.addIgnoredResource(fmt.Sprintf("databricks_notebook. path=%s", r.ID))
+				}
 				return err
 			}
 			var fileExtension string
@@ -1558,6 +1557,9 @@ var resourcesMap map[string]importable = map[string]importable{
 			notebooksAPI := workspace.NewNotebooksAPI(ic.Context, ic.Client)
 			contentB64, err := notebooksAPI.Export(r.ID, "AUTO")
 			if err != nil {
+				if apierr.IsMissing(err) {
+					ic.addIgnoredResource(fmt.Sprintf("databricks_workspace_file. path=%s", r.ID))
+				}
 				return err
 			}
 			objectId := r.Data.Get("object_id").(int)
@@ -2609,9 +2611,6 @@ var resourcesMap map[string]importable = map[string]importable{
 			}
 			return common.StructToData(pList, s, r.Data)
 		},
-		Ignore: func(ic *importContext, r *resource) bool {
-			return r.Data.Get("grant.#").(int) == 0
-		},
 		Depends: []reference{
 			{Path: "catalog", Resource: "databricks_catalog"},
 			{Path: "schema", Resource: "databricks_schema"},

exporter/importables_test.go

@@ -100,15 +100,6 @@ func TestInstancePool(t *testing.T) {
 	err := resourcesMap["databricks_instance_pool"].Import(ic, r)
 	assert.NoError(t, err)
 	assert.True(t, ic.testEmits["databricks_permissions[inst_pool_def] (id: /instance-pools/abc)"])
-
-	// Check ignore function
-	assert.True(t, resourcesMap["databricks_instance_pool"].Ignore(ic, r))
-	assert.Equal(t, 1, len(ic.ignoredResources))
-	assert.Contains(t, ic.ignoredResources, "databricks_instance_pool. id=abc")
-	//
-	d.Set("instance_pool_name", "test")
-	assert.False(t, resourcesMap["databricks_instance_pool"].Ignore(ic, r))
-	assert.Equal(t, 1, len(ic.ignoredResources))
 }
 
 func TestClusterPolicy(t *testing.T) {
@@ -987,6 +978,30 @@ func TestRepoListFails(t *testing.T) {
 	})
 }
 
+func TestNotebookWorkspaceFileImportNotFound(t *testing.T) {
+	qa.HTTPFixturesApply(t, []qa.HTTPFixture{
+		{
+			ReuseRequest: true,
+			MatchAny:     true,
+			Status:       404,
+			Response:     apierr.NotFound("nope"),
+		},
+	}, func(ctx context.Context, client *common.DatabricksClient) {
+		ic := importContextForTestWithClient(ctx, client)
+		err := resourcesMap["databricks_notebook"].Import(ic, &resource{
+			ID: "/abc",
+		})
+		assert.EqualError(t, err, "nope")
+		assert.Contains(t, ic.ignoredResources, "databricks_notebook. path=/abc")
+
+		err = resourcesMap["databricks_workspace_file"].Import(ic, &resource{
+			ID: "/def",
+		})
+		assert.EqualError(t, err, "nope")
+		assert.Contains(t, ic.ignoredResources, "databricks_workspace_file. path=/def")
+	})
+}
+
 func testGenerate(t *testing.T, fixtures []qa.HTTPFixture, services string, asAdmin bool, cb func(*importContext)) {
 	qa.HTTPFixturesApply(t, fixtures, func(ctx context.Context, client *common.DatabricksClient) {
 		ic := importContextForTestWithClient(ctx, client)
@@ -2179,9 +2194,6 @@ func TestImportGrants(t *testing.T) {
 	err := resourcesMap["databricks_grants"].Import(ic, r)
 	assert.NoError(t, err)
 
-	// Test ignore function
-	assert.True(t, resourcesMap["databricks_grants"].Ignore(ic, r))
-
 	var pList tfcatalog.PermissionsList
 	common.DataToStructPointer(r.Data, s, &pList)
 	assert.Empty(t, pList.Assignments)

exporter/model.go

@@ -343,12 +343,16 @@ func (r *resource) ImportResource(ic *importContext) {
 			return pr.ReadContext(ctx, r.Data, ic.Client)
 		},
 			fmt.Sprintf("reading %s#%s", r.Resource, r.ID))
-		if dia != nil {
+		if dia.HasError() {
 			log.Printf("[ERROR] Error reading %s#%s: %v", r.Resource, r.ID, dia)
 			return
 		}
 		if r.Data.Id() == "" {
-			r.Data.SetId(r.ID)
+			if r.Resource != "databricks_permissions" && r.Resource != "databricks_grants" {
+				log.Printf("[WARN] %s %s has empty ID because it's deleted or empty", r.Resource, r.ID)
+				ic.addIgnoredResource(fmt.Sprintf("%s. id=%s", r.Resource, r.ID))
+			}
+			return
 		}
 	}
 	r.Name = ic.ResourceName(r)

exporter/util.go

@@ -1418,7 +1418,7 @@ func generateIgnoreObjectWithoutName(resourceType string) func(ic *importContext
 	return func(ic *importContext, r *resource) bool {
 		res := (r.Data != nil && r.Data.Get("name").(string) == "")
 		if res {
-			ic.addIgnoredResource(fmt.Sprintf("%s. ID=%s", resourceType, r.ID))
+			ic.addIgnoredResource(fmt.Sprintf("%s. id=%s", resourceType, r.ID))
 		}
 		return res
 	}

exporter/util_test.go

@@ -406,7 +406,7 @@ func TestIgnoreObjectWithEmptyName(t *testing.T) {
 	require.NotNil(t, ignoreFunc)
 	assert.True(t, ignoreFunc(ic, r))
 	require.Equal(t, 1, len(ic.ignoredResources))
-	assert.Contains(t, ic.ignoredResources, "databricks_volume. ID=vtest")
+	assert.Contains(t, ic.ignoredResources, "databricks_volume. id=vtest")
 
 	d.Set("name", "test")
 	assert.False(t, ignoreFunc(ic, r))