Exporter: Don't list directories in the incremental mode (#3569)

* Exporter: Don't list directories in the incremental mode

Instead, rely on behavior or them emitted as dependencies for notebooks

* Emit directories only if service is enabled

* Add a note about directories listing in incremental mode

Author: alexott

docs/guides/experimental-exporter.md

@@ -64,7 +64,7 @@ Services are just logical groups of resources used for filtering and organizatio
 
 * `access` - [databricks_permissions](../resources/permissions.md), [databricks_instance_profile](../resources/instance_profile.md), [databricks_ip_access_list](../resources/ip_access_list.md), [databricks_mws_permission_assignment](../resources/mws_permission_assignment.md) and [databricks_access_control_rule_set](../resources/access_control_rule_set.md).
 * `compute` - **listing** [databricks_cluster](../resources/cluster.md).
-* `directories` - **listing** [databricks_directory](../resources/directory.md).
+* `directories` - **listing** [databricks_directory](../resources/directory.md).  *Please note that directories aren't listed when running in the incremental mode! Only directories with updated notebooks will be emitted.*
 * `dlt` - **listing** [databricks_pipeline](../resources/pipeline.md).
 * `groups` - **listing** [databricks_group](../data-sources/group.md) with [membership](../resources/group_member.md) and [data access](../resources/group_instance_profile.md).
 * `jobs` - **listing** [databricks_job](../resources/job.md). Usually, there are more automated workflows than interactive clusters, so they get their own file in this tool's output.  *Please note that workflows deployed and maintained via [Databricks Asset Bundles](https://docs.databricks.com/en/dev-tools/bundles/index.html) aren't exported!*

exporter/importables.go

@@ -2053,6 +2053,9 @@ var resourcesMap map[string]importable = map[string]importable{
 		// TODO: think if we really need this, we need directories only for permissions,
 		// and only when they are different from parents & notebooks
 		List: func(ic *importContext) error {
+			if ic.incremental {
+				return nil
+			}
 			directoryList := ic.getAllDirectories()
 			for offset, directory := range directoryList {
 				if strings.HasPrefix(directory.Path, "/Repos") {

exporter/util.go

@@ -1087,6 +1087,9 @@ func (ic *importContext) emitSqlParentDirectory(parent string) {
 }
 
 func (ic *importContext) shouldSkipWorkspaceObject(object workspace.ObjectStatus, updatedSinceMs int64) bool {
+	if ic.incremental && object.ObjectType == workspace.Directory {
+		return true
+	}
 	if !(object.ObjectType == workspace.Notebook || object.ObjectType == workspace.File) ||
 		strings.HasPrefix(object.Path, "/Repos") {
 		// log.Printf("[DEBUG] Skipping unsupported entry %v", object)
@@ -1155,8 +1158,10 @@ func listNotebooksAndWorkspaceFiles(ic *importContext) error {
 	updatedSinceMs := ic.getUpdatedSinceMs()
 	allObjects := ic.getAllWorkspaceObjects(func(objects []workspace.ObjectStatus) {
 		for _, object := range objects {
-			if object.ObjectType == workspace.Directory && object.Path != "/" && !ic.incremental {
-				objectsChannel <- object
+			if object.ObjectType == workspace.Directory {
+				if !ic.incremental && object.Path != "/" && ic.isServiceEnabled("directories") {
+					objectsChannel <- object
+				}
 			} else {
 				if ic.shouldSkipWorkspaceObject(object, updatedSinceMs) {
 					continue
@@ -1179,7 +1184,11 @@ func listNotebooksAndWorkspaceFiles(ic *importContext) error {
 			if ic.shouldSkipWorkspaceObject(object, updatedSinceMs) {
 				continue
 			}
-			emitWorkpaceObject(ic, object)
+			if object.ObjectType == workspace.Directory && !ic.incremental && ic.isServiceEnabled("directories") && object.Path != "/" {
+				emitWorkpaceObject(ic, object)
+			} else if (object.ObjectType == workspace.Notebook || object.ObjectType == workspace.File) && ic.isServiceEnabled("notebooks") {
+				emitWorkpaceObject(ic, object)
+			}
 		}
 	}
 	return nil

exporter/util_test.go

@@ -429,3 +429,13 @@ func TestEmitWorkspaceObjectParentDirectory(t *testing.T) {
 	assert.True(t, exists)
 	assert.Equal(t, dirPath, dir)
 }
+
+func TestDirectoryIncrementalMode(t *testing.T) {
+	ic := importContextForTest()
+	ic.incremental = true
+
+	// test direct listing
+	assert.Nil(t, resourcesMap["databricks_directory"].List(ic))
+	// test emit during workspace listing
+	assert.True(t, ic.shouldSkipWorkspaceObject(workspace.ObjectStatus{ObjectType: workspace.Directory}, 111111))
+}