Exporter: relax handling of problematic files/directories (#2258)

The new UI has a bug that allows creation of directories with `/` character in the name -
it breaks the Workspace List API, and prevents exporting of the workspace objects.   The
temporary solution is explicitly add the `ignoreErrors` flag when using exporter.

Author: alexott

exporter/context.go

@@ -392,7 +392,8 @@ func (ic *importContext) HasInState(r *resource, onlyAdded bool) bool {
 			continue
 		}
 		for _, i := range sr.Instances {
-			if i.Attributes[k].(string) == v {
+			tv, ok := i.Attributes[k].(string)
+			if ok && tv == v {
 				return true
 			}
 		}

exporter/exporter_test.go

@@ -1628,6 +1628,12 @@ func TestImportingDLTPipelines(t *testing.T) {
 					},
 				},
 			},
+			{
+				Method:       "GET",
+				Resource:     "/api/2.0/workspace/get-status?path=%2FUsers%2Fuser%40domain.com",
+				Response:     map[string]any{},
+				ReuseRequest: true,
+			},
 			{
 				Method:   "GET",
 				Resource: "/api/2.0/permissions/repos/123",

exporter/importables.go

@@ -1150,20 +1150,20 @@ var resourcesMap map[string]importable = map[string]importable{
 		},
 		List: func(ic *importContext) error {
 			notebooksAPI := workspace.NewNotebooksAPI(ic.Context, ic.Client)
-			notebookList, err := notebooksAPI.List("/", true)
+			notebookList, err := notebooksAPI.List("/", true, true)
 			if err != nil {
 				return err
 			}
 			for offset, notebook := range notebookList {
 				if strings.HasPrefix(notebook.Path, "/Repos") {
 					continue
 				}
-				// TODO: emit permissions for notebook folders if non-default,
-				// as per-notebook permission entry would be a noise in the state
-				ic.Emit(&resource{
-					Resource: "databricks_notebook",
-					ID:       notebook.Path,
-				})
+				if notebook.ObjectType == workspace.Notebook {
+					ic.Emit(&resource{
+						Resource: "databricks_notebook",
+						ID:       notebook.Path,
+					})
+				}
 				if offset%50 == 0 {
 					log.Printf("[INFO] Scanned %d of %d notebooks",
 						offset+1, len(notebookList))
@@ -1204,9 +1204,8 @@ var resourcesMap map[string]importable = map[string]importable{
 
 			if ic.meAdmin {
 				ic.Emit(&resource{
-					Resource:  "databricks_directory",
-					Attribute: "path",
-					Value:     directoryPath,
+					Resource: "databricks_directory",
+					ID:       directoryPath,
 				})
 			}
 

exporter/util.go

@@ -143,7 +143,7 @@ func (ic *importContext) emitNotebookOrRepo(path string) {
 func (ic *importContext) getAllDirectories() []workspace.ObjectStatus {
 	if len(ic.allDirectories) == 0 {
 		notebooksAPI := workspace.NewNotebooksAPI(ic.Context, ic.Client)
-		ic.allDirectories, _ = notebooksAPI.ListDirectories("/", true)
+		ic.allDirectories, _ = notebooksAPI.ListDirectories("/", true, true)
 	}
 	return ic.allDirectories
 }

workspace/data_notebook_paths.go

@@ -13,7 +13,7 @@ func DataSourceNotebookPaths() *schema.Resource {
 		ReadContext: func(ctx context.Context, d *schema.ResourceData, m any) diag.Diagnostics {
 			path := d.Get("path").(string)
 			recursive := d.Get("recursive").(bool)
-			notebookList, err := NewNotebooksAPI(ctx, m).List(path, recursive)
+			notebookList, err := NewNotebooksAPI(ctx, m).List(path, recursive, false)
 			if err != nil {
 				return diag.FromErr(err)
 			}

workspace/resource_notebook.go

@@ -133,10 +133,10 @@ func (a NotebooksAPI) Mkdirs(path string) error {
 // List will list all objects in a path on the workspace
 // and with the recursive flag it will recursively list
 // all the objects
-func (a NotebooksAPI) List(path string, recursive bool) ([]ObjectStatus, error) {
+func (a NotebooksAPI) List(path string, recursive bool, ignoreErrors bool) ([]ObjectStatus, error) {
 	if recursive {
 		var paths []ObjectStatus
-		err := a.recursiveAddPaths(path, &paths)
+		err := a.recursiveAddPaths(path, &paths, ignoreErrors)
 		if err != nil {
 			return nil, err
 		}
@@ -145,28 +145,28 @@ func (a NotebooksAPI) List(path string, recursive bool) ([]ObjectStatus, error)
 	return a.list(path)
 }
 
-func (a NotebooksAPI) recursiveAddPaths(path string, pathList *[]ObjectStatus) error {
+func (a NotebooksAPI) recursiveAddPaths(path string, pathList *[]ObjectStatus, ignoreErrors bool) error {
 	notebookInfoList, err := a.list(path)
-	if err != nil {
+	if err != nil && !ignoreErrors {
 		return err
 	}
 	for _, v := range notebookInfoList {
 		if v.ObjectType == Notebook || v.ObjectType == File {
 			*pathList = append(*pathList, v)
 		} else if v.ObjectType == Directory {
-			err := a.recursiveAddPaths(v.Path, pathList)
+			err := a.recursiveAddPaths(v.Path, pathList, ignoreErrors)
 			if err != nil {
 				return err
 			}
 		}
 	}
-	return err
+	return nil
 }
 
-func (a NotebooksAPI) ListDirectories(path string, recursive bool) ([]ObjectStatus, error) {
+func (a NotebooksAPI) ListDirectories(path string, recursive bool, ignoreErrors bool) ([]ObjectStatus, error) {
 	if recursive {
 		var paths []ObjectStatus
-		err := a.recursiveAddDirectoryPaths(path, &paths)
+		err := a.recursiveAddDirectoryPaths(path, &paths, ignoreErrors)
 		if err != nil {
 			return nil, err
 		}
@@ -175,21 +175,21 @@ func (a NotebooksAPI) ListDirectories(path string, recursive bool) ([]ObjectStat
 	return a.list(path)
 }
 
-func (a NotebooksAPI) recursiveAddDirectoryPaths(path string, pathList *[]ObjectStatus) error {
+func (a NotebooksAPI) recursiveAddDirectoryPaths(path string, pathList *[]ObjectStatus, ignoreErrors bool) error {
 	directoryInfoList, err := a.list(path)
-	if err != nil {
+	if err != nil && !ignoreErrors {
 		return err
 	}
 	for _, v := range directoryInfoList {
 		if v.ObjectType == Directory {
 			*pathList = append(*pathList, v)
-			err := a.recursiveAddDirectoryPaths(v.Path, pathList)
+			err := a.recursiveAddDirectoryPaths(v.Path, pathList, ignoreErrors)
 			if err != nil {
 				return err
 			}
 		}
 	}
-	return err
+	return nil
 }
 
 type ObjectList struct {

workspace/resource_notebook_test.go

@@ -457,7 +457,7 @@ func TestListDirectories(t *testing.T) {
 	ctx := context.Background()
 	notebooksAPI := NewNotebooksAPI(ctx, client)
 
-	directoryList, err := notebooksAPI.ListDirectories("/", true)
+	directoryList, err := notebooksAPI.ListDirectories("/", true, false)
 	var paths []string
 	for _, directory := range directoryList {
 		paths = append(paths, directory.Path)
@@ -546,7 +546,7 @@ func TestListDirectories_NoneRecursive(t *testing.T) {
 	require.NoError(t, err)
 	ctx := context.Background()
 	notebooksAPI := NewNotebooksAPI(ctx, client)
-	directoryList_not_recursive, err := notebooksAPI.ListDirectories("/", false)
+	directoryList_not_recursive, err := notebooksAPI.ListDirectories("/", false, false)
 	var paths []string
 	for _, directory := range directoryList_not_recursive {
 		paths = append(paths, directory.Path)
@@ -586,7 +586,7 @@ func TestListDirectoriesRecursive_Error(t *testing.T) {
 	require.NoError(t, err)
 	ctx := context.Background()
 	notebooksAPI := NewNotebooksAPI(ctx, client)
-	directoryList, err := notebooksAPI.ListDirectories("/", true)
+	directoryList, err := notebooksAPI.ListDirectories("/", true, false)
 	assert.Error(t, err, err)
 	assert.Nil(t, directoryList)
 }