Exporter: Ignore workspace assets of deleted users and service principals (#2980)

* Exporter: detecting deleted users/service principals & not exporting their data

Initial work, tests should be fixed.

* Fix tests

* Add a command-line flag to export assets of deleted users

* Fix formatting

* Fix tests after the Go SDK upgrade

Author: alexott

docs/guides/experimental-exporter.md

@@ -45,6 +45,7 @@ All arguments are optional, and they tune what code is being generated.
 * `-skip-interactive` - optionally run in a non-interactive mode.
 * `-includeUserDomains` - optionally include domain name into generated resource name for `databricks_user` resource.
 * `-importAllUsers` - optionally include all users and service principals even if they are only part of the `users` group.
+* `-exportDeletedUsersAssets` - optionally include assets of deleted users and service principals.
 * `-incremental` - experimental option for incremental export of modified resources and merging with existing resources. *Please note that only a limited set of resources (notebooks, SQL queries/dashboards/alerts, ...) provides information about the last modified date - all other resources will be re-exported again! Also, it's impossible to detect the deletion of the resources, so you must do periodic full export if resources are deleted!*   **Requires** `-updated-since` option if no `exporter-run-stats.json` file exists in the output directory.
 * `-updated-since` - timestamp (in ISO8601 format supported by Go language) for exporting of resources modified since a given timestamp. I.e., `2023-07-24T00:00:00Z`. If not specified, the exporter will try to load the last run timestamp from the `exporter-run-stats.json` file generated during the export and use it.
 * `-notebooksFormat` - optional format for exporting of notebooks. Supported values are `SOURCE` (default), `DBC`, `JUPYTER`.  This option could be used to export notebooks with embedded dashboards.

exporter/command.go

@@ -100,6 +100,8 @@ func Run(args ...string) error {
 	flags.BoolVar(&ic.includeUserDomains, "includeUserDomains", false, "Include domain portion in `databricks_user` resource name")
 	flags.BoolVar(&ic.importAllUsers, "importAllUsers", false,
 		"Import all users and service principals, even if they aren't referenced in any resource")
+	flags.BoolVar(&ic.exportDeletedUsersAssets, "exportDeletedUsersAssets", false,
+		"Export assets (notebooks, etc.) of deleted users & service principals")
 	flags.StringVar(&ic.Directory, "directory", cwd,
 		"Directory to generate sources in. Defaults to current directory.")
 	flags.Int64Var(&ic.lastActiveDays, "last-active-days", 3650,

exporter/context.go

@@ -80,25 +80,26 @@ type importContext struct {
 	Scope importedResources
 
 	// command-line resources (immutable, or set by the single thread)
-	includeUserDomains  bool
-	importAllUsers      bool
-	debug               bool
-	incremental         bool
-	mounts              bool
-	noFormat            bool
-	services            string
-	listing             string
-	match               string
-	lastActiveDays      int64
-	lastActiveMs        int64
-	generateDeclaration bool
-	meAdmin             bool
-	prefix              string
-	accountLevel        bool
-	shImports           map[string]bool
-	notebooksFormat     string
-	updatedSinceStr     string
-	updatedSinceMs      int64
+	includeUserDomains       bool
+	importAllUsers           bool
+	exportDeletedUsersAssets bool
+	debug                    bool
+	incremental              bool
+	mounts                   bool
+	noFormat                 bool
+	services                 string
+	listing                  string
+	match                    string
+	lastActiveDays           int64
+	lastActiveMs             int64
+	generateDeclaration      bool
+	meAdmin                  bool
+	prefix                   string
+	accountLevel             bool
+	shImports                map[string]bool
+	notebooksFormat          string
+	updatedSinceStr          string
+	updatedSinceMs           int64
 
 	waitGroup *sync.WaitGroup
 

exporter/exporter_test.go

@@ -1559,6 +1559,10 @@ func TestImportingRepos(t *testing.T) {
 	qa.HTTPFixturesApply(t,
 		[]qa.HTTPFixture{
 			meAdminFixture,
+			userListIdUsernameFixture,
+			userListIdUsernameFixture2,
+			userListFixture,
+			userReadFixture,
 			{
 				Method:   "GET",
 				Resource: "/api/2.0/repos?",
@@ -1900,6 +1904,18 @@ func TestImportingDLTPipelines(t *testing.T) {
 					Content: "spark.range(10)",
 				},
 			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/preview/scim/v2/Users?attributes=userName%2Cid",
+				Response: scim.UserList{
+					Resources: []scim.User{
+						{
+							ID:       "id",
+							UserName: "id",
+						},
+					},
+				},
+			},
 			{
 				Method:   "GET",
 				Resource: "/api/2.0/instance-profiles/list",
@@ -1971,10 +1987,13 @@ func TestImportingDLTPipelinesMatchingOnly(t *testing.T) {
 			meAdminFixture,
 			emptyRepos,
 			emptyIpAccessLIst,
+			userListIdUsernameFixture,
+			userListIdUsernameFixture2,
+			userListFixture,
+			userReadFixture,
 			{
 				Method:   "GET",
 				Resource: "/api/2.0/pipelines?max_results=50",
-
 				Response: pipelines.PipelineListResponse{
 					Statuses: []pipelines.PipelineStateInfo{
 						{

exporter/importables.go

@@ -45,6 +45,7 @@ var (
 	fileNameNormalizationRegex   = regexp.MustCompile(`[^-_\w/.@]`)
 	jobClustersRegex             = regexp.MustCompile(`^((job_cluster|task)\.[0-9]+\.new_cluster\.[0-9]+\.)`)
 	dltClusterRegex              = regexp.MustCompile(`^(cluster\.[0-9]+\.)`)
+	userDirRegex                 = regexp.MustCompile(`^(/Users/[^/]+)(/.*)?$`)
 	secretPathRegex              = regexp.MustCompile(`^\{\{secrets\/([^\/]+)\/([^}]+)\}\}$`)
 	sqlParentRegexp              = regexp.MustCompile(`^folders/(\d+)$`)
 	dltDefaultStorageRegex       = regexp.MustCompile(`^dbfs:/pipelines/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
@@ -642,10 +643,7 @@ var resourcesMap map[string]importable = map[string]importable{
 				}
 				if typ == "fixed" && strings.HasPrefix(k, "init_scripts.") &&
 					strings.HasSuffix(k, ".workspace.destination") {
-					ic.Emit(&resource{
-						Resource: "databricks_workspace_file",
-						ID:       eitherString(value, defaultValue),
-					})
+					ic.maybeEmitWorkspaceObject("databricks_workspace_file", eitherString(value, defaultValue))
 				}
 				if typ == "fixed" && (strings.HasPrefix(k, "spark_conf.") || strings.HasPrefix(k, "spark_env_vars.")) {
 					either := eitherString(value, defaultValue)
@@ -1941,14 +1939,10 @@ var resourcesMap map[string]importable = map[string]importable{
 				if res := ignoreIdeFolderRegex.FindStringSubmatch(directory.Path); res != nil {
 					continue
 				}
-				// TODO: don't emit directories for deleted users/SPs (how to identify them?)
-				ic.Emit(&resource{
-					Resource: "databricks_directory",
-					ID:       directory.Path,
-				})
+				ic.maybeEmitWorkspaceObject("databricks_directory", directory.Path)
+
 				if offset%50 == 0 {
-					log.Printf("[INFO] Scanned %d of %d directories",
-						offset+1, len(directoryList))
+					log.Printf("[INFO] Scanned %d of %d directories", offset+1, len(directoryList))
 				}
 			}
 			return nil

exporter/importables_test.go

@@ -34,15 +34,16 @@ import (
 func importContextForTest() *importContext {
 	p := provider.DatabricksProvider()
 	return &importContext{
-		Importables: resourcesMap,
-		Resources:   p.ResourcesMap,
-		Files:       map[string]*hclwrite.File{},
-		testEmits:   map[string]bool{},
-		nameFixes:   nameFixes,
-		waitGroup:   &sync.WaitGroup{},
-		allUsers:    map[string]scim.User{},
-		allSps:      map[string]scim.User{},
-		channels:    makeResourcesChannels(p),
+		Importables:              resourcesMap,
+		Resources:                p.ResourcesMap,
+		Files:                    map[string]*hclwrite.File{},
+		testEmits:                map[string]bool{},
+		nameFixes:                nameFixes,
+		waitGroup:                &sync.WaitGroup{},
+		allUsers:                 map[string]scim.User{},
+		allSps:                   map[string]scim.User{},
+		channels:                 makeResourcesChannels(p),
+		exportDeletedUsersAssets: false,
 	}
 }
 

exporter/util.go

@@ -41,10 +41,7 @@ func (ic *importContext) emitInitScripts(initScripts []clusters.InitScriptStorag
 			})
 		}
 		if is.Workspace != nil {
-			ic.Emit(&resource{
-				Resource: "databricks_workspace_file",
-				ID:       is.Workspace.Destination,
-			})
+			ic.maybeEmitWorkspaceObject("", is.Workspace.Destination)
 		}
 	}
 
@@ -112,27 +109,33 @@ func (ic *importContext) emitUserOrServicePrincipal(userOrSPName string) {
 	if userOrSPName == "" {
 		return
 	}
-	// TODO: think about another way of checking for a user. ideally we need to check against the
-	// list of users/SPs obtained via SCIM API - this will be done in the refactoring requested by the SCIM team
-	if strings.Contains(userOrSPName, "@") {
-		ic.Emit(&resource{
-			Resource:  "databricks_user",
-			Attribute: "user_name",
-			Value:     strings.ToLower(userOrSPName),
-		})
-	} else if common.StringIsUUID(userOrSPName) {
-		ic.Emit(&resource{
-			Resource:  "databricks_service_principal",
-			Attribute: "application_id",
-			Value:     userOrSPName,
-		})
+	if common.StringIsUUID(userOrSPName) {
+		user, err := ic.findSpnByAppID(userOrSPName)
+		if err != nil {
+			log.Printf("[ERROR] Can't find SP with application ID %s", userOrSPName)
+		} else {
+			ic.Emit(&resource{
+				Resource: "databricks_service_principal",
+				ID:       user.ID,
+			})
+		}
+	} else {
+		user, err := ic.findUserByName(strings.ToLower(userOrSPName))
+		if err != nil {
+			log.Printf("[ERROR] Can't find user with name %s", userOrSPName)
+		} else {
+			ic.Emit(&resource{
+				Resource: "databricks_user",
+				ID:       user.ID,
+			})
+		}
 	}
 }
 
 func (ic *importContext) emitUserOrServicePrincipalForPath(path, prefix string) {
 	if strings.HasPrefix(path, prefix) {
 		parts := strings.SplitN(path, "/", 4)
-		if len(parts) >= 3 {
+		if len(parts) >= 3 && parts[2] != "" {
 			ic.emitUserOrServicePrincipal(parts[2])
 		}
 	}
@@ -143,11 +146,16 @@ func (ic *importContext) IsUserOrServicePrincipalDirectory(path, prefix string)
 		return false
 	}
 	parts := strings.SplitN(path, "/", 4)
-	if len(parts) == 3 || (len(parts) == 4 && parts[3] == "") {
-		// TODO: think about another way of checking for a user. ideally we need to check against the
-		// list of users/SPs obtained via SCIM API - this will be done in the refactoring requested by the SCIM team
+	if (len(parts) == 3 || (len(parts) == 4 && parts[3] == "")) && parts[2] != "" {
 		userOrSPName := parts[2]
-		return strings.Contains(userOrSPName, "@") || common.StringIsUUID(userOrSPName)
+		var err error
+		if common.StringIsUUID(userOrSPName) {
+			_, err = ic.findSpnByAppID(userOrSPName)
+		} else {
+			_, err = ic.findUserByName(strings.ToLower(userOrSPName))
+		}
+		return err == nil
+
 	}
 	return false
 }
@@ -160,10 +168,7 @@ func (ic *importContext) emitNotebookOrRepo(path string) {
 			Value:     strings.Join(strings.SplitN(path, "/", 5)[:4], "/"),
 		})
 	} else {
-		ic.Emit(&resource{
-			Resource: "databricks_notebook",
-			ID:       path,
-		})
+		ic.maybeEmitWorkspaceObject("databricks_notebook", path)
 	}
 }
 
@@ -795,6 +800,26 @@ func wsObjectGetModifiedAt(obs workspace.ObjectStatus) int64 {
 	return obs.ModifiedAt
 }
 
+func (ic *importContext) shouldEmitForPath(path string) bool {
+	if !ic.exportDeletedUsersAssets && strings.HasPrefix(path, "/Users/") {
+		userDir := userDirRegex.ReplaceAllString(path, "$1")
+		return ic.IsUserOrServicePrincipalDirectory(userDir, "/Users")
+	}
+	return true
+}
+
+func (ic *importContext) maybeEmitWorkspaceObject(resourceType, path string) {
+	if ic.shouldEmitForPath(path) {
+		ic.Emit(&resource{
+			Resource:    resourceType,
+			ID:          path,
+			Incremental: ic.incremental,
+		})
+	} else {
+		log.Printf("[DEBUG] Not emitting a workspace object %s for deleted user", path)
+	}
+}
+
 func createListWorkspaceObjectsFunc(objType string, resourceType string, objName string) func(ic *importContext) error {
 	return func(ic *importContext) error {
 		// TODO: can we pass a visitor here, that will emit corresponding object earlier?
@@ -816,11 +841,7 @@ func createListWorkspaceObjectsFunc(objType string, resourceType string, objName
 			if !ic.MatchesName(object.Path) {
 				continue
 			}
-			ic.Emit(&resource{
-				Resource:    resourceType,
-				ID:          object.Path,
-				Incremental: ic.incremental,
-			})
+			ic.maybeEmitWorkspaceObject(resourceType, object.Path)
 
 			if offset%50 == 0 {
 				log.Printf("[INFO] Scanned %d of %d %ss", offset+1, len(objectsList), objName)