Exporter: generate references that are matching only to a prefix (#1890)

* Exporter: generate references that are matching only to a prefix

There are cases when reference isn't a full match of a given value but a prefix of it. For
example, user's notebook has a path of `/Users/user@domain/notebook`.  To make it working
correctly we need to emit user or service principal, and then make a correct reference
with dependency on emitted user or notebook, like,
`${databricks_user.user_domain.home}/notebook`.

This PR adds a new field to dependency specification: `MatchType` that may have following values:

- `prefix` - value of attribute of resource is a prefix of a given attribute
- `exact` (or empty string) - must have exact matching (previous behaviour)

this fixes #1777

* improving test coverage & emit user/sp for repos

* Address PR feedback

Author: alexott

exporter/context.go

@@ -301,7 +301,7 @@ func (ic *importContext) MatchesName(n string) bool {
 	return strings.Contains(strings.ToLower(n), strings.ToLower(ic.match))
 }
 
-func (ic *importContext) Find(r *resource, pick string) hcl.Traversal {
+func (ic *importContext) Find(r *resource, pick string, matchType MatchType) (string, hcl.Traversal) {
 	for _, sr := range ic.State.Resources {
 		if sr.Type != r.Resource {
 			continue
@@ -313,24 +313,36 @@ func (ic *importContext) Find(r *resource, pick string) hcl.Traversal {
 					r.Attribute, r.Resource, r.Name, r.ID)
 				continue
 			}
-			if v.(string) == r.Value {
-				if sr.Mode == "data" {
-					return hcl.Traversal{
-						hcl.TraverseRoot{Name: "data"},
-						hcl.TraverseAttr{Name: sr.Type},
-						hcl.TraverseAttr{Name: sr.Name},
-						hcl.TraverseAttr{Name: pick},
-					}
-				}
-				return hcl.Traversal{
-					hcl.TraverseRoot{Name: sr.Type},
+			strValue := v.(string)
+			matched := false
+			switch matchType {
+			case MatchExact:
+				matched = (strValue == r.Value)
+			case MatchPrefix:
+				matched = strings.HasPrefix(r.Value, strValue)
+			default:
+				log.Printf("[WARN] Unsupported match type: %s", matchType)
+			}
+			if !matched {
+				continue
+			}
+			if sr.Mode == "data" {
+				return strValue, hcl.Traversal{
+					hcl.TraverseRoot{Name: "data"},
+					hcl.TraverseAttr{Name: sr.Type},
 					hcl.TraverseAttr{Name: sr.Name},
 					hcl.TraverseAttr{Name: pick},
 				}
 			}
+			return strValue, hcl.Traversal{
+				hcl.TraverseRoot{Name: sr.Type},
+				hcl.TraverseAttr{Name: sr.Name},
+				hcl.TraverseAttr{Name: pick},
+			}
+
 		}
 	}
-	return nil
+	return "", nil
 }
 
 func (ic *importContext) Has(r *resource) bool {
@@ -486,6 +498,35 @@ func (ic *importContext) Emit(r *resource) {
 	ic.Add(r)
 }
 
+func (ic *importContext) getTraversalTokens(ref reference, value string) hclwrite.Tokens {
+	matchType := ref.MatchTypeValue()
+	attr := ref.MatchAttribute()
+	attrValue, traversal := ic.Find(&resource{
+		Resource:  ref.Resource,
+		Attribute: attr,
+		Value:     value,
+	}, attr, matchType)
+	// at least one invocation of ic.Find will assign Nil to traversal if resource with value is not found
+	if traversal == nil {
+		return nil
+	}
+	switch matchType {
+	case MatchExact:
+		return hclwrite.TokensForTraversal(traversal)
+	case MatchPrefix:
+		rest := value[len(attrValue):]
+		tokens := hclwrite.Tokens{&hclwrite.Token{Type: hclsyntax.TokenOQuote, Bytes: []byte{'"', '$', '{'}}}
+		tokens = append(tokens, hclwrite.TokensForTraversal(traversal)...)
+		tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenCQuote, Bytes: []byte{'}'}})
+		tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenQuotedLit, Bytes: []byte(rest)})
+		tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenCQuote, Bytes: []byte{'"'}})
+		return tokens
+	default:
+		log.Printf("[WARN] Unsupported match type: %s", ref.MatchType)
+	}
+	return nil
+}
+
 // TODO: move to IC
 var dependsRe = regexp.MustCompile(`(\.[\d]+)`)
 
@@ -506,18 +547,9 @@ func (ic *importContext) reference(i importable, path []string, value string) hc
 		if d.Variable {
 			return ic.variable(fmt.Sprintf("%s_%s", path[0], value), "")
 		}
-		attr := "id"
-		if d.Match != "" {
-			attr = d.Match
-		}
-		traversal := ic.Find(&resource{
-			Resource:  d.Resource,
-			Attribute: attr,
-			Value:     value,
-		}, attr)
-		//at least one invocation of ic.Find will assign Nil to traversal if resource with value is not found
-		if traversal != nil {
-			return hclwrite.TokensForTraversal(traversal)
+
+		if tokens := ic.getTraversalTokens(d, value); tokens != nil {
+			return tokens
 		}
 	}
 	return hclwrite.TokensForValue(cty.StringVal(value))

exporter/context_test.go

@@ -17,7 +17,7 @@ func TestMatchesName(t *testing.T) {
 }
 
 func TestImportContextFindSkips(t *testing.T) {
-	assert.Nil(t, (&importContext{
+	_, traversal := (&importContext{
 		State: stateApproximation{
 			Resources: []resourceApproximation{
 				{
@@ -36,7 +36,8 @@ func TestImportContextFindSkips(t *testing.T) {
 		Resource:  "a",
 		Attribute: "b",
 		Name:      "c",
-	}, "x"))
+	}, "x", "")
+	assert.Nil(t, traversal)
 }
 
 func TestImportContextHas(t *testing.T) {

exporter/exporter_test.go

@@ -1533,6 +1533,51 @@ func TestImportingDLTPipelines(t *testing.T) {
 					},
 				},
 			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/permissions/repos/123",
+				Response: getJSONObject("test-data/get-repo-permissions.json"),
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/workspace/get-status?path=%2FRepos%2Fuser%40domain.com%2Frepo",
+				Response: workspace.ObjectStatus{
+					ObjectID:   123,
+					ObjectType: "REPO",
+					Path:       "/Repos/[email protected]/repo",
+				},
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/repos/123",
+				Response: repos.ReposInformation{
+					ID:           123,
+					Url:          "https://github.com/user/test.git",
+					Provider:     "gitHub",
+					Path:         "/Repos/[email protected]/repo",
+					HeadCommitID: "1124323423abc23424",
+					Branch:       "releases",
+				},
+				ReuseRequest: true,
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/preview/scim/v2/Users?filter=userName%20eq%20%27user%40domain.com%27",
+				Response: scim.UserList{
+					Resources: []scim.User{
+						{ID: "123", DisplayName: "[email protected]", UserName: "[email protected]"},
+					},
+					StartIndex:   1,
+					TotalResults: 1,
+					ItemsPerPage: 1,
+				},
+				ReuseRequest: true,
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/preview/scim/v2/Users/123",
+				Response: scim.User{ID: "123", DisplayName: "[email protected]", UserName: "[email protected]"},
+			},
 			{
 				Method:   "GET",
 				Resource: "/api/2.0/pipelines/123",
@@ -1597,7 +1642,7 @@ func TestImportingDLTPipelines(t *testing.T) {
 			ic := newImportContext(client)
 			ic.Directory = tmpDir
 			ic.listing = "dlt"
-			ic.services = "dlt,access,notebooks,secrets"
+			ic.services = "dlt,access,notebooks,users,repos,secrets"
 
 			err := ic.Run()
 			assert.NoError(t, err)

exporter/importables.go

@@ -41,6 +41,7 @@ var (
 	nameNormalizationRegex    = regexp.MustCompile(`\W+`)
 	jobClustersRegex          = regexp.MustCompile(`^((job_cluster|task)\.[0-9]+\.new_cluster\.[0-9]+\.)`)
 	dltClusterRegex           = regexp.MustCompile(`^(cluster\.[0-9]+\.)`)
+	uuidRegex                 = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
 	predefinedClusterPolicies = []string{"Personal Compute", "Job Compute", "Power User Compute", "Shared Compute"}
 	secretPathRegex           = regexp.MustCompile(`^\{\{secrets\/([^\/]+)\/([^}]+)\}\}$`)
 )
@@ -287,6 +288,7 @@ var resourcesMap map[string]importable = map[string]importable{
 			{Path: "spark_python_task.parameters", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
 			{Path: "spark_jar_task.jar_uri", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
 			{Path: "notebook_task.notebook_path", Resource: "databricks_notebook"},
+			{Path: "notebook_task.notebook_path", Resource: "databricks_repo", Match: "path", MatchType: MatchPrefix},
 			{Path: "pipeline_task.pipeline_id", Resource: "databricks_pipeline"},
 			{Path: "task.library.jar", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
 			{Path: "task.library.whl", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
@@ -295,6 +297,7 @@ var resourcesMap map[string]importable = map[string]importable{
 			{Path: "task.spark_python_task.parameters", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
 			{Path: "task.spark_jar_task.jar_uri", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
 			{Path: "task.notebook_task.notebook_path", Resource: "databricks_notebook"},
+			{Path: "task.notebook_task.notebook_path", Resource: "databricks_repo", Match: "path", MatchType: MatchPrefix},
 			{Path: "task.pipeline_task.pipeline_id", Resource: "databricks_pipeline"},
 			{Path: "task.sql_task.query.query_id", Resource: "databricks_sql_query"},
 			{Path: "task.sql_task.dashboard.dashboard_id", Resource: "databricks_sql_dashboard"},
@@ -359,10 +362,7 @@ var resourcesMap map[string]importable = map[string]importable{
 				}
 			}
 			if job.NotebookTask != nil {
-				ic.Emit(&resource{
-					Resource: "databricks_notebook",
-					ID:       job.NotebookTask.NotebookPath,
-				})
+				ic.emitNotebookOrRepo(job.NotebookTask.NotebookPath)
 			}
 			if job.PipelineTask != nil {
 				ic.Emit(&resource{
@@ -373,10 +373,7 @@ var resourcesMap map[string]importable = map[string]importable{
 			// Support for multitask jobs
 			for _, task := range job.Tasks {
 				if task.NotebookTask != nil {
-					ic.Emit(&resource{
-						Resource: "databricks_notebook",
-						ID:       task.NotebookTask.NotebookPath,
-					})
+					ic.emitNotebookOrRepo(task.NotebookTask.NotebookPath)
 				}
 				if task.PipelineTask != nil {
 					ic.Emit(&resource{
@@ -958,7 +955,21 @@ var resourcesMap map[string]importable = map[string]importable{
 			if name == "" {
 				return d.Id()
 			}
-			return strings.TrimPrefix(name, "/")
+			return nameNormalizationRegex.ReplaceAllString(name[7:], "_")
+		},
+		Search: func(ic *importContext, r *resource) error {
+			reposAPI := repos.NewReposAPI(ic.Context, ic.Client)
+			notebooksAPI := workspace.NewNotebooksAPI(ic.Context, ic.Client)
+			repoDir, err := notebooksAPI.Read(r.Value)
+			if err != nil {
+				return err
+			}
+			repo, err := reposAPI.Read(fmt.Sprintf("%d", repoDir.ObjectID))
+			if err != nil {
+				return err
+			}
+			r.ID = fmt.Sprintf("%d", repo.ID)
+			return nil
 		},
 		List: func(ic *importContext) error {
 			repoList, err := repos.NewReposAPI(ic.Context, ic.Client).ListAll()
@@ -977,6 +988,7 @@ var resourcesMap map[string]importable = map[string]importable{
 			return nil
 		},
 		Import: func(ic *importContext, r *resource) error {
+			ic.emitUserOrServicePrincipalForPath(r.Data.Get("path").(string), "/Repos")
 			if ic.meAdmin {
 				ic.Emit(&resource{
 					Resource: "databricks_permissions",
@@ -986,6 +998,17 @@ var resourcesMap map[string]importable = map[string]importable{
 			}
 			return nil
 		},
+		ShouldOmitField: func(ic *importContext, pathString string, as *schema.Schema, d *schema.ResourceData) bool {
+			if pathString == "path" {
+				return false
+			}
+			return defaultShouldOmitFieldFunc(ic, pathString, as, d)
+		},
+
+		Depends: []reference{
+			{Path: "path", Resource: "databricks_user", Match: "repos", MatchType: MatchPrefix},
+			{Path: "path", Resource: "databricks_service_principal", Match: "repos", MatchType: MatchPrefix},
+		},
 	},
 	"databricks_workspace_conf": {
 		Service: "workspace",
@@ -1094,6 +1117,7 @@ var resourcesMap map[string]importable = map[string]importable{
 			return nil
 		},
 		Import: func(ic *importContext, r *resource) error {
+			ic.emitUserOrServicePrincipalForPath(r.ID, "/Users")
 			notebooksAPI := workspace.NewNotebooksAPI(ic.Context, ic.Client)
 			contentB64, err := notebooksAPI.Export(r.ID, "SOURCE")
 			if err != nil {
@@ -1118,6 +1142,8 @@ var resourcesMap map[string]importable = map[string]importable{
 		},
 		Depends: []reference{
 			{Path: "source", File: true},
+			{Path: "path", Resource: "databricks_user", Match: "home", MatchType: MatchPrefix},
+			{Path: "path", Resource: "databricks_service_principal", Match: "home", MatchType: MatchPrefix},
 		},
 	},
 	"databricks_sql_query": {
@@ -1391,10 +1417,7 @@ var resourcesMap map[string]importable = map[string]importable{
 			common.DataToStructPointer(r.Data, s, &pipeline)
 			for _, lib := range pipeline.Libraries {
 				if lib.Notebook != nil {
-					ic.Emit(&resource{
-						Resource: "databricks_notebook",
-						ID:       lib.Notebook.Path,
-					})
+					ic.emitNotebookOrRepo(lib.Notebook.Path)
 				}
 				ic.emitIfDbfsFile(lib.Jar)
 				ic.emitIfDbfsFile(lib.Whl)
@@ -1452,6 +1475,7 @@ var resourcesMap map[string]importable = map[string]importable{
 			{Path: "cluster.instance_pool_id", Resource: "databricks_instance_pool"},
 			{Path: "cluster.driver_instance_pool_id", Resource: "databricks_instance_pool"},
 			{Path: "library.notebook.path", Resource: "databricks_notebook"},
+			{Path: "library.notebook.path", Resource: "databricks_repo", Match: "path", MatchType: MatchPrefix},
 			{Path: "library.jar", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
 			{Path: "library.whl", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
 		},

exporter/model.go

@@ -56,12 +56,36 @@ type importable struct {
 	ApiVersion common.ApiVersion
 }
 
+type MatchType string
+
+const (
+	// MatchExact is to specify that whole value should match
+	MatchExact = "exact"
+	// MatchPrefix is to specify that prefix of value should match
+	MatchPrefix = "prefix"
+)
+
 type reference struct {
-	Path     string
-	Resource string
-	Match    string
-	Variable bool
-	File     bool
+	Path      string
+	Resource  string
+	Match     string
+	MatchType MatchType // type of match, `prefix` - reference is embedded into string, `` (or `exact`) - full match
+	Variable  bool
+	File      bool
+}
+
+func (r reference) MatchAttribute() string {
+	if r.Match != "" {
+		return r.Match
+	}
+	return "id"
+}
+
+func (r reference) MatchTypeValue() MatchType {
+	if r.MatchType == "" {
+		return MatchExact
+	}
+	return r.MatchType
 }
 
 type resource struct {

exporter/test-data/get-dlt-pipeline.json

@@ -40,6 +40,11 @@
         "notebook": {
           "path": "/Users/[email protected]/Test DLT"
         }
+      },
+      {
+        "notebook": {
+          "path": "/Repos/[email protected]/repo/Test DLT"
+        }
       }
     ],
     "name": "Test DLT",