Skip to content

Commit 13a47d0

Browse files
alexottalexott
authored
Add MatchRegexp match type for partial matches (#2102)
First use is for creating references to parent folder in the DBSQL queries/dashboards
1 parent e1dd5c2 commit 13a47d0

File tree

10 files changed

+165
-17
lines changed

10 files changed

+165
-17
lines changed

‎exporter/context.go‎

Lines changed: 39 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"github.com/databricks/terraform-provider-databricks/common"
1717
"github.com/databricks/terraform-provider-databricks/provider"
1818
"github.com/databricks/terraform-provider-databricks/scim"
19+
"github.com/databricks/terraform-provider-databricks/workspace"
1920

2021
"github.com/hashicorp/hcl/v2"
2122
"github.com/hashicorp/hcl/v2/hclsyntax"
@@ -66,6 +67,7 @@ type importContext struct {
6667
testEmits map[string]bool
6768
sqlDatasources map[string]string
6869
workspaceConfKeys map[string]any
70+
allDirectories []workspace.ObjectStatus
6971

7072
includeUserDomains bool
7173
importAllUsers bool
@@ -132,9 +134,9 @@ func newImportContext(c *common.DatabricksClient) *importContext {
132134
nameFixes: nameFixes,
133135
hclFixes: []regexFix{ // Be careful with that! it may break working code
134136
},
135-
allUsers: []scim.User{},
136-
variables: map[string]string{},
137-
137+
allUsers: []scim.User{},
138+
variables: map[string]string{},
139+
allDirectories: []workspace.ObjectStatus{},
138140
workspaceConfKeys: workspaceConfKeys,
139141
}
140142
}
@@ -300,11 +302,24 @@ func (ic *importContext) MatchesName(n string) bool {
300302
return strings.Contains(strings.ToLower(n), strings.ToLower(ic.match))
301303
}
302304

303-
func (ic *importContext) Find(r *resource, pick string, matchType MatchType) (string, hcl.Traversal) {
305+
func (ic *importContext) Find(r *resource, pick string, ref reference) (string, hcl.Traversal) {
304306
for _, sr := range ic.State.Resources {
305307
if sr.Type != r.Resource {
306308
continue
307309
}
310+
// optimize performance by avoiding doing regexp matching multiple times
311+
matchValue := ""
312+
if ref.MatchType == MatchRegexp {
313+
if ref.Regexp == nil {
314+
log.Printf("[WARN] you must provide regular expression for 'regexp' match type")
315+
continue
316+
}
317+
res := ref.Regexp.FindStringSubmatch(r.Value)
318+
if len(res) < 2 {
319+
log.Printf("[WARN] no match for regexp: %v in string %s", ref.Regexp, r.Value)
320+
}
321+
matchValue = res[1]
322+
}
308323
for _, i := range sr.Instances {
309324
v := i.Attributes[r.Attribute]
310325
if v == nil {
@@ -314,13 +329,15 @@ func (ic *importContext) Find(r *resource, pick string, matchType MatchType) (st
314329
}
315330
strValue := v.(string)
316331
matched := false
317-
switch matchType {
318-
case MatchExact:
332+
switch ref.MatchType {
333+
case MatchExact, MatchDefault:
319334
matched = (strValue == r.Value)
320335
case MatchPrefix:
321336
matched = strings.HasPrefix(r.Value, strValue)
337+
case MatchRegexp:
338+
matched = (matchValue == strValue)
322339
default:
323-
log.Printf("[WARN] Unsupported match type: %s", matchType)
340+
log.Printf("[WARN] Unsupported match type: %s", ref.MatchType)
324341
}
325342
if !matched {
326343
continue
@@ -511,13 +528,13 @@ func (ic *importContext) getTraversalTokens(ref reference, value string) hclwrit
511528
Resource: ref.Resource,
512529
Attribute: attr,
513530
Value: value,
514-
}, attr, matchType)
531+
}, attr, ref)
515532
// at least one invocation of ic.Find will assign Nil to traversal if resource with value is not found
516533
if traversal == nil {
517534
return nil
518535
}
519536
switch matchType {
520-
case MatchExact:
537+
case MatchExact, MatchDefault:
521538
return hclwrite.TokensForTraversal(traversal)
522539
case MatchPrefix:
523540
rest := value[len(attrValue):]
@@ -527,6 +544,19 @@ func (ic *importContext) getTraversalTokens(ref reference, value string) hclwrit
527544
tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenQuotedLit, Bytes: []byte(rest)})
528545
tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenCQuote, Bytes: []byte{'"'}})
529546
return tokens
547+
case MatchRegexp:
548+
indices := ref.Regexp.FindStringSubmatchIndex(value)
549+
if len(indices) == 4 {
550+
tokens := hclwrite.Tokens{&hclwrite.Token{Type: hclsyntax.TokenOQuote, Bytes: []byte{'"'}}}
551+
tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenQuotedLit, Bytes: []byte(value[0:indices[2]])})
552+
tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenOQuote, Bytes: []byte{'$', '{'}})
553+
tokens = append(tokens, hclwrite.TokensForTraversal(traversal)...)
554+
tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenCQuote, Bytes: []byte{'}'}})
555+
tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenQuotedLit, Bytes: []byte(value[indices[3]:])})
556+
tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenCQuote, Bytes: []byte{'"'}})
557+
return tokens
558+
}
559+
log.Printf("[WARN] Can't match found data in '%s'. Indices: %v", value, indices)
530560
default:
531561
log.Printf("[WARN] Unsupported match type: %s", ref.MatchType)
532562
}

‎exporter/context_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ func TestImportContextFindSkips(t *testing.T) {
3636
Resource: "a",
3737
Attribute: "b",
3838
Name: "c",
39-
}, "x", "")
39+
}, "x", reference{})
4040
assert.Nil(t, traversal)
4141
}
4242

‎exporter/exporter_test.go‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1440,6 +1440,38 @@ func TestImportingSqlObjects(t *testing.T) {
14401440
emptyRepos,
14411441
emptyIpAccessLIst,
14421442
emptyGlobalSQLConfig,
1443+
{
1444+
Method: "GET",
1445+
Resource: "/api/2.0/workspace/list?path=%2F",
1446+
Response: workspace.ObjectList{
1447+
Objects: []workspace.ObjectStatus{
1448+
{
1449+
Path: "/Shared",
1450+
ObjectID: 4451965692354143,
1451+
ObjectType: workspace.Directory,
1452+
},
1453+
},
1454+
},
1455+
},
1456+
{
1457+
Method: "GET",
1458+
Resource: "/api/2.0/workspace/list?path=%2FShared",
1459+
Response: workspace.ObjectList{},
1460+
},
1461+
{
1462+
Method: "GET",
1463+
Resource: "/api/2.0/workspace/get-status?path=%2FShared",
1464+
Response: workspace.ObjectStatus{
1465+
Path: "/Shared",
1466+
ObjectID: 4451965692354143,
1467+
ObjectType: workspace.Directory,
1468+
},
1469+
},
1470+
{
1471+
Method: "GET",
1472+
Resource: "/api/2.0/permissions/directories/4451965692354143",
1473+
Response: getJSONObject("test-data/get-directory-permissions.json"),
1474+
},
14431475
{
14441476
Method: "GET",
14451477
Resource: "/api/2.0/global-init-scripts",
@@ -1512,8 +1544,8 @@ func TestImportingSqlObjects(t *testing.T) {
15121544

15131545
ic := newImportContext(client)
15141546
ic.Directory = tmpDir
1515-
ic.listing = "sql-dashboards,sql-queries,sql-endpoints,access"
1516-
ic.services = "sql-dashboards,sql-queries,sql-endpoints,access"
1547+
ic.listing = "sql-dashboards,sql-queries,sql-endpoints"
1548+
ic.services = "sql-dashboards,sql-queries,sql-endpoints,access,notebooks"
15171549

15181550
err := ic.Run()
15191551
assert.NoError(t, err)
@@ -1525,6 +1557,7 @@ func TestImportingDLTPipelines(t *testing.T) {
15251557
[]qa.HTTPFixture{
15261558
meAdminFixture,
15271559
emptyRepos,
1560+
emptyWorkspace,
15281561
emptyIpAccessLIst,
15291562
{
15301563
Method: "GET",

‎exporter/importables.go‎

Lines changed: 40 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ var (
4646
uuidRegex = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
4747
predefinedClusterPolicies = []string{"Personal Compute", "Job Compute", "Power User Compute", "Shared Compute"}
4848
secretPathRegex = regexp.MustCompile(`^\{\{secrets\/([^\/]+)\/([^}]+)\}\}$`)
49+
sqlParentRegexp = regexp.MustCompile(`^folders/(\d+)$`)
4950
)
5051

5152
func generateMountBody(ic *importContext, body *hclwrite.Body, r *resource) error {
@@ -1244,6 +1245,16 @@ var resourcesMap map[string]importable = map[string]importable{
12441245
})
12451246
}
12461247
}
1248+
if query.Parent != "" {
1249+
res := sqlParentRegexp.FindStringSubmatch(query.Parent)
1250+
if len(res) > 1 {
1251+
ic.Emit(&resource{
1252+
Resource: "databricks_directory",
1253+
Attribute: "object_id",
1254+
Value: res[1],
1255+
})
1256+
}
1257+
}
12471258
if ic.meAdmin {
12481259
ic.Emit(&resource{
12491260
Resource: "databricks_permissions",
@@ -1256,6 +1267,8 @@ var resourcesMap map[string]importable = map[string]importable{
12561267
Depends: []reference{
12571268
{Path: "data_source_id", Resource: "databricks_sql_endpoint", Match: "data_source_id"},
12581269
{Path: "parameter.query.query_id", Resource: "databricks_sql_query", Match: "id"},
1270+
{Path: "parent", Resource: "databricks_directory", Match: "object_id", MatchType: MatchRegexp,
1271+
Regexp: sqlParentRegexp},
12591272
},
12601273
},
12611274
"databricks_sql_endpoint": {
@@ -1364,6 +1377,16 @@ var resourcesMap map[string]importable = map[string]importable{
13641377
if err != nil {
13651378
return err
13661379
}
1380+
if dashboard.Parent != "" {
1381+
res := sqlParentRegexp.FindStringSubmatch(dashboard.Parent)
1382+
if len(res) > 1 {
1383+
ic.Emit(&resource{
1384+
Resource: "databricks_directory",
1385+
Attribute: "object_id",
1386+
Value: res[1],
1387+
})
1388+
}
1389+
}
13671390
for _, rv := range dashboard.Widgets {
13681391
var widget sql_api.Widget
13691392
err = json.Unmarshal(rv, &widget)
@@ -1416,6 +1439,10 @@ var resourcesMap map[string]importable = map[string]importable{
14161439
}
14171440
return nil
14181441
},
1442+
Depends: []reference{
1443+
{Path: "parent", Resource: "databricks_directory", Match: "object_id", MatchType: MatchRegexp,
1444+
Regexp: sqlParentRegexp},
1445+
},
14191446
},
14201447
"databricks_sql_widget": {
14211448
Service: "sql-dashboards",
@@ -1549,12 +1576,22 @@ var resourcesMap map[string]importable = map[string]importable{
15491576
}
15501577
return name
15511578
},
1552-
List: func(ic *importContext) error {
1553-
notebooksAPI := workspace.NewNotebooksAPI(ic.Context, ic.Client)
1554-
directoryList, err := notebooksAPI.ListDirectories("/", true)
1579+
Search: func(ic *importContext, r *resource) error {
1580+
directoryList := ic.getAllDirectories()
1581+
objId, err := strconv.ParseInt(r.Value, 10, 64)
15551582
if err != nil {
15561583
return err
15571584
}
1585+
for _, directory := range directoryList {
1586+
if directory.ObjectID == objId {
1587+
r.ID = directory.Path
1588+
return nil
1589+
}
1590+
}
1591+
return fmt.Errorf("can't find directory with object_id: %s", r.Value)
1592+
},
1593+
List: func(ic *importContext) error {
1594+
directoryList := ic.getAllDirectories()
15581595
for offset, directory := range directoryList {
15591596
if strings.HasPrefix(directory.Path, "/Repos") {
15601597
continue
@@ -1571,7 +1608,6 @@ var resourcesMap map[string]importable = map[string]importable{
15711608
return nil
15721609
},
15731610
Import: func(ic *importContext, r *resource) error {
1574-
15751611
ic.emitUserOrServicePrincipalForPath(r.ID, "/Users")
15761612
splits := strings.Split(r.Name, "_")
15771613
directoryId := splits[len(splits)-1]

‎exporter/model.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,17 +61,22 @@ type MatchType string
6161
const (
6262
// MatchExact is to specify that whole value should match
6363
MatchExact = "exact"
64+
// MatchDefault - same meaning as MatchExact
65+
MatchDefault = ""
6466
// MatchPrefix is to specify that prefix of value should match
6567
MatchPrefix = "prefix"
68+
// MatchRegexp is to specify that the group extracted from value should match
69+
MatchRegexp = "regexp"
6670
)
6771

6872
type reference struct {
6973
Path string
7074
Resource string
7175
Match string
72-
MatchType MatchType // type of match, `prefix` - reference is embedded into string, `` (or `exact`) - full match
76+
MatchType MatchType // type of match, `prefix` - reference is embedded into string, `` (or `exact`) - full match, ...
7377
Variable bool
7478
File bool
79+
Regexp *regexp.Regexp // regular expression must define a group that will be used to extract value to match
7580
}
7681

7782
func (r reference) MatchAttribute() string {

‎exporter/test-data/get-directory-permissions.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
{
2+
"access_control_list": [
3+
{
4+
"all_permissions": [
5+
{
6+
"inherited": true,
7+
"inherited_from_object": [
8+
"/directories/"
9+
],
10+
"permission_level": "CAN_MANAGE"
11+
}
12+
],
13+
"user_name": "[email protected]"
14+
},
15+
{
16+
"all_permissions": [
17+
{
18+
"inherited": true,
19+
"inherited_from_object": [
20+
"/directories/"
21+
],
22+
"Permission_level": "CAN_MANAGE"
23+
}
24+
],
25+
"group_name": "admins"
26+
}
27+
],
28+
"object_id": "/4451965692354143/4451965692354143",
29+
"object_type": "directory"
30+
}

‎exporter/test-data/get-sql-dashboard.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@
1010
"is_favorite": true,
1111
"layout": [],
1212
"name": "Test",
13+
"parent": "folders/4451965692354143",
1314
"options": {
15+
"parent": "folders/4451965692354143",
1416
"refresh_schedules": []
1517
},
1618
"permission_tier": "CAN_MANAGE",

‎exporter/test-data/get-sql-dashboards.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
"layout": [],
1616
"name": "Test",
1717
"options": {
18+
"parent": "folders/4451965692354143",
1819
"refresh_schedules": []
1920
},
2021
"refresh_schedules": [],

‎exporter/test-data/get-sql-query.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717
},
1818
"latest_query_data_id": "e39090e8-6cc0-4a6d-b989-6c7dc75a7a8f",
1919
"name": "Jobs per day per status last 30 days",
20+
"parent": "folders/4451965692354143",
2021
"options": {
22+
"parent": "folders/4451965692354143",
2123
"apply_auto_limit": true,
2224
"parameters": []
2325
},

‎exporter/util.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"github.com/databricks/terraform-provider-databricks/scim"
2121
"github.com/databricks/terraform-provider-databricks/sql"
2222
"github.com/databricks/terraform-provider-databricks/storage"
23+
"github.com/databricks/terraform-provider-databricks/workspace"
2324

2425
"github.com/hashicorp/hcl/v2/hclwrite"
2526
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
@@ -139,6 +140,14 @@ func (ic *importContext) emitNotebookOrRepo(path string) {
139140
}
140141
}
141142

143+
func (ic *importContext) getAllDirectories() []workspace.ObjectStatus {
144+
if len(ic.allDirectories) == 0 {
145+
notebooksAPI := workspace.NewNotebooksAPI(ic.Context, ic.Client)
146+
ic.allDirectories, _ = notebooksAPI.ListDirectories("/", true)
147+
}
148+
return ic.allDirectories
149+
}
150+
142151
func (ic *importContext) emitGroups(u scim.User, principal string) {
143152
for _, g := range u.Groups {
144153
if g.Type != "direct" {

0 commit comments

Comments
 (0)