Skip to content

Commit 65564f1

Browse files
alexottalexott
authored
Exporter: phase 1 of support for Account-level exports (#2205)
Implemented export of users/groups/service principals on account level, including generation of corresponding role objects to support `account_admin` role. To support role export, was need to make roles exclusion for users/service principals configurable. Current limitation: not all users & service principals could be exported - will require addition of the dedicated `List` implementation for these resources, as `account users` group isn't returned by the API.
1 parent eb3bf25 commit 65564f1

14 files changed

+105
-44
lines changed

‎docs/guides/experimental-exporter.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ Exporter aims to generate HCL code for the most of resources within the Databric
8282
| [databricks_group](../resources/group.md) | Yes |
8383
| [databricks_group_instance_profile](../resources/group_instance_profile.md) | Yes |
8484
| [databricks_group_member](../resources/group_member.md) | Yes |
85+
| [databricks_group_role](../resources/group_role.md) | Yes |
8586
| [databricks_instance_pool](../resources/instance_pool.md) | Yes |
8687
| [databricks_instance_profile](../resources/instance_profile.md) | Yes |
8788
| [databricks_ip_access_list](../resources/ip_access_list.md) | Yes |
@@ -98,6 +99,7 @@ Exporter aims to generate HCL code for the most of resources within the Databric
9899
| [databricks_secret_acl](../resources/secret_acl.md) | Yes |
99100
| [databricks_secret_scope](../resources/secret_scope.md) | Yes |
100101
| [databricks_service_principal](../resources/service_principal.md) | Yes |
102+
| [databricks_service_principal_role](../resources/service_principal_role.md) | Yes |
101103
| [databricks_sql_alert](../resources/sql_alert.md) | Yes |
102104
| [databricks_sql_dashboard](../resources/sql_dashboard.md) | Yes |
103105
| [databricks_sql_endpoint](../resources/sql_endpoint.md) | Yes |
@@ -109,4 +111,5 @@ Exporter aims to generate HCL code for the most of resources within the Databric
109111
| [databricks_token](../resources/token.md) | Not Applicable |
110112
| [databricks_user](../resources/user.md) | Yes |
111113
| [databricks_user_instance_profile](../resources/user_instance_profile.md) | No (Deprecated) |
114+
| [databricks_user_role](../resources/user_role.md) | Yes |
112115
| [databricks_workspace_conf](../resources/workspace_conf.md) | Yes (partial) |

‎exporter/context.go‎

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ type importContext struct {
8080
generateDeclaration bool
8181
meAdmin bool
8282
prefix string
83+
accountLevel bool
8384
}
8485

8586
type mount struct {
@@ -161,16 +162,23 @@ func (ic *importContext) Run() error {
161162
if err != nil {
162163
return err
163164
}
164-
me, err := w.CurrentUser.Me(ic.Context)
165-
if err != nil {
166-
return err
167-
}
168-
for _, g := range me.Groups {
169-
if g.Display == "admins" {
170-
ic.meAdmin = true
171-
break
165+
166+
ic.accountLevel = ic.Client.Config.IsAccountClient()
167+
if ic.accountLevel {
168+
ic.meAdmin = true
169+
} else {
170+
me, err := w.CurrentUser.Me(ic.Context)
171+
if err != nil {
172+
return err
173+
}
174+
for _, g := range me.Groups {
175+
if g.Display == "admins" {
176+
ic.meAdmin = true
177+
break
178+
}
172179
}
173180
}
181+
174182
for resourceName, ir := range ic.Importables {
175183
if ir.List == nil {
176184
continue
@@ -180,6 +188,10 @@ func (ic *importContext) Run() error {
180188
resourceName, ir.Service)
181189
continue
182190
}
191+
if ic.accountLevel && !ir.AccountLevel {
192+
log.Printf("[DEBUG] %s (%s service) is not account level", resourceName, ir.Service)
193+
continue
194+
}
183195
if err := ir.List(ic); err != nil {
184196
log.Printf("[ERROR] %s (%s service) listing failed: %s",
185197
resourceName, ir.Service, err)
@@ -474,6 +486,12 @@ func (ic *importContext) Emit(r *resource) {
474486
log.Printf("[ERROR] %s is not available for import", r)
475487
return
476488
}
489+
if ic.accountLevel && !ir.AccountLevel {
490+
log.Printf("[DEBUG] %s (%s service) is not part of the account level export",
491+
r.Resource, ir.Service)
492+
return
493+
494+
}
477495
if !strings.Contains(ic.services, ir.Service) {
478496
log.Printf("[DEBUG] %s (%s service) is not part of the import",
479497
r.Resource, ir.Service)

‎exporter/exporter_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -360,7 +360,7 @@ func TestImportingUsersGroupsSecretScopes(t *testing.T) {
360360
},
361361
{
362362
Method: "GET",
363-
Resource: "/api/2.0/preview/scim/v2/ServicePrincipals?excludedAttributes=roles&filter=applicationId%20eq%20%27spn%27",
363+
Resource: "/api/2.0/preview/scim/v2/ServicePrincipals?filter=applicationId%20eq%20%27spn%27",
364364
Response: scim.User{ID: "321", DisplayName: "spn", ApplicationID: "spn",
365365
Groups: []scim.ComplexValue{
366366
{Display: "admins", Value: "a", Ref: "Groups/a", Type: "direct"},
@@ -426,7 +426,7 @@ func TestImportingUsersGroupsSecretScopes(t *testing.T) {
426426
},
427427
{
428428
Method: "GET",
429-
Resource: "/api/2.0/preview/scim/v2/Users?excludedAttributes=roles&filter=userName%20eq%20%27test%40test.com%27",
429+
Resource: "/api/2.0/preview/scim/v2/Users?filter=userName%20eq%20%27test%40test.com%27",
430430
Response: scim.UserList{
431431
Resources: []scim.User{
432432
{ID: "123", DisplayName: "[email protected]", UserName: "[email protected]"},
@@ -1316,7 +1316,7 @@ func TestImportingUser(t *testing.T) {
13161316
{
13171317
Method: "GET",
13181318
ReuseRequest: true,
1319-
Resource: "/api/2.0/preview/scim/v2/Users?excludedAttributes=roles&filter=userName%20eq%20%27me%27",
1319+
Resource: "/api/2.0/preview/scim/v2/Users?filter=userName%20eq%20%27me%27",
13201320
Response: scim.UserList{
13211321
Resources: []scim.User{
13221322
{
@@ -1658,7 +1658,7 @@ func TestImportingDLTPipelines(t *testing.T) {
16581658
},
16591659
{
16601660
Method: "GET",
1661-
Resource: "/api/2.0/preview/scim/v2/Users?excludedAttributes=roles&filter=userName%20eq%20%27user%40domain.com%27",
1661+
Resource: "/api/2.0/preview/scim/v2/Users?filter=userName%20eq%20%27user%40domain.com%27",
16621662
Response: scim.UserList{
16631663
Resources: []scim.User{
16641664
{ID: "123", DisplayName: "[email protected]", UserName: "[email protected]"},

‎exporter/importables.go‎

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -196,12 +196,29 @@ var resourcesMap map[string]importable = map[string]importable{
196196
},
197197
},
198198
"databricks_group_role": {
199-
Service: "access",
199+
Service: "access",
200+
AccountLevel: true,
200201
Depends: []reference{
201202
{Path: "group_id", Resource: "databricks_group"},
202203
{Path: "role", Resource: "databricks_instance_profile", Match: "instance_profile_arn"},
203204
},
204205
},
206+
"databricks_user_role": {
207+
Service: "access",
208+
AccountLevel: true,
209+
Depends: []reference{
210+
{Path: "user_id", Resource: "databricks_user"},
211+
{Path: "role", Resource: "databricks_instance_profile", Match: "instance_profile_arn"},
212+
},
213+
},
214+
"databricks_service_principal_role": {
215+
Service: "access",
216+
AccountLevel: true,
217+
Depends: []reference{
218+
{Path: "service_principal_id", Resource: "databricks_group"},
219+
{Path: "role", Resource: "databricks_instance_profile", Match: "instance_profile_arn"},
220+
},
221+
},
205222
"databricks_library": {
206223
Service: "compute",
207224
Depends: []reference{
@@ -531,7 +548,8 @@ var resourcesMap map[string]importable = map[string]importable{
531548
Body: resourceOrDataBlockBody,
532549
},
533550
"databricks_group": {
534-
Service: "groups",
551+
Service: "groups",
552+
AccountLevel: true,
535553
Name: func(ic *importContext, d *schema.ResourceData) string {
536554
return d.Get("display_name").(string) + "_" + d.Id()
537555
},
@@ -590,16 +608,7 @@ var resourcesMap map[string]importable = map[string]importable{
590608
if r.ID != g.ID {
591609
continue
592610
}
593-
for _, instanceProfile := range g.Roles {
594-
ic.Emit(&resource{
595-
Resource: "databricks_instance_profile",
596-
ID: instanceProfile.Value,
597-
})
598-
ic.Emit(&resource{
599-
Resource: "databricks_group_role",
600-
ID: fmt.Sprintf("%s|%s", g.ID, instanceProfile.Value),
601-
})
602-
}
611+
ic.emitRoles("group", g.ID, g.Roles)
603612
if g.DisplayName == "users" && !ic.importAllUsers {
604613
log.Printf("[INFO] Skipping import of entire user directory ...")
605614
continue
@@ -656,7 +665,8 @@ var resourcesMap map[string]importable = map[string]importable{
656665
Body: resourceOrDataBlockBody,
657666
},
658667
"databricks_group_member": {
659-
Service: "groups",
668+
Service: "groups",
669+
AccountLevel: true,
660670
Depends: []reference{
661671
{Path: "group_id", Resource: "databricks_group"},
662672
{Path: "member_id", Resource: "databricks_user"},
@@ -665,7 +675,8 @@ var resourcesMap map[string]importable = map[string]importable{
665675
},
666676
},
667677
"databricks_user": {
668-
Service: "users",
678+
Service: "users",
679+
AccountLevel: true,
669680
Name: func(ic *importContext, d *schema.ResourceData) string {
670681
s := d.Get("user_name").(string)
671682
// if CLI argument includeUserDomains is set then it includes domain portion as well
@@ -694,11 +705,13 @@ var resourcesMap map[string]importable = map[string]importable{
694705
userName = u.UserName
695706
}
696707
ic.emitGroups(u, userName)
708+
ic.emitRoles("user", u.ID, u.Roles)
697709
return nil
698710
},
699711
},
700712
"databricks_service_principal": {
701-
Service: "users",
713+
Service: "users",
714+
AccountLevel: true,
702715
Name: func(ic *importContext, d *schema.ResourceData) string {
703716
name := d.Get("display_name").(string)
704717
if name == "" {
@@ -738,6 +751,7 @@ var resourcesMap map[string]importable = map[string]importable{
738751
spnName = u.ApplicationID
739752
}
740753
ic.emitGroups(u, spnName)
754+
ic.emitRoles("service_principal", u.ID, u.Roles)
741755
return nil
742756
},
743757
},

‎exporter/importables_test.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ func TestGroup(t *testing.T) {
113113
Roles: []scim.ComplexValue{
114114
{
115115
Value: "abc",
116+
Type: "direct",
116117
},
117118
},
118119
Members: []scim.ComplexValue{
@@ -444,7 +445,7 @@ func TestUserSearchFails(t *testing.T) {
444445
{
445446
ReuseRequest: true,
446447
Method: "GET",
447-
Resource: "/api/2.0/preview/scim/v2/Users?excludedAttributes=roles&filter=userName%20eq%20%27dbc%27",
448+
Resource: "/api/2.0/preview/scim/v2/Users?filter=userName%20eq%20%27dbc%27",
448449
Status: 404,
449450
Response: apierr.NotFound("nope"),
450451
},
@@ -473,7 +474,7 @@ func TestSpnSearchFails(t *testing.T) {
473474
{
474475
ReuseRequest: true,
475476
Method: "GET",
476-
Resource: "/api/2.0/preview/scim/v2/ServicePrincipals?excludedAttributes=roles&filter=applicationId%20eq%20%27dbc%27",
477+
Resource: "/api/2.0/preview/scim/v2/ServicePrincipals?filter=applicationId%20eq%20%27dbc%27",
477478
Status: 404,
478479
Response: apierr.NotFound("nope"),
479480
},
@@ -502,7 +503,7 @@ func TestSpnSearchSuccess(t *testing.T) {
502503
{
503504
ReuseRequest: true,
504505
Method: "GET",
505-
Resource: "/api/2.0/preview/scim/v2/ServicePrincipals?excludedAttributes=roles&filter=applicationId%20eq%20%27dbc%27",
506+
Resource: "/api/2.0/preview/scim/v2/ServicePrincipals?filter=applicationId%20eq%20%27dbc%27",
506507
Response: scim.UserList{Resources: []scim.User{
507508
{ID: "321", DisplayName: "spn", ApplicationID: "dbc"},
508509
}},
@@ -556,7 +557,7 @@ func TestUserImportSkipNonDirectGroups(t *testing.T) {
556557
{
557558
ReuseRequest: true,
558559
Method: "GET",
559-
Resource: "/api/2.0/preview/scim/v2/Users?excludedAttributes=roles&filter=userName%20eq%20%27dbc%27",
560+
Resource: "/api/2.0/preview/scim/v2/Users?filter=userName%20eq%20%27dbc%27",
560561
Response: scim.UserList{
561562
Resources: []scim.User{
562563
{

‎exporter/model.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ type importable struct {
5454
ShouldOmitField func(ic *importContext, pathString string, as *schema.Schema, d *schema.ResourceData) bool
5555
// Defines which API version should be used for this specific resource
5656
ApiVersion common.ApiVersion
57+
// Defines if specific service is account level
58+
AccountLevel bool
5759
}
5860

5961
type MatchType string

‎exporter/util.go‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,25 @@ func (ic *importContext) emitGroups(u scim.User, principal string) {
166166
}
167167
}
168168

169+
func (ic *importContext) emitRoles(objType string, id string, roles []scim.ComplexValue) {
170+
log.Printf("[DEBUG] emitting roles for object type: %s, ID: %s, roles: %v", objType, id, roles)
171+
for _, role := range roles {
172+
if role.Type != "direct" {
173+
continue
174+
}
175+
if !ic.accountLevel {
176+
ic.Emit(&resource{
177+
Resource: "databricks_instance_profile",
178+
ID: role.Value,
179+
})
180+
}
181+
ic.Emit(&resource{
182+
Resource: fmt.Sprintf("databricks_%s_role", objType),
183+
ID: fmt.Sprintf("%s|%s", id, role.Value),
184+
})
185+
}
186+
}
187+
169188
func (ic *importContext) importLibraries(d *schema.ResourceData, s map[string]*schema.Schema) error {
170189
var cll libraries.ClusterLibraryList
171190
common.DataToStructPointer(d, s, &cll)
@@ -212,7 +231,7 @@ func (ic *importContext) cacheGroups() error {
212231

213232
func (ic *importContext) findUserByName(name string) (u scim.User, err error) {
214233
a := scim.NewUsersAPI(ic.Context, ic.Client)
215-
users, err := a.Filter(fmt.Sprintf("userName eq '%s'", name))
234+
users, err := a.Filter(fmt.Sprintf("userName eq '%s'", name), false)
216235
if err != nil {
217236
return
218237
}
@@ -226,7 +245,7 @@ func (ic *importContext) findUserByName(name string) (u scim.User, err error) {
226245

227246
func (ic *importContext) findSpnByAppID(applicationID string) (u scim.User, err error) {
228247
a := scim.NewServicePrincipalsAPI(ic.Context, ic.Client)
229-
users, err := a.Filter(fmt.Sprintf("applicationId eq '%s'", strings.ReplaceAll(applicationID, "'", "")))
248+
users, err := a.Filter(fmt.Sprintf("applicationId eq '%s'", strings.ReplaceAll(applicationID, "'", "")), false)
230249
if err != nil {
231250
return
232251
}

‎scim/data_service_principal.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ func DataSourceServicePrincipal() *schema.Resource {
2323
return common.DataResource(spnData{}, func(ctx context.Context, e any, c *common.DatabricksClient) error {
2424
response := e.(*spnData)
2525
spnAPI := NewServicePrincipalsAPI(ctx, c)
26-
spList, err := spnAPI.Filter(fmt.Sprintf("applicationId eq '%s'", response.ApplicationID))
26+
spList, err := spnAPI.Filter(fmt.Sprintf("applicationId eq '%s'", response.ApplicationID), true)
2727
if err != nil {
2828
return err
2929
}

‎scim/data_service_principals.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ func DataSourceServicePrincipals() *schema.Resource {
2424
if response.DisplayNameContains != "" {
2525
filter = fmt.Sprintf("displayName co '%s'", response.DisplayNameContains)
2626
}
27-
spList, err := spnAPI.Filter(filter)
27+
spList, err := spnAPI.Filter(filter, true)
2828
if err != nil {
2929
return err
3030
}

‎scim/data_user.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ func getUser(usersAPI UsersAPI, id, name string) (user User, err error) {
1313
if id != "" {
1414
return usersAPI.Read(id, "userName,displayName,externalId,applicationId")
1515
}
16-
userList, err := usersAPI.Filter(fmt.Sprintf("userName eq '%s'", name))
16+
userList, err := usersAPI.Filter(fmt.Sprintf("userName eq '%s'", name), true)
1717
if err != nil {
1818
return
1919
}

0 commit comments

Comments
 (0)