Skip to content

Commit 881b26e

Browse files
alexottalexott
authored
Switch to databricks_mount in the exporter (#1006)
1 parent 9af3fc8 commit 881b26e

File tree

3 files changed

+135
-170
lines changed

3 files changed

+135
-170
lines changed

‎exporter/exporter_test.go‎

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ func TestImportingMounts(t *testing.T) {
104104
Status: "Finished",
105105
Results: &common.CommandResults{
106106
ResultType: "text",
107-
Data: `{"foo": "s3a://foo", "bar": "abfss://[email protected]/thing", "third": "adls://foo.bar.com/path"}
107+
Data: `{"foo": "s3a://foo", "bar": "abfss://[email protected]/thing", "third": "adls://foo3.bar.com/path", "fourth":"wasbs://[email protected]/dir", "fifth":"gs://foo5", "sixth":"abc://foo5"}
108108
and some chatty messages`,
109109
},
110110
},
@@ -172,22 +172,14 @@ func TestImportingMounts(t *testing.T) {
172172
ic.listing = "mounts"
173173
ic.mounts = true
174174

175-
err := ic.Importables["databricks_aws_s3_mount"].List(ic)
175+
err := ic.Importables["databricks_mount"].List(ic)
176176
assert.NoError(t, err)
177177

178-
err = ic.Importables["databricks_azure_adls_gen2_mount"].List(ic)
179-
assert.NoError(t, err)
180-
err = ic.Importables["databricks_azure_adls_gen2_mount"].Body(ic,
181-
hclwrite.NewEmptyFile().Body(), ic.Scope[1])
182-
assert.NoError(t, err)
183-
184-
err = ic.Importables["databricks_azure_adls_gen1_mount"].List(ic)
185-
assert.NoError(t, err)
186-
err = ic.Importables["databricks_azure_adls_gen1_mount"].Body(ic,
187-
hclwrite.NewEmptyFile().Body(), ic.Scope[2])
188-
assert.NoError(t, err)
189-
190-
//Run("-listing", "mounts", "-mounts")
178+
for i := 0; i < len(ic.Scope); i++ {
179+
err = ic.Importables["databricks_mount"].Body(ic,
180+
hclwrite.NewEmptyFile().Body(), ic.Scope[i])
181+
assert.NoError(t, err)
182+
}
191183
})
192184
}
193185

‎exporter/importables.go‎

Lines changed: 126 additions & 153 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,101 @@ import (
2727
)
2828

2929
var (
30-
//adlsGen2Regex = regexp.MustCompile(`^((?:abfs|wasb)s?)://([^@]+)@([^.]+)\.(?:[^/]+)(/.*)?$`)
3130
adlsGen2Regex = regexp.MustCompile(`^(abfss?)://([^@]+)@([^.]+)\.(?:[^/]+)(/.*)?$`)
3231
adlsGen1Regex = regexp.MustCompile(`^(adls?)://([^.]+)\.(?:[^/]+)(/.*)?$`)
32+
wasbsRegex = regexp.MustCompile(`^(wasbs?)://([^@]+)@([^.]+)\.(?:[^/]+)(/.*)?$`)
33+
s3Regex = regexp.MustCompile(`^(s3a?)://([^/]+)(/.*)?$`)
34+
gsRegex = regexp.MustCompile(`^gs://([^/]+)(/.*)?$`)
3335
)
3436

37+
func generateMountBody(ic *importContext, body *hclwrite.Body, r *resource) error {
38+
mount := ic.mountMap[r.ID]
39+
40+
b := body.AppendNewBlock("resource", []string{r.Resource, r.Name}).Body()
41+
b.SetAttributeValue("name", cty.StringVal(strings.Replace(r.ID, "/mnt/", "", 1)))
42+
if res := s3Regex.FindStringSubmatch(mount.URL); res != nil {
43+
block := b.AppendNewBlock("s3", nil).Body()
44+
block.SetAttributeValue("bucket_name", cty.StringVal(res[2]))
45+
if mount.InstanceProfile != "" {
46+
block.SetAttributeValue("instance_profile", cty.StringVal(mount.InstanceProfile))
47+
} else if mount.ClusterID != "" {
48+
b.SetAttributeValue("cluster_id", cty.StringVal(mount.ClusterID))
49+
}
50+
} else if res := gsRegex.FindStringSubmatch(mount.URL); res != nil {
51+
block := b.AppendNewBlock("gs", nil).Body()
52+
block.SetAttributeValue("bucket_name", cty.StringVal(res[1]))
53+
if mount.ClusterID != "" {
54+
b.SetAttributeValue("cluster_id", cty.StringVal(mount.ClusterID))
55+
}
56+
} else if res := adlsGen2Regex.FindStringSubmatch(mount.URL); res != nil {
57+
containerName := res[2]
58+
storageAccountName := res[3]
59+
block := b.AppendNewBlock("abfs", nil).Body()
60+
block.SetAttributeValue("container_name", cty.StringVal(containerName))
61+
block.SetAttributeValue("storage_account_name", cty.StringVal(storageAccountName))
62+
if res[4] != "" && res[4] != "/" {
63+
block.SetAttributeValue("directory", cty.StringVal(res[4]))
64+
}
65+
66+
varName := ic.regexFix("_"+storageAccountName+"_"+containerName+"_abfs", ic.nameFixes)
67+
textStr := fmt.Sprintf(" for mounting ADLSv2 resource %s://%s@%s",
68+
res[1], containerName, storageAccountName)
69+
70+
block.SetAttributeRaw("client_id", ic.variable(
71+
"client_id"+varName, "Client ID"+textStr))
72+
block.SetAttributeRaw("tenant_id", ic.variable(
73+
"tenant_id"+varName, "Tenant ID"+textStr))
74+
block.SetAttributeRaw("client_secret_scope", ic.variable(
75+
"client_secret_scope"+varName,
76+
"Secret scope name that stores app client secret"+textStr))
77+
block.SetAttributeRaw("client_secret_key", ic.variable(
78+
"client_secret_key"+varName,
79+
"Key in secret scope that stores app client secret"+textStr))
80+
} else if res := adlsGen1Regex.FindStringSubmatch(mount.URL); res != nil {
81+
block := b.AppendNewBlock("adl", nil).Body()
82+
storageResourceName := res[2]
83+
block.SetAttributeValue("storage_resource_name", cty.StringVal(storageResourceName))
84+
if res[3] != "" && res[3] != "/" {
85+
block.SetAttributeValue("directory", cty.StringVal(res[3]))
86+
}
87+
varName := ic.regexFix("_"+storageResourceName+"_adl", ic.nameFixes)
88+
textStr := fmt.Sprintf(" for mounting ADLSv1 resource %s://%s", res[1], storageResourceName)
89+
90+
block.SetAttributeRaw("client_id", ic.variable("client_id"+varName, "Client ID"+textStr))
91+
block.SetAttributeRaw("tenant_id", ic.variable("tenant_id"+varName, "Tenant IDs"+textStr))
92+
block.SetAttributeRaw("client_secret_scope", ic.variable(
93+
"client_secret_scope"+varName, "Secret scope name that stores app client secret"+textStr))
94+
block.SetAttributeRaw("client_secret_key", ic.variable(
95+
"client_secret_key"+varName, "Key in secret scope that stores app client secret"+textStr))
96+
} else if res := wasbsRegex.FindStringSubmatch(mount.URL); res != nil {
97+
containerName := res[2]
98+
storageAccountName := res[3]
99+
block := b.AppendNewBlock("wasb", nil).Body()
100+
block.SetAttributeValue("container_name", cty.StringVal(containerName))
101+
block.SetAttributeValue("storage_account_name", cty.StringVal(storageAccountName))
102+
if res[4] != "" && res[4] != "/" {
103+
block.SetAttributeValue("directory", cty.StringVal(res[4]))
104+
}
105+
block.SetAttributeValue("auth_type", cty.StringVal("ACCESS_KEY"))
106+
107+
varName := ic.regexFix("_"+storageAccountName+"_"+containerName+"_wasb", ic.nameFixes)
108+
textStr := fmt.Sprintf(" for mounting WASB resource %s://%s@%s",
109+
res[1], containerName, storageAccountName)
110+
111+
block.SetAttributeRaw("token_secret_scope", ic.variable(
112+
"client_secret_scope"+varName,
113+
"Secret scope name that stores app client secret"+textStr))
114+
block.SetAttributeRaw("token_secret_key", ic.variable(
115+
"client_secret_key"+varName,
116+
"Key in secret scope that stores app client secret"+textStr))
117+
} else {
118+
return fmt.Errorf("no matching handler for: %s", mount.URL)
119+
}
120+
body.AppendNewline()
121+
122+
return nil
123+
}
124+
35125
var resourcesMap map[string]importable = map[string]importable{
36126
"databricks_dbfs_file": {
37127
Service: "storage",
@@ -560,8 +650,9 @@ var resourcesMap map[string]importable = map[string]importable{
560650
{Path: "principal", Resource: "databricks_user", Match: "user_name"},
561651
},
562652
},
563-
"databricks_aws_s3_mount": {
653+
"databricks_mount": {
564654
Service: "mounts",
655+
Body: generateMountBody,
565656
List: func(ic *importContext) error {
566657
if !ic.mounts {
567658
return nil
@@ -570,174 +661,56 @@ var resourcesMap map[string]importable = map[string]importable{
570661
return err
571662
}
572663
for mountName, source := range ic.mountMap {
573-
if !strings.HasPrefix(source.URL, "s3a://") {
574-
continue
575-
}
576664
if !ic.MatchesName(mountName) {
577665
continue
578666
}
579-
log.Printf("[INFO] Emitting databricks_aws_s3_mount: %s",
580-
source.URL)
581-
attrs := map[string]string{
582-
"s3_bucket_name": strings.ReplaceAll(source.URL, "s3a://", ""),
583-
"mount_name": mountName,
584-
}
585-
if source.InstanceProfile != "" {
586-
attrs["instance_profile"] = source.InstanceProfile
587-
ic.Emit(&resource{
588-
Resource: "databricks_instance_profile",
589-
ID: source.InstanceProfile,
590-
})
591-
} else if source.ClusterID != "" {
592-
attrs["cluster_id"] = source.ClusterID
593-
ic.Emit(&resource{
594-
Resource: "databricks_cluster",
595-
ID: source.ClusterID,
596-
})
667+
if strings.HasPrefix(source.URL, "s3a://") {
668+
log.Printf("[INFO] Emitting databricks_mount: %s", source.URL)
669+
if source.InstanceProfile != "" {
670+
ic.Emit(&resource{
671+
Resource: "databricks_instance_profile",
672+
ID: source.InstanceProfile,
673+
})
674+
} else if source.ClusterID != "" {
675+
ic.Emit(&resource{
676+
Resource: "databricks_cluster",
677+
ID: source.ClusterID,
678+
})
679+
}
680+
} else if strings.HasPrefix(source.URL, "gs://") {
681+
if source.ClusterID != "" {
682+
ic.Emit(&resource{
683+
Resource: "databricks_cluster",
684+
ID: source.ClusterID,
685+
})
686+
}
687+
} else if res := adlsGen2Regex.FindStringSubmatch(source.URL); res != nil {
688+
} else if res := adlsGen1Regex.FindStringSubmatch(source.URL); res != nil {
689+
} else if res := wasbsRegex.FindStringSubmatch(source.URL); res != nil {
690+
} else {
691+
log.Printf("[INFO] No matching handler for: %s", source.URL)
692+
continue
597693
}
694+
log.Printf("[INFO] Emitting databricks_mount: %s", source.URL)
598695
ic.Emit(&resource{
696+
Resource: "databricks_mount",
599697
ID: mountName,
600-
Resource: "databricks_aws_s3_mount",
601-
Data: ic.Resources["databricks_aws_s3_mount"].Data(
698+
Data: ic.Resources["databricks_mount"].Data(
602699
&terraform.InstanceState{
603700
ID: mountName,
604-
Attributes: attrs,
701+
Attributes: map[string]string{},
605702
}),
606703
})
704+
607705
}
608706
return nil
609707
},
610708
Depends: []reference{
611-
{Path: "s3_bucket_name", Resource: "aws_s3_bucket", Match: "bucket"},
709+
{Path: "s3_bucket_name", Resource: "aws_s3_bucket", Match: "bucket"}, // this should be changed somehow & avoid clashes with GCS bucket_name
612710
{Path: "instance_profile", Resource: "databricks_instance_profile"},
613711
{Path: "cluster_id", Resource: "databricks_cluster"},
614-
},
615-
},
616-
"databricks_azure_adls_gen2_mount": {
617-
Service: "mounts",
618-
List: func(ic *importContext) error {
619-
if !ic.mounts {
620-
return nil
621-
}
622-
if err := ic.refreshMounts(); err != nil {
623-
return err
624-
}
625-
for mountName, source := range ic.mountMap {
626-
if res := adlsGen2Regex.FindStringSubmatch(source.URL); res == nil {
627-
log.Printf("[DEBUG] skipping %s mounted at %s", source, mountName)
628-
continue
629-
}
630-
if !ic.MatchesName(mountName) {
631-
continue
632-
}
633-
ic.Emit(&resource{
634-
Resource: "databricks_azure_adls_gen2_mount",
635-
ID: mountName,
636-
Data: ic.Resources["databricks_azure_adls_gen2_mount"].Data(
637-
&terraform.InstanceState{
638-
ID: mountName,
639-
// don't open another command/context
640-
Attributes: map[string]string{},
641-
}),
642-
})
643-
}
644-
return nil
645-
},
646-
Body: func(ic *importContext, body *hclwrite.Body, r *resource) error {
647-
b := body.AppendNewBlock("resource", []string{r.Resource, r.Name}).Body()
648-
649-
mount := ic.mountMap[r.ID]
650-
res := adlsGen2Regex.FindStringSubmatch(mount.URL)
651-
if res == nil {
652-
return fmt.Errorf("can't extract ADLSv2 information from string '%s'", mount)
653-
}
654-
containerName := res[2]
655-
storageAccountName := res[3]
656-
b.SetAttributeValue("container_name", cty.StringVal(containerName))
657-
b.SetAttributeValue("storage_account_name", cty.StringVal(storageAccountName))
658-
if res[4] != "" && res[4] != "/" {
659-
b.SetAttributeValue("directory", cty.StringVal(res[4]))
660-
}
661-
b.SetAttributeValue("mount_name", cty.StringVal(strings.Replace(r.ID, "/mnt/", "", 1)))
662-
663-
varName := "_" + storageAccountName + "_" + containerName
664-
textStr := fmt.Sprintf(" for mounting ADLSv2 resource %s://%s@%s",
665-
res[1], containerName, storageAccountName)
666-
667-
b.SetAttributeRaw("client_id", ic.variable(
668-
"client_id"+varName, "Client ID"+textStr))
669-
b.SetAttributeRaw("tenant_id", ic.variable(
670-
"tenant_id"+varName, "Tenant ID"+textStr))
671-
b.SetAttributeRaw("client_secret_scope", ic.variable(
672-
"client_secret_scope"+varName,
673-
"Secret scope name that stores app client secret"+textStr))
674-
b.SetAttributeRaw("client_secret_key", ic.variable(
675-
"client_secret_key"+varName,
676-
"Key in secret scope that stores app client secret"+textStr))
677-
678-
return nil
679-
},
680-
Depends: []reference{
681-
{Path: "storage_account_name", Resource: "azurerm_storage_account", Match: "name"},
712+
{Path: "storage_account_name", Resource: "azurerm_storage_account", Match: "name"}, // similarly for WASBS vs ABFSS
682713
{Path: "container_name", Resource: "azurerm_storage_container", Match: "name"},
683-
},
684-
},
685-
"databricks_azure_adls_gen1_mount": {
686-
Service: "mounts",
687-
List: func(ic *importContext) error {
688-
if !ic.mounts {
689-
return nil
690-
}
691-
if err := ic.refreshMounts(); err != nil {
692-
return err
693-
}
694-
for mountName, source := range ic.mountMap {
695-
if res := adlsGen1Regex.FindStringSubmatch(source.URL); res == nil {
696-
continue
697-
}
698-
if !ic.MatchesName(mountName) {
699-
continue
700-
}
701-
ic.Emit(&resource{
702-
Resource: "databricks_azure_adls_gen1_mount",
703-
ID: mountName,
704-
Data: ic.Resources["databricks_azure_adls_gen2_mount"].Data(
705-
&terraform.InstanceState{
706-
ID: mountName,
707-
// don't open another command/context
708-
Attributes: map[string]string{},
709-
}),
710-
})
711-
}
712-
return nil
713-
},
714-
Body: func(ic *importContext, body *hclwrite.Body, r *resource) error {
715-
b := body.AppendNewBlock("resource", []string{r.Resource, r.Name}).Body()
716-
717-
mount := ic.mountMap[r.ID]
718-
res := adlsGen1Regex.FindStringSubmatch(mount.URL)
719-
if res == nil {
720-
return fmt.Errorf("can't extract ADLSv1 information from string '%s'", mount)
721-
}
722-
storageResourceName := res[2]
723-
b.SetAttributeValue("storage_resource_name", cty.StringVal(storageResourceName))
724-
if res[3] != "" && res[3] != "/" {
725-
b.SetAttributeValue("directory", cty.StringVal(res[3]))
726-
}
727-
b.SetAttributeValue("mount_name", cty.StringVal(strings.Replace(r.ID, "/mnt/", "", 1)))
728-
varName := "_" + storageResourceName
729-
textStr := fmt.Sprintf(" for mounting ADLSv1 resource %s://%s", res[1], storageResourceName)
730-
731-
b.SetAttributeRaw("client_id", ic.variable("client_id"+varName, "Client ID"+textStr))
732-
b.SetAttributeRaw("tenant_id", ic.variable("tenant_id"+varName, "Tenant IDs"+textStr))
733-
b.SetAttributeRaw("client_secret_scope", ic.variable(
734-
"client_secret_scope"+varName, "Secret scope name that stores app client secret"+textStr))
735-
b.SetAttributeRaw("client_secret_key", ic.variable(
736-
"client_secret_key"+varName, "Key in secret scope that stores app client secret"+textStr))
737-
738-
return nil
739-
},
740-
Depends: []reference{
741714
{Path: "storage_resource_name", Resource: "azurerm_data_lake_store", Match: "name"},
742715
},
743716
},

‎exporter/importables_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -519,11 +519,11 @@ func TestAwsS3MountProfile(t *testing.T) {
519519
URL: "s3a://def",
520520
InstanceProfile: "bcd",
521521
}
522-
err := resourcesMap["databricks_aws_s3_mount"].List(ic)
522+
err := resourcesMap["databricks_mount"].List(ic)
523523
assert.NoError(t, err)
524524
assert.Len(t, ic.testEmits, 2)
525525
assert.True(t, ic.testEmits["databricks_instance_profile[<unknown>] (id: bcd)"])
526-
assert.True(t, ic.testEmits["databricks_aws_s3_mount[<unknown>] (id: /mnt/abc)"])
526+
assert.True(t, ic.testEmits["databricks_mount[<unknown>] (id: /mnt/abc)"])
527527
}
528528

529529
func TestGlobalInitScriptNameFromId(t *testing.T) {

0 commit comments

Comments
 (0)