Skip to content

Commit 1ca9c22

Browse files
stikkireddystikkireddy
authored
Merge pull request #106 from storey247/ds/fixmountdirectoryerror
Fix issue where file system mount to invalid directory caused plan to fail
2 parents 8b42563 + cf84787 commit 1ca9c22

9 files changed

+288
-50
lines changed

‎client/service/commands.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,11 @@ type CommandsAPI struct {
1616
Client *DBApiClient
1717
}
1818

19+
// CommandExecutor creates a spark context and executes a command and then closes context
20+
type CommandExecutor interface {
21+
Execute(clusterID, language, commandStr string) (model.Command, error)
22+
}
23+
1924
// Execute creates a spark context and executes a command and then closes context
2025
func (a CommandsAPI) Execute(clusterID, language, commandStr string) (model.Command, error) {
2126
var resp model.Command

‎client/service/commands_test.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ import (
77
"github.com/databrickslabs/databricks-terraform/client/model"
88
)
99

10+
// Test interface compliance
11+
var _ CommandExecutor = (*CommandsAPI)(nil)
12+
1013
func TestCommandsAPI_Execute(t *testing.T) {
1114
type context struct {
1215
Language string `json:"language,omitempty"`

‎databricks/mounts.go‎

Lines changed: 33 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,13 @@ func NewAWSIamMount(s3BucketName string, mountName string) *AWSIamMount {
3030
}
3131

3232
// Create creates an aws iam mount given a cluster ID
33-
func (m AWSIamMount) Create(client *service.DBApiClient, clusterID string) error {
33+
func (m AWSIamMount) Create(exec service.CommandExecutor, clusterID string) error {
3434
iamMountCommand := fmt.Sprintf(`
3535
dbutils.fs.mount("s3a://%s", "/mnt/%s")
3636
dbutils.fs.ls("/mnt/%s")
3737
dbutils.notebook.exit("success")
3838
`, m.S3BucketName, m.MountName, m.MountName)
39-
resp, err := client.Commands().Execute(clusterID, "python", iamMountCommand)
39+
resp, err := exec.Execute(clusterID, "python", iamMountCommand)
4040
if err != nil {
4141
return err
4242
}
@@ -48,13 +48,13 @@ dbutils.notebook.exit("success")
4848
}
4949

5050
// Delete deletes an aws iam mount given a cluster ID
51-
func (m AWSIamMount) Delete(client *service.DBApiClient, clusterID string) error {
51+
func (m AWSIamMount) Delete(exec service.CommandExecutor, clusterID string) error {
5252
iamMountCommand := fmt.Sprintf(`
5353
dbutils.fs.unmount("/mnt/%s")
5454
dbutils.fs.refreshMounts()
5555
dbutils.notebook.exit("success")
5656
`, m.MountName)
57-
resp, err := client.Commands().Execute(clusterID, "python", iamMountCommand)
57+
resp, err := exec.Execute(clusterID, "python", iamMountCommand)
5858
if err != nil {
5959
return err
6060
}
@@ -66,14 +66,14 @@ dbutils.notebook.exit("success")
6666
}
6767

6868
// Read verifies an aws iam mount given a cluster ID
69-
func (m AWSIamMount) Read(client *service.DBApiClient, clusterID string) (string, error) {
69+
func (m AWSIamMount) Read(exec service.CommandExecutor, clusterID string) (string, error) {
7070
iamMountCommand := fmt.Sprintf(`
7171
dbutils.fs.ls("/mnt/%s")
7272
for mount in dbutils.fs.mounts():
7373
if mount.mountPoint == "/mnt/%s":
7474
dbutils.notebook.exit(mount.source)
7575
`, m.MountName, m.MountName)
76-
resp, err := client.Commands().Execute(clusterID, "python", iamMountCommand)
76+
resp, err := exec.Execute(clusterID, "python", iamMountCommand)
7777
if err != nil {
7878
return "", err
7979
}
@@ -109,7 +109,7 @@ func NewAzureBlobMount(containerName string, storageAccountName string, director
109109
}
110110

111111
// Create creates a azure blob storage mount given a cluster id
112-
func (m AzureBlobMount) Create(client *service.DBApiClient, clusterID string) error {
112+
func (m AzureBlobMount) Create(exec service.CommandExecutor, clusterID string) error {
113113
var confKey string
114114

115115
if m.AuthType == "SAS" {
@@ -133,7 +133,7 @@ except Exception as e:
133133
raise e
134134
dbutils.notebook.exit("success")
135135
`, m.ContainerName, m.StorageAccountName, m.Directory, m.MountName, confKey, m.SecretScope, m.SecretKey)
136-
resp, err := client.Commands().Execute(clusterID, "python", iamMountCommand)
136+
resp, err := exec.Execute(clusterID, "python", iamMountCommand)
137137
if err != nil {
138138
return err
139139
}
@@ -145,13 +145,13 @@ dbutils.notebook.exit("success")
145145
}
146146

147147
// Delete deletes a azure blob storage mount given a cluster id
148-
func (m AzureBlobMount) Delete(client *service.DBApiClient, clusterID string) error {
148+
func (m AzureBlobMount) Delete(exec service.CommandExecutor, clusterID string) error {
149149
iamMountCommand := fmt.Sprintf(`
150150
dbutils.fs.unmount("/mnt/%s")
151151
dbutils.fs.refreshMounts()
152152
dbutils.notebook.exit("success")
153153
`, m.MountName)
154-
resp, err := client.Commands().Execute(clusterID, "python", iamMountCommand)
154+
resp, err := exec.Execute(clusterID, "python", iamMountCommand)
155155
if err != nil {
156156
return err
157157
}
@@ -163,14 +163,13 @@ dbutils.notebook.exit("success")
163163
}
164164

165165
// Read verifies a azure blob storage mount given a cluster id
166-
func (m AzureBlobMount) Read(client *service.DBApiClient, clusterID string) (string, error) {
166+
func (m AzureBlobMount) Read(exec service.CommandExecutor, clusterID string) (string, error) {
167167
iamMountCommand := fmt.Sprintf(`
168-
dbutils.fs.ls("/mnt/%s")
169168
for mount in dbutils.fs.mounts():
170-
if mount.mountPoint == "/mnt/%s":
171-
dbutils.notebook.exit(mount.source)
172-
`, m.MountName, m.MountName)
173-
resp, err := client.Commands().Execute(clusterID, "python", iamMountCommand)
169+
if mount.mountPoint == "/mnt/%s":
170+
dbutils.notebook.exit(mount.source)
171+
`, m.MountName)
172+
resp, err := exec.Execute(clusterID, "python", iamMountCommand)
174173
if err != nil {
175174
return "", err
176175
}
@@ -214,7 +213,7 @@ func NewAzureADLSGen1Mount(storageResource string, directory string, mountName s
214213
}
215214

216215
// Create creates a azure datalake gen 1 storage mount given a cluster id
217-
func (m AzureADLSGen1Mount) Create(client *service.DBApiClient, clusterID string) error {
216+
func (m AzureADLSGen1Mount) Create(exec service.CommandExecutor, clusterID string) error {
218217
iamMountCommand := fmt.Sprintf(`
219218
for mount in dbutils.fs.mounts():
220219
if mount.mountPoint == "/mnt/%[8]s" and mount.source=="adl://%[6]s.azuredatalakestore.net%[7]s":
@@ -235,7 +234,7 @@ except Exception as e:
235234
raise e
236235
dbutils.notebook.exit("success")
237236
`, m.PrefixType, m.ClientID, m.SecretScope, m.SecretKey, m.TenantID, m.StorageResource, m.Directory, m.MountName, m.MountName)
238-
resp, err := client.Commands().Execute(clusterID, "python", iamMountCommand)
237+
resp, err := exec.Execute(clusterID, "python", iamMountCommand)
239238
if err != nil {
240239
return err
241240
}
@@ -247,13 +246,13 @@ dbutils.notebook.exit("success")
247246
}
248247

249248
// Delete deletes a azure datalake gen 1 storage mount given a cluster id
250-
func (m AzureADLSGen1Mount) Delete(client *service.DBApiClient, clusterID string) error {
249+
func (m AzureADLSGen1Mount) Delete(exec service.CommandExecutor, clusterID string) error {
251250
iamMountCommand := fmt.Sprintf(`
252251
dbutils.fs.unmount("/mnt/%s")
253252
dbutils.fs.refreshMounts()
254253
dbutils.notebook.exit("success")
255254
`, m.MountName)
256-
resp, err := client.Commands().Execute(clusterID, "python", iamMountCommand)
255+
resp, err := exec.Execute(clusterID, "python", iamMountCommand)
257256
if err != nil {
258257
return err
259258
}
@@ -265,14 +264,13 @@ dbutils.notebook.exit("success")
265264
}
266265

267266
// Read verifies the azure datalake gen 1 storage mount given a cluster id
268-
func (m AzureADLSGen1Mount) Read(client *service.DBApiClient, clusterID string) (string, error) {
267+
func (m AzureADLSGen1Mount) Read(exec service.CommandExecutor, clusterID string) (string, error) {
269268
iamMountCommand := fmt.Sprintf(`
270-
dbutils.fs.ls("/mnt/%s")
271269
for mount in dbutils.fs.mounts():
272-
if mount.mountPoint == "/mnt/%s":
273-
dbutils.notebook.exit(mount.source)
274-
`, m.MountName, m.MountName)
275-
resp, err := client.Commands().Execute(clusterID, "python", iamMountCommand)
270+
if mount.mountPoint == "/mnt/%s":
271+
dbutils.notebook.exit(mount.source)
272+
`, m.MountName)
273+
resp, err := exec.Execute(clusterID, "python", iamMountCommand)
276274
if err != nil {
277275
return "", err
278276
}
@@ -316,7 +314,7 @@ func NewAzureADLSGen2Mount(containerName string, storageAccountName string, dire
316314
}
317315

318316
// Create creates a azure datalake gen 2 storage mount
319-
func (m AzureADLSGen2Mount) Create(client *service.DBApiClient, clusterID string) error {
317+
func (m AzureADLSGen2Mount) Create(exec service.CommandExecutor, clusterID string) error {
320318
iamMountCommand := fmt.Sprintf(`
321319
for mount in dbutils.fs.mounts():
322320
if mount.mountPoint == "/mnt/%[9]s" and mount.source=="abfss://%[6]s@%[7]s.dfs.core.windows.net%[8]s":
@@ -342,7 +340,7 @@ except Exception as e:
342340
raise e
343341
dbutils.notebook.exit("success")
344342
`, m.ClientID, m.SecretScope, m.SecretKey, m.TenantID, m.InitializeFileSystem, m.ContainerName, m.StorageAccountName, m.Directory, m.MountName)
345-
resp, err := client.Commands().Execute(clusterID, "python", iamMountCommand)
343+
resp, err := exec.Execute(clusterID, "python", iamMountCommand)
346344
if err != nil {
347345
return err
348346
}
@@ -354,13 +352,13 @@ dbutils.notebook.exit("success")
354352
}
355353

356354
// Delete deletes a azure datalake gen 2 storage mount
357-
func (m AzureADLSGen2Mount) Delete(client *service.DBApiClient, clusterID string) error {
355+
func (m AzureADLSGen2Mount) Delete(exec service.CommandExecutor, clusterID string) error {
358356
iamMountCommand := fmt.Sprintf(`
359357
dbutils.fs.unmount("/mnt/%s")
360358
dbutils.fs.refreshMounts()
361359
dbutils.notebook.exit("success")
362360
`, m.MountName)
363-
resp, err := client.Commands().Execute(clusterID, "python", iamMountCommand)
361+
resp, err := exec.Execute(clusterID, "python", iamMountCommand)
364362
if err != nil {
365363
return err
366364
}
@@ -372,14 +370,13 @@ dbutils.notebook.exit("success")
372370
}
373371

374372
// Read verifies the azure datalake gen 2 storage mount
375-
func (m AzureADLSGen2Mount) Read(client *service.DBApiClient, clusterID string) (string, error) {
373+
func (m AzureADLSGen2Mount) Read(exec service.CommandExecutor, clusterID string) (string, error) {
376374
iamMountCommand := fmt.Sprintf(`
377-
dbutils.fs.ls("/mnt/%s")
378375
for mount in dbutils.fs.mounts():
379-
if mount.mountPoint == "/mnt/%s":
380-
dbutils.notebook.exit(mount.source)
381-
`, m.MountName, m.MountName)
382-
resp, err := client.Commands().Execute(clusterID, "python", iamMountCommand)
376+
if mount.mountPoint == "/mnt/%s":
377+
dbutils.notebook.exit(mount.source)
378+
`, m.MountName)
379+
resp, err := exec.Execute(clusterID, "python", iamMountCommand)
383380
if err != nil {
384381
return "", err
385382
}

0 commit comments

Comments
 (0)