[Fix] databricks_mount: remove redundant listing of mounted directories (#4590)

Directory listings can take very long when there are lots of files in
the mounted directory. This slows down both the state refresh and the
actual mounting up to the point that the operation times out.
Fixes
[4509](#4509)

## Changes
<!-- Summary of your changes that are easy to understand -->
Remove directory listing of mounted directories.


## Tests
<!-- 
How is this tested? Please see the checklist below and also describe any
other relevant tests
-->

- [x] `make test` run locally
- [ ] relevant change in `docs/` folder
- [ ] covered with integration tests in `internal/acceptance`
- [ ] using Go SDK
- [ ] using TF Plugin Framework

---------

Co-authored-by: Alex Ott <[email protected]>

Author: KoenDeCausmaecker-TomTom

NEXT_CHANGELOG.md

@@ -9,6 +9,7 @@
 ### Bug Fixes
 
  * Recreate `databricks_access_control_rule_set` when the `name` changes ([#4572](https://github.com/databricks/terraform-provider-databricks/pull/4572)).
+ * Avoid timeouts during `databricks_mount` state refresh and creation ([#4590](https://github.com/databricks/terraform-provider-databricks/pull/4590)).
 
 ### Documentation
 

storage/mounts.go

@@ -79,14 +79,19 @@ func (mp MountPoint) Mount(mo Mount, client *common.DatabricksClient) (source st
 	sparkConfRe := regexp.MustCompile(`"\{\{sparkconf/([^\}]+)\}\}"`)
 	extraConfigs = sparkConfRe.ReplaceAll(extraConfigs, []byte(`spark.conf.get("$1")`))
 	command := fmt.Sprintf(`
+		def check_path(path_string):
+			fs = sc._jvm.org.apache.hadoop.fs.FileSystem.get(sc._jsc.hadoopConfiguration())
+			path = spark.sparkContext._jvm.org.apache.hadoop.fs.Path(f"dbfs:{path_string}")
+			fs.exists(path)
+
 		def safe_mount(mount_point, mount_source, configs, encryptionType):
 			for mount in dbutils.fs.mounts():
 				if mount.mountPoint == mount_point and mount.source == mount_source:
 					return
 			try:
 				dbutils.fs.mount(mount_source, mount_point, extra_configs=configs, encryption_type=encryptionType)
 				dbutils.fs.refreshMounts()
-				dbutils.fs.ls(mount_point)
+				check_path(mount_point)
 				return mount_source
 			except Exception as e:
 				try:

storage/mounts_test.go

@@ -97,14 +97,19 @@ func TestMountPoint_Mount(t *testing.T) {
 	expectedMountConfig := `{"fake-key":"fake-value"}`
 	mountName := "this_mount"
 	expectedCommand := fmt.Sprintf(`
+		def check_path(path_string):
+			fs = sc._jvm.org.apache.hadoop.fs.FileSystem.get(sc._jsc.hadoopConfiguration())
+			path = spark.sparkContext._jvm.org.apache.hadoop.fs.Path(f"dbfs:{path_string}")
+			fs.exists(path)
+
 		def safe_mount(mount_point, mount_source, configs, encryptionType):
 			for mount in dbutils.fs.mounts():
 				if mount.mountPoint == mount_point and mount.source == mount_source:
 					return
 			try:
 				dbutils.fs.mount(mount_source, mount_point, extra_configs=configs, encryption_type=encryptionType)
 				dbutils.fs.refreshMounts()
-				dbutils.fs.ls(mount_point)
+				check_path(mount_point)
 				return mount_source
 			except Exception as e:
 				try: