direct: Added retries for cluster update calls (#3837)

## Changes
Added retries for cluster update calls

## Why
Copying the logic from TF to make sure that when we issue an update from
direct, it waits for the cluster to reach an expected state first

https://github.com/databricks/terraform-provider-databricks/blob/main/clusters/resource_cluster.go#L624-L635

## Tests
Added an acceptance test

<!-- If your PR needs to be included in the release notes for next
release,
add a separate entry in NEXT_CHANGELOG.md as part of your PR. -->

Author: andrewnester

acceptance/bundle/resources/clusters/deploy/data_security_mode/out.direct-exp.txt

@@ -8,13 +8,5 @@ Deployment complete!
 >>> errcode [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
 Deploying resources...
-Error: cannot update resources.clusters.test_cluster: updating id=[CLUSTER-ID]: Cluster [CLUSTER-ID] is in unexpected state Pending. (400 INVALID_STATE)
-
-Endpoint: POST [DATABRICKS_URL]/api/2.1/clusters/edit
-HTTP Status: 400 Bad Request
-API error_code: INVALID_STATE
-API message: Cluster [CLUSTER-ID] is in unexpected state Pending.
-
 Updating deployment state...
-
-Exit code: 1
+Deployment complete!

acceptance/bundle/resources/clusters/deploy/data_security_mode/out.requests.direct-exp.txt

@@ -2,14 +2,11 @@ envsubst < databricks.yml.tmpl > databricks.yml
 
 cleanup() {
     trace $CLI bundle destroy --auto-approve
-    rm -f out.requests.txt
 }
 trap cleanup EXIT
 
 trace $CLI bundle deploy > out.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1
-print_requests.py --get //clusters > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt
 
 trace $CLI bundle plan >> out.plan.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1
 
 trace errcode $CLI bundle deploy >> out.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1
-print_requests.py --get //clusters > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt

acceptance/bundle/resources/clusters/deploy/data_security_mode/out.requests.terraform.txt

@@ -1,6 +1,6 @@
 Local = false
 Cloud = true
-RecordRequests = true
+RecordRequests = false
 
 Ignore = [
     "databricks.yml",

acceptance/bundle/resources/clusters/deploy/data_security_mode/script

@@ -0,0 +1,15 @@
+bundle:
+  name: test-deploy-cluster-update-after-create
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  clusters:
+    test_cluster:
+      cluster_name: test-cluster-$UNIQUE_NAME
+      spark_version: $DEFAULT_SPARK_VERSION
+      node_type_id: $NODE_TYPE_ID
+      num_workers: 2
+      spark_conf:
+        "spark.executor.memory": "2g"

acceptance/bundle/resources/clusters/deploy/data_security_mode/test.toml

@@ -0,0 +1 @@
+print("Hello World!")

acceptance/bundle/resources/clusters/deploy/update-after-create/databricks.yml.tmpl

@@ -0,0 +1,34 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Cluster should exist after bundle deployment:
+{
+  "cluster_name": "test-cluster-[UNIQUE_NAME]",
+  "num_workers": 2
+}
+
+=== Updating cluster should call update API
+
+>>> [CLI] bundle plan
+update clusters.test_cluster
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete cluster test_cluster
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/clusters/deploy/update-after-create/hello_world.py

@@ -0,0 +1,18 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+trace $CLI bundle deploy
+
+title "Cluster should exist after bundle deployment:\n"
+CLUSTER_ID=$($CLI bundle summary -o json | jq -r '.resources.clusters.test_cluster.id')
+$CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers}'
+
+title "Updating cluster should call update API\n"
+update_file.py databricks.yml "\"spark.executor.memory\": \"2g\"" "\"spark.executor.memory\": \"4g\""
+trace $CLI bundle plan
+trace $CLI bundle deploy