direct: Added support for clusters deployment

acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/test.toml

@@ -1,5 +1,3 @@
-EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"]  # clusters not implemented
-
 [EnvMatrix]
 DATA_SECURITY_MODE = [
     "USER_ISOLATION",

acceptance/bundle/override/clusters/output.txt

@@ -5,6 +5,7 @@
     "max_workers": 7,
     "min_workers": 2
   },
+  "autotermination_minutes": 60,
   "cluster_name": "foo",
   "node_type_id": "[NODE_TYPE_ID]",
   "num_workers": 2,
@@ -20,6 +21,7 @@
     "max_workers": 3,
     "min_workers": 1
   },
+  "autotermination_minutes": 60,
   "cluster_name": "foo-override",
   "node_type_id": "m5.xlarge",
   "num_workers": 3,

acceptance/bundle/override/merge-string-map/output.txt

@@ -7,6 +7,7 @@ Warning: expected map, found string
 {
   "clusters": {
     "my_cluster": {
+      "autotermination_minutes": 60,
       "spark_version": "25"
     }
   }

acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/databricks.yml.tmpl

@@ -0,0 +1,13 @@
+bundle:
+  name: test-deploy-cluster-autoscale
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  clusters:
+    test_cluster:
+      cluster_name: test-cluster-$UNIQUE_NAME
+      spark_version: $DEFAULT_SPARK_VERSION
+      node_type_id: $NODE_TYPE_ID
+      num_workers: 2

acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/hello_world.py

@@ -0,0 +1 @@
+print("Hello World!")

acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct-exp.json

@@ -0,0 +1,172 @@
+{
+  "plan": {
+    "resources.clusters.test_cluster": {
+      "action": "create",
+      "new_state": {
+        "config": {
+          "autotermination_minutes": 60,
+          "cluster_name": "test-cluster-[UNIQUE_NAME]",
+          "node_type_id": "[NODE_TYPE_ID]",
+          "num_workers": 2,
+          "spark_version": "13.3.x-snapshot-scala2.12"
+        }
+      }
+    }
+  }
+}
+{
+  "plan": {
+    "resources.clusters.test_cluster": {
+      "action": "update",
+      "new_state": {
+        "config": {
+          "autoscale": {
+            "max_workers": 4,
+            "min_workers": 2
+          },
+          "autotermination_minutes": 60,
+          "cluster_name": "test-cluster-[UNIQUE_NAME]",
+          "node_type_id": "[NODE_TYPE_ID]",
+          "spark_version": "13.3.x-snapshot-scala2.12"
+        }
+      },
+      "remote_state": {
+        "autotermination_minutes": 60,
+        "cluster_id": "[CLUSTER_ID]",
+        "cluster_name": "test-cluster-[UNIQUE_NAME]",
+        "node_type_id": "[NODE_TYPE_ID]",
+        "num_workers": 2,
+        "spark_version": "13.3.x-snapshot-scala2.12"
+      },
+      "changes": {
+        "local": {
+          "autoscale": {
+            "action": "update"
+          },
+          "num_workers": {
+            "action": "update"
+          }
+        }
+      }
+    }
+  }
+}
+{
+  "plan": {
+    "resources.clusters.test_cluster": {
+      "action": "update",
+      "new_state": {
+        "config": {
+          "autoscale": {
+            "max_workers": 5,
+            "min_workers": 3
+          },
+          "autotermination_minutes": 60,
+          "cluster_name": "test-cluster-[UNIQUE_NAME]",
+          "node_type_id": "[NODE_TYPE_ID]",
+          "spark_version": "13.3.x-snapshot-scala2.12"
+        }
+      },
+      "remote_state": {
+        "autoscale": {
+          "max_workers": 4,
+          "min_workers": 2
+        },
+        "autotermination_minutes": 60,
+        "cluster_id": "[CLUSTER_ID]",
+        "cluster_name": "test-cluster-[UNIQUE_NAME]",
+        "node_type_id": "[NODE_TYPE_ID]",
+        "spark_version": "13.3.x-snapshot-scala2.12"
+      },
+      "changes": {
+        "local": {
+          "autoscale.max_workers": {
+            "action": "update"
+          },
+          "autoscale.min_workers": {
+            "action": "update"
+          }
+        }
+      }
+    }
+  }
+}
+{
+  "plan": {
+    "resources.clusters.test_cluster": {
+      "action": "resize",
+      "new_state": {
+        "config": {
+          "autoscale": {
+            "max_workers": 6,
+            "min_workers": 4
+          },
+          "autotermination_minutes": 60,
+          "cluster_name": "test-cluster-[UNIQUE_NAME]",
+          "node_type_id": "[NODE_TYPE_ID]",
+          "spark_version": "13.3.x-snapshot-scala2.12"
+        }
+      },
+      "remote_state": {
+        "autoscale": {
+          "max_workers": 5,
+          "min_workers": 3
+        },
+        "autotermination_minutes": 60,
+        "cluster_id": "[CLUSTER_ID]",
+        "cluster_name": "test-cluster-[UNIQUE_NAME]",
+        "node_type_id": "[NODE_TYPE_ID]",
+        "spark_version": "13.3.x-snapshot-scala2.12",
+        "state": "RUNNING"
+      },
+      "changes": {
+        "local": {
+          "autoscale.max_workers": {
+            "action": "resize"
+          },
+          "autoscale.min_workers": {
+            "action": "resize"
+          }
+        }
+      }
+    }
+  }
+}
+{
+  "plan": {
+    "resources.clusters.test_cluster": {
+      "action": "resize",
+      "new_state": {
+        "config": {
+          "autotermination_minutes": 60,
+          "cluster_name": "test-cluster-[UNIQUE_NAME]",
+          "node_type_id": "[NODE_TYPE_ID]",
+          "num_workers": 3,
+          "spark_version": "13.3.x-snapshot-scala2.12"
+        }
+      },
+      "remote_state": {
+        "autoscale": {
+          "max_workers": 6,
+          "min_workers": 4
+        },
+        "autotermination_minutes": 60,
+        "cluster_id": "[CLUSTER_ID]",
+        "cluster_name": "test-cluster-[UNIQUE_NAME]",
+        "node_type_id": "[NODE_TYPE_ID]",
+        "spark_version": "13.3.x-snapshot-scala2.12",
+        "state": "RUNNING"
+      },
+      "changes": {
+        "local": {
+          "autoscale": {
+            "action": "resize"
+          },
+          "num_workers": {
+            "action": "resize"
+          }
+        }
+      }
+    }
+  }
+}

acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct-exp.txt

@@ -0,0 +1,15 @@
+create clusters.test_cluster
+
+Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged
+update clusters.test_cluster
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged
+update clusters.test_cluster
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged
+resize clusters.test_cluster
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged
+resize clusters.test_cluster
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged

acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.json

@@ -0,0 +1,35 @@
+{
+  "plan": {
+    "resources.clusters.test_cluster": {
+      "action": "create"
+    }
+  }
+}
+{
+  "plan": {
+    "resources.clusters.test_cluster": {
+      "action": "update"
+    }
+  }
+}
+{
+  "plan": {
+    "resources.clusters.test_cluster": {
+      "action": "update"
+    }
+  }
+}
+{
+  "plan": {
+    "resources.clusters.test_cluster": {
+      "action": "update"
+    }
+  }
+}
+{
+  "plan": {
+    "resources.clusters.test_cluster": {
+      "action": "update"
+    }
+  }
+}

acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.txt

@@ -0,0 +1,15 @@
+create clusters.test_cluster
+
+Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged
+update clusters.test_cluster
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged
+update clusters.test_cluster
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged
+update clusters.test_cluster
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged
+update clusters.test_cluster
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged