Fixed #637 - Recreate mounting clusters when they are deleted

Author: nfx

storage/mounts.go

@@ -107,38 +107,43 @@ func NewMountPoint(executor common.CommandExecutor, name, clusterID string) Moun
 	}
 }
 
+func getOrCreateMountingCluster(clustersAPI compute.ClustersAPI) (string, error) {
+	cluster, err := clustersAPI.GetOrCreateRunningCluster("terraform-mount", compute.Cluster{
+		NumWorkers:  0,
+		ClusterName: "terraform-mount",
+		SparkVersion: clustersAPI.LatestSparkVersionOrDefault(
+			compute.SparkVersionRequest{
+				Latest:          true,
+				LongTermSupport: true,
+			}),
+		NodeTypeID: clustersAPI.GetSmallestNodeType(
+			compute.NodeTypeRequest{
+				LocalDisk: true,
+			}),
+		AutoterminationMinutes: 10,
+		SparkConf: map[string]string{
+			"spark.master":                     "local[*]",
+			"spark.databricks.cluster.profile": "singleNode",
+		},
+		CustomTags: map[string]string{
+			"ResourceClass": "SingleNode",
+		},
+	})
+	if err != nil {
+		return "", err
+	}
+	return cluster.ClusterID, nil
+}
+
 func getMountingClusterID(ctx context.Context, client *common.DatabricksClient, clusterID string) (string, error) {
 	clustersAPI := compute.NewClustersAPI(ctx, client)
 	if clusterID == "" {
-		r := compute.Cluster{
-			NumWorkers:  0,
-			ClusterName: "terraform-mount",
-			SparkVersion: clustersAPI.LatestSparkVersionOrDefault(
-				compute.SparkVersionRequest{
-					Latest:          true,
-					LongTermSupport: true,
-				}),
-			NodeTypeID: clustersAPI.GetSmallestNodeType(
-				compute.NodeTypeRequest{
-					LocalDisk: true,
-				}),
-
-			AutoterminationMinutes: 10,
-			SparkConf: map[string]string{
-				"spark.master":                     "local[*]",
-				"spark.databricks.cluster.profile": "singleNode",
-			},
-			CustomTags: map[string]string{
-				"ResourceClass": "SingleNode",
-			},
-		}
-		cluster, err := clustersAPI.GetOrCreateRunningCluster("terraform-mount", r)
-		if err != nil {
-			return "", err
-		}
-		return cluster.ClusterID, nil
+		return getOrCreateMountingCluster(clustersAPI)
 	}
 	clusterInfo, err := clustersAPI.Get(clusterID)
+	if e, ok := err.(common.APIError); ok && e.IsMissing() {
+		return getOrCreateMountingCluster(clustersAPI)
+	}
 	if err != nil {
 		return "", err
 	}

storage/mounts_test.go

@@ -213,3 +213,91 @@ func TestMountPoint_Delete(t *testing.T) {
 		return expectedCommandResp, mp.Delete()
 	}, nil, mountName, expectedCommand)
 }
+
+func TestDeletedMountClusterRecreates(t *testing.T) {
+	qa.HTTPFixturesApply(t, []qa.HTTPFixture{
+		{
+			Method:   "GET",
+			Resource: "/api/2.0/clusters/get?cluster_id=abc",
+			Status:   404,
+		},
+		{
+			Method:       "GET",
+			ReuseRequest: true,
+			Resource:     "/api/2.0/clusters/list",
+			Response:     map[string]interface{}{},
+		},
+		{
+			Method:       "GET",
+			ReuseRequest: true,
+			Resource:     "/api/2.0/clusters/spark-versions",
+			Response: compute.SparkVersionsList{
+				SparkVersions: []compute.SparkVersion{
+					{
+						Version:     "7.1.x-cpu-ml-scala2.12",
+						Description: "7.1 ML (includes Apache Spark 3.0.0, Scala 2.12)",
+					},
+				},
+			},
+		},
+		{
+			Method:       "GET",
+			ReuseRequest: true,
+			Resource:     "/api/2.0/clusters/list-node-types",
+			Response: compute.NodeTypeList{
+				NodeTypes: []compute.NodeType{
+					{
+						NodeTypeID:     "Standard_F4s",
+						InstanceTypeID: "Standard_F4s",
+						MemoryMB:       8192,
+						NumCores:       4,
+						NodeInstanceType: &compute.NodeInstanceType{
+							LocalDisks:      1,
+							InstanceTypeID:  "Standard_F4s",
+							LocalDiskSizeGB: 16,
+							LocalNVMeDisks:  0,
+						},
+					},
+				},
+			},
+		},
+		{
+			Method:       "POST",
+			ReuseRequest: true,
+			Resource:     "/api/2.0/clusters/create",
+			ExpectedRequest: compute.Cluster{
+				AutoterminationMinutes: 10,
+				ClusterName:            "terraform-mount",
+				NodeTypeID:             "Standard_F4s",
+				SparkVersion:           "7.3.x-scala2.12",
+				CustomTags: map[string]string{
+					"ResourceClass": "SingleNode",
+				},
+				SparkConf: map[string]string{
+					"spark.databricks.cluster.profile":       "singleNode",
+					"spark.master":                           "local[*]",
+				},
+			},
+			Response: compute.ClusterID{
+				ClusterID: "bcd",
+			},
+		},
+		{
+			Method:       "GET",
+			ReuseRequest: true,
+			Resource:     "/api/2.0/clusters/get?cluster_id=bcd",
+			Response: compute.ClusterInfo{
+				ClusterID: "bcd",
+				State:     "RUNNING",
+				SparkConf: map[string]string{
+					"spark.databricks.acl.dfAclsEnabled": "true",
+					"spark.databricks.cluster.profile":   "singleNode",
+				},
+			},
+		},
+	}, func(ctx context.Context, client *common.DatabricksClient) {
+		clusterID, err := getMountingClusterID(ctx, client, "abc")
+		assert.NoError(t, err)
+		assert.Equal(t, "bcd", clusterID)
+	})
+}