Add autoscale mode configuration to databricks_pipeline resource (#1600)

Author: alexott

docs/resources/pipeline.md

@@ -54,7 +54,7 @@ The following arguments are supported:
 * `storage` - A location on DBFS or cloud storage where output data and metadata required for pipeline execution are stored. By default, tables are stored in a subdirectory of this location. *Change of this parameter forces recreation of the pipeline.*
 * `configuration` - An optional list of values to apply to the entire pipeline. Elements must be formatted as key:value pairs.
 * `library` blocks - Specifies pipeline code and required artifacts. Syntax resembles [library](cluster.md#library-configuration-block) configuration block with the addition of a special `notebook` type of library that should have the `path` attribute. *Right now only the `notebook` type is supported.*
-* `cluster` blocks - [Clusters](cluster.md) to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that DLT pipeline clusters are supporting only subset of attributes as described in [documentation](https://docs.databricks.com/data-engineering/delta-live-tables/delta-live-tables-api-guide.html#pipelinesnewcluster).*
+* `cluster` blocks - [Clusters](cluster.md) to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that DLT pipeline clusters are supporting only subset of attributes as described in [documentation](https://docs.databricks.com/data-engineering/delta-live-tables/delta-live-tables-api-guide.html#pipelinesnewcluster).*  Also, note that `autoscale` block is extended with the `mode` parameter that controls the autoscaling algorithm (possible values are `ENHANCED` for new, enhanced autoscaling algorithm, or `LEGACY` for old algorithm).
 * `continuous` - A flag indicating whether to run the pipeline continuously. The default value is `false`.
 * `development` - A flag indicating whether to run the pipeline in development mode. The default value is `false`.
 * `photon` - A flag indicating whether to use Photon engine. The default value is `false`.

pipelines/resource_pipeline.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
@@ -19,15 +20,22 @@ import (
 // DefaultTimeout is the default amount of time that Terraform will wait when creating, updating and deleting pipelines.
 const DefaultTimeout = 20 * time.Minute
 
+// dltAutoScale is a struct the describes auto scaling for DLT clusters
+type dltAutoScale struct {
+	MinWorkers int32  `json:"min_workers,omitempty"`
+	MaxWorkers int32  `json:"max_workers,omitempty"`
+	Mode       string `json:"mode,omitempty"`
+}
+
 // We separate this struct from Cluster for two reasons:
 // 1. Pipeline clusters include a `Label` field.
 // 2. Spark version is not required (and shouldn't be specified) for pipeline clusters.
 // 3. num_workers is optional, and there is no single-node support for pipelines clusters.
 type pipelineCluster struct {
 	Label string `json:"label,omitempty"` // used only by pipelines
 
-	NumWorkers int32               `json:"num_workers,omitempty" tf:"group:size"`
-	Autoscale  *clusters.AutoScale `json:"autoscale,omitempty" tf:"group:size"`
+	NumWorkers int32         `json:"num_workers,omitempty" tf:"group:size"`
+	Autoscale  *dltAutoScale `json:"autoscale,omitempty" tf:"group:size"`
 
 	NodeTypeID           string                  `json:"node_type_id,omitempty" tf:"group:node_type,computed"`
 	DriverNodeTypeID     string                  `json:"driver_node_type_id,omitempty" tf:"computed"`
@@ -266,12 +274,21 @@ func suppressStorageDiff(k, old, new string, d *schema.ResourceData) bool {
 	return false
 }
 
+func AutoscaleModeDiffSuppress(k, old, new string, d *schema.ResourceData) bool {
+	if strings.EqualFold(old, new) {
+		log.Printf("[INFO] Suppressing diff on autoscale mode")
+		return true
+	}
+	return false
+}
+
 func adjustPipelineResourceSchema(m map[string]*schema.Schema) map[string]*schema.Schema {
 	cluster, _ := m["cluster"].Elem.(*schema.Resource)
 	clustersSchema := cluster.Schema
 	clustersSchema["spark_conf"].DiffSuppressFunc = clusters.SparkConfDiffSuppressFunc
 	common.MustSchemaPath(clustersSchema,
 		"aws_attributes", "zone_id").DiffSuppressFunc = clusters.ZoneDiffSuppress
+	common.MustSchemaPath(clustersSchema, "autoscale", "mode").DiffSuppressFunc = AutoscaleModeDiffSuppress
 
 	awsAttributes, _ := clustersSchema["aws_attributes"].Elem.(*schema.Resource)
 	awsAttributesSchema := awsAttributes.Schema