Updated databricks_pipeline resource to match latest APIs (#1368)

Author: alexott

docs/resources/pipeline.md

@@ -42,26 +42,25 @@ resource "databricks_pipeline" "this" {
     }
   }
 
-  filters {
-    include = ["com.databricks.include"]
-    exclude = ["com.databricks.exclude"]
-  }
-
   continuous = false
 }
 ```
 
 ## Argument Reference
 
-The following arguments are required:
+The following arguments are supported:
 
 * `name` - A user-friendly name for this pipeline. The name can be used to identify pipeline jobs in the UI.
-* `storage` - A location on DBFS or cloud storage where output data and metadata required for pipeline execution are stored. By default, tables are stored in a subdirectory of this location.
+* `storage` - A location on DBFS or cloud storage where output data and metadata required for pipeline execution are stored. By default, tables are stored in a subdirectory of this location. *Change of this parameter forces recreation of the pipeline.*
 * `configuration` - An optional list of values to apply to the entire pipeline. Elements must be formatted as key:value pairs.
-* `library` blocks - Specifies pipeline code and required artifacts. Syntax resembles [library](cluster.md#library-configuration-block) configuration block with the addition of a special `notebook` type of library that should have the `path` attribute.
-* `cluster` blocks - [Clusters](cluster.md) to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline.
+* `library` blocks - Specifies pipeline code and required artifacts. Syntax resembles [library](cluster.md#library-configuration-block) configuration block with the addition of a special `notebook` type of library that should have the `path` attribute. *Right now only the `notebook` type is supported.*
+* `cluster` blocks - [Clusters](cluster.md) to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that DLT pipeline clusters are supporting only subset of attributes as described in [documentation](https://docs.databricks.com/data-engineering/delta-live-tables/delta-live-tables-api-guide.html#pipelinesnewcluster).*
 * `continuous` - A flag indicating whether to run the pipeline continuously. The default value is `false`.
+* `development` - A flag indicating whether to run the pipeline in development mode. The default value is `false`.
+* `photon` - A flag indicating whether to use Photon engine. The default value is `false`.
 * `target` - The name of a database for persisting pipeline output data. Configuring the target setting allows you to view and query the pipeline output data from the Databricks UI.
+* `edition` - optional name of the [product edition](https://docs.databricks.com/data-engineering/delta-live-tables/delta-live-tables-concepts.html#editions). Supported values are: `core`, `pro`, `advanced` (default).
+* `channel` - optional name of the release channel for Spark version used by DLT pipeline.  Supported values are: `current` (default) and `preview`.
 
 ## Import
 

pipelines/resource_pipeline.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
 
 	"github.com/databrickslabs/terraform-provider-databricks/clusters"
 	"github.com/databrickslabs/terraform-provider-databricks/common"
@@ -27,10 +28,12 @@ type pipelineCluster struct {
 	NumWorkers int32               `json:"num_workers,omitempty" tf:"group:size"`
 	Autoscale  *clusters.AutoScale `json:"autoscale,omitempty" tf:"group:size"`
 
-	NodeTypeID       string                  `json:"node_type_id,omitempty" tf:"group:node_type,computed"`
-	DriverNodeTypeID string                  `json:"driver_node_type_id,omitempty" tf:"conflicts:instance_pool_id,computed"`
-	InstancePoolID   string                  `json:"instance_pool_id,omitempty" tf:"group:node_type"`
-	AwsAttributes    *clusters.AwsAttributes `json:"aws_attributes,omitempty" tf:"conflicts:instance_pool_id"`
+	NodeTypeID           string                  `json:"node_type_id,omitempty" tf:"group:node_type,computed"`
+	DriverNodeTypeID     string                  `json:"driver_node_type_id,omitempty" tf:"computed"`
+	InstancePoolID       string                  `json:"instance_pool_id,omitempty" tf:"group:node_type"`
+	DriverInstancePoolID string                  `json:"driver_instance_pool_id,omitempty"`
+	AwsAttributes        *clusters.AwsAttributes `json:"aws_attributes,omitempty" tf:"suppress_diff"`
+	GcpAttributes        *clusters.GcpAttributes `json:"gcp_attributes,omitempty" tf:"suppress_diff"`
 
 	SparkConf    map[string]string `json:"spark_conf,omitempty"`
 	SparkEnvVars map[string]string `json:"spark_env_vars,omitempty"`
@@ -60,14 +63,18 @@ type filters struct {
 type pipelineSpec struct {
 	ID                  string            `json:"id,omitempty" tf:"computed"`
 	Name                string            `json:"name,omitempty"`
-	Storage             string            `json:"storage,omitempty"`
+	Storage             string            `json:"storage,omitempty" tf:"force_new"`
 	Configuration       map[string]string `json:"configuration,omitempty"`
 	Clusters            []pipelineCluster `json:"clusters,omitempty" tf:"slice_set,alias:cluster"`
 	Libraries           []pipelineLibrary `json:"libraries,omitempty" tf:"slice_set,alias:library"`
-	Filters             *filters          `json:"filters"`
+	Filters             *filters          `json:"filters,omitempty"`
 	Continuous          bool              `json:"continuous,omitempty"`
+	Development         bool              `json:"development,omitempty"`
 	AllowDuplicateNames bool              `json:"allow_duplicate_names,omitempty"`
 	Target              string            `json:"target,omitempty"`
+	Photon              bool              `json:"photon,omitempty"`
+	Edition             string            `json:"edition,omitempty" tf:"suppress_diff,default:advanced"`
+	Channel             string            `json:"channel,omitempty" tf:"suppress_diff,default:current"`
 }
 
 type createPipelineResponse struct {
@@ -206,18 +213,26 @@ func adjustPipelineResourceSchema(m map[string]*schema.Schema) map[string]*schem
 
 	awsAttributes, _ := clustersSchema["aws_attributes"].Elem.(*schema.Resource)
 	awsAttributesSchema := awsAttributes.Schema
-	delete(awsAttributesSchema, "first_on_demand")
 	delete(awsAttributesSchema, "availability")
 	delete(awsAttributesSchema, "spot_bid_price_percent")
 	delete(awsAttributesSchema, "ebs_volume_type")
 	delete(awsAttributesSchema, "ebs_volume_count")
 	delete(awsAttributesSchema, "ebs_volume_size")
 
+	gcpAttributes, _ := clustersSchema["gcp_attributes"].Elem.(*schema.Resource)
+	gcpAttributesSchema := gcpAttributes.Schema
+	delete(gcpAttributesSchema, "use_preemptible_executors")
+	delete(gcpAttributesSchema, "availability")
+	delete(gcpAttributesSchema, "boot_disk_size")
+	delete(gcpAttributesSchema, "zone_id")
+
 	m["library"].MinItems = 1
 	m["url"] = &schema.Schema{
 		Type:     schema.TypeString,
 		Computed: true,
 	}
+	m["channel"].ValidateFunc = validation.StringInSlice([]string{"current", "preview"}, true)
+	m["edition"].ValidateFunc = validation.StringInSlice([]string{"pro", "core", "advanced"}, true)
 
 	return m
 }

pipelines/resource_pipeline_test.go

@@ -4,7 +4,6 @@ import (
 	"testing"
 
 	"github.com/databrickslabs/terraform-provider-databricks/common"
-	"github.com/databrickslabs/terraform-provider-databricks/libraries"
 
 	"github.com/databrickslabs/terraform-provider-databricks/qa"
 	"github.com/stretchr/testify/assert"
@@ -27,11 +26,8 @@ var basicPipelineSpec = pipelineSpec{
 	},
 	Libraries: []pipelineLibrary{
 		{
-			Jar: "dbfs:/pipelines/code/abcde.jar",
-		},
-		{
-			Maven: &libraries.Maven{
-				Coordinates: "com.microsoft.azure:azure-eventhubs-spark_2.12:2.3.18",
+			Notebook: &notebookLibrary{
+				Path: "/Test",
 			},
 		},
 	},
@@ -97,11 +93,8 @@ func TestResourcePipelineCreate(t *testing.T) {
 		  }
 		}
 		library {
-		  jar = "dbfs:/pipelines/code/abcde.jar"
-		}
-		library {
-		  maven {
-			coordinates = "com.microsoft.azure:azure-eventhubs-spark_2.12:2.3.18"
+		  notebook {
+			path = "/Test"
 		  }
 		}
 		filters {
@@ -132,7 +125,9 @@ func TestResourcePipelineCreate_Error(t *testing.T) {
 		HCL: `name = "test"
 		storage = "/test/storage"
 		library {
-			jar = "jar"
+			notebook {
+				path = "/Test"
+			}
 		}
 		filters {
 			include = ["a"]
@@ -181,7 +176,9 @@ func TestResourcePipelineCreate_ErrorWhenWaitingFailedCleanup(t *testing.T) {
 		HCL: `name = "test"
 		storage = "/test/storage"
 		library {
-			jar = "jar"
+			notebook {
+				path = "/Test"
+			}
 		}
 		filters {
 			include = ["a"]
@@ -230,7 +227,9 @@ func TestResourcePipelineCreate_ErrorWhenWaitingSuccessfulCleanup(t *testing.T)
 		HCL: `name = "test"
 		storage = "/test/storage"
 		library {
-			jar = "jar"
+			notebook {
+				path = "/Test"
+			}
 		}
 		filters {
 			include = ["a"]
@@ -316,14 +315,16 @@ func TestResourcePipelineUpdate(t *testing.T) {
 		Storage: "/test/storage",
 		Libraries: []pipelineLibrary{
 			{
-				Maven: &libraries.Maven{
-					Coordinates: "coordinates",
+				Notebook: &notebookLibrary{
+					Path: "/Test",
 				},
 			},
 		},
 		Filters: &filters{
 			Include: []string{"com.databricks.include"},
 		},
+		Channel: "current",
+		Edition: "advanced",
 	}
 	d, err := qa.ResourceFixture{
 		Fixtures: []qa.HTTPFixture{
@@ -355,13 +356,17 @@ func TestResourcePipelineUpdate(t *testing.T) {
 		HCL: `name = "test"
 		storage = "/test/storage"
 		library {
-			maven {
-				coordinates = "coordinates"
+			notebook {
+				path = "/Test"
 			}
 		}
 		filters {
 			include = [ "com.databricks.include" ]
 		}`,
+		InstanceState: map[string]string{
+			"name":    "test",
+			"storage": "/test/storage",
+		},
 		Update: true,
 		ID:     "abcd",
 	}.Apply(t)
@@ -386,15 +391,19 @@ func TestResourcePipelineUpdate_Error(t *testing.T) {
 		HCL: `name = "test"
 		storage = "/test/storage"
 		library {
-			maven {
-				coordinates = "coordinates"
+			notebook {
+				path = "/Test"
 			}
 		}
 		filters {
 			include = [ "com.databricks.include" ]
 		}`,
 		Update: true,
-		ID:     "abcd",
+		InstanceState: map[string]string{
+			"name":    "test",
+			"storage": "/test/storage",
+		},
+		ID: "abcd",
 	}.Apply(t)
 	qa.AssertErrorStartsWith(t, err, "Internal error happened")
 	assert.Equal(t, "abcd", d.Id())
@@ -408,14 +417,16 @@ func TestResourcePipelineUpdate_FailsAfterUpdate(t *testing.T) {
 		Storage: "/test/storage",
 		Libraries: []pipelineLibrary{
 			{
-				Maven: &libraries.Maven{
-					Coordinates: "coordinates",
+				Notebook: &notebookLibrary{
+					Path: "/Test",
 				},
 			},
 		},
 		Filters: &filters{
 			Include: []string{"com.databricks.include"},
 		},
+		Channel: "current",
+		Edition: "advanced",
 	}
 	d, err := qa.ResourceFixture{
 		Fixtures: []qa.HTTPFixture{
@@ -438,15 +449,19 @@ func TestResourcePipelineUpdate_FailsAfterUpdate(t *testing.T) {
 		HCL: `name = "test"
 		storage = "/test/storage"
 		library {
-			maven {
-				coordinates = "coordinates"
+			notebook {
+				path = "/Test"
 			}
 		}
 		filters {
 			include = [ "com.databricks.include" ]
 		}`,
 		Update: true,
-		ID:     "abcd",
+		InstanceState: map[string]string{
+			"name":    "test",
+			"storage": "/test/storage",
+		},
+		ID: "abcd",
 	}.Apply(t)
 	qa.AssertErrorStartsWith(t, err, "pipeline abcd has failed")
 	assert.Equal(t, "abcd", d.Id(), "Id should be the same as in reading")