Add file library type to databricks_pipeline resource (#1975)

* Add `file` library type to `databricks_pipeline` resource

* handle references for file path in exporter as well

Author: alexott

docs/resources/pipeline.md

@@ -12,6 +12,10 @@ resource "databricks_notebook" "dlt_demo" {
   #...
 }
 
+resource "databricks_repo" "dlt_demo" {
+  #...
+}
+
 resource "databricks_pipeline" "this" {
   name    = "Pipeline Name"
   storage = "/test/first-pipeline"
@@ -42,6 +46,12 @@ resource "databricks_pipeline" "this" {
     }
   }
 
+  library {
+    file {
+      path = "${databricks_repo.dlt_demo.path}/pipeline.sql"
+    }
+  }
+
   continuous = false
 }
 ```
@@ -53,7 +63,7 @@ The following arguments are supported:
 * `name` - A user-friendly name for this pipeline. The name can be used to identify pipeline jobs in the UI.
 * `storage` - A location on DBFS or cloud storage where output data and metadata required for pipeline execution are stored. By default, tables are stored in a subdirectory of this location. *Change of this parameter forces recreation of the pipeline.*
 * `configuration` - An optional list of values to apply to the entire pipeline. Elements must be formatted as key:value pairs.
-* `library` blocks - Specifies pipeline code and required artifacts. Syntax resembles [library](cluster.md#library-configuration-block) configuration block with the addition of a special `notebook` type of library that should have the `path` attribute. *Right now only the `notebook` type is supported.*
+* `library` blocks - Specifies pipeline code and required artifacts. Syntax resembles [library](cluster.md#library-configuration-block) configuration block with the addition of a special `notebook` & `file` library types that should have the `path` attribute. *Right now only the `notebook` & `file` types are supported.*
 * `cluster` blocks - [Clusters](cluster.md) to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that DLT pipeline clusters are supporting only subset of attributes as described in [documentation](https://docs.databricks.com/data-engineering/delta-live-tables/delta-live-tables-api-guide.html#pipelinesnewcluster).*  Also, note that `autoscale` block is extended with the `mode` parameter that controls the autoscaling algorithm (possible values are `ENHANCED` for new, enhanced autoscaling algorithm, or `LEGACY` for old algorithm).
 * `continuous` - A flag indicating whether to run the pipeline continuously. The default value is `false`.
 * `development` - A flag indicating whether to run the pipeline in development mode. The default value is `true`.

exporter/importables.go

@@ -1432,6 +1432,9 @@ var resourcesMap map[string]importable = map[string]importable{
 				if lib.Notebook != nil {
 					ic.emitNotebookOrRepo(lib.Notebook.Path)
 				}
+				if lib.File != nil {
+					ic.emitNotebookOrRepo(lib.File.Path)
+				}
 				ic.emitIfDbfsFile(lib.Jar)
 				ic.emitIfDbfsFile(lib.Whl)
 			}
@@ -1489,6 +1492,8 @@ var resourcesMap map[string]importable = map[string]importable{
 			{Path: "cluster.driver_instance_pool_id", Resource: "databricks_instance_pool"},
 			{Path: "library.notebook.path", Resource: "databricks_notebook"},
 			{Path: "library.notebook.path", Resource: "databricks_repo", Match: "path", MatchType: MatchPrefix},
+			{Path: "library.file.path", Resource: "databricks_notebook"},
+			{Path: "library.file.path", Resource: "databricks_repo", Match: "path", MatchType: MatchPrefix},
 			{Path: "library.jar", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
 			{Path: "library.whl", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
 		},

exporter/test-data/get-dlt-pipeline.json

@@ -42,7 +42,7 @@
         }
       },
       {
-        "notebook": {
+        "file": {
           "path": "/Repos/[email protected]/repo/Test DLT"
         }
       }

pipelines/resource_pipeline.go

@@ -63,11 +63,16 @@ type NotebookLibrary struct {
 	Path string `json:"path"`
 }
 
+type FileLibrary struct {
+	Path string `json:"path"`
+}
+
 type PipelineLibrary struct {
 	Jar      string           `json:"jar,omitempty"`
 	Maven    *libraries.Maven `json:"maven,omitempty"`
 	Whl      string           `json:"whl,omitempty"`
 	Notebook *NotebookLibrary `json:"notebook,omitempty"`
+	File     *FileLibrary     `json:"file,omitempty"`
 }
 
 type filters struct {
@@ -79,6 +84,7 @@ type PipelineSpec struct {
 	ID                  string            `json:"id,omitempty" tf:"computed"`
 	Name                string            `json:"name,omitempty"`
 	Storage             string            `json:"storage,omitempty" tf:"force_new"`
+	Catalog             string            `json:"catalog,omitempty" tf:"force_new"`
 	Configuration       map[string]string `json:"configuration,omitempty"`
 	Clusters            []pipelineCluster `json:"clusters,omitempty" tf:"alias:cluster"`
 	Libraries           []PipelineLibrary `json:"libraries,omitempty" tf:"slice_set,alias:library"`
@@ -307,6 +313,8 @@ func adjustPipelineResourceSchema(m map[string]*schema.Schema) map[string]*schem
 	m["edition"].ValidateFunc = validation.StringInSlice([]string{"pro", "core", "advanced"}, true)
 
 	m["storage"].DiffSuppressFunc = suppressStorageDiff
+	m["storage"].ConflictsWith = []string{"catalog"}
+	m["catalog"].ConflictsWith = []string{"storage"}
 
 	return m
 }