Add support for pipeline tagging (#3086)

## Changes

Add support for pipeline tagging to Databricks Bundles


## Why

Tagging allows users to tag their pipelines (which makes them filterable
through the UI), and propagates the same set of tags to compute.

## Tests

Acceptance test, manual test. Depends on
#3087

---------

Co-authored-by: Pieter Noordhuis <[email protected]>

Author: akolar-db

NEXT_CHANGELOG.md

@@ -11,5 +11,6 @@
 ### CLI
 
 ### Bundles
+* Add support for tagging pipelines ([#3086](https://github.com/databricks/cli/pull/3086))
 
 ### API Changes

acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json

@@ -143,7 +143,10 @@
         ],
         "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_pipeline",
         "permissions": [],
-        "schema": "project_name_[UNIQUE_NAME]_dev"
+        "schema": "project_name_[UNIQUE_NAME]_dev",
+        "tags": {
+          "dev": "[USERNAME]"
+        }
       }
     }
   },

acceptance/bundle/templates/default-python/integration_classic/output.txt

@@ -74,11 +74,11 @@ Resources:
 +        "id": "[UUID]",
          "libraries": [
            {
-@@ -144,5 +147,6 @@
-         "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_pipeline",
-         "permissions": [],
--        "schema": "project_name_[UNIQUE_NAME]_dev"
-+        "schema": "project_name_[UNIQUE_NAME]_dev",
+@@ -147,5 +150,6 @@
+         "tags": {
+           "dev": "[USERNAME]"
+-        }
++        },
 +        "url": "[DATABRICKS_URL]/pipelines/[UUID]"
        }
      }
@@ -195,7 +195,7 @@ Validation OK!
 +          "pause_status": "UNPAUSED",
            "periodic": {
              "interval": 1,
-@@ -127,22 +114,21 @@
+@@ -127,25 +114,21 @@
          "channel": "CURRENT",
          "configuration": {
 -          "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src"
@@ -219,11 +219,14 @@ Validation OK!
 -        "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_pipeline",
 +        "name": "project_name_[UNIQUE_NAME]_pipeline",
          "permissions": [],
--        "schema": "project_name_[UNIQUE_NAME]_dev"
+-        "schema": "project_name_[UNIQUE_NAME]_dev",
+-        "tags": {
+-          "dev": "[USERNAME]"
+-        }
 +        "schema": "project_name_[UNIQUE_NAME]_prod"
        }
      }
-@@ -154,10 +140,10 @@
+@@ -157,10 +140,10 @@
    },
    "workspace": {
 -    "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts",
@@ -367,7 +370,7 @@ Resources:
 -        "development": true,
          "edition": "ADVANCED",
          "id": "[UUID]",
-@@ -141,11 +127,11 @@
+@@ -141,14 +127,11 @@
            {
              "notebook": {
 -              "path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/dlt_pipeline"
@@ -379,10 +382,13 @@ Resources:
 +        "name": "project_name_[UNIQUE_NAME]_pipeline",
          "permissions": [],
 -        "schema": "project_name_[UNIQUE_NAME]_dev",
+-        "tags": {
+-          "dev": "[USERNAME]"
+-        },
 +        "schema": "project_name_[UNIQUE_NAME]_prod",
          "url": "[DATABRICKS_URL]/pipelines/[UUID]"
        }
-@@ -158,10 +144,10 @@
+@@ -161,10 +144,10 @@
    },
    "workspace": {
 -    "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts",

acceptance/bundle/templates/experimental-jobs-as-code/output.txt

@@ -104,6 +104,9 @@ Warning: Ignoring Databricks CLI version constraint for development build. Requi
       ],
       "name": "[dev [USERNAME]] my_jobs_as_code_pipeline",
       "permissions": [],
+      "tags": {
+        "dev": "[USERNAME]"
+      },
       "target": "my_jobs_as_code_dev"
     }
   }

acceptance/pipelines/e2e/output.txt

@@ -54,7 +54,10 @@ View your pipeline my_project_pipeline_2 here: [DATABRICKS_URL]/pipelines/[UUID]
     "edition":"ADVANCED",
     "id":"[UUID]",
     "name":"[dev [USERNAME]] my_project_pipeline_2",
-    "storage":"dbfs:/pipelines/[UUID]"
+    "storage":"dbfs:/pipelines/[UUID]",
+    "tags": {
+      "dev":"[USERNAME]"
+    }
   },
   "state":"IDLE"
 }

bundle/config/mutator/resourcemutator/apply_presets.go

@@ -105,7 +105,18 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 		if t.TriggerPauseStatus == config.Paused {
 			p.Continuous = false
 		}
-		// As of 2024-06, pipelines don't yet support tags
+
+		if len(tags) > 0 {
+			if p.Tags == nil {
+				// Note: only create this map if tags is not empty, to avoid inserting "tags: {}" entry in the config
+				p.Tags = make(map[string]string, len(tags))
+			}
+			for _, tag := range tags {
+				if p.Tags[tag.Key] == "" {
+					p.Tags[tag.Key] = tag.Value
+				}
+			}
+		}
 	}
 
 	// Models presets: Prefix, Tags