Add pipeline info to metadata.json (#3677)

## Changes
Output `id` and `relative_path` for each pipeline to `metadata.json`
during deploy

## Why
To enable DABs integrations in the Lakeflow UI

## Tests
Updated existing test

<!-- If your PR needs to be included in the release notes for next
release,
add a separate entry in NEXT_CHANGELOG.md as part of your PR. -->

Author: rclarey

bundle/deploy/metadata/compute.go

@@ -35,7 +35,7 @@ func (m *compute) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
 	}
 
 	// Set job config paths in metadata
-	jobsMetadata := make(map[string]*metadata.Job)
+	jobsMetadata := make(map[string]*metadata.Resource)
 	for name, job := range b.Config.Resources.Jobs {
 		// Compute config file path the job is defined in, relative to the bundle
 		// root
@@ -45,13 +45,31 @@ func (m *compute) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
 			return diag.Errorf("failed to compute relative path for job %s: %v", name, err)
 		}
 		// Metadata for the job
-		jobsMetadata[name] = &metadata.Job{
+		jobsMetadata[name] = &metadata.Resource{
 			ID:           job.ID,
 			RelativePath: filepath.ToSlash(relativePath),
 		}
 	}
 	b.Metadata.Config.Resources.Jobs = jobsMetadata
 
+	// Set pipeline config paths in metadata
+	pipelinesMetadata := make(map[string]*metadata.Resource)
+	for name, pipeline := range b.Config.Resources.Pipelines {
+		// Compute config file path the pipeline is defined in, relative to the bundle
+		// root
+		l := b.Config.GetLocation("resources.pipelines." + name)
+		relativePath, err := filepath.Rel(b.BundleRootPath, l.File)
+		if err != nil {
+			return diag.Errorf("failed to compute relative path for pipeline %s: %v", name, err)
+		}
+		// Metadata for the pipeline
+		pipelinesMetadata[name] = &metadata.Resource{
+			ID:           pipeline.ID,
+			RelativePath: filepath.ToSlash(relativePath),
+		}
+	}
+	b.Metadata.Config.Resources.Pipelines = pipelinesMetadata
+
 	// Set file upload destination of the bundle in metadata
 	b.Metadata.Config.Workspace.FilePath = b.Config.Workspace.FilePath
 	// In source-linked deployment files are not copied and resources use source files, therefore we use sync path as file path in metadata

bundle/deploy/metadata/compute_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/databricks/cli/bundle/metadata"
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/databricks/databricks-sdk-go/service/pipelines"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -49,15 +50,27 @@ func TestComputeMetadataMutator(t *testing.T) {
 					},
 				},
 				Pipelines: map[string]*resources.Pipeline{
-					"my-pipeline": {},
+					"my-pipeline-1": {
+						BaseResource: resources.BaseResource{ID: "3333"},
+						CreatePipeline: pipelines.CreatePipeline{
+							Name: "My Pipeline One",
+						},
+					},
+					"my-pipeline-2": {
+						BaseResource: resources.BaseResource{ID: "4444"},
+						CreatePipeline: pipelines.CreatePipeline{
+							Name: "My Pipeline Two",
+						},
+					},
 				},
 			},
 		},
 	}
 
 	bundletest.SetLocation(b, "resources.jobs.my-job-1", []dyn.Location{{File: "a/b/c"}})
 	bundletest.SetLocation(b, "resources.jobs.my-job-2", []dyn.Location{{File: "d/e/f"}})
-	bundletest.SetLocation(b, "resources.pipelines.my-pipeline", []dyn.Location{{File: "abc"}})
+	bundletest.SetLocation(b, "resources.pipelines.my-pipeline-1", []dyn.Location{{File: "x/y/z"}})
+	bundletest.SetLocation(b, "resources.pipelines.my-pipeline-2", []dyn.Location{{File: "u/v/w"}})
 
 	expectedMetadata := metadata.Metadata{
 		Version: metadata.Version,
@@ -74,7 +87,7 @@ func TestComputeMetadataMutator(t *testing.T) {
 				},
 			},
 			Resources: metadata.Resources{
-				Jobs: map[string]*metadata.Job{
+				Jobs: map[string]*metadata.Resource{
 					"my-job-1": {
 						RelativePath: "a/b/c",
 						ID:           "1111",
@@ -84,6 +97,16 @@ func TestComputeMetadataMutator(t *testing.T) {
 						ID:           "2222",
 					},
 				},
+				Pipelines: map[string]*metadata.Resource{
+					"my-pipeline-1": {
+						RelativePath: "x/y/z",
+						ID:           "3333",
+					},
+					"my-pipeline-2": {
+						RelativePath: "u/v/w",
+						ID:           "4444",
+					},
+				},
 			},
 		},
 	}

bundle/metadata/metadata.go

@@ -14,7 +14,7 @@ type Workspace struct {
 	FilePath string `json:"file_path"`
 }
 
-type Job struct {
+type Resource struct {
 	ID string `json:"id,omitempty"`
 
 	// Relative path from the bundle root to the configuration file that holds
@@ -23,7 +23,8 @@ type Job struct {
 }
 
 type Resources struct {
-	Jobs map[string]*Job `json:"jobs,omitempty"`
+	Jobs      map[string]*Resource `json:"jobs,omitempty"`
+	Pipelines map[string]*Resource `json:"pipelines,omitempty"`
 }
 
 type Config struct {