Allow referencing job libraries outside bundle root without the need to specify sync root (#2842)

andrewnester · web-flow · commit 863f940676f5 · 2025-09-03T14:08:42.000+02:00
## Changes
Allow referencing job libraries outside bundle root without the need to
specify sync root.

Previously it was failing with error indicating the libraries are not
sync root, now it does not fail anymore.

## Why
Libraries has their own upload cycle and do not rely on general bundle
sync mechanism therefore it is not necessary to specify or check that
libraries are in sync root.

## Tests
Added acceptance tests

&lt;!-- If your PR needs to be included in the release notes for next
release,
add a separate entry in NEXT_CHANGELOG.md as part of your PR. --&gt;
diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md
@@ -16,6 +16,7 @@
 * Update default-python template to make DB Connect work out of the box for unit tests, using uv to install dependencies ([#3254](https://github.com/databricks/cli/pull/3254))
 * Add support for `TaskRetryMode` for continuous jobs ([#3529](https://github.com/databricks/cli/pull/3529))
 * Add support for specifying database instance as an application resource ([#3529](https://github.com/databricks/cli/pull/3529))
+* Allow referencing job libraries outside bundle root without the need to specify sync root ([#2842](https://github.com/databricks/cli/pull/2842))
 * Add top level `run_as` support for Lakeflow Declarative Pipelines ([#3307](https://github.com/databricks/cli/pull/3307))
 
 ### API Changes
diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/bundle/databricks.yml b/acceptance/bundle/libraries/outside_of_bundle_root/bundle/databricks.yml
@@ -0,0 +1,28 @@
+bundle:
+  name: outside_of_bundle_root
+
+variables:
+  cluster:
+    default:
+      spark_version: 15.4.x-scala2.12
+      node_type_id: i3.xlarge
+      data_security_mode: SINGLE_USER
+      num_workers: 0
+      spark_conf:
+        spark.master: "local[*, 4]"
+        spark.databricks.cluster.profile: singleNode
+      custom_tags:
+        ResourceClass: SingleNode
+
+resources:
+  jobs:
+    test:
+      name: "test"
+      tasks:
+        - task_key: task1
+          new_cluster: ${var.cluster}
+          python_wheel_task:
+            entry_point: main
+            package_name: my_default_python
+          libraries:
+            - whl: ../*.whl
diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/out.test.toml b/acceptance/bundle/libraries/outside_of_bundle_root/out.test.toml
@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]
diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/output.txt b/acceptance/bundle/libraries/outside_of_bundle_root/output.txt
@@ -0,0 +1,28 @@
+
+>>> [CLI] bundle validate -o json
+[
+  {
+    "whl": "../*.whl"
+  }
+]
+
+>>> [CLI] bundle deploy
+Uploading ../test.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/outside_of_bundle_root/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Check that the job libraries are uploaded and the path is correct in the job
+>>> cat out.requests.txt
+[
+  {
+    "whl": "/Workspace/Users/[USERNAME]/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl"
+  }
+]
+
+>>> cat out.requests.txt
+{
+  "method": "POST",
+  "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl"
+}
diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/script b/acceptance/bundle/libraries/outside_of_bundle_root/script
@@ -0,0 +1,12 @@
+cd bundle
+
+trace $CLI bundle validate -o json | jq '.resources.jobs.test.tasks[0].libraries'
+trace $CLI bundle deploy
+
+cd ..
+
+title "Check that the job libraries are uploaded and the path is correct in the job"
+trace cat out.requests.txt | jq 'select(.path == "/api/2.2/jobs/create")' | jq '.body.tasks[0].libraries'
+trace cat out.requests.txt | jq 'select(.path | test("/api/2.0/workspace-files/import-file/Workspace/Users/.*/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl"))'
+
+rm out.requests.txt
diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/test.toml b/acceptance/bundle/libraries/outside_of_bundle_root/test.toml
@@ -0,0 +1,6 @@
+Cloud = false
+RecordRequests = true
+
+Ignore = [
+    '.databricks',
+]
diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/test.whl b/acceptance/bundle/libraries/outside_of_bundle_root/test.whl
diff --git a/acceptance/bundle/paths/fallback/output.job.json b/acceptance/bundle/paths/fallback/output.job.json
@@ -24,7 +24,7 @@
         "whl": "dist/wheel1.whl"
       },
       {
-        "whl": "dist/wheel2.whl"
+        "whl": "../dist/wheel2.whl"
       }
     ],
     "python_wheel_task": {
@@ -39,7 +39,7 @@
         "jar": "target/jar1.jar"
       },
       {
-        "jar": "target/jar2.jar"
+        "jar": "../target/jar2.jar"
       }
     ],
     "spark_jar_task": {
diff --git a/acceptance/bundle/paths/fallback/output.txt b/acceptance/bundle/paths/fallback/output.txt
@@ -21,12 +21,6 @@ Error: path ../src/sql.sql is defined relative to the [TEST_TMP_DIR]/resources d
   in override_job.yml:24:25
      resources/my_job.yml:27:21
 
-Error: path ../dist/wheel2.whl is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:33:24). Please update the path to be relative to the file where it is defined or use earlier version of CLI (0.261.0 or earlier).
-  in override_job.yml:33:24
-
-Error: path ../target/jar2.jar is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:41:24). Please update the path to be relative to the file where it is defined or use earlier version of CLI (0.261.0 or earlier).
-  in override_job.yml:41:24
-
 Error: path ../src/notebook2.py is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_pipeline.yml:13:23). Please update the path to be relative to the file where it is defined or use earlier version of CLI (0.261.0 or earlier).
   in override_pipeline.yml:13:23
 
diff --git a/acceptance/bundle/paths/outside_root_no_sync/bundle/databricks.yml b/acceptance/bundle/paths/outside_root_no_sync/bundle/databricks.yml
@@ -0,0 +1,11 @@
+bundle:
+  name: outside_root_no_sync
+
+resources:
+  jobs:
+    my_job:
+      name: include_outside_root
+      tasks:
+        - task_key: task_key
+          notebook_task:
+            notebook_path: "../src/notebook.py"
diff --git a/acceptance/bundle/paths/outside_root_no_sync/out.test.toml b/acceptance/bundle/paths/outside_root_no_sync/out.test.toml
@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]
diff --git a/acceptance/bundle/paths/outside_root_no_sync/output.txt b/acceptance/bundle/paths/outside_root_no_sync/output.txt
@@ -0,0 +1,28 @@
+
+>>> [CLI] bundle validate --output json
+Error: path [TEST_TMP_DIR]/src/notebook.py is not contained in sync root path
+
+{
+  "resources": {
+    "jobs": {
+      "my_job": {
+        "max_concurrent_runs": 1,
+        "name": "include_outside_root",
+        "permissions": [],
+        "queue": {
+          "enabled": true
+        },
+        "tasks": [
+          {
+            "notebook_task": {
+              "notebook_path": "../src/notebook.py"
+            },
+            "task_key": "task_key"
+          }
+        ]
+      }
+    }
+  }
+}
+
+Exit code: 1
diff --git a/acceptance/bundle/paths/outside_root_no_sync/script b/acceptance/bundle/paths/outside_root_no_sync/script
@@ -0,0 +1,2 @@
+cd bundle
+trace $CLI bundle validate --output json | jq 'pick(.resources)'
diff --git a/acceptance/bundle/paths/outside_root_no_sync/src/notebook.py b/acceptance/bundle/paths/outside_root_no_sync/src/notebook.py
@@ -0,0 +1,2 @@
+# Databricks notebook source
+1 + 1
diff --git a/acceptance/bundle/paths/outside_root_no_sync/test.toml b/acceptance/bundle/paths/outside_root_no_sync/test.toml
@@ -0,0 +1,3 @@
+[[Repls]]
+Old = '\\'
+New = '/'
diff --git a/bundle/config/mutator/paths/job_libraries_paths_visitor.go b/bundle/config/mutator/paths/job_libraries_paths_visitor.go
@@ -0,0 +1,100 @@
+package paths
+
+import (
+	"github.com/databricks/cli/bundle/libraries"
+	"github.com/databricks/cli/libs/dyn"
+)
+
+func jobTaskLibrariesRewritePatterns(base dyn.Pattern) []jobRewritePattern {
+	return []jobRewritePattern{
+		{
+			base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("whl")),
+			TranslateModeLocalRelative,
+			noSkipRewrite,
+		},
+		{
+			base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("jar")),
+			TranslateModeLocalRelative,
+			noSkipRewrite,
+		},
+	}
+}
+
+func jobLibrariesRewritePatterns() []jobRewritePattern {
+	// Base pattern to match all tasks in all jobs.
+	base := dyn.NewPattern(
+		dyn.Key("resources"),
+		dyn.Key("jobs"),
+		dyn.AnyKey(),
+		dyn.Key("tasks"),
+		dyn.AnyIndex(),
+	)
+
+	// Compile list of patterns and their respective rewrite functions.
+	jobEnvironmentsPatterns := []jobRewritePattern{
+		{
+			dyn.NewPattern(
+				dyn.Key("resources"),
+				dyn.Key("jobs"),
+				dyn.AnyKey(),
+				dyn.Key("environments"),
+				dyn.AnyIndex(),
+				dyn.Key("spec"),
+				dyn.Key("dependencies"),
+				dyn.AnyIndex(),
+			),
+			TranslateModeLocalRelativeWithPrefix,
+			func(s string) bool {
+				return !libraries.IsLibraryLocal(s)
+			},
+		},
+	}
+
+	jobEnvironmentsWithRequirementsPatterns := []jobRewritePattern{
+		{
+			dyn.NewPattern(
+				dyn.Key("resources"),
+				dyn.Key("jobs"),
+				dyn.AnyKey(),
+				dyn.Key("environments"),
+				dyn.AnyIndex(),
+				dyn.Key("spec"),
+				dyn.Key("dependencies"),
+				dyn.AnyIndex(),
+			),
+			TranslateModeEnvironmentRequirements,
+			func(s string) bool {
+				_, ok := libraries.IsLocalRequirementsFile(s)
+				return !ok
+			},
+		},
+	}
+
+	taskPatterns := jobTaskLibrariesRewritePatterns(base)
+	forEachPatterns := jobTaskLibrariesRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task")))
+	allPatterns := append(taskPatterns, jobEnvironmentsPatterns...)
+	allPatterns = append(allPatterns, jobEnvironmentsWithRequirementsPatterns...)
+	allPatterns = append(allPatterns, forEachPatterns...)
+	return allPatterns
+}
+
+// VisitJobLibrariesPaths visits all libraries related paths in job resources and applies a function to each path.
+func VisitJobLibrariesPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) {
+	var err error
+	newValue := value
+
+	for _, rewritePattern := range jobLibrariesRewritePatterns() {
+		newValue, err = dyn.MapByPattern(newValue, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+			if rewritePattern.skipRewrite(v.MustString()) {
+				return v, nil
+			}
+
+			return fn(p, rewritePattern.mode, v)
+		})
+		if err != nil {
+			return dyn.InvalidValue, err
+		}
+	}
+
+	return newValue, nil
+}
diff --git a/bundle/config/mutator/paths/job_libraries_paths_visitor_test.go b/bundle/config/mutator/paths/job_libraries_paths_visitor_test.go
@@ -0,0 +1,116 @@
+package paths
+
+import (
+	"testing"
+
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/databricks-sdk-go/service/compute"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestVisitJobLibrariesPaths(t *testing.T) {
+	task0 := jobs.Task{
+		NotebookTask: &jobs.NotebookTask{
+			NotebookPath: "abc",
+		},
+	}
+	task1 := jobs.Task{
+		SparkPythonTask: &jobs.SparkPythonTask{
+			PythonFile: "abc",
+		},
+	}
+	task2 := jobs.Task{
+		DbtTask: &jobs.DbtTask{
+			ProjectDirectory: "abc",
+		},
+	}
+	task3 := jobs.Task{
+		SqlTask: &jobs.SqlTask{
+			File: &jobs.SqlTaskFile{
+				Path: "abc",
+			},
+		},
+	}
+	task4 := jobs.Task{
+		Libraries: []compute.Library{
+			{Whl: "dist/foo.whl"},
+		},
+	}
+	task5 := jobs.Task{
+		Libraries: []compute.Library{
+			{Jar: "dist/foo.jar"},
+		},
+	}
+	task6 := jobs.Task{
+		Libraries: []compute.Library{
+			{Requirements: "requirements.txt"},
+		},
+	}
+
+	job0 := &resources.Job{
+		JobSettings: jobs.JobSettings{
+			Tasks: []jobs.Task{
+				task0,
+				task1,
+				task2,
+				task3,
+				task4,
+				task5,
+				task6,
+			},
+		},
+	}
+
+	root := config.Root{
+		Resources: config.Resources{
+			Jobs: map[string]*resources.Job{
+				"job0": job0,
+			},
+		},
+	}
+
+	actual := collectVisitedPaths(t, root, VisitJobLibrariesPaths)
+	expected := []dyn.Path{
+		dyn.MustPathFromString("resources.jobs.job0.tasks[4].libraries[0].whl"),
+		dyn.MustPathFromString("resources.jobs.job0.tasks[5].libraries[0].jar"),
+	}
+
+	assert.ElementsMatch(t, expected, actual)
+}
+
+func TestVisitJobLibrariesPaths_environments(t *testing.T) {
+	environment0 := jobs.JobEnvironment{
+		Spec: &compute.Environment{
+			Dependencies: []string{
+				"dist_0/*.whl",
+				"dist_1/*.whl",
+			},
+		},
+	}
+	job0 := &resources.Job{
+		JobSettings: jobs.JobSettings{
+			Environments: []jobs.JobEnvironment{
+				environment0,
+			},
+		},
+	}
+
+	root := config.Root{
+		Resources: config.Resources{
+			Jobs: map[string]*resources.Job{
+				"job0": job0,
+			},
+		},
+	}
+
+	actual := collectVisitedPaths(t, root, VisitJobLibrariesPaths)
+	expected := []dyn.Path{
+		dyn.MustPathFromString("resources.jobs.job0.environments[0].spec.dependencies[0]"),
+		dyn.MustPathFromString("resources.jobs.job0.environments[0].spec.dependencies[1]"),
+	}
+
+	assert.ElementsMatch(t, expected, actual)
+}
diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go
diff --git a/bundle/config/mutator/paths/job_paths_visitor_test.go b/bundle/config/mutator/paths/job_paths_visitor_test.go
diff --git a/bundle/config/mutator/paths/visitor.go b/bundle/config/mutator/paths/visitor.go
diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go
diff --git a/bundle/config/mutator/translate_paths_jobs.go b/bundle/config/mutator/translate_paths_jobs.go

Original file line number	Diff line number	Diff line change
`@@ -24,7 +24,7 @@`
`24`	`24`	`"whl": "dist/wheel1.whl"`
`25`	`25`	`},`
`26`	`26`	`{`
`27`		`- "whl": "dist/wheel2.whl"`
	`27`	`+ "whl": "../dist/wheel2.whl"`
`28`	`28`	`}`
`29`	`29`	`],`
`30`	`30`	`"python_wheel_task": {`
`@@ -39,7 +39,7 @@`
`39`	`39`	`"jar": "target/jar1.jar"`
`40`	`40`	`},`
`41`	`41`	`{`
`42`		`- "jar": "target/jar2.jar"`
	`42`	`+ "jar": "../target/jar2.jar"`
`43`	`43`	`}`
`44`	`44`	`],`
`45`	`45`	`"spark_jar_task": {`