fix: allow file:/// paths to reference assets in container images

Signed-off-by: sshpuntoff <[email protected]>

Author: sshpuntoff

NEXT_CHANGELOG.md

@@ -12,5 +12,6 @@
 * Update templates to use serverless environment version 4 and matching Python version ([#3897](https://github.com/databricks/cli/pull/3897))
 * Add a language prompt to the `default-minimal` template ([#3918](https://github.com/databricks/cli/pull/3918))
 * Add `default-scala` template for Scala projects with SBT build configuration and example code ([#3906](https://github.com/databricks/cli/pull/3906))
+* Allow `file://` URIs in job libraries to reference runtime filesystem paths (e.g., JARs pre-installed on clusters via init scripts). These paths are no longer treated as local files to upload. ([#3884](https://github.com/databricks/cli/pull/3884))
 
 ### API Changes

bundle/libraries/local_path.go

@@ -16,17 +16,18 @@ import (
 // - myfile.txt
 // - ./myfile.txt
 // - ../myfile.txt
-// - file:///foo/bar/myfile.txt
 //
 // The following paths are considered remote:
 //
+// - file:///opt/spark/jars/myfile.jar
 // - dbfs:/mnt/myfile.txt
 // - s3:/mybucket/myfile.txt
 // - /Users/[email protected]/myfile.txt
 func IsLocalPath(p string) bool {
-	// If the path has the explicit file scheme, it's a local path.
+	// If the path has the file:// scheme, it's a runtime path (remote).
+	// Users should use relative paths without scheme for local files to upload.
 	if strings.HasPrefix(p, "file://") {
-		return true
+		return false
 	}
 
 	// If the path has another scheme, it's a remote path.

bundle/libraries/local_path_test.go

@@ -8,22 +8,30 @@ import (
 )
 
 func TestIsLocalPath(t *testing.T) {
-	// Relative paths, paths with the file scheme, and Windows paths.
+	// Relative paths and Windows paths (local files to upload).
 	assert.True(t, IsLocalPath("some/local/path"))
 	assert.True(t, IsLocalPath("./some/local/path"))
-	assert.True(t, IsLocalPath("file://path/to/package"))
 	assert.True(t, IsLocalPath("C:\\path\\to\\package"))
 	assert.True(t, IsLocalPath("myfile.txt"))
 	assert.True(t, IsLocalPath("./myfile.txt"))
 	assert.True(t, IsLocalPath("../myfile.txt"))
-	assert.True(t, IsLocalPath("file:///foo/bar/myfile.txt"))
 
-	// Absolute paths.
+	// Absolute paths without scheme (remote).
 	assert.False(t, IsLocalPath("/some/full/path"))
 	assert.False(t, IsLocalPath("/Workspace/path/to/package"))
 	assert.False(t, IsLocalPath("/Users/path/to/package"))
 
-	// Paths with schemes.
+	// file:// URIs are runtime paths (remote - not uploaded).
+	assert.False(t, IsLocalPath("file:///foo/bar/myfile.txt"))
+	assert.False(t, IsLocalPath("file:///opt/spark/jars/driver.jar"))
+	assert.False(t, IsLocalPath("file:///"))
+	assert.False(t, IsLocalPath("file:///absolute/path"))
+	assert.False(t, IsLocalPath("file://path/to/package"))
+	assert.False(t, IsLocalPath("file://foo/bar/myfile.txt"))
+	assert.False(t, IsLocalPath("file://./relative.jar"))
+	assert.False(t, IsLocalPath("file://../lib/package.whl"))
+
+	// Paths with other schemes (remote).
 	assert.False(t, IsLocalPath("dbfs://path/to/package"))
 	assert.False(t, IsLocalPath("dbfs:/path/to/package"))
 	assert.False(t, IsLocalPath("s3://path/to/package"))
@@ -47,7 +55,6 @@ func TestIsLibraryLocal(t *testing.T) {
 		{path: ".\\..\\local\\*.whl", expected: true},
 		{path: "../../local/*.whl", expected: true},
 		{path: "..\\..\\local\\*.whl", expected: true},
-		{path: "file://path/to/package/whl.whl", expected: true},
 		{path: "local/foo-bar.whl", expected: true},
 
 		{path: "", expected: false},