Skip to content

Commit 4769cc7

Browse files
authored
Better error message for users trying to use abfss or s3 schemes with SDK DBUtils (#634)
## Changes DBUtils in the SDK only supports the `dbfs` and `file` schemes. Within DBFS, only paths in the root filesystem are supported, not mountpoints. Other schemes are not supported. To minimize customer confusion, we can check for this case during all DBUtils FS operations when constructing the path. ## Tests Added a unit test for this case. - [ ] `make test` run locally - [ ] `make fmt` applied - [ ] relevant integration tests applied
1 parent 19228af commit 4769cc7

File tree

2 files changed

+20
-8
lines changed

2 files changed

+20
-8
lines changed

databricks/sdk/mixins/files.py

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
from types import TracebackType
1212
from typing import (TYPE_CHECKING, AnyStr, BinaryIO, Generator, Iterable,
1313
Iterator, Type, Union)
14+
from urllib import parse
1415

1516
from .._property import _cached_property
1617
from ..errors import NotFound
@@ -570,15 +571,19 @@ def exists(self, path: str) -> bool:
570571
p = self._path(path)
571572
return p.exists()
572573

574+
__ALLOWED_SCHEMES = [None, 'file', 'dbfs']
575+
573576
def _path(self, src):
574-
src = str(src)
575-
if src.startswith('file:'):
576-
return _LocalPath(src)
577-
if src.startswith('dbfs:'):
578-
src = src[len('dbfs:'):]
579-
if src.startswith('/Volumes'):
580-
return _VolumesPath(self._files_api, src)
581-
return _DbfsPath(self._dbfs_api, src)
577+
src = parse.urlparse(str(src))
578+
if src.scheme and src.scheme not in self.__ALLOWED_SCHEMES:
579+
raise ValueError(
580+
f'unsupported scheme "{src.scheme}". DBUtils in the SDK only supports local, root DBFS, and '
581+
'UC Volumes paths, not external locations or DBFS mount points.')
582+
if src.scheme == 'file':
583+
return _LocalPath(src.geturl())
584+
if src.path.startswith('/Volumes'):
585+
return _VolumesPath(self._files_api, src.geturl())
586+
return _DbfsPath(self._dbfs_api, src.geturl())
582587

583588
def copy(self, src: str, dst: str, *, recursive=False, overwrite=False):
584589
"""Copy files between DBFS and local filesystems"""

tests/test_dbfs_mixins.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,3 +63,10 @@ def test_moving_local_dir_to_dbfs(config, tmp_path, mocker):
6363
def test_fs_path(config, path, expected_type):
6464
dbfs_ext = DbfsExt(config)
6565
assert isinstance(dbfs_ext._path(path), expected_type)
66+
67+
68+
def test_fs_path_invalid(config):
69+
dbfs_ext = DbfsExt(config)
70+
with pytest.raises(ValueError) as e:
71+
dbfs_ext._path('s3://path/to/file')
72+
assert 'unsupported scheme "s3"' in str(e.value)

0 commit comments

Comments
 (0)