Skip to content

Fixes for _un_chain #1736

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Oct 23, 2024
Merged

Fixes for _un_chain #1736

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e936502
Check if cls is a caching file system.
groutr Oct 22, 2024
046c989
Expand statement for better readability/performance.
groutr Oct 22, 2024
77aa0f6
Test all builtin names for caches.
groutr Oct 22, 2024
03224ee
lint
martindurant Oct 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 14 additions & 10 deletions fsspec/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -329,12 +329,19 @@ def open_files(


def _un_chain(path, kwargs):
x = re.compile(".*[^a-z]+.*") # test for non protocol-like single word
bits = (
[p if "://" in p or x.match(p) else p + "://" for p in path.split("::")]
if "::" in path
else [path]
)
# Avoid a circular import
from fsspec.implementations.cached import CachingFileSystem

if "::" in path:
x = re.compile(".*[^a-z]+.*") # test for non protocol-like single word
bits = []
for p in path.split("::"):
if "://" in p or x.match(p):
bits.append(p)
else:
bits.append(p + "://")
else:
bits = [path]
# [[url, protocol, kwargs], ...]
out = []
previous_bit = None
Expand All @@ -351,10 +358,7 @@ def _un_chain(path, kwargs):
**kws,
)
bit = cls._strip_protocol(bit)
if (
protocol in {"blockcache", "filecache", "simplecache"}
and "target_protocol" not in kw
):
if "target_protocol" not in kw and issubclass(cls, CachingFileSystem):
bit = previous_bit
out.append((bit, protocol, kw))
previous_bit = bit
Expand Down
24 changes: 15 additions & 9 deletions fsspec/implementations/tests/test_cached.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,7 @@ def test_blockcache_workflow(ftp_writable, tmp_path, force_save_pickle):
assert f.read(5) == b"test\n"


@pytest.mark.parametrize("impl", ["filecache", "blockcache"])
@pytest.mark.parametrize("impl", ["filecache", "blockcache", "cached"])
def test_workflow(ftp_writable, impl):
host, port, user, pw = ftp_writable
fs = FTPFileSystem(host, port, user, pw)
Expand Down Expand Up @@ -295,7 +295,7 @@ def test_workflow(ftp_writable, impl):
) # new value, because we overwrote the cached location


@pytest.mark.parametrize("impl", ["simplecache", "blockcache"])
@pytest.mark.parametrize("impl", ["simplecache", "blockcache", "cached"])
def test_glob(ftp_writable, impl):
host, port, user, pw = ftp_writable
fs = FTPFileSystem(host, port, user, pw)
Expand Down Expand Up @@ -622,7 +622,7 @@ def open_raise(*_, **__):
assert "Cache save failed due to interpreter shutdown" in caplog.text


@pytest.mark.parametrize("impl", ["filecache", "simplecache", "blockcache"])
@pytest.mark.parametrize("impl", ["filecache", "simplecache", "blockcache", "cached"])
def test_local_filecache_creates_dir_if_needed(impl):
import tempfile

Expand Down Expand Up @@ -875,7 +875,7 @@ def test_filecache_with_checks():
assert fs.cat(f1) == data * 2 # changed, since origin changed


@pytest.mark.parametrize("impl", ["filecache", "simplecache", "blockcache"])
@pytest.mark.parametrize("impl", ["filecache", "simplecache", "blockcache", "cached"])
@pytest.mark.parametrize("fs", ["local", "multi"], indirect=["fs"])
def test_filecache_takes_fs_instance(impl, fs):
origin = tempfile.mkdtemp()
Expand All @@ -889,7 +889,7 @@ def test_filecache_takes_fs_instance(impl, fs):
assert fs2.cat(f1) == data


@pytest.mark.parametrize("impl", ["filecache", "simplecache", "blockcache"])
@pytest.mark.parametrize("impl", ["filecache", "simplecache", "blockcache", "cached"])
@pytest.mark.parametrize("fs", ["local", "multi"], indirect=["fs"])
def test_filecache_serialization(impl, fs):
fs1 = fsspec.filesystem(impl, fs=fs)
Expand Down Expand Up @@ -1031,7 +1031,9 @@ def test_multi_cache(protocol):
assert f.read() == b"hello"


@pytest.mark.parametrize("protocol", ["simplecache", "filecache", "blockcache"])
@pytest.mark.parametrize(
"protocol", ["simplecache", "filecache", "blockcache", "cached"]
)
def test_multi_cat(protocol, ftp_writable):
host, port, user, pw = ftp_writable
fs = FTPFileSystem(host, port, user, pw)
Expand Down Expand Up @@ -1064,7 +1066,9 @@ def test_multi_cache_chain(protocol):
assert files[0].read() == b"hello"


@pytest.mark.parametrize("protocol", ["blockcache", "simplecache", "filecache"])
@pytest.mark.parametrize(
"protocol", ["blockcache", "cached", "simplecache", "filecache"]
)
def test_strip(protocol):
fs = fsspec.filesystem(protocol, target_protocol="memory")
url1 = "memory://afile"
Expand Down Expand Up @@ -1235,9 +1239,11 @@ def test_cache_dir_auto_deleted(temp_cache, tmpdir):
assert local.exists(cache_dir)


@pytest.mark.parametrize("protocol", ["filecache", "blockcache", "simplecache"])
@pytest.mark.parametrize(
"protocol", ["filecache", "blockcache", "cached", "simplecache"]
)
def test_cache_size(tmpdir, protocol):
if win and protocol == "blockcache":
if win and protocol in {"blockcache", "cached"}:
pytest.skip("Windows file locking affects blockcache size tests")

source = os.path.join(tmpdir, "source")
Expand Down
Loading