From 59a87b4b1c9f3228bc0f80c4005a7769ec6832a0 Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Thu, 21 Nov 2024 12:57:56 -0600 Subject: [PATCH 1/3] Ensure reference FS wraps any sync filesystems --- fsspec/implementations/reference.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fsspec/implementations/reference.py b/fsspec/implementations/reference.py index 5c5372cda..60fb5d57b 100644 --- a/fsspec/implementations/reference.py +++ b/fsspec/implementations/reference.py @@ -20,6 +20,7 @@ from fsspec.asyn import AsyncFileSystem from fsspec.callbacks import DEFAULT_CALLBACK from fsspec.core import filesystem, open, split_protocol +from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper from fsspec.utils import isfilelike, merge_offset_ranges, other_paths logger = logging.getLogger("fsspec.reference") @@ -757,6 +758,10 @@ def __init__( self.fss[remote_protocol] = fs self.fss[None] = fs or filesystem("file") # default one + # Wrap any non-async filesystems to ensure async methods are available below + for k, f in self.fss.items(): + if not f.async_impl: + self.fss[k] = AsyncFileSystemWrapper(f) def _cat_common(self, path, start=None, end=None): path = self._strip_protocol(path) From 77d4eeb68a9dc1aeee8a7a4894adc7c6a6132cbf Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Wed, 4 Dec 2024 12:57:24 -0600 Subject: [PATCH 2/3] Add protocol to async wrapper FS --- fsspec/implementations/asyn_wrapper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fsspec/implementations/asyn_wrapper.py b/fsspec/implementations/asyn_wrapper.py index 0d462652b..e15500123 100644 --- a/fsspec/implementations/asyn_wrapper.py +++ b/fsspec/implementations/asyn_wrapper.py @@ -46,6 +46,7 @@ def __init__(self, sync_fs, *args, **kwargs): super().__init__(*args, **kwargs) self.asynchronous = True self.fs = sync_fs + self.protocol = self.fs.protocol self._wrap_all_sync_methods() @property From 4a99145ea2ed1de5726284a6b3d4f270bd99fb9c Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Wed, 4 Dec 2024 16:26:26 -0600 Subject: [PATCH 3/3] Update reference FS tests for async wrapper --- fsspec/implementations/asyn_wrapper.py | 12 ++++++------ fsspec/implementations/tests/test_reference.py | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/fsspec/implementations/asyn_wrapper.py b/fsspec/implementations/asyn_wrapper.py index e15500123..43604a28c 100644 --- a/fsspec/implementations/asyn_wrapper.py +++ b/fsspec/implementations/asyn_wrapper.py @@ -45,27 +45,27 @@ class AsyncFileSystemWrapper(AsyncFileSystem): def __init__(self, sync_fs, *args, **kwargs): super().__init__(*args, **kwargs) self.asynchronous = True - self.fs = sync_fs - self.protocol = self.fs.protocol + self.sync_fs = sync_fs + self.protocol = self.sync_fs.protocol self._wrap_all_sync_methods() @property def fsid(self): - return f"async_{self.fs.fsid}" + return f"async_{self.sync_fs.fsid}" def _wrap_all_sync_methods(self): """ Wrap all synchronous methods of the underlying filesystem with asynchronous versions. """ - for method_name in dir(self.fs): + for method_name in dir(self.sync_fs): if method_name.startswith("_"): continue - attr = inspect.getattr_static(self.fs, method_name) + attr = inspect.getattr_static(self.sync_fs, method_name) if isinstance(attr, property): continue - method = getattr(self.fs, method_name) + method = getattr(self.sync_fs, method_name) if callable(method) and not asyncio.iscoroutinefunction(method): async_method = async_wrapper(method, obj=self) setattr(self, f"_{method_name}", async_method) diff --git a/fsspec/implementations/tests/test_reference.py b/fsspec/implementations/tests/test_reference.py index 0063d11b0..dd9ff688f 100644 --- a/fsspec/implementations/tests/test_reference.py +++ b/fsspec/implementations/tests/test_reference.py @@ -458,7 +458,8 @@ def test_fss_has_defaults(m): assert fs.fss["memory"].protocol == "memory" fs = fsspec.filesystem("reference", fs=m, fo={}) - assert fs.fss[None] is m + # Default behavior here wraps synchronous filesystems to enable the async API + assert fs.fss[None].sync_fs is m fs = fsspec.filesystem("reference", fs={"memory": m}, fo={}) assert fs.fss["memory"] is m