Merge pull request #379 from gerritholl/shallow-find

Test for #378 to verify find respects maxdepth.

Author: martindurant

s3fs/core.py

@@ -3,7 +3,6 @@
 import logging
 import os
 import socket
-import time
 from typing import Tuple, Optional
 import weakref
 
@@ -204,9 +203,7 @@ async def _call_s3(self, method, *akwarglist, **kwargs):
                                                        **kwargs)
         for i in range(self.retries):
             try:
-                out = await method(**additional_kwargs)
-                locals().pop("err", None)  # break cycle following retry
-                return out
+                return await method(**additional_kwargs)
             except S3_RETRYABLE_ERRORS as e:
                 logger.debug("Retryable error: %s" % e)
                 err = e
@@ -446,6 +443,9 @@ async def _find(self, path, maxdepth=None, withdirs=None, detail=False):
         bucket, key, _ = self.split_path(path)
         if not bucket:
             raise ValueError("Cannot traverse all of S3")
+        if maxdepth:
+            return super().find(bucket + "/" + key, maxdepth=maxdepth, withdirs=withdirs,
+                                detail=detail)
         # TODO: implement find from dircache, if all listings are present
         # if refresh is False:
         #     out = incomplete_tree_dirs(self.dircache, path)
@@ -855,7 +855,7 @@ def isdir(self, path):
             return False
 
         # This only returns things within the path and NOT the path object itself
-        return bool(sync(self.loop, self._lsdir, path))
+        return bool(maybe_sync(self._lsdir, self, path))
 
     def ls(self, path, detail=False, refresh=False, **kwargs):
         """ List single "directory" with or without details
@@ -873,9 +873,9 @@ def ls(self, path, detail=False, refresh=False, **kwargs):
             additional arguments passed on
         """
         path = self._strip_protocol(path).rstrip('/')
-        files = sync(self.loop, self._ls, path, refresh=refresh)
+        files = maybe_sync(self._ls, self, path, refresh=refresh)
         if not files:
-            files = sync(self.loop, self._ls, self._parent(path), refresh=refresh)
+            files = maybe_sync(self._ls, self, self._parent(path), refresh=refresh)
             files = [o for o in files if o['name'].rstrip('/') == path
                      and o['type'] != 'directory']
         if detail:
@@ -1080,7 +1080,7 @@ def url(self, path, expires=3600, **kwargs):
             the number of seconds this signature will be good for.
         """
         bucket, key, version_id = self.split_path(path)
-        return sync(self.loop, self.s3.generate_presigned_url,
+        return maybe_sync(self.s3.generate_presigned_url, self,
             ClientMethod='get_object',
             Params=dict(Bucket=bucket, Key=key, **version_id_kw(version_id), **kwargs),
             ExpiresIn=expires)
@@ -1274,7 +1274,7 @@ def rm(self, path, recursive=False, **kwargs):
             bucket, key, _ = self.split_path(path)
             if not key and self.is_bucket_versioned(bucket):
                 # special path to completely remove versioned bucket
-                sync(self.loop, self._rm_versioned_bucket_contents, bucket)
+                maybe_sync(self._rm_versioned_bucket_contents, self, bucket)
         super().rm(path, recursive=recursive, **kwargs)
 
     def invalidate_cache(self, path=None):

s3fs/tests/test_s3fs.py

@@ -1679,3 +1679,15 @@ async def async_wrapper():
         await s3._s3.close()
 
     asyncio.run(_())
+
+
+def test_shallow_find(s3):
+    """Test that find method respects maxdepth.
+
+    Verify that the ``find`` method respects the ``maxdepth`` parameter.  With
+    ``maxdepth=1``, the results of ``find`` should be the same as those of
+    ``ls``, without returning subdirectories.  See also issue 378.
+    """
+
+    assert s3.ls(test_bucket_name) == s3.find(test_bucket_name, maxdepth=1, withdirs=True)
+    assert s3.ls(test_bucket_name) == s3.glob(test_bucket_name + "/*")