Merge branch 'fsspec:master' into master

Author: CyrilJl

fsspec/asyn.py

@@ -816,11 +816,9 @@ async def _glob(self, path, maxdepth=None, **kwargs):
             p: info
             for p, info in sorted(allpaths.items())
             if pattern.match(
-                (
-                    p + "/"
-                    if append_slash_to_dirname and info["type"] == "directory"
-                    else p
-                )
+                p + "/"
+                if append_slash_to_dirname and info["type"] == "directory"
+                else p
             )
         }
 

fsspec/caching.py

@@ -7,9 +7,9 @@
 import os
 import threading
 import warnings
+from concurrent.futures import Future, ThreadPoolExecutor
 from itertools import groupby
 from operator import itemgetter
-from concurrent.futures import Future, ThreadPoolExecutor
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -87,12 +87,7 @@ def _log_stats(self) -> str:
         if self.hit_count == 0 and self.miss_count == 0:
             # a cache that does nothing, this is for logs only
             return ""
-        return " , %s: %d hits, %d misses, %d total requested bytes" % (
-            self.name,
-            self.hit_count,
-            self.miss_count,
-            self.total_requested_bytes,
-        )
+        return f" , {self.name}: {self.hit_count} hits, {self.miss_count} misses, {self.total_requested_bytes} total requested bytes"
 
     def __repr__(self) -> str:
         # TODO: use rich for better formatting

fsspec/core.py

@@ -680,9 +680,7 @@ def get_fs_token_paths(
         elif not isinstance(paths, list):
             paths = list(paths)
     else:
-        if "w" in mode and expand:
-            paths = _expand_paths(paths, name_function, num)
-        elif "x" in mode and expand:
+        if ("w" in mode or "x" in mode) and expand:
             paths = _expand_paths(paths, name_function, num)
         elif "*" in paths:
             paths = [f for f in sorted(fs.glob(paths)) if not fs.isdir(f)]

fsspec/implementations/asyn_wrapper.py

@@ -1,6 +1,7 @@
 import asyncio
-import inspect
 import functools
+import inspect
+
 from fsspec.asyn import AsyncFileSystem
 
 

fsspec/implementations/cached.py

@@ -612,7 +612,7 @@ def cat(
         **kwargs,
     ):
         paths = self.expand_path(
-            path, recursive=recursive, maxdepth=kwargs.get("maxdepth", None)
+            path, recursive=recursive, maxdepth=kwargs.get("maxdepth")
         )
         getpaths = []
         storepaths = []

fsspec/implementations/ftp.py

@@ -387,7 +387,7 @@ def _mlsd2(ftp, path="."):
                 "size": split_line[4],
             },
         )
-        if "d" == this[1]["unix.mode"][0]:
+        if this[1]["unix.mode"][0] == "d":
             this[1]["type"] = "dir"
         else:
             this[1]["type"] = "file"

fsspec/implementations/reference.py

@@ -5,8 +5,8 @@
 import logging
 import math
 import os
-from itertools import chain
 from functools import lru_cache
+from itertools import chain
 from typing import TYPE_CHECKING, Literal
 
 import fsspec.core
@@ -176,7 +176,7 @@ def open_refs(field, record):
             try:
                 df = self.pd.read_parquet(data, engine=self.engine)
                 refs = {c: df[c].to_numpy() for c in df.columns}
-            except IOError:
+            except OSError:
                 refs = None
             return refs
 
@@ -431,7 +431,7 @@ def write(self, field, record, base_url=None, storage_options=None):
         if len(partition) < self.record_size:
             try:
                 original = self.open_refs(field, record)
-            except IOError:
+            except OSError:
                 pass
 
         if original:

fsspec/implementations/tests/test_asyn_wrapper.py

@@ -1,10 +1,12 @@
 import asyncio
-import pytest
 import os
 
+import pytest
+
 import fsspec
 from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper
 from fsspec.implementations.local import LocalFileSystem
+
 from .test_local import csv_files, filetexts
 
 

fsspec/implementations/tests/test_local.py

@@ -247,7 +247,7 @@ def test_not_found():
 def test_isfile():
     fs = LocalFileSystem()
     with filetexts(files, mode="b"):
-        for f in files.keys():
+        for f in files:
             assert fs.isfile(f)
             assert fs.isfile(f"file://{f}")
         assert not fs.isfile("not-a-file")
@@ -257,7 +257,7 @@ def test_isfile():
 def test_isdir():
     fs = LocalFileSystem()
     with filetexts(files, mode="b"):
-        for f in files.keys():
+        for f in files:
             assert fs.isdir(os.path.dirname(os.path.abspath(f)))
             assert not fs.isdir(f)
         assert not fs.isdir("not-a-dir")

fsspec/implementations/tests/test_smb.py

@@ -29,7 +29,7 @@
 
 
 def stop_docker(container):
-    cmd = shlex.split('docker ps -a -q --filter "name=%s"' % container)
+    cmd = shlex.split(f'docker ps -a -q --filter "name={container}"')
     cid = subprocess.check_output(cmd).strip().decode()
     if cid:
         subprocess.call(["docker", "rm", "-f", "-v", cid])