Merge branch 'master' of github.com:fsspec/filesystem_spec into patch-1

OneSizeFitsQuorum · OneSizeFitsQuorum · commit d80b6556243e · 2025-11-17T10:16:59.000+08:00
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -14,18 +14,18 @@ jobs:
       fail-fast: false
       matrix:
         PY:
-          - "3.9"
           - "3.10"
           - "3.11"
           - "3.12"
           - "3.13"
+          - "3.14"
 
     env:
       CIRUN: true
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
 
@@ -50,7 +50,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
 
@@ -81,7 +81,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
 
@@ -124,7 +124,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Setup conda
         uses: conda-incubator/setup-miniconda@v3
@@ -145,5 +145,5 @@ jobs:
         shell: bash -l {0}
         run: |
           cd ${{ matrix.FRIEND }}
-          pytest -v
+          pytest -v -W ignore::pytest.PytestRemovedIn9Warning
           cd ..
diff --git a/.github/workflows/pypipublish.yaml b/.github/workflows/pypipublish.yaml
@@ -8,9 +8,9 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v6
         with:
           python-version: "3.x"
       - name: Install dependencies
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -13,9 +13,8 @@ repos:
       - id: check-json
       - id: check-yaml
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.12.2
+    rev: v0.14.3
     hooks:
-      # Run the linter.
       - id: ruff-check
         args: [ --fix, "--show-fixes"]
       - id: ruff-format
diff --git a/README.md b/README.md
@@ -47,7 +47,7 @@ CI runtime. For local use, pick a version suitable for you.
 
 ```bash
 # For a new environment (mamba / conda).
-mamba create -n fsspec -c conda-forge  python=3.9 -y
+mamba create -n fsspec -c conda-forge  python=3.10 -y
 conda activate fsspec
 
 # Standard dev install with docs and tests.
diff --git a/docs/environment.yml b/docs/environment.yml
@@ -2,4 +2,4 @@ name: fsspec
 channels:
   - defaults
 dependencies:
-  - python=3.9
+  - python=3.10
diff --git a/fsspec/caching.py b/fsspec/caching.py
@@ -8,18 +8,11 @@
 import threading
 import warnings
 from collections import OrderedDict
+from collections.abc import Callable
 from concurrent.futures import Future, ThreadPoolExecutor
 from itertools import groupby
 from operator import itemgetter
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Callable,
-    ClassVar,
-    Generic,
-    NamedTuple,
-    TypeVar,
-)
+from typing import TYPE_CHECKING, Any, ClassVar, Generic, NamedTuple, TypeVar
 
 if TYPE_CHECKING:
     import mmap
diff --git a/fsspec/implementations/arrow.py b/fsspec/implementations/arrow.py
@@ -205,11 +205,11 @@ def modified(self, path):
         return self.fs.get_file_info(path).mtime
 
     def cat_file(self, path, start=None, end=None, **kwargs):
-        kwargs["seekable"] = start not in [None, 0]
+        kwargs.setdefault("seekable", start not in [None, 0])
         return super().cat_file(path, start=None, end=None, **kwargs)
 
     def get_file(self, rpath, lpath, **kwargs):
-        kwargs["seekable"] = False
+        kwargs.setdefault("seekable", False)
         super().get_file(rpath, lpath, **kwargs)
 
 
diff --git a/fsspec/implementations/cache_metadata.py b/fsspec/implementations/cache_metadata.py
@@ -15,9 +15,7 @@
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
-    from typing import Any, Literal
-
-    from typing_extensions import TypeAlias
+    from typing import Any, Literal, TypeAlias
 
     from .cached import CachingFileSystem
 
diff --git a/fsspec/implementations/cached.py b/fsspec/implementations/cached.py
@@ -6,8 +6,9 @@
 import tempfile
 import time
 import weakref
+from collections.abc import Callable
 from shutil import rmtree
-from typing import TYPE_CHECKING, Any, Callable, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar
 
 from fsspec import filesystem
 from fsspec.callbacks import DEFAULT_CALLBACK
diff --git a/fsspec/implementations/data.py b/fsspec/implementations/data.py
@@ -1,6 +1,5 @@
 import base64
 import io
-from typing import Optional
 from urllib.parse import unquote
 
 from fsspec import AbstractFileSystem
@@ -50,7 +49,7 @@ def _open(
         return io.BytesIO(self.cat_file(path))
 
     @staticmethod
-    def encode(data: bytes, mime: Optional[str] = None):
+    def encode(data: bytes, mime: str | None = None):
         """Format the given data into data-URL syntax
 
         This version always base64 encodes, even when the data is ascii/url-safe.
diff --git a/fsspec/implementations/libarchive.py b/fsspec/implementations/libarchive.py
@@ -195,7 +195,7 @@ def _open(
         if mode != "rb":
             raise NotImplementedError
 
-        data = bytes()
+        data = b""
         with self._open_archive() as arc:
             for entry in arc:
                 if entry.pathname != path:
diff --git a/fsspec/implementations/reference.py b/fsspec/implementations/reference.py
@@ -219,7 +219,7 @@ def create(root, storage_options=None, fs=None, record_size=10000, **kwargs):
         fs.pipe("/".join([root, ".zmetadata"]), json.dumps(met).encode())
         return LazyReferenceMapper(root, fs, **kwargs)
 
-    @lru_cache()
+    @lru_cache
     def listdir(self):
         """List top-level directories"""
         dirs = (p.rsplit("/", 1)[0] for p in self.zmetadata if not p.startswith(".z"))
diff --git a/fsspec/implementations/tests/test_arrow.py b/fsspec/implementations/tests/test_arrow.py
@@ -268,3 +268,91 @@ def test_get_kwargs_from_urls_hadoop_fs():
     assert kwargs["host"] == "localhost"
     assert kwargs["port"] == 8020
     assert "replication" not in kwargs
+
+
+def test_get_file_seekable_default(fs, remote_dir, tmp_path):
+    """Test that get_file defaults to seekable=False but allows override."""
+    data = b"test data for seekable"
+
+    # Create a test file
+    with fs.open(remote_dir + "/test_file.txt", "wb") as f:
+        f.write(data)
+
+    # Test default behavior (seekable=False)
+    local_file = tmp_path / "test_default.txt"
+    fs.get_file(remote_dir + "/test_file.txt", str(local_file))
+    with open(local_file, "rb") as f:
+        assert f.read() == data
+
+    # Test with explicit seekable=True
+    local_file_seekable = tmp_path / "test_seekable.txt"
+    fs.get_file(remote_dir + "/test_file.txt", str(local_file_seekable), seekable=True)
+    with open(local_file_seekable, "rb") as f:
+        assert f.read() == data
+
+    # Test with explicit seekable=False
+    local_file_not_seekable = tmp_path / "test_not_seekable.txt"
+    fs.get_file(
+        remote_dir + "/test_file.txt", str(local_file_not_seekable), seekable=False
+    )
+    with open(local_file_not_seekable, "rb") as f:
+        assert f.read() == data
+
+
+def test_cat_file_seekable_override(fs, remote_dir):
+    """Test that cat_file allows seekable to be overridden."""
+    data = b"test data for cat_file seekable"
+
+    # Create a test file
+    with fs.open(remote_dir + "/test_cat.txt", "wb") as f:
+        f.write(data)
+
+    # Test default behavior - when start is None, seekable should default to False
+    result = fs.cat_file(remote_dir + "/test_cat.txt")
+    assert result == data
+
+    # Test with explicit seekable=True even when start is None
+    result = fs.cat_file(remote_dir + "/test_cat.txt", seekable=True)
+    assert result == data
+
+    # Test with explicit seekable=False
+    result = fs.cat_file(remote_dir + "/test_cat.txt", seekable=False)
+    assert result == data
+
+
+def test_seekable_true_allows_size_method(fs, remote_dir):
+    """Test that size() method works when seekable=True."""
+    data = b"test data for size method" * 10
+
+    # Create a test file
+    test_file = remote_dir + "/test_size.txt"
+    with fs.open(test_file, "wb") as f:
+        f.write(data)
+
+    # Open with seekable=True - size() should work
+    with fs.open(test_file, "rb", seekable=True) as f:
+        assert f.seekable() is True
+        # Verify size() method works and returns correct size
+        file_size = f.size()
+        assert file_size == len(data)
+        # Also verify we can read the data
+        assert f.read() == data
+
+
+def test_seekable_false_prevents_size_method(fs, remote_dir):
+    """Test that size() method raises OSError when seekable=False."""
+    data = b"test data for size method" * 10
+
+    # Create a test file
+    test_file = remote_dir + "/test_size.txt"
+    with fs.open(test_file, "wb") as f:
+        f.write(data)
+
+    # Open with seekable=False - size() should raise OSError
+    with fs.open(test_file, "rb", seekable=False) as f:
+        assert f.seekable() is False
+        # Verify size() raises OSError
+        with pytest.raises(OSError, match="only valid on seekable files"):
+            f.size()
+        # Verify we can still read the data
+        assert f.read() == data
diff --git a/fsspec/implementations/tests/test_dbfs.py b/fsspec/implementations/tests/test_dbfs.py
@@ -23,16 +23,14 @@
 """
 
 import os
-import sys
 from urllib.parse import urlparse
 
 import numpy
 import pytest
 
 import fsspec
 
-if sys.version_info >= (3, 10):
-    pytest.skip("These tests need to be re-recorded.", allow_module_level=True)
+pytest.skip("These tests need to be re-recorded.", allow_module_level=True)
 
 DUMMY_INSTANCE = "my_instance.com"
 INSTANCE = os.getenv("DBFS_INSTANCE", DUMMY_INSTANCE)
diff --git a/fsspec/json.py b/fsspec/json.py
@@ -1,13 +1,8 @@
 import json
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
 from contextlib import suppress
 from pathlib import PurePath
-from typing import (
-    Any,
-    Callable,
-    ClassVar,
-    Optional,
-)
+from typing import Any, ClassVar
 
 from .registry import _import_class, get_filesystem_class
 from .spec import AbstractFileSystem
@@ -45,12 +40,12 @@ class FilesystemJSONDecoder(json.JSONDecoder):
     def __init__(
         self,
         *,
-        object_hook: Optional[Callable[[dict[str, Any]], Any]] = None,
-        parse_float: Optional[Callable[[str], Any]] = None,
-        parse_int: Optional[Callable[[str], Any]] = None,
-        parse_constant: Optional[Callable[[str], Any]] = None,
+        object_hook: Callable[[dict[str, Any]], Any] | None = None,
+        parse_float: Callable[[str], Any] | None = None,
+        parse_int: Callable[[str], Any] | None = None,
+        parse_constant: Callable[[str], Any] | None = None,
         strict: bool = True,
-        object_pairs_hook: Optional[Callable[[list[tuple[str, Any]]], Any]] = None,
+        object_pairs_hook: Callable[[list[tuple[str, Any]]], Any] | None = None,
     ) -> None:
         self.original_object_hook = object_hook
 
diff --git a/fsspec/utils.py b/fsspec/utils.py
@@ -7,23 +7,16 @@
 import re
 import sys
 import tempfile
-from collections.abc import Iterable, Iterator, Sequence
+from collections.abc import Callable, Iterable, Iterator, Sequence
 from functools import partial
 from hashlib import md5
 from importlib.metadata import version
-from typing import (
-    IO,
-    TYPE_CHECKING,
-    Any,
-    Callable,
-    TypeVar,
-)
+from typing import IO, TYPE_CHECKING, Any, TypeVar
 from urllib.parse import urlsplit
 
 if TYPE_CHECKING:
     import pathlib
-
-    from typing_extensions import TypeGuard
+    from typing import TypeGuard
 
     from fsspec.spec import AbstractFileSystem
 
diff --git a/pyproject.toml b/pyproject.toml
@@ -9,18 +9,18 @@ description = "File-system specification"
 readme = "README.md"
 license = "BSD-3-Clause"
 license-files = ["LICENSE"]
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 maintainers = [{ name = "Martin Durant", email = "mdurant@anaconda.com" }]
 keywords = ["file"]
 classifiers = [
     "Development Status :: 4 - Beta",
     "Intended Audience :: Developers",
     "Operating System :: OS Independent",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
 ]
 
 [project.optional-dependencies]
@@ -194,6 +194,8 @@ ignore = [
 	"B026",
 	# No explicit `stacklevel` keyword argument found
 	"B028",
+    # `zip` without explicit `strict` keyword
+    "B905",
     # Assigning lambda expression
     "E731",
     # Ambiguous variable names
@@ -220,8 +222,6 @@ ignore = [
     "SIM115",
     "SIM117",
     "TC003",
-    # https://github.com/astral-sh/ruff/issues/7871
-    "UP038",
     # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules
     "W191",
     "E111",
