Fix python311 compatibility (#69)

* Add Python 3.11 testing

* upath.core: fix python3.11 compatibility

mkdir was the one method broken by moving away from _accessors
in Python3.11

* upath.core: explictly disable more methods broken by Python3.11

* tests: add tests for mkdir exist_ok

* tests: cloud filesystems require a file for dirs

* tests: mark unsupported http tests as skipped

* upath.implementations.hdfs: fix mkdir to correctly handle exist_ok

* upath.core: catch FileExistsError in mkdir and fix mode kwarg

* upath.implementations.cloud: fix exist_ok=False for mkdir

* ci: use newest setup-python and checkout actions

* black formatting

* nox: add workaround for missing py311 aiohttp wheels

* noxfile: lint only the upath package

* tests: better name for empty dir support flag

Co-authored-by: Norman Rzepka <[email protected]>

Author: ap--

.github/workflows/python.yml

@@ -7,13 +7,13 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9, "3.10"]
+        python-version: [3.7, 3.8, 3.9, "3.10", "3.11-dev"]
         os: [ubuntu-latest, windows-latest]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

noxfile.py

@@ -1,3 +1,5 @@
+import sys
+
 import nox
 from pathlib import Path
 
@@ -17,7 +19,7 @@ def black(session):
 @nox.session()
 def lint(session):
     session.install("flake8")
-    session.run(*"flake8".split())
+    session.run("flake8", "upath")
 
 
 @nox.session()
@@ -27,6 +29,12 @@ def install(session):
 
 @nox.session()
 def smoke(session):
+    if (3, 10) < sys.version_info <= (3, 11, 0, "final"):
+        # workaround for missing aiohttp wheels for py3.11
+        session.install("aiohttp", "--no-binary", "aiohttp", env={
+            "AIOHTTP_NO_EXTENSIONS": "1"
+        })
+
     session.install(
         "pytest",
         "adlfs",

setup.py

@@ -16,5 +16,5 @@
     description="pathlib api extended to use fsspec backends",
     long_description=long_description,
     long_description_content_type="text/markdown",
-    license="MIT"
+    license="MIT",
 )

upath/core.py

@@ -29,7 +29,7 @@ def __init__(self, parsed_url: ParseResult, **kwargs):
     def _format_path(self, path: "UPath") -> str:
         return path.path
 
-    def open(self, path, mode='r', *args, **kwargs):
+    def open(self, path, mode="r", *args, **kwargs):
         return self._fs.open(self._format_path(path), mode, *args, **kwargs)
 
     def stat(self, path, **kwargs):
@@ -185,6 +185,9 @@ def parent(self):
     def stat(self):
         return self._accessor.stat(self)
 
+    def samefile(self, other_path):
+        raise NotImplementedError
+
     def iterdir(self):
         """Iterate over the files in this directory.  Does not yield any
         result for the special paths '.' and '..'.
@@ -223,18 +226,36 @@ def relative_to(self, *other):
         output._kwargs = self._kwargs
         return output
 
+    def _scandir(self):
+        # provided in Python3.11 but not required in fsspec glob implementation
+        raise NotImplementedError
+
     def glob(self, pattern):
         path_pattern = self.joinpath(pattern)
         for name in self._accessor.glob(self, path_pattern):
             name = self._sub_path(name)
             name = name.split(self._flavour.sep)
             yield self._make_child(name)
 
+    def rglob(self, pattern):
+        path_pattern = self.joinpath("**", pattern)
+        for name in self._accessor.glob(self, path_pattern):
+            name = self._sub_path(name)
+            name = name.split(self._flavour.sep)
+            yield self._make_child(name)
+
     def _sub_path(self, name):
         # only want the path name with iterdir
         sp = self.path
         return re.sub(f"^({sp}|{sp[1:]})/", "", name)
 
+    def absolute(self):
+        # fsspec paths are always absolute
+        return self
+
+    def resolve(self, strict=False):
+        raise NotImplementedError
+
     def exists(self):
         """
         Whether this path exists.
@@ -290,6 +311,9 @@ def is_block_device(self):
     def is_char_device(self):
         return False
 
+    def is_absolute(self):
+        return True
+
     def unlink(self, missing_ok=False):
         if not self.exists():
             if not missing_ok:
@@ -308,13 +332,25 @@ def rmdir(self, recursive=True):
             raise NotDirectoryError
         self._accessor.rm(self, recursive=recursive)
 
-    def chmod(self, mod):
+    def chmod(self, mode, *, follow_symlinks=True):
         raise NotImplementedError
 
     def rename(self, target):
         # can be implemented, but may be tricky
         raise NotImplementedError
 
+    def replace(self, target):
+        raise NotImplementedError
+
+    def symlink_to(self, target, target_is_directory=False):
+        raise NotImplementedError
+
+    def hardlink_to(self, target):
+        raise NotImplementedError
+
+    def link_to(self, target):
+        raise NotImplementedError
+
     def cwd(self):
         raise NotImplementedError
 
@@ -342,6 +378,28 @@ def readlink(self):
     def touch(self, truncate=True, **kwargs):
         self._accessor.touch(self, truncate=truncate, **kwargs)
 
+    def mkdir(self, mode=0o777, parents=False, exist_ok=False):
+        """
+        Create a new directory at this given path.
+        """
+        if parents:
+            self._accessor.mkdir(
+                self,
+                create_parents=True,
+                exist_ok=exist_ok,
+                mode=mode,
+            )
+        else:
+            try:
+                self._accessor.mkdir(
+                    self,
+                    create_parents=False,
+                    mode=mode,
+                )
+            except FileExistsError:
+                if not exist_ok or not self.is_dir():
+                    raise
+
     @classmethod
     def _from_parts(cls, args, url=None, **kwargs):
         obj = object.__new__(cls)
@@ -426,7 +484,7 @@ def with_suffix(self, suffix):
         f = self._flavour
         if f.sep in suffix or f.altsep and f.altsep in suffix:
             raise ValueError("Invalid suffix %r" % (suffix,))
-        if suffix and not suffix.startswith('.') or suffix == '.':
+        if suffix and not suffix.startswith(".") or suffix == ".":
             raise ValueError("Invalid suffix %r" % (suffix))
         name = self.name
         if not name:
@@ -435,17 +493,24 @@ def with_suffix(self, suffix):
         if not old_suffix:
             name = name + suffix
         else:
-            name = name[:-len(old_suffix)] + suffix
-        return self._from_parsed_parts(self._drv, self._root,
-                                       self._parts[:-1] + [name], url=self._url)
+            name = name[: -len(old_suffix)] + suffix
+        return self._from_parsed_parts(
+            self._drv, self._root, self._parts[:-1] + [name], url=self._url
+        )
 
     def with_name(self, name):
         """Return a new path with the file name changed."""
         if not self.name:
             raise ValueError("%r has an empty name" % (self,))
         drv, root, parts = self._flavour.parse_parts((name,))
-        if (not name or name[-1] in [self._flavour.sep, self._flavour.altsep]
-                or drv or root or len(parts) != 1):
+        if (
+            not name
+            or name[-1] in [self._flavour.sep, self._flavour.altsep]
+            or drv
+            or root
+            or len(parts) != 1
+        ):
             raise ValueError("Invalid name %r" % (name))
-        return self._from_parsed_parts(self._drv, self._root,
-                                       self._parts[:-1] + [name], url=self._url)
+        return self._from_parsed_parts(
+            self._drv, self._root, self._parts[:-1] + [name], url=self._url
+        )

upath/implementations/cloud.py

@@ -9,6 +9,15 @@ def _format_path(self, path):
         """
         return f"{path._url.netloc}/{path.path.lstrip('/')}"
 
+    def mkdir(self, path, create_parents=True, **kwargs):
+        if (
+            not create_parents
+            and not kwargs.get("exist_ok", False)
+            and self._fs.exists(self._format_path(path))
+        ):
+            raise FileExistsError
+        return super().mkdir(path, create_parents=create_parents, **kwargs)
+
 
 # project is not part of the path, but is part of the credentials
 class CloudPath(upath.core.UPath):

upath/implementations/hdfs.py

@@ -10,6 +10,15 @@ def touch(self, path, **kwargs):
         kwargs.pop("truncate", None)
         super().touch(path, **kwargs)
 
+    def mkdir(self, path, create_parents=True, **kwargs):
+        pth = self._format_path(path)
+        if create_parents:
+            return self._fs.makedirs(pth, **kwargs)
+        else:
+            if not kwargs.get("exist_ok", False) and self._fs.exists(pth):
+                raise FileExistsError
+            return self._fs.mkdir(pth, create_parents=create_parents, **kwargs)
+
 
 class HDFSPath(upath.core.UPath):
     _default_accessor = _HDFSAccessor

upath/tests/cases.py

@@ -9,6 +9,8 @@
 
 
 class BaseTests:
+    SUPPORTS_EMPTY_DIRS = True
+
     def test_cwd(self):
         with pytest.raises(NotImplementedError):
             self.path.cwd()
@@ -124,8 +126,25 @@ def test_lstat(self):
     def test_mkdir(self):
         new_dir = self.path.joinpath("new_dir")
         new_dir.mkdir()
+        if not self.SUPPORTS_EMPTY_DIRS:
+            new_dir.joinpath(".file").touch()
         assert new_dir.exists()
 
+    def test_mkdir_exists_ok_true(self):
+        new_dir = self.path.joinpath("new_dir_may_exists")
+        new_dir.mkdir()
+        if not self.SUPPORTS_EMPTY_DIRS:
+            new_dir.joinpath(".file").touch()
+        new_dir.mkdir(exist_ok=True)
+
+    def test_mkdir_exists_ok_false(self):
+        new_dir = self.path.joinpath("new_dir_may_not_exists")
+        new_dir.mkdir()
+        if not self.SUPPORTS_EMPTY_DIRS:
+            new_dir.joinpath(".file").touch()
+        with pytest.raises(FileExistsError):
+            new_dir.mkdir(exist_ok=False)
+
     def test_open(self):
         pass
 

upath/tests/implementations/test_azure.py

@@ -10,6 +10,8 @@
 @skip_on_windows
 @pytest.mark.usefixtures("path")
 class TestAzurePath(BaseTests):
+    SUPPORTS_EMPTY_DIRS = False
+
     @pytest.fixture(autouse=True, scope="function")
     def path(self, azurite_credentials, azure_fixture):
         account_name, connection_string = azurite_credentials
@@ -24,14 +26,8 @@ def path(self, azurite_credentials, azure_fixture):
     def test_is_AzurePath(self):
         assert isinstance(self.path, AzurePath)
 
-    def test_mkdir(self):
-        new_dir = self.path / "new_dir"
-        new_dir.mkdir()
-        (new_dir / "test.txt").touch()
-        assert new_dir.exists()
-
     def test_rmdir(self):
-        new_dir = self.path / "new_dir"
+        new_dir = self.path / "new_dir_rmdir"
         new_dir.mkdir()
         path = new_dir / "test.txt"
         path.write_text("hello")

upath/tests/implementations/test_gcs.py

@@ -10,6 +10,8 @@
 @skip_on_windows
 @pytest.mark.usefixtures("path")
 class TestGCSPath(BaseTests):
+    SUPPORTS_EMPTY_DIRS = False
+
     @pytest.fixture(autouse=True, scope="function")
     def path(self, gcs_fixture):
         path, endpoint_url = gcs_fixture
@@ -19,11 +21,6 @@ def path(self, gcs_fixture):
     def test_is_GCSPath(self):
         assert isinstance(self.path, GCSPath)
 
-    def test_mkdir(self):
-        new_dir = self.path.joinpath("new_dir")
-        new_dir.joinpath("test.txt").touch()
-        assert new_dir.exists()
-
     def test_rmdir(self):
         dirname = "rmdir_test"
         mock_dir = self.path.joinpath(dirname)

upath/tests/implementations/test_http.py

@@ -34,15 +34,27 @@ def path(self, http_fixture):
     def test_work_at_root(self):
         assert "folder" in (f.name for f in self.path.parent.iterdir())
 
+    @pytest.mark.skip
     def test_mkdir(self):
         pass
 
+    @pytest.mark.skip
+    def test_mkdir_exists_ok_false(self):
+        pass
+
+    @pytest.mark.skip
+    def test_mkdir_exists_ok_true(self):
+        pass
+
+    @pytest.mark.skip
     def test_touch_unlink(self):
         pass
 
+    @pytest.mark.skip
     def test_write_bytes(self, pathlib_base):
         pass
 
+    @pytest.mark.skip
     def test_write_text(self, pathlib_base):
         pass