Add SFTPPath implementation (#265)

* tests: add sshpath tests

* upath: add SFTPPath implementation

* tests: add paramiko dependency

* upath.implementations.sftp: remove monkeypatch

* tests: add missing protocols to registry tests

* tests: fix sftp tests for fsspec<2022.10.0

* tests: xfail mkdir sftp tests on old fsspec

Author: ap--

pyproject.toml

@@ -51,6 +51,7 @@ dev = [
     "s3fs",
     "moto[s3,server]",
     "webdav4[fsspec]",
+    "paramiko",
     "wsgidav",
     "cheroot",
     # "hadoop-test-cluster",

upath/_flavour.py

@@ -109,6 +109,8 @@ class WrappedFileSystemFlavour:  # (pathlib_abc.FlavourBase)
             "https",
             "s3",
             "s3a",
+            "sftp",
+            "ssh",
             "smb",
             "gs",
             "gcs",

upath/implementations/sftp.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+import sys
+from typing import Any
+from typing import Generator
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+from upath import UPath
+
+_unset: Any = object()
+
+
+class SFTPPath(UPath):
+    __slots__ = ()
+
+    def iterdir(self) -> Generator[Self, None, None]:
+        if not self.is_dir():
+            raise NotADirectoryError(str(self))
+        else:
+            return super().iterdir()

upath/registry.py

@@ -76,6 +76,8 @@ class _Registry(MutableMapping[str, "type[upath.UPath]"]):
         "memory": "upath.implementations.memory.MemoryPath",
         "s3": "upath.implementations.cloud.S3Path",
         "s3a": "upath.implementations.cloud.S3Path",
+        "sftp": "upath.implementations.sftp.SFTPPath",
+        "ssh": "upath.implementations.sftp.SFTPPath",
         "webdav": "upath.implementations.webdav.WebdavPath",
         "webdav+http": "upath.implementations.webdav.WebdavPath",
         "webdav+https": "upath.implementations.webdav.WebdavPath",

upath/tests/conftest.py

@@ -16,6 +16,7 @@
 from fsspec.registry import _registry
 from fsspec.registry import register_implementation
 from fsspec.utils import stringify_path
+from packaging.version import Version
 
 from .utils import posixify
 
@@ -464,3 +465,61 @@ def smb_fixture(local_testdir, smb_url, smb_container):
     smb.put(local_testdir, "/home/testdir", recursive=True)
     yield url
     smb.delete("/home/testdir", recursive=True)
+
+
+@pytest.fixture(scope="module")
+def ssh_container():
+    if shutil.which("docker") is None:
+        pytest.skip("docker not installed")
+
+    name = "fsspec_test_ssh"
+    stop_docker(name)
+    cmd = (
+        "docker run"
+        " -d"
+        f" --name {name}"
+        " -e USER_NAME=user"
+        " -e PASSWORD_ACCESS=true"
+        " -e USER_PASSWORD=pass"
+        " -p 2222:2222"
+        " linuxserver/openssh-server:latest"
+    )
+    try:
+        subprocess.run(shlex.split(cmd))
+        time.sleep(1)
+        yield {
+            "host": "localhost",
+            "port": 2222,
+            "username": "user",
+            "password": "pass",
+        }
+    finally:
+        stop_docker(name)
+
+
+@pytest.fixture
+def ssh_fixture(ssh_container, local_testdir, monkeypatch):
+    pytest.importorskip("paramiko", reason="sftp tests require paramiko")
+
+    cls = fsspec.get_filesystem_class("ssh")
+    if cls.put != fsspec.AbstractFileSystem.put:
+        monkeypatch.setattr(cls, "put", fsspec.AbstractFileSystem.put)
+    if Version(fsspec.__version__) < Version("2022.10.0"):
+        from fsspec.callbacks import _DEFAULT_CALLBACK
+
+        monkeypatch.setattr(_DEFAULT_CALLBACK, "relative_update", lambda *args: None)
+
+    fs = fsspec.filesystem(
+        "ssh",
+        host=ssh_container["host"],
+        port=ssh_container["port"],
+        username=ssh_container["username"],
+        password=ssh_container["password"],
+    )
+    fs.put(local_testdir, "/app/testdir", recursive=True)
+    try:
+        yield "ssh://{username}:{password}@{host}:{port}/app/testdir/".format(
+            **ssh_container
+        )
+    finally:
+        fs.delete("/app/testdir", recursive=True)

upath/tests/implementations/test_sftp.py

@@ -0,0 +1,40 @@
+import pytest
+
+from upath import UPath
+from upath.tests.cases import BaseTests
+from upath.tests.utils import skip_on_windows
+from upath.tests.utils import xfail_if_version
+
+_xfail_old_fsspec = xfail_if_version(
+    "fsspec",
+    lt="2022.7.0",
+    reason="fsspec<2022.7.0 sftp does not support create_parents",
+)
+
+
+@skip_on_windows
+class TestUPathSFTP(BaseTests):
+
+    @pytest.fixture(autouse=True)
+    def path(self, ssh_fixture):
+        self.path = UPath(ssh_fixture)
+
+    @_xfail_old_fsspec
+    def test_mkdir(self):
+        super().test_mkdir()
+
+    @_xfail_old_fsspec
+    def test_mkdir_exists_ok_true(self):
+        super().test_mkdir_exists_ok_true()
+
+    @_xfail_old_fsspec
+    def test_mkdir_exists_ok_false(self):
+        super().test_mkdir_exists_ok_false()
+
+    @_xfail_old_fsspec
+    def test_mkdir_parents_true_exists_ok_false(self):
+        super().test_mkdir_parents_true_exists_ok_false()
+
+    @_xfail_old_fsspec
+    def test_mkdir_parents_true_exists_ok_true(self):
+        super().test_mkdir_parents_true_exists_ok_true()

upath/tests/test_registry.py

@@ -22,7 +22,9 @@
     "memory",
     "s3",
     "s3a",
+    "sftp",
     "smb",
+    "ssh",
     "webdav",
     "webdav+http",
     "webdav+https",