Merge pull request #8 from Quansight/refactor

refactors code to move implimentations into their own folders

Author: andrewfulton9

upath/core.py

@@ -3,7 +3,6 @@
 import urllib
 
 from upath.registry import _registry
-from upath.universal_path import UniversalPath
 
 
 class UPath(pathlib.Path):
@@ -30,10 +29,7 @@ def __new__(cls, *args, **kwargs):
                     )
                 self._init()
             else:
-                if parsed_url.scheme in _registry:
-                    cls = _registry[parsed_url.scheme]
-                else:
-                    cls = UniversalPath
+                cls = _registry[parsed_url.scheme]
                 kwargs["_url"] = parsed_url
                 new_args.insert(0, parsed_url.path)
                 args = tuple(new_args)

upath/implementations/hdfs.py

@@ -0,0 +1,28 @@
+from upath.universal_path import _FSSpecAccessor, UniversalPath
+
+
+class _HDFSAccessor(_FSSpecAccessor):
+    def __init__(self, parsed_url, *args, **kwargs):
+        super().__init__(parsed_url, *args, **kwargs)
+        self._fs.root_marker = "/"
+
+    def argument_upath_self_to_filepath(self, func):
+        """if arguments are passed to the wrapped function, and if the first
+        argument is a UniversalPath instance, that argument is replaced with
+        the UniversalPath's path attribute
+        """
+
+        def wrapper(*args, **kwargs):
+            args, kwargs = self._format_path(args, kwargs)
+            print(args, kwargs)
+            if "trunicate" in kwargs:
+                kwargs.pop("trunicate")
+            if func.__name__ == "mkdir":
+                args = args[:1]
+            return func(*args, **kwargs)
+
+        return wrapper
+
+
+class HDFSPath(UniversalPath):
+    _default_accessor = _HDFSAccessor

upath/implementations/s3.py

@@ -0,0 +1,10 @@
+from upath.universal_path import _FSSpecAccessor, UniversalPath
+
+
+class _S3Accessor(_FSSpecAccessor):
+    def __init__(self, parsed_url, *args, **kwargs):
+        super().__init__(parsed_url, *args, **kwargs)
+
+
+class S3Path(UniversalPath):
+    _default_accessor = _S3Accessor

upath/registry.py

@@ -1,3 +1,25 @@
-from upath.implementations import http
+import warnings
 
-_registry = {"http": http.HTTPPath}
+from upath.universal_path import UniversalPath
+from upath.implementations import http, hdfs, s3
+
+
+class _Registry:
+    http = http.HTTPPath
+    hdfs = hdfs.HDFSPath
+    s3 = s3.S3Path
+
+    def __getitem__(self, item):
+        implimented_path = getattr(self, item, None)
+        if not implimented_path:
+            warning_str = (
+                f"{item} filesystem path not explicitely implimented. "
+                "falling back to default implimentation UniversalPath. "
+                "This filesystem may not be tested"
+            )
+            warnings.warn(warning_str, UserWarning)
+            return UniversalPath
+        return implimented_path
+
+
+_registry = _Registry()

upath/tests/conftest.py

@@ -73,6 +73,11 @@ def local_testdir(tempdir, clear_registry):
     shutil.rmtree(tempdir)
 
 
+@pytest.fixture()
+def pathlib_base(local_testdir):
+    return Path(local_testdir)
+
+
 @pytest.fixture(scope="session")
 def htcluster():
     proc = subprocess.Popen(

upath/tests/implementations/test_hdfs.py

@@ -0,0 +1,23 @@
+"""see upath/tests/conftest.py for fixtures
+"""
+import pytest  # noqa: F401
+
+from upath import UPath
+from upath.implementations.hdfs import HDFSPath
+from upath.tests.test_core import TestUpath
+
+
+@pytest.mark.hdfs
+class TestUPathHDFS(TestUpath):
+    @pytest.fixture(autouse=True)
+    def path(self, local_testdir, hdfs):
+        host, user, port = hdfs
+        path = f"hdfs:{local_testdir}"
+        self.path = UPath(path, host=host, user=user, port=port)
+
+    def test_is_HDFSPath(self):
+        assert isinstance(self.path, HDFSPath)
+
+    def test_chmod(self):
+        # todo
+        pass

upath/tests/implementations/test_s3.py

@@ -0,0 +1,54 @@
+"""see upath/tests/conftest.py for fixtures
+"""
+import pytest  # noqa: F401
+
+from upath import UPath
+from upath.errors import NotDirectoryError
+from upath.implementations.s3 import S3Path
+from upath.tests.test_core import TestUpath
+
+
+class TestUPathS3(TestUpath):
+    @pytest.fixture(autouse=True)
+    def path(self, local_testdir, s3):
+        anon, s3so = s3
+        path = f"s3:{local_testdir}"
+        self.path = UPath(path, anon=anon, **s3so)
+
+    def test_is_S3Path(self):
+        assert isinstance(self.path, S3Path)
+
+    def test_chmod(self):
+        # todo
+        pass
+
+    def test_mkdir(self):
+        new_dir = self.path.joinpath("new_dir")
+        # new_dir.mkdir()
+        # mkdir doesnt really do anything. A directory only exists in s3
+        # if some file or something is written to it
+        new_dir.joinpath("test.txt").touch()
+        assert new_dir.exists()
+
+    def test_rmdir(self, local_testdir):
+        dirname = "rmdir_test"
+        mock_dir = self.path.joinpath(dirname)
+        mock_dir.joinpath("test.txt").touch()
+        mock_dir.rmdir()
+        assert not mock_dir.exists()
+        with pytest.raises(NotDirectoryError):
+            self.path.joinpath("file1.txt").rmdir()
+
+    def test_touch_unlink(self):
+        path = self.path.joinpath("test_touch.txt")
+        path.touch()
+        assert path.exists()
+        path.unlink()
+        assert not path.exists()
+
+        # should raise FileNotFoundError since file is missing
+        with pytest.raises(FileNotFoundError):
+            path.unlink()
+
+        # file doesn't exists, but missing_ok is True
+        path.unlink(missing_ok=True)

upath/tests/test_core.py

@@ -1,29 +1,32 @@
 import pathlib
 from pathlib import Path
+import warnings
 
 import pytest
 
 from upath import UPath
-from upath.errors import NotDirectoryError
-
-
-@pytest.fixture()
-def pathlib_base(local_testdir):
-    return Path(local_testdir)
 
 
 def test_posix_path(local_testdir):
     assert isinstance(UPath(local_testdir), pathlib.PosixPath)
 
 
+def test_UPath_warning():
+    with warnings.catch_warnings(record=True) as w:
+        path = UPath("mock:/")  # noqa: F841
+        assert len(w) == 1
+        assert issubclass(w[-1].category, UserWarning)
+        assert "mock" in str(w[-1].message)
+
+
 class TestUpath:
     @pytest.fixture(autouse=True)
     def path(self, local_testdir):
-        self.path = UPath(f"mock:{local_testdir}")
-        print(self.path)
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            self.path = UPath(f"mock:{local_testdir}")
 
     def test_cwd(self):
-        print("test_cwd")
         with pytest.raises(NotImplementedError):
             self.path.cwd()
 
@@ -118,7 +121,6 @@ def test_lstat(self):
     def test_mkdir(self):
         new_dir = self.path.joinpath("new_dir")
         new_dir.mkdir()
-        print(new_dir._accessor.info(new_dir))
         assert new_dir.exists()
 
     def test_open(self):
@@ -196,62 +198,6 @@ def test_write_text(self, pathlib_base):
         assert path.read_text() == s
 
 
-@pytest.mark.hdfs
-class TestUPathHDFS(TestUpath):
-    @pytest.fixture(autouse=True)
-    def path(self, local_testdir, hdfs):
-        host, user, port = hdfs
-        path = f"hdfs:{local_testdir}"
-        self.path = UPath(path, host=host, user=user, port=port)
-
-    def test_chmod(self):
-        # todo
-        pass
-
-
-class TestUPathS3(TestUpath):
-    @pytest.fixture(autouse=True)
-    def path(self, local_testdir, s3):
-        anon, s3so = s3
-        path = f"s3:{local_testdir}"
-        self.path = UPath(path, anon=anon, **s3so)
-
-    def test_chmod(self):
-        # todo
-        pass
-
-    def test_mkdir(self):
-        new_dir = self.path.joinpath("new_dir")
-        # new_dir.mkdir()
-        # mkdir doesnt really do anything. A directory only exists in s3
-        # if some file or something is written to it
-        new_dir.joinpath("test.txt").touch()
-        assert new_dir.exists()
-
-    def test_rmdir(self, local_testdir):
-        dirname = "rmdir_test"
-        mock_dir = self.path.joinpath(dirname)
-        mock_dir.joinpath("test.txt").touch()
-        mock_dir.rmdir()
-        assert not mock_dir.exists()
-        with pytest.raises(NotDirectoryError):
-            self.path.joinpath("file1.txt").rmdir()
-
-    def test_touch_unlink(self):
-        path = self.path.joinpath("test_touch.txt")
-        path.touch()
-        assert path.exists()
-        path.unlink()
-        assert not path.exists()
-
-        # should raise FileNotFoundError since file is missing
-        with pytest.raises(FileNotFoundError):
-            path.unlink()
-
-        # file doesn't exists, but missing_ok is True
-        path.unlink(missing_ok=True)
-
-
 @pytest.mark.hdfs
 def test_multiple_backend_paths(local_testdir, s3, hdfs):
     anon, s3so = s3

upath/universal_path.py

@@ -16,8 +16,6 @@ def __init__(self, parsed_url, *args, **kwargs):
         )
         url_kwargs.update(kwargs)
         self._fs = cls(**url_kwargs)
-        if self._url.scheme in ["hdfs"]:
-            self._fs.root_marker = "/"
 
     def argument_upath_self_to_filepath(self, func):
         """if arguments are passed to the wrapped function, and if the first
@@ -26,33 +24,33 @@ def argument_upath_self_to_filepath(self, func):
         """
 
         def wrapper(*args, **kwargs):
-            if args:
-                args = list(args)
-                first_arg = args.pop(0)
-                if not kwargs.get("path"):
-                    if isinstance(first_arg, UniversalPath):
-                        first_arg = first_arg.path
-                        if not self._fs.root_marker and first_arg.startswith(
-                            "/"
-                        ):
-                            first_arg = first_arg[1:]
-                        args.insert(0, first_arg)
-                    args = tuple(args)
-                else:
-                    if not self._fs.root_marker and kwargs["path"].startswith(
-                        "/"
-                    ):
-                        kwargs["path"] = kwargs["path"][1:]
-                if self._url.scheme == "hdfs":
-                    if "trunicate" in kwargs:
-                        kwargs.pop("trunicate")
-                    if func.__name__ == "mkdir":
-                        args = args[:1]
-
+            args, kwargs = self._format_path(args, kwargs)
             return func(*args, **kwargs)
 
         return wrapper
 
+    def _format_path(self, args, kwargs):
+        """formats the path properly for the filesystem backend."""
+        args = list(args)
+        first_arg = args.pop(0)
+        if not kwargs.get("path"):
+            if isinstance(first_arg, UniversalPath):
+                first_arg = self._remove_root_slash(first_arg.path)
+                args.insert(0, first_arg)
+            args = tuple(args)
+        else:
+            kwargs["path"] = self._remove_root_slash(kwargs["path"])
+        return args, kwargs
+
+    def _remove_root_slash(self, s):
+        """If the filesystem backend doesn't have a root_marker, strip the
+        leading slash of a path
+        """
+        if not self._fs.root_marker and s.startswith("/"):
+            return s[1:]
+        else:
+            return s
+
     def __getattribute__(self, item):
         class_attrs = ["_url", "_fs", "__class__"]
         if item in class_attrs:
@@ -62,6 +60,8 @@ def __getattribute__(self, item):
             "__init__",
             "__getattribute__",
             "argument_upath_self_to_filepath",
+            "_format_path",
+            "_remove_root_slash",
         ]
         if item in class_methods:
             return lambda *args, **kwargs: getattr(self.__class__, item)(
@@ -134,8 +134,8 @@ def _init(self, *args, template=None, **kwargs):
 
     def __getattribute__(self, item):
         if item == "__class__":
-            return UniversalPath
-        if item in getattr(UniversalPath, "not_implemented"):
+            return super().__getattribute__("__class__")
+        if item in getattr(self.__class__, "not_implemented"):
             raise NotImplementedError(f"UniversalPath has no attribute {item}")
         else:
             return super().__getattribute__(item)
@@ -254,7 +254,7 @@ def _from_parts_init(cls, args, init=False):
     def _from_parts(self, args, init=True):
         # We need to call _parse_args on the instance, so as to get the
         # right flavour.
-        obj = object.__new__(UniversalPath)
+        obj = object.__new__(self.__class__)
         drv, root, parts = self._parse_args(args)
         obj._drv = drv
         obj._root = root
@@ -264,7 +264,7 @@ def _from_parts(self, args, init=True):
         return obj
 
     def _from_parsed_parts(self, drv, root, parts, init=True):
-        obj = object.__new__(UniversalPath)
+        obj = object.__new__(self.__class__)
         obj._drv = drv
         obj._root = root
         obj._parts = parts