Merge pull request #58 from leowe/http-sub-path

Fix iterdir for http-path and refactorings

Author: andrewfulton9

noxfile.py

@@ -27,7 +27,9 @@ def install(session):
 
 @nox.session(python=False)
 def smoke(session):
-    session.install(*"pytest aiohttp requests gcsfs".split())
+    session.install(
+        *"pytest aiohttp requests gcsfs s3fs moto[s3,server]".split()
+    )
     session.run(*"pytest --skiphdfs -vv upath".split())
 
 

upath/core.py

@@ -1,5 +1,6 @@
 import pathlib
 import re
+import sys
 from typing import Union
 import urllib
 
@@ -88,21 +89,14 @@ def __getattribute__(self, item):
 
 class UPath(pathlib.Path):
 
-    __slots__ = ("_url", "_kwargs", "_closed", "_accessor")
+    __slots__ = (
+        "_url",
+        "_kwargs",
+        "_accessor",  # overwritten because of default in Python 3.10
+    )
     _flavour = pathlib._posix_flavour
     _default_accessor = _FSSpecAccessor
 
-    not_implemented = [
-        "cwd",
-        "home",
-        "expanduser",
-        "group",
-        "lchmod",
-        "lstat",
-        "owner",
-        "readlink",
-    ]
-
     def __new__(cls, *args, **kwargs) -> Union["UPath", pathlib.Path]:
         args_list = list(args)
         first = args_list.pop(0)
@@ -116,7 +110,6 @@ def __new__(cls, *args, **kwargs) -> Union["UPath", pathlib.Path]:
             )
 
             new_kwargs = getattr(other, "_kwargs", {}).copy()
-            new_kwargs.pop("_url", None)
             new_kwargs.update(kwargs)
 
             return other.__class__(
@@ -136,44 +129,38 @@ def __new__(cls, *args, **kwargs) -> Union["UPath", pathlib.Path]:
             import upath.registry
 
             cls = upath.registry._registry[parsed_url.scheme]
-            kwargs["_url"] = parsed_url
             args_list.insert(0, parsed_url.path)
-            return cls._from_parts(tuple(args_list), **kwargs)
+            return cls._from_parts(tuple(args_list), url=parsed_url, **kwargs)
 
         # treat as local filesystem, return PosixPath or WindowsPath
         return pathlib.Path(*args, **kwargs)
 
     def __getattr__(self, item):
         if item == "_accessor":
             # cache the _accessor attribute on first access
-            kw = self._kwargs.copy()
-            kw.pop("_url", None)
-            self._accessor = _accessor = self._default_accessor(self._url, **kw)
+            kwargs = self._kwargs.copy()
+            self._accessor = _accessor = self._default_accessor(
+                self._url, **kwargs
+            )
             return _accessor
         else:
             raise AttributeError(item)
 
-    def __getattribute__(self, item):
-        if item == "__class__":
-            return super().__getattribute__("__class__")
-        if item in getattr(self.__class__, "not_implemented"):
-            raise NotImplementedError(f"UPath has no attribute {item}")
-        else:
-            return super().__getattribute__(item)
-
     def _make_child(self, args):
-        drv, root, parts = self._parse_args(args, **self._kwargs)
+        drv, root, parts = self._parse_args(args)
         drv, root, parts = self._flavour.join_parsed_parts(
             self._drv, self._root, self._parts, drv, root, parts
         )
-        return self._from_parsed_parts(drv, root, parts, **self._kwargs)
+        return self._from_parsed_parts(
+            drv, root, parts, url=self._url, **self._kwargs
+        )
 
     def _make_child_relpath(self, part):
         # This is an optimization used for dir walking.  `part` must be
         # a single part relative to this path.
         parts = self._parts + [part]
         return self._from_parsed_parts(
-            self._drv, self._root, parts, **self._kwargs
+            self._drv, self._root, parts, url=self._url, **self._kwargs
         )
 
     def _format_parsed_parts(self, drv, root, parts):
@@ -213,7 +200,9 @@ def parent(self):
         parts = self._parts
         if len(parts) == 1 and (drv or root):
             return self
-        return self._from_parsed_parts(drv, root, parts[:-1], **self._kwargs)
+        return self._from_parsed_parts(
+            drv, root, parts[:-1], url=self._url, **self._kwargs
+        )
 
     def stat(self):
         return self._accessor.stat(self)
@@ -222,8 +211,6 @@ def iterdir(self):
         """Iterate over the files in this directory.  Does not yield any
         result for the special paths '.' and '..'.
         """
-        if self._closed:
-            self._raise_closed()
         for name in self._accessor.listdir(self):
             # fsspec returns dictionaries
             if isinstance(name, dict):
@@ -234,8 +221,29 @@ def iterdir(self):
             # only want the path name with iterdir
             name = self._sub_path(name)
             yield self._make_child_relpath(name)
-            if self._closed:
-                self._raise_closed()
+
+    def relative_to(self, *other):
+        for other_item in other:
+            if not isinstance(other_item, self.__class__) and not isinstance(
+                other_item, str
+            ):
+                raise ValueError(
+                    f"{repr(self)} and {repr(other_item)} are "
+                    "not of compatible classes."
+                )
+            if not isinstance(other_item, str) and (
+                other_item._url.scheme != self._url.scheme
+                or other_item._url.netloc != self._url.netloc
+                or other_item._kwargs != self._kwargs
+            ):
+                raise ValueError(
+                    f"{self} and {other_item} do not share the same "
+                    "base URL and storage options."
+                )
+        output = super().relative_to(*other)
+        output._url = self._url
+        output._kwargs = self._kwargs
+        return output
 
     def glob(self, pattern):
         path = self.joinpath(pattern)
@@ -263,24 +271,33 @@ def exists(self):
             return self._accessor.exists(self)
 
     def is_dir(self):
-        info = self._accessor.info(self)
-        if info["type"] == "directory":
-            return True
+        try:
+            info = self._accessor.info(self)
+            if info["type"] == "directory":
+                return True
+        except FileNotFoundError:
+            return False
         return False
 
     def is_file(self):
-        info = self._accessor.info(self)
-        if info["type"] == "file":
-            return True
+        try:
+            info = self._accessor.info(self)
+            if info["type"] == "file":
+                return True
+        except FileNotFoundError:
+            return False
         return False
 
     def is_mount(self):
         return False
 
     def is_symlink(self):
-        info = self._accessor.info(self)
-        if "islink" in info:
-            return info["islink"]
+        try:
+            info = self._accessor.info(self)
+            if "islink" in info:
+                return info["islink"]
+        except FileNotFoundError:
+            return False
         return False
 
     def is_socket(self):
@@ -295,16 +312,6 @@ def is_block_device(self):
     def is_char_device(self):
         return False
 
-    def chmod(self, mod):
-        raise NotImplementedError
-
-    def rename(self, target):
-        # can be implemented, but may be tricky
-        raise NotImplementedError
-
-    def touch(self, trunicate=True, **kwargs):
-        self._accessor.touch(self, trunicate=trunicate, **kwargs)
-
     def unlink(self, missing_ok=False):
         if not self.exists():
             if not missing_ok:
@@ -323,45 +330,80 @@ def rmdir(self, recursive=True):
             raise NotDirectoryError
         self._accessor.rm(self, recursive=recursive)
 
-    @classmethod
-    def _parse_args(cls, args, **kwargs):
-        return super(UPath, cls)._parse_args(args)
+    def chmod(self, mod):
+        raise NotImplementedError
+
+    def rename(self, target):
+        # can be implemented, but may be tricky
+        raise NotImplementedError
+
+    def cwd(self):
+        raise NotImplementedError
+
+    def home(self):
+        raise NotImplementedError
+
+    def expanduser(self):
+        raise NotImplementedError
+
+    def group(self):
+        raise NotImplementedError
+
+    def lchmod(self, mode):
+        raise NotImplementedError
+
+    def lstat(self):
+        raise NotImplementedError
+
+    def owner(self):
+        raise NotImplementedError
+
+    def readlink(self):
+        raise NotImplementedError
+
+    def touch(self, trunicate=True, **kwargs):
+        self._accessor.touch(self, trunicate=trunicate, **kwargs)
 
     @classmethod
-    def _from_parts(cls, args, **kwargs):
+    def _from_parts(cls, args, url=None, **kwargs):
         obj = object.__new__(cls)
-        drv, root, parts = obj._parse_args(args, **kwargs)
+        drv, root, parts = obj._parse_args(args)
         obj._drv = drv
-        obj._parts = parts
-        obj._closed = False
+        if sys.version_info < (3, 9):
+            obj._closed = False
+        obj._url = url
         obj._kwargs = kwargs.copy()
-        obj._url = kwargs.pop("_url", None) or None
 
         if not root:
             if not parts:
                 root = "/"
+                parts = ["/"]
             elif parts[0] == "/":
-                root = parts.pop(0)
+                root = parts[1:]
         obj._root = root
+        obj._parts = parts
 
         return obj
 
     @classmethod
-    def _from_parsed_parts(cls, drv, root, parts, **kwargs):
+    def _from_parsed_parts(cls, drv, root, parts, url=None, **kwargs):
         obj = object.__new__(cls)
         obj._drv = drv
         obj._parts = parts
-        obj._closed = False
+        if sys.version_info < (3, 9):
+            obj._closed = False
+        obj._url = url
         obj._kwargs = kwargs.copy()
-        obj._url = kwargs.pop("_url", None) or None
 
         if not root:
             if not parts:
                 root = "/"
             elif parts[0] == "/":
                 root = parts.pop(0)
-        obj._root = root
+        if len(obj._parts) == 0 or obj._parts[0] != root:
+            obj._parts.insert(0, root)
 
+        obj._root = root
         return obj
 
     @property
@@ -380,7 +422,6 @@ def __truediv__(self, key):
         )
 
         kwargs = self._kwargs.copy()
-        kwargs.pop("_url")
 
         # Create a new object
         out = self.__class__(
@@ -390,16 +431,11 @@ def __truediv__(self, key):
         return out
 
     def __setstate__(self, state):
-        kwargs = state["_kwargs"].copy()
-        kwargs["_url"] = self._url
-        self._kwargs = kwargs
+        self._kwargs = state["_kwargs"].copy()
 
     def __reduce__(self):
-        kwargs = self._kwargs.copy()
-        kwargs.pop("_url", None)
-
         return (
             self.__class__,
             (self._format_parsed_parts(self._drv, self._root, self._parts),),
-            {"_kwargs": kwargs},
+            {"_kwargs": self._kwargs.copy()},
         )

upath/implementations/gcs.py

@@ -1,5 +1,4 @@
 import upath.core
-import os
 import re
 
 
@@ -11,7 +10,7 @@ def _format_path(self, s):
         """
         netloc has already been set to project via `GCSPath._from_parts`
         """
-        s = os.path.join(self._url.netloc, s.lstrip("/"))
+        s = f"{self._url.netloc}/{s.lstrip('/')}"
         return s
 
 
@@ -20,25 +19,26 @@ class GCSPath(upath.core.UPath):
     _default_accessor = _GCSAccessor
 
     @classmethod
-    def _from_parts(cls, args, **kwargs):
-        obj = super()._from_parts(args, **kwargs)
-        if kwargs.get("bucket") and kwargs.get("_url"):
-            bucket = obj._kwargs.pop("bucket")
-            obj._url = obj._url._replace(netloc=bucket)
+    def _from_parts(cls, args, url=None, **kwargs):
+        if kwargs.get("bucket") and url is not None:
+            bucket = kwargs.pop("bucket")
+            url = url._replace(netloc=bucket)
+        obj = super()._from_parts(args, url, **kwargs)
         return obj
 
     @classmethod
-    def _from_parsed_parts(cls, drv, root, parts, **kwargs):
-        obj = super()._from_parsed_parts(drv, root, parts, **kwargs)
-        if kwargs.get("bucket") and kwargs.get("_url"):
-            bucket = obj._kwargs.pop("bucket")
-            obj._url = obj._url._replace(netloc=bucket)
+    def _from_parsed_parts(cls, drv, root, parts, url=None, **kwargs):
+        if kwargs.get("bucket") and url is not None:
+            bucket = kwargs.pop("bucket")
+            url = url._replace(netloc=bucket)
+        obj = super()._from_parsed_parts(drv, root, parts, url, **kwargs)
         return obj
 
     def _sub_path(self, name):
-        """gcs returns path as `{bucket}/<path>` with listdir
-        and glob, so here we can add the netloc to the sub string
-        so it gets subbed out as well
+        """
+        `gcsfs` returns the full path as `<bucket>/<path>` with `listdir` and
+        `glob`. However, in `iterdir` and `glob` we only want the relative path
+        to `self`.
         """
         sp = self.path
         subed = re.sub(f"^({self._url.netloc})?/?({sp}|{sp[1:]})/?", "", name)