Merge remote-tracking branch 'upstream/main' into copy-upath

Author: normanrz

.github/workflows/python.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.7, 3.8, 3.9, "3.10"]
         os: [ubuntu-latest, windows-latest]
 
     steps:

environment.yml

@@ -3,15 +3,14 @@ channels:
     - defaults
     - conda-forge
 dependencies:
-    - python==3.8
-    - fsspec==2021.11.1
+    - python==3.10
+    - fsspec
     # optional
     - requests
     - s3fs
-    - jupyter   
+    - jupyter
     - ipython
     - pytest
-    - vcrpy
     - pylint
     - flake8
     - pyarrow

upath/__init__.py

@@ -1,5 +1,5 @@
 """Pathlib API extended to use fsspec backends"""
-__version__ = "0.0.12"
+__version__ = "0.0.15"
 
 from upath.core import UPath
 

upath/core.py

@@ -102,7 +102,7 @@ def __subclasscheck__(cls, subclass):
 
 class UPath(pathlib.Path, PureUPath, metaclass=UPathMeta):
 
-    __slots__ = ("_url", "_kwargs", "_closed", "fs")
+    __slots__ = ("_url", "_kwargs", "_closed", "_accessor")
 
     not_implemented = [
         "cwd",
@@ -158,45 +158,32 @@ def __new__(cls, *args, **kwargs):
                     if os.name == "nt"
                     else pathlib.PosixPath
                 )
-                self = cls._from_parts(args, init=False)
+                self = cls._from_parts(args)
                 if not self._flavour.is_supported:
                     raise NotImplementedError(
                         "cannot instantiate %r on your system" % (cls.__name__,)
                     )
-                self._init()
             else:
                 import upath.registry
 
                 cls = upath.registry._registry[parsed_url.scheme]
                 kwargs["_url"] = parsed_url
                 args_list.insert(0, parsed_url.path)
                 args = tuple(args_list)
-                self = cls._from_parts_init(args, init=False)
-                self._init(*args, **kwargs)
+                self = cls._from_parts(args, **kwargs)
         else:
             self = super().__new__(*args, **kwargs)
         return self
 
-    def _init(self, *args, template=None, **kwargs):
-        self._closed = False
-        if not kwargs:
-            kwargs = dict(**self._kwargs)
+    def __getattr__(self, item):
+        if item == "_accessor":
+            # cache the _accessor attribute on first access
+            kw = self._kwargs.copy()
+            kw.pop("_url", None)
+            self._accessor = _accessor = self._default_accessor(self._url, **kw)
+            return _accessor
         else:
-            self._kwargs = dict(**kwargs)
-        self._url = kwargs.pop("_url") if kwargs.get("_url") else None
-
-        if not self._root:
-            if not self._parts:
-                self._root = "/"
-            elif self._parts[0] == "/":
-                self._root = self._parts.pop(0)
-        if getattr(self, "_str", None):
-            delattr(self, "_str")
-        if template is not None:
-            self._accessor = template._accessor
-        else:
-            self._accessor = self._default_accessor(self._url, *args, **kwargs)
-        self.fs = self._accessor._fs
+            raise AttributeError(item)
 
     def __getattribute__(self, item):
         if item == "__class__":
@@ -206,6 +193,21 @@ def __getattribute__(self, item):
         else:
             return super().__getattribute__(item)
 
+    def _make_child(self, args):
+        drv, root, parts = self._parse_args(args, **self._kwargs)
+        drv, root, parts = self._flavour.join_parsed_parts(
+            self._drv, self._root, self._parts, drv, root, parts
+        )
+        return self._from_parsed_parts(drv, root, parts, **self._kwargs)
+
+    def _make_child_relpath(self, part):
+        # This is an optimization used for dir walking.  `part` must be
+        # a single part relative to this path.
+        parts = self._parts + [part]
+        return self._from_parsed_parts(
+            self._drv, self._root, parts, **self._kwargs
+        )
+
     def _format_parsed_parts(self, drv, root, parts):
         if parts:
             join_parts = parts[1:] if parts[0] == "/" else parts
@@ -235,6 +237,19 @@ def path(self):
     def open(self, *args, **kwargs):
         return self._accessor.open(self, *args, **kwargs)
 
+    @property
+    def parent(self):
+        """The logical parent of the path."""
+        drv = self._drv
+        root = self._root
+        parts = self._parts
+        if len(parts) == 1 and (drv or root):
+            return self
+        return self._from_parsed_parts(drv, root, parts[:-1], **self._kwargs)
+
+    def stat(self):
+        return self._accessor.stat(self)
+
     def iterdir(self):
         """Iterate over the files in this directory.  Does not yield any
         result for the special paths '.' and '..'.
@@ -320,30 +335,50 @@ def rmdir(self, recursive=True):
         self._accessor.rm(self, recursive=recursive)
 
     @classmethod
-    def _from_parts_init(cls, args, init=False):
-        return super()._from_parts(args, init=init)
-
-    def _from_parts(self, args, init=True):
-        # We need to call _parse_args on the instance, so as to get the
-        # right flavour.
-        obj = object.__new__(self.__class__)
-        drv, root, parts = self._parse_args(args)
+    def _parse_args(cls, args, **kwargs):
+        return super(UPath, cls)._parse_args(args)
+
+    @classmethod
+    def _from_parts(cls, args, **kwargs):
+        obj = object.__new__(cls)
+        drv, root, parts = obj._parse_args(args, **kwargs)
         obj._drv = drv
-        obj._root = root
         obj._parts = parts
-        if init:
-            obj._init(**self._kwargs)
+        obj._closed = False
+        obj._kwargs = kwargs.copy()
+        obj._url = kwargs.pop("_url", None) or None
+
+        if not root:
+            if not parts:
+                root = "/"
+            elif parts[0] == "/":
+                root = parts.pop(0)
+        obj._root = root
+
         return obj
 
-    def _from_parsed_parts(self, drv, root, parts, init=True):
-        obj = object.__new__(self.__class__)
+    @classmethod
+    def _from_parsed_parts(cls, drv, root, parts, **kwargs):
+        obj = object.__new__(cls)
         obj._drv = drv
-        obj._root = root
         obj._parts = parts
-        if init:
-            obj._init(**self._kwargs)
+        obj._closed = False
+        obj._kwargs = kwargs.copy()
+        obj._url = kwargs.pop("_url", None) or None
+
+        if not root:
+            if not parts:
+                root = "/"
+            elif parts[0] == "/":
+                root = parts.pop(0)
+        obj._root = root
+
         return obj
 
+    @property
+    def fs(self):
+        return self._accessor._fs
+
     def __truediv__(self, key):
         # Add `/` root if not present
         if len(self._parts) == 0:
@@ -369,9 +404,6 @@ def __setstate__(self, state):
         kwargs = state["_kwargs"].copy()
         kwargs["_url"] = self._url
         self._kwargs = kwargs
-        # _init needs to be called again, because when __new__ called _init,
-        # the _kwargs were not yet set
-        self._init()
 
     def __reduce__(self):
         kwargs = self._kwargs.copy()

upath/implementations/gcs.py

@@ -9,7 +9,7 @@ def __init__(self, parsed_url, *args, **kwargs):
 
     def _format_path(self, s):
         """
-        netloc has already been set to project via `GCSPath._init`
+        netloc has already been set to project via `GCSPath._from_parts`
         """
         s = os.path.join(self._url.netloc, s.lstrip("/"))
         return s
@@ -19,12 +19,21 @@ def _format_path(self, s):
 class GCSPath(upath.core.UPath):
     _default_accessor = _GCSAccessor
 
-    def _init(self, *args, template=None, **kwargs):
-        # ensure that the bucket is part of the netloc path
+    @classmethod
+    def _from_parts(cls, args, **kwargs):
+        obj = super()._from_parts(args, **kwargs)
         if kwargs.get("bucket") and kwargs.get("_url"):
-            bucket = kwargs.pop("bucket")
-            kwargs["_url"] = kwargs["_url"]._replace(netloc=bucket)
-        super()._init(*args, template=template, **kwargs)
+            bucket = obj._kwargs.pop("bucket")
+            obj._url = obj._url._replace(netloc=bucket)
+        return obj
+
+    @classmethod
+    def _from_parsed_parts(cls, drv, root, parts, **kwargs):
+        obj = super()._from_parsed_parts(drv, root, parts, **kwargs)
+        if kwargs.get("bucket") and kwargs.get("_url"):
+            bucket = obj._kwargs.pop("bucket")
+            obj._url = obj._url._replace(netloc=bucket)
+        return obj
 
     def _sub_path(self, name):
         """gcs returns path as `{bucket}/<path>` with listdir

upath/implementations/s3.py

@@ -19,13 +19,21 @@ def _format_path(self, s):
 class S3Path(upath.core.UPath):
     _default_accessor = _S3Accessor
 
-    def _init(self, *args, template=None, **kwargs):
-        # ensure that the bucket is part of the netloc path
+    @classmethod
+    def _from_parts(cls, args, **kwargs):
+        obj = super()._from_parts(args, **kwargs)
         if kwargs.get("bucket") and kwargs.get("_url"):
-            bucket = kwargs.pop("bucket")
-            kwargs["_url"] = kwargs["_url"]._replace(netloc=bucket)
+            bucket = obj._kwargs.pop("bucket")
+            obj._url = obj._url._replace(netloc=bucket)
+        return obj
 
-        super()._init(*args, template=template, **kwargs)
+    @classmethod
+    def _from_parsed_parts(cls, drv, root, parts, **kwargs):
+        obj = super()._from_parsed_parts(drv, root, parts, **kwargs)
+        if kwargs.get("bucket") and kwargs.get("_url"):
+            bucket = obj._kwargs.pop("bucket")
+            obj._url = obj._url._replace(netloc=bucket)
+        return obj
 
     def _sub_path(self, name):
         """s3fs returns path as `{bucket}/<path>` with listdir