Add storage options and protocol (#135)

* tests: add storage-options and protocol tests

* upath: implement storage_options and protocol

* upath.implementations.webdav: fix protocol and storage_options

* upath: allow overriding .path in subclasses

* tests: fix azure protocol test

* upath.implementations.http: fix .path

* upath: update readme

* typing: fix errors

Author: ap--

README.md

@@ -125,6 +125,29 @@ If a local path is provided, `UPath` will return a `PosixUPath` or `WindowsUPath
 These two subclasses are 100% compatible with the `PosixPath` and `WindowsPath` classes of their
 specific Python version, and are tested against all relevant tests of the CPython pathlib test-suite.
 
+### UPath public class API
+
+`UPath`'s public class interface is identical to `pathlib.Path` with the addition of the following attributes:
+
+- `UPath(...).protocol: str` the filesystem_spec protocol _(note: for `PosixUPath` and `WindowsUPath` it's an empty string)_
+- `UPath(...).storage_options: dict[str, Any]` the storage options for instantiating the filesystem_spec class
+- `UPath(...).path: str` the filesystem_spec compatible path for use with filesystem instances
+- `UPath(...).fs: AbstractFileSystem` convenience attribute to access an instantiated filesystem
+
+the first three provide a public interface to access a file via fsspec as follows:
+
+```python
+from upath import UPath
+from fsspec import filesystem
+
+p = UPath("s3://bucket/file.txt", anon=True)
+
+fs = filesystem(p.protocol, **p.storage_options)  # equivalent to p.fs
+with fs.open(p.path) as f:
+    data = f.read()
+```
+
+
 ## Contributing
 
 Contributions are very welcome.

upath/core.py

@@ -44,7 +44,7 @@ def __init__(self, parsed_url: SplitResult | None, **kwargs: Any) -> None:
         self._fs = cls(**url_kwargs)
 
     def _format_path(self, path: UPath) -> str:
-        return path.path
+        return path._path
 
     def open(self, path, mode="r", *args, **kwargs):
         return self._fs.open(self._format_path(path), mode, *args, **kwargs)
@@ -206,6 +206,44 @@ def __new__(cls: type[PT], *args: str | PathLike, **kwargs: Any) -> PT:
             args_list, url=parsed_url, **kwargs
         )
 
+    @property
+    def protocol(self) -> str:
+        """The filesystem_spec protocol
+
+        For local paths protocol is either 'file' if the UPath instance
+        is backed by fsspec or '' if it's backed by stdlib pathlib. For
+        both `fsspec.get_filesystem_class` returns `LocalFileSystem`.
+        """
+        if self._url is None:
+            return ""
+        return self._url.scheme
+
+    @property
+    def storage_options(self) -> dict[str, Any]:
+        """The filesystem_spec storage options dictionary
+
+        Accessing `.storage_options` does not instantiate the
+        corresponding fsspec filesystem class.
+        """
+        return {
+            key: value
+            for key, value in self._kwargs.items()
+            if key not in {"scheme", "netloc", "url"}
+        }
+
+    @property
+    def fs(self) -> AbstractFileSystem:
+        """The filesystem_spec filesystem instance"""
+        return self._accessor._fs
+
+    @property
+    def path(self) -> str:
+        """The filesystem_spec path for use with a filesystem instance
+
+        Note: for some file systems this can be prefixed by the protocol.
+        """
+        return self._path
+
     def __getattr__(self, item: str) -> Any:
         if item == "_accessor":
             # cache the _accessor attribute on first access
@@ -258,7 +296,7 @@ def _format_parsed_parts(
         return formatted
 
     @property
-    def path(self) -> str:
+    def _path(self) -> str:
         if self._parts:
             join_parts = self._parts[1:] if self._parts[0] == "/" else self._parts
             path: str = self._flavour.join(join_parts)
@@ -349,7 +387,7 @@ def rglob(self: PT, pattern: str) -> Generator[PT, None, None]:
 
     def _sub_path(self, name):
         # only want the path name with iterdir
-        sp = self.path
+        sp = self._path
         return re.sub(f"^({sp}|{sp[1:]})/", "", name)
 
     def absolute(self: PT) -> PT:
@@ -631,10 +669,6 @@ def __str__(self) -> str:
             )
             return self._str
 
-    @property
-    def fs(self) -> AbstractFileSystem:
-        return self._accessor._fs
-
     def __truediv__(self: PT, key: str | PathLike) -> PT:
         # Add `/` root if not present
         if len(self._parts) == 0:

upath/implementations/cloud.py

@@ -10,7 +10,7 @@ def _format_path(self, path):
         """
         netloc has already been set to project via `CloudPath._from_parts`
         """
-        return f"{path._url.netloc}/{path.path.lstrip('/')}"
+        return f"{path._url.netloc}/{path._path.lstrip('/')}"
 
     def mkdir(self, path, create_parents=True, **kwargs):
         _path = self._format_path(path)
@@ -49,7 +49,7 @@ def _sub_path(self, name):
         `listdir` and `glob`. However, in `iterdir` and `glob` we only want the
         relative path to `self`.
         """
-        sp = re.escape(self.path)
+        sp = re.escape(self._path)
         netloc = self._url.netloc
         return re.sub(
             f"^({netloc})?/?({sp}|{sp[1:]})/?",
@@ -71,6 +71,12 @@ def joinpath(self, *args):
             self._kwargs["bucket"] = bucket
             return super().joinpath(*tuple(args_list))
 
+    @property
+    def path(self) -> str:
+        if self._url is None:
+            raise RuntimeError(str(self))
+        return f"{self._url.netloc}{super()._path}"
+
 
 class GCSPath(CloudPath):
     pass

upath/implementations/http.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from urllib.parse import urlunsplit
+
 from fsspec.asyn import sync
 
 import upath.core
@@ -43,7 +45,7 @@ def _sub_path(self, name):
         relative path to `self`.
         """
         complete_address = self._format_parsed_parts(
-            None, None, [self.path], url=self._url, **self._kwargs
+            None, None, [self._path], url=self._url, **self._kwargs
         )
 
         if name.startswith(complete_address):
@@ -83,3 +85,10 @@ def resolve(
                     break
 
         return resolved_path
+
+    @property
+    def path(self) -> str:
+        # http filesystems use the full url as path
+        if self._url is None:
+            raise RuntimeError(str(self))
+        return urlunsplit(self._url)

upath/implementations/webdav.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from typing import Any
 from urllib.parse import ParseResult
 from urllib.parse import urlunsplit
 
@@ -49,3 +50,20 @@ def _sub_path(self, name):
         name = name.strip("/")
 
         return name
+
+    @property
+    def protocol(self) -> str:
+        if self._url is None:
+            raise RuntimeError(str(self))
+        return self._url.scheme.split("+")[0]
+
+    @property
+    def storage_options(self) -> dict[str, Any]:
+        if self._url is None:
+            raise RuntimeError(str(self))
+        sopts = super().storage_options
+        http_protocol = self._url.scheme.split("+")[1]
+        assert http_protocol in {"http", "https"}
+        base_url = urlunsplit(self._url._replace(scheme=http_protocol, path=""))
+        sopts["base_url"] = base_url
+        return sopts

upath/tests/cases.py

@@ -4,6 +4,7 @@
 from pathlib import Path
 
 import pytest
+from fsspec import filesystem
 
 from upath import UPath
 
@@ -416,3 +417,24 @@ def test_as_uri(self):
         uri = p0.as_uri()
         p1 = UPath(uri, **p0.fs.storage_options)
         assert p0 == p1
+
+    def test_protocol(self):
+        protocol = self.path.protocol
+        protocols = [p] if isinstance((p := type(self.path.fs).protocol), str) else p
+        print(protocol, protocols)
+        assert protocol in protocols
+
+    def test_storage_options(self):
+        storage_options = self.path.storage_options
+        assert storage_options == self.path.fs.storage_options
+
+    def test_read_with_fsspec(self):
+        p = self.path.joinpath("file2.txt")
+
+        protocol = p.protocol
+        storage_options = p.storage_options
+        path = p.path
+
+        fs = filesystem(protocol, **storage_options)
+        with fs.open(path) as f:
+            assert f.read() == b"hello world"

upath/tests/implementations/test_azure.py

@@ -44,3 +44,8 @@ def test_makedirs_exist_ok_false(self):
 
     def test_rglob(self, pathlib_base):
         return super().test_rglob(pathlib_base)
+
+    def test_protocol(self):
+        # test all valid protocols for azure...
+        protocol = self.path.protocol
+        assert protocol in ["abfs", "abfss", "adl", "az"]

upath/tests/implementations/test_webdav.py

@@ -12,3 +12,11 @@ def path(self, webdav_fixture):
 
     def test_fsspec_compat(self):
         pass
+
+    def test_storage_options(self):
+        # we need to add base_url to storage options for webdav filesystems,
+        # to be able to serialize the http protocol to string...
+        storage_options = self.path.storage_options
+        base_url = storage_options.pop("base_url")
+        assert storage_options == self.path.fs.storage_options
+        assert base_url == self.path.fs.client.base_url