Upath annotate storage options for implementations (#432)

* upath.types: add storage_options submodule

* upath.types: fix *PathLike types

* upath.implementations: add storage-options types

* typesafety: add tests ensuring that storage_options are typed

* upath.types: fix missing import

Author: ap--

typesafety/test_upath_types.yml

@@ -129,7 +129,7 @@
     reveal_type(p)  # N: Revealed type is "{{ cls_fqn }}"
 
 - case: upath__new__empty_protocol
-  disable_cache: true
+  disable_cache: false
   skip: sys.platform == "win32"
   main: |
     from upath.core import UPath
@@ -138,10 +138,149 @@
     reveal_type(p)  # N: Revealed type is "upath.implementations.local.PosixUPath"
 
 - case: get_upath_class_pathlib_win
-  disable_cache: true
+  disable_cache: false
   skip: sys.platform != "win32"
   main: |
     from upath.core import UPath
 
     p = UPath("", protocol="")
     reveal_type(p)  # N: Revealed type is "upath.implementations.local.WindowsUPath"
+
+- case: upath_constructor_sopts_correct_type
+  disable_cache: false
+  parametrized:
+    - module: upath.implementations.cached
+      cls: SimpleCachePath
+      supported_example_name: check_files
+      supported_example_value: True
+    - module: upath.implementations.cloud
+      cls: GCSPath
+      supported_example_name: project
+      supported_example_value: '"my-project"'
+    - module: upath.implementations.cloud
+      cls: S3Path
+      supported_example_name: anon
+      supported_example_value: True
+    - module: upath.implementations.cloud
+      cls: AzurePath
+      supported_example_name: account_name
+      supported_example_value: '"myaccount"'
+    - module: upath.implementations.data
+      cls: DataPath
+      supported_example_name: use_listings_cache
+      supported_example_value: False
+    - module: upath.implementations.github
+      cls: GitHubPath
+      supported_example_name: org
+      supported_example_value: '"fsspec"'
+    - module: upath.implementations.hdfs
+      cls: HDFSPath
+      supported_example_name: host
+      supported_example_value: '"localhost"'
+    - module: upath.implementations.http
+      cls: HTTPPath
+      supported_example_name: simple_links
+      supported_example_value: True
+    - module: upath.implementations.local
+      cls: FilePath
+      supported_example_name: auto_mkdir
+      supported_example_value: True
+    - module: upath.implementations.memory
+      cls: MemoryPath
+      supported_example_name: use_listings_cache
+      supported_example_value: True
+    - module: upath.implementations.sftp
+      cls: SFTPPath
+      supported_example_name: host
+      supported_example_value: '"sftp.example.com"'
+    - module: upath.implementations.smb
+      cls: SMBPath
+      supported_example_name: timeout
+      supported_example_value: 60
+    - module: upath.implementations.webdav
+      cls: WebdavPath
+      supported_example_name: base_url
+      supported_example_value: '"https://webdav.example.com"'
+  main: |
+    from {{ module }} import {{ cls }}
+
+    p = {{ cls }}(".", {{ supported_example_name }}={{ supported_example_value }})
+    reveal_type(p)  # N: Revealed type is "{{ module }}.{{ cls }}"
+
+- case: upath_constructor_sopts_incorrect_type
+  disable_cache: false
+  parametrized:
+    - module: upath.implementations.cached
+      cls: SimpleCachePath
+      supported_example_name: check_files
+      unsupported_example_value: '"blub"'
+    - module: upath.implementations.cloud
+      cls: GCSPath
+      supported_example_name: version_aware
+      unsupported_example_value: '"yes"'
+    - module: upath.implementations.cloud
+      cls: S3Path
+      supported_example_name: anon
+      unsupported_example_value: '"blah"'
+    - module: upath.implementations.cloud
+      cls: AzurePath
+      supported_example_name: account_name
+      unsupported_example_value: '123'
+    - module: upath.implementations.data
+      cls: DataPath
+      supported_example_name: use_listings_cache
+      unsupported_example_value: '"blub"'
+    - module: upath.implementations.github
+      cls: GitHubPath
+      supported_example_name: repo
+      unsupported_example_value: 'True'
+    - module: upath.implementations.hdfs
+      cls: HDFSPath
+      supported_example_name: host
+      unsupported_example_value: '8020'
+    - module: upath.implementations.http
+      cls: HTTPPath
+      supported_example_name: simple_links
+      unsupported_example_value: '"no"'
+    - module: upath.implementations.local
+      cls: FilePath
+      supported_example_name: auto_mkdir
+      unsupported_example_value: '"abc"'
+    - module: upath.implementations.memory
+      cls: MemoryPath
+      supported_example_name: use_listings_cache
+      unsupported_example_value: '{}'
+    - module: upath.implementations.sftp
+      cls: SFTPPath
+      supported_example_name: host
+      unsupported_example_value: '[]'
+    - module: upath.implementations.smb
+      cls: SMBPath
+      supported_example_name: host
+      unsupported_example_value: 'False'
+    - module: upath.implementations.webdav
+      cls: WebdavPath
+      supported_example_name: base_url
+      unsupported_example_value: '123'
+  main: |
+    from {{ module }} import {{ cls }}
+
+    p = {{ cls }}(".", {{ supported_example_name }}={{ unsupported_example_value }})  # ER: Argument "{{ supported_example_name }}" to "{{ cls }}" has incompatible type.*
+
+# FIXME: mypy emits a 'defined here' note when emitting the error below.
+#        seems to be a limitation/bug in pytest-mypy-plugins that I can't match it
+#
+# - case: upath_constructor_sopts_incorrect_name
+#   disable_cache: false
+#   regex: yes
+#   parametrized:
+#     - module: upath.implementations.cached
+#       cls: SimpleCachePath
+#       unsupported_example_name: idontexist
+#   main: |
+#     from {{ module }} import {{ cls }}
+#
+#     p = {{ cls }}(".", {{ unsupported_example_name }}=1)
+#   out: |
+#     \\.\\..*
+#     main:3: error: Unexpected keyword argument "idontexist" to "{{ cls }}".*

upath/implementations/cached.py

@@ -1,7 +1,35 @@
 from __future__ import annotations
 
+import sys
+from typing import TYPE_CHECKING
+
 from upath.core import UPath
+from upath.types import JoinablePathLike
+
+if TYPE_CHECKING:
+    from typing import Literal
+
+    if sys.version_info >= (3, 11):
+        from typing import Unpack
+    else:
+        from typing_extensions import Unpack
+
+    from upath._chain import FSSpecChainParser
+    from upath.types.storage_options import SimpleCacheStorageOptions
+
+
+__all__ = ["SimpleCachePath"]
 
 
 class SimpleCachePath(UPath):
-    pass
+    __slots__ = ()
+
+    if TYPE_CHECKING:
+
+        def __init__(
+            self,
+            *args: JoinablePathLike,
+            protocol: Literal["simplecache"] | None = ...,
+            chain_parser: FSSpecChainParser = ...,
+            **storage_options: Unpack[SimpleCacheStorageOptions],
+        ) -> None: ...

upath/implementations/cloud.py

@@ -5,15 +5,25 @@
 from typing import TYPE_CHECKING
 from typing import Any
 
+from upath._chain import DEFAULT_CHAIN_PARSER
 from upath._flavour import upath_strip_protocol
 from upath.core import UPath
 from upath.types import JoinablePathLike
 
 if TYPE_CHECKING:
+    from typing import Literal
+
     if sys.version_info >= (3, 11):
         from typing import Self
+        from typing import Unpack
     else:
         from typing_extensions import Self
+        from typing_extensions import Unpack
+
+    from upath._chain import FSSpecChainParser
+    from upath.types.storage_options import AzureBlobStorageOptions
+    from upath.types.storage_options import GCSStorageOptions
+    from upath.types.storage_options import S3StorageOptions
 
 __all__ = [
     "CloudPath",
@@ -84,10 +94,13 @@ class GCSPath(CloudPath):
     def __init__(
         self,
         *args: JoinablePathLike,
-        protocol: str | None = None,
-        **storage_options: Any,
+        protocol: Literal["gcs", "gs"] | None = None,
+        chain_parser: FSSpecChainParser = DEFAULT_CHAIN_PARSER,
+        **storage_options: Unpack[GCSStorageOptions],
     ) -> None:
-        super().__init__(*args, protocol=protocol, **storage_options)
+        super().__init__(
+            *args, protocol=protocol, chain_parser=chain_parser, **storage_options
+        )
         if not self.drive and len(self.parts) > 1:
             raise ValueError("non key-like path provided (bucket/container missing)")
 
@@ -114,10 +127,13 @@ class S3Path(CloudPath):
     def __init__(
         self,
         *args: JoinablePathLike,
-        protocol: str | None = None,
-        **storage_options: Any,
+        protocol: Literal["s3", "s3a"] | None = None,
+        chain_parser: FSSpecChainParser = DEFAULT_CHAIN_PARSER,
+        **storage_options: Unpack[S3StorageOptions],
     ) -> None:
-        super().__init__(*args, protocol=protocol, **storage_options)
+        super().__init__(
+            *args, protocol=protocol, chain_parser=chain_parser, **storage_options
+        )
         if not self.drive and len(self.parts) > 1:
             raise ValueError("non key-like path provided (bucket/container missing)")
 
@@ -128,9 +144,12 @@ class AzurePath(CloudPath):
     def __init__(
         self,
         *args: JoinablePathLike,
-        protocol: str | None = None,
-        **storage_options: Any,
+        protocol: Literal["abfs", "abfss", "adl", "az"] | None = None,
+        chain_parser: FSSpecChainParser = DEFAULT_CHAIN_PARSER,
+        **storage_options: Unpack[AzureBlobStorageOptions],
     ) -> None:
-        super().__init__(*args, protocol=protocol, **storage_options)
+        super().__init__(
+            *args, protocol=protocol, chain_parser=chain_parser, **storage_options
+        )
         if not self.drive and len(self.parts) > 1:
             raise ValueError("non key-like path provided (bucket/container missing)")

upath/implementations/data.py

@@ -8,13 +8,33 @@
 from upath.types import JoinablePathLike
 
 if TYPE_CHECKING:
-    if sys.version_info > (3, 11):
+    from typing import Literal
+
+    if sys.version_info >= (3, 11):
         from typing import Self
+        from typing import Unpack
     else:
         from typing_extensions import Self
+        from typing_extensions import Unpack
+
+    from upath._chain import FSSpecChainParser
+    from upath.types.storage_options import DataStorageOptions
+
+__all__ = ["DataPath"]
 
 
 class DataPath(UPath):
+    __slots__ = ()
+
+    if TYPE_CHECKING:
+
+        def __init__(
+            self,
+            *args: JoinablePathLike,
+            protocol: Literal["data"] | None = ...,
+            chain_parser: FSSpecChainParser = ...,
+            **storage_options: Unpack[DataStorageOptions],
+        ) -> None: ...
 
     @property
     def parts(self) -> Sequence[str]:

upath/implementations/github.py

@@ -9,20 +9,42 @@
 from collections.abc import Sequence
 from typing import TYPE_CHECKING
 
-import upath.core
+from upath.core import UPath
+from upath.types import JoinablePathLike
 
 if TYPE_CHECKING:
-    if sys.version_info > (3, 11):
+    from typing import Literal
+
+    if sys.version_info >= (3, 11):
         from typing import Self
+        from typing import Unpack
     else:
         from typing_extensions import Self
+        from typing_extensions import Unpack
+
+    from upath._chain import FSSpecChainParser
+    from upath.types.storage_options import GithubStorageOptions
+
+__all__ = ["GitHubPath"]
 
 
-class GitHubPath(upath.core.UPath):
+class GitHubPath(UPath):
     """
     GitHubPath supporting the fsspec.GitHubFileSystem
     """
 
+    __slots__ = ()
+
+    if TYPE_CHECKING:
+
+        def __init__(
+            self,
+            *args: JoinablePathLike,
+            protocol: Literal["github"] | None = ...,
+            chain_parser: FSSpecChainParser = ...,
+            **storage_options: Unpack[GithubStorageOptions],
+        ) -> None: ...
+
     @property
     def path(self) -> str:
         pth = super().path

upath/implementations/hdfs.py

@@ -5,19 +5,37 @@
 from typing import TYPE_CHECKING
 
 from upath.core import UPath
+from upath.types import JoinablePathLike
 
 if TYPE_CHECKING:
-    if sys.version_info > (3, 11):
+    from typing import Literal
+
+    if sys.version_info >= (3, 11):
         from typing import Self
+        from typing import Unpack
     else:
         from typing_extensions import Self
+        from typing_extensions import Unpack
+
+    from upath._chain import FSSpecChainParser
+    from upath.types.storage_options import HDFSStorageOptions
 
 __all__ = ["HDFSPath"]
 
 
 class HDFSPath(UPath):
     __slots__ = ()
 
+    if TYPE_CHECKING:
+
+        def __init__(
+            self,
+            *args: JoinablePathLike,
+            protocol: Literal["hdfs"] | None = ...,
+            chain_parser: FSSpecChainParser = ...,
+            **storage_options: Unpack[HDFSStorageOptions],
+        ) -> None: ...
+
     def mkdir(
         self, mode: int = 0o777, parents: bool = False, exist_ok: bool = False
     ) -> None: