Access config values out from stores (#210)

* Access config values out from stores

* Update `from_url` typing

* doc note

Author: kylebarron

docs/api/store/aws.md

@@ -1,6 +1,9 @@
 # AWS S3
 
 ::: obstore.store.S3Store
+::: obstore.store.S3ConfigInput
+    options:
+        show_if_no_docstring: true
 ::: obstore.store.S3Config
     options:
         show_if_no_docstring: true

docs/api/store/azure.md

@@ -1,6 +1,9 @@
 # Microsoft Azure
 
 ::: obstore.store.AzureStore
+::: obstore.store.AzureConfigInput
+    options:
+        show_if_no_docstring: true
 ::: obstore.store.AzureConfig
     options:
         show_if_no_docstring: true

docs/api/store/gcs.md

@@ -1,6 +1,9 @@
 # Google Cloud Storage
 
 ::: obstore.store.GCSStore
+::: obstore.store.GCSConfigInput
+    options:
+        show_if_no_docstring: true
 ::: obstore.store.GCSConfig
     options:
         show_if_no_docstring: true

obstore/python/obstore/store/__init__.pyi

@@ -3,11 +3,14 @@ from pathlib import Path
 from typing import Any, Unpack, overload
 
 from ._aws import S3Config as S3Config
+from ._aws import S3ConfigInput as S3ConfigInput
 from ._aws import S3Store as S3Store
 from ._azure import AzureConfig as AzureConfig
+from ._azure import AzureConfigInput as AzureConfigInput
 from ._azure import AzureStore as AzureStore
 from ._client import ClientConfig as ClientConfig
 from ._gcs import GCSConfig as GCSConfig
+from ._gcs import GCSConfigInput as GCSConfigInput
 from ._gcs import GCSStore as GCSStore
 from ._http import HTTPStore as HTTPStore
 from ._retry import BackoffConfig as BackoffConfig
@@ -17,28 +20,28 @@ from ._retry import RetryConfig as RetryConfig
 def from_url(
     url: str,
     *,
-    config: S3Config | None = None,
+    config: S3Config | S3ConfigInput | None = None,
     client_options: ClientConfig | None = None,
     retry_config: RetryConfig | None = None,
-    **kwargs: Unpack[S3Config],
+    **kwargs: Unpack[S3ConfigInput],
 ) -> ObjectStore: ...
 @overload
 def from_url(
     url: str,
     *,
-    config: GCSConfig | None = None,
+    config: GCSConfig | GCSConfigInput | None = None,
     client_options: ClientConfig | None = None,
     retry_config: RetryConfig | None = None,
-    **kwargs: Unpack[GCSConfig],
+    **kwargs: Unpack[GCSConfigInput],
 ) -> ObjectStore: ...
 @overload
 def from_url(
     url: str,
     *,
-    config: AzureConfig | None = None,
+    config: AzureConfig | AzureConfigInput | None = None,
     client_options: ClientConfig | None = None,
     retry_config: RetryConfig | None = None,
-    **kwargs: Unpack[AzureConfig],
+    **kwargs: Unpack[AzureConfigInput],
 ) -> ObjectStore: ...
 @overload
 def from_url(
@@ -53,7 +56,7 @@ def from_url(
 def from_url(
     url: str,
     *,
-    config: S3Config | GCSConfig | AzureConfig | None = None,
+    config: S3ConfigInput | GCSConfigInput | AzureConfigInput | None = None,
     client_options: ClientConfig | None = None,
     retry_config: RetryConfig | None = None,
     **kwargs: Any,
@@ -80,6 +83,10 @@ def from_url(
     - `amazonaws.com` -> [`S3Store`][obstore.store.S3Store]
     - `r2.cloudflarestorage.com` -> [`S3Store`][obstore.store.S3Store]
 
+    !!! note
+        For best static typing, use the constructors on individual store classes
+        directly.
+
     Args:
         url: well-known storage URL.
 

obstore/python/obstore/store/_aws.pyi

@@ -11,12 +11,84 @@ from ._retry import RetryConfig
 # Note: we removed `bucket` because it overlaps with an existing named arg in the
 # constructors
 class S3Config(TypedDict, total=False):
-    """Configuration parameters for S3Store.
+    """Configuration parameters returned from [S3Store.config][obstore.store.S3Store.config].
 
-    There are duplicates of many parameters, and parameters can be either upper or lower
-    case. Not all parameters are required.
+    Note that this is a strict subset of the keys allowed for _input_ into the store,
+    see [S3ConfigInput][obstore.store.S3ConfigInput].
     """
 
+    aws_access_key_id: str
+    """AWS Access Key"""
+    aws_bucket: str
+    """Bucket name"""
+    aws_checksum_algorithm: str
+    """
+    Sets the [checksum algorithm] which has to be used for object integrity check during upload.
+
+    [checksum algorithm]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+    """
+    aws_conditional_put: str
+    """
+    See [`S3ConfigInput.aws_conditional_put`][obstore.store.S3ConfigInput.aws_conditional_put].
+    """
+    aws_container_credentials_relative_uri: str
+    """
+    See [`S3ConfigInput.aws_container_credentials_relative_uri`][obstore.store.S3ConfigInput.aws_container_credentials_relative_uri].
+    """
+    aws_copy_if_not_exists: str
+    """
+    See [`S3ConfigInput.aws_copy_if_not_exists`][obstore.store.S3ConfigInput.aws_copy_if_not_exists].
+    """
+    aws_default_region: str
+    """Default region"""
+    aws_disable_tagging: bool
+    """Disable tagging objects. This can be desirable if not supported by the backing store."""
+    aws_endpoint: str
+    """Sets custom endpoint for communicating with AWS S3."""
+    aws_imdsv1_fallback: str
+    """Fall back to ImdsV1"""
+    aws_metadata_endpoint: str
+    """Set the instance metadata endpoint"""
+    aws_region: str
+    """Region"""
+    aws_request_payer: bool
+    """If `True`, enable operations on requester-pays buckets."""
+    aws_s3_express: bool
+    """Enable Support for S3 Express One Zone"""
+    aws_secret_access_key: str
+    """Secret Access Key"""
+    aws_server_side_encryption: str
+    """
+    See [`S3ConfigInput.aws_server_side_encryption`][obstore.store.S3ConfigInput.aws_server_side_encryption].
+    """
+    aws_session_token: str
+    """Token to use for requests (passed to underlying provider)"""
+    aws_skip_signature: bool
+    """If `True`, S3Store will not fetch credentials and will not sign requests."""
+    aws_sse_bucket_key_enabled: bool
+    """
+    If set to `True`, will use the bucket's default KMS key for server-side encryption.
+    If set to `False`, will disable the use of the bucket's default KMS key for server-side encryption.
+    """
+    aws_sse_customer_key_base64: str
+    """
+    The base64 encoded, 256-bit customer encryption key to use for server-side
+    encryption. If set, the server side encryption config value must be `"sse-c"`.
+    """
+    aws_sse_kms_key_id: str
+    """
+    The KMS key ID to use for server-side encryption.
+
+    If set, the server side encryption config value must be `"aws:kms"` or `"aws:kms:dsse"`.
+    """
+    aws_token: str
+    """Token to use for requests (passed to underlying provider)"""
+    aws_unsigned_payload: bool
+    """Avoid computing payload checksum when calculating signature."""
+    aws_virtual_hosted_style_request: bool
+    """If virtual hosted style request has to be used."""
+
+class S3ConfigInput(TypedDict, total=False):
     access_key_id: str
     """AWS Access Key"""
     aws_access_key_id: str
@@ -152,6 +224,7 @@ class S3Config(TypedDict, total=False):
     """Avoid computing payload checksum when calculating signature."""
     aws_virtual_hosted_style_request: bool
     """If virtual hosted style request has to be used."""
+
     bucket_name: str
     """Bucket name"""
     checksum_algorithm: str
@@ -491,8 +564,8 @@ class S3Store:
 
     All constructors will check for environment variables. All environment variables
     starting with `AWS_` will be evaluated. Names must match keys from
-    [`S3Config`][obstore.store.S3Config]. Only upper-case environment variables are
-    accepted.
+    [`S3ConfigInput`][obstore.store.S3ConfigInput]. Only upper-case environment
+    variables are accepted.
 
     Some examples of variables extracted from environment:
 
@@ -522,10 +595,10 @@ class S3Store:
         bucket: str | None = None,
         *,
         prefix: str | None = None,
-        config: S3Config | None = None,
+        config: S3Config | S3ConfigInput | None = None,
         client_options: ClientConfig | None = None,
         retry_config: RetryConfig | None = None,
-        **kwargs: Unpack[S3Config],
+        **kwargs: Unpack[S3ConfigInput],
     ) -> None:
         """Create a new S3Store.
 
@@ -549,10 +622,10 @@ class S3Store:
         bucket: str | None = None,
         *,
         prefix: str | None = None,
-        config: S3Config | None = None,
+        config: S3Config | S3ConfigInput | None = None,
         client_options: ClientConfig | None = None,
         retry_config: RetryConfig | None = None,
-        **kwargs: Unpack[S3Config],
+        **kwargs: Unpack[S3ConfigInput],
     ) -> S3Store:
         """Construct a new S3Store with credentials inferred from a boto3 Session.
 
@@ -590,10 +663,10 @@ class S3Store:
         cls,
         url: str,
         *,
-        config: S3Config | None = None,
+        config: S3Config | S3ConfigInput | None = None,
         client_options: ClientConfig | None = None,
         retry_config: RetryConfig | None = None,
-        **kwargs: Unpack[S3Config],
+        **kwargs: Unpack[S3ConfigInput],
     ) -> S3Store:
         """Parse available connection info from a well-known storage URL.
 
@@ -620,3 +693,15 @@ class S3Store:
 
     def __getnewargs_ex__(self): ...
     def __repr__(self) -> str: ...
+    @property
+    def prefix(self) -> str | None:
+        """Get the prefix applied to all operations in this store, if any."""
+    @property
+    def config(self) -> S3Config:
+        """Get the underlying S3 config parameters."""
+    @property
+    def client_options(self) -> ClientConfig | None:
+        """Get the store's client configuration."""
+    @property
+    def retry_config(self) -> RetryConfig | None:
+        """Get the store's retry configuration."""

obstore/python/obstore/store/_azure.pyi

@@ -3,7 +3,62 @@ from typing import TypedDict, Unpack
 from ._client import ClientConfig
 from ._retry import RetryConfig
 
+# TODO:
+# azure_storage_authority_host
+# azure_fabric_token_service_url
+# azure_fabric_workload_host
+# "azure_fabric_session_token",
+# "azure_fabric_cluster_identifier",
 class AzureConfig(TypedDict, total=False):
+    """Configuration parameters returned from [AzureStore.config][obstore.store.AzureStore.config].
+
+    Note that this is a strict subset of the keys allowed for _input_ into the store,
+    see [AzureConfigInput][obstore.store.AzureConfigInput].
+    """
+
+    azure_storage_account_name: str
+    """The name of the azure storage account"""
+    azure_storage_account_key: str
+    """Master key for accessing storage account"""
+    azure_storage_client_id: str
+    """Service principal client id for authorizing requests"""
+    azure_storage_client_secret: str
+    """Service principal client secret for authorizing requests"""
+    azure_storage_tenant_id: str
+    """Tenant id used in oauth flows"""
+    azure_storage_sas_key: str
+    """
+    Shared access signature.
+
+    The signature is expected to be percent-encoded, `much `like they are provided in
+    the azure storage explorer or azure portal.
+    """
+    azure_storage_token: str
+    """Bearer token"""
+    azure_storage_use_emulator: bool
+    """Use object store with azurite storage emulator"""
+    azure_use_fabric_endpoint: bool
+    """Use object store with url scheme account.dfs.fabric.microsoft.com"""
+    azure_storage_endpoint: str
+    """Override the endpoint used to communicate with blob storage"""
+    azure_msi_endpoint: str
+    """Endpoint to request a imds managed identity token"""
+    azure_object_id: str
+    """Object id for use with managed identity authentication"""
+    azure_msi_resource_id: str
+    """Msi resource id for use with managed identity authentication"""
+    azure_federated_token_file: str
+    """File containing token for Azure AD workload identity federation"""
+    azure_use_azure_cli: bool
+    """Use azure cli for acquiring access token"""
+    azure_skip_signature: bool
+    """Skip signing requests"""
+    azure_container_name: str
+    """Container name"""
+    azure_disable_tagging: bool
+    """Disables tagging objects"""
+
+class AzureConfigInput(TypedDict, total=False):
     """Configuration parameters for AzureStore.
 
     There are duplicates of many parameters, and parameters can be either upper or lower
@@ -282,7 +337,7 @@ class AzureStore:
         container: str | None = None,
         *,
         prefix: str | None = None,
-        config: AzureConfig | None = None,
+        config: AzureConfig | AzureConfigInput | None = None,
         client_options: ClientConfig | None = None,
         retry_config: RetryConfig | None = None,
         **kwargs: Unpack[AzureConfig],
@@ -307,7 +362,7 @@ class AzureStore:
         url: str,
         *,
         prefix: str | None = None,
-        config: AzureConfig | None = None,
+        config: AzureConfig | AzureConfigInput | None = None,
         client_options: ClientConfig | None = None,
         retry_config: RetryConfig | None = None,
         **kwargs: Unpack[AzureConfig],
@@ -344,3 +399,15 @@ class AzureStore:
 
     def __getnewargs_ex__(self): ...
     def __repr__(self) -> str: ...
+    @property
+    def prefix(self) -> str | None:
+        """Get the prefix applied to all operations in this store, if any."""
+    @property
+    def config(self) -> AzureConfig:
+        """Get the underlying Azure config parameters."""
+    @property
+    def client_options(self) -> ClientConfig | None:
+        """Get the store's client configuration."""
+    @property
+    def retry_config(self) -> RetryConfig | None:
+        """Get the store's retry configuration."""