Add top-level from_url function

docs/api/store/index.md

@@ -1,3 +1,4 @@
 # ObjectStore
 
+::: obstore.store.new_store
 ::: obstore.store.ObjectStore

obstore/python/obstore/store/__init__.pyi

@@ -1,5 +1,6 @@
 # TODO: move to reusable types package
 from pathlib import Path
+from typing import Any, Unpack, overload
 
 from ._aws import S3Config as S3Config
 from ._aws import S3Store as S3Store
@@ -13,6 +14,40 @@ from ._prefix import PrefixStore as PrefixStore
 from ._retry import BackoffConfig as BackoffConfig
 from ._retry import RetryConfig as RetryConfig
 
+@overload
+def new_store(
+    url: str, *, config: S3Config | None = None, **kwargs: Unpack[S3Config]
+) -> PrefixStore: ...
+@overload
+def new_store(
+    url: str, *, config: GCSConfig | None = None, **kwargs: Unpack[GCSConfig]
+) -> PrefixStore: ...
+@overload
+def new_store(
+    url: str, *, config: AzureConfig | None = None, **kwargs: Unpack[AzureConfig]
+) -> PrefixStore: ...
+def new_store(
+    url: str, *, config: S3Config | GCSConfig | AzureConfig | None = None, **kwargs: Any
+) -> PrefixStore:
+    """Easy construction of store by URL, identifying the relevant store.
+
+    Supported formats:
+
+    - `file:///path/to/my/file` -> [`LocalStore`][obstore.store.LocalStore]
+    - `memory:///` -> [`MemoryStore`][obstore.store.MemoryStore]
+    - `s3://bucket/path` -> [`S3Store`][obstore.store.S3Store] (also supports `s3a`)
+    - `gs://bucket/path` -> [`GCSStore`][obstore.store.GCSStore]
+    - `az://account/container/path` -> [`AzureStore`][obstore.store.AzureStore] (also supports `adl`, `azure`, `abfs`, `abfss`)
+    - `http://mydomain/path` -> [`HTTPStore`][obstore.store.HTTPStore]
+    - `https://mydomain/path` -> [`HTTPStore`][obstore.store.HTTPStore]
+
+    There are also special cases for AWS and Azure for `https://{host?}/path` paths:
+
+    - `dfs.core.windows.net`, `blob.core.windows.net`, `dfs.fabric.microsoft.com`, `blob.fabric.microsoft.com` -> [`AzureStore`][obstore.store.AzureStore]
+    - `amazonaws.com` -> [`S3Store`][obstore.store.S3Store]
+    - `r2.cloudflarestorage.com` -> [`S3Store`][obstore.store.S3Store]
+    """
+
 class LocalStore:
     """
     Local filesystem storage providing an ObjectStore interface to files on local disk.

obstore/src/signer.rs

@@ -13,9 +13,8 @@ use pyo3::exceptions::PyValueError;
 use pyo3::intern;
 use pyo3::prelude::*;
 use pyo3::pybacked::PyBackedStr;
-use pyo3::types::PyString;
 use pyo3_object_store::{
-    PyAzureStore, PyGCSStore, PyObjectStoreError, PyObjectStoreResult, PyS3Store,
+    PyAzureStore, PyGCSStore, PyObjectStoreError, PyObjectStoreResult, PyS3Store, PyUrl,
 };
 use url::Url;
 
@@ -129,18 +128,6 @@ impl<'py> FromPyObject<'py> for PyMethod {
     }
 }
 
-pub(crate) struct PyUrl(url::Url);
-
-impl<'py> IntoPyObject<'py> for PyUrl {
-    type Target = PyString;
-    type Output = Bound<'py, PyString>;
-    type Error = std::convert::Infallible;
-
-    fn into_pyobject(self, py: Python<'py>) -> Result<Self::Output, Self::Error> {
-        String::from(self.0).into_pyobject(py)
-    }
-}
-
 #[derive(IntoPyObject)]
 pub(crate) struct PyUrls(Vec<PyUrl>);
 
@@ -164,12 +151,12 @@ pub(crate) fn sign(
     py.allow_threads(|| match paths {
         PyPaths::One(path) => {
             let url = runtime.block_on(store.signed_url(method, &path, expires_in))?;
-            Ok(PySignResult::One(PyUrl(url)))
+            Ok(PySignResult::One(PyUrl::new(url)))
         }
         PyPaths::Many(paths) => {
             let urls = runtime.block_on(store.signed_urls(method, &paths, expires_in))?;
             Ok(PySignResult::Many(PyUrls(
-                urls.into_iter().map(PyUrl).collect(),
+                urls.into_iter().map(PyUrl::new).collect(),
             )))
         }
     })
@@ -191,15 +178,15 @@ pub(crate) fn sign_async(
                     .signed_url(method, &path, expires_in)
                     .await
                     .map_err(PyObjectStoreError::ObjectStoreError)?;
-                Ok(PySignResult::One(PyUrl(url)))
+                Ok(PySignResult::One(PyUrl::new(url)))
             }
             PyPaths::Many(paths) => {
                 let urls = store
                     .signed_urls(method, &paths, expires_in)
                     .await
                     .map_err(PyObjectStoreError::ObjectStoreError)?;
                 Ok(PySignResult::Many(PyUrls(
-                    urls.into_iter().map(PyUrl).collect(),
+                    urls.into_iter().map(PyUrl::new).collect(),
                 )))
             }
         }

pyo3-object_store/src/api.rs

@@ -3,7 +3,8 @@ use pyo3::prelude::*;
 
 use crate::error::*;
 use crate::{
-    PyAzureStore, PyGCSStore, PyHttpStore, PyLocalStore, PyMemoryStore, PyPrefixStore, PyS3Store,
+    new_store, PyAzureStore, PyGCSStore, PyHttpStore, PyLocalStore, PyMemoryStore, PyPrefixStore,
+    PyS3Store,
 };
 
 /// Export the default Python API as a submodule named `store` within the given parent module
@@ -46,6 +47,7 @@ pub fn register_store_module(
 
     let child_module = PyModule::new(parent_module.py(), "store")?;
 
+    child_module.add_wrapped(wrap_pyfunction!(new_store))?;
     child_module.add_class::<PyAzureStore>()?;
     child_module.add_class::<PyGCSStore>()?;
     child_module.add_class::<PyHttpStore>()?;

pyo3-object_store/src/aws.rs

@@ -123,7 +123,7 @@ impl PyS3Store {
 
     #[classmethod]
     #[pyo3(signature = (url, *, config=None, client_options=None, retry_config=None, **kwargs))]
-    fn from_url(
+    pub(crate) fn from_url(
         _cls: &Bound<PyType>,
         url: &str,
         config: Option<PyAmazonS3Config>,
@@ -161,7 +161,13 @@ impl<'py> FromPyObject<'py> for PyAmazonS3ConfigKey {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+impl AsRef<str> for PyAmazonS3ConfigKey {
+    fn as_ref(&self) -> &str {
+        self.0.as_ref()
+    }
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
 pub struct PyAmazonS3Config(HashMap<PyAmazonS3ConfigKey, PyConfigValue>);
 
 impl<'py> FromPyObject<'py> for PyAmazonS3Config {

pyo3-object_store/src/azure.rs

@@ -59,7 +59,7 @@ impl PyAzureStore {
 
     #[classmethod]
     #[pyo3(signature = (url, *, config=None, client_options=None, retry_config=None, **kwargs))]
-    fn from_url(
+    pub(crate) fn from_url(
         _cls: &Bound<PyType>,
         url: &str,
         config: Option<PyAzureConfig>,
@@ -97,7 +97,13 @@ impl<'py> FromPyObject<'py> for PyAzureConfigKey {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+impl AsRef<str> for PyAzureConfigKey {
+    fn as_ref(&self) -> &str {
+        self.0.as_ref()
+    }
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
 pub struct PyAzureConfig(HashMap<PyAzureConfigKey, PyConfigValue>);
 
 impl<'py> FromPyObject<'py> for PyAzureConfig {

pyo3-object_store/src/config.rs

@@ -24,3 +24,9 @@ impl<'py> FromPyObject<'py> for PyConfigValue {
         }
     }
 }
+
+impl From<PyConfigValue> for String {
+    fn from(value: PyConfigValue) -> Self {
+        value.0
+    }
+}

pyo3-object_store/src/gcp.rs

@@ -59,7 +59,7 @@ impl PyGCSStore {
 
     #[classmethod]
     #[pyo3(signature = (url, *, config=None, client_options=None, retry_config=None, **kwargs))]
-    fn from_url(
+    pub(crate) fn from_url(
         _cls: &Bound<PyType>,
         url: &str,
         config: Option<PyGoogleConfig>,
@@ -97,7 +97,13 @@ impl<'py> FromPyObject<'py> for PyGoogleConfigKey {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+impl AsRef<str> for PyGoogleConfigKey {
+    fn as_ref(&self) -> &str {
+        self.0.as_ref()
+    }
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
 pub struct PyGoogleConfig(HashMap<PyGoogleConfigKey, PyConfigValue>);
 
 impl<'py> FromPyObject<'py> for PyGoogleConfig {

pyo3-object_store/src/http.rs

@@ -33,7 +33,7 @@ impl PyHttpStore {
 impl PyHttpStore {
     #[classmethod]
     #[pyo3(signature = (url, *, client_options=None, retry_config=None))]
-    fn from_url(
+    pub(crate) fn from_url(
         _cls: &Bound<PyType>,
         url: &str,
         client_options: Option<PyClientOptions>,

pyo3-object_store/src/lib.rs

@@ -13,7 +13,9 @@ mod local;
 mod memory;
 mod prefix;
 mod retry;
+mod simple;
 mod store;
+mod url;
 
 pub use api::{register_exceptions_module, register_store_module};
 pub use aws::PyS3Store;
@@ -25,4 +27,6 @@ pub use http::PyHttpStore;
 pub use local::PyLocalStore;
 pub use memory::PyMemoryStore;
 pub use prefix::PyPrefixStore;
+pub use simple::new_store;
 pub use store::PyObjectStore;
+pub use url::PyUrl;

pyo3-object_store/src/local.rs

@@ -49,7 +49,7 @@ impl PyLocalStore {
     }
 
     #[classmethod]
-    fn from_url(_cls: &Bound<PyType>, url: &str) -> PyObjectStoreResult<Self> {
+    pub(crate) fn from_url(_cls: &Bound<PyType>, url: &str) -> PyObjectStoreResult<Self> {
         let url = Url::parse(url).map_err(|err| PyValueError::new_err(err.to_string()))?;
         let (scheme, path) = ObjectStoreScheme::parse(&url).map_err(object_store::Error::from)?;
 

pyo3-object_store/src/memory.rs

@@ -15,6 +15,12 @@ impl AsRef<Arc<InMemory>> for PyMemoryStore {
     }
 }
 
+impl From<Arc<InMemory>> for PyMemoryStore {
+    fn from(value: Arc<InMemory>) -> Self {
+        Self(value)
+    }
+}
+
 impl<'py> PyMemoryStore {
     /// Consume self and return the underlying [`InMemory`].
     pub fn into_inner(self) -> Arc<InMemory> {

pyo3-object_store/src/prefix.rs

@@ -17,6 +17,12 @@ impl AsRef<Arc<PrefixStore<Arc<dyn ObjectStore>>>> for PyPrefixStore {
     }
 }
 
+impl From<PrefixStore<Arc<dyn ObjectStore>>> for PyPrefixStore {
+    fn from(value: PrefixStore<Arc<dyn ObjectStore>>) -> Self {
+        PyPrefixStore(Arc::new(value))
+    }
+}
+
 #[pymethods]
 impl PyPrefixStore {
     #[new]

pyo3-object_store/src/simple.rs

@@ -0,0 +1,99 @@
+use std::sync::Arc;
+
+use object_store::memory::InMemory;
+use object_store::ObjectStoreScheme;
+use pyo3::prelude::*;
+use pyo3::types::PyType;
+use pyo3::IntoPyObjectExt;
+
+use crate::error::ObstoreError;
+use crate::url::PyUrl;
+use crate::{
+    PyAzureStore, PyGCSStore, PyHttpStore, PyLocalStore, PyMemoryStore, PyObjectStoreResult,
+    PyS3Store,
+};
+
+/// Simple construction of stores by url.
+// Note: We don't extract the PyObject in the function signature because it's possible that
+// AWS/Azure/Google config keys could overlap. And so we don't want to accidentally parse a config
+// as an AWS config before knowing that the URL scheme is AWS.
+#[pyfunction]
+#[pyo3(signature = (url, *, config=None, **kwargs))]
+pub fn new_store(
+    py: Python,
+    url: PyUrl,
+    config: Option<Bound<PyAny>>,
+    kwargs: Option<Bound<PyAny>>,
+) -> PyObjectStoreResult<PyObject> {
+    let url = url.into_inner();
+    let (scheme, _) = ObjectStoreScheme::parse(&url).map_err(object_store::Error::from)?;
+    match scheme {
+        ObjectStoreScheme::AmazonS3 => {
+            let store = PyS3Store::from_url(
+                &PyType::new::<PyS3Store>(py),
+                url.as_str(),
+                config.map(|x| x.extract()).transpose()?,
+                None,
+                None,
+                kwargs.map(|x| x.extract()).transpose()?,
+            )?;
+            Ok(store.into_pyobject(py)?.into_py_any(py)?)
+        }
+        ObjectStoreScheme::GoogleCloudStorage => {
+            let store = PyGCSStore::from_url(
+                &PyType::new::<PyGCSStore>(py),
+                url.as_str(),
+                config.map(|x| x.extract()).transpose()?,
+                None,
+                None,
+                kwargs.map(|x| x.extract()).transpose()?,
+            )?;
+            Ok(store.into_pyobject(py)?.into_py_any(py)?)
+        }
+        ObjectStoreScheme::MicrosoftAzure => {
+            let store = PyAzureStore::from_url(
+                &PyType::new::<PyAzureStore>(py),
+                url.as_str(),
+                config.map(|x| x.extract()).transpose()?,
+                None,
+                None,
+                kwargs.map(|x| x.extract()).transpose()?,
+            )?;
+            Ok(store.into_pyobject(py)?.into_py_any(py)?)
+        }
+        ObjectStoreScheme::Http => {
+            raise_if_config_passed(config, kwargs, "http")?;
+            let store =
+                PyHttpStore::from_url(&PyType::new::<PyHttpStore>(py), url.as_str(), None, None)?;
+            Ok(store.into_pyobject(py)?.into_py_any(py)?)
+        }
+        ObjectStoreScheme::Local => {
+            raise_if_config_passed(config, kwargs, "local")?;
+            let store = PyLocalStore::from_url(&PyType::new::<PyLocalStore>(py), url.as_str())?;
+            Ok(store.into_pyobject(py)?.into_py_any(py)?)
+        }
+        ObjectStoreScheme::Memory => {
+            raise_if_config_passed(config, kwargs, "memory")?;
+            let store: PyMemoryStore = Arc::new(InMemory::new()).into();
+            Ok(store.into_pyobject(py)?.into_py_any(py)?)
+        }
+        scheme => {
+            return Err(ObstoreError::new_err(format!("Unknown URL scheme {:?}", scheme,)).into());
+        }
+    }
+}
+
+fn raise_if_config_passed(
+    config: Option<Bound<PyAny>>,
+    kwargs: Option<Bound<PyAny>>,
+    scheme: &str,
+) -> PyObjectStoreResult<()> {
+    if config.is_some() || kwargs.is_some() {
+        return Err(ObstoreError::new_err(format!(
+            "Cannot pass config or keyword parameters for scheme {:?}",
+            scheme,
+        ))
+        .into());
+    }
+    Ok(())
+}

pyo3-object_store/src/store.rs

@@ -53,7 +53,6 @@ impl<'py> FromPyObject<'py> for PyObjectStore {
                 return Err(PyValueError::new_err("You must use an object store instance exported from **the same library** as this function. They cannot be used across libraries.\nThis is because object store instances are compiled with a specific version of Rust and Python." ));
             }
 
-            // TODO: Check for fsspec
             Err(PyValueError::new_err(format!(
                 "Expected an object store instance, got {}",
                 ob.repr()?