feat: refactor

.github/workflows/ci.yaml

@@ -27,5 +27,7 @@ jobs:
       - uses: Swatinem/rust-cache@v2
       - name: Sync
         run: uv sync
+      - name: Lint
+        run: scripts/lint
       - name: Test
-        run: uv run pytest
+        run: scripts/test

CHANGELOG.md

@@ -10,6 +10,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 - DuckDB client ([#15](https://github.com/gadomski/stacrs/pull/15))
 
+### Changed
+
+- `read` and `write` are now async ([#18](https://github.com/gadomski/stacrs/pull/18))
+
 ## [0.3.0] - 2024-11-21
 
 ### Removed

Cargo.toml

@@ -11,9 +11,11 @@ crate-type = ["cdylib"]
 duckdb = { version = "1.1.1", features = ["bundled"] }
 geojson = "0.24.1"
 pyo3 = "0.23.3"
+pyo3-async-runtimes = { version = "0.23.0", features = ["tokio", "tokio-runtime"] }
 pyo3-log = "0.12.1"
 pythonize = "0.23.0"
-serde_json = "1.0.134"
+serde = "1.0.217"
+serde_json = "1.0.135"
 stac = { version = "0.11.1", features = [
     "geoparquet-compression",
     "object-store-all",

README.md

@@ -74,8 +74,7 @@ Then:
 ```shell
 git clone [email protected]:gadomski/stacrs.git
 cd stacrs
-uv sync  # This will take a little while while the Rust dependencies build
-uv run pytest
+scripts/test  # This will take a little while while the Rust dependencies build, especially DuckDB
 ```
 
 See [CONTRIBUTING.md](./CONTRIBUTING.md) for more information about contributing to this project.

docs/api/duckdb.md

@@ -0,0 +1,7 @@
+---
+description: Query stac-geoparquet with DuckDB
+---
+
+# DuckDB
+
+::: stacrs.DuckdbClient

docs/example.ipynb

@@ -134,7 +134,8 @@
     }
    ],
    "source": [
-    "stacrs.search_to(\"items-compressed.parquet\",\n",
+    "stacrs.search_to(\n",
+    "    \"items-compressed.parquet\",\n",
     "    \"https://landsatlook.usgs.gov/stac-server\",\n",
     "    collections=\"landsat-c2l2-sr\",\n",
     "    intersects={\"type\": \"Point\", \"coordinates\": [-105.119, 40.173]},\n",

mkdocs.yml

@@ -20,10 +20,10 @@ nav:
   - Example notebook: example.ipynb
   - API:
       - api/index.md
+      - duckdb: api/duckdb.md
       - migrate: api/migrate.md
       - read: api/read.md
       - search: api/search.md
-      - validate: api/validate.md
       - version: api/version.md
       - write: api/write.md
   - CONTRIBUTING.md
@@ -37,10 +37,16 @@ plugins:
         python:
           load_external_modules: false
           options:
+            allow_inspection: false
+            docstring_section_style: list
+            docstring_style: google
+            line_length: 80
+            separate_signature: true
             show_root_heading: true
-            show_signature: true
             show_signature_annotations: true
-            separate_signature: true
+            show_source: false
+            show_symbol_type_toc: true
+            signature_crossrefs: true
   - search
   - social:
       cards_layout_options:

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "stacrs"
-description = "A no-dependency Python package for STAC, using Rust under the hood."
+description = "A small, no-dependency Python package for STAC, using Rust under the hood."
 readme = "README.md"
 authors = [{ name = "Pete Gadomski", email = "[email protected]" }]
 requires-python = ">=3.10"
@@ -26,10 +26,17 @@ Repository = "https://github.com/gadomski/stacrs"
 Documentation = "https://gadom.ski/stacrs"
 Issues = "https://github.com/gadomski/stacrs/issues"
 
+[tool.mypy]
+files = "**/*.py"
+
 [[tool.mypy.overrides]]
 module = "pyarrow.*"
 ignore_missing_imports = true
 
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
+
 [tool.uv]
 dev-dependencies = [
     "maturin>=1.7.4",
@@ -38,7 +45,10 @@ dev-dependencies = [
     "mkdocs-material[imaging]>=9.5.45",
     "mkdocstrings[python]>=0.27.0",
     "mypy>=1.11.2",
+    "pyarrow>=18.0.0",
+    "pystac[validation]>=1.11.0",
     "pytest>=8.3.3",
+    "pytest-asyncio>=0.25.1",
     "ruff>=0.6.9",
     "stac-geoparquet>=0.6.0",
 ]

scripts/format

@@ -0,0 +1,6 @@
+#!/usr/bin/env sh
+
+set -e
+
+uv run ruff check --fix
+uv run ruff format

scripts/lint

@@ -0,0 +1,7 @@
+#!/usr/bin/env sh
+
+set -e 
+
+uv run ruff check
+uv run ruff format --check
+uv run mypy

scripts/test

@@ -2,4 +2,5 @@
 
 set -e
 
-maturin dev --uv && pytest "$@"
+uv run maturin dev --uv
+uv run pytest "$@"

src/duckdb.rs

@@ -1,4 +1,4 @@
-use crate::{Error, Result};
+use crate::Result;
 use pyo3::{
     exceptions::PyException,
     prelude::*,
@@ -36,7 +36,7 @@ impl DuckdbClient {
         filter: Option<StringOrDict>,
         query: Option<Bound<'py, PyDict>>,
         kwargs: Option<Bound<'py, PyDict>>,
-    ) -> PyResult<Bound<'py, PyDict>> {
+    ) -> Result<Bound<'py, PyDict>> {
         let search = stac_api::python::search(
             intersects,
             ids,
@@ -56,18 +56,20 @@ impl DuckdbClient {
                 .0
                 .lock()
                 .map_err(|err| PyException::new_err(err.to_string()))?;
-            client.search(&href, search).map_err(Error::from)?
+            client.search(&href, search)?
         };
         let dict = pythonize::pythonize(py, &item_collection)?;
-        dict.extract()
+        let dict = dict.extract()?;
+        Ok(dict)
     }
 
-    fn get_collections<'py>(&self, py: Python<'py>, href: String) -> PyResult<Bound<'py, PyList>> {
+    fn get_collections<'py>(&self, py: Python<'py>, href: String) -> Result<Bound<'py, PyList>> {
         let client = self
             .0
             .lock()
             .map_err(|err| PyException::new_err(err.to_string()))?;
-        let collections = client.collections(&href).map_err(Error::from)?;
-        pythonize::pythonize(py, &collections)?.extract()
+        let collections = client.collections(&href)?;
+        let collections = pythonize::pythonize(py, &collections)?.extract()?;
+        Ok(collections)
     }
 }

src/error.rs

@@ -1,14 +1,26 @@
-use pyo3::{exceptions::PyException, PyErr};
+use pyo3::{
+    create_exception,
+    exceptions::{PyException, PyIOError},
+    PyErr,
+};
 use thiserror::Error;
 
+create_exception!(stacrs, StacrsError, PyException);
+
 #[derive(Debug, Error)]
 pub enum Error {
     #[error(transparent)]
     Geojson(#[from] geojson::Error),
 
+    #[error(transparent)]
+    Io(#[from] std::io::Error),
+
     #[error(transparent)]
     Pythonize(#[from] pythonize::PythonizeError),
 
+    #[error(transparent)]
+    Py(#[from] PyErr),
+
     #[error(transparent)]
     SerdeJson(#[from] serde_json::Error),
 
@@ -23,14 +35,11 @@ pub enum Error {
 }
 
 impl From<Error> for PyErr {
-    fn from(value: Error) -> Self {
-        match value {
-            Error::Geojson(err) => PyException::new_err(err.to_string()),
-            Error::Pythonize(err) => PyException::new_err(err.to_string()),
-            Error::SerdeJson(err) => PyException::new_err(err.to_string()),
-            Error::Stac(err) => PyException::new_err(err.to_string()),
-            Error::StacApi(err) => PyException::new_err(err.to_string()),
-            Error::StacDuckdb(err) => PyException::new_err(err.to_string()),
+    fn from(err: Error) -> Self {
+        match err {
+            Error::Py(err) => err,
+            Error::Io(err) => PyIOError::new_err(err.to_string()),
+            _ => StacrsError::new_err(err.to_string()),
         }
     }
 }

src/lib.rs

@@ -1,4 +1,4 @@
-#![deny(unused_crate_dependencies, warnings)]
+#![deny(unused_crate_dependencies)]
 
 mod duckdb;
 mod error;
@@ -14,17 +14,32 @@ use pyo3::prelude::*;
 
 type Result<T> = std::result::Result<T, Error>;
 
-/// A collection of functions for working with STAC, using Rust under the hood.
 #[pymodule]
-fn stacrs(m: &Bound<'_, PyModule>) -> PyResult<()> {
+fn stacrs(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
     pyo3_log::init();
+
+    m.add("StacrsError", py.get_type::<error::StacrsError>())?;
+
     m.add_class::<duckdb::DuckdbClient>()?;
+
     m.add_function(wrap_pyfunction!(migrate::migrate, m)?)?;
     m.add_function(wrap_pyfunction!(migrate::migrate_href, m)?)?;
     m.add_function(wrap_pyfunction!(read::read, m)?)?;
     m.add_function(wrap_pyfunction!(search::search, m)?)?;
     m.add_function(wrap_pyfunction!(search::search_to, m)?)?;
     m.add_function(wrap_pyfunction!(version::version, m)?)?;
     m.add_function(wrap_pyfunction!(write::write, m)?)?;
+
     Ok(())
 }
+
+struct Json<T: serde::Serialize>(T);
+
+impl<'py, T: serde::Serialize> IntoPyObject<'py> for Json<T> {
+    type Error = pythonize::PythonizeError;
+    type Output = Bound<'py, PyAny>;
+    type Target = PyAny;
+    fn into_pyobject(self, py: Python<'py>) -> std::result::Result<Self::Output, Self::Error> {
+        pythonize::pythonize(py, &self.0)
+    }
+}

src/migrate.rs

@@ -1,4 +1,4 @@
-use crate::Error;
+use crate::{Error, Result};
 use pyo3::{prelude::*, types::PyDict};
 use stac::{Migrate, Value};
 
@@ -7,15 +7,16 @@ use stac::{Migrate, Value};
 pub fn migrate<'py>(
     value: &Bound<'py, PyDict>,
     version: Option<&str>,
-) -> PyResult<Bound<'py, PyDict>> {
+) -> Result<Bound<'py, PyDict>> {
     let py = value.py();
     let value: Value = pythonize::depythonize(value)?;
     let version = version
         .map(|version| version.parse().unwrap())
         .unwrap_or_default();
     let value = value.migrate(&version).map_err(Error::from)?;
     let value = pythonize::pythonize(py, &value)?;
-    value.downcast_into().map_err(PyErr::from)
+    let value = value.extract()?;
+    Ok(value)
 }
 
 #[pyfunction]
@@ -24,12 +25,13 @@ pub fn migrate_href<'py>(
     py: Python<'py>,
     href: &str,
     version: Option<&str>,
-) -> PyResult<Bound<'py, PyDict>> {
+) -> Result<Bound<'py, PyDict>> {
     let value: Value = stac::read(href).map_err(Error::from)?;
     let version = version
         .map(|version| version.parse().unwrap())
         .unwrap_or_default();
-    let value = value.migrate(&version).map_err(Error::from)?;
+    let value = value.migrate(&version)?;
     let value = pythonize::pythonize(py, &value)?;
-    value.downcast_into().map_err(PyErr::from)
+    let value = value.extract()?;
+    Ok(value)
 }