diff --git a/CHANGELOG.md b/CHANGELOG.md index 21ce4d7..9ad2c6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - `type` field to geoparquet writes ([#136](https://github.com/stac-utils/rustac-py/pull/136), ) +### Fixed + +- Error instead of panic for cql ([#138](https://github.com/stac-utils/rustac-py/pull/138), ) + ## [0.8.0] - 2025-05-13 ### Added diff --git a/Cargo.lock b/Cargo.lock index 05d48a4..a274076 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -674,9 +674,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.22" +version = "1.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32db95edf998450acc7881c932f94cd9b05c87b4b2599e8bab064753da4acfd1" +checksum = "5f4ac86a9e5bc1e2b3449ab9d7d3a6a405e3d1bb28d7b9be8614f55846ae3766" dependencies = [ "jobserver", "libc", @@ -840,9 +840,9 @@ dependencies = [ [[package]] name = "cql2" -version = "0.3.7-beta.0" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d741b16ae47ecd1d7fccf6f34e24c7a23f957d100e21e7f17ba8b17bbd4bb68a" +checksum = "c9120f33809d40dd1202bd3fb9832321bebc05d048d2c537a4076ed0d2ac5241" dependencies = [ "geo", "geo-types", @@ -1056,9 +1056,9 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" +checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" dependencies = [ "libc", "windows-sys 0.59.0", @@ -1785,9 +1785,9 @@ checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] name = "icu_properties" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2549ca8c7241c82f59c80ba2a6f415d931c5b58d24fb8412caa1a1f02c49139a" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" dependencies = [ "displaydoc", "icu_collections", @@ -1801,9 +1801,9 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8197e866e47b68f8f7d95249e172903bec06004b18b2937f1095d40a0c57de04" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" [[package]] name = "icu_provider" @@ -2037,7 +2037,6 @@ dependencies = [ "referencing 0.30.0", "regex", "regex-syntax 0.8.5", - "reqwest", "serde", "serde_json", "uuid-simd", @@ -2647,7 +2646,7 @@ dependencies = [ [[package]] name = "pgstac" version = "0.3.0" -source = "git+https://github.com/stac-utils/rustac?branch=main#46d6642a25f5f8ad31836d664b9fd50bf214ee84" +source = "git+https://github.com/stac-utils/rustac?branch=main#8a33be4288cad2979b2c261082bdfef494d3e732" dependencies = [ "serde", "serde_json", @@ -2910,9 +2909,9 @@ dependencies = [ [[package]] name = "pyo3-log" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079e412e909af5d6be7c04a7f29f6a2837a080410e1c529c9dee2c367383db4" +checksum = "45192e5e4a4d2505587e27806c7b710c231c40c56f3bfc19535d0bb25df52264" dependencies = [ "arc-swap", "log", @@ -3384,7 +3383,7 @@ dependencies = [ [[package]] name = "rustac" version = "0.5.3" -source = "git+https://github.com/stac-utils/rustac?branch=main#46d6642a25f5f8ad31836d664b9fd50bf214ee84" +source = "git+https://github.com/stac-utils/rustac?branch=main#8a33be4288cad2979b2c261082bdfef494d3e732" dependencies = [ "anyhow", "axum", @@ -3393,6 +3392,7 @@ dependencies = [ "stac", "stac-api", "stac-duckdb", + "stac-io", "stac-server", "tokio", "tracing", @@ -3408,6 +3408,7 @@ dependencies = [ "duckdb", "geoarrow-array", "geojson", + "object_store", "pyo3", "pyo3-arrow", "pyo3-async-runtimes", @@ -3420,6 +3421,7 @@ dependencies = [ "stac", "stac-api", "stac-duckdb", + "stac-io", "thiserror 2.0.12", "tokio", "tracing", @@ -3789,7 +3791,7 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stac" version = "0.12.0" -source = "git+https://github.com/stac-utils/rustac?branch=main#46d6642a25f5f8ad31836d664b9fd50bf214ee84" +source = "git+https://github.com/stac-utils/rustac?branch=main#8a33be4288cad2979b2c261082bdfef494d3e732" dependencies = [ "arrow-array", "arrow-cast", @@ -3797,7 +3799,6 @@ dependencies = [ "arrow-schema", "bytes", "chrono", - "fluent-uri", "geo", "geo-traits", "geo-types", @@ -3806,17 +3807,13 @@ dependencies = [ "geoarrow-schema", "geojson", "indexmap 2.9.0", - "jsonschema 0.30.0", "log", "mime", - "object_store", "parquet", - "reqwest", "serde", "serde_json", "stac-derive", "thiserror 2.0.12", - "tokio", "tracing", "url", ] @@ -3824,7 +3821,7 @@ dependencies = [ [[package]] name = "stac-api" version = "0.7.1" -source = "git+https://github.com/stac-utils/rustac?branch=main#46d6642a25f5f8ad31836d664b9fd50bf214ee84" +source = "git+https://github.com/stac-utils/rustac?branch=main#8a33be4288cad2979b2c261082bdfef494d3e732" dependencies = [ "async-stream", "chrono", @@ -3849,7 +3846,7 @@ dependencies = [ [[package]] name = "stac-derive" version = "0.2.0" -source = "git+https://github.com/stac-utils/rustac?branch=main#46d6642a25f5f8ad31836d664b9fd50bf214ee84" +source = "git+https://github.com/stac-utils/rustac?branch=main#8a33be4288cad2979b2c261082bdfef494d3e732" dependencies = [ "quote", "syn 2.0.101", @@ -3858,7 +3855,7 @@ dependencies = [ [[package]] name = "stac-duckdb" version = "0.1.1" -source = "git+https://github.com/stac-utils/rustac?branch=main#46d6642a25f5f8ad31836d664b9fd50bf214ee84" +source = "git+https://github.com/stac-utils/rustac?branch=main#8a33be4288cad2979b2c261082bdfef494d3e732" dependencies = [ "arrow-array", "chrono", @@ -3867,6 +3864,7 @@ dependencies = [ "geo", "geoarrow-array", "geojson", + "getrandom 0.3.3", "log", "serde_json", "stac", @@ -3874,10 +3872,29 @@ dependencies = [ "thiserror 2.0.12", ] +[[package]] +name = "stac-io" +version = "0.1.0" +source = "git+https://github.com/stac-utils/rustac?branch=main#8a33be4288cad2979b2c261082bdfef494d3e732" +dependencies = [ + "bytes", + "fluent-uri", + "jsonschema 0.30.0", + "object_store", + "parquet", + "reqwest", + "serde", + "serde_json", + "stac", + "thiserror 2.0.12", + "tracing", + "url", +] + [[package]] name = "stac-server" version = "0.3.4" -source = "git+https://github.com/stac-utils/rustac?branch=main#46d6642a25f5f8ad31836d664b9fd50bf214ee84" +source = "git+https://github.com/stac-utils/rustac?branch=main#8a33be4288cad2979b2c261082bdfef494d3e732" dependencies = [ "axum", "bb8", @@ -4768,15 +4785,15 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.61.0" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ "windows-implement", "windows-interface", "windows-link", "windows-result", - "windows-strings 0.4.0", + "windows-strings 0.4.2", ] [[package]] @@ -4820,9 +4837,9 @@ dependencies = [ [[package]] name = "windows-result" -version = "0.3.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ "windows-link", ] @@ -4838,9 +4855,9 @@ dependencies = [ [[package]] name = "windows-strings" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" dependencies = [ "windows-link", ] diff --git a/Cargo.toml b/Cargo.toml index c14d318..6c8fdcb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,14 +27,15 @@ serde = "1.0.217" serde_json = { version = "1.0.138", features = ["preserve_order"] } stac = { features = [ "geoparquet-compression", - "object-store-all", +], git = "https://github.com/stac-utils/rustac", branch = "main" } +stac-io = { features = [ + "store-all", ], git = "https://github.com/stac-utils/rustac", branch = "main" } stac-api = { features = [ "client", ], git = "https://github.com/stac-utils/rustac", branch = "main" } rustac = { git = "https://github.com/stac-utils/rustac", features = [ "pgstac", - "duckdb", ], branch = "main" } stac-duckdb = { git = "https://github.com/stac-utils/rustac", branch = "main" } thiserror = "2.0.12" @@ -42,6 +43,7 @@ tokio = { version = "1.44.0", features = ["rt-multi-thread"] } pyo3-log = "0.12.1" tracing = "0.1.41" pyo3-object_store = "0.2.0" +object_store = "0.12.1" [build-dependencies] cargo-lock = "10" diff --git a/src/error.rs b/src/error.rs index 37789fc..aa44e2e 100644 --- a/src/error.rs +++ b/src/error.rs @@ -32,11 +32,17 @@ pub enum Error { #[error(transparent)] Stac(#[from] stac::Error), + #[error(transparent)] + StacIo(#[from] stac_io::Error), + #[error(transparent)] StacApi(#[from] stac_api::Error), #[error(transparent)] StacDuckdb(#[from] stac_duckdb::Error), + + #[error(transparent)] + TokioTaskJon(#[from] tokio::task::JoinError), } impl From for PyErr { diff --git a/src/read.rs b/src/read.rs index 48b127d..5cc1262 100644 --- a/src/read.rs +++ b/src/read.rs @@ -1,7 +1,8 @@ use crate::{Error, Json}; use pyo3::{Bound, PyResult, Python, pyfunction, types::PyAny}; use pyo3_object_store::AnyObjectStore; -use stac::{Format, Link, Links, SelfHref, Value}; +use stac::{Link, Links, SelfHref, Value}; +use stac_io::Format; #[pyfunction] #[pyo3(signature = (href, *, format=None, store=None, set_self_link=true))] diff --git a/src/search.rs b/src/search.rs index 239135c..bbf6a7d 100644 --- a/src/search.rs +++ b/src/search.rs @@ -4,8 +4,8 @@ use pyo3::prelude::*; use pyo3::{Bound, FromPyObject, PyErr, PyResult, exceptions::PyValueError, types::PyDict}; use pyo3_object_store::AnyObjectStore; use stac::Bbox; -use stac::Format; use stac_api::{Fields, Filter, Items, Search, Sortby}; +use stac_io::Format; #[pyfunction] #[pyo3(signature = (href, *, intersects=None, ids=None, collections=None, max_items=None, limit=None, bbox=None, datetime=None, include=None, exclude=None, sortby=None, filter=None, query=None, use_duckdb=None, **kwargs))] diff --git a/src/walk.rs b/src/walk.rs index 5e0d1ff..13feb82 100644 --- a/src/walk.rs +++ b/src/walk.rs @@ -1,28 +1,43 @@ use crate::{Error, Json, Result}; +use object_store::ObjectStore; use pyo3::{ Bound, Py, PyAny, PyResult, Python, exceptions::PyStopAsyncIteration, pyclass, pyfunction, pymethods, types::PyDict, }; -use stac::{Container, Item, Links, Node, SelfHref, Value}; +use pyo3_object_store::AnyObjectStore; +use stac::{Item, Links, SelfHref, Value}; +use stac_io::Format; use std::collections::VecDeque; use std::sync::Arc; use tokio::sync::Mutex; +use tokio::task::JoinSet; #[pyfunction] -pub fn walk(container: Bound<'_, PyDict>) -> Result { +#[pyo3(signature = (container, store=None))] +pub fn walk(container: Bound<'_, PyDict>, store: Option) -> Result { let mut value: Value = pythonize::depythonize(&container)?; if let Some(link) = value.link("self").cloned() { *value.self_href_mut() = Some(link.href); } - let container: Container = value.try_into()?; - let node = Node::from(container); let mut walks = VecDeque::new(); - walks.push_back(node); - Ok(Walk(Arc::new(Mutex::new(walks)))) + walks.push_back(value); + let store = if let Some(store) = store { + Some(store.into_dyn()) + } else { + None + }; + Ok(Walk { + values: Arc::new(Mutex::new(walks)), + store, + }) } #[pyclass] -pub struct Walk(Arc>>); +#[derive(Clone)] +pub struct Walk { + values: Arc>>, + store: Option>, +} #[pymethods] impl Walk { @@ -31,26 +46,58 @@ impl Walk { } fn __anext__<'py>(&self, py: Python<'py>) -> PyResult> { - let nodes = self.0.clone(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { next_walk(nodes).await }) + let walk = self.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { walk.next().await }) } } -type WalkStep = (Value, Vec, VecDeque); - -async fn next_walk(nodes: Arc>>) -> PyResult> { - let mut nodes = nodes.lock().await; - match nodes.pop_front() { - Some(node) => { - let mut node = node.resolve().await.map_err(Error::from)?; - let items = std::mem::take(&mut node.items); - let mut children = Vec::with_capacity(node.children.len()); - for child in node.children { - children.push(child.value.clone()); - nodes.push_back(child); +impl Walk { + async fn next(self) -> PyResult> { + let value = { + let mut values = self.values.lock().await; + values.pop_front() + }; + match value { + Some(value) => { + let mut items = Vec::new(); + let mut children = Vec::new(); + let mut join_set = JoinSet::new(); + for mut link in value.links().iter().cloned() { + if link.is_child() || link.is_item() { + let store = self.store.clone(); + if let Some(self_href) = value.self_href() { + link.make_absolute(self_href).map_err(Error::from)?; + } + join_set.spawn(async move { + if let Some(store) = store { + Format::json() + .get_store::(store, link.href.as_str()) + .await + } else { + Format::json() + .get_opts(link.href.as_str(), [] as [(&str, &str); 0]) + .await + } + }); + } + } + { + let mut values = self.values.lock().await; + while let Some(result) = join_set.join_next().await { + let value = result.map_err(Error::from)?.map_err(Error::from)?; + if let Value::Item(item) = value { + items.push(item); + } else if value.is_catalog() || value.is_collection() { + children.push(value.clone()); + values.push_back(value.clone()); + } + } + } + Ok(Json((value, children, items))) } - Ok(Json((node.value.into(), children, items))) + _ => Err(PyStopAsyncIteration::new_err("done walking")), } - _ => Err(PyStopAsyncIteration::new_err("done walking")), } } + +type WalkStep = (Value, Vec, Vec); diff --git a/src/write.rs b/src/write.rs index dc63f30..997d53d 100644 --- a/src/write.rs +++ b/src/write.rs @@ -2,7 +2,8 @@ use crate::{Error, Json, Result}; use pyo3::{Bound, PyAny, PyResult, Python, pyfunction}; use pyo3_object_store::AnyObjectStore; use serde_json::Value; -use stac::{Format, Item, ItemCollection}; +use stac::{Item, ItemCollection}; +use stac_io::Format; #[pyfunction] #[pyo3(signature = (href, value, *, format=None, store=None))] diff --git a/tests/test_search.py b/tests/test_search.py index 3a7a103..665129c 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -3,8 +3,10 @@ from typing import Any import pyarrow.parquet +import pytest import rustac import stac_geoparquet.arrow +from rustac import RustacError from rustac.store import MemoryStore @@ -92,3 +94,22 @@ async def test_list_sortby(data: Path) -> None: assert first["id"] <= second["id"] else: assert first["properties"]["datetime"] >= second["properties"]["datetime"] + + +async def test_cql(data: Path) -> None: + # https://github.com/stac-utils/rustac-py/issues/135 + with pytest.raises(RustacError, match="eq is not a valid operator"): + await rustac.search( + str(data / "100-sentinel-2-items.parquet"), + filter={ + "op": "and", + "args": [ + # eq is cql, not cql2 + { + "op": "eq", + "args": [{"property": "platform"}, "made-up-platform"], + }, + ], + }, + max_items=1, + )