From f63d8bd6935fc2fb458846b000ba7bd16a66a64f Mon Sep 17 00:00:00 2001 From: Pete Gadomski Date: Mon, 19 May 2025 12:40:03 -0600 Subject: [PATCH 1/3] --wip-- [skip ci] --- Cargo.lock | 161 ++++++++++++++++++++++++------------------- Cargo.toml | 6 +- src/error.rs | 6 ++ src/read.rs | 3 +- src/search.rs | 2 +- src/walk.rs | 93 ++++++++++++++++++------- src/write.rs | 3 +- tests/test_search.py | 14 ++++ 8 files changed, 189 insertions(+), 99 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6530a4c..a274076 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -167,9 +167,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3095aaf545942ff5abd46654534f15b03a90fba78299d661e045e5d587222f0d" +checksum = "b1bb018b6960c87fd9d025009820406f74e83281185a8bdcb44880d2aa5c9a87" dependencies = [ "arrow-arith", "arrow-array", @@ -185,9 +185,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00752064ff47cee746e816ddb8450520c3a52cbad1e256f6fa861a35f86c45e7" +checksum = "44de76b51473aa888ecd6ad93ceb262fb8d40d1f1154a4df2f069b3590aa7575" dependencies = [ "arrow-array", "arrow-buffer", @@ -199,9 +199,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cebfe926794fbc1f49ddd0cdaf898956ca9f6e79541efce62dabccfd81380472" +checksum = "29ed77e22744475a9a53d00026cf8e166fe73cf42d89c4c4ae63607ee1cfcc3f" dependencies = [ "ahash 0.8.12", "arrow-buffer", @@ -216,9 +216,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0303c7ec4cf1a2c60310fc4d6bbc3350cd051a17bf9e9c0a8e47b4db79277824" +checksum = "b0391c96eb58bf7389171d1e103112d3fc3e5625ca6b372d606f2688f1ea4cce" dependencies = [ "bytes", "half", @@ -227,9 +227,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335f769c5a218ea823d3760a743feba1ef7857cba114c01399a891c2fff34285" +checksum = "f39e1d774ece9292697fcbe06b5584401b26bd34be1bec25c33edae65c2420ff" dependencies = [ "arrow-array", "arrow-buffer", @@ -248,9 +248,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8affacf3351a24039ea24adab06f316ded523b6f8c3dbe28fbac5f18743451b" +checksum = "cf75ac27a08c7f48b88e5c923f267e980f27070147ab74615ad85b5c5f90473d" dependencies = [ "arrow-buffer", "arrow-schema", @@ -260,9 +260,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69880a9e6934d9cba2b8630dd08a3463a91db8693b16b499d54026b6137af284" +checksum = "a222f0d93772bd058d1268f4c28ea421a603d66f7979479048c429292fac7b2e" dependencies = [ "arrow-array", "arrow-buffer", @@ -273,9 +273,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8dafd17a05449e31e0114d740530e0ada7379d7cb9c338fd65b09a8130960b0" +checksum = "9085342bbca0f75e8cb70513c0807cc7351f1fbf5cb98192a67d5e3044acb033" dependencies = [ "arrow-array", "arrow-buffer", @@ -295,9 +295,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "895644523af4e17502d42c3cb6b27cb820f0cb77954c22d75c23a85247c849e1" +checksum = "ab2f1065a5cad7b9efa9e22ce5747ce826aa3855766755d4904535123ef431e7" dependencies = [ "arrow-array", "arrow-buffer", @@ -308,9 +308,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9be8a2a4e5e7d9c822b2b8095ecd77010576d824f654d347817640acfc97d229" +checksum = "3703a0e3e92d23c3f756df73d2dc9476873f873a76ae63ef9d3de17fda83b2d8" dependencies = [ "arrow-array", "arrow-buffer", @@ -321,18 +321,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7450c76ab7c5a6805be3440dc2e2096010da58f7cab301fdc996a4ee3ee74e49" +checksum = "73a47aa0c771b5381de2b7f16998d351a6f4eb839f1e13d48353e17e873d969b" dependencies = [ "bitflags", ] [[package]] name = "arrow-select" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa5f5a93c75f46ef48e4001535e7b6c922eeb0aa20b73cf58d09e13d057490d8" +checksum = "24b7b85575702b23b85272b01bc1c25a01c9b9852305e5d0078c79ba25d995d4" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -344,9 +344,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7005d858d84b56428ba2a98a107fe88c0132c61793cf6b8232a1f9bfc0452b" +checksum = "9260fddf1cdf2799ace2b4c2fc0356a9789fa7551e0953e35435536fecefebbd" dependencies = [ "arrow-array", "arrow-buffer", @@ -533,9 +533,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" [[package]] name = "bitvec" @@ -589,9 +589,9 @@ dependencies = [ [[package]] name = "brotli" -version = "7.0.0" +version = "8.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" +checksum = "9991eea70ea4f293524138648e41ee89b0b2b12ddef3b255effa43c8056e0e0d" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -600,9 +600,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "4.0.3" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a334ef7c9e23abf0ce748e8cd309037da93e606ad52eb372e4ce327a0dcfbdfd" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -674,9 +674,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.22" +version = "1.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32db95edf998450acc7881c932f94cd9b05c87b4b2599e8bab064753da4acfd1" +checksum = "5f4ac86a9e5bc1e2b3449ab9d7d3a6a405e3d1bb28d7b9be8614f55846ae3766" dependencies = [ "jobserver", "libc", @@ -840,9 +840,9 @@ dependencies = [ [[package]] name = "cql2" -version = "0.3.7-beta.0" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d741b16ae47ecd1d7fccf6f34e24c7a23f957d100e21e7f17ba8b17bbd4bb68a" +checksum = "c9120f33809d40dd1202bd3fb9832321bebc05d048d2c537a4076ed0d2ac5241" dependencies = [ "geo", "geo-types", @@ -1056,9 +1056,9 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" +checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" dependencies = [ "libc", "windows-sys 0.59.0", @@ -1785,9 +1785,9 @@ checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] name = "icu_properties" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2549ca8c7241c82f59c80ba2a6f415d931c5b58d24fb8412caa1a1f02c49139a" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" dependencies = [ "displaydoc", "icu_collections", @@ -1801,9 +1801,9 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8197e866e47b68f8f7d95249e172903bec06004b18b2937f1095d40a0c57de04" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" [[package]] name = "icu_provider" @@ -2037,7 +2037,6 @@ dependencies = [ "referencing 0.30.0", "regex", "regex-syntax 0.8.5", - "reqwest", "serde", "serde_json", "uuid-simd", @@ -2226,9 +2225,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "matrixmultiply" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" dependencies = [ "autocfg", "rawpointer", @@ -2445,9 +2444,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9ce831b09395f933addbc56d894d889e4b226eba304d4e7adbab591e26daf1e" +checksum = "d94ac16b433c0ccf75326388c893d2835ab7457ea35ab8ba5d745c053ef5fa16" dependencies = [ "async-trait", "base64", @@ -2465,7 +2464,7 @@ dependencies = [ "parking_lot", "percent-encoding", "quick-xml", - "rand 0.8.5", + "rand 0.9.1", "reqwest", "ring", "rustls-pemfile", @@ -2477,6 +2476,8 @@ dependencies = [ "tracing", "url", "walkdir", + "wasm-bindgen-futures", + "web-time", ] [[package]] @@ -2537,9 +2538,9 @@ dependencies = [ [[package]] name = "parquet" -version = "55.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd31a8290ac5b19f09ad77ee7a1e6a541f1be7674ad410547d5f1eef6eef4a9c" +checksum = "be7b2d778f6b841d37083ebdf32e33a524acde1266b5884a8ca29bf00dfa1231" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -2645,7 +2646,7 @@ dependencies = [ [[package]] name = "pgstac" version = "0.3.0" -source = "git+https://github.com/stac-utils/rustac?branch=main#2e988e14947ebc90ca7043a9faa0066abe9da78b" +source = "git+https://github.com/stac-utils/rustac?branch=main#8a33be4288cad2979b2c261082bdfef494d3e732" dependencies = [ "serde", "serde_json", @@ -2908,9 +2909,9 @@ dependencies = [ [[package]] name = "pyo3-log" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079e412e909af5d6be7c04a7f29f6a2837a080410e1c529c9dee2c367383db4" +checksum = "45192e5e4a4d2505587e27806c7b710c231c40c56f3bfc19535d0bb25df52264" dependencies = [ "arc-swap", "log", @@ -3382,7 +3383,7 @@ dependencies = [ [[package]] name = "rustac" version = "0.5.3" -source = "git+https://github.com/stac-utils/rustac?branch=main#2e988e14947ebc90ca7043a9faa0066abe9da78b" +source = "git+https://github.com/stac-utils/rustac?branch=main#8a33be4288cad2979b2c261082bdfef494d3e732" dependencies = [ "anyhow", "axum", @@ -3391,6 +3392,7 @@ dependencies = [ "stac", "stac-api", "stac-duckdb", + "stac-io", "stac-server", "tokio", "tracing", @@ -3406,6 +3408,7 @@ dependencies = [ "duckdb", "geoarrow-array", "geojson", + "object_store", "pyo3", "pyo3-arrow", "pyo3-async-runtimes", @@ -3418,6 +3421,7 @@ dependencies = [ "stac", "stac-api", "stac-duckdb", + "stac-io", "thiserror 2.0.12", "tokio", "tracing", @@ -3787,7 +3791,7 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stac" version = "0.12.0" -source = "git+https://github.com/stac-utils/rustac?branch=main#2e988e14947ebc90ca7043a9faa0066abe9da78b" +source = "git+https://github.com/stac-utils/rustac?branch=main#8a33be4288cad2979b2c261082bdfef494d3e732" dependencies = [ "arrow-array", "arrow-cast", @@ -3795,7 +3799,6 @@ dependencies = [ "arrow-schema", "bytes", "chrono", - "fluent-uri", "geo", "geo-traits", "geo-types", @@ -3804,17 +3807,13 @@ dependencies = [ "geoarrow-schema", "geojson", "indexmap 2.9.0", - "jsonschema 0.30.0", "log", "mime", - "object_store", "parquet", - "reqwest", "serde", "serde_json", "stac-derive", "thiserror 2.0.12", - "tokio", "tracing", "url", ] @@ -3822,7 +3821,7 @@ dependencies = [ [[package]] name = "stac-api" version = "0.7.1" -source = "git+https://github.com/stac-utils/rustac?branch=main#2e988e14947ebc90ca7043a9faa0066abe9da78b" +source = "git+https://github.com/stac-utils/rustac?branch=main#8a33be4288cad2979b2c261082bdfef494d3e732" dependencies = [ "async-stream", "chrono", @@ -3847,7 +3846,7 @@ dependencies = [ [[package]] name = "stac-derive" version = "0.2.0" -source = "git+https://github.com/stac-utils/rustac?branch=main#2e988e14947ebc90ca7043a9faa0066abe9da78b" +source = "git+https://github.com/stac-utils/rustac?branch=main#8a33be4288cad2979b2c261082bdfef494d3e732" dependencies = [ "quote", "syn 2.0.101", @@ -3856,7 +3855,7 @@ dependencies = [ [[package]] name = "stac-duckdb" version = "0.1.1" -source = "git+https://github.com/stac-utils/rustac?branch=main#2e988e14947ebc90ca7043a9faa0066abe9da78b" +source = "git+https://github.com/stac-utils/rustac?branch=main#8a33be4288cad2979b2c261082bdfef494d3e732" dependencies = [ "arrow-array", "chrono", @@ -3865,6 +3864,7 @@ dependencies = [ "geo", "geoarrow-array", "geojson", + "getrandom 0.3.3", "log", "serde_json", "stac", @@ -3872,10 +3872,29 @@ dependencies = [ "thiserror 2.0.12", ] +[[package]] +name = "stac-io" +version = "0.1.0" +source = "git+https://github.com/stac-utils/rustac?branch=main#8a33be4288cad2979b2c261082bdfef494d3e732" +dependencies = [ + "bytes", + "fluent-uri", + "jsonschema 0.30.0", + "object_store", + "parquet", + "reqwest", + "serde", + "serde_json", + "stac", + "thiserror 2.0.12", + "tracing", + "url", +] + [[package]] name = "stac-server" version = "0.3.4" -source = "git+https://github.com/stac-utils/rustac?branch=main#2e988e14947ebc90ca7043a9faa0066abe9da78b" +source = "git+https://github.com/stac-utils/rustac?branch=main#8a33be4288cad2979b2c261082bdfef494d3e732" dependencies = [ "axum", "bb8", @@ -4766,15 +4785,15 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.61.0" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ "windows-implement", "windows-interface", "windows-link", "windows-result", - "windows-strings 0.4.0", + "windows-strings 0.4.2", ] [[package]] @@ -4818,9 +4837,9 @@ dependencies = [ [[package]] name = "windows-result" -version = "0.3.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ "windows-link", ] @@ -4836,9 +4855,9 @@ dependencies = [ [[package]] name = "windows-strings" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" dependencies = [ "windows-link", ] diff --git a/Cargo.toml b/Cargo.toml index c14d318..6c8fdcb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,14 +27,15 @@ serde = "1.0.217" serde_json = { version = "1.0.138", features = ["preserve_order"] } stac = { features = [ "geoparquet-compression", - "object-store-all", +], git = "https://github.com/stac-utils/rustac", branch = "main" } +stac-io = { features = [ + "store-all", ], git = "https://github.com/stac-utils/rustac", branch = "main" } stac-api = { features = [ "client", ], git = "https://github.com/stac-utils/rustac", branch = "main" } rustac = { git = "https://github.com/stac-utils/rustac", features = [ "pgstac", - "duckdb", ], branch = "main" } stac-duckdb = { git = "https://github.com/stac-utils/rustac", branch = "main" } thiserror = "2.0.12" @@ -42,6 +43,7 @@ tokio = { version = "1.44.0", features = ["rt-multi-thread"] } pyo3-log = "0.12.1" tracing = "0.1.41" pyo3-object_store = "0.2.0" +object_store = "0.12.1" [build-dependencies] cargo-lock = "10" diff --git a/src/error.rs b/src/error.rs index 37789fc..aa44e2e 100644 --- a/src/error.rs +++ b/src/error.rs @@ -32,11 +32,17 @@ pub enum Error { #[error(transparent)] Stac(#[from] stac::Error), + #[error(transparent)] + StacIo(#[from] stac_io::Error), + #[error(transparent)] StacApi(#[from] stac_api::Error), #[error(transparent)] StacDuckdb(#[from] stac_duckdb::Error), + + #[error(transparent)] + TokioTaskJon(#[from] tokio::task::JoinError), } impl From for PyErr { diff --git a/src/read.rs b/src/read.rs index 48b127d..5cc1262 100644 --- a/src/read.rs +++ b/src/read.rs @@ -1,7 +1,8 @@ use crate::{Error, Json}; use pyo3::{Bound, PyResult, Python, pyfunction, types::PyAny}; use pyo3_object_store::AnyObjectStore; -use stac::{Format, Link, Links, SelfHref, Value}; +use stac::{Link, Links, SelfHref, Value}; +use stac_io::Format; #[pyfunction] #[pyo3(signature = (href, *, format=None, store=None, set_self_link=true))] diff --git a/src/search.rs b/src/search.rs index 239135c..bbf6a7d 100644 --- a/src/search.rs +++ b/src/search.rs @@ -4,8 +4,8 @@ use pyo3::prelude::*; use pyo3::{Bound, FromPyObject, PyErr, PyResult, exceptions::PyValueError, types::PyDict}; use pyo3_object_store::AnyObjectStore; use stac::Bbox; -use stac::Format; use stac_api::{Fields, Filter, Items, Search, Sortby}; +use stac_io::Format; #[pyfunction] #[pyo3(signature = (href, *, intersects=None, ids=None, collections=None, max_items=None, limit=None, bbox=None, datetime=None, include=None, exclude=None, sortby=None, filter=None, query=None, use_duckdb=None, **kwargs))] diff --git a/src/walk.rs b/src/walk.rs index 5e0d1ff..13feb82 100644 --- a/src/walk.rs +++ b/src/walk.rs @@ -1,28 +1,43 @@ use crate::{Error, Json, Result}; +use object_store::ObjectStore; use pyo3::{ Bound, Py, PyAny, PyResult, Python, exceptions::PyStopAsyncIteration, pyclass, pyfunction, pymethods, types::PyDict, }; -use stac::{Container, Item, Links, Node, SelfHref, Value}; +use pyo3_object_store::AnyObjectStore; +use stac::{Item, Links, SelfHref, Value}; +use stac_io::Format; use std::collections::VecDeque; use std::sync::Arc; use tokio::sync::Mutex; +use tokio::task::JoinSet; #[pyfunction] -pub fn walk(container: Bound<'_, PyDict>) -> Result { +#[pyo3(signature = (container, store=None))] +pub fn walk(container: Bound<'_, PyDict>, store: Option) -> Result { let mut value: Value = pythonize::depythonize(&container)?; if let Some(link) = value.link("self").cloned() { *value.self_href_mut() = Some(link.href); } - let container: Container = value.try_into()?; - let node = Node::from(container); let mut walks = VecDeque::new(); - walks.push_back(node); - Ok(Walk(Arc::new(Mutex::new(walks)))) + walks.push_back(value); + let store = if let Some(store) = store { + Some(store.into_dyn()) + } else { + None + }; + Ok(Walk { + values: Arc::new(Mutex::new(walks)), + store, + }) } #[pyclass] -pub struct Walk(Arc>>); +#[derive(Clone)] +pub struct Walk { + values: Arc>>, + store: Option>, +} #[pymethods] impl Walk { @@ -31,26 +46,58 @@ impl Walk { } fn __anext__<'py>(&self, py: Python<'py>) -> PyResult> { - let nodes = self.0.clone(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { next_walk(nodes).await }) + let walk = self.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { walk.next().await }) } } -type WalkStep = (Value, Vec, VecDeque); - -async fn next_walk(nodes: Arc>>) -> PyResult> { - let mut nodes = nodes.lock().await; - match nodes.pop_front() { - Some(node) => { - let mut node = node.resolve().await.map_err(Error::from)?; - let items = std::mem::take(&mut node.items); - let mut children = Vec::with_capacity(node.children.len()); - for child in node.children { - children.push(child.value.clone()); - nodes.push_back(child); +impl Walk { + async fn next(self) -> PyResult> { + let value = { + let mut values = self.values.lock().await; + values.pop_front() + }; + match value { + Some(value) => { + let mut items = Vec::new(); + let mut children = Vec::new(); + let mut join_set = JoinSet::new(); + for mut link in value.links().iter().cloned() { + if link.is_child() || link.is_item() { + let store = self.store.clone(); + if let Some(self_href) = value.self_href() { + link.make_absolute(self_href).map_err(Error::from)?; + } + join_set.spawn(async move { + if let Some(store) = store { + Format::json() + .get_store::(store, link.href.as_str()) + .await + } else { + Format::json() + .get_opts(link.href.as_str(), [] as [(&str, &str); 0]) + .await + } + }); + } + } + { + let mut values = self.values.lock().await; + while let Some(result) = join_set.join_next().await { + let value = result.map_err(Error::from)?.map_err(Error::from)?; + if let Value::Item(item) = value { + items.push(item); + } else if value.is_catalog() || value.is_collection() { + children.push(value.clone()); + values.push_back(value.clone()); + } + } + } + Ok(Json((value, children, items))) } - Ok(Json((node.value.into(), children, items))) + _ => Err(PyStopAsyncIteration::new_err("done walking")), } - _ => Err(PyStopAsyncIteration::new_err("done walking")), } } + +type WalkStep = (Value, Vec, Vec); diff --git a/src/write.rs b/src/write.rs index dc63f30..997d53d 100644 --- a/src/write.rs +++ b/src/write.rs @@ -2,7 +2,8 @@ use crate::{Error, Json, Result}; use pyo3::{Bound, PyAny, PyResult, Python, pyfunction}; use pyo3_object_store::AnyObjectStore; use serde_json::Value; -use stac::{Format, Item, ItemCollection}; +use stac::{Item, ItemCollection}; +use stac_io::Format; #[pyfunction] #[pyo3(signature = (href, value, *, format=None, store=None))] diff --git a/tests/test_search.py b/tests/test_search.py index 3a7a103..f5cb348 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -92,3 +92,17 @@ async def test_list_sortby(data: Path) -> None: assert first["id"] <= second["id"] else: assert first["properties"]["datetime"] >= second["properties"]["datetime"] + + +async def test_cql(data: Path) -> None: + await rustac.search( + str(data / "100-sentinel-2-items.parquet"), + filter={ + "op": "and", + "args": [ + # eq is cql, not cql2 + {"op": "eq", "args": [{"property": "forecast:horizon"}, "PT48H"]}, + ], + }, + max_items=1, + ) From db1b1a4c359443aeb066cddb85744d1b0f679f39 Mon Sep 17 00:00:00 2001 From: Pete Gadomski Date: Mon, 19 May 2025 12:43:03 -0600 Subject: [PATCH 2/3] fix: cql errors --- tests/test_search.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/tests/test_search.py b/tests/test_search.py index f5cb348..665129c 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -3,8 +3,10 @@ from typing import Any import pyarrow.parquet +import pytest import rustac import stac_geoparquet.arrow +from rustac import RustacError from rustac.store import MemoryStore @@ -95,14 +97,19 @@ async def test_list_sortby(data: Path) -> None: async def test_cql(data: Path) -> None: - await rustac.search( - str(data / "100-sentinel-2-items.parquet"), - filter={ - "op": "and", - "args": [ - # eq is cql, not cql2 - {"op": "eq", "args": [{"property": "forecast:horizon"}, "PT48H"]}, - ], - }, - max_items=1, - ) + # https://github.com/stac-utils/rustac-py/issues/135 + with pytest.raises(RustacError, match="eq is not a valid operator"): + await rustac.search( + str(data / "100-sentinel-2-items.parquet"), + filter={ + "op": "and", + "args": [ + # eq is cql, not cql2 + { + "op": "eq", + "args": [{"property": "platform"}, "made-up-platform"], + }, + ], + }, + max_items=1, + ) From fb83ffe97936838da642e6c71c9a9622d6ef2869 Mon Sep 17 00:00:00 2001 From: Pete Gadomski Date: Mon, 19 May 2025 12:45:25 -0600 Subject: [PATCH 3/3] chore: update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21ce4d7..9ad2c6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - `type` field to geoparquet writes ([#136](https://github.com/stac-utils/rustac-py/pull/136), ) +### Fixed + +- Error instead of panic for cql ([#138](https://github.com/stac-utils/rustac-py/pull/138), ) + ## [0.8.0] - 2025-05-13 ### Added