diff --git a/datatypes/src/primitives/query_rectangle.rs b/datatypes/src/primitives/query_rectangle.rs index 64a1a47a0..ac9257bdd 100644 --- a/datatypes/src/primitives/query_rectangle.rs +++ b/datatypes/src/primitives/query_rectangle.rs @@ -176,6 +176,14 @@ impl BandSelection { pub fn is_single(&self) -> bool { self.count() == 1 } + + pub fn contains(&self, band: u32) -> bool { + self.0.contains(&band) + } + + pub fn contains_all(&self, bands: &[u32]) -> bool { + bands.iter().all(|band| self.contains(*band)) + } } impl From for BandSelection { @@ -184,6 +192,12 @@ impl From for BandSelection { } } +impl AsRef<[u32]> for BandSelection { + fn as_ref(&self) -> &[u32] { + self.as_slice() + } +} + impl TryFrom> for BandSelection { type Error = crate::error::Error; @@ -200,11 +214,19 @@ impl TryFrom<[u32; N]> for BandSelection { } } +impl TryFrom<&[u32]> for BandSelection { + type Error = crate::error::Error; + + fn try_from(value: &[u32]) -> Result { + Self::new(value.to_vec()) + } +} + impl QueryAttributeSelection for BandSelection {} #[derive(Clone, Debug, PartialEq, Serialize)] pub struct BandSelectionIter { - band_selection: BandSelection, + pub band_selection: BandSelection, next_index: usize, } diff --git a/datatypes/src/primitives/time_gap_fill_iter.rs b/datatypes/src/primitives/time_gap_fill_iter.rs index 53b32f334..bf142e278 100644 --- a/datatypes/src/primitives/time_gap_fill_iter.rs +++ b/datatypes/src/primitives/time_gap_fill_iter.rs @@ -1364,7 +1364,7 @@ mod tests { } #[test] - fn time_irregular_all_caes() { + fn time_irregular_all_cases() { let intervals = vec![ TimeInterval::new_unchecked( TimeInstance::from_millis(15).unwrap(), @@ -1471,4 +1471,50 @@ mod tests { ] ); } + + #[test] + fn irregular_fill_min_max_time() { + let intervals = vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(15).unwrap(), + TimeInstance::from_millis(25).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(45).unwrap(), + TimeInstance::from_millis(55).unwrap(), + ), + ]; + + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_irregular_range_fill(TimeInterval::default()); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![ + TimeInterval::new_unchecked( + TimeInstance::MIN, + TimeInstance::from_millis(15).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(15).unwrap(), + TimeInstance::from_millis(25).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(25).unwrap(), + TimeInstance::from_millis(45).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(45).unwrap(), + TimeInstance::from_millis(55).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(55).unwrap(), + TimeInstance::MAX + ), + ] + ); + } } diff --git a/datatypes/src/raster/grid_spatial.rs b/datatypes/src/raster/grid_spatial.rs index 59e7b8b7c..f07239d36 100644 --- a/datatypes/src/raster/grid_spatial.rs +++ b/datatypes/src/raster/grid_spatial.rs @@ -284,7 +284,7 @@ impl GeoTransformAccess for SpatialGridDefinition { } } -#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)] pub struct TilingSpatialGridDefinition { // Don't make this public to avoid leaking inner element_grid_definition: SpatialGridDefinition, diff --git a/datatypes/src/raster/mod.rs b/datatypes/src/raster/mod.rs index 63b120ebd..297d82bb4 100755 --- a/datatypes/src/raster/mod.rs +++ b/datatypes/src/raster/mod.rs @@ -54,6 +54,7 @@ pub use raster_properties::{ RasterProperties, RasterPropertiesEntry, RasterPropertiesEntryType, RasterPropertiesKey, }; pub use raster_traits::{CoordinatePixelAccess, GeoTransformAccess, Raster}; +pub use util::{TileIdxBandCrossProductIter, TileInformationBandCrossProductIter}; mod arrow_conversion; mod band_names; @@ -79,3 +80,4 @@ mod raster_traits; mod tiling; mod typed_raster_conversion; mod typed_raster_tile; +mod util; diff --git a/datatypes/src/raster/tiling.rs b/datatypes/src/raster/tiling.rs index be66439ae..b2cccb64a 100644 --- a/datatypes/src/raster/tiling.rs +++ b/datatypes/src/raster/tiling.rs @@ -10,7 +10,7 @@ use crate::{ use serde::{Deserialize, Serialize}; /// The static parameters required to create a `TilingStrategy` -#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq)] pub struct TilingSpecification { pub tile_size_in_pixels: GridShape2D, } diff --git a/datatypes/src/raster/util.rs b/datatypes/src/raster/util.rs new file mode 100644 index 000000000..183da8f89 --- /dev/null +++ b/datatypes/src/raster/util.rs @@ -0,0 +1,121 @@ +use crate::{ + primitives::{BandSelection, BandSelectionIter}, + raster::{GridBoundingBox2D, GridIdx2D, GridIdx2DIter, TileInformation, TileInformationIter}, +}; + +#[derive(Clone, Debug)] +pub struct TileIdxBandCrossProductIter { + tile_iter: GridIdx2DIter, + band_iter: BandSelectionIter, // TODO: maybe change this to actual attributes from ResultDescriptor not the Selection? + current_tile: Option, +} + +impl TileIdxBandCrossProductIter { + pub fn new(tile_iter: GridIdx2DIter, band_iter: BandSelectionIter) -> Self { + let mut tile_iter = tile_iter; + let current_tile = tile_iter.next(); + Self { + tile_iter, + band_iter, + current_tile, + } + } + + pub fn grid_bounds(&self) -> GridBoundingBox2D { + self.tile_iter.grid_bounds + } + + pub fn band_selection(&self) -> &BandSelection { + &self.band_iter.band_selection + } + + pub fn with_grid_bounds_and_selection( + bounds: GridBoundingBox2D, + band_selection: BandSelection, + ) -> Self { + let tile_iter = GridIdx2DIter::new(&bounds); + let band_iter = BandSelectionIter::new(band_selection); + Self::new(tile_iter, band_iter) + } + + pub fn reset(&mut self) { + self.band_iter.reset(); + self.tile_iter.reset(); + self.current_tile = self.tile_iter.next(); + } +} + +impl Iterator for TileIdxBandCrossProductIter { + type Item = (GridIdx2D, u32); + + fn next(&mut self) -> Option { + let current_t = self.current_tile; + + match (current_t, self.band_iter.next()) { + (None, _) => None, + (Some(t), Some(b)) => Some((t, b)), + (Some(_t), None) => { + self.band_iter.reset(); + self.current_tile = self.tile_iter.next(); + self.current_tile.map(|t| { + ( + t, + self.band_iter + .next() + .expect("There must be at least one band"), + ) + }) + } + } + } +} + +#[derive(Clone, Debug)] +pub struct TileInformationBandCrossProductIter { + tile_iter: TileInformationIter, + band_iter: BandSelectionIter, + current_tile: Option, +} + +impl TileInformationBandCrossProductIter { + pub fn new(tile_iter: TileInformationIter, band_iter: BandSelectionIter) -> Self { + let mut tile_iter = tile_iter; + let current_tile = tile_iter.next(); + Self { + tile_iter, + band_iter, + current_tile, + } + } + + pub fn reset(&mut self) { + self.band_iter.reset(); + self.tile_iter.reset(); + self.current_tile = self.tile_iter.next(); + } +} + +impl Iterator for TileInformationBandCrossProductIter { + type Item = (TileInformation, u32); + + fn next(&mut self) -> Option { + let current_t = self.current_tile; + + match (current_t, self.band_iter.next()) { + (None, _) => None, + (Some(t), Some(b)) => Some((t, b)), + (Some(_t), None) => { + self.band_iter.reset(); + self.current_tile = self.tile_iter.next(); + self.current_tile.map(|t| { + ( + t, + self.band_iter + .next() + .expect("There must be at least one band"), + ) + }) + } + } + } +} diff --git a/datatypes/src/util/test.rs b/datatypes/src/util/test.rs index 963fc9619..54141ccb9 100644 --- a/datatypes/src/util/test.rs +++ b/datatypes/src/util/test.rs @@ -1,8 +1,11 @@ use float_cmp::approx_eq; -use crate::raster::{ - EmptyGrid, GeoTransform, Grid, GridIndexAccess, GridOrEmpty, GridSize, MaskedGrid, Pixel, - RasterTile2D, grid_idx_iter_2d, +use crate::{ + primitives::TimeInterval, + raster::{ + EmptyGrid, GeoTransform, Grid, GridIdx2D, GridIndexAccess, GridOrEmpty, GridSize, + MaskedGrid, Pixel, RasterTile2D, grid_idx_iter_2d, + }, }; use std::panic; @@ -103,6 +106,10 @@ pub fn assert_eq_two_list_of_tiles( list_b: &[RasterTile2D

], compare_cache_hint: bool, ) { + fn tile_pos(tile: &RasterTile2D) -> (TimeInterval, GridIdx2D, u32) { + (tile.time, tile.tile_position, tile.band) + } + assert_eq!( list_a.len(), list_b.len(), @@ -156,10 +163,12 @@ pub fn assert_eq_two_list_of_tiles( assert_eq!( a.grid_array.is_empty(), b.grid_array.is_empty(), - "grid shape of tile {} input_a is_empty: {:?}, input_b is_empty: {:?}", + "grid shape of tile {} input_a is_empty: {:?}, input_b is_empty: {:?}, a info: {:?}, b info: {:?}", i, a.grid_array.is_empty(), b.grid_array.is_empty(), + tile_pos(a), + tile_pos(b) ); if !a.grid_array.is_empty() { let mat_a = a.grid_array.clone().into_materialized_masked_grid(); diff --git a/operators/src/adapters/mod.rs b/operators/src/adapters/mod.rs index 7918878a5..4dd4a4a91 100644 --- a/operators/src/adapters/mod.rs +++ b/operators/src/adapters/mod.rs @@ -2,27 +2,22 @@ mod band_extractor; mod feature_collection_merger; mod raster_stacker; mod raster_subquery; -mod raster_time; mod raster_time_substream; mod simple_raster_stacker; -mod sparse_tiles_fill_adapter; mod stream_statistics_adapter; mod time_stream_merge; use band_extractor::BandExtractor; pub use feature_collection_merger::FeatureCollectionChunkMerger; -pub use raster_stacker::{RasterStackerAdapter, RasterStackerSource}; +pub use raster_stacker::{ + PartialQueryRect, QueryWrapper, RasterStackerAdapter, RasterStackerSource, +}; pub use raster_subquery::{ FoldTileAccu, FoldTileAccuMut, RasterSubQueryAdapter, SubQueryTileAggregator, TileReprojectionSubQuery, TileReprojectionSubqueryGridInfo, fold_by_coordinate_lookup_future, }; -pub use raster_time::{QueryWrapper, Queryable, RasterArrayTimeAdapter, RasterTimeAdapter}; pub use simple_raster_stacker::{ - SimpleRasterStackerAdapter, SimpleRasterStackerSource, stack_individual_aligned_raster_bands, -}; -pub(crate) use sparse_tiles_fill_adapter::{ - FillerTileCacheExpirationStrategy, FillerTimeBounds, SparseTilesFillAdapter, - SparseTilesFillAdapterError, + SimpleRasterStackerAdapter, SimpleRasterStackerError, SimpleRasterStackerSource, }; pub use stream_statistics_adapter::StreamStatisticsAdapter; pub use time_stream_merge::TimeIntervalStreamMerge; diff --git a/operators/src/adapters/raster_stacker.rs b/operators/src/adapters/raster_stacker.rs index 4c40a8d11..7611ee0fe 100644 --- a/operators/src/adapters/raster_stacker.rs +++ b/operators/src/adapters/raster_stacker.rs @@ -1,16 +1,57 @@ +use crate::engine::{QueryContext, RasterQueryProcessor}; use crate::util::Result; -use futures::future::JoinAll; -use futures::stream::{Fuse, FusedStream, Stream}; +use futures::future::{BoxFuture, JoinAll}; +use futures::stream::{BoxStream, Fuse, FusedStream, Stream}; use futures::{Future, StreamExt, ready}; use geoengine_datatypes::primitives::{BandSelection, RasterQueryRectangle, TimeInterval}; use geoengine_datatypes::raster::{ - GridBoundingBox2D, GridIdx2D, GridSize, Pixel, RasterTile2D, TileInformation, TilingStrategy, + GridBoundingBox2D, GridBounds, GridIdx2D, Pixel, RasterTile2D, TileIdxBandCrossProductIter, + TilingStrategy, }; use pin_project::pin_project; use std::pin::Pin; use std::task::{Context, Poll}; -use super::Queryable; +/// A wrapper around a `QueryProcessor` and a `QueryContext` that allows querying +/// with only a `QueryRectangle`. +pub struct QueryWrapper<'a, P, T> +where + P: RasterQueryProcessor, + T: Pixel, +{ + pub p: &'a P, + pub ctx: &'a dyn QueryContext, +} + +/// This trait allows hiding the concrete type of the `QueryProcessor` from the +/// `RasterTimeAdapter` and allows querying with only a `QueryRectangle`. +/// Notice, that the `query` function is not async, but return a `Future`. +/// This is necessary because there are no async function traits or async closures yet. +pub trait Queryable +where + T: Pixel, +{ + /// the type of the stream produced by the `QueryProcessor` + type Stream; + /// the type of the future produced by the `QueryProcessor`. We need both types + /// to correctly specify trait bounds in the `RasterTimeAdapter` + type Output; + + fn query(&self, rect: RasterQueryRectangle) -> Self::Output; +} + +impl<'a, P, T> Queryable for QueryWrapper<'a, P, T> +where + P: RasterQueryProcessor, + T: Pixel, +{ + type Stream = BoxStream<'a, Result>>; + type Output = BoxFuture<'a, Result>; + + fn query(&self, rect: RasterQueryRectangle) -> Self::Output { + self.p.raster_query(rect, self.ctx) + } +} #[pin_project(project = ArrayStateProjection)] enum State @@ -42,8 +83,9 @@ enum StreamState { first_tiles: Vec>, time_slice: TimeInterval, current_stream: usize, - current_band: usize, - current_spatial_tile: usize, + current_stream_band_pos: usize, + current_tile: GridIdx2D, + current_out_band: u32, }, } @@ -61,7 +103,6 @@ impl From<(Q, Vec)> for RasterStackerSource { } } } - #[derive(Debug)] pub struct PartialQueryRect { pub spatial_bounds: GridBoundingBox2D, @@ -72,6 +113,13 @@ impl PartialQueryRect { fn raster_query_rectangle(&self, attributes: BandSelection) -> RasterQueryRectangle { RasterQueryRectangle::new(self.spatial_bounds, self.time_interval, attributes) } + + pub fn new(spatial_bounds: GridBoundingBox2D, time_interval: TimeInterval) -> Self { + PartialQueryRect { + spatial_bounds, + time_interval, + } + } } impl From for PartialQueryRect { @@ -99,7 +147,10 @@ where state: State, // the current query rectangle, which is advanced over time by increasing the start time query_rect: PartialQueryRect, - num_spatial_tiles: Option, + + tiling_strategy: TilingStrategy, + // this iterator helps to keep track of the elements being produced + tile_band_iter: TileIdxBandCrossProductIter, } impl RasterStackerAdapter @@ -109,42 +160,26 @@ where F::Stream: Stream>>, F::Output: Future>, { - pub fn new(queryables: Vec>, query_rect: PartialQueryRect) -> Self { + pub fn new( + queryables: Vec>, + query_rect: PartialQueryRect, + tiling_strategy: TilingStrategy, + ) -> Self { + let number_of_bands: usize = queryables.iter().map(|s| s.band_idxs.len()).sum(); + + let tile_band_iter = TileIdxBandCrossProductIter::with_grid_bounds_and_selection( + tiling_strategy.global_pixel_grid_bounds_to_tile_grid_bounds(query_rect.spatial_bounds), + BandSelection::first_n(number_of_bands as u32), + ); + Self { sources: queryables, query_rect, state: State::Initial, - num_spatial_tiles: None, + tiling_strategy, + tile_band_iter, } } - - fn number_of_tiles_in_grid_bounds( - tile_info: &TileInformation, - grid_bounds: GridBoundingBox2D, - ) -> usize { - let strat = TilingStrategy { - tile_size_in_pixels: tile_info.tile_size_in_pixels, - geo_transform: tile_info.global_geo_transform, - }; - strat - .global_pixel_grid_bounds_to_tile_grid_bounds(grid_bounds) - .number_of_elements() - } - - fn grid_idx_for_nth_tile( - tile_info: &TileInformation, - pixel_bounds: GridBoundingBox2D, - n: usize, - ) -> Option { - let strat = TilingStrategy { - tile_size_in_pixels: tile_info.tile_size_in_pixels, - geo_transform: tile_info.global_geo_transform, - }; - - strat - .tile_idx_iterator_from_grid_bounds(pixel_bounds) - .nth(n) - } } impl Stream for RasterStackerAdapter @@ -162,7 +197,8 @@ where sources, mut state, query_rect, - num_spatial_tiles, + tiling_strategy: _tiling_strategy, + tile_band_iter, } = self.project(); loop { @@ -266,34 +302,42 @@ where }); } - *num_spatial_tiles = Some(Self::number_of_tiles_in_grid_bounds( - &ok_tiles[0].tile_information(), - query_rect.spatial_bounds, //TODO: use direct mehtod instead of conversion - )); + let (first_tile, first_band) = tile_band_iter + .next() + .expect("There must be at least one tile and band"); *stream_state = StreamState::ProducingTimeSlice { first_tiles: ok_tiles, time_slice: time, current_stream: 0, - current_band: 0, - current_spatial_tile: 0, + current_stream_band_pos: 0, + current_tile: first_tile, + current_out_band: first_band, } } StreamState::ProducingTimeSlice { first_tiles, time_slice, current_stream, - current_band, - current_spatial_tile, + current_stream_band_pos, + current_tile, + current_out_band, } => { - let tile = if *current_spatial_tile == 0 && *current_band == 0 { + let tile = if *current_tile == tile_band_iter.grid_bounds().min_index() + && *current_stream_band_pos == 0 + { // consume tiles that were already computed first Some(Ok(first_tiles[*current_stream].clone())) // TODO: avoid clone and instead consume the tile } else { ready!(Pin::new(&mut streams[*current_stream]).poll_next(cx)) }; - let mut tile = match tile { + let mut tile: geoengine_datatypes::raster::BaseTile< + geoengine_datatypes::raster::GridOrEmpty< + geoengine_datatypes::raster::GridShape<[usize; 2]>, + T, + >, + > = match tile { Some(Ok(tile)) => tile, Some(Err(e)) => { state.set(State::Finished); @@ -305,87 +349,95 @@ where } }; - debug_assert_eq!( - tile.band, *current_band as u32, - "RasterStacker got tile with unexpected band index: expected {}, got {} for source {}", - current_band, tile.band, current_stream - ); - - debug_assert!( - tile.time.contains(time_slice), - "RasterStacker got tile with unexpected time: time slice [{}, {}) not contained in tile time [{}, {}) for source {}", - time_slice.start().as_datetime_string(), - time_slice.end().as_datetime_string(), - tile.time.start().as_datetime_string(), - tile.time.end().as_datetime_string(), - current_stream - ); - - debug_assert_eq!( - Some(tile.tile_position), - Self::grid_idx_for_nth_tile( - &tile.tile_information(), - query_rect.spatial_bounds, - *current_spatial_tile - ), - "RasteStacker got tile with unexpected tile_position: expected {:?}, got {:?} for source {}", - Self::grid_idx_for_nth_tile( - &tile.tile_information(), - query_rect.spatial_bounds, - *current_spatial_tile - ), - tile.tile_position, - current_stream - ); - - tile.band = sources - .iter() - .take(*current_stream) - .map(|b| b.band_idxs.len() as u32) - .sum::() - + *current_band as u32; - tile.time = *time_slice; + #[cfg(debug_assertions)] + { + debug_assert!( + !tile.time.is_instant(), + "Tile time must be intervals with length > 0. Got an instant {}", + tile.time + ); - // make progress - *current_band += 1; - if *current_band >= sources[*current_stream].band_idxs.len() { - *current_band = 0; - *current_stream += 1; - } + let source_bands = sources[*current_stream].band_idxs.as_slice(); + let current_souce_band = source_bands[*current_stream_band_pos]; + + debug_assert_eq!( + tile.band, + current_souce_band, + "RasterStacker got tile with unexpected band index: expected {}, got {} for source {} selection {:?} pos {}", + current_souce_band, + tile.band, + current_stream, + source_bands, + current_stream_band_pos + ); - if *current_stream >= streams.len() { - *current_stream = 0; - *current_band = 0; - *current_spatial_tile += 1; + debug_assert!( + tile.time.contains(time_slice), + "RasterStacker got tile with unexpected time: time slice [{}, {}) not contained in tile time [{}, {}) for source {}", + time_slice.start().as_datetime_string(), + time_slice.end().as_datetime_string(), + tile.time.start().as_datetime_string(), + tile.time.end().as_datetime_string(), + current_stream + ); + + debug_assert_eq!( + tile.tile_position, *current_tile, + "RasteStacker got tile with unexpected tile_position: expected (state) {:?}, got (tile) {:?} for source {}", + current_tile, tile.tile_position, current_stream + ); } - if *current_spatial_tile >= num_spatial_tiles.unwrap_or_default() { - *current_spatial_tile = 0; - *current_band = 0; - *current_stream = 0; + tile.band = *current_out_band; + tile.time = *time_slice; - let mut new_start = time_slice.end(); + let next_step = tile_band_iter.next(); - if new_start == query_rect.time_interval.start() { - // in the case that the time interval has no length, i.e. start=end, - // we have to advance `new_start` to prevent infinite loops. - // Otherwise, the new query rectangle would be equal to the previous one. - new_start += 1; - } + match next_step { + Some((next_tile, next_out_band)) if next_tile == *current_tile => { + // the next tile is in the same spatial tile, just a different band - if new_start >= query_rect.time_interval.end() { - // the query window is exhausted, end the stream - state.set(State::Finished); - } else { - // advance the query rectangle and reset the state so that the sources are queried again for the next time step - query_rect.time_interval = TimeInterval::new_unchecked( - new_start, - query_rect.time_interval.end(), + // make progress + *current_out_band = next_out_band; + *current_stream_band_pos += 1; + if *current_stream_band_pos + >= sources[*current_stream].band_idxs.len() + { + *current_stream_band_pos = 0; + *current_stream += 1; + } + debug_assert!( + *current_stream < sources.len(), + "Current stream must not excede the number of streams" ); + } + Some((next_tile, next_band)) => { + // the next tile is in a different spatial tile. We need to start with 0 here... + *current_stream_band_pos = 0; + *current_stream = 0; + *current_tile = next_tile; + *current_out_band = next_band; + } + None => { + // this is either a new TimeStep OR finish! + let new_start = time_slice.end(); - state.set(State::Initial); + if new_start >= query_rect.time_interval.end() { + // the query window is exhausted, end the stream + state.set(State::Finished); + } else { + // advance the query rectangle and reset the state so that the sources are queried again for the next time step + query_rect.time_interval = TimeInterval::new_unchecked( + new_start, + query_rect.time_interval.end(), + ); + tile_band_iter.reset(); // reset iter to start at first tile / band + + state.set(State::Initial); + } } } + return Poll::Ready(Some(Ok(tile))); } }, @@ -406,11 +458,10 @@ mod tests { TilesEqualIgnoringCacheHint, }, spatial_reference::SpatialReference, - util::test::TestDefault, + util::test::{TestDefault, assert_eq_two_list_of_tiles}, }; use crate::{ - adapters::QueryWrapper, engine::{ MockExecutionContext, RasterBandDescriptor, RasterBandDescriptors, RasterOperator, RasterResultDescriptor, SpatialGridDescriptor, TimeDescriptor, WorkflowOperatorPath, @@ -582,10 +633,13 @@ mod tests { ) .into(), ], - PartialQueryRect { - spatial_bounds: GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), - time_interval: TimeInterval::new_unchecked(0, 10), - }, + PartialQueryRect::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 10), + ), + result_descriptor + .tiling_grid_definition(exe_ctx.tiling_specification) + .generate_data_tiling_strategy(), ); let result = stacker.collect::>().await; @@ -693,10 +747,13 @@ mod tests { ) .into(), ], - PartialQueryRect { - spatial_bounds: GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), - time_interval: TimeInterval::new_unchecked(0, 10), - }, + PartialQueryRect::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 10), + ), + result_descriptor + .tiling_grid_definition(exe_ctx.tiling_specification) + .generate_data_tiling_strategy(), ); let result = stacker.collect::>().await; @@ -970,10 +1027,13 @@ mod tests { ) .into(), ], - PartialQueryRect { - spatial_bounds: GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), - time_interval: TimeInterval::new_unchecked(0, 10), - }, + PartialQueryRect::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 10), + ), + result_descriptor_1 + .tiling_grid_definition(exe_ctx.tiling_specification) + .generate_data_tiling_strategy(), ); let result = stacker.collect::>().await; @@ -1021,7 +1081,7 @@ mod tests { let result_descriptor_2 = RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_regular_with_epoch(None, TimeStep::millis(5).unwrap()), + time: TimeDescriptor::new_irregular(None), spatial_grid: SpatialGridDescriptor::source_from_parts( GeoTransform::test_default(), GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), @@ -1264,10 +1324,13 @@ mod tests { ) .into(), ], - PartialQueryRect { - spatial_bounds: GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), - time_interval: TimeInterval::new_unchecked(0, 10), - }, + PartialQueryRect::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 10), + ), + result_descriptor_1 + .tiling_grid_definition(exe_ctx.tiling_specification) + .generate_data_tiling_strategy(), ); let result = stacker.collect::>().await; @@ -1546,7 +1609,7 @@ mod tests { let result_descriptor_2 = RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_regular_with_epoch(None, TimeStep::millis(5).unwrap()), + time: TimeDescriptor::new_irregular(None), spatial_grid: SpatialGridDescriptor::source_from_parts( GeoTransform::test_default(), GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), @@ -1562,7 +1625,7 @@ mod tests { let result_descriptor_3 = RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_regular_with_epoch(None, TimeStep::millis(5).unwrap()), + time: TimeDescriptor::new_irregular(None), spatial_grid: SpatialGridDescriptor::source_from_parts( GeoTransform::test_default(), GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), @@ -1924,10 +1987,13 @@ mod tests { ) .into(), ], - PartialQueryRect { - spatial_bounds: GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), - time_interval: TimeInterval::new_unchecked(0, 10), - }, + PartialQueryRect::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 10), + ), + result_descriptor_1 + .tiling_grid_definition(exe_ctx.tiling_specification) + .generate_data_tiling_strategy(), ); let result = stacker.collect::>().await; @@ -2316,6 +2382,6 @@ mod tests { }, ]; - assert!(expected.tiles_equal_ignoring_cache_hint(&result)); + assert_eq_two_list_of_tiles(&expected, &result, false); } } diff --git a/operators/src/adapters/raster_subquery/raster_subquery_adapter.rs b/operators/src/adapters/raster_subquery/raster_subquery_adapter.rs index 6678eb282..179aab5ff 100644 --- a/operators/src/adapters/raster_subquery/raster_subquery_adapter.rs +++ b/operators/src/adapters/raster_subquery/raster_subquery_adapter.rs @@ -12,7 +12,9 @@ use futures::{ use geoengine_datatypes::primitives::RasterQueryRectangle; use geoengine_datatypes::primitives::TimeInterval; use geoengine_datatypes::primitives::{BandSelectionIter, CacheHint}; -use geoengine_datatypes::raster::{GridOrEmpty, TileInformationIter, TilingStrategy}; +use geoengine_datatypes::raster::{ + GridOrEmpty, TileInformationBandCrossProductIter, TilingStrategy, +}; use geoengine_datatypes::raster::{Pixel, RasterTile2D, TileInformation}; use rayon::ThreadPool; use std::pin::Pin; @@ -45,56 +47,6 @@ type IntoTileFuture<'a, T> = BoxFuture<'a, Result>>; /// This is done using a `TileSubQuery`. /// The sub-query is resolved for each produced tile. -#[derive(Clone, Debug)] -struct TileBandCrossProductIter { - tile_iter: TileInformationIter, - band_iter: BandSelectionIter, - current_tile: Option, -} - -impl TileBandCrossProductIter { - fn new(tile_iter: TileInformationIter, band_iter: BandSelectionIter) -> Self { - let mut tile_iter = tile_iter; - let current_tile = tile_iter.next(); - Self { - tile_iter, - band_iter, - current_tile, - } - } - - fn reset(&mut self) { - self.band_iter.reset(); - self.tile_iter.reset(); - self.current_tile = self.tile_iter.next(); - } -} - -impl Iterator for TileBandCrossProductIter { - type Item = (TileInformation, u32); - - fn next(&mut self) -> Option { - let current_t = self.current_tile; - - match (current_t, self.band_iter.next()) { - (None, _) => None, - (Some(t), Some(b)) => Some((t, b)), - (Some(_t), None) => { - self.band_iter.reset(); - self.current_tile = self.tile_iter.next(); - self.current_tile.map(|t| { - ( - t, - self.band_iter - .next() - .expect("There must be at least one band"), - ) - }) - } - } - } -} - #[pin_project(project=StateInnerProjection)] #[derive(Debug, Clone)] enum StateInner { @@ -173,7 +125,7 @@ where /// This `TimeInterval` is the time currently worked on current_time: TimeInterval, - band_tile_iter: TileBandCrossProductIter, + band_tile_iter: TileInformationBandCrossProductIter, /// This current state of the adapter #[pin] @@ -205,7 +157,7 @@ where let tile_iter = tiling_strategy .tile_information_iterator_from_pixel_bounds(query_rect_to_answer.spatial_bounds()); let band_iter = BandSelectionIter::new(query_rect_to_answer.attributes().clone()); - let band_tile_iter = TileBandCrossProductIter::new(tile_iter, band_iter); + let band_tile_iter = TileInformationBandCrossProductIter::new(tile_iter, band_iter); Self { current_time: TimeInterval::default(), // This is overwritten in the first poll_next call! diff --git a/operators/src/adapters/raster_time.rs b/operators/src/adapters/raster_time.rs deleted file mode 100644 index 78289606c..000000000 --- a/operators/src/adapters/raster_time.rs +++ /dev/null @@ -1,2354 +0,0 @@ -use crate::engine::{QueryContext, RasterQueryProcessor}; -use crate::util::Result; -use crate::util::stream_zip::StreamArrayZip; -use futures::future::{self, BoxFuture, Join, JoinAll}; -use futures::stream::{BoxStream, FusedStream, Zip}; -use futures::{Future, Stream, StreamExt, ready}; -use geoengine_datatypes::primitives::{RasterQueryRectangle, TimeInterval}; -use geoengine_datatypes::raster::{ - GridBoundingBox2D, GridSize, Pixel, RasterTile2D, TileInformation, TilingStrategy, -}; -use pin_project::pin_project; -use std::cmp::min; -use std::pin::Pin; -use std::task::{Context, Poll}; - -#[allow(clippy::large_enum_variant)] // TODO: investigate if we should larger `Box` variants -#[pin_project(project = StateProjection)] -enum State -where - T1: Pixel, - T2: Pixel, - F1: Queryable, - F2: Queryable, - F1::Stream: Stream>>, - F2::Stream: Stream>>, - F1::Output: Future>, - F2::Output: Future>, -{ - Initial, - AwaitingQuery { - #[pin] - query_a_b_fut: Join, - }, - ConsumingStream { - #[pin] - stream: Zip, - current_spatial_tile: usize, - }, - Finished, -} - -/// Merges two raster sources by aligning the temporal validity. -/// -/// # Assumptions -/// * Assumes that the raster tiles already align spatially. -/// * Assumes that raster tiles are contiguous temporally, with no-data-tiles filling gaps. -/// -/// # Notice -/// Potentially queries the same tiles multiple times from its sources. -#[pin_project(project = RasterTimeAdapterProjection)] -pub struct RasterTimeAdapter -where - T1: Pixel, - T2: Pixel, - F1: Queryable, - F2: Queryable, - F1::Stream: Stream>>, - F2::Stream: Stream>>, - F1::Output: Future>, - F2::Output: Future>, -{ - // the first source (wrapped QueryProcessor) - source_a: F1, - // the second source (wrapped QueryProcessor) - source_b: F2, - #[pin] - state: State, - // the current query rectangle, which is advanced over time by increasing the start time - query_rect: RasterQueryRectangle, - num_spatial_tiles: Option, -} - -impl RasterTimeAdapter -where - T1: Pixel, - T2: Pixel, - F1: Queryable, - F2: Queryable, - F1::Stream: Stream>>, - F2::Stream: Stream>>, - F1::Output: Future>, - F2::Output: Future>, -{ - pub fn new(source_a: F1, source_b: F2, query_rect: RasterQueryRectangle) -> Self { - // TODO: need to ensure all sources are single-band - Self { - source_a, - source_b, - query_rect, - state: State::Initial, - num_spatial_tiles: None, - } - } - - fn align_tiles( - mut tile_a: RasterTile2D, - mut tile_b: RasterTile2D, - ) -> (RasterTile2D, RasterTile2D) { - // TODO: scale data if measurement unit requires it? - let time = tile_a.time.intersect(&tile_b.time).unwrap_or_else(|| { - panic!( - "intervals must overlap: ({}/{}) <-> ({}/{})\nThis is a bug and most likely means an operator or adapter has a faulty implementation.", - tile_a.time.start().as_datetime_string(), - tile_a.time.end().as_datetime_string(), - tile_b.time.start().as_datetime_string(), - tile_b.time.end().as_datetime_string() - ) - }); - tile_a.time = time; - tile_b.time = time; - (tile_a, tile_b) - } - - fn number_of_tiles_in_pixel_grid_bounds( - tile_info: &TileInformation, - grid_bounds: GridBoundingBox2D, - ) -> usize { - let strat = TilingStrategy { - tile_size_in_pixels: tile_info.tile_size_in_pixels, - geo_transform: tile_info.global_geo_transform, - }; - strat - .global_pixel_grid_bounds_to_tile_grid_bounds(grid_bounds) - .number_of_elements() - } -} - -#[pin_project(project = ArrayStateProjection)] -enum ArrayState -where - T: Pixel, - F: Queryable, - F::Stream: Stream>>, - F::Output: Future>, -{ - Initial, - AwaitingQuery { - #[pin] - query_futures: JoinAll, // TODO: use join construct on array if available - }, - ConsumingStream { - #[pin] - stream: StreamArrayZip, - current_spatial_tile: usize, - }, - Finished, -} - -/// Merges `N` raster sources by aligning the temporal validity. -/// -/// # Assumptions -/// * Assumes that the raster tiles already align spatially. -/// * Assumes that raster tiles are contiguous temporally, with no-data-tiles filling gaps. -/// -/// # Notice -/// Potentially queries the same tiles multiple times from its sources. -#[pin_project(project = RasterArrayTimeAdapterProjection)] -pub struct RasterArrayTimeAdapter -where - T: Pixel, - F: Queryable, - F::Stream: Stream>>, - F::Output: Future>, -{ - // the sources (wrapped `QueryProcessor`s) - sources: [F; N], - #[pin] - state: ArrayState, - // the current query rectangle, which is advanced over time by increasing the start time - query_rect: RasterQueryRectangle, - num_spatial_tiles: Option, -} - -impl RasterArrayTimeAdapter -where - T: Pixel, - F: Queryable, - F::Stream: Stream>>, - F::Output: Future>, -{ - pub fn new(sources: [F; N], query_rect: RasterQueryRectangle) -> Self { - // TODO: need to ensure all sources are single-band - Self { - sources, - query_rect, - state: ArrayState::Initial, - num_spatial_tiles: None, - } - } - - fn align_tiles(mut tiles: [RasterTile2D; N]) -> [RasterTile2D; N] { - let mut iter = tiles.iter(); - - // first time as initial time - let mut time = iter.next().expect("RasterArrayTimeAdapter: N > 0").time; - - for tile in iter { - time = time.intersect(&tile.time).unwrap_or_else(|| { - panic!( - "intervals must overlap: ({}/{}) <-> ({}/{})\nThis is a bug and most likely means an operator or adapter has a faulty implementation.", - time.start().as_datetime_string(), - time.end().as_datetime_string(), - tile.time.start().as_datetime_string(), - tile.time.end().as_datetime_string() - ) - }); - } - - for tile in &mut tiles { - tile.time = time; - } - - tiles - } - - fn number_of_tiles_in_grid_bounds( - tile_info: &TileInformation, - grid_bounds: GridBoundingBox2D, - ) -> usize { - RasterTimeAdapter::::number_of_tiles_in_pixel_grid_bounds( - tile_info, - grid_bounds, - ) - } -} - -impl Stream for RasterTimeAdapter -where - T1: Pixel, - T2: Pixel, - F1: Queryable, - F2: Queryable, - F1::Stream: Stream>>, - F2::Stream: Stream>>, - F1::Output: Future>, - F2::Output: Future>, -{ - type Item = Result<(RasterTile2D, RasterTile2D)>; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - // The adapter aligns the two input time series by querying both sources simultaneously. - // All spatial tiles for the current time step are aligned, s.t. they are both valid - // for the same time (minimum of both). Then both queries are reset to start at the end - // of the previous time step. - - let RasterTimeAdapterProjection { - source_a, - source_b, - mut state, - query_rect, - num_spatial_tiles, - } = self.project(); - - loop { - match state.as_mut().project() { - StateProjection::Initial => { - let fut1 = source_a.query(query_rect.clone()); - let fut2 = source_b.query(query_rect.clone()); - let join = future::join(fut1, fut2); - - state.set(State::AwaitingQuery { - query_a_b_fut: join, - }); - } - StateProjection::AwaitingQuery { query_a_b_fut } => { - let queries = ready!(query_a_b_fut.poll(cx)); - - match queries { - (Ok(stream_a), Ok(stream_b)) => { - // both sources produced an output, set the stream to be consumed - state.set(State::ConsumingStream { - stream: stream_a.zip(stream_b), - current_spatial_tile: 0, - }); - } - (Err(e), _) | (_, Err(e)) => { - // at least one source failed, output error and end the stream - state.set(State::Finished); - return Poll::Ready(Some(Err(e))); - } - } - } - StateProjection::ConsumingStream { - stream, - current_spatial_tile, - } => { - match ready!(stream.poll_next(cx)) { - Some((Ok(tile_a), Ok(tile_b))) => { - // TODO: calculate at start when tiling info is available before querying first tile - let num_spatial_tiles = *num_spatial_tiles.get_or_insert_with(|| { - Self::number_of_tiles_in_pixel_grid_bounds( - &tile_a.tile_information(), - query_rect.spatial_bounds(), // TODO: this should be calculated from the tile grid bounds and not the spatial bounds. - ) - }); - - if *current_spatial_tile + 1 >= num_spatial_tiles { - // time slice ended => query next time slice of sources - - if tile_a.time.end() == tile_b.time.end() { - // the tiles are currently aligned, continue with next tile of the stream - *current_spatial_tile = 0; - } else { - // the tiles are not aligned. We need to reset the stream with the next time step - // and replay the spatial tiles of the source that is still valid with the next - // time step of the other one - - // advance current query rectangle - let mut new_start = min(tile_a.time.end(), tile_b.time.end()); - - if new_start == query_rect.time_interval().start() { - // in the case that the time interval has no length, i.e. start=end, - // we have to advance `new_start` to prevent infinite loops. - // Otherwise, the new query rectangle would be equal to the previous one. - new_start += 1; - } - - if new_start >= query_rect.time_interval().end() { - // the query window is exhausted, end the stream - state.set(State::Finished); - } else { - *query_rect.time_interval_mut() = - TimeInterval::new_unchecked( - new_start, - query_rect.time_interval().end(), - ); - - state.set(State::Initial); - } - } - } else { - *current_spatial_tile += 1; - } - return Poll::Ready(Some(Ok(Self::align_tiles(tile_a, tile_b)))); - } - Some((Ok(_), Err(e)) | (Err(e), Ok(_) | Err(_))) => { - state.set(State::Finished); - return Poll::Ready(Some(Err(e))); - } - None => { - state.set(State::Finished); - return Poll::Ready(None); - } - } - } - StateProjection::Finished => return Poll::Ready(None), - } - } - } -} - -impl Stream for RasterArrayTimeAdapter -where - T: Pixel, - F: Queryable, - F::Stream: Stream>> + Unpin, - F::Output: Future>, -{ - type Item = Result<[RasterTile2D; N]>; - - #[allow(clippy::too_many_lines)] // TODO: refactor - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - // The adapter aligns multiple two input time series by querying both sources simultaneously. - // All spatial tiles for the current time step are aligned, s.t. they are both valid - // for the same time (minimum of both). Then both queries are reset to start at the end - // of the previous time step. - - let RasterArrayTimeAdapterProjection { - sources, - mut state, - query_rect, - num_spatial_tiles, - } = self.project(); - - loop { - match state.as_mut().project() { - ArrayStateProjection::Initial => { - let array_of_futures = sources - .iter() - .map(|source| source.query(query_rect.clone())) - .collect::>(); - - let query_futures = futures::future::join_all(array_of_futures); - - state.set(ArrayState::AwaitingQuery { query_futures }); - } - ArrayStateProjection::AwaitingQuery { query_futures } => { - let queries = ready!(query_futures.poll(cx)); - - let mut ok_queries = Vec::with_capacity(N); - for query in queries { - match query { - Ok(query) => ok_queries.push(query), - Err(e) => { - // at least one source failed, output error and end the stream - state.set(ArrayState::Finished); - return Poll::Ready(Some(Err(e))); - } - } - } - - // all sources produced an output, set the stream to be consumed - let Ok(streams) = ok_queries.try_into() else { - unreachable!("RasterArrayTimeAdapter: ok_queries.len() != N"); - }; - state.set(ArrayState::ConsumingStream { - stream: StreamArrayZip::new(streams), - current_spatial_tile: 0, - }); - } - ArrayStateProjection::ConsumingStream { - stream, - current_spatial_tile, - } => { - let tiles: Option<[Result>; N]> = ready!(stream.poll_next(cx)); - - // 1. CHECK IF POLL WAS NONE - let tiles: [Result>; N] = if let Some(tiles) = tiles { - tiles - } else { - state.set(ArrayState::Finished); - return Poll::Ready(None); - }; - - // 2. CHECK IF SOME RESULT HAD AN ERROR - let mut ok_tiles = Vec::with_capacity(N); - for tile in tiles { - match tile { - Ok(tile) => ok_tiles.push(tile), - Err(e) => { - // at least one stream failed, output error and end the stream - state.set(ArrayState::Finished); - return Poll::Ready(Some(Err(e))); - } - } - } - let tiles: [RasterTile2D; N] = ok_tiles - .try_into() - .expect("RasterArrayTimeAdapter: ok_tiles.len() != N"); - - // 3. PROCEED WITH VALID TILES - - // TODO: calculate at start when tiling info is available before querying first tile - let num_spatial_tiles = *num_spatial_tiles.get_or_insert_with(|| { - Self::number_of_tiles_in_grid_bounds( - &tiles[0].tile_information(), - query_rect.spatial_bounds(), - ) - }); - - if *current_spatial_tile + 1 >= num_spatial_tiles { - // time slice ended => query next time slice of sources - - let first_time_end = tiles[0].time.end(); - if tiles - .iter() - .skip(1) - .all(|tile| tile.time.end() == first_time_end) - { - // the tiles are currently aligned, continue with next tile of the stream - *current_spatial_tile = 0; - } else { - // the tiles are not aligned. We need to reset the stream with the next time step - // and replay the spatial tiles of the source that is still valid with the next - // time step of the other one - - // advance current query rectangle - let mut new_start = tiles - .iter() - .map(|tile| tile.time.end()) - .min() - .expect("N > 0"); - - if new_start == query_rect.time_interval().start() { - // in the case that the time interval has no length, i.e. start=end, - // we have to advance `new_start` to prevent infinite loops. - // Otherwise, the new query rectangle would be equal to the previous one. - new_start += 1; - } - - if new_start >= query_rect.time_interval().end() { - // the query window is exhausted, end the stream - state.set(ArrayState::Finished); - } else { - *query_rect.time_interval_mut() = TimeInterval::new_unchecked( - new_start, - query_rect.time_interval().end(), - ); - - state.set(ArrayState::Initial); - } - } - } else { - *current_spatial_tile += 1; - } - return Poll::Ready(Some(Ok(Self::align_tiles(tiles)))); - } - ArrayStateProjection::Finished => return Poll::Ready(None), - } - } - } -} - -impl FusedStream for RasterTimeAdapter -where - T1: Pixel, - T2: Pixel, - F1: Queryable, - F2: Queryable, - F1::Stream: Stream>>, - F2::Stream: Stream>>, - F1::Output: Future>, - F2::Output: Future>, -{ - fn is_terminated(&self) -> bool { - matches!(self.state, State::Finished) - } -} - -impl FusedStream for RasterArrayTimeAdapter -where - T: Pixel, - F: Queryable, - F::Stream: Stream>> + Unpin, - F::Output: Future>, -{ - fn is_terminated(&self) -> bool { - matches!(self.state, ArrayState::Finished) - } -} - -/// A wrapper around a `QueryProcessor` and a `QueryContext` that allows querying -/// with only a `QueryRectangle`. -pub struct QueryWrapper<'a, P, T> -where - P: RasterQueryProcessor, - T: Pixel, -{ - pub p: &'a P, - pub ctx: &'a dyn QueryContext, -} - -/// This trait allows hiding the concrete type of the `QueryProcessor` from the -/// `RasterTimeAdapter` and allows querying with only a `QueryRectangle`. -/// Notice, that the `query` function is not async, but return a `Future`. -/// This is necessary because there are no async function traits or async closures yet. -pub trait Queryable -where - T: Pixel, -{ - /// the type of the stream produced by the `QueryProcessor` - type Stream; - /// the type of the future produced by the `QueryProcessor`. We need both types - /// to correctly specify trait bounds in the `RasterTimeAdapter` - type Output; - - fn query(&self, rect: RasterQueryRectangle) -> Self::Output; -} - -impl<'a, P, T> Queryable for QueryWrapper<'a, P, T> -where - P: RasterQueryProcessor, - T: Pixel, -{ - type Stream = BoxStream<'a, Result>>; - type Output = BoxFuture<'a, Result>; - - fn query(&self, rect: RasterQueryRectangle) -> Self::Output { - self.p.raster_query(rect, self.ctx) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::engine::{ - MockExecutionContext, RasterBandDescriptors, RasterOperator, RasterResultDescriptor, - SpatialGridDescriptor, TimeDescriptor, WorkflowOperatorPath, - }; - use crate::mock::{MockRasterSource, MockRasterSourceParams}; - use futures::StreamExt; - use geoengine_datatypes::primitives::{BandSelection, CacheHint, TimeStep}; - use geoengine_datatypes::raster::{ - BoundedGrid, EmptyGrid, GeoTransform, Grid, GridShape2D, RasterDataType, RasterProperties, - SpatialGridDefinition, TilingSpecification, - }; - use geoengine_datatypes::spatial_reference::SpatialReference; - use geoengine_datatypes::util::test::TestDefault; - - #[tokio::test] - #[allow(clippy::too_many_lines)] - async fn adapter() { - let mrs1 = MockRasterSource { - params: MockRasterSourceParams:: { - data: vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![1, 2, 3, 4, 5, 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![7, 8, 9, 10, 11, 12]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![13, 14, 15, 16, 17, 18]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![19, 20, 21, 22, 23, 24]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_regular_with_epoch( - None, - TimeStep::millis(5).unwrap(), - ), - spatial_grid: SpatialGridDescriptor::new_source(SpatialGridDefinition::new( - GeoTransform::new((0., -3.).into(), 1., -1.), - GridShape2D::new_2d(3, 4).bounding_box(), - )), - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let mrs2 = MockRasterSource { - params: MockRasterSourceParams { - data: vec![ - RasterTile2D:: { - time: TimeInterval::new_unchecked(0, 3), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![101, 102, 103, 104, 105, 106]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 3), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![107, 108, 109, 110, 111, 112]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(3, 6), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![113, 114, 115, 116, 117, 118]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(3, 6), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![119, 120, 121, 122, 123, 124]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(6, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![125, 126, 127, 128, 129, 130]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(6, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![131, 132, 133, 134, 135, 136]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_irregular(None), - spatial_grid: SpatialGridDescriptor::new_source(SpatialGridDefinition::new( - GeoTransform::new((0., -3.).into(), 1., -1.), - GridShape2D::new_2d(3, 4).bounding_box(), - )), - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let exe_ctx = - MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); - let query_rect = RasterQueryRectangle::new( - GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), - TimeInterval::new_unchecked(0, 10), - BandSelection::first(), - ); - let query_ctx = exe_ctx.mock_query_context_test_default(); - - let qp1 = mrs1 - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap() - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - let qp2 = mrs2 - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap() - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - let source_a = QueryWrapper { - p: &qp1, - ctx: &query_ctx, - }; - - let source_b = QueryWrapper { - p: &qp2, - ctx: &query_ctx, - }; - - let adapter = RasterTimeAdapter::new(source_a, source_b, query_rect); - - let result = adapter - .map(Result::unwrap) - .collect::, RasterTile2D)>>() - .await; - - let times: Vec<_> = result.iter().map(|(a, b)| (a.time, b.time)).collect(); - assert_eq!( - ×, - &[ - ( - TimeInterval::new_unchecked(0, 3), - TimeInterval::new_unchecked(0, 3) - ), - ( - TimeInterval::new_unchecked(0, 3), - TimeInterval::new_unchecked(0, 3) - ), - ( - TimeInterval::new_unchecked(3, 5), - TimeInterval::new_unchecked(3, 5) - ), - ( - TimeInterval::new_unchecked(3, 5), - TimeInterval::new_unchecked(3, 5) - ), - ( - TimeInterval::new_unchecked(5, 6), - TimeInterval::new_unchecked(5, 6) - ), - ( - TimeInterval::new_unchecked(5, 6), - TimeInterval::new_unchecked(5, 6) - ), - ( - TimeInterval::new_unchecked(6, 10), - TimeInterval::new_unchecked(6, 10) - ), - ( - TimeInterval::new_unchecked(6, 10), - TimeInterval::new_unchecked(6, 10) - ) - ] - ); - } - - #[tokio::test] - #[allow(clippy::too_many_lines)] - async fn array_adapter() { - let mrs1 = MockRasterSource { - params: MockRasterSourceParams:: { - data: vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![1, 2, 3, 4, 5, 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![7, 8, 9, 10, 11, 12]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![13, 14, 15, 16, 17, 18]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![19, 20, 21, 22, 23, 24]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_regular_with_epoch( - None, - TimeStep::millis(5).unwrap(), - ), - spatial_grid: SpatialGridDescriptor::source_from_parts( - GeoTransform::new((0., -3.).into(), 1., -1.), - GridShape2D::new_2d(3, 4).bounding_box(), - ), - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let mrs2 = MockRasterSource { - params: MockRasterSourceParams { - data: vec![ - RasterTile2D:: { - time: TimeInterval::new_unchecked(0, 3), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![101, 102, 103, 104, 105, 106]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 3), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![107, 108, 109, 110, 111, 112]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(3, 6), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![113, 114, 115, 116, 117, 118]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(3, 6), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![119, 120, 121, 122, 123, 124]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(6, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![125, 126, 127, 128, 129, 130]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(6, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![131, 132, 133, 134, 135, 136]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_irregular(None), - spatial_grid: SpatialGridDescriptor::source_from_parts( - GeoTransform::new((0., -3.).into(), 1., -1.), - GridShape2D::new_2d(3, 4).bounding_box(), - ), - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let exe_ctx = - MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); - let query_rect = RasterQueryRectangle::new( - GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), - TimeInterval::new_unchecked(0, 10), - BandSelection::first(), - ); - let query_ctx = exe_ctx.mock_query_context_test_default(); - - let qp1 = mrs1 - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap() - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - let qp2 = mrs2 - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap() - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - let source_a = QueryWrapper { - p: &qp1, - ctx: &query_ctx, - }; - - let source_b = QueryWrapper { - p: &qp2, - ctx: &query_ctx, - }; - - let adapter = RasterArrayTimeAdapter::new([source_a, source_b], query_rect); - - let result = adapter - .map(Result::unwrap) - .collect::; 2]>>() - .await; - - let times: Vec<_> = result.iter().map(|[a, b]| (a.time, b.time)).collect(); - assert_eq!( - ×, - &[ - ( - TimeInterval::new_unchecked(0, 3), - TimeInterval::new_unchecked(0, 3) - ), - ( - TimeInterval::new_unchecked(0, 3), - TimeInterval::new_unchecked(0, 3) - ), - ( - TimeInterval::new_unchecked(3, 5), - TimeInterval::new_unchecked(3, 5) - ), - ( - TimeInterval::new_unchecked(3, 5), - TimeInterval::new_unchecked(3, 5) - ), - ( - TimeInterval::new_unchecked(5, 6), - TimeInterval::new_unchecked(5, 6) - ), - ( - TimeInterval::new_unchecked(5, 6), - TimeInterval::new_unchecked(5, 6) - ), - ( - TimeInterval::new_unchecked(6, 10), - TimeInterval::new_unchecked(6, 10) - ), - ( - TimeInterval::new_unchecked(6, 10), - TimeInterval::new_unchecked(6, 10) - ) - ] - ); - } - - #[tokio::test] - #[allow(clippy::too_many_lines)] - async fn already_aligned() { - let mrs1 = MockRasterSource { - params: MockRasterSourceParams:: { - data: vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![1, 2, 3, 4, 5, 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![7, 8, 9, 10, 11, 12]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![13, 14, 15, 16, 17, 18]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![19, 20, 21, 22, 23, 24]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_regular_with_epoch( - None, - TimeStep::millis(5).unwrap(), - ), - spatial_grid: SpatialGridDescriptor::source_from_parts( - GeoTransform::new((0., -3.).into(), 1., -1.), - GridShape2D::new_2d(3, 4).bounding_box(), - ), - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let mrs2 = MockRasterSource { - params: MockRasterSourceParams:: { - data: vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![1, 2, 3, 4, 5, 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![7, 8, 9, 10, 11, 12]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![13, 14, 15, 16, 17, 18]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![19, 20, 21, 22, 23, 24]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_regular_with_epoch( - None, - TimeStep::millis(5).unwrap(), - ), - spatial_grid: SpatialGridDescriptor::source_from_parts( - GeoTransform::new((0., -3.).into(), 1., -1.), - GridShape2D::new_2d(3, 4).bounding_box(), - ), - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let exe_ctx = - MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); - let query_rect = RasterQueryRectangle::new( - GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), - TimeInterval::new_unchecked(0, 10), - BandSelection::first(), - ); - let query_ctx = exe_ctx.mock_query_context_test_default(); - - let qp1 = mrs1 - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap() - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - let qp2 = mrs2 - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap() - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - let source_a = QueryWrapper { - p: &qp1, - ctx: &query_ctx, - }; - - let source_b = QueryWrapper { - p: &qp2, - ctx: &query_ctx, - }; - - let adapter = RasterTimeAdapter::new(source_a, source_b, query_rect); - - let result = adapter - .map(Result::unwrap) - .collect::, RasterTile2D)>>() - .await; - - let times: Vec<_> = result.iter().map(|(a, b)| (a.time, b.time)).collect(); - assert_eq!( - ×, - &[ - ( - TimeInterval::new_unchecked(0, 5), - TimeInterval::new_unchecked(0, 5) - ), - ( - TimeInterval::new_unchecked(0, 5), - TimeInterval::new_unchecked(0, 5) - ), - ( - TimeInterval::new_unchecked(5, 10), - TimeInterval::new_unchecked(5, 10) - ), - ( - TimeInterval::new_unchecked(5, 10), - TimeInterval::new_unchecked(5, 10) - ), - ] - ); - } - - #[tokio::test] - #[allow(clippy::too_many_lines)] - async fn array_already_aligned() { - let mrs1 = MockRasterSource { - params: MockRasterSourceParams:: { - data: vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![1, 2, 3, 4, 5, 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![7, 8, 9, 10, 11, 12]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![13, 14, 15, 16, 17, 18]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![19, 20, 21, 22, 23, 24]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_regular_with_epoch( - None, - TimeStep::millis(5).unwrap(), - ), - spatial_grid: SpatialGridDescriptor::source_from_parts( - GeoTransform::new((0., -3.).into(), 1., -1.), - GridShape2D::new_2d(3, 4).bounding_box(), - ), - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let mrs2 = MockRasterSource { - params: MockRasterSourceParams:: { - data: vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![1, 2, 3, 4, 5, 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![7, 8, 9, 10, 11, 12]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![13, 14, 15, 16, 17, 18]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![19, 20, 21, 22, 23, 24]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_regular_with_epoch( - None, - TimeStep::millis(5).unwrap(), - ), - spatial_grid: SpatialGridDescriptor::source_from_parts( - GeoTransform::new((0., -3.).into(), 1., -1.), - GridShape2D::new_2d(3, 4).bounding_box(), - ), - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let exe_ctx = - MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); - let query_rect = RasterQueryRectangle::new( - GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), - TimeInterval::new_unchecked(0, 10), - BandSelection::first(), - ); - let query_ctx = exe_ctx.mock_query_context_test_default(); - - let qp1 = mrs1 - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap() - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - let qp2 = mrs2 - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap() - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - let source_a = QueryWrapper { - p: &qp1, - ctx: &query_ctx, - }; - - let source_b = QueryWrapper { - p: &qp2, - ctx: &query_ctx, - }; - - let adapter = RasterArrayTimeAdapter::new([source_a, source_b], query_rect); - - let result = adapter - .map(Result::unwrap) - .collect::; 2]>>() - .await; - - let times: Vec<_> = result.iter().map(|[a, b]| (a.time, b.time)).collect(); - assert_eq!( - ×, - &[ - ( - TimeInterval::new_unchecked(0, 5), - TimeInterval::new_unchecked(0, 5) - ), - ( - TimeInterval::new_unchecked(0, 5), - TimeInterval::new_unchecked(0, 5) - ), - ( - TimeInterval::new_unchecked(5, 10), - TimeInterval::new_unchecked(5, 10) - ), - ( - TimeInterval::new_unchecked(5, 10), - TimeInterval::new_unchecked(5, 10) - ), - ] - ); - } - - #[tokio::test] - #[allow(clippy::too_many_lines)] - async fn query_contained_in_tile() { - let mrs1 = MockRasterSource { - params: MockRasterSourceParams:: { - data: vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![1, 2, 3, 4, 5, 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![7, 8, 9, 10, 11, 12]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_regular_with_epoch( - None, - TimeStep::millis(10).unwrap(), - ), - spatial_grid: SpatialGridDescriptor::source_from_parts( - GeoTransform::new((0., -3.).into(), 1., -1.), - GridShape2D::new_2d(3, 4).bounding_box(), - ), - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let mrs2 = MockRasterSource { - params: MockRasterSourceParams:: { - data: vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(2, 4), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![1, 2, 3, 4, 5, 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(2, 4), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![7, 8, 9, 10, 11, 12]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(4, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: EmptyGrid::new([3, 2].into()).into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(4, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: EmptyGrid::new([3, 2].into()).into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_irregular(None), - spatial_grid: SpatialGridDescriptor::source_from_parts( - GeoTransform::new((0., -3.).into(), 1., -1.), - GridShape2D::new_2d(3, 4).bounding_box(), - ), - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let exe_ctx = - MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); - let query_rect = RasterQueryRectangle::new( - GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), - TimeInterval::new_unchecked(2, 4), - BandSelection::first(), - ); - let query_ctx = exe_ctx.mock_query_context_test_default(); - - let qp1 = mrs1 - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap() - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - let qp2 = mrs2 - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap() - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - let source_a = QueryWrapper { - p: &qp1, - ctx: &query_ctx, - }; - - let source_b = QueryWrapper { - p: &qp2, - ctx: &query_ctx, - }; - - let adapter = RasterTimeAdapter::new(source_a, source_b, query_rect); - - let result = adapter - .map(Result::unwrap) - .collect::, RasterTile2D)>>() - .await; - - let times: Vec<_> = result.iter().map(|(a, b)| (a.time, b.time)).collect(); - assert_eq!( - ×, - &[ - // first spatial tile - ( - TimeInterval::new_unchecked(2, 4), - TimeInterval::new_unchecked(2, 4) - ), - // second spatial tile - ( - TimeInterval::new_unchecked(2, 4), - TimeInterval::new_unchecked(2, 4) - ), - ] - ); - } - - #[tokio::test] - #[allow(clippy::too_many_lines)] - async fn query_contained_in_tile_array() { - let no_data_value = 0; - let mrs1 = MockRasterSource { - params: MockRasterSourceParams:: { - data: vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![1, 2, 3, 4, 5, 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![7, 8, 9, 10, 11, 12]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_regular_with_epoch( - None, - TimeStep::millis(10).unwrap(), - ), - spatial_grid: SpatialGridDescriptor::source_from_parts( - GeoTransform::new((0., -3.).into(), 1., -1.), - GridShape2D::new_2d(3, 4).bounding_box(), - ), - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let mrs2 = MockRasterSource { - params: MockRasterSourceParams:: { - data: vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(2, 4), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![1, 2, 3, 4, 5, 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(2, 4), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![7, 8, 9, 10, 11, 12]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(4, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![no_data_value; 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(4, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![no_data_value; 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_irregular(None), - spatial_grid: SpatialGridDescriptor::source_from_parts( - GeoTransform::new((0., -3.).into(), 1., -1.), - GridShape2D::new_2d(3, 4).bounding_box(), - ), - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let exe_ctx = - MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); - let query_rect = RasterQueryRectangle::new( - GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), - TimeInterval::new_unchecked(2, 4), - BandSelection::first(), - ); - let query_ctx = exe_ctx.mock_query_context_test_default(); - - let qp1 = mrs1 - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap() - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - let qp2 = mrs2 - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap() - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - let source_a = QueryWrapper { - p: &qp1, - ctx: &query_ctx, - }; - - let source_b = QueryWrapper { - p: &qp2, - ctx: &query_ctx, - }; - - let adapter = RasterArrayTimeAdapter::new([source_a, source_b], query_rect); - - let result = adapter - .map(Result::unwrap) - .collect::; 2]>>() - .await; - - let times: Vec<_> = result.iter().map(|[a, b]| (a.time, b.time)).collect(); - assert_eq!( - ×, - &[ - // first spatial tile - ( - TimeInterval::new_unchecked(2, 4), - TimeInterval::new_unchecked(2, 4) - ), - // second spatial tile - ( - TimeInterval::new_unchecked(2, 4), - TimeInterval::new_unchecked(2, 4) - ), - ] - ); - } - - #[tokio::test] - #[allow(clippy::too_many_lines)] - async fn both_tiles_longer_valid_than_query() { - let mrs1 = MockRasterSource { - params: MockRasterSourceParams:: { - data: vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![1, 2, 3, 4, 5, 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![7, 8, 9, 10, 11, 12]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(10, 20), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: EmptyGrid::new([3, 2].into()).into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(10, 20), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: EmptyGrid::new([3, 2].into()).into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_regular_with_epoch( - None, - TimeStep::millis(10).unwrap(), - ), - spatial_grid: SpatialGridDescriptor::source_from_parts( - GeoTransform::new((0., -3.).into(), 1., -1.), - GridShape2D::new_2d(3, 4).bounding_box(), - ), - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let mrs2 = MockRasterSource { - params: MockRasterSourceParams:: { - data: vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(2, 9), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![1, 2, 3, 4, 5, 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(2, 9), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![7, 8, 9, 10, 11, 12]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(9, 20), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: EmptyGrid::new([3, 2].into()).into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(9, 20), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: EmptyGrid::new([3, 2].into()).into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_irregular(None), - spatial_grid: SpatialGridDescriptor::source_from_parts( - GeoTransform::new((0., -3.).into(), 1., -1.), - GridShape2D::new_2d(3, 4).bounding_box(), - ), - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let exe_ctx = - MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); - let query_rect = RasterQueryRectangle::new( - GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), - TimeInterval::new_unchecked(2, 8), - BandSelection::first(), - ); - let query_ctx = exe_ctx.mock_query_context_test_default(); - - let qp1 = mrs1 - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap() - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - let qp2 = mrs2 - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap() - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - let source_a = QueryWrapper { - p: &qp1, - ctx: &query_ctx, - }; - - let source_b = QueryWrapper { - p: &qp2, - ctx: &query_ctx, - }; - - let adapter = RasterTimeAdapter::new(source_a, source_b, query_rect); - - let result = adapter - .map(Result::unwrap) - .collect::, RasterTile2D)>>() - .await; - - let times: Vec<_> = result.iter().map(|(a, b)| (a.time, b.time)).collect(); - assert_eq!( - ×, - &[ - // first spatial tile - ( - TimeInterval::new_unchecked(2, 9), - TimeInterval::new_unchecked(2, 9) - ), - // second spatial tile - ( - TimeInterval::new_unchecked(2, 9), - TimeInterval::new_unchecked(2, 9) - ), - ] - ); - } - - #[tokio::test] - #[allow(clippy::too_many_lines)] - async fn array_tiles_longer_valid_than_query() { - let no_data_value = 0; - let mrs1 = MockRasterSource { - params: MockRasterSourceParams:: { - data: vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![1, 2, 3, 4, 5, 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![7, 8, 9, 10, 11, 12]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(10, 20), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![no_data_value; 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(10, 20), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![no_data_value; 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_regular_with_epoch( - None, - TimeStep::millis(10).unwrap(), - ), - spatial_grid: SpatialGridDescriptor::source_from_parts( - GeoTransform::new((0., -3.).into(), 1., -1.), - GridShape2D::new_2d(3, 4).bounding_box(), - ), - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let mrs2 = MockRasterSource { - params: MockRasterSourceParams:: { - data: vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(2, 9), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![1, 2, 3, 4, 5, 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(2, 9), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![7, 8, 9, 10, 11, 12]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(9, 20), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![no_data_value; 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(9, 20), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![no_data_value; 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_irregular(None), - spatial_grid: SpatialGridDescriptor::source_from_parts( - GeoTransform::new((0., -3.).into(), 1., -1.), - GridShape2D::new_2d(3, 4).bounding_box(), - ), - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let exe_ctx = - MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); - let query_rect = RasterQueryRectangle::new( - GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), - TimeInterval::new_unchecked(2, 8), - BandSelection::first(), - ); - let query_ctx = exe_ctx.mock_query_context_test_default(); - - let qp1 = mrs1 - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap() - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - let qp2 = mrs2 - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap() - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - let source_a = QueryWrapper { - p: &qp1, - ctx: &query_ctx, - }; - - let source_b = QueryWrapper { - p: &qp2, - ctx: &query_ctx, - }; - - let adapter = RasterArrayTimeAdapter::new([source_a, source_b], query_rect); - - let result = adapter - .map(Result::unwrap) - .collect::; 2]>>() - .await; - - let times: Vec<_> = result.iter().map(|[a, b]| (a.time, b.time)).collect(); - assert_eq!( - ×, - &[ - // first spatial tile - ( - TimeInterval::new_unchecked(2, 9), - TimeInterval::new_unchecked(2, 9) - ), - // second spatial tile - ( - TimeInterval::new_unchecked(2, 9), - TimeInterval::new_unchecked(2, 9) - ), - ] - ); - } - - #[tokio::test] - #[allow(clippy::too_many_lines)] - async fn array_tiles_without_lengths() { - // FIXME: this shuld be illegal, because the tiles have no length - let raster_source_a = MockRasterSource { - params: MockRasterSourceParams:: { - data: vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(1, 1), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![1, 2, 3, 4, 5, 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(1, 1), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![7, 8, 9, 10, 11, 12]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(2, 2), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![13, 14, 15, 16, 17, 18]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(2, 2), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![19, 20, 21, 22, 23, 24]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_irregular(None), - spatial_grid: SpatialGridDescriptor::source_from_parts( - GeoTransform::new((0., -3.).into(), 1., -1.), - GridShape2D::new_2d(3, 4).bounding_box(), - ), - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let raster_source_b = MockRasterSource { - params: MockRasterSourceParams:: { - data: vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![1, 2, 3, 4, 5, 6]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([3, 2].into(), vec![7, 8, 9, 10, 11, 12]) - .unwrap() - .into(), - properties: RasterProperties::default(), - cache_hint: CacheHint::default(), - }, - ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: TimeDescriptor::new_regular_with_epoch( - None, - TimeStep::millis(10).unwrap(), - ), - spatial_grid: SpatialGridDescriptor::source_from_parts( - GeoTransform::new((0., -3.).into(), 1., -1.), - GridShape2D::new_2d(3, 4).bounding_box(), - ), - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let exe_ctx = - MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); - let query_rect = RasterQueryRectangle::new( - GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), - TimeInterval::new_unchecked(1, 3), - BandSelection::first(), - ); - let query_ctx = exe_ctx.mock_query_context_test_default(); - - let query_processor_a = raster_source_a - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap() - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - let query_processor_b = raster_source_b - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap() - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - let source_a = QueryWrapper { - p: &query_processor_a, - ctx: &query_ctx, - }; - - let source_b = QueryWrapper { - p: &query_processor_b, - ctx: &query_ctx, - }; - - let adapter = RasterArrayTimeAdapter::new([source_a, source_b], query_rect); - - let result = adapter - .map(Result::unwrap) - .collect::; 2]>>() - .await; - - let times: Vec<_> = result.iter().map(|[a, b]| (a.time, b.time)).collect(); - assert_eq!( - times, - [ - ( - TimeInterval::new_unchecked(1, 1), - TimeInterval::new_unchecked(1, 1), - ), - ( - TimeInterval::new_unchecked(1, 1), - TimeInterval::new_unchecked(1, 1), - ), - ( - TimeInterval::new_unchecked(2, 2), - TimeInterval::new_unchecked(2, 2), - ), - ( - TimeInterval::new_unchecked(2, 2), - TimeInterval::new_unchecked(2, 2), - ), - ] - ); - } -} diff --git a/operators/src/adapters/simple_raster_stacker.rs b/operators/src/adapters/simple_raster_stacker.rs index 1951e7551..f9e42a1ba 100644 --- a/operators/src/adapters/simple_raster_stacker.rs +++ b/operators/src/adapters/simple_raster_stacker.rs @@ -1,14 +1,46 @@ -use crate::error::{AtLeastOneStreamRequired, Error}; +use crate::engine::RasterQueryProcessor; +use crate::error::Error; use crate::util::Result; use futures::future::join_all; use futures::stream::{BoxStream, Stream}; -use futures::{Future, ready}; +use futures::{TryFutureExt, ready}; use geoengine_datatypes::primitives::{BandSelection, RasterQueryRectangle, TimeInterval}; use geoengine_datatypes::raster::{Pixel, RasterTile2D}; use pin_project::pin_project; -use snafu::ensure; +use snafu::{Snafu, ensure}; use std::pin::Pin; use std::task::{Context, Poll}; +use strum::IntoStaticStr; + +#[derive(Debug, Snafu, IntoStaticStr)] +#[snafu(context(suffix(false)))] // disables default `Snafu` suffix +pub enum SimpleRasterStackerError { + #[snafu(display( + "BandSelection count exceeds band input count: {} > {}", + selection_bands, + source_bands + ))] + BandInputAndSelectionMissmatch { + source_bands: usize, + selection_bands: usize, + }, + AtLeastOneStreamRequired, + + InputsNotTemporalAligned, + + CreateSourceStreams { + source: Box, + }, + + CreateSingleBandStream { + source: Box, + }, + + UnAlignedElements { + t1: TimeInterval, + t2: TimeInterval, + }, +} /// Stacks the bands of the input raster streams to create a single raster stream with all the combined bands. /// @@ -19,47 +51,271 @@ use std::task::{Context, Poll}; pub struct SimpleRasterStackerAdapter { #[pin] streams: Vec, - batch_size_per_stream: Vec, - current_stream: usize, - current_stream_item: usize, + current_idx: usize, current_time: Option, finished: bool, + band_tracking: BandTracker, +} + +pub struct BandTracker { + source_band_idxs: Vec, + band_selection: BandSelection, +} + +impl BandTracker { + pub fn new( + source_bands: Vec, + out_bands: BandSelection, + ) -> Result { + ensure!(!source_bands.is_empty(), AtLeastOneStreamRequired); + ensure!(out_bands.count() > 0, AtLeastOneStreamRequired); + + let bt = Self { + source_band_idxs: source_bands, + band_selection: out_bands, + }; + + ensure!( + bt.num_total_source_bands() == bt.band_selection.as_slice().len(), + BandInputAndSelectionMissmatch { + selection_bands: bt.band_selection.as_slice().len(), + source_bands: bt.num_total_source_bands() + } + ); + + Ok(bt) + } + + pub fn num_total_source_bands(&self) -> usize { + self.source_band_idxs.iter().map(|s| s.as_vec().len()).sum() + } + + pub fn num_sources(&self) -> usize { + self.source_band_idxs.len() + } + + pub fn num_source_bands(&self, source: usize) -> usize { + self.source_band_idxs[source].as_slice().len() + } + + pub fn band_idx_to_source_band_idx(&self, band_idx: usize) -> (usize, usize) { + debug_assert!( + band_idx < self.num_total_source_bands(), + "Band must be in range of all source bands" + ); + + let mut starts_at = 0; + let mut source_idx = 0; + for (i, num_bands) in self + .source_band_idxs + .iter() + .map(|s| s.as_slice().len()) + .enumerate() + { + let next_starts_at = starts_at + num_bands; + if (starts_at..next_starts_at).contains(&band_idx) { + source_idx = i; + break; + } + starts_at = next_starts_at; + } + + let source_band_idx = band_idx - starts_at; + + debug_assert!( + source_idx < self.num_sources(), + "Source {source_idx} must be in range {0}", + self.num_sources() + ); + debug_assert!( + source_band_idx < self.num_source_bands(source_idx), + "Band {source_band_idx} must be in range of source bands {0}", + self.num_source_bands(source_idx) + ); + + (source_idx, source_band_idx) + } + + pub fn source_in_and_out_band(&self, band_idx: usize) -> (usize, u32, u32) { + let (source, in_band_idx) = self.band_idx_to_source_band_idx(band_idx); + let in_band = self.source_band_idxs[source].as_slice()[in_band_idx]; + let out_band = self.band_selection.as_slice()[band_idx]; + + (source, in_band, out_band) + } } pub struct SimpleRasterStackerSource { pub stream: S, - pub num_bands: usize, + pub band_idxs: BandSelection, } -impl From<(S, usize)> for SimpleRasterStackerSource { - fn from(value: (S, usize)) -> Self { - debug_assert!(value.1 > 0, "At least one band required"); - Self { +impl SimpleRasterStackerSource { + pub fn num_bands(&self) -> usize { + self.band_idxs.as_slice().len() + } +} + +impl> TryFrom<(S, I)> for SimpleRasterStackerSource { + type Error = I::Error; + + fn try_from(value: (S, I)) -> Result { + value.1.try_into().map(|bs| SimpleRasterStackerSource { stream: value.0, - num_bands: value.1, - } + band_idxs: bs, + }) } } impl SimpleRasterStackerAdapter { - pub fn new(streams: Vec>) -> Result { + pub fn new( + streams: Vec>, + out_bands: BandSelection, + ) -> Result { ensure!(!streams.is_empty(), AtLeastOneStreamRequired); + let (streams, bandsel): (Vec, Vec) = streams + .into_iter() + .map(|s| (s.stream, s.band_idxs)) + .collect(); + + let bt = BandTracker::new(bandsel, out_bands)?; + Ok(SimpleRasterStackerAdapter { - batch_size_per_stream: streams.iter().map(|s| s.num_bands).collect(), - streams: streams.into_iter().map(|s| s.stream).collect(), - current_stream: 0, - current_stream_item: 0, + streams, + current_idx: 0, current_time: None, finished: false, + band_tracking: bt, }) } + + /// A helper method that computes a function on multiple bands (that are already aligned) individually and then stacks the result into a multi-band raster. + pub async fn stack_individual_aligned_raster_bands<'a, F, Fut, P>( + query: &RasterQueryRectangle, + ctx: &'a dyn crate::engine::QueryContext, + create_single_bands_stream_fn: F, + ) -> Result, SimpleRasterStackerError> + where + S: Stream>> + Unpin, + F: Fn(RasterQueryRectangle, &'a dyn crate::engine::QueryContext) -> Fut, + Fut: Future>, + P: Pixel, + { + // compute the aggreation for each band separately and stack the streams to get a multi band raster tile stream + let band_streams = join_all(query.attributes().as_slice().iter().map(|band| { + let band_idx = BandSelection::new_single(*band); + let query = query.select_attributes(band_idx.clone()); + + async { + create_single_bands_stream_fn(query, ctx) + .await + .map_err(|e| SimpleRasterStackerError::CreateSingleBandStream { + source: Box::new(e), + }) + .map(|s| SimpleRasterStackerSource { + stream: s, + band_idxs: band_idx, + }) + } + })) + .await + .into_iter() + .collect::, _>>()?; + + SimpleRasterStackerAdapter::new(band_streams, query.attributes().clone()) + } + + /// This method queries multiple sources for the requested bands and stacks the results into a multi band raster tile stream. + /// The order of the bands is determined by the order of the sources. The query attributes are used to select the relevant bands from the sources. + pub async fn stack_selected_regular_aligned_raster_bands<'a, P, R>( + query: &RasterQueryRectangle, + ctx: &'a dyn crate::engine::QueryContext, + sources: &'a [R], + ) -> Result< + SimpleRasterStackerAdapter>>>, + SimpleRasterStackerError, + > + where + R: RasterQueryProcessor + 'a, + { + // check that inputs are temp aligned + let _regular_time = sources + .iter() + .map(|r| r.raster_result_descriptor().time.dimension.unwrap_regular()) + .reduce(|a, f| match (a, f) { + (Some(at), Some(ft)) if at.compatible_with(ft) => Some(at), + _ => None, + }) + .ok_or(SimpleRasterStackerError::AtLeastOneStreamRequired)? + .ok_or(SimpleRasterStackerError::InputsNotTemporalAligned)?; + + let bands_per_source = sources + .iter() + .map(|rq| rq.result_descriptor().bands.count()) + .collect::>(); + + let total_bands: u32 = bands_per_source.iter().sum(); + + let temp_bt = BandTracker::new( + bands_per_source + .iter() + .map(|&num_bands| BandSelection::first_n(num_bands)) + .collect(), + BandSelection::first_n(total_bands), + )?; + + // Build source bands mapping + let mut source_bands: Vec> = vec![Vec::new(); sources.len()]; + for out_band in query.attributes().as_slice() { + let (source_idx, in_band, check_out_band) = + temp_bt.source_in_and_out_band(*out_band as usize); + + debug_assert_eq!(check_out_band, *out_band, "Band tracking mismatch"); + + source_bands[source_idx].push(in_band); + } + + // Query relevant sources + let (futures, band_lists): (Vec<_>, Vec<_>) = (0..sources.len()) + .filter_map(|source_idx| { + if source_bands[source_idx].is_empty() { + return None; + } + let bands = source_bands[source_idx].clone(); + let band_selection = BandSelection::new_unchecked(bands.clone()); + Some(( + sources[source_idx] + .raster_query(query.select_attributes(band_selection), ctx) + .map_err(|e| SimpleRasterStackerError::CreateSourceStreams { + source: Box::new(e), + }), + bands, + )) + }) + .unzip(); + + // Await and construct sources + let stacker_sources = join_all(futures) + .await + .into_iter() + .zip(band_lists) + .map(|(stream_result, bands)| { + stream_result.map(|stream| SimpleRasterStackerSource { + stream, + band_idxs: BandSelection::new_unchecked(bands), + }) + }) + .collect::, _>>()?; + + SimpleRasterStackerAdapter::new(stacker_sources, query.attributes().clone()) + } } impl Stream for SimpleRasterStackerAdapter where S: Stream>> + Unpin, - T: Send + Sync, + T: Send + Sync + Pixel, { type Item = Result>; @@ -70,34 +326,40 @@ where let SimpleRasterStackerAdapterProjection { mut streams, - batch_size_per_stream, - current_stream, - current_stream_item, + current_idx, current_time, - finished: _, + finished, + band_tracking, } = self.as_mut().project(); - let stream = &mut streams[*current_stream]; + let (stream_idx, in_band, out_band) = band_tracking.source_in_and_out_band(*current_idx); + let stream = &mut streams[stream_idx]; let item = ready!(Pin::new(stream).poll_next(cx)); let Some(mut item) = item else { // if one input stream ends, end the output stream + *finished = true; return Poll::Ready(None); }; if let Ok(tile) = item.as_mut() { - // compute output band number from its place among all bands of all inputs - let band = batch_size_per_stream - .iter() - .take(*current_stream) - .sum::() - + *current_stream_item; - tile.band = band as u32; + debug_assert!( + tile.band == in_band, + "Expected stream {stream_idx} band {in_band} got {0} to produce out band {out_band}", + tile.band + ); + + tile.band = out_band; - // TODO: replace time check with temporal alignment if let Some(time) = current_time { - if band == 0 { + if *current_idx == 0 { + debug_assert!( + *time == tile.time || time.end() == tile.time.start(), + "Time hole discovered! Time of last tile: {time}, time of current tile: {0}. Stream index: {stream_idx}, In band: {in_band}, Out band: {out_band}", + tile.time + ); + // save the first bands time *current_time = Some(tile.time); } else { @@ -105,7 +367,7 @@ where if tile.time != *time { return Poll::Ready(Some(Err( Error::InputStreamsMustBeTemporallyAligned { - stream_index: *current_stream, + stream_index: stream_idx, expected: *time, found: tile.time, }, @@ -119,50 +381,15 @@ where } // next item in stream, or go to next stream - *current_stream_item += 1; - if *current_stream_item >= batch_size_per_stream[*current_stream] { - *current_stream_item = 0; - *current_stream = (*current_stream + 1) % streams.len(); + *current_idx += 1; + if *current_idx >= band_tracking.num_total_source_bands() { + *current_idx = 0; } Poll::Ready(Some(item)) } } -/// A helper method that computes a function on multiple bands (that are already aligned) individually and then stacks the result into a multi-band raster. -pub async fn stack_individual_aligned_raster_bands<'a, F, Fut, P>( - query: &RasterQueryRectangle, - ctx: &'a dyn crate::engine::QueryContext, - create_single_bands_stream_fn: F, -) -> Result>>> -where - F: Fn(RasterQueryRectangle, &'a dyn crate::engine::QueryContext) -> Fut, - Fut: Future>>>>, - P: Pixel, -{ - if query.attributes().count() == 1 { - // special case of single band query requires no tile stacking - return create_single_bands_stream_fn(query.clone(), ctx).await; - } - - // compute the aggreation for each band separately and stack the streams to get a multi band raster tile stream - let band_streams = join_all(query.attributes().as_slice().iter().map(|band| { - let query = query.select_attributes(BandSelection::new_single(*band)); - - async { - Ok(SimpleRasterStackerSource { - stream: create_single_bands_stream_fn(query, ctx).await?, - num_bands: 1, - }) - } - })) - .await - .into_iter() - .collect::>>()?; - - Ok(Box::pin(SimpleRasterStackerAdapter::new(band_streams)?)) -} - #[cfg(test)] mod tests { use futures::{StreamExt, stream}; @@ -174,6 +401,87 @@ mod tests { use super::*; + #[test] + fn band_tracker_works() { + let bt = BandTracker::new( + vec![BandSelection::first_n(2), BandSelection::first_n(3)], + BandSelection::first_n(5), + ) + .unwrap(); + + assert_eq!(bt.num_total_source_bands(), 5); + assert_eq!(bt.num_sources(), 2); + assert_eq!(bt.num_source_bands(0), 2); + assert_eq!(bt.num_source_bands(1), 3); + + assert_eq!(bt.band_idx_to_source_band_idx(0), (0, 0)); + assert_eq!(bt.band_idx_to_source_band_idx(1), (0, 1)); + assert_eq!(bt.band_idx_to_source_band_idx(2), (1, 0)); + assert_eq!(bt.band_idx_to_source_band_idx(3), (1, 1)); + assert_eq!(bt.band_idx_to_source_band_idx(4), (1, 2)); + + assert_eq!(bt.source_in_and_out_band(0), (0, 0, 0)); + assert_eq!(bt.source_in_and_out_band(1), (0, 1, 1)); + assert_eq!(bt.source_in_and_out_band(2), (1, 0, 2)); + assert_eq!(bt.source_in_and_out_band(3), (1, 1, 3)); + assert_eq!(bt.source_in_and_out_band(4), (1, 2, 4)); + } + + #[test] + fn band_tracker_with_gaps_works() { + let bt = BandTracker::new( + vec![BandSelection::first_n(2), BandSelection::first_n(3)], + BandSelection::new(vec![0, 2, 4, 6, 8]).unwrap(), + ) + .unwrap(); + + assert_eq!(bt.num_total_source_bands(), 5); + assert_eq!(bt.num_sources(), 2); + assert_eq!(bt.num_source_bands(0), 2); + assert_eq!(bt.num_source_bands(1), 3); + + assert_eq!(bt.band_idx_to_source_band_idx(0), (0, 0)); + assert_eq!(bt.band_idx_to_source_band_idx(1), (0, 1)); + assert_eq!(bt.band_idx_to_source_band_idx(2), (1, 0)); + assert_eq!(bt.band_idx_to_source_band_idx(3), (1, 1)); + assert_eq!(bt.band_idx_to_source_band_idx(4), (1, 2)); + + assert_eq!(bt.source_in_and_out_band(0), (0, 0, 0)); + assert_eq!(bt.source_in_and_out_band(1), (0, 1, 2)); + assert_eq!(bt.source_in_and_out_band(2), (1, 0, 4)); + assert_eq!(bt.source_in_and_out_band(3), (1, 1, 6)); + assert_eq!(bt.source_in_and_out_band(4), (1, 2, 8)); + } + + #[test] + fn band_tracker_with_input_gaps_works() { + let bt = BandTracker::new( + vec![ + BandSelection::new(vec![0, 2]).unwrap(), + BandSelection::new(vec![1, 3, 5]).unwrap(), + ], + BandSelection::first_n(5), + ) + .unwrap(); + + assert_eq!(bt.num_total_source_bands(), 5); + assert_eq!(bt.num_sources(), 2); + assert_eq!(bt.num_source_bands(0), 2); + assert_eq!(bt.num_source_bands(1), 3); + + assert_eq!(bt.band_idx_to_source_band_idx(0), (0, 0)); + assert_eq!(bt.band_idx_to_source_band_idx(1), (0, 1)); + assert_eq!(bt.band_idx_to_source_band_idx(2), (1, 0)); + assert_eq!(bt.band_idx_to_source_band_idx(3), (1, 1)); + assert_eq!(bt.band_idx_to_source_band_idx(4), (1, 2)); + + assert_eq!(bt.source_in_and_out_band(0), (0, 0, 0)); + assert_eq!(bt.source_in_and_out_band(1), (0, 2, 1)); + assert_eq!(bt.source_in_and_out_band(2), (1, 1, 2)); + assert_eq!(bt.source_in_and_out_band(3), (1, 3, 3)); + assert_eq!(bt.source_in_and_out_band(4), (1, 5, 4)); + } + #[tokio::test] #[allow(clippy::too_many_lines)] async fn it_stacks() { @@ -268,8 +576,14 @@ mod tests { let stream = stream::iter(data.clone().into_iter().map(Result::Ok)).boxed(); let stream2 = stream::iter(data2.clone().into_iter().map(Result::Ok)).boxed(); - let stacker = - SimpleRasterStackerAdapter::new(vec![(stream, 1).into(), (stream2, 1).into()]).unwrap(); + let stacker = SimpleRasterStackerAdapter::new( + vec![ + (stream, 0).try_into().unwrap(), + (stream2, 0).try_into().unwrap(), + ], + BandSelection::first_n(2), + ) + .unwrap(); let result = stacker.collect::>().await; let result = result.into_iter().collect::>>().unwrap(); @@ -462,8 +776,14 @@ mod tests { let stream = stream::iter(data.clone().into_iter().map(Result::Ok)).boxed(); let stream2 = stream::iter(data2.clone().into_iter().map(Result::Ok)).boxed(); - let stacker = - SimpleRasterStackerAdapter::new(vec![(stream, 2).into(), (stream2, 2).into()]).unwrap(); + let stacker = SimpleRasterStackerAdapter::new( + vec![ + (stream, [0, 1]).try_into().unwrap(), + (stream2, [0, 1]).try_into().unwrap(), + ], + BandSelection::first_n(4), + ) + .unwrap(); let result = stacker.collect::>().await; let result = result.into_iter().collect::>>().unwrap(); @@ -514,8 +834,14 @@ mod tests { let stream = stream::iter(data.clone().into_iter().map(Result::Ok)).boxed(); let stream2 = stream::iter(data2.clone().into_iter().map(Result::Ok)).boxed(); - let stacker = - SimpleRasterStackerAdapter::new(vec![(stream, 1).into(), (stream2, 1).into()]).unwrap(); + let stacker = SimpleRasterStackerAdapter::new( + vec![ + (stream, 0).try_into().unwrap(), + (stream2, 0).try_into().unwrap(), + ], + BandSelection::first_n(2), + ) + .unwrap(); let result = stacker.collect::>().await; diff --git a/operators/src/adapters/sparse_tiles_fill_adapter.rs b/operators/src/adapters/sparse_tiles_fill_adapter.rs deleted file mode 100644 index d1e2747f1..000000000 --- a/operators/src/adapters/sparse_tiles_fill_adapter.rs +++ /dev/null @@ -1,2513 +0,0 @@ -use crate::util::Result; -use futures::{Stream, ready}; -use geoengine_datatypes::{ - primitives::{CacheExpiration, CacheHint, TimeInstance, TimeInterval}, - raster::{ - EmptyGrid2D, GeoTransform, GridBoundingBox2D, GridBounds, GridIdx2D, GridShape2D, GridStep, - Pixel, RasterTile2D, - }, -}; -use pin_project::pin_project; -use snafu::Snafu; -use std::{pin::Pin, task::Poll}; - -#[derive(Debug, Snafu)] -pub enum SparseTilesFillAdapterError { - #[snafu(display( - "Received tile TimeInterval ({}) starts before current TimeInterval: {}. This is probably a bug in a child operator.", - tile_start, - current_start - ))] - TileTimeIntervalStartBeforeCurrentStart { - tile_start: TimeInterval, - current_start: TimeInterval, - }, - #[snafu(display( - "Received tile TimeInterval ({}) length differs from received TimeInterval with equal start: {}. This is probably a bug in a child operator.", - tile_interval, - current_interval - ))] - TileTimeIntervalLengthMissmatch { - tile_interval: TimeInterval, - current_interval: TimeInterval, - }, - #[snafu(display( - "Received tile TimeInterval ({}) is the first in a grid run but does no time progress. (Old time: {}). This is probably a bug in a child operator.", - tile_interval, - current_interval - ))] - GridWrapMustDoTimeProgress { - tile_interval: TimeInterval, - current_interval: TimeInterval, - }, -} - -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -enum State { - Initial, - PollingForNextTile, - FillAndProduceNextTile, - FillToEnd, - Ended, -} - -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub struct FillerTimeBounds { - start: TimeInstance, - end: TimeInstance, -} - -impl From for FillerTimeBounds { - fn from(time: TimeInterval) -> FillerTimeBounds { - FillerTimeBounds { - start: time.start(), - end: time.end(), - } - } -} - -impl FillerTimeBounds { - fn start(&self) -> TimeInstance { - self.start - } - fn end(&self) -> TimeInstance { - self.end - } - - pub fn new(start: TimeInstance, end: TimeInstance) -> Self { - Self::new_unchecked(start, end) - } - - pub fn new_unchecked(start: TimeInstance, end: TimeInstance) -> Self { - Self { start, end } - } -} - -#[derive(Debug, PartialEq, Clone)] -struct StateContainer { - current_idx: GridIdx2D, - current_band_idx: u32, - current_time: Option, - next_tile: Option>, - no_data_grid: EmptyGrid2D, - grid_bounds: GridBoundingBox2D, - num_bands: u32, - global_geo_transform: GeoTransform, - state: State, - cache_hint: FillerTileCacheHintProvider, - requested_time_bounds: TimeInterval, - data_time_bounds: FillerTimeBounds, -} - -struct GridIdxAndBand { - idx: GridIdx2D, - band_idx: u32, -} - -impl StateContainer { - /// Create a new no-data `RasterTile2D` with `GridIdx` and time from the current state - fn current_no_data_tile(&self) -> RasterTile2D { - RasterTile2D::new( - self.current_time - .expect("time must exist when a tile is stored."), - self.current_idx, - self.current_band_idx, - self.global_geo_transform, - self.no_data_grid.into(), - self.cache_hint.next_hint(), - ) - } - - /// Check if the next tile to produce is the stored one - fn stored_tile_is_next(&self) -> bool { - if let Some(t) = &self.next_tile { - // The stored tile is the one we are looking for if the tile position is the next to produce - self.grid_idx_and_band_is_the_next_to_produce(t.tile_position, t.band) && - // and the time equals the current state time - (self.time_equals_current_state(t.time) - // or it starts a new time step, and the time directly follows the current state time - || (self.current_idx_and_band_is_first_in_grid_run() && self.time_directly_following_current_state(t.time))) - } else { - false - } - } - - /// Get the next `GridIdxAndBand` following the current state `GridIdx` and band. None if the current `GridIdx` is the max `GridIdx` of the grid. - fn maybe_next_idx_and_band(&self) -> Option { - if self.current_band_idx + 1 < self.num_bands { - // next band - Some(GridIdxAndBand { - idx: self.current_idx, - band_idx: self.current_band_idx + 1, - }) - } else { - // next tile - self.grid_bounds - .inc_idx_unchecked(self.current_idx, 1) - .map(|idx| GridIdxAndBand { idx, band_idx: 0 }) - } - } - - /// Get the next `GridIdxAndBand` following the current state `GridIdx` and band. Returns the minimal `GridIdx` if the current `GridIdx` is the max `GridIdx`. - fn wrapped_next_idx_and_band(&self) -> GridIdxAndBand { - if self.current_band_idx + 1 < self.num_bands { - // next band - GridIdxAndBand { - idx: self.current_idx, - band_idx: self.current_band_idx + 1, - } - } else { - // next tile - GridIdxAndBand { - idx: self - .grid_bounds - .inc_idx_unchecked(self.current_idx, 1) - .unwrap_or_else(|| self.min_index()), - band_idx: 0, - } - } - } - - /// Get the minimal `GridIdx` of the grid. - fn min_index(&self) -> GridIdx2D { - self.grid_bounds.min_index() - } - - /// Get the maximal `GridIdx` of the grid. - fn max_index(&self) -> GridIdx2D { - self.grid_bounds.max_index() - } - - /// Check if any tile is stored in the state. - fn is_any_tile_stored(&self) -> bool { - self.next_tile.is_some() - } - - /// Check if a `TimeInterval` starts before the current state `TimeInterval`. - fn time_starts_before_current_state(&self, time: TimeInterval) -> bool { - time.start() - < self - .current_time - .expect("state time is set on initialize") - .start() - } - - /// Check if a `TimeInterval` start equals the start of the current state `TimeInterval`. - fn time_starts_equals_current_state(&self, time: TimeInterval) -> bool { - time.start() - == self - .current_time - .expect("state time is set on initialize") - .start() - } - - /// Check if a `TimeInterval` duration equals the duration of the current state `TimeInterval`. - fn time_duration_equals_current_state(&self, time: TimeInterval) -> bool { - time.duration_ms() - == self - .current_time - .expect("state time is set on initialize") - .duration_ms() - } - - /// Check if a `TimeInterval` equals the current state `TimeInterval`. - fn time_equals_current_state(&self, time: TimeInterval) -> bool { - time == self.current_time.expect("state time is set on initialize") - } - - /// Check if a `GridIdx` is the next to produce i.e. the current state `GridIdx`. - fn grid_idx_and_band_is_the_next_to_produce(&self, tile_idx: GridIdx2D, band_idx: u32) -> bool { - tile_idx == self.current_idx && band_idx == self.current_band_idx - } - - /// Check if a `TimeInterval` is directly connected to the end of the current state `TimeInterval`. - fn time_directly_following_current_state(&self, time: TimeInterval) -> bool { - let current_time = self.current_time.expect("state time is set on initialize"); - if current_time.is_instant() { - current_time.end() + 1 == time.start() - } else { - current_time.end() == time.start() - } - } - - fn next_time_interval_from_stored_tile(&self) -> Option { - // we wrapped around. We need to do time progress. - if let Some(tile) = &self.next_tile { - let stored_tile_time = tile.time; - - if self.time_directly_following_current_state(stored_tile_time) { - Some(stored_tile_time) - } else if let Some(current_time) = self.current_time { - // we need to fill a time gap - if current_time.is_instant() { - TimeInterval::new(current_time.end() + 1, stored_tile_time.start()).ok() - } else { - TimeInterval::new(current_time.end(), stored_tile_time.start()).ok() - } - } else { - None - } - } else { - None - } - } - - /// Check if the current state `GridIdx` is the first of a grid run i.e. it equals the minimal `GridIdx` . - fn current_idx_and_band_is_first_in_grid_run(&self) -> bool { - self.current_idx == self.min_index() && self.current_band_idx == 0 - } - - /// Check if the current state `GridIdx` is the first of a grid run i.e. it equals the minimal `GridIdx` . - fn current_idx_and_band_is_last_in_grid_run(&self) -> bool { - self.current_idx == self.max_index() && self.current_band_idx == self.num_bands - 1 - } - - fn set_current_time_from_initial_tile(&mut self, first_tile_time: TimeInterval) { - // if we know a bound we must use it to set the current time - let start_data_bound = self.data_time_bounds.start(); - let requested_start = self.requested_time_bounds.start(); - - debug_assert!( - start_data_bound <= requested_start, - "The data bound hint start should be <= the requested start. " - ); - - if requested_start < first_tile_time.start() { - tracing::debug!( - "The initial tile starts ({}) after the requested start bound ({}), setting the current time to the data start bound ({}) --> filling", - first_tile_time.start(), - requested_start, - start_data_bound - ); - self.current_time = Some(TimeInterval::new_unchecked( - start_data_bound, - first_tile_time.start(), - )); - return; - } - if start_data_bound > first_tile_time.start() { - tracing::debug!( - "The initial tile time start ({}) is before the exprected time bounds ({}). This means the data overflows the filler start bound.", - first_tile_time.start(), - start_data_bound - ); - } - self.current_time = Some(first_tile_time); - } - - fn set_current_time_from_data_time_bounds(&mut self) { - assert!(self.state == State::FillToEnd); - self.current_time = Some(TimeInterval::new_unchecked( - self.data_time_bounds.start(), - self.data_time_bounds.end(), - )); - } - - fn update_current_time(&mut self, new_time: TimeInterval) { - debug_assert!( - !new_time.is_instant(), - "Tile time is the data validity and must not be an instant!" - ); - - if let Some(old_time) = self.current_time { - if old_time == new_time { - return; - } - - debug_assert!( - old_time.end() <= new_time.start(), - "Time progress must be positive" - ); - - debug_assert!( - !old_time.is_instant() || old_time.end() < new_time.start(), - "Instant progress must be > 1" - ); - } - self.current_time = Some(new_time); - } - - fn current_time_fill_to_end_interval(&mut self) { - let current_time: TimeInterval = self - .current_time - .expect("current time must exist for fill to end state."); - debug_assert!(current_time.end() <= self.requested_time_bounds.end()); - - debug_assert!(current_time.end() < self.data_time_bounds.end()); - - let new_time = if current_time.is_instant() { - TimeInterval::new_unchecked(current_time.end() + 1, self.data_time_bounds.end()) - } else { - TimeInterval::new_unchecked(current_time.end(), self.data_time_bounds.end()) - }; - self.update_current_time(new_time); - } - - fn current_time_is_valid_end_bound(&self) -> bool { - let time_requested_end = self.requested_time_bounds.end(); - let time_bounds_end = self.data_time_bounds.end(); - let current_time = self.current_time.expect("state time is set on initialize"); - - if current_time.end() < time_requested_end { - return false; - } - if current_time.end() > time_bounds_end { - tracing::debug!( - "The current time end ({}) is after the exprected time bounds ({}). This means the data overflows the filler end bound.", - current_time.end(), - time_bounds_end - ); - } - - true - } - - fn store_tile(&mut self, tile: RasterTile2D) { - debug_assert!(self.next_tile.is_none()); - let current_time = self - .current_time - .expect("Time must be set when the first tile arrives"); - debug_assert!(current_time.start() <= tile.time.start()); - debug_assert!( - current_time.start() < tile.time.start() - || (self.current_idx.y() < tile.tile_position.y() - || (self.current_idx.y() == tile.tile_position.y() - && self.current_idx.x() < tile.tile_position.x())) - ); - self.next_tile = Some(tile); - } -} - -#[pin_project(project=SparseTilesFillAdapterProjection)] -pub struct SparseTilesFillAdapter { - #[pin] - stream: S, - - sc: StateContainer, -} - -impl SparseTilesFillAdapter -where - T: Pixel, - S: Stream>>, -{ - #[allow(clippy::too_many_arguments)] - pub fn new( - stream: S, - tile_grid_bounds: GridBoundingBox2D, - num_bands: u32, - global_geo_transform: GeoTransform, - tile_shape: GridShape2D, - cache_expiration: FillerTileCacheExpirationStrategy, // Specifies the cache expiration for the produced filler tiles. Set this to unlimited if the filler tiles will always be empty - requested_time_bounds: TimeInterval, - data_time_bounds: FillerTimeBounds, - ) -> Self { - debug_assert!( - data_time_bounds.start <= requested_time_bounds.start(), - "Data time bounds hint start should be <= requested time start." - ); - debug_assert!( - requested_time_bounds.end() <= data_time_bounds.end, - "Data time bounds hint end should be >= requested time end." - ); - - SparseTilesFillAdapter { - stream, - sc: StateContainer { - current_idx: tile_grid_bounds.min_index(), - current_band_idx: 0, - current_time: None, - global_geo_transform, - grid_bounds: tile_grid_bounds, - num_bands, - next_tile: None, - no_data_grid: EmptyGrid2D::new(tile_shape), - state: State::Initial, - cache_hint: cache_expiration.into(), - requested_time_bounds, - data_time_bounds, - }, - } - } - - // TODO: return Result with SparseTilesFillAdapterError and map it to Error in the poll_next method if possible - #[allow(clippy::too_many_lines, clippy::missing_panics_doc)] - pub fn next_step( - self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll>>> { - // TODO: we should check for unexpected bands in the input stream (like we do for time and space). - - let min_idx = self.sc.min_index(); - let wrapped_next = self.sc.wrapped_next_idx_and_band(); - let wrapped_next_idx = wrapped_next.idx; - let wrapped_next_band = wrapped_next.band_idx; - - let mut this = self.project(); - - match this.sc.state { - // this is the initial state. - State::Initial => { - // there must not be a grid stored - debug_assert!(!this.sc.is_any_tile_stored()); - // poll for a first (input) tile - let result_tile = match ready!(this.stream.as_mut().poll_next(cx)) { - Some(Ok(tile)) => { - // now we have to inspect the time we got and the bound we need to fill. If there are bounds known, then we need to check if the tile starts with the bounds. - this.sc.set_current_time_from_initial_tile(tile.time); - - this.sc - .cache_hint - .update_cache_expiration(tile.cache_hint.expires()); // save the expiration date to be used for the filler tiles, depending on the expiration strategy - - if this.sc.time_equals_current_state(tile.time) - && this.sc.grid_idx_and_band_is_the_next_to_produce( - tile.tile_position, - tile.band, - ) - { - this.sc.state = State::PollingForNextTile; // return the received tile and set state to polling for the next tile - tile - } else { - this.sc.store_tile(tile); - this.sc.state = State::FillAndProduceNextTile; // save the tile and go to fill mode - this.sc.current_no_data_tile() - } - } - // an error ouccured, stop producing anything and return the error. - Some(Err(e)) => { - this.sc.state = State::Ended; - return Poll::Ready(Some(Err(e))); - } - // the source never produced a tile. - None => { - debug_assert!(this.sc.current_idx == min_idx); - this.sc.state = State::FillToEnd; - this.sc.set_current_time_from_data_time_bounds(); - this.sc.current_no_data_tile() - } - }; - // move the current_idx. There is no need to do time progress here. Either a new tile triggers that or it is never needed for an empty source. - this.sc.current_idx = wrapped_next_idx; - this.sc.current_band_idx = wrapped_next_band; - Poll::Ready(Some(Ok(result_tile))) - } - // this is the state where we are waiting for the next tile to arrive. - State::PollingForNextTile => { - // there must not be a tile stored - debug_assert!(!this.sc.is_any_tile_stored()); - - // There are four cases we have to handle: - // 1. The received tile has a TimeInterval that starts before the current tile. This must not happen and is probably a bug in the source. - // 2. The received tile has a TimeInterval that matches the TimeInterval of the state. This TimeInterval is currently produced. - // 3. The received tile has a TimeInterval that directly continues the current TimeInterval. - // This TimeInterval will be produced once all tiles of the current grid have been produced. - // 4. The received tile has a TimeInterval that starts after the current TimeInterval and is not directly connected to the current TimeInterval. - // Before this TimeInterval is produced, an intermediate TimeInterval is produced. - - let res = match ready!(this.stream.as_mut().poll_next(cx)) { - Some(Ok(tile)) => { - // 1. The start of the recieved TimeInterval MUST NOT BE before the start of the current TimeInterval. - if this.sc.time_starts_before_current_state(tile.time) { - this.sc.state = State::Ended; - return Poll::Ready(Some(Err( - SparseTilesFillAdapterError::TileTimeIntervalStartBeforeCurrentStart { - current_start: this.sc.current_time.expect("time is set in initial step"), - tile_start: tile.time, - } - .into(), - ))); - } - if tile.time.start() >= this.sc.requested_time_bounds.end() { - tracing::warn!( - "The tile time start ({}) is outside of the requested time bounds ({})!", - tile.time.start(), - this.sc.requested_time_bounds.end() - ); - } - - // 1. b) This is a new grid run but the time is not increased - if this.sc.current_idx_and_band_is_first_in_grid_run() - && this.sc.time_equals_current_state(tile.time) - { - this.sc.state = State::Ended; - return Poll::Ready(Some(Err( - SparseTilesFillAdapterError::GridWrapMustDoTimeProgress { - current_interval: this - .sc - .current_time - .expect("time is set in initial step"), - tile_interval: tile.time, - } - .into(), - ))); - } - - // 2. a) The received TimeInterval with start EQUAL to the current TimeInterval MUST NOT have a different duration / end. - if this.sc.time_starts_equals_current_state(tile.time) - && !this.sc.time_duration_equals_current_state(tile.time) - { - this.sc.state = State::Ended; - return Poll::Ready(Some(Err( - SparseTilesFillAdapterError::TileTimeIntervalLengthMissmatch { - tile_interval: tile.time, - current_interval: this - .sc - .current_time - .expect("time is set in initial step"), - } - .into(), - ))); - } - - this.sc - .cache_hint - .update_cache_expiration(tile.cache_hint.expires()); // save the expiration date to be used for the filler tiles, depending on the expiration strategy - - // 2 b) The received TimeInterval with start EQUAL to the current TimeInterval MUST NOT have a different duration / end. - let next_tile = if this.sc.time_equals_current_state(tile.time) { - if this.sc.grid_idx_and_band_is_the_next_to_produce( - tile.tile_position, - tile.band, - ) { - // the tile is the next to produce. Return it and set state to polling for the next tile. - this.sc.state = State::PollingForNextTile; - tile - } else { - // the tile is not the next to produce. Save it and go to fill mode. - this.sc.store_tile(tile); - this.sc.state = State::FillAndProduceNextTile; - this.sc.current_no_data_tile() - } - } - // 3. The received tile has a TimeInterval that directly continues the current TimeInterval. - else if this.sc.time_directly_following_current_state(tile.time) { - // if the current_idx is the first in a new grid run then it is the first one with the new TimeInterval. - // this switches the time in the state to the time of the new tile. - if this.sc.current_idx_and_band_is_first_in_grid_run() { - if this.sc.grid_idx_and_band_is_the_next_to_produce( - tile.tile_position, - tile.band, - ) { - // return the tile and set state to polling for the next tile. - this.sc.update_current_time(tile.time); - this.sc.state = State::PollingForNextTile; - tile - } else { - // save the tile and go to fill mode. - this.sc.update_current_time(tile.time); - this.sc.store_tile(tile); - this.sc.state = State::FillAndProduceNextTile; - this.sc.current_no_data_tile() - } - } else { - // the received tile is in a new TimeInterval but we still need to finish the current one. Store tile and go to fill mode. - this.sc.store_tile(tile); - this.sc.state = State::FillAndProduceNextTile; - this.sc.current_no_data_tile() - } - } - // 4. The received tile has a TimeInterval that starts after the current TimeInterval and is not directly connected to the current TimeInterval. - else { - // if the current_idx is the first in a new grid run then it is the first one with a new TimeInterval. - // We need to generate a fill TimeInterval since current and tile TimeInterval are not connected. - if this.sc.current_idx_and_band_is_first_in_grid_run() { - this.sc.update_current_time(TimeInterval::new( - this.sc - .current_time - .expect("time is set in initial state") - .end(), - tile.time.start(), - )?); - this.sc.store_tile(tile); - this.sc.state = State::FillAndProduceNextTile; - this.sc.current_no_data_tile() - } else { - // the received tile is in a new TimeInterval but we still need to finish the current one. Store tile and go to fill mode. - this.sc.store_tile(tile); - this.sc.state = State::FillAndProduceNextTile; - this.sc.current_no_data_tile() - } - }; - Some(Ok(next_tile)) - } - - // an error ouccured, stop producing anything and return the error. - Some(Err(e)) => { - this.sc.state = State::Ended; - return Poll::Ready(Some(Err(e))); - } - // the source is empty (now). Remember that. - None => { - if this.sc.current_idx_and_band_is_first_in_grid_run() - && this.sc.current_time_is_valid_end_bound() - { - // there was a tile and it flipped the state index to the first one. => we are done. - this.sc.state = State::Ended; - None - } else if this.sc.current_idx_and_band_is_last_in_grid_run() - && this.sc.current_time_is_valid_end_bound() - { - // this is the last tile - this.sc.state = State::Ended; - Some(Ok(this.sc.current_no_data_tile())) - } else if this.sc.current_idx_and_band_is_last_in_grid_run() { - // there was a tile and it was the last one but it does not cover the time bounds. => go to fill to end mode. - this.sc.state = State::FillToEnd; - let no_data_tile = this.sc.current_no_data_tile(); - this.sc.current_time_fill_to_end_interval(); - Some(Ok(no_data_tile)) - } else if this.sc.current_idx_and_band_is_first_in_grid_run() { - // there was a tile and it was the last one but it does not cover the time bounds. => go to fill to end mode. - this.sc.state = State::FillToEnd; - this.sc.current_time_fill_to_end_interval(); - Some(Ok(this.sc.current_no_data_tile())) - } else { - // there was a tile and it was not the last one. => go to fill to end mode. - this.sc.state = State::FillToEnd; - Some(Ok(this.sc.current_no_data_tile())) - } - } - }; - // move the current_idx and band. There is no need to do time progress here. Either a new tile sets that or it is not needed to fill to the end of the grid. - - this.sc.current_idx = wrapped_next_idx; - this.sc.current_band_idx = wrapped_next_band; - - if this.sc.current_idx_and_band_is_first_in_grid_run() - && let Some(next_time) = this.sc.next_time_interval_from_stored_tile() - { - this.sc.update_current_time(next_time); - } - - Poll::Ready(res) - } - // the tile to produce is the the one stored - State::FillAndProduceNextTile if this.sc.stored_tile_is_next() => { - // take the tile (replace in state with NONE) - let next_tile = this.sc.next_tile.take().expect("checked by case"); - debug_assert!(this.sc.current_idx == next_tile.tile_position); - - this.sc.update_current_time(next_tile.time); - this.sc.current_idx = wrapped_next_idx; - this.sc.current_band_idx = wrapped_next_band; - this.sc.state = State::PollingForNextTile; - - Poll::Ready(Some(Ok(next_tile))) - } - // this is the state where we produce fill tiles until another state is reached. - State::FillAndProduceNextTile => { - let (next_idx, next_band, next_time) = match this.sc.maybe_next_idx_and_band() { - // the next GridIdx is in the current TimeInterval - Some(idx) => ( - idx.idx, - idx.band_idx, - this.sc.current_time.expect("time is set by initial step"), - ), - // the next GridIdx is in the next TimeInterval - None => ( - this.sc.min_index(), - 0, - this.sc - .next_time_interval_from_stored_tile() - .expect("there must be a way to determine the new time"), - ), - }; - - let no_data_tile = this.sc.current_no_data_tile(); - - this.sc.update_current_time(next_time); - this.sc.current_idx = next_idx; - this.sc.current_band_idx = next_band; - - Poll::Ready(Some(Ok(no_data_tile))) - } - // this is the last tile to produce ever - State::FillToEnd - if this.sc.current_idx_and_band_is_last_in_grid_run() - && this.sc.current_time_is_valid_end_bound() => - { - this.sc.state = State::Ended; - Poll::Ready(Some(Ok(this.sc.current_no_data_tile()))) - } - State::FillToEnd if this.sc.current_idx_and_band_is_last_in_grid_run() => { - // we have reached the end of the time interval but the bounds are not covered yet. We need to create a new intermediate TimeInterval. - let no_data_tile = this.sc.current_no_data_tile(); - this.sc.current_idx = wrapped_next_idx; - this.sc.current_band_idx = wrapped_next_band; - this.sc.current_time_fill_to_end_interval(); - Poll::Ready(Some(Ok(no_data_tile))) - } - - // there are more tiles to produce to fill the grid in this time step - State::FillToEnd => { - let no_data_tile = this.sc.current_no_data_tile(); - this.sc.current_idx = wrapped_next_idx; - this.sc.current_band_idx = wrapped_next_band; - Poll::Ready(Some(Ok(no_data_tile))) - } - State::Ended => Poll::Ready(None), - } - } -} - -impl Stream for SparseTilesFillAdapter -where - T: Pixel, - S: Stream>>, -{ - type Item = Result>; - - #[allow(clippy::too_many_lines)] - fn poll_next( - self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - if self.sc.state == State::Ended { - return Poll::Ready(None); - } - - self.next_step(cx) - } -} - -/// The strategy determines how to set the cache expiration for filler tiles produced by the adapter. -/// -/// It can either be a fixed value for all produced tiles, or it can be derived from the surrounding tiles that are not produced by the adapter. -/// In the latter case it will use the cache expiration of the preceeding tile if it is available, otherwise it will use the cache expiration of the following tile. -#[derive(Debug, Clone, PartialEq)] -#[allow(dead_code)] -pub enum FillerTileCacheExpirationStrategy { - NoCache, - FixedValue(CacheExpiration), - DerivedFromSurroundingTiles, -} - -#[derive(Debug, Clone, PartialEq)] -struct FillerTileCacheHintProvider { - strategy: FillerTileCacheExpirationStrategy, - state: FillerTileCacheHintState, -} - -#[derive(Debug, Clone, PartialEq)] -enum FillerTileCacheHintState { - Initial, - FirstTile(CacheExpiration), - OtherTile { - current_cache_expiration: CacheExpiration, - next_cache_expiration: CacheExpiration, - }, -} - -impl FillerTileCacheHintState { - fn cache_expiration(&self) -> Option { - match self { - FillerTileCacheHintState::Initial => None, - FillerTileCacheHintState::FirstTile(expiration) => Some(*expiration), - FillerTileCacheHintState::OtherTile { - current_cache_expiration, - next_cache_expiration: _, - } => Some(*current_cache_expiration), - } - } -} - -impl FillerTileCacheHintProvider { - fn update_cache_expiration(&mut self, expiration: CacheExpiration) { - self.state = match self.state { - FillerTileCacheHintState::Initial => FillerTileCacheHintState::FirstTile(expiration), - FillerTileCacheHintState::FirstTile(current_cache_expiration) => { - FillerTileCacheHintState::OtherTile { - current_cache_expiration, - next_cache_expiration: expiration, - } - } - FillerTileCacheHintState::OtherTile { - current_cache_expiration: _, - next_cache_expiration, - } => FillerTileCacheHintState::OtherTile { - current_cache_expiration: next_cache_expiration, - next_cache_expiration: expiration, - }, - }; - } - - fn next_hint(&self) -> CacheHint { - match self.strategy { - FillerTileCacheExpirationStrategy::NoCache => CacheHint::no_cache(), - FillerTileCacheExpirationStrategy::FixedValue(expiration) => expiration.into(), - FillerTileCacheExpirationStrategy::DerivedFromSurroundingTiles => self - .state - .cache_expiration() - .map_or(CacheHint::no_cache(), Into::into), - } - } -} - -impl From for FillerTileCacheHintProvider { - fn from(value: FillerTileCacheExpirationStrategy) -> Self { - Self { - strategy: value, - state: FillerTileCacheHintState::Initial, - } - } -} - -#[cfg(test)] -mod tests { - use futures::{StreamExt, stream}; - use geoengine_datatypes::{ - primitives::{CacheHint, TimeInterval}, - raster::Grid, - util::test::TestDefault, - }; - - use super::*; - - #[tokio::test] - async fn test_gap_overlaps_time_step() { - let data = vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - // GAP - // GAP - // GAP - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![19, 20, 21, 22]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - // GAP - ]; - - let result_data = data.into_iter().map(Ok); - - let in_stream = stream::iter(result_data); - let grid_bounding_box = GridBoundingBox2D::new([-1, 0], [0, 1]).unwrap(); - let global_geo_transform = GeoTransform::test_default(); - let tile_shape = [2, 2].into(); - - let adapter = SparseTilesFillAdapter::new( - in_stream, - grid_bounding_box, - 1, - global_geo_transform, - tile_shape, - FillerTileCacheExpirationStrategy::NoCache, - TimeInterval::new_unchecked(5, 10), - FillerTimeBounds::from(TimeInterval::new_unchecked(5, 10)), - ); - - let tiles: Vec>> = adapter.collect().await; - - let tile_time_positions: Vec<(GridIdx2D, TimeInterval)> = tiles - .into_iter() - .map(|t| { - let g = t.unwrap(); - (g.tile_position, g.time) - }) - .collect(); - - let expected_positions = vec![ - ([-1, 0].into(), TimeInterval::new_unchecked(0, 5)), - ([-1, 1].into(), TimeInterval::new_unchecked(0, 5)), - ([0, 0].into(), TimeInterval::new_unchecked(0, 5)), - ([0, 1].into(), TimeInterval::new_unchecked(0, 5)), - ([-1, 0].into(), TimeInterval::new_unchecked(5, 10)), - ([-1, 1].into(), TimeInterval::new_unchecked(5, 10)), - ([0, 0].into(), TimeInterval::new_unchecked(5, 10)), - ([0, 1].into(), TimeInterval::new_unchecked(5, 10)), - ]; - - assert_eq!(tile_time_positions, expected_positions); - } - - #[tokio::test] - async fn test_empty() { - let data = vec![]; - // GAP - // GAP - // GAP - // GAP - - let result_data = data.into_iter().map(Ok); - - let in_stream = stream::iter(result_data); - let grid_bounding_box = GridBoundingBox2D::new([-1, 0], [0, 1]).unwrap(); - let global_geo_transform = GeoTransform::test_default(); - let tile_shape = [2, 2].into(); - - let adapter = SparseTilesFillAdapter::new( - in_stream, - grid_bounding_box, - 1, - global_geo_transform, - tile_shape, - FillerTileCacheExpirationStrategy::NoCache, - TimeInterval::default(), - FillerTimeBounds::from(TimeInterval::default()), - ); - - let tiles: Vec>> = adapter.collect().await; - - let tile_time_positions: Vec<(GridIdx2D, TimeInterval)> = tiles - .into_iter() - .map(|t| { - let g = t.unwrap(); - (g.tile_position, g.time) - }) - .collect(); - - let expected_positions = vec![ - ([-1, 0].into(), TimeInterval::default()), - ([-1, 1].into(), TimeInterval::default()), - ([0, 0].into(), TimeInterval::default()), - ([0, 1].into(), TimeInterval::default()), - ]; - - assert_eq!(tile_time_positions, expected_positions); - } - - #[tokio::test] - async fn test_gaps_at_begin() { - let data = vec![ - // GAP - // GAP - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [0, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - // GAP - // GAP - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [0, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![19, 20, 21, 22]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - ]; - - let result_data = data.into_iter().map(Ok); - - let in_stream = stream::iter(result_data); - let grid_bounding_box = GridBoundingBox2D::new([-1, 0], [0, 1]).unwrap(); - let global_geo_transform = GeoTransform::test_default(); - let tile_shape = [2, 2].into(); - - let adapter = SparseTilesFillAdapter::new( - in_stream, - grid_bounding_box, - 1, - global_geo_transform, - tile_shape, - FillerTileCacheExpirationStrategy::NoCache, - TimeInterval::new_unchecked(0, 10), - FillerTimeBounds::from(TimeInterval::new_unchecked(0, 10)), - ); - - let tiles: Vec>> = adapter.collect().await; - - let tile_time_positions: Vec<(GridIdx2D, TimeInterval)> = tiles - .into_iter() - .map(|t| { - let g = t.unwrap(); - (g.tile_position, g.time) - }) - .collect(); - - let expected_positions = vec![ - ([-1, 0].into(), TimeInterval::new_unchecked(0, 5)), - ([-1, 1].into(), TimeInterval::new_unchecked(0, 5)), - ([0, 0].into(), TimeInterval::new_unchecked(0, 5)), - ([0, 1].into(), TimeInterval::new_unchecked(0, 5)), - ([-1, 0].into(), TimeInterval::new_unchecked(5, 10)), - ([-1, 1].into(), TimeInterval::new_unchecked(5, 10)), - ([0, 0].into(), TimeInterval::new_unchecked(5, 10)), - ([0, 1].into(), TimeInterval::new_unchecked(5, 10)), - ]; - - assert_eq!(tile_time_positions, expected_positions); - } - - #[tokio::test] - #[allow(clippy::too_many_lines)] - async fn test_single_gap_at_end() { - let data = vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 1, 1, 1]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![2, 2, 2, 2]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![3, 3, 3, 3]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - // GAP - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![101, 101, 101, 110]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![102, 102, 102, 102]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![103, 103, 103, 103]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [0, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![104, 104, 104, 104]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - ]; - - let result_data = data.into_iter().map(Ok); - - let in_stream = stream::iter(result_data); - let grid_bounding_box = GridBoundingBox2D::new([-1, 0], [0, 1]).unwrap(); - let global_geo_transform = GeoTransform::test_default(); - let tile_shape = [2, 2].into(); - - let adapter = SparseTilesFillAdapter::new( - in_stream, - grid_bounding_box, - 1, - global_geo_transform, - tile_shape, - FillerTileCacheExpirationStrategy::NoCache, - TimeInterval::new_unchecked(0, 10), - FillerTimeBounds::from(TimeInterval::new_unchecked(0, 10)), - ); - - let tiles: Vec>> = adapter.collect().await; - - let tile_time_positions: Vec<(GridIdx2D, TimeInterval)> = tiles - .into_iter() - .map(|t| { - let g = t.unwrap(); - (g.tile_position, g.time) - }) - .collect(); - - let expected_positions = vec![ - ([-1, 0].into(), TimeInterval::new_unchecked(0, 5)), - ([-1, 1].into(), TimeInterval::new_unchecked(0, 5)), - ([0, 0].into(), TimeInterval::new_unchecked(0, 5)), - ([0, 1].into(), TimeInterval::new_unchecked(0, 5)), - ([-1, 0].into(), TimeInterval::new_unchecked(5, 10)), - ([-1, 1].into(), TimeInterval::new_unchecked(5, 10)), - ([0, 0].into(), TimeInterval::new_unchecked(5, 10)), - ([0, 1].into(), TimeInterval::new_unchecked(5, 10)), - ]; - - assert_eq!(tile_time_positions, expected_positions); - } - - #[tokio::test] - async fn test_gaps_at_end() { - let data = vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - // GAP - // GAP - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![19, 20, 21, 22]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - // GAP - // GAP - ]; - - let result_data = data.into_iter().map(Ok); - - let in_stream = stream::iter(result_data); - let grid_bounding_box = GridBoundingBox2D::new([-1, 0], [0, 1]).unwrap(); - let global_geo_transform = GeoTransform::test_default(); - let tile_shape = [2, 2].into(); - - let adapter = SparseTilesFillAdapter::new( - in_stream, - grid_bounding_box, - 1, - global_geo_transform, - tile_shape, - FillerTileCacheExpirationStrategy::NoCache, - TimeInterval::new_unchecked(0, 10), - FillerTimeBounds::from(TimeInterval::new_unchecked(0, 10)), - ); - - let tiles: Vec>> = adapter.collect().await; - - let tile_time_positions: Vec<(GridIdx2D, TimeInterval)> = tiles - .into_iter() - .map(|t| { - let g = t.unwrap(); - (g.tile_position, g.time) - }) - .collect(); - - let expected_positions = vec![ - ([-1, 0].into(), TimeInterval::new_unchecked(0, 5)), - ([-1, 1].into(), TimeInterval::new_unchecked(0, 5)), - ([0, 0].into(), TimeInterval::new_unchecked(0, 5)), - ([0, 1].into(), TimeInterval::new_unchecked(0, 5)), - ([-1, 0].into(), TimeInterval::new_unchecked(5, 10)), - ([-1, 1].into(), TimeInterval::new_unchecked(5, 10)), - ([0, 0].into(), TimeInterval::new_unchecked(5, 10)), - ([0, 1].into(), TimeInterval::new_unchecked(5, 10)), - ]; - - assert_eq!(tile_time_positions, expected_positions); - } - - #[tokio::test] - async fn test_one_cell_grid() { - let data = vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - ]; - - let result_data = data.into_iter().map(Ok); - - let in_stream = stream::iter(result_data); - let grid_bounding_box = GridBoundingBox2D::new([0, 0], [0, 0]).unwrap(); - let global_geo_transform = GeoTransform::test_default(); - let tile_shape = [2, 2].into(); - - let adapter = SparseTilesFillAdapter::new( - in_stream, - grid_bounding_box, - 1, - global_geo_transform, - tile_shape, - FillerTileCacheExpirationStrategy::NoCache, - TimeInterval::new_unchecked(0, 10), - FillerTimeBounds::from(TimeInterval::new_unchecked(0, 10)), - ); - - let tiles: Vec>> = adapter.collect().await; - - let tile_time_positions: Vec<(GridIdx2D, TimeInterval)> = tiles - .into_iter() - .map(|t| { - let g = t.unwrap(); - (g.tile_position, g.time) - }) - .collect(); - - let expected_positions = vec![ - ([0, 0].into(), TimeInterval::new_unchecked(0, 5)), - ([0, 0].into(), TimeInterval::new_unchecked(5, 10)), - ]; - - assert_eq!(tile_time_positions, expected_positions); - } - - #[allow(clippy::too_many_lines)] - #[tokio::test] - async fn test_no_gaps() { - let data = vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [0, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![19, 20, 21, 22]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [0, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![19, 20, 21, 22]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - ]; - - let result_data = data.into_iter().map(Ok); - - let in_stream = stream::iter(result_data); - let grid_bounding_box = GridBoundingBox2D::new([-1, 0], [0, 1]).unwrap(); - let global_geo_transform = GeoTransform::test_default(); - let tile_shape = [2, 2].into(); - - let adapter = SparseTilesFillAdapter::new( - in_stream, - grid_bounding_box, - 1, - global_geo_transform, - tile_shape, - FillerTileCacheExpirationStrategy::NoCache, - TimeInterval::new_unchecked(0, 10), - FillerTimeBounds::from(TimeInterval::new_unchecked(0, 10)), - ); - - let tiles: Vec>> = adapter.collect().await; - - let tile_time_positions: Vec<(GridIdx2D, TimeInterval)> = tiles - .into_iter() - .map(|t| { - let g = t.unwrap(); - (g.tile_position, g.time) - }) - .collect(); - - let expected_positions = vec![ - ([-1, 0].into(), TimeInterval::new_unchecked(0, 5)), - ([-1, 1].into(), TimeInterval::new_unchecked(0, 5)), - ([0, 0].into(), TimeInterval::new_unchecked(0, 5)), - ([0, 1].into(), TimeInterval::new_unchecked(0, 5)), - ([-1, 0].into(), TimeInterval::new_unchecked(5, 10)), - ([-1, 1].into(), TimeInterval::new_unchecked(5, 10)), - ([0, 0].into(), TimeInterval::new_unchecked(5, 10)), - ([0, 1].into(), TimeInterval::new_unchecked(5, 10)), - ]; - - assert_eq!(tile_time_positions, expected_positions); - } - - #[tokio::test] - async fn test_min_max_time() { - let data = vec![ - RasterTile2D { - time: TimeInterval::default(), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - // GAP - // GAP - RasterTile2D { - time: TimeInterval::default(), - tile_position: [0, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - ]; - - let result_data = data.into_iter().map(Ok); - - let in_stream = stream::iter(result_data); - let grid_bounding_box = GridBoundingBox2D::new([-1, 0], [0, 1]).unwrap(); - let global_geo_transform = GeoTransform::test_default(); - let tile_shape = [2, 2].into(); - - let adapter = SparseTilesFillAdapter::new( - in_stream, - grid_bounding_box, - 1, - global_geo_transform, - tile_shape, - FillerTileCacheExpirationStrategy::NoCache, - TimeInterval::default(), - FillerTimeBounds::from(TimeInterval::default()), - ); - - let tiles: Vec>> = adapter.collect().await; - - let tile_time_positions: Vec<(GridIdx2D, TimeInterval)> = tiles - .into_iter() - .map(|t| { - let g = t.unwrap(); - (g.tile_position, g.time) - }) - .collect(); - - let expected_positions = vec![ - ([-1, 0].into(), TimeInterval::default()), - ([-1, 1].into(), TimeInterval::default()), - ([0, 0].into(), TimeInterval::default()), - ([0, 1].into(), TimeInterval::default()), - ]; - - assert_eq!(tile_time_positions, expected_positions); - } - - #[tokio::test] - async fn test_error() { - let data = vec![ - Ok(RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }), - Err(crate::error::Error::NoSpatialBoundsAvailable), - ]; - - let result_data = data.into_iter(); - - let in_stream = stream::iter(result_data); - let grid_bounding_box = GridBoundingBox2D::new([-1, 0], [0, 1]).unwrap(); - let global_geo_transform = GeoTransform::test_default(); - let tile_shape = [2, 2].into(); - - let adapter = SparseTilesFillAdapter::new( - in_stream, - grid_bounding_box, - 1, - global_geo_transform, - tile_shape, - FillerTileCacheExpirationStrategy::NoCache, - TimeInterval::new_unchecked(0, 5), - FillerTimeBounds::from(TimeInterval::new_unchecked(0, 5)), - ); - - let tiles: Vec>> = adapter.collect().await; - - assert_eq!(tiles.len(), 2); - assert!(tiles[0].is_ok()); - assert!(tiles[1].is_err()); - } - - #[allow(clippy::too_many_lines)] - #[tokio::test] - async fn test_timeinterval_gap() { - let data = vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [0, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(10, 15), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(10, 15), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![19, 20, 21, 22]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(10, 15), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(10, 15), - tile_position: [0, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![19, 20, 21, 22]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - ]; - - let result_data = data.into_iter().map(Ok); - - let in_stream = stream::iter(result_data); - let grid_bounding_box = GridBoundingBox2D::new([-1, 0], [0, 1]).unwrap(); - let global_geo_transform = GeoTransform::test_default(); - let tile_shape = [2, 2].into(); - - let adapter = SparseTilesFillAdapter::new( - in_stream, - grid_bounding_box, - 1, - global_geo_transform, - tile_shape, - FillerTileCacheExpirationStrategy::NoCache, - TimeInterval::new_unchecked(0, 15), - FillerTimeBounds::from(TimeInterval::new_unchecked(0, 15)), - ); - - let tiles: Vec>> = adapter.collect().await; - - let tile_time_positions: Vec<(GridIdx2D, TimeInterval)> = tiles - .into_iter() - .map(|t| { - let g = t.unwrap(); - (g.tile_position, g.time) - }) - .collect(); - - let expected_positions = vec![ - ([-1, 0].into(), TimeInterval::new_unchecked(0, 5)), - ([-1, 1].into(), TimeInterval::new_unchecked(0, 5)), - ([0, 0].into(), TimeInterval::new_unchecked(0, 5)), - ([0, 1].into(), TimeInterval::new_unchecked(0, 5)), - ([-1, 0].into(), TimeInterval::new_unchecked(5, 10)), - ([-1, 1].into(), TimeInterval::new_unchecked(5, 10)), - ([0, 0].into(), TimeInterval::new_unchecked(5, 10)), - ([0, 1].into(), TimeInterval::new_unchecked(5, 10)), - ([-1, 0].into(), TimeInterval::new_unchecked(10, 15)), - ([-1, 1].into(), TimeInterval::new_unchecked(10, 15)), - ([0, 0].into(), TimeInterval::new_unchecked(10, 15)), - ([0, 1].into(), TimeInterval::new_unchecked(10, 15)), - ]; - - assert_eq!(tile_time_positions, expected_positions); - } - - #[allow(clippy::too_many_lines)] - #[tokio::test] - async fn test_timeinterval_gap_and_end_start_gap() { - let data = vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(10, 15), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - ]; - - let result_data = data.into_iter().map(Ok); - - let in_stream = stream::iter(result_data); - let grid_bounding_box = GridBoundingBox2D::new([-1, 0], [0, 1]).unwrap(); - let global_geo_transform = GeoTransform::test_default(); - let tile_shape = [2, 2].into(); - - let adapter = SparseTilesFillAdapter::new( - in_stream, - grid_bounding_box, - 1, - global_geo_transform, - tile_shape, - FillerTileCacheExpirationStrategy::NoCache, - TimeInterval::new_unchecked(0, 15), - FillerTimeBounds::from(TimeInterval::new_unchecked(0, 15)), - ); - - let tiles: Vec>> = adapter.collect().await; - - let tile_time_positions: Vec<(GridIdx2D, TimeInterval, bool)> = tiles - .into_iter() - .map(|t| { - let g = t.unwrap(); - (g.tile_position, g.time, g.is_empty()) - }) - .collect(); - - let expected_positions = vec![ - ([-1, 0].into(), TimeInterval::new_unchecked(0, 5), true), - ([-1, 1].into(), TimeInterval::new_unchecked(0, 5), false), - ([0, 0].into(), TimeInterval::new_unchecked(0, 5), true), - ([0, 1].into(), TimeInterval::new_unchecked(0, 5), true), - ([-1, 0].into(), TimeInterval::new_unchecked(5, 10), true), - ([-1, 1].into(), TimeInterval::new_unchecked(5, 10), true), - ([0, 0].into(), TimeInterval::new_unchecked(5, 10), true), - ([0, 1].into(), TimeInterval::new_unchecked(5, 10), true), - ([-1, 0].into(), TimeInterval::new_unchecked(10, 15), true), - ([-1, 1].into(), TimeInterval::new_unchecked(10, 15), true), - ([0, 0].into(), TimeInterval::new_unchecked(10, 15), false), - ([0, 1].into(), TimeInterval::new_unchecked(10, 15), true), - ]; - - assert_eq!(tile_time_positions, expected_positions); - } - - #[tokio::test] - async fn it_sets_no_cache() { - let cache_hint1 = CacheHint::seconds(0); - let cache_hint2 = CacheHint::seconds(60 * 60 * 24); - - let data = vec![ - // GAP - // GAP - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: cache_hint1, - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [0, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: cache_hint2, - }, - // GAP - // GAP - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: cache_hint1, - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [0, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![19, 20, 21, 22]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: cache_hint1, - }, - ]; - - let result_data = data.into_iter().map(Ok); - - let in_stream = stream::iter(result_data); - let grid_bounding_box = GridBoundingBox2D::new([-1, 0], [0, 1]).unwrap(); - let global_geo_transform = GeoTransform::test_default(); - let tile_shape = [2, 2].into(); - - let adapter = SparseTilesFillAdapter::new( - in_stream, - grid_bounding_box, - 1, - global_geo_transform, - tile_shape, - FillerTileCacheExpirationStrategy::NoCache, - TimeInterval::new_unchecked(0, 10), - FillerTimeBounds::from(TimeInterval::new_unchecked(0, 10)), - ); - - let tiles: Vec>> = adapter.collect().await; - - let cache_expirations: Vec = tiles - .into_iter() - .map(|t| { - let g = t.unwrap(); - g.cache_hint.is_expired() - }) - .collect(); - - let expected_expirations = vec![ - true, // GAP - true, // GAP - cache_hint1.is_expired(), // data - cache_hint2.is_expired(), // data - true, // GAP - true, // GAP - cache_hint1.is_expired(), // data - cache_hint1.is_expired(), // data - ]; - - assert_eq!(cache_expirations, expected_expirations); - } - - #[tokio::test] - async fn it_derives_cache_hint_from_surrounding_tiles() { - let cache_hint1 = CacheHint::seconds(60); - let cache_hint2 = CacheHint::seconds(60 * 60 * 24); - - let data = vec![ - // GAP - // GAP - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: cache_hint1, - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [0, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: cache_hint2, - }, - // GAP - // GAP - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: cache_hint1, - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [0, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![19, 20, 21, 22]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: cache_hint1, - }, - ]; - - let result_data = data.into_iter().map(Ok); - - let in_stream = stream::iter(result_data); - let grid_bounding_box = GridBoundingBox2D::new([-1, 0], [0, 1]).unwrap(); - let global_geo_transform = GeoTransform::test_default(); - let tile_shape = [2, 2].into(); - - let adapter = SparseTilesFillAdapter::new( - in_stream, - grid_bounding_box, - 1, - global_geo_transform, - tile_shape, - FillerTileCacheExpirationStrategy::DerivedFromSurroundingTiles, - TimeInterval::new_unchecked(0, 10), - FillerTimeBounds::from(TimeInterval::new_unchecked(0, 10)), - ); - - let tiles: Vec>> = adapter.collect().await; - - let cache_expirations: Vec = tiles - .into_iter() - .map(|t| { - let g = t.unwrap(); - g.cache_hint.expires() - }) - .collect(); - - let expected_expirations = vec![ - cache_hint1.expires(), // GAP (use first real tile) - cache_hint1.expires(), // GAP (use first real tile) - cache_hint1.expires(), // data - cache_hint2.expires(), // data - cache_hint2.expires(), // GAP (use previous tile) - cache_hint2.expires(), // GAP (use previous tile) - cache_hint1.expires(), // data - cache_hint1.expires(), // data - ]; - - assert_eq!(cache_expirations, expected_expirations); - } - - #[tokio::test] - async fn it_derives_cache_hint_from_no_tiles() { - let data = vec![ - // no surrounding tiles - ]; - - let result_data = data.into_iter().map(Ok); - - let in_stream = stream::iter(result_data); - let grid_bounding_box = GridBoundingBox2D::new([-1, 0], [0, 1]).unwrap(); - let global_geo_transform = GeoTransform::test_default(); - let tile_shape = [2, 2].into(); - - let adapter = SparseTilesFillAdapter::new( - in_stream, - grid_bounding_box, - 1, - global_geo_transform, - tile_shape, - FillerTileCacheExpirationStrategy::DerivedFromSurroundingTiles, - TimeInterval::new_unchecked(0, 10), - FillerTimeBounds::from(TimeInterval::new_unchecked(0, 10)), - ); - - let tiles: Vec>> = adapter.collect().await; - - assert_eq!(tiles.len(), 4); - - // as there are no surrounding tiles, we have no further information and fall back to not caching - for tile in tiles { - assert!(tile.as_ref().unwrap().cache_hint.is_expired()); - } - } - - #[tokio::test] - async fn it_fills_gap_around_timestep() { - let cache_hint1 = CacheHint::seconds(0); - let cache_hint2 = CacheHint::seconds(60 * 60 * 24); - - let data = vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: cache_hint1, - }, - // GAP t [0,5) pos [0, 0] - // GAP t [5,10) pos [-1, 0] - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: cache_hint2, - }, - ]; - - let result_data = data.into_iter().map(Ok); - - let in_stream = stream::iter(result_data); - let grid_bounding_box = GridBoundingBox2D::new([-1, 0], [0, 0]).unwrap(); - let global_geo_transform = GeoTransform::test_default(); - let tile_shape = [2, 2].into(); - - let adapter = SparseTilesFillAdapter::new( - in_stream, - grid_bounding_box, - 1, - global_geo_transform, - tile_shape, - FillerTileCacheExpirationStrategy::NoCache, - TimeInterval::new_unchecked(0, 10), - FillerTimeBounds::from(TimeInterval::new_unchecked(0, 10)), - ); - - let tiles: Vec>> = adapter.collect().await; - let tiles = tiles.into_iter().map(Result::unwrap).collect::>(); - - assert_eq!(tiles[0].time, TimeInterval::new_unchecked(0, 5)); - assert_eq!(tiles[0].tile_position, [-1, 0].into()); - assert!(!tiles[0].is_empty()); - - assert_eq!(tiles[1].time, TimeInterval::new_unchecked(0, 5)); - assert_eq!(tiles[1].tile_position, [0, 0].into()); - assert!(tiles[1].is_empty()); - - assert_eq!(tiles[2].time, TimeInterval::new_unchecked(5, 10)); - assert_eq!(tiles[2].tile_position, [-1, 0].into()); - assert!(tiles[2].is_empty()); - - assert_eq!(tiles[3].time, TimeInterval::new_unchecked(5, 10)); - assert_eq!(tiles[3].tile_position, [0, 0].into()); - assert!(!tiles[3].is_empty()); - - assert_eq!(tiles.len(), 4); - } - - #[tokio::test] - async fn it_fills_multiband_streams() { - let cache_hint1 = CacheHint::seconds(0); - let cache_hint2 = CacheHint::seconds(60 * 60 * 24); - - let data = vec![ - // GAP t [0,5) pos[-1, 0] b 0 - // GAP t [0,5) pos[-1, 0] b 1 - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: cache_hint1, - }, - // GAP t [0,5) pos [0, 0] b 1 - // GAP t [5,10) pos [-1, 0] b 0 - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 0].into(), - band: 1, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: cache_hint2, - }, - // GAP t [5,10) pos [0, 0] b 0 - // GAP t [5,10) pos [0, 0] b 1 - ]; - - let result_data = data.into_iter().map(Ok); - - let in_stream = stream::iter(result_data); - let grid_bounding_box = GridBoundingBox2D::new([-1, 0], [0, 0]).unwrap(); - let global_geo_transform = GeoTransform::test_default(); - let tile_shape = [2, 2].into(); - - let adapter = SparseTilesFillAdapter::new( - in_stream, - grid_bounding_box, - 2, - global_geo_transform, - tile_shape, - FillerTileCacheExpirationStrategy::NoCache, - TimeInterval::new_unchecked(0, 10), - FillerTimeBounds::from(TimeInterval::new_unchecked(0, 10)), - ); - - let tiles: Vec>> = adapter.collect().await; - let tiles = tiles.into_iter().map(Result::unwrap).collect::>(); - - assert_eq!(tiles[0].time, TimeInterval::new_unchecked(0, 5)); - assert_eq!(tiles[0].tile_position, [-1, 0].into()); - assert_eq!(tiles[0].band, 0); - assert!(tiles[0].is_empty()); - - assert_eq!(tiles[1].time, TimeInterval::new_unchecked(0, 5)); - assert_eq!(tiles[1].tile_position, [-1, 0].into()); - assert_eq!(tiles[1].band, 1); - assert!(tiles[1].is_empty()); - - assert_eq!(tiles[2].time, TimeInterval::new_unchecked(0, 5)); - assert_eq!(tiles[2].tile_position, [0, 0].into()); - assert_eq!(tiles[2].band, 0); - assert!(!tiles[2].is_empty()); - - assert_eq!(tiles[3].time, TimeInterval::new_unchecked(0, 5)); - assert_eq!(tiles[3].tile_position, [0, 0].into()); - assert_eq!(tiles[3].band, 1); - assert!(tiles[3].is_empty()); - - assert_eq!(tiles[4].time, TimeInterval::new_unchecked(5, 10)); - assert_eq!(tiles[4].tile_position, [-1, 0].into()); - assert_eq!(tiles[4].band, 0); - assert!(tiles[4].is_empty()); - - assert_eq!(tiles[5].time, TimeInterval::new_unchecked(5, 10)); - assert_eq!(tiles[5].tile_position, [-1, 0].into()); - assert_eq!(tiles[5].band, 1); - assert!(!tiles[5].is_empty()); - - assert_eq!(tiles[6].time, TimeInterval::new_unchecked(5, 10)); - assert_eq!(tiles[6].tile_position, [0, 0].into()); - assert_eq!(tiles[6].band, 0); - assert!(tiles[6].is_empty()); - - assert_eq!(tiles[7].time, TimeInterval::new_unchecked(5, 10)); - assert_eq!(tiles[7].tile_position, [0, 0].into()); - assert_eq!(tiles[7].band, 1); - assert!(tiles[7].is_empty()); - - assert_eq!(tiles.len(), 8); - } - - #[tokio::test] - async fn it_detects_non_increasing_intervals() { - let data = vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [0, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![19, 20, 21, 22]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [0, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![19, 20, 21, 22]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - ]; - - let result_data = data.into_iter().map(Ok); - - let in_stream = stream::iter(result_data); - let grid_bounding_box = GridBoundingBox2D::new([-1, 0], [0, 1]).unwrap(); - let global_geo_transform = GeoTransform::test_default(); - let tile_shape = [2, 2].into(); - - let adapter = SparseTilesFillAdapter::new( - in_stream, - grid_bounding_box, - 1, - global_geo_transform, - tile_shape, - FillerTileCacheExpirationStrategy::NoCache, - TimeInterval::new_unchecked(0, 5), - FillerTimeBounds::from(TimeInterval::new_unchecked(0, 5)), - ); - - let tiles: Vec>> = adapter.collect().await; - - assert_eq!(tiles.len(), 5); - assert!(tiles[4].is_err()); - } - - #[tokio::test] - async fn it_detects_non_increasing_instants() { - let data = vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 0), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 0), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 0), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 0), - tile_position: [0, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 0), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 0), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![19, 20, 21, 22]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 0), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 0), - tile_position: [0, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![19, 20, 21, 22]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - ]; - - let result_data = data.into_iter().map(Ok); - - let in_stream = stream::iter(result_data); - let grid_bounding_box = GridBoundingBox2D::new([-1, 0], [0, 1]).unwrap(); - let global_geo_transform = GeoTransform::test_default(); - let tile_shape = [2, 2].into(); - - let adapter = SparseTilesFillAdapter::new( - in_stream, - grid_bounding_box, - 1, - global_geo_transform, - tile_shape, - FillerTileCacheExpirationStrategy::NoCache, - TimeInterval::new_unchecked(0, 1), - FillerTimeBounds::from(TimeInterval::new_unchecked(0, 1)), - ); - - let tiles: Vec>> = adapter.collect().await; - - assert_eq!(tiles.len(), 5); - assert!(tiles[4].is_err()); - } - - #[allow(clippy::too_many_lines)] - #[tokio::test] - async fn test_timeinterval_fill_data_bounds() { - let data = vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(10, 15), - tile_position: [0, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - ]; - - let result_data = data.into_iter().map(Ok); - - let in_stream = stream::iter(result_data); - let grid_bounding_box = GridBoundingBox2D::new([-1, 0], [0, 1]).unwrap(); - let global_geo_transform = GeoTransform::test_default(); - let tile_shape = [2, 2].into(); - - let adapter = SparseTilesFillAdapter::new( - in_stream, - grid_bounding_box, - 1, - global_geo_transform, - tile_shape, - FillerTileCacheExpirationStrategy::NoCache, - TimeInterval::new_unchecked(0, 20), - FillerTimeBounds::from(TimeInterval::default()), - ); - - let tiles: Vec>> = adapter.collect().await; - - let tile_time_positions: Vec<(GridIdx2D, TimeInterval, bool)> = tiles - .into_iter() - .map(|t| { - let g = t.unwrap(); - (g.tile_position, g.time, g.is_empty()) - }) - .collect(); - - let expected_positions = vec![ - ( - [-1, 0].into(), - TimeInterval::new_unchecked(TimeInstance::MIN, 5), - true, - ), - ( - [-1, 1].into(), - TimeInterval::new_unchecked(TimeInstance::MIN, 5), - true, - ), - ( - [0, 0].into(), - TimeInterval::new_unchecked(TimeInstance::MIN, 5), - true, - ), - ( - [0, 1].into(), - TimeInterval::new_unchecked(TimeInstance::MIN, 5), - true, - ), - ([-1, 0].into(), TimeInterval::new_unchecked(5, 10), true), - ([-1, 1].into(), TimeInterval::new_unchecked(5, 10), false), - ([0, 0].into(), TimeInterval::new_unchecked(5, 10), true), - ([0, 1].into(), TimeInterval::new_unchecked(5, 10), true), - ([-1, 0].into(), TimeInterval::new_unchecked(10, 15), true), - ([-1, 1].into(), TimeInterval::new_unchecked(10, 15), true), - ([0, 0].into(), TimeInterval::new_unchecked(10, 15), false), - ([0, 1].into(), TimeInterval::new_unchecked(10, 15), true), - ( - [-1, 0].into(), - TimeInterval::new_unchecked(15, TimeInstance::MAX), - true, - ), - ( - [-1, 1].into(), - TimeInterval::new_unchecked(15, TimeInstance::MAX), - true, - ), - ( - [0, 0].into(), - TimeInterval::new_unchecked(15, TimeInstance::MAX), - true, - ), - ( - [0, 1].into(), - TimeInterval::new_unchecked(15, TimeInstance::MAX), - true, - ), - ]; - - assert_eq!(tile_time_positions, expected_positions); - } -} diff --git a/operators/src/cache/cache_operator.rs b/operators/src/cache/cache_operator.rs index bd46f7b68..02233401a 100644 --- a/operators/src/cache/cache_operator.rs +++ b/operators/src/cache/cache_operator.rs @@ -457,7 +457,7 @@ where impl

ResultStreamWrapper for RasterTile2D

where P: 'static + Pixel, - RasterTile2D

: CacheElement, + RasterTile2D

: CacheElement, Self::ResultStream: Send + Sync, { fn wrap_result_stream<'a>( @@ -482,7 +482,7 @@ mod tests { source::{GdalSource, GdalSourceParameters}, util::gdal::add_ndvi_dataset, }; - use futures::StreamExt; + use futures::{StreamExt, TryStreamExt}; use geoengine_datatypes::{ primitives::{BandSelection, RasterQueryRectangle, TimeInterval}, raster::{GridBoundingBox2D, RasterDataType, RenameBands, TilesEqualIgnoringCacheHint}, @@ -635,9 +635,9 @@ mod tests { // only keep the second band for comparison let tiles = tiles .into_iter() - .filter_map(|mut tile| { + .filter_map(|tile| { if tile.band == 1 { - tile.band = 0; + //tile.band = 0; Some(tile) } else { None @@ -672,4 +672,153 @@ mod tests { assert!(tiles.tiles_equal_ignoring_cache_hint(&tiles_from_cache)); } + + #[allow(clippy::too_many_lines)] + #[tokio::test] + async fn it_reuses_multi_band_subset() { + let mut exe_ctx = MockExecutionContext::test_default(); + + let ndvi_id = add_ndvi_dataset(&mut exe_ctx); + + let source_operator_a = GdalSource { + params: GdalSourceParameters::new(ndvi_id.clone()), + }; + + let source_operator_b = GdalSource { + params: GdalSourceParameters::new(ndvi_id.clone()), + }; + + let stacked_operator = RasterStacker { + params: RasterStackerParams { + rename_bands: RenameBands::Rename(vec!["band_0".to_string(), "band_1".to_string()]), + }, + sources: MultipleRasterSources { + rasters: vec![source_operator_a.boxed(), source_operator_b.boxed()], + }, + }; + + let operator = stacked_operator + .boxed() + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + let cached_op = InitializedCacheOperator::new(operator); + + let processor = cached_op.query_processor().unwrap().get_u8().unwrap(); + + let tile_cache = Arc::new(SharedCache::test_default()); + + let query_ctx = exe_ctx.mock_query_context_with_query_extensions( + ChunkByteSize::test_default(), + Some(tile_cache), + None, + None, + ); + + // query the first two bands + let stream = processor + .query( + RasterQueryRectangle::new( + GridBoundingBox2D::new([-90, -180], [89, 179]).unwrap(), + TimeInterval::default(), + BandSelection::new(vec![0, 1]).unwrap(), + ), + &query_ctx, + ) + .await + .unwrap(); + + // now count the tiles per band by folding over the stream + let tile_per_band_count: (usize, usize) = stream + .fold((0usize, 0usize), |mut acc, tile_result| async move { + match tile_result { + Ok(tile) => { + if tile.band == 0 { + acc.0 += 1; + } else if tile.band == 1 { + acc.1 += 1; + } + } + Err(e) => { + panic!("Error in tile stream: {e}") + } + } + acc + }) + .await; + + assert!( + tile_per_band_count.0 > 0, + "There should be tiles for band 0" + ); + assert_eq!( + tile_per_band_count.0, tile_per_band_count.1, + "Both bands should have the same number of tiles" + ); + + // wait for the cache to be filled, which happens asynchronously + tokio::time::sleep(std::time::Duration::from_millis(1000)).await; + + // delete the dataset to make sure the result is served from the cache + exe_ctx.delete_meta_data(&ndvi_id); + + // now query only the second band + let stream_from_cache_0 = processor + .query( + RasterQueryRectangle::new( + GridBoundingBox2D::new([-90, -180], [89, 179]).unwrap(), + TimeInterval::default(), + BandSelection::first_n(1), + ), + &query_ctx, + ) + .await + .unwrap(); + + let band_0_count = stream_from_cache_0 + .inspect_ok(|tile| { + assert_eq!(tile.band, 0, "Only band 0 should be present in the result"); + }) + .count() + .await; + + assert!( + band_0_count > 0, + "There should be tiles for band 0 in the cache result" + ); + assert_eq!( + band_0_count, tile_per_band_count.0, + "The number of tiles for band 0 should match the previous query" + ); + + // now query only the second band + let stream_from_cache_1 = processor + .query( + RasterQueryRectangle::new( + GridBoundingBox2D::new([-90, -180], [89, 179]).unwrap(), + TimeInterval::default(), + BandSelection::new_single(1), + ), + &query_ctx, + ) + .await + .unwrap(); + + let band_1_count = stream_from_cache_1 + .inspect_ok(|tile| { + assert_eq!(tile.band, 1, "Only band 1 should be present in the result"); + }) + .count() + .await; + + assert!( + band_1_count > 1, + "There should be tiles for band 1 in the cache result" + ); + assert_eq!( + band_1_count, tile_per_band_count.1, + "The number of tiles for band 1 should match the previous query" + ); + } } diff --git a/operators/src/cache/cache_tiles.rs b/operators/src/cache/cache_tiles.rs index 0dfb02a0c..2ae3e4bb0 100644 --- a/operators/src/cache/cache_tiles.rs +++ b/operators/src/cache/cache_tiles.rs @@ -212,7 +212,7 @@ where .global_pixel_bounds() .intersects(&query.spatial_bounds()) && self.time.intersects(&query.time_interval()) - && query.attributes().as_slice().contains(&self.band) + && query.attributes().contains(self.band) } } diff --git a/operators/src/cache/shared_cache.rs b/operators/src/cache/shared_cache.rs index 6b3ee2cdc..e7a6235a2 100644 --- a/operators/src/cache/shared_cache.rs +++ b/operators/src/cache/shared_cache.rs @@ -474,6 +474,15 @@ where .ok_or(CacheError::QueryNotFoundInLandingZone)?; let res = cache.add_query_element_to_landing_zone(query_id, landing_zone_element); + #[cfg(debug_assertions)] + match res.as_ref() { + Err(CacheError::TileExpiredBeforeInsertion) => { + tracing::trace!("Element expired before insertion."); + } + Err(er) => tracing::debug!("Error on insert query element: {er}"), + _ => {} + } + // if we cant add the element to the landing zone, we remove the query from the landing zone if res.is_err() { let _old_entry = cache.remove_query_from_landing_zone(query_id); @@ -858,6 +867,7 @@ impl CacheQueryMatch for RasterQueryRectangle { cache_spatial_query.contains(&query_spatial_query) && self.time_interval().contains(&query.time_interval()) + && self.attributes().contains_all(query.attributes().as_ref()) } } diff --git a/operators/src/error.rs b/operators/src/error.rs index df5ca69d8..527a5e73e 100644 --- a/operators/src/error.rs +++ b/operators/src/error.rs @@ -1,3 +1,4 @@ +use crate::adapters::SimpleRasterStackerError; use crate::engine::SpatialGridDescriptor; use crate::optimization::OptimizationError; use crate::processing::BandNeighborhoodAggregateError; @@ -335,10 +336,6 @@ pub enum Error { source: crate::util::statistics::StatisticsError, }, - #[snafu(display("SparseTilesFillAdapter error: {}", source))] - SparseTilesFillAdapter { - source: crate::adapters::SparseTilesFillAdapterError, - }, #[snafu(display("Expression error: {source}"), context(false))] ExpressionOperator { source: crate::processing::RasterExpressionError, @@ -534,12 +531,11 @@ pub enum Error { Optimization { source: OptimizationError, }, -} -impl From for Error { - fn from(source: crate::adapters::SparseTilesFillAdapterError) -> Self { - Error::SparseTilesFillAdapter { source } - } + #[snafu(display("Error in the SimpleRasterStacker: {source}"))] + SimpleRasterStacker { + source: SimpleRasterStackerError, + }, } impl From for Error { diff --git a/operators/src/mock/mock_raster_source.rs b/operators/src/mock/mock_raster_source.rs index 345591fb9..7911dbb3b 100644 --- a/operators/src/mock/mock_raster_source.rs +++ b/operators/src/mock/mock_raster_source.rs @@ -1,6 +1,3 @@ -use crate::adapters::{ - FillerTileCacheExpirationStrategy, FillerTimeBounds, SparseTilesFillAdapter, -}; use crate::engine::{ BoxRasterQueryProcessor, CanonicOperatorName, InitializedRasterOperator, OperatorData, OperatorName, QueryProcessor, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, @@ -12,15 +9,17 @@ use crate::processing::{ }; use crate::util::Result; use async_trait::async_trait; +use futures::TryStreamExt; use futures::{stream, stream::StreamExt}; use geoengine_datatypes::dataset::NamedData; use geoengine_datatypes::primitives::{ - BandSelection, CacheExpiration, RasterQueryRectangle, SpatialResolution, TimeFilledItem, - TimeInstance, TryIrregularTimeFillIterExt, TryRegularTimeFillIterExt, + BandSelection, CacheHint, RasterQueryRectangle, RegularTimeDimension, SpatialResolution, + TimeFilledItem, TimeInstance, TimeInterval, TryIrregularTimeFillIterExt, + TryRegularTimeFillIterExt, }; use geoengine_datatypes::raster::{ - GridBoundingBox2D, GridIntersection, GridShape2D, GridShapeAccess, GridSize, Pixel, - RasterTile2D, TilingSpecification, + GridBoundingBox2D, GridOrEmpty, GridShape2D, GridShapeAccess, GridSize, Pixel, RasterTile2D, + TileIdxBandCrossProductIter, TilingSpecification, }; use itertools::Itertools; use serde::{Deserialize, Serialize}; @@ -39,6 +38,16 @@ pub enum MockRasterSourceError { tiling_specification_yx: GridShape2D, tile_size_yx: GridShape2D, }, + #[snafu(display( + "A tile has a time interval of {} with len {} which is not valid in the specified regular dimension {:?}", + tile_time, + tile_time.duration_ms(), + time_dim + ))] + InvalidTimeIntervalInRegularDim { + tile_time: TimeInterval, + time_dim: RegularTimeDimension, + }, } #[derive(Debug, Clone)] @@ -81,6 +90,17 @@ where ); } + if let Some(regular) = result_descriptor.time.dimension.unwrap_regular() { + for x in &data { + if !regular.valid_interval(x.time) { + return Err(MockRasterSourceError::InvalidTimeIntervalInRegularDim { + tile_time: x.time, + time_dim: regular, + }); + } + } + } + Ok(Self { result_descriptor, data, @@ -118,75 +138,50 @@ where async fn _query<'a>( &'a self, query: RasterQueryRectangle, - _ctx: &'a dyn crate::engine::QueryContext, + ctx: &'a dyn crate::engine::QueryContext, ) -> Result>> { - let mut known_time_start: Option = None; - let mut known_time_end: Option = None; - let qt = query.time_interval(); - let qg = query.spatial_bounds(); - let parts: Vec> = self - .data - .iter() - .inspect(|m| { - let time_interval = m.time; - - if time_interval.contains(&qt) { - let t1 = time_interval.start(); - let t2 = time_interval.end(); - known_time_start = Some(t1); - known_time_end = Some(t2); - return; - } - - if time_interval.end() <= qt.start() { - let t1 = time_interval.end(); - known_time_start = known_time_start.map(|old| old.max(t1)).or(Some(t1)); - } else if time_interval.start() <= qt.start() { - let t1 = time_interval.start(); - known_time_start = known_time_start.map(|old| old.max(t1)).or(Some(t1)); - } - - if time_interval.start() >= qt.end() { - let t2 = time_interval.start(); - known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); - } else if time_interval.end() >= qt.end() { - let t2 = time_interval.end(); - known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); - } - }) - .filter(move |t| { - t.time.intersects(&qt) - && t.tile_information() - .global_pixel_bounds() - .intersects(&query.spatial_bounds()) + let tiling_strat = self + .result_descriptor + .tiling_grid_definition(ctx.tiling_specification()) + .generate_data_tiling_strategy(); + + let tiling_bounds = + tiling_strat.global_pixel_grid_bounds_to_tile_grid_bounds(query.spatial_bounds()); + + let times = self.time_query(query.time_interval(), ctx).await?; + + let ttb = times + .map_ok(move |time| { + stream::iter( + TileIdxBandCrossProductIter::with_grid_bounds_and_selection( + tiling_bounds, + query.attributes().clone(), + ) + .map(move |tile_band| { + crate::util::Result::<_, crate::error::Error>::Ok((time, tile_band)) + }), + ) }) - .cloned() - .collect(); - - // if we found no time bound we can assume that there is no data - let known_time_before = known_time_start.unwrap_or(TimeInstance::MIN); - let known_time_after = known_time_end.unwrap_or(TimeInstance::MAX); - - let inner_stream = stream::iter(parts.into_iter().map(Result::Ok)); + .try_flatten(); + + let tile_stream = ttb.map_ok(move |(time, (tile, band))| { + self.data + .iter() + .find(|data| data.band == band && data.tile_position == tile && data.time == time) + .cloned() + .unwrap_or_else(|| { + RasterTile2D::new( + time, + tile, + band, + tiling_strat.geo_transform, + GridOrEmpty::new_empty_shape(tiling_strat.tile_size_in_pixels), + CacheHint::no_cache(), + ) + }) + }); - let tiling_grid_spec = self - .result_descriptor - .tiling_grid_definition(self.tiling_specification); - - let tiling_strategy = tiling_grid_spec.generate_data_tiling_strategy(); - - // use SparseTilesFillAdapter to fill all the gaps - Ok(SparseTilesFillAdapter::new( - inner_stream, - tiling_strategy.global_pixel_grid_bounds_to_tile_grid_bounds(qg), - self.result_descriptor.bands.count(), - tiling_strategy.geo_transform, - tiling_strategy.tile_size_in_pixels, - FillerTileCacheExpirationStrategy::FixedValue(CacheExpiration::max()), // cache forever because we know all mock data - qt, - FillerTimeBounds::new(known_time_before, known_time_after), - ) - .boxed()) + Ok(tile_stream.boxed()) } fn result_descriptor(&self) -> &Self::ResultDescription { @@ -206,21 +201,68 @@ where query: geoengine_datatypes::primitives::TimeInterval, _ctx: &'a dyn crate::engine::QueryContext, ) -> Result>> { - let unique_times = self + let unique_times: Vec = self .data .iter() - .map(|tile| tile.time) + .map(|tile| &tile.time) .unique_by(|t| *t) - .filter(move |t| t.intersects(&query)) - .map(Ok); + .copied() + .collect(); + + let intersecting_times: Vec = unique_times + .iter() + .filter(|t| t.intersects(&query.time())) + .copied() + .collect(); let times = match self.result_descriptor.time.dimension { geoengine_datatypes::primitives::TimeDimension::Irregular => { - let times = unique_times.try_time_irregular_range_fill(query.time()); + let r_start = if let Some(first) = intersecting_times.first() { + if first.start() <= query.time().start() { + first.start() + } else { + let before = unique_times + .iter() + .rfind(|t| t.end() <= query.time().start()); + before.map_or( + TimeInstance::MIN, + geoengine_datatypes::primitives::TimeInterval::end, + ) + } + } else { + TimeInstance::MIN + }; + + let r_end = if let Some(last) = intersecting_times.last() { + if last.end() >= query.time().end() { + last.end() + } else { + let follow = unique_times + .iter() + .filter(|t| t.start() >= query.time().end()) + .nth(0); + follow.map_or( + TimeInstance::MAX, + geoengine_datatypes::primitives::TimeInterval::start, + ) + } + } else { + TimeInstance::MAX + }; + + let fill_range = TimeInterval::new(r_start, r_end)?; + + let times = intersecting_times + .into_iter() + .map(Result::Ok) + .try_time_irregular_range_fill(fill_range); // Fills min max stream::iter(times).boxed() } geoengine_datatypes::primitives::TimeDimension::Regular(regular_dim) => { - let times = unique_times.try_time_regular_range_fill(regular_dim, query.time()); + let times = intersecting_times + .into_iter() + .map(Result::Ok) + .try_time_regular_range_fill(regular_dim, query.time()); stream::iter(times).boxed() } }; @@ -415,7 +457,9 @@ mod tests { MockExecutionContext, QueryProcessor, RasterBandDescriptors, SpatialGridDescriptor, TimeDescriptor, }; - use geoengine_datatypes::primitives::{BandSelection, CacheHint, TimeInterval, TimeStep}; + use geoengine_datatypes::primitives::{ + BandSelection, CacheHint, TimeInstance, TimeInterval, TimeStep, + }; use geoengine_datatypes::raster::{ BoundedGrid, GeoTransform, Grid, Grid2D, GridBoundingBox2D, MaskedGrid, RasterDataType, RasterProperties, TileInformation, @@ -611,6 +655,7 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: TimeDescriptor::new_regular_with_epoch( + // This is a regular TimeDescriptor! Some(TimeInterval::new_unchecked(1, 3)), TimeStep::millis(1).unwrap(), ), @@ -650,6 +695,134 @@ mod tests { let result = result_stream.map(Result::unwrap).collect::>().await; + assert_eq!( + result.iter().map(|tile| tile.time).collect::>(), + [ + TimeInterval::new_unchecked(0, 1), + TimeInterval::new_unchecked(0, 1), + TimeInterval::new_unchecked(1, 2), + TimeInterval::new_unchecked(1, 2), + TimeInterval::new_unchecked(2, 3), + TimeInterval::new_unchecked(2, 3), + TimeInterval::new_unchecked(3, 4), + TimeInterval::new_unchecked(3, 4), + ] + ); + + // QUERY 2 + + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(2, 4), + BandSelection::first(), + ); + + let result_stream = query_processor.query(query_rect, &query_ctx).await.unwrap(); + + let result = result_stream.map(Result::unwrap).collect::>().await; + + assert_eq!( + result.iter().map(|tile| tile.time).collect::>(), + [ + TimeInterval::new_unchecked(2, 3), + TimeInterval::new_unchecked(2, 3), + TimeInterval::new_unchecked(3, 4), + TimeInterval::new_unchecked(3, 4), + ] + ); + } + + #[allow(clippy::too_many_lines)] + #[tokio::test] + async fn query_interval_larger_then_data_range_irregular() { + let raster_source = MockRasterSource { + params: MockRasterSourceParams:: { + data: vec![ + RasterTile2D { + time: TimeInterval::new_unchecked(1, 2), + tile_position: [-1, 0].into(), + band: 0, + global_geo_transform: TestDefault::test_default(), + grid_array: Grid::new([3, 2].into(), vec![1, 2, 3, 4, 5, 6]) + .unwrap() + .into(), + properties: RasterProperties::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(1, 2), + tile_position: [-1, 1].into(), + band: 0, + global_geo_transform: TestDefault::test_default(), + grid_array: Grid::new([3, 2].into(), vec![7, 8, 9, 10, 11, 12]) + .unwrap() + .into(), + properties: RasterProperties::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(2, 3), + tile_position: [-1, 0].into(), + band: 0, + global_geo_transform: TestDefault::test_default(), + grid_array: Grid::new([3, 2].into(), vec![13, 14, 15, 16, 17, 18]) + .unwrap() + .into(), + properties: RasterProperties::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(2, 3), + tile_position: [-1, 1].into(), + band: 0, + global_geo_transform: TestDefault::test_default(), + grid_array: Grid::new([3, 2].into(), vec![19, 20, 21, 22, 23, 24]) + .unwrap() + .into(), + properties: RasterProperties::default(), + cache_hint: CacheHint::default(), + }, + ], + result_descriptor: RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(None), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), + }, + }, + } + .boxed(); + + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + + let query_processor = raster_source + .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) + .await + .unwrap() + .query_processor() + .unwrap() + .get_u8() + .unwrap(); + + let query_ctx = execution_context.mock_query_context_test_default(); + + // QUERY 1 + + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 4), + BandSelection::first(), + ); + + let result_stream = query_processor.query(query_rect, &query_ctx).await.unwrap(); + + let result = result_stream.map(Result::unwrap).collect::>().await; + assert_eq!( result.iter().map(|tile| tile.time).collect::>(), [ diff --git a/operators/src/processing/interpolation/mod.rs b/operators/src/processing/interpolation/mod.rs index a22d0405b..587ebd211 100644 --- a/operators/src/processing/interpolation/mod.rs +++ b/operators/src/processing/interpolation/mod.rs @@ -675,8 +675,8 @@ mod tests { Coordinate2D, RasterQueryRectangle, SpatialResolution, TimeInterval, TimeStep, }, raster::{ - Grid2D, GridOrEmpty, RasterDataType, RasterTile2D, RenameBands, TileInformation, - TilingSpecification, + Grid2D, GridOrEmpty, GridShape2D, RasterDataType, RasterTile2D, RenameBands, + TileInformation, TilingSpecification, }, spatial_reference::SpatialReference, util::test::TestDefault, @@ -793,6 +793,7 @@ mod tests { Ok(()) } + #[allow(clippy::too_many_lines)] fn make_raster(cache_hint: CacheHint) -> Box { // test raster: // [0, 10) @@ -803,6 +804,62 @@ mod tests { // || 8 | 7 || 6 | 5 || // || 4 | 3 || 2 | 1 || let raster_tiles = vec![ + // we need to add no-data tiles explicit to force the cahe_hint from the mock source! + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(0, 10), + TileInformation { + global_tile_position: [-2, -1].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(0, 10), + TileInformation { + global_tile_position: [-2, 0].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(0, 10), + TileInformation { + global_tile_position: [-2, 1].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(0, 10), + TileInformation { + global_tile_position: [-2, 2].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(0, 10), + TileInformation { + global_tile_position: [-1, -1].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), RasterTile2D::::new_with_tile_info( TimeInterval::new_unchecked(0, 10), TileInformation { @@ -825,6 +882,116 @@ mod tests { GridOrEmpty::from(Grid2D::new([2, 2].into(), vec![3, 4, 7, 8]).unwrap()), cache_hint, ), + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(0, 10), + TileInformation { + global_tile_position: [-1, 2].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(0, 10), + TileInformation { + global_tile_position: [0, -1].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(0, 10), + TileInformation { + global_tile_position: [0, 0].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(0, 10), + TileInformation { + global_tile_position: [0, 1].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(0, 10), + TileInformation { + global_tile_position: [0, 2].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(10, 20), + TileInformation { + global_tile_position: [-2, -1].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(10, 20), + TileInformation { + global_tile_position: [-2, 0].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(10, 20), + TileInformation { + global_tile_position: [-2, 1].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(10, 20), + TileInformation { + global_tile_position: [-2, 2].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(10, 20), + TileInformation { + global_tile_position: [-1, -1].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), RasterTile2D::new_with_tile_info( TimeInterval::new_unchecked(10, 20), TileInformation { @@ -847,6 +1014,61 @@ mod tests { GridOrEmpty::from(Grid2D::new([2, 2].into(), vec![6, 5, 2, 1]).unwrap()), cache_hint, ), + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(10, 20), + TileInformation { + global_tile_position: [-1, 2].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(10, 20), + TileInformation { + global_tile_position: [0, -1].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(10, 20), + TileInformation { + global_tile_position: [0, 0].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(10, 20), + TileInformation { + global_tile_position: [0, 1].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), + RasterTile2D::new_with_tile_info( + TimeInterval::new_unchecked(10, 20), + TileInformation { + global_tile_position: [0, 2].into(), + tile_size_in_pixels: [2, 2].into(), + global_geo_transform: TestDefault::test_default(), + }, + 0, + GridOrEmpty::new_empty_shape(GridShape2D::new([2, 2])), + cache_hint, + ), ]; let result_descriptor = RasterResultDescriptor { diff --git a/operators/src/processing/neighborhood_aggregate/mod.rs b/operators/src/processing/neighborhood_aggregate/mod.rs index d652be7d1..969a3bb55 100644 --- a/operators/src/processing/neighborhood_aggregate/mod.rs +++ b/operators/src/processing/neighborhood_aggregate/mod.rs @@ -4,7 +4,7 @@ mod tile_sub_query; use self::aggregate::{AggregateFunction, Neighborhood, StandardDeviation, Sum}; use self::tile_sub_query::NeighborhoodAggregateTileNeighborhood; use crate::adapters::RasterSubQueryAdapter; -use crate::adapters::stack_individual_aligned_raster_bands; +use crate::adapters::SimpleRasterStackerAdapter; use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, Operator, OperatorName, QueryContext, QueryProcessor, RasterOperator, RasterQueryProcessor, @@ -13,6 +13,7 @@ use crate::engine::{ use crate::optimization::OptimizationError; use crate::util::Result; use async_trait::async_trait; +use futures::StreamExt; use futures::stream::BoxStream; use geoengine_datatypes::primitives::{BandSelection, RasterQueryRectangle, SpatialResolution}; use geoengine_datatypes::raster::{ @@ -279,29 +280,35 @@ where query: RasterQueryRectangle, ctx: &'a dyn QueryContext, ) -> Result>> { - stack_individual_aligned_raster_bands(&query, ctx, |query, ctx| async move { - let sub_query = - NeighborhoodAggregateTileNeighborhood::::new(self.neighborhood.clone()); - - let time_stream = self.source.time_query(query.time_interval(), ctx).await?; - - let tiling_strat = self - .source - .result_descriptor() - .tiling_grid_definition(self.tiling_specification) - .generate_data_tiling_strategy(); - - let sq = RasterSubQueryAdapter::<'a, P, _, _, _>::new( - &self.source, - query, - tiling_strat, - ctx, - sub_query, - time_stream, - ); - Ok(sq.box_pin()) - }) + SimpleRasterStackerAdapter::stack_individual_aligned_raster_bands( + &query, + ctx, + |query, ctx| async move { + let sub_query = + NeighborhoodAggregateTileNeighborhood::::new(self.neighborhood.clone()); + + let time_stream = self.source.time_query(query.time_interval(), ctx).await?; + + let tiling_strat = self + .source + .result_descriptor() + .tiling_grid_definition(self.tiling_specification) + .generate_data_tiling_strategy(); + + let sq = RasterSubQueryAdapter::<'a, P, _, _, _>::new( + &self.source, + query, + tiling_strat, + ctx, + sub_query, + time_stream, + ); + Ok(sq.box_pin()) + }, + ) .await + .map_err(|e| crate::error::Error::SimpleRasterStacker { source: e }) + .map(StreamExt::boxed) } fn result_descriptor(&self) -> &RasterResultDescriptor { diff --git a/operators/src/processing/raster_stacker.rs b/operators/src/processing/raster_stacker.rs index 15b792b07..4ffe43b97 100644 --- a/operators/src/processing/raster_stacker.rs +++ b/operators/src/processing/raster_stacker.rs @@ -1,4 +1,7 @@ -use crate::adapters::{QueryWrapper, RasterStackerAdapter, RasterStackerSource}; +use crate::adapters::{ + PartialQueryRect, QueryWrapper, RasterStackerAdapter, RasterStackerSource, + SimpleRasterStackerAdapter, SimpleRasterStackerError, +}; use crate::engine::{ BoxRasterQueryProcessor, CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, MultipleRasterSources, Operator, OperatorName, QueryContext, @@ -11,6 +14,7 @@ use crate::error::{ use crate::optimization::OptimizationError; use crate::util::Result; use async_trait::async_trait; +use futures::StreamExt; use futures::stream::BoxStream; use geoengine_datatypes::primitives::{BandSelection, RasterQueryRectangle, SpatialResolution}; use geoengine_datatypes::raster::{ @@ -313,30 +317,6 @@ impl RasterStackerProcessor { } } -/// compute the bands in the input source from the bands in a query that uses multiple sources -fn map_query_bands_to_source_bands( - query_bands: &BandSelection, - bands_per_source: &[u32], - source_index: usize, -) -> Option { - let source_start: u32 = bands_per_source.iter().take(source_index).sum(); - let source_bands = bands_per_source[source_index]; - let source_end = source_start + source_bands; - - let bands = query_bands - .as_slice() - .iter() - .filter(|output_band| **output_band >= source_start && **output_band < source_end) - .map(|output_band| output_band - source_start) - .collect::>(); - - if bands.is_empty() { - return None; - } - - Some(BandSelection::new_unchecked(bands)) -} - #[async_trait] impl QueryProcessor for RasterStackerProcessor where @@ -352,14 +332,36 @@ where query: RasterQueryRectangle, ctx: &'a dyn QueryContext, ) -> Result>>> { + // First try to create simple raster stacker for temporal aligned data + let sdp = SimpleRasterStackerAdapter::< + SimpleRasterStackerAdapter>, + >::stack_selected_regular_aligned_raster_bands(&query, ctx, &self.sources) + .await; + + let x = match sdp { + Ok(p) => Ok(Some(p)), + Err(SimpleRasterStackerError::InputsNotTemporalAligned) => Ok(None), + Err(e) => Err(crate::error::Error::SimpleRasterStacker { source: e }), + }?; + + if let Some(sdp) = x { + tracing::trace!("Using regular time aligned stacker processor"); + return Ok(Box::pin(sdp)); + } + + // if the simple stacker can not be used, try to use the more complex stacker + + tracing::trace!("Using non-regular time aligned stacker processor"); + let mut sources = vec![]; + let tiling_strat = self + .result_descriptor + .tiling_grid_definition(ctx.tiling_specification()) + .generate_data_tiling_strategy(); for (idx, source) in self.sources.iter().enumerate() { - let Some(bands) = - map_query_bands_to_source_bands(query.attributes(), &self.bands_per_source, idx) - else { - continue; - }; + // FIXME: find a better way to do the selection and avoid work done without benefit. + let bands = BandSelection::first_n(self.bands_per_source[idx]); sources.push(RasterStackerSource { queryable: QueryWrapper { p: source, ctx }, @@ -367,7 +369,29 @@ where }); } - let output = RasterStackerAdapter::new(sources, query.into()); + #[cfg(debug_assertions)] + { + let num_input_bands = self.bands_per_source.iter().sum::() as usize; + let num_query_bands = query.attributes().as_vec().len(); + + let fact = num_input_bands as f32 / num_query_bands as f32; + + tracing::debug!( + "StackerAdapter queries {num_input_bands} to produce {num_query_bands}. This is {fact}x the work required." + ); + } + + let query_band_selection = query.attributes().clone(); + let partial_query = PartialQueryRect::from(query); + let output = + RasterStackerAdapter::new(sources, partial_query, tiling_strat).filter_map(move |o| { + let pred = match o { + Ok(tile) if query_band_selection.contains(tile.band) => Some(Ok(tile)), + Ok(_) => None, + Err(e) => Some(Err(e)), + }; + std::future::ready(pred) + }); Ok(Box::pin(output)) } @@ -412,7 +436,7 @@ mod tests { TilesEqualIgnoringCacheHint, }, spatial_reference::SpatialReference, - util::test::TestDefault, + util::test::{TestDefault, assert_eq_two_list_of_tiles}, }; use crate::{ @@ -428,31 +452,23 @@ mod tests { use super::*; - #[test] - fn it_maps_query_bands_to_source_bands() { - assert_eq!( - map_query_bands_to_source_bands(&0.into(), &[2, 1], 0), - Some(0.into()) - ); - assert_eq!(map_query_bands_to_source_bands(&0.into(), &[2, 1], 1), None); - assert_eq!( - map_query_bands_to_source_bands(&2.into(), &[2, 1], 1), - Some(0.into()) - ); - - assert_eq!( - map_query_bands_to_source_bands(&[1, 2].try_into().unwrap(), &[2, 2], 0), - Some(1.into()) - ); - assert_eq!( - map_query_bands_to_source_bands(&[1, 2, 3].try_into().unwrap(), &[2, 2], 1), - Some([0, 1].try_into().unwrap()) - ); + #[tokio::test] + async fn it_stacks() { + it_stacks_impl(crate::engine::TimeDescriptor::new_irregular(None)).await; } #[tokio::test] + + async fn it_stacks_regular() { + it_stacks_impl(crate::engine::TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 5)), + TimeStep::millis(5).unwrap(), + )) + .await; + } + #[allow(clippy::too_many_lines)] - async fn it_stacks() { + async fn it_stacks_impl(time_desc: crate::engine::TimeDescriptor) { let data: Vec> = vec![ RasterTile2D { time: TimeInterval::new_unchecked(0, 5), @@ -541,13 +557,10 @@ mod tests { }, ]; - let result_descriptor = RasterResultDescriptor { + let result_descriptor1 = RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: crate::engine::TimeDescriptor::new_regular_with_epoch( - Some(TimeInterval::new_unchecked(0, 5)), - TimeStep::millis(10).unwrap(), - ), + time: time_desc, spatial_grid: SpatialGridDescriptor::source_from_parts( GeoTransform::test_default(), GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), @@ -558,7 +571,7 @@ mod tests { let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: result_descriptor.clone(), + result_descriptor: result_descriptor1.clone(), }, } .boxed(); @@ -566,7 +579,7 @@ mod tests { let mrs2 = MockRasterSource { params: MockRasterSourceParams { data: data2.clone(), - result_descriptor, + result_descriptor: result_descriptor1, }, } .boxed(); @@ -622,8 +635,21 @@ mod tests { } #[tokio::test] - #[allow(clippy::too_many_lines)] async fn it_stacks_stacks() { + it_stacks_stacks_impl(crate::engine::TimeDescriptor::new_irregular(None)).await; + } + + #[tokio::test] + async fn it_stacks_stacks_regular() { + it_stacks_stacks_impl(crate::engine::TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 10)), + TimeStep::millis(5).unwrap(), + )) + .await; + } + + #[allow(clippy::too_many_lines)] + async fn it_stacks_stacks_impl(time_desc: crate::engine::TimeDescriptor) { let data: Vec> = vec![ RasterTile2D { time: TimeInterval::new_unchecked(0, 5), @@ -797,10 +823,7 @@ mod tests { let result_descriptor = RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: crate::engine::TimeDescriptor::new_regular_with_epoch( - Some(TimeInterval::new_unchecked(0, 10)), - TimeStep::millis(10).unwrap(), - ), + time: time_desc, spatial_grid: SpatialGridDescriptor::source_from_parts( GeoTransform::test_default(), GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), @@ -886,8 +909,21 @@ mod tests { } #[tokio::test] - #[allow(clippy::too_many_lines)] async fn it_selects_band_from_stack() { + it_selects_band_from_stack_impl(crate::engine::TimeDescriptor::new_irregular(None)).await; + } + + #[tokio::test] + async fn it_selects_band_from_stack_regular() { + it_selects_band_from_stack_impl(crate::engine::TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 10)), + TimeStep::millis(5).unwrap(), + )) + .await; + } + + #[allow(clippy::too_many_lines)] + async fn it_selects_band_from_stack_impl(time_desc: crate::engine::TimeDescriptor) { let data: Vec> = vec![ RasterTile2D { time: TimeInterval::new_unchecked(0, 5), @@ -979,10 +1015,7 @@ mod tests { let result_descriptor = RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: crate::engine::TimeDescriptor::new_regular_with_epoch( - Some(TimeInterval::new_unchecked(0, 10)), - TimeStep::millis(10).unwrap(), - ), + time: time_desc, spatial_grid: SpatialGridDescriptor::source_from_parts( GeoTransform::test_default(), GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), @@ -1044,7 +1077,16 @@ mod tests { .await; let result = result.into_iter().collect::>>().unwrap(); - assert!(data2.tiles_equal_ignoring_cache_hint(&result)); + let expected_band1: Vec<_> = data2 + .iter() + .map(|t| { + let mut t_1 = t.clone(); + t_1.band = 1; + t_1 + }) + .collect(); + + assert_eq_two_list_of_tiles(&result, &expected_band1, false); } #[tokio::test] @@ -1096,11 +1138,6 @@ mod tests { let processor = operator.query_processor().unwrap().get_u8().unwrap(); - let mut exe_ctx = MockExecutionContext::test_default(); - exe_ctx.tiling_specification.tile_size_in_pixels = GridShape { - shape_array: [2, 2], - }; - let query_ctx = exe_ctx.mock_query_context_test_default(); // query both bands @@ -1139,9 +1176,10 @@ mod tests { .unwrap() .collect::>() .await; - let result = result.into_iter().collect::>>().unwrap(); + let result_0 = result.into_iter().collect::>>().unwrap(); - assert!(!result.is_empty()); + assert!(!result_0.is_empty()); + assert!(result_0.iter().all(|t| t.band == 0)); // query only second band let query_rect = RasterQueryRectangle::new( @@ -1159,9 +1197,12 @@ mod tests { .unwrap() .collect::>() .await; - let result = result.into_iter().collect::>>().unwrap(); + let result_1 = result.into_iter().collect::>>().unwrap(); - assert!(!result.is_empty()); + assert!(!result_1.is_empty()); + assert!(result_1.iter().all(|t| t.band == 1)); + + assert_eq!(result_0.len(), result_1.len()); } #[test] diff --git a/operators/src/processing/raster_vector_join/aggregated.rs b/operators/src/processing/raster_vector_join/aggregated.rs index 165e97974..05833d6a9 100644 --- a/operators/src/processing/raster_vector_join/aggregated.rs +++ b/operators/src/processing/raster_vector_join/aggregated.rs @@ -106,7 +106,7 @@ where let raster_query = RasterQueryRectangle::new( pixel_bounds, time_span.time_interval, - BandSelection::first(), // FIXME: this should prop. use all bands? + BandSelection::first_n(rd.bands.count()), ); let mut rasters = raster_processor.raster_query(raster_query, ctx).await?; diff --git a/operators/src/processing/raster_vector_join/non_aggregated.rs b/operators/src/processing/raster_vector_join/non_aggregated.rs index 064af12f7..746b6087c 100644 --- a/operators/src/processing/raster_vector_join/non_aggregated.rs +++ b/operators/src/processing/raster_vector_join/non_aggregated.rs @@ -139,7 +139,7 @@ where let query = RasterQueryRectangle::new( pixel_bounds, time_interval, - BandSelection::first_n(column_names.len() as u32), + BandSelection::first_n(rd.bands.count()), ); call_on_generic_raster_processor!(raster_processor, raster_processor => { @@ -985,7 +985,7 @@ mod tests { spatial_reference: SpatialReference::epsg_4326().into(), time: TimeDescriptor::new_regular_with_epoch( Some(TimeInterval::new(0, 20).unwrap()), - TimeStep::seconds(10).unwrap(), + TimeStep::millis(10).unwrap(), ), spatial_grid: SpatialGridDescriptor::source_from_parts( GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), diff --git a/operators/src/processing/temporal_raster_aggregation/temporal_aggregation_operator.rs b/operators/src/processing/temporal_raster_aggregation/temporal_aggregation_operator.rs index 81b4ecf66..e931204a5 100644 --- a/operators/src/processing/temporal_raster_aggregation/temporal_aggregation_operator.rs +++ b/operators/src/processing/temporal_raster_aggregation/temporal_aggregation_operator.rs @@ -9,7 +9,7 @@ use super::first_last_subquery::{ TemporalRasterAggregationSubQueryNoDataOnly, first_tile_fold_future, last_tile_fold_future, }; use super::subquery::GlobalStateTemporalRasterAggregationSubQuery; -use crate::adapters::stack_individual_aligned_raster_bands; +use crate::adapters::SimpleRasterStackerAdapter; use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedSources, Operator, QueryProcessor, RasterOperator, SingleRasterSource, TimeDescriptor, WorkflowOperatorPath, @@ -503,11 +503,17 @@ where query: RasterQueryRectangle, ctx: &'a dyn crate::engine::QueryContext, ) -> Result>> { - stack_individual_aligned_raster_bands(&query, ctx, |query, ctx| async { - self.create_subquery_adapter_stream_for_single_band(query, ctx) - .await - }) + SimpleRasterStackerAdapter::stack_individual_aligned_raster_bands( + &query, + ctx, + |query, ctx| async { + self.create_subquery_adapter_stream_for_single_band(query, ctx) + .await + }, + ) .await + .map_err(|e| crate::error::Error::SimpleRasterStacker { source: e }) + .map(StreamExt::boxed) } fn result_descriptor(&self) -> &Self::ResultDescription { diff --git a/operators/src/util/raster_stream_to_geotiff.rs b/operators/src/util/raster_stream_to_geotiff.rs index 359a8920a..d4a8f9af5 100644 --- a/operators/src/util/raster_stream_to_geotiff.rs +++ b/operators/src/util/raster_stream_to_geotiff.rs @@ -1613,11 +1613,7 @@ mod tests { 1, GridBoundingBox2D::new([-4, -4], [4, 4]).unwrap(), GeoTransform::test_default(), - TimeDescriptor::new_regular( - Some(time_bounds), - time_bounds.start(), - time_step.try_into().unwrap(), - ), + TimeDescriptor::new_irregular(Some(time_bounds)), ); let query_time = TimeInterval::new(data[0].time.start(), data[1].time.end()).unwrap(); diff --git a/operators/src/util/raster_stream_to_png.rs b/operators/src/util/raster_stream_to_png.rs index 0bbf08ffc..6c87b71a9 100644 --- a/operators/src/util/raster_stream_to_png.rs +++ b/operators/src/util/raster_stream_to_png.rs @@ -138,6 +138,11 @@ async fn single_band_colorizer_to_png_bytes let output_tile: BoxFuture, CacheHint)>> = Box::pin(tile_stream.fold(accu, |accu, tile| { + #[cfg(debug_assertions)] + if let Err(er) = tile.as_ref() { + tracing::debug!("Error tile passed to single_[..]_to_png: {er}"); + } + let result: Result<(GridOrEmpty, CacheHint)> = blit_tile(accu, tile); @@ -179,13 +184,17 @@ async fn multi_band_colorizer_to_png_bytes( let output_tile = Box::pin(tile_stream.try_chunks(rgb_channel_count).fold( accu, |raster2d, chunk| async move { + #[cfg(debug_assertions)] + if let Err(er) = chunk.as_ref() { + tracing::debug!("Error chunk passed to multi_[..]_to_png: {er}"); + } + let chunk = chunk.boxed_context(error::QueryDidNotProduceNextChunk)?; if chunk.len() != rgb_channel_count { return Err(PngCreationError::RgbChunkIsNotEnoughBands)?; } - // TODO: spawn blocking task let rgb_tile = crate::util::spawn_blocking(move || { compute_rgb_tile( [ diff --git a/services/src/datasets/external/netcdfcf/mod.rs b/services/src/datasets/external/netcdfcf/mod.rs index 4f4e45daf..ffb7386c9 100644 --- a/services/src/datasets/external/netcdfcf/mod.rs +++ b/services/src/datasets/external/netcdfcf/mod.rs @@ -2283,7 +2283,7 @@ mod tests { }, sources: Expression { params: ExpressionParams { - expression: "if A is NODATA {NODATA} else {A}".to_string(), // FIXME: was "A" because nodata pixels would be skipped. --> The landcover pixels overlapping are NODATA, but why? + expression: "if A is NODATA {NODATA} else {A}".to_string(), // FIXME: was "A" because nodata pixels would be skipped. --> The landcover pixels overlapping are NODATA, but why? Because ther is something wrong here! output_type: RasterDataType::F64, output_band: None, map_no_data: true,