Skip to content

Commit 0adef45

Browse files
mykhailo-kuchmamykhailo-kuchma
authored
Fix the prefetch for the aggregated tiles. (#1083)
Fix the prefetch API behavior for the aggregated tiles when the aggregated tile is too far away. A new approach is to download and store quadtrees of lower levels until the needed aggregated tile is placed in sub quads (which means that we can find it later). Relates-To: OLPEDGE-2312 Signed-off-by: Mykhailo Kuchma <[email protected]>
1 parent 65bd7a0 commit 0adef45

13 files changed

+393
-161
lines changed

‎olp-cpp-sdk-dataservice-read/src/ProtectDependencyResolver.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ bool ProtectDependencyResolver::AddDataHandle(
8686
bool ProtectDependencyResolver::ProcessTileKeyInCache(
8787
const geo::TileKey& tile) {
8888
read::QuadTreeIndex cached_tree;
89-
if (partitions_cache_repository_.FindQuadTree(version_, tile, cached_tree) &&
89+
if (partitions_cache_repository_.FindQuadTree(tile, version_, cached_tree) &&
9090
AddDataHandle(tile, cached_tree)) {
9191
auto root_tile = cached_tree.GetRootTile();
9292
// add quad tree to list for protection

‎olp-cpp-sdk-dataservice-read/src/VersionedLayerClientImpl.cpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -393,9 +393,12 @@ client::CancellationToken VersionedLayerClientImpl::PrefetchTiles(
393393
OLP_SDK_LOG_DEBUG_F(kLogTag, "PrefetchTiles, subquads=%zu, key=%s",
394394
sliced_tiles.size(), key.c_str());
395395

396+
const bool aggregation_enabled = request.GetDataAggregationEnabled();
397+
396398
auto query = [=](geo::TileKey root,
397399
client::CancellationContext inner_context) mutable {
398400
return repository.GetVersionedSubQuads(root, kQuadTreeDepth, version,
401+
aggregation_enabled,
399402
inner_context);
400403
};
401404

@@ -607,7 +610,7 @@ bool VersionedLayerClientImpl::RemoveFromCache(const geo::TileKey& tile) {
607610
return false;
608611
}
609612

610-
if (partitions_cache_repository.FindQuadTree(version, tile, cached_tree)) {
613+
if (partitions_cache_repository.FindQuadTree(tile, version, cached_tree)) {
611614
auto data = cached_tree.Find(tile, false);
612615
if (!data) {
613616
return true;
@@ -668,7 +671,7 @@ bool VersionedLayerClientImpl::IsCached(const geo::TileKey& tile,
668671
repository::PartitionsCacheRepository partitions_repo(catalog_, layer_id_,
669672
cache);
670673

671-
if (partitions_repo.FindQuadTree(version, tile, cached_tree)) {
674+
if (partitions_repo.FindQuadTree(tile, version, cached_tree)) {
672675
auto data = cached_tree.Find(tile, aggregated);
673676
if (data) {
674677
repository::DataCacheRepository data_repo(catalog_, cache);

‎olp-cpp-sdk-dataservice-read/src/VolatileLayerClientImpl.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919

2020
#include "VolatileLayerClientImpl.h"
2121

22+
#include <iterator>
23+
2224
#include <olp/core/cache/DefaultCache.h>
2325
#include <olp/core/client/CancellationContext.h>
2426
#include <olp/core/client/OlpClientSettingsFactory.h>

‎olp-cpp-sdk-dataservice-read/src/repositories/PartitionsCacheRepository.cpp‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ void PartitionsCacheRepository::Put(const model::Partitions& partitions,
9191
OLP_SDK_LOG_DEBUG_F(kLogTag, "Put -> '%s'", key.c_str());
9292

9393
cache_->Put(key, partition,
94-
[&]() { return olp::serializer::serialize(partition); },
94+
[&]() { return serializer::serialize(partition); },
9595
expiry.get_value_or(default_expiry_));
9696

9797
if (layer_metadata) {
@@ -104,7 +104,7 @@ void PartitionsCacheRepository::Put(const model::Partitions& partitions,
104104
OLP_SDK_LOG_DEBUG_F(kLogTag, "Put -> '%s'", key.c_str());
105105

106106
cache_->Put(key, partition_ids,
107-
[&]() { return olp::serializer::serialize(partition_ids); },
107+
[&]() { return serializer::serialize(partition_ids); },
108108
expiry.get_value_or(default_expiry_));
109109
}
110110
}
@@ -173,7 +173,7 @@ void PartitionsCacheRepository::Put(
173173
OLP_SDK_LOG_DEBUG_F(kLogTag, "Put -> '%s'", key.c_str());
174174

175175
cache_->Put(key, layer_versions,
176-
[&]() { return olp::serializer::serialize(layer_versions); },
176+
[&]() { return serializer::serialize(layer_versions); },
177177
default_expiry_);
178178
}
179179

@@ -291,26 +291,25 @@ bool PartitionsCacheRepository::GetPartitionHandle(
291291
}
292292

293293
std::string PartitionsCacheRepository::CreateQuadKey(
294-
olp::geo::TileKey key, int32_t depth,
294+
geo::TileKey key, int32_t depth,
295295
const boost::optional<int64_t>& version) const {
296296
return catalog_ + "::" + layer_id_ + "::" + key.ToHereTile() +
297297
"::" + (version ? std::to_string(*version) + "::" : "") +
298298
std::to_string(depth) + "::quadtree";
299299
}
300300

301-
bool PartitionsCacheRepository::FindQuadTree(boost::optional<int64_t> version,
302-
const olp::geo::TileKey& tile_key,
301+
bool PartitionsCacheRepository::FindQuadTree(geo::TileKey key,
302+
boost::optional<int64_t> version,
303303
read::QuadTreeIndex& tree) {
304-
auto max_depth =
305-
std::min<std::uint32_t>(tile_key.Level(), kMaxQuadTreeIndexDepth);
304+
auto max_depth = std::min<std::uint32_t>(key.Level(), kMaxQuadTreeIndexDepth);
306305
for (auto i = 0u; i <= max_depth; ++i) {
307-
const auto& root_tile_key = tile_key.ChangedLevelBy(-i);
306+
const auto& root_tile_key = key.ChangedLevelBy(-i);
308307
QuadTreeIndex cached_tree;
309308
if (Get(root_tile_key, kMaxQuadTreeIndexDepth, version, cached_tree)) {
310309
OLP_SDK_LOG_DEBUG_F(kLogTag,
311310
"FindQuadTree found in cache, tile='%s', "
312311
"root='%s', depth='%" PRId32 "'",
313-
tile_key.ToHereTile().c_str(),
312+
key.ToHereTile().c_str(),
314313
root_tile_key.ToHereTile().c_str(),
315314
kMaxQuadTreeIndexDepth);
316315
tree = std::move(cached_tree);
@@ -322,6 +321,12 @@ bool PartitionsCacheRepository::FindQuadTree(boost::optional<int64_t> version,
322321
return false;
323322
}
324323

324+
bool PartitionsCacheRepository::ContainsTree(
325+
geo::TileKey key, int32_t depth,
326+
const boost::optional<int64_t>& version) const {
327+
return cache_->Contains(CreateQuadKey(key, depth, version));
328+
}
329+
325330
} // namespace repository
326331
} // namespace read
327332
} // namespace dataservice

‎olp-cpp-sdk-dataservice-read/src/repositories/PartitionsCacheRepository.h‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -83,13 +83,15 @@ class PartitionsCacheRepository final {
8383
const boost::optional<int64_t>& catalog_version,
8484
std::string& data_handle);
8585

86-
std::string CreateQuadKey(olp::geo::TileKey key, int32_t depth,
86+
std::string CreateQuadKey(geo::TileKey key, int32_t depth,
8787
const boost::optional<int64_t>& version) const;
8888

89-
bool FindQuadTree(boost::optional<int64_t> version,
90-
const olp::geo::TileKey& tile_key,
89+
bool FindQuadTree(geo::TileKey key, boost::optional<int64_t> version,
9190
read::QuadTreeIndex& tree);
9291

92+
bool ContainsTree(geo::TileKey key, int32_t depth,
93+
const boost::optional<int64_t>& version) const;
94+
9395
private:
9496
const std::string catalog_;
9597
const std::string layer_id_;

‎olp-cpp-sdk-dataservice-read/src/repositories/PartitionsRepository.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -333,7 +333,7 @@ QuadTreeIndexResponse PartitionsRepository::GetQuadTreeIndexForTile(
333333
// Look for QuadTree covering the tile in the cache
334334
if (fetch_option != OnlineOnly && fetch_option != CacheWithUpdate) {
335335
read::QuadTreeIndex cached_tree;
336-
if (cache_.FindQuadTree(version, tile_key, cached_tree)) {
336+
if (cache_.FindQuadTree(tile_key, version, cached_tree)) {
337337
OLP_SDK_LOG_DEBUG_F(kLogTag,
338338
"GetQuadTreeIndexForTile found in cache, "
339339
"tile='%s', depth='%" PRId32 "'",

‎olp-cpp-sdk-dataservice-read/src/repositories/PrefetchTilesRepository.cpp‎

Lines changed: 97 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,18 @@ namespace repository {
4444
namespace {
4545
constexpr auto kLogTag = "PrefetchTilesRepository";
4646
constexpr std::uint32_t kMaxQuadTreeIndexDepth = 4u;
47+
48+
SubQuadsResult FlattenTree(const QuadTreeIndex& tree) {
49+
SubQuadsResult result;
50+
auto index_data = tree.GetIndexData();
51+
std::transform(index_data.begin(), index_data.end(),
52+
std::inserter(result, result.end()),
53+
[](const QuadTreeIndex::IndexData& data) {
54+
return std::make_pair(data.tile_key, data.data_handle);
55+
});
56+
return result;
57+
}
58+
4759
} // namespace
4860

4961
PrefetchTilesRepository::PrefetchTilesRepository(
@@ -159,72 +171,57 @@ RootTilesForRequest PrefetchTilesRepository::GetSlicedTiles(
159171

160172
SubQuadsResponse PrefetchTilesRepository::GetVersionedSubQuads(
161173
geo::TileKey tile, int32_t depth, std::int64_t version,
162-
client::CancellationContext context) {
174+
bool aggregation_enabled, client::CancellationContext context) {
163175
OLP_SDK_LOG_TRACE_F(kLogTag, "GetSubQuads(%s, %" PRId64 ", %" PRId32 ")",
164176
tile.ToHereTile().c_str(), version, depth);
165-
auto get_sub_quads = [](const QuadTreeIndex& tree) -> SubQuadsResult {
166-
SubQuadsResult result;
167-
auto index_data = tree.GetIndexData();
168-
std::transform(
169-
index_data.begin(), index_data.end(),
170-
std::inserter(result, result.end()),
171-
[](const QuadTreeIndex::IndexData& data) -> SubQuadsResult::value_type {
172-
return {data.tile_key, data.data_handle};
173-
});
174-
return result;
175-
};
176-
177177
// check if quad tree with requested tile and depth already in cache
178-
QuadTreeIndex cached_tree;
179-
if (cache_repository_.Get(tile, depth, version, cached_tree)) {
178+
QuadTreeIndex quad_tree;
179+
client::NetworkStatistics network_stats;
180+
181+
if (cache_repository_.Get(tile, depth, version, quad_tree)) {
180182
OLP_SDK_LOG_DEBUG_F(kLogTag,
181183
"GetSubQuads found in cache, tile='%s', "
182184
"depth='%" PRId32 "'",
183185
tile.ToHereTile().c_str(), depth);
186+
} else {
187+
QuadTreeResponse response =
188+
DownloadVersionedQuadTree(tile, depth, version, context);
184189

185-
return get_sub_quads(cached_tree);
186-
}
190+
network_stats = GetNetworkStatistics(response);
187191

188-
auto query_api = lookup_client_.LookupApi("query", "v1",
189-
client::OnlineIfNotFound, context);
192+
if (!response.IsSuccessful()) {
193+
return {response.GetError(), network_stats};
194+
}
190195

191-
if (!query_api.IsSuccessful()) {
192-
return query_api.GetError();
196+
quad_tree = response.MoveResult();
193197
}
194198

195-
auto tile_key = tile.ToHereTile();
196-
197-
OLP_SDK_LOG_INFO_F(kLogTag,
198-
"GetSubQuads execute(%s, %" PRId64 ", %" PRId32 ")",
199-
tile_key.c_str(), version, depth);
200-
201-
auto quad_tree = QueryApi::QuadTreeIndex(query_api.GetResult(), layer_id_,
202-
tile_key, version, depth,
203-
boost::none, billing_tag_, context);
204-
205-
if (quad_tree.status != olp::http::HttpStatusCode::OK) {
206-
OLP_SDK_LOG_WARNING_F(kLogTag,
207-
"GetSubQuads failed(%s, %" PRId64 ", %" PRId32 ")",
208-
tile_key.c_str(), version, depth);
209-
return {{quad_tree.status, quad_tree.response.str()},
210-
quad_tree.GetNetworkStatistics()};
199+
// Currently there is no better way to correctly handle the prefetch of
200+
// aggregated tiles, we download parent trees until the tile or it's parent is
201+
// found in subtiles. In this way we make sure that all tiles within requested
202+
// tree have aggregated parent downloaded and cached. This may cause
203+
// additional or duplicate download request.
204+
if (aggregation_enabled) {
205+
auto root = quad_tree.GetRootTile();
206+
auto root_index = quad_tree.Find(root, true);
207+
if (root_index) {
208+
const auto& aggregated_tile_key = root_index->tile_key;
209+
210+
while (root.Level() > aggregated_tile_key.Level()) {
211+
root = root.ChangedLevelBy(-kMaxQuadTreeIndexDepth - 1);
212+
213+
if (!cache_repository_.ContainsTree(root, kMaxQuadTreeIndexDepth,
214+
version)) {
215+
QuadTreeResponse response =
216+
DownloadVersionedQuadTree(root, depth, version, context);
217+
218+
network_stats += GetNetworkStatistics(response);
219+
}
220+
}
221+
}
211222
}
212223

213-
QuadTreeIndex tree(tile, depth, quad_tree.response);
214-
215-
if (tree.IsNull()) {
216-
OLP_SDK_LOG_WARNING_F(kLogTag,
217-
"QuadTreeIndex failed, hrn='%s', "
218-
"layer='%s', root='%s', version='%" PRId64
219-
"', depth='%" PRId32 "'",
220-
catalog_.ToString().c_str(), layer_id_.c_str(),
221-
tile_key.c_str(), version, depth);
222-
return {{client::ErrorCode::Unknown, "Failed to parse quad tree response"},
223-
quad_tree.GetNetworkStatistics()};
224-
}
225-
// add to cache
226-
cache_repository_.Put(tile, depth, tree, version);
227-
return {get_sub_quads(tree), quad_tree.GetNetworkStatistics()};
224+
return {FlattenTree(quad_tree), network_stats};
228225
}
229226

230227
SubQuadsResponse PrefetchTilesRepository::GetVolatileSubQuads(
@@ -368,6 +365,54 @@ SubQuadsResult PrefetchTilesRepository::FilterTilesByList(
368365
return tiles;
369366
}
370367

368+
PrefetchTilesRepository::QuadTreeResponse
369+
PrefetchTilesRepository::DownloadVersionedQuadTree(
370+
geo::TileKey tile, int32_t depth, std::int64_t version,
371+
client::CancellationContext context) {
372+
auto query_api = lookup_client_.LookupApi("query", "v1",
373+
client::OnlineIfNotFound, context);
374+
375+
if (!query_api.IsSuccessful()) {
376+
return query_api.GetError();
377+
}
378+
379+
auto tile_key = tile.ToHereTile();
380+
381+
OLP_SDK_LOG_INFO_F(kLogTag,
382+
"GetSubQuads execute(%s, %" PRId64 ", %" PRId32 ")",
383+
tile_key.c_str(), version, depth);
384+
385+
auto quad_tree = QueryApi::QuadTreeIndex(query_api.GetResult(), layer_id_,
386+
tile_key, version, depth,
387+
boost::none, billing_tag_, context);
388+
389+
if (quad_tree.status != olp::http::HttpStatusCode::OK) {
390+
OLP_SDK_LOG_WARNING_F(kLogTag,
391+
"GetSubQuads failed(%s, %" PRId64 ", %" PRId32 ")",
392+
tile_key.c_str(), version, depth);
393+
return {{quad_tree.status, quad_tree.response.str()},
394+
quad_tree.GetNetworkStatistics()};
395+
}
396+
397+
QuadTreeIndex tree(tile, depth, quad_tree.response);
398+
399+
if (tree.IsNull()) {
400+
OLP_SDK_LOG_WARNING_F(kLogTag,
401+
"QuadTreeIndex failed, hrn='%s', "
402+
"layer='%s', root='%s', version='%" PRId64
403+
"', depth='%" PRId32 "'",
404+
catalog_.ToString().c_str(), layer_id_.c_str(),
405+
tile_key.c_str(), version, depth);
406+
return {{client::ErrorCode::Unknown, "Failed to parse quad tree response"},
407+
quad_tree.GetNetworkStatistics()};
408+
}
409+
410+
// add to cache
411+
cache_repository_.Put(tile, depth, tree, version);
412+
413+
return {std::move(tree), quad_tree.GetNetworkStatistics()};
414+
}
415+
371416
} // namespace repository
372417
} // namespace read
373418
} // namespace dataservice

‎olp-cpp-sdk-dataservice-read/src/repositories/PrefetchTilesRepository.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ class PrefetchTilesRepository {
100100

101101
SubQuadsResponse GetVersionedSubQuads(geo::TileKey tile, int32_t depth,
102102
std::int64_t version,
103+
bool aggregation_enabled,
103104
client::CancellationContext context);
104105

105106
SubQuadsResponse GetVolatileSubQuads(geo::TileKey tile, int32_t depth,
@@ -110,6 +111,13 @@ class PrefetchTilesRepository {
110111
RootTilesForRequest::iterator subtree_to_split,
111112
const geo::TileKey& tile_key, std::uint32_t min);
112113

114+
using QuadTreeResponse = ExtendedApiResponse<QuadTreeIndex, client::ApiError,
115+
client::NetworkStatistics>;
116+
117+
QuadTreeResponse DownloadVersionedQuadTree(
118+
geo::TileKey tile, int32_t depth, std::int64_t version,
119+
client::CancellationContext context);
120+
113121
private:
114122
client::HRN catalog_;
115123
std::string layer_id_;

‎tests/integration/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ set(OLP_SDK_INTEGRATIONAL_TESTS_SOURCES
2323
./olp-cpp-sdk-dataservice-read/VolatileLayerClientCacheTest.cpp
2424
./olp-cpp-sdk-dataservice-read/VersionedLayerClientCacheTest.cpp
2525
./olp-cpp-sdk-dataservice-read/VersionedLayerGetAggregatedDataTest.cpp
26+
./olp-cpp-sdk-dataservice-read/VersionedLayerPrefetch.cpp
27+
./olp-cpp-sdk-dataservice-read/VersionedLayerTestBase.cpp
2628
./olp-cpp-sdk-dataservice-read/CatalogClientTestBase.cpp
2729
./olp-cpp-sdk-dataservice-read/CatalogClientTest.cpp
2830
./olp-cpp-sdk-dataservice-read/HttpResponses.h

0 commit comments

Comments
 (0)