Skip to content

Commit 4d49b18

Browse files
author
Liubov Didkivska
authored
Add API to remove partition/tilekey from cache (#747)
* Add API to remove partition/tilekey. Add RemoveFromCache functions to VolatileLayerClient. Add integration test to check if partition is not longer avialible in cache after deletion. Add integration test to check if partition related to removed TileKey is not longer avialible in cache after deletion. Relates-To: OLPEDGE-1663 Signed-off-by: Liubov Didkivska <[email protected]>
1 parent 0f79d85 commit 4d49b18

File tree

8 files changed

+285
-5
lines changed

8 files changed

+285
-5
lines changed

‎olp-cpp-sdk-dataservice-read/include/olp/dataservice/read/VolatileLayerClient.h‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include <olp/core/client/CancellationToken.h>
2727
#include <olp/core/client/HRN.h>
2828
#include <olp/core/client/OlpClientSettings.h>
29+
#include <olp/core/geo/tiling/TileKey.h>
2930
#include <olp/dataservice/read/DataRequest.h>
3031
#include <olp/dataservice/read/PartitionsRequest.h>
3132
#include <olp/dataservice/read/Types.h>
@@ -175,6 +176,24 @@ class DATASERVICE_READ_API VolatileLayerClient final {
175176
*/
176177
olp::client::CancellableFuture<DataResponse> GetData(DataRequest request);
177178

179+
/**
180+
* @brief Removes the partition from the mutable disk cache.
181+
*
182+
* @param partition_id The partition ID that should be removed.
183+
*
184+
* @return True if partition data is removed successfully; false otherwise.
185+
*/
186+
bool RemoveFromCache(const std::string& partition_id);
187+
188+
/**
189+
* @brief Removes the tile from the mutable disk cache.
190+
*
191+
* @param tile The tile key that should be removed.
192+
*
193+
* @return True if tile data is removed successfully; false otherwise.
194+
*/
195+
bool RemoveFromCache(const geo::TileKey& tile);
196+
178197
private:
179198
std::unique_ptr<VolatileLayerClientImpl> impl_;
180199
};

‎olp-cpp-sdk-dataservice-read/src/VolatileLayerClient.cpp‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,15 @@ olp::client::CancellableFuture<DataResponse> VolatileLayerClient::GetData(
6363
DataRequest request) {
6464
return impl_->GetData(std::move(request));
6565
}
66+
67+
bool VolatileLayerClient::RemoveFromCache(const std::string& partition_id) {
68+
return impl_->RemoveFromCache(partition_id);
69+
}
70+
71+
bool VolatileLayerClient::RemoveFromCache(const geo::TileKey& tile) {
72+
return impl_->RemoveFromCache(tile);
73+
}
74+
6675
} // namespace read
6776
} // namespace dataservice
6877
} // namespace olp

‎olp-cpp-sdk-dataservice-read/src/VolatileLayerClientImpl.cpp‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,10 @@
2828

2929
#include "Common.h"
3030
#include "repositories/CatalogRepository.h"
31+
#include "repositories/DataCacheRepository.h"
3132
#include "repositories/DataRepository.h"
3233
#include "repositories/ExecuteOrSchedule.inl"
34+
#include "repositories/PartitionsCacheRepository.h"
3335
#include "repositories/PartitionsRepository.h"
3436

3537
namespace olp {
@@ -124,6 +126,29 @@ client::CancellableFuture<DataResponse> VolatileLayerClientImpl::GetData(
124126
return olp::client::CancellableFuture<DataResponse>(token, promise);
125127
}
126128

129+
bool VolatileLayerClientImpl::RemoveFromCache(const std::string& partition_id) {
130+
repository::PartitionsCacheRepository cache_repository(catalog_,
131+
settings_.cache);
132+
boost::optional<model::Partition> partition;
133+
if (!cache_repository.ClearPartitionMetadata(boost::none, partition_id,
134+
layer_id_, partition)) {
135+
return false;
136+
}
137+
138+
if (!partition) {
139+
// partition are not stored in cache
140+
return true;
141+
}
142+
143+
repository::DataCacheRepository data_repository(catalog_, settings_.cache);
144+
return data_repository.Clear(layer_id_, partition.get().GetDataHandle());
145+
}
146+
147+
bool VolatileLayerClientImpl::RemoveFromCache(const geo::TileKey& tile) {
148+
auto partition_id = tile.ToHereTile();
149+
return RemoveFromCache(partition_id);
150+
}
151+
127152
} // namespace read
128153
} // namespace dataservice
129154
} // namespace olp

‎olp-cpp-sdk-dataservice-read/src/VolatileLayerClientImpl.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include <olp/core/client/CancellationToken.h>
2323
#include <olp/core/client/HRN.h>
2424
#include <olp/core/client/OlpClientSettings.h>
25+
#include <olp/core/geo/tiling/TileKey.h>
2526
#include <olp/dataservice/read/DataRequest.h>
2627
#include <olp/dataservice/read/PartitionsRequest.h>
2728
#include <olp/dataservice/read/Types.h>
@@ -59,6 +60,10 @@ class VolatileLayerClientImpl {
5960

6061
virtual client::CancellableFuture<DataResponse> GetData(DataRequest request);
6162

63+
virtual bool RemoveFromCache(const std::string& partition_id);
64+
65+
virtual bool RemoveFromCache(const geo::TileKey& tile);
66+
6267
private:
6368
client::HRN catalog_;
6469
std::string layer_id_;

‎olp-cpp-sdk-dataservice-read/src/repositories/PartitionsCacheRepository.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -207,8 +207,8 @@ void PartitionsCacheRepository::ClearPartitions(
207207
}
208208

209209
bool PartitionsCacheRepository::ClearPartitionMetadata(
210-
int64_t catalog_version, const std::string& partition_id,
211-
const std::string& layer_id,
210+
const boost::optional<int64_t>& catalog_version,
211+
const std::string& partition_id, const std::string& layer_id,
212212
boost::optional<model::Partition>& out_partition) {
213213
std::string hrn(hrn_.ToCatalogHRNString());
214214
auto key = CreateKey(hrn, layer_id, partition_id, catalog_version);

‎olp-cpp-sdk-dataservice-read/src/repositories/PartitionsCacheRepository.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ class PartitionsCacheRepository final {
6363
const std::vector<std::string>& partitionIds,
6464
const std::string& layer_id);
6565

66-
bool ClearPartitionMetadata(int64_t catalog_version,
66+
bool ClearPartitionMetadata(const boost::optional<int64_t>& catalog_version,
6767
const std::string& partition_id,
6868
const std::string& layer_id,
6969
boost::optional<model::Partition>& out_partition);

‎olp-cpp-sdk-dataservice-read/tests/VolatileLayerClientImplTest.cpp‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -328,4 +328,130 @@ TEST(VolatileLayerClientImplTest, GetDataCancellableFutureCancel) {
328328
EXPECT_EQ(data_response.GetError().GetErrorCode(),
329329
olp::client::ErrorCode::Cancelled);
330330
}
331+
332+
TEST(VolatileLayerClientImplTest, RemoveFromCachePartition) {
333+
olp::client::OlpClientSettings settings;
334+
std::shared_ptr<CacheMock> cache_mock = std::make_shared<CacheMock>();
335+
settings.cache = cache_mock;
336+
337+
// successfull mock cache calls
338+
auto found_cache_response = [](const std::string& /*key*/,
339+
const olp::cache::Decoder& /*encoder*/) {
340+
auto partition = model::Partition();
341+
partition.SetPartition(kPartitionId);
342+
partition.SetDataHandle(kBlobDataHandle);
343+
return partition;
344+
};
345+
346+
auto partition_cache_remove = [&](const std::string& prefix) {
347+
std::string expected_prefix = kHrn.ToCatalogHRNString() + "::" + kLayerId +
348+
"::" + kPartitionId + "::partition";
349+
EXPECT_EQ(prefix, expected_prefix);
350+
return true;
351+
};
352+
353+
auto data_cache_remove = [&](const std::string& prefix) {
354+
std::string expected_prefix = kHrn.ToCatalogHRNString() + "::" + kLayerId +
355+
"::" + kBlobDataHandle + "::Data";
356+
EXPECT_EQ(prefix, expected_prefix);
357+
return true;
358+
};
359+
360+
VolatileLayerClientImpl client(kHrn, kLayerId, settings);
361+
{
362+
SCOPED_TRACE("Successfull remove partition from cache");
363+
EXPECT_CALL(*cache_mock, Get(_, _)).WillOnce(found_cache_response);
364+
EXPECT_CALL(*cache_mock, RemoveKeysWithPrefix(_))
365+
.WillOnce(partition_cache_remove)
366+
.WillOnce(data_cache_remove);
367+
ASSERT_TRUE(client.RemoveFromCache(kPartitionId));
368+
}
369+
{
370+
SCOPED_TRACE("Remove not existing partition from cache");
371+
EXPECT_CALL(*cache_mock, Get(_, _))
372+
.WillOnce([](const std::string&, const olp::cache::Decoder&) {
373+
return boost::any();
374+
});
375+
ASSERT_TRUE(client.RemoveFromCache(kPartitionId));
376+
}
377+
{
378+
SCOPED_TRACE("Partition cache failure");
379+
EXPECT_CALL(*cache_mock, Get(_, _)).WillOnce(found_cache_response);
380+
EXPECT_CALL(*cache_mock, RemoveKeysWithPrefix(_))
381+
.WillOnce([](const std::string&) { return false; });
382+
ASSERT_FALSE(client.RemoveFromCache(kPartitionId));
383+
}
384+
{
385+
SCOPED_TRACE("Data cache failure");
386+
EXPECT_CALL(*cache_mock, Get(_, _)).WillOnce(found_cache_response);
387+
EXPECT_CALL(*cache_mock, RemoveKeysWithPrefix(_))
388+
.WillOnce(partition_cache_remove)
389+
.WillOnce([](const std::string&) { return false; });
390+
ASSERT_FALSE(client.RemoveFromCache(kPartitionId));
391+
}
392+
}
393+
394+
TEST(VolatileLayerClientImplTest, RemoveFromCacheTileKey) {
395+
olp::client::OlpClientSettings settings;
396+
std::shared_ptr<CacheMock> cache_mock = std::make_shared<CacheMock>();
397+
settings.cache = cache_mock;
398+
399+
// successfull mock cache calls
400+
auto found_cache_response = [](const std::string& /*key*/,
401+
const olp::cache::Decoder& /*encoder*/) {
402+
auto partition = model::Partition();
403+
partition.SetPartition(kPartitionId);
404+
partition.SetDataHandle(kBlobDataHandle);
405+
return partition;
406+
};
407+
408+
auto partition_cache_remove = [&](const std::string& prefix) {
409+
std::string expected_prefix = kHrn.ToCatalogHRNString() + "::" + kLayerId +
410+
"::" + kPartitionId + "::partition";
411+
EXPECT_EQ(prefix, expected_prefix);
412+
return true;
413+
};
414+
415+
auto data_cache_remove = [&](const std::string& prefix) {
416+
std::string expected_prefix = kHrn.ToCatalogHRNString() + "::" + kLayerId +
417+
"::" + kBlobDataHandle + "::Data";
418+
EXPECT_EQ(prefix, expected_prefix);
419+
return true;
420+
};
421+
422+
auto tile_key = olp::geo::TileKey::FromHereTile(kPartitionId);
423+
VolatileLayerClientImpl client(kHrn, kLayerId, settings);
424+
{
425+
SCOPED_TRACE("Successfull remove partition from cache");
426+
EXPECT_CALL(*cache_mock, Get(_, _)).WillOnce(found_cache_response);
427+
EXPECT_CALL(*cache_mock, RemoveKeysWithPrefix(_))
428+
.WillOnce(partition_cache_remove)
429+
.WillOnce(data_cache_remove);
430+
ASSERT_TRUE(client.RemoveFromCache(tile_key));
431+
}
432+
{
433+
SCOPED_TRACE("Remove not existing partition from cache");
434+
EXPECT_CALL(*cache_mock, Get(_, _))
435+
.WillOnce([](const std::string&, const olp::cache::Decoder&) {
436+
return boost::any();
437+
});
438+
ASSERT_TRUE(client.RemoveFromCache(tile_key));
439+
}
440+
{
441+
SCOPED_TRACE("Partition cache failure");
442+
EXPECT_CALL(*cache_mock, Get(_, _)).WillOnce(found_cache_response);
443+
EXPECT_CALL(*cache_mock, RemoveKeysWithPrefix(_))
444+
.WillOnce([](const std::string&) { return false; });
445+
ASSERT_FALSE(client.RemoveFromCache(tile_key));
446+
}
447+
{
448+
SCOPED_TRACE("Data cache failure");
449+
EXPECT_CALL(*cache_mock, Get(_, _)).WillOnce(found_cache_response);
450+
EXPECT_CALL(*cache_mock, RemoveKeysWithPrefix(_))
451+
.WillOnce(partition_cache_remove)
452+
.WillOnce([](const std::string&) { return false; });
453+
ASSERT_FALSE(client.RemoveFromCache(tile_key));
454+
}
455+
}
456+
331457
} // namespace

‎tests/integration/olp-cpp-sdk-dataservice-read/VolatileLayerClientTest.cpp‎

Lines changed: 98 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -717,7 +717,7 @@ TEST_F(DataserviceReadVolatileLayerClientTest, GetVolatileDataHandle) {
717717

718718
ASSERT_TRUE(data_response.IsSuccessful())
719719
<< ApiErrorToString(data_response.GetError());
720-
ASSERT_LT(0, data_response.GetResult()->size());
720+
ASSERT_FALSE(data_response.GetResult()->empty());
721721
std::string data_string(data_response.GetResult()->begin(),
722722
data_response.GetResult()->end());
723723
ASSERT_EQ("someData", data_string);
@@ -754,7 +754,7 @@ TEST_F(DataserviceReadVolatileLayerClientTest, GetVolatileDataByPartitionId) {
754754

755755
ASSERT_TRUE(data_response.IsSuccessful())
756756
<< ApiErrorToString(data_response.GetError());
757-
ASSERT_LT(0, data_response.GetResult()->size());
757+
ASSERT_FALSE(data_response.GetResult()->empty());
758758
std::string data_string(data_response.GetResult()->begin(),
759759
data_response.GetResult()->end());
760760
ASSERT_EQ("someData", data_string);
@@ -827,4 +827,100 @@ TEST_F(DataserviceReadVolatileLayerClientTest,
827827
data_response.GetError().GetErrorCode());
828828
}
829829

830+
TEST_F(DataserviceReadVolatileLayerClientTest, RemoveFromCachePartition) {
831+
olp::client::HRN hrn(GetTestCatalog());
832+
833+
auto partition_id = "269";
834+
EXPECT_CALL(*network_mock_,
835+
Send(IsGetRequest(URL_LATEST_CATALOG_VERSION), _, _, _, _))
836+
.Times(0);
837+
838+
EXPECT_CALL(*network_mock_,
839+
Send(IsGetRequest(URL_QUERY_VOLATILE_PARTITION_269), _, _, _, _))
840+
.WillOnce(ReturnHttpResponse(olp::http::NetworkResponse().WithStatus(
841+
olp::http::HttpStatusCode::OK),
842+
HTTP_RESPONSE_PARTITIONS_V2));
843+
844+
EXPECT_CALL(*network_mock_,
845+
Send(IsGetRequest(URL_VOLATILE_BLOB_DATA), _, _, _, _))
846+
.WillOnce(ReturnHttpResponse(olp::http::NetworkResponse().WithStatus(
847+
olp::http::HttpStatusCode::OK),
848+
"someData"));
849+
850+
olp::dataservice::read::VolatileLayerClient client(hrn, "testlayer_volatile",
851+
settings_);
852+
853+
auto request = olp::dataservice::read::DataRequest();
854+
request.WithPartitionId(partition_id);
855+
856+
auto future = client.GetData(request);
857+
858+
auto data_response = future.GetFuture().get();
859+
860+
ASSERT_TRUE(data_response.IsSuccessful())
861+
<< ApiErrorToString(data_response.GetError());
862+
ASSERT_FALSE(data_response.GetResult()->empty());
863+
std::string data_string(data_response.GetResult()->begin(),
864+
data_response.GetResult()->end());
865+
ASSERT_EQ("someData", data_string);
866+
867+
// remove the data from cache
868+
ASSERT_TRUE(client.RemoveFromCache(partition_id));
869+
870+
// check the data is not available in cache
871+
request.WithFetchOption(CacheOnly);
872+
future = client.GetData(request);
873+
data_response = future.GetFuture().get();
874+
ASSERT_FALSE(data_response.IsSuccessful())
875+
<< ApiErrorToString(data_response.GetError());
876+
}
877+
878+
TEST_F(DataserviceReadVolatileLayerClientTest, RemoveFromCacheTileKey) {
879+
olp::client::HRN hrn(GetTestCatalog());
880+
881+
auto partition_id = "269";
882+
EXPECT_CALL(*network_mock_,
883+
Send(IsGetRequest(URL_LATEST_CATALOG_VERSION), _, _, _, _))
884+
.Times(0);
885+
886+
EXPECT_CALL(*network_mock_,
887+
Send(IsGetRequest(URL_QUERY_VOLATILE_PARTITION_269), _, _, _, _))
888+
.WillOnce(ReturnHttpResponse(olp::http::NetworkResponse().WithStatus(
889+
olp::http::HttpStatusCode::OK),
890+
HTTP_RESPONSE_PARTITIONS_V2));
891+
892+
EXPECT_CALL(*network_mock_,
893+
Send(IsGetRequest(URL_VOLATILE_BLOB_DATA), _, _, _, _))
894+
.WillOnce(ReturnHttpResponse(olp::http::NetworkResponse().WithStatus(
895+
olp::http::HttpStatusCode::OK),
896+
"someData"));
897+
898+
olp::dataservice::read::VolatileLayerClient client(hrn, "testlayer_volatile",
899+
settings_);
900+
901+
auto request =
902+
olp::dataservice::read::DataRequest().WithPartitionId(partition_id);
903+
904+
auto future = client.GetData(request);
905+
906+
auto data_response = future.GetFuture().get();
907+
908+
ASSERT_TRUE(data_response.IsSuccessful())
909+
<< ApiErrorToString(data_response.GetError());
910+
ASSERT_FALSE(data_response.GetResult()->empty());
911+
std::string data_string(data_response.GetResult()->begin(),
912+
data_response.GetResult()->end());
913+
ASSERT_EQ("someData", data_string);
914+
915+
// remove the data from cache
916+
auto tile_key = olp::geo::TileKey::FromHereTile(partition_id);
917+
ASSERT_TRUE(client.RemoveFromCache(tile_key));
918+
919+
// check the data is not available in cache
920+
request.WithFetchOption(CacheOnly);
921+
future = client.GetData(request);
922+
data_response = future.GetFuture().get();
923+
ASSERT_FALSE(data_response.IsSuccessful())
924+
<< ApiErrorToString(data_response.GetError());
925+
}
830926
} // namespace

0 commit comments

Comments
 (0)