Add multi partitions Protect and Release (#1464)

The main goal is to decrease number of calls and number of log messages

Relates-To: OLPEDGE-2855

Signed-off-by: Rustam Gamidov <[email protected]>

Author: rustam-gamidov-here

olp-cpp-sdk-dataservice-read/include/olp/dataservice/read/VersionedLayerClient.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019-2023 HERE Europe B.V.
+ * Copyright (C) 2019-2024 HERE Europe B.V.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -286,7 +286,7 @@ class DATASERVICE_READ_API VersionedLayerClient final {
    * @brief Fetches a list of partitions of the given generic layer
    * asynchronously. Client does not cache the partitions, instead every
    * partition is passed to the provided callback.
-   * 
+   *
    * @note API is considered experimental and a subject to change.
    *
    * @param partitions_request The `PartitionsRequest` instance that contains
@@ -549,6 +549,29 @@ class DATASERVICE_READ_API VersionedLayerClient final {
    */
   bool Protect(const std::string& partition_id);
 
+  /**
+   * @brief Protect partitions from eviction.
+   *
+   * Protecting partitions means that its data and metadata keys are added to
+   * the protected list and stored in the cache. These keys are removed from the
+   * LRU cache, so they could not be evicted. Also, they do not expire.
+   *
+   * @note Before calling the API, specify a catalog version. You can set it
+   * using the constructor or after the first online request.
+   *
+   * @note You can only protect partitions which data handles are present in the
+   * cache at the time of the call.
+   *
+   * @note Please do not call `Protect` while the `Release` call for the same
+   * catalog and layer is in progress.
+   *
+   * @param partition_ids Partition ids to be protected.
+   *
+   * @return True if partition keys were successfully added to the protected
+   * list; false otherwise.
+   */
+  bool Protect(const std::vector<std::string>& partition_ids);
+
   /**
    * @brief Removes a list of tiles from protection.
    *
@@ -591,6 +614,27 @@ class DATASERVICE_READ_API VersionedLayerClient final {
    */
   bool Release(const std::string& partition_id);
 
+  /**
+   * @brief Removes partitions from protection.
+   *
+   * Releasing partition ids removes data handle and metadata keys from the
+   * protected list. The keys are added to the LRU cache, so they could be
+   * evicted. Expiration value is restored, and related to partition keys can
+   * expire.
+   *
+   * @note Before calling the API, specify a catalog version. You can set it
+   * using the constructor or after the first online request.
+   *
+   * @note Please make sure that `Protect` will not be called for the same
+   * catalog and layer while the `Release` call is in progress.
+   *
+   * @param partition_ids Partition ids to be removed from protection.
+   *
+   * @return True if keys related to partitions were successfully removed from
+   * the protected list; false otherwise.
+   */
+  bool Release(const std::vector<std::string>& partition_ids);
+
  private:
   std::unique_ptr<VersionedLayerClientImpl> impl_;
 };

olp-cpp-sdk-dataservice-read/src/VersionedLayerClient.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019-2023 HERE Europe B.V.
+ * Copyright (C) 2019-2024 HERE Europe B.V.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -156,11 +156,21 @@ bool VersionedLayerClient::Release(const TileKeys& tiles) {
 }
 
 bool VersionedLayerClient::Protect(const std::string& partition_id) {
-  return impl_->Protect(partition_id);
+  return impl_->Protect({partition_id});
 }
 
 bool VersionedLayerClient::Release(const std::string& partition_id) {
-  return impl_->Release(partition_id);
+  return impl_->Release({partition_id});
+}
+
+bool VersionedLayerClient::Protect(
+    const std::vector<std::string>& partition_ids) {
+  return impl_->Protect(partition_ids);
+}
+
+bool VersionedLayerClient::Release(
+    const std::vector<std::string>& partition_ids) {
+  return impl_->Release(partition_ids);
 }
 
 }  // namespace read

olp-cpp-sdk-dataservice-read/src/VersionedLayerClientImpl.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019-2023 HERE Europe B.V.
+ * Copyright (C) 2019-2024 HERE Europe B.V.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -910,7 +910,8 @@ bool VersionedLayerClientImpl::Release(const TileKeys& tiles) {
   return settings_.cache->Release(keys_to_release);
 }
 
-bool VersionedLayerClientImpl::Protect(const std::string& partition_id) {
+bool VersionedLayerClientImpl::Protect(
+    const std::vector<std::string>& partition_ids) {
   if (!settings_.cache) {
     return {};
   }
@@ -925,10 +926,11 @@ bool VersionedLayerClientImpl::Protect(const std::string& partition_id) {
   repository::PartitionsCacheRepository repository(catalog_, layer_id_,
                                                    settings_.cache);
 
-  return repository.Protect(partition_id, version);
+  return repository.Protect(partition_ids, version);
 }
 
-bool VersionedLayerClientImpl::Release(const std::string& partition_id) {
+bool VersionedLayerClientImpl::Release(
+    const std::vector<std::string>& partition_ids) {
   if (!settings_.cache) {
     return {};
   }
@@ -943,7 +945,7 @@ bool VersionedLayerClientImpl::Release(const std::string& partition_id) {
   repository::PartitionsCacheRepository repository(catalog_, layer_id_,
                                                    settings_.cache);
 
-  return repository.Release(partition_id, version);
+  return repository.Release(partition_ids, version);
 }
 
 }  // namespace read

olp-cpp-sdk-dataservice-read/src/VersionedLayerClientImpl.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019-2023 HERE Europe B.V.
+ * Copyright (C) 2019-2024 HERE Europe B.V.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -119,9 +119,9 @@ class VersionedLayerClientImpl {
 
   virtual bool Release(const TileKeys& tiles);
 
-  virtual bool Protect(const std::string& partition_id);
+  virtual bool Protect(const std::vector<std::string>& partition_ids);
 
-  virtual bool Release(const std::string& partition_id);
+  virtual bool Release(const std::vector<std::string>& partition_ids);
 
  private:
   CatalogVersionResponse GetVersion(boost::optional<std::string> billing_tag,

olp-cpp-sdk-dataservice-read/src/repositories/PartitionsCacheRepository.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019-2023 HERE Europe B.V.
+ * Copyright (C) 2019-2024 HERE Europe B.V.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -357,8 +357,18 @@ PartitionsCacheRepository::CreatePartitionKeys(
 }
 
 bool PartitionsCacheRepository::Protect(
-    const std::string& partition_id, const boost::optional<int64_t>& version) {
-  const auto keys = CreatePartitionKeys(partition_id, version);
+    const std::vector<std::string>& partition_ids,
+    const boost::optional<int64_t>& version) {
+  cache::KeyValueCache::KeyListType keys;
+  keys.reserve(partition_ids.size() * 2u);
+  std::for_each(partition_ids.cbegin(), partition_ids.cend(),
+                [&](const std::string& partition_id) {
+                  auto partition_keys =
+                      CreatePartitionKeys(partition_id, version);
+                  std::move(partition_keys.begin(), partition_keys.end(),
+                            std::back_inserter(keys));
+                });
+
   if (keys.empty()) {
     return false;
   }
@@ -367,8 +377,18 @@ bool PartitionsCacheRepository::Protect(
 }
 
 bool PartitionsCacheRepository::Release(
-    const std::string& partition_id, const boost::optional<int64_t>& version) {
-  const auto keys = CreatePartitionKeys(partition_id, version);
+    const std::vector<std::string>& partition_ids,
+    const boost::optional<int64_t>& version) {
+  cache::KeyValueCache::KeyListType keys;
+  keys.reserve(partition_ids.size() * 2u);
+  std::for_each(partition_ids.cbegin(), partition_ids.cend(),
+                [&](const std::string& partition_id) {
+                  auto partition_keys =
+                      CreatePartitionKeys(partition_id, version);
+                  std::move(partition_keys.begin(), partition_keys.end(),
+                            std::back_inserter(keys));
+                });
+
   if (keys.empty()) {
     return false;
   }

olp-cpp-sdk-dataservice-read/src/repositories/PartitionsCacheRepository.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019-2021 HERE Europe B.V.
+ * Copyright (C) 2019-2024 HERE Europe B.V.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -92,10 +92,10 @@ class PartitionsCacheRepository final {
   bool ContainsTree(geo::TileKey key, int32_t depth,
                     const boost::optional<int64_t>& version) const;
 
-  bool Protect(const std::string& partition_id,
+  bool Protect(const std::vector<std::string>& partition_ids,
                const boost::optional<int64_t>& version);
 
-  bool Release(const std::string& partition_id,
+  bool Release(const std::vector<std::string>& partition_ids,
                const boost::optional<int64_t>& version);
 
  private:

olp-cpp-sdk-dataservice-read/tests/VersionedLayerClientImplTest.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019-2022 HERE Europe B.V.
+ * Copyright (C) 2019-2024 HERE Europe B.V.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -384,6 +384,20 @@ TEST(VersionedLayerClientTest, ProtectThenReleasePartition) {
     ASSERT_TRUE(client.Protect(kPartitionId));
   }
 
+  {
+    SCOPED_TRACE("Successful protect partitions");
+
+    EXPECT_CALL(*cache_mock, Get(_, _)).WillOnce(found_cache_response);
+    EXPECT_CALL(*cache_mock, Protect(_)).WillOnce(partition_keys);
+    ASSERT_TRUE(client.Protect(std::vector<std::string>{kPartitionId}));
+  }
+
+  {
+    SCOPED_TRACE("Protect empty partitions");
+
+    ASSERT_FALSE(client.Protect(std::vector<std::string>{}));
+  }
+
   {
     SCOPED_TRACE("Successful release partition");
 
@@ -392,6 +406,20 @@ TEST(VersionedLayerClientTest, ProtectThenReleasePartition) {
     ASSERT_TRUE(client.Release(kPartitionId));
   }
 
+  {
+    SCOPED_TRACE("Successful release partitions");
+
+    EXPECT_CALL(*cache_mock, Get(_, _)).WillOnce(found_cache_response);
+    EXPECT_CALL(*cache_mock, Release(_)).WillOnce(partition_keys);
+    ASSERT_TRUE(client.Release(std::vector<std::string>{kPartitionId}));
+  }
+
+  {
+    SCOPED_TRACE("Release empty partitions");
+
+    ASSERT_FALSE(client.Release(std::vector<std::string>{}));
+  }
+
   {
     SCOPED_TRACE("Protect not existing partition");
     EXPECT_CALL(*cache_mock, Get(_, _))