Change the GetPartitions API (#1426)

Change the GetPartitions API, now it supports requests with more than
100 partitions which is an backend API limit.

Relates-To: OLPSUP-25510

Signed-off-by: Mykhailo Kuchma <[email protected]>

Author: mykhailo-kuchma

olp-cpp-sdk-dataservice-read/src/repositories/PartitionsRepository.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019-2022 HERE Europe B.V.
+ * Copyright (C) 2019-2023 HERE Europe B.V.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -43,6 +43,7 @@ namespace repository = olp::dataservice::read::repository;
 
 constexpr auto kLogTag = "PartitionsRepository";
 constexpr auto kAggregateQuadTreeDepth = 4;
+constexpr auto kQueryRequestLimit = 100;
 
 using LayerVersionReponse = client::ApiResponse<int64_t, client::ApiError>;
 using LayerVersionCallback = std::function<void(LayerVersionReponse)>;
@@ -195,12 +196,6 @@ PartitionsRepository::GetVersionedPartitionsExtendedResponse(
                                        boost::none, fail_on_cache_error);
 }
 
-PartitionsResponse PartitionsRepository::GetVersionedPartitions(
-    const PartitionsRequest& request, int64_t version,
-    client::CancellationContext context) {
-  return GetPartitions(request, version, std::move(context));
-}
-
 PartitionsResponse PartitionsRepository::GetVolatilePartitions(
     const PartitionsRequest& request, client::CancellationContext context) {
   auto catalog_request = CatalogRequest()
@@ -220,8 +215,8 @@ PartitionsResponse PartitionsRepository::GetVolatilePartitions(
     return expiry_response.GetError();
   }
 
-  return GetPartitions(request, boost::none, context,
-                       expiry_response.MoveResult());
+  return GetPartitionsExtendedResponse(request, boost::none, context,
+                                       expiry_response.MoveResult());
 }
 
 QueryApi::PartitionsExtendedResponse
@@ -291,9 +286,15 @@ PartitionsRepository::GetPartitionsExtendedResponse(
       return query_api.GetError();
     }
 
-    response = QueryApi::GetPartitionsbyId(
-        query_api.GetResult(), layer_id_, partition_ids, version,
-        request.GetAdditionalFields(), request.GetBillingTag(), context);
+    if (partition_ids.size() <= kQueryRequestLimit) {
+      response = QueryApi::GetPartitionsbyId(
+          query_api.GetResult(), layer_id_, partition_ids, version,
+          request.GetAdditionalFields(), request.GetBillingTag(), context);
+    } else {
+      response = QueryPartitionsInBatches(
+          query_api.GetResult(), partition_ids, version,
+          request.GetAdditionalFields(), request.GetBillingTag(), context);
+    }
   }
   // Save all partitions only when downloaded via metadata API
   const bool is_layer_metadata = partition_ids.empty();
@@ -324,13 +325,6 @@ PartitionsRepository::GetPartitionsExtendedResponse(
   return response;
 }
 
-PartitionsResponse PartitionsRepository::GetPartitions(
-    const PartitionsRequest& request, boost::optional<std::int64_t> version,
-    client::CancellationContext context, boost::optional<time_t> expiry) {
-  return GetPartitionsExtendedResponse(request, std::move(version),
-                                       std::move(context), std::move(expiry));
-}
-
 PartitionsResponse PartitionsRepository::GetPartitionById(
     const DataRequest& request, boost::optional<int64_t> version,
     client::CancellationContext context) {
@@ -584,6 +578,52 @@ PartitionResponse PartitionsRepository::GetTile(
                            quad_tree_response.GetPayload());
 }
 
+QueryApi::PartitionsExtendedResponse
+PartitionsRepository::QueryPartitionsInBatches(
+    const client::OlpClient& client,
+    const PartitionsRequest::PartitionIds& partitions,
+    boost::optional<std::int64_t> version,
+    const PartitionsRequest::AdditionalFields& additional_fields,
+    boost::optional<std::string> billing_tag,
+    client::CancellationContext context) {
+  std::vector<model::Partition> aggregated_partitions;
+  aggregated_partitions.reserve(partitions.size());
+
+  client::NetworkStatistics aggregated_network_statistics;
+
+  PartitionsRequest::PartitionIds batch;
+  batch.reserve(kQueryRequestLimit);
+
+  for (size_t i = 0; i < partitions.size(); i += kQueryRequestLimit) {
+    batch.clear();
+
+    std::copy(partitions.begin() + i,
+              partitions.begin() +
+                  std::min(partitions.size(), i + kQueryRequestLimit),
+              std::back_inserter(batch));
+
+    auto query_response =
+        QueryApi::GetPartitionsbyId(client, layer_id_, batch, version,
+                                    additional_fields, billing_tag, context);
+    if (!query_response) {
+      return query_response.GetError();
+    }
+
+    auto partitions = query_response.MoveResult();
+    auto& mutable_partitions = partitions.GetMutablePartitions();
+    std::move(mutable_partitions.begin(), mutable_partitions.end(),
+              std::back_inserter(aggregated_partitions));
+
+    aggregated_network_statistics += query_response.GetPayload();
+  }
+
+  model::Partitions result_partitions;
+  result_partitions.GetMutablePartitions().swap(aggregated_partitions);
+
+  return QueryApi::PartitionsExtendedResponse(std::move(result_partitions),
+                                              aggregated_network_statistics);
+}
+
 }  // namespace repository
 }  // namespace read
 }  // namespace dataservice

olp-cpp-sdk-dataservice-read/src/repositories/PartitionsRepository.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019-2022 HERE Europe B.V.
+ * Copyright (C) 2019-2023 HERE Europe B.V.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -57,10 +57,6 @@ class PartitionsRepository {
                        client::ApiLookupClient client,
                        NamedMutexStorage storage = NamedMutexStorage());
 
-  PartitionsResponse GetVersionedPartitions(
-      const read::PartitionsRequest& request, std::int64_t version,
-      client::CancellationContext context);
-
   PartitionsResponse GetVolatilePartitions(
       const read::PartitionsRequest& request,
       client::CancellationContext context);
@@ -93,19 +89,21 @@ class PartitionsRepository {
       boost::optional<std::vector<std::string>> additional_fields =
           boost::none);
 
-  PartitionsResponse GetPartitions(
-      const read::PartitionsRequest& request,
-      boost::optional<std::int64_t> version,
-      client::CancellationContext context,
-      boost::optional<time_t> expiry = boost::none);
-
   QueryApi::PartitionsExtendedResponse GetPartitionsExtendedResponse(
       const read::PartitionsRequest& request,
       boost::optional<std::int64_t> version,
       client::CancellationContext context,
       boost::optional<time_t> expiry = boost::none,
       bool fail_on_cache_error = false);
 
+  QueryApi::PartitionsExtendedResponse QueryPartitionsInBatches(
+      const client::OlpClient& client,
+      const PartitionsRequest::PartitionIds& partitions,
+      boost::optional<std::int64_t> version,
+      const PartitionsRequest::AdditionalFields& additional_fields,
+      boost::optional<std::string> billing_tag,
+      client::CancellationContext context);
+
   const client::HRN catalog_;
   const std::string layer_id_;
   client::OlpClientSettings settings_;

olp-cpp-sdk-dataservice-read/tests/PartitionsRepositoryTest.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019-2022 HERE Europe B.V.
+ * Copyright (C) 2019-2023 HERE Europe B.V.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -625,8 +625,8 @@ TEST_F(PartitionsRepositoryTest, GetVersionedPartitions) {
     request.WithPartitionIds({kPartitionId, kInvalidPartitionId});
     request.WithFetchOption(read::CacheOnly);
 
-    auto response =
-        repository.GetVersionedPartitions(request, kVersion, context);
+    auto response = repository.GetVersionedPartitionsExtendedResponse(
+        request, kVersion, context);
 
     ASSERT_FALSE(response.IsSuccessful());
     EXPECT_TRUE(response.GetResult().GetPartitions().empty());
@@ -658,8 +658,8 @@ TEST_F(PartitionsRepositoryTest, GetVersionedPartitions) {
     read::PartitionsRequest request;
     request.WithPartitionIds({kPartitionId});
 
-    auto response =
-        repository.GetVersionedPartitions(request, kVersion, context);
+    auto response = repository.GetVersionedPartitionsExtendedResponse(
+        request, kVersion, context);
 
     ASSERT_TRUE(response.IsSuccessful()) << response.GetError().GetMessage();
     EXPECT_EQ(response.GetResult().GetPartitions().size(), 1);
@@ -690,15 +690,16 @@ TEST_F(PartitionsRepositoryTest, GetVersionedPartitions) {
                                                 settings, lookup_client);
     read::PartitionsRequest request;
 
-    auto response =
-        repository.GetVersionedPartitions(request, kVersion, context);
+    auto response = repository.GetVersionedPartitionsExtendedResponse(
+        request, kVersion, context);
 
     ASSERT_TRUE(response.IsSuccessful()) << response.GetError().GetMessage();
     EXPECT_TRUE(response.GetResult().GetPartitions().empty());
 
     request.WithFetchOption(read::CacheOnly);
 
-    response = repository.GetVersionedPartitions(request, kVersion, context);
+    response = repository.GetVersionedPartitionsExtendedResponse(
+        request, kVersion, context);
 
     ASSERT_TRUE(response.IsSuccessful()) << response.GetError().GetMessage();
     EXPECT_TRUE(response.GetResult().GetPartitions().empty());
@@ -817,7 +818,8 @@ TEST_F(PartitionsRepositoryTest, AdditionalFields) {
                                 read::PartitionsRequest::kCrc,
                                 read::PartitionsRequest::kDataSize});
 
-  auto response = repository.GetVersionedPartitions(request, kVersion, context);
+  auto response = repository.GetVersionedPartitionsExtendedResponse(
+      request, kVersion, context);
 
   ASSERT_TRUE(response.IsSuccessful());
   auto result = response.GetResult();
@@ -830,8 +832,8 @@ TEST_F(PartitionsRepositoryTest, AdditionalFields) {
 
   request.WithFetchOption(read::CacheOnly);
 
-  auto response_2 =
-      repository.GetVersionedPartitions(request, kVersion, context);
+  auto response_2 = repository.GetVersionedPartitionsExtendedResponse(
+      request, kVersion, context);
 
   ASSERT_TRUE(response_2.IsSuccessful());