Add GetPartitions extended response (#1079)

Add MetadataApi and QueryApi metods wich returns
extended responce. Add GetPartitionsExtendedResponse to
PartitionsRepository, which returns extended response.
Add PrefetchPartitionsRequest and PrefetchPartitionsStatus.

Relates-To: OLPEDGE-1902

Signed-off-by: Liubov Didkivska <[email protected]>

Author: Liubov Didkivska

olp-cpp-sdk-dataservice-read/include/olp/dataservice/read/PrefetchPartitionsRequest.h

@@ -0,0 +1,168 @@
+/*
+ * Copyright (C) 2020 HERE Europe B.V.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+#pragma once
+
+#include <sstream>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <olp/core/thread/TaskScheduler.h>
+#include <olp/dataservice/read/DataServiceReadApi.h>
+#include <olp/dataservice/read/FetchOptions.h>
+#include <boost/optional.hpp>
+
+namespace olp {
+namespace dataservice {
+namespace read {
+
+/**
+ * @brief Encapsulates the fields required to prefetch a list of partitions for
+ * the given catalog and layer.
+ */
+class DATASERVICE_READ_API PrefetchPartitionsRequest final {
+ public:
+  /// The alias type of the vector of partitions ids
+  using PartitionIds = std::vector<std::string>;
+
+  /**
+   * @brief Sets the list of partitions.
+   *
+   * When the list is empty, the GetPartitions method will download the whole
+   * layer metadata. Additionally, a single request supports up to 100
+   * partitions. If partitions list has more than 100, it will be splitted
+   * internally to multiple requests.
+   *
+   * @param partitions The list of partitions to request.
+   *
+   * @return A reference to the updated `PrefetchPartitionsRequest` instance.
+   */
+  inline PrefetchPartitionsRequest& WithPartitionIds(
+      PartitionIds partition_ids) {
+    partition_ids_ = std::move(partition_ids);
+    return *this;
+  }
+
+  /**
+   * @brief Gets the list of the partitions.
+   *
+   * @return The vector of strings that represent partitions.
+   */
+  inline const PartitionIds& GetPartitionIds() const { return partition_ids_; }
+
+  /**
+   * @brief Gets the billing tag to group billing records together.
+   *
+   * The billing tag is an optional free-form tag that is used for grouping
+   * billing records together. If supplied, it must be 4–16 characters
+   * long and contain only alphanumeric ASCII characters [A-Za-z0-9].
+   *
+   * @return The `BillingTag` string or `boost::none` if the billing tag is not
+   * set.
+   */
+  inline const boost::optional<std::string>& GetBillingTag() const {
+    return billing_tag_;
+  }
+
+  /**
+   * @brief Sets the billing tag for the request.
+   *
+   * @see `GetBillingTag()` for information on usage and format.
+   *
+   * @param billingTag The `BillingTag` string or `boost::none`.
+   *
+   * @return A reference to the updated `PrefetchTilesRequest` instance.
+   */
+  inline PrefetchPartitionsRequest& WithBillingTag(
+      boost::optional<std::string> billing_tag) {
+    billing_tag_ = billing_tag;
+    return *this;
+  }
+
+  /**
+   * @brief Sets the billing tag for the request.
+   *
+   * @see `GetBillingTag()` for information on usage and format.
+   *
+   * @param billingTag The rvalue reference to the `BillingTag` string or
+   * `boost::none`.
+   *
+   * @return A reference to the updated `PrefetchTilesRequest` instance.
+   */
+  inline PrefetchPartitionsRequest& WithBillingTag(std::string&& billing_tag) {
+    billing_tag_ = std::move(billing_tag);
+    return *this;
+  }
+
+  /**
+   * @brief Gets the request priority.
+   *
+   * The default priority is `Priority::LOW`.
+   *
+   * @return The request priority.
+   */
+  inline uint32_t GetPriority() const { return priority_; }
+
+  /**
+   * @brief Sets the priority of the prefetch request.
+   *
+   * @param priority The priority of the request.
+   *
+   * @return A reference to the updated `PrefetchPartitionsRequest` instance.
+   */
+  inline PrefetchPartitionsRequest& WithPriority(uint32_t priority) {
+    priority_ = priority;
+    return *this;
+  }
+
+  /**
+   * @brief Creates a readable format for the request.
+   *
+   * @param layer_id The ID of the layer that is used for the request.
+   *
+   * @return A string representation of the request.
+   */
+  std::string CreateKey(const std::string& layer_id,
+                        boost::optional<int64_t> version = boost::none) const {
+    std::stringstream out;
+    out << layer_id;
+    if (version) {
+      out << "@" << version.get();
+    }
+    out << "^" << partition_ids_.size();
+    if (partition_ids_.size() > 0) {
+      out << "[" << partition_ids_.front() << ", ...]";
+    }
+    if (GetBillingTag()) {
+      out << "$" << GetBillingTag().get();
+    }
+    out << "*" << priority_;
+    return out.str();
+  }
+
+ private:
+  PartitionIds partition_ids_;
+  boost::optional<std::string> billing_tag_;
+  uint32_t priority_{thread::LOW};
+};
+
+}  // namespace read
+}  // namespace dataservice
+}  // namespace olp

olp-cpp-sdk-dataservice-read/include/olp/dataservice/read/PrefetchStatus.h

@@ -37,6 +37,19 @@ struct PrefetchStatus {
   size_t bytes_transferred;
 };
 
+/*
+ * @brief PrefetchPartitionsStatus structure represent the progress of prefetch
+ * operation for partitions.
+ */
+struct PrefetchPartitionsStatus {
+  /// Partitions available (prefetched).
+  size_t prefetched_partitions;
+  /// Total number of partitions to prefetch during prefetch operation.
+  size_t total_partitions_to_prefetch;
+  /// Total bytes tranferred during API calls.
+  size_t bytes_transferred;
+};
+
 }  // namespace read
 }  // namespace dataservice
 }  // namespace olp

olp-cpp-sdk-dataservice-read/include/olp/dataservice/read/Types.h

@@ -96,6 +96,9 @@ using PrefetchStatusCallback = std::function<void(PrefetchStatus)>;
 using PrefetchPartitionsResponse = Response<PrefetchPartitionsResult>;
 /// The callback type of the prefetch completion.
 using PrefetchPartitionsResponseCallback = Callback<PrefetchPartitionsResult>;
+/// The callback type for the prefetch status update.
+using PrefetchPartitionsStatusCallback =
+    std::function<void(PrefetchPartitionsStatus)>;
 
 /// The subscribe ID type of the stream layer client.
 using SubscriptionId = std::string;

olp-cpp-sdk-dataservice-read/src/generated/api/MetadataApi.cpp

@@ -80,7 +80,7 @@ MetadataApi::LayerVersionsResponse MetadataApi::GetLayerVersions(
   return parser::parse_result<LayerVersionsResponse>(api_response.response);
 }
 
-MetadataApi::PartitionsResponse MetadataApi::GetPartitions(
+MetadataApi::PartitionsExtendedResponse MetadataApi::GetPartitions(
     const client::OlpClient& client, const std::string& layer_id,
     boost::optional<std::int64_t> version,
     const std::vector<std::string>& additional_fields,
@@ -107,14 +107,27 @@ MetadataApi::PartitionsResponse MetadataApi::GetPartitions(
 
   std::string metadataUri = "/layers/" + layer_id + "/partitions";
 
-  auto api_response = client.CallApi(metadataUri, "GET", query_params,
-                                     header_params, {}, nullptr, "", context);
+  auto http_response = client.CallApi(metadataUri, "GET", query_params,
+                                      header_params, {}, nullptr, "", context);
 
-  if (api_response.status != http::HttpStatusCode::OK) {
-    return {{api_response.status, api_response.response.str()}};
+  if (http_response.status != olp::http::HttpStatusCode::OK) {
+    return {{http_response.status, http_response.response.str()},
+            http_response.GetNetworkStatistics()};
+  }
+
+  using PartitionsResponse =
+      client::ApiResponse<model::Partitions, client::ApiError>;
+
+  auto partitions_response =
+      parser::parse_result<PartitionsResponse>(http_response.response);
+
+  if (!partitions_response.IsSuccessful()) {
+    return {{partitions_response.GetError()},
+            http_response.GetNetworkStatistics()};
   }
 
-  return parser::parse_result<PartitionsResponse>(api_response.response);
+  return {partitions_response.MoveResult(),
+          http_response.GetNetworkStatistics()};
 }
 
 MetadataApi::CatalogVersionResponse MetadataApi::GetLatestCatalogVersion(

olp-cpp-sdk-dataservice-read/src/generated/api/MetadataApi.h

@@ -25,6 +25,7 @@
 #include <olp/core/client/ApiError.h>
 #include <olp/core/client/ApiResponse.h>
 #include <boost/optional.hpp>
+#include "ExtendedApiResponse.h"
 #include "generated/model/LayerVersions.h"
 #include "olp/dataservice/read/model/Partitions.h"
 #include "olp/dataservice/read/model/VersionInfos.h"
@@ -45,15 +46,17 @@ class MetadataApi {
  public:
   using VersionsResponse =
       client::ApiResponse<model::VersionInfos, client::ApiError>;
-  using PartitionsResponse =
-      client::ApiResponse<model::Partitions, client::ApiError>;
 
   using CatalogVersionResponse =
       client::ApiResponse<model::VersionResponse, client::ApiError>;
 
   using LayerVersionsResponse =
       client::ApiResponse<model::LayerVersions, client::ApiError>;
 
+  using PartitionsExtendedResponse =
+      ExtendedApiResponse<model::Partitions, client::ApiError,
+                          client::NetworkStatistics>;
+
   /**
    * @brief Retrieves the latest metadata version for each layer of a specified
    * catalog metadata version.
@@ -91,9 +94,10 @@ class MetadataApi {
    * characters, contain only alpha/numeric ASCII characters  [A-Za-z0-9].
    * @param context A CancellationContext, which can be used to cancel request.
    *
-   * @return The Partitions response.
+   * @return  The result of this operation as an extended client::ApiResponse
+   * object with \c model::Partitions as a result.
    */
-  static PartitionsResponse GetPartitions(
+  static PartitionsExtendedResponse GetPartitions(
       const client::OlpClient& client, const std::string& layerId,
       boost::optional<int64_t> version,
       const std::vector<std::string>& additional_fields,

olp-cpp-sdk-dataservice-read/src/generated/api/QueryApi.cpp

@@ -59,7 +59,7 @@ namespace olp {
 namespace dataservice {
 namespace read {
 
-QueryApi::PartitionsResponse QueryApi::GetPartitionsbyId(
+QueryApi::PartitionsExtendedResponse QueryApi::GetPartitionsbyId(
     const client::OlpClient& client, const std::string& layer_id,
     const std::vector<std::string>& partitions,
     boost::optional<int64_t> version,
@@ -86,17 +86,30 @@ QueryApi::PartitionsResponse QueryApi::GetPartitionsbyId(
 
   std::string metadata_uri = "/layers/" + layer_id + "/partitions";
 
-  client::HttpResponse response = client.CallApi(
+  client::HttpResponse http_response = client.CallApi(
       metadata_uri, "GET", std::move(query_params), std::move(header_params),
       {}, nullptr, std::string{}, std::move(context));
-  if (response.status != olp::http::HttpStatusCode::OK) {
-    return client::ApiError(response.status, response.response.str());
+
+  OLP_SDK_LOG_TRACE_F(kLogTag, "GetPartitionsbyId, layer_id=%s, status=%d",
+                      layer_id.c_str(), http_response.status);
+
+  if (http_response.status != olp::http::HttpStatusCode::OK) {
+    return {{http_response.status, http_response.response.str()},
+            http_response.GetNetworkStatistics()};
   }
+  using PartitionsResponse =
+      client::ApiResponse<model::Partitions, client::ApiError>;
 
-  OLP_SDK_LOG_TRACE_F(kLogTag, "GetPartitionsbyId, uri=%s, status=%d",
-                      metadata_uri.c_str(), response.status);
+  auto partitions_response =
+      parser::parse_result<PartitionsResponse>(http_response.response);
+
+  if (!partitions_response.IsSuccessful()) {
+    return {{partitions_response.GetError()},
+            http_response.GetNetworkStatistics()};
+  }
 
-  return parser::parse_result<PartitionsResponse>(response.response);
+  return {partitions_response.MoveResult(),
+          http_response.GetNetworkStatistics()};
 }
 
 olp::client::HttpResponse QueryApi::QuadTreeIndex(

olp-cpp-sdk-dataservice-read/src/generated/api/QueryApi.h

@@ -27,6 +27,7 @@
 #include <olp/core/client/CancellationContext.h>
 #include <olp/core/client/HttpResponse.h>
 #include <boost/optional.hpp>
+#include "ExtendedApiResponse.h"
 #include "generated/model/Index.h"
 #include "olp/dataservice/read/model/Partitions.h"
 
@@ -42,11 +43,13 @@ namespace read {
  */
 class QueryApi {
  public:
-  using PartitionsResponse =
-      client::ApiResponse<model::Partitions, client::ApiError>;
   using QuadTreeIndexResponse =
       client::ApiResponse<model::Index, client::ApiError>;
 
+  using PartitionsExtendedResponse =
+      ExtendedApiResponse<model::Partitions, client::ApiError,
+                          client::NetworkStatistics>;
+
   /**
    * @brief Call to synchronously retrieve metadata for specified partitions in
    * a specified layer.
@@ -66,10 +69,10 @@ class QueryApi {
    * response.
    * @param context A CancellationContext instance which can be used to cancel
    * call of this method.
-   * @param The result of this operation as a client::ApiResponse object with \c
-   * model::Partitions as a result.
+   * @return  The result of this operation as an extended client::ApiResponse
+   * object with \c model::Partitions as a result.
    */
-  static PartitionsResponse GetPartitionsbyId(
+  static PartitionsExtendedResponse GetPartitionsbyId(
       const client::OlpClient& client, const std::string& layer_id,
       const std::vector<std::string>& partitions,
       boost::optional<int64_t> version,
@@ -104,8 +107,7 @@ class QueryApi {
    * release.
    * @param context A CancellationContext instance which can be used to cancel
    * call of this method.
-   * @param The result of this operation as a client::ApiResponse object with \c
-   * model::Index as a result
+   * @return HttpResponse of this operation.
    **/
   static olp::client::HttpResponse QuadTreeIndex(
       const client::OlpClient& client, const std::string& layer_id,