Add additional fields support to PartitionsRequest (#696)

mykhailo-kuchma · web-flow · commit a9730c2ded1e · 2020-03-03T16:06:07.000+02:00
Add CRC field support to partition model.

Change QueryApi and MetadataApi to accept additional headers by const ref.

This gives to user functionality to access partition metadata fields:
* data size
* compressed data size
* checksum
* crc

Resolves: OLPEDGE-1019

Signed-off-by: Mykhailo Kuchma &lt;ext-mykhailo.kuchma@here.com&gt;
diff --git a/olp-cpp-sdk-dataservice-read/include/olp/dataservice/read/PartitionsRequest.h b/olp-cpp-sdk-dataservice-read/include/olp/dataservice/read/PartitionsRequest.h
@@ -41,6 +41,22 @@ class DATASERVICE_READ_API PartitionsRequest final {
   /// The alias type of the vector of partitions ids
   using PartitionIds = std::vector<std::string>;
 
+  /// Additional field to request partition data size, see GetPartitions
+  static constexpr const char* kDataSize = "dataSize";
+
+  /// Additional field to request partition checksum, see GetPartitions
+  static constexpr const char* kChecksum = "checksum";
+
+  /// Additional field to request partition compressed data size, see
+  /// GetPartitions
+  static constexpr const char* kCompressedDataSize = "compressedDataSize";
+
+  /// Additional field to request partition crc, see GetPartitions
+  static constexpr const char* kCrc = "crc";
+
+  /// The alias type of the set of additional fields
+  using AdditionalFields = std::vector<std::string>;
+
   /**
    * @brief Sets the list of partitions.
    *
@@ -64,6 +80,35 @@ class DATASERVICE_READ_API PartitionsRequest final {
    */
   inline const PartitionIds& GetPartitionIds() const { return partition_ids_; }
 
+  /**
+   * @brief Sets the list of additional fields.
+   *
+   * When specified, the result metadata will include the additional information
+   * requested. The supported fields are:
+   *  - dataSize
+   *  - checksum
+   *  - compressedDataSize
+   *  - crc
+   *
+   * @param fields The list of additional fields.
+   *
+   * @return A reference to the updated `PartitionsRequest` instance.
+   */
+  inline PartitionsRequest& WithAdditionalFields(
+      AdditionalFields additional_fields) {
+    additional_fields_ = std::move(additional_fields);
+    return *this;
+  }
+
+  /**
+   * @brief Gets the list of additional fields.
+   *
+   * @return The set of additional fields.
+   */
+  inline const AdditionalFields& GetAdditionalFields() const {
+    return additional_fields_;
+  }
+
   /**
    * @brief Sets the catalog metadata version.
    *
@@ -187,6 +232,7 @@ class DATASERVICE_READ_API PartitionsRequest final {
 
  private:
   PartitionIds partition_ids_;
+  AdditionalFields additional_fields_;
   boost::optional<int64_t> catalog_version_;
   boost::optional<std::string> billing_tag_;
   FetchOptions fetch_option_{OnlineIfNotFound};
diff --git a/olp-cpp-sdk-dataservice-read/include/olp/dataservice/read/model/Partitions.h b/olp-cpp-sdk-dataservice-read/include/olp/dataservice/read/model/Partitions.h
@@ -44,6 +44,7 @@ class DATASERVICE_READ_API Partition {
   boost::optional<int64_t> compressed_data_size_;
   std::string data_handle_;
   boost::optional<int64_t> data_size_;
+  boost::optional<std::string> crc_;
   std::string partition_;
   boost::optional<int64_t> version_;
 
@@ -182,6 +183,35 @@ class DATASERVICE_READ_API Partition {
     this->data_size_ = value;
   }
 
+  /**
+   * Optional value for the CRC of the partition data in bytes.
+   *
+   * The response only includes the data size if you specify crc in the
+   * additionalFields query parameter, and if crc was specified in the partition
+   * metadata when it was published.
+   *
+   * @return The partition CRC.
+   */
+  const boost::optional<std::string>& GetCrc() const { return crc_; }
+  /**
+   * @brief (Optional) Gets a mutable reference to the partition crc.
+   *
+   * @see `GetCrc` for information on the crc.
+   *
+   * @return The mutable reference to the partition crc.
+   */
+  boost::optional<std::string>& GetMutableCrc() { return crc_; }
+  /**
+   * @brief (Optional) Sets the partition crc.
+   *
+   * @see `GetCrc` for information on the crc.
+   *
+   * @param value The partition crc.
+   */
+  void SetCrc(boost::optional<std::string> value) {
+    this->crc_ = std::move(value);
+  }
+
   /**
    * @brief Gets the partition key.
    *
diff --git a/olp-cpp-sdk-dataservice-read/src/generated/api/MetadataApi.cpp b/olp-cpp-sdk-dataservice-read/src/generated/api/MetadataApi.cpp
@@ -84,7 +84,7 @@ MetadataApi::LayerVersionsResponse MetadataApi::GetLayerVersions(
 MetadataApi::PartitionsResponse MetadataApi::GetPartitions(
     const OlpClient& client, const std::string& layer_id,
     boost::optional<int64_t> version,
-    boost::optional<std::vector<std::string>> additional_fields,
+    const std::vector<std::string>& additional_fields,
     boost::optional<std::string> range,
     boost::optional<std::string> billing_tag,
     const client::CancellationContext& context) {
@@ -95,9 +95,9 @@ MetadataApi::PartitionsResponse MetadataApi::GetPartitions(
   }
 
   std::multimap<std::string, std::string> query_params;
-  if (additional_fields) {
+  if (!additional_fields.empty()) {
     query_params.emplace("additionalFields",
-                         concatStringArray(*additional_fields, ","));
+                         concatStringArray(additional_fields, ","));
   }
   if (billing_tag) {
     query_params.emplace("billingTag", *billing_tag);
diff --git a/olp-cpp-sdk-dataservice-read/src/generated/api/MetadataApi.h b/olp-cpp-sdk-dataservice-read/src/generated/api/MetadataApi.h
@@ -93,7 +93,7 @@ class MetadataApi {
   static PartitionsResponse GetPartitions(
       const client::OlpClient& client, const std::string& layerId,
       boost::optional<int64_t> version,
-      boost::optional<std::vector<std::string>> additional_fields,
+      const std::vector<std::string>& additional_fields,
       boost::optional<std::string> range,
       boost::optional<std::string> billing_tag,
       const client::CancellationContext& context);
diff --git a/olp-cpp-sdk-dataservice-read/src/generated/api/QueryApi.cpp b/olp-cpp-sdk-dataservice-read/src/generated/api/QueryApi.cpp
@@ -60,7 +60,7 @@ QueryApi::PartitionsResponse QueryApi::GetPartitionsbyId(
     const client::OlpClient& client, const std::string& layer_id,
     const std::vector<std::string>& partitions,
     boost::optional<int64_t> version,
-    boost::optional<std::vector<std::string>> additional_fields,
+    const std::vector<std::string>& additional_fields,
     boost::optional<std::string> billing_tag,
     client::CancellationContext context) {
   std::multimap<std::string, std::string> header_params;
@@ -70,9 +70,9 @@ QueryApi::PartitionsResponse QueryApi::GetPartitionsbyId(
   for (const auto& partition : partitions) {
     query_params.emplace("partition", partition);
   }
-  if (additional_fields) {
+  if (!additional_fields.empty()) {
     query_params.insert(std::make_pair(
-        "additionalFields", ConcatStringArray(*additional_fields, ",")));
+        "additionalFields", ConcatStringArray(additional_fields, ",")));
   }
   if (billing_tag) {
     query_params.insert(std::make_pair("billingTag", *billing_tag));
diff --git a/olp-cpp-sdk-dataservice-read/src/generated/api/QueryApi.h b/olp-cpp-sdk-dataservice-read/src/generated/api/QueryApi.h
@@ -72,7 +72,7 @@ class QueryApi {
       const client::OlpClient& client, const std::string& layer_id,
       const std::vector<std::string>& partition,
       boost::optional<int64_t> version,
-      boost::optional<std::vector<std::string>> additional_fields,
+      const std::vector<std::string>& additional_fields,
       boost::optional<std::string> billing_tag,
       client::CancellationContext context);
 
diff --git a/olp-cpp-sdk-dataservice-read/src/generated/parser/PartitionsParser.cpp b/olp-cpp-sdk-dataservice-read/src/generated/parser/PartitionsParser.cpp
@@ -31,6 +31,7 @@ void from_json(const rapidjson::Value& value, model::Partition& x) {
       parse<boost::optional<int64_t>>(value, "compressedDataSize"));
   x.SetDataHandle(parse<std::string>(value, "dataHandle"));
   x.SetDataSize(parse<boost::optional<int64_t>>(value, "dataSize"));
+  x.SetCrc(parse<boost::optional<std::string>>(value, "crc"));
   x.SetPartition(parse<std::string>(value, "partition"));
   x.SetVersion(parse<boost::optional<int64_t>>(value, "version"));
 }
diff --git a/olp-cpp-sdk-dataservice-read/src/generated/serializer/PartitionsSerializer.cpp b/olp-cpp-sdk-dataservice-read/src/generated/serializer/PartitionsSerializer.cpp
@@ -33,6 +33,7 @@ void to_json(const dataservice::read::model::Partition& x,
   serialize("compressedDataSize", x.GetCompressedDataSize(), value, allocator);
   serialize("dataHandle", x.GetDataHandle(), value, allocator);
   serialize("dataSize", x.GetDataSize(), value, allocator);
+  serialize("crc", x.GetCrc(), value, allocator);
   serialize("partition", x.GetPartition(), value, allocator);
   serialize("version", x.GetVersion(), value, allocator);
 }
diff --git a/olp-cpp-sdk-dataservice-read/src/repositories/PartitionsRepository.cpp b/olp-cpp-sdk-dataservice-read/src/repositories/PartitionsRepository.cpp
@@ -141,7 +141,7 @@ PartitionsResponse PartitionsRepository::GetPartitions(
   PartitionsResponse response;
 
   const auto& partition_ids = request.GetPartitionIds();
-
+  
   if (partition_ids.empty()) {
     auto metadata_api =
         ApiClientLookup::LookupApi(catalog, cancellation_context, "metadata",
@@ -152,8 +152,9 @@ PartitionsResponse PartitionsRepository::GetPartitions(
     }
 
     response = MetadataApi::GetPartitions(
-        metadata_api.GetResult(), layer, request.GetVersion(), boost::none,
-        boost::none, request.GetBillingTag(), cancellation_context);
+        metadata_api.GetResult(), layer, request.GetVersion(),
+        request.GetAdditionalFields(), boost::none, request.GetBillingTag(),
+        cancellation_context);
   } else {
     auto query_api =
         ApiClientLookup::LookupApi(catalog, cancellation_context, "query", "v1",
@@ -165,7 +166,8 @@ PartitionsResponse PartitionsRepository::GetPartitions(
 
     response = QueryApi::GetPartitionsbyId(
         query_api.GetResult(), layer, partition_ids, request.GetVersion(),
-        boost::none, request.GetBillingTag(), cancellation_context);
+        request.GetAdditionalFields(), request.GetBillingTag(),
+        cancellation_context);
   }
 
   // Save all partitions only when downloaded via metadata API
@@ -235,8 +237,8 @@ PartitionsResponse PartitionsRepository::GetPartitionById(
   const client::OlpClient& client = query_api.GetResult();
 
   PartitionsResponse query_response = QueryApi::GetPartitionsbyId(
-      client, layer, partitions, version, boost::none,
-      data_request.GetBillingTag(), cancellation_context);
+      client, layer, partitions, version, {}, data_request.GetBillingTag(),
+      cancellation_context);
 
   if (query_response.IsSuccessful()) {
     OLP_SDK_LOG_INFO_F(kLogTag, "put '%s' to cache",
diff --git a/olp-cpp-sdk-dataservice-read/tests/ParserTest.cpp b/olp-cpp-sdk-dataservice-read/tests/ParserTest.cpp
@@ -296,6 +296,7 @@ TEST(ParserTest, Partitions) {
         \"compressedDataSize\": 1024,\
         \"dataHandle\": \"1b2ca68f-d4a0-4379-8120-cd025640510c\",\
         \"dataSize\": 1024,\
+        \"crc\": \"291f66\",\
         \"partition\": \"314010583\",\
         \"version\": 2\
       }\
@@ -318,6 +319,8 @@ TEST(ParserTest, Partitions) {
   ASSERT_EQ(1024, *partitions.GetPartitions().at(0).GetCompressedDataSize());
   ASSERT_EQ("1b2ca68f-d4a0-4379-8120-cd025640510c",
             partitions.GetPartitions().at(0).GetDataHandle());
+  ASSERT_TRUE(partitions.GetPartitions().at(0).GetCrc());
+  ASSERT_EQ("291f66", *partitions.GetPartitions().at(0).GetCrc());
   ASSERT_TRUE(partitions.GetPartitions().at(0).GetDataSize());
   ASSERT_EQ(1024, *partitions.GetPartitions().at(0).GetDataSize());
   ASSERT_EQ("314010583", partitions.GetPartitions().at(0).GetPartition());
diff --git a/olp-cpp-sdk-dataservice-read/tests/PartitionsRepositoryTest.cpp b/olp-cpp-sdk-dataservice-read/tests/PartitionsRepositoryTest.cpp
@@ -26,6 +26,7 @@
 #include <olp/core/cache/CacheSettings.h>
 #include <olp/core/client/OlpClientSettings.h>
 #include <olp/core/client/OlpClientSettingsFactory.h>
+#include <olp/core/utils/Url.h>
 #include <olp/dataservice/read/DataRequest.h>
 #include <olp/dataservice/read/PartitionsRequest.h>
 #include <olp/dataservice/read/TileRequest.h>
@@ -57,17 +58,32 @@ const std::string kOlpSdkHttpResponseLookupQuery =
     R"jsonString([{"api":"query","version":"v1","baseURL":"https://query.data.api.platform.here.com/metadata/v1/catalogs/hereos-internal-test-v2","parameters":{}}])jsonString";
 const std::string kCacheKeyMetadata = kCatalog + "::query::v1::api";
 
-const std::string kOlpSdkUrlPartitionByIdNoVersion =
+const std::string kOlpSdkUrlPartitionByIdBase =
     R"(https://query.data.api.platform.here.com/metadata/v1/catalogs/hereos-internal-test-v2/layers/)" +
-    kVersionedLayerId + R"(/partitions?partition=)" + kPartitionId;
+    kVersionedLayerId + R"(/partitions)";
+
+const std::string kOlpSdkUrlPartitionByIdNoVersion =
+    kOlpSdkUrlPartitionByIdBase + R"(?partition=)" + kPartitionId;
+
 const std::string kOlpSdkUrlPartitionById = kOlpSdkUrlPartitionByIdNoVersion +
                                             R"(&version=)" +
                                             std::to_string(kVersion);
+
+const std::string kOlpSdkUrlPartitionByIdWithAdditionalParams =
+    kOlpSdkUrlPartitionByIdBase + R"(?additionalFields=)" +
+    olp::utils::Url::Encode(R"(checksum,compressedDataSize,crc,dataSize)") +
+    R"(&partition=)" + kPartitionId + R"(&version=)" + std::to_string(kVersion);
+
 const std::string kOlpSdkHttpResponsePartitionById =
     R"jsonString({ "partitions": [{"version":42,"partition":")jsonString" +
     kPartitionId +
     R"jsonString(","layer":"olp-cpp-sdk-ingestion-test-volatile-layer","dataHandle":"PartitionsRepositoryTest-partitionId"}]})jsonString";
 
+const std::string kOlpSdkHttpResponsePartitionByIdWithAdditionalFields =
+    R"jsonString({ "partitions": [{"version":42,"partition":")jsonString" +
+    kPartitionId +
+    R"jsonString(","layer":"olp-cpp-sdk-ingestion-test-volatile-layer","dataHandle":"PartitionsRepositoryTest-partitionId","checksum":"xxx","compressedDataSize":15,"dataSize":10,"crc":"yyy"}]})jsonString";
+
 const std::string kOlpSdkHttpResponseEmptyPartitionList =
     R"({ "partitions": [] })";
 
@@ -114,6 +130,7 @@ const std::string kSubQuads =
 
 const std::string kBlobDataHandle23618364 =
     R"(f9a9fd8e-eb1b-48e5-bfdb-4392b3826443)";
+
 class PartitionsRepositoryTest : public ::testing::Test,
                                  protected repository::PartitionsRepository {};
 
@@ -737,6 +754,72 @@ TEST_F(PartitionsRepositoryTest, GetVolatilePartitions) {
   }
 }
 
+TEST_F(PartitionsRepositoryTest, AdditionalFields) {
+  using namespace testing;
+
+  std::shared_ptr<cache::KeyValueCache> default_cache =
+      olp::client::OlpClientSettingsFactory::CreateDefaultCache({});
+
+  auto mock_network = std::make_shared<NetworkMock>();
+  const auto catalog = HRN::FromString(kCatalog);
+
+  EXPECT_CALL(*mock_network,
+              Send(IsGetRequest(kOlpSdkUrlLookupQuery), _, _, _, _))
+      .WillOnce(ReturnHttpResponse(olp::http::NetworkResponse().WithStatus(
+                                       olp::http::HttpStatusCode::OK),
+                                   kOlpSdkHttpResponseLookupQuery));
+
+  EXPECT_CALL(*mock_network,
+              Send(IsGetRequest(kOlpSdkUrlPartitionByIdWithAdditionalParams), _,
+                   _, _, _))
+      .WillOnce(ReturnHttpResponse(
+          olp::http::NetworkResponse().WithStatus(
+              olp::http::HttpStatusCode::OK),
+          kOlpSdkHttpResponsePartitionByIdWithAdditionalFields));
+
+  OlpClientSettings settings;
+  settings.cache = default_cache;
+  settings.network_request_handler = mock_network;
+
+  CancellationContext context;
+  PartitionsRequest request;
+
+  request.WithPartitionIds({kPartitionId});
+  request.WithVersion(4);
+  request.WithAdditionalFields(
+      {PartitionsRequest::kChecksum, PartitionsRequest::kCompressedDataSize,
+       PartitionsRequest::kCrc, PartitionsRequest::kDataSize});
+
+  auto response = repository::PartitionsRepository::GetVersionedPartitions(
+      catalog, kVersionedLayerId, context, request, settings);
+
+  ASSERT_TRUE(response.IsSuccessful());
+  auto result = response.GetResult();
+  auto partitions = result.GetPartitions();
+  ASSERT_EQ(partitions.size(), 1);
+  EXPECT_EQ(partitions[0].GetDataSize().value_or(0), 10);
+  EXPECT_EQ(partitions[0].GetCompressedDataSize().value_or(0), 15);
+  EXPECT_EQ(partitions[0].GetChecksum().value_or(""), "xxx");
+  EXPECT_EQ(partitions[0].GetCrc().value_or(""), "yyy");
+
+  request.WithFetchOption(FetchOptions::CacheOnly);
+
+  auto response_2 = repository::PartitionsRepository::GetVersionedPartitions(
+      catalog, kVersionedLayerId, context, request, settings);
+
+  ASSERT_TRUE(response_2.IsSuccessful());
+
+  auto cached_result = response_2.GetResult();
+  auto cached_partitions = cached_result.GetPartitions();
+  ASSERT_EQ(cached_partitions.size(), 1);
+
+  EXPECT_EQ(partitions[0].GetDataSize(), cached_partitions[0].GetDataSize());
+  EXPECT_EQ(partitions[0].GetCompressedDataSize(),
+            cached_partitions[0].GetCompressedDataSize());
+  EXPECT_EQ(partitions[0].GetChecksum(), cached_partitions[0].GetChecksum());
+  EXPECT_EQ(partitions[0].GetCrc(), cached_partitions[0].GetCrc());
+}
+
 TEST_F(PartitionsRepositoryTest, CheckCashedPartitions) {
   using namespace testing;
   std::shared_ptr<cache::KeyValueCache> default_cache =
diff --git a/tests/functional/olp-cpp-sdk-dataservice-read/ApiTest.cpp b/tests/functional/olp-cpp-sdk-dataservice-read/ApiTest.cpp

Original file line number	Diff line number	Diff line change
`@@ -31,6 +31,7 @@ void from_json(const rapidjson::Value& value, model::Partition& x) {`
`31`	`31`	`parse<boost::optional<int64_t>>(value, "compressedDataSize"));`
`32`	`32`	`x.SetDataHandle(parse<std::string>(value, "dataHandle"));`
`33`	`33`	`x.SetDataSize(parse<boost::optional<int64_t>>(value, "dataSize"));`
	`34`	`+ x.SetCrc(parse<boost::optional<std::string>>(value, "crc"));`
`34`	`35`	`x.SetPartition(parse<std::string>(value, "partition"));`
`35`	`36`	`x.SetVersion(parse<boost::optional<int64_t>>(value, "version"));`
`36`	`37`	`}`
Original file line number	Diff line number	Diff line change
`@@ -33,6 +33,7 @@ void to_json(const dataservice::read::model::Partition& x,`
`33`	`33`	`serialize("compressedDataSize", x.GetCompressedDataSize(), value, allocator);`
`34`	`34`	`serialize("dataHandle", x.GetDataHandle(), value, allocator);`
`35`	`35`	`serialize("dataSize", x.GetDataSize(), value, allocator);`
	`36`	`+ serialize("crc", x.GetCrc(), value, allocator);`
`36`	`37`	`serialize("partition", x.GetPartition(), value, allocator);`
`37`	`38`	`serialize("version", x.GetVersion(), value, allocator);`
`38`	`39`	`}`