feat(storage): Add full object checksum validation at finalization for JSON path

Author: v-pratap

.bazelignore

@@ -5,3 +5,4 @@ cmake-build-debug/
 cmake-build-coverage/
 cmake-build-release/
 .build/
+_build/

google/cloud/storage/internal/object_requests.h

@@ -534,6 +534,9 @@ class UploadChunkRequest
   HashValues const& known_object_hashes() const { return known_object_hashes_; }
 
   HashFunction& hash_function() const { return *hash_function_; }
+  std::shared_ptr<HashFunction> hash_function_ptr() const {
+    return hash_function_;
+  }
 
   bool last_chunk() const { return upload_size_.has_value(); }
   std::size_t payload_size() const { return TotalBytes(payload_); }

google/cloud/storage/internal/object_write_streambuf.cc

@@ -144,22 +144,26 @@ void ObjectWriteStreambuf::FlushFinal() {
 
   // Calculate the portion of the buffer that needs to be uploaded, if any.
   auto const actual_size = put_area_size();
+ HashValues final_hashes = known_hashes_;
+ if (hash_function_) {
+   hash_function_->Update(committed_size_, {pbase(), actual_size});
+   final_hashes = hash_function_->Finish();
+   hash_function_.reset();
+ }
 
   // After this point the session will be closed, and no more calls to the hash
   // function are possible.
   auto upload_request = UploadChunkRequest(upload_id_, committed_size_,
                                            {ConstBuffer(pbase(), actual_size)},
-                                           hash_function_, known_hashes_);
+                                           hash_function_, final_hashes);
   request_.ForEachOption(internal::CopyCommonOptions(upload_request));
   OptionsSpan const span(span_options_);
   auto response = connection_->UploadChunk(upload_request);
   if (!response) {
     last_status_ = std::move(response).status();
     return;
   }
-
-  auto function = std::move(hash_function_);
-  hash_values_ = std::move(*function).Finish();
+ hash_values_ = final_hashes;
 
   committed_size_ = response->committed_size.value_or(0);
   metadata_ = std::move(response->payload);
@@ -203,6 +207,9 @@ void ObjectWriteStreambuf::FlushRoundChunk(ConstBufferSequence buffers) {
   auto upload_request =
       UploadChunkRequest(upload_id_, committed_size_, payload, hash_function_);
   request_.ForEachOption(internal::CopyCommonOptions(upload_request));
+  upload_request.ForEachOption([](auto const& opt) {
+    std::cout << "DEBUG: option=" << opt << "\n";
+  });
   OptionsSpan const span(span_options_);
   auto response = connection_->UploadChunk(upload_request);
   if (!response) {

google/cloud/storage/internal/rest/stub.cc

@@ -770,12 +770,23 @@ StatusOr<QueryResumableUploadResponse> RestStub::UploadChunk(
   // default (at least in this case), and that wastes bandwidth as the content
   // length is known.
   builder.AddHeader("Transfer-Encoding", {});
-  auto offset = request.offset();
-  for (auto const& b : request.payload()) {
-    request.hash_function().Update(offset,
-                                   absl::string_view{b.data(), b.size()});
-    offset += b.size();
-  }
+ auto hash_function = request.hash_function_ptr();
+ if (hash_function) {
+   auto offset = request.offset();
+   for (auto const& b : request.payload()) {
+     hash_function->Update(offset, absl::string_view{b.data(), b.size()});
+     offset += b.size();
+   }
+ }
+ if (request.last_chunk()) {
+   auto const& hashes = request.known_object_hashes();
+   if (!hashes.crc32c.empty()) {
+     builder.AddHeader("x-goog-hash", "crc32c=" + hashes.crc32c);
+   }
+   if (!hashes.md5.empty()) {
+     builder.AddHeader("x-goog-hash", "md5=" + hashes.md5);
+   }
+ }
 
   auto failure_predicate = [](rest::HttpStatusCode code) {
     return (code != rest::HttpStatusCode::kResumeIncomplete &&

google/cloud/storage/quickstart/quickstart.cc

@@ -18,19 +18,25 @@
 #include <string>
 
 int main(int argc, char* argv[]) {
-  if (argc != 2) {
-    std::cerr << "Missing bucket name.\n";
-    std::cerr << "Usage: quickstart <bucket-name>\n";
-    return 1;
-  }
-  std::string const bucket_name = argv[1];
+  // if (argc != 2) {
+  //   std::cerr << "Missing bucket name.\n";
+  //   std::cerr << "Usage: quickstart <bucket-name>\n";
+  //   return 1;
+  // }
+  std::string const bucket_name = "vaibhav-test-001";
 
   // Create a client to communicate with Google Cloud Storage. This client
   // uses the default configuration for authentication and project id.
   auto client = google::cloud::storage::Client();
 
-  auto writer = client.WriteObject(bucket_name, "quickstart.txt");
-  writer << "Hello World!";
+  auto writer = client.WriteObject(bucket_name, "quickstart1.txt");
+  std::string x ="Hello World1!";
+  for(int i=0;i<17;i++){
+    x+=x;
+  }
+  for(int i=0;i<15;i++){
+  writer << x.data();
+  }
   writer.Close();
   if (!writer.metadata()) {
     std::cerr << "Error creating object: " << writer.metadata().status()
@@ -39,14 +45,14 @@ int main(int argc, char* argv[]) {
   }
   std::cout << "Successfully created object: " << *writer.metadata() << "\n";
 
-  auto reader = client.ReadObject(bucket_name, "quickstart.txt");
-  if (!reader) {
-    std::cerr << "Error reading object: " << reader.status() << "\n";
-    return 1;
-  }
+  // auto reader = client.ReadObject(bucket_name, "quickstart.txt");
+  // if (!reader) {
+  //   std::cerr << "Error reading object: " << reader.status() << "\n";
+  //   return 1;
+  // }
 
-  std::string contents{std::istreambuf_iterator<char>{reader}, {}};
-  std::cout << contents << "\n";
+  // std::string contents{std::istreambuf_iterator<char>{reader}, {}};
+  // std::cout << contents << "\n";
 
   return 0;
 }