cleanup(storage): report transfer size in benchmark (#8447)

For the most part this just creates a placeholder to report the transfer
size. While the transfer size and object size could be different in case
of (unrecoverable) errors during download, the real motivation is to
support partial downloads in a future PR.

Author: coryan

google/cloud/storage/benchmarks/throughput_experiment.cc

@@ -16,9 +16,6 @@
 #include "google/cloud/storage/benchmarks/benchmark_utils.h"
 #include "google/cloud/storage/client.h"
 #include "google/cloud/storage/grpc_plugin.h"
-#if GOOGLE_CLOUD_CPP_STORAGE_HAVE_GRPC
-#include "google/cloud/storage/internal/grpc_client.h"
-#endif  // GOOGLE_CLOUD_CPP_STORAGE_HAVE_GRPC
 #include "google/cloud/grpc_error_delegate.h"
 #include "google/cloud/internal/getenv.h"
 #include "absl/algorithm/container.h"
@@ -72,6 +69,7 @@ class UploadObject : public ThroughputExperiment {
           gcs::DisableMD5Hash(!config.enable_md5), api_selector);
       auto const usage = timer.Sample();
       return ThroughputResult{kOpInsert,
+                              config.object_size,
                               config.object_size,
                               config.app_buffer_size,
                               config.lib_buffer_size,
@@ -99,6 +97,7 @@ class UploadObject : public ThroughputExperiment {
     auto const usage = timer.Sample();
 
     return ThroughputResult{kOpWrite,
+                            config.object_size,
                             config.object_size,
                             config.app_buffer_size,
                             config.lib_buffer_size,
@@ -143,12 +142,14 @@ class DownloadObject : public ThroughputExperiment {
         bucket_name, object_name,
         gcs::DisableCrc32cChecksum(!config.enable_crc32c),
         gcs::DisableMD5Hash(!config.enable_md5), api_selector);
-    for (std::uint64_t num_read = 0; reader.read(buffer.data(), buffer.size());
-         num_read += reader.gcount()) {
+    std::int64_t transfer_size = 0;
+    while (reader.read(buffer.data(), buffer.size())) {
+      transfer_size += reader.gcount();
     }
     auto const usage = timer.Sample();
     return ThroughputResult{config.op,
                             config.object_size,
+                            transfer_size,
                             config.app_buffer_size,
                             config.lib_buffer_size,
                             config.enable_crc32c,
@@ -228,6 +229,7 @@ class DownloadObjectLibcurl : public ThroughputExperiment {
     curl_slist_free_all(slist1);
     auto const usage = timer.Sample();
     return ThroughputResult{config.op,
+                            config.object_size,
                             config.object_size,
                             config.app_buffer_size,
                             config.lib_buffer_size,
@@ -301,6 +303,7 @@ class DownloadObjectRawGrpc : public ThroughputExperiment {
 
     return ThroughputResult{config.op,
                             config.object_size,
+                            bytes_received,
                             config.app_buffer_size,
                             /*lib_buffer_size=*/0,
                             /*crc_enabled=*/false,

google/cloud/storage/benchmarks/throughput_result.cc

@@ -39,14 +39,22 @@ std::string QuoteCsv(T const& element) {
 }  // namespace
 
 void PrintAsCsv(std::ostream& os, ThroughputResult const& r) {
-  os << ToString(r.op) << ',' << r.object_size << ',' << r.app_buffer_size
-     << ',' << r.lib_buffer_size << ',' << r.crc_enabled << ',' << r.md5_enabled
-     << ',' << ToString(r.api) << ',' << r.elapsed_time.count() << ','
-     << r.cpu_time.count() << ',' << QuoteCsv(r.status) << '\n';
+  os << ToString(r.op)                 // force clang to keep one field per-line
+     << ',' << r.object_size           //
+     << ',' << r.transfer_size         //
+     << ',' << r.app_buffer_size       //
+     << ',' << r.lib_buffer_size       //
+     << ',' << r.crc_enabled           //
+     << ',' << r.md5_enabled           //
+     << ',' << ToString(r.api)         //
+     << ',' << r.elapsed_time.count()  //
+     << ',' << r.cpu_time.count()      //
+     << ',' << QuoteCsv(r.status)      //
+     << '\n';
 }
 
 void PrintThroughputResultHeader(std::ostream& os) {
-  os << "Op,ObjectSize,AppBufferSize,LibBufferSize"
+  os << "Op,ObjectSize,TransferSize,AppBufferSize,LibBufferSize"
      << ",Crc32cEnabled,MD5Enabled,ApiName"
      << ",ElapsedTimeUs,CpuTimeUs,Status\n";
 }

google/cloud/storage/benchmarks/throughput_result.h

@@ -61,8 +61,10 @@ struct ThroughputResult {
   OpType op;
   /// The total size of the object involved in this experiment. Currently also
   /// represents the number of bytes transferred.
-  // TODO(#4349) - use a separate field to represent the bytes transferred
   std::int64_t object_size;
+  /// The size of the transfer. For uploads this is always equal to the object
+  /// size. For downloads this can be smaller than the object size.
+  std::int64_t transfer_size;
   /// The size of the application buffer (for .read() or .write() calls).
   std::size_t app_buffer_size;
   /// The size of the library buffers (if any).

google/cloud/storage/benchmarks/throughput_result_test.cc

@@ -31,7 +31,7 @@ MATCHER_P(
     "status field from PrintAsCsv is properly quoted and contains substr") {
   // The status field is the 10th value.
   std::size_t pos = 0;
-  for (int i = 0; i < 9; ++i) {
+  for (int i = 0; i < 10; ++i) {
     pos = arg.find(',', pos);
     if (pos == std::string::npos) {
       *result_listener << "Couldn't find status field: " << arg;
@@ -68,7 +68,7 @@ TEST(ThroughputResult, HeaderMatches) {
   auto const header = std::move(header_stream).str();
 
   auto const line = ToString(ThroughputResult{
-      kOpInsert, /*object_size=*/3 * kMiB,
+      kOpInsert, /*object_size=*/5 * kMiB, /*transfer_size=*/3 * kMiB,
       /*app_buffer_size=*/2 * kMiB, /*lib_buffer_size=*/4 * kMiB,
       /*crc_enabled=*/true, /*md5_enabled=*/false, ApiName::kApiGrpc,
       std::chrono::microseconds(234000), std::chrono::microseconds(345000),
@@ -86,6 +86,7 @@ TEST(ThroughputResult, HeaderMatches) {
   // We don't want to create a change detector test, but we can verify the basic
   // fields are formatted correctly.
   EXPECT_THAT(*line, HasSubstr(ToString(kOpInsert)));
+  EXPECT_THAT(*line, HasSubstr("," + std::to_string(5 * kMiB) + ","));
   EXPECT_THAT(*line, HasSubstr("," + std::to_string(3 * kMiB) + ","));
   EXPECT_THAT(*line, HasSubstr("," + std::to_string(2 * kMiB) + ","));
   EXPECT_THAT(*line, HasSubstr("," + std::to_string(4 * kMiB) + ","));