Merge branch 'main' into use_latest_write_handle

Author: shubham-up-47

ci/cloudbuild/builds/lib/integration.sh

@@ -31,7 +31,7 @@ source module ci/lib/io.sh
 export PATH="${HOME}/.local/bin:${PATH}"
 python3 -m pip uninstall -y --quiet googleapis-storage-testbench
 python3 -m pip install --upgrade --user --quiet --disable-pip-version-check \
-  "git+https://github.com/googleapis/storage-testbench@v0.52.0"
+  "git+https://github.com/googleapis/storage-testbench@v0.59.0"
 
 # Some of the tests will need a valid roots.pem file.
 rm -f /dev/shm/roots.pem

google/cloud/storage/client.h

@@ -21,6 +21,7 @@
 #include "google/cloud/storage/internal/signed_url_requests.h"
 #include "google/cloud/storage/internal/storage_connection.h"
 #include "google/cloud/storage/internal/tuple_filter.h"
+#include "google/cloud/storage/list_buckets_extended_reader.h"
 #include "google/cloud/storage/list_buckets_reader.h"
 #include "google/cloud/storage/list_hmac_keys_reader.h"
 #include "google/cloud/storage/list_objects_and_prefixes_reader.h"
@@ -366,6 +367,56 @@ class Client {
                                  std::forward<Options>(options)...);
   }
 
+  /**
+   * Fetches the list of buckets and unreachable resources for the default
+   * project.
+   *
+   * This function will return an error if it cannot determine the "default"
+   * project. The default project is found by looking, in order, for:
+   * - Any parameters of type `OverrideDefaultProject`, with a value.
+   * - Any `google::cloud::storage::ProjectIdOption` value in any parameters of
+   *   type `google::cloud::Options{}`.
+   * - Any `google::cloud::storage::ProjectIdOption` value provided in the
+   *   `google::cloud::Options{}` passed to the constructor.
+   * - The value from the `GOOGLE_CLOUD_PROJECT` environment variable.
+   *
+   * @param options a list of optional query parameters and/or request headers.
+   *     Valid types for this operation include `MaxResults`, `Prefix`,
+   *     `Projection`, `UserProject`, `OverrideDefaultProject`, and
+   *     `ReturnPartialSuccess`.
+   *
+   * @par Idempotency
+   * This is a read-only operation and is always idempotent.
+   *
+   * @par Example
+   * @snippet storage_bucket_samples.cc list buckets partial success
+   */
+  template <typename... Options>
+  ListBucketsExtendedReader ListBucketsExtended(Options&&... options) {
+    auto opts = SpanOptions(std::forward<Options>(options)...);
+    auto project_id = storage_internal::RequestProjectId(
+        GCP_ERROR_INFO(), opts, std::forward<Options>(options)...);
+    if (!project_id) {
+      return google::cloud::internal::MakeErrorPaginationRange<
+          ListBucketsExtendedReader>(std::move(project_id).status());
+    }
+    google::cloud::internal::OptionsSpan const span(std::move(opts));
+
+    internal::ListBucketsRequest request(*std::move(project_id));
+    request.set_multiple_options(std::forward<Options>(options)...);
+    auto& client = connection_;
+    return google::cloud::internal::MakePaginationRange<
+        ListBucketsExtendedReader>(
+        request,
+        [client](internal::ListBucketsRequest const& r) {
+          return client->ListBuckets(r);
+        },
+        [](internal::ListBucketsResponse res) {
+          return std::vector<BucketsExtended>{BucketsExtended{
+              std::move(res.items), std::move(res.unreachable)}};
+        });
+  }
+
   /**
    * Creates a new Google Cloud Storage bucket using the default project.
    *

google/cloud/storage/examples/storage_bucket_samples.cc

@@ -14,6 +14,7 @@
 
 #include "google/cloud/storage/client.h"
 #include "google/cloud/storage/examples/storage_examples_common.h"
+#include "google/cloud/storage/list_buckets_extended_reader.h"
 #include "google/cloud/internal/getenv.h"
 #include <functional>
 #include <iostream>
@@ -48,6 +49,35 @@ void ListBuckets(google::cloud::storage::Client client,
   (std::move(client));
 }
 
+void ListBucketsPartialSuccess(google::cloud::storage::Client client,
+                               std::vector<std::string> const& /*argv*/) {
+  //! [list buckets partial success] [START
+  //! storage_list_buckets_partial_success]
+  namespace gcs = ::google::cloud::storage;
+  using ::google::cloud::StatusOr;
+  [](gcs::Client client) {
+    int count = 0;
+    gcs::ListBucketsExtendedReader bucket_list = client.ListBucketsExtended();
+    for (auto&& result : bucket_list) {
+      if (!result) throw std::move(result).status();
+
+      for (auto const& bucket_metadata : result->buckets) {
+        std::cout << bucket_metadata.name() << "\n";
+        ++count;
+      }
+      for (auto const& unreachable : result->unreachable) {
+        std::cout << "Unreachable location: " << unreachable << "\n";
+      }
+    }
+
+    if (count == 0) {
+      std::cout << "No buckets in default project\n";
+    }
+  }
+  //! [list buckets partial success] [END storage_list_buckets_partial_success]
+  (std::move(client));
+}
+
 void ListBucketsForProject(google::cloud::storage::Client client,
                            std::vector<std::string> const& argv) {
   //! [list buckets for project]
@@ -683,6 +713,9 @@ void RunAll(std::vector<std::string> const& argv) {
   std::cout << "\nRunning ListBuckets() example" << std::endl;
   ListBuckets(client, {});
 
+  std::cout << "\nRunning ListBucketsPartialSuccess() example" << std::endl;
+  ListBucketsPartialSuccess(client, {});
+
   std::cout << "\nRunning CreateBucket() example" << std::endl;
   CreateBucket(client, {bucket_name});
 
@@ -726,6 +759,8 @@ int main(int argc, char* argv[]) {
 
   examples::Example example({
       examples::CreateCommandEntry("list-buckets", {}, ListBuckets),
+      examples::CreateCommandEntry("list-buckets-partial-success", {},
+                                   ListBucketsPartialSuccess),
       examples::CreateCommandEntry("list-buckets-for-project", {"<project-id>"},
                                    ListBucketsForProject),
       make_entry("create-bucket", {}, CreateBucket),

google/cloud/storage/examples/storage_client_initialization_samples.cc

@@ -137,7 +137,6 @@ void SetClientUniverseDomain(std::vector<std::string> const& argv) {
         " <bucket-name> <object-name>"};
   }
   //! [START storage_set_client_universe_domain] [set-client-universe-domain]
-  namespace g = ::google::cloud;
   namespace gcs = ::google::cloud::storage;
   [](std::string const& bucket_name, std::string const& object_name) {
     google::cloud::Options options;

google/cloud/storage/google_cloud_cpp_storage.bzl

@@ -116,6 +116,7 @@ google_cloud_cpp_storage_hdrs = [
     "internal/unified_rest_credentials.h",
     "internal/well_known_parameters_impl.h",
     "lifecycle_rule.h",
+    "list_buckets_extended_reader.h",
     "list_buckets_reader.h",
     "list_hmac_keys_reader.h",
     "list_objects_and_prefixes_reader.h",

google/cloud/storage/google_cloud_cpp_storage.cmake

@@ -198,6 +198,7 @@ add_library(
     internal/win32/hash_function_impl.cc
     lifecycle_rule.cc
     lifecycle_rule.h
+    list_buckets_extended_reader.h
     list_buckets_reader.cc
     list_buckets_reader.h
     list_hmac_keys_reader.cc
@@ -506,6 +507,7 @@ if (BUILD_TESTING)
         internal/tuple_filter_test.cc
         internal/unified_rest_credentials_test.cc
         lifecycle_rule_test.cc
+        list_buckets_extended_reader_test.cc
         list_buckets_reader_test.cc
         list_hmac_keys_reader_test.cc
         list_objects_and_prefixes_reader_test.cc

google/cloud/storage/internal/async/writer_connection_impl.cc

@@ -133,7 +133,8 @@ AsyncWriterConnectionImpl::Finalize(
 
   auto p = WritePayloadImpl::GetImpl(payload);
   auto size = p.size();
-  auto action = request_.has_append_object_spec()
+  auto action = request_.has_append_object_spec() ||
+                        request_.write_object_spec().appendable()
                     ? PartialUpload::kFinalize
                     : PartialUpload::kFinalizeWithChecksum;
   auto coro = PartialUpload::Call(impl_, hash_function_, std::move(write),

google/cloud/storage/internal/async/writer_connection_impl_test.cc

@@ -649,6 +649,56 @@ TEST(AsyncWriterConnectionTest, UnexpectedQueryFailsWithoutError) {
   EXPECT_THAT(query.get(), StatusIs(StatusCode::kInternal));
 }
 
+TEST(AsyncWriterConnectionTest, FinalizeAppendableNoChecksum) {
+  AsyncSequencer<bool> sequencer;
+  auto mock = std::make_unique<MockStream>();
+  EXPECT_CALL(*mock, Cancel).Times(1);
+  EXPECT_CALL(*mock, Write)
+      .WillOnce([&](Request const& request, grpc::WriteOptions wopt) {
+        EXPECT_TRUE(request.finish_write());
+        EXPECT_TRUE(wopt.is_last_message());
+        EXPECT_EQ(request.common_object_request_params().encryption_algorithm(),
+                  "test-only-algo");
+        EXPECT_FALSE(request.has_object_checksums());
+        return sequencer.PushBack("Write");
+      });
+  EXPECT_CALL(*mock, Read).WillOnce([&]() {
+    return sequencer.PushBack("Read").then([](auto f) {
+      if (!f.get()) return absl::optional<Response>();
+      return absl::make_optional(MakeTestResponse());
+    });
+  });
+  EXPECT_CALL(*mock, Finish).WillOnce([&] {
+    return sequencer.PushBack("Finish").then([](auto f) {
+      if (f.get()) return Status{};
+      return PermanentError();
+    });
+  });
+  auto hash = std::make_shared<MockHashFunction>();
+  EXPECT_CALL(*hash, Update(_, An<absl::Cord const&>(), _)).Times(1);
+  EXPECT_CALL(*hash, Finish).Times(0);
+
+  auto request = MakeRequest();
+  request.mutable_write_object_spec()->set_appendable(true);
+  auto tested = std::make_unique<AsyncWriterConnectionImpl>(
+      TestOptions(), std::move(request), std::move(mock), hash, 1024);
+  auto response = tested->Finalize(WritePayload{});
+  auto next = sequencer.PopFrontWithName();
+  ASSERT_THAT(next.second, "Write");
+  next.first.set_value(true);
+  next = sequencer.PopFrontWithName();
+  ASSERT_THAT(next.second, "Read");
+  next.first.set_value(true);
+  auto object = response.get();
+  EXPECT_THAT(object, IsOkAndHolds(IsProtoEqual(MakeTestObject())))
+      << "=" << object->DebugString();
+
+  tested = {};
+  next = sequencer.PopFrontWithName();
+  ASSERT_THAT(next.second, "Finish");
+  next.first.set_value(true);
+}
+
 TEST(AsyncWriterConnectionTest, WriteHandleIsUpdatedAfterQuery) {
   AsyncSequencer<bool> sequencer;
   auto mock = std::make_unique<MockStream>();

google/cloud/storage/internal/bucket_requests.cc

@@ -243,6 +243,12 @@ StatusOr<ListBucketsResponse> ListBucketsResponse::FromHttpResponse(
     result.items.emplace_back(std::move(*parsed));
   }
 
+  if (json.count("unreachable") != 0) {
+    for (auto const& kv : json["unreachable"].items()) {
+      result.unreachable.emplace_back(kv.value().get<std::string>());
+    }
+  }
+
   return result;
 }
 
@@ -256,6 +262,9 @@ std::ostream& operator<<(std::ostream& os, ListBucketsResponse const& r) {
      << ", items={";
   std::copy(r.items.begin(), r.items.end(),
             std::ostream_iterator<BucketMetadata>(os, "\n  "));
+  os << "}, unreachable={";
+  std::copy(r.unreachable.begin(), r.unreachable.end(),
+            std::ostream_iterator<std::string>(os, "\n "));
   return os << "}}";
 }
 

google/cloud/storage/internal/bucket_requests.h

@@ -37,14 +37,18 @@ namespace internal {
  */
 class ListBucketsRequest
     : public GenericRequest<ListBucketsRequest, MaxResults, Prefix, Projection,
-                            UserProject, OverrideDefaultProject> {
+                            UserProject, OverrideDefaultProject,
+                            ReturnPartialSuccess> {
  public:
   ListBucketsRequest() = default;
   explicit ListBucketsRequest(std::string project_id)
       : project_id_(std::move(project_id)) {}
 
   std::string const& project_id() const { return project_id_; }
   std::string const& page_token() const { return page_token_; }
+  bool return_partial_success() const {
+    return GetOption<ReturnPartialSuccess>().value_or(false);
+  }
   ListBucketsRequest& set_page_token(std::string page_token) {
     page_token_ = std::move(page_token);
     return *this;
@@ -65,6 +69,7 @@ struct ListBucketsResponse {
 
   std::string next_page_token;
   std::vector<BucketMetadata> items;
+  std::vector<std::string> unreachable;
 };
 
 std::ostream& operator<<(std::ostream& os, ListBucketsResponse const& r);