feat(storage): add support for partial list bucket (#15763)

Author: bajajneha27

ci/cloudbuild/builds/lib/integration.sh

@@ -31,7 +31,7 @@ source module ci/lib/io.sh
 export PATH="${HOME}/.local/bin:${PATH}"
 python3 -m pip uninstall -y --quiet googleapis-storage-testbench
 python3 -m pip install --upgrade --user --quiet --disable-pip-version-check \
-  "git+https://github.com/googleapis/storage-testbench@v0.52.0"
+  "git+https://github.com/googleapis/storage-testbench@v0.59.0"
 
 # Some of the tests will need a valid roots.pem file.
 rm -f /dev/shm/roots.pem

google/cloud/storage/client.h

@@ -21,6 +21,7 @@
 #include "google/cloud/storage/internal/signed_url_requests.h"
 #include "google/cloud/storage/internal/storage_connection.h"
 #include "google/cloud/storage/internal/tuple_filter.h"
+#include "google/cloud/storage/list_buckets_extended_reader.h"
 #include "google/cloud/storage/list_buckets_reader.h"
 #include "google/cloud/storage/list_hmac_keys_reader.h"
 #include "google/cloud/storage/list_objects_and_prefixes_reader.h"
@@ -366,6 +367,56 @@ class Client {
                                  std::forward<Options>(options)...);
   }
 
+  /**
+   * Fetches the list of buckets and unreachable resources for the default
+   * project.
+   *
+   * This function will return an error if it cannot determine the "default"
+   * project. The default project is found by looking, in order, for:
+   * - Any parameters of type `OverrideDefaultProject`, with a value.
+   * - Any `google::cloud::storage::ProjectIdOption` value in any parameters of
+   *   type `google::cloud::Options{}`.
+   * - Any `google::cloud::storage::ProjectIdOption` value provided in the
+   *   `google::cloud::Options{}` passed to the constructor.
+   * - The value from the `GOOGLE_CLOUD_PROJECT` environment variable.
+   *
+   * @param options a list of optional query parameters and/or request headers.
+   *     Valid types for this operation include `MaxResults`, `Prefix`,
+   *     `Projection`, `UserProject`, `OverrideDefaultProject`, and
+   *     `ReturnPartialSuccess`.
+   *
+   * @par Idempotency
+   * This is a read-only operation and is always idempotent.
+   *
+   * @par Example
+   * @snippet storage_bucket_samples.cc list buckets partial success
+   */
+  template <typename... Options>
+  ListBucketsExtendedReader ListBucketsExtended(Options&&... options) {
+    auto opts = SpanOptions(std::forward<Options>(options)...);
+    auto project_id = storage_internal::RequestProjectId(
+        GCP_ERROR_INFO(), opts, std::forward<Options>(options)...);
+    if (!project_id) {
+      return google::cloud::internal::MakeErrorPaginationRange<
+          ListBucketsExtendedReader>(std::move(project_id).status());
+    }
+    google::cloud::internal::OptionsSpan const span(std::move(opts));
+
+    internal::ListBucketsRequest request(*std::move(project_id));
+    request.set_multiple_options(std::forward<Options>(options)...);
+    auto& client = connection_;
+    return google::cloud::internal::MakePaginationRange<
+        ListBucketsExtendedReader>(
+        request,
+        [client](internal::ListBucketsRequest const& r) {
+          return client->ListBuckets(r);
+        },
+        [](internal::ListBucketsResponse res) {
+          return std::vector<BucketsExtended>{BucketsExtended{
+              std::move(res.items), std::move(res.unreachable)}};
+        });
+  }
+
   /**
    * Creates a new Google Cloud Storage bucket using the default project.
    *

google/cloud/storage/examples/storage_bucket_samples.cc

@@ -14,6 +14,7 @@
 
 #include "google/cloud/storage/client.h"
 #include "google/cloud/storage/examples/storage_examples_common.h"
+#include "google/cloud/storage/list_buckets_extended_reader.h"
 #include "google/cloud/internal/getenv.h"
 #include <functional>
 #include <iostream>
@@ -48,6 +49,35 @@ void ListBuckets(google::cloud::storage::Client client,
   (std::move(client));
 }
 
+void ListBucketsPartialSuccess(google::cloud::storage::Client client,
+                               std::vector<std::string> const& /*argv*/) {
+  //! [list buckets partial success] [START
+  //! storage_list_buckets_partial_success]
+  namespace gcs = ::google::cloud::storage;
+  using ::google::cloud::StatusOr;
+  [](gcs::Client client) {
+    int count = 0;
+    gcs::ListBucketsExtendedReader bucket_list = client.ListBucketsExtended();
+    for (auto&& result : bucket_list) {
+      if (!result) throw std::move(result).status();
+
+      for (auto const& bucket_metadata : result->buckets) {
+        std::cout << bucket_metadata.name() << "\n";
+        ++count;
+      }
+      for (auto const& unreachable : result->unreachable) {
+        std::cout << "Unreachable location: " << unreachable << "\n";
+      }
+    }
+
+    if (count == 0) {
+      std::cout << "No buckets in default project\n";
+    }
+  }
+  //! [list buckets partial success] [END storage_list_buckets_partial_success]
+  (std::move(client));
+}
+
 void ListBucketsForProject(google::cloud::storage::Client client,
                            std::vector<std::string> const& argv) {
   //! [list buckets for project]
@@ -683,6 +713,9 @@ void RunAll(std::vector<std::string> const& argv) {
   std::cout << "\nRunning ListBuckets() example" << std::endl;
   ListBuckets(client, {});
 
+  std::cout << "\nRunning ListBucketsPartialSuccess() example" << std::endl;
+  ListBucketsPartialSuccess(client, {});
+
   std::cout << "\nRunning CreateBucket() example" << std::endl;
   CreateBucket(client, {bucket_name});
 
@@ -726,6 +759,8 @@ int main(int argc, char* argv[]) {
 
   examples::Example example({
       examples::CreateCommandEntry("list-buckets", {}, ListBuckets),
+      examples::CreateCommandEntry("list-buckets-partial-success", {},
+                                   ListBucketsPartialSuccess),
       examples::CreateCommandEntry("list-buckets-for-project", {"<project-id>"},
                                    ListBucketsForProject),
       make_entry("create-bucket", {}, CreateBucket),

google/cloud/storage/examples/storage_client_initialization_samples.cc

@@ -137,7 +137,6 @@ void SetClientUniverseDomain(std::vector<std::string> const& argv) {
         " <bucket-name> <object-name>"};
   }
   //! [START storage_set_client_universe_domain] [set-client-universe-domain]
-  namespace g = ::google::cloud;
   namespace gcs = ::google::cloud::storage;
   [](std::string const& bucket_name, std::string const& object_name) {
     google::cloud::Options options;

google/cloud/storage/google_cloud_cpp_storage.bzl

@@ -116,6 +116,7 @@ google_cloud_cpp_storage_hdrs = [
     "internal/unified_rest_credentials.h",
     "internal/well_known_parameters_impl.h",
     "lifecycle_rule.h",
+    "list_buckets_extended_reader.h",
     "list_buckets_reader.h",
     "list_hmac_keys_reader.h",
     "list_objects_and_prefixes_reader.h",

google/cloud/storage/google_cloud_cpp_storage.cmake

@@ -198,6 +198,7 @@ add_library(
     internal/win32/hash_function_impl.cc
     lifecycle_rule.cc
     lifecycle_rule.h
+    list_buckets_extended_reader.h
     list_buckets_reader.cc
     list_buckets_reader.h
     list_hmac_keys_reader.cc
@@ -506,6 +507,7 @@ if (BUILD_TESTING)
         internal/tuple_filter_test.cc
         internal/unified_rest_credentials_test.cc
         lifecycle_rule_test.cc
+        list_buckets_extended_reader_test.cc
         list_buckets_reader_test.cc
         list_hmac_keys_reader_test.cc
         list_objects_and_prefixes_reader_test.cc

google/cloud/storage/internal/bucket_requests.cc

@@ -243,6 +243,12 @@ StatusOr<ListBucketsResponse> ListBucketsResponse::FromHttpResponse(
     result.items.emplace_back(std::move(*parsed));
   }
 
+  if (json.count("unreachable") != 0) {
+    for (auto const& kv : json["unreachable"].items()) {
+      result.unreachable.emplace_back(kv.value().get<std::string>());
+    }
+  }
+
   return result;
 }
 
@@ -256,6 +262,9 @@ std::ostream& operator<<(std::ostream& os, ListBucketsResponse const& r) {
      << ", items={";
   std::copy(r.items.begin(), r.items.end(),
             std::ostream_iterator<BucketMetadata>(os, "\n  "));
+  os << "}, unreachable={";
+  std::copy(r.unreachable.begin(), r.unreachable.end(),
+            std::ostream_iterator<std::string>(os, "\n "));
   return os << "}}";
 }
 

google/cloud/storage/internal/bucket_requests.h

@@ -37,14 +37,18 @@ namespace internal {
  */
 class ListBucketsRequest
     : public GenericRequest<ListBucketsRequest, MaxResults, Prefix, Projection,
-                            UserProject, OverrideDefaultProject> {
+                            UserProject, OverrideDefaultProject,
+                            ReturnPartialSuccess> {
  public:
   ListBucketsRequest() = default;
   explicit ListBucketsRequest(std::string project_id)
       : project_id_(std::move(project_id)) {}
 
   std::string const& project_id() const { return project_id_; }
   std::string const& page_token() const { return page_token_; }
+  bool return_partial_success() const {
+    return GetOption<ReturnPartialSuccess>().value_or(false);
+  }
   ListBucketsRequest& set_page_token(std::string page_token) {
     page_token_ = std::move(page_token);
     return *this;
@@ -65,6 +69,7 @@ struct ListBucketsResponse {
 
   std::string next_page_token;
   std::vector<BucketMetadata> items;
+  std::vector<std::string> unreachable;
 };
 
 std::ostream& operator<<(std::ostream& os, ListBucketsResponse const& r);

google/cloud/storage/internal/bucket_requests_test.cc

@@ -134,6 +134,16 @@ TEST(ListBucketsRequestTest, OStream) {
   EXPECT_THAT(os.str(), HasSubstr("prefix=foo-bar-baz"));
 }
 
+TEST(ListBucketsRequestTest, PartialSuccess) {
+  ListBucketsRequest request("project-to-list");
+  request.set_multiple_options(ReturnPartialSuccess(true));
+
+  std::ostringstream os;
+  os << request;
+  EXPECT_THAT(os.str(), HasSubstr("ListBucketsRequest"));
+  EXPECT_THAT(os.str(), HasSubstr("returnPartialSuccess=true"));
+}
+
 TEST(ListBucketsResponseTest, Parse) {
   std::string bucket1 = R"""({
       "kind": "storage#bucket",

google/cloud/storage/internal/connection_impl_bucket_test.cc

@@ -66,6 +66,30 @@ TEST(StorageConnectionImpl, ListBucketPermanentFailure) {
   EXPECT_THAT(permanent.captured_authority_options(), RetryLoopUsesOptions());
 }
 
+TEST(StorageConnectionImpl, ListBucketsPartialResult) {
+  auto mock = std::make_unique<MockGenericStub>();
+  EXPECT_CALL(*mock, options);
+  EXPECT_CALL(*mock, ListBuckets)
+      .WillOnce([](rest_internal::RestContext&, Options const&,
+                   ListBucketsRequest const& r) {
+        EXPECT_TRUE(r.return_partial_success());
+        ListBucketsResponse response;
+        response.items.emplace_back(BucketMetadata{}.set_name("bucket2"));
+        response.unreachable.emplace_back("projects/_/buckets/bucket1");
+        return response;
+      });
+  auto client =
+      StorageConnectionImpl::Create(std::move(mock), RetryTestOptions());
+  google::cloud::internal::OptionsSpan span(client->options());
+  auto response =
+      client->ListBuckets(ListBucketsRequest("test-project")
+                              .set_option(ReturnPartialSuccess(true)));
+  ASSERT_TRUE(response.ok());
+  EXPECT_EQ(1, response->items.size());
+  EXPECT_THAT(response->unreachable,
+              ::testing::ElementsAre("projects/_/buckets/bucket1"));
+}
+
 TEST(StorageConnectionImpl, CreateBucketTooManyFailures) {
   auto transient = MockRetryClientFunction(TransientError());
   auto mock = std::make_unique<MockGenericStub>();