Skip to content

Commit 8a0ff6c

Browse files
bajajneha27bajajneha27
committed
feat(storage): add support for partial list bucket
1 parent 887d7d0 commit 8a0ff6c

File tree

12 files changed

+227
-3
lines changed

12 files changed

+227
-3
lines changed

‎ci/cloudbuild/builds/lib/integration.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ source module ci/lib/io.sh
3131
export PATH="${HOME}/.local/bin:${PATH}"
3232
python3 -m pip uninstall -y --quiet googleapis-storage-testbench
3333
python3 -m pip install --upgrade --user --quiet --disable-pip-version-check \
34-
"git+https://github.com/googleapis/storage-testbench@v0.52.0"
34+
"git+https://github.com/googleapis/storage-testbench@v0.56.0"
3535

3636
# Some of the tests will need a valid roots.pem file.
3737
rm -f /dev/shm/roots.pem

‎google/cloud/storage/client.h‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include "google/cloud/storage/internal/signed_url_requests.h"
2222
#include "google/cloud/storage/internal/storage_connection.h"
2323
#include "google/cloud/storage/internal/tuple_filter.h"
24+
#include "google/cloud/storage/list_buckets_partial_reader.h"
2425
#include "google/cloud/storage/list_buckets_reader.h"
2526
#include "google/cloud/storage/list_hmac_keys_reader.h"
2627
#include "google/cloud/storage/list_objects_and_prefixes_reader.h"
@@ -366,6 +367,56 @@ class Client {
366367
std::forward<Options>(options)...);
367368
}
368369

370+
/**
371+
* Fetches the list of buckets and unreachable resources for the default
372+
* project.
373+
*
374+
* This function will return an error if it cannot determine the "default"
375+
* project. The default project is found by looking, in order, for:
376+
* - Any parameters of type `OverrideDefaultProject`, with a value.
377+
* - Any `google::cloud::storage::ProjectIdOption` value in any parameters of
378+
* type `google::cloud::Options{}`.
379+
* - Any `google::cloud::storage::ProjectIdOption` value provided in the
380+
* `google::cloud::Options{}` passed to the constructor.
381+
* - The value from the `GOOGLE_CLOUD_PROJECT` environment variable.
382+
*
383+
* @param options a list of optional query parameters and/or request headers.
384+
* Valid types for this operation include `MaxResults`, `Prefix`,
385+
* `Projection`, `UserProject`, `OverrideDefaultProject`, and
386+
* `ReturnPartialSuccess`.
387+
*
388+
* @par Idempotency
389+
* This is a read-only operation and is always idempotent.
390+
*
391+
* @par Example
392+
* @snippet storage_bucket_samples.cc list buckets partial result
393+
*/
394+
template <typename... Options>
395+
ListBucketsPartialReader ListBucketsPartial(Options&&... options) {
396+
auto opts = SpanOptions(std::forward<Options>(options)...);
397+
auto project_id = storage_internal::RequestProjectId(
398+
GCP_ERROR_INFO(), opts, std::forward<Options>(options)...);
399+
if (!project_id) {
400+
return google::cloud::internal::MakeErrorPaginationRange<
401+
ListBucketsPartialReader>(std::move(project_id).status());
402+
}
403+
google::cloud::internal::OptionsSpan const span(std::move(opts));
404+
405+
internal::ListBucketsRequest request(*std::move(project_id));
406+
request.set_multiple_options(std::forward<Options>(options)...);
407+
auto& client = connection_;
408+
return google::cloud::internal::MakePaginationRange<
409+
ListBucketsPartialReader>(
410+
request,
411+
[client](internal::ListBucketsRequest const& r) {
412+
return client->ListBuckets(r);
413+
},
414+
[](internal::ListBucketsResponse r) {
415+
return std::vector<BucketsPartial>{
416+
BucketsPartial{std::move(r.items), std::move(r.unreachable)}};
417+
});
418+
}
419+
369420
/**
370421
* Creates a new Google Cloud Storage bucket using the default project.
371422
*

‎google/cloud/storage/examples/storage_bucket_samples.cc‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
#include "google/cloud/storage/client.h"
1616
#include "google/cloud/storage/examples/storage_examples_common.h"
17+
#include "google/cloud/storage/list_buckets_partial_reader.h"
1718
#include "google/cloud/internal/getenv.h"
1819
#include <functional>
1920
#include <iostream>
@@ -48,6 +49,34 @@ void ListBuckets(google::cloud::storage::Client client,
4849
(std::move(client));
4950
}
5051

52+
void ListBucketsPartial(google::cloud::storage::Client client,
53+
std::vector<std::string> const& /*argv*/) {
54+
//! [list buckets partial result] [START list_buckets_partial]
55+
namespace gcs = ::google::cloud::storage;
56+
using ::google::cloud::StatusOr;
57+
[](gcs::Client client) {
58+
int count = 0;
59+
gcs::ListBucketsPartialReader bucket_list = client.ListBucketsPartial();
60+
for (auto&& result : bucket_list) {
61+
if (!result) throw std::move(result).status();
62+
63+
for (auto const& bucket_metadata : result->buckets) {
64+
std::cout << bucket_metadata.name() << "\n";
65+
++count;
66+
}
67+
for (auto const& unreachable : result->unreachable) {
68+
std::cout << "Unreachable location: " << unreachable << "\n";
69+
}
70+
}
71+
72+
if (count == 0) {
73+
std::cout << "No buckets in default project\n";
74+
}
75+
}
76+
//! [list buckets partial result] [END list_buckets_partial]
77+
(std::move(client));
78+
}
79+
5180
void ListBucketsForProject(google::cloud::storage::Client client,
5281
std::vector<std::string> const& argv) {
5382
//! [list buckets for project]
@@ -683,6 +712,9 @@ void RunAll(std::vector<std::string> const& argv) {
683712
std::cout << "\nRunning ListBuckets() example" << std::endl;
684713
ListBuckets(client, {});
685714

715+
std::cout << "\nRunning ListBucketsPartial() example" << std::endl;
716+
ListBucketsPartial(client, {});
717+
686718
std::cout << "\nRunning CreateBucket() example" << std::endl;
687719
CreateBucket(client, {bucket_name});
688720

@@ -726,6 +758,8 @@ int main(int argc, char* argv[]) {
726758

727759
examples::Example example({
728760
examples::CreateCommandEntry("list-buckets", {}, ListBuckets),
761+
examples::CreateCommandEntry("list-buckets-partial", {},
762+
ListBucketsPartial),
729763
examples::CreateCommandEntry("list-buckets-for-project", {"<project-id>"},
730764
ListBucketsForProject),
731765
make_entry("create-bucket", {}, CreateBucket),

‎google/cloud/storage/internal/bucket_requests.cc‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,12 @@ StatusOr<ListBucketsResponse> ListBucketsResponse::FromHttpResponse(
243243
result.items.emplace_back(std::move(*parsed));
244244
}
245245

246+
if (json.count("unreachable") != 0) {
247+
for (auto const& kv : json["unreachable"].items()) {
248+
result.unreachable.emplace_back(kv.value().get<std::string>());
249+
}
250+
}
251+
246252
return result;
247253
}
248254

@@ -256,6 +262,9 @@ std::ostream& operator<<(std::ostream& os, ListBucketsResponse const& r) {
256262
<< ", items={";
257263
std::copy(r.items.begin(), r.items.end(),
258264
std::ostream_iterator<BucketMetadata>(os, "\n "));
265+
os << "}, unreachable={";
266+
std::copy(r.unreachable.begin(), r.unreachable.end(),
267+
std::ostream_iterator<std::string>(os, "\n "));
259268
return os << "}}";
260269
}
261270

‎google/cloud/storage/internal/bucket_requests.h‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,14 +37,16 @@ namespace internal {
3737
*/
3838
class ListBucketsRequest
3939
: public GenericRequest<ListBucketsRequest, MaxResults, Prefix, Projection,
40-
UserProject, OverrideDefaultProject> {
40+
UserProject, OverrideDefaultProject,
41+
ReturnPartialSuccess> {
4142
public:
4243
ListBucketsRequest() = default;
4344
explicit ListBucketsRequest(std::string project_id)
4445
: project_id_(std::move(project_id)) {}
4546

4647
std::string const& project_id() const { return project_id_; }
4748
std::string const& page_token() const { return page_token_; }
49+
bool const& return_partial_success() const { return return_partial_success_; }
4850
ListBucketsRequest& set_page_token(std::string page_token) {
4951
page_token_ = std::move(page_token);
5052
return *this;
@@ -53,10 +55,17 @@ class ListBucketsRequest
5355
private:
5456
std::string project_id_;
5557
std::string page_token_;
58+
bool return_partial_success_;
5659
};
5760

5861
std::ostream& operator<<(std::ostream& os, ListBucketsRequest const& r);
5962

63+
struct ListBucketsResult {
64+
std::string page_token;
65+
std::vector<BucketMetadata> items;
66+
std::vector<std::string> unreachable;
67+
};
68+
6069
struct ListBucketsResponse {
6170
static StatusOr<ListBucketsResponse> FromHttpResponse(
6271
std::string const& payload);
@@ -65,6 +74,7 @@ struct ListBucketsResponse {
6574

6675
std::string next_page_token;
6776
std::vector<BucketMetadata> items;
77+
std::vector<std::string> unreachable;
6878
};
6979

7080
std::ostream& operator<<(std::ostream& os, ListBucketsResponse const& r);

‎google/cloud/storage/internal/bucket_requests_test.cc‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,16 @@ TEST(ListBucketsRequestTest, OStream) {
134134
EXPECT_THAT(os.str(), HasSubstr("prefix=foo-bar-baz"));
135135
}
136136

137+
TEST(ListBucketsRequestTest, PartialSuccess) {
138+
ListBucketsRequest request("project-to-list");
139+
request.set_multiple_options(ReturnPartialSuccess(true));
140+
141+
std::ostringstream os;
142+
os << request;
143+
EXPECT_THAT(os.str(), HasSubstr("ListBucketsRequest"));
144+
EXPECT_THAT(os.str(), HasSubstr("returnPartialSuccess=1"));
145+
}
146+
137147
TEST(ListBucketsResponseTest, Parse) {
138148
std::string bucket1 = R"""({
139149
"kind": "storage#bucket",

‎google/cloud/storage/internal/grpc/bucket_request_parser_test.cc‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ TEST(GrpcBucketRequestParser, ListBucketsRequestAllOptions) {
187187
page_size: 123
188188
page_token: "test-token"
189189
prefix: "test-prefix"
190+
return_partial_success: "true"
190191
read_mask { paths: [ "*" ] }
191192
)pb",
192193
&expected));
@@ -196,7 +197,8 @@ TEST(GrpcBucketRequestParser, ListBucketsRequestAllOptions) {
196197
req.set_multiple_options(
197198
storage::MaxResults(123), storage::Prefix("test-prefix"),
198199
storage::Projection("full"), storage::UserProject("test-user-project"),
199-
storage::QuotaUser("test-quota-user"), storage::UserIp("test-user-ip"));
200+
storage::QuotaUser("test-quota-user"), storage::UserIp("test-user-ip"),
201+
storage::ReturnPartialSuccess(true));
200202

201203
auto const actual = ToProto(req);
202204
EXPECT_THAT(actual, IsProtoEqual(expected));
@@ -214,6 +216,8 @@ TEST(GrpcBucketRequestParser, ListBucketsResponse) {
214216
name: "projects/_/buckets/test-bucket-2"
215217
bucket_id: "test-bucket-2"
216218
}
219+
unreachable { "projects/_/buckets/bucket1"
220+
"projects/1234567/locations/location1" }
217221
next_page_token: "test-token"
218222
)pb",
219223
&input));

‎google/cloud/storage/internal/rest/stub.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,8 @@ StatusOr<ListBucketsResponse> RestStub::ListBuckets(
186186
if (!request.page_token().empty()) {
187187
builder.AddQueryParameter("pageToken", request.page_token());
188188
}
189+
builder.AddQueryParameter("returnPartialSuccess",
190+
(request.return_partial_success() ? "1" : "0"));
189191
return ParseFromRestResponse<ListBucketsResponse>(
190192
storage_rest_client_->Get(context, std::move(builder).BuildRequest()));
191193
}

‎google/cloud/storage/internal/rest/stub_test.cc‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,26 @@ TEST(RestStubTest, ListBucketsOmitsPageTokenWhenEmptyInRequest) {
230230
tested->ListBuckets(context, TestOptions(), request);
231231
}
232232

233+
TEST(RestStubTest, ListBucketsReturnPartialSuccess) {
234+
auto mock = std::make_shared<MockRestClient>();
235+
bool return_partial_success = true;
236+
ListBucketsRequest request("test-project-id");
237+
request.set_multiple_options(
238+
storage::ReturnPartialSuccess(return_partial_success));
239+
240+
EXPECT_CALL(*mock,
241+
Get(ExpectedContext(),
242+
ResultOf(
243+
"request parameters contain 'returnPartialSuccess'",
244+
[](RestRequest const& r) { return r.parameters(); },
245+
Contains(Pair("returnPartialSuccess", "1")))))
246+
.WillOnce(Return(PermanentError()));
247+
248+
auto tested = std::make_unique<RestStub>(Options{}, mock, mock);
249+
auto context = TestContext();
250+
tested->ListBuckets(context, TestOptions(), request);
251+
}
252+
233253
TEST(RestStubTest, GetBucketMetadata) {
234254
auto mock = std::make_shared<MockRestClient>();
235255
EXPECT_CALL(*mock, Get(ExpectedContext(), ExpectedRequest()))

‎google/cloud/storage/list_buckets_partial_reader.h‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
// Copyright 2025 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_LIST_BUCKETS_PARTIAL_READER_H
16+
#define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_LIST_BUCKETS_PARTIAL_READER_H
17+
18+
#include "google/cloud/storage/bucket_metadata.h"
19+
#include "google/cloud/storage/version.h"
20+
#include "google/cloud/internal/pagination_range.h"
21+
#include <string>
22+
#include <vector>
23+
24+
namespace google {
25+
namespace cloud {
26+
namespace storage {
27+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
28+
29+
struct BucketsPartial {
30+
std::vector<BucketMetadata> buckets;
31+
std::vector<std::string> unreachable;
32+
};
33+
34+
using ListBucketsPartialReader =
35+
google::cloud::internal::PaginationRange<BucketsPartial>;
36+
37+
using ListBucketsPartialIterator = ListBucketsPartialReader::iterator;
38+
39+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
40+
} // namespace storage
41+
} // namespace cloud
42+
} // namespace google
43+
44+
#endif // GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_LIST_BUCKETS_PARTIAL_READER_H

0 commit comments

Comments
 (0)