Skip to content

Commit d4f3ebf

Browse files
coryancoryan
authored
feat(storage): support AuthorityOption (#8462)
If present, use the `AuthorityOption` to configure the `Host: ` header or the `.set_authority()` attribute in `grpc::ClientContext`. By default, configure `AuthorityOption` to be `storage.googleapis.com`, applications can override this when initializing the `storage::Client`. There is already an issue open to add per-call `Options` that would provide more fine-grained control of this field.
1 parent af04273 commit d4f3ebf

File tree

5 files changed

+99
-21
lines changed

5 files changed

+99
-21
lines changed

‎google/cloud/internal/rest_options.h‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,21 @@ struct UserIpOption {
3232
using Type = std::string;
3333
};
3434

35-
/// Configure the REST endpoint for the GCS client library.
35+
/**
36+
* Configure the REST endpoint for the GCS client library.
37+
*
38+
* This endpoint must include the URL scheme (`http` or `https`) and `authority`
39+
* (host and port) used to access the GCS service, for example:
40+
* https://storage.googleapis.com
41+
* When using emulators or testbench it can be of the form:
42+
* http://localhost:8080/my-gcs-emulator-path
43+
*
44+
* @note The `Host` header is based on the `authority` component of the URL.
45+
* Applications can override this default value using
46+
* `google::cloud::AuthorityOption`
47+
*
48+
* @see https://en.wikipedia.org/wiki/Uniform_Resource_Identifier#URLs_and_URNs
49+
*/
3650
struct RestEndpointOption {
3751
using Type = std::string;
3852
};

‎google/cloud/storage/internal/curl_client.cc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,9 +110,11 @@ std::string HostHeader(Options const& options, char const* service) {
110110
// header based on the URL. In most cases this is the correct value. The main
111111
// exception are applications using `VPC-SC`:
112112
// https://cloud.google.com/vpc/docs/configure-private-google-access
113-
// In those cases the application would target an URL like
113+
// In those cases the application would target a URL like
114114
// `https://restricted.googleapis.com`, or `https://private.googleapis.com`,
115115
// or their own proxy, and need to provide the target's service host.
116+
auto const& auth = options.get<AuthorityOption>();
117+
if (!auth.empty()) return absl::StrCat("Host: ", auth);
116118
auto const& endpoint = options.get<RestEndpointOption>();
117119
if (absl::StrContains(endpoint, "googleapis.com")) {
118120
return absl::StrCat("Host: ", service, ".googleapis.com");

‎google/cloud/storage/internal/curl_client_test.cc‎

Lines changed: 23 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -101,38 +101,42 @@ class CurlClientTest : public ::testing::Test,
101101
TEST(CurlClientStandaloneFunctions, HostHeader) {
102102
struct Test {
103103
std::string endpoint;
104+
std::string authority;
104105
std::string service;
105106
std::string expected;
106107
} cases[] = {
107-
{"https://storage.googleapis.com", "storage",
108+
{"https://storage.googleapis.com", "", "storage",
108109
"Host: storage.googleapis.com"},
109-
{"https://storage.googleapis.com:443", "storage",
110+
{"https://storage.googleapis.com", "auth", "storage", "Host: auth"},
111+
{"https://storage.googleapis.com:443", "", "storage",
110112
"Host: storage.googleapis.com"},
111-
{"https://restricted.googleapis.com", "storage",
113+
{"https://restricted.googleapis.com", "", "storage",
112114
"Host: storage.googleapis.com"},
113-
{"https://private.googleapis.com", "storage",
115+
{"https://private.googleapis.com", "", "storage",
114116
"Host: storage.googleapis.com"},
115-
{"https://restricted.googleapis.com", "iamcredentials",
117+
{"https://restricted.googleapis.com", "", "iamcredentials",
116118
"Host: iamcredentials.googleapis.com"},
117-
{"https://private.googleapis.com", "iamcredentials",
119+
{"https://private.googleapis.com", "", "iamcredentials",
118120
"Host: iamcredentials.googleapis.com"},
119-
{"http://localhost:8080", "", ""},
120-
{"http://[::1]", "", ""},
121-
{"http://[::1]/", "", ""},
122-
{"http://[::1]/foo/bar", "", ""},
123-
{"http://[::1]:8080/", "", ""},
124-
{"http://[::1]:8080/foo/bar", "", ""},
125-
{"http://localhost:8080", "", ""},
126-
{"https://storage-download.127.0.0.1.nip.io/xmlapi/", "", ""},
127-
{"https://gcs.127.0.0.1.nip.io/storage/v1/", "", ""},
128-
{"https://gcs.127.0.0.1.nip.io:4443/upload/storage/v1/", "", ""},
129-
{"https://gcs.127.0.0.1.nip.io:4443/upload/storage/v1/", "", ""},
121+
{"http://localhost:8080", "", "", ""},
122+
{"http://localhost:8080", "auth", "", "Host: auth"},
123+
{"http://[::1]", "", "", ""},
124+
{"http://[::1]/", "", "", ""},
125+
{"http://[::1]/foo/bar", "", "", ""},
126+
{"http://[::1]:8080/", "", "", ""},
127+
{"http://[::1]:8080/foo/bar", "", "", ""},
128+
{"http://localhost:8080", "", "", ""},
129+
{"https://storage-download.127.0.0.1.nip.io/xmlapi/", "", "", ""},
130+
{"https://gcs.127.0.0.1.nip.io/storage/v1/", "", "", ""},
131+
{"https://gcs.127.0.0.1.nip.io:4443/upload/storage/v1/", "", "", ""},
132+
{"https://gcs.127.0.0.1.nip.io:4443/upload/storage/v1/", "", "", ""},
130133
};
131134

132135
for (auto const& test : cases) {
133136
SCOPED_TRACE("Testing for " + test.endpoint + ", " + test.service);
134-
auto const actual = HostHeader(
135-
Options{}.set<RestEndpointOption>(test.endpoint), test.service.c_str());
137+
auto options = Options{}.set<RestEndpointOption>(test.endpoint);
138+
if (!test.authority.empty()) options.set<AuthorityOption>(test.authority);
139+
auto const actual = HostHeader(options, test.service.c_str());
136140
EXPECT_EQ(test.expected, actual);
137141
}
138142
}

‎google/cloud/storage/internal/grpc_client.cc‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
4242
namespace internal {
4343

4444
using ::google::cloud::internal::MakeBackgroundThreadsFactory;
45+
using ::google::cloud::internal::OptionsSpan;
4546

4647
int DefaultGrpcNumChannels() {
4748
auto constexpr kMinimumChannels = 4;
@@ -59,6 +60,9 @@ Options DefaultOptionsGrpc(Options options) {
5960
if (!options.has<EndpointOption>()) {
6061
options.set<EndpointOption>("storage.googleapis.com");
6162
}
63+
if (!options.has<AuthorityOption>()) {
64+
options.set<AuthorityOption>("storage.googleapis.com");
65+
}
6266
using google::cloud::internal::GetEnv;
6367
auto env = GetEnv("CLOUD_STORAGE_GRPC_ENDPOINT");
6468
if (env.has_value()) options.set<EndpointOption>(*env);
@@ -110,6 +114,7 @@ std::unique_ptr<GrpcClient::WriteObjectStream> GrpcClient::CreateUploadWriter(
110114

111115
StatusOr<ResumableUploadResponse> GrpcClient::QueryResumableUpload(
112116
QueryResumableUploadRequest const& request) {
117+
OptionsSpan span(options_);
113118
grpc::ClientContext context;
114119
ApplyQueryParameters(context, request, "resource");
115120
auto const timeout = options_.get<TransferStallTimeoutOption>();
@@ -153,6 +158,7 @@ ClientOptions const& GrpcClient::client_options() const {
153158

154159
StatusOr<ListBucketsResponse> GrpcClient::ListBuckets(
155160
ListBucketsRequest const& request) {
161+
OptionsSpan span(options_);
156162
auto proto = GrpcBucketRequestParser::ToProto(request);
157163
grpc::ClientContext context;
158164
ApplyQueryParameters(context, request);
@@ -163,6 +169,7 @@ StatusOr<ListBucketsResponse> GrpcClient::ListBuckets(
163169

164170
StatusOr<BucketMetadata> GrpcClient::CreateBucket(
165171
CreateBucketRequest const& request) {
172+
OptionsSpan span(options_);
166173
auto proto = GrpcBucketRequestParser::ToProto(request);
167174
grpc::ClientContext context;
168175
ApplyQueryParameters(context, request);
@@ -173,6 +180,7 @@ StatusOr<BucketMetadata> GrpcClient::CreateBucket(
173180

174181
StatusOr<BucketMetadata> GrpcClient::GetBucketMetadata(
175182
GetBucketMetadataRequest const& request) {
183+
OptionsSpan span(options_);
176184
auto proto = GrpcBucketRequestParser::ToProto(request);
177185
grpc::ClientContext context;
178186
ApplyQueryParameters(context, request);
@@ -183,6 +191,7 @@ StatusOr<BucketMetadata> GrpcClient::GetBucketMetadata(
183191

184192
StatusOr<EmptyResponse> GrpcClient::DeleteBucket(
185193
DeleteBucketRequest const& request) {
194+
OptionsSpan span(options_);
186195
auto proto = GrpcBucketRequestParser::ToProto(request);
187196
grpc::ClientContext context;
188197
ApplyQueryParameters(context, request);
@@ -193,6 +202,7 @@ StatusOr<EmptyResponse> GrpcClient::DeleteBucket(
193202

194203
StatusOr<BucketMetadata> GrpcClient::UpdateBucket(
195204
UpdateBucketRequest const& request) {
205+
OptionsSpan span(options_);
196206
auto proto = GrpcBucketRequestParser::ToProto(request);
197207
grpc::ClientContext context;
198208
ApplyQueryParameters(context, request);
@@ -203,6 +213,7 @@ StatusOr<BucketMetadata> GrpcClient::UpdateBucket(
203213

204214
StatusOr<BucketMetadata> GrpcClient::PatchBucket(
205215
PatchBucketRequest const& request) {
216+
OptionsSpan span(options_);
206217
auto proto = GrpcBucketRequestParser::ToProto(request);
207218
if (!proto) return std::move(proto).status();
208219
grpc::ClientContext context;
@@ -217,6 +228,7 @@ StatusOr<BucketMetadata> GrpcClient::PatchBucket(
217228

218229
StatusOr<IamPolicy> GrpcClient::GetBucketIamPolicy(
219230
GetBucketIamPolicyRequest const& request) {
231+
OptionsSpan span(options_);
220232
auto proto = GrpcBucketRequestParser::ToProto(request);
221233
grpc::ClientContext context;
222234
ApplyQueryParameters(context, request);
@@ -234,6 +246,7 @@ StatusOr<IamPolicy> GrpcClient::GetBucketIamPolicy(
234246

235247
StatusOr<NativeIamPolicy> GrpcClient::GetNativeBucketIamPolicy(
236248
GetBucketIamPolicyRequest const& request) {
249+
OptionsSpan span(options_);
237250
auto proto = GrpcBucketRequestParser::ToProto(request);
238251
grpc::ClientContext context;
239252
ApplyQueryParameters(context, request);
@@ -247,6 +260,7 @@ StatusOr<NativeIamPolicy> GrpcClient::GetNativeBucketIamPolicy(
247260

248261
StatusOr<IamPolicy> GrpcClient::SetBucketIamPolicy(
249262
SetBucketIamPolicyRequest const& request) {
263+
OptionsSpan span(options_);
250264
auto proto = GrpcBucketRequestParser::ToProto(request);
251265
grpc::ClientContext context;
252266
ApplyQueryParameters(context, request);
@@ -264,6 +278,7 @@ StatusOr<IamPolicy> GrpcClient::SetBucketIamPolicy(
264278

265279
StatusOr<NativeIamPolicy> GrpcClient::SetNativeBucketIamPolicy(
266280
SetNativeBucketIamPolicyRequest const& request) {
281+
OptionsSpan span(options_);
267282
auto proto = GrpcBucketRequestParser::ToProto(request);
268283
grpc::ClientContext context;
269284
ApplyQueryParameters(context, request);
@@ -274,6 +289,7 @@ StatusOr<NativeIamPolicy> GrpcClient::SetNativeBucketIamPolicy(
274289

275290
StatusOr<TestBucketIamPermissionsResponse> GrpcClient::TestBucketIamPermissions(
276291
TestBucketIamPermissionsRequest const& request) {
292+
OptionsSpan span(options_);
277293
auto proto = GrpcBucketRequestParser::ToProto(request);
278294
grpc::ClientContext context;
279295
ApplyQueryParameters(context, request);
@@ -284,6 +300,7 @@ StatusOr<TestBucketIamPermissionsResponse> GrpcClient::TestBucketIamPermissions(
284300

285301
StatusOr<BucketMetadata> GrpcClient::LockBucketRetentionPolicy(
286302
LockBucketRetentionPolicyRequest const& request) {
303+
OptionsSpan span(options_);
287304
auto proto = GrpcBucketRequestParser::ToProto(request);
288305
grpc::ClientContext context;
289306
ApplyQueryParameters(context, request);
@@ -294,6 +311,7 @@ StatusOr<BucketMetadata> GrpcClient::LockBucketRetentionPolicy(
294311

295312
StatusOr<ObjectMetadata> GrpcClient::InsertObjectMedia(
296313
InsertObjectMediaRequest const& request) {
314+
OptionsSpan span(options_);
297315
auto r = GrpcObjectRequestParser::ToProto(request);
298316
if (!r) return std::move(r).status();
299317
auto proto_request = *r;
@@ -345,6 +363,7 @@ StatusOr<ObjectMetadata> GrpcClient::InsertObjectMedia(
345363

346364
StatusOr<ObjectMetadata> GrpcClient::CopyObject(
347365
CopyObjectRequest const& request) {
366+
OptionsSpan span(options_);
348367
auto proto = GrpcObjectRequestParser::ToProto(request);
349368
grpc::ClientContext context;
350369
ApplyQueryParameters(context, request, "resource");
@@ -360,6 +379,7 @@ StatusOr<ObjectMetadata> GrpcClient::CopyObject(
360379

361380
StatusOr<ObjectMetadata> GrpcClient::GetObjectMetadata(
362381
GetObjectMetadataRequest const& request) {
382+
OptionsSpan span(options_);
363383
auto proto = GrpcObjectRequestParser::ToProto(request);
364384
grpc::ClientContext context;
365385
ApplyQueryParameters(context, request);
@@ -370,6 +390,7 @@ StatusOr<ObjectMetadata> GrpcClient::GetObjectMetadata(
370390

371391
StatusOr<std::unique_ptr<ObjectReadSource>> GrpcClient::ReadObject(
372392
ReadObjectRangeRequest const& request) {
393+
OptionsSpan span(options_);
373394
// With the REST API this condition was detected by the server as an error,
374395
// generally we prefer the server to detect errors because its answers are
375396
// authoritative. In this case, the server cannot: with gRPC '0' is the same
@@ -396,6 +417,7 @@ StatusOr<std::unique_ptr<ObjectReadSource>> GrpcClient::ReadObject(
396417

397418
StatusOr<ListObjectsResponse> GrpcClient::ListObjects(
398419
ListObjectsRequest const& request) {
420+
OptionsSpan span(options_);
399421
auto proto = GrpcObjectRequestParser::ToProto(request);
400422
grpc::ClientContext context;
401423
ApplyQueryParameters(context, request);
@@ -406,6 +428,7 @@ StatusOr<ListObjectsResponse> GrpcClient::ListObjects(
406428

407429
StatusOr<EmptyResponse> GrpcClient::DeleteObject(
408430
DeleteObjectRequest const& request) {
431+
OptionsSpan span(options_);
409432
auto proto = GrpcObjectRequestParser::ToProto(request);
410433
grpc::ClientContext context;
411434
ApplyQueryParameters(context, request);
@@ -416,6 +439,7 @@ StatusOr<EmptyResponse> GrpcClient::DeleteObject(
416439

417440
StatusOr<ObjectMetadata> GrpcClient::UpdateObject(
418441
UpdateObjectRequest const& request) {
442+
OptionsSpan span(options_);
419443
auto proto = GrpcObjectRequestParser::ToProto(request);
420444
if (!proto) return std::move(proto).status();
421445
grpc::ClientContext context;
@@ -427,6 +451,7 @@ StatusOr<ObjectMetadata> GrpcClient::UpdateObject(
427451

428452
StatusOr<ObjectMetadata> GrpcClient::PatchObject(
429453
PatchObjectRequest const& request) {
454+
OptionsSpan span(options_);
430455
auto proto = GrpcObjectRequestParser::ToProto(request);
431456
if (!proto) return std::move(proto).status();
432457
grpc::ClientContext context;
@@ -438,6 +463,7 @@ StatusOr<ObjectMetadata> GrpcClient::PatchObject(
438463

439464
StatusOr<ObjectMetadata> GrpcClient::ComposeObject(
440465
ComposeObjectRequest const& request) {
466+
OptionsSpan span(options_);
441467
auto proto = GrpcObjectRequestParser::ToProto(request);
442468
if (!proto) return std::move(proto).status();
443469
grpc::ClientContext context;
@@ -449,6 +475,7 @@ StatusOr<ObjectMetadata> GrpcClient::ComposeObject(
449475

450476
StatusOr<RewriteObjectResponse> GrpcClient::RewriteObject(
451477
RewriteObjectRequest const& request) {
478+
OptionsSpan span(options_);
452479
auto proto = GrpcObjectRequestParser::ToProto(request);
453480
if (!proto) return std::move(proto).status();
454481
grpc::ClientContext context;
@@ -460,6 +487,7 @@ StatusOr<RewriteObjectResponse> GrpcClient::RewriteObject(
460487

461488
StatusOr<std::unique_ptr<ResumableUploadSession>>
462489
GrpcClient::CreateResumableSession(ResumableUploadRequest const& request) {
490+
OptionsSpan span(options_);
463491
auto session_id = request.GetOption<UseResumableUploadSession>().value_or("");
464492
if (!session_id.empty()) {
465493
return FullyRestoreResumableSession(request, session_id);
@@ -579,6 +607,7 @@ StatusOr<ObjectAccessControl> GrpcClient::PatchDefaultObjectAcl(
579607

580608
StatusOr<ServiceAccount> GrpcClient::GetServiceAccount(
581609
GetProjectServiceAccountRequest const& request) {
610+
OptionsSpan span(options_);
582611
auto proto = GrpcServiceAccountParser::ToProto(request);
583612
grpc::ClientContext context;
584613
ApplyQueryParameters(context, request);

0 commit comments

Comments
 (0)