Skip to content

Commit 60a7f72

Browse files
yuvalifyuvalif
committed
rgw/logging: rollover objects when conf changes
and return the name of the flushed object to the client Fixes: https://tracker.ceph.com/issues/72940 Signed-off-by: Yuval Lifshitz <[email protected]>
1 parent 098432f commit 60a7f72

File tree

5 files changed

+81
-14
lines changed

5 files changed

+81
-14
lines changed

‎doc/radosgw/s3/bucketops.rst‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -651,7 +651,7 @@ Parameters
651651
Response Entities
652652
~~~~~~~~~~~~~~~~~
653653

654-
Response is XML encoded in the body of the request, in the following format:
654+
The response is XML encoded in the body of the request, in the following format:
655655

656656
::
657657

@@ -790,7 +790,7 @@ Parameters are XML encoded in the body of the request, in the following format:
790790
| | | between different source buckets writing log records to the same log bucket. | |
791791
+-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
792792
| ``LoggingType`` | String | The type of logging. Valid values are: | No |
793-
| | | ``Standard`` (default) all bucket operations are logged after being perfomed. | |
793+
| | | ``Standard`` (default) all bucket operations are logged after being performed. | |
794794
| | | The log record will contain all fields. | |
795795
| | | ``Journal`` only operations that modify and object are logged. | |
796796
| | | Will record the minimum subset of fields in the log record that is needed | |
@@ -800,6 +800,18 @@ Parameters are XML encoded in the body of the request, in the following format:
800800
| | | object added to the log bucket. Default is 3600 seconds (1 hour). | |
801801
+-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
802802

803+
Response Entities
804+
~~~~~~~~~~~~~~~~~
805+
806+
The response is XML encoded in the body of the request, only if a configuration change triggers flushing of the current logging object.
807+
In this case it will return the name of the flushed logging object in following format:
808+
809+
::
810+
811+
<PostBucketLoggingOutput xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
812+
<FlushedLoggingObject>string</FlushedLoggingObject>
813+
</PostBucketLoggingOutput>
814+
803815

804816
HTTP Response
805817
~~~~~~~~~~~~~
@@ -864,7 +876,7 @@ Syntax
864876
Response Entities
865877
~~~~~~~~~~~~~~~~~
866878

867-
Response header contains ``Last-Modified`` date/time of the logging configuration.
879+
The response header contains ``Last-Modified`` date/time of the logging configuration.
868880
Logging configuration is XML encoded in the body of the response, in the following format:
869881

870882
::
@@ -931,7 +943,7 @@ Syntax
931943
Response Entities
932944
~~~~~~~~~~~~~~~~~
933945

934-
Response is XML encoded in the body of the request, in the following format:
946+
The response is XML encoded in the body of the request, in the following format:
935947

936948
::
937949

‎examples/rgw/boto3/service-2.sdk-extras.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,17 @@
2424
"output": {"shape": "PostBucketLoggingOutput"},
2525
"documentationUrl":"https://docs.ceph.com/docs/master/radosgw/s3/bucketops/#post-bucket-logging",
2626
"documentation":"<p>Flushes the logging objects of the buckets.</p>"
27+
},
28+
"PutBucketLogging":{
29+
"name":"PutBucketLogging",
30+
"http":{
31+
"method":"PUT",
32+
"requestUri":"/{Bucket}?logging"
33+
},
34+
"input":{"shape":"PutBucketLoggingRequest"},
35+
"output": {"shape": "PutBucketLoggingOutput"},
36+
"documentationUrl":"https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLogging.html",
37+
"documentation":"<p>Put bucket logging configuration on source bucket.</p>"
2738
},
2839
"GetUsageStats":{
2940
"name":"GetUsageStats",
@@ -402,6 +413,15 @@
402413
}
403414
}
404415
},
416+
"PutBucketLoggingOutput": {
417+
"type":"structure",
418+
"members":{
419+
"FlushedLoggingObject": {
420+
"shape":"FlushedLoggingObject",
421+
"documentation":"<p>Name of the pending logging object that was flushed.</p>"
422+
}
423+
}
424+
},
405425
"FlushedLoggingObject":{
406426
"type":"string"
407427
},

‎src/rgw/rgw_bucket_logging.cc‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -837,14 +837,15 @@ int bucket_deletion_cleanup(const DoutPrefixProvider* dpp,
837837
}
838838
}
839839

840-
return source_bucket_cleanup(dpp, driver, bucket, false, y);
840+
return source_bucket_cleanup(dpp, driver, bucket, false, y, nullptr);
841841
}
842842

843843
int source_bucket_cleanup(const DoutPrefixProvider* dpp,
844844
sal::Driver* driver,
845845
sal::Bucket* bucket,
846846
bool remove_attr,
847-
optional_yield y) {
847+
optional_yield y,
848+
std::string* last_committed) {
848849
std::optional<configuration> conf;
849850
if (const int ret = retry_raced_bucket_write(dpp, bucket, [dpp, bucket, &conf, remove_attr, y] {
850851
auto& attrs = bucket->get_attrs();
@@ -879,7 +880,7 @@ int source_bucket_cleanup(const DoutPrefixProvider* dpp,
879880
return 0;
880881
}
881882
const auto& info = bucket->get_info();
882-
if (const int ret = commit_logging_object(*conf, dpp, driver, info.bucket.tenant, y, nullptr); ret < 0) {
883+
if (const int ret = commit_logging_object(*conf, dpp, driver, info.bucket.tenant, y, last_committed); ret < 0) {
883884
ldpp_dout(dpp, 5) << "WARNING: could not commit pending logging object of bucket '" <<
884885
bucket->get_key() << "' during cleanup. ret = " << ret << dendl;
885886
} else {

‎src/rgw/rgw_bucket_logging.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,11 +251,13 @@ int bucket_deletion_cleanup(const DoutPrefixProvider* dpp,
251251
// in addition:
252252
// any pending log objects should be comitted to the log bucket
253253
// and the log bucket should be updated to remove the bucket as a source
254+
// if "last_committed" is not null, it will be set to the name of the last committed object
254255
int source_bucket_cleanup(const DoutPrefixProvider* dpp,
255256
sal::Driver* driver,
256257
sal::Bucket* bucket,
257258
bool remove_attr,
258-
optional_yield y);
259+
optional_yield y,
260+
std::string* last_committed);
259261

260262
// verify that the target bucket has the correct policy to allow the source bucket to log to it
261263
// note that this function adds entries to the request state environment

‎src/rgw/rgw_rest_bucket_logging.cc‎

Lines changed: 38 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ class RGWGetBucketLoggingOp : public RGWOp {
111111
}
112112

113113
void send_response() override {
114+
set_req_state_err(s, op_ret);
114115
dump_errno(s);
115116
if (mtime) {
116117
dump_last_modified(s, *mtime);
@@ -123,6 +124,7 @@ class RGWGetBucketLoggingOp : public RGWOp {
123124
s->formatter->close_section();
124125
rgw_flush_formatter_and_reset(s, s->formatter);
125126
}
127+
126128
const char* name() const override { return "get_bucket_logging"; }
127129
std::string canonical_name() const override { return fmt::format("REST.{}.LOGGING", s->info.method); }
128130
RGWOpType get_type() override { return RGW_OP_GET_BUCKET_LOGGING; }
@@ -136,6 +138,7 @@ class RGWPutBucketLoggingOp : public RGWDefaultResponseOp {
136138
// and usd in execute()
137139
rgw::bucketlogging::configuration configuration;
138140
std::unique_ptr<rgw::sal::Bucket> target_bucket;
141+
std::string old_obj; // used when conf change triggers a rollover
139142

140143
int init_processing(optional_yield y) override {
141144
if (const auto ret = verify_bucket_logging_params(this, s); ret < 0) {
@@ -234,7 +237,7 @@ class RGWPutBucketLoggingOp : public RGWDefaultResponseOp {
234237
}
235238

236239
if (!configuration.enabled) {
237-
op_ret = rgw::bucketlogging::source_bucket_cleanup(this, driver, src_bucket.get(), true, y);
240+
op_ret = rgw::bucketlogging::source_bucket_cleanup(this, driver, src_bucket.get(), true, y, &old_obj);
238241
return;
239242
}
240243

@@ -306,12 +309,27 @@ class RGWPutBucketLoggingOp : public RGWDefaultResponseOp {
306309
}
307310
} else if (*old_conf != configuration) {
308311
// conf changed - do cleanup
309-
if (const auto ret = commit_logging_object(*old_conf, target_bucket, this, y, nullptr); ret < 0) {
310-
ldpp_dout(this, 1) << "WARNING: could not commit pending logging object when updating logging configuration of bucket '" <<
311-
src_bucket->get_key() << "', ret = " << ret << dendl;
312+
RGWObjVersionTracker objv_tracker;
313+
std::string obj_name;
314+
const auto region = driver->get_zone()->get_zonegroup().get_api_name();
315+
if (const auto ret = rollover_logging_object(*old_conf,
316+
target_bucket,
317+
obj_name,
318+
this,
319+
region,
320+
src_bucket,
321+
y,
322+
false, // rollover should happen even if commit failed
323+
&objv_tracker,
324+
&old_obj); ret < 0) {
325+
ldpp_dout(this, 1) << "WARNING: failed to flush pending logging object '" << obj_name << "'"
326+
<< " to target bucket '" << target_bucket_id << "'. "
327+
<< " last committed object is '" << old_obj <<
328+
"' when updating logging configuration of bucket '" << src_bucket->get_key() << ". error: " << ret << dendl;
312329
} else {
313-
ldpp_dout(this, 20) << "INFO: committed pending logging object when updating logging configuration of bucket '" <<
314-
src_bucket->get_key() << "'" << dendl;
330+
ldpp_dout(this, 20) << "INFO: flushed pending logging object '" << old_obj
331+
<< "' to target bucket '" << target_bucket_id << "' when updating logging configuration of bucket '"
332+
<< src_bucket->get_key() << "'" << dendl;
315333
}
316334
if (old_conf->target_bucket != configuration.target_bucket) {
317335
rgw_bucket old_target_bucket_id;
@@ -334,6 +352,19 @@ class RGWPutBucketLoggingOp : public RGWDefaultResponseOp {
334352
ldpp_dout(this, 20) << "INFO: logging configuration of bucket '" << src_bucket_id << "' did not change" << dendl;
335353
}
336354
}
355+
356+
void send_response() override {
357+
set_req_state_err(s, op_ret);
358+
dump_errno(s);
359+
end_header(s, this, to_mime_type(s->format));
360+
if (!old_obj.empty()) {
361+
dump_start(s);
362+
s->formatter->open_object_section_in_ns("PutBucketLoggingOutput", XMLNS_AWS_S3);
363+
s->formatter->dump_string("FlushedLoggingObject", old_obj);
364+
s->formatter->close_section();
365+
rgw_flush_formatter_and_reset(s, s->formatter);
366+
}
367+
}
337368
};
338369

339370
// Post /<bucket name>/?logging
@@ -407,6 +438,7 @@ class RGWPostBucketLoggingOp : public RGWDefaultResponseOp {
407438
}
408439

409440
void send_response() override {
441+
set_req_state_err(s, op_ret);
410442
dump_errno(s);
411443
end_header(s, this, to_mime_type(s->format));
412444
dump_start(s);

0 commit comments

Comments
 (0)