Skip to content

Commit a0a8d7c

Browse files
cbodleycbodley
authored
Merge pull request ceph#58082 from jzhu116-bloomberg/wip-66516
rgw/multipart: use cls_version to avoid racing between part upload and multipart complete Reviewed-by: Casey Bodley <[email protected]>
2 parents 91fd5ef + 451b70d commit a0a8d7c

21 files changed

+407
-110
lines changed

‎src/rgw/driver/daos/rgw_sal_daos.cc‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1196,7 +1196,8 @@ int DaosObject::DaosDeleteOp::delete_obj(const DoutPrefixProvider* dpp,
11961196
}
11971197

11981198
int DaosObject::delete_object(const DoutPrefixProvider* dpp, optional_yield y,
1199-
uint32_t flags) {
1199+
uint32_t flags, std::list<rgw_obj_index_key>* remove_objs,
1200+
RGWObjVersionTracker* objv) {
12001201
ldpp_dout(dpp, 20) << "DEBUG: delete_object" << dendl;
12011202
DaosObject::DaosDeleteOp del_op(this);
12021203
del_op.params.bucket_owner = bucket->get_info().owner;
@@ -1677,7 +1678,8 @@ int DaosMultipartUpload::complete(
16771678
map<int, string>& part_etags, list<rgw_obj_index_key>& remove_objs,
16781679
uint64_t& accounted_size, bool& compressed, RGWCompressionInfo& cs_info,
16791680
off_t& off, std::string& tag, ACLOwner& owner, uint64_t olh_epoch,
1680-
rgw::sal::Object* target_obj) {
1681+
rgw::sal::Object* target_obj,
1682+
prefix_map_t& processed_prefixes) {
16811683
ldpp_dout(dpp, 20) << "DEBUG: complete" << dendl;
16821684
char final_etag[CEPH_CRYPTO_MD5_DIGESTSIZE];
16831685
char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16];
@@ -1922,6 +1924,15 @@ int DaosMultipartUpload::complete(
19221924
return ret;
19231925
}
19241926

1927+
int DaosMultipartUpload::cleanup_orphaned_parts(const DoutPrefixProvider *dpp,
1928+
CephContext *cct, optional_yield y,
1929+
const rgw_obj& obj,
1930+
std::list<rgw_obj_index_key>& remove_objs,
1931+
prefix_map_t& processed_prefixes)
1932+
{
1933+
return -ENOTSUP;
1934+
}
1935+
19251936
int DaosMultipartUpload::get_info(const DoutPrefixProvider* dpp,
19261937
optional_yield y, rgw_placement_rule** rule,
19271938
rgw::sal::Attrs* attrs) {

‎src/rgw/driver/daos/rgw_sal_daos.h‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -596,7 +596,8 @@ class DaosObject : public StoreObject {
596596
virtual ~DaosObject();
597597

598598
virtual int delete_object(const DoutPrefixProvider* dpp, optional_yield y,
599-
uint32_t flags) override;
599+
uint32_t flags, std::list<rgw_obj_index_key>* remove_objs,
600+
RGWObjVersionTracker* objv) override;
600601
virtual int copy_object(
601602
const ACLOwner& owner, const rgw_user& remote_user,
602603
req_info* info, const rgw_zone_id& source_zone,
@@ -861,7 +862,13 @@ class DaosMultipartUpload : public StoreMultipartUpload {
861862
uint64_t& accounted_size, bool& compressed,
862863
RGWCompressionInfo& cs_info, off_t& off,
863864
std::string& tag, ACLOwner& owner, uint64_t olh_epoch,
864-
rgw::sal::Object* target_obj) override;
865+
rgw::sal::Object* target_obj,
866+
prefix_map_t& processed_prefixes) override;
867+
virtual int cleanup_orphaned_parts(const DoutPrefixProvider *dpp,
868+
CephContext *cct, optional_yield y,
869+
const rgw_obj& obj,
870+
std::list<rgw_obj_index_key>& remove_objs,
871+
prefix_map_t& processed_prefixes) override;
865872
virtual int get_info(const DoutPrefixProvider* dpp, optional_yield y,
866873
rgw_placement_rule** rule,
867874
rgw::sal::Attrs* attrs = nullptr) override;

‎src/rgw/driver/motr/rgw_sal_motr.cc‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -585,7 +585,7 @@ int MotrBucket::remove(const DoutPrefixProvider *dpp, bool delete_children, opti
585585

586586
std::unique_ptr<rgw::sal::Object> object = get_object(key);
587587

588-
ret = object->delete_object(dpp, null_yield, rgw::sal::FLAG_LOG_OP);
588+
ret = object->delete_object(dpp, null_yield, rgw::sal::FLAG_LOG_OP, nullptr, nullptr);
589589
if (ret < 0 && ret != -ENOENT) {
590590
ldpp_dout(dpp, 0) << "ERROR: remove_bucket rgw_remove_object failed rc=" << ret << dendl;
591591
return ret;
@@ -1502,7 +1502,11 @@ int MotrObject::MotrDeleteOp::delete_obj(const DoutPrefixProvider* dpp, optional
15021502
return 0;
15031503
}
15041504

1505-
int MotrObject::delete_object(const DoutPrefixProvider* dpp, optional_yield y, uint32_t flags)
1505+
int MotrObject::delete_object(const DoutPrefixProvider* dpp,
1506+
optional_yield y,
1507+
uint32_t flags,
1508+
std::list<rgw_obj_index_key>* remove_objs,
1509+
RGWObjVersionTracker* objv)
15061510
{
15071511
MotrObject::MotrDeleteOp del_op(this);
15081512
del_op.params.bucket_owner = bucket->get_info().owner;
@@ -2668,7 +2672,8 @@ int MotrMultipartUpload::complete(const DoutPrefixProvider *dpp,
26682672
RGWCompressionInfo& cs_info, off_t& off,
26692673
std::string& tag, ACLOwner& owner,
26702674
uint64_t olh_epoch,
2671-
rgw::sal::Object* target_obj)
2675+
rgw::sal::Object* target_obj,
2676+
prefix_map_t& processed_prefixes)
26722677
{
26732678
char final_etag[CEPH_CRYPTO_MD5_DIGESTSIZE];
26742679
char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16];
@@ -2878,6 +2883,15 @@ int MotrMultipartUpload::complete(const DoutPrefixProvider *dpp,
28782883
M0_IC_DEL, meta_obj->get_key().get_oid(), bl);
28792884
}
28802885

2886+
int MotrMultipartUpload::cleanup_orphaned_parts(const DoutPrefixProvider *dpp,
2887+
CephContext *cct, optional_yield y,
2888+
const rgw_obj& obj,
2889+
std::list<rgw_obj_index_key>& remove_objs,
2890+
prefix_map_t& processed_prefixes)
2891+
{
2892+
return -ENOTSUP;
2893+
}
2894+
28812895
int MotrMultipartUpload::get_info(const DoutPrefixProvider *dpp, optional_yield y, rgw_placement_rule** rule, rgw::sal::Attrs* attrs)
28822896
{
28832897
if (!rule && !attrs) {

‎src/rgw/driver/motr/rgw_sal_motr.h‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -657,7 +657,9 @@ class MotrObject : public StoreObject {
657657

658658
virtual int delete_object(const DoutPrefixProvider* dpp,
659659
optional_yield y,
660-
uint32_t flags) override;
660+
uint32_t flags,
661+
td::list<rgw_obj_index_key>* remove_objs,
662+
GWObjVersionTracker* objv) override;
661663
virtual int copy_object(const ACLOwner& owner,
662664
const rgw_user& remote_user,
663665
req_info* info, const rgw_zone_id& source_zone,
@@ -933,7 +935,13 @@ class MotrMultipartUpload : public StoreMultipartUpload {
933935
RGWCompressionInfo& cs_info, off_t& off,
934936
std::string& tag, ACLOwner& owner,
935937
uint64_t olh_epoch,
936-
rgw::sal::Object* target_obj) override;
938+
rgw::sal::Object* target_obj,
939+
prefix_map_t& processed_prefixes) override;
940+
virtual int cleanup_orphaned_parts(const DoutPrefixProvider *dpp,
941+
CephContext *cct, optional_yield y,
942+
const rgw_obj& obj,
943+
std::list<rgw_obj_index_key>& remove_objs,
944+
prefix_map_t& processed_prefixes) override;
937945
virtual int get_info(const DoutPrefixProvider *dpp, optional_yield y, rgw_placement_rule** rule, rgw::sal::Attrs* attrs = nullptr) override;
938946
virtual std::unique_ptr<Writer> get_writer(const DoutPrefixProvider *dpp,
939947
optional_yield y,

‎src/rgw/driver/posix/rgw_sal_posix.cc‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2730,7 +2730,9 @@ int POSIXBucket::rename(const DoutPrefixProvider* dpp, optional_yield y, Object*
27302730

27312731
int POSIXObject::delete_object(const DoutPrefixProvider* dpp,
27322732
optional_yield y,
2733-
uint32_t flags)
2733+
uint32_t flags,
2734+
std::list<rgw_obj_index_key>* remove_objs,
2735+
RGWObjVersionTracker* objv)
27342736
{
27352737
POSIXBucket *b = static_cast<POSIXBucket*>(get_bucket());
27362738
if (!b) {
@@ -3505,7 +3507,7 @@ int POSIXObject::POSIXReadOp::get_attr(const DoutPrefixProvider* dpp, const char
35053507
int POSIXObject::POSIXDeleteOp::delete_obj(const DoutPrefixProvider* dpp,
35063508
optional_yield y, uint32_t flags)
35073509
{
3508-
return source->delete_object(dpp, y, flags);
3510+
return source->delete_object(dpp, y, flags, nullptr, nullptr);
35093511
}
35103512

35113513
int POSIXObject::copy(const DoutPrefixProvider *dpp, optional_yield y,
@@ -3703,7 +3705,8 @@ int POSIXMultipartUpload::complete(const DoutPrefixProvider *dpp,
37033705
RGWCompressionInfo& cs_info, off_t& ofs,
37043706
std::string& tag, ACLOwner& owner,
37053707
uint64_t olh_epoch,
3706-
rgw::sal::Object* target_obj)
3708+
rgw::sal::Object* target_obj,
3709+
prefix_map_t& processed_prefixes)
37073710
{
37083711
char final_etag[CEPH_CRYPTO_MD5_DIGESTSIZE];
37093712
char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16];
@@ -3852,6 +3855,15 @@ int POSIXMultipartUpload::complete(const DoutPrefixProvider *dpp,
38523855
return 0;
38533856
}
38543857

3858+
int POSIXMultipartUpload::cleanup_orphaned_parts(const DoutPrefixProvider *dpp,
3859+
CephContext *cct, optional_yield y,
3860+
const rgw_obj& obj,
3861+
std::list<rgw_obj_index_key>& remove_objs,
3862+
prefix_map_t& processed_prefixes)
3863+
{
3864+
return -ENOTSUP;
3865+
}
3866+
38553867
int POSIXMultipartUpload::get_info(const DoutPrefixProvider *dpp, optional_yield y,
38563868
rgw_placement_rule** rule, rgw::sal::Attrs* attrs)
38573869
{

‎src/rgw/driver/posix/rgw_sal_posix.h‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -632,7 +632,9 @@ class POSIXObject : public StoreObject {
632632

633633
virtual int delete_object(const DoutPrefixProvider* dpp,
634634
optional_yield y,
635-
uint32_t flags) override;
635+
uint32_t flags,
636+
std::list<rgw_obj_index_key>* remove_objs,
637+
RGWObjVersionTracker* objv) override;
636638
virtual int copy_object(const ACLOwner& owner,
637639
const rgw_user& remote_user,
638640
req_info* info, const rgw_zone_id& source_zone,
@@ -879,7 +881,13 @@ class POSIXMultipartUpload : public StoreMultipartUpload {
879881
RGWCompressionInfo& cs_info, off_t& ofs,
880882
std::string& tag, ACLOwner& owner,
881883
uint64_t olh_epoch,
882-
rgw::sal::Object* target_obj) override;
884+
rgw::sal::Object* target_obj,
885+
prefix_map_t& processed_prefixes) override;
886+
virtual int cleanup_orphaned_parts(const DoutPrefixProvider *dpp,
887+
CephContext *cct, optional_yield y,
888+
const rgw_obj& obj,
889+
std::list<rgw_obj_index_key>& remove_objs,
890+
prefix_map_t& processed_prefixes) override;
883891
virtual int get_info(const DoutPrefixProvider *dpp, optional_yield y,
884892
rgw_placement_rule** rule, rgw::sal::Attrs* attrs) override;
885893

‎src/rgw/driver/rados/rgw_bucket.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ int rgw_remove_object(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver, r
149149

150150
std::unique_ptr<rgw::sal::Object> object = bucket->get_object(key);
151151

152-
return object->delete_object(dpp, y, rgw::sal::FLAG_LOG_OP);
152+
return object->delete_object(dpp, y, rgw::sal::FLAG_LOG_OP, nullptr, nullptr);
153153
}
154154

155155
static void set_err_msg(std::string *sink, std::string msg)

‎src/rgw/driver/rados/rgw_object_expirer_core.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@ int RGWObjectExpirer::garbage_single_object(const DoutPrefixProvider *dpp, objex
225225

226226
std::unique_ptr<rgw::sal::Object> obj = bucket->get_object(key);
227227
obj->set_atomic();
228-
ret = obj->delete_object(dpp, null_yield, rgw::sal::FLAG_LOG_OP);
228+
ret = obj->delete_object(dpp, null_yield, rgw::sal::FLAG_LOG_OP, nullptr, nullptr);
229229

230230
return ret;
231231
}

‎src/rgw/driver/rados/rgw_putobj_processor.cc‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
#include "services/svc_zone.h"
2323
#include "rgw_sal_rados.h"
2424

25+
#include "cls/version/cls_version_client.h"
26+
2527
#define dout_subsys ceph_subsys_rgw
2628

2729
using namespace std;
@@ -573,7 +575,9 @@ int MultipartObjectProcessor::complete(
573575
}
574576

575577
librados::ObjectWriteOperation op;
578+
op.assert_exists();
576579
cls_rgw_mp_upload_part_info_update(op, p, info);
580+
cls_version_inc(op);
577581
r = rgw_rados_operate(rctx.dpp, meta_obj_ref.ioctx, meta_obj_ref.obj.oid, &op, rctx.y);
578582
ldpp_dout(rctx.dpp, 20) << "Update meta: " << meta_obj_ref.obj.oid << " part " << p << " prefix " << info.manifest.get_prefix() << " return " << r << dendl;
579583

@@ -588,8 +592,10 @@ int MultipartObjectProcessor::complete(
588592
op = librados::ObjectWriteOperation{};
589593
op.assert_exists(); // detect races with abort
590594
op.omap_set(m);
595+
cls_version_inc(op);
591596
r = rgw_rados_operate(rctx.dpp, meta_obj_ref.ioctx, meta_obj_ref.obj.oid, &op, rctx.y);
592597
}
598+
593599
if (r < 0) {
594600
return r == -ENOENT ? -ERR_NO_SUCH_UPLOAD : r;
595601
}

‎src/rgw/driver/rados/rgw_rados.cc‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5857,6 +5857,10 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi
58575857

58585858
store->remove_rgw_head_obj(op);
58595859

5860+
if (params.check_objv != nullptr) {
5861+
cls_version_check(op, *params.check_objv, VER_COND_EQ);
5862+
}
5863+
58605864
auto& ioctx = ref.ioctx;
58615865
r = rgw_rados_operate(dpp, ioctx, ref.obj.oid, &op, y);
58625866

@@ -6051,7 +6055,7 @@ int RGWRados::get_obj_state_impl(const DoutPrefixProvider *dpp, RGWObjectCtx *oc
60516055
int r = -ENOENT;
60526056

60536057
if (!assume_noent) {
6054-
r = RGWRados::raw_obj_stat(dpp, raw_obj, &s->size, &s->mtime, &s->epoch, &s->attrset, (s->prefetch_data ? &s->data : NULL), NULL, y);
6058+
r = RGWRados::raw_obj_stat(dpp, raw_obj, &s->size, &s->mtime, &s->epoch, &s->attrset, (s->prefetch_data ? &s->data : NULL), &s->objv_tracker, y);
60556059
}
60566060

60576061
if (r == -ENOENT) {
@@ -6784,6 +6788,10 @@ int RGWRados::Object::Read::prepare(optional_yield y, const DoutPrefixProvider *
67846788
return -ENOENT;
67856789
}
67866790

6791+
if (params.objv_tracker) {
6792+
*params.objv_tracker = astate->objv_tracker;
6793+
}
6794+
67876795
RGWBucketInfo& bucket_info = source->get_bucket_info();
67886796

67896797
if (params.part_num) {
@@ -8781,6 +8789,7 @@ int RGWRados::raw_obj_stat(const DoutPrefixProvider *dpp,
87818789
if (first_chunk) {
87828790
op.read(0, cct->_conf->rgw_max_chunk_size, first_chunk, NULL);
87838791
}
8792+
87848793
bufferlist outbl;
87858794
r = rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, &outbl, y);
87868795

0 commit comments

Comments
 (0)