Skip to content

Commit f00ac7c

Browse files
soumyakodurisoumyakoduri
committed
rgw/cloud-restore: Fixing issues with initializing and resetting FIFO
In addition, added some more debug statements and done code cleanup Reviewed-by: Adam Emerson <[email protected]> Reviewed-by: Jiffin Tony Thottan <[email protected]> Signed-off-by: Soumya Koduri <[email protected]>
1 parent 9974f51 commit f00ac7c

File tree

12 files changed

+188
-76
lines changed

12 files changed

+188
-76
lines changed

‎src/rgw/driver/rados/rgw_lc_tier.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -292,7 +292,7 @@ int rgw_cloud_tier_restore_object(RGWLCCloudTierCtx& tier_ctx,
292292
}
293293

294294
// now send HEAD request and verify if restore is complete on glacier/tape endpoint
295-
static constexpr int MAX_RETRIES = 10;
295+
static constexpr int MAX_RETRIES = 2;
296296
uint32_t retries = 0;
297297
do {
298298
ret = rgw_cloud_tier_get_object(tier_ctx, true, headers, nullptr, etag,

‎src/rgw/driver/rados/rgw_rados.cc‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1379,7 +1379,12 @@ int RGWRados::init_complete(const DoutPrefixProvider *dpp, optional_yield y, rgw
13791379
lc->start_processor();
13801380

13811381
restore = new RGWRestore();
1382-
restore->initialize(cct, this->driver);
1382+
ret = restore->initialize(cct, this->driver);
1383+
1384+
if (ret < 0) {
1385+
ldpp_dout(dpp, 0) << "ERROR: failed to initialize restore thread" << dendl;
1386+
return ret;
1387+
}
13831388

13841389
if (use_restore_thread)
13851390
restore->start_processor();

‎src/rgw/driver/rados/rgw_sal_rados.cc‎

Lines changed: 61 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2026,10 +2026,9 @@ std::unique_ptr<Lifecycle> RadosStore::get_lifecycle(void)
20262026
return std::make_unique<RadosLifecycle>(this);
20272027
}
20282028

2029-
std::unique_ptr<Restore> RadosStore::get_restore(const int n_objs,
2030-
const std::vector<std::string_view>& obj_names)
2029+
std::unique_ptr<Restore> RadosStore::get_restore(void)
20312030
{
2032-
return std::make_unique<RadosRestore>(this, n_objs, obj_names);
2031+
return std::make_unique<RadosRestore>(this);
20332032
}
20342033

20352034
bool RadosStore::process_expired_objects(const DoutPrefixProvider *dpp,
@@ -4673,12 +4672,45 @@ std::unique_ptr<RestoreSerializer> RadosRestore::get_serializer(
46734672
return std::make_unique<RadosRestoreSerializer>(store, oid, lock_name, cookie);
46744673
}
46754674

4675+
int RadosRestore::initialize(const DoutPrefixProvider* dpp, optional_yield y,
4676+
int n_objs, std::vector<std::string>& o_names)
4677+
{
4678+
int ret = 0;
4679+
num_objs = n_objs;
4680+
obj_names = o_names;
4681+
4682+
for (auto i=0; i < num_objs; i++) {
4683+
std::unique_ptr<rgw::cls::fifo::FIFO> fifo_tmp;
4684+
ret = rgw::cls::fifo::FIFO::create(dpp, ioctx, obj_names[i], &fifo_tmp, y);
4685+
4686+
ldpp_dout(dpp, 20) << "creating fifo object for index=" << i
4687+
<< ", objname=" << obj_names[i] <<
4688+
" returned ret=" << ret << dendl;
4689+
4690+
if (ret) {
4691+
return ret;
4692+
}
4693+
4694+
fifos.push_back(std::move(fifo_tmp));
4695+
}
4696+
4697+
return ret;
4698+
}
4699+
4700+
void RadosRestore::finalize() {
4701+
obj_names.clear();
4702+
fifos.clear();
4703+
}
4704+
46764705
int RadosRestore::add_entry(const DoutPrefixProvider* dpp, optional_yield y,
46774706
int index, const RGWRestoreEntry& entry) {
46784707
bufferlist bl;
46794708

46804709
encode(entry, bl);
46814710

4711+
ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__
4712+
<< "Adding entry:(" << entry.bucket << "," << entry.obj_key << ") to FIFO:" << obj_names[index] << dendl;
4713+
46824714
auto ret = push(dpp, y, index, std::move(bl));
46834715
if (ret < 0) {
46844716
ldpp_dout(dpp, -1) << "ERROR: push() returned " << ret << dendl;
@@ -4701,6 +4733,9 @@ int RadosRestore::add_entries(const DoutPrefixProvider* dpp, optional_yield y,
47014733

47024734
}
47034735

4736+
ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__
4737+
<< "Adding " << restore_entries.size() << " entries to FIFO:" << obj_names[index] << dendl;
4738+
47044739
int ret = push(dpp, y, index, std::move(ent_list));
47054740

47064741
if (ret < 0) {
@@ -4713,7 +4748,10 @@ int RadosRestore::add_entries(const DoutPrefixProvider* dpp, optional_yield y,
47134748

47144749
int RadosRestore::push(const DoutPrefixProvider *dpp, optional_yield y,
47154750
int index, std::vector<ceph::buffer::list>&& items) {
4716-
auto r = fifos[index].push(dpp, items, y);
4751+
ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__
4752+
<< "Pushing entries to FIFO:" << obj_names[index] << dendl;
4753+
4754+
auto r = fifos[index]->push(dpp, items, y);
47174755
if (r < 0) {
47184756
ldpp_dout(dpp, -1) << __PRETTY_FUNCTION__
47194757
<< ": unable to push to FIFO: " << obj_names[index]
@@ -4724,7 +4762,10 @@ int RadosRestore::push(const DoutPrefixProvider *dpp, optional_yield y,
47244762

47254763
int RadosRestore::push(const DoutPrefixProvider *dpp, optional_yield y,
47264764
int index, ceph::buffer::list&& bl) {
4727-
auto r = fifos[index].push(dpp, std::move(bl), y);
4765+
ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__
4766+
<< "Pushing entry to FIFO:" << obj_names[index] << dendl;
4767+
4768+
auto r = fifos[index]->push(dpp, std::move(bl), y);
47284769
if (r < 0) {
47294770
ldpp_dout(dpp, -1) << __PRETTY_FUNCTION__
47304771
<< ": unable to push to FIFO: " << obj_names[index]
@@ -4768,8 +4809,10 @@ int RadosRestore::list(const DoutPrefixProvider *dpp, optional_yield y,
47684809
std::vector<rgw::cls::fifo::list_entry> restore_entries;
47694810
bool more = false;
47704811

4771-
auto r = fifos[index].list(dpp, max_entries, marker, &restore_entries, &more, y);
4812+
ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__
4813+
<< "Listing entries from FIFO:" << obj_names[index] << dendl;
47724814

4815+
auto r = fifos[index]->list(dpp, max_entries, marker, &restore_entries, &more, y);
47734816
if (r < 0) {
47744817
ldpp_dout(dpp, -1) << __PRETTY_FUNCTION__
47754818
<< ": unable to list FIFO: " << obj_names[index]
@@ -4804,6 +4847,11 @@ int RadosRestore::list(const DoutPrefixProvider *dpp, optional_yield y,
48044847
*out_marker = restore_entries.back().marker;
48054848
}
48064849

4850+
ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__
4851+
<< "Listing from FIFO:" << obj_names[index] << ", returned:"
4852+
<< restore_entries.size() << " entries, truncated:" << more
4853+
<< ", out_marker:" << (out_marker ? *out_marker : "") << dendl;
4854+
48074855
return 0;
48084856
}
48094857

@@ -4818,7 +4866,10 @@ int RadosRestore::trim_entries(const DoutPrefixProvider *dpp, optional_yield y,
48184866

48194867
int RadosRestore::trim(const DoutPrefixProvider *dpp, optional_yield y,
48204868
int index, const std::string_view& marker) {
4821-
auto r = fifos[index].trim(dpp, marker, false, y);
4869+
ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__
4870+
<< "Trimming FIFO:" << obj_names[index] << " upto marker:" << marker << dendl;
4871+
4872+
auto r = fifos[index]->trim(dpp, marker, false, y);
48224873
if (r < 0) {
48234874
ldpp_dout(dpp, -1) << __PRETTY_FUNCTION__
48244875
<< ": unable to trim FIFO: " << obj_names[index]
@@ -4838,14 +4889,16 @@ int RadosRestore::is_empty(const DoutPrefixProvider *dpp, optional_yield y) {
48384889
bool more = false;
48394890

48404891
for (auto shard = 0u; shard < fifos.size(); ++shard) {
4841-
auto r = fifos[shard].list(dpp, 1, {}, &restore_entries, &more, y);
4892+
auto r = fifos[shard]->list(dpp, 1, {}, &restore_entries, &more, y);
48424893
if (r < 0) {
48434894
ldpp_dout(dpp, -1) << __PRETTY_FUNCTION__
48444895
<< ": unable to list FIFO: " << obj_names[shard]
48454896
<< ": " << cpp_strerror(-r) << dendl;
48464897
return r;
48474898
}
48484899
if (!restore_entries.empty()) {
4900+
ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__
4901+
<< "Entries found in FIFO:" << obj_names[shard] << dendl;
48494902
return 0;
48504903
}
48514904
}

‎src/rgw/driver/rados/rgw_sal_rados.h‎

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
#include "rgw_putobj_processor.h"
2828
#include "services/svc_tier_rados.h"
2929
#include "cls/lock/cls_lock_client.h"
30-
#include "rgw_log_backing.h"
3130

3231
namespace rgw { namespace sal {
3332

@@ -283,8 +282,7 @@ class RadosStore : public StoreDriver {
283282
virtual int list_all_zones(const DoutPrefixProvider* dpp, std::list<std::string>& zone_ids) override;
284283
virtual int cluster_stat(RGWClusterStat& stats) override;
285284
virtual std::unique_ptr<Lifecycle> get_lifecycle(void) override;
286-
virtual std::unique_ptr<Restore> get_restore(const int n_objs,
287-
const std::vector<std::string_view>& obj_names) override;
285+
virtual std::unique_ptr<Restore> get_restore(void) override;
288286
virtual bool process_expired_objects(const DoutPrefixProvider *dpp, optional_yield y) override;
289287
virtual std::unique_ptr<Notification> get_notification(rgw::sal::Object* obj, rgw::sal::Object* src_obj, req_state* s, rgw::notify::EventType event_type, optional_yield y, const std::string* object_name=nullptr) override;
290288
virtual std::unique_ptr<Notification> get_notification(
@@ -961,23 +959,24 @@ class RadosRestoreSerializer : public StoreRestoreSerializer {
961959
}
962960
};
963961

964-
class RadosRestore : public Restore {
962+
class RadosRestore : public StoreRestore {
965963
RadosStore* store;
966-
const int num_objs;
967-
const std::vector<std::string_view>& obj_names;
968964
librados::IoCtx& ioctx;
969-
ceph::containers::tiny_vector<LazyFIFO> fifos;
965+
int num_objs;
966+
std::vector<std::string> obj_names;
967+
std::vector<std::unique_ptr<rgw::cls::fifo::FIFO>> fifos;
970968

971969
public:
972-
RadosRestore(RadosStore* _st, const int n_objs, const std::vector<std::string_view>& o_names) : store(_st),
973-
num_objs(n_objs), obj_names(o_names),
974-
ioctx(*store->getRados()->get_restore_pool_ctx()),
975-
fifos(num_objs,
976-
[&](const size_t ix, auto emplacer) {
977-
emplacer.emplace(ioctx, std::string(obj_names[ix]));
978-
}) {}
979-
980-
~RadosRestore() override = default;
970+
RadosRestore(RadosStore* _st) : store(_st),
971+
ioctx(*store->getRados()->get_restore_pool_ctx()) {}
972+
973+
~RadosRestore() override {
974+
finalize();
975+
}
976+
977+
virtual int initialize(const DoutPrefixProvider* dpp, optional_yield y,
978+
int n_objs, std::vector<std::string>& obj_names) override;
979+
void finalize();
981980

982981
virtual int add_entry(const DoutPrefixProvider* dpp, optional_yield y,
983982
int index, const RGWRestoreEntry& r_entry) override;

0 commit comments

Comments
 (0)