Skip to content

Commit 84c478f

Browse files
yuhaijun999ketor
yuhaijun999
authored and
ketor
committed
[feat][store] Add restore feature first edition.
1 parent c73df13 commit 84c478f

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+6209
-158
lines changed

‎dingo-store-proto‎

‎src/br/backup.cc‎

Lines changed: 45 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -434,12 +434,21 @@ butil::Status Backup::DoRun() {
434434

435435
std::map<std::string, std::string> kvs;
436436

437-
{ kvs.emplace(meta_meta->file_name(), meta_meta->SerializeAsString()); }
437+
if (meta_meta) {
438+
kvs.emplace(meta_meta->file_name(), meta_meta->SerializeAsString());
439+
}
438440

439-
{ kvs.emplace(meta_data->file_name(), meta_data->SerializeAsString()); }
441+
if (meta_data) {
442+
kvs.emplace(meta_data->file_name(), meta_data->SerializeAsString());
443+
}
440444

441-
{ kvs.emplace(dingodb::Constant::kIdEpochTypeAndValueKey, id_epoch_type_and_value->SerializeAsString()); }
442-
{ kvs.emplace(dingodb::Constant::kTableIncrementKey, table_increment_group->SerializeAsString()); }
445+
if (id_epoch_type_and_value) {
446+
kvs.emplace(dingodb::Constant::kIdEpochTypeAndValueKey, id_epoch_type_and_value->SerializeAsString());
447+
}
448+
449+
if (table_increment_group) {
450+
kvs.emplace(dingodb::Constant::kTableIncrementKey, table_increment_group->SerializeAsString());
451+
}
443452

444453
dingodb::pb::common::VersionInfo version_info = dingodb::GetVersionInfo();
445454

@@ -501,17 +510,25 @@ butil::Status Backup::DoRun() {
501510
DINGO_LOG(INFO) << dingodb::Constant::kBackupBackupParamKey << " :";
502511
DINGO_LOG(INFO) << backup_param.DebugString() << std::endl;
503512

504-
DINGO_LOG(INFO) << dingodb::Constant::kIdEpochTypeAndValueKey << " :";
505-
DINGO_LOG(INFO) << id_epoch_type_and_value->DebugString() << std::endl;
513+
if (id_epoch_type_and_value) {
514+
DINGO_LOG(INFO) << dingodb::Constant::kIdEpochTypeAndValueKey << " :";
515+
DINGO_LOG(INFO) << id_epoch_type_and_value->DebugString() << std::endl;
516+
}
506517

507-
DINGO_LOG(INFO) << dingodb::Constant::kTableIncrementKey << " :";
508-
DINGO_LOG(INFO) << table_increment_group->DebugString() << std::endl;
518+
if (table_increment_group) {
519+
DINGO_LOG(INFO) << dingodb::Constant::kTableIncrementKey << " :";
520+
DINGO_LOG(INFO) << table_increment_group->DebugString() << std::endl;
521+
}
509522

510-
DINGO_LOG(INFO) << meta_meta->file_name() << " :";
511-
DINGO_LOG(INFO) << meta_meta->DebugString() << std::endl;
523+
if (meta_meta) {
524+
DINGO_LOG(INFO) << meta_meta->file_name() << " :";
525+
DINGO_LOG(INFO) << meta_meta->DebugString() << std::endl;
526+
}
512527

513-
DINGO_LOG(INFO) << meta_data->file_name() << " :";
514-
DINGO_LOG(INFO) << meta_data->DebugString() << std::endl;
528+
if (meta_data) {
529+
DINGO_LOG(INFO) << meta_data->file_name() << " :";
530+
DINGO_LOG(INFO) << meta_data->DebugString() << std::endl;
531+
}
515532
DINGO_LOG(INFO) << dingodb::Constant::kBackupMetaName
516533
<< "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<";
517534
}
@@ -535,17 +552,25 @@ butil::Status Backup::DoRun() {
535552
writer << dingodb::Constant::kBackupBackupParamKey << " : " << std::endl;
536553
writer << backup_param.DebugString() << std::endl;
537554

538-
writer << dingodb::Constant::kIdEpochTypeAndValueKey << " : " << std::endl;
539-
writer << id_epoch_type_and_value->DebugString() << std::endl;
555+
if (id_epoch_type_and_value) {
556+
writer << dingodb::Constant::kIdEpochTypeAndValueKey << " : " << std::endl;
557+
writer << id_epoch_type_and_value->DebugString() << std::endl;
558+
}
540559

541-
writer << dingodb::Constant::kTableIncrementKey << " : " << std::endl;
542-
writer << table_increment_group->DebugString() << std::endl;
560+
if (table_increment_group) {
561+
writer << dingodb::Constant::kTableIncrementKey << " : " << std::endl;
562+
writer << table_increment_group->DebugString() << std::endl;
563+
}
543564

544-
writer << meta_meta->file_name() << " : " << std::endl;
545-
writer << meta_meta->DebugString() << std::endl;
565+
if (meta_meta) {
566+
writer << meta_meta->file_name() << " : " << std::endl;
567+
writer << meta_meta->DebugString() << std::endl;
568+
}
546569

547-
writer << meta_data->file_name() << " : " << std::endl;
548-
writer << meta_data->DebugString() << std::endl;
570+
if (meta_data) {
571+
writer << meta_data->file_name() << " : " << std::endl;
572+
writer << meta_data->DebugString() << std::endl;
573+
}
549574

550575
if (writer.is_open()) {
551576
writer.close();

‎src/br/backup_data.cc‎

Lines changed: 52 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -208,46 +208,53 @@ butil::Status BackupData::Finish() {
208208

209209
std::map<std::string, std::string> kvs;
210210

211-
std::shared_ptr<std::vector<dingodb::pb::common::BackupMeta>> sql_backup_meta = backup_sql_data_->GetBackupMeta();
212-
std::shared_ptr<std::vector<dingodb::pb::common::BackupMeta>> sdk_backup_meta = backup_sdk_data_->GetBackupMeta();
213-
for (const auto& meta : *sql_backup_meta) {
214-
kvs.emplace(meta.file_name(), meta.SerializeAsString());
211+
if (backup_sql_data_) {
212+
std::shared_ptr<std::vector<dingodb::pb::common::BackupMeta>> sql_backup_meta = backup_sql_data_->GetBackupMeta();
213+
for (const auto& meta : *sql_backup_meta) {
214+
kvs.emplace(meta.file_name(), meta.SerializeAsString());
215+
}
215216
}
216217

217-
for (const auto& meta : *sdk_backup_meta) {
218-
kvs.emplace(meta.file_name(), meta.SerializeAsString());
218+
if (backup_sdk_data_) {
219+
std::shared_ptr<std::vector<dingodb::pb::common::BackupMeta>> sdk_backup_meta = backup_sdk_data_->GetBackupMeta();
220+
for (const auto& meta : *sdk_backup_meta) {
221+
kvs.emplace(meta.file_name(), meta.SerializeAsString());
222+
}
219223
}
220224

221-
rocksdb::Options options;
222-
std::shared_ptr<SstFileWriter> sst = std::make_shared<SstFileWriter>(options);
225+
if (!kvs.empty()) {
226+
rocksdb::Options options;
227+
std::shared_ptr<SstFileWriter> sst = std::make_shared<SstFileWriter>(options);
223228

224-
status = sst->SaveFile(kvs, file_path);
225-
if (!status.ok()) {
226-
DINGO_LOG(ERROR) << status.error_cstr();
227-
return status;
228-
}
229+
status = sst->SaveFile(kvs, file_path);
230+
if (!status.ok()) {
231+
DINGO_LOG(ERROR) << status.error_cstr();
232+
return status;
233+
}
229234

230-
std::string hash_code;
231-
status = dingodb::Helper::CalSha1CodeWithFileEx(file_path, hash_code);
232-
if (!status.ok()) {
233-
DINGO_LOG(ERROR) << status.error_cstr();
234-
return status;
235-
}
235+
std::string hash_code;
236+
status = dingodb::Helper::CalSha1CodeWithFileEx(file_path, hash_code);
237+
if (!status.ok()) {
238+
DINGO_LOG(ERROR) << status.error_cstr();
239+
return status;
240+
}
236241

237-
if (!backup_meta_) {
238-
backup_meta_ = std::make_shared<dingodb::pb::common::BackupMeta>();
239-
}
242+
if (!backup_meta_) {
243+
backup_meta_ = std::make_shared<dingodb::pb::common::BackupMeta>();
244+
}
240245

241-
backup_meta_->set_remark("backup sdk data and sql data. save dingodb::pb::common::BackupDataFileValueSstMetaGroup. ");
242-
backup_meta_->set_exec_node(dingodb::Constant::kBackupRegionName);
243-
backup_meta_->set_dir_name("");
244-
backup_meta_->set_file_size(sst->GetSize());
245-
backup_meta_->set_encryption(hash_code);
246-
backup_meta_->set_file_name(file_name);
246+
backup_meta_->set_remark(
247+
"backup sdk data and sql data. save dingodb::pb::common::BackupDataFileValueSstMetaGroup. ");
248+
backup_meta_->set_exec_node(dingodb::Constant::kBackupRegionName);
249+
backup_meta_->set_dir_name("");
250+
backup_meta_->set_file_size(sst->GetSize());
251+
backup_meta_->set_encryption(hash_code);
252+
backup_meta_->set_file_name(file_name);
247253

248-
if (FLAGS_br_log_switch_backup_detail) {
249-
DINGO_LOG(INFO) << backup_meta_->DebugString() << std::endl;
250-
}
254+
if (FLAGS_br_log_switch_backup_detail) {
255+
DINGO_LOG(INFO) << backup_meta_->DebugString() << std::endl;
256+
}
257+
} // if (!kvs.empty()) {
251258

252259
return butil::Status::OK();
253260
}
@@ -274,6 +281,20 @@ butil::Status BackupData::GetAllRegionMapFromCoordinator() {
274281
return butil::Status(response.error().errcode(), response.error().errmsg());
275282
}
276283

284+
for (const auto& region : response.regionmap().regions()) {
285+
if (region.id() == 80081) {
286+
if (region.definition().has_index_parameter()) {
287+
DINGO_LOG(INFO) << "has_index_parameter ***************meta store region id : " << region.id()
288+
<< ", name : " << region.definition().name()
289+
<< ", index_parameter : " << region.definition().index_parameter().DebugString();
290+
} else {
291+
DINGO_LOG(INFO) << "no_index_parameter ***************meta store region id : " << region.id()
292+
<< ", name : " << region.definition().name()
293+
<< ", index_parameter : " << region.definition().index_parameter().DebugString();
294+
}
295+
}
296+
}
297+
277298
region_map_ = std::make_shared<dingodb::pb::common::RegionMap>(response.regionmap());
278299

279300
if (FLAGS_br_log_switch_backup_detail) {

‎src/br/backup_data.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,4 +74,4 @@ class BackupData : public std::enable_shared_from_this<BackupData> {
7474

7575
} // namespace br
7676

77-
#endif // DINGODB_BR_BACKUP_SDK_DATA_H_
77+
#endif // DINGODB_BR_BACKUP_DATA_H_

‎src/br/backup_data_base.cc‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,11 @@ BackupDataBase::BackupDataBase(ServerInteractionPtr coordinator_interaction, Ser
4444
already_handle_store_regions_(0),
4545
already_handle_index_regions_(0),
4646
already_handle_document_regions_(0),
47-
name_(name) {}
47+
name_(name) {
48+
bthread_mutex_init(&mutex_, nullptr);
49+
}
4850

49-
BackupDataBase::~BackupDataBase() = default;
51+
BackupDataBase::~BackupDataBase() { bthread_mutex_destroy(&mutex_); }
5052

5153
void BackupDataBase::SetRegionMap(std::shared_ptr<dingodb::pb::common::RegionMap> region_map) {
5254
region_map_ = region_map;
@@ -274,7 +276,10 @@ butil::Status BackupDataBase::DoBackupRegionInternal(
274276
ServerInteractionPtr interaction, const std::string& service_name,
275277
std::shared_ptr<std::vector<dingodb::pb::common::Region>> wait_for_handle_regions,
276278
std::atomic<int64_t>& already_handle_regions,
277-
std::shared_ptr<std::map<int64_t, dingodb::pb::common::BackupDataFileValueSstMetaGroup>> save_region_map) {
279+
std::shared_ptr<std::map<int64_t, dingodb::pb::common::BackupDataFileValueSstMetaGroup>> save_region_map,
280+
std::atomic<bool>& is_thread_exit) {
281+
butil::Status status;
282+
is_thread_exit = false;
278283
for (const auto& region : *wait_for_handle_regions) {
279284
if (is_need_exit_) {
280285
break;
@@ -296,13 +301,13 @@ butil::Status BackupDataBase::DoBackupRegionInternal(
296301

297302
DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << name_ << " " << request.DebugString();
298303

299-
butil::Status status = interaction->SendRequest(service_name, "BackupData", request, response);
304+
status = interaction->SendRequest(service_name, "BackupData", request, response);
300305
if (!status.ok()) {
301306
is_need_exit_ = true;
302307
std::string s = fmt::format("Fail to backup region, region_id={}, status={}", region.id(), status.error_cstr());
303308
DINGO_LOG(ERROR) << s;
304-
last_error_ = status;
305-
return status;
309+
status = butil::Status(status.error_code(), s);
310+
break;
306311
}
307312

308313
if (response.error().errcode() != dingodb::pb::error::OK) {
@@ -311,17 +316,24 @@ butil::Status BackupDataBase::DoBackupRegionInternal(
311316
fmt::format("Fail to backup region, region_id={}, error={}", region.id(), response.error().errmsg());
312317
DINGO_LOG(ERROR) << s;
313318
status = butil::Status(response.error().errcode(), s);
314-
last_error_ = status;
315-
return status;
319+
break;
316320
}
317321

318322
DINGO_LOG_IF(INFO, FLAGS_br_log_switch_backup_detail_detail) << name_ << " " << response.DebugString();
319323

320-
save_region_map->insert({region.id(), response.sst_metas()});
321-
324+
if (response.sst_metas().backup_data_file_value_sst_metas_size() > 0) {
325+
save_region_map->insert({region.id(), response.sst_metas()});
326+
}
322327
already_handle_regions++;
323328
}
324-
return butil::Status::OK();
329+
330+
is_thread_exit = true;
331+
{
332+
BAIDU_SCOPED_LOCK(mutex_);
333+
last_error_ = status;
334+
}
335+
336+
return status;
325337
}
326338

327339
} // namespace br

‎src/br/backup_data_base.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,8 @@ class BackupDataBase {
6666
ServerInteractionPtr interaction, const std::string& service_name,
6767
std::shared_ptr<std::vector<dingodb::pb::common::Region>> wait_for_handle_regions,
6868
std::atomic<int64_t>& already_handle_regions,
69-
std::shared_ptr<std::map<int64_t, dingodb::pb::common::BackupDataFileValueSstMetaGroup>> save_region_map);
69+
std::shared_ptr<std::map<int64_t, dingodb::pb::common::BackupDataFileValueSstMetaGroup>> save_region_map,
70+
std::atomic<bool>& is_thread_exit);
7071

7172
private:
7273
ServerInteractionPtr coordinator_interaction_;
@@ -102,6 +103,8 @@ class BackupDataBase {
102103

103104
std::shared_ptr<std::vector<dingodb::pb::common::BackupMeta>> backup_data_base_;
104105

106+
bthread_mutex_t mutex_;
107+
105108
// last error
106109
butil::Status last_error_;
107110
};

0 commit comments

Comments
 (0)