Skip to content

Commit 506e3dc

Browse files
visualYJDrock-git
visualYJD
authored and
rock-git
committed
[fix][store] remove useless code which coordinator push store
operations. 1.Remove useless code. 2.Format debug info. 3.Rename joblist to job.
1 parent 51b83a1 commit 506e3dc

33 files changed

+1075
-1751
lines changed

‎conf/coordinator.template.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ coordinator:
1212
peers: $COORDINATOR_RAFT_PEERS$
1313
push_interval_s: 1
1414
update_state_interval_s: 10
15-
job_list_interval_s: 1
15+
job_interval_s: 1
1616
calc_metrics_interval_s: 60
1717
recycle_orphan_interval_s: 60
1818
remove_watch_interval_s: 10
@@ -21,8 +21,8 @@ coordinator:
2121
auto_compaction: true
2222
compaction_interval_s: 300
2323
compaction_retention_rev_count: 1000
24-
recycle_job_list_interval_s: 600
25-
reserve_job_list_recent_day: 7
24+
recycle_job_interval_s: 600
25+
reserve_job_recent_day: 7
2626
balance_leader_inspection_time_period: 0,23 # format: start_hour,end_hour;start_hour,end_hour e.g. 2,4;5,6;8,9
2727
balance_region_inspection_time_period: 0,23 # format: start_hour,end_hour;start_hour,end_hour e.g. 2,4;5,6;8,9
2828
balance_region_default_region_count_ratio: 0.8

‎src/client/coordinator_client_function_coor.cc‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ DECLARE_int64(peer_add_store_id);
5252
DECLARE_int64(peer_del_store_id);
5353
DECLARE_int64(store_id);
5454
DECLARE_int64(region_id);
55-
DECLARE_int64(job_list_id);
55+
DECLARE_int64(job_id);
5656
DECLARE_int64(region_cmd_id);
5757
DECLARE_string(store_ids);
5858
DECLARE_int64(index);
@@ -1820,11 +1820,11 @@ void SendGetStoreOperation(std::shared_ptr<dingodb::CoordinatorInteraction> coor
18201820
DINGO_LOG(INFO) << "SendRequest status=" << status;
18211821

18221822
for (const auto& it : response.store_operations()) {
1823-
DINGO_LOG(INFO) << "store_id=" << it.id() << " store_operation=" << it.DebugString();
1823+
DINGO_LOG(INFO) << "store_id=" << it.store_id() << " store_operation=" << it.DebugString();
18241824
}
18251825

18261826
for (const auto& it : response.store_operations()) {
1827-
DINGO_LOG(INFO) << "store_id=" << it.id() << " cmd_count=" << it.region_cmds_size();
1827+
DINGO_LOG(INFO) << "store_id=" << it.store_id() << " cmd_count=" << it.region_cmds_size();
18281828
}
18291829
}
18301830

@@ -1849,7 +1849,7 @@ void SendAddStoreOperation(std::shared_ptr<dingodb::CoordinatorInteraction> coor
18491849
dingodb::pb::coordinator::AddStoreOperationResponse response;
18501850

18511851
if (!FLAGS_id.empty()) {
1852-
request.mutable_store_operation()->set_id(std::stoll(FLAGS_id));
1852+
request.mutable_store_operation()->set_store_id(std::stoll(FLAGS_id));
18531853
} else {
18541854
DINGO_LOG(ERROR) << "id is empty (this is store_id)";
18551855
return;
@@ -1898,7 +1898,7 @@ void SendGetJobList(std::shared_ptr<dingodb::CoordinatorInteraction> coordinator
18981898
dingodb::pb::coordinator::GetJobListResponse response;
18991899

19001900
if (!FLAGS_id.empty()) {
1901-
request.set_job_list_id(std::stoll(FLAGS_id));
1901+
request.set_job_id(std::stoll(FLAGS_id));
19021902
}
19031903

19041904
request.set_include_archive(FLAGS_include_archive);
@@ -1913,8 +1913,8 @@ void SendGetJobList(std::shared_ptr<dingodb::CoordinatorInteraction> coordinator
19131913
DINGO_LOG_IF(INFO, !status.ok()) << "SendRequest status=" << status;
19141914
DINGO_LOG_IF(INFO, response.error().errcode() != 0) << "error: " << response.error().ShortDebugString();
19151915

1916-
for (const auto& job_list : response.job_lists()) {
1917-
DINGO_LOG(INFO) << "job_list: " << (FLAGS_show_pretty ? job_list.DebugString() : job_list.ShortDebugString());
1916+
for (const auto& job : response.job_list()) {
1917+
DINGO_LOG(INFO) << "job: " << (FLAGS_show_pretty ? job.DebugString() : job.ShortDebugString());
19181918
}
19191919
}
19201920

@@ -1923,10 +1923,10 @@ void SendCleanJobList(std::shared_ptr<dingodb::CoordinatorInteraction> coordinat
19231923
dingodb::pb::coordinator::CleanJobListResponse response;
19241924

19251925
if (FLAGS_id.empty()) {
1926-
DINGO_LOG(ERROR) << "id is empty, if you want to clean all job_list, set --id=0";
1926+
DINGO_LOG(ERROR) << "id is empty, if you want to clean all job, set --id=0";
19271927
return;
19281928
}
1929-
request.set_job_list_id(std::stoll(FLAGS_id));
1929+
request.set_job_id(std::stoll(FLAGS_id));
19301930

19311931
auto status = coordinator_interaction->SendRequest("CleanJobList", request, response);
19321932
DINGO_LOG(INFO) << "SendRequest status=" << status;
@@ -2096,8 +2096,8 @@ void SendUpdateRegionCmdStatus(std::shared_ptr<dingodb::CoordinatorInteraction>
20962096
dingodb::pb::coordinator::UpdateRegionCmdStatusRequest request;
20972097
dingodb::pb::coordinator::UpdateRegionCmdStatusResponse response;
20982098

2099-
if (FLAGS_job_list_id <= 0) {
2100-
DINGO_LOG(ERROR) << "job_list_id is empty";
2099+
if (FLAGS_job_id <= 0) {
2100+
DINGO_LOG(ERROR) << "job_id is empty";
21012101
return;
21022102
}
21032103

@@ -2121,7 +2121,7 @@ void SendUpdateRegionCmdStatus(std::shared_ptr<dingodb::CoordinatorInteraction>
21212121
return;
21222122
}
21232123

2124-
request.set_job_list_id(FLAGS_job_list_id);
2124+
request.set_job_id(FLAGS_job_id);
21252125
request.set_region_cmd_id(FLAGS_region_cmd_id);
21262126
request.set_status(static_cast<dingodb::pb::coordinator::RegionCmdStatus>(FLAGS_status));
21272127
request.mutable_error()->set_errcode(static_cast<dingodb::pb::error::Errno>(FLAGS_errcode));

‎src/client/dingodb_client.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ DEFINE_int64(start_region_cmd_id, 0, "start_region_cmd_id");
6767
DEFINE_int64(end_region_cmd_id, 0, "end_region_cmd_id");
6868
DEFINE_int64(region_id, 0, "region_id");
6969
DEFINE_int64(region_cmd_id, 0, "region_cmd_id");
70-
DEFINE_int64(job_list_id, 0, "job_list_id");
70+
DEFINE_int64(job_id, 0, "job_id");
7171
DEFINE_string(store_ids, "1001,1002,1003", "store_ids splited by ,");
7272
DEFINE_int64(index, 0, "index");
7373
DEFINE_int32(service_type, 0, "service type for getting leader, 0: meta or coordinator, 2: auto increment");

‎src/client/store_client_function.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3842,7 +3842,7 @@ uint32_t SendGetJobList() {
38423842

38433843
InteractionManager::GetInstance().SendRequestWithoutContext("CoordinatorService", "GetJobList", request, response);
38443844

3845-
return response.job_lists().size();
3845+
return response.job_list().size();
38463846
}
38473847

38483848
struct BatchPutGetParam {

‎src/client_v2/coordinator.cc‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ uint32_t SendGetJobList() {
113113

114114
InteractionManager::GetInstance().SendRequestWithoutContext("CoordinatorService", "GetJobList", request, response);
115115

116-
return response.job_lists().size();
116+
return response.job_list().size();
117117
}
118118

119119
dingodb::pb::common::StoreMap SendGetStoreMap() {
@@ -2126,11 +2126,11 @@ void RunGetStoreOperation(GetStoreOperationOption const &opt) {
21262126
DINGO_LOG(INFO) << "SendRequest status=" << status;
21272127

21282128
for (const auto &it : response.store_operations()) {
2129-
DINGO_LOG(INFO) << "store_id=" << it.id() << " store_operation=" << it.DebugString();
2129+
DINGO_LOG(INFO) << "store_id=" << it.store_id() << " store_operation=" << it.DebugString();
21302130
}
21312131

21322132
for (const auto &it : response.store_operations()) {
2133-
DINGO_LOG(INFO) << "store_id=" << it.id() << " cmd_count=" << it.region_cmds_size();
2133+
DINGO_LOG(INFO) << "store_id=" << it.store_id() << " cmd_count=" << it.region_cmds_size();
21342134
}
21352135
}
21362136

@@ -2157,7 +2157,7 @@ void RunGetJobList(GetJobListOptions const &opt) {
21572157
dingodb::pb::coordinator::GetJobListRequest request;
21582158
dingodb::pb::coordinator::GetJobListResponse response;
21592159

2160-
request.set_job_list_id(opt.id);
2160+
request.set_job_id(opt.id);
21612161
request.set_include_archive(opt.include_archive);
21622162
request.set_archive_limit(opt.limit);
21632163
request.set_archive_start_id(opt.start_id);
@@ -2169,8 +2169,8 @@ void RunGetJobList(GetJobListOptions const &opt) {
21692169
std::cout << "Get task list failed, error:" << response.error().ShortDebugString() << std::endl;
21702170
return;
21712171
}
2172-
for (const auto &job_list : response.job_lists()) {
2173-
std::cout << "job_list: " << job_list.DebugString() << std::endl;
2172+
for (const auto &job : response.job_list()) {
2173+
std::cout << "job: " << job.DebugString() << std::endl;
21742174
}
21752175
} else {
21762176
Pretty::Show(response);
@@ -2181,8 +2181,7 @@ void SetUpCleanJobList(CLI::App &app) {
21812181
auto opt = std::make_shared<CleanJobListOption>();
21822182
auto *cmd = app.add_subcommand("CleanJobList", "Clean task list")->group("Coordinator Command");
21832183
cmd->add_option("--coor_url", opt->coor_url, "Coordinator url, default:file://./coor_list");
2184-
cmd->add_option("--id", opt->id, "Request parameter task id, if you want to clean all job_list, set --id=0")
2185-
->required();
2184+
cmd->add_option("--id", opt->id, "Request parameter task id, if you want to clean all job, set --id=0")->required();
21862185
cmd->callback([opt]() { RunCleanJobList(*opt); });
21872186
}
21882187

@@ -2193,7 +2192,7 @@ void RunCleanJobList(CleanJobListOption const &opt) {
21932192
dingodb::pb::coordinator::CleanJobListRequest request;
21942193
dingodb::pb::coordinator::CleanJobListResponse response;
21952194

2196-
request.set_job_list_id(opt.id);
2195+
request.set_job_id(opt.id);
21972196

21982197
auto status =
21992198
CoordinatorInteraction::GetInstance().GetCoorinatorInteraction()->SendRequest("CleanJobList", request, response);
@@ -2211,7 +2210,7 @@ void SetUpUpdateRegionCmdStatus(CLI::App &app) {
22112210
auto opt = std::make_shared<UpdateRegionCmdStatusOptions>();
22122211
auto *cmd = app.add_subcommand("UpdateRegionCmdStatus", "Update region cmd status")->group("Coordinator Command");
22132212
cmd->add_option("--coor_url", opt->coor_url, "Coordinator url, default:file://./coor_list");
2214-
cmd->add_option("--job_list_id", opt->job_list_id, "Request parameter job_list_id")->required();
2213+
cmd->add_option("--job_id", opt->job_id, "Request parameter job_id")->required();
22152214
cmd->add_option("--region_cmd_id", opt->region_cmd_id, "Request parameter region cmd id")->required();
22162215
cmd->add_option("--status", opt->status, "Request parameter status, must be 1 [DONE] or 2 [FAIL]")->required();
22172216
cmd->add_option("--errcode", opt->errcode, "Request parameter errcode ")->required();
@@ -2225,7 +2224,7 @@ void RunUpdateRegionCmdStatus(UpdateRegionCmdStatusOptions const &opt) {
22252224
}
22262225
dingodb::pb::coordinator::UpdateRegionCmdStatusRequest request;
22272226
dingodb::pb::coordinator::UpdateRegionCmdStatusResponse response;
2228-
request.set_job_list_id(opt.job_list_id);
2227+
request.set_job_id(opt.job_id);
22292228
request.set_region_cmd_id(opt.region_cmd_id);
22302229
request.set_status(static_cast<dingodb::pb::coordinator::RegionCmdStatus>(opt.status));
22312230
request.mutable_error()->set_errcode(static_cast<dingodb::pb::error::Errno>(opt.errcode));
@@ -2277,7 +2276,7 @@ void RunAddStoreOperation(AddStoreOperationOptions const &opt) {
22772276
dingodb::pb::coordinator::AddStoreOperationRequest request;
22782277
dingodb::pb::coordinator::AddStoreOperationResponse response;
22792278

2280-
request.mutable_store_operation()->set_id(opt.id);
2279+
request.mutable_store_operation()->set_store_id(opt.id);
22812280
auto *region_cmd = request.mutable_store_operation()->add_region_cmds();
22822281
region_cmd->set_region_id(opt.region_id);
22832282
region_cmd->set_region_cmd_type(::dingodb::pb::coordinator::RegionCmdType::CMD_NONE);

‎src/client_v2/coordinator.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -427,7 +427,7 @@ void RunCleanJobList(CleanJobListOption const &opt);
427427

428428
struct UpdateRegionCmdStatusOptions {
429429
std::string coor_url;
430-
int64_t job_list_id;
430+
int64_t job_id;
431431
int64_t region_cmd_id;
432432
int64_t status;
433433
int64_t errcode;

‎src/client_v2/pretty.cc‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1369,23 +1369,23 @@ void Pretty::Show(dingodb::pb::coordinator::GetJobListResponse& response) {
13691369
ftxui::paragraph("CreateTime"),
13701370
ftxui::paragraph("FinishTime"),
13711371
}};
1372-
if (response.job_lists_size() == 0) {
1372+
if (response.job_list_size() == 0) {
13731373
std::cout << "Task list is empty." << std::endl;
13741374
return;
13751375
}
1376-
for (auto const& job_list : response.job_lists()) {
1376+
for (auto const& job : response.job_list()) {
13771377
std::vector<ftxui::Element> row = {
1378-
ftxui::paragraph(fmt::format("{}", job_list.id())),
1379-
ftxui::paragraph(fmt::format("{}", job_list.name())),
1380-
ftxui::paragraph(fmt::format("{}", job_list.next_step())),
1381-
ftxui::paragraph(fmt::format("{}", job_list.tasks_size())),
1382-
ftxui::paragraph(job_list.create_time()),
1383-
ftxui::paragraph(job_list.finish_time()),
1378+
ftxui::paragraph(fmt::format("{}", job.id())),
1379+
ftxui::paragraph(fmt::format("{}", job.name())),
1380+
ftxui::paragraph(fmt::format("{}", job.next_step())),
1381+
ftxui::paragraph(fmt::format("{}", job.tasks_size())),
1382+
ftxui::paragraph(job.create_time()),
1383+
ftxui::paragraph(job.finish_time()),
13841384
};
13851385
rows.push_back(row);
13861386
}
13871387
PrintTable(rows);
1388-
std::cout << "Sumary: total_job_list_size: " << response.job_lists_size() << std::endl;
1388+
std::cout << "Sumary: total_job_size: " << response.job_list_size() << std::endl;
13891389
}
13901390

13911391
void Pretty::Show(dingodb::pb::coordinator::GetExecutorMapResponse& response) {

‎src/common/constant.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -253,7 +253,7 @@ class Constant {
253253

254254
static const uint32_t kLeaderNumWeightDefaultValue = 1;
255255

256-
static const uint32_t kReserveJobListRecentDayDefaultValue = 7;
256+
static const uint32_t kReserveJobRecentDayDefaultValue = 7;
257257

258258
// diskann
259259
// min count of vectors for diskann

‎src/config/config_helper.cc‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -242,14 +242,14 @@ uint32_t ConfigHelper::GetLeaderNumWeight() {
242242
return (num <= 0) ? Constant::kLeaderNumWeightDefaultValue : num;
243243
}
244244

245-
uint32_t ConfigHelper::GetReserveJobListRecentDay() {
245+
uint32_t ConfigHelper::GetReserveJobRecentDay() {
246246
auto config = ConfigManager::GetInstance().GetRoleConfig();
247247
if (config == nullptr) {
248-
return Constant::kReserveJobListRecentDayDefaultValue;
248+
return Constant::kReserveJobRecentDayDefaultValue;
249249
}
250250

251-
int num = config->GetInt("coordinator.reserve_job_list_recent_day");
252-
return (num <= 0) ? Constant::kReserveJobListRecentDayDefaultValue : num;
251+
int num = config->GetInt("coordinator.reserve_job_recent_day");
252+
return (num <= 0) ? Constant::kReserveJobRecentDayDefaultValue : num;
253253
}
254254

255255
std::string ConfigHelper::GetBalanceLeaderInspectionTimePeriod() {
@@ -334,7 +334,7 @@ double ConfigHelper::GetRaftWorkerThreadRatio() {
334334
return config->GetDouble("server.raft_worker_thread_ratio");
335335
}
336336

337-
std::string ConfigHelper::GetBlockCacheValue(){
337+
std::string ConfigHelper::GetBlockCacheValue() {
338338
auto config = ConfigManager::GetInstance().GetRoleConfig();
339339
if (config == nullptr) {
340340
return Constant::kBlockCacheDefaultValue;

‎src/config/config_helper.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ class ConfigHelper {
4141

4242
static uint32_t GetLeaderNumWeight();
4343

44-
static uint32_t GetReserveJobListRecentDay();
44+
static uint32_t GetReserveJobRecentDay();
4545

4646
static std::string GetBalanceLeaderInspectionTimePeriod();
4747

0 commit comments

Comments
 (0)