diff --git a/conf/pika.conf b/conf/pika.conf index 4f51f9cdbd..4a68be3544 100644 --- a/conf/pika.conf +++ b/conf/pika.conf @@ -681,30 +681,25 @@ wash-data: true # obd-compact https://github.com/OpenAtomFoundation/pika/issues/2255 compaction-strategy : obd-compact -# For OBD_Compact +################################## Configure For OBD_Compact ##################### # According to the number of sst files in rocksdb, # compact every `compact-every-num-of-files` file. compact-every-num-of-files : 10 -# For OBD_Compact -# In another search, if the file creation time is -# greater than `force-compact-file-age-seconds`, -# a compaction of the upper and lower boundaries -# of the file will be performed at the same time -# `compact-every-num-of-files` -1 -force-compact-file-age-seconds : 300 +# Extends the time a file must exist (20 minutes) before triggering forced compaction +force-compact-file-age-seconds : 1200 -# For OBD_Compact -# According to the number of sst files in rocksdb, -# compact every `compact-every-num-of-files` file. -force-compact-min-delete-ratio : 10 +# The forced deletion ratio threshold - compaction will be triggered when +# the deletion markers percentage in SST files reaches this value +force-compact-min-delete-ratio : 50 -# For OBD_Compact -# According to the number of sst files in rocksdb, -# compact every `compact-every-num-of-files` file. -dont-compact-sst-created-in-seconds : 20 +# Files created within the last 5 minutes will not be compacted +dont-compact-sst-created-in-seconds : 300 -# For OBD_Compact -# According to the number of sst files in rocksdb, -# compact every `compact-every-num-of-files` file. -best-delete-min-ratio : 10 \ No newline at end of file +# The optimal deletion ratio - defines the best threshold percentage +# for deletion markers in SST files to optimize compaction efficiency +best-delete-min-ratio : 30 + +# Trigger the obd-compact task periodically according to `obd-compact-interval` +# obd-compact-interval >= 600. +obd-compact-interval : 600 \ No newline at end of file diff --git a/include/pika_conf.h b/include/pika_conf.h index ecd42e0f91..1c1b6b4861 100644 --- a/include/pika_conf.h +++ b/include/pika_conf.h @@ -143,6 +143,10 @@ class PikaConf : public pstd::BaseConf { std::shared_lock l(rwlock_); return best_delete_min_ratio_; } + int obd_compact_interval () { + std::shared_lock l(rwlock_); + return obd_compact_interval_; + } CompactionStrategy compaction_strategy() { std::shared_lock l(rwlock_); return compaction_strategy_; @@ -984,6 +988,11 @@ class PikaConf : public pstd::BaseConf { ConfigRewrite(); } + void SetObdCompactInterval(const int value) { + std::unique_lock l(rwlock_); + obd_compact_interval_ = value < 600 ? 600 : value; + } + size_t GetUnfinishedFullSyncCount() { std::shared_lock l(rwlock_); return internal_used_unfinished_full_sync_.size(); @@ -1038,6 +1047,7 @@ class PikaConf : public pstd::BaseConf { int force_compact_min_delete_ratio_; int dont_compact_sst_created_in_seconds_; int best_delete_min_ratio_; + int obd_compact_interval_; CompactionStrategy compaction_strategy_; int64_t resume_check_interval_ = 60; // seconds diff --git a/include/pika_server.h b/include/pika_server.h index df75229188..0bc3fc4e67 100644 --- a/include/pika_server.h +++ b/include/pika_server.h @@ -550,6 +550,7 @@ class PikaServer : public pstd::noncopyable { */ bool have_scheduled_crontask_ = false; struct timeval last_check_compact_time_; + struct timeval last_strategy_compact_time_; /* * ResumeDB used diff --git a/src/pika_admin.cc b/src/pika_admin.cc index 3275e6cb7b..7e3dd548eb 100644 --- a/src/pika_admin.cc +++ b/src/pika_admin.cc @@ -2308,6 +2308,12 @@ void ConfigCmd::ConfigGet(std::string& ret) { EncodeNumber(&config_body, g_pika_conf->db_statistics_level()); } + if (pstd::stringmatch(pattern.data(), "obd-compact-interval", 1)) { + elements += 2; + EncodeString(&config_body, "obd-compact-interval"); + EncodeNumber(&config_body, g_pika_conf->obd_compact_interval()); + } + std::stringstream resp; resp << "*" << std::to_string(elements) << "\r\n" << config_body; ret = resp.str(); @@ -2370,6 +2376,7 @@ void ConfigCmd::ConfigSet(std::shared_ptr db) { "zset-cache-field-num-per-key", "cache-lfu-decay-time", "max-conn-rbuf-size", + "obd-compact-interval", }); res_.AppendStringVector(replyVt); return; @@ -3092,6 +3099,13 @@ void ConfigCmd::ConfigSet(std::shared_ptr db) { } g_pika_conf->SetMaxConnRbufSize(static_cast(ival)); res_.AppendStringRaw("+OK\r\n"); + } else if (set_item == "obd-compact-interval") { + if (pstd::string2int(value.data(), value.size(), &ival) == 0 || ival < 0) { + res_.AppendStringRaw("-ERR Invalid argument \'" + value + "\' for CONFIG SET 'obd-compact-interval'\r\n"); + return; + } + g_pika_conf->SetObdCompactInterval(ival); + res_.AppendStringRaw("+OK\r\n"); } else { res_.AppendStringRaw("-ERR Unsupported CONFIG parameter: " + set_item + "\r\n"); } diff --git a/src/pika_conf.cc b/src/pika_conf.cc index 0469a9d498..7d4ee9fa2c 100644 --- a/src/pika_conf.cc +++ b/src/pika_conf.cc @@ -334,6 +334,11 @@ int PikaConf::Load() { if (best_delete_min_ratio_ < 10) { best_delete_min_ratio_ = 10; } + + GetConfInt("obd-compact-interval", &obd_compact_interval_); + if (obd_compact_interval_ < 600) { + obd_compact_interval_ = 600; + } std::string cs_; GetConfStr("compaction-strategy", &cs_); @@ -882,7 +887,10 @@ int PikaConf::ConfigRewrite() { if (best_delete_min_ratio_ < 10) { best_delete_min_ratio_ = 10; } - + SetConfInt("obd-compact-interval", obd_compact_interval_); + if (obd_compact_interval_ < 600) { + obd_compact_interval_ = 600; + } std::string cs_; SetConfStr("compaction-strategy", cs_); if (cs_ == "full-compact") { diff --git a/src/pika_server.cc b/src/pika_server.cc index b205f3e34b..2a6efae563 100644 --- a/src/pika_server.cc +++ b/src/pika_server.cc @@ -46,6 +46,7 @@ PikaServer::PikaServer() : exit_(false), slow_cmd_thread_pool_flag_(g_pika_conf->slow_cmd_pool()), last_check_compact_time_({0, 0}), + last_strategy_compact_time_({0, 0}), last_check_resume_time_({0, 0}), repl_state_(PIKA_REPL_NO_CONNECT), role_(PIKA_ROLE_SINGLE) { @@ -1230,10 +1231,19 @@ void PikaServer::AutoCompactRange() { } } - if (g_pika_conf->compaction_strategy() == PikaConf::FullCompact) { - DoSameThingEveryDB(TaskType::kCompactAll); - } else if (g_pika_conf->compaction_strategy() == PikaConf::OldestOrBestDeleteRatioSstCompact) { - DoSameThingEveryDB(TaskType::kCompactOldestOrBestDeleteRatioSst); + + struct timeval strategy_now; + gettimeofday(&strategy_now, nullptr); + if (last_strategy_compact_time_.tv_sec == 0){ + gettimeofday(&last_strategy_compact_time_, nullptr); + } + if (last_strategy_compact_time_.tv_sec > 0 && + strategy_now.tv_sec - last_strategy_compact_time_.tv_sec >= g_pika_conf->obd_compact_interval()) { + gettimeofday(&last_strategy_compact_time_, nullptr); + if (g_pika_conf->compaction_strategy() == PikaConf::OldestOrBestDeleteRatioSstCompact) { + LOG(INFO) << "[Compaction]schedule OldestOrBestDeleteRatioSstCompact, interval: " << g_pika_conf->obd_compact_interval() << " seconds"; + DoSameThingEveryDB(TaskType::kCompactOldestOrBestDeleteRatioSst); + } } } @@ -1500,6 +1510,7 @@ void PikaServer::InitStorageOptions() { // For Storage compaction storage_options_.compact_param_.best_delete_min_ratio_ = g_pika_conf->best_delete_min_ratio(); + storage_options_.compact_param_.obd_compact_interval_ = g_pika_conf->obd_compact_interval(); storage_options_.compact_param_.dont_compact_sst_created_in_seconds_ = g_pika_conf->dont_compact_sst_created_in_seconds(); storage_options_.compact_param_.force_compact_file_age_seconds_ = g_pika_conf->force_compact_file_age_seconds(); storage_options_.compact_param_.force_compact_min_delete_ratio_ = g_pika_conf->force_compact_min_delete_ratio(); diff --git a/src/storage/include/storage/storage.h b/src/storage/include/storage/storage.h index dd41b3ea94..dba4c4e06a 100644 --- a/src/storage/include/storage/storage.h +++ b/src/storage/include/storage/storage.h @@ -81,6 +81,7 @@ struct StorageOptions { int force_compact_min_delete_ratio_; int dont_compact_sst_created_in_seconds_; int best_delete_min_ratio_; + int obd_compact_interval_; }; CompactParam compact_param_; Status ResetOptions(const OptionType& option_type, const std::unordered_map& options_map);