Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
05f3001
Test feasibility.
xuchenhao Dec 4, 2025
7bbd855
Implement the basic functionality.
xuchenhao Dec 8, 2025
f66fa1b
Add some descriptions.
xuchenhao Dec 8, 2025
c105ac6
Add FileCacheAdmissionManager.java.
xuchenhao Dec 8, 2025
ea95ae3
Optimize code logic.
xuchenhao Dec 9, 2025
ff250e3
Update the order of rule matching.
xuchenhao Dec 11, 2025
8661457
Update the default value of file_cache_admission.
xuchenhao Dec 11, 2025
e5198b8
Update the level of LOG.
xuchenhao Dec 11, 2025
9acf780
Merge remote-tracking branch 'refs/remotes/origin/master' into file_c…
xuchenhao Dec 15, 2025
c9e4353
Updat the rule matching order.
xuchenhao Dec 15, 2025
613dab5
Add ut cases.
xuchenhao Dec 15, 2025
4ef6f8f
Update the measurement of cost.
xuchenhao Dec 16, 2025
54689a2
Merge branch 'master' into file_cache_admission_control
xuchenhao Dec 17, 2025
f31901d
Merge branch 'master' into file_cache_admission_control
xuchenhao Dec 17, 2025
e8fc91b
Merge branch 'master' into file_cache_admission_control
xuchenhao Dec 17, 2025
bb3d217
Merge branch 'master' into file_cache_admission_control
xuchenhao Dec 17, 2025
7d7b9ea
Merge branch 'master' into file_cache_admission_control
xuchenhao Dec 17, 2025
12de857
Merge branch 'master' into file_cache_admission_control
xuchenhao Dec 18, 2025
bb5c886
Update pom.xml
xuchenhao Dec 18, 2025
8d27a8e
Store admission rules in JSON files.
xuchenhao Dec 23, 2025
6022fa8
Update FileCacheAdmissionManager.
xuchenhao Dec 24, 2025
6f9d4ec
Update export_mysql_rule_to_json.sh.
xuchenhao Dec 24, 2025
d49d833
Add license.
xuchenhao Dec 24, 2025
17f688e
Merge branch 'master' into file_cache_admission_control
xuchenhao Dec 25, 2025
9bdf111
Update FileCacheAdmissionManager.java
xuchenhao Dec 25, 2025
8849330
Update pom.xml
xuchenhao Dec 25, 2025
cea5272
Merge branch 'master' into file_cache_admission_control
xuchenhao Dec 25, 2025
4a959e6
Update FileCacheAdmissionManager.java
xuchenhao Dec 26, 2025
8368506
Update export_mysql_rule_to_json.sh
xuchenhao Dec 26, 2025
42de0ee
Enhanced compatibility.
xuchenhao Jan 5, 2026
a795d77
Merge branch 'master' into file_cache_admission_control
xuchenhao Jan 6, 2026
cd0acbc
Merge branch 'master' into file_cache_admission_control
xuchenhao Jan 6, 2026
65665f9
Merge branch 'master' into file_cache_admission_control
xuchenhao Jan 6, 2026
2fdb764
Refactored variable names.
xuchenhao Jan 8, 2026
1289199
Merge branch 'master' into file_cache_admission_control
xuchenhao Jan 8, 2026
1262552
Merge branch 'master' into file_cache_admission_control
xuchenhao Jan 9, 2026
786c2e2
Merge branch 'master' into file_cache_admission_control
xuchenhao Jan 9, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion be/src/io/file_factory.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ io::FileReaderOptions FileFactory::get_reader_options(RuntimeState* state,
};
if (config::enable_file_cache && state != nullptr &&
state->query_options().__isset.enable_file_cache &&
state->query_options().enable_file_cache) {
state->query_options().enable_file_cache && fd.file_cache_admission) {
opts.cache_type = io::FileCachePolicy::FILE_BLOCK_CACHE;
}
if (state != nullptr && state->query_options().__isset.file_cache_base_path &&
Expand Down
1 change: 1 addition & 0 deletions be/src/io/file_factory.h
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ struct FileDescription {
// because for a hive table, differenet partitions may have different
// locations(or fs), so different files may have different fs.
std::string fs_name;
bool file_cache_admission = true;
};

} // namespace io
Expand Down
1 change: 1 addition & 0 deletions be/src/vec/exec/format/csv/csv_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,7 @@ void CsvReader::_init_file_description() {
if (_range.__isset.fs_name) {
_file_description.fs_name = _range.fs_name;
}
_file_description.file_cache_admission = _range.file_cache_admission;
}

Status CsvReader::init_reader(bool is_load) {
Expand Down
1 change: 1 addition & 0 deletions be/src/vec/exec/format/json/new_json_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@ void NewJsonReader::_init_file_description() {
if (_range.__isset.fs_name) {
_file_description.fs_name = _range.fs_name;
}
_file_description.file_cache_admission = _range.file_cache_admission;
}

Status NewJsonReader::init_reader(
Expand Down
1 change: 1 addition & 0 deletions be/src/vec/exec/format/orc/vorc_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1437,6 +1437,7 @@ void OrcReader::_init_file_description() {
if (_scan_range.__isset.fs_name) {
_file_description.fs_name = _scan_range.fs_name;
}
_file_description.file_cache_admission = _scan_range.file_cache_admission;
}

DataTypePtr OrcReader::convert_to_doris_type(const orc::Type* orc_type) {
Expand Down
1 change: 1 addition & 0 deletions be/src/vec/exec/format/parquet/vparquet_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,7 @@ void ParquetReader::_init_file_description() {
if (_scan_range.__isset.fs_name) {
_file_description.fs_name = _scan_range.fs_name;
}
_file_description.file_cache_admission = _scan_range.file_cache_admission;
}

Status ParquetReader::init_reader(
Expand Down
24 changes: 24 additions & 0 deletions fe/fe-common/src/main/java/org/apache/doris/common/Config.java
Original file line number Diff line number Diff line change
Expand Up @@ -3750,6 +3750,30 @@ public static int metaServiceRpcRetryTimes() {
})
public static int first_error_msg_max_length = 256;

@ConfField(mutable = false, description = {
"是否启用文件缓存准入控制",
"Whether to enable file cache admission control"
})
public static boolean enable_file_cache_admission_control = false;

@ConfField(mutable = true, description = {
"存储准入规则的JSON文件路径",
"JSON file path for storing admission rules"
})
public static String file_cache_admission_control_json_file_path = "";

@ConfField(mutable = true, description = {
"准入规则自动刷新间隔(秒),设置为0关闭自动刷新",
"Auto-refresh interval for admission policies (seconds). Set to 0 to disable auto refresh"
})
public static int file_cache_admission_control_fresh_interval_s = 300;

@ConfField(mutable = true, description = {
"当没有匹配的准入规则时的默认行为:true=允许,false=拒绝",
"Default behavior when no matching admission rule is found: true=allow, false=deny"
})
public static boolean file_cache_admission_control_default_allow = false;

@ConfField
public static String cloud_snapshot_handler_class = "org.apache.doris.cloud.snapshot.CloudSnapshotHandler";
@ConfField
Expand Down
9 changes: 9 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/DorisFE.java
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.apache.doris.common.util.JdkUtils;
import org.apache.doris.common.util.NetUtils;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.FileCacheAdmissionManager;
import org.apache.doris.httpv2.HttpServer;
import org.apache.doris.journal.bdbje.BDBDebugger;
import org.apache.doris.journal.bdbje.BDBTool;
Expand Down Expand Up @@ -160,6 +161,10 @@ public static void start(String dorisHomeDir, String pidDir, String[] args, Star
serverReady.set(false);
gracefulShutdown();

if (Config.enable_file_cache_admission_control) {
FileCacheAdmissionManager.getInstance().shutdown();
}

// Shutdown HTTP server after main process graceful shutdown is complete
if (httpServer != null) {
httpServer.shutdown();
Expand Down Expand Up @@ -221,6 +226,10 @@ public static void start(String dorisHomeDir, String pidDir, String[] args, Star
Env.getCurrentEnv().initialize(args);
Env.getCurrentEnv().waitForReady();

if (Config.enable_file_cache_admission_control) {
FileCacheAdmissionManager.getInstance().loadOnStartup();
}

// init and start:
// 1. HttpServer for HTTP Server
// 2. FeServer for Thrift Server
Expand Down
Loading