Skip to content

Commit 065dcc4

Browse files
airborne12liaoxin01
authored andcommitted
[refact](inverted index) refact compound idx writer (apache#59219)
### What problem does this PR solve? This PR refactors the compound index writer for the inverted index V2 format to optimize performance, particularly for cloud storage systems like S3. The main improvement is bypassing unnecessary directory operations by directly creating the output stream using an existing file writer. Key changes: Added a static factory method FSIndexOutputV2::create() for direct instantiation without requiring a Directory object Refactored IndexStorageFormatV2::create_output_stream() to return only an IndexOutput instead of a Directory-IndexOutput pair Removed the cloud mode check in DorisFSDirectoryFactory::getDirectory() to simplify code (directory creation is safe for all file systems)
1 parent dd3b3ef commit 065dcc4

File tree

1 file changed

+19
-22
lines changed

1 file changed

+19
-22
lines changed

be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp

Lines changed: 19 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -925,28 +925,25 @@ DorisFSDirectory* DorisFSDirectoryFactory::getDirectory(const io::FileSystemSPtr
925925
if (config::inverted_index_ram_dir_enable && can_use_ram_dir) {
926926
dir = _CLNEW DorisRAMFSDirectory();
927927
} else {
928-
// cloud mode does not need to create directory
929-
if (!config::is_cloud_mode()) {
930-
bool exists = false;
931-
auto st = _fs->exists(file, &exists);
932-
DBUG_EXECUTE_IF("DorisFSDirectoryFactory::getDirectory_exists_status_is_not_ok", {
933-
st = Status::Error<ErrorCode::INTERNAL_ERROR>(
934-
"debug point: "
935-
"DorisFSDirectoryFactory::getDirectory_exists_status_is_not_ok");
936-
})
937-
LOG_AND_THROW_IF_ERROR(st, "Get directory exists IO error");
938-
if (!exists) {
939-
st = _fs->create_directory(file);
940-
DBUG_EXECUTE_IF(
941-
"DorisFSDirectoryFactory::getDirectory_create_directory_status_is_not_ok", {
942-
st = Status::Error<ErrorCode::INTERNAL_ERROR>(
943-
"debug point: "
944-
"DorisFSDirectoryFactory::getDirectory_create_directory_status_"
945-
"is_"
946-
"not_ok");
947-
})
948-
LOG_AND_THROW_IF_ERROR(st, "Get directory create directory IO error");
949-
}
928+
bool exists = false;
929+
auto st = _fs->exists(file, &exists);
930+
DBUG_EXECUTE_IF("DorisFSDirectoryFactory::getDirectory_exists_status_is_not_ok", {
931+
st = Status::Error<ErrorCode::INTERNAL_ERROR>(
932+
"debug point: "
933+
"DorisFSDirectoryFactory::getDirectory_exists_status_is_not_ok");
934+
})
935+
LOG_AND_THROW_IF_ERROR(st, "Get directory exists IO error");
936+
if (!exists) {
937+
st = _fs->create_directory(file);
938+
DBUG_EXECUTE_IF(
939+
"DorisFSDirectoryFactory::getDirectory_create_directory_status_is_not_ok", {
940+
st = Status::Error<ErrorCode::INTERNAL_ERROR>(
941+
"debug point: "
942+
"DorisFSDirectoryFactory::getDirectory_create_directory_status_"
943+
"is_"
944+
"not_ok");
945+
})
946+
LOG_AND_THROW_IF_ERROR(st, "Get directory create directory IO error");
950947
}
951948
dir = _CLNEW DorisFSDirectory();
952949
}

0 commit comments

Comments
 (0)