Skip to content

Commit 2eef794

Browse files
yiguoleiyiguolei
authored
[be](refactor)refactor doris BE code structure (apache#61107)
### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test <!-- At least one of them must be included. --> - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason <!-- Add your reason? --> - Behavior changed: - [ ] No. - [ ] Yes. <!-- Explain the behavior change --> - Does this need documentation? - [ ] No. - [ ] Yes. <!-- Add document PR link here. eg: apache/doris-website#1214 --> ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->
1 parent 10b052f commit 2eef794

File tree

3,228 files changed

+16745
-16911
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

3,228 files changed

+16745
-16911
lines changed

‎be/CMakeLists.txt‎

Lines changed: 13 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -529,21 +529,19 @@ set(DORIS_LINK_LIBS
529529
${WL_START_GROUP}
530530
Agent
531531
Common
532+
Core
532533
Exec
533534
Exprs
535+
Format
534536
IO
535-
Olap
537+
Storage
536538
Runtime
537-
RuntimeFilter
538539
Service
539540
Udf
540541
Util
541542
DorisGen
542-
Webserver
543-
Geo
544-
GeoType
545-
Vec
546-
Pipeline
543+
Load
544+
InformationSchema
547545
Cloud
548546
${WL_END_GROUP}
549547
CommonCPP
@@ -953,22 +951,24 @@ endfunction(pch_reuse target)
953951

954952
add_subdirectory(${SRC_DIR}/agent)
955953
add_subdirectory(${SRC_DIR}/common)
954+
add_subdirectory(${SRC_DIR}/core)
956955
add_subdirectory(${SRC_DIR}/exec)
957956
add_subdirectory(${SRC_DIR}/exprs)
957+
add_subdirectory(${SRC_DIR}/format)
958958
add_subdirectory(${SRC_DIR}/gen_cpp)
959-
add_subdirectory(${SRC_DIR}/geo)
960-
add_subdirectory(${SRC_DIR}/http)
961959
add_subdirectory(${SRC_DIR}/io)
962-
add_subdirectory(${SRC_DIR}/olap/rowset/segment_v2/ann_index)
963-
add_subdirectory(${SRC_DIR}/olap)
960+
add_subdirectory(${SRC_DIR}/storage/index/ann)
961+
add_subdirectory(${SRC_DIR}/storage)
964962
add_subdirectory(${SRC_DIR}/runtime)
965-
add_subdirectory(${SRC_DIR}/runtime_filter)
966963
add_subdirectory(${SRC_DIR}/service) # this include doris_be
967964
add_subdirectory(${SRC_DIR}/udf)
968965
add_subdirectory(${SRC_DIR}/cloud)
966+
add_subdirectory(${SRC_DIR}/load)
967+
add_subdirectory(${SRC_DIR}/information_schema)
969968

970969
option(BUILD_META_TOOL "Build meta tool" OFF)
971-
if (BUILD_META_TOOL)
970+
option(BUILD_INDEX_TOOL "Build index tool" OFF)
971+
if (BUILD_META_TOOL OR BUILD_INDEX_TOOL)
972972
add_subdirectory(${SRC_DIR}/tools)
973973
endif()
974974

@@ -983,14 +983,7 @@ if (BUILD_FILE_CACHE_MICROBENCH_TOOL)
983983
DESTINATION ${OUTPUT_DIR}/bin)
984984
endif()
985985

986-
option(BUILD_INDEX_TOOL "Build index tool" OFF)
987-
if (BUILD_INDEX_TOOL)
988-
add_subdirectory(${SRC_DIR}/index-tools)
989-
endif()
990-
991986
add_subdirectory(${SRC_DIR}/util)
992-
add_subdirectory(${SRC_DIR}/vec)
993-
add_subdirectory(${SRC_DIR}/pipeline)
994987

995988
# this include doris_be_test
996989
if (MAKE_TEST)

‎be/benchmark/benchmark_plain_text_line_reader.hpp‎

Lines changed: 62 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -16,88 +16,98 @@
1616
// under the License.
1717

1818
#include <benchmark/benchmark.h>
19+
1920
#include <string>
2021
#include <vector>
2122

2223
#include "vec/exec/format/file_reader/new_plain_text_line_reader.h"
2324

2425
namespace doris {
2526

26-
static std::string create_test_data(size_t length, const std::string& delimiter = "", char fill_char = 'a') {
27+
static std::string create_test_data(size_t length, const std::string& delimiter = "",
28+
char fill_char = 'a') {
2729
return std::string(length, fill_char) + delimiter;
2830
}
2931

3032
static void BM_FindLfCrlfLineSep(benchmark::State& state) {
3133
size_t data_size = state.range(0);
3234
size_t delimiter_type = state.range(1);
33-
35+
3436
std::string test_data;
35-
switch(delimiter_type) {
36-
case 0: // No delimiter
37-
test_data = create_test_data(data_size);
38-
break;
39-
case 1: // Delimiter is \n
40-
test_data = create_test_data(data_size, "\n");
41-
break;
42-
case 2: // Delimiter is \r\n
43-
test_data = create_test_data(data_size, "\r\n");
44-
break;
45-
default:
46-
test_data = create_test_data(data_size);
47-
break;
37+
switch (delimiter_type) {
38+
case 0: // No delimiter
39+
test_data = create_test_data(data_size);
40+
break;
41+
case 1: // Delimiter is \n
42+
test_data = create_test_data(data_size, "\n");
43+
break;
44+
case 2: // Delimiter is \r\n
45+
test_data = create_test_data(data_size, "\r\n");
46+
break;
47+
default:
48+
test_data = create_test_data(data_size);
49+
break;
4850
}
4951

5052
PlainTextLineReaderCtx ctx("\n", 1, false);
5153
const auto* data = reinterpret_cast<const uint8_t*>(test_data.c_str());
5254
const size_t size = test_data.size();
53-
55+
5456
for (auto _ : state) {
5557
const auto* result = ctx.find_lf_crlf_line_sep(data, size);
5658
benchmark::DoNotOptimize(result);
5759
}
58-
60+
5961
state.SetBytesProcessed(state.iterations() * test_data.size());
60-
62+
6163
std::string label = "size_" + std::to_string(data_size);
6264
switch (delimiter_type) {
63-
case 0: label += "_delim_no"; break;
64-
case 1: label += "_delim_lf"; break;
65-
case 2: label += "_delim_crlf"; break;
66-
default: label += "_delim_no"; break;
65+
case 0:
66+
label += "_delim_no";
67+
break;
68+
case 1:
69+
label += "_delim_lf";
70+
break;
71+
case 2:
72+
label += "_delim_crlf";
73+
break;
74+
default:
75+
label += "_delim_no";
76+
break;
6777
}
6878
state.SetLabel(label);
6979
}
7080

7181
BENCHMARK(BM_FindLfCrlfLineSep)
72-
->Unit(benchmark::kNanosecond)
73-
->Args({16, 0}) // 16 bytes, no delimiter
74-
->Args({16, 1}) // 16 bytes, delimiter is \n
75-
->Args({16, 2}) // 16 bytes, delimiter is \r\n
76-
->Args({32, 0}) // 32 bytes, no delimiter
77-
->Args({32, 1}) // 32 bytes, delimiter is \n
78-
->Args({32, 2}) // 32 bytes, delimiter is \r\n
79-
->Args({64, 0}) // 64 bytes, no delimiter
80-
->Args({64, 1}) // 64 bytes, delimiter is \n
81-
->Args({64, 2}) // 64 bytes, delimiter is \r\n
82-
->Args({128, 0}) // 128 bytes, no delimiter
83-
->Args({128, 1}) // 128 bytes, delimiter is \n
84-
->Args({128, 2}) // 128 bytes, delimiter is \r\n
85-
->Args({256, 0}) // 256 bytes, no delimiter
86-
->Args({256, 1}) // 256 bytes, delimiter is \n
87-
->Args({256, 2}) // 256 bytes, delimiter is \r\n
88-
->Args({512, 0}) // 512 bytes, no delimiter
89-
->Args({512, 1}) // 512 bytes, delimiter is \n
90-
->Args({512, 2}) // 512 bytes, delimiter is \r\n
91-
->Args({1024, 0}) // 1KB, no delimiter
92-
->Args({1024, 1}) // 1KB, delimiter is \n
93-
->Args({1024, 2}) // 1KB, delimiter is \r\n
94-
->Args({64 * 1024, 0}) // 64KB, no delimiter
95-
->Args({64 * 1024, 1}) // 64KB, delimiter is \n
96-
->Args({64 * 1024, 2}) // 64KB, delimiter is \r\n
97-
->Args({1024 * 1024, 0}) // 1MB, no delimiter
98-
->Args({1024 * 1024, 1}) // 1MB, delimiter is \n
99-
->Args({1024 * 1024, 2}) // 1MB, delimiter is \r\n
100-
->Repetitions(5)
101-
->DisplayAggregatesOnly();
82+
->Unit(benchmark::kNanosecond)
83+
->Args({16, 0}) // 16 bytes, no delimiter
84+
->Args({16, 1}) // 16 bytes, delimiter is \n
85+
->Args({16, 2}) // 16 bytes, delimiter is \r\n
86+
->Args({32, 0}) // 32 bytes, no delimiter
87+
->Args({32, 1}) // 32 bytes, delimiter is \n
88+
->Args({32, 2}) // 32 bytes, delimiter is \r\n
89+
->Args({64, 0}) // 64 bytes, no delimiter
90+
->Args({64, 1}) // 64 bytes, delimiter is \n
91+
->Args({64, 2}) // 64 bytes, delimiter is \r\n
92+
->Args({128, 0}) // 128 bytes, no delimiter
93+
->Args({128, 1}) // 128 bytes, delimiter is \n
94+
->Args({128, 2}) // 128 bytes, delimiter is \r\n
95+
->Args({256, 0}) // 256 bytes, no delimiter
96+
->Args({256, 1}) // 256 bytes, delimiter is \n
97+
->Args({256, 2}) // 256 bytes, delimiter is \r\n
98+
->Args({512, 0}) // 512 bytes, no delimiter
99+
->Args({512, 1}) // 512 bytes, delimiter is \n
100+
->Args({512, 2}) // 512 bytes, delimiter is \r\n
101+
->Args({1024, 0}) // 1KB, no delimiter
102+
->Args({1024, 1}) // 1KB, delimiter is \n
103+
->Args({1024, 2}) // 1KB, delimiter is \r\n
104+
->Args({64 * 1024, 0}) // 64KB, no delimiter
105+
->Args({64 * 1024, 1}) // 64KB, delimiter is \n
106+
->Args({64 * 1024, 2}) // 64KB, delimiter is \r\n
107+
->Args({1024 * 1024, 0}) // 1MB, no delimiter
108+
->Args({1024 * 1024, 1}) // 1MB, delimiter is \n
109+
->Args({1024 * 1024, 2}) // 1MB, delimiter is \r\n
110+
->Repetitions(5)
111+
->DisplayAggregatesOnly();
102112

103113
} // namespace doris

‎be/src/agent/agent_server.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,11 @@
3737
#include "common/config.h"
3838
#include "common/logging.h"
3939
#include "common/status.h"
40-
#include "olap/olap_define.h"
41-
#include "olap/options.h"
42-
#include "olap/snapshot_manager.h"
43-
#include "olap/storage_engine.h"
4440
#include "runtime/exec_env.h"
41+
#include "storage/olap_define.h"
42+
#include "storage/options.h"
43+
#include "storage/snapshot/snapshot_manager.h"
44+
#include "storage/storage_engine.h"
4545
#include "util/work_thread_pool.hpp"
4646

4747
namespace doris {

‎be/src/agent/heartbeat_server.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,12 @@
3232
#include "cloud/config.h"
3333
#include "common/config.h"
3434
#include "common/status.h"
35-
#include "olap/storage_engine.h"
3635
#include "runtime/cluster_info.h"
3736
#include "runtime/exec_env.h"
3837
#include "runtime/fragment_mgr.h"
3938
#include "runtime/heartbeat_flags.h"
4039
#include "service/backend_options.h"
40+
#include "storage/storage_engine.h"
4141
#include "util/debug_util.h"
4242
#include "util/mem_info.h"
4343
#include "util/network_util.h"

‎be/src/agent/task_worker_pool.cpp‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
#include "cloud/config.h"
5656
#include "common/config.h"
5757
#include "common/logging.h"
58+
#include "common/metrics/doris_metrics.h"
5859
#include "common/status.h"
5960
#include "io/fs/file_system.h"
6061
#include "io/fs/hdfs_file_system.h"
@@ -63,36 +64,35 @@
6364
#include "io/fs/path.h"
6465
#include "io/fs/remote_file_system.h"
6566
#include "io/fs/s3_file_system.h"
66-
#include "olap/cumulative_compaction_time_series_policy.h"
67-
#include "olap/data_dir.h"
68-
#include "olap/olap_common.h"
69-
#include "olap/rowset/rowset_meta.h"
70-
#include "olap/snapshot_manager.h"
71-
#include "olap/storage_engine.h"
72-
#include "olap/storage_policy.h"
73-
#include "olap/tablet.h"
74-
#include "olap/tablet_manager.h"
75-
#include "olap/tablet_meta.h"
76-
#include "olap/tablet_schema.h"
77-
#include "olap/task/engine_batch_load_task.h"
78-
#include "olap/task/engine_checksum_task.h"
79-
#include "olap/task/engine_clone_task.h"
80-
#include "olap/task/engine_cloud_index_change_task.h"
81-
#include "olap/task/engine_index_change_task.h"
82-
#include "olap/task/engine_publish_version_task.h"
83-
#include "olap/task/engine_storage_migration_task.h"
84-
#include "olap/txn_manager.h"
85-
#include "olap/utils.h"
8667
#include "runtime/exec_env.h"
8768
#include "runtime/fragment_mgr.h"
8869
#include "runtime/index_policy/index_policy_mgr.h"
8970
#include "runtime/memory/global_memory_arbitrator.h"
9071
#include "runtime/snapshot_loader.h"
9172
#include "runtime/user_function_cache.h"
9273
#include "service/backend_options.h"
74+
#include "storage/compaction/cumulative_compaction_time_series_policy.h"
75+
#include "storage/data_dir.h"
76+
#include "storage/olap_common.h"
77+
#include "storage/rowset/rowset_meta.h"
78+
#include "storage/snapshot/snapshot_manager.h"
79+
#include "storage/storage_engine.h"
80+
#include "storage/storage_policy.h"
81+
#include "storage/tablet/tablet.h"
82+
#include "storage/tablet/tablet_manager.h"
83+
#include "storage/tablet/tablet_meta.h"
84+
#include "storage/tablet/tablet_schema.h"
85+
#include "storage/task/engine_batch_load_task.h"
86+
#include "storage/task/engine_checksum_task.h"
87+
#include "storage/task/engine_clone_task.h"
88+
#include "storage/task/engine_cloud_index_change_task.h"
89+
#include "storage/task/engine_index_change_task.h"
90+
#include "storage/task/engine_publish_version_task.h"
91+
#include "storage/task/engine_storage_migration_task.h"
92+
#include "storage/txn/txn_manager.h"
93+
#include "storage/utils.h"
9394
#include "util/brpc_client_cache.h"
9495
#include "util/debug_points.h"
95-
#include "util/doris_metrics.h"
9696
#include "util/jni-util.h"
9797
#include "util/mem_info.h"
9898
#include "util/random.h"

‎be/src/agent/utils.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@
4242

4343
#include "common/config.h"
4444
#include "common/status.h"
45-
#include "runtime/client_cache.h"
4645
#include "runtime/cluster_info.h"
46+
#include "util/client_cache.h"
4747

4848
namespace doris {
4949
class TConfirmUnusedRemoteFilesRequest;

‎be/src/agent/utils.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
#include <string>
2424

2525
#include "common/status.h"
26-
#include "runtime/client_cache.h"
26+
#include "util/client_cache.h"
2727

2828
namespace doris {
2929
class TConfirmUnusedRemoteFilesRequest;

‎be/src/cloud/cloud_backend_service.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@
2929
#include "common/status.h"
3030
#include "io/cache/block_file_cache_downloader.h"
3131
#include "io/cache/block_file_cache_factory.h"
32-
#include "runtime/stream_load/stream_load_context.h"
33-
#include "runtime/stream_load/stream_load_recorder.h"
32+
#include "load/stream_load/stream_load_context.h"
33+
#include "load/stream_load/stream_load_recorder.h"
3434
#include "util/brpc_client_cache.h" // BrpcClientCache
3535
#include "util/stack_util.h"
3636
#include "util/thrift_server.h"

‎be/src/cloud/cloud_base_compaction.cpp‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,20 @@
1717

1818
#include "cloud/cloud_base_compaction.h"
1919

20+
#include <gen_cpp/cloud.pb.h>
21+
2022
#include <boost/container_hash/hash.hpp>
2123

2224
#include "cloud/cloud_meta_mgr.h"
2325
#include "cloud/config.h"
2426
#include "common/config.h"
27+
#include "core/value/vdatetime_value.h"
2528
#include "cpp/sync_point.h"
26-
#include "gen_cpp/cloud.pb.h"
27-
#include "olap/compaction.h"
28-
#include "olap/task/engine_checksum_task.h"
2929
#include "service/backend_options.h"
30+
#include "storage/compaction/compaction.h"
31+
#include "storage/task/engine_checksum_task.h"
3032
#include "util/thread.h"
3133
#include "util/uuid_generator.h"
32-
#include "vec/runtime/vdatetime_value.h"
3334

3435
namespace doris {
3536
using namespace ErrorCode;

‎be/src/cloud/cloud_base_compaction.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121

2222
#include "cloud/cloud_storage_engine.h"
2323
#include "cloud/cloud_tablet.h"
24-
#include "olap/compaction.h"
24+
#include "storage/compaction/compaction.h"
2525

2626
namespace doris {
2727

0 commit comments

Comments
 (0)