Skip to content

Commit ce7938f

Browse files
xy720xy720
committed
save
1 parent e52956b commit ce7938f

File tree

9 files changed

+439
-1
lines changed

9 files changed

+439
-1
lines changed

‎be/src/cloud/cloud_snapshot_mgr.cpp‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,9 @@ Status CloudSnapshotMgr::convert_rowsets(
145145
cooldown_meta_id->set_hi(0);
146146
cooldown_meta_id->set_lo(0);
147147

148+
TabletSchemaSPtr target_tablet_schema = std::make_shared<TabletSchema>();
149+
target_tablet_schema->copy_from(*target_tablet->tablet_schema());
150+
148151
TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
149152
tablet_schema->init_from_pb(in.schema());
150153

@@ -155,10 +158,17 @@ Status CloudSnapshotMgr::convert_rowsets(
155158
RETURN_IF_ERROR(_create_rowset_meta(new_rowset_meta_pb, rowset_meta_pb, tablet_id,
156159
target_tablet, storage_resource, tablet_schema,
157160
file_mapping, rowset_id_mapping));
161+
if (new_rowset_meta_pb->has_tablet_schema() && new_rowset_meta_pb->tablet_schema().index_size() > 0) {
162+
RETURN_IF_ERROR(_rename_index_ids(*new_rowset_meta_pb->mutable_tablet_schema(), target_tablet_schema));
163+
}
158164
Version rowset_version = {rowset_meta_pb.start_version(), rowset_meta_pb.end_version()};
159165
rs_version_map[rowset_version] = new_rowset_meta_pb;
160166
}
161167

168+
if (out->schema().index_size() > 0) {
169+
RETURN_IF_ERROR(_rename_index_ids(*out->mutable_schema(), target_tablet_schema));
170+
}
171+
162172
if (!rowset_id_mapping.empty() && in.has_delete_bitmap()) {
163173
const auto& old_del_bitmap_pb = in.delete_bitmap();
164174
DeleteBitmapPB* new_del_bitmap_pb = out->mutable_delete_bitmap();
@@ -178,6 +188,7 @@ Status CloudSnapshotMgr::convert_rowsets(
178188
new_del_bitmap_pb->set_rowset_ids(cast_set<int>(i), it->second.to_string());
179189
}
180190
}
191+
181192
return Status::OK();
182193
}
183194

@@ -271,5 +282,22 @@ Status CloudSnapshotMgr::_create_rowset_meta(
271282
return Status::OK();
272283
}
273284

285+
Status CloudSnapshotMgr::_rename_index_ids(TabletSchemaPB& schema_pb, const TabletSchemaSPtr& tablet_schema) const {
286+
for (int i = 0; i < schema_pb.index_size(); ++i) {
287+
TabletIndexPB* index_pb = schema_pb.mutable_index(i);
288+
for (int32_t col_unique_id : index_pb->col_unique_id()) {
289+
auto local_index = tablet_schema->get_index(col_unique_id, index_pb->index_type(),
290+
index_pb->index_suffix_name());
291+
if (local_index) {
292+
if (index_pb->index_id() != local_index->index_id()) {
293+
index_pb->set_index_id(local_index->index_id());
294+
}
295+
break;
296+
}
297+
}
298+
}
299+
return Status::OK();
300+
}
301+
274302
#include "common/compile_check_end.h"
275303
} // namespace doris

‎be/src/cloud/cloud_snapshot_mgr.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,8 @@ class CloudSnapshotMgr {
9090
std::unordered_map<std::string, std::string>& file_mapping,
9191
std::unordered_map<RowsetId, RowsetId>& rowset_id_mapping);
9292

93+
Status _rename_index_ids(TabletSchemaPB& schema_pb, const TabletSchemaSPtr& tablet_schema) const;
94+
9395
private:
9496
CloudStorageEngine& _engine;
9597
std::atomic<uint64_t> _snapshot_base_id {0};

‎be/src/olap/snapshot_manager.cpp‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,14 @@ Result<std::vector<PendingRowsetGuard>> SnapshotManager::convert_rowset_ids(
183183
"clone dir not existed when convert rowsetids. clone_dir={}", clone_dir));
184184
}
185185

186+
TabletSharedPtr target_tablet = _engine.tablet_manager()->get_tablet(tablet_id);
187+
if (target_tablet == nullptr) {
188+
return unexpected(Status::Error<INVALID_ARGUMENT>("failed to get tablet. tablet={}", tablet_id));
189+
}
190+
191+
TabletSchemaSPtr target_tablet_schema = std::make_shared<TabletSchema>();
192+
target_tablet_schema->copy_from(*target_tablet->tablet_schema());
193+
186194
// load original tablet meta
187195
auto cloned_meta_file = fmt::format("{}/{}.hdr", clone_dir, tablet_id);
188196
TabletMetaPB cloned_tablet_meta_pb;
@@ -241,6 +249,10 @@ Result<std::vector<PendingRowsetGuard>> SnapshotManager::convert_rowset_ids(
241249
rowset_meta->set_partition_id(partition_id);
242250
}
243251

252+
if (rowset_meta->has_tablet_schema() && rowset_meta->tablet_schema().index_size() > 0) {
253+
RETURN_IF_ERROR_RESULT(_rename_index_ids(*rowset_meta->mutable_tablet_schema(), target_tablet_schema));
254+
}
255+
244256
Version rowset_version = {visible_rowset.start_version(), visible_rowset.end_version()};
245257
rs_version_map[rowset_version] = rowset_meta;
246258
}
@@ -277,6 +289,14 @@ Result<std::vector<PendingRowsetGuard>> SnapshotManager::convert_rowset_ids(
277289
if (partition_id != -1) {
278290
rowset_meta->set_partition_id(partition_id);
279291
}
292+
293+
if (rowset_meta->has_tablet_schema() && rowset_meta->tablet_schema().index_size() > 0) {
294+
RETURN_IF_ERROR_RESULT(_rename_index_ids(*rowset_meta->mutable_tablet_schema(), target_tablet_schema));
295+
}
296+
}
297+
298+
if (new_tablet_meta_pb.schema().index_size() > 0) {
299+
RETURN_IF_ERROR_RESULT(_rename_index_ids(*new_tablet_meta_pb.mutable_schema(), target_tablet_schema));
280300
}
281301

282302
if (!rowset_id_mapping.empty() && cloned_tablet_meta_pb.has_delete_bitmap()) {
@@ -346,6 +366,23 @@ Status SnapshotManager::_rename_rowset_id(const RowsetMetaPB& rs_meta_pb,
346366
return Status::OK();
347367
}
348368

369+
Status SnapshotManager::_rename_index_ids(TabletSchemaPB& schema_pb, const TabletSchemaSPtr& tablet_schema) const {
370+
for (int i = 0; i < schema_pb.index_size(); ++i) {
371+
TabletIndexPB* index_pb = schema_pb.mutable_index(i);
372+
for (int32_t col_unique_id : index_pb->col_unique_id()) {
373+
auto local_index = tablet_schema->get_index(col_unique_id, index_pb->index_type(),
374+
index_pb->index_suffix_name());
375+
if (local_index) {
376+
if (index_pb->index_id() != local_index->index_id()) {
377+
index_pb->set_index_id(local_index->index_id());
378+
}
379+
break;
380+
}
381+
}
382+
}
383+
return Status::OK();
384+
}
385+
349386
// get snapshot path: curtime.seq.timeout
350387
// eg: 20190819221234.3.86400
351388
Status SnapshotManager::_calc_snapshot_id_path(const TabletSharedPtr& tablet, int64_t timeout_s,

‎be/src/olap/snapshot_manager.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,8 @@ class SnapshotManager {
133133
TabletSchemaSPtr tablet_schema, const RowsetId& next_id,
134134
RowsetMetaPB* new_rs_meta_pb);
135135

136+
Status _rename_index_ids(TabletSchemaPB& schema_pb, const TabletSchemaSPtr& tablet_schema) const;
137+
136138
StorageEngine& _engine;
137139
std::atomic<uint64_t> _snapshot_base_id {0};
138140

‎be/src/olap/tablet_schema.cpp‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1694,6 +1694,18 @@ const TabletIndex* TabletSchema::get_ngram_bf_index(int32_t col_unique_id) const
16941694
return nullptr;
16951695
}
16961696

1697+
const TabletIndex* TabletSchema::get_index(int32_t col_unique_id, IndexType index_type,
1698+
const std::string& suffix_path) const {
1699+
IndexKey index_key(index_type, col_unique_id, suffix_path);
1700+
auto it = _col_id_suffix_to_index.find(index_key);
1701+
if (it != _col_id_suffix_to_index.end()) {
1702+
if (!it->second.empty() && it->second[0] < _indexes.size()) {
1703+
return _indexes[it->second[0]].get();
1704+
}
1705+
}
1706+
return nullptr;
1707+
}
1708+
16971709
vectorized::Block TabletSchema::create_block(
16981710
const std::vector<uint32_t>& return_columns,
16991711
const std::unordered_set<uint32_t>* tablet_columns_need_convert_null) const {

‎be/src/olap/tablet_schema.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -535,6 +535,7 @@ class TabletSchema : public MetadataAdder<TabletSchema> {
535535

536536
bool has_ngram_bf_index(int32_t col_unique_id) const;
537537
const TabletIndex* get_ngram_bf_index(int32_t col_unique_id) const;
538+
const TabletIndex* get_index(int32_t col_unique_id, IndexType index_type, const std::string& suffix_path) const;
538539
void update_indexes_from_thrift(const std::vector<doris::TOlapTableIndex>& indexes);
539540
// If schema version is not set, it should be -1
540541
int32_t schema_version() const { return _schema_version; }

‎be/test/cloud/cloud_snapshot_mgr_test.cpp‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,38 @@ TEST_F(CloudSnapshotMgrTest, TestConvertRowsets) {
6161
input_meta_pb.set_schema_hash(123456);
6262
*input_meta_pb.mutable_tablet_uid() = TabletUid::gen_uid().to_proto();
6363

64+
TabletSchemaPB* input_schema = input_meta_pb.mutable_schema();
65+
input_schema->set_keys_type(KeysType::DUP_KEYS);
66+
input_schema->set_num_short_key_columns(1);
67+
input_schema->set_num_rows_per_row_block(1024);
68+
input_schema->set_compress_kind(COMPRESS_LZ4);
69+
70+
ColumnPB* col1 = input_schema->add_column();
71+
col1->set_unique_id(1);
72+
col1->set_name("col1");
73+
col1->set_type("INT");
74+
col1->set_is_key(true);
75+
col1->set_aggregation("NONE");
76+
77+
ColumnPB* col2 = input_schema->add_column();
78+
col2->set_unique_id(2);
79+
col2->set_name("col2");
80+
col2->set_type("VARCHAR");
81+
col2->set_is_key(false);
82+
col2->set_aggregation("REPLACE");
83+
84+
doris::TabletIndexPB* index1 = input_schema->add_index();
85+
index1->set_index_id(1001);
86+
index1->set_index_name("test_index1");
87+
index1->set_index_type(IndexType::BITMAP);
88+
index1->add_col_unique_id(2);
89+
90+
doris::TabletIndexPB* index2 = input_schema->add_index();
91+
index2->set_index_id(1002);
92+
index2->set_index_name("test_index2");
93+
index2->set_index_type(IndexType::INVERTED);
94+
index2->add_col_unique_id(2);
95+
6496
RowsetMetaPB* rowset_meta = input_meta_pb.add_rs_metas();
6597
RowsetId rowset_id;
6698
rowset_id.init(10000);
@@ -78,8 +110,21 @@ TEST_F(CloudSnapshotMgrTest, TestConvertRowsets) {
78110
rowset_meta->set_rowset_state(RowsetStatePB::VISIBLE);
79111
rowset_meta->set_newest_write_timestamp(1678901234567890);
80112

113+
TabletSchemaPB* rowset_schema = rowset_meta->mutable_tablet_schema();
114+
rowset_schema->CopyFrom(*input_schema);
115+
81116
auto tablet_meta = std::make_shared<TabletMeta>();
82117
tablet_meta->init_from_pb(input_meta_pb);
118+
119+
TabletSchemaSPtr local_tablet_schema = std::make_shared<TabletSchema>();
120+
local_tablet_schema->init_from_pb(input_meta_pb.schema());
121+
122+
// reset the index id
123+
input_schema->mutable_index(0)->set_index_id(2001);
124+
input_schema->mutable_index(1)->set_index_id(2002);
125+
rowset_meta->mutable_tablet_schema()->mutable_index(0)->set_index_id(2001);
126+
rowset_meta->mutable_tablet_schema()->mutable_index(1)->set_index_id(2002);
127+
83128
CloudTabletSPtr target_tablet = std::make_shared<CloudTablet>(*_engine, tablet_meta);
84129
StorageResource storage_resource {_fs};
85130
std::unordered_map<std::string, std::string> file_mapping;
@@ -92,6 +137,16 @@ TEST_F(CloudSnapshotMgrTest, TestConvertRowsets) {
92137
EXPECT_EQ(output_meta_pb.rs_metas(0).tablet_id(), 3000);
93138
EXPECT_EQ(output_meta_pb.rs_metas(0).rowset_id(), 0);
94139
EXPECT_NE(output_meta_pb.rs_metas(0).rowset_id_v2(), rowset_id.to_string());
140+
EXPECT_TRUE(output_meta_pb.has_schema());
141+
EXPECT_EQ(output_meta_pb.schema().index_size(), 2);
142+
EXPECT_EQ(output_meta_pb.schema().index(0).index_id(), 1001);
143+
EXPECT_EQ(output_meta_pb.schema().index(0).index_name(), "test_index1");
144+
EXPECT_EQ(output_meta_pb.schema().index(1).index_id(), 1002);
145+
EXPECT_EQ(output_meta_pb.schema().index(1).index_name(), "test_index2");
146+
EXPECT_TRUE(output_meta_pb.rs_metas(0).has_tablet_schema());
147+
EXPECT_EQ(output_meta_pb.rs_metas(0).tablet_schema().index_size(), 2);
148+
EXPECT_EQ(output_meta_pb.rs_metas(0).tablet_schema().index(0).index_id(), 1001);
149+
EXPECT_EQ(output_meta_pb.rs_metas(0).tablet_schema().index(1).index_id(), 1002);
95150
EXPECT_EQ(output_meta_pb.rs_metas(0).num_segments(), 3);
96151
EXPECT_EQ(output_meta_pb.rs_metas(0).num_rows(), 100);
97152
EXPECT_EQ(output_meta_pb.rs_metas(0).start_version(), 100);
@@ -105,4 +160,55 @@ TEST_F(CloudSnapshotMgrTest, TestConvertRowsets) {
105160
EXPECT_TRUE(status.ok());
106161
}
107162

163+
TEST_F(CloudSnapshotMgrTest, TestRenameIndexIds) {
164+
TabletSchemaPB source_schema_pb;
165+
source_schema_pb.set_keys_type(KeysType::DUP_KEYS);
166+
source_schema_pb.set_num_short_key_columns(1);
167+
source_schema_pb.set_num_rows_per_row_block(1024);
168+
source_schema_pb.set_compress_kind(COMPRESS_LZ4);
169+
170+
ColumnPB* col1 = source_schema_pb.add_column();
171+
col1->set_unique_id(1);
172+
col1->set_name("col1");
173+
col1->set_type("INT");
174+
col1->set_is_key(true);
175+
col1->set_aggregation("NONE");
176+
177+
ColumnPB* col2 = source_schema_pb.add_column();
178+
col2->set_unique_id(2);
179+
col2->set_name("col2");
180+
col2->set_type("VARCHAR");
181+
col2->set_is_key(false);
182+
col2->set_aggregation("REPLACE");
183+
184+
doris::TabletIndexPB* index1 = source_schema_pb.add_index();
185+
index1->set_index_id(1001);
186+
index1->set_index_name("test_index1");
187+
index1->set_index_type(IndexType::BITMAP);
188+
index1->add_col_unique_id(2);
189+
190+
doris::TabletIndexPB* index2 = source_schema_pb.add_index();
191+
index2->set_index_id(1002);
192+
index2->set_index_name("test_index2");
193+
index2->set_index_type(IndexType::INVERTED);
194+
index2->add_col_unique_id(2);
195+
196+
TabletSchemaSPtr local_tablet_schema = std::make_shared<TabletSchema>();
197+
local_tablet_schema->init_from_pb(source_schema_pb);
198+
199+
// reset the index id
200+
source_schema_pb.mutable_index(0)->set_index_id(2001);
201+
source_schema_pb.mutable_index(1)->set_index_id(2002);
202+
203+
Status status = _snapshot_mgr->_rename_index_ids(source_schema_pb, local_tablet_schema);
204+
EXPECT_TRUE(status.ok());
205+
EXPECT_EQ(source_schema_pb.index_size(), 2);
206+
EXPECT_EQ(source_schema_pb.index(0).index_id(), 1001);
207+
EXPECT_EQ(source_schema_pb.index(0).index_name(), "test_index1");
208+
EXPECT_EQ(source_schema_pb.index(0).index_type(), IndexType::BITMAP);
209+
EXPECT_EQ(source_schema_pb.index(1).index_id(), 1002);
210+
EXPECT_EQ(source_schema_pb.index(1).index_name(), "test_index2");
211+
EXPECT_EQ(source_schema_pb.index(1).index_type(), IndexType::INVERTED);
212+
}
213+
108214
} // namespace doris

‎be/test/olap/tablet_schema_test.cpp‎

Lines changed: 91 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -803,4 +803,94 @@ TEST_F(TabletSchemaTest, test_tablet_schema_need_record_variant_extended_schema)
803803
EXPECT_FALSE(schema_multiple_variants.need_record_variant_extended_schema());
804804
}
805805

806-
} // namespace doris
806+
TEST_F(TabletSchemaTest, test_tablet_schema_get_index) {
807+
TabletSchema schema;
808+
809+
TabletColumn col1;
810+
col1.set_unique_id(14001);
811+
col1.set_name("test_col1");
812+
col1.set_type(FieldType::OLAP_FIELD_TYPE_STRING);
813+
schema.append_column(col1);
814+
815+
TabletColumn col2;
816+
col2.set_unique_id(14002);
817+
col2.set_name("test_col2");
818+
col2.set_type(FieldType::OLAP_FIELD_TYPE_VARCHAR);
819+
schema.append_column(col2);
820+
821+
TabletIndex inverted_index;
822+
TabletIndexPB inverted_index_pb;
823+
inverted_index_pb.set_index_id(5001);
824+
inverted_index_pb.set_index_name("inverted_idx");
825+
inverted_index_pb.set_index_type(IndexType::INVERTED);
826+
inverted_index_pb.add_col_unique_id(14001);
827+
inverted_index.init_from_pb(inverted_index_pb);
828+
829+
TabletIndex bitmap_index;
830+
TabletIndexPB bitmap_index_pb;
831+
bitmap_index_pb.set_index_id(5002);
832+
bitmap_index_pb.set_index_name("bitmap_idx");
833+
bitmap_index_pb.set_index_type(IndexType::BITMAP);
834+
bitmap_index_pb.add_col_unique_id(14001);
835+
bitmap_index.init_from_pb(bitmap_index_pb);
836+
837+
TabletIndex ann_index;
838+
TabletIndexPB ann_index_pb;
839+
ann_index_pb.set_index_id(5003);
840+
ann_index_pb.set_index_name("ann_idx");
841+
ann_index_pb.set_index_type(IndexType::ANN);
842+
ann_index_pb.add_col_unique_id(14002);
843+
ann_index.init_from_pb(ann_index_pb);
844+
845+
TabletIndex ngram_bf_index;
846+
TabletIndexPB ngram_bf_index_pb;
847+
ngram_bf_index_pb.set_index_id(5004);
848+
ngram_bf_index_pb.set_index_name("ngram_bf_idx");
849+
ngram_bf_index_pb.set_index_type(IndexType::NGRAM_BF);
850+
ngram_bf_index_pb.add_col_unique_id(14002);
851+
ngram_bf_index.init_from_pb(ngram_bf_index_pb);
852+
853+
schema.append_index(std::move(inverted_index));
854+
schema.append_index(std::move(bitmap_index));
855+
schema.append_index(std::move(ann_index));
856+
schema.append_index(std::move(ngram_bf_index));
857+
858+
const TabletIndex* found_inverted = schema.get_index(14001, IndexType::INVERTED, "");
859+
EXPECT_NE(nullptr, found_inverted);
860+
EXPECT_EQ("inverted_idx", found_inverted->index_name());
861+
EXPECT_EQ(5001, found_inverted->index_id());
862+
const TabletIndex* found_bitmap = schema.get_index(14001, IndexType::BITMAP, "");
863+
EXPECT_NE(nullptr, found_bitmap);
864+
EXPECT_EQ("bitmap_idx", found_bitmap->index_name());
865+
EXPECT_EQ(5002, found_bitmap->index_id());
866+
const TabletIndex* found_ann = schema.get_index(14002, IndexType::ANN, "");
867+
EXPECT_NE(nullptr, found_ann);
868+
EXPECT_EQ("ann_idx", found_ann->index_name());
869+
EXPECT_EQ(5003, found_ann->index_id());
870+
const TabletIndex* found_ngram_bf = schema.get_index(14002, IndexType::NGRAM_BF, "");
871+
EXPECT_NE(nullptr, found_ngram_bf);
872+
EXPECT_EQ("ngram_bf_idx", found_ngram_bf->index_name());
873+
EXPECT_EQ(5004, found_ngram_bf->index_id());
874+
const TabletIndex* not_found = schema.get_index(99999, IndexType::INVERTED, "");
875+
EXPECT_EQ(nullptr, not_found);
876+
const TabletIndex* empty_suffix = schema.get_index(14001, IndexType::INVERTED, "");
877+
EXPECT_NE(nullptr, empty_suffix);
878+
EXPECT_EQ("inverted_idx", empty_suffix->index_name());
879+
const TabletIndex* with_suffix = schema.get_index(14001, IndexType::INVERTED, "test_suffix");
880+
EXPECT_EQ(nullptr, with_suffix);
881+
882+
EXPECT_TRUE(found_inverted->is_inverted_index());
883+
EXPECT_EQ(IndexType::INVERTED, found_inverted->index_type());
884+
EXPECT_EQ(IndexType::BITMAP, found_bitmap->index_type());
885+
EXPECT_EQ(IndexType::ANN, found_ann->index_type());
886+
EXPECT_EQ(IndexType::NGRAM_BF, found_ngram_bf->index_type());
887+
888+
const auto& inverted_col_ids = found_inverted->col_unique_ids();
889+
EXPECT_EQ(1, inverted_col_ids.size());
890+
EXPECT_EQ(14001, inverted_col_ids[0]);
891+
const auto& ann_col_ids = found_ann->col_unique_ids();
892+
EXPECT_EQ(1, ann_col_ids.size());
893+
EXPECT_EQ(14002, ann_col_ids[0]);
894+
}
895+
896+
} // namespace doris

0 commit comments

Comments
 (0)