Skip to content

Commit bc7b943

Browse files
committed
[fix](schema-change) Fix sync MV loss due to index change (apache#54285)
### What problem does this PR solve? Problem Summary: Creation of shadow index did not take define stmt and where clause of MV into account, therefore, after index creation, MV info will be lost. This bug only affect index change on MV related columns. Other types of heavy sc are not allowed on MV related columns.
1 parent ef2c200 commit bc7b943

File tree

3 files changed

+83
-2
lines changed

3 files changed

+83
-2
lines changed

fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.apache.doris.catalog.KeysType;
3131
import org.apache.doris.catalog.MaterializedIndex;
3232
import org.apache.doris.catalog.MaterializedIndex.IndexState;
33+
import org.apache.doris.catalog.MaterializedIndexMeta;
3334
import org.apache.doris.catalog.OlapTable;
3435
import org.apache.doris.catalog.OlapTable.OlapTableState;
3536
import org.apache.doris.catalog.Partition;
@@ -450,12 +451,15 @@ protected void addShadowIndexToCatalog(OlapTable tbl) {
450451
}
451452

452453
for (long shadowIdxId : indexIdMap.keySet()) {
454+
MaterializedIndexMeta originalIndexMeta = tbl.getIndexMetaByIndexId(indexIdMap.get(shadowIdxId));
453455
tbl.setIndexMeta(shadowIdxId, indexIdToName.get(shadowIdxId), indexSchemaMap.get(shadowIdxId),
454456
indexSchemaVersionAndHashMap.get(shadowIdxId).schemaVersion,
455457
indexSchemaVersionAndHashMap.get(shadowIdxId).schemaHash,
456458
indexShortKeyMap.get(shadowIdxId), TStorageType.COLUMN,
457-
tbl.getKeysTypeByIndexId(indexIdMap.get(shadowIdxId)),
458-
indexChange ? indexes : tbl.getIndexMetaByIndexId(indexIdMap.get(shadowIdxId)).getIndexes());
459+
tbl.getKeysTypeByIndexId(indexIdMap.get(shadowIdxId)), originalIndexMeta.getDefineStmt(),
460+
indexChange ? indexes : originalIndexMeta.getIndexes());
461+
MaterializedIndexMeta shadowIndexMeta = tbl.getIndexMetaByIndexId(shadowIdxId);
462+
shadowIndexMeta.setWhereClause(originalIndexMeta.getWhereClause());
459463
}
460464

461465
tbl.rebuildFullSchema();
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !sql --
3+
test_index_change_after_mv test_index_change_after_mv_1 \n CREATE MATERIALIZED VIEW test_index_change_after_mv_1 AS SELECT name, age, grade FROM test_index_change_after_mv WHERE name = "steve" AND age = 0\n
4+
Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
import org.awaitility.Awaitility
19+
20+
import static java.util.concurrent.TimeUnit.SECONDS
21+
22+
suite("test_index_change_after_mv") {
23+
def tableName = "test_index_change_after_mv"
24+
sql """ DROP TABLE IF EXISTS ${tableName} """
25+
sql """
26+
CREATE TABLE IF NOT EXISTS ${tableName} (
27+
name varchar(50),
28+
age int NOT NULL,
29+
grade int NOT NULL,
30+
studentInfo char(100),
31+
tearchComment string,
32+
)
33+
DUPLICATE KEY(`name`)
34+
DISTRIBUTED BY HASH(`name`) BUCKETS 10
35+
properties("replication_num" = "1");
36+
"""
37+
38+
def mvName = """${tableName}_1"""
39+
40+
sql """ INSERT INTO ${tableName} VALUES("steve", 0, 2, "xxx", "xxx") """
41+
42+
sql """
43+
CREATE MATERIALIZED VIEW ${mvName} AS SELECT name, age, grade FROM ${tableName} WHERE name = "steve" AND age = 0
44+
"""
45+
46+
def getJobState = { tblName ->
47+
def jobStateResult = sql """ SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${tblName}' ORDER BY CreateTime DESC LIMIT 1; """
48+
return jobStateResult[0][8]
49+
}
50+
51+
def max_try_secs = 60
52+
Awaitility.await().atMost(max_try_secs, SECONDS).pollInterval(2, SECONDS).until{
53+
String res = getJobState(tableName)
54+
if (res == "FINISHED" || res == "CANCELLED") {
55+
assertEquals("FINISHED", res)
56+
sleep(3000)
57+
return true
58+
}
59+
return false
60+
}
61+
62+
sql """
63+
CREATE INDEX ${tableName}_idx ON ${tableName}(age) USING INVERTED COMMENT 'inverted index age_idx_1'
64+
"""
65+
66+
waitForSchemaChangeDone({
67+
sql " SHOW ALTER TABLE COLUMN WHERE TableName='${tableName}' ORDER BY createtime DESC LIMIT 1 "
68+
})
69+
70+
qt_sql """
71+
SHOW CREATE MATERIALIZED VIEW ${mvName} on ${tableName}
72+
"""
73+
}

0 commit comments

Comments
 (0)