Skip to content

Commit 4451f5f

Browse files
LiShuMingLiShuMing
authored
[BugFix] Fix possible PartitionColumnMinMaxRewriteRule bugs caused by Partition.hasStorageData (StarRocks#69751)
Signed-off-by: shuming.li <ming.moriarty@gmail.com>
1 parent d4fde94 commit 4451f5f

File tree

13 files changed

+1288
-12
lines changed

13 files changed

+1288
-12
lines changed

‎fe/fe-core/src/main/java/com/starrocks/catalog/PhysicalPartition.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -642,7 +642,7 @@ public boolean hasStorageData() {
642642
// The fe unit test need to check the selected index id without any data.
643643
// So if set FeConstants.runningUnitTest, we can ensure that the number of partitions is not empty,
644644
// And the test case can continue to execute the logic of 'select best roll up'
645-
return ((visibleVersion != PARTITION_INIT_VERSION)
645+
return ((dataVersion != PARTITION_INIT_VERSION && visibleVersion != PARTITION_INIT_VERSION)
646646
|| FeConstants.runningUnitTest);
647647
}
648648

‎fe/fe-core/src/test/java/com/starrocks/http/StarRocksHttpTestCase.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ public static OlapTable newEmptyTable(String name) {
187187
Partition partition = new Partition(testPartitionId, testPhysicalPartitionId,
188188
"testPartition", baseIndex, distributionInfo);
189189
partition.getDefaultPhysicalPartition().updateVisibleVersion(testStartVersion);
190+
partition.getDefaultPhysicalPartition().setDataVersion(testStartVersion);
190191
partition.getDefaultPhysicalPartition().setNextVersion(testStartVersion + 1);
191192

192193
// table
@@ -244,6 +245,7 @@ public static OlapTable newTable(String name, long replicaDataSize) {
244245
Partition partition = new Partition(testPartitionId, testPhysicalPartitionId,
245246
"testPartition", baseIndex, distributionInfo);
246247
partition.getDefaultPhysicalPartition().updateVisibleVersion(testStartVersion);
248+
partition.getDefaultPhysicalPartition().setDataVersion(testStartVersion);
247249
partition.getDefaultPhysicalPartition().setNextVersion(testStartVersion + 1);
248250

249251
// table

‎fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePartialPartitionTest.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -730,13 +730,18 @@ public void testPartitionQueryRewriteSkipEmptyPartitions() throws Exception {
730730
MaterializedView mv = starRocksAssert.getMv("test", "test_loose_mv");
731731
mv.getPartition("p19910330").getDefaultPhysicalPartition()
732732
.setVisibleVersion(Partition.PARTITION_INIT_VERSION, System.currentTimeMillis());
733+
mv.getPartition("p19910330").getDefaultPhysicalPartition()
734+
.setDataVersion(Partition.PARTITION_INIT_VERSION);
733735
String query5 = "select id_date, sum(t1b) from table_with_day_partition" +
734736
" where id_date >= '1991-03-30' and id_date < '1991-04-03' group by id_date";
735-
FeConstants.runningUnitTest = false;
736-
String plan = getFragmentPlan(query5);
737-
FeConstants.runningUnitTest = true;
738-
PlanTestBase.assertContains(plan, "test_loose_mv", "partitions=3/4",
739-
"table_with_day_partition", "partitions=1/4", "UNION");
737+
try {
738+
FeConstants.runningUnitTest = false;
739+
String plan = getFragmentPlan(query5);
740+
PlanTestBase.assertContains(plan, "test_loose_mv", "partitions=3/4",
741+
"table_with_day_partition", "partitions=1/4", "UNION");
742+
} finally {
743+
FeConstants.runningUnitTest = true;
744+
}
740745
dropMv("test", "test_loose_mv");
741746
}
742747
}

‎fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -354,8 +354,11 @@ public void testLogicalOlapTableEmptyPartition() {
354354
Partition partition3 = partitions.get(2);
355355
// mock one empty partition
356356
partition1.getDefaultPhysicalPartition().setVisibleVersion(Partition.PARTITION_INIT_VERSION, System.currentTimeMillis());
357+
partition1.getDefaultPhysicalPartition().setDataVersion(Partition.PARTITION_INIT_VERSION);
357358
partition2.getDefaultPhysicalPartition().setVisibleVersion(2, System.currentTimeMillis());
359+
partition2.getDefaultPhysicalPartition().setDataVersion(2);
358360
partition3.getDefaultPhysicalPartition().setVisibleVersion(2, System.currentTimeMillis());
361+
partition3.getDefaultPhysicalPartition().setDataVersion(2);
359362
List<Long> partitionIds = partitions.stream().filter(partition -> !(partition.getName().equalsIgnoreCase("p1"))).
360363
mapToLong(Partition::getId).boxed().collect(Collectors.toList());
361364

‎fe/fe-core/src/test/java/com/starrocks/sql/plan/ExternalTableTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,7 @@ public void testJoinWithMysqlTable() throws Exception {
266266
OlapTable tbl = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), "jointest");
267267
for (Partition partition : tbl.getPartitions()) {
268268
partition.getDefaultPhysicalPartition().updateVisibleVersion(2);
269+
partition.getDefaultPhysicalPartition().setDataVersion(2);
269270
for (MaterializedIndex mIndex : partition.getDefaultPhysicalPartition()
270271
.getLatestMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE)) {
271272
mIndex.setRowCount(10000);

‎fe/fe-core/src/test/java/com/starrocks/sql/plan/PartitionPruneTest.java‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -634,12 +634,16 @@ public void testMinMaxRangePartitionPruneWithEmptyPartition() throws Exception {
634634
" \"replication_num\" = \"1\"\n" +
635635
");");
636636
starRocksAssert.getCtx().executeSql("insert into t4_range_minmax values('2025-04-29', 1, 'bar')");
637-
FeConstants.runningUnitTest = false;
638-
starRocksAssert.getTable("test", "t4_range_minmax")
639-
.getPartition("p20250428").getDefaultPhysicalPartition().updateVisibleVersion(1);
640-
starRocksAssert.query("select min(dt), max(dt) from t4_range_minmax").explainContains("partitions=1/2");
641-
FeConstants.runningUnitTest = true;
642-
637+
try {
638+
FeConstants.runningUnitTest = false;
639+
starRocksAssert.getTable("test", "t4_range_minmax")
640+
.getPartition("p20250428").getDefaultPhysicalPartition().updateVisibleVersion(1);
641+
starRocksAssert.getTable("test", "t4_range_minmax")
642+
.getPartition("p20250428").getDefaultPhysicalPartition().setDataVersion(1);
643+
starRocksAssert.query("select min(dt), max(dt) from t4_range_minmax").explainContains("partitions=1/2");
644+
} finally {
645+
FeConstants.runningUnitTest = true;
646+
}
643647
}
644648

645649
@Test

‎fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1344,6 +1344,7 @@ public static boolean matchPlanWithoutId(String expect, String actual) {
13441344

13451345
public static void setPartitionVersion(Partition partition, long version) {
13461346
partition.getDefaultPhysicalPartition().setVisibleVersion(version, System.currentTimeMillis());
1347+
partition.getDefaultPhysicalPartition().setDataVersion(version);
13471348
MaterializedIndex baseIndex = partition.getDefaultPhysicalPartition().getLatestBaseIndex();
13481349
List<Tablet> tablets = baseIndex.getTablets();
13491350
for (Tablet tablet : tablets) {

‎test/sql/test_partition_storage_data/R/test_minmax_partition_column_rewrite.sql‎

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
-- name: test_minmax_partition_column_rewrite
2+
DROP DATABASE IF EXISTS test_minmax_partition_column_rewrite;
3+
-- result:
4+
-- !result
5+
CREATE DATABASE test_minmax_partition_column_rewrite;
6+
-- result:
7+
-- !result
8+
USE test_minmax_partition_column_rewrite;
9+
-- result:
10+
-- !result
11+
CREATE TABLE `t1_partial_empty` (
12+
`dt` date NOT NULL COMMENT "",
13+
`id` int(11) NULL COMMENT "",
14+
`value` bigint NULL COMMENT ""
15+
) ENGINE=OLAP
16+
DUPLICATE KEY(`dt`)
17+
PARTITION BY RANGE(`dt`)
18+
(
19+
PARTITION p202501 VALUES [("2025-01-01"), ("2025-02-01")),
20+
PARTITION p202502 VALUES [("2025-02-01"), ("2025-03-01")),
21+
PARTITION p202503 VALUES [("2025-03-01"), ("2025-04-01"))
22+
)
23+
DISTRIBUTED BY HASH(`id`)
24+
PROPERTIES (
25+
"replication_num" = "1"
26+
);
27+
-- result:
28+
-- !result
29+
INSERT INTO t1_partial_empty VALUES('2025-02-15', 1, 100);
30+
-- result:
31+
-- !result
32+
SELECT MIN(dt) FROM t1_partial_empty;
33+
-- result:
34+
2025-02-15
35+
-- !result
36+
SELECT MAX(dt) FROM t1_partial_empty;
37+
-- result:
38+
2025-02-15
39+
-- !result
40+
SELECT MIN(dt), MAX(dt) FROM t1_partial_empty;
41+
-- result:
42+
2025-02-15 2025-02-15
43+
-- !result
44+
CREATE TABLE `t2_schema_change_minmax` (
45+
`dt` date NOT NULL COMMENT "",
46+
`id` int(11) NULL COMMENT ""
47+
) ENGINE=OLAP
48+
DUPLICATE KEY(`dt`)
49+
PARTITION BY RANGE(`dt`)
50+
(
51+
PARTITION p202501 VALUES [("2025-01-01"), ("2025-02-01")),
52+
PARTITION p202502 VALUES [("2025-02-01"), ("2025-03-01"))
53+
)
54+
DISTRIBUTED BY HASH(`id`)
55+
PROPERTIES (
56+
"replication_num" = "1"
57+
);
58+
-- result:
59+
-- !result
60+
INSERT INTO t2_schema_change_minmax VALUES('2025-01-15', 1);
61+
-- result:
62+
-- !result
63+
INSERT INTO t2_schema_change_minmax VALUES('2025-02-15', 2);
64+
-- result:
65+
-- !result
66+
SELECT MIN(dt), MAX(dt) FROM t2_schema_change_minmax;
67+
-- result:
68+
2025-01-15 2025-02-15
69+
-- !result
70+
ALTER TABLE t2_schema_change_minmax ADD COLUMN new_col INT NULL DEFAULT "0";
71+
-- result:
72+
-- !result
73+
function: wait_alter_table_finish()
74+
-- result:
75+
None
76+
-- !result
77+
SELECT MIN(dt), MAX(dt) FROM t2_schema_change_minmax;
78+
-- result:
79+
2025-01-15 2025-02-15
80+
-- !result
81+
INSERT INTO t2_schema_change_minmax VALUES('2025-01-20', 3, 100);
82+
-- result:
83+
-- !result
84+
INSERT INTO t2_schema_change_minmax VALUES('2025-02-20', 4, 200);
85+
-- result:
86+
-- !result
87+
SELECT MIN(dt), MAX(dt) FROM t2_schema_change_minmax;
88+
-- result:
89+
2025-01-15 2025-02-20
90+
-- !result
91+
CREATE TABLE `t3_list_minmax` (
92+
`c1` int NOT NULL,
93+
`c2` int,
94+
`c3` varchar(100)
95+
) ENGINE=OLAP
96+
DUPLICATE KEY(`c1`)
97+
PARTITION BY LIST(`c1`)
98+
(
99+
PARTITION p1 VALUES IN ('1', '2', '3'),
100+
PARTITION p2 VALUES IN ('4', '5', '6'),
101+
PARTITION p3 VALUES IN ('7', '8', '9')
102+
)
103+
DISTRIBUTED BY HASH(`c1`)
104+
PROPERTIES (
105+
"replication_num" = "1"
106+
);
107+
-- result:
108+
-- !result
109+
INSERT INTO t3_list_minmax VALUES(1, 100, 'a'), (5, 200, 'b'), (9, 300, 'c');
110+
-- result:
111+
-- !result
112+
SELECT MIN(c1), MAX(c1) FROM t3_list_minmax;
113+
-- result:
114+
1 9
115+
-- !result
116+
ALTER TABLE t3_list_minmax ADD COLUMN new_col DOUBLE NULL;
117+
-- result:
118+
-- !result
119+
function: wait_alter_table_finish()
120+
-- result:
121+
None
122+
-- !result
123+
SELECT MIN(c1), MAX(c1) FROM t3_list_minmax;
124+
-- result:
125+
1 9
126+
-- !result
127+
CREATE TABLE `t4_prune_with_minmax` (
128+
`dt` date NOT NULL COMMENT "",
129+
`id` int(11) NULL COMMENT ""
130+
) ENGINE=OLAP
131+
DUPLICATE KEY(`dt`)
132+
PARTITION BY RANGE(`dt`)
133+
(
134+
PARTITION p202501 VALUES [("2025-01-01"), ("2025-02-01")),
135+
PARTITION p202502 VALUES [("2025-02-01"), ("2025-03-01")),
136+
PARTITION p202503 VALUES [("2025-03-01"), ("2025-04-01")),
137+
PARTITION p202504 VALUES [("2025-04-01"), ("2025-05-01"))
138+
)
139+
DISTRIBUTED BY HASH(`id`)
140+
PROPERTIES (
141+
"replication_num" = "1"
142+
);
143+
-- result:
144+
-- !result
145+
INSERT INTO t4_prune_with_minmax VALUES('2025-01-15', 1);
146+
-- result:
147+
-- !result
148+
INSERT INTO t4_prune_with_minmax VALUES('2025-02-15', 2);
149+
-- result:
150+
-- !result
151+
INSERT INTO t4_prune_with_minmax VALUES('2025-03-15', 3);
152+
-- result:
153+
-- !result
154+
INSERT INTO t4_prune_with_minmax VALUES('2025-04-15', 4);
155+
-- result:
156+
-- !result
157+
SELECT MIN(dt) FROM t4_prune_with_minmax;
158+
-- result:
159+
2025-01-15
160+
-- !result
161+
SELECT MAX(dt) FROM t4_prune_with_minmax;
162+
-- result:
163+
2025-04-15
164+
-- !result
165+
ALTER TABLE t4_prune_with_minmax ADD COLUMN col1 STRING NULL;
166+
-- result:
167+
-- !result
168+
function: wait_alter_table_finish()
169+
-- result:
170+
None
171+
-- !result
172+
SELECT MIN(dt) FROM t4_prune_with_minmax;
173+
-- result:
174+
2025-01-15
175+
-- !result
176+
SELECT MAX(dt) FROM t4_prune_with_minmax;
177+
-- result:
178+
2025-04-15
179+
-- !result
180+
CREATE TABLE `t5_filter_minmax` (
181+
`dt` date NOT NULL COMMENT "",
182+
`id` int(11) NULL COMMENT "",
183+
`value` bigint NULL COMMENT ""
184+
) ENGINE=OLAP
185+
DUPLICATE KEY(`dt`)
186+
PARTITION BY RANGE(`dt`)
187+
(
188+
PARTITION p202501 VALUES [("2025-01-01"), ("2025-02-01")),
189+
PARTITION p202502 VALUES [("2025-02-01"), ("2025-03-01"))
190+
)
191+
DISTRIBUTED BY HASH(`id`)
192+
PROPERTIES (
193+
"replication_num" = "1"
194+
);
195+
-- result:
196+
-- !result
197+
INSERT INTO t5_filter_minmax VALUES('2025-01-15', 1, 100);
198+
-- result:
199+
-- !result
200+
INSERT INTO t5_filter_minmax VALUES('2025-02-15', 2, 200);
201+
-- result:
202+
-- !result
203+
SELECT MIN(dt), MAX(dt) FROM t5_filter_minmax;
204+
-- result:
205+
2025-01-15 2025-02-15
206+
-- !result
207+
ALTER TABLE t5_filter_minmax ADD COLUMN col1 INT NULL;
208+
-- result:
209+
-- !result
210+
function: wait_alter_table_finish()
211+
-- result:
212+
None
213+
-- !result
214+
SELECT MIN(dt), MAX(dt) FROM t5_filter_minmax;
215+
-- result:
216+
2025-01-15 2025-02-15
217+
-- !result
218+
DROP DATABASE IF EXISTS test_minmax_partition_column_rewrite;
219+
-- result:
220+
-- !result

0 commit comments

Comments
 (0)