Skip to content

Commit 728e38a

Browse files
suxiaogang223suxiaogang223
committed
Clean HMS DLA MTMV dead paths and add bridge tests
1 parent dd2e0c2 commit 728e38a

File tree

6 files changed

+97
-106
lines changed

6 files changed

+97
-106
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSDlaTable.java‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -19,17 +19,12 @@
1919

2020
import org.apache.doris.catalog.Column;
2121
import org.apache.doris.catalog.MTMV;
22-
import org.apache.doris.catalog.PartitionItem;
2322
import org.apache.doris.catalog.PartitionType;
24-
import org.apache.doris.common.AnalysisException;
2523
import org.apache.doris.common.DdlException;
2624
import org.apache.doris.datasource.mvcc.MvccSnapshot;
2725
import org.apache.doris.mtmv.MTMVBaseTableIf;
28-
import org.apache.doris.mtmv.MTMVRefreshContext;
29-
import org.apache.doris.mtmv.MTMVSnapshotIf;
3026

3127
import java.util.List;
32-
import java.util.Map;
3328
import java.util.Optional;
3429
import java.util.Set;
3530

@@ -53,23 +48,12 @@ public HMSDlaTable(HMSExternalTable table) {
5348
this.hmsTable = table;
5449
}
5550

56-
abstract Map<String, PartitionItem> getAndCopyPartitionItems(Optional<MvccSnapshot> snapshot)
57-
throws AnalysisException;
58-
5951
abstract PartitionType getPartitionType(Optional<MvccSnapshot> snapshot);
6052

6153
abstract Set<String> getPartitionColumnNames(Optional<MvccSnapshot> snapshot);
6254

6355
abstract List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot);
6456

65-
abstract MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context,
66-
Optional<MvccSnapshot> snapshot) throws AnalysisException;
67-
68-
abstract MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, Optional<MvccSnapshot> snapshot)
69-
throws AnalysisException;
70-
71-
abstract MTMVSnapshotIf getTableSnapshot(Optional<MvccSnapshot> snapshot) throws AnalysisException;
72-
7357
abstract boolean isPartitionColumnAllowNull();
7458

7559
@Override

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveDlaTable.java‎

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,12 @@
1818
package org.apache.doris.datasource.hive;
1919

2020
import org.apache.doris.catalog.Column;
21-
import org.apache.doris.catalog.PartitionItem;
2221
import org.apache.doris.catalog.PartitionType;
23-
import org.apache.doris.common.AnalysisException;
2422
import org.apache.doris.datasource.SchemaCacheValue;
25-
import org.apache.doris.datasource.metacache.EngineMtmvSupport;
2623
import org.apache.doris.datasource.mvcc.MvccSnapshot;
27-
import org.apache.doris.mtmv.MTMVRefreshContext;
28-
import org.apache.doris.mtmv.MTMVSnapshotIf;
2924

3025
import java.util.Collections;
3126
import java.util.List;
32-
import java.util.Map;
3327
import java.util.Optional;
3428
import java.util.Set;
3529
import java.util.stream.Collectors;
@@ -58,29 +52,6 @@ public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
5852
.orElse(Collections.emptyList());
5953
}
6054

61-
@Override
62-
public Map<String, PartitionItem> getAndCopyPartitionItems(Optional<MvccSnapshot> snapshot)
63-
throws AnalysisException {
64-
return EngineMtmvSupport.getAndCopyPartitionItems(hmsTable, snapshot);
65-
}
66-
67-
@Override
68-
public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context,
69-
Optional<MvccSnapshot> snapshot) throws AnalysisException {
70-
return EngineMtmvSupport.getPartitionSnapshot(hmsTable, partitionName, snapshot);
71-
}
72-
73-
@Override
74-
public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, Optional<MvccSnapshot> snapshot)
75-
throws AnalysisException {
76-
return getTableSnapshot(snapshot);
77-
}
78-
79-
@Override
80-
public MTMVSnapshotIf getTableSnapshot(Optional<MvccSnapshot> snapshot) throws AnalysisException {
81-
return EngineMtmvSupport.getTableSnapshot(hmsTable, snapshot);
82-
}
83-
8455
@Override
8556
public boolean isPartitionColumnAllowNull() {
8657
return true;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveEngineCache.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -90,8 +90,7 @@ public void invalidateTable(ExternalTable table) {
9090
@Override
9191
public EnginePartitionInfo getPartitionInfo(ExternalTable table, Optional<MvccSnapshot> snapshot) {
9292
HMSExternalTable hmsTable = asHiveTable(table);
93-
return new HivePartition(metaStoreCache.getPartitionValues(
94-
hmsTable, hmsTable.getPartitionColumnTypes(snapshot)));
93+
return new HivePartition(hmsTable.getHivePartitionValues(snapshot));
9594
}
9695

9796
@Override
@@ -100,8 +99,7 @@ public EngineSnapshot getSnapshot(ExternalTable table, Optional<MvccSnapshot> sn
10099
if (hmsTable.getPartitionType(snapshot) == PartitionType.UNPARTITIONED) {
101100
return new HiveSnapshotMeta(hmsTable.getName(), hmsTable.getLastDdlTime());
102101
}
103-
HiveMetaStoreCache.HivePartitionValues partitionValues = metaStoreCache.getPartitionValues(
104-
hmsTable, hmsTable.getPartitionColumnTypes(snapshot));
102+
HiveMetaStoreCache.HivePartitionValues partitionValues = hmsTable.getHivePartitionValues(snapshot);
105103
List<HivePartition> partitions = metaStoreCache.getAllPartitionsWithCache(
106104
hmsTable, new ArrayList<>(partitionValues.getPartitionValuesMap().values()));
107105
if (CollectionUtils.isEmpty(partitions)) {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HudiDlaTable.java‎

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -19,22 +19,16 @@
1919

2020
import org.apache.doris.catalog.Column;
2121
import org.apache.doris.catalog.Env;
22-
import org.apache.doris.catalog.PartitionItem;
2322
import org.apache.doris.catalog.PartitionType;
24-
import org.apache.doris.common.AnalysisException;
2523
import org.apache.doris.datasource.CacheException;
2624
import org.apache.doris.datasource.ExternalSchemaCache;
2725
import org.apache.doris.datasource.SchemaCacheValue;
2826
import org.apache.doris.datasource.hudi.HudiMvccSnapshot;
2927
import org.apache.doris.datasource.hudi.HudiSchemaCacheKey;
3028
import org.apache.doris.datasource.hudi.HudiUtils;
31-
import org.apache.doris.datasource.metacache.EngineMtmvSupport;
3229
import org.apache.doris.datasource.mvcc.MvccSnapshot;
33-
import org.apache.doris.mtmv.MTMVRefreshContext;
34-
import org.apache.doris.mtmv.MTMVSnapshotIf;
3530

3631
import java.util.List;
37-
import java.util.Map;
3832
import java.util.Optional;
3933
import java.util.Set;
4034
import java.util.stream.Collectors;
@@ -61,29 +55,6 @@ public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
6155
return getHudiSchemaCacheValue(snapshot).getPartitionColumns();
6256
}
6357

64-
@Override
65-
public Map<String, PartitionItem> getAndCopyPartitionItems(Optional<MvccSnapshot> snapshot)
66-
throws AnalysisException {
67-
return EngineMtmvSupport.getAndCopyPartitionItems(hmsTable, snapshot);
68-
}
69-
70-
@Override
71-
public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context,
72-
Optional<MvccSnapshot> snapshot) throws AnalysisException {
73-
return EngineMtmvSupport.getPartitionSnapshot(hmsTable, partitionName, snapshot);
74-
}
75-
76-
@Override
77-
public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, Optional<MvccSnapshot> snapshot)
78-
throws AnalysisException {
79-
return getTableSnapshot(snapshot);
80-
}
81-
82-
@Override
83-
public MTMVSnapshotIf getTableSnapshot(Optional<MvccSnapshot> snapshot) throws AnalysisException {
84-
return EngineMtmvSupport.getTableSnapshot(hmsTable, snapshot);
85-
}
86-
8758
@Override
8859
public boolean isPartitionColumnAllowNull() {
8960
return true;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/IcebergDlaTable.java‎

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -18,22 +18,16 @@
1818
package org.apache.doris.datasource.hive;
1919

2020
import org.apache.doris.catalog.Column;
21-
import org.apache.doris.catalog.PartitionItem;
2221
import org.apache.doris.catalog.PartitionType;
23-
import org.apache.doris.common.AnalysisException;
2422
import org.apache.doris.datasource.iceberg.IcebergUtils;
25-
import org.apache.doris.datasource.metacache.EngineMtmvSupport;
2623
import org.apache.doris.datasource.mvcc.MvccSnapshot;
27-
import org.apache.doris.mtmv.MTMVRefreshContext;
28-
import org.apache.doris.mtmv.MTMVSnapshotIf;
2924

3025
import com.google.common.collect.Sets;
3126
import org.apache.iceberg.PartitionField;
3227
import org.apache.iceberg.PartitionSpec;
3328
import org.apache.iceberg.Table;
3429

3530
import java.util.List;
36-
import java.util.Map;
3731
import java.util.Optional;
3832
import java.util.Set;
3933
import java.util.stream.Collectors;
@@ -47,11 +41,6 @@ public IcebergDlaTable(HMSExternalTable table) {
4741
super(table);
4842
}
4943

50-
@Override
51-
public Map<String, PartitionItem> getAndCopyPartitionItems(Optional<MvccSnapshot> snapshot) {
52-
return EngineMtmvSupport.getAndCopyPartitionItems(hmsTable, snapshot);
53-
}
54-
5544
@Override
5645
public PartitionType getPartitionType(Optional<MvccSnapshot> snapshot) {
5746
return isValidRelatedTable() ? PartitionType.RANGE : PartitionType.UNPARTITIONED;
@@ -67,23 +56,6 @@ public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
6756
return IcebergUtils.getIcebergPartitionColumns(snapshot, hmsTable);
6857
}
6958

70-
@Override
71-
public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context,
72-
Optional<MvccSnapshot> snapshot) throws AnalysisException {
73-
return EngineMtmvSupport.getPartitionSnapshot(hmsTable, partitionName, snapshot);
74-
}
75-
76-
@Override
77-
public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, Optional<MvccSnapshot> snapshot)
78-
throws AnalysisException {
79-
return EngineMtmvSupport.getTableSnapshot(hmsTable, snapshot);
80-
}
81-
82-
@Override
83-
public MTMVSnapshotIf getTableSnapshot(Optional<MvccSnapshot> snapshot) throws AnalysisException {
84-
return EngineMtmvSupport.getTableSnapshot(hmsTable, snapshot);
85-
}
86-
8759
@Override
8860
boolean isPartitionColumnAllowNull() {
8961
return true;

‎fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EngineMtmvSupportTest.java‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.datasource.metacache;
19+
20+
import org.apache.doris.catalog.Env;
21+
import org.apache.doris.catalog.PartitionItem;
22+
import org.apache.doris.datasource.ExternalCatalog;
23+
import org.apache.doris.datasource.ExternalMetaCacheMgr;
24+
import org.apache.doris.datasource.TablePartitionValues;
25+
import org.apache.doris.datasource.hive.HMSExternalTable;
26+
import org.apache.doris.datasource.hive.HMSExternalTable.DLAType;
27+
import org.apache.doris.datasource.hive.HiveEngineCache;
28+
import org.apache.doris.datasource.hudi.source.HudiEngineCache;
29+
import org.apache.doris.mtmv.MTMVMaxTimestampSnapshot;
30+
import org.apache.doris.mtmv.MTMVSnapshotIf;
31+
import org.apache.doris.mtmv.MTMVTimestampSnapshot;
32+
33+
import org.junit.Assert;
34+
import org.junit.Test;
35+
import org.mockito.MockedStatic;
36+
import org.mockito.Mockito;
37+
38+
import java.util.Optional;
39+
40+
public class EngineMtmvSupportTest {
41+
42+
@Test
43+
public void testGetTableSnapshotForHive() throws Exception {
44+
Env env = Mockito.mock(Env.class);
45+
ExternalMetaCacheMgr metaCacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
46+
ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class);
47+
HMSExternalTable table = Mockito.mock(HMSExternalTable.class);
48+
HiveEngineCache hiveEngineCache = Mockito.mock(HiveEngineCache.class);
49+
50+
Mockito.when(env.getExtMetaCacheMgr()).thenReturn(metaCacheMgr);
51+
Mockito.when(table.getCatalog()).thenReturn(catalog);
52+
Mockito.when(table.getDlaType()).thenReturn(DLAType.HIVE);
53+
Mockito.when(metaCacheMgr.getHiveEngineCache(catalog)).thenReturn(hiveEngineCache);
54+
Mockito.when(hiveEngineCache.getSnapshot(table, Optional.empty()))
55+
.thenReturn(new HiveEngineCache.HiveSnapshotMeta("p=1", 123L));
56+
57+
try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
58+
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
59+
MTMVSnapshotIf snapshot = EngineMtmvSupport.getTableSnapshot(table, Optional.empty());
60+
Assert.assertTrue(snapshot instanceof MTMVMaxTimestampSnapshot);
61+
Assert.assertEquals(123L, snapshot.getSnapshotVersion());
62+
}
63+
}
64+
65+
@Test
66+
public void testGetPartitionSnapshotForHudiFallbackToTableTimestamp() throws Exception {
67+
Env env = Mockito.mock(Env.class);
68+
ExternalMetaCacheMgr metaCacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
69+
ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class);
70+
HMSExternalTable table = Mockito.mock(HMSExternalTable.class);
71+
HudiEngineCache hudiEngineCache = Mockito.mock(HudiEngineCache.class);
72+
73+
TablePartitionValues partitionValues = new TablePartitionValues();
74+
partitionValues.getPartitionNameToIdMap().put("p=1", 1L);
75+
partitionValues.getPartitionIdToNameMap().put(1L, "p=1");
76+
partitionValues.getIdToPartitionItem().put(1L, Mockito.mock(PartitionItem.class));
77+
partitionValues.getPartitionNameToLastModifiedMap().put("p=1", 0L);
78+
79+
Mockito.when(env.getExtMetaCacheMgr()).thenReturn(metaCacheMgr);
80+
Mockito.when(table.getCatalog()).thenReturn(catalog);
81+
Mockito.when(table.getDlaType()).thenReturn(DLAType.HUDI);
82+
Mockito.when(metaCacheMgr.getHudiEngineCache(catalog)).thenReturn(hudiEngineCache);
83+
Mockito.when(hudiEngineCache.getPartitionInfo(table, Optional.empty()))
84+
.thenReturn(new HudiEngineCache.HudiPartition(partitionValues));
85+
Mockito.when(hudiEngineCache.getSnapshot(table, Optional.empty()))
86+
.thenReturn(new HudiEngineCache.HudiSnapshotMeta(456L));
87+
88+
try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
89+
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
90+
MTMVSnapshotIf snapshot = EngineMtmvSupport.getPartitionSnapshot(table, "p=1", Optional.empty());
91+
Assert.assertTrue(snapshot instanceof MTMVTimestampSnapshot);
92+
Assert.assertEquals(456L, snapshot.getSnapshotVersion());
93+
}
94+
}
95+
}

0 commit comments

Comments
 (0)