Skip to content

Commit 4f3457b

Browse files
authored
[opt](analyze) support iceberg external table in analyze (#59477)
### What problem does this PR solve? Problem Summary: master #59473
1 parent 0296fae commit 4f3457b

File tree

3 files changed

+40
-11
lines changed

3 files changed

+40
-11
lines changed

fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -594,9 +594,7 @@ protected List<AutoAnalysisPendingJob> getPendingJobs(Map<TableNameInfo, Set<Pai
594594
for (Entry<TableNameInfo, Set<Pair<String, String>>> entry : jobMap.entrySet()) {
595595
TableNameInfo table = entry.getKey();
596596
if (tableNameInfo == null
597-
|| tableNameInfo.getCtl() == null && tableNameInfo.getDb() == null
598-
&& tableNameInfo.getTbl() == null
599-
|| tableNameInfo.equals(table)) {
597+
|| matchesFilter(tableNameInfo, table)) {
600598
result.add(new AutoAnalysisPendingJob(table.getCtl(),
601599
table.getDb(), table.getTbl(), entry.getValue(), priority));
602600
}
@@ -605,6 +603,29 @@ protected List<AutoAnalysisPendingJob> getPendingJobs(Map<TableNameInfo, Set<Pai
605603
return result;
606604
}
607605

606+
private boolean matchesFilter(TableNameInfo filter, TableNameInfo target) {
607+
if (StringUtils.isEmpty(filter.getCtl())
608+
&& StringUtils.isEmpty(filter.getDb())
609+
&& StringUtils.isEmpty(filter.getTbl())) {
610+
return true;
611+
}
612+
613+
if (!StringUtils.isEmpty(filter.getCtl())
614+
&& !filter.getCtl().equals(target.getCtl())) {
615+
return false;
616+
}
617+
if (!StringUtils.isEmpty(filter.getDb())
618+
&& !filter.getDb().equals(target.getDb())) {
619+
return false;
620+
}
621+
if (!StringUtils.isEmpty(filter.getTbl())
622+
&& !filter.getTbl().equals(target.getTbl())) {
623+
return false;
624+
}
625+
626+
return true;
627+
}
628+
608629
public List<AnalysisInfo> findAnalysisJobs(String state, String ctl, String db,
609630
String table, long jobId, boolean isAuto) {
610631
TableIf tbl = null;

fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@
2525
import org.apache.doris.common.DdlException;
2626
import org.apache.doris.common.Pair;
2727
import org.apache.doris.common.util.MasterDaemon;
28+
import org.apache.doris.datasource.ExternalTable;
2829
import org.apache.doris.datasource.hive.HMSExternalTable;
30+
import org.apache.doris.datasource.iceberg.IcebergExternalTable;
2931
import org.apache.doris.info.TableNameInfo;
3032
import org.apache.doris.persist.TableStatsDeletionLog;
3133
import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
@@ -148,6 +150,9 @@ protected void processOneJob(TableIf table, Set<Pair<String, String>> columns, J
148150
if (StatisticsUtil.enablePartitionAnalyze() && table.isPartitionedTable()) {
149151
analysisMethod = AnalysisMethod.FULL;
150152
}
153+
if (table instanceof ExternalTable) { // External table only support full analyze now
154+
analysisMethod = AnalysisMethod.FULL;
155+
}
151156
boolean isSampleAnalyze = analysisMethod.equals(AnalysisMethod.SAMPLE);
152157
OlapTable olapTable = table instanceof OlapTable ? (OlapTable) table : null;
153158
AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
@@ -229,9 +234,9 @@ protected boolean supportAutoAnalyze(TableIf tableIf) {
229234
if (tableIf == null) {
230235
return false;
231236
}
232-
return tableIf instanceof OlapTable
237+
return tableIf instanceof OlapTable || tableIf instanceof IcebergExternalTable
233238
|| tableIf instanceof HMSExternalTable
234-
&& ((HMSExternalTable) tableIf).getDlaType().equals(HMSExternalTable.DLAType.HIVE);
239+
&& ((HMSExternalTable) tableIf).getDlaType().equals(HMSExternalTable.DLAType.HIVE);
235240
}
236241

237242
protected AnalysisInfo createAnalyzeJobForTbl(TableIf table, Set<Pair<String, String>> jobColumns,

fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import org.apache.doris.datasource.InternalCatalog;
5656
import org.apache.doris.datasource.hive.HMSExternalTable;
5757
import org.apache.doris.datasource.hive.HMSExternalTable.DLAType;
58+
import org.apache.doris.datasource.iceberg.IcebergExternalTable;
5859
import org.apache.doris.info.TableNameInfo;
5960
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
6061
import org.apache.doris.nereids.trees.expressions.literal.IPv4Literal;
@@ -1125,15 +1126,17 @@ public static boolean needAnalyzeColumn(TableIf table, Pair<String, String> colu
11251126
// 3. Check partition
11261127
return needAnalyzePartition(olapTable, tableStatsStatus, columnStatsMeta);
11271128
} else {
1128-
// Now, we only support Hive external table auto analyze.
1129-
if (!(table instanceof HMSExternalTable)) {
1129+
if (!(table instanceof HMSExternalTable || (table instanceof IcebergExternalTable))) {
11301130
return false;
11311131
}
1132-
HMSExternalTable hmsTable = (HMSExternalTable) table;
1133-
if (!hmsTable.getDlaType().equals(DLAType.HIVE)) {
1134-
return false;
1132+
if (table instanceof HMSExternalTable) {
1133+
HMSExternalTable hmsTable = (HMSExternalTable) table;
1134+
if (!hmsTable.getDlaType().equals(DLAType.HIVE)) {
1135+
return false;
1136+
}
11351137
}
1136-
// External is hard to calculate change rate, use time interval to control analyze frequency.
1138+
// External is hard to calculate change rate, use time interval to control
1139+
// analyze frequency.
11371140
return System.currentTimeMillis()
11381141
- tableStatsStatus.lastAnalyzeTime > StatisticsUtil.getExternalTableAutoAnalyzeIntervalInMillis();
11391142
}

0 commit comments

Comments
 (0)