From 7e2d8d1c21f3182871507fd0ac75c4433005a993 Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Wed, 7 Jan 2026 09:56:54 +0800 Subject: [PATCH] [fix](catalog) update the table's last update time after related operations. (#59387) We should update the external table's last update time after operations like schema change, insert or refresh. So the the SQL cache feature can get the right time to validate the cache. Also merge `schemaUpdateTime` and `eventUpdateTime` into one `updateTime`, and move it to `ExternalTable`. So that all kinds of external table can use this field. --- .../java/org/apache/doris/alter/Alter.java | 7 ++- .../apache/doris/catalog/RefreshManager.java | 19 +++--- .../apache/doris/datasource/CatalogMgr.java | 4 +- .../doris/datasource/ExternalCatalog.java | 41 +++++++------ .../doris/datasource/ExternalObjectLog.java | 7 ++- .../doris/datasource/ExternalTable.java | 21 +++---- .../datasource/hive/HMSExternalDatabase.java | 2 +- .../datasource/hive/HMSExternalTable.java | 18 +----- .../datasource/hive/HiveMetadataOps.java | 4 +- .../iceberg/IcebergExternalCatalog.java | 20 +++--- .../iceberg/IcebergMetadataOps.java | 43 +++++++------ .../operations/ExternalMetadataOps.java | 18 +++--- .../datasource/test/TestExternalTable.java | 1 + .../plans/commands/ExecuteActionCommand.java | 6 +- .../commands/insert/HiveInsertExecutor.java | 7 ++- .../doris/persist/TruncateTableInfo.java | 9 ++- .../apache/doris/qe/cache/CacheAnalyzer.java | 2 +- .../doris/catalog/RefreshTableTest.java | 16 ++--- .../doris/external/hms/HmsCatalogTest.java | 15 ++--- .../apache/doris/qe/HmsQueryCacheTest.java | 12 ++-- .../iceberg/iceberg_branch_tag_operate.groovy | 13 +++- .../iceberg/iceberg_schema_change_ddl.groovy | 7 +++ .../test_external_table_update_time.groovy | 61 +++++++++++++++++++ 23 files changed, 218 insertions(+), 135 deletions(-) create mode 100644 regression-test/suites/external_table_p0/test_external_table_update_time.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index 9150c60879d5da..1c3f94cce68c95 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -392,6 +392,7 @@ private void setExternalTableAutoAnalyzePolicy(ExternalTable table, List alterClauses) throws UserException { + long updateTime = System.currentTimeMillis(); for (AlterClause alterClause : alterClauses) { if (alterClause instanceof ModifyTablePropertiesClause) { setExternalTableAutoAnalyzePolicy(table, alterClauses); @@ -438,7 +439,7 @@ private void processAlterTableForExternalTable( AddPartitionFieldClause addPartitionField = (AddPartitionFieldClause) alterClause; if (table instanceof IcebergExternalTable) { ((IcebergExternalCatalog) table.getCatalog()).addPartitionField( - (IcebergExternalTable) table, addPartitionField); + (IcebergExternalTable) table, addPartitionField, updateTime); } else { throw new UserException("ADD PARTITION KEY is only supported for Iceberg tables"); } @@ -446,7 +447,7 @@ private void processAlterTableForExternalTable( DropPartitionFieldClause dropPartitionField = (DropPartitionFieldClause) alterClause; if (table instanceof IcebergExternalTable) { ((IcebergExternalCatalog) table.getCatalog()).dropPartitionField( - (IcebergExternalTable) table, dropPartitionField); + (IcebergExternalTable) table, dropPartitionField, updateTime); } else { throw new UserException("DROP PARTITION KEY is only supported for Iceberg tables"); } @@ -454,7 +455,7 @@ private void processAlterTableForExternalTable( ReplacePartitionFieldClause replacePartitionField = (ReplacePartitionFieldClause) alterClause; if (table instanceof IcebergExternalTable) { ((IcebergExternalCatalog) table.getCatalog()).replacePartitionField( - (IcebergExternalTable) table, replacePartitionField); + (IcebergExternalTable) table, replacePartitionField, updateTime); } else { throw new UserException("REPLACE PARTITION KEY is only supported for Iceberg tables"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java index 2802055a8873c3..54237f2d38edb0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java @@ -146,10 +146,10 @@ public void handleRefreshTable(String catalogName, String dbName, String tableNa } return; } - refreshTableInternal((ExternalDatabase) db, (ExternalTable) table, 0); - + long updateTime = System.currentTimeMillis(); + refreshTableInternal((ExternalDatabase) db, (ExternalTable) table, updateTime); ExternalObjectLog log = ExternalObjectLog.createForRefreshTable(catalog.getId(), db.getFullName(), - table.getName()); + table.getName(), updateTime); Env.getCurrentEnv().getEditLog().logRefreshExternalTable(log); } @@ -194,6 +194,9 @@ public void replayRefreshTable(ExternalObjectLog log) { HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() .getMetaStoreCache((HMSExternalCatalog) catalog); cache.refreshAffectedPartitionsCache((HMSExternalTable) table.get(), modifiedPartNames, newPartNames); + if (table.get() instanceof HMSExternalTable && log.getLastUpdateTime() > 0) { + ((HMSExternalTable) table.get()).setUpdateTime(log.getLastUpdateTime()); + } LOG.info("replay refresh partitions for table {}, " + "modified partitions count: {}, " + "new partitions count: {}", @@ -229,12 +232,12 @@ public void refreshExternalTableFromEvent(String catalogName, String dbName, Str public void refreshTableInternal(ExternalDatabase db, ExternalTable table, long updateTime) { table.unsetObjectCreated(); - if (table instanceof HMSExternalTable && updateTime > 0) { - ((HMSExternalTable) table).setEventUpdateTime(updateTime); + if (updateTime > 0) { + table.setUpdateTime(updateTime); } Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(table); - LOG.info("refresh table {}, id {} from db {} in catalog {}", - table.getName(), table.getId(), db.getFullName(), db.getCatalog().getName()); + LOG.info("refresh table {}, id {} from db {} in catalog {}, update time: {}", + table.getName(), table.getId(), db.getFullName(), db.getCatalog().getName(), updateTime); } // Refresh partition @@ -268,7 +271,7 @@ public void refreshPartitions(String catalogName, String dbName, String tableNam } Env.getCurrentEnv().getExtMetaCacheMgr().invalidatePartitionsCache((ExternalTable) table, partitionNames); - ((HMSExternalTable) table).setEventUpdateTime(updateTime); + ((HMSExternalTable) table).setUpdateTime(updateTime); } public void addToRefreshMap(long catalogId, Integer[] sec) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 692d32c5931afc..16eeebd7dffab8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -726,7 +726,7 @@ public void addExternalPartitions(String catalogName, String dbName, String tabl HMSExternalTable hmsTable = (HMSExternalTable) table; Env.getCurrentEnv().getExtMetaCacheMgr().addPartitionsCache(catalog.getId(), hmsTable, partitionNames); - hmsTable.setEventUpdateTime(updateTime); + hmsTable.setUpdateTime(updateTime); } public void dropExternalPartitions(String catalogName, String dbName, String tableName, @@ -757,7 +757,7 @@ public void dropExternalPartitions(String catalogName, String dbName, String tab HMSExternalTable hmsTable = (HMSExternalTable) table; Env.getCurrentEnv().getExtMetaCacheMgr().dropPartitionsCache(catalog.getId(), hmsTable, partitionNames); - hmsTable.setEventUpdateTime(updateTime); + hmsTable.setUpdateTime(updateTime); } public void registerCatalogRefreshListener(Env env) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 629af2c2e26a36..c2f5a3e9c7079a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -1137,8 +1137,9 @@ public void truncateTable(String dbName, String tableName, PartitionNamesInfo pa partitions = partitionNamesInfo.getPartitionNames(); } ExternalTable dorisTable = getDbOrDdlException(dbName).getTableOrDdlException(tableName); - metadataOps.truncateTable(dorisTable, partitions); - TruncateTableInfo info = new TruncateTableInfo(getName(), dbName, tableName, partitions); + long updateTime = System.currentTimeMillis(); + metadataOps.truncateTable(dorisTable, partitions, updateTime); + TruncateTableInfo info = new TruncateTableInfo(getName(), dbName, tableName, partitions, updateTime); Env.getCurrentEnv().getEditLog().logTruncateTable(info); } catch (Exception e) { LOG.warn("Failed to truncate table {}.{} in catalog {}", dbName, tableName, getName(), e); @@ -1148,7 +1149,7 @@ public void truncateTable(String dbName, String tableName, PartitionNamesInfo pa public void replayTruncateTable(TruncateTableInfo info) { if (metadataOps != null) { - metadataOps.afterTruncateTable(info.getDb(), info.getTable()); + metadataOps.afterTruncateTable(info.getDb(), info.getTable(), info.getUpdateTime()); } } @@ -1320,11 +1321,11 @@ public void resetMetaCacheNames() { } // log the refresh external table operation - private void logRefreshExternalTable(ExternalTable dorisTable) { + private void logRefreshExternalTable(ExternalTable dorisTable, long updateTime) { Env.getCurrentEnv().getEditLog() .logRefreshExternalTable( ExternalObjectLog.createForRefreshTable(dorisTable.getCatalog().getId(), - dorisTable.getDbName(), dorisTable.getName())); + dorisTable.getDbName(), dorisTable.getName(), updateTime)); } @Override @@ -1336,8 +1337,9 @@ public void addColumn(TableIf dorisTable, Column column, ColumnPosition position throw new DdlException("Add column operation is not supported for catalog: " + getName()); } try { - metadataOps.addColumn(externalTable, column, position); - logRefreshExternalTable(externalTable); + long updateTime = System.currentTimeMillis(); + metadataOps.addColumn(externalTable, column, position, updateTime); + logRefreshExternalTable(externalTable, updateTime); } catch (Exception e) { LOG.warn("Failed to add column {} to table {}.{} in catalog {}", column.getName(), externalTable.getDbName(), externalTable.getName(), getName(), e); @@ -1354,8 +1356,9 @@ public void addColumns(TableIf dorisTable, List columns) throws UserExce throw new DdlException("Add columns operation is not supported for catalog: " + getName()); } try { - metadataOps.addColumns(externalTable, columns); - logRefreshExternalTable(externalTable); + long updateTime = System.currentTimeMillis(); + metadataOps.addColumns(externalTable, columns, updateTime); + logRefreshExternalTable(externalTable, updateTime); } catch (Exception e) { LOG.warn("Failed to add columns to table {}.{} in catalog {}", externalTable.getDbName(), externalTable.getName(), getName(), e); @@ -1372,8 +1375,9 @@ public void dropColumn(TableIf dorisTable, String columnName) throws UserExcepti throw new DdlException("Drop column operation is not supported for catalog: " + getName()); } try { - metadataOps.dropColumn(externalTable, columnName); - logRefreshExternalTable(externalTable); + long updateTime = System.currentTimeMillis(); + metadataOps.dropColumn(externalTable, columnName, updateTime); + logRefreshExternalTable(externalTable, updateTime); } catch (Exception e) { LOG.warn("Failed to drop column {} from table {}.{} in catalog {}", columnName, externalTable.getDbName(), externalTable.getName(), getName(), e); @@ -1390,8 +1394,9 @@ public void renameColumn(TableIf dorisTable, String oldName, String newName) thr throw new DdlException("Rename column operation is not supported for catalog: " + getName()); } try { - metadataOps.renameColumn(externalTable, oldName, newName); - logRefreshExternalTable(externalTable); + long updateTime = System.currentTimeMillis(); + metadataOps.renameColumn(externalTable, oldName, newName, updateTime); + logRefreshExternalTable(externalTable, updateTime); } catch (Exception e) { LOG.warn("Failed to rename column {} to {} in table {}.{} in catalog {}", oldName, newName, externalTable.getDbName(), externalTable.getName(), getName(), e); @@ -1408,8 +1413,9 @@ public void modifyColumn(TableIf dorisTable, Column column, ColumnPosition colum throw new DdlException("Modify column operation is not supported for catalog: " + getName()); } try { - metadataOps.modifyColumn(externalTable, column, columnPosition); - logRefreshExternalTable(externalTable); + long updateTime = System.currentTimeMillis(); + metadataOps.modifyColumn(externalTable, column, columnPosition, updateTime); + logRefreshExternalTable(externalTable, updateTime); } catch (Exception e) { LOG.warn("Failed to modify column {} in table {}.{} in catalog {}", column.getName(), externalTable.getDbName(), externalTable.getName(), getName(), e); @@ -1426,8 +1432,9 @@ public void reorderColumns(TableIf dorisTable, List newOrder) throws Use throw new DdlException("Reorder columns operation is not supported for catalog: " + getName()); } try { - metadataOps.reorderColumns(externalTable, newOrder); - logRefreshExternalTable(externalTable); + long updateTime = System.currentTimeMillis(); + metadataOps.reorderColumns(externalTable, newOrder, updateTime); + logRefreshExternalTable(externalTable, updateTime); } catch (Exception e) { LOG.warn("Failed to reorder columns in table {}.{} in catalog {}", externalTable.getDbName(), externalTable.getName(), getName(), e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java index 43a0675d783c41..84f0605b8d8557 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java @@ -75,22 +75,25 @@ public static ExternalObjectLog createForRefreshDb(long catalogId, String dbName return externalObjectLog; } - public static ExternalObjectLog createForRefreshTable(long catalogId, String dbName, String tblName) { + public static ExternalObjectLog createForRefreshTable(long catalogId, String dbName, String tblName, + long updateTime) { ExternalObjectLog externalObjectLog = new ExternalObjectLog(); externalObjectLog.setCatalogId(catalogId); externalObjectLog.setDbName(dbName); externalObjectLog.setTableName(tblName); + externalObjectLog.setLastUpdateTime(updateTime); return externalObjectLog; } public static ExternalObjectLog createForRefreshPartitions(long catalogId, String dbName, String tblName, - List modifiedPartNames, List newPartNames) { + List modifiedPartNames, List newPartNames, long updateTime) { ExternalObjectLog externalObjectLog = new ExternalObjectLog(); externalObjectLog.setCatalogId(catalogId); externalObjectLog.setDbName(dbName); externalObjectLog.setTableName(tblName); externalObjectLog.setPartitionNames(modifiedPartNames); externalObjectLog.setNewPartitionNames(newPartNames); + externalObjectLog.setLastUpdateTime(updateTime); return externalObjectLog; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java index cf74ef2fb54898..4a81bb3c7f7193 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java @@ -85,8 +85,8 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable { @SerializedName(value = "ta") private final TableAttributes tableAttributes = new TableAttributes(); - // this field will be refreshed after reloading schema - protected volatile long schemaUpdateTime; + // record the table update time, like insert/alter/delete + protected volatile long updateTime = 0; protected long dbId; protected boolean objectCreated; @@ -276,16 +276,15 @@ public long getCreateTime() { return 0; } - // return schema update time as default - // override this method if there is some other kinds of update time - // use getSchemaUpdateTime if just need the schema update time @Override + // Returns the table update time, tracking when the table was last modified + // (for example, by insert, alter, or refresh operations). public long getUpdateTime() { - return this.schemaUpdateTime; + return updateTime; } - public void setUpdateTime(long schemaUpdateTime) { - this.schemaUpdateTime = schemaUpdateTime; + public void setUpdateTime(long updateTime) { + this.updateTime = updateTime; } @Override @@ -345,7 +344,7 @@ public Optional getColumnStatistic(String colName) { * @return */ public Optional initSchemaAndUpdateTime(SchemaCacheKey key) { - schemaUpdateTime = System.currentTimeMillis(); + setUpdateTime(System.currentTimeMillis()); return initSchema(key); } @@ -484,10 +483,6 @@ public int hashCode() { return Objects.hashCode(name, db); } - public long getSchemaUpdateTime() { - return schemaUpdateTime; - } - public long getDbId() { return dbId; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java index 467887fd476de3..ab0884d0ce1637 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java @@ -51,7 +51,7 @@ public boolean registerTable(TableIf tableIf) { super.registerTable(tableIf); HMSExternalTable table = getTableNullable(tableIf.getName()); if (table != null) { - table.setEventUpdateTime(tableIf.getUpdateTime()); + table.setUpdateTime(tableIf.getUpdateTime()); } return true; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index 8e8556c8716976..47ed5ff6a6a8d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -190,9 +190,6 @@ public class HMSExternalTable extends ExternalTable implements MTMVRelatedTableI private HMSDlaTable dlaTable; - // record the event update time when enable hms event listener - protected volatile long eventUpdateTime; - public enum DLAType { UNKNOWN, HIVE, HUDI, ICEBERG } @@ -648,11 +645,11 @@ public Set getPartitionNames() { public Optional initSchemaAndUpdateTime(SchemaCacheKey key) { Table table = loadHiveTable(); // try to use transient_lastDdlTime from hms client - schemaUpdateTime = MapUtils.isNotEmpty(table.getParameters()) + setUpdateTime(MapUtils.isNotEmpty(table.getParameters()) && table.getParameters().containsKey(TBL_PROP_TRANSIENT_LAST_DDL_TIME) ? Long.parseLong(table.getParameters().get(TBL_PROP_TRANSIENT_LAST_DDL_TIME)) * 1000 // use current timestamp if lastDdlTime does not exist (hive views don't have this prop) - : System.currentTimeMillis(); + : System.currentTimeMillis()); return initSchema(key); } @@ -902,17 +899,6 @@ private void setStatData(Column col, ColumnStatisticsData data, ColumnStatisticB builder.setMaxValue(Double.POSITIVE_INFINITY); } - public void setEventUpdateTime(long updateTime) { - this.eventUpdateTime = updateTime; - } - - @Override - // get the max value of `schemaUpdateTime` and `eventUpdateTime` - // eventUpdateTime will be refreshed after processing events with hms event listener enabled - public long getUpdateTime() { - return Math.max(this.schemaUpdateTime, this.eventUpdateTime); - } - @Override public void gsonPostProcess() throws IOException { super.gsonPostProcess(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java index b1ede3926be5c4..7d8501f449fb50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java @@ -337,7 +337,7 @@ public void truncateTableImpl(ExternalTable dorisTable, List partitions) } @Override - public void afterTruncateTable(String dbName, String tblName) { + public void afterTruncateTable(String dbName, String tblName, long updateTime) { try { // Invalidate cache. Optional> db = catalog.getDbForReplay(dbName); @@ -345,7 +345,7 @@ public void afterTruncateTable(String dbName, String tblName) { Optional tbl = db.get().getTableForReplay(tblName); if (tbl.isPresent()) { Env.getCurrentEnv().getRefreshManager() - .refreshTableInternal(db.get(), (ExternalTable) tbl.get(), 0); + .refreshTableInternal(db.get(), (ExternalTable) tbl.get(), updateTime); } } } catch (Exception e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index 25bf959bdd5b00..76cd2776ab2d75 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -231,46 +231,48 @@ public boolean viewExists(String dbName, String viewName) { /** * Add partition field to Iceberg table for partition evolution */ - public void addPartitionField(IcebergExternalTable table, AddPartitionFieldClause clause) throws UserException { + public void addPartitionField(IcebergExternalTable table, AddPartitionFieldClause clause, long updateTime) + throws UserException { makeSureInitialized(); if (metadataOps == null) { throw new UserException("Add partition field operation is not supported for catalog: " + getName()); } - ((IcebergMetadataOps) metadataOps).addPartitionField(table, clause); + ((IcebergMetadataOps) metadataOps).addPartitionField(table, clause, updateTime); Env.getCurrentEnv().getEditLog() .logRefreshExternalTable( ExternalObjectLog.createForRefreshTable(table.getCatalog().getId(), - table.getDbName(), table.getName())); + table.getDbName(), table.getName(), updateTime)); } /** * Drop partition field from Iceberg table for partition evolution */ - public void dropPartitionField(IcebergExternalTable table, DropPartitionFieldClause clause) throws UserException { + public void dropPartitionField(IcebergExternalTable table, DropPartitionFieldClause clause, long updateTime) + throws UserException { makeSureInitialized(); if (metadataOps == null) { throw new UserException("Drop partition field operation is not supported for catalog: " + getName()); } - ((IcebergMetadataOps) metadataOps).dropPartitionField(table, clause); + ((IcebergMetadataOps) metadataOps).dropPartitionField(table, clause, updateTime); Env.getCurrentEnv().getEditLog() .logRefreshExternalTable( ExternalObjectLog.createForRefreshTable(table.getCatalog().getId(), - table.getDbName(), table.getName())); + table.getDbName(), table.getName(), updateTime)); } /** * Replace partition field in Iceberg table for partition evolution */ public void replacePartitionField(IcebergExternalTable table, - ReplacePartitionFieldClause clause) throws UserException { + ReplacePartitionFieldClause clause, long updateTime) throws UserException { makeSureInitialized(); if (metadataOps == null) { throw new UserException("Replace partition field operation is not supported for catalog: " + getName()); } - ((IcebergMetadataOps) metadataOps).replacePartitionField(table, clause); + ((IcebergMetadataOps) metadataOps).replacePartitionField(table, clause, updateTime); Env.getCurrentEnv().getEditLog() .logRefreshExternalTable( ExternalObjectLog.createForRefreshTable(table.getCatalog().getId(), - table.getDbName(), table.getName())); + table.getDbName(), table.getName(), updateTime)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index c52c5068003246..c2795a76706e31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -623,19 +623,19 @@ private void applyPosition(UpdateSchema updateSchema, ColumnPosition position, S } } - private void refreshTable(ExternalTable dorisTable) { + private void refreshTable(ExternalTable dorisTable, long updateTime) { Optional> db = dorisCatalog.getDbForReplay(dorisTable.getRemoteDbName()); if (db.isPresent()) { Optional tbl = db.get().getTableForReplay(dorisTable.getRemoteName()); if (tbl.isPresent()) { Env.getCurrentEnv().getRefreshManager() - .refreshTableInternal(db.get(), (ExternalTable) tbl.get(), System.currentTimeMillis()); + .refreshTableInternal(db.get(), (ExternalTable) tbl.get(), updateTime); } } } @Override - public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position) + public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { validateCommonColumnInfo(column); Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); @@ -650,11 +650,11 @@ public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition po throw new UserException("Failed to add column: " + column.getName() + " to table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); } - refreshTable(dorisTable); + refreshTable(dorisTable, updateTime); } @Override - public void addColumns(ExternalTable dorisTable, List columns) throws UserException { + public void addColumns(ExternalTable dorisTable, List columns, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); UpdateSchema updateSchema = icebergTable.updateSchema(); for (Column column : columns) { @@ -667,11 +667,11 @@ public void addColumns(ExternalTable dorisTable, List columns) throws Us throw new UserException("Failed to add columns to table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); } - refreshTable(dorisTable); + refreshTable(dorisTable, updateTime); } @Override - public void dropColumn(ExternalTable dorisTable, String columnName) throws UserException { + public void dropColumn(ExternalTable dorisTable, String columnName, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); UpdateSchema updateSchema = icebergTable.updateSchema(); updateSchema.deleteColumn(columnName); @@ -681,11 +681,12 @@ public void dropColumn(ExternalTable dorisTable, String columnName) throws UserE throw new UserException("Failed to drop column: " + columnName + " from table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); } - refreshTable(dorisTable); + refreshTable(dorisTable, updateTime); } @Override - public void renameColumn(ExternalTable dorisTable, String oldName, String newName) throws UserException { + public void renameColumn(ExternalTable dorisTable, String oldName, String newName, long updateTime) + throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); UpdateSchema updateSchema = icebergTable.updateSchema(); updateSchema.renameColumn(oldName, newName); @@ -695,11 +696,11 @@ public void renameColumn(ExternalTable dorisTable, String oldName, String newNam throw new UserException("Failed to rename column: " + oldName + " to " + newName + " in table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); } - refreshTable(dorisTable); + refreshTable(dorisTable, updateTime); } @Override - public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition position) + public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); validateForModifyColumn(column, icebergTable); @@ -720,7 +721,7 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition throw new UserException("Failed to modify column: " + column.getName() + " in table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); } - refreshTable(dorisTable); + refreshTable(dorisTable, updateTime); } private void validateForModifyColumn(Column column, Table icebergTable) throws UserException { @@ -752,7 +753,7 @@ private void validateCommonColumnInfo(Column column) throws UserException { } @Override - public void reorderColumns(ExternalTable dorisTable, List newOrder) throws UserException { + public void reorderColumns(ExternalTable dorisTable, List newOrder, long updateTime) throws UserException { if (newOrder == null || newOrder.isEmpty()) { throw new UserException("Reorder column failed, new order is empty."); } @@ -768,7 +769,7 @@ public void reorderColumns(ExternalTable dorisTable, List newOrder) thro throw new UserException("Failed to reorder columns in table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); } - refreshTable(dorisTable); + refreshTable(dorisTable, updateTime); } public ExecutionAuthenticator getExecutionAuthenticator() { @@ -810,7 +811,8 @@ private Term getTransform(String transformName, String columnName, Integer trans /** * Add partition field to Iceberg table for partition evolution */ - public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldClause clause) throws UserException { + public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldClause clause, long updateTime) + throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); @@ -832,7 +834,7 @@ public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldClause throw new UserException("Failed to add partition field to table: " + icebergTable.name() + ", error message is: " + ExceptionUtils.getRootCauseMessage(e), e); } - refreshTable(dorisTable); + refreshTable(dorisTable, updateTime); // Reset cached isValidRelatedTable flag after partition evolution ((IcebergExternalTable) dorisTable).setIsValidRelatedTableCached(false); } @@ -840,7 +842,8 @@ public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldClause /** * Drop partition field from Iceberg table for partition evolution */ - public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldClause clause) throws UserException { + public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldClause clause, long updateTime) + throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); @@ -860,7 +863,7 @@ public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldClaus throw new UserException("Failed to drop partition field from table: " + icebergTable.name() + ", error message is: " + ExceptionUtils.getRootCauseMessage(e), e); } - refreshTable(dorisTable); + refreshTable(dorisTable, updateTime); // Reset cached isValidRelatedTable flag after partition evolution ((IcebergExternalTable) dorisTable).setIsValidRelatedTableCached(false); } @@ -868,7 +871,7 @@ public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldClaus /** * Replace partition field in Iceberg table for partition evolution */ - public void replacePartitionField(ExternalTable dorisTable, ReplacePartitionFieldClause clause) + public void replacePartitionField(ExternalTable dorisTable, ReplacePartitionFieldClause clause, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); @@ -903,7 +906,7 @@ public void replacePartitionField(ExternalTable dorisTable, ReplacePartitionFiel throw new UserException("Failed to replace partition field in table: " + icebergTable.name() + ", error message is: " + ExceptionUtils.getRootCauseMessage(e), e); } - refreshTable(dorisTable); + refreshTable(dorisTable, updateTime); // Reset cached isValidRelatedTable flag after partition evolution ((IcebergExternalTable) dorisTable).setIsValidRelatedTableCached(false); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java index 35fcea590486e6..3407e8e7cec09b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java @@ -140,14 +140,14 @@ default void afterRenameTable(String dbName, String oldName, String newName) { * @param dorisTable * @param partitions */ - default void truncateTable(ExternalTable dorisTable, List partitions) throws DdlException { + default void truncateTable(ExternalTable dorisTable, List partitions, long updateTime) throws DdlException { truncateTableImpl(dorisTable, partitions); - afterTruncateTable(dorisTable.getDbName(), dorisTable.getName()); + afterTruncateTable(dorisTable.getDbName(), dorisTable.getName(), updateTime); } void truncateTableImpl(ExternalTable dorisTable, List partitions) throws DdlException; - default void afterTruncateTable(String dbName, String tblName) { + default void afterTruncateTable(String dbName, String tblName, long updateTime) { } /** @@ -223,7 +223,7 @@ default void dropBranch(ExternalTable dorisTable, DropBranchInfo branchInfo) * @param position * @throws UserException */ - default void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position) + default void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { throw new UnsupportedOperationException("Add column operation is not supported for this table type."); } @@ -235,7 +235,7 @@ default void addColumn(ExternalTable dorisTable, Column column, ColumnPosition p * @param columns * @throws UserException */ - default void addColumns(ExternalTable dorisTable, List columns) + default void addColumns(ExternalTable dorisTable, List columns, long updateTime) throws UserException { throw new UnsupportedOperationException("Add columns operation is not supported for this table type."); } @@ -247,7 +247,7 @@ default void addColumns(ExternalTable dorisTable, List columns) * @param columnName * @throws UserException */ - default void dropColumn(ExternalTable dorisTable, String columnName) + default void dropColumn(ExternalTable dorisTable, String columnName, long updateTime) throws UserException { throw new UnsupportedOperationException("Drop column operation is not supported for this table type."); } @@ -260,7 +260,7 @@ default void dropColumn(ExternalTable dorisTable, String columnName) * @param newName * @throws UserException */ - default void renameColumn(ExternalTable dorisTable, String oldName, String newName) + default void renameColumn(ExternalTable dorisTable, String oldName, String newName, long updateTime) throws UserException { throw new UnsupportedOperationException("Rename column operation is not supported for this table type."); } @@ -273,7 +273,7 @@ default void renameColumn(ExternalTable dorisTable, String oldName, String newNa * @param position * @throws UserException */ - default void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition position) + default void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { throw new UnsupportedOperationException("Modify column operation is not supported for this table type."); } @@ -285,7 +285,7 @@ default void modifyColumn(ExternalTable dorisTable, Column column, ColumnPositio * @param newOrder * @throws UserException */ - default void reorderColumns(ExternalTable dorisTable, List newOrder) + default void reorderColumns(ExternalTable dorisTable, List newOrder, long updateTime) throws UserException { throw new UnsupportedOperationException("Reorder columns operation is not supported for this table type."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java index 6d08b10403bb76..d3277fe0756137 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java @@ -41,6 +41,7 @@ public TestExternalTable(long id, String name, String remoteName, TestExternalCa public synchronized void makeSureInitialized() { super.makeSureInitialized(); this.objectCreated = true; + setUpdateTime(System.currentTimeMillis()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteActionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteActionCommand.java index e23d63b3aaab2c..394b4c25751a25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteActionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteActionCommand.java @@ -105,7 +105,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { action.validate(tableNameInfo, ctx.getCurrentUserIdentity()); ResultSet resultSet = action.execute(table); - logRefreshTable(table); + logRefreshTable(table, System.currentTimeMillis()); if (resultSet != null) { executor.sendResultSet(resultSet); } @@ -150,7 +150,7 @@ public Optional getWhereCondition() { * @param table the table to log * @throws UserException if the table type is not supported */ - private void logRefreshTable(TableIf table) throws UserException { + private void logRefreshTable(TableIf table, long updateTime) throws UserException { if (table instanceof ExternalTable) { ExternalTable externalTable = (ExternalTable) table; Env.getCurrentEnv().getEditLog() @@ -158,7 +158,7 @@ private void logRefreshTable(TableIf table) throws UserException { ExternalObjectLog.createForRefreshTable( externalTable.getCatalog().getId(), externalTable.getDbName(), - externalTable.getName())); + externalTable.getName(), updateTime)); } else { // support more table in future throw new UserException("Unsupported table type: " + table.getClass().getName() + " for refresh table"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java index 9f8be52afec30d..fef4d283165cbe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java @@ -98,6 +98,8 @@ protected void doAfterCommit() throws DdlException { } // Write edit log to notify other FEs + long updateTime = System.currentTimeMillis(); + hmsTable.setUpdateTime(updateTime); ExternalObjectLog log; if (!modifiedPartNames.isEmpty() || !newPartNames.isEmpty()) { // Partition-level refresh for other FEs @@ -106,13 +108,14 @@ protected void doAfterCommit() throws DdlException { table.getDatabase().getFullName(), table.getName(), modifiedPartNames, - newPartNames); + newPartNames, + updateTime); } else { // Full table refresh for other FEs log = ExternalObjectLog.createForRefreshTable( hmsTable.getCatalog().getId(), table.getDatabase().getFullName(), - table.getName()); + table.getName(), updateTime); } Env.getCurrentEnv().getEditLog().logRefreshExternalTable(log); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java index b9d77ca6818444..b846d1acbdc1a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java @@ -58,6 +58,8 @@ public class TruncateTableInfo implements Writable { private boolean force = true; // older version it was forced always. @SerializedName(value = "ur") private Map updateRecords; + @SerializedName(value = "ut") + private long updateTime; public TruncateTableInfo() { @@ -82,11 +84,12 @@ public TruncateTableInfo(long dbId, String db, long tblId, String table, List partNames) { + public TruncateTableInfo(String ctl, String db, String table, List partNames, long updateTime) { this.ctl = ctl; this.db = db; this.table = table; this.extPartNames = partNames; + this.updateTime = updateTime; } public String getCtl() { @@ -137,6 +140,10 @@ public Map getUpdateRecords() { return updateRecords; } + public long getUpdateTime() { + return updateTime; + } + public static TruncateTableInfo read(DataInput in) throws IOException { String json = Text.readString(in); return GsonUtils.GSON.fromJson(json, TruncateTableInfo.class); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java index 32703c682a304a..c9dd169bc78a12 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java @@ -136,7 +136,7 @@ private void checkCacheConfig() { enableSqlCache = true; } } - // alread remove the entrance of partition cache, so we force set to false + // already remove the entrance of partition cache, so we force set to false enablePartitionCache = false; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java index 0a1cd323d8fb9b..a5f2465c6e734f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java @@ -88,12 +88,12 @@ public void testRefreshTable() throws Exception { CatalogIf test1 = env.getCatalogMgr().getCatalog("test1"); TestExternalTable table = (TestExternalTable) test1.getDbNullable("db1").getTable("tbl11").get(); Assertions.assertFalse(table.isObjectCreated()); - long l1 = table.getSchemaUpdateTime(); - Assertions.assertTrue(l1 == 0); + long l1 = table.getUpdateTime(); + // Assertions.assertEquals(0, l1); table.makeSureInitialized(); Assertions.assertTrue(table.isObjectCreated()); - long l2 = table.getSchemaUpdateTime(); - Assertions.assertTrue(l2 == l1); + long l2 = table.getUpdateTime(); + Assertions.assertTrue(l2 >= l1); TableNameInfo tableNameInfo = new TableNameInfo("test1", "db1", "tbl11"); try { Env.getCurrentEnv().getRefreshManager() @@ -102,15 +102,15 @@ public void testRefreshTable() throws Exception { // Do nothing } Assertions.assertFalse(table.isObjectCreated()); - long l3 = table.getSchemaUpdateTime(); - Assertions.assertTrue(l3 == l2); + long l3 = table.getUpdateTime(); + Assertions.assertTrue(l3 >= l2); table.getFullSchema(); // only table.getFullSchema() can change table.lastUpdateTime - long l4 = table.getSchemaUpdateTime(); + long l4 = table.getUpdateTime(); Assertions.assertTrue(l4 > l3); // updateTime is equal to schema update time as default long l5 = table.getUpdateTime(); - Assertions.assertTrue(l5 == l4); + Assertions.assertTrue(l5 >= l4); // external info schema db ExternalInfoSchemaDatabase infoDb = (ExternalInfoSchemaDatabase) test1.getDbNullable(InfoSchemaDb.DATABASE_NAME); diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java index 7d358073ab89fa..43d5669e6ad74a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java @@ -106,8 +106,7 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { Deencapsulation.setField(db, "initialized", true); Deencapsulation.setField(tbl, "objectCreated", true); - Deencapsulation.setField(tbl, "schemaUpdateTime", NOW); - Deencapsulation.setField(tbl, "eventUpdateTime", 0); + Deencapsulation.setField(tbl, "updateTime", NOW); Deencapsulation.setField(tbl, "catalog", hmsCatalog); Deencapsulation.setField(tbl, "dbName", "hms_db"); Deencapsulation.setField(tbl, "name", "hms_tbl"); @@ -163,8 +162,7 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { }; Deencapsulation.setField(view1, "objectCreated", true); - Deencapsulation.setField(view1, "schemaUpdateTime", NOW); - Deencapsulation.setField(view1, "eventUpdateTime", 0); + Deencapsulation.setField(view1, "updateTime", NOW); Deencapsulation.setField(view1, "catalog", hmsCatalog); Deencapsulation.setField(view1, "dbName", "hms_db"); Deencapsulation.setField(view1, "name", "hms_view1"); @@ -215,8 +213,7 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { }; Deencapsulation.setField(view2, "objectCreated", true); - Deencapsulation.setField(view2, "schemaUpdateTime", NOW); - Deencapsulation.setField(view2, "eventUpdateTime", 0); + Deencapsulation.setField(view2, "updateTime", NOW); Deencapsulation.setField(view2, "catalog", hmsCatalog); Deencapsulation.setField(view2, "dbName", "hms_db"); Deencapsulation.setField(view2, "name", "hms_view2"); @@ -268,8 +265,7 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { }; Deencapsulation.setField(view3, "objectCreated", true); - Deencapsulation.setField(view3, "schemaUpdateTime", NOW); - Deencapsulation.setField(view3, "eventUpdateTime", 0); + Deencapsulation.setField(view3, "updateTime", NOW); Deencapsulation.setField(view3, "catalog", hmsCatalog); Deencapsulation.setField(view3, "dbName", "hms_db"); Deencapsulation.setField(view3, "name", "hms_view3"); @@ -321,8 +317,7 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { }; Deencapsulation.setField(view4, "objectCreated", true); - Deencapsulation.setField(view4, "schemaUpdateTime", NOW); - Deencapsulation.setField(view4, "eventUpdateTime", 0); + Deencapsulation.setField(view4, "updateTime", NOW); Deencapsulation.setField(view4, "catalog", hmsCatalog); Deencapsulation.setField(view4, "dbName", "hms_db"); Deencapsulation.setField(view4, "name", "hms_view4"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java index bbca8c183f1539..2e69f7e31f9905 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java @@ -136,8 +136,7 @@ private void init(HMSExternalCatalog hmsCatalog) { setField(db, "initialized", true); setField(tbl, "objectCreated", true); - setField(tbl, "schemaUpdateTime", NOW); - setField(tbl, "eventUpdateTime", 0); + setField(tbl, "updateTime", NOW); setField(tbl, "catalog", hmsCatalog); setField(tbl, "dbName", "hms_db"); setField(tbl, "name", "hms_tbl"); @@ -159,8 +158,7 @@ private void init(HMSExternalCatalog hmsCatalog) { .thenReturn(Optional.empty()); setField(tbl2, "objectCreated", true); - setField(tbl2, "schemaUpdateTime", NOW); - setField(tbl2, "eventUpdateTime", 0); + setField(tbl2, "updateTime", NOW); setField(tbl2, "catalog", hmsCatalog); setField(tbl2, "dbName", "hms_db"); setField(tbl2, "name", "hms_tbl2"); @@ -178,11 +176,11 @@ private void init(HMSExternalCatalog hmsCatalog) { Mockito.when(tbl2.getDatabase()).thenReturn(db); Mockito.when(tbl2.getSupportedSysTables()).thenReturn(SupportedSysTables.HIVE_SUPPORTED_SYS_TABLES); Mockito.when(tbl2.getUpdateTime()).thenReturn(NOW); - Mockito.when(tbl2.getSchemaUpdateTime()).thenReturn(NOW); + Mockito.when(tbl2.getUpdateTime()).thenReturn(NOW); // mock initSchemaAndUpdateTime and do nothing Mockito.when(tbl2.initSchemaAndUpdateTime(Mockito.any(ExternalSchemaCache.SchemaCacheKey.class))) .thenReturn(Optional.empty()); - Mockito.doNothing().when(tbl2).setEventUpdateTime(Mockito.anyLong()); + Mockito.doNothing().when(tbl2).setUpdateTime(Mockito.anyLong()); setField(view1, "objectCreated", true); @@ -259,7 +257,7 @@ public void testHitSqlCacheByNereidsAfterPartitionUpdateTimeChanged() { SqlCache sqlCache1 = (SqlCache) ca.getCache(); // latestTime is equals to the schema update time if not set partition update time - Assert.assertEquals(tbl2.getSchemaUpdateTime(), sqlCache1.getLatestTime()); + Assert.assertEquals(tbl2.getUpdateTime(), sqlCache1.getLatestTime()); // wait a second and set partition update time try { diff --git a/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy b/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy index f5d38350e7a1fb..b8838eadbaa269 100644 --- a/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy +++ b/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy @@ -53,6 +53,9 @@ suite("iceberg_branch_tag_operate", "p0,external,doris,external_docker,external_ sql """ alter table test_branch_tag_operate create branch b1 """ sql """ alter table test_branch_tag_operate create branch if not exists b1 """ + def result = sql """select UPDATE_TIME from information_schema.tables where TABLE_SCHEMA="test_db" and TABLE_NAME='test_branch_tag_operate'""" + def update_time1 = result[0][0]; + sleep(1000) test { sql """ alter table test_branch_tag_operate create or replace branch b1 """ @@ -72,7 +75,11 @@ suite("iceberg_branch_tag_operate", "p0,external,doris,external_docker,external_ sql """ insert into test_branch_tag_operate values (3) """ sql """ insert into test_branch_tag_operate values (4) """ sql """ insert into test_branch_tag_operate values (5) """ - + result = sql """select UPDATE_TIME from information_schema.tables where TABLE_SCHEMA="test_db" and TABLE_NAME='test_branch_tag_operate'""" + def update_time2 = result[0][0]; + logger.info("get update times " + update_time1 + " vs. " + update_time2) + assertTrue(update_time2 > update_time1); + sleep(1000) List> snapshots = sql """ select snapshot_id from iceberg_meta("table" = "${catalog_name}.test_db.test_branch_tag_operate", "query_type" = "snapshots") order by committed_at; """ String s0 = snapshots.get(0)[0] @@ -137,6 +144,10 @@ suite("iceberg_branch_tag_operate", "p0,external,doris,external_docker,external_ exception "Cannot set b8 to unknown snapshot: 11223344" } + result = sql """select UPDATE_TIME from information_schema.tables where TABLE_SCHEMA="test_db" and TABLE_NAME='test_branch_tag_operate'""" + def update_time3 = result[0][0]; + logger.info("get update times " + update_time2 + " vs. " + update_time3) + assertTrue(update_time3 > update_time2); // tag sql """ alter table test_branch_tag_operate create tag t2 as of version ${s0} """ diff --git a/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy b/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy index bc6002500ea756..33ce064811021e 100644 --- a/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy +++ b/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy @@ -73,6 +73,9 @@ suite("iceberg_schema_change_ddl", "p0,external,doris,external_docker,external_d (2, 'Bob', 30, 87.2), (3, 'Charlie', 22, 92.8) """ + def result = sql """select UPDATE_TIME from information_schema.tables where TABLE_SCHEMA="iceberg_schema_change_ddl_db" and TABLE_NAME='${table_name}'""" + def update_time1 = result[0][0]; + sleep(1000) // Verify initial state qt_init_1 """ DESC ${table_name} """ @@ -104,6 +107,10 @@ suite("iceberg_schema_change_ddl", "p0,external,doris,external_docker,external_d // Test 3: ADD complex type column sql """ ALTER TABLE ${table_name} ADD COLUMN address STRUCT """ + result = sql """select UPDATE_TIME from information_schema.tables where TABLE_SCHEMA="iceberg_schema_change_ddl_db" and TABLE_NAME='${table_name}'""" + def update_time2 = result[0][0]; + logger.info("get update times " + update_time1 + " vs. " + update_time2) + assertTrue(update_time2 > update_time1); qt_add_multi_1 """ DESC ${table_name} """ diff --git a/regression-test/suites/external_table_p0/test_external_table_update_time.groovy b/regression-test/suites/external_table_p0/test_external_table_update_time.groovy new file mode 100644 index 00000000000000..9d18db8aa03727 --- /dev/null +++ b/regression-test/suites/external_table_p0/test_external_table_update_time.groovy @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_external_table_update_time", "p0,external,hive,external_docker,external_docker_hive") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable Hive test.") + return + } + + for (String hivePrefix : ["hive3"]) { + String extHiveHmsHost = context.config.otherConfigs.get("externalEnvIp") + String extHiveHmsPort = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String catalog_name = hivePrefix + "_test_update_time_ctl" + sql """drop catalog if exists ${catalog_name};""" + sql """ + create catalog if not exists ${catalog_name} properties ( + 'type'='hms', + 'hadoop.username' = 'hadoop', + 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}' + ); + """ + logger.info("catalog " + catalog_name + " created") + sql """switch ${catalog_name};""" + + sql "drop database if exists test_update_time_db force"; + sql "create database test_update_time_db"; + sql "use test_update_time_db"; + sql "create table test_update_time_tbl(k1 int)" + sql "insert into test_update_time_tbl values(1)" + def result = sql """select UPDATE_TIME from information_schema.tables where TABLE_SCHEMA="test_update_time_db" and TABLE_NAME="test_update_time_tbl"""" + def update_time1 = result[0][0]; + sleep(2000); + sql "insert into test_update_time_tbl values(2)" + result = sql """select UPDATE_TIME from information_schema.tables where TABLE_SCHEMA="test_update_time_db" and TABLE_NAME="test_update_time_tbl"""" + def update_time2 = result[0][0]; + logger.info("get update times " + update_time1 + " vs. " + update_time2) + assertTrue(update_time2 > update_time1); + sleep(2000); + sql "truncate table test_update_time_tbl"; + result = sql """select UPDATE_TIME from information_schema.tables where TABLE_SCHEMA="test_update_time_db" and TABLE_NAME="test_update_time_tbl"""" + def update_time3 = result[0][0]; + logger.info("get update times " + update_time2 + " vs. " + update_time3) + assertTrue(update_time3 > update_time2); + } +} +