diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java index 22483ab43a68d..8b0cd73241723 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java @@ -113,6 +113,7 @@ public void testManageDatabase() { statement.execute("alter database if exists test1 set properties ttl='INF'"); statement.execute("alter database test set properties ttl=default"); + statement.execute("alter database test set properties need_last_cache=false"); String[] databaseNames = new String[] {"test"}; String[] TTLs = new String[] {"INF"}; @@ -168,6 +169,7 @@ public void testManageDatabase() { assertTrue(resultSet.getInt(7) >= defaultSchemaRegionGroupNum[cnt]); assertEquals(dataRegionGroupNum[cnt], resultSet.getInt(8)); assertTrue(resultSet.getInt(9) >= defaultDataRegionGroupNum[cnt]); + assertFalse(resultSet.getBoolean(10)); cnt++; } assertEquals(databaseNames.length, cnt); @@ -318,6 +320,37 @@ public void testShowCreateDatabase() throws SQLException { } } + @Test + public void testNeedLastCacheDatabaseProperty() throws SQLException { + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("create database need_cache_false with (need_last_cache=false)"); + statement.execute("create database need_cache_default"); + + assertDatabaseNeedLastCache(statement, "need_cache_false", false); + assertDatabaseNeedLastCache(statement, "need_cache_default", true); + + statement.execute("alter database need_cache_false set properties need_last_cache=true"); + assertDatabaseNeedLastCache(statement, "need_cache_false", true); + + statement.execute("alter database need_cache_false set properties need_last_cache=false"); + assertDatabaseNeedLastCache(statement, "need_cache_false", false); + + statement.execute("alter database need_cache_false set properties need_last_cache=default"); + assertDatabaseNeedLastCache(statement, "need_cache_false", true); + + try { + statement.execute("create database need_cache_invalid with (need_last_cache=1)"); + fail("non-boolean need_last_cache should be rejected"); + } catch (final SQLException e) { + assertEquals( + "701: need_last_cache value must be a BooleanLiteral, but now is LongLiteral, value: 1", + e.getMessage()); + } + } + } + @Test public void testShowCreatePipe() throws SQLException { try (final Connection connection = @@ -343,6 +376,30 @@ public void testShowCreateInformationSchemaDatabase() throws SQLException { } } + private static void assertDatabaseNeedLastCache( + final Statement statement, final String database, final boolean expected) + throws SQLException { + try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES DETAILS")) { + boolean found = false; + while (resultSet.next()) { + if (!database.equals(resultSet.getString("Database"))) { + continue; + } + found = true; + assertEquals(expected, resultSet.getBoolean("NeedLastCache")); + } + assertTrue(found); + } + + TestUtils.assertResultSetEqual( + statement.executeQuery( + "select database, need_last_cache from information_schema.databases where database = '" + + database + + "'"), + "database,need_last_cache,", + Collections.singleton(database + "," + expected + ",")); + } + private static void assertShowCreateSystemDatabaseFails( final Statement statement, final String database) throws SQLException { try { @@ -500,7 +557,8 @@ public void testInformationSchema() throws SQLException { "schema_region_group_num,INT32,ATTRIBUTE,", "max_schema_region_group_num,INT32,ATTRIBUTE,", "data_region_group_num,INT32,ATTRIBUTE,", - "max_data_region_group_num,INT32,ATTRIBUTE,"))); + "max_data_region_group_num,INT32,ATTRIBUTE,", + "need_last_cache,BOOLEAN,ATTRIBUTE,"))); TestUtils.assertResultSetEqual( statement.executeQuery("desc tables"), "ColumnName,DataType,Category,", @@ -511,7 +569,8 @@ public void testInformationSchema() throws SQLException { "ttl(ms),STRING,ATTRIBUTE,", "status,STRING,ATTRIBUTE,", "comment,STRING,ATTRIBUTE,", - "table_type,STRING,ATTRIBUTE,"))); + "table_type,STRING,ATTRIBUTE,", + "need_last_cache,BOOLEAN,ATTRIBUTE,"))); TestUtils.assertResultSetEqual( statement.executeQuery("desc columns"), "ColumnName,DataType,Category,", @@ -723,6 +782,7 @@ public void testInformationSchema() throws SQLException { for (int columnIndex = 3; columnIndex <= 9; columnIndex++) { assertNull(resultSet.getObject(columnIndex)); } + assertFalse(resultSet.getBoolean(10)); } else { assertEquals("test", resultSet.getString(1)); assertEquals("INF", resultSet.getString(2)); @@ -733,6 +793,7 @@ public void testInformationSchema() throws SQLException { assertTrue(resultSet.getInt(7) >= 1); assertEquals(0, resultSet.getInt(8)); assertTrue(resultSet.getInt(9) >= 2); + assertTrue(resultSet.getBoolean(10)); } cnt++; } @@ -740,32 +801,32 @@ public void testInformationSchema() throws SQLException { } TestUtils.assertResultSetEqual( statement.executeQuery("show devices from tables where status = 'USING'"), - "database,table_name,ttl(ms),status,comment,table_type,", + "database,table_name,ttl(ms),status,comment,table_type,need_last_cache,", new HashSet<>( Arrays.asList( - "information_schema,databases,INF,USING,null,SYSTEM VIEW,", - "information_schema,tables,INF,USING,null,SYSTEM VIEW,", - "information_schema,columns,INF,USING,null,SYSTEM VIEW,", - "information_schema,queries,INF,USING,null,SYSTEM VIEW,", - "information_schema,regions,INF,USING,null,SYSTEM VIEW,", - "information_schema,topics,INF,USING,null,SYSTEM VIEW,", - "information_schema,pipe_plugins,INF,USING,null,SYSTEM VIEW,", - "information_schema,pipes,INF,USING,null,SYSTEM VIEW,", - "information_schema,services,INF,USING,null,SYSTEM VIEW,", - "information_schema,subscriptions,INF,USING,null,SYSTEM VIEW,", - "information_schema,views,INF,USING,null,SYSTEM VIEW,", - "information_schema,functions,INF,USING,null,SYSTEM VIEW,", - "information_schema,configurations,INF,USING,null,SYSTEM VIEW,", - "information_schema,keywords,INF,USING,null,SYSTEM VIEW,", - "information_schema,nodes,INF,USING,null,SYSTEM VIEW,", - "information_schema,table_disk_usage,INF,USING,null,SYSTEM VIEW,", - "information_schema,config_nodes,INF,USING,null,SYSTEM VIEW,", - "information_schema,data_nodes,INF,USING,null,SYSTEM VIEW,", - "information_schema,connections,INF,USING,null,SYSTEM VIEW,", - "information_schema,current_queries,INF,USING,null,SYSTEM VIEW,", - "information_schema,queries_costs_histogram,INF,USING,null,SYSTEM VIEW,", - "test,test,INF,USING,test,BASE TABLE,", - "test,view_table,100,USING,null,VIEW FROM TREE,"))); + "information_schema,databases,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,tables,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,columns,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,queries,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,regions,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,topics,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,pipe_plugins,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,pipes,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,services,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,subscriptions,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,views,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,functions,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,configurations,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,keywords,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,nodes,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,table_disk_usage,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,config_nodes,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,data_nodes,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,connections,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,current_queries,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,queries_costs_histogram,INF,USING,null,SYSTEM VIEW,false,", + "test,test,INF,USING,test,BASE TABLE,true,", + "test,view_table,100,USING,null,VIEW FROM TREE,true,"))); TestUtils.assertResultSetEqual( statement.executeQuery("count devices from tables where status = 'USING'"), "count(devices),", @@ -872,6 +933,13 @@ public void testMixedDatabase() throws SQLException { try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { statement.execute("create database root.test"); + statement.execute("alter database root.test WITH NEED_LAST_CACHE=false"); + try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES DETAILS root.test")) { + assertTrue(resultSet.next()); + assertEquals("root.test", resultSet.getString("Database")); + assertFalse(resultSet.getBoolean("NeedLastCache")); + assertFalse(resultSet.next()); + } Assert.assertThrows( IoTDBSQLException.class, () -> @@ -995,8 +1063,9 @@ public void testDBAuth() throws SQLException { Collections.singleton("information_schema,INF,null,null,null,")); TestUtils.assertResultSetEqual( userStmt.executeQuery("select * from information_schema.databases"), - "database,ttl(ms),schema_replication_factor,data_replication_factor,time_partition_interval,schema_region_group_num,max_schema_region_group_num,data_region_group_num,max_data_region_group_num,", - Collections.singleton("information_schema,INF,null,null,null,null,null,null,null,")); + "database,ttl(ms),schema_replication_factor,data_replication_factor,time_partition_interval,schema_region_group_num,max_schema_region_group_num,data_region_group_num,max_data_region_group_num,need_last_cache,", + Collections.singleton( + "information_schema,INF,null,null,null,null,null,null,null,false,")); } try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java index c3a72d3c23a59..e3f9f3173523f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java @@ -135,6 +135,7 @@ public void testManageTable() { String[] ttls = new String[] {"INF"}; String[] statuses = new String[] {"USING"}; String[] comments = new String[] {"test"}; + String[] needLastCaches = new String[] {"true"}; statement.execute("use test2"); @@ -154,6 +155,7 @@ public void testManageTable() { assertEquals(ttls[cnt], resultSet.getString(2)); assertEquals(statuses[cnt], resultSet.getString(3)); assertEquals(comments[cnt], resultSet.getString(4)); + assertEquals(needLastCaches[cnt], resultSet.getString(6)); cnt++; } assertEquals(tableNames.length, cnt); @@ -198,6 +200,9 @@ public void testManageTable() { statement.execute("comment on table test1.table1 is 'new_test'"); comments = new String[] {"new_test"}; + + statement.execute("alter table test1.table1 set properties need_last_cache=false"); + needLastCaches = new String[] {"false"}; // using SHOW tables from try (final ResultSet resultSet = statement.executeQuery("SHOW tables details from test1")) { int cnt = 0; @@ -211,6 +216,7 @@ public void testManageTable() { assertEquals(tableNames[cnt], resultSet.getString(1)); assertEquals(ttls[cnt], resultSet.getString(2)); assertEquals(comments[cnt], resultSet.getString(4)); + assertEquals(needLastCaches[cnt], resultSet.getString(6)); cnt++; } assertEquals(tableNames.length, cnt); @@ -314,7 +320,7 @@ public void testManageTable() { statement.executeQuery("show create table table2"), "Table,Create Table,", Collections.singleton( - "table2,CREATE TABLE \"table2\" (\"t1\" TIMESTAMP TIME,\"region_id\" STRING TAG,\"plant_id\" STRING TAG,\"color\" STRING ATTRIBUTE,\"temperature\" FLOAT FIELD,\"speed\" DOUBLE FIELD COMMENT 'fast') WITH (ttl=6600000),")); + "table2,CREATE TABLE \"table2\" (\"t1\" TIMESTAMP TIME,\"region_id\" STRING TAG,\"plant_id\" STRING TAG,\"color\" STRING ATTRIBUTE,\"temperature\" FLOAT FIELD,\"speed\" DOUBLE FIELD COMMENT 'fast') WITH (ttl=6600000, need_last_cache=true),")); try { statement.execute("alter table table2 add column speed DOUBLE FIELD"); @@ -478,7 +484,7 @@ public void testManageTable() { } // After - statement.execute("COMMENT ON COLUMN table2.region_id IS '重庆'"); + statement.execute("COMMENT ON COLUMN table2.region_id IS '閲嶅簡'"); statement.execute("COMMENT ON COLUMN table2.region_id IS NULL"); statement.execute("COMMENT ON COLUMN test2.table2.t1 IS 'recent'"); statement.execute("COMMENT ON COLUMN test2.table2.region_id IS ''"); @@ -626,13 +632,122 @@ public void testManageTable() { statement.executeQuery("show create table test100"), "Table,Create Table,", Collections.singleton( - "test100,CREATE TABLE \"test100\" (\"t1\" TIMESTAMP TIME) WITH (ttl='INF'),")); + "test100,CREATE TABLE \"test100\" (\"t1\" TIMESTAMP TIME) WITH (ttl='INF', need_last_cache=true),")); } catch (final SQLException e) { e.printStackTrace(); fail(e.getMessage()); } } + @Test + public void testNeedLastCacheTableAndViewProperty() throws Exception { + try (final Connection treeConnection = EnvFactory.getEnv().getConnection(); + final Statement treeStatement = treeConnection.createStatement()) { + treeStatement.execute("create database root.need_cache_view_source"); + treeStatement.execute("create timeseries root.need_cache_view_source.d1.s1 int32"); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("create database need_cache_db with (need_last_cache=false)"); + statement.execute("create database need_cache_default_db"); + statement.execute("use need_cache_db"); + + statement.execute( + "create table inherited_table(time timestamp time, device_id string tag, temperature float field)"); + statement.execute( + "create table override_table(time timestamp time, device_id string tag, temperature float field) with (need_last_cache=true)"); + statement.execute( + "create table explicit_false_table(time timestamp time, device_id string tag, temperature float field) with (need_last_cache=false)"); + + assertTableNeedLastCache(statement, "need_cache_db", "inherited_table", false); + assertTableNeedLastCache(statement, "need_cache_db", "override_table", true); + assertTableNeedLastCache(statement, "need_cache_db", "explicit_false_table", false); + + TestUtils.assertResultSetEqual( + statement.executeQuery("show create table explicit_false_table"), + "Table,Create Table,", + Collections.singleton( + "explicit_false_table,CREATE TABLE \"explicit_false_table\" (\"time\" TIMESTAMP TIME,\"device_id\" STRING TAG,\"temperature\" FLOAT FIELD) WITH (ttl='INF', need_last_cache=false),")); + + statement.execute("alter table inherited_table set properties need_last_cache=true"); + assertTableNeedLastCache(statement, "need_cache_db", "inherited_table", true); + + statement.execute("alter table inherited_table set properties need_last_cache=default"); + assertTableNeedLastCache(statement, "need_cache_db", "inherited_table", false); + + statement.execute("alter table override_table set properties need_last_cache=false"); + assertTableNeedLastCache(statement, "need_cache_db", "override_table", false); + + statement.execute("alter table override_table set properties need_last_cache=true"); + assertTableNeedLastCache(statement, "need_cache_db", "override_table", true); + + statement.execute("use need_cache_default_db"); + statement.execute( + "create table default_reset_table(time timestamp time, device_id string tag, temperature float field) with (need_last_cache=false)"); + statement.execute("alter table default_reset_table set properties need_last_cache=default"); + assertTableNeedLastCache(statement, "need_cache_default_db", "default_reset_table", true); + + statement.execute("use need_cache_db"); + try { + statement.execute( + "create view invalid_need_cache_view (tag1 string tag, s1 int32 field) restrict with (ttl=100, need_last_cache=false) as root.need_cache_view_source.**"); + fail("tree view need_last_cache should be rejected"); + } catch (final SQLException e) { + assertEquals( + "701: The tree view does not support need_last_cache property.", e.getMessage()); + } + + statement.execute( + "create view explicit_view (tag1 string tag, s1 int32 field) restrict with (ttl=100) as root.need_cache_view_source.**"); + + TestUtils.assertResultSetEqual( + statement.executeQuery("show create view explicit_view"), + "View,Create View,", + Collections.singleton( + "explicit_view,CREATE VIEW \"explicit_view\" (\"time\" TIMESTAMP TIME,\"tag1\" STRING TAG,\"s1\" INT32 FIELD) RESTRICT WITH (ttl=100) AS root.\"need_cache_view_source\".**,")); + + try { + statement.execute("alter view explicit_view set properties need_last_cache=false"); + fail("tree view need_last_cache alter should be rejected"); + } catch (final SQLException e) { + assertEquals( + "701: The tree view does not support need_last_cache property.", e.getMessage()); + } + } + } + + private static void assertTableNeedLastCache( + final Statement statement, + final String database, + final String table, + final boolean expectedNeedLastCache) + throws SQLException { + try (final ResultSet resultSet = + statement.executeQuery("show tables details from " + database)) { + boolean found = false; + while (resultSet.next()) { + if (!table.equals(resultSet.getString("TableName"))) { + continue; + } + found = true; + assertEquals(expectedNeedLastCache, resultSet.getBoolean("NeedLastCache")); + } + assertTrue(found); + } + + TestUtils.assertResultSetEqual( + statement.executeQuery( + "select database, table_name, need_last_cache from information_schema.tables where database = '" + + database + + "' and table_name = '" + + table + + "'"), + "database,table_name,need_last_cache,", + Collections.singleton(database + "," + table + "," + expectedNeedLastCache + ",")); + } + @Test public void testTableAuth() throws SQLException { try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -649,7 +764,7 @@ public void testTableAuth() throws SQLException { Assert.assertThrows(SQLException.class, () -> userStmt.execute("select * from db.test")); TestUtils.assertResultSetEqual( userStmt.executeQuery("select * from information_schema.tables where database = 'db'"), - "database,table_name,ttl(ms),status,comment,table_type,", + "database,table_name,ttl(ms),status,comment,table_type,need_last_cache,", Collections.emptySet()); TestUtils.assertResultSetEqual( userStmt.executeQuery("select * from information_schema.columns where database = 'db'"), @@ -775,10 +890,10 @@ public void testTreeViewTable() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { statement.execute("create database root.another"); - statement.execute("create database root.`重庆`.`1`.b"); - statement.execute("create timeSeries root.`重庆`.`1`.b.`2`.S1 int32"); - statement.execute("create timeSeries root.`重庆`.`1`.b.`2`.s2 string"); - statement.execute("create timeSeries root.`重庆`.`1`.b.S1 int32"); + statement.execute("create database root.`閲嶅簡`.`1`.b"); + statement.execute("create timeSeries root.`閲嶅簡`.`1`.b.`2`.S1 int32"); + statement.execute("create timeSeries root.`閲嶅簡`.`1`.b.`2`.s2 string"); + statement.execute("create timeSeries root.`閲嶅簡`.`1`.b.S1 int32"); } catch (SQLException e) { fail(e.getMessage()); } @@ -797,13 +912,13 @@ public void testTreeViewTable() throws Exception { "701: Cannot specify view pattern to match more than one tree database.", e.getMessage()); } - statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.\"重庆\".\"1\".**"); + statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.\"閲嶅簡\".\"1\".**"); statement.execute("drop view tree_table"); } try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { - statement.execute("create timeSeries root.`重庆`.`1`.b.`1`.s1 int32"); + statement.execute("create timeSeries root.`閲嶅簡`.`1`.b.`1`.s1 int32"); } catch (SQLException e) { fail(e.getMessage()); } @@ -814,7 +929,7 @@ public void testTreeViewTable() throws Exception { statement.execute("use tree_view_db"); try { - statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.\"重庆\".\"1\".**"); + statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.\"閲嶅簡\".\"1\".**"); fail(); } catch (final SQLException e) { final Set result = @@ -828,13 +943,13 @@ public void testTreeViewTable() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { - statement.execute("drop timeSeries root.`重庆`.`1`.b.`1`.s1"); + statement.execute("drop timeSeries root.`閲嶅簡`.`1`.b.`1`.s1"); statement.execute("create device template t1 (S1 boolean, s9 int32)"); - statement.execute("set schema template t1 to root.`重庆`.`1`.b.`1`"); - statement.execute("create timeSeries root.`重庆`.`1`.b.`2`.f.g.h.S1 int32"); + statement.execute("set schema template t1 to root.`閲嶅簡`.`1`.b.`1`"); + statement.execute("create timeSeries root.`閲嶅簡`.`1`.b.`2`.f.g.h.S1 int32"); // Put schema cache - statement.execute("select S1, s2 from root.`重庆`.`1`.b.`2`"); + statement.execute("select S1, s2 from root.`閲嶅簡`.`1`.b.`2`"); } catch (SQLException e) { fail(e.getMessage()); } @@ -845,7 +960,7 @@ public void testTreeViewTable() throws Exception { statement.execute("use tree_view_db"); try { - statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.\"重庆\".\"1\".**"); + statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.\"閲嶅簡\".\"1\".**"); fail(); } catch (final SQLException e) { assertEquals( @@ -855,7 +970,7 @@ public void testTreeViewTable() throws Exception { try { statement.execute( - "create view tree_table (tag1 tag, tag2 tag, S1 field) as root.\"重庆\".\"1\".**"); + "create view tree_table (tag1 tag, tag2 tag, S1 field) as root.\"閲嶅簡\".\"1\".**"); fail(); } catch (final SQLException e) { assertEquals( @@ -866,7 +981,7 @@ public void testTreeViewTable() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { - statement.execute("create timeSeries root.`重庆`.`1`.b.e.s1 int32"); + statement.execute("create timeSeries root.`閲嶅簡`.`1`.b.e.s1 int32"); } catch (SQLException e) { fail(e.getMessage()); } @@ -887,7 +1002,7 @@ public void testTreeViewTable() throws Exception { // Temporary try { statement.execute( - "create or replace view tree_table (tag1 tag, tag2 tag, S1 int32 field, s3 boolean from S1) as root.\"重庆\".\"1\".**"); + "create or replace view tree_table (tag1 tag, tag2 tag, S1 int32 field, s3 boolean from S1) as root.\"閲嶅簡\".\"1\".**"); fail(); } catch (final SQLException e) { assertEquals( @@ -896,14 +1011,14 @@ public void testTreeViewTable() throws Exception { try { statement.execute( - "create or replace view tree_table (tag1 tag, tag2 tag, S1 int32 field, s3 from s2, s8 field) as root.\"重庆\".\"1\".**"); + "create or replace view tree_table (tag1 tag, tag2 tag, S1 int32 field, s3 from s2, s8 field) as root.\"閲嶅簡\".\"1\".**"); fail(); } catch (final SQLException e) { assertEquals("528: Measurements not found for s8, cannot auto detect", e.getMessage()); } statement.execute( - "create or replace view tree_table (tag1 tag, tag2 tag, S1 int32 field, s3 from s2) as root.\"重庆\".\"1\".**"); + "create or replace view tree_table (tag1 tag, tag2 tag, S1 int32 field, s3 from s2) as root.\"閲嶅簡\".\"1\".**"); // Cannot be written try { @@ -932,8 +1047,8 @@ public void testTreeViewTable() throws Exception { TestUtils.assertResultSetEqual( statement.executeQuery("show tables details"), - "TableName,TTL(ms),Status,Comment,TableType,", - Collections.singleton("view_table,100,USING,comment,VIEW FROM TREE,")); + "TableName,TTL(ms),Status,Comment,TableType,NeedLastCache,", + Collections.singleton("view_table,100,USING,comment,VIEW FROM TREE,true,")); TestUtils.assertResultSetEqual( statement.executeQuery("desc view_table"), @@ -971,7 +1086,7 @@ public void testTreeViewTable() throws Exception { final Statement statement = connection.createStatement()) { // Test create & replace + restrict statement.execute( - "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.`重庆`.`1`.**"); + "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.`閲嶅簡`.`1`.**"); fail(); } catch (final SQLException e) { assertTrue( @@ -992,7 +1107,7 @@ public void testTreeViewTable() throws Exception { .getConnection("testUser", "testUser123456", BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { statement.execute( - "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.\"重庆\".\"1\".**"); + "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.\"閲嶅簡\".\"1\".**"); fail(); } catch (final SQLException e) { assertEquals( @@ -1013,7 +1128,7 @@ public void testTreeViewTable() throws Exception { .getConnection("testUser", "testUser123456", BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { statement.execute( - "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.\"重庆\".\"1\".**"); + "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.\"閲嶅簡\".\"1\".**"); fail(); } catch (final SQLException e) { assertEquals( @@ -1023,7 +1138,7 @@ public void testTreeViewTable() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { - statement.execute("grant read_schema on root.`重庆`.** to user testUser"); + statement.execute("grant read_schema on root.`閲嶅簡`.** to user testUser"); } catch (final SQLException e) { fail(e.getMessage()); } @@ -1033,7 +1148,7 @@ public void testTreeViewTable() throws Exception { .getConnection("testUser", "testUser123456", BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { statement.execute( - "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.\"重庆\".\"1\".**"); + "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.\"閲嶅簡\".\"1\".**"); fail(); } catch (final SQLException e) { assertEquals( @@ -1043,7 +1158,7 @@ public void testTreeViewTable() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { - statement.execute("grant read_data on root.`重庆`.** to user testUser"); + statement.execute("grant read_data on root.`閲嶅簡`.** to user testUser"); } catch (final SQLException e) { fail(e.getMessage()); } @@ -1053,7 +1168,7 @@ public void testTreeViewTable() throws Exception { final Statement statement = connection.createStatement()) { statement.execute("alter database tree_view_db set properties ttl=100"); statement.execute( - "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict as root.\"重庆\".\"1\".**"); + "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict as root.\"閲嶅簡\".\"1\".**"); TestUtils.assertResultSetEqual( statement.executeQuery("show tables from tree_view_db"), @@ -1077,14 +1192,14 @@ public void testTreeViewTable() throws Exception { statement.executeQuery("show create view view_table"), "View,Create View,", Collections.singleton( - "view_table,CREATE VIEW \"view_table\" (\"time\" TIMESTAMP TIME,\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" STRING FIELD FROM \"s2\") RESTRICT WITH (ttl=100) AS root.\"重庆\".\"1\".**,")); + "view_table,CREATE VIEW \"view_table\" (\"time\" TIMESTAMP TIME,\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" STRING FIELD FROM \"s2\") RESTRICT WITH (ttl=100) AS root.\"閲嶅簡\".\"1\".**,")); // Can also use "show create table" TestUtils.assertResultSetEqual( statement.executeQuery("show create table view_table"), "View,Create View,", Collections.singleton( - "view_table,CREATE VIEW \"view_table\" (\"time\" TIMESTAMP TIME,\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" STRING FIELD FROM \"s2\") RESTRICT WITH (ttl=100) AS root.\"重庆\".\"1\".**,")); + "view_table,CREATE VIEW \"view_table\" (\"time\" TIMESTAMP TIME,\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" STRING FIELD FROM \"s2\") RESTRICT WITH (ttl=100) AS root.\"閲嶅簡\".\"1\".**,")); statement.execute("create table a ()"); try { diff --git a/iotdb-client/client-cpp/examples/README.md b/iotdb-client/client-cpp/examples/README.md index 763ec693bee23..f45d54c99039d 100644 --- a/iotdb-client/client-cpp/examples/README.md +++ b/iotdb-client/client-cpp/examples/README.md @@ -149,9 +149,9 @@ iotdb_session.dll **Prerequisites on the target PC** - **64-bit Windows** (examples are built x64). -- **[Microsoft Visual C++ Redistributable for Visual Studio 2015–2022](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist)** +- **[Microsoft Visual C++ Redistributable for Visual Studio 2015–2022](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist)** (x64). The SDK and examples are built with **`/MD`**; the redistributable - supplies `vcruntime140.dll`, `msvcp140.dll`, etc. + supplies `vcruntime140.dll`, `msvcp140.dll`, etc. Installing this package is enough—you do **not** need Visual Studio or the IoTDB SDK on the target machine. diff --git a/iotdb-client/client-cpp/examples/README_zh.md b/iotdb-client/client-cpp/examples/README_zh.md index 4adc38a3fc73b..2a660b4b6e98a 100644 --- a/iotdb-client/client-cpp/examples/README_zh.md +++ b/iotdb-client/client-cpp/examples/README_zh.md @@ -146,9 +146,9 @@ iotdb_session.dll **目标机器前置条件** - **64 位 Windows**（示例为 x64 构建）。 -- 安装 **[Microsoft Visual C++ 2015–2022 可再发行组件包（x64）](https://learn.microsoft.com/zh-cn/cpp/windows/latest-supported-vc-redist)**。 +- 安装 **[Microsoft Visual C++ 2015–2022 可再发行组件包（x64）](https://learn.microsoft.com/zh-cn/cpp/windows/latest-supported-vc-redist)**。 SDK 与示例均使用 **`/MD`**（动态 CRT），该安装包提供 `vcruntime140.dll`、 - `msvcp140.dll` 等运行时。 + `msvcp140.dll` 等运行时。 **仅安装此 Redistributable 即可**，目标机 **不需要** Visual Studio，也 **不需要** IoTDB SDK 头文件或 Thrift/Boost。 diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 index 135b674554b52..9613a453c96ef 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 @@ -156,6 +156,7 @@ keyWords | MODELS | MODIFY | NAN + | NEED_LAST_CACHE | NODEID | NODES | NONE diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 index 5353535ad344e..4ea045632410c 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 @@ -116,6 +116,7 @@ databaseAttributesClause databaseAttributeClause : databaseAttributeKey operator_eq INTEGER_LITERAL + | NEED_LAST_CACHE operator_eq boolean_literal ; databaseAttributeKey diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 index 4d915459c39a8..5ba86f80f8afb 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 @@ -1239,6 +1239,10 @@ MAX_DATA_REGION_GROUP_NUM : M A X '_' D A T A '_' R E G I O N '_' G R O U P '_' N U M ; +NEED_LAST_CACHE + : N E E D '_' L A S T '_' C A C H E + ; + CURRENT_TIMESTAMP : C U R R E N T '_' T I M E S T A M P ; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index b352b067a4dd6..7f4bd8497ec70 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -154,6 +154,14 @@ public class ClusterSchemaManager { private static final String CONSENSUS_WRITE_ERROR = "Failed in the write API executing the consensus layer due to: "; + public static boolean isNeedLastCacheEnabled(final TDatabaseSchema databaseSchema) { + return !databaseSchema.isSetNeedLastCache() || databaseSchema.isNeedLastCache(); + } + + private static boolean needInvalidateLastCache(final TDatabaseSchema after) { + return after.isSetNeedLastCache() && !after.isNeedLastCache(); + } + public ClusterSchemaManager( final IManager configManager, final ClusterSchemaInfo clusterSchemaInfo, @@ -229,7 +237,9 @@ public TSStatus alterDatabase( TSStatus result; final TDatabaseSchema databaseSchema = databaseSchemaPlan.getSchema(); - if (!isDatabaseExist(databaseSchema.getName())) { + try { + getDatabaseSchemaByName(databaseSchema.getName()); + } catch (final DatabaseNotExistsException e) { // Reject if Database doesn't exist result = new TSStatus(TSStatusCode.DATABASE_NOT_EXIST.getStatusCode()); result.setMessage( @@ -268,11 +278,16 @@ public TSStatus alterDatabase( isGeneratedByPipe ? new PipeEnrichedPlan(databaseSchemaPlan) : databaseSchemaPlan); - PartitionMetrics.bindDatabaseReplicationFactorMetricsWhenUpdate( - MetricService.getInstance(), - databaseSchemaPlan.getSchema().getName(), - databaseSchemaPlan.getSchema().getDataReplicationFactor(), - databaseSchemaPlan.getSchema().getSchemaReplicationFactor()); + if (result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + if (needInvalidateLastCache(databaseSchema)) { + invalidateLastCache(databaseSchema.getName()); + } + PartitionMetrics.bindDatabaseReplicationFactorMetricsWhenUpdate( + MetricService.getInstance(), + databaseSchemaPlan.getSchema().getName(), + databaseSchemaPlan.getSchema().getDataReplicationFactor(), + databaseSchemaPlan.getSchema().getSchemaReplicationFactor()); + } return result; } catch (final ConsensusException e) { LOGGER.warn(CONSENSUS_WRITE_ERROR, e); @@ -382,6 +397,7 @@ public TShowDatabaseResp showDatabase(final GetDatabasePlan getDatabasePlan) { databaseInfo.setDataReplicationFactor(databaseSchema.getDataReplicationFactor()); databaseInfo.setTimePartitionOrigin(databaseSchema.getTimePartitionOrigin()); databaseInfo.setTimePartitionInterval(databaseSchema.getTimePartitionInterval()); + databaseInfo.setNeedLastCache(isNeedLastCacheEnabled(databaseSchema)); databaseInfo.setMaxSchemaRegionNum( getMaxRegionGroupNum(database, TConsensusGroupType.SchemaRegion)); databaseInfo.setMaxDataRegionNum( @@ -908,6 +924,10 @@ public static TSStatus enrichDatabaseSchemaWithDefaultProperties( .MESSAGE_FAILED_CREATE_DATABASE_TIMEPARTITIONINTERVAL_SHOULD_POSITIVE_BB1B473F); } + if (!databaseSchema.isSetNeedLastCache()) { + databaseSchema.setNeedLastCache(true); + } + if (isSystemDatabase || isAuditDatabase) { databaseSchema.setMinSchemaRegionGroupNum(1); } else if (!databaseSchema.isSetMinSchemaRegionGroupNum()) { @@ -1713,6 +1733,13 @@ public synchronized Pair updateTableProperties( return result.get(); } + if (isTableView && updatedProperties.containsKey(TsTable.NEED_LAST_CACHE_PROPERTY)) { + return new Pair<>( + RpcUtils.getStatus( + TSStatusCode.SEMANTIC_ERROR, TreeViewSchema.UNSUPPORTED_NEED_LAST_CACHE_PROPERTY), + null); + } + updatedProperties .keySet() .removeIf( @@ -1723,20 +1750,55 @@ public synchronized Pair updateTableProperties( return new Pair<>(RpcUtils.SUCCESS_STATUS, null); } + final TDatabaseSchema databaseSchema; + try { + databaseSchema = + updatedProperties.containsKey(TsTable.NEED_LAST_CACHE_PROPERTY) + && Objects.isNull(updatedProperties.get(TsTable.NEED_LAST_CACHE_PROPERTY)) + ? getDatabaseSchemaByName(database) + : null; + } catch (final DatabaseNotExistsException e) { + throw new MetadataException(e); + } + final TsTable updatedTable = new TsTable(originalTable); updatedProperties.forEach( (k, v) -> { originalProperties.put(k, originalTable.getPropValue(k).orElse(null)); if (Objects.nonNull(v)) { updatedTable.addProp(k, v); + } else if (TsTable.NEED_LAST_CACHE_PROPERTY.equals(k) + && Objects.nonNull(databaseSchema) + && databaseSchema.isSetNeedLastCache()) { + updatedTable.addProp(k, String.valueOf(databaseSchema.isNeedLastCache())); } else { updatedTable.removeProp(k); } }); - return new Pair<>(RpcUtils.SUCCESS_STATUS, updatedTable); } + private void invalidateLastCache(final String database) { + final Map dataNodeLocationMap = + getNodeManager().getRegisteredDataNodeLocations(); + final DataNodeAsyncRequestContext clientHandler = + new DataNodeAsyncRequestContext<>( + CnToDnAsyncRequestType.INVALIDATE_LAST_CACHE, database, dataNodeLocationMap); + CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler); + clientHandler + .getResponseMap() + .forEach( + (dataNodeId, status) -> { + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + LOGGER.warn( + "Failed to invalidate last cache of database {} on DataNode {}, status: {}", + database, + dataNodeId, + status); + } + }); + } + public static Optional> checkTable4View( final String database, final TsTable table, final boolean isView) { if (!isView && TreeViewSchema.isTreeViewTable(table)) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index d7bdf1d91e057..e8a7a29d0b3a7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -141,6 +141,7 @@ import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE_PATTERN; import static org.apache.iotdb.commons.schema.table.Audit.TABLE_MODEL_AUDIT_DATABASE; import static org.apache.iotdb.commons.schema.table.Audit.TREE_MODEL_AUDIT_DATABASE; +import static org.apache.iotdb.commons.schema.table.TsTable.NEED_LAST_CACHE_PROPERTY; import static org.apache.iotdb.commons.schema.table.TsTable.TTL_PROPERTY; /** @@ -260,6 +261,14 @@ public TSStatus alterDatabase(final DatabaseSchemaPlan plan) { currentSchema.getTTL()); } + if (alterSchema.isSetNeedLastCache()) { + currentSchema.setNeedLastCache(alterSchema.isNeedLastCache()); + LOGGER.info( + "[SetNeedLastCache] The need last cache flag of Database: {} is adjusted to: {}", + currentSchema.getName(), + currentSchema.isNeedLastCache()); + } + mTree .getDatabaseNodeByDatabasePath(partialPathName) .getAsMNode() @@ -1278,6 +1287,11 @@ public ShowTableResp showTables(final ShowTablePlan plan) { TreeViewSchema.isTreeViewTable(pair.getLeft()) ? TableType.VIEW_FROM_TREE.ordinal() : TableType.BASE_TABLE.ordinal()); + info.setNeedLastCache( + pair.getLeft() + .getPropValue(NEED_LAST_CACHE_PROPERTY) + .map(Boolean::parseBoolean) + .orElse(true)); return info; }) .collect(Collectors.toList()) @@ -1326,6 +1340,11 @@ public ShowTable4InformationSchemaResp showTables4InformationSchema() { TreeViewSchema.isTreeViewTable(pair.getLeft()) ? TableType.VIEW_FROM_TREE.ordinal() : TableType.BASE_TABLE.ordinal()); + info.setNeedLastCache( + pair.getLeft() + .getPropValue(NEED_LAST_CACHE_PROPERTY) + .map(Boolean::parseBoolean) + .orElse(true)); return info; }) .collect(Collectors.toList())))); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java index cf8d9d5e2ed3a..8640543d15478 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java @@ -947,6 +947,11 @@ public void setTableProperties( && databaseNode.getDatabaseSchema().isSetTTL() && databaseNode.getDatabaseSchema().getTTL() != Long.MAX_VALUE) { table.addProp(k, String.valueOf(databaseNode.getDatabaseSchema().getTTL())); + } else if (k.equals(TsTable.NEED_LAST_CACHE_PROPERTY) + && databaseNode.getDatabaseSchema().isSetNeedLastCache()) { + table.addProp( + TsTable.NEED_LAST_CACHE_PROPERTY, + String.valueOf(databaseNode.getDatabaseSchema().isNeedLastCache())); } else { table.removeProp(k); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java index 05e0facb3018e..29cb315751e4c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java @@ -133,6 +133,10 @@ protected void checkTableExistence(final ConfigNodeProcedureEnv env) { && schema.getTTL() != Long.MAX_VALUE) { table.addProp(TsTable.TTL_PROPERTY, String.valueOf(schema.getTTL())); } + if (!table.getPropValue(TsTable.NEED_LAST_CACHE_PROPERTY).isPresent() + && schema.isSetNeedLastCache()) { + table.addProp(TsTable.NEED_LAST_CACHE_PROPERTY, String.valueOf(schema.isNeedLastCache())); + } setNextState(CreateTableState.PRE_CREATE); } } catch (final MetadataException | DatabaseNotExistsException e) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java index 88a163157293a..f699a88a6edf2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java @@ -72,8 +72,17 @@ public CreateTableViewProcedure( @Override protected void checkTableExistence(final ConfigNodeProcedureEnv env) { + if (table.getPropValue(TsTable.NEED_LAST_CACHE_PROPERTY).isPresent()) { + setFailure( + new ProcedureException( + new IoTDBException( + TreeViewSchema.UNSUPPORTED_NEED_LAST_CACHE_PROPERTY, + TSStatusCode.SEMANTIC_ERROR.getStatusCode()))); + return; + } if (!replace) { super.checkTableExistence(env); + table.removeProp(TsTable.NEED_LAST_CACHE_PROPERTY); } else { try { final Optional> oldTableAndStatus = diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java index 261f60ed65a1d..2a9459daaf353 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.confignode.manager; import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.junit.Assert; import org.junit.Test; @@ -37,4 +38,18 @@ public void testCalcMaxRegionGroupNum() { // (resourceWeight * resource) / (createdStorageGroupNum * replicationFactor) Assert.assertEquals(20, ClusterSchemaManager.calcMaxRegionGroupNum(3, 1.0, 120, 2, 3, 5)); } + + @Test + public void testNeedLastCacheDefaultsToTrueWhenUnset() { + final TDatabaseSchema unsetSchema = new TDatabaseSchema(); + Assert.assertTrue(ClusterSchemaManager.isNeedLastCacheEnabled(unsetSchema)); + + final TDatabaseSchema enabledSchema = new TDatabaseSchema(); + enabledSchema.setNeedLastCache(true); + Assert.assertTrue(ClusterSchemaManager.isNeedLastCacheEnabled(enabledSchema)); + + final TDatabaseSchema disabledSchema = new TDatabaseSchema(); + disabledSchema.setNeedLastCache(false); + Assert.assertFalse(ClusterSchemaManager.isNeedLastCacheEnabled(disabledSchema)); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java index 7ca14b38284ef..85e08b5a2ec7d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java @@ -350,6 +350,7 @@ protected void constructLine() { columnBuilders[6].writeInt(currentDatabase.getMaxSchemaRegionNum()); columnBuilders[7].writeInt(currentDatabase.getDataRegionNum()); columnBuilders[8].writeInt(currentDatabase.getMaxDataRegionNum()); + columnBuilders[9].writeBoolean(currentDatabase.isNeedLastCache()); resultBuilder.declarePosition(); currentDatabase = null; } @@ -404,6 +405,7 @@ private TableSupplier(final List dataTypes, final UserEntity userEnt table.getTableName(), table.getPropValue(TTL_PROPERTY).orElse(TTL_INFINITE)); info.setState(TableNodeStatus.USING.ordinal()); + info.setNeedLastCache(false); return info; }) .collect(Collectors.toList())); @@ -438,6 +440,7 @@ protected void constructLine() { columnBuilders[5].writeBinary( new Binary(TableType.BASE_TABLE.getName(), TSFileConfig.STRING_CHARSET)); } + columnBuilders[6].writeBoolean(currentTable.isNeedLastCache()); resultBuilder.declarePosition(); currentTable = null; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java index 7bb7478950b9c..99d7de4defdbd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java @@ -358,6 +358,10 @@ public SchemaPartition getSchemaPartition( return getOrCreateSchemaPartition(database, deviceIDs, false, null); } + public boolean needLastCache(final String database) { + return partitionCache.isNeedLastCache(database); + } + private SchemaPartition getOrCreateSchemaPartition( final String database, final @Nullable List deviceIDs, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java index 0549ec3964743..99f1b9f99c3b3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq; +import org.apache.tsfile.annotations.TableModel; import org.apache.tsfile.file.metadata.IDeviceID; import javax.annotation.Nullable; @@ -123,6 +124,7 @@ SchemaPartition getOrCreateSchemaPartition( * *

The device id shall be [table, seg1, ....] */ + @TableModel SchemaPartition getSchemaPartition( final String database, final @Nullable List deviceIDs); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java index dc896ceca812e..c221a67333870 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java @@ -60,6 +60,7 @@ import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.protocol.session.SessionManager; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceSchemaCacheManager; import org.apache.iotdb.db.schemaengine.schemaregion.utils.MetaUtils; import org.apache.iotdb.db.service.metrics.CacheMetrics; import org.apache.iotdb.rpc.TSStatusCode; @@ -67,6 +68,8 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import org.apache.thrift.TException; +import org.apache.tsfile.annotations.TableModel; +import org.apache.tsfile.annotations.TreeModel; import org.apache.tsfile.file.metadata.IDeviceID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -92,6 +95,10 @@ public class PartitionCache { IoTDBDescriptor.getInstance().getMemoryConfig(); private static final List ROOT_PATH = Arrays.asList("root", "**"); + private static boolean isNeedLastCacheEnabled(final TDatabaseSchema databaseSchema) { + return !databaseSchema.isSetNeedLastCache() || databaseSchema.isNeedLastCache(); + } + /** calculate slotId by device */ private final String seriesSlotExecutorName = config.getSeriesPartitionExecutorClass(); @@ -99,7 +106,7 @@ public class PartitionCache { private final SeriesPartitionExecutor partitionExecutor; /** the cache of database */ - private final Set databaseCache = new HashSet<>(); + private final Map database2NeedLastCacheCache = new HashMap<>(); /** database -> schemaPartitionTable */ private final Cache schemaPartitionCache; @@ -200,7 +207,7 @@ public void put(final IDeviceID device, final String databaseName) { * @return database name, return {@code null} if cache miss */ private String getDatabaseName(final IDeviceID deviceID) { - for (final String database : databaseCache) { + for (final String database : database2NeedLastCacheCache.keySet()) { if (PathUtils.isStartWith(deviceID, database)) { return database; } @@ -217,7 +224,7 @@ private String getDatabaseName(final IDeviceID deviceID) { private boolean containsDatabase(final String database) { databaseCacheLock.readLock().lock(); try { - return databaseCache.contains(database); + return database2NeedLastCacheCache.containsKey(database); } finally { databaseCacheLock.readLock().unlock(); } @@ -246,7 +253,7 @@ private void fetchDatabaseAndUpdateCache( if (databaseSchemaResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { // update all database into cache - updateDatabaseCache(databaseSchemaResp.getDatabaseSchemaMap().keySet()); + updateDatabaseCache(databaseSchemaResp.getDatabaseSchemaMap()); getDatabaseMap(result, deviceIDs, true); } } @@ -255,19 +262,39 @@ private void fetchDatabaseAndUpdateCache( } } + @TreeModel + public boolean isNeedLastCache(final String database) { + Boolean needLastCache = database2NeedLastCacheCache.get(database); + if (Objects.nonNull(needLastCache)) { + return needLastCache; + } + try { + fetchDatabaseAndUpdateCache(false); + } catch (final TException | ClientManagerException e) { + logger.warn( + "Failed to get need_last_cache info for database {}, will put cache anyway, exception: {}", + database, + e.getMessage()); + return true; + } + needLastCache = database2NeedLastCacheCache.get(database); + return Objects.isNull(needLastCache) || needLastCache; + } + /** get all database from configNode and update database cache. */ - private void fetchDatabaseAndUpdateCache() throws ClientManagerException, TException { + private void fetchDatabaseAndUpdateCache(final boolean isTableModel) + throws ClientManagerException, TException { databaseCacheLock.writeLock().lock(); try (final ConfigNodeClient client = configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final TGetDatabaseReq req = new TGetDatabaseReq(ROOT_PATH, SchemaConstant.ALL_MATCH_SCOPE_BINARY) - .setIsTableModel(true) + .setIsTableModel(isTableModel) .setCanSeeAuditDB(true); final TDatabaseSchemaResp databaseSchemaResp = client.getMatchedDatabaseSchemas(req); if (databaseSchemaResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { // update all database into cache - updateDatabaseCache(databaseSchemaResp.getDatabaseSchemaMap().keySet()); + updateDatabaseCache(databaseSchemaResp.getDatabaseSchemaMap()); } } finally { databaseCacheLock.writeLock().unlock(); @@ -525,13 +552,14 @@ private void getDatabaseCacheResult( } } + @TableModel public void checkAndAutoCreateDatabase( final String database, final boolean isAutoCreate, final String userName) { boolean isExisted = containsDatabase(database); if (!isExisted) { try { // try to fetch database from config node when miss - fetchDatabaseAndUpdateCache(); + fetchDatabaseAndUpdateCache(true); isExisted = containsDatabase(database); if (!isExisted && isAutoCreate) { // try to auto create database of failed device @@ -550,12 +578,33 @@ public void checkAndAutoCreateDatabase( /** * update database cache * - * @param databaseNames the database names that need to update + * @param databases the database names + */ + public void updateDatabaseCache(final Set databases) { + databaseCacheLock.writeLock().lock(); + try { + databases.forEach(database -> database2NeedLastCacheCache.put(database, true)); + } finally { + databaseCacheLock.writeLock().unlock(); + } + } + + /** + * update database cache + * + * @param databaseMap the database names and need last cache that need to update */ - public void updateDatabaseCache(final Set databaseNames) { + public void updateDatabaseCache(final Map databaseMap) { databaseCacheLock.writeLock().lock(); try { - databaseCache.addAll(databaseNames); + databaseMap.forEach( + (database, schema) -> { + final boolean needLastCache = isNeedLastCacheEnabled(schema); + database2NeedLastCacheCache.put(database, needLastCache); + if (!needLastCache) { + TreeDeviceSchemaCacheManager.getInstance().invalidateDatabaseLastCache(database); + } + }); } finally { databaseCacheLock.writeLock().unlock(); } @@ -565,7 +614,7 @@ public void updateDatabaseCache(final Set databaseNames) { public void removeFromDatabaseCache() { databaseCacheLock.writeLock().lock(); try { - databaseCache.clear(); + database2NeedLastCacheCache.clear(); } finally { databaseCacheLock.writeLock().unlock(); } @@ -1097,7 +1146,7 @@ public void invalidAllCache() { public String toString() { return "PartitionCache{" + ", databaseCache=" - + databaseCache + + database2NeedLastCacheCache + ", replicaSetCache=" + groupIdToReplicaSetMap + ", schemaPartitionCache=" diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 46634c3136964..272a6a308d1b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -37,6 +37,7 @@ import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.queryengine.plan.relational.metadata.QualifiedObjectName; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.DataType; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Literal; @@ -288,9 +289,11 @@ import static org.apache.iotdb.commons.executable.ExecutableManager.isUriTrusted; import static org.apache.iotdb.commons.queryengine.plan.relational.type.InternalTypeManager.getTSDataType; import static org.apache.iotdb.commons.queryengine.plan.relational.type.TypeSignatureTranslator.toTypeSignature; +import static org.apache.iotdb.commons.schema.table.TsTable.NEED_LAST_CACHE_PROPERTY; import static org.apache.iotdb.commons.schema.table.TsTable.TABLE_ALLOWED_PROPERTIES; import static org.apache.iotdb.commons.schema.table.TsTable.TIME_COLUMN_NAME; import static org.apache.iotdb.commons.schema.table.TsTable.TTL_PROPERTY; +import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AbstractDatabaseTask.NEED_LAST_CACHE_KEY; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.MAX_DATA_REGION_GROUP_NUM_KEY; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.MAX_SCHEMA_REGION_GROUP_NUM_KEY; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.TIME_PARTITION_INTERVAL_KEY; @@ -367,6 +370,11 @@ private IConfigTask visitDatabaseStatement( schema.setTTL(Long.MAX_VALUE); } break; + case NEED_LAST_CACHE_KEY: + if (node.getType() == DatabaseSchemaStatement.DatabaseSchemaStatementType.ALTER) { + schema.setNeedLastCache(true); + } + break; default: throw new SemanticException( DataNodeQueryMessages.UNSUPPORTED_DATABASE_PROPERTY_KEY + key); @@ -402,6 +410,9 @@ private IConfigTask visitDatabaseStatement( schema.setMaxDataRegionGroupNum( parseIntFromLiteral(value, MAX_DATA_REGION_GROUP_NUM_KEY)); break; + case NEED_LAST_CACHE_KEY: + schema.setNeedLastCache(parseBooleanFromLiteral(value, NEED_LAST_CACHE_KEY)); + break; default: throw new SemanticException( DataNodeQueryMessages.UNSUPPORTED_DATABASE_PROPERTY_KEY + key); @@ -582,6 +593,7 @@ public IConfigTask visitCreateTable(final CreateTable node, final MPPQueryContex @Override public IConfigTask visitCreateView(final CreateView node, final MPPQueryContext context) { + validateTreeViewProperties(node.getProperties()); final Pair databaseTablePair = parseTable4CreateTableOrView(node, context); final TsTable table = databaseTablePair.getRight(); accessControl.checkCanCreateViewFromTreePath(node.getPrefixPath(), context); @@ -835,13 +847,17 @@ public IConfigTask visitSetProperties(final SetProperties node, final MPPQueryCo accessControl.checkCanAlterTable( context.getSession().getUserName(), new QualifiedObjectName(database, tableName), context); + final boolean isTreeView = node.getType() == SetProperties.Type.TREE_VIEW; + if (isTreeView) { + validateTreeViewProperties(node.getProperties()); + } return new AlterTableSetPropertiesTask( database, tableName, convertPropertiesToMap(node.getProperties(), true), context.getQueryId().getId(), node.ifExists(), - node.getType() == SetProperties.Type.TREE_VIEW); + isTreeView); } @Override @@ -912,6 +928,15 @@ public Pair splitQualifiedName(final QualifiedName name) { return new Pair<>(database, name.getSuffix()); } + private void validateTreeViewProperties(final List propertyList) { + for (final Property property : propertyList) { + final String key = property.getName().getValue().toLowerCase(Locale.ENGLISH); + if (NEED_LAST_CACHE_PROPERTY.equals(key)) { + throw new SemanticException(TreeViewSchema.UNSUPPORTED_NEED_LAST_CACHE_PROPERTY); + } + } + } + private Map convertPropertiesToMap( final List propertyList, final boolean serializeDefault) { final Map map = new HashMap<>(); @@ -924,17 +949,27 @@ private Map convertPropertiesToMap( if (TABLE_ALLOWED_PROPERTIES.contains(key)) { if (!property.isSetToDefault()) { final Expression value = property.getNonDefaultValue(); - final Optional strValue = parseStringFromLiteralIfBinary(value); - if (strValue.isPresent()) { - if (!strValue.get().equalsIgnoreCase(TTL_INFINITE)) { - throw new SemanticException( - DataNodeQueryMessages.TTL_VALUE_MUST_BE_INF_OR_A_LONG_LITERAL_BUT_NOW_IS + value); - } - map.put(key, strValue.get().toUpperCase(Locale.ENGLISH)); - continue; + switch (key) { + case TTL_PROPERTY: + final Optional strValue = parseStringFromLiteralIfBinary(value); + if (strValue.isPresent()) { + if (!strValue.get().equalsIgnoreCase(TTL_INFINITE)) { + throw new SemanticException( + DataNodeQueryMessages.TTL_VALUE_MUST_BE_INF_OR_A_LONG_LITERAL_BUT_NOW_IS + + value); + } + map.put(key, strValue.get().toUpperCase(Locale.ENGLISH)); + continue; + } + map.put(key, String.valueOf(parseLongFromLiteral(value, TTL_PROPERTY))); + break; + case NEED_LAST_CACHE_PROPERTY: + map.put( + key, String.valueOf(parseBooleanFromLiteral(value, NEED_LAST_CACHE_PROPERTY))); + break; + default: + break; } - // TODO: support validation for other properties - map.put(key, String.valueOf(parseLongFromLiteral(value, TTL_PROPERTY))); } else if (serializeDefault) { map.put(key, null); } @@ -1113,6 +1148,18 @@ public IConfigTask visitSetSystemStatus(SetSystemStatus node, MPPQueryContext co return new SetSystemStatusTask(((SetSystemStatusStatement) node.getInnerTreeStatement())); } + private boolean parseBooleanFromLiteral(final Object value, final String name) { + if (!(value instanceof BooleanLiteral)) { + throw new SemanticException( + name + + " value must be a BooleanLiteral, but now is " + + (Objects.nonNull(value) ? value.getClass().getSimpleName() : null) + + ", value: " + + value); + } + return ((BooleanLiteral) value).getValue(); + } + private Optional parseStringFromLiteralIfBinary(final Object value) { return value instanceof Literal && ((Literal) value).getTsValue() instanceof Binary ? Optional.of( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java index 31c96887600c5..581b1ed2b9980 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java @@ -73,6 +73,9 @@ public static TDatabaseSchema constructDatabaseSchema( if (databaseSchemaStatement.getMaxDataRegionGroupNum() != null) { databaseSchema.setMaxDataRegionGroupNum(databaseSchemaStatement.getMaxDataRegionGroupNum()); } + if (databaseSchemaStatement.isSetNeedLastCache()) { + databaseSchema.setNeedLastCache(databaseSchemaStatement.isNeedLastCache()); + } databaseSchema.setIsTableModel(false); return databaseSchema; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractDatabaseTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractDatabaseTask.java index 59b50536c15ba..c1b6ba6bfc329 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractDatabaseTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractDatabaseTask.java @@ -29,6 +29,7 @@ public abstract class AbstractDatabaseTask implements IConfigTask { public static final String TIME_PARTITION_INTERVAL_KEY = "time_partition_interval"; public static final String MAX_SCHEMA_REGION_GROUP_NUM_KEY = "max_schema_region_group_num"; public static final String MAX_DATA_REGION_GROUP_NUM_KEY = "max_data_region_group_num"; + public static final String NEED_LAST_CACHE_KEY = "need_last_cache"; // Deprecated public static final String SCHEMA_REPLICATION_FACTOR_KEY = "schema_replication_factor"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateTableTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateTableTask.java index 229cf9aa5f45b..98a0d7b6feb02 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateTableTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateTableTask.java @@ -138,12 +138,16 @@ private static String getShowCreateTableSQL(final TsTable table) { if (table.getPropValue(TsTable.COMMENT_KEY).isPresent()) { builder.append(" COMMENT ").append(getString(table.getPropValue(TsTable.COMMENT_KEY).get())); } - String ttlString = table.getPropValue(TsTable.TTL_PROPERTY).orElse(TTL_INFINITE); if (ttlString.equals(TTL_INFINITE)) { ttlString = "'" + ttlString + "'"; } - builder.append(" WITH (ttl=").append(ttlString).append(")"); + builder + .append(" WITH (ttl=") + .append(ttlString) + .append(", need_last_cache=") + .append(table.getPropValue(TsTable.NEED_LAST_CACHE_PROPERTY).orElse("true")) + .append(")"); return builder.toString(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java index 128458888dc97..05999e3f69dce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java @@ -169,6 +169,7 @@ private static void buildTSBlockForDetails( builder.getColumnBuilder(6).writeInt(storageGroupInfo.getMaxSchemaRegionNum()); builder.getColumnBuilder(7).writeInt(storageGroupInfo.getDataRegionNum()); builder.getColumnBuilder(8).writeInt(storageGroupInfo.getMaxDataRegionNum()); + builder.getColumnBuilder(9).writeBoolean(storageGroupInfo.isNeedLastCache()); builder.declarePosition(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java index 228b1dd266acb..2ddf07f9e0ff2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java @@ -104,6 +104,9 @@ public static void buildTsBlock( ? TableType.values()[tableInfo.getType()].getName() : TableType.BASE_TABLE.getName(), TSFileConfig.STRING_CHARSET)); + builder + .getColumnBuilder(5) + .writeBoolean(!tableInfo.isSetNeedLastCache() || tableInfo.isNeedLastCache()); builder.declarePosition(); }); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 6ddd95593a1ec..575426592c9c7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -3016,6 +3016,13 @@ private void parseDatabaseAttributesClause( ctx.databaseAttributeClause()) { final IoTDBSqlParser.DatabaseAttributeKeyContext attributeKey = attribute.databaseAttributeKey(); + if (attributeKey == null) { + if (attribute.NEED_LAST_CACHE() != null) { + databaseSchemaStatement.setNeedLastCache( + Boolean.parseBoolean(attribute.boolean_literal().getText())); + } + continue; + } if (attributeKey.TTL() != null) { final long ttl = Long.parseLong(attribute.INTEGER_LITERAL().getText()); databaseSchemaStatement.setTtl(ttl); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index d49314d002a3d..883fb3f8f96e9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -66,6 +66,7 @@ import org.apache.iotdb.udf.api.relational.ScalarFunction; import org.apache.iotdb.udf.api.relational.TableFunction; +import org.apache.tsfile.annotations.TableModel; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.common.type.ObjectType; import org.apache.tsfile.read.common.type.Type; @@ -1696,6 +1697,7 @@ public SchemaPartition getOrCreateSchemaPartition( return partitionFetcher.getOrCreateSchemaPartition(database, deviceIDList, userName); } + @TableModel @Override public SchemaPartition getSchemaPartition( final String database, final List deviceIDList) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java index ec24b0a004e87..09592e27df399 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternUtil; import org.apache.iotdb.commons.queryengine.plan.relational.metadata.QualifiedObjectName; +import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.conf.DataNodeMemoryConfig; @@ -108,24 +109,44 @@ public class TableDeviceSchemaCache { private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(false); - private final IMemoryBlock memoryBlock; + @Nullable private final IMemoryBlock memoryBlock; private TableDeviceSchemaCache() { - memoryBlock = + this( memoryConfig .getSchemaCacheMemoryManager() - .exactAllocate(DataNodeMemoryConfig.SCHEMA_CACHE, MemoryBlockType.STATIC); + .exactAllocate(DataNodeMemoryConfig.SCHEMA_CACHE, MemoryBlockType.STATIC), + true); + } + + private TableDeviceSchemaCache(final IMemoryBlock memoryBlock, final boolean bindMetrics) { + this(memoryBlock.getTotalMemorySizeInBytes(), memoryBlock, bindMetrics); + } + + private TableDeviceSchemaCache( + final long memoryCapacity, + final @Nullable IMemoryBlock memoryBlock, + final boolean bindMetrics) { + this.memoryBlock = memoryBlock; dualKeyCache = new DualKeyCacheBuilder() .cacheEvictionPolicy( DualKeyCachePolicy.valueOf(config.getDataNodeSchemaCacheEvictionPolicy())) - .memoryCapacity(memoryBlock.getTotalMemorySizeInBytes()) + .memoryCapacity(memoryCapacity) .firstKeySizeComputer(TableId::estimateSize) .secondKeySizeComputer(deviceID -> (int) deviceID.ramBytesUsed()) .valueSizeComputer(TableDeviceCacheEntry::estimateSize) .build(); - memoryBlock.allocate(memoryBlock.getTotalMemorySizeInBytes()); - MetricService.getInstance().addMetricSet(new TableDeviceSchemaCacheMetrics(this)); + if (Objects.nonNull(this.memoryBlock)) { + this.memoryBlock.allocate(this.memoryBlock.getTotalMemorySizeInBytes()); + } + if (bindMetrics) { + MetricService.getInstance().addMetricSet(new TableDeviceSchemaCacheMetrics(this)); + } + } + + static TableDeviceSchemaCache createForTest(final long memoryCapacity) { + return new TableDeviceSchemaCache(memoryCapacity, null, false); } public static TableDeviceSchemaCache getInstance() { @@ -238,8 +259,9 @@ public void initOrInvalidateLastCache( readWriteLock.readLock().lock(); try { // Avoid stale table - if (Objects.isNull( - DataNodeTableCache.getInstance().getTable(database, deviceId.getTableName(), false))) { + final TsTable table = + DataNodeTableCache.getInstance().getTable(database, deviceId.getTableName(), false); + if (Objects.isNull(table) || Boolean.FALSE.equals(table.getCachedNeedLastCache())) { return; } dualKeyCache.update( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java index 62f30c8e4c176..67d8692a856b9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.i18n.DataNodeSchemaMessages; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.common.schematree.IMeasurementSchemaInfo; +import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaComputation; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.db.schemaengine.template.ITemplateManager; @@ -338,6 +339,10 @@ public void updateLastCacheIfExists( final @Nonnull TimeValuePair[] timeValuePairs, final boolean isAligned, final IMeasurementSchema[] measurementSchemas) { + if (!ClusterPartitionFetcher.getInstance().needLastCache(database)) { + return; + } + tableDeviceSchemaCache.updateLastCache( database, deviceID, measurements, timeValuePairs, isAligned, measurementSchemas, false); } @@ -363,6 +368,10 @@ public void updateLastCacheIfExists( * @param measurementPath the fetched {@link MeasurementPath} */ public void declareLastCache(final String database, final MeasurementPath measurementPath) { + if (!ClusterPartitionFetcher.getInstance().needLastCache(database)) { + return; + } + tableDeviceSchemaCache.updateLastCache( database, measurementPath.getIDeviceID(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DatabaseSchemaStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DatabaseSchemaStatement.java index e27dde87b41b8..dc18d56055649 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DatabaseSchemaStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DatabaseSchemaStatement.java @@ -39,6 +39,8 @@ public class DatabaseSchemaStatement extends Statement implements IConfigStateme private Integer maxSchemaRegionGroupNum = null; private Integer maxDataRegionGroupNum = null; private boolean enablePrintExceptionLog = true; + private boolean needLastCache = true; + private boolean isNeedLastCacheSet = false; // Deprecated private Integer schemaReplicationFactor = null; @@ -118,6 +120,19 @@ public void setEnablePrintExceptionLog(final boolean enablePrintExceptionLog) { this.enablePrintExceptionLog = enablePrintExceptionLog; } + public boolean isNeedLastCache() { + return needLastCache; + } + + public boolean isSetNeedLastCache() { + return isNeedLastCacheSet; + } + + public void setNeedLastCache(final boolean needLastCache) { + this.needLastCache = needLastCache; + this.isNeedLastCacheSet = true; + } + @Override public R accept(final StatementVisitor visitor, final C context) { switch (subType) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java index 2367aa0c31264..0167480576280 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java @@ -104,6 +104,7 @@ public void buildTSBlock( builder.getColumnBuilder(6).writeInt(databaseInfo.getMaxSchemaRegionNum()); builder.getColumnBuilder(7).writeInt(databaseInfo.getDataRegionNum()); builder.getColumnBuilder(8).writeInt(databaseInfo.getMaxDataRegionNum()); + builder.getColumnBuilder(9).writeBoolean(databaseInfo.isNeedLastCache()); } builder.declarePosition(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index 142e355371581..79d299df5189d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodeSchemaMessages; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.ClusterConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.utils.Pair; @@ -144,6 +145,11 @@ public void preUpdateTable(String database, final TsTable table, final String ol } }); LOGGER.info(DataNodeSchemaMessages.PRE_UPDATE_TABLE_SUCCESS, database, table.getTableName()); + // Since a pre-updated table can be used for query planning before commit-release, stale + // last cache should stop serving as soon as need_last_cache turns off. + if (!table.getCachedNeedLastCache()) { + TableDeviceSchemaCache.getInstance().invalidateLastCache(database, table.getTableName()); + } // If rename table if (Objects.nonNull(oldName)) { @@ -229,6 +235,18 @@ private void removeTableFromPreUpdateMap(final String database, final String tab return Objects.nonNull(tableVersionPair) ? tableVersionPair.getLeft() : null; } + private @Nullable TsTable getTableFromCache(final String database, final String tableName) { + final Map tableMap = databaseTableMap.get(database); + return Objects.nonNull(tableMap) ? tableMap.get(tableName) : null; + } + + private void invalidateLastCacheIfDisabled( + final String database, final String tableName, final TsTable currentTable) { + if (Objects.nonNull(currentTable) && !currentTable.getCachedNeedLastCache()) { + TableDeviceSchemaCache.getInstance().invalidateLastCache(database, tableName); + } + } + @Override public void commitUpdateTable( String database, final String tableName, final @Nullable String oldName) { @@ -259,6 +277,7 @@ public void commitUpdateTable( databaseTableMap .computeIfAbsent(database, k -> new ConcurrentHashMap<>()) .put(tableName, newTable); + invalidateLastCacheIfDisabled(database, tableName, newTable); if (LOGGER.isDebugEnabled()) { LOGGER.debug( DataNodeSchemaMessages.COMMIT_UPDATE_TABLE_SUCCESS_WITH_DETAIL, @@ -468,6 +487,7 @@ private void updateTable( databaseTableMap .computeIfAbsent(database, k -> new ConcurrentHashMap<>()) .put(tableName, tsTable); + invalidateLastCacheIfDisabled(database, tableName, tsTable); } else if (databaseTableMap.containsKey(database)) { databaseTableMap.get(database).remove(tableName); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java index 443352ae919c3..50d36d5a98133 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java @@ -75,10 +75,11 @@ public static void buildDatabaseTsBlock( builder.getColumnBuilder(4).appendNull(); if (details) { for (int columnIndex = 5; - columnIndex < builder.getValueColumnBuilders().length; + columnIndex < builder.getValueColumnBuilders().length - 1; columnIndex++) { builder.getColumnBuilder(columnIndex).appendNull(); } + builder.getColumnBuilder(builder.getValueColumnBuilders().length - 1).writeBoolean(false); } builder.declarePosition(); } @@ -125,6 +126,7 @@ public static boolean mayShowTable( builder .getColumnBuilder(4) .writeBinary(new Binary(TableType.SYSTEM_VIEW.getName(), TSFileConfig.STRING_CHARSET)); + builder.getColumnBuilder(5).writeBoolean(false); } builder.declarePosition(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/TreeDeviceSchemaCacheManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/TreeDeviceSchemaCacheManagerTest.java index f9188aea9afba..1454f8eef41a8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/TreeDeviceSchemaCacheManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/TreeDeviceSchemaCacheManagerTest.java @@ -24,8 +24,10 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.commons.schema.template.Template; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.partition.PartitionCache; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.SchemaCacheEntry; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; @@ -274,6 +276,33 @@ public void testUpdateLastCache() throws IllegalPathException { Assert.assertEquals(0, TableDeviceSchemaCache.getInstance().getMemoryUsage()); } + @Test + public void testLastCacheIsInvalidatedWhenDatabaseNeedLastCacheSetFalse() + throws IllegalPathException { + final String database = "root.db"; + final PartialPath device = new PartialPath("root.db.d"); + final MeasurementSchema s1 = new MeasurementSchema("s1", TSDataType.INT32); + final TimeValuePair tv1 = new TimeValuePair(1, new TsPrimitiveType.TsInt(1)); + final MeasurementPath measurementPath = new MeasurementPath(device.concatNode("s1"), s1); + + treeDeviceSchemaCacheManager.declareLastCache(database, measurementPath); + treeDeviceSchemaCacheManager.updateLastCacheIfExists( + database, + IDeviceID.Factory.DEFAULT_FACTORY.create( + StringArrayDeviceID.splitDeviceIdString(device.getNodes())), + new String[] {"s1"}, + new TimeValuePair[] {tv1}, + false, + new MeasurementSchema[] {s1}); + Assert.assertEquals(tv1, treeDeviceSchemaCacheManager.getLastCache(measurementPath)); + + final TDatabaseSchema disabledSchema = new TDatabaseSchema(); + disabledSchema.setNeedLastCache(false); + new PartitionCache().updateDatabaseCache(Collections.singletonMap(database, disabledSchema)); + + Assert.assertNull(treeDeviceSchemaCacheManager.getLastCache(measurementPath)); + } + @Test public void testUpdateLastCacheWithAliasDoesNotCopyMeasurements() throws IllegalPathException { final String database = "root.db"; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java index e08a2b610c996..6bffb381b1eab 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.commons.partition.executor.SeriesPartitionExecutor; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -53,6 +54,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class PartitionCacheTest { @@ -62,7 +64,7 @@ public class PartitionCacheTest { SeriesPartitionExecutor.getSeriesPartitionExecutor( config.getSeriesPartitionExecutorClass(), config.getSeriesPartitionSlotNum()); - private static final Set storageGroups = new HashSet<>(); + private static final Set databases = new HashSet<>(); private static final Map> schemaPartitionTable = new HashMap<>(); private static final Map< @@ -85,7 +87,7 @@ public class PartitionCacheTest { storageGroupNumber++) { // init each database String storageGroupName = getDatabaseName(storageGroupNumber); - storageGroups.add(storageGroupName); + databases.add(storageGroupName); if (!schemaPartitionTable.containsKey(storageGroupName)) { schemaPartitionTable.put(storageGroupName, new HashMap<>()); } @@ -148,7 +150,7 @@ private static String getDeviceName(String storageGroupName, int deviceNumber) { @Before public void setUp() throws Exception { partitionCache = new PartitionCache(); - partitionCache.updateDatabaseCache(storageGroups); + partitionCache.updateDatabaseCache(databases); partitionCache.updateSchemaPartitionCache(schemaPartitionTable); partitionCache.updateDataPartitionCache(dataPartitionTable); partitionCache.updateGroupIdToReplicaSetMap(100, consensusGroupIdToRegionReplicaSet); @@ -243,6 +245,29 @@ public void testStorageGroupCache() { assertEquals(0, deviceToStorageGroupMap.size()); } + @Test + public void testNeedLastCacheDefaultsToTrueWhenUnset() { + final Map databaseSchemaMap = new HashMap<>(); + + final TDatabaseSchema unsetSchema = new TDatabaseSchema(); + databaseSchemaMap.put("root.unset", unsetSchema); + + final TDatabaseSchema enabledSchema = new TDatabaseSchema(); + enabledSchema.setNeedLastCache(true); + databaseSchemaMap.put("root.enabled", enabledSchema); + + final TDatabaseSchema disabledSchema = new TDatabaseSchema(); + disabledSchema.setNeedLastCache(false); + databaseSchemaMap.put("root.disabled", disabledSchema); + + partitionCache.updateDatabaseCache(databaseSchemaMap); + + assertTrue(partitionCache.isNeedLastCache("root.unset")); + assertTrue(partitionCache.isNeedLastCache("root.enabled")); + assertFalse(partitionCache.isNeedLastCache("root.disabled")); + assertTrue(partitionCache.isNeedLastCache("root.missing")); + } + @Test public void testRegionReplicaSetCache() { // test update regionReplicaSetCache with small timestamp diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTaskTest.java new file mode 100644 index 0000000000000..41703a4583493 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTaskTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.DatabaseSchemaStatement; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class DatabaseSchemaTaskTest { + + @Test + public void testConstructDatabaseSchemaDoesNotSetNeedLastCacheWhenAbsent() throws Exception { + final DatabaseSchemaStatement statement = + new DatabaseSchemaStatement(DatabaseSchemaStatement.DatabaseSchemaStatementType.ALTER); + statement.setDatabasePath(new PartialPath("root.sg")); + + final TDatabaseSchema databaseSchema = DatabaseSchemaTask.constructDatabaseSchema(statement); + + assertFalse(databaseSchema.isSetNeedLastCache()); + } + + @Test + public void testConstructDatabaseSchemaSetsNeedLastCacheWhenPresent() throws Exception { + final DatabaseSchemaStatement statement = + new DatabaseSchemaStatement(DatabaseSchemaStatement.DatabaseSchemaStatementType.ALTER); + statement.setDatabasePath(new PartialPath("root.sg")); + statement.setNeedLastCache(false); + + final TDatabaseSchema databaseSchema = DatabaseSchemaTask.constructDatabaseSchema(statement); + + assertTrue(databaseSchema.isSetNeedLastCache()); + assertEquals(false, databaseSchema.isNeedLastCache()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java index e808652a42a9c..2b8909f38613e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java @@ -19,15 +19,14 @@ package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache; -import org.apache.iotdb.commons.memory.MemoryManager; import org.apache.iotdb.commons.schema.column.ColumnHeader; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.AttributeColumnSchema; import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; import org.apache.iotdb.commons.schema.table.column.TimeColumnSchema; -import org.apache.iotdb.db.conf.DataNodeMemoryConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.partition.PartitionCache; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.tsfile.common.conf.TSFileConfig; @@ -58,15 +57,11 @@ public class TableDeviceSchemaCacheTest { - private static final DataNodeMemoryConfig memoryConfig = - IoTDBDescriptor.getInstance().getMemoryConfig(); - - private static long originMemConfig; - private static final String database1 = "sg1"; private static final String database2 = "sg2"; private static final String table1 = "t1"; private static final String table2 = "t2"; + private static final String tableNeedLastCacheDisabled = "t_need_last_cache_disabled"; private static final String attributeName1 = "type"; private static final String attributeName2 = "cycle"; private static final String measurement1 = "s0"; @@ -146,24 +141,45 @@ public static void prepareEnvironment() { DataNodeTableCache.getInstance().preUpdateTable(database1, testTable2, null); DataNodeTableCache.getInstance().commitUpdateTable(database1, table2, null); - originMemConfig = memoryConfig.getSchemaCacheMemoryManager().getTotalMemorySizeInBytes(); - changeSchemaCacheMemorySize(1300L); + final TsTable testTableNeedLastCacheDisabled = new TsTable(tableNeedLastCacheDisabled); + columnHeaderList.forEach( + columnHeader -> + testTableNeedLastCacheDisabled.addColumnSchema( + new TagColumnSchema(columnHeader.getColumnName(), columnHeader.getColumnType()))); + testTableNeedLastCacheDisabled.addColumnSchema( + new AttributeColumnSchema(attributeName1, TSDataType.STRING)); + testTableNeedLastCacheDisabled.addColumnSchema( + new AttributeColumnSchema(attributeName2, TSDataType.STRING)); + testTableNeedLastCacheDisabled.addColumnSchema(new TimeColumnSchema("time", TSDataType.INT64)); + testTableNeedLastCacheDisabled.addColumnSchema( + new FieldColumnSchema( + measurement1, TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); + testTableNeedLastCacheDisabled.addColumnSchema( + new FieldColumnSchema( + measurement2, TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); + testTableNeedLastCacheDisabled.addColumnSchema( + new FieldColumnSchema( + measurement3, TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); + testTableNeedLastCacheDisabled.addColumnSchema( + new FieldColumnSchema( + measurement4, TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); + testTableNeedLastCacheDisabled.addColumnSchema( + new FieldColumnSchema( + measurement5, TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); + testTableNeedLastCacheDisabled.addColumnSchema( + new FieldColumnSchema( + measurement6, TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); + testTableNeedLastCacheDisabled.addProp( + TsTable.NEED_LAST_CACHE_PROPERTY, Boolean.FALSE.toString()); + DataNodeTableCache.getInstance() + .preUpdateTable(database1, testTableNeedLastCacheDisabled, null); + DataNodeTableCache.getInstance().commitUpdateTable(database1, tableNeedLastCacheDisabled, null); } @AfterClass public static void clearEnvironment() { DataNodeTableCache.getInstance().invalid(database1); DataNodeTableCache.getInstance().invalid(database2); - changeSchemaCacheMemorySize(originMemConfig); - } - - private static void changeSchemaCacheMemorySize(long size) { - MemoryManager memoryManager = memoryConfig.getSchemaEngineMemoryManager(); - MemoryManager schemaCacheMemoryManager = memoryConfig.getSchemaCacheMemoryManager(); - schemaCacheMemoryManager.clearAll(); - memoryManager.releaseChildMemoryManager("schemaCache"); - schemaCacheMemoryManager = memoryManager.getOrCreateMemoryManager("schemaCache", size); - memoryConfig.setSchemaCacheMemoryManager(schemaCacheMemoryManager); } @After @@ -171,9 +187,13 @@ public void rollback() { TableDeviceSchemaCache.getInstance().invalidateAll(); } + private TableDeviceSchemaCache createTestCache() { + return TableDeviceSchemaCache.createForTest(1300L); + } + @Test public void testDeviceCache() { - final TableDeviceSchemaCache cache = TableDeviceSchemaCache.getInstance(); + final TableDeviceSchemaCache cache = createTestCache(); final Map attributeMap = new HashMap<>(); attributeMap.put(attributeName1, new Binary("new", TSFileConfig.STRING_CHARSET)); @@ -257,7 +277,7 @@ public void testDeviceCache() { @Test public void testLastCache() { - final TableDeviceSchemaCache cache = TableDeviceSchemaCache.getInstance(); + final TableDeviceSchemaCache cache = createTestCache(); final String[] device0 = new String[] {"hebei", "p_1", "d_0"}; @@ -556,7 +576,7 @@ public void testUpdateNonExistWhenWriting() { final TimeValuePair[] testTimeValuePairs = new TimeValuePair[] {tv3, tv3, tv3, tv3}; // Test disable put cache by writing - final TableDeviceSchemaCache cache = TableDeviceSchemaCache.getInstance(); + final TableDeviceSchemaCache cache = createTestCache(); cache.updateLastCacheIfExists( database1, @@ -592,6 +612,124 @@ public void testUpdateNonExistWhenWriting() { cache.getLastEntry(database1, convertTagValuesToDeviceID(table1, device0), "s2")); } + @Test + public void testLastCacheIsInvalidatedWhenTableNeedLastCacheTurnsFalse() { + final String[] device0 = new String[] {"hebei", "p_1", "d_0"}; + final IDeviceID deviceID = convertTagValuesToDeviceID(table1, device0); + final String[] measurements = new String[] {measurement1}; + final TimeValuePair[] values = + new TimeValuePair[] {new TimeValuePair(1L, new TsPrimitiveType.TsInt(1))}; + + final TableDeviceSchemaCache cache = TableDeviceSchemaCache.getInstance(); + updateLastCache4Query(cache, database1, deviceID, measurements, values); + Assert.assertEquals(values[0], cache.getLastEntry(database1, deviceID, measurement1)); + + final TsTable disabledTable = + new TsTable(DataNodeTableCache.getInstance().getTable(database1, table1)); + disabledTable.addProp(TsTable.NEED_LAST_CACHE_PROPERTY, Boolean.FALSE.toString()); + DataNodeTableCache.getInstance().preUpdateTable(database1, disabledTable, null); + DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null); + + try { + Assert.assertNull(cache.getLastEntry(database1, deviceID, measurement1)); + } finally { + final TsTable enabledTable = new TsTable(disabledTable); + enabledTable.removeProp(TsTable.NEED_LAST_CACHE_PROPERTY); + DataNodeTableCache.getInstance().preUpdateTable(database1, enabledTable, null); + DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null); + } + } + + @Test + public void testLastCacheIsInvalidatedWhenTableNeedLastCacheSetFalseWithoutCachedTable() { + final String database = "sg_missing_table_cache"; + final String tableName = "t_missing_table_cache"; + final String[] device0 = new String[] {"hebei", "p_1", "d_0"}; + final IDeviceID deviceID = convertTagValuesToDeviceID(tableName, device0); + final String[] measurements = new String[] {measurement1}; + final TimeValuePair[] values = + new TimeValuePair[] {new TimeValuePair(1L, new TsPrimitiveType.TsInt(1))}; + + final TsTable enabledTable = + new TsTable(DataNodeTableCache.getInstance().getTable(database1, table1)); + enabledTable.renameTable(tableName); + DataNodeTableCache.getInstance().preUpdateTable(database, enabledTable, null); + DataNodeTableCache.getInstance().commitUpdateTable(database, tableName, null); + + final TableDeviceSchemaCache cache = TableDeviceSchemaCache.getInstance(); + updateLastCache4Query(cache, database, deviceID, measurements, values); + Assert.assertEquals(values[0], cache.getLastEntry(database, deviceID, measurement1)); + + try { + DataNodeTableCache.getInstance().invalid(database); + final TsTable disabledTable = new TsTable(enabledTable); + disabledTable.addProp(TsTable.NEED_LAST_CACHE_PROPERTY, Boolean.FALSE.toString()); + DataNodeTableCache.getInstance().preUpdateTable(database, disabledTable, null); + + Assert.assertNull(cache.getLastEntry(database, deviceID, measurement1)); + } finally { + DataNodeTableCache.getInstance().invalid(database); + } + } + + @Test + public void testLastCacheIsInvalidatedWhenDatabaseNeedLastCacheTurnsFalse() { + final String[] device0 = new String[] {"hebei", "p_1", "d_0"}; + final IDeviceID deviceInDatabase1 = convertTagValuesToDeviceID(table1, device0); + final IDeviceID deviceInDatabase2 = convertTagValuesToDeviceID(table1, device0); + final String[] measurements = new String[] {measurement1}; + final TimeValuePair[] values = + new TimeValuePair[] {new TimeValuePair(1L, new TsPrimitiveType.TsInt(1))}; + + final TableDeviceSchemaCache cache = TableDeviceSchemaCache.getInstance(); + updateLastCache4Query(cache, database1, deviceInDatabase1, measurements, values); + updateLastCache4Query(cache, database2, deviceInDatabase2, measurements, values); + Assert.assertEquals(values[0], cache.getLastEntry(database1, deviceInDatabase1, measurement1)); + Assert.assertEquals(values[0], cache.getLastEntry(database2, deviceInDatabase2, measurement1)); + + final PartitionCache partitionCache = new PartitionCache(); + try { + final TDatabaseSchema enabledSchema = new TDatabaseSchema(); + enabledSchema.setNeedLastCache(true); + partitionCache.updateDatabaseCache(Collections.singletonMap(database1, enabledSchema)); + Assert.assertEquals( + values[0], cache.getLastEntry(database1, deviceInDatabase1, measurement1)); + + final TDatabaseSchema disabledSchema = new TDatabaseSchema(); + disabledSchema.setNeedLastCache(false); + partitionCache.updateDatabaseCache(Collections.singletonMap(database1, disabledSchema)); + + Assert.assertNull(cache.getLastEntry(database1, deviceInDatabase1, measurement1)); + Assert.assertEquals( + values[0], cache.getLastEntry(database2, deviceInDatabase2, measurement1)); + } finally { + partitionCache.invalidAllCache(); + } + } + + @Test + public void testNeedLastCacheFalseSkipsInitLastCache() { + final String[] device0 = new String[] {"hebei", "p_1", "d_0"}; + final String[] measurements = new String[] {measurement1}; + final TimeValuePair[] values = + new TimeValuePair[] {new TimeValuePair(1L, new TsPrimitiveType.TsInt(1))}; + + final TableDeviceSchemaCache cache = createTestCache(); + final IDeviceID disabledDevice = + convertTagValuesToDeviceID(tableNeedLastCacheDisabled, device0); + + cache.initOrInvalidateLastCache(database1, disabledDevice, measurements, false); + cache.updateLastCacheIfExists(database1, disabledDevice, measurements, values); + + Assert.assertNull(cache.getLastEntry(database1, disabledDevice, measurement1)); + + final IDeviceID enabledDevice = convertTagValuesToDeviceID(table1, device0); + cache.initOrInvalidateLastCache(database1, enabledDevice, measurements, false); + cache.updateLastCacheIfExists(database1, enabledDevice, measurements, values); + + Assert.assertEquals(values[0], cache.getLastEntry(database1, enabledDevice, measurement1)); + } + private void updateLastCache4Query( final TableDeviceSchemaCache cache, final String database, diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java index 18653a52c5708..73bc61f271188 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java @@ -258,6 +258,7 @@ private ColumnHeaderConstant() { "max_schema_region_group_num"; public static final String DATA_REGION_GROUP_NUM_TABLE_MODEL = "data_region_group_num"; public static final String MAX_DATA_REGION_GROUP_NUM_TABLE_MODEL = "max_data_region_group_num"; + public static final String NEED_LAST_CACHE_TABLE_MODEL = "need_last_cache"; public static final String REGION_ID_TABLE_MODEL = "region_id"; public static final String DATANODE_ID_TABLE_MODEL = "datanode_id"; @@ -345,6 +346,7 @@ private ColumnHeaderConstant() { public static final String PRIVILEGES = "Privileges"; public static final String COMMENT = "Comment"; public static final String TABLE_TYPE = "TableType"; + public static final String NEED_LAST_CACHE = "NeedLastCache"; public static final String VIEW = "View"; public static final String CREATE_VIEW = "Create View"; @@ -432,7 +434,8 @@ private ColumnHeaderConstant() { new ColumnHeader(SCHEMA_REGION_GROUP_NUM, TSDataType.INT32), new ColumnHeader(MAX_SCHEMA_REGION_GROUP_NUM, TSDataType.INT32), new ColumnHeader(DATA_REGION_GROUP_NUM, TSDataType.INT32), - new ColumnHeader(MAX_DATA_REGION_GROUP_NUM, TSDataType.INT32)); + new ColumnHeader(MAX_DATA_REGION_GROUP_NUM, TSDataType.INT32), + new ColumnHeader(NEED_LAST_CACHE, TSDataType.BOOLEAN)); public static final List showChildPathsColumnHeaders = ImmutableList.of( @@ -755,7 +758,8 @@ private ColumnHeaderConstant() { new ColumnHeader(SCHEMA_REGION_GROUP_NUM, TSDataType.INT32), new ColumnHeader(MAX_SCHEMA_REGION_GROUP_NUM, TSDataType.INT32), new ColumnHeader(DATA_REGION_GROUP_NUM, TSDataType.INT32), - new ColumnHeader(MAX_DATA_REGION_GROUP_NUM, TSDataType.INT32)); + new ColumnHeader(MAX_DATA_REGION_GROUP_NUM, TSDataType.INT32), + new ColumnHeader(NEED_LAST_CACHE, TSDataType.BOOLEAN)); public static final List describeTableColumnHeaders = ImmutableList.of( @@ -804,7 +808,8 @@ private ColumnHeaderConstant() { new ColumnHeader(COLUMN_TTL, TSDataType.TEXT), new ColumnHeader(STATUS, TSDataType.TEXT), new ColumnHeader(COMMENT, TSDataType.TEXT), - new ColumnHeader(TABLE_TYPE, TSDataType.TEXT)); + new ColumnHeader(TABLE_TYPE, TSDataType.TEXT), + new ColumnHeader(NEED_LAST_CACHE, TSDataType.BOOLEAN)); public static final List LIST_USER_OR_ROLE_PRIVILEGES_COLUMN_HEADERS = ImmutableList.of( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java index 32f51a173bec4..e929ec83dbfcb 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java @@ -115,6 +115,10 @@ public class InformationSchema { databaseTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.MAX_DATA_REGION_GROUP_NUM_TABLE_MODEL, TSDataType.INT32)); + databaseTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.NEED_LAST_CACHE_TABLE_MODEL, TSDataType.BOOLEAN)); + databaseTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(DATABASES, databaseTable); final TsTable tableTable = new TsTable(TABLES); @@ -134,6 +138,10 @@ public class InformationSchema { ColumnHeaderConstant.COMMENT.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); tableTable.addColumnSchema( new AttributeColumnSchema(ColumnHeaderConstant.TABLE_TYPE_TABLE_MODEL, TSDataType.STRING)); + tableTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.NEED_LAST_CACHE_TABLE_MODEL, TSDataType.BOOLEAN)); + tableTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(TABLES, tableTable); final TsTable columnTable = new TsTable(COLUMNS); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java index 207e54720d762..2acbb993254d7 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java @@ -33,6 +33,8 @@ public class TreeViewSchema { public static final String ORIGINAL_NAME = "__original_name"; public static final String TREE_PATH_PATTERN = "__tree_path_pattern"; public static final String RESTRICT = "__restrict"; + public static final String UNSUPPORTED_NEED_LAST_CACHE_PROPERTY = + "The tree view does not support need_last_cache property."; public static boolean isTreeViewTable(final TsTable table) { return table.getPropValue(TREE_PATH_PATTERN).isPresent(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index 833c37a59407f..53b4db6b8a7cc 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -41,8 +41,10 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -63,7 +65,10 @@ public class TsTable { public static final String TIME_COLUMN_NAME = "time"; public static final String COMMENT_KEY = "__comment"; public static final String TTL_PROPERTY = "ttl"; - public static final Set TABLE_ALLOWED_PROPERTIES = Collections.singleton(TTL_PROPERTY); + public static final String NEED_LAST_CACHE_PROPERTY = "need_last_cache"; + public static final Set TABLE_ALLOWED_PROPERTIES = + Collections.unmodifiableSet( + new HashSet<>(Arrays.asList(TTL_PROPERTY, NEED_LAST_CACHE_PROPERTY))); private static final String OBJECT_STRING_ERROR = "When there are object fields, the %s %s shall not be '.', '..' or contain './', '.\\'."; protected String tableName; @@ -90,6 +95,7 @@ public class TsTable { // Cache, avoid string parsing private transient long ttlValue = Long.MIN_VALUE; + private transient Boolean needLastCache = null; private transient int tagNums = 0; private transient int fieldNum = 0; @@ -331,6 +337,14 @@ public long getTableTTL() { : Long.MAX_VALUE; } + public boolean getCachedNeedLastCache() { + if (needLastCache == null) { + needLastCache = + getPropValue(NEED_LAST_CACHE_PROPERTY).map(Boolean::parseBoolean).orElse(true); + } + return needLastCache; + } + public Map getProps() { readWriteLock.readLock().lock(); try { @@ -366,6 +380,7 @@ public void addProp(final String key, final String value) { props = new HashMap<>(); } props.put(key, value); + invalidateCachedPropValue(key); }); } @@ -376,9 +391,18 @@ public void removeProp(final String key) { return; } props.remove(key); + invalidateCachedPropValue(key); }); } + private void invalidateCachedPropValue(final String key) { + if (TTL_PROPERTY.equals(key)) { + ttlValue = Long.MIN_VALUE; + } else if (NEED_LAST_CACHE_PROPERTY.equals(key)) { + needLastCache = null; + } + } + public void serialize(final OutputStream stream) throws IOException { ReadWriteIOUtils.write(tableName, stream); ReadWriteIOUtils.write(columnSchemaMap.size(), stream); diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index a414584f80ec9..78322e616aad0 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -234,6 +234,7 @@ struct TDatabaseSchema { 9: optional i32 maxDataRegionGroupNum 10: optional i64 timePartitionOrigin 11: optional bool isTableModel + 12: optional bool needLastCache } // Schema @@ -747,6 +748,7 @@ struct TDatabaseInfo { 9: required i32 dataRegionNum 11: required i32 maxDataRegionNum 12: optional i64 timePartitionOrigin + 13: optional bool needLastCache } struct TGetDatabaseReq { @@ -1302,6 +1304,7 @@ struct TTableInfo { 3: optional i32 state 4: optional string comment 5: optional i32 type + 6: optional bool needLastCache } struct TCreateTableViewReq {