Merge branch 'master' of https://github.com/apache/iotdb into fix-audit-logger

Author: Caideyipi

integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java

@@ -137,4 +137,10 @@ public DataNodeConfig setDataNodeMemoryProportion(String dataNodeMemoryProportio
     setProperty("datanode_memory_proportion", dataNodeMemoryProportion);
     return this;
   }
+
+  @Override
+  public DataNodeConfig setQueryCostStatWindow(int queryCostStatWindow) {
+    setProperty("query_cost_stat_window", String.valueOf(queryCostStatWindow));
+    return this;
+  }
 }

integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java

@@ -93,4 +93,9 @@ public DataNodeConfig setDeleteWalFilesPeriodInMs(long deleteWalFilesPeriodInMs)
   public DataNodeConfig setDataNodeMemoryProportion(String dataNodeMemoryProportion) {
     return this;
   }
+
+  @Override
+  public DataNodeConfig setQueryCostStatWindow(int queryCostStatWindow) {
+    return this;
+  }
 }

integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java

@@ -51,4 +51,6 @@ DataNodeConfig setLoadTsFileAnalyzeSchemaMemorySizeInBytes(
   DataNodeConfig setDeleteWalFilesPeriodInMs(long deleteWalFilesPeriodInMs);
 
   DataNodeConfig setDataNodeMemoryProportion(String dataNodeMemoryProportion);
+
+  DataNodeConfig setQueryCostStatWindow(int queryCostStatWindow);
 }

integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeClusterConfigIT.java

@@ -107,7 +107,4 @@ private void aiNodeRegisterAndRemoveTest(Statement statement) throws SQLExceptio
     }
     Assert.fail("The target AINode is not removed successfully after all retries.");
   }
-
-  // TODO: We might need to add remove unknown test in the future, but current infrastructure is too
-  // hard to implement it.
 }

integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeConcurrentForecastIT.java

@@ -49,7 +49,7 @@ public class AINodeConcurrentForecastIT {
   private static final Logger LOGGER = LoggerFactory.getLogger(AINodeConcurrentForecastIT.class);
 
   private static final String FORECAST_TABLE_FUNCTION_SQL_TEMPLATE =
-      "SELECT * FROM FORECAST(model_id=>'%s', input=>(SELECT time,s FROM root.AI) ORDER BY time, forecast_length=>%d)";
+      "SELECT * FROM FORECAST(model_id=>'%s', input=>(SELECT time,s FROM root.AI) ORDER BY time, output_length=>%d)";
 
   @BeforeClass
   public static void setUp() throws Exception {

integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeModelManageIT.java

@@ -131,15 +131,15 @@ private void userDefinedModelManagementTest(Statement statement)
   public void dropBuiltInModelErrorTestInTree() throws SQLException {
     try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT);
         Statement statement = connection.createStatement()) {
-      errorTest(statement, "drop model sundial", "1510: Cannot delete built-in model: sundial");
+      errorTest(statement, "drop model sundial", "1506: Cannot delete built-in model: sundial");
     }
   }
 
   @Test
   public void dropBuiltInModelErrorTestInTable() throws SQLException {
     try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
         Statement statement = connection.createStatement()) {
-      errorTest(statement, "drop model sundial", "1510: Cannot delete built-in model: sundial");
+      errorTest(statement, "drop model sundial", "1506: Cannot delete built-in model: sundial");
     }
   }
 

integration-test/src/test/java/org/apache/iotdb/ainode/utils/AINodeTestUtils.java

@@ -20,6 +20,7 @@
 package org.apache.iotdb.ainode.utils;
 
 import com.google.common.collect.ImmutableSet;
+import org.junit.Assert;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -34,6 +35,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -101,6 +103,7 @@ public static void errorTest(Statement statement, String sql, String errorMessag
   public static void concurrentInference(
       Statement statement, String sql, int threadCnt, int loop, int expectedOutputLength)
       throws InterruptedException {
+    AtomicBoolean allPass = new AtomicBoolean(true);
     Thread[] threads = new Thread[threadCnt];
     for (int i = 0; i < threadCnt; i++) {
       threads[i] =
@@ -113,12 +116,23 @@ public static void concurrentInference(
                       while (resultSet.next()) {
                         outputCnt++;
                       }
-                      assertEquals(expectedOutputLength, outputCnt);
+                      if (expectedOutputLength != outputCnt) {
+                        allPass.set(false);
+                        fail(
+                            "Output count mismatch for SQL: "
+                                + sql
+                                + ". Expected: "
+                                + expectedOutputLength
+                                + ", but got: "
+                                + outputCnt);
+                      }
                     } catch (SQLException e) {
+                      allPass.set(false);
                       fail(e.getMessage());
                     }
                   }
                 } catch (Exception e) {
+                  allPass.set(false);
                   fail(e.getMessage());
                 }
               });
@@ -130,6 +144,7 @@ public static void concurrentInference(
         fail("Thread timeout after 10 minutes");
       }
     }
+    Assert.assertTrue(allPass.get());
   }
 
   public static void checkModelOnSpecifiedDevice(Statement statement, String modelId, String device)

integration-test/src/test/java/org/apache/iotdb/db/it/audit/IoTDBAuditLogBasicIT.java

@@ -32,6 +32,7 @@
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
@@ -52,6 +53,7 @@
  * This test class ensures the audit log behave exactly the same as we expected, including the
  * number, sequence and content of the audit logs.
  */
+@Ignore
 @RunWith(IoTDBTestRunner.class)
 @Category({LocalStandaloneIT.class})
 public class IoTDBAuditLogBasicIT {

integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/informationschema/IoTDBCurrentQueriesIT.java

@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.relational.it.query.recent.informationschema;
+
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.db.queryengine.execution.QueryState;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
+import org.apache.iotdb.itbase.env.BaseEnv;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.END_TIME_TABLE_MODEL;
+import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.NUMS;
+import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.STATEMENT_TABLE_MODEL;
+import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.STATE_TABLE_MODEL;
+import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.USER_TABLE_MODEL;
+import static org.apache.iotdb.commons.schema.table.InformationSchema.getSchemaTables;
+import static org.apache.iotdb.db.it.utils.TestUtils.createUser;
+import static org.apache.iotdb.itbase.env.BaseEnv.TABLE_SQL_DIALECT;
+import static org.junit.Assert.fail;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({TableLocalStandaloneIT.class})
+// This IT will run at least 60s, so we only run it in 1C1D
+public class IoTDBCurrentQueriesIT {
+  private static final int CURRENT_QUERIES_COLUMN_NUM =
+      getSchemaTables().get("current_queries").getColumnNum();
+  private static final int QUERIES_COSTS_HISTOGRAM_COLUMN_NUM =
+      getSchemaTables().get("queries_costs_histogram").getColumnNum();
+  private static final String ADMIN_NAME =
+      CommonDescriptor.getInstance().getConfig().getDefaultAdminName();
+  private static final String ADMIN_PWD =
+      CommonDescriptor.getInstance().getConfig().getAdminPassword();
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    EnvFactory.getEnv().initClusterEnvironment();
+    createUser("test", "test123123456");
+  }
+
+  @AfterClass
+  public static void tearDown() throws Exception {
+    EnvFactory.getEnv().cleanClusterEnvironment();
+  }
+
+  @Test
+  public void testCurrentQueries() {
+    try {
+      Assert.assertEquals(3, QUERIES_COSTS_HISTOGRAM_COLUMN_NUM);
+
+      Connection connection =
+          EnvFactory.getEnv().getConnection(ADMIN_NAME, ADMIN_PWD, BaseEnv.TABLE_SQL_DIALECT);
+      Statement statement = connection.createStatement();
+      statement.execute("USE information_schema");
+      statement.execute("set configuration \"query_cost_stat_window\"='1'");
+
+      // 1. query current_queries table
+      String sql = "SELECT * FROM current_queries WHERE state='RUNNING'";
+      ResultSet resultSet = statement.executeQuery(sql);
+      ResultSetMetaData metaData = resultSet.getMetaData();
+      Assert.assertEquals(CURRENT_QUERIES_COLUMN_NUM, metaData.getColumnCount());
+      int rowNum = 0;
+      while (resultSet.next()) {
+        Assert.assertEquals(QueryState.RUNNING.name(), resultSet.getString(STATE_TABLE_MODEL));
+        Assert.assertEquals(null, resultSet.getString(END_TIME_TABLE_MODEL));
+        Assert.assertEquals(sql, resultSet.getString(STATEMENT_TABLE_MODEL));
+        Assert.assertEquals(ADMIN_NAME, resultSet.getString(USER_TABLE_MODEL));
+        rowNum++;
+      }
+      Assert.assertEquals(1, rowNum);
+      resultSet.close();
+
+      // 2. query queries_costs_histogram table
+      sql = "SELECT * FROM queries_costs_histogram";
+      resultSet = statement.executeQuery(sql);
+      metaData = resultSet.getMetaData();
+      Assert.assertEquals(QUERIES_COSTS_HISTOGRAM_COLUMN_NUM, metaData.getColumnCount());
+      rowNum = 0;
+      int queriesCount = 0;
+      while (resultSet.next()) {
+        int nums = resultSet.getInt(NUMS);
+        if (nums > 0) {
+          queriesCount++;
+        }
+        rowNum++;
+      }
+      Assert.assertEquals(1, queriesCount);
+      Assert.assertEquals(61, rowNum);
+
+      // 3. requery current_queries table
+      sql = "SELECT * FROM current_queries WHERE state='FINISHED'";
+      resultSet = statement.executeQuery(sql);
+      metaData = resultSet.getMetaData();
+      Assert.assertEquals(CURRENT_QUERIES_COLUMN_NUM, metaData.getColumnCount());
+      rowNum = 0;
+      int finishedQueries = 0;
+      while (resultSet.next()) {
+        if (QueryState.FINISHED.name().equals(resultSet.getString(STATE_TABLE_MODEL))) {
+          finishedQueries++;
+        }
+        rowNum++;
+      }
+      // two rows in the result, 2 FINISHED
+      Assert.assertEquals(2, rowNum);
+      Assert.assertEquals(2, finishedQueries);
+      resultSet.close();
+
+      // 4. test the expired QueryInfo was evicted
+      Thread.sleep(61_001);
+      sql = "SELECT * FROM current_queries";
+      resultSet = statement.executeQuery(sql);
+      rowNum = 0;
+      while (resultSet.next()) {
+        rowNum++;
+      }
+      // one row in the result, current query
+      Assert.assertEquals(1, rowNum);
+      resultSet.close();
+
+      sql = "SELECT * FROM queries_costs_histogram";
+      resultSet = statement.executeQuery(sql);
+      queriesCount = 0;
+      while (resultSet.next()) {
+        int nums = resultSet.getInt(NUMS);
+        if (nums > 0) {
+          queriesCount++;
+        }
+      }
+      // the last current_queries table query was recorded, others are evicted
+      Assert.assertEquals(1, queriesCount);
+    } catch (Exception e) {
+      fail(e.getMessage());
+    }
+
+    // 5. test privilege
+    testPrivilege();
+
+    // 6. test more configurations
+    testMoreConfigurations();
+  }
+
+  private void testPrivilege() {
+    // 1. test current_queries table
+    try (Connection connection =
+            EnvFactory.getEnv().getConnection("test", "test123123456", TABLE_SQL_DIALECT);
+        Statement statement = connection.createStatement()) {
+      String sql = "SELECT * FROM information_schema.current_queries";
+
+      // another user executes a query
+      try (Connection connection2 =
+          EnvFactory.getEnv().getConnection(ADMIN_NAME, ADMIN_PWD, BaseEnv.TABLE_SQL_DIALECT)) {
+        ResultSet resultSet = connection2.createStatement().executeQuery(sql);
+        resultSet.close();
+      } catch (Exception e) {
+        fail(e.getMessage());
+      }
+
+      // current user query current_queries table
+      ResultSet resultSet = statement.executeQuery(sql);
+      int rowNum = 0;
+      while (resultSet.next()) {
+        rowNum++;
+      }
+      // only current query in the result
+      Assert.assertEquals(1, rowNum);
+    } catch (SQLException e) {
+      fail(e.getMessage());
+    }
+
+    // 2. test queries_costs_histogram table
+    try (Connection connection =
+            EnvFactory.getEnv().getConnection("test", "test123123456", TABLE_SQL_DIALECT);
+        Statement statement = connection.createStatement()) {
+      statement.executeQuery("SELECT * FROM information_schema.queries_costs_histogram");
+    } catch (SQLException e) {
+      Assert.assertEquals(
+          "803: Access Denied: No permissions for this operation, please add privilege SYSTEM",
+          e.getMessage());
+    }
+  }
+
+  private void testMoreConfigurations() {
+    try {
+      Connection connection =
+          EnvFactory.getEnv().getConnection(ADMIN_NAME, ADMIN_PWD, BaseEnv.TABLE_SQL_DIALECT);
+      Statement statement = connection.createStatement();
+      statement.execute("USE information_schema");
+
+      statement.execute("set configuration \"query_cost_stat_window\"='0'");
+      Thread.sleep(1_001);
+
+      // query_cost_stat_window = 0, history queries are cleared
+      String sql = "SELECT * FROM current_queries WHERE state='FINISHED'";
+      ResultSet resultSet = statement.executeQuery(sql);
+      ResultSetMetaData metaData = resultSet.getMetaData();
+      Assert.assertEquals(CURRENT_QUERIES_COLUMN_NUM, metaData.getColumnCount());
+      int rowNum = 0;
+      while (resultSet.next()) {
+        rowNum++;
+      }
+      Assert.assertEquals(0, rowNum);
+      resultSet.close();
+
+      statement.execute("set configuration \"query_cost_stat_window\"='1040000000'");
+      // make query_cost_stat_window very large but not overflow
+      resultSet = statement.executeQuery(sql);
+      while (resultSet.next()) {
+        rowNum++;
+      }
+      resultSet.close();
+
+      resultSet = statement.executeQuery(sql);
+      rowNum = 0;
+      while (resultSet.next()) {
+        rowNum++;
+      }
+      // the history SQL is recorded
+      Assert.assertEquals(1, rowNum);
+      resultSet.close();
+
+      // make query_cost_stat_window overflow
+      try {
+        statement.execute("set configuration \"query_cost_stat_window\"='10400000000'");
+      } catch (Exception e) {
+        Assert.assertTrue(
+            e.getMessage()
+                .contains("java.lang.NumberFormatException: For input string: \"10400000000\""));
+      }
+    } catch (Exception e) {
+      fail(e.getMessage());
+    }
+  }
+}