feat(test): display query_id on sqllogictest failure

tests/sqllogictests/Cargo.toml

@@ -23,6 +23,7 @@ cookie = { workspace = true }
 databend-common-exception = { workspace = true }
 env_logger = { workspace = true }
 futures-util = { workspace = true }
+glob = { workspace = true }
 mysql_async = { workspace = true }
 rand = { workspace = true }
 recursive = { workspace = true }
@@ -39,5 +40,8 @@ tokio = { workspace = true }
 url = { workspace = true }
 walkdir = { workspace = true }
 
+[dev-dependencies]
+tempfile = { workspace = true }
+
 [lints]
 workspace = true

tests/sqllogictests/README.md

@@ -15,13 +15,34 @@ Run all tests with specific handler.
 databend-sqllogictests --handlers <handler_name>
 ```
 ---
-Run tests under specific directory.
+Run tests by glob pattern.
 
+```shell
+databend-sqllogictests --run 'tests/sqllogictests/suites/base/**/*.test'
+```
+or
+```shell
+databend-sqllogictests --run '**/suites/base/**/*'
+```
+
+
+---
+Run a small set of files or directories with multiple glob patterns.
+```shell
+databend-sqllogictests --run 'tests/sqllogictests/suites/base/00_dummy/*.test,tests/sqllogictests/suites/base/03_common/*.test'
+```
+---
+Skip part of the matched files.
+```shell
+databend-sqllogictests --run 'tests/sqllogictests/suites/base/**/*.test' --skip 'tests/sqllogictests/suites/base/01_system'
+```
+---
+Run tests under a specific directory name found under `--suites`.
 ```shell
 databend-sqllogictests --run_dir <dir_name>
 ```
 ---
-Run tests under specific file. This is the most commonly used command because users do not need to run all tests at a time and only need to run their newly added test files or test files with changes
+Run tests under a specific file name found under `--suites`.
 ```shell
 databend-sqllogictests --run_file <file_name>
 ```
@@ -44,7 +65,7 @@ databend-sqllogictests --help
 ### Parallel
 If you want to run test files in parallel, please add the following args:
 ```shell
-databend-sqllogictest --enable_sandbox --parallel <number>
+databend-sqllogictests --enable_sandbox --parallel <number>
 ```
 
 When start databend query, please add `--internal-enable-sandbox-tenant` config.
@@ -73,4 +94,4 @@ For more information about arguments, such as <type_string>, <sort_mode>, <label
 
 ### Aditional features
 
-- sql with regexp pattern `\$RAND_(\d+)_(\d+)` will be replaced by a random number from the range.
+- sql with regexp pattern `\$RAND_(\d+)_(\d+)` will be replaced by a random number from the range.

tests/sqllogictests/src/arg.rs

@@ -17,35 +17,63 @@ use clap::Parser;
 // Add options when run sqllogictest, such as specific dir or file
 #[derive(Parser, Debug, Clone)]
 pub struct SqlLogicTestArgs {
+    #[arg(
+        long = "run",
+        use_value_delimiter = true,
+        value_delimiter = ',',
+        conflicts_with_all = ["dir", "file", "skipped_dir", "skipped_file"],
+        help = "Run sqllogictests by glob patterns."
+    )]
+    pub run: Option<Vec<String>>,
+
+    #[arg(
+        long = "skip",
+        use_value_delimiter = true,
+        value_delimiter = ',',
+        requires = "run",
+        conflicts_with_all = ["dir", "file", "skipped_dir", "skipped_file"],
+        help = "Skip sqllogictests by glob patterns. Requires --run"
+    )]
+    pub skip: Option<Vec<String>>,
+
+    // Choose suits to run
+    #[arg(
+        short = 'u',
+        long = "suites",
+        help = "The tests to be run will come from under suits",
+        default_value = "tests/sqllogictests/suites"
+    )]
+    pub suites: String,
+
     // Set specific dir to run
     #[arg(
         short = 'd',
         long = "run_dir",
-        help = "Run sqllogictests in specific directory, the arg is optional"
+        help = "Run sqllogictests in a specific directory name found under --suites, the arg is optional"
     )]
     pub dir: Option<String>,
 
     // Set specific test file to run
     #[arg(
         short = 'f',
         long = "run_file",
-        help = "Run sqllogictests in specific test file, the arg is optional"
+        help = "Run sqllogictests in a specific test file name found under --suites, the arg is optional"
     )]
     pub file: Option<String>,
 
     // Set specific dir to skip
     #[arg(
         short = 's',
         long = "skip_dir",
-        help = "Skip sqllogictests in specific directory, the arg is optional"
+        help = "Skip sqllogictests in specific directory names found under --suites, the arg is optional"
     )]
     pub skipped_dir: Option<String>,
 
     // Set specific file to skip
     #[arg(
         short = 'x',
         long = "skip_file",
-        help = "Skip sqllogictests in specific test file, the arg is optional"
+        help = "Skip sqllogictests in specific test file names found under --suites, the arg is optional"
     )]
     pub skipped_file: Option<String>,
 
@@ -59,15 +87,6 @@ pub struct SqlLogicTestArgs {
     )]
     pub handlers: Option<Vec<String>>,
 
-    // Choose suits to run
-    #[arg(
-        short = 'u',
-        long = "suites",
-        help = "The tests to be run will come from under suits",
-        default_value = "tests/sqllogictests/suites"
-    )]
-    pub suites: String,
-
     // If enable complete mode
     #[arg(
         short = 'c',

tests/sqllogictests/src/diagnostics.rs

@@ -0,0 +1,91 @@
+// Copyright 2021 Datafuse Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use sqllogictest::Runner;
+
+use crate::report::NonDefaultSetting;
+use crate::util::ColumnType;
+
+const LAST_QUERY_ID_COLUMN_TYPES: &str = "T";
+const LAST_QUERY_ID_SQL: &str = "SELECT LAST_QUERY_ID()";
+const NON_DEFAULT_SETTINGS_COLUMN_TYPES: &str = "TTTT";
+const NON_DEFAULT_SETTINGS_SQL: &str = "SELECT name, value, default, level \
+     FROM system.settings \
+     WHERE value <> default \
+     ORDER BY name";
+
+pub(crate) struct FailureDiagnostics {
+    pub(crate) query_id: Option<String>,
+    pub(crate) non_default_settings: Vec<NonDefaultSetting>,
+}
+
+pub(crate) trait DiagnosticsQueryExecutor {
+    async fn query_rows(&mut self, column_types: &str, sql: &str) -> Option<Vec<Vec<String>>>;
+}
+
+impl<D, M> DiagnosticsQueryExecutor for Runner<D, M>
+where
+    D: sqllogictest::AsyncDB<ColumnType = ColumnType>,
+    M: sqllogictest::MakeConnection<Conn = D>,
+{
+    async fn query_rows(&mut self, column_types: &str, sql: &str) -> Option<Vec<Vec<String>>> {
+        let script = format!("query {column_types}\n{sql}\n----\n");
+        let records = sqllogictest::parse::<ColumnType>(&script).ok()?;
+        let record = records.into_iter().next()?;
+        if let sqllogictest::RecordOutput::Query { rows, .. } = self.apply_record(record).await {
+            Some(rows)
+        } else {
+            None
+        }
+    }
+}
+
+pub(crate) async fn capture_failure_diagnostics(
+    executor: &mut impl DiagnosticsQueryExecutor,
+) -> FailureDiagnostics {
+    let query_id = executor
+        .query_rows(LAST_QUERY_ID_COLUMN_TYPES, LAST_QUERY_ID_SQL)
+        .await
+        .and_then(extract_last_query_id);
+    let non_default_settings = executor
+        .query_rows(NON_DEFAULT_SETTINGS_COLUMN_TYPES, NON_DEFAULT_SETTINGS_SQL)
+        .await
+        .map(extract_non_default_settings)
+        .unwrap_or_default();
+
+    FailureDiagnostics {
+        query_id,
+        non_default_settings,
+    }
+}
+
+fn extract_last_query_id(rows: Vec<Vec<String>>) -> Option<String> {
+    rows.into_iter()
+        .next()
+        .and_then(|row| row.into_iter().next())
+}
+
+fn extract_non_default_settings(rows: Vec<Vec<String>>) -> Vec<NonDefaultSetting> {
+    rows.into_iter()
+        .filter_map(|row| match row.as_slice() {
+            [name, value, default_value, level] => Some(NonDefaultSetting::new(
+                name.clone(),
+                value.clone(),
+                default_value.clone(),
+                level.clone(),
+            )),
+            _ => None,
+        })
+        .collect()
+}