Merge branch 'main' into bump-deps

Author: mergify[bot]

README.md

@@ -127,7 +127,8 @@ docker run --net=host  datafuselabs/databend
 - [How to Load Data from Remote Files](https://databend.rs/doc/load-data/http)
 - [How to Load Data from Amazon S3](https://databend.rs/doc/load-data/s3)
 - [How to Load Data from Databend Stages](https://databend.rs/doc/load-data/stage)
-
+* [Querying Data in Staged Files](https://databend.rs/doc/load-data/querying-stage)
+* [Transforming Data During a Load](http://databend.rs/doc/load-data/data-load-transform)
 
 ### Unloading Data from Databend
 

docs/doc/01-guides/index.md

@@ -30,7 +30,8 @@ These tutorials are intended to help you get started with Databend:
 * [How to Load Data from Amazon S3](../12-load-data/01-s3.md)
 * [How to Load Data from Local File System](../12-load-data/02-local.md)
 * [How to Load Data from Remote Files](../12-load-data/04-http.md)
-
+* [Querying Data in Staged Files](../12-load-data/05-querying-stage.md)
+* [Transforming Data During a Load](../12-load-data/06-data-load-transform.md)
 
 ## Unloading Data from Databend
 

docs/doc/12-load-data/05-querying-stage.md

@@ -0,0 +1,140 @@
+---
+title: Querying Data in Staged Files
+description:  Learn how to use standard SQL to query data files in internal or external storage stages with Databend
+---
+
+Databend supports using standard SQL to query data files located in an internal stage or named external stage (Amazon S3, Google Cloud Storage, or Microsoft Azure). This can be useful for inspecting or viewing the contents of the staged files, particularly before loading or after unloading data.
+
+The schema is automatically detected, same as [infer_schema](../15-sql-functions/112-table-functions/infer_schema.md).
+
+## Query Syntax and Parameters
+
+```sql
+SELECT <columns> FROM
+{@<stage_name>[/<path>] | '<uri>'} [(
+  [ PARTTERN => '<regex_pattern>']
+  [ FILE_FORMAT => '<format_name>']
+  [ FILES => ( 'file_name' [ , 'file_name' ... ] ) ]
+  [ ENDPOINT_URL => <'url'> ]
+  [ AWS_KEY_ID => <'aws_key_id'> ]
+  [ AWS_KEY_SECRET => <'aws_key_secret'> ]
+  [ ACCESS_KEY_ID => <'access_key_id'> ]
+  [ ACCESS_KEY_SECRET => <'access_key_secret'> ]
+  [ SECRET_ACCESS_KEY => <'secret_access_key'> ]
+  [ SESSION_TOKEN => <'session_token'> ]
+  [ REGION => <'region'> ]
+  [ ENABLE_VIRTUAL_HOST_STYLE => true|false ]
+)]
+```
+
+The function parameters are as follows:
+
+### FILE_FORMAT = '<format_name>'
+
+`<format_name>` should be one of the following:
+
+1. A built-in file format (see [Input & Output File Formats](../13-sql-reference/50-file-format-options.md).
+2. A named file format created by [CREATE FILE FORMAT](../14-sql-commands/00-ddl/100-file-format/01-ddl-create-file-format.md).
+
+If not specified for named stages, the format of the stage should be used.
+
+:::caution
+
+Only parquet file format is currently supported.
+
+:::
+
+### PATTERN = '<regex_pattern>'
+
+A [PCRE2](https://www.pcre.org/current/doc/html/)-based regular expression pattern string, enclosed in single quotes, specifying the file names to match. Click [here](#loading-data-with-pattern-matching) to see an example. For PCRE2 syntax, see http://www.pcre.org/current/doc/html/pcre2syntax.html.
+
+
+### FILES  = ( 'file1' [ , 'file2' ... ] )
+
+Specifies a list of one or more files names (separated by commas) to be read.
+
+### Connection Options for `<uri>` only
+
+These include:
+
+- ENDPOINT_URL
+- AWS_KEY_ID
+- AWS_SECRET_KEY
+- ACCESS_KEY_ID
+- ACCESS_KEY_SECRET
+- SECRET_ACCESS_KEY
+- SESSION_TOKEN
+- REGION
+- ENABLE_VIRTUAL_HOST_STYLE
+
+They are explained in [Create Stage](../14-sql-commands/00-ddl/40-stage/01-ddl-create-stage.md).
+
+## Query Examples
+
+### Example 1: Querying Columns in a Parquet File
+
+Let's assume you have a Parquet file called "example.parquet" with the following data:
+
+```text
+
+| name  | age | city   |
+|-------|-----|--------|
+| Alice | 28  | London |
+| Bob   | 35  | Berlin |
+| Carol | 42  | Paris  |
+```
+
+You can query the "name" and "age" columns from this file using the following query:
+```sql
+SELECT name, age FROM @internal_stage/example.parquet;
+```
+
+### Example 2: Querying Data using Pattern Matching
+
+Suppose you have a directory with several Parquet files, and you only want to query the files that end with ".parquet". You can do this using the PATTERN parameter.
+
+Assuming the following file structure:
+
+```text
+data/
+  ├── 2022-01-01.parquet
+  ├── 2022-01-02.parquet
+  ├── 2022-01-03.parquet
+  ├── 2022-01-04.parquet
+  └── 2022-01-05.parquet
+```
+
+You can query the "name" and "age" columns from all files in the "data" folder using the following query:
+
+```sql
+SELECT name, age FROM @internal_stage/data/
+(file_format => 'parquet', pattern => '.*parquet');
+```
+
+### Example 3: Querying Data in an External Stage
+
+Let's assume you have data in an Amazon S3 bucket and you want to query it using Databend. You can use the following query:
+```sql
+SELECT *  FROM 's3://bucket/' 
+(
+ access_key_id => 'your-access-key-id', 
+ secret_access_key => 'your-secret-access-key',
+ endpoint_url => 'your-object-storage-endpoint',
+ file_format => 'parquet',
+ pattern => '.*parquet'
+);  
+```
+
+### Example 4: Querying Data from a URI
+
+You can also use a URI to query data files from a remote location, like this:
+
+```sql
+SELECT count(*), author FROM 'https://datafuse-1253727613.cos.ap-hongkong.myqcloud.com/data/books.parquet'
+(file_format => 'parquet')
+GROUP BY author;
+```
+
+## Conclusion
+
+We hope this document has provided you with a better understanding of how to use standard SQL to query data files in an internal or external storage stage with Databend. By using the query function, you can easily inspect or view the contents of staged files, making it easier to load and unload data. The examples we provided should help you get started with using this powerful feature.

…c/12-load-data/05-data-load-transform.md …c/12-load-data/06-data-load-transform.mddocs/doc/12-load-data/05-data-load-transform.md renamed to docs/doc/12-load-data/06-data-load-transform.md docs/doc/12-load-data/05-data-load-transform.md renamed to docs/doc/12-load-data/06-data-load-transform.md

@@ -2074,19 +2074,47 @@ impl<'ast> Visitor<'ast> for AstFormatVisitor {
                 FormatTreeNode::with_children(selection_format_ctx, vec![selection_child]);
             children.push(selection_node);
         }
-        if !stmt.group_by.is_empty() {
-            let mut group_by_list_children = Vec::with_capacity(stmt.group_by.len());
-            for group_by in stmt.group_by.iter() {
-                self.visit_expr(group_by);
-                group_by_list_children.push(self.children.pop().unwrap());
+        match &stmt.group_by {
+            Some(GroupBy::Normal(exprs)) => {
+                let mut group_by_list_children = Vec::with_capacity(exprs.len());
+                for group_by in exprs.iter() {
+                    self.visit_expr(group_by);
+                    group_by_list_children.push(self.children.pop().unwrap());
+                }
+                let group_by_list_name = "GroupByList".to_string();
+                let group_by_list_format_ctx = AstFormatContext::with_children(
+                    group_by_list_name,
+                    group_by_list_children.len(),
+                );
+                let group_by_list_node =
+                    FormatTreeNode::with_children(group_by_list_format_ctx, group_by_list_children);
+                children.push(group_by_list_node);
             }
-            let group_by_list_name = "GroupByList".to_string();
-            let group_by_list_format_ctx =
-                AstFormatContext::with_children(group_by_list_name, group_by_list_children.len());
-            let group_by_list_node =
-                FormatTreeNode::with_children(group_by_list_format_ctx, group_by_list_children);
-            children.push(group_by_list_node);
+            Some(GroupBy::GroupingSets(sets)) => {
+                let mut grouping_sets = Vec::with_capacity(sets.len());
+                for set in sets.iter() {
+                    let mut grouping_set = Vec::with_capacity(set.len());
+                    for expr in set.iter() {
+                        self.visit_expr(expr);
+                        grouping_set.push(self.children.pop().unwrap());
+                    }
+                    let name = "GroupingSet".to_string();
+                    let grouping_set_format_ctx =
+                        AstFormatContext::with_children(name, grouping_set.len());
+                    let grouping_set_node =
+                        FormatTreeNode::with_children(grouping_set_format_ctx, grouping_set);
+                    grouping_sets.push(grouping_set_node);
+                }
+                let group_by_list_name = "GroupByList".to_string();
+                let group_by_list_format_ctx =
+                    AstFormatContext::with_children(group_by_list_name, grouping_sets.len());
+                let group_by_list_node =
+                    FormatTreeNode::with_children(group_by_list_format_ctx, grouping_sets);
+                children.push(group_by_list_node);
+            }
+            _ => {}
         }
+
         if let Some(having) = &stmt.having {
             self.visit_expr(having);
             let having_child = self.children.pop().unwrap();

src/query/ast/src/ast/format/ast_format.rs

@@ -21,6 +21,7 @@ use crate::ast::format::syntax::interweave_comma;
 use crate::ast::format::syntax::parenthenized;
 use crate::ast::format::syntax::NEST_FACTOR;
 use crate::ast::Expr;
+use crate::ast::GroupBy;
 use crate::ast::JoinCondition;
 use crate::ast::JoinOperator;
 use crate::ast::OrderByExpr;
@@ -194,12 +195,19 @@ fn pretty_selection(selection: Option<Expr>) -> RcDoc<'static> {
     }
 }
 
-fn pretty_group_by(group_by: Vec<Expr>) -> RcDoc<'static> {
-    if !group_by.is_empty() {
-        RcDoc::line()
+fn pretty_group_set(set: Vec<Expr>) -> RcDoc<'static> {
+    RcDoc::nil()
+        .append(RcDoc::text("("))
+        .append(inline_comma(set.into_iter().map(pretty_expr)))
+        .append(RcDoc::text(")"))
+}
+
+fn pretty_group_by(group_by: Option<GroupBy>) -> RcDoc<'static> {
+    match group_by {
+        Some(GroupBy::Normal(exprs)) => RcDoc::line()
             .append(
                 RcDoc::text("GROUP BY").append(
-                    if group_by.len() > 1 {
+                    if exprs.len() > 1 {
                         RcDoc::line()
                     } else {
                         RcDoc::space()
@@ -208,12 +216,20 @@ fn pretty_group_by(group_by: Vec<Expr>) -> RcDoc<'static> {
                 ),
             )
             .append(
-                interweave_comma(group_by.into_iter().map(pretty_expr))
+                interweave_comma(exprs.into_iter().map(pretty_expr))
+                    .nest(NEST_FACTOR)
+                    .group(),
+            ),
+        Some(GroupBy::GroupingSets(sets)) => RcDoc::line()
+            .append(RcDoc::text("GROUP BY GROUPING SETS (").append(RcDoc::line().nest(NEST_FACTOR)))
+            .append(
+                interweave_comma(sets.into_iter().map(pretty_group_set))
                     .nest(NEST_FACTOR)
                     .group(),
             )
-    } else {
-        RcDoc::nil()
+            .append(RcDoc::line())
+            .append(RcDoc::text(")")),
+        _ => RcDoc::nil(),
     }
 }
 

src/query/ast/src/ast/format/syntax/query.rs

@@ -82,11 +82,22 @@ pub struct SelectStmt {
     // `WHERE` clause
     pub selection: Option<Expr>,
     // `GROUP BY` clause
-    pub group_by: Vec<Expr>,
+    pub group_by: Option<GroupBy>,
     // `HAVING` clause
     pub having: Option<Expr>,
 }
 
+/// Group by Clause.
+#[derive(Debug, Clone, PartialEq)]
+pub enum GroupBy {
+    /// GROUP BY expr [, expr]*
+    Normal(Vec<Expr>),
+    /// GROUP BY GROUPING SETS ( GroupSet [, GroupSet]* )
+    ///
+    /// GroupSet := (expr [, expr]*) | expr
+    GroupingSets(Vec<Vec<Expr>>),
+}
+
 /// A relational set expression, like `SELECT ... FROM ... {UNION|EXCEPT|INTERSECT} SELECT ... FROM ...`
 #[derive(Debug, Clone, PartialEq)]
 pub enum SetExpr {
@@ -442,9 +453,25 @@ impl Display for SelectStmt {
         }
 
         // GROUP BY clause
-        if !self.group_by.is_empty() {
+        if self.group_by.is_some() {
             write!(f, " GROUP BY ")?;
-            write_comma_separated_list(f, &self.group_by)?;
+            match self.group_by.as_ref().unwrap() {
+                GroupBy::Normal(exprs) => {
+                    write_comma_separated_list(f, exprs)?;
+                }
+                GroupBy::GroupingSets(sets) => {
+                    write!(f, "GROUPING SETS (")?;
+                    for (i, set) in sets.iter().enumerate() {
+                        if i > 0 {
+                            write!(f, ", ")?;
+                        }
+                        write!(f, "(")?;
+                        write_comma_separated_list(f, set)?;
+                        write!(f, ")")?;
+                    }
+                    write!(f, ")")?;
+                }
+            }
         }
 
         // HAVING clause

src/query/ast/src/ast/query.rs

@@ -555,7 +555,7 @@ pub enum SetOperationElement {
         select_list: Box<Vec<SelectTarget>>,
         from: Box<Vec<TableReference>>,
         selection: Box<Option<Expr>>,
-        group_by: Box<Vec<Expr>>,
+        group_by: Option<GroupBy>,
         having: Box<Option<Expr>>,
     },
     SetOperation {
@@ -565,6 +565,25 @@ pub enum SetOperationElement {
     Group(SetExpr),
 }
 
+pub fn group_by_items(i: Input) -> IResult<GroupBy> {
+    let normal = map(rule! { ^#comma_separated_list1(expr) }, |groups| {
+        GroupBy::Normal(groups)
+    });
+    let group_set = alt((
+        map(rule! {"(" ~ ")"}, |(_, _)| vec![]), // empty grouping set
+        map(
+            rule! {"(" ~ #comma_separated_list1(expr) ~ ")"},
+            |(_, sets, _)| sets,
+        ),
+        map(rule! { #expr }, |e| vec![e]),
+    ));
+    let group_sets = map(
+        rule! { GROUPING ~ SETS ~ "(" ~ ^#comma_separated_list1(group_set) ~ ")"  },
+        |(_, _, _, sets, _)| GroupBy::GroupingSets(sets),
+    );
+    rule!(#group_sets | #normal)(i)
+}
+
 pub fn set_operation_element(i: Input) -> IResult<WithSpan<SetOperationElement>> {
     let set_operator = map(
         rule! {
@@ -588,7 +607,7 @@ pub fn set_operation_element(i: Input) -> IResult<WithSpan<SetOperationElement>>
              SELECT ~ DISTINCT? ~ ^#comma_separated_list1(select_target)
                 ~ ( FROM ~ ^#comma_separated_list1(table_reference) )?
                 ~ ( WHERE ~ ^#expr )?
-                ~ ( GROUP ~ ^BY ~ ^#comma_separated_list1(expr) )?
+                ~ ( GROUP ~ ^BY ~ ^#group_by_items )?
                 ~ ( HAVING ~ ^#expr )?
         },
         |(
@@ -609,11 +628,7 @@ pub fn set_operation_element(i: Input) -> IResult<WithSpan<SetOperationElement>>
                         .unwrap_or_default(),
                 ),
                 selection: Box::new(opt_where_block.map(|(_, selection)| selection)),
-                group_by: Box::new(
-                    opt_group_by_block
-                        .map(|(_, _, group_by)| group_by)
-                        .unwrap_or_default(),
-                ),
+                group_by: opt_group_by_block.map(|(_, _, group_by)| group_by),
                 having: Box::new(opt_having_block.map(|(_, having)| having)),
             }
         },
@@ -667,7 +682,7 @@ impl<'a, I: Iterator<Item = WithSpan<'a, SetOperationElement>>> PrattParser<I>
                 select_list: *select_list,
                 from: *from,
                 selection: *selection,
-                group_by: *group_by,
+                group_by,
                 having: *having,
             })),
             _ => unreachable!(),