Support optional semicolon between statements (apache#1937)

Author: yoavcloud

src/parser/mod.rs

@@ -222,13 +222,17 @@ pub struct ParserOptions {
     /// Controls how literal values are unescaped. See
     /// [`Tokenizer::with_unescape`] for more details.
     pub unescape: bool,
+    /// Controls if the parser expects a semi-colon token
+    /// between statements. Default is `true`.
+    pub require_semicolon_stmt_delimiter: bool,
 }
 
 impl Default for ParserOptions {
     fn default() -> Self {
         Self {
             trailing_commas: false,
             unescape: true,
+            require_semicolon_stmt_delimiter: true,
         }
     }
 }
@@ -467,6 +471,10 @@ impl<'a> Parser<'a> {
                 expecting_statement_delimiter = false;
             }
 
+            if !self.options.require_semicolon_stmt_delimiter {
+                expecting_statement_delimiter = false;
+            }
+
             match self.peek_token().token {
                 Token::EOF => break,
 

src/test_utils.rs

@@ -294,6 +294,11 @@ pub fn all_dialects() -> TestedDialects {
     ])
 }
 
+// Returns all available dialects with the specified parser options
+pub fn all_dialects_with_options(options: ParserOptions) -> TestedDialects {
+    TestedDialects::new_with_options(all_dialects().dialects, options)
+}
+
 /// Returns all dialects matching the given predicate.
 pub fn all_dialects_where<F>(predicate: F) -> TestedDialects
 where

tests/sqlparser_common.rs

@@ -40,8 +40,9 @@ use sqlparser::parser::{Parser, ParserError, ParserOptions};
 use sqlparser::tokenizer::Tokenizer;
 use sqlparser::tokenizer::{Location, Span};
 use test_utils::{
-    all_dialects, all_dialects_where, alter_table_op, assert_eq_vec, call, expr_from_projection,
-    join, number, only, table, table_alias, table_from_name, TestedDialects,
+    all_dialects, all_dialects_where, all_dialects_with_options, alter_table_op, assert_eq_vec,
+    call, expr_from_projection, join, number, only, table, table_alias, table_from_name,
+    TestedDialects,
 };
 
 #[macro_use]
@@ -16115,3 +16116,20 @@ fn test_select_exclude() {
         ParserError::ParserError("Expected: end of statement, found: EXCLUDE".to_string())
     );
 }
+
+#[test]
+fn test_no_semicolon_required_between_statements() {
+    let sql = r#"
+SELECT * FROM tbl1
+SELECT * FROM tbl2
+    "#;
+
+    let dialects = all_dialects_with_options(ParserOptions {
+        trailing_commas: false,
+        unescape: true,
+        require_semicolon_stmt_delimiter: false,
+    });
+    let stmts = dialects.parse_sql_statements(sql).unwrap();
+    assert_eq!(stmts.len(), 2);
+    assert!(stmts.iter().all(|s| matches!(s, Statement::Query { .. })));
+}

tests/sqlparser_mssql.rs

@@ -32,7 +32,7 @@ use sqlparser::ast::DeclareAssignment::MsSqlAssignment;
 use sqlparser::ast::Value::SingleQuotedString;
 use sqlparser::ast::*;
 use sqlparser::dialect::{GenericDialect, MsSqlDialect};
-use sqlparser::parser::{Parser, ParserError};
+use sqlparser::parser::{Parser, ParserError, ParserOptions};
 
 #[test]
 fn parse_mssql_identifiers() {
@@ -2327,6 +2327,18 @@ fn ms() -> TestedDialects {
     TestedDialects::new(vec![Box::new(MsSqlDialect {})])
 }
 
+// MS SQL dialect with support for optional semi-colon statement delimiters
+fn tsql() -> TestedDialects {
+    TestedDialects::new_with_options(
+        vec![Box::new(MsSqlDialect {})],
+        ParserOptions {
+            trailing_commas: false,
+            unescape: true,
+            require_semicolon_stmt_delimiter: false,
+        },
+    )
+}
+
 fn ms_and_generic() -> TestedDialects {
     TestedDialects::new(vec![Box::new(MsSqlDialect {}), Box::new(GenericDialect {})])
 }
@@ -2483,3 +2495,15 @@ fn parse_mssql_grant() {
 fn parse_mssql_deny() {
     ms().verified_stmt("DENY SELECT ON my_table TO public, db_admin");
 }
+
+#[test]
+fn test_tsql_no_semicolon_delimiter() {
+    let sql = r#"
+DECLARE @X AS NVARCHAR(MAX)='x'
+DECLARE @Y AS NVARCHAR(MAX)='y'
+    "#;
+
+    let stmts = tsql().parse_sql_statements(sql).unwrap();
+    assert_eq!(stmts.len(), 2);
+    assert!(stmts.iter().all(|s| matches!(s, Statement::Declare { .. })));
+}

tests/sqlparser_mysql.rs

@@ -1442,6 +1442,7 @@ fn parse_escaped_quote_identifiers_with_no_escape() {
             ParserOptions {
                 trailing_commas: false,
                 unescape: false,
+                require_semicolon_stmt_delimiter: true,
             }
         )
         .verified_stmt(sql),