Add remaining path expressions to parser (#124)

* Add remaining path navigation to parser

This PR includes changes for enabling Parser to parse
Path Navigations as specified in Section 4 of PartiQL
Specification. It also ensures the following tests go through:
https://github.com/partiql/partiql-tests/blob/c76715ddb95ba6eec2d34ca500077b6e7eba9e35/partiql-test-data/pass/parser/primitives/path-expression.ion

The following PR in `partiql-tests` is created for adding additional conformance
tests for path expressions:
partiql/partiql-tests#13

In addition:
- the PR introduces `dbl_quoted` property to `SymbolPrimitive`
in order to cater for symbol primitives that are double quoted, e.g.
`AS "date".

- Splits `Identifier` to `QuotedIdentifier` and `UnquotedIdentifer`
  in order for the parser to populate the  AST with the correct
   CaseSensitivity.

Author: am357

partiql-ast/src/ast.rs

@@ -19,11 +19,14 @@ pub trait ToAstNode: Sized {
     /// further [AstNode] construction.
     /// ## Example:
     /// ```
+    /// use partiql_ast::ast;
     /// use partiql_ast::ast::{SymbolPrimitive, ToAstNode};
+    /// use partiql_ast::ast::CaseSensitivity::CaseInsensitive;
     /// use partiql_source_map::location::{ByteOffset, BytePosition, Location, ToLocated};
     ///
     /// let p = SymbolPrimitive {
-    ///     value: "symbol2".to_string()
+    ///     value: "symbol2".to_string(),
+    ///     case: Some(ast::CaseSensitivity::CaseInsensitive)
     ///  };
     ///
     /// let node = p
@@ -576,7 +579,6 @@ pub enum PathStep {
 #[derive(Clone, Debug, PartialEq)]
 pub struct PathExpr {
     pub index: Box<Expr>,
-    pub case: CaseSensitivity,
 }
 
 /// Is used to determine if variable lookup should be case-sensitive or not.
@@ -871,6 +873,8 @@ pub struct CustomType {
 #[derive(Clone, Debug, PartialEq)]
 pub struct SymbolPrimitive {
     pub value: String,
+    // Optional because string literal symbols don't have case sensitivity
+    pub case: Option<CaseSensitivity>,
 }
 
 #[derive(Clone, Debug, PartialEq)]

partiql-ast/tests/test_ast.rs

@@ -22,6 +22,7 @@ fn test_ast_init() {
 
     let span_only = ast::SymbolPrimitive {
         value: "symbol1".to_string(),
+        case: Some(CaseSensitivity::CaseInsensitive),
     }
     .to_node()
     .location(Location {

partiql-parser/src/lexer.rs

@@ -462,17 +462,21 @@ pub enum Token<'input> {
 
     // unquoted identifiers
     #[regex("[a-zA-Z_$][a-zA-Z0-9_$]*", |lex| lex.slice())]
+    UnquotedIdent(&'input str),
+
     // quoted identifiers (quoted with double quotes)
     #[regex(r#""([^"\\]|\\t|\\u|\\n|\\")*""#,
             |lex| lex.slice().trim_matches('"'))]
-    Identifier(&'input str),
+    QuotedIdent(&'input str),
 
     // unquoted @identifiers
     #[regex("@[a-zA-Z_$][a-zA-Z0-9_$]*", |lex| &lex.slice()[1..])]
+    UnquotedAtIdentifier(&'input str),
+
     // quoted @identifiers (quoted with double quotes)
     #[regex(r#"@"([^"\\]|\\t|\\u|\\n|\\")*""#,
             |lex| lex.slice()[1..].trim_matches('"'))]
-    AtIdentifier(&'input str),
+    QuotedAtIdentifier(&'input str),
 
     #[regex("[0-9]+", |lex| lex.slice())]
     Int(&'input str),
@@ -645,8 +649,10 @@ impl<'input> fmt::Display for Token<'input> {
             Token::Caret => write!(f, "^"),
             Token::Period => write!(f, "."),
             Token::DblPipe => write!(f, "||"),
-            Token::Identifier(id) => write!(f, "<{}:IDENT>", id),
-            Token::AtIdentifier(id) => write!(f, "<{}:@IDENT>", id),
+            Token::UnquotedIdent(id) => write!(f, "<{}:UNQUOTED_IDENT>", id),
+            Token::QuotedIdent(id) => write!(f, "<{}:QUOTED_IDENT>", id),
+            Token::UnquotedAtIdentifier(id) => write!(f, "<{}:UNQUOTED_ATIDENT>", id),
+            Token::QuotedAtIdentifier(id) => write!(f, "<{}:QUOTED_ATIDENT>", id),
             Token::Int(txt) => write!(f, "<{}:INT>", txt),
             Token::ExpReal(txt) => write!(f, "<{}:REAL>", txt),
             Token::Real(txt) => write!(f, "<{}:REAL>", txt),
@@ -730,7 +736,7 @@ mod tests {
     fn display() -> Result<(), ParseError<'static, BytePosition>> {
         let symbols =
             "( [ { } ] ) << >> ; , < > <= >= != <> = == - + * % / ^ . || : --foo /*block*/";
-        let primitives = "ident @atident";
+        let primitives = r#"unquoted_ident "quoted_ident" @unquoted_atident @"quoted_atident""#;
         let keywords =
             "WiTH Where Value uSiNg Unpivot UNION True Select right Preserve pivoT Outer Order Or \
              On Offset Nulls Null Not Natural Missing Limit Like Left Lateral Last Join \
@@ -753,10 +759,11 @@ mod tests {
             "PIVOT", ">", "OUTER", "<=", "ORDER", ">=", "OR", "!=", "ON", "<>", "OFFSET",
             "=", "NULLS", "==", "NULL", "-", "NOT", "+", "NATURAL", "*", "MISSING", "%",
             "LIMIT", "/", "LIKE", "^", "LEFT", ".", "LATERAL", "||", "LAST", ":", "JOIN",
-            "--", "INTERSECT", "/**/", "IS", "<ident:IDENT>", "INNER", "<atident:@IDENT>", "IN",
-            "HAVING", "GROUP", "FROM", "FULL", "FIRST", "FALSE", "EXCEPT", "ESCAPE", "DESC",
-            "CROSS", "BY", "BETWEEN", "AT", "AS", "AND", "ASC", "ALL", "VALUES", "CASE", "WHEN",
-            "THEN", "ELSE", "END",
+            "--", "INTERSECT", "/**/","IS", "<unquoted_ident:UNQUOTED_IDENT>", "INNER",
+            "<quoted_ident:QUOTED_IDENT>", "IN", "<unquoted_atident:UNQUOTED_ATIDENT>", "HAVING",
+            "<quoted_atident:QUOTED_ATIDENT>", "GROUP", "FROM", "FULL", "FIRST", "FALSE", "EXCEPT",
+            "ESCAPE", "DESC", "CROSS", "BY", "BETWEEN", "AT", "AS", "AND", "ASC", "ALL", "VALUES",
+            "CASE", "WHEN", "THEN", "ELSE", "END",
         ];
         let displayed = toks
             .into_iter()
@@ -840,20 +847,22 @@ mod tests {
 
     #[test]
     fn select() -> Result<(), ParseError<'static, BytePosition>> {
-        let query = "SELECT g\nFROM data\nGROUP BY a";
+        let query = r#"SELECT g
+            FROM "data"
+            GROUP BY a"#;
         let mut offset_tracker = LineOffsetTracker::default();
         let lexer = PartiqlLexer::new(query, &mut offset_tracker);
         let toks: Vec<_> = lexer.collect::<Result<_, _>>()?;
 
         assert_eq!(
             vec![
                 Token::Select,
-                Token::Identifier("g"),
+                Token::UnquotedIdent("g"),
                 Token::From,
-                Token::Identifier("data"),
+                Token::QuotedIdent("data"),
                 Token::Group,
                 Token::By,
-                Token::Identifier("a")
+                Token::UnquotedIdent("a")
             ],
             toks.into_iter().map(|(_s, t, _e)| t).collect::<Vec<_>>()
         );
@@ -873,7 +882,7 @@ mod tests {
         );
         assert_eq!(
             LineAndColumn::from(offset_tracker.at(query, 19.into()).unwrap()),
-            LineAndColumn::new(3, 1).unwrap()
+            LineAndColumn::new(2, 11).unwrap()
         );
 
         let offset_r_a = query.rfind('a').unwrap();
@@ -900,12 +909,12 @@ mod tests {
         assert_eq!(
             vec![
                 Token::Select,
-                Token::Identifier("🐈"),
+                Token::QuotedIdent("🐈"),
                 Token::From,
-                Token::Identifier("❤ℝ"),
+                Token::QuotedIdent("❤ℝ"),
                 Token::Group,
                 Token::By,
-                Token::Identifier("🧸")
+                Token::QuotedIdent("🧸")
             ],
             toks.into_iter().map(|(_s, t, _e)| t).collect::<Vec<_>>()
         );
@@ -981,9 +990,9 @@ mod tests {
             vec![
                 Token::Select,
                 Token::CommentLine("--comment"),
-                Token::AtIdentifier("g"),
+                Token::UnquotedAtIdentifier("g"),
                 Token::From,
-                Token::AtIdentifier("foo"),
+                Token::QuotedAtIdentifier("foo"),
             ],
             toks.into_iter().map(|(_s, t, _e)| t).collect::<Vec<_>>()
         );
@@ -1002,7 +1011,7 @@ mod tests {
             vec![
                 Token::Select,
                 Token::CommentBlock("/*comment*/"),
-                Token::Identifier("g"),
+                Token::UnquotedIdent("g"),
             ],
             toks.into_iter().map(|(_s, t, _e)| t).collect::<Vec<_>>()
         );

partiql-parser/src/parse/mod.rs

@@ -294,6 +294,66 @@ mod tests {
         }
     }
 
+    mod pathexpr {
+        use super::*;
+
+        #[test]
+        fn nested() {
+            parse!(r#"a.b"#);
+            parse!(r#"a.b.c['item']."d"[5].e['s'].f[1+2]"#);
+            parse!(r#"a.b.*"#);
+            parse!(r#"a.b[*]"#);
+            parse!(r#"@a.b[*]"#);
+            parse!(r#"@"a".b[*]"#);
+            parse!(r#"tables.items[*].product.*.nest"#);
+        }
+
+        #[test]
+        fn tuple() {
+            parse!(r#"{'a':1 , 'data': 2}.a"#);
+            parse!(r#"{'a':1 , 'data': 2}.'a'"#);
+            parse!(r#"{'A':1 , 'data': 2}."A""#);
+            parse!(r#"{'A':1 , 'data': 2}['a']"#);
+            parse!(r#"{'attr': 1, 'b':2}[v || w]"#);
+            parse!(r#"{'a':1, 'b':2}.*"#);
+        }
+
+        #[test]
+        fn array() {
+            parse!(r#"[1,2,3][0]"#);
+            parse!(r#"[1,2,3][1 + 1]"#);
+            parse!(r#"[1,2,3][*]"#);
+        }
+
+        #[test]
+        fn query() {
+            parse!(r#"(SELECT a FROM table).a"#);
+            parse!(r#"(SELECT a FROM table).'a'"#);
+            parse!(r#"(SELECT a FROM table)."a""#);
+            parse!(r#"(SELECT a FROM table)['a']"#);
+            parse!(r#"(SELECT a FROM table).*"#);
+            parse!(r#"(SELECT a FROM table)[*]"#);
+        }
+
+        #[test]
+        fn function_call() {
+            parse!(r#"foo(x, y).a"#);
+            parse!(r#"foo(x, y).*"#);
+            parse!(r#"foo(x, y)[*]"#);
+            parse!(r#"foo(x, y)[5]"#);
+            parse!(r#"foo(x, y).a.*"#);
+        }
+
+        #[test]
+        #[should_panic]
+        fn erroneous() {
+            parse!(r#"a.b.['item']"#);
+            parse!(r#"a.b.{'a': 1, 'b': 2}.a"#);
+            parse!(r#"a.b.[1, 2, 3][2]"#);
+            parse!(r#"a.b.[*]"#);
+        }
+    }
+
     mod sfw {
         use super::*;