Parse concrete syntax with a proper grammar (#741)

Concrete syntax has a hacky implementation based on string matching. If
we want to expand the grammar to support constraints and other
constructs, it's worthwhile to build a proper grammar. This will allow
us to expand it in the future to use things like:

- `where len(:[name]) > 2` for example to select nodes only with two or
more children;
- `where :[name] matches regex` with regex to add constraints on
template variables.

In this PR I essentially wrote the paper, but didn't touch the matching
algorithm. In summary:

- Added Pest grammar (concrete_syntax.pest)
- Added parser that converts strings to build an actual AST
- Moved the matching algorithm to
[interpreter.rs](https://github.com/danieltrt/piranha/blob/a993eb15fd9f2317f928c6a94c96577ffb73336d/src/models/concrete_syntax/interpreter.rs#L44)

Author: danieltrt

Cargo.toml

@@ -70,6 +70,8 @@ pyo3 = "0.20.0"
 pyo3-log = "0.9.0"
 glob = "0.3.1"
 lazy_static = "1.4.0"
+pest = "2.8.1"
+pest_derive = "2.8.1"
 
 [features]
 extension-module = ["pyo3/extension-module"]

mod.rs

@@ -0,0 +1,18 @@
+/*
+ Copyright (c) 2023 Uber Technologies, Inc.
+
+ <p>Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ except in compliance with the License. You may obtain a copy of the License at
+ <p>http://www.apache.org/licenses/LICENSE-2.0
+
+ <p>Unless required by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ express or implied. See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+//! Concrete Syntax Module
+//!
+//! This module provides concrete syntax pattern matching capabilities for Piranha.
+//! It includes a parser for concrete syntax patterns and an interpreter that can
+//! match these patterns against tree-sitter ASTs.

src/df/utils.rs

@@ -34,6 +34,7 @@ pub fn get_capture_groups_from_matcher(node: &Rule) -> Vec<String> {
   match &node.query().pattern_type() {
     PatternType::Tsq => get_capture_groups_from_tsq(node.query().pattern()),
     PatternType::Regex => get_capture_groups_from_regex(node.query().extract_regex().unwrap()),
+    PatternType::Cs => vec![],
     PatternType::Unknown => vec![],
   }
 }
@@ -47,6 +48,7 @@ pub fn get_capture_group_usage_from_matcher(node: &Rule) -> Vec<String> {
   match &node.query().pattern_type() {
     PatternType::Tsq => get_capture_group_usage_from_tsq(node.query().pattern()),
     PatternType::Regex => get_capture_group_usage_from_regex(node.query().pattern()),
+    PatternType::Cs => vec![],
     PatternType::Unknown => vec![],
   }
 }

src/lib.rs

@@ -21,7 +21,6 @@ use models::{
 extern crate lazy_static;
 pub mod df;
 pub mod models;
-
 #[cfg(test)]
 mod tests;
 pub mod utilities;

src/models/capture_group_patterns.rs

@@ -11,7 +11,8 @@ Copyright (c) 2023 Uber Technologies, Inc.
  limitations under the License.
 */
 
-use crate::models::concrete_syntax::get_all_matches_for_concrete_syntax;
+use crate::models::concrete_syntax::interpreter::get_all_matches_for_concrete_syntax;
+use crate::models::concrete_syntax::parser::ConcreteSyntax;
 use crate::{
   models::Validator,
   utilities::{
@@ -26,8 +27,6 @@ use serde_derive::Deserialize;
 use std::collections::HashMap;
 use tree_sitter::{Node, Query};
 
-#[derive(Debug)]
-pub struct ConcreteSyntax(pub String);
 use super::{
   default_configs::{CONCRETE_SYNTAX_QUERY_PREFIX, REGEX_QUERY_PREFIX},
   matches::Match,
@@ -36,6 +35,7 @@ use super::{
 pub enum PatternType {
   Tsq,
   Regex,
+  Cs,
   Unknown,
 }
 
@@ -59,13 +59,14 @@ impl CGPattern {
 
   pub(crate) fn extract_concrete_syntax(&self) -> ConcreteSyntax {
     let mut _val = &self.pattern()[CONCRETE_SYNTAX_QUERY_PREFIX.len()..];
-    ConcreteSyntax(_val.to_string())
+    ConcreteSyntax::parse(_val).unwrap()
   }
 
   pub(crate) fn pattern_type(&self) -> PatternType {
     match self.0.as_str() {
       pattern if pattern.starts_with("rgx") => PatternType::Regex,
       pattern if pattern.trim().starts_with('(') => PatternType::Tsq,
+      pattern if pattern.trim().starts_with("cs") => PatternType::Cs,
       _ => PatternType::Unknown,
     }
   }

src/models/concrete_syntax/concrete_syntax.pest

@@ -0,0 +1,15 @@
+// Concrete Syntax Grammar for Piranha
+concrete_syntax = { SOI ~ pattern ~ EOI }
+pattern = { (element)+ }
+
+// An element is either a capture or literal text
+element = _{ capture | literal_text | whitespace }
+
+// Captures: :[name], :[name+], :[name*], @name
+capture = { (":[" ~ identifier ~ capture_mode? ~ "]") | "@"~identifier } // FIXME: Should remove @ from the grammar, because literals may be parsed incorrectly
+capture_mode = { "+" | "*" }
+identifier = { (ASCII_ALPHA | "_") ~ (ASCII_ALPHANUMERIC | "_")* }
+
+// Literal text - single word/token without whitespace
+literal_text = { (!( ":[" | whitespace ) ~ ANY)+ }
+whitespace = _{ (" " | "\t" | "\r" | "\n")+ }