syntax: support LookArounds in AST and HIR

Author: shilangyu

regex-syntax/src/ast/mod.rs

@@ -477,6 +477,9 @@ pub enum Ast {
     Dot(Box<Span>),
     /// A single zero-width assertion.
     Assertion(Box<Assertion>),
+    #[cfg(feature = "look-behinds")]
+    /// A single look-around regular expression.
+    LookAround(Box<LookAround>),
     /// A single Unicode character class, e.g., `\pL` or `\p{Greek}`.
     ClassUnicode(Box<ClassUnicode>),
     /// A single perl character class, e.g., `\d` or `\W`.
@@ -521,6 +524,12 @@ impl Ast {
         Ast::Assertion(Box::new(e))
     }
 
+    /// Create a "look-around" AST item.
+    #[cfg(feature = "look-behinds")]
+    pub fn lookaround(e: LookAround) -> Ast {
+        Ast::LookAround(Box::new(e))
+    }
+
     /// Create a "Unicode class" AST item.
     pub fn class_unicode(e: ClassUnicode) -> Ast {
         Ast::ClassUnicode(Box::new(e))
@@ -564,6 +573,8 @@ impl Ast {
             Ast::Literal(ref x) => &x.span,
             Ast::Dot(ref span) => span,
             Ast::Assertion(ref x) => &x.span,
+            #[cfg(feature = "look-behinds")]
+            Ast::LookAround(ref x) => &x.span,
             Ast::ClassUnicode(ref x) => &x.span,
             Ast::ClassPerl(ref x) => &x.span,
             Ast::ClassBracketed(ref x) => &x.span,
@@ -598,6 +609,8 @@ impl Ast {
             | Ast::Group(_)
             | Ast::Alternation(_)
             | Ast::Concat(_) => true,
+            #[cfg(feature = "look-behinds")]
+            Ast::LookAround(_) => true,
         }
     }
 }
@@ -1342,6 +1355,28 @@ pub enum AssertionKind {
     WordBoundaryEndHalf,
 }
 
+/// A single zero-width look-around.
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
+pub struct LookAround {
+    /// The span of this look-around.
+    pub span: Span,
+    /// The look-around kind, e.g. negative/positive look-behind.
+    pub kind: LookAroundKind,
+    /// The regular expression inside the look-around.
+    pub ast: Box<Ast>,
+}
+
+/// A look-around kind.
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
+pub enum LookAroundKind {
+    /// `(?<=...)`
+    PositiveLookBehind,
+    /// `(?<!...)`
+    NegativeLookBehind,
+}
+
 /// A repetition operation applied to a regular expression.
 #[derive(Clone, Debug, Eq, PartialEq)]
 #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
@@ -1647,6 +1682,8 @@ impl Drop for Ast {
             | Ast::ClassBracketed(_) => return,
             Ast::Repetition(ref x) if !x.ast.has_subexprs() => return,
             Ast::Group(ref x) if !x.ast.has_subexprs() => return,
+            #[cfg(feature = "look-behinds")]
+            Ast::LookAround(ref x) if !x.ast.has_subexprs() => return,
             Ast::Alternation(ref x) if x.asts.is_empty() => return,
             Ast::Concat(ref x) if x.asts.is_empty() => return,
             _ => {}
@@ -1673,6 +1710,10 @@ impl Drop for Ast {
                 Ast::Group(ref mut x) => {
                     stack.push(mem::replace(&mut x.ast, empty_ast()));
                 }
+                #[cfg(feature = "look-behinds")]
+                Ast::LookAround(ref mut x) => {
+                    stack.push(mem::replace(&mut x.ast, empty_ast()));
+                }
                 Ast::Alternation(ref mut x) => {
                     stack.extend(x.asts.drain(..));
                 }

regex-syntax/src/ast/parse.rs

@@ -2328,6 +2328,8 @@ impl<'p, 's, P: Borrow<Parser>> ast::Visitor for NestLimiter<'p, 's, P> {
             Ast::ClassBracketed(ref x) => &x.span,
             Ast::Repetition(ref x) => &x.span,
             Ast::Group(ref x) => &x.span,
+            #[cfg(feature = "look-behinds")]
+            Ast::LookAround(ref x) => &x.span,
             Ast::Alternation(ref x) => &x.span,
             Ast::Concat(ref x) => &x.span,
         };
@@ -2354,6 +2356,11 @@ impl<'p, 's, P: Borrow<Parser>> ast::Visitor for NestLimiter<'p, 's, P> {
                 self.decrement_depth();
                 Ok(())
             }
+            #[cfg(feature = "look-behinds")]
+            Ast::LookAround(_) => {
+                self.decrement_depth();
+                Ok(())
+            }
         }
     }
 

regex-syntax/src/ast/print.rs

@@ -80,6 +80,8 @@ impl<W: fmt::Write> Visitor for Writer<W> {
     fn visit_pre(&mut self, ast: &Ast) -> fmt::Result {
         match *ast {
             Ast::Group(ref x) => self.fmt_group_pre(x),
+            #[cfg(feature = "look-behinds")]
+            Ast::LookAround(ref x) => self.fmt_lookaround_pre(x),
             Ast::ClassBracketed(ref x) => self.fmt_class_bracketed_pre(x),
             _ => Ok(()),
         }
@@ -92,6 +94,8 @@ impl<W: fmt::Write> Visitor for Writer<W> {
             Ast::Literal(ref x) => self.fmt_literal(x),
             Ast::Dot(_) => self.wtr.write_str("."),
             Ast::Assertion(ref x) => self.fmt_assertion(x),
+            #[cfg(feature = "look-behinds")]
+            Ast::LookAround(ref x) => self.fmt_lookaround_post(x),
             Ast::ClassPerl(ref x) => self.fmt_class_perl(x),
             Ast::ClassUnicode(ref x) => self.fmt_class_unicode(x),
             Ast::ClassBracketed(ref x) => self.fmt_class_bracketed_post(x),
@@ -174,6 +178,20 @@ impl<W: fmt::Write> Writer<W> {
         self.wtr.write_str(")")
     }
 
+    #[cfg(feature = "look-behinds")]
+    fn fmt_lookaround_pre(&mut self, ast: &ast::LookAround) -> fmt::Result {
+        use crate::ast::LookAroundKind::*;
+        match ast.kind {
+            PositiveLookBehind => self.wtr.write_str("(?<="),
+            NegativeLookBehind => self.wtr.write_str("(?<!"),
+        }
+    }
+
+    #[cfg(feature = "look-behinds")]
+    fn fmt_lookaround_post(&mut self, _ast: &ast::LookAround) -> fmt::Result {
+        self.wtr.write_str(")")
+    }
+
     fn fmt_repetition(&mut self, ast: &ast::Repetition) -> fmt::Result {
         use crate::ast::RepetitionKind::*;
         match ast.op.kind {
@@ -511,6 +529,13 @@ mod tests {
         roundtrip("(a)");
     }
 
+    #[cfg(feature = "look-behinds")]
+    #[test]
+    fn print_lookaround() {
+        roundtrip("(?<=a)");
+        roundtrip("(?<!a)");
+    }
+
     #[test]
     fn print_class() {
         roundtrip(r"[abc]");

regex-syntax/src/ast/visitor.rs

@@ -140,6 +140,10 @@ enum Frame<'a> {
     /// A stack frame allocated just before descending into a group's child
     /// node.
     Group(&'a ast::Group),
+    /// A stack frame allocated just before descending into a look-around's
+    /// child node.
+    #[cfg(feature = "look-behinds")]
+    LookAround(&'a ast::LookAround),
     /// The stack frame used while visiting every child node of a concatenation
     /// of expressions.
     Concat {
@@ -270,6 +274,8 @@ impl<'a> HeapVisitor<'a> {
             }
             Ast::Repetition(ref x) => Some(Frame::Repetition(x)),
             Ast::Group(ref x) => Some(Frame::Group(x)),
+            #[cfg(feature = "look-behinds")]
+            Ast::LookAround(ref x) => Some(Frame::LookAround(x)),
             Ast::Concat(ref x) if x.asts.is_empty() => None,
             Ast::Concat(ref x) => {
                 Some(Frame::Concat { head: &x.asts[0], tail: &x.asts[1..] })
@@ -289,6 +295,8 @@ impl<'a> HeapVisitor<'a> {
         match induct {
             Frame::Repetition(_) => None,
             Frame::Group(_) => None,
+            #[cfg(feature = "look-behinds")]
+            Frame::LookAround(_) => None,
             Frame::Concat { tail, .. } => {
                 if tail.is_empty() {
                     None
@@ -444,6 +452,8 @@ impl<'a> Frame<'a> {
         match *self {
             Frame::Repetition(rep) => &rep.ast,
             Frame::Group(group) => &group.ast,
+            #[cfg(feature = "look-behinds")]
+            Frame::LookAround(look) => &look.ast,
             Frame::Concat { head, .. } => head,
             Frame::Alternation { head, .. } => head,
         }

regex-syntax/src/hir/literal.rs

@@ -173,6 +173,8 @@ impl Extractor {
 
         match *hir.kind() {
             Empty | Look(_) => Seq::singleton(self::Literal::exact(vec![])),
+            #[cfg(feature = "look-behinds")]
+            LookAround(_) => Seq::singleton(self::Literal::exact(vec![])),
             Literal(hir::Literal(ref bytes)) => {
                 let mut seq =
                     Seq::singleton(self::Literal::exact(bytes.to_vec()));
@@ -2453,6 +2455,22 @@ mod tests {
         assert_eq!(expected, e(r"^aZ*b"));
     }
 
+    #[test]
+    #[cfg(feature = "look-behinds")]
+    fn lookaround() {
+        assert_eq!(exact([E("ab")]), e(r"a(?<=qwa)b"));
+        assert_eq!(exact([E("ab")]), e(r"a(?<!qw1)b"));
+
+        assert_eq!(exact([E("ab")]), e(r"(?<=qwe)ab"));
+        assert_eq!(exact([E("ab")]), e(r"(?<!qwe)ab"));
+
+        assert_eq!(exact([E("ab")]), e(r"ab(?<=qab)"));
+        assert_eq!(exact([E("ab")]), e(r"ab(?<!qwe)"));
+
+        let expected = (seq([I("aZ"), E("ab")]), seq([I("Zb"), E("ab")]));
+        assert_eq!(expected, e(r"(?<=foo)aZ*b"));
+    }
+
     #[test]
     fn repetition() {
         assert_eq!(exact(["a", ""]), e(r"a?"));