Parse Oniguruma recursion levels

Parse an additional `+n` or `-n` suffix on
references in conditionals and the `\k`
backreference, which is valid Oniguruma that allows
a recursion level to be specified for the reference.

Author: hamishknight

Sources/_MatchingEngine/Regex/AST/AST.swift

@@ -207,12 +207,19 @@ extension AST {
     }
     public var kind: Kind
 
+    /// An additional specifier supported by Oniguruma that specifies what
+    /// recursion level the group being referenced belongs to.
+    public var recursionLevel: Located<Int>?
+
     /// The location of the inner numeric or textual reference, e.g the location
-    /// of '-2' in '\g{-2}'.
+    /// of '-2' in '\g{-2}'. Note this includes the recursion level for e.g
+    /// '\k<a+2>'.
     public var innerLoc: SourceLocation
 
-    public init(_ kind: Kind, innerLoc: SourceLocation) {
+    public init(_ kind: Kind, recursionLevel: Located<Int>? = nil,
+                innerLoc: SourceLocation) {
       self.kind = kind
+      self.recursionLevel = recursionLevel
       self.innerLoc = innerLoc
     }
 

Sources/_MatchingEngine/Regex/AST/Atom.swift

@@ -76,6 +76,32 @@ extension AST {
   }
 }
 
+extension AST.Atom {
+  private var _associatedValue: Any? {
+    switch kind {
+    case .char(let v):                  return v
+    case .scalar(let v):                return v
+    case .property(let v):              return v
+    case .escaped(let v):               return v
+    case .keyboardControl(let v):       return v
+    case .keyboardMeta(let v):          return v
+    case .keyboardMetaControl(let v):   return v
+    case .namedCharacter(let v):        return v
+    case .backreference(let v):         return v
+    case .subpattern(let v):            return v
+    case .callout(let v):               return v
+    case .backtrackingDirective(let v): return v
+    case .any:                          return nil
+    case .startOfLine:                  return nil
+    case .endOfLine:                    return nil
+    }
+  }
+
+  func `as`<T>(_ t: T.Type = T.self) -> T? {
+    _associatedValue as? T
+  }
+}
+
 extension AST.Atom {
 
   // TODO: We might scrap this and break out a few categories so

Sources/_MatchingEngine/Regex/Parse/LexicalAnalysis.swift

@@ -865,8 +865,6 @@ extension Source {
           return .recursionCheck
         }
 
-        // TODO: Oniguruma can also parse an additional recursion level for
-        // group-matched checks.
         if let open = src.tryEat(anyOf: "<", "'") {
           // In PCRE, this can only be a named reference. In Oniguruma, it can
           // also be a numbered reference.
@@ -884,9 +882,9 @@ extension Source {
         }
 
         // If we have a numbered reference, this is a check to see if a group
-        // matched.
-        if let numRef = try src.lexNumberedReference() {
-          return .groupMatched(numRef)
+        // matched. Oniguruma also permits a recursion level here.
+        if let num = try src.lexNumberedReference(allowRecursionLevel: true) {
+          return .groupMatched(num)
         }
 
         // PCRE and .NET also allow a named reference to be parsed here. PCRE
@@ -896,9 +894,9 @@ extension Source {
         // FIXME: This should apply to future groups too.
         // TODO: We should probably advise users to use the more explicit
         // syntax.
-        if let nameRef = src.lexNamedReference(endingWith: ")",
-                                               eatEnding: false),
-           context.isPriorGroupRef(nameRef.kind) {
+        let nameRef = src.lexNamedReference(
+          endingWith: ")", eatEnding: false, allowRecursionLevel: true)
+        if let nameRef = nameRef, context.isPriorGroupRef(nameRef.kind) {
           return .groupMatched(nameRef)
         }
         return nil
@@ -1052,10 +1050,10 @@ extension Source {
 
   /// Try to lex an absolute or relative numbered reference.
   ///
-  ///     NumberRef -> ('+' | '-')? <Decimal Number>
+  ///     NumberRef -> ('+' | '-')? <Decimal Number> RecursionLevel?
   ///
   private mutating func lexNumberedReference(
-    allowWholePatternRef: Bool = false
+    allowWholePatternRef: Bool = false, allowRecursionLevel: Bool = false
   ) throws -> AST.Reference? {
     let kind = try recordLoc { src -> AST.Reference.Kind? in
       // Note this logic should match canLexNumberedReference.
@@ -1074,7 +1072,22 @@ extension Source {
     guard allowWholePatternRef || kind.value != .recurseWholePattern else {
       throw ParseError.cannotReferToWholePattern
     }
-    return .init(kind.value, innerLoc: kind.location)
+    let recLevel = allowRecursionLevel ? try lexRecursionLevel() : nil
+    let loc = recLevel?.location.union(with: kind.location) ?? kind.location
+    return .init(kind.value, recursionLevel: recLevel, innerLoc: loc)
+  }
+
+  /// Try to consume a recursion level for a group reference.
+  ///
+  ///     RecursionLevel -> '+' <Int> | '-' <Int>
+  ///
+  private mutating func lexRecursionLevel(
+  ) throws -> Located<Int>? {
+    try recordLoc { src in
+      if src.tryEat("+") { return try src.expectNumber().value }
+      if src.tryEat("-") { return try -src.expectNumber().value }
+      return nil
+    }
   }
 
   /// Checks whether a numbered reference can be lexed.
@@ -1087,19 +1100,34 @@ extension Source {
 
   /// Eat a named reference up to a given closing delimiter.
   private mutating func expectNamedReference(
-    endingWith end: String, eatEnding: Bool = true
+    endingWith end: String, eatEnding: Bool = true,
+    allowRecursionLevel: Bool = false
   ) throws -> AST.Reference {
-    let str = try expectGroupName(endingWith: end, eatEnding: eatEnding)
-    return .init(.named(str.value), innerLoc: str.location)
+    // Note we don't want to eat the ending as we may also want to parse a
+    // recursion level.
+    let str = try expectGroupName(endingWith: end, eatEnding: false)
+
+    // If we're allowed to, try parse a recursion level.
+    let recLevel = allowRecursionLevel ? try lexRecursionLevel() : nil
+    let loc = recLevel?.location.union(with: str.location) ?? str.location
+
+    if eatEnding {
+      try expect(sequence: end)
+    }
+    return .init(.named(str.value), recursionLevel: recLevel, innerLoc: loc)
   }
 
   /// Try to consume a named reference up to a closing delimiter, returning
   /// `nil` if the characters aren't valid for a named reference.
   private mutating func lexNamedReference(
-    endingWith end: String, eatEnding: Bool = true
+    endingWith end: String, eatEnding: Bool = true,
+    allowRecursionLevel: Bool = false
   ) -> AST.Reference? {
     tryEating { src in
-      try? src.expectNamedReference(endingWith: end, eatEnding: eatEnding)
+      try? src.expectNamedReference(
+        endingWith: end, eatEnding: eatEnding,
+        allowRecursionLevel: allowRecursionLevel
+      )
     }
   }
 
@@ -1109,17 +1137,22 @@ extension Source {
   ///
   private mutating func expectNamedOrNumberedReference(
     endingWith ending: String, eatEnding: Bool = true,
-    allowWholePatternRef: Bool = false
+    allowWholePatternRef: Bool = false, allowRecursionLevel: Bool = false
   ) throws -> AST.Reference {
-    if let numbered = try lexNumberedReference(
-      allowWholePatternRef: allowWholePatternRef
-    ) {
+    let num = try lexNumberedReference(
+      allowWholePatternRef: allowWholePatternRef,
+      allowRecursionLevel: allowRecursionLevel
+    )
+    if let num = num {
       if eatEnding {
         try expect(sequence: ending)
       }
-      return numbered
+      return num
     }
-    return try expectNamedReference(endingWith: ending, eatEnding: eatEnding)
+    return try expectNamedReference(
+      endingWith: ending, eatEnding: eatEnding,
+      allowRecursionLevel: allowRecursionLevel
+    )
   }
 
   private static func getClosingDelimiter(
@@ -1183,10 +1216,9 @@ extension Source {
             // Perl only accept named references here, but Oniguruma and .NET
             // also accepts numbered references. This shouldn't be an ambiguity
             // as named references may not begin with a digit, '-', or '+'.
-            let ref = try src.expectNamedOrNumberedReference(
-              endingWith: closing)
-
-            return .backreference(ref)
+            // Oniguruma also allows a recursion level to be specified.
+            return .backreference(try src.expectNamedOrNumberedReference(
+              endingWith: closing, allowRecursionLevel: true))
           }
           // Perl/.NET also allow a named references with the '{' delimiter.
           if src.tryEat("{") {
@@ -1210,10 +1242,10 @@ extension Source {
         // here.
         if firstChar != "0", let numAndLoc = try src.lexNumber() {
           let num = numAndLoc.value
-          let loc = numAndLoc.location
+          let ref = AST.Reference(.absolute(num), innerLoc: numAndLoc.location)
           if num < 10 || firstChar == "8" || firstChar == "9" ||
-              context.isPriorGroupRef(.absolute(num)) {
-            return .backreference(.init(.absolute(num), innerLoc: loc))
+              context.isPriorGroupRef(ref.kind) {
+            return .backreference(ref)
           }
           return nil
         }

Sources/_MatchingEngine/Regex/Parse/SourceLocation.swift

@@ -36,6 +36,12 @@ extension Source {
     }
     public var isFake: Bool { self == Self.fake }
     public var isReal: Bool { !isFake }
+
+    /// Returns the smallest location that contains both this location and
+    /// another.
+    public func union(with other: Location) -> SourceLocation {
+      .init(min(start, other.start) ..< max(end, other.end))
+    }
   }
 }
 public typealias SourceLocation = Source.Location

Sources/_MatchingEngine/Regex/Printing/DumpAST.swift

@@ -149,7 +149,11 @@ extension AST.Atom.Callout: _ASTPrintable {
 
 extension AST.Reference: _ASTPrintable {
   public var _dumpBase: String {
-    "\(kind)"
+    var result = "\(kind)"
+    if let recursionLevel = recursionLevel {
+      result += "\(recursionLevel.value)"
+    }
+    return result
   }
 }
 

Sources/_StringProcessing/ASTBuilder.swift

@@ -139,17 +139,21 @@ func unsetMatchingOptions(
   unsetMatchingOptions(adding: adding)
 }
 
-func ref(_ i: Int) -> AST.Reference {
-  .init(.absolute(i), innerLoc: .fake)
+func ref(_ i: Int, recursionLevel: Int? = nil) -> AST.Reference {
+  .init(.absolute(i), recursionLevel: recursionLevel.map { .init(faking: $0) },
+        innerLoc: .fake)
 }
-func ref(plus n: Int) -> AST.Reference {
-  .init(.relative(n), innerLoc: .fake)
+func ref(plus n: Int, recursionLevel: Int? = nil) -> AST.Reference {
+  .init(.relative(n), recursionLevel: recursionLevel.map { .init(faking: $0) },
+        innerLoc: .fake)
 }
-func ref(minus n: Int) -> AST.Reference {
-  .init(.relative(-n), innerLoc: .fake)
+func ref(minus n: Int, recursionLevel: Int? = nil) -> AST.Reference {
+  .init(.relative(-n), recursionLevel: recursionLevel.map { .init(faking: $0) },
+        innerLoc: .fake)
 }
-func ref(_ s: String) -> AST.Reference {
-  .init(.named(s), innerLoc: .fake)
+func ref(_ s: String, recursionLevel: Int? = nil) -> AST.Reference {
+  .init(.named(s), recursionLevel: recursionLevel.map { .init(faking: $0) },
+        innerLoc: .fake)
 }
 func conditional(
   _ cond: AST.Conditional.Condition.Kind, trueBranch: AST, falseBranch: AST
@@ -286,8 +290,10 @@ func scalar_m(_ s: Unicode.Scalar) -> AST.CustomCharacterClass.Member {
   atom_m(.scalar(s))
 }
 
-func backreference(_ r: AST.Reference.Kind) -> AST {
-  atom(.backreference(.init(r, innerLoc: .fake)))
+func backreference(_ r: AST.Reference.Kind, recursionLevel: Int? = nil) -> AST {
+  atom(.backreference(.init(
+    r, recursionLevel: recursionLevel.map { .init(faking: $0) }, innerLoc: .fake
+  )))
 }
 func subpattern(_ r: AST.Reference.Kind) -> AST {
   atom(.subpattern(.init(r, innerLoc: .fake)))