Correct \x lexing

As per PCRE, we eat up to 2 digits, with a fun
behavior where no digits is interpreted as 0.

Author: hamishknight

Sources/_MatchingEngine/Regex/Parse/LexicalAnalysis.swift

@@ -231,7 +231,7 @@ extension Source {
   ///     UniScalar -> 'u{' HexDigit{1...} '}'
   ///                | 'u'  HexDigit{4}
   ///                | 'x{' HexDigit{1...} '}'
-  ///                | 'x'  HexDigit{2}
+  ///                | 'x'  HexDigit{0...2}
   ///                | 'U'  HexDigit{8}
   ///                | 'o{' OctalDigit{1...} '}'
   ///                | OctalDigit{1...3}
@@ -240,15 +240,26 @@ extension Source {
     escapedCharacter base: Character
   ) throws -> Located<Unicode.Scalar> {
     try recordLoc { src in
+      // TODO: PCRE offers a different behavior if PCRE2_ALT_BSUX is set.
       switch base {
       // Hex numbers.
-      case "u", "x":
-        if src.tryEat("{") {
-          let str = try src.lexUntil(eating: "}").value
-          return try Source.validateUnicodeScalar(str, .hex)
+      case "u" where src.tryEat("{"), "x" where src.tryEat("{"):
+        let str = try src.lexUntil(eating: "}").value
+        return try Source.validateUnicodeScalar(str, .hex)
+
+      case "x":
+        // \x expects *up to* 2 digits.
+        guard let digits = src.tryEatPrefix(maxLength: 2, \.isHexDigit) else {
+          // In PCRE, \x without any valid hex digits is \u{0}.
+          // TODO: This doesn't appear to be followed by ICU or Oniguruma, so
+          // could be changed to throw an error if we had a parsing mode for
+          // them.
+          return Unicode.Scalar(0)
         }
-        let numDigits = base == "u" ? 4 : 2
-        return try src.expectUnicodeScalar(numDigits: numDigits).value
+        return try Source.validateUnicodeScalar(digits.string, .hex)
+
+      case "u":
+        return try src.expectUnicodeScalar(numDigits: 4).value
       case "U":
         return try src.expectUnicodeScalar(numDigits: 8).value
 

Tests/RegexTests/LexTests.swift

@@ -83,9 +83,6 @@ extension RegexTests {
       _ = scalars
     }
 
-// FIXME:
-//    diagnoseUniScalar(
-//      "12ab", base: "x", expectedDigits: 2)
     diagnoseUniScalar(
       "12", base: "u", expectedDigits: 4)
     diagnoseUniScalar(

Tests/RegexTests/ParseTests.swift

@@ -270,6 +270,13 @@ extension RegexTests {
     parseTest(#"[\7777]"#, charClass(scalar_m("\u{1FF}"), "7"))
     parseTest(#"[\181]"#, charClass(scalar_m("\u{1}"), "8", "1"))
 
+    // We take *up to* the first two valid digits for \x. No valid digits is 0.
+    parseTest(#"\x"#, scalar("\u{0}"))
+    parseTest(#"\x5"#, scalar("\u{5}"))
+    parseTest(#"\xX"#, concat(scalar("\u{0}"), "X"))
+    parseTest(#"\x5X"#, concat(scalar("\u{5}"), "X"))
+    parseTest(#"\x12ab"#, concat(scalar("\u{12}"), "a", "b"))
+
     // MARK: Character classes
 
     parseTest(#"abc\d"#, concat("a", "b", "c", escaped(.decimalDigit)))