fix(parser): some special chars unescaped in brackets

Some special chars that should appear literally in a regex don't need
to be escaped when inside square brackets, e.g. "$" or ".". Previously,
the parser would fail when encountering these.

That brings parse errors in the parser benchmark down to 304.

Author: gruhn

benchmark/parser-bench.js

@@ -19,22 +19,27 @@ export function* readDataset() {
 
 let hasError = 0
 let noError = 0
+let totalParseTime = 0
 
 for (const { regex, flags } of readDataset()) {
   try {
-    const time = performance.now()
     // parseRegexString(regex)
-    const regexp = RB(new RegExp(regex, flags))
+    const regexp = new RegExp(regex, flags)
     console.log('====', regexp, '====')
-    for (const word of RB(regexp).enumerate().take(10)) {
-      console.log(JSON.stringify(word))
-    }
-    console.log(`time: ${Math.round(performance.now() - time)}ms`)
+
+    const timeStart = performance.now()
+    const parsed = RB(regexp)
+    const timeEnd = performance.now()
+
+    console.log(`time: ${Math.round(timeEnd - timeStart)}ms`)
+    totalParseTime += timeEnd - timeStart
     noError++
   } catch (e) {
     // console.error(new RegExp(regex, flags))
     hasError++
   }
 }
 
-console.debug(hasError, '/', hasError + noError)
+console.log('error ratio:', hasError, '/', hasError + noError)
+console.log('total parse time:', Math.round(totalParseTime), 'ms')
+

src/code-point-range.ts

@@ -1,5 +1,4 @@
-import { isSingleton } from './char-set'
-import { assert, checkedAllCases } from './utils'
+import { assert } from './utils'
 
 export type CodePointRange = { start: number, end: number }
 
@@ -106,14 +105,34 @@ export function difference(rangeA: CodePointRange, rangeB: CodePointRange): [] |
   return union(before, after)
 }
 
-export function isMetaChar(char: string): boolean {
-  return /^[.^$*+?()[\]{\|\\\/]$/.test(char)
+/**
+ * Returns true iff the given char must always be escaped to occur literally
+ * in a regular expression. Some special chars like `$` don't need to be 
+ * escaped when inside brackets (e.g. `/[$]/`). But `/`, `\` and `]` must 
+ * even be escaped when inside brackets. 
+ */
+export function mustAlwaysBeEscaped(char: string) {
+  return '\\\]\/'.includes(char)
+}
+
+/**
+ * Returns true iff the given char must be escaped to occur literally
+ * in a regular expression, unless within square brackets. That's true
+ * for special chars like `$`. Outside brackets we have to write `\$`. 
+ * Inside brackets `[$]` is allowed.
+ */
+export function mustBeEscapedOrInBrackets(char: string) {
+  return '.^$*+?()[{-|'.includes(char)
+}
+
+export function neverMustBeEscaped(char: string) {
+  return !mustAlwaysBeEscaped(char) && !mustBeEscapedOrInBrackets(char)
 }
 
 function codePointToString(codePoint: number): string {
   const char = String.fromCharCode(codePoint)
 
-  if (isMetaChar(char)) 
+  if (mustAlwaysBeEscaped(char) || mustBeEscapedOrInBrackets(char)) 
     // e.g. \$ \+ \. 
     return '\\' + char
   else if (codePoint > 126)

src/regex-parser.ts

@@ -42,13 +42,10 @@ const wildcard = P.string('.').map(
   () => RE.literal(CharSet.wildcard({ dotAll: false }))
 )
 
-const singleChar = P.satisfy(char => !Range.isMetaChar(char))
+const unescapedChar = P.satisfy(Range.neverMustBeEscaped)
 
-const codePoint = singleChar.map(char => {
-  const result = char.codePointAt(0)!
-  assert(result !== undefined)
-  return result
-})
+const unescapedCharInsideBrackets = P.satisfy(Range.mustBeEscapedOrInBrackets)
+  .map(CharSet.singleton)
 
 export class UnsupportedSyntaxError extends Error {}
 
@@ -77,9 +74,9 @@ const escapeSequence = P.string('\\').andThen(_ => P.anyChar).map(escapedChar =>
 })
 
 const codePointRange: P.Parser<CharSet.CharSet> =
-  codePoint.andThen(start =>
-    P.optional(P.string('-').andThen(_ => codePoint))
-     .map(end => CharSet.fromRange({ start, end: end ?? start }))
+  unescapedChar.andThen(start =>
+    P.optional(P.string('-').andThen(_ => unescapedChar))
+     .map(end => CharSet.charRange(start, end ?? start))
   )
 
 const charSet = P.choice([
@@ -88,7 +85,7 @@ const charSet = P.choice([
     P.string('['),
     P.string(']'),
     P.optional(P.string('^')).andThen(negated =>
-      P.many(P.choice([escapeSequence, codePointRange])).map(
+      P.many(P.choice([escapeSequence, codePointRange, unescapedCharInsideBrackets])).map(
         sets => {
           if (negated === undefined)
             return sets.reduce(CharSet.union, CharSet.empty)
@@ -98,7 +95,7 @@ const charSet = P.choice([
       )
     )
   ),
-  singleChar.map(CharSet.singleton),
+  unescapedChar.map(CharSet.singleton),
 ])
 
 const group = P.between(

test/regex-parser.spec.ts

@@ -29,12 +29,21 @@ describe('parseRegexString', () => {
     [/^\W$/, RE.literal(CharSet.nonWordChars)],
     [/^\n$/, RE.literal(CharSet.singleton('\n'))],
     [/^\.$/, RE.literal(CharSet.singleton('.'))],
+    // char class from range:
     [/^[a-z]$/, RE.literal(CharSet.charRange('a', 'z'))],
+    // negative char class:
     [/^[^abc]$/, RE.literal(CharSet.complement(CharSet.fromArray(['a', 'b', 'c'])))],
-    [/^(?:ab)$/, RE.string('ab')], // non-capturing groups
-    [/^(?=^a$)a$/, RE.intersection(RE.string('a'), RE.string('a'))], // positive lookahead
-    [/^(?!^a$)b$/, RE.intersection(RE.complement(RE.string('a')), RE.string('b'))], // negative lookahead
+    // non-capturing groups
+    [/^(?:ab)$/, RE.string('ab')],
+    // positive lookahead
+    [/^(?=^a$)a$/, RE.intersection(RE.string('a'), RE.string('a'))], 
+    // negative lookahead
+    [/^(?!^a$)b$/, RE.intersection(RE.complement(RE.string('a')), RE.string('b'))], 
     [/^(?!^a$)b|c$/, RE.union(RE.intersection(RE.complement(RE.string('a')), RE.string('b')), RE.string('c'))],
+    // some special chars don't need escape when inside brackets:
+    [/^[.^$*+?()[{-|]$/, RE.literal(CharSet.fromArray([...'.^$*+?()[{-|']))],
+    // other special chars need escaped even inside brackets:
+    [/^[\\\]\/]$/, RE.literal(CharSet.fromArray([...'\\]/']))],
   ])('can parse %s', (regexp, expected) => {
     expect(parseRegExp(regexp)).toEqual(expected)
   })