Merge pull request #3411 from Earlopain/parser-translator-better-regexp-handling

Better handle regexp in the parser translator

Author: kddnewton

lib/prism/translation/parser/compiler.rb

@@ -1507,7 +1507,7 @@ def visit_regular_expression_node(node)
             elsif node.content.include?("\n")
               string_nodes_from_line_continuations(node.unescaped, node.content, node.content_loc.start_offset, node.opening)
             else
-              [builder.string_internal(token(node.content_loc))]
+              [builder.string_internal([node.unescaped, srange(node.content_loc)])]
             end
 
           builder.regexp_compose(

lib/prism/translation/parser/lexer.rb

@@ -637,18 +637,34 @@ def trim_heredoc_whitespace(string, heredoc)
         DELIMITER_SYMETRY = { "[" => "]", "(" => ")", "{" => "}", "<" => ">" }.freeze
         private_constant :DELIMITER_SYMETRY
 
+
+        # https://github.com/whitequark/parser/blob/v3.3.6.0/lib/parser/lexer-strings.rl#L14
+        REGEXP_META_CHARACTERS = ["\\", "$", "(", ")", "*", "+", ".", "<", ">", "?", "[", "]", "^", "{", "|", "}"]
+        private_constant :REGEXP_META_CHARACTERS
+
         # Apply Ruby string escaping rules
         def unescape_string(string, quote)
           # In single-quoted heredocs, everything is taken literally.
           return string if quote == "<<'"
 
-          # TODO: Implement regexp escaping
-          return string if quote == "/" || quote.start_with?("%r")
-
           # OPTIMIZATION: Assume that few strings need escaping to speed up the common case.
           return string unless string.include?("\\")
 
-          if interpolation?(quote)
+          # Enclosing character for the string. `"` for `"foo"`, `{` for `%w{foo}`, etc.
+          delimiter = quote[-1]
+
+          if regexp?(quote)
+            # Should be escaped handled to single-quoted heredocs. The only character that is
+            # allowed to be escaped is the delimiter, except when that also has special meaning
+            # in the regexp. Since all the symetry delimiters have special meaning, they don't need
+            # to be considered separately.
+            if REGEXP_META_CHARACTERS.include?(delimiter)
+              string
+            else
+              # There can never be an even amount of backslashes. It would be a syntax error.
+              string.gsub(/\\(#{Regexp.escape(delimiter)})/, '\1')
+            end
+          elsif interpolation?(quote)
             # Appending individual escape sequences may force the string out of its intended
             # encoding. Start out with binary and force it back later.
             result = "".b
@@ -693,12 +709,6 @@ def unescape_string(string, quote)
 
             result
           else
-            if quote == "'"
-              delimiter = "'"
-            else
-              delimiter = quote[2]
-            end
-
             delimiters = Regexp.escape("#{delimiter}#{DELIMITER_SYMETRY[delimiter]}")
             string.gsub(/\\([\\#{delimiters}])/, '\1')
           end
@@ -730,6 +740,11 @@ def interpolation?(quote)
           quote != "'" && !quote.start_with?("%q", "%w", "%i")
         end
 
+        # Regexp allow interpolation but are handled differently during unescaping
+        def regexp?(quote)
+          quote == "/" || quote.start_with?("%r")
+        end
+
         # Determine if the string is part of a %-style array.
         def percent_array?(quote)
           quote.start_with?("%w", "%W", "%i", "%I")

test/prism/ruby/parser_test.rb

@@ -69,13 +69,15 @@ class ParserTest < TestCase
 
       # https://github.com/whitequark/parser/issues/950
       "whitequark/dedenting_interpolating_heredoc_fake_line_continuation.txt",
+
+      # Contains an escaped multibyte character. This is supposed to drop to backslash
+      "seattlerb/regexp_escape_extended.txt",
     ]
 
     # These files are either failing to parse or failing to translate, so we'll
     # skip them for now.
     skip_all = skip_incorrect | [
       "unescaping.txt",
-      "seattlerb/bug190.txt",
       "seattlerb/heredoc_with_extra_carriage_returns_windows.txt",
       "seattlerb/heredoc_with_only_carriage_returns_windows.txt",
       "seattlerb/heredoc_with_only_carriage_returns.txt",