[Bug #20990] Fix tokenizing "\M-\[mbchar]" and "\C-\[mbchar]"

Author: tompng

parse.y

@@ -8152,11 +8152,15 @@ read_escape(struct parser_params *p, int flags, const char *begin)
             goto eof;
         }
         if ((c = nextc(p)) == '\\') {
-            switch (peekc(p)) {
+            switch (c = peekc(p)) {
               case 'u': case 'U':
                 nextc(p);
                 goto eof;
             }
+            if (!ISASCII(c) && c != -1) {
+                tokskip_mbchar(p);
+                goto eof;
+            }
             return read_escape(p, flags|ESCAPE_META, begin) | 0x80;
         }
         else if (c == -1) goto eof;
@@ -8185,11 +8189,15 @@ read_escape(struct parser_params *p, int flags, const char *begin)
       case 'c':
         if (flags & ESCAPE_CONTROL) goto eof;
         if ((c = nextc(p))== '\\') {
-            switch (peekc(p)) {
+            switch (c = peekc(p)) {
               case 'u': case 'U':
                 nextc(p);
                 goto eof;
             }
+            if (!ISASCII(c) && c != -1) {
+                tokskip_mbchar(p);
+                goto eof;
+            }
             c = read_escape(p, flags|ESCAPE_CONTROL, begin);
         }
         else if (c == '?')

test/ripper/test_lexer.rb

@@ -390,6 +390,30 @@ def test_invalid_escape_ctrl_meta_mbchar
     assert_lexer(expected, code)
   end
 
+  def test_invalid_escape_ctrl_backslash_mbchar
+    code = %["\\C-\\\u{3042}"]
+    expected = [
+      [[1, 0], :on_tstring_beg, '"', state(:EXPR_BEG)],
+      [[1, 1], :on_tstring_content, "\\C-\\", state(:EXPR_BEG)],
+      [[1, 5], :on_tstring_content, "\u{3042}", state(:EXPR_BEG)],
+      [[1, 8], :on_tstring_end, '"', state(:EXPR_END)],
+    ]
+
+    assert_lexer(expected, code)
+  end
+
+  def test_invalid_escape_meta_backslash_mbchar
+    code = %["\\M-\\\u{3042}"]
+    expected = [
+      [[1, 0], :on_tstring_beg, '"', state(:EXPR_BEG)],
+      [[1, 1], :on_tstring_content, "\\M-\\", state(:EXPR_BEG)],
+      [[1, 5], :on_tstring_content, "\u{3042}", state(:EXPR_BEG)],
+      [[1, 8], :on_tstring_end, '"', state(:EXPR_END)],
+    ]
+
+    assert_lexer(expected, code)
+  end
+
   def test_invalid_escape_string
     code = "\"hello\\x world"
     expected = [