Apply encoding compatibility patch

Author: Charlie Somerville

enc/trans/single_byte.trans

@@ -2,11 +2,13 @@
 
 <%
   us_ascii_map = [["{00-7f}", :nomap]]
+  binary_map   = [["{00-ff}", :nomap]]
 
   transcode_tblgen "US-ASCII", "UTF-8", us_ascii_map
   transcode_tblgen "UTF-8", "US-ASCII", us_ascii_map
   transcode_tblgen "ASCII-8BIT", "UTF-8", us_ascii_map
   transcode_tblgen "UTF-8", "ASCII-8BIT", us_ascii_map
+  transcode_tblgen "UTF-8-COMPAT", "ASCII-8BIT", binary_map, '{00-ff}'
 
   CONTROL1_TO_UCS_TBL = (0x80..0x9f).map {|c| ["%02X" % c, c] }
 
@@ -84,8 +86,14 @@
 
 <%= transcode_generated_code %>
 
+extern rb_encoding_compat;
+
 TRANS_INIT(single_byte)
 {
+    if (rb_encoding_compat) {
+        ((struct rb_transcoder *)&rb_from_ASCII_8BIT)->conv_tree_start = from_UTF_8_COMPAT_to_ASCII_8BIT;
+        ((struct rb_transcoder *)  &rb_to_ASCII_8BIT)->conv_tree_start = from_UTF_8_COMPAT_to_ASCII_8BIT;
+    }
 <%= transcode_register_code %>
 }
 

encoding.c

@@ -36,6 +36,8 @@ void rb_encdb_set_unicode(int index);
 #pragma GCC visibility pop
 #endif
 
+int rb_encoding_compat;
+
 static ID id_encoding;
 VALUE rb_cEncoding;
 static VALUE rb_encoding_list;
@@ -843,6 +845,14 @@ rb_enc_compatible(VALUE str1, VALUE str2)
 	    if (cr2 == ENC_CODERANGE_7BIT) {
 		return enc1;
 	    }
+            if (rb_encoding_compat) {
+	        if (idx1 == ENCINDEX_UTF_8 && idx2 == ENCINDEX_ASCII) {
+	            return enc2;
+	        }
+	        else if (idx1 == ENCINDEX_ASCII && idx2 == ENCINDEX_UTF_8) {
+	            return enc1;
+	        }
+	    }
 	}
 	if (cr1 == ENC_CODERANGE_7BIT)
 	    return enc2;
@@ -935,7 +945,10 @@ rb_enc_codepoint_len(const char *p, const char *e, int *len_p, rb_encoding *enc)
         rb_raise(rb_eArgError, "empty string");
     r = rb_enc_precise_mbclen(p, e, enc);
     if (!MBCLEN_CHARFOUND_P(r)) {
-	rb_raise(rb_eArgError, "invalid byte sequence in %s", rb_enc_name(enc));
+        if (rb_encoding_compat && enc == rb_utf8_encoding() && MBCLEN_CHARFOUND_P(r = rb_enc_precise_mbclen(p, e, rb_ascii8bit_encoding())))
+          enc = rb_ascii8bit_encoding();
+        else
+	  rb_raise(rb_eArgError, "invalid byte sequence in %s", rb_enc_name(enc));
     }
     if (len_p) *len_p = MBCLEN_CHARFOUND_LEN(r);
     return rb_enc_mbc_to_codepoint(p, e, enc);
@@ -1420,6 +1433,12 @@ get_default_internal(VALUE klass)
     return rb_enc_default_internal();
 }
 
+static VALUE
+rb_enc_compat_mode_enabled_p(VALUE klass)
+{
+    return rb_encoding_compat ? Qtrue : Qfalse;
+}
+
 void
 rb_enc_set_default_internal(VALUE encoding)
 {
@@ -1872,6 +1891,9 @@ Init_Encoding(void)
     for (i = 0; i < enc_table.count; ++i) {
 	rb_ary_push(list, enc_new(enc_table.list[i].enc));
     }
+
+    rb_const_set(rb_cEncoding, rb_intern_const("COMPAT_MODE_AVAILABLE"), Qtrue);
+    rb_define_singleton_method(rb_cEncoding, "compat_mode_enabled?", rb_enc_compat_mode_enabled_p, 0);
 }
 
 /* locale insensitive ctype functions */

include/ruby/encoding.h

@@ -77,6 +77,7 @@ extern "C" {
         ENC_CODERANGE_SET(rb_encoding_coderange_obj, (cr)); \
     } while (0)
 
+extern int rb_encoding_compat;
 typedef OnigEncodingType rb_encoding;
 
 int rb_char_to_option_kcode(int c, int *option, int *kcode);

re.c

@@ -1247,15 +1247,16 @@ static rb_encoding*
 rb_reg_prepare_enc(VALUE re, VALUE str, int warn)
 {
     rb_encoding *enc = 0;
+    enc = rb_enc_get(str);
 
     if (rb_enc_str_coderange(str) == ENC_CODERANGE_BROKEN) {
-        rb_raise(rb_eArgError,
-            "invalid byte sequence in %s",
-            rb_enc_name(rb_enc_get(str)));
+        if (!(rb_encoding_compat && enc == rb_utf8_encoding()))
+            rb_raise(rb_eArgError,
+                "invalid byte sequence in %s",
+                rb_enc_name(rb_enc_get(str)));
     }
 
     rb_reg_check(re);
-    enc = rb_enc_get(str);
     if (!rb_enc_str_asciicompat_p(str)) {
         if (RREGEXP(re)->ptr->enc != enc) {
 	    reg_enc_error(re, str);
@@ -1265,15 +1266,21 @@ rb_reg_prepare_enc(VALUE re, VALUE str, int warn)
         if (RREGEXP(re)->ptr->enc != enc &&
 	    (!rb_enc_asciicompat(RREGEXP(re)->ptr->enc) ||
 	     rb_enc_str_coderange(str) != ENC_CODERANGE_7BIT)) {
+            if (rb_encoding_compat &&
+                ((RREGEXP(re)->ptr->enc == rb_ascii8bit_encoding() && enc == rb_utf8_encoding()) ||
+                 (enc == rb_ascii8bit_encoding() && RREGEXP(re)->ptr->enc == rb_utf8_encoding()))) {
+                return rb_ascii8bit_encoding();
+            }
 	    reg_enc_error(re, str);
 	}
 	enc = RREGEXP(re)->ptr->enc;
     }
     if (warn && (RBASIC(re)->flags & REG_ENCODING_NONE) &&
 	enc != rb_ascii8bit_encoding() &&
 	rb_enc_str_coderange(str) != ENC_CODERANGE_7BIT) {
-	rb_warn("regexp match /.../n against to %s string",
-		rb_enc_name(enc));
+        if (!(rb_encoding_compat && enc == rb_utf8_encoding()))
+	    rb_warn("regexp match /.../n against to %s string",
+	        rb_enc_name(enc));
     }
     return enc;
 }

ruby.c

@@ -1098,6 +1098,9 @@ proc_options(long argc, char **argv, struct cmdline_options *opt, int envopt)
 		set_source_encoding_once(opt, s, 0);
 	    }
 #endif
+	    else if (strcmp("encoding-compatibility", s) == 0) {
+                rb_encoding_compat = 1;
+            }
 	    else if (strcmp("version", s) == 0) {
 		if (envopt) goto noenvopt_long;
 		opt->dump |= DUMP_BIT(version);

string.c

@@ -2019,7 +2019,12 @@ rb_enc_cr_str_buf_cat(VALUE str, const char *ptr, long len,
     if (ptr_cr_ret)
         *ptr_cr_ret = ptr_cr;
 
-    if (str_encindex != ptr_encindex &&
+    if (rb_encoding_compat &&
+        ((str_encindex == rb_utf8_encindex() && ptr_encindex == rb_ascii8bit_encindex()) ||
+         (str_encindex == rb_ascii8bit_encindex() && ptr_encindex == rb_utf8_encindex()))) {
+        /* fall through to conditional below */
+    }
+    else if (str_encindex != ptr_encindex &&
         str_cr != ENC_CODERANGE_7BIT &&
         ptr_cr != ENC_CODERANGE_7BIT) {
       incompatible:
@@ -2028,7 +2033,14 @@ rb_enc_cr_str_buf_cat(VALUE str, const char *ptr, long len,
             rb_enc_name(rb_enc_from_index(ptr_encindex)));
     }
 
-    if (str_cr == ENC_CODERANGE_UNKNOWN) {
+    if (rb_encoding_compat &&
+        str_encindex != ptr_encindex &&
+        str_cr != ENC_CODERANGE_7BIT && ptr_cr != ENC_CODERANGE_7BIT) {
+        /* from fall through above */
+        res_encindex = rb_ascii8bit_encindex();
+        res_cr = ENC_CODERANGE_VALID;
+    }
+    else if (str_cr == ENC_CODERANGE_UNKNOWN) {
         res_encindex = str_encindex;
         res_cr = ENC_CODERANGE_UNKNOWN;
     }
@@ -2240,6 +2252,8 @@ rb_str_hash(VALUE str)
     if (e && rb_enc_str_coderange(str) == ENC_CODERANGE_7BIT) {
 	e = 0;
     }
+    if (rb_encoding_compat && e == rb_utf8_encindex() || e == rb_ascii8bit_encindex())
+      e = 0;
     return rb_memhash((const void *)RSTRING_PTR(str), RSTRING_LEN(str)) ^ e;
 }
 
@@ -2294,6 +2308,11 @@ rb_str_comparable(VALUE str1, VALUE str2)
 	if (rb_enc_asciicompat(rb_enc_from_index(idx1)))
 	    return TRUE;
     }
+    if (rb_encoding_compat &&
+        ((idx1 == rb_utf8_encindex() && idx2 == rb_ascii8bit_encindex()) ||
+         (idx1 == rb_ascii8bit_encindex() && idx2 == rb_utf8_encindex()))) {
+        return TRUE;
+    }
     return FALSE;
 }
 
@@ -6034,7 +6053,8 @@ rb_str_split_m(int argc, VALUE *argv, VALUE str)
 	long slen = RSTRING_LEN(spat);
 
 	if (is_broken_string(str)) {
-	    rb_raise(rb_eArgError, "invalid byte sequence in %s", rb_enc_name(STR_ENC_GET(str)));
+	    if (!(rb_encoding_compat && STR_ENC_GET(str) == rb_utf8_encoding()))
+	        rb_raise(rb_eArgError, "invalid byte sequence in %s", rb_enc_name(STR_ENC_GET(str)));
 	}
 	if (is_broken_string(spat)) {
 	    rb_raise(rb_eArgError, "invalid byte sequence in %s", rb_enc_name(STR_ENC_GET(spat)));