[mypyc] Support unicode surrogates in string literals (#18209)

Previously surrogates would trigger a compiler crash.

Author: JukkaL

mypyc/codegen/literals.py

@@ -230,7 +230,7 @@ def format_int(n: int) -> bytes:
 
 
 def format_str_literal(s: str) -> bytes:
-    utf8 = s.encode("utf-8")
+    utf8 = s.encode("utf-8", errors="surrogatepass")
     return format_int(len(utf8)) + utf8
 
 

mypyc/lib-rt/misc_ops.c

@@ -563,7 +563,7 @@ int CPyStatics_Initialize(PyObject **statics,
             while (num-- > 0) {
                 size_t len;
                 data = parse_int(data, &len);
-                PyObject *obj = PyUnicode_FromStringAndSize(data, len);
+                PyObject *obj = PyUnicode_DecodeUTF8(data, len, "surrogatepass");
                 if (obj == NULL) {
                     return -1;
                 }

mypyc/test-data/run-strings.test

@@ -639,3 +639,12 @@ def test_encode() -> None:
     assert u'\u00E1'.encode('latin1') == b'\xe1'
     with assertRaises(UnicodeEncodeError):
         u.encode('latin1')
+
+[case testUnicodeSurrogate]
+def f() -> str:
+    return "\ud800"
+
+def test_surrogate() -> None:
+    assert ord(f()) == 0xd800
+    assert ord("\udfff") == 0xdfff
+    assert repr("foobar\x00\xab\ud912\U00012345") == r"'foobar\x00«\ud912𒍅'"