fix race

Author: ZeroIntensity

Lib/test/test_capi/test_unicode.py

@@ -1,7 +1,7 @@
 import unittest
 import sys
 from test import support
-from test.support import import_helper
+from test.support import threading_helper
 
 try:
     import _testcapi
@@ -1005,6 +1005,23 @@ def test_asutf8(self):
         self.assertRaises(TypeError, unicode_asutf8, [], 0)
         # CRASHES unicode_asutf8(NULL, 0)
 
+    @unittest.skipIf(_testcapi is None, 'need _testcapi module')
+    @threading_helper.requires_working_threading()
+    def test_asutf8_race(self):
+        from _testcapi import unicode_asutf8
+        from threading import Thread
+
+        data = "😊"
+
+        def worker():
+            for _ in range(1000):
+                self.assertEqual(unicode_asutf8(data, 5), b'\xf0\x9f\x98\x8a\0')
+
+        threads = [Thread(target=worker) for _ in range(10)]
+        with threading_helper.start_threads(threads):
+            pass
+
+
     @support.cpython_only
     @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
     def test_asutf8andsize(self):

Objects/unicodeobject.c

@@ -118,14 +118,14 @@ NOTE: In the interpreter's initialization phase, some globals are currently
     (assert(_PyUnicode_CHECK(op)),                      \
      PyUnicode_IS_COMPACT_ASCII(op) ?                   \
          ((char*)(_PyASCIIObject_CAST(op) + 1)) :       \
-         _PyUnicode_UTF8(op))
+        FT_ATOMIC_LOAD_PTR(_PyUnicode_UTF8(op)))
 #define _PyUnicode_UTF8_LENGTH(op)                      \
     (_PyCompactUnicodeObject_CAST(op)->utf8_length)
 #define PyUnicode_UTF8_LENGTH(op)                       \
     (assert(_PyUnicode_CHECK(op)),                      \
      PyUnicode_IS_COMPACT_ASCII(op) ?                   \
          _PyASCIIObject_CAST(op)->length :              \
-         _PyUnicode_UTF8_LENGTH(op))
+         FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyUnicode_UTF8_LENGTH(op)))
 
 #define _PyUnicode_LENGTH(op)                           \
     (_PyASCIIObject_CAST(op)->length)
@@ -667,16 +667,16 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content)
             CHECK(ascii->state.compact == 0);
             CHECK(data != NULL);
             if (ascii->state.ascii) {
-                CHECK(compact->utf8 == data);
-                CHECK(compact->utf8_length == ascii->length);
+                CHECK(FT_ATOMIC_LOAD_PTR(compact->utf8) == data);
+                CHECK(FT_ATOMIC_LOAD_SSIZE_RELAXED(compact->utf8_length) == ascii->length);
             }
             else {
-                CHECK(compact->utf8 != data);
+                CHECK(FT_ATOMIC_LOAD_PTR(compact->utf8) != data);
             }
         }
 
-        if (compact->utf8 == NULL)
-            CHECK(compact->utf8_length == 0);
+        if (FT_ATOMIC_LOAD_PTR(compact->utf8) == NULL)
+            CHECK(FT_ATOMIC_LOAD_SSIZE_RELAXED(compact->utf8_length) == 0);
     }
 
     /* check that the best kind is used: O(n) operation */
@@ -5823,7 +5823,11 @@ unicode_fill_utf8(PyObject *unicode)
 {
     /* the string cannot be ASCII, or PyUnicode_UTF8() would be set */
     assert(!PyUnicode_IS_ASCII(unicode));
-
+    int ret = 0;
+    Py_BEGIN_CRITICAL_SECTION(unicode);
+    if (FT_ATOMIC_LOAD_PTR(_PyUnicode_UTF8(unicode))) {
+        goto exit;
+    }
     int kind = PyUnicode_KIND(unicode);
     const void *data = PyUnicode_DATA(unicode);
     Py_ssize_t size = PyUnicode_GET_LENGTH(unicode);
@@ -5849,7 +5853,8 @@ unicode_fill_utf8(PyObject *unicode)
     }
     if (end == NULL) {
         _PyBytesWriter_Dealloc(&writer);
-        return -1;
+        ret = -1;
+        goto exit;
     }
 
     const char *start = writer.use_small_buffer ? writer.small_buffer :
@@ -5860,14 +5865,17 @@ unicode_fill_utf8(PyObject *unicode)
     if (cache == NULL) {
         _PyBytesWriter_Dealloc(&writer);
         PyErr_NoMemory();
-        return -1;
+        ret = -1;
+        goto exit;
     }
-    _PyUnicode_UTF8(unicode) = cache;
-    _PyUnicode_UTF8_LENGTH(unicode) = len;
     memcpy(cache, start, len);
     cache[len] = '\0';
+    FT_ATOMIC_STORE_SSIZE_RELAXED(_PyUnicode_UTF8_LENGTH(unicode), len);
+    FT_ATOMIC_STORE_PTR(_PyUnicode_UTF8(unicode), cache);
     _PyBytesWriter_Dealloc(&writer);
-    return 0;
+exit:;
+    Py_END_CRITICAL_SECTION();
+    return ret;
 }
 
 PyObject *