Address feedback; Rename to PyUCS4_*, define macro and test small buffer case

Author: lysnikolaou

Doc/c-api/unicode.rst

@@ -307,7 +307,7 @@ These APIs can be used for fast direct character conversions:
    possible.  This function does not raise exceptions.
 
 
-.. c:function:: Py_ssize_t PyUnicode_ToLower(Py_UCS4 ch, Py_UCS4 *buffer, Py_ssize_t size)
+.. c:function:: Py_ssize_t PyUCS4_ToLower(Py_UCS4 ch, Py_UCS4 *buffer, Py_ssize_t size)
 
    Convert *ch* to lower case, store result in *buffer*, which should be
    able to hold as many characters needed for *ch* to be lower cased
@@ -318,7 +318,7 @@ These APIs can be used for fast direct character conversions:
    .. versionadded:: next
 
 
-.. c:function:: Py_ssize_t PyUnicode_ToUpper(Py_UCS4 ch, Py_UCS4 *buffer, Py_ssize_t size)
+.. c:function:: Py_ssize_t PyUCS4_ToUpper(Py_UCS4 ch, Py_UCS4 *buffer, Py_ssize_t size)
 
    Convert *ch* to upper case, store result in *buffer*, which should be
    able to hold as many characters needed for *ch* to be upper cased
@@ -329,7 +329,7 @@ These APIs can be used for fast direct character conversions:
    .. versionadded:: next
 
 
-.. c:function:: Py_ssize_t PyUnicode_ToTitle(Py_UCS4 ch, Py_UCS4 *buffer, Py_ssize_t size)
+.. c:function:: Py_ssize_t PyUCS4_ToTitle(Py_UCS4 ch, Py_UCS4 *buffer, Py_ssize_t size)
 
    Convert *ch* to title case, store result in *buffer*, which should be
    able to hold as many characters needed for *ch* to be title cased
@@ -340,7 +340,7 @@ These APIs can be used for fast direct character conversions:
    .. versionadded:: next
 
 
-.. c:function:: Py_ssize_t PyUnicode_ToFolded(Py_UCS4 ch, Py_UCS4 *buffer, Py_ssize_t size)
+.. c:function:: Py_ssize_t PyUCS4_ToFolded(Py_UCS4 ch, Py_UCS4 *buffer, Py_ssize_t size)
 
    Foldcase *ch*, store result in *buffer*, which should be
    able to hold as many characters needed for *ch* to be foldcased

Include/cpython/unicodeobject.h

@@ -733,25 +733,25 @@ PyAPI_FUNC(int) _PyUnicode_IsAlpha(
     Py_UCS4 ch       /* Unicode character */
     );
 
-PyAPI_FUNC(Py_ssize_t) PyUnicode_ToLower(
+PyAPI_FUNC(Py_ssize_t) PyUCS4_ToLower(
     Py_UCS4 ch,     /* Unicode character */
     Py_UCS4 *res,   /* Output buffer */
     Py_ssize_t size        /* Buffer size */
     );
 
-PyAPI_FUNC(Py_ssize_t) PyUnicode_ToUpper(
+PyAPI_FUNC(Py_ssize_t) PyUCS4_ToUpper(
     Py_UCS4 ch,     /* Unicode character */
     Py_UCS4 *res,   /* Output buffer */
     Py_ssize_t size        /* Buffer size */
     );
 
-PyAPI_FUNC(Py_ssize_t) PyUnicode_ToTitle(
+PyAPI_FUNC(Py_ssize_t) PyUCS4_ToTitle(
     Py_UCS4 ch,     /* Unicode character */
     Py_UCS4 *res,   /* Output buffer */
     Py_ssize_t size        /* Buffer size */
     );
 
-PyAPI_FUNC(Py_ssize_t) PyUnicode_ToFolded(
+PyAPI_FUNC(Py_ssize_t) PyUCS4_ToFolded(
     Py_UCS4 ch,     /* Unicode character */
     Py_UCS4 *res,   /* Output buffer */
     Py_ssize_t size        /* Buffer size */
@@ -792,6 +792,8 @@ static inline int Py_UNICODE_ISSPACE(Py_UCS4 ch) {
 
 #define Py_UNICODE_ISALPHA(ch) _PyUnicode_IsAlpha(ch)
 
+#define PyUCS4_CASE_CONVERSION_BUFFER_SIZE 3
+
 static inline int Py_UNICODE_ISALNUM(Py_UCS4 ch) {
    return (Py_UNICODE_ISALPHA(ch)
            || Py_UNICODE_ISDECIMAL(ch)

Lib/test/test_capi/test_unicode.py

@@ -1771,7 +1771,7 @@ def test_tolower(self):
     @unittest.skipIf(_testcapi is None, 'need _testcapi module')
     def test_toupper(self):
         import string
-        from _testcapi import unicode_toupper
+        from _testcapi import unicode_toupper, unicode_toupper_buffer_too_small
 
         for i, c in enumerate(string.ascii_lowercase):
             with self.subTest(c):
@@ -1782,6 +1782,10 @@ def test_toupper(self):
         self.assertEqual(unicode_toupper("ß"), "SS")
         self.assertEqual(unicode_toupper("ΐ"), "Ϊ́")
 
+        # Test unicode character with smaller buffer
+        with self.assertRaisesRegex(ValueError, "output buffer is too small"):
+            unicode_toupper_buffer_too_small("ß")
+
     @support.cpython_only
     @unittest.skipIf(_testcapi is None, 'need _testcapi module')
     def test_totitle(self):

Misc/NEWS.d/next/C_API/2025-07-01-14-56-41.gh-issue-76535.9cwObj.rst

@@ -1 +1 @@
-Make :c:func:`PyUnicode_ToLower`, :c:func:`PyUnicode_ToUpper`, :c:func:`PyUnicode_ToTitle` and :c:func:`PyUnicode_ToFolded` public.
+Make :c:func:`PyUCS4_ToLower`, :c:func:`PyUCS4_ToUpper`, :c:func:`PyUCS4_ToTitle` and :c:func:`PyUCS4_ToFolded` public.

Modules/_testcapi/unicode.c

@@ -221,7 +221,8 @@ unicode_copycharacters(PyObject *self, PyObject *args)
 }
 
 static PyObject *
-unicode_case_operation(PyObject *str, Py_ssize_t (*function)(Py_UCS4, Py_UCS4 *, Py_ssize_t))
+unicode_case_operation(PyObject *str, Py_ssize_t (*function)(Py_UCS4, Py_UCS4 *, Py_ssize_t),
+                       Py_UCS4 *buf, Py_ssize_t size)
 {
     if (!PyUnicode_Check(str)) {
         PyErr_Format(PyExc_TypeError, "expect str type, got %T", str);
@@ -235,42 +236,53 @@ unicode_case_operation(PyObject *str, Py_ssize_t (*function)(Py_UCS4, Py_UCS4 *,
 
     Py_UCS4 c = PyUnicode_READ_CHAR(str, 0);
 
-    Py_UCS4 buf[3];
-    Py_ssize_t chars = function(c, buf, Py_ARRAY_LENGTH(buf));
+    Py_ssize_t chars = function(c, buf, size);
     if (chars < 0) {
         return NULL;
     }
 
     return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, chars);
 }
 
-/* Test PyUnicode_ToLower() */
+/* Test PyUCS4_ToLower() */
 static PyObject *
 unicode_tolower(PyObject *self, PyObject *arg)
 {
-    return unicode_case_operation(arg, PyUnicode_ToLower);
+    Py_UCS4 buf[PyUCS4_CASE_CONVERSION_BUFFER_SIZE];
+    return unicode_case_operation(arg, PyUCS4_ToLower, buf, PyUCS4_CASE_CONVERSION_BUFFER_SIZE);
 }
 
-/* Test PyUnicode_ToUpper() */
+
+/* Test PyUCS4_ToUpper() */
 static PyObject *
 unicode_toupper(PyObject *self, PyObject *arg)
 {
-    return unicode_case_operation(arg, PyUnicode_ToUpper);
+    Py_UCS4 buf[PyUCS4_CASE_CONVERSION_BUFFER_SIZE];
+    return unicode_case_operation(arg, PyUCS4_ToUpper, buf, PyUCS4_CASE_CONVERSION_BUFFER_SIZE);
 }
 
+/* Test PyUCS4_ToUpper() with a small buffer */
+static PyObject *
+unicode_toupper_buffer_too_small(PyObject *self, PyObject *arg)
+{
+    Py_UCS4 buf;
+    return unicode_case_operation(arg, PyUCS4_ToUpper, &buf, 1);
+}
 
-/* Test PyUnicode_ToLower() */
+/* Test PyUCS4_ToLower() */
 static PyObject *
 unicode_totitle(PyObject *self, PyObject *arg)
 {
-    return unicode_case_operation(arg, PyUnicode_ToTitle);
+    Py_UCS4 buf[PyUCS4_CASE_CONVERSION_BUFFER_SIZE];
+    return unicode_case_operation(arg, PyUCS4_ToTitle, buf, PyUCS4_CASE_CONVERSION_BUFFER_SIZE);
 }
 
-/* Test PyUnicode_ToLower() */
+/* Test PyUCS4_ToLower() */
 static PyObject *
 unicode_tofolded(PyObject *self, PyObject *arg)
 {
-    return unicode_case_operation(arg, PyUnicode_ToFolded);
+    Py_UCS4 buf[PyUCS4_CASE_CONVERSION_BUFFER_SIZE];
+    return unicode_case_operation(arg, PyUCS4_ToFolded, buf, PyUCS4_CASE_CONVERSION_BUFFER_SIZE);
 }
 
 
@@ -633,6 +645,7 @@ static PyMethodDef TestMethods[] = {
     {"unicode_GET_CACHED_HASH",  unicode_GET_CACHED_HASH,        METH_O},
     {"unicode_tolower",          unicode_tolower,                METH_O},
     {"unicode_toupper",          unicode_toupper,                METH_O},
+    {"unicode_toupper_buffer_too_small",    unicode_toupper_buffer_too_small,   METH_O},
     {"unicode_totitle",          unicode_totitle,                METH_O},
     {"unicode_tofolded",         unicode_tofolded,               METH_O},
     {NULL},

Objects/unicodectype.c

@@ -198,7 +198,7 @@ Py_UCS4 _PyUnicode_ToLowercase(Py_UCS4 ch)
     return ch + ctype->lower;
 }
 
-Py_ssize_t PyUnicode_ToLower(Py_UCS4 ch, Py_UCS4 *res, Py_ssize_t size)
+Py_ssize_t PyUCS4_ToLower(Py_UCS4 ch, Py_UCS4 *res, Py_ssize_t size)
 {
     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
 
@@ -224,7 +224,7 @@ Py_ssize_t PyUnicode_ToLower(Py_UCS4 ch, Py_UCS4 *res, Py_ssize_t size)
     return 1;
 }
 
-Py_ssize_t PyUnicode_ToTitle(Py_UCS4 ch, Py_UCS4 *res, Py_ssize_t size)
+Py_ssize_t PyUCS4_ToTitle(Py_UCS4 ch, Py_UCS4 *res, Py_ssize_t size)
 {
     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
 
@@ -250,7 +250,7 @@ Py_ssize_t PyUnicode_ToTitle(Py_UCS4 ch, Py_UCS4 *res, Py_ssize_t size)
     return 1;
 }
 
-Py_ssize_t PyUnicode_ToUpper(Py_UCS4 ch, Py_UCS4 *res, Py_ssize_t size)
+Py_ssize_t PyUCS4_ToUpper(Py_UCS4 ch, Py_UCS4 *res, Py_ssize_t size)
 {
     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
 
@@ -276,7 +276,7 @@ Py_ssize_t PyUnicode_ToUpper(Py_UCS4 ch, Py_UCS4 *res, Py_ssize_t size)
     return 1;
 }
 
-Py_ssize_t PyUnicode_ToFolded(Py_UCS4 ch, Py_UCS4 *res, Py_ssize_t size)
+Py_ssize_t PyUCS4_ToFolded(Py_UCS4 ch, Py_UCS4 *res, Py_ssize_t size)
 {
     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
 
@@ -294,7 +294,7 @@ Py_ssize_t PyUnicode_ToFolded(Py_UCS4 ch, Py_UCS4 *res, Py_ssize_t size)
         return n;
     }
 
-    return PyUnicode_ToLower(ch, res, size);
+    return PyUCS4_ToLower(ch, res, size);
 }
 
 int _PyUnicode_IsCased(Py_UCS4 ch)

Objects/unicodeobject.c

@@ -10045,7 +10045,7 @@ lower_ucs4(int kind, const void *data, Py_ssize_t length, Py_ssize_t i,
         mapped[0] = handle_capital_sigma(kind, data, length, i);
         return 1;
     }
-    return PyUnicode_ToLower(c, mapped, mapped_size);
+    return PyUCS4_ToLower(c, mapped, mapped_size);
 }
 
 static Py_ssize_t
@@ -10055,7 +10055,7 @@ do_capitalize(int kind, const void *data, Py_ssize_t length, Py_UCS4 *res, Py_UC
     Py_UCS4 c, mapped[3];
 
     c = PyUnicode_READ(kind, data, 0);
-    n_res = PyUnicode_ToTitle(c, mapped, Py_ARRAY_LENGTH(mapped));
+    n_res = PyUCS4_ToTitle(c, mapped, Py_ARRAY_LENGTH(mapped));
     assert(n_res >= 1);
     for (j = 0; j < n_res; j++) {
         *maxchar = Py_MAX(*maxchar, mapped[j]);
@@ -10084,7 +10084,7 @@ do_swapcase(int kind, const void *data, Py_ssize_t length, Py_UCS4 *res, Py_UCS4
             n_res = lower_ucs4(kind, data, length, i, c, mapped, Py_ARRAY_LENGTH(mapped));
         }
         else if (Py_UNICODE_ISLOWER(c)) {
-            n_res = PyUnicode_ToUpper(c, mapped, Py_ARRAY_LENGTH(mapped));
+            n_res = PyUCS4_ToUpper(c, mapped, Py_ARRAY_LENGTH(mapped));
         }
         else {
             n_res = 1;
@@ -10111,7 +10111,7 @@ do_upper_or_lower(int kind, const void *data, Py_ssize_t length, Py_UCS4 *res,
         if (lower)
             n_res = lower_ucs4(kind, data, length, i, c, mapped, Py_ARRAY_LENGTH(mapped));
         else
-            n_res = PyUnicode_ToUpper(c, mapped, Py_ARRAY_LENGTH(mapped));
+            n_res = PyUCS4_ToUpper(c, mapped, Py_ARRAY_LENGTH(mapped));
         assert(n_res >= 1);
         for (j = 0; j < n_res; j++) {
             *maxchar = Py_MAX(*maxchar, mapped[j]);
@@ -10141,7 +10141,7 @@ do_casefold(int kind, const void *data, Py_ssize_t length, Py_UCS4 *res, Py_UCS4
     for (i = 0; i < length; i++) {
         Py_UCS4 c = PyUnicode_READ(kind, data, i);
         Py_UCS4 mapped[3];
-        Py_ssize_t j, n_res = PyUnicode_ToFolded(c, mapped, Py_ARRAY_LENGTH(mapped));
+        Py_ssize_t j, n_res = PyUCS4_ToFolded(c, mapped, Py_ARRAY_LENGTH(mapped));
         assert(n_res >= 1);
         for (j = 0; j < n_res; j++) {
             *maxchar = Py_MAX(*maxchar, mapped[j]);
@@ -10166,7 +10166,7 @@ do_title(int kind, const void *data, Py_ssize_t length, Py_UCS4 *res, Py_UCS4 *m
         if (previous_is_cased)
             n_res = lower_ucs4(kind, data, length, i, c, mapped, Py_ARRAY_LENGTH(mapped));
         else
-            n_res = PyUnicode_ToTitle(c, mapped, Py_ARRAY_LENGTH(mapped));
+            n_res = PyUCS4_ToTitle(c, mapped, Py_ARRAY_LENGTH(mapped));
         assert(n_res >= 1);
         for (j = 0; j < n_res; j++) {
             *maxchar = Py_MAX(*maxchar, mapped[j]);