gh-103997: Fix and test _PyUnicode_Dedent

Include/internal/pycore_unicodeobject.h

@@ -259,9 +259,11 @@ extern Py_ssize_t _PyUnicode_InsertThousandsGrouping(
 
 /* Dedent a string.
    Behaviour is expected to be an exact match of `textwrap.dedent`.
-   Return a new reference on success, NULL with exception set on error.
+   Return a new reference on success, NULL with an exception set on error.
+
+   Export for test_capi.test_unicode
    */
-extern PyObject* _PyUnicode_Dedent(PyObject *unicode);
+PyAPI_FUNC(PyObject*) _PyUnicode_Dedent(PyObject *unicode);
 
 /* --- Misc functions ----------------------------------------------------- */
 

Lib/test/test_capi/test_unicode.py

@@ -1074,6 +1074,95 @@ def test_transform_decimal_and_space(self):
         self.assertRaises(SystemError, transform_decimal, [])
         # CRASHES transform_decimal(NULL)
 
+    @support.cpython_only
+    @unittest.skipIf(_testinternalcapi is None,'need _testinternalcapi module')
+    def test_dedent(self):
+        from _testinternalcapi import _PyUnicode_Dedent as dedent
+        self.assertEqual('hello\nworld', dedent('  hello\n  world'))
+        self.assertEqual('hello\nmy\n  friend', dedent('  hello\n  my\n    friend'))
+
+        # Only spaces.
+        text = "    "
+        expect = ""
+        self.assertEqual(expect, dedent(text))
+
+        # Only tabs.
+        text = "\t\t\t\t"
+        expect = ""
+        self.assertEqual(expect, dedent(text))
+
+        # A mixture.
+        text = " \t  \t\t  \t "
+        expect = ""
+        self.assertEqual(expect, dedent(text))
+
+        # ASCII whitespace.
+        text = "\f\n\r\t\v "
+        expect = "\n"
+        self.assertEqual(expect, dedent(text))
+
+        # One newline.
+        text = "\n"
+        expect = "\n"
+        self.assertEqual(expect, dedent(text))
+
+        # Windows-style newlines.
+        text = "\r\n" * 5
+        expect = "\n" * 5
+        self.assertEqual(expect, dedent(text))
+
+        # Whitespace mixture.
+        text = "    \n\t\n  \n\t\t\n\n\n       "
+        expect = "\n\n\n\n\n\n"
+        self.assertEqual(expect, dedent(text))
+
+        # Lines consisting only of whitespace are always normalised
+        text = "a\n \n\t\n"
+        expect = "a\n\n\n"
+        self.assertEqual(expect, dedent(text))
+
+        # Whitespace characters on non-empty lines are retained
+        text = "a\r\n\r\n\r\n"
+        expect = "a\r\n\n\n"
+        self.assertEqual(expect, dedent(text))
+
+        # Uneven indentation with declining indent level.
+        text = "     Foo\n    Bar\n"  # 5 spaces, then 4
+        expect = " Foo\nBar\n"
+        self.assertEqual(expect, dedent(text))
+
+        # Declining indent level with blank line.
+        text = "     Foo\n\n    Bar\n"  # 5 spaces, blank, then 4
+        expect = " Foo\n\nBar\n"
+        self.assertEqual(expect, dedent(text))
+
+        # Declining indent level with whitespace only line.
+        text = "     Foo\n    \n    Bar\n"  # 5 spaces, then 4, then 4
+        expect = " Foo\n\nBar\n"
+        self.assertEqual(expect, dedent(text))
+
+        text = "  hello\tthere\n  how are\tyou?"
+        expect = "hello\tthere\nhow are\tyou?"
+        self.assertEqual(expect, dedent(text))
+
+        # dedent() only removes whitespace that can be uniformly removed!
+        text = "\thello there\n\thow are you?"
+        expect = "hello there\nhow are you?"
+        self.assertEqual(expect, dedent(text))
+
+        text = '''\
+        def foo():
+            while 1:
+                return foo
+        '''
+        expect = '''\
+def foo():
+    while 1:
+        return foo
+'''
+        self.assertEqual(expect, dedent(text))
+
+
     @support.cpython_only
     @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
     def test_concat(self):

Misc/NEWS.d/next/Core_and_Builtins/2025-09-07-13-36-15.gh-issue-103997.jIPHCc.rst

@@ -0,0 +1 @@
+:option:`-c` now dedents like :func:`textwrap.dedent`

Modules/_testinternalcapi.c

@@ -34,7 +34,7 @@
 #include "pycore_pyerrors.h"      // _PyErr_ChainExceptions1()
 #include "pycore_pylifecycle.h"   // _PyInterpreterConfig_InitFromDict()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
-#include "pycore_unicodeobject.h" // _PyUnicode_TransformDecimalAndSpaceToASCII()
+#include "pycore_unicodeobject.h" // _PyUnicode_TransformDecimalAndSpaceToASCII() / _PyUnicode_Dedent()
 
 #include "clinic/_testinternalcapi.c.h"
 
@@ -1416,6 +1416,17 @@ unicode_transformdecimalandspacetoascii(PyObject *self, PyObject *arg)
     return _PyUnicode_TransformDecimalAndSpaceToASCII(arg);
 }
 
+/* Test _PyUnicode_Dedent() */
+static PyObject *
+unicode_dedent(PyObject *self, PyObject *arg)
+{
+    if (arg == Py_None) {
+        arg = NULL;
+    }
+    return _PyUnicode_Dedent(arg);
+}
+
+
 static PyObject *
 test_pyobject_is_freed(const char *test_name, PyObject *op)
 {
@@ -2422,6 +2433,7 @@ static PyMethodDef module_functions[] = {
     {"_PyTraceMalloc_GetTraceback", tracemalloc_get_traceback, METH_VARARGS},
     {"test_tstate_capi", test_tstate_capi, METH_NOARGS, NULL},
     {"_PyUnicode_TransformDecimalAndSpaceToASCII", unicode_transformdecimalandspacetoascii, METH_O},
+    {"_PyUnicode_Dedent", unicode_dedent, METH_O},
     {"check_pyobject_forbidden_bytes_is_freed",
                             check_pyobject_forbidden_bytes_is_freed, METH_NOARGS},
     {"check_pyobject_freed_is_freed", check_pyobject_freed_is_freed, METH_NOARGS},

Objects/unicodeobject.c

@@ -14309,9 +14309,9 @@ unicode_getnewargs(PyObject *v, PyObject *Py_UNUSED(ignored))
 }
 
 /*
-This function searchs the longest common leading whitespace
+This function searches the longest common leading whitespace
 of all lines in the [src, end).
-It returns the length of the common leading whitespace and sets `output` to
+It returns the length of the common leading whitespace and sets *output* to
 point to the beginning of the common leading whitespace if length > 0.
 */
 static Py_ssize_t
@@ -14331,9 +14331,7 @@ search_longest_common_leading_whitespace(
 
         // scan the whole line
         while (iter < end && *iter != '\n') {
-            if (!leading_whitespace_end && *iter != ' ' && *iter != '\t') {
-                /* `iter` points to the first non-whitespace character
-                   in this line */
+            if (!leading_whitespace_end && !Py_ISSPACE(Py_CHARMASK(*iter))) {
                 if (iter == line_start) {
                     // some line has no indent, fast exit!
                     return 0;
@@ -14389,8 +14387,8 @@ search_longest_common_leading_whitespace(
 }
 
 /* Dedent a string.
-   Behaviour is expected to be an exact match of `textwrap.dedent`.
-   Return a new reference on success, NULL with exception set on error.
+   Behaviour is expected to be an exact match of textwrap.dedent.
+   Return a new reference on success, NULL with an exception set on error.
    */
 PyObject *
 _PyUnicode_Dedent(PyObject *unicode)
@@ -14410,13 +14408,9 @@ _PyUnicode_Dedent(PyObject *unicode)
     // [whitespace_start, whitespace_start + whitespace_len)
     // describes the current longest common leading whitespace
     const char *whitespace_start = NULL;
-    Py_ssize_t whitespace_len = search_longest_common_leading_whitespace(
+    const Py_ssize_t whitespace_len = search_longest_common_leading_whitespace(
         src, end, &whitespace_start);
 
-    if (whitespace_len == 0) {
-        return Py_NewRef(unicode);
-    }
-
     // now we should trigger a dedent
     char *dest = PyMem_Malloc(src_len);
     if (!dest) {
@@ -14431,25 +14425,26 @@ _PyUnicode_Dedent(PyObject *unicode)
 
         // iterate over a line to find the end of a line
         while (iter < end && *iter != '\n') {
-            if (in_leading_space && *iter != ' ' && *iter != '\t') {
+            if (in_leading_space && !Py_ISSPACE(Py_CHARMASK(*iter))) {
                 in_leading_space = false;
             }
             ++iter;
         }
 
         // invariant: *iter == '\n' or iter == end
-        bool append_newline = iter < end;
+        const bool append_newline = iter < end;
 
         // if this line has all white space, write '\n' and continue
-        if (in_leading_space && append_newline) {
-            *dest_iter++ = '\n';
+        if (in_leading_space) {
+            if (append_newline) {
+                *dest_iter++ = '\n';
+            }
             continue;
         }
 
         /* copy [new_line_start + whitespace_len, iter) to buffer, then
             conditionally append '\n' */
-
-        Py_ssize_t new_line_len = iter - line_start - whitespace_len;
+        const Py_ssize_t new_line_len = iter - line_start - whitespace_len;
         assert(new_line_len >= 0);
         memcpy(dest_iter, line_start + whitespace_len, new_line_len);