gh-103997: Automatically dedent the argument to "-c"

Include/internal/pycore_bytesobject.h

@@ -122,6 +122,31 @@ PyAPI_FUNC(void*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer,
     const void *bytes,
     Py_ssize_t size);
 
+
+/** Dedent a UTF-8 encoded string.
+ * behavior is expected to match `textwrap.dedent`
+ *
+ * return value:
+ * 0, no need to dedent, writer untouched
+ * 1, success
+ * -1, failure
+ *
+ * str is the beginning of the string to dedent.
+ * expecting (str != NULL)
+ *
+ * len is the length of the string to dedent.
+ * expecting (len >= 0)
+ *
+ * writer is a _PyBytesWriter object to write the dedented string.
+ * expecting (writer != NULL)
+ *
+ * p points to a char* indicating the current position in the _PyBytesWriter.
+ * It is updated to the new position after writing the dedented string on exit.
+ * expecting (p != NULL && *p != NULL)
+ */
+PyAPI_FUNC(int)
+_PyBytes_Dedent(const char *str, Py_ssize_t len, _PyBytesWriter *writer,
+                char **p);
 #ifdef __cplusplus
 }
 #endif

Lib/test/test_cmd_line.py

@@ -901,6 +901,75 @@ def res2int(res):
         )
         self.assertEqual(res2int(res), (6000, 6000))
 
+    def test_cmd_dedent(self):
+        # test that -c auto-dedents its arguments
+        from textwrap import dedent
+        test_cases = [
+            {
+                'code': '''
+                    print('space-auto-dedent')
+                ''',
+                'expected': b'space-auto-dedent',
+            },
+            {
+                'code': dedent('''
+                ^^^print('tab-auto-dedent')
+                ''').replace('^', '\t'),
+                'expected': b'tab-auto-dedent',
+            },
+            {
+                'code': dedent('''
+                ^^if 1:
+                ^^^^print('mixed-auto-dedent-1')
+                ^^print('mixed-auto-dedent-2')
+                ''').replace('^', '\t \t'),
+                'expected': b'mixed-auto-dedent-1\nmixed-auto-dedent-2',
+            },
+            {
+                'code': '''
+                    data = """$
+
+                    this data has an empty newline above and a newline with spaces below $
+                                            $
+                    """$
+                    if 1:         $
+                        print(repr(data))$
+                '''.replace('$', ''),
+                # Note: entirely blank lines are normalized to \n, even if they
+                # are part of a data string. This is consistent with
+                # textwrap.dedent behavior, but might not be intuitive.
+                'expected': b"'\\n\\nthis data has an empty newline above and a newline with spaces below \\n\\n'",
+            },
+        ]
+        for case in test_cases:
+            # Run the auto-dedent case
+            args1 = sys.executable, '-c', case['code']
+            proc1 = subprocess.run(args1, stdout=subprocess.PIPE)
+            self.assertEqual(proc1.returncode, 0, proc1)
+            output1 = proc1.stdout.strip()
+
+            # Manually dedent beforehand, check the result is the same.
+            args2 = sys.executable, '-c', dedent(case['code'])
+            proc2 = subprocess.run(args2, stdout=subprocess.PIPE)
+            self.assertEqual(proc2.returncode, 0, proc2)
+            output2 = proc2.stdout.strip()
+
+            self.assertEqual(output1, output2)
+            self.assertEqual(output1.replace(b'\r\n', b'\n'), case['expected'])
+
+    def test_cmd_dedent_failcase(self):
+        # Mixing tabs and spaces is not allowed
+        from textwrap import dedent
+        template = dedent(
+            '''
+            -+if 1:
+            +-++ print('will fail')
+            ''')
+        code = template.replace('-', ' ').replace('+', '\t')
+        assert_python_failure('-c', code)
+        code = template.replace('-', '\t').replace('+', ' ')
+        assert_python_failure('-c', code)
+
 
 @unittest.skipIf(interpreter_requires_environment(),
                  'Cannot run -I tests when PYTHON env vars are required.')

Misc/NEWS.d/next/Core and Builtins/2023-04-29-23-15-38.gh-issue-103997.BS3uVt.rst

@@ -0,0 +1,7 @@
+String arguments passed to  "-c" are now automatically dedented as if by
+:func:`textwrap.dedent`. This allows "python -c" invocations to be indented
+ in shell scripts without causing indentation errors.
+
+Add a private API :c:func:`_PyBytes_Dedent`.
+
+(Patch by Jon Crall and Steven Sun)

Modules/main.c

@@ -7,6 +7,7 @@
 #include "pycore_pathconfig.h"    // _PyPathConfig_ComputeSysPath0()
 #include "pycore_pylifecycle.h"   // _Py_PreInitializeFromPyArgv()
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
+#include "pycore_bytesobject.h"   // _PyBytesWriter, _PyBytes_Dedent()
 
 /* Includes for exit_sigint() */
 #include <stdio.h>                // perror()
@@ -228,6 +229,37 @@ pymain_import_readline(const PyConfig *config)
     }
 }
 
+/* Strip common leading whitespace, just as textwrap.dedent.
+   It returns a new reference. */
+static PyObject *
+dedent_utf8_bytes(PyObject *bytes)
+{
+    assert(bytes == NULL || !PyBytes_CheckExact(bytes->ob_type));
+
+    Py_ssize_t nchars;
+    char *start;
+    if (PyBytes_AsStringAndSize(bytes, &start, &nchars) != 0) {
+        return NULL;
+    }
+
+    _PyBytesWriter writer;
+    _PyBytesWriter_Init(&writer);
+    char *p = _PyBytesWriter_Alloc(&writer, nchars);
+    if (p == NULL) {
+        return NULL;
+    }
+
+    int ret = _PyBytes_Dedent(start, nchars, &writer, &p);
+    if (ret < 0) {
+        return NULL;
+    }
+    if (ret == 0) {
+        Py_INCREF(bytes);
+        _PyBytesWriter_Dealloc(&writer);
+        return bytes;
+    }
+    return _PyBytesWriter_Finish(&writer, p);
+}
 
 static int
 pymain_run_command(wchar_t *command)
@@ -250,6 +282,13 @@ pymain_run_command(wchar_t *command)
         goto error;
     }
 
+    PyObject *new_bytes = dedent_utf8_bytes(bytes);
+    if (new_bytes == NULL) {
+        Py_DECREF(bytes);
+        goto error;
+    }
+    Py_SETREF(bytes, new_bytes);
+
     PyCompilerFlags cf = _PyCompilerFlags_INIT;
     cf.cf_flags |= PyCF_IGNORE_COOKIE;
     ret = PyRun_SimpleStringFlags(PyBytes_AsString(bytes), &cf);

Objects/bytesobject.c

@@ -3535,6 +3535,8 @@ _PyBytesWriter_WriteBytes(_PyBytesWriter *writer, void *ptr,
 }
 
 
+/* Algorithms on bytes */
+
 void
 _PyBytes_Repeat(char* dest, Py_ssize_t len_dest,
     const char* src, Py_ssize_t len_src)
@@ -3558,3 +3560,148 @@ _PyBytes_Repeat(char* dest, Py_ssize_t len_dest,
     }
 }
 
+/** Dedent a UTF-8 encoded string.
+ * behavior is expected to match `textwrap.dedent`
+ *
+ * return value:
+ * 0, no need to dedent, writer untouched
+ * 1, success
+ * -1, failure
+ *
+ * str is the beginning of the string to dedent.
+ * expecting (str != NULL)
+ *
+ * len is the length of the string to dedent.
+ * expecting (len >= 0)
+ *
+ * writer is a _PyBytesWriter object to write the dedented string.
+ * expecting (writer != NULL)
+ *
+ * p points to a char* indicating the current position in the _PyBytesWriter.
+ * It is updated to the new position after writing the dedented string on exit.
+ * expecting (p != NULL && *p != NULL)
+ */
+int
+_PyBytes_Dedent(const char *str, Py_ssize_t len, _PyBytesWriter *writer,
+                char **p)
+{
+    assert(str);
+    assert(p != NULL && *p != NULL);
+    assert(writer);
+
+    if (len <= 0)
+        return 0;
+
+    const char *end = str + len;
+    assert(str < end); // prevent overflow when len is too large
+
+    const char *candidate_start = NULL;
+    Py_ssize_t candidate_len = 0;
+
+    for (const char *iter = str; iter < end; ++iter) {
+        const char *line_start = iter;
+        const char *leading_whitespace_end = NULL;
+
+        // scan the whole line
+        while (iter < end && *iter != '\n') {
+            if (!leading_whitespace_end && *iter != ' ' && *iter != '\t') {
+                if (iter == line_start) {
+                    // some line has no indent, fast exit!
+                    return 0;
+                }
+                leading_whitespace_end = iter;
+            }
+            ++iter;
+        }
+
+        // if this line has all white space, skip it
+        if (!leading_whitespace_end) {
+            continue;
+        }
+
+        if (!candidate_start) {
+            candidate_start = line_start;
+            candidate_len = leading_whitespace_end - line_start;
+            assert(candidate_len > 0);
+        } else {
+            /* We then compare with the current longest leading whitespace.
+
+               [line_start, leading_whitespace_end) is the leading whitespace of
+               this line,
+
+               [candidate_start, candidate_start + candidate_len)
+               is the leading whitespace of the current longest leading
+               whitespace. */
+            Py_ssize_t new_candidate_len = 0;
+
+            for (const char *candidate_iter = candidate_start,
+                            *line_iter = line_start;
+                 candidate_iter < candidate_start + candidate_len &&
+                 line_iter < leading_whitespace_end;
+                 ++candidate_iter, ++line_iter) {
+                if (*candidate_iter != *line_iter) {
+                    break;
+                }
+                ++new_candidate_len;
+            }
+
+            candidate_len = new_candidate_len;
+            if (candidate_len == 0) {
+                return 0;
+            }
+        }
+    }
+
+    assert(candidate_len >= 0);
+    if (candidate_len == 0) {
+        return 0;
+    }
+
+    // trigger a dedent
+
+    // prepare the writer
+    char *p_ = _PyBytesWriter_Prepare(writer, *p, len);
+    if (p_ == NULL) {
+        *p = NULL;
+        return -1;
+    }
+
+    for (const char *iter = str; iter < end; ++iter) {
+        const char *line_start = iter;
+        bool in_leading_space = true;
+
+        // iterate over a line to find the end of a line
+        while (iter < end && *iter != '\n') {
+            if (in_leading_space && *iter != ' ' && *iter != '\t') {
+                in_leading_space = false;
+            }
+            ++iter;
+        }
+
+        // invariant: *iter == '\n' or iter == end
+        bool append_newline = iter < end;
+
+        // if this line has all white space, write '\n'
+        if (in_leading_space && append_newline) {
+            *p_++ = '\n';
+            continue;
+        }
+
+        /* copy [new_line_start + candidate_len, iter) to buffer, then
+            conditionally append '\n' */
+
+        Py_ssize_t new_line_len = iter - line_start - candidate_len;
+        assert(new_line_len >= 0);
+
+        memcpy(p_, line_start + candidate_len, new_line_len);
+
+        p_ += new_line_len;
+
+        if (append_newline) {
+            *p_++ = '\n';
+        }
+    }
+
+    *p = p_;
+    return 1;
+}