Improve error messages for stack overflow

Author: markshannon

Include/cpython/pystate.h

@@ -113,6 +113,7 @@ struct _ts {
     int py_recursion_limit;
 
     // These are addresses, but we need to convert to ints to avoid UB.
+    uintptr_t c_stack_top;
     uintptr_t c_stack_soft_limit;
     uintptr_t c_stack_hard_limit;
     int recursion_headroom; /* Allow 50 more calls to handle any errors. */
@@ -204,48 +205,6 @@ struct _ts {
     PyObject *threading_local_sentinel;
 };
 
-
-#if defined(__s390x__)
-#  define Py_C_STACK_SIZE 320000
-#elif defined(_WIN32) && defined(_M_ARM64)
-#  define Py_C_STACK_SIZE 400000
-#elif defined(_WIN32)
-#  define Py_C_STACK_SIZE 1200000
-#elif defined(__ANDROID__)
-   // On an ARM64 emulator, API level 34 was OK with 10000, but API level 21
-   // crashed in test_compiler_recursion_limit.
-#  define Py_C_STACK_SIZE 1200000
-#elif defined(__sparc__)
-   // test_descr crashed on sparc64 with >7000 but let's keep a margin of error.
-#  define Py_C_STACK_SIZE 1600000
-#elif defined(__wasi__)
-   // Based on wasmtime 16.
-#  define Py_C_STACK_SIZE 2000000
-#elif defined(__hppa__) || defined(__powerpc64__)
-   // test_descr crashed with >8000 but let's keep a margin of error.
-#  define Py_C_STACK_SIZE 2000000
-#else
-   // This value is duplicated in Lib/test/support/__init__.py
-#  define Py_C_STACK_SIZE 5000000
-#endif
-
-
-#ifdef Py_DEBUG
-   // A debug build is likely built with low optimization level which implies
-   // higher stack memory usage than a release build: use a lower limit.
-#  if defined(__has_feature)  /* Clang */
-    // Clang debug builds use a lot of stack space
-#    define Py_C_RECURSION_LIMIT (Py_C_STACK_SIZE / 4000)
-#  else
-#    define Py_C_RECURSION_LIMIT (Py_C_STACK_SIZE / 2000)
-#  endif
-#elif defined(_Py_ADDRESS_SANITIZER)
-#  define Py_C_RECURSION_LIMIT (Py_C_STACK_SIZE / 600)
-#else
-#  define Py_C_RECURSION_LIMIT (Py_C_STACK_SIZE / 300)
-#endif
-
-
 /* other API */
 
 /* Similar to PyThreadState_Get(), but don't issue a fatal error
@@ -259,7 +218,6 @@ _PyThreadState_UncheckedGet(void)
     return PyThreadState_GetUnchecked();
 }
 
-
 // Disable tracing and profiling.
 PyAPI_FUNC(void) PyThreadState_EnterTracing(PyThreadState *tstate);
 

Modules/_testcapimodule.c

@@ -3213,7 +3213,6 @@ PyInit__testcapi(void)
     PyModule_AddObject(m, "instancemethod", (PyObject *)&PyInstanceMethod_Type);
 
     PyModule_AddIntConstant(m, "the_number_three", 3);
-    PyModule_AddIntMacro(m, Py_C_RECURSION_LIMIT);
     PyModule_AddObject(m, "INT32_MIN", PyLong_FromInt32(INT32_MIN));
     PyModule_AddObject(m, "INT32_MAX", PyLong_FromInt32(INT32_MAX));
     PyModule_AddObject(m, "UINT32_MAX", PyLong_FromUInt32(UINT32_MAX));

Python/ceval.c

@@ -318,11 +318,33 @@ _Py_EnterRecursiveCallUnchecked(PyThreadState *tstate)
     }
 }
 
+#if defined(__s390x__)
+#  define Py_C_STACK_SIZE 320000
+#elif defined(_WIN32) && defined(_M_ARM64)
+#  define Py_C_STACK_SIZE 400000
+#elif defined(_WIN32)
+#  define Py_C_STACK_SIZE 1200000
+#elif defined(__ANDROID__)
+#  define Py_C_STACK_SIZE 1200000
+#elif defined(__sparc__)
+   // test_descr crashed on sparc64 with >7000 but let's keep a margin of error.
+#  define Py_C_STACK_SIZE 1600000
+#elif defined(__wasi__)
+   // Based on wasmtime 16.
+#  define Py_C_STACK_SIZE 2000000
+#elif defined(__hppa__) || defined(__powerpc64__)
+   // test_descr crashed with >8000 but let's keep a margin of error.
+#  define Py_C_STACK_SIZE 2000000
+#else
+#  define Py_C_STACK_SIZE 5000000
+#endif
+
 void
 _Py_InitializeRecursionCheck(PyThreadState *tstate)
 {
     char here;
     uintptr_t here_addr = (uintptr_t)&here;
+    tstate->c_stack_top = here_addr;
 #ifdef USE_STACKCHECK
     if (_PyOS_CheckStack(PYOS_STACK_MARGIN * 2) == 0) {
         tstate->c_stack_soft_limit = here_addr - PYOS_STACK_MARGIN_BYTES;
@@ -364,15 +386,20 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where)
     assert(tstate->c_stack_hard_limit != 0);
     if (here_addr < tstate->c_stack_hard_limit) {
         /* Overflowing while handling an overflow. Give up. */
-        Py_FatalError("Cannot recover from stack overflow.");
+        int kbytes_used = (tstate->c_stack_top - here_addr)/1024;
+        char buffer[80];
+        snprintf(buffer, 80, "Unrecoverable stack overflow (used %dkb)%s", kbytes_used, where);
+        Py_FatalError(buffer);
     }
     if (tstate->recursion_headroom) {
         return 0;
     }
     else {
+        int kbytes_used = (tstate->c_stack_top - here_addr)/1024;
         tstate->recursion_headroom++;
         _PyErr_Format(tstate, PyExc_RecursionError,
-                    "maximum recursion depth exceeded%s",
+                    "Stack overflow (used %dkb)S%s",
+                    kbytes_used,
                     where);
         tstate->recursion_headroom--;
         return -1;