Skip to content

GH-113655: Use PyOS_CheckStack on those platforms that support it #113701

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 6 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 21 additions & 2 deletions Include/cpython/pystate.h
Original file line number Diff line number Diff line change
Expand Up @@ -214,20 +214,39 @@ struct _ts {

};

#if defined(__has_feature) /* Clang */
# if __has_feature(address_sanitizer) /* is ASAN enabled? */
# define HAS_ASAN 1
# endif
#endif

#ifdef Py_DEBUG
// A debug build is likely built with low optimization level which implies
// higher stack memory usage than a release build: use a lower limit.
# define Py_C_RECURSION_LIMIT 500

# if defined(USE_STACKCHECK)
# define Py_C_RECURSION_LIMIT 300
# else
# define Py_C_RECURSION_LIMIT 500
# endif
#elif defined(__wasi__)
// WASI has limited call stack. Python's recursion limit depends on code
// layout, optimization, and WASI runtime. Wasmtime can handle about 700
// recursions, sometimes less. 500 is a more conservative limit.
# define Py_C_RECURSION_LIMIT 500
#elif defined(__s390x__)
# define Py_C_RECURSION_LIMIT 1200
#elif defined(USE_STACKCHECK)
# define Py_C_RECURSION_LIMIT 4000
#elif defined (HAS_ASAN)
# define Py_C_RECURSION_LIMIT 7000
#else
// This value is duplicated in Lib/test/support/__init__.py
# define Py_C_RECURSION_LIMIT 8000
# define Py_C_RECURSION_LIMIT 10000
#endif

#ifdef HAS_ASAN
# undef HAS_ASAN
#endif


Expand Down
9 changes: 0 additions & 9 deletions Include/internal/pycore_ceval.h
Original file line number Diff line number Diff line change
Expand Up @@ -135,18 +135,9 @@ extern void _PyEval_DeactivateOpCache(void);

/* --- _Py_EnterRecursiveCall() ----------------------------------------- */

#ifdef USE_STACKCHECK
/* With USE_STACKCHECK macro defined, trigger stack checks in
_Py_CheckRecursiveCall() on every 64th call to _Py_EnterRecursiveCall. */
static inline int _Py_MakeRecCheck(PyThreadState *tstate) {
return (tstate->c_recursion_remaining-- <= 0
|| (tstate->c_recursion_remaining & 63) == 0);
}
#else
static inline int _Py_MakeRecCheck(PyThreadState *tstate) {
return tstate->c_recursion_remaining-- <= 0;
}
#endif

// Export for '_json' shared extension, used via _Py_EnterRecursiveCall()
// static inline function.
Expand Down
22 changes: 19 additions & 3 deletions Include/pythonrun.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,32 @@ PyAPI_FUNC(void) PyErr_Display(PyObject *, PyObject *, PyObject *);
PyAPI_FUNC(void) PyErr_DisplayException(PyObject *);
#endif

/* C frame size approximations in "pointers".
* Note: These are for "typical" C frames, we expect PyEval_EvalDefault
* to use three times as much stack space.
* TO DO: We should determine the numbers more accurately at build time.
*/
#ifdef Py_DEBUG
/* Debug frames are larger, very much so for Clang -O0. */
# ifdef __clang__
# define Py_C_FRAME_SIZE 500
# else
# define Py_C_FRAME_SIZE 200
# endif
#else
# define Py_C_FRAME_SIZE 50
#endif


/* Stuff with no proper home (yet) */
PyAPI_DATA(int) (*PyOS_InputHook)(void);

/* Stack size, in "pointers" (so we get extra safety margins
on 64-bit platforms). On a 32-bit platform, this translates
to an 8k margin. */
#define PYOS_STACK_MARGIN 2048
to an 40k margin. */
#define PYOS_STACK_MARGIN 10000

#if defined(WIN32) && !defined(MS_WIN64) && !defined(_M_ARM) && defined(_MSC_VER) && _MSC_VER >= 1300
#if defined(WIN32) && !defined(_M_ARM) && defined(_MSC_VER) && _MSC_VER >= 1300
/* Enable stack checking under Microsoft C */
// When changing the platforms, ensure PyOS_CheckStack() docs are still correct
#define USE_STACKCHECK
Expand Down
5 changes: 4 additions & 1 deletion Lib/test/support/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2377,7 +2377,10 @@ def _get_c_recursion_limit():
return _testcapi.Py_C_RECURSION_LIMIT
except (ImportError, AttributeError):
# Originally taken from Include/cpython/pystate.h .
return 8000
if sys.platform == 'win32':
return 4000
else:
return 10000

# The default C recursion limit.
Py_C_RECURSION_LIMIT = _get_c_recursion_limit()
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/test_dictviews.py
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ def test_recursive_repr(self):

def test_deeply_nested_repr(self):
d = {}
for i in range(Py_C_RECURSION_LIMIT//2 + 100):
for i in range(50_000):
d = {42: d.values()}
self.assertRaises(RecursionError, repr, d)

Expand Down
2 changes: 1 addition & 1 deletion Lib/test/test_exception_group.py
Original file line number Diff line number Diff line change
Expand Up @@ -460,7 +460,7 @@ def test_basics_split_by_predicate__match(self):
class DeepRecursionInSplitAndSubgroup(unittest.TestCase):
def make_deep_eg(self):
e = TypeError(1)
for i in range(Py_C_RECURSION_LIMIT + 1):
for i in range(50_000):
e = ExceptionGroup('eg', [e])
return e

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Use the PyOS_CheckStack to dynamically check for C stack usage on platforms
that support it (just Windows for now).
13 changes: 6 additions & 7 deletions Python/ceval.c
Original file line number Diff line number Diff line change
Expand Up @@ -279,13 +279,6 @@ Py_SetRecursionLimit(int new_limit)
int
_Py_CheckRecursiveCall(PyThreadState *tstate, const char *where)
{
#ifdef USE_STACKCHECK
if (PyOS_CheckStack()) {
++tstate->c_recursion_remaining;
_PyErr_SetString(tstate, PyExc_MemoryError, "Stack overflow");
return -1;
}
#endif
if (tstate->recursion_headroom) {
if (tstate->c_recursion_remaining < -50) {
/* Overflowing while handling an overflow. Give up. */
Expand All @@ -294,6 +287,12 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where)
}
else {
if (tstate->c_recursion_remaining <= 0) {
#ifdef USE_STACKCHECK
if (PyOS_CheckStack() == 0) {
tstate->c_recursion_remaining += (int)(PYOS_STACK_MARGIN/Py_C_FRAME_SIZE);
return 0;
}
#endif
tstate->recursion_headroom++;
_PyErr_Format(tstate, PyExc_RecursionError,
"maximum recursion depth exceeded%s",
Expand Down
2 changes: 1 addition & 1 deletion Python/pythonrun.c
Original file line number Diff line number Diff line change
Expand Up @@ -1525,7 +1525,7 @@ _Py_SourceAsString(PyObject *cmd, const char *funcname, const char *what, PyComp
#include <excpt.h>

/*
* Return non-zero when we run out of memory on the stack; zero otherwise.
* Return non-zero when we are low on memory on the stack; zero otherwise.
*/
int
PyOS_CheckStack(void)
Expand Down