From dbf2efcabbb9d519f44647a5c074c17688e64209 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 31 Dec 2023 23:05:35 +0000 Subject: [PATCH 1/6] Use PyOS_CheckStack on those platforms that support it (currently just Windows) --- Include/cpython/pystate.h | 4 +++- Include/internal/pycore_ceval.h | 9 -------- Include/pythonrun.h | 22 ++++++++++++++++--- Lib/test/support/__init__.py | 5 ++++- ...-12-31-23-14-13.gh-issue-113655.1IIsob.rst | 2 ++ Python/ceval.c | 4 ++++ Python/pythonrun.c | 2 +- 7 files changed, 33 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-12-31-23-14-13.gh-issue-113655.1IIsob.rst diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index ed7dd829d4b6f0..90eaa0a2af43fa 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -225,9 +225,11 @@ struct _ts { # define Py_C_RECURSION_LIMIT 500 #elif defined(__s390x__) # define Py_C_RECURSION_LIMIT 1200 +#elif defined(USE_STACKCHECK) +# define Py_C_RECURSION_LIMIT 4000 #else // This value is duplicated in Lib/test/support/__init__.py -# define Py_C_RECURSION_LIMIT 8000 +# define Py_C_RECURSION_LIMIT 10000 #endif diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a357bfa3a26064..70ce2af4464fef 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -135,18 +135,9 @@ extern void _PyEval_DeactivateOpCache(void); /* --- _Py_EnterRecursiveCall() ----------------------------------------- */ -#ifdef USE_STACKCHECK -/* With USE_STACKCHECK macro defined, trigger stack checks in - _Py_CheckRecursiveCall() on every 64th call to _Py_EnterRecursiveCall. */ -static inline int _Py_MakeRecCheck(PyThreadState *tstate) { - return (tstate->c_recursion_remaining-- <= 0 - || (tstate->c_recursion_remaining & 63) == 0); -} -#else static inline int _Py_MakeRecCheck(PyThreadState *tstate) { return tstate->c_recursion_remaining-- <= 0; } -#endif // Export for '_json' shared extension, used via _Py_EnterRecursiveCall() // static inline function. diff --git a/Include/pythonrun.h b/Include/pythonrun.h index 154c7450cb934f..685480eedc91cd 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -17,16 +17,32 @@ PyAPI_FUNC(void) PyErr_Display(PyObject *, PyObject *, PyObject *); PyAPI_FUNC(void) PyErr_DisplayException(PyObject *); #endif +/* C frame size approximations in "pointers". + * Note: These are for "typical" C frames, we expect PyEval_EvalDefault + * to use three times as much stack space. + * TO DO: We should determine the numbers more accurately at build time. + */ +#ifdef Py_DEBUG +/* Debug frames are larger, very much so for Clang -O0. */ +# ifdef __clang__ +# define Py_C_FRAME_SIZE 400 +# else +# define Py_C_FRAME_SIZE 100; +# endif +#else +# define Py_C_FRAME_SIZE 25; +#endif + /* Stuff with no proper home (yet) */ PyAPI_DATA(int) (*PyOS_InputHook)(void); /* Stack size, in "pointers" (so we get extra safety margins on 64-bit platforms). On a 32-bit platform, this translates - to an 8k margin. */ -#define PYOS_STACK_MARGIN 2048 + to an 40k margin. */ +#define PYOS_STACK_MARGIN 10000 -#if defined(WIN32) && !defined(MS_WIN64) && !defined(_M_ARM) && defined(_MSC_VER) && _MSC_VER >= 1300 +#if defined(WIN32) && !defined(_M_ARM) && defined(_MSC_VER) && _MSC_VER >= 1300 /* Enable stack checking under Microsoft C */ // When changing the platforms, ensure PyOS_CheckStack() docs are still correct #define USE_STACKCHECK diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index e5fb725a30b5b8..8344dd1849c61d 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2377,7 +2377,10 @@ def _get_c_recursion_limit(): return _testcapi.Py_C_RECURSION_LIMIT except (ImportError, AttributeError): # Originally taken from Include/cpython/pystate.h . - return 8000 + if sys.platform == 'win32': + return 4000 + else: + return 10000 # The default C recursion limit. Py_C_RECURSION_LIMIT = _get_c_recursion_limit() diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-31-23-14-13.gh-issue-113655.1IIsob.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-31-23-14-13.gh-issue-113655.1IIsob.rst new file mode 100644 index 00000000000000..8a0c1ad9e889cd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-31-23-14-13.gh-issue-113655.1IIsob.rst @@ -0,0 +1,2 @@ +Use the PyOS_CheckStack to dynamically check for C stack usage on platforms +that support it (just Windows for now). diff --git a/Python/ceval.c b/Python/ceval.c index 1fea9747488102..4a67b342e60b7d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -285,6 +285,10 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) _PyErr_SetString(tstate, PyExc_MemoryError, "Stack overflow"); return -1; } + else { + tstate->c_recursion_remaining += (int)(PYOS_STACK_MARGIN/Py_C_FRAME_SIZE); + return 0; + } #endif if (tstate->recursion_headroom) { if (tstate->c_recursion_remaining < -50) { diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 5f305aa00e08b9..ed3f7fcd7e0e64 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1525,7 +1525,7 @@ _Py_SourceAsString(PyObject *cmd, const char *funcname, const char *what, PyComp #include /* - * Return non-zero when we run out of memory on the stack; zero otherwise. + * Return non-zero when we are low on memory on the stack; zero otherwise. */ int PyOS_CheckStack(void) From 90a3a535d4049bedbe7d53c4202b2cc5cbcc5763 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 31 Dec 2023 23:44:06 +0000 Subject: [PATCH 2/6] Call PyOS_CheckStack only when recursion counter hits zero --- Python/ceval.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 4a67b342e60b7d..1713905ea2d41e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -279,17 +279,6 @@ Py_SetRecursionLimit(int new_limit) int _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) { -#ifdef USE_STACKCHECK - if (PyOS_CheckStack()) { - ++tstate->c_recursion_remaining; - _PyErr_SetString(tstate, PyExc_MemoryError, "Stack overflow"); - return -1; - } - else { - tstate->c_recursion_remaining += (int)(PYOS_STACK_MARGIN/Py_C_FRAME_SIZE); - return 0; - } -#endif if (tstate->recursion_headroom) { if (tstate->c_recursion_remaining < -50) { /* Overflowing while handling an overflow. Give up. */ @@ -298,6 +287,12 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) } else { if (tstate->c_recursion_remaining <= 0) { +#ifdef USE_STACKCHECK + if (PyOS_CheckStack() == 0) { + tstate->c_recursion_remaining += (int)(PYOS_STACK_MARGIN/Py_C_FRAME_SIZE); + return 0; + } +#endif tstate->recursion_headroom++; _PyErr_Format(tstate, PyExc_RecursionError, "maximum recursion depth exceeded%s", From 6e39238385c957ee7ea3f4152692ae9f0d66f6ee Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 31 Dec 2023 23:48:28 +0000 Subject: [PATCH 3/6] Remove spurious semicolons --- Include/pythonrun.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/pythonrun.h b/Include/pythonrun.h index 685480eedc91cd..5d585a6c7c9541 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -27,10 +27,10 @@ PyAPI_FUNC(void) PyErr_DisplayException(PyObject *); # ifdef __clang__ # define Py_C_FRAME_SIZE 400 # else -# define Py_C_FRAME_SIZE 100; +# define Py_C_FRAME_SIZE 100 # endif #else -# define Py_C_FRAME_SIZE 25; +# define Py_C_FRAME_SIZE 25 #endif From 6cff5048b0a80e8b9b9273547ef5f729f84897c6 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 1 Jan 2024 01:28:28 +0000 Subject: [PATCH 4/6] Adjust test to force overflow --- Lib/test/test_dictviews.py | 2 +- Lib/test/test_exception_group.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index cad568b6ac4c2d..bbb0593dea2510 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -279,7 +279,7 @@ def test_recursive_repr(self): def test_deeply_nested_repr(self): d = {} - for i in range(Py_C_RECURSION_LIMIT//2 + 100): + for i in range(20_000): d = {42: d.values()} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 20122679223843..99092d9a101226 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -460,7 +460,7 @@ def test_basics_split_by_predicate__match(self): class DeepRecursionInSplitAndSubgroup(unittest.TestCase): def make_deep_eg(self): e = TypeError(1) - for i in range(Py_C_RECURSION_LIMIT + 1): + for i in range(20_000): e = ExceptionGroup('eg', [e]) return e From 03b6964048032840456d5ff83fab7cf0850ebb31 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 1 Jan 2024 01:35:37 +0000 Subject: [PATCH 5/6] Adjust some stack and frame sizes --- Include/cpython/pystate.h | 12 ++++++++++++ Include/pythonrun.h | 6 +++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 90eaa0a2af43fa..b6deff69800e8b 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -214,6 +214,12 @@ struct _ts { }; +#if defined(__has_feature) /* Clang */ +# if __has_feature(address_sanitizer) /* is ASAN enabled? */ +# define HAS_ASAN 1 +# endif +#endif + #ifdef Py_DEBUG // A debug build is likely built with low optimization level which implies // higher stack memory usage than a release build: use a lower limit. @@ -227,11 +233,17 @@ struct _ts { # define Py_C_RECURSION_LIMIT 1200 #elif defined(USE_STACKCHECK) # define Py_C_RECURSION_LIMIT 4000 +#elif defined (HAS_ASAN) +# define Py_C_RECURSION_LIMIT 7000 #else // This value is duplicated in Lib/test/support/__init__.py # define Py_C_RECURSION_LIMIT 10000 #endif +#ifdef HAS_ASAN +# undef HAS_ASAN +#endif + /* other API */ diff --git a/Include/pythonrun.h b/Include/pythonrun.h index 5d585a6c7c9541..34b0cfbc7b5a1b 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -25,12 +25,12 @@ PyAPI_FUNC(void) PyErr_DisplayException(PyObject *); #ifdef Py_DEBUG /* Debug frames are larger, very much so for Clang -O0. */ # ifdef __clang__ -# define Py_C_FRAME_SIZE 400 +# define Py_C_FRAME_SIZE 500 # else -# define Py_C_FRAME_SIZE 100 +# define Py_C_FRAME_SIZE 150 # endif #else -# define Py_C_FRAME_SIZE 25 +# define Py_C_FRAME_SIZE 50 #endif From acaee079d58d73e77dab72765bc9742e0e97dc37 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 1 Jan 2024 12:25:33 +0000 Subject: [PATCH 6/6] Reduce base C recursion limit on Windows debug builds a bit --- Include/cpython/pystate.h | 7 ++++++- Include/pythonrun.h | 2 +- Lib/test/test_dictviews.py | 2 +- Lib/test/test_exception_group.py | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index b6deff69800e8b..746df95ccbe97c 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -223,7 +223,12 @@ struct _ts { #ifdef Py_DEBUG // A debug build is likely built with low optimization level which implies // higher stack memory usage than a release build: use a lower limit. -# define Py_C_RECURSION_LIMIT 500 + +# if defined(USE_STACKCHECK) +# define Py_C_RECURSION_LIMIT 300 +# else +# define Py_C_RECURSION_LIMIT 500 +# endif #elif defined(__wasi__) // WASI has limited call stack. Python's recursion limit depends on code // layout, optimization, and WASI runtime. Wasmtime can handle about 700 diff --git a/Include/pythonrun.h b/Include/pythonrun.h index 34b0cfbc7b5a1b..f43e37b49def5d 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -27,7 +27,7 @@ PyAPI_FUNC(void) PyErr_DisplayException(PyObject *); # ifdef __clang__ # define Py_C_FRAME_SIZE 500 # else -# define Py_C_FRAME_SIZE 150 +# define Py_C_FRAME_SIZE 200 # endif #else # define Py_C_FRAME_SIZE 50 diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index bbb0593dea2510..2f7d64a93a21ed 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -279,7 +279,7 @@ def test_recursive_repr(self): def test_deeply_nested_repr(self): d = {} - for i in range(20_000): + for i in range(50_000): d = {42: d.values()} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 99092d9a101226..20e5b806c224a7 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -460,7 +460,7 @@ def test_basics_split_by_predicate__match(self): class DeepRecursionInSplitAndSubgroup(unittest.TestCase): def make_deep_eg(self): e = TypeError(1) - for i in range(20_000): + for i in range(50_000): e = ExceptionGroup('eg', [e]) return e