diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 628f2e0996e469..95fad893786d8d 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -251,12 +251,24 @@ struct _ts { /* WASI has limited call stack. Python's recursion limit depends on code layout, optimization, and WASI runtime. Wasmtime can handle about 700 recursions, sometimes less. 500 is a more conservative limit. */ -#ifndef C_RECURSION_LIMIT -# ifdef __wasi__ +#ifdef Py_DEBUG +# if defined(__wasi__) +# define C_RECURSION_LIMIT 150 +# else +# define C_RECURSION_LIMIT 500 +# endif +#else +# if defined(__wasi__) # define C_RECURSION_LIMIT 500 +# elif defined(__s390x__) +# define C_RECURSION_LIMIT 800 +# elif defined(_WIN32) +# define C_RECURSION_LIMIT 3000 +# elif defined(_Py_ADDRESS_SANITIZER) +# define C_RECURSION_LIMIT 4000 # else - // This value is duplicated in Lib/test/support/__init__.py -# define C_RECURSION_LIMIT 1500 + // This value is duplicated in Lib/test/support/__init__.py +# define C_RECURSION_LIMIT 10000 # endif #endif diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 8c4b4e023f633f..cb5a84aa74e05f 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2112,13 +2112,13 @@ def set_recursion_limit(limit): finally: sys.setrecursionlimit(original_limit) -def infinite_recursion(max_depth=100): - """Set a lower limit for tests that interact with infinite recursions - (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some - debug windows builds, due to not enough functions being inlined the - stack size might not handle the default recursion limit (1000). See - bpo-11105 for details.""" - if max_depth < 3: +def infinite_recursion(max_depth=None): + if max_depth is None: + # Pick a number large enough to cause problems + # but not take too long for code that can handle + # very deep recursion. + max_depth = 20_000 + elif max_depth < 3: raise ValueError("max_depth must be at least 3, got {max_depth}") depth = get_recursion_depth() depth = max(depth - 1, 1) # Ignore infinite_recursion() frame. @@ -2362,7 +2362,22 @@ def adjust_int_max_str_digits(max_digits): EXCEEDS_RECURSION_LIMIT = 5000 # The default C recursion limit (from Include/cpython/pystate.h). -C_RECURSION_LIMIT = 1500 +if Py_DEBUG: + if is_wasi: + C_RECURSION_LIMIT = 150 + else: + C_RECURSION_LIMIT = 500 +else: + if is_wasi: + C_RECURSION_LIMIT = 500 + elif hasattr(os, 'uname') and os.uname().machine == 's390x': + C_RECURSION_LIMIT = 800 + elif sys.platform.startswith('win'): + C_RECURSION_LIMIT = 3000 + elif check_sanitizer(address=True): + C_RECURSION_LIMIT = 4000 + else: + C_RECURSION_LIMIT = 10000 #Windows doesn't have os.uname() but it doesn't support s390x. skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 3ba7cf7b04266f..9736208a92fc6e 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1087,9 +1087,9 @@ def next(self): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") @support.cpython_only def test_ast_recursion_limit(self): - fail_depth = support.EXCEEDS_RECURSION_LIMIT + fail_depth = support.C_RECURSION_LIMIT + 1 crash_depth = 100_000 - success_depth = 1200 + success_depth = int(support.C_RECURSION_LIMIT * 0.9) def check_limit(prefix, repeated): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index ec8dc29d36c16a..46abf40605f491 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,5 +1,5 @@ import unittest -from test.support import cpython_only, requires_limited_api, skip_on_s390x +from test.support import cpython_only, requires_limited_api, skip_on_s390x, is_wasi, Py_DEBUG try: import _testcapi except ImportError: @@ -932,6 +932,7 @@ def test_multiple_values(self): class TestRecursion(unittest.TestCase): @skip_on_s390x + @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") def test_super_deep(self): def recurse(n): diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 42df670fe00e0a..ed23d31c0d3ecd 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -607,9 +607,9 @@ def test_compiler_recursion_limit(self): # Expected limit is C_RECURSION_LIMIT * 2 # Duplicating the limit here is a little ugly. # Perhaps it should be exposed somewhere... - fail_depth = C_RECURSION_LIMIT * 2 + 1 + fail_depth = C_RECURSION_LIMIT + 1 crash_depth = C_RECURSION_LIMIT * 100 - success_depth = int(C_RECURSION_LIMIT * 1.8) + success_depth = int(C_RECURSION_LIMIT * 0.9) def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index bf9332e40aeaf2..b3e317bd7990fd 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -352,7 +352,7 @@ def blowstack(fxn, arg, compare_to): # Make sure that calling isinstance with a deeply nested tuple for its # argument will raise RecursionError eventually. tuple_arg = (compare_to,) - for cnt in range(support.EXCEEDS_RECURSION_LIMIT): + for cnt in range(support.C_RECURSION_LIMIT * 2): tuple_arg = (tuple_arg,) fxn(arg, tuple_arg) diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index 3f10f16d71996d..fa46050658afe0 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -908,7 +908,7 @@ def test_cycles(self): self.assertIs(b['x'], b) def test_deep_nesting(self): - tests = [50, 100_000] if support.is_wasi else [50, 300, 100_000] + tests = [50, 100_000] if support.is_wasi else [50, 600, 100_000] for N in tests: chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)] try: diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 7e16e94aa110b2..196fd60d1973f6 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2965,16 +2965,18 @@ def test_trace_unpack_long_sequence(self): self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1}) def test_trace_lots_of_globals(self): + count = min(1000, int(support.C_RECURSION_LIMIT * 0.8)) + code = """if 1: def f(): return ( {} ) - """.format("\n+\n".join(f"var{i}\n" for i in range(1000))) - ns = {f"var{i}": i for i in range(1000)} + """.format("\n+\n".join(f"var{i}\n" for i in range(count))) + ns = {f"var{i}": i for i in range(count)} exec(code, ns) counts = self.count_traces(ns["f"]) - self.assertEqual(counts, {'call': 1, 'line': 2000, 'return': 1}) + self.assertEqual(counts, {'call': 1, 'line': count * 2, 'return': 1}) class TestEdgeCases(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-04-01-26-20.gh-issue-112215.NyRPXM.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-04-01-26-20.gh-issue-112215.NyRPXM.rst new file mode 100644 index 00000000000000..4f6c356e414928 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-04-01-26-20.gh-issue-112215.NyRPXM.rst @@ -0,0 +1,2 @@ +Change the C recursion limits to more closely reflect the underlying +platform limits. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index d42c26396d5de2..2c34f5c1bc2675 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1393,15 +1393,14 @@ class PartingShots(StaticVisitor): int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return NULL; } struct validator vstate; - vstate.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + vstate.recursion_limit = C_RECURSION_LIMIT; int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; vstate.recursion_depth = starting_recursion_depth; PyObject *result = ast2obj_mod(state, &vstate, t); diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 6c95f07c386fd0..ecaff2041f9565 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -13152,15 +13152,14 @@ PyObject* PyAST_mod2obj(mod_ty t) int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return NULL; } struct validator vstate; - vstate.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + vstate.recursion_limit = C_RECURSION_LIMIT; int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; vstate.recursion_depth = starting_recursion_depth; PyObject *result = ast2obj_mod(state, &vstate, t); diff --git a/Python/ast.c b/Python/ast.c index 82d7beec0ee510..76f6556ded098b 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1038,9 +1038,6 @@ validate_type_params(struct validator *state, asdl_type_param_seq *tps) } -/* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 2 - int _PyAST_Validate(mod_ty mod) { @@ -1057,9 +1054,9 @@ _PyAST_Validate(mod_ty mod) } /* Be careful here to prevent overflow. */ int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; state.recursion_depth = starting_recursion_depth; - state.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + state.recursion_limit = C_RECURSION_LIMIT; switch (mod->kind) { case Module_kind: diff --git a/Python/ast_opt.c b/Python/ast_opt.c index f8c4a9513236b9..e881b7fe2d4413 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -1102,9 +1102,6 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat #undef CALL_OPT #undef CALL_SEQ -/* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 2 - int _PyAST_Optimize(mod_ty mod, PyArena *arena, _PyASTOptimizeState *state) { @@ -1118,9 +1115,9 @@ _PyAST_Optimize(mod_ty mod, PyArena *arena, _PyASTOptimizeState *state) } /* Be careful here to prevent overflow. */ int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; state->recursion_depth = starting_recursion_depth; - state->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + state->recursion_limit = C_RECURSION_LIMIT; int ret = astfold_mod(mod, arena, state); assert(ret || PyErr_Occurred()); diff --git a/Python/symtable.c b/Python/symtable.c index a5c6b465b71ddd..4cf6ad3cf5ca6f 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -281,11 +281,6 @@ symtable_new(void) return NULL; } -/* Using a scaling factor means this should automatically adjust when - the recursion limit is adjusted for small or large C stack allocations. -*/ -#define COMPILER_STACK_FRAME_SCALE 2 - struct symtable * _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) { @@ -312,9 +307,9 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) } /* Be careful here to prevent overflow. */ int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; st->recursion_depth = starting_recursion_depth; - st->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + st->recursion_limit = C_RECURSION_LIMIT; /* Make the initial symbol information gathering pass */ if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) {