From 1079703aa3c99d216b7f32b3742312e7526fedd4 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Jan 2024 14:09:35 +0000 Subject: [PATCH 01/14] Set recursion limit lower for Windows --- Include/cpython/pystate.h | 4 +++- Lib/test/support/__init__.py | 5 ++++- Lib/test/test_functools.py | 10 +++++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index ed7dd829d4b6f0..f0aa08478ff99e 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -225,9 +225,11 @@ struct _ts { # define Py_C_RECURSION_LIMIT 500 #elif defined(__s390x__) # define Py_C_RECURSION_LIMIT 1200 +#elif defined(_WIN32) +# define Py_C_RECURSION_LIMIT 4000 #else // This value is duplicated in Lib/test/support/__init__.py -# define Py_C_RECURSION_LIMIT 8000 +# define Py_C_RECURSION_LIMIT 10000 #endif diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index e5fb725a30b5b8..8344dd1849c61d 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2377,7 +2377,10 @@ def _get_c_recursion_limit(): return _testcapi.Py_C_RECURSION_LIMIT except (ImportError, AttributeError): # Originally taken from Include/cpython/pystate.h . - return 8000 + if sys.platform == 'win32': + return 4000 + else: + return 10000 # The default C recursion limit. Py_C_RECURSION_LIMIT = _get_c_recursion_limit() diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 0ef45d3c670e85..32235e7c463555 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1876,7 +1876,15 @@ def fib(n): if not support.Py_DEBUG: with support.infinite_recursion(): - fib(2500) + if sys.platform == 'win32': + fib(1200) + else: + fib(3000) + if self.module == c_functools: + fib.cache_clear() + with support.infinite_recursion(): + with self.assertRaises(RecursionError): + fib(10000) @py_functools.lru_cache() From 48ff1160247d41373b4d9bab5d2a7066e497fe16 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Jan 2024 01:28:35 +0000 Subject: [PATCH 02/14] Add news --- .../2024-01-11-01-28-25.gh-issue-113655.Mfioxp.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-01-11-01-28-25.gh-issue-113655.Mfioxp.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-11-01-28-25.gh-issue-113655.Mfioxp.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-11-01-28-25.gh-issue-113655.Mfioxp.rst new file mode 100644 index 00000000000000..30f1b0a76612db --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-11-01-28-25.gh-issue-113655.Mfioxp.rst @@ -0,0 +1,3 @@ +Set the C recursion limit to 4000 on Windows, and 10000 on Linux/OSX. This +seems to be near the sweet spot to maintain safety, but not compromise +backwards compatibility. From b797d8cdef56807c4d56107d30fd99b31b870c25 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Jan 2024 01:50:51 +0000 Subject: [PATCH 03/14] Lower recursion limit for address sanitizer --- Include/cpython/pystate.h | 41 ++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index f0aa08478ff99e..f051401a2fcb97 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -214,22 +214,31 @@ struct _ts { }; -#ifdef Py_DEBUG - // A debug build is likely built with low optimization level which implies - // higher stack memory usage than a release build: use a lower limit. -# define Py_C_RECURSION_LIMIT 500 -#elif defined(__wasi__) - // WASI has limited call stack. Python's recursion limit depends on code - // layout, optimization, and WASI runtime. Wasmtime can handle about 700 - // recursions, sometimes less. 500 is a more conservative limit. -# define Py_C_RECURSION_LIMIT 500 -#elif defined(__s390x__) -# define Py_C_RECURSION_LIMIT 1200 -#elif defined(_WIN32) -# define Py_C_RECURSION_LIMIT 4000 -#else - // This value is duplicated in Lib/test/support/__init__.py -# define Py_C_RECURSION_LIMIT 10000 + +#if defined(__has_feature) /* Clang */ +# if __has_feature(address_sanitizer) /* is ASAN enabled? */ +# define Py_C_RECURSION_LIMIT 7000 +# endif +#endif + +#ifndef Py_C_RECURSION_LIMIT +# ifdef Py_DEBUG + // A debug build is likely built with low optimization level which implies + // higher stack memory usage than a release build: use a lower limit. +# define Py_C_RECURSION_LIMIT 500 +# elif defined(__wasi__) + // WASI has limited call stack. Python's recursion limit depends on code + // layout, optimization, and WASI runtime. Wasmtime can handle about 700 + // recursions, sometimes less. 500 is a more conservative limit. +# define Py_C_RECURSION_LIMIT 500 +# elif defined(__s390x__) +# define Py_C_RECURSION_LIMIT 1200 +# elif defined(_WIN32) +# define Py_C_RECURSION_LIMIT 4000 +# else + // This value is duplicated in Lib/test/support/__init__.py +# define Py_C_RECURSION_LIMIT 10000 +# endif #endif From 0f11e2143b8c3deb83ed08e29dfe3af18ff083ea Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Jan 2024 02:21:30 +0000 Subject: [PATCH 04/14] Further lower C recursion limit for address sanitizer. --- Include/cpython/pystate.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index f051401a2fcb97..453bd0bb2a79ed 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -217,7 +217,7 @@ struct _ts { #if defined(__has_feature) /* Clang */ # if __has_feature(address_sanitizer) /* is ASAN enabled? */ -# define Py_C_RECURSION_LIMIT 7000 +# define Py_C_RECURSION_LIMIT 5000 # endif #endif From ea1223866e7c36ca213e360af8fcf8fea98b6165 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Jan 2024 02:27:47 +0000 Subject: [PATCH 05/14] Account for GCC when testing for address sanitizer --- Include/cpython/pystate.h | 43 ++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 453bd0bb2a79ed..3662c6d2740a05 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -214,31 +214,24 @@ struct _ts { }; - -#if defined(__has_feature) /* Clang */ -# if __has_feature(address_sanitizer) /* is ASAN enabled? */ -# define Py_C_RECURSION_LIMIT 5000 -# endif -#endif - -#ifndef Py_C_RECURSION_LIMIT -# ifdef Py_DEBUG - // A debug build is likely built with low optimization level which implies - // higher stack memory usage than a release build: use a lower limit. -# define Py_C_RECURSION_LIMIT 500 -# elif defined(__wasi__) - // WASI has limited call stack. Python's recursion limit depends on code - // layout, optimization, and WASI runtime. Wasmtime can handle about 700 - // recursions, sometimes less. 500 is a more conservative limit. -# define Py_C_RECURSION_LIMIT 500 -# elif defined(__s390x__) -# define Py_C_RECURSION_LIMIT 1200 -# elif defined(_WIN32) -# define Py_C_RECURSION_LIMIT 4000 -# else - // This value is duplicated in Lib/test/support/__init__.py -# define Py_C_RECURSION_LIMIT 10000 -# endif +#ifdef Py_DEBUG + // A debug build is likely built with low optimization level which implies + // higher stack memory usage than a release build: use a lower limit. +# define Py_C_RECURSION_LIMIT 500 +#elif defined(__wasi__) + // WASI has limited call stack. Python's recursion limit depends on code + // layout, optimization, and WASI runtime. Wasmtime can handle about 700 + // recursions, sometimes less. 500 is a more conservative limit. +# define Py_C_RECURSION_LIMIT 500 +#elif defined(__s390x__) +# define Py_C_RECURSION_LIMIT 1200 +#elif defined(_WIN32) +# define Py_C_RECURSION_LIMIT 4000 +#elif defined(_Py_ADDRESS_SANITIZER) +# define Py_C_RECURSION_LIMIT 7000 +#else + // This value is duplicated in Lib/test/support/__init__.py +# define Py_C_RECURSION_LIMIT 10000 #endif From c5d0a504ef738f65af25954dc59fba873829284a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Jan 2024 05:47:42 +0000 Subject: [PATCH 06/14] Give address sanitizer build a bit more C recursion depth --- Include/cpython/pystate.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 3662c6d2740a05..b51b5c38c604d1 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -228,7 +228,7 @@ struct _ts { #elif defined(_WIN32) # define Py_C_RECURSION_LIMIT 4000 #elif defined(_Py_ADDRESS_SANITIZER) -# define Py_C_RECURSION_LIMIT 7000 +# define Py_C_RECURSION_LIMIT 8000 #else // This value is duplicated in Lib/test/support/__init__.py # define Py_C_RECURSION_LIMIT 10000 From eaadfd0b27f90113d468cd38097a04e24cd91122 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Jan 2024 06:18:16 +0000 Subject: [PATCH 07/14] Revert change to test --- Lib/test/test_functools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 32235e7c463555..0d4a57c62bfc6a 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1879,7 +1879,7 @@ def fib(n): if sys.platform == 'win32': fib(1200) else: - fib(3000) + fib(2500) if self.module == c_functools: fib.cache_clear() with support.infinite_recursion(): From 82acac8e8380cf008dfdce45d97aec3efb2494e8 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Jan 2024 07:51:20 +0000 Subject: [PATCH 08/14] Remove COMPILER_STACK_FRAME_SCALE and tweak recursion limits --- Include/cpython/pystate.h | 2 +- .../pycore_global_objects_fini_generated.h | 6 ++++++ Include/internal/pycore_global_strings.h | 6 ++++++ .../internal/pycore_runtime_init_generated.h | 6 ++++++ .../internal/pycore_unicodeobject_generated.h | 18 ++++++++++++++++++ Lib/test/test_compile.py | 8 +++----- Lib/test/test_functools.py | 6 ++---- Parser/asdl_c.py | 5 ++--- Python/Python-ast.c | 5 ++--- Python/ast.c | 8 ++------ Python/ast_opt.c | 7 ++----- Python/symtable.c | 9 ++------- 12 files changed, 52 insertions(+), 34 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index b51b5c38c604d1..2bd34416d6e952 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -228,7 +228,7 @@ struct _ts { #elif defined(_WIN32) # define Py_C_RECURSION_LIMIT 4000 #elif defined(_Py_ADDRESS_SANITIZER) -# define Py_C_RECURSION_LIMIT 8000 +# define Py_C_RECURSION_LIMIT 4000 #else // This value is duplicated in Lib/test/support/__init__.py # define Py_C_RECURSION_LIMIT 10000 diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 1fd67ceb3c05fb..9c69c7c967fb3e 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -787,8 +787,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_child)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aggregate_class)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argdefs)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(args)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arguments)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argv)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(as_integer_ratio)); @@ -912,6 +914,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(errors)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(event)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eventmask)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_type)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(excepthook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exception)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(existing_file_name)); @@ -1165,6 +1169,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seek)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seekable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(selectors)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(self)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(send)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sep)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sequence)); @@ -1227,6 +1232,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timetuple)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(top)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(trace_callback)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(traceback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(trailers)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(translate)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(true)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index da1f9b67bdfb6a..42220c93a01bfa 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -276,8 +276,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(after_in_child) STRUCT_FOR_ID(after_in_parent) STRUCT_FOR_ID(aggregate_class) + STRUCT_FOR_ID(alias) STRUCT_FOR_ID(append) STRUCT_FOR_ID(argdefs) + STRUCT_FOR_ID(args) STRUCT_FOR_ID(arguments) STRUCT_FOR_ID(argv) STRUCT_FOR_ID(as_integer_ratio) @@ -401,6 +403,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(errors) STRUCT_FOR_ID(event) STRUCT_FOR_ID(eventmask) + STRUCT_FOR_ID(exc_type) + STRUCT_FOR_ID(exc_value) STRUCT_FOR_ID(excepthook) STRUCT_FOR_ID(exception) STRUCT_FOR_ID(existing_file_name) @@ -654,6 +658,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(seek) STRUCT_FOR_ID(seekable) STRUCT_FOR_ID(selectors) + STRUCT_FOR_ID(self) STRUCT_FOR_ID(send) STRUCT_FOR_ID(sep) STRUCT_FOR_ID(sequence) @@ -716,6 +721,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(timetuple) STRUCT_FOR_ID(top) STRUCT_FOR_ID(trace_callback) + STRUCT_FOR_ID(traceback) STRUCT_FOR_ID(trailers) STRUCT_FOR_ID(translate) STRUCT_FOR_ID(true) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index e285d02b48a9f4..20dc11f2996e79 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -785,8 +785,10 @@ extern "C" { INIT_ID(after_in_child), \ INIT_ID(after_in_parent), \ INIT_ID(aggregate_class), \ + INIT_ID(alias), \ INIT_ID(append), \ INIT_ID(argdefs), \ + INIT_ID(args), \ INIT_ID(arguments), \ INIT_ID(argv), \ INIT_ID(as_integer_ratio), \ @@ -910,6 +912,8 @@ extern "C" { INIT_ID(errors), \ INIT_ID(event), \ INIT_ID(eventmask), \ + INIT_ID(exc_type), \ + INIT_ID(exc_value), \ INIT_ID(excepthook), \ INIT_ID(exception), \ INIT_ID(existing_file_name), \ @@ -1163,6 +1167,7 @@ extern "C" { INIT_ID(seek), \ INIT_ID(seekable), \ INIT_ID(selectors), \ + INIT_ID(self), \ INIT_ID(send), \ INIT_ID(sep), \ INIT_ID(sequence), \ @@ -1225,6 +1230,7 @@ extern "C" { INIT_ID(timetuple), \ INIT_ID(top), \ INIT_ID(trace_callback), \ + INIT_ID(traceback), \ INIT_ID(trailers), \ INIT_ID(translate), \ INIT_ID(true), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 2f9874029fb8f1..a82d7f480b766c 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -669,12 +669,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(aggregate_class); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(alias); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(append); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(argdefs); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(args); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(arguments); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1044,6 +1050,12 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(eventmask); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(exc_type); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(exc_value); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(excepthook); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1803,6 +1815,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(selectors); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(self); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(send); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1989,6 +2004,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(trace_callback); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(traceback); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(trailers); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 45fd30987760cb..2377caf7fbe15f 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -623,12 +623,10 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): - # Expected limit is Py_C_RECURSION_LIMIT * 2 - # Duplicating the limit here is a little ugly. - # Perhaps it should be exposed somewhere... - fail_depth = Py_C_RECURSION_LIMIT * 2 + 1 + # Expected limit is Py_C_RECURSION_LIMIT + fail_depth = Py_C_RECURSION_LIMIT + 1 crash_depth = Py_C_RECURSION_LIMIT * 100 - success_depth = int(Py_C_RECURSION_LIMIT * 1.8) + success_depth = int(Py_C_RECURSION_LIMIT * 0.8) def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 0d4a57c62bfc6a..7c66b906d308ba 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1875,11 +1875,9 @@ def fib(n): return fib(n-1) + fib(n-2) if not support.Py_DEBUG: + depth = support.Py_C_RECURSION_LIMIT*2//7 with support.infinite_recursion(): - if sys.platform == 'win32': - fib(1200) - else: - fib(2500) + fib(depth) if self.module == c_functools: fib.cache_clear() with support.infinite_recursion(): diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 4bb337349748cf..ce92672bf00776 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1388,15 +1388,14 @@ class PartingShots(StaticVisitor): int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return NULL; } struct validator vstate; - vstate.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + vstate.recursion_limit = Py_C_RECURSION_LIMIT; int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; vstate.recursion_depth = starting_recursion_depth; PyObject *result = ast2obj_mod(state, &vstate, t); diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 699e1c157c591c..d77e986ba067a3 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -13149,15 +13149,14 @@ PyObject* PyAST_mod2obj(mod_ty t) int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return NULL; } struct validator vstate; - vstate.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + vstate.recursion_limit = Py_C_RECURSION_LIMIT; int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; vstate.recursion_depth = starting_recursion_depth; PyObject *result = ast2obj_mod(state, &vstate, t); diff --git a/Python/ast.c b/Python/ast.c index 5f46d4149c2ed0..71b09d889f17c1 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1037,10 +1037,6 @@ validate_type_params(struct validator *state, asdl_type_param_seq *tps) return 1; } - -/* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 2 - int _PyAST_Validate(mod_ty mod) { @@ -1057,9 +1053,9 @@ _PyAST_Validate(mod_ty mod) } /* Be careful here to prevent overflow. */ int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; state.recursion_depth = starting_recursion_depth; - state.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + state.recursion_limit = Py_C_RECURSION_LIMIT; switch (mod->kind) { case Module_kind: diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 04d7ae6eaafbc0..41e906c66e8eec 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -1100,9 +1100,6 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat #undef CALL_OPT #undef CALL_SEQ -/* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 2 - int _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features) { @@ -1120,9 +1117,9 @@ _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features) } /* Be careful here to prevent overflow. */ int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; state.recursion_depth = starting_recursion_depth; - state.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + state.recursion_limit = Py_C_RECURSION_LIMIT; int ret = astfold_mod(mod, arena, &state); assert(ret || PyErr_Occurred()); diff --git a/Python/symtable.c b/Python/symtable.c index 83137b491f282c..743029956e32fa 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -386,11 +386,6 @@ symtable_new(void) return NULL; } -/* Using a scaling factor means this should automatically adjust when - the recursion limit is adjusted for small or large C stack allocations. -*/ -#define COMPILER_STACK_FRAME_SCALE 2 - struct symtable * _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) { @@ -417,9 +412,9 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) } /* Be careful here to prevent overflow. */ int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining; - starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE; + starting_recursion_depth = recursion_depth; st->recursion_depth = starting_recursion_depth; - st->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE; + st->recursion_limit = Py_C_RECURSION_LIMIT; /* Make the initial symbol information gathering pass */ if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) { From 3f3ec467225194de3dd36619301cc9aefc0bb3e9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Jan 2024 07:56:49 +0000 Subject: [PATCH 09/14] Lower C recursion limit for s390 back to 1000. --- Include/cpython/pystate.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 2bd34416d6e952..04730ad9736478 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -224,7 +224,7 @@ struct _ts { // recursions, sometimes less. 500 is a more conservative limit. # define Py_C_RECURSION_LIMIT 500 #elif defined(__s390x__) -# define Py_C_RECURSION_LIMIT 1200 +# define Py_C_RECURSION_LIMIT 1000 #elif defined(_WIN32) # define Py_C_RECURSION_LIMIT 4000 #elif defined(_Py_ADDRESS_SANITIZER) From cc149d86a38823da495805926cf2bef06f00ac9f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Jan 2024 08:38:28 +0000 Subject: [PATCH 10/14] Adjust a couple of tests --- Lib/test/test_ast.py | 2 +- Lib/test/test_sys_settrace.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 64fcb02309de77..3789ac22e3899c 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1126,7 +1126,7 @@ def next(self): def test_ast_recursion_limit(self): fail_depth = support.EXCEEDS_RECURSION_LIMIT crash_depth = 100_000 - success_depth = 1200 + success_depth = int(support.Py_C_RECURSION_LIMIT * 0.8) if _testinternalcapi is not None: remaining = _testinternalcapi.get_c_recursion_remaining() success_depth = min(success_depth, remaining) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index fc5ca72236b1fb..ae6e192a7ab6ef 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -3037,10 +3037,8 @@ def test_trace_unpack_long_sequence(self): self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1}) def test_trace_lots_of_globals(self): - count = 1000 - if _testinternalcapi is not None: - remaining = _testinternalcapi.get_c_recursion_remaining() - count = min(count, remaining) + + count = min(1000, int(support.Py_C_RECURSION_LIMIT * 0.8)) code = """if 1: def f(): From 4a22c7e5addc22d4fc803d94dd81a7352b8ef4a2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Jan 2024 10:19:17 +0000 Subject: [PATCH 11/14] Add a bit more info to the faulthandler traceback --- Python/traceback.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Python/traceback.c b/Python/traceback.c index abd429ac6c1f71..a2e745dc414840 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -965,7 +965,11 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) unsigned int depth = 0; while (1) { if (MAX_FRAME_DEPTH <= depth) { - PUTS(fd, " ...\n"); + if (MAX_FRAME_DEPTH < depth) { + PUTS(fd, "plus "); + _Py_DumpDecimal(fd, depth); + PUTS(fd, "frames\n"); + } break; } dump_frame(fd, frame); From f4a46b78f09859636be3220dd2789ac3cd019a0b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Jan 2024 10:45:59 +0000 Subject: [PATCH 12/14] Fix whitespace --- Python/traceback.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/traceback.c b/Python/traceback.c index a2e745dc414840..7a188e56c939c0 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -968,7 +968,7 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) if (MAX_FRAME_DEPTH < depth) { PUTS(fd, "plus "); _Py_DumpDecimal(fd, depth); - PUTS(fd, "frames\n"); + PUTS(fd, " frames\n"); } break; } From adae6cf25ab394e63a04a50c55ba1c0d27f03702 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 12 Jan 2024 13:52:52 +0000 Subject: [PATCH 13/14] Regenerate global string on Windows --- .../pycore_global_objects_fini_generated.h | 6 ------ Include/internal/pycore_global_strings.h | 6 ------ .../internal/pycore_runtime_init_generated.h | 6 ------ .../internal/pycore_unicodeobject_generated.h | 18 ------------------ 4 files changed, 36 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 9c69c7c967fb3e..1fd67ceb3c05fb 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -787,10 +787,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_child)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aggregate_class)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argdefs)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(args)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arguments)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argv)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(as_integer_ratio)); @@ -914,8 +912,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(errors)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(event)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eventmask)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_type)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(excepthook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exception)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(existing_file_name)); @@ -1169,7 +1165,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seek)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seekable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(selectors)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(self)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(send)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sep)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sequence)); @@ -1232,7 +1227,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timetuple)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(top)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(trace_callback)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(traceback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(trailers)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(translate)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(true)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 42220c93a01bfa..da1f9b67bdfb6a 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -276,10 +276,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(after_in_child) STRUCT_FOR_ID(after_in_parent) STRUCT_FOR_ID(aggregate_class) - STRUCT_FOR_ID(alias) STRUCT_FOR_ID(append) STRUCT_FOR_ID(argdefs) - STRUCT_FOR_ID(args) STRUCT_FOR_ID(arguments) STRUCT_FOR_ID(argv) STRUCT_FOR_ID(as_integer_ratio) @@ -403,8 +401,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(errors) STRUCT_FOR_ID(event) STRUCT_FOR_ID(eventmask) - STRUCT_FOR_ID(exc_type) - STRUCT_FOR_ID(exc_value) STRUCT_FOR_ID(excepthook) STRUCT_FOR_ID(exception) STRUCT_FOR_ID(existing_file_name) @@ -658,7 +654,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(seek) STRUCT_FOR_ID(seekable) STRUCT_FOR_ID(selectors) - STRUCT_FOR_ID(self) STRUCT_FOR_ID(send) STRUCT_FOR_ID(sep) STRUCT_FOR_ID(sequence) @@ -721,7 +716,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(timetuple) STRUCT_FOR_ID(top) STRUCT_FOR_ID(trace_callback) - STRUCT_FOR_ID(traceback) STRUCT_FOR_ID(trailers) STRUCT_FOR_ID(translate) STRUCT_FOR_ID(true) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 20dc11f2996e79..e285d02b48a9f4 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -785,10 +785,8 @@ extern "C" { INIT_ID(after_in_child), \ INIT_ID(after_in_parent), \ INIT_ID(aggregate_class), \ - INIT_ID(alias), \ INIT_ID(append), \ INIT_ID(argdefs), \ - INIT_ID(args), \ INIT_ID(arguments), \ INIT_ID(argv), \ INIT_ID(as_integer_ratio), \ @@ -912,8 +910,6 @@ extern "C" { INIT_ID(errors), \ INIT_ID(event), \ INIT_ID(eventmask), \ - INIT_ID(exc_type), \ - INIT_ID(exc_value), \ INIT_ID(excepthook), \ INIT_ID(exception), \ INIT_ID(existing_file_name), \ @@ -1167,7 +1163,6 @@ extern "C" { INIT_ID(seek), \ INIT_ID(seekable), \ INIT_ID(selectors), \ - INIT_ID(self), \ INIT_ID(send), \ INIT_ID(sep), \ INIT_ID(sequence), \ @@ -1230,7 +1225,6 @@ extern "C" { INIT_ID(timetuple), \ INIT_ID(top), \ INIT_ID(trace_callback), \ - INIT_ID(traceback), \ INIT_ID(trailers), \ INIT_ID(translate), \ INIT_ID(true), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index a82d7f480b766c..2f9874029fb8f1 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -669,18 +669,12 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(aggregate_class); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(alias); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(append); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(argdefs); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(args); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(arguments); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1050,12 +1044,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(eventmask); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(exc_type); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(exc_value); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(excepthook); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1815,9 +1803,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(selectors); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(self); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(send); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -2004,9 +1989,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(trace_callback); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(traceback); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(trailers); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); From 1c3e63af37d120782a456c58faa4f88c58ed3d39 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Jan 2024 16:48:19 +0000 Subject: [PATCH 14/14] Further lower C recursion limit for s390x --- Include/cpython/pystate.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 04730ad9736478..10913943c1140d 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -224,7 +224,7 @@ struct _ts { // recursions, sometimes less. 500 is a more conservative limit. # define Py_C_RECURSION_LIMIT 500 #elif defined(__s390x__) -# define Py_C_RECURSION_LIMIT 1000 +# define Py_C_RECURSION_LIMIT 800 #elif defined(_WIN32) # define Py_C_RECURSION_LIMIT 4000 #elif defined(_Py_ADDRESS_SANITIZER)