From 5e50b7ede2acdb739c7897e26bca2898ee0dc34e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 15 Dec 2023 08:22:45 +0000 Subject: [PATCH 01/10] Try increasing the C recursion limit --- Include/cpython/pystate.h | 2 +- Lib/test/support/__init__.py | 17 +++-------------- Lib/test/test_ast.py | 2 +- Lib/test/test_json/test_recursion.py | 6 +++--- Lib/test/test_support.py | 2 +- Lib/test/test_xml_etree.py | 2 +- 6 files changed, 10 insertions(+), 21 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 56172d231c44f4..4d5901791176df 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -225,7 +225,7 @@ struct _ts { # define Py_C_RECURSION_LIMIT 500 #else // This value is duplicated in Lib/test/support/__init__.py -# define Py_C_RECURSION_LIMIT 1500 +# define Py_C_RECURSION_LIMIT 4500 #endif diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index b605951320dc8b..ba1d8ea768225d 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2122,19 +2122,8 @@ def set_recursion_limit(limit): sys.setrecursionlimit(original_limit) def infinite_recursion(max_depth=None): - """Set a lower limit for tests that interact with infinite recursions - (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some - debug windows builds, due to not enough functions being inlined the - stack size might not handle the default recursion limit (1000). See - bpo-11105 for details.""" if max_depth is None: - if not python_is_optimized() or Py_DEBUG: - # Python built without compiler optimizations or in debug mode - # usually consumes more stack memory per function call. - # Unoptimized number based on what works under a WASI debug build. - max_depth = 50 - else: - max_depth = 100 + max_depth = 20_000 elif max_depth < 3: raise ValueError("max_depth must be at least 3, got {max_depth}") depth = get_recursion_depth() @@ -2374,7 +2363,7 @@ def adjust_int_max_str_digits(max_digits): sys.set_int_max_str_digits(current) #For recursion tests, easily exceeds default recursion limit -EXCEEDS_RECURSION_LIMIT = 5000 +EXCEEDS_RECURSION_LIMIT = 10_000 def _get_c_recursion_limit(): try: @@ -2382,7 +2371,7 @@ def _get_c_recursion_limit(): return _testcapi.Py_C_RECURSION_LIMIT except (ImportError, AttributeError): # Originally taken from Include/cpython/pystate.h . - return 1500 + return 4500 # The default C recursion limit. Py_C_RECURSION_LIMIT = _get_c_recursion_limit() diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 64fcb02309de77..35011dfe972188 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -19,7 +19,7 @@ from test import support from test.support.import_helper import import_fresh_module -from test.support import os_helper, script_helper +from test.support import os_helper, script_helper, Py_DEBUG from test.support.ast_helper import ASTTestMixin def to_tuple(t): diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index 9919d7fbe54ef7..164ff2013eb552 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -85,10 +85,10 @@ def test_highly_nested_objects_encoding(self): for x in range(100000): l, d = [l], {'k':d} with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(5000): self.dumps(l) with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(5000): self.dumps(d) def test_endless_recursion(self): @@ -99,7 +99,7 @@ def default(self, o): return [o] with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(1000): EndlessJSONEncoder(check_circular=False).encode(5j) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index c34b0e5e015702..d160cbf0645b47 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -630,7 +630,7 @@ def recursive_function(depth): if depth: recursive_function(depth - 1) - for max_depth in (5, 25, 250): + for max_depth in (5, 25, 250, 2500): with support.infinite_recursion(max_depth): available = support.get_recursion_available() diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index b9e7937b0bbc00..80ee064896f59a 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -2535,7 +2535,7 @@ def __eq__(self, o): e.extend([ET.Element('bar')]) self.assertRaises(ValueError, e.remove, X('baz')) - @support.infinite_recursion(25) + @support.infinite_recursion() def test_recursive_repr(self): # Issue #25455 e = ET.Element('foo') From eefd3d6790ea8e28c3097601c5360606b4e3643a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 15 Dec 2023 10:30:31 +0000 Subject: [PATCH 02/10] Change max C recursion to 7000 --- Include/cpython/pystate.h | 2 +- Lib/test/support/__init__.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 4d5901791176df..c602942712c241 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -225,7 +225,7 @@ struct _ts { # define Py_C_RECURSION_LIMIT 500 #else // This value is duplicated in Lib/test/support/__init__.py -# define Py_C_RECURSION_LIMIT 4500 +# define Py_C_RECURSION_LIMIT 7000 #endif diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index ba1d8ea768225d..d00e7c568bb456 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2123,6 +2123,9 @@ def set_recursion_limit(limit): def infinite_recursion(max_depth=None): if max_depth is None: + # Pick a number large enough to cause problems + # but not take too long for code that can handle + # very deep recursion. max_depth = 20_000 elif max_depth < 3: raise ValueError("max_depth must be at least 3, got {max_depth}") @@ -2362,8 +2365,6 @@ def adjust_int_max_str_digits(max_digits): finally: sys.set_int_max_str_digits(current) -#For recursion tests, easily exceeds default recursion limit -EXCEEDS_RECURSION_LIMIT = 10_000 def _get_c_recursion_limit(): try: @@ -2371,11 +2372,14 @@ def _get_c_recursion_limit(): return _testcapi.Py_C_RECURSION_LIMIT except (ImportError, AttributeError): # Originally taken from Include/cpython/pystate.h . - return 4500 + return 7000 # The default C recursion limit. Py_C_RECURSION_LIMIT = _get_c_recursion_limit() +#For recursion tests, easily exceeds default recursion limit +EXCEEDS_RECURSION_LIMIT = Py_C_RECURSION_LIMIT * 3 + #Windows doesn't have os.uname() but it doesn't support s390x. skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', 'skipped on s390x') From 90f696d0afffbd29f64f24081855e91c02dd4bac Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 15 Dec 2023 13:23:41 +0000 Subject: [PATCH 03/10] Up C recursion limit to 10_000 --- Include/cpython/pystate.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index c602942712c241..01df96a0968131 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -225,7 +225,7 @@ struct _ts { # define Py_C_RECURSION_LIMIT 500 #else // This value is duplicated in Lib/test/support/__init__.py -# define Py_C_RECURSION_LIMIT 7000 +# define Py_C_RECURSION_LIMIT 10000 #endif From 3b743852d3f911d103eebec2e6b6b5b1f7780c63 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 15 Dec 2023 13:55:00 +0000 Subject: [PATCH 04/10] Reduce recursion depth to 9000 --- Include/cpython/pystate.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 01df96a0968131..38ef3e88df88e8 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -225,7 +225,7 @@ struct _ts { # define Py_C_RECURSION_LIMIT 500 #else // This value is duplicated in Lib/test/support/__init__.py -# define Py_C_RECURSION_LIMIT 10000 +# define Py_C_RECURSION_LIMIT 9000 #endif From 1f4e3a321a02dd7aced0d3cb135fe0cde7f447fb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 15 Dec 2023 14:57:31 +0000 Subject: [PATCH 05/10] Reduce recursion depth to 8000 --- Include/cpython/pystate.h | 4 +++- Lib/test/support/__init__.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 38ef3e88df88e8..1614ff8f85580a 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -223,9 +223,11 @@ struct _ts { // layout, optimization, and WASI runtime. Wasmtime can handle about 700 // recursions, sometimes less. 500 is a more conservative limit. # define Py_C_RECURSION_LIMIT 500 +#elif defined(__s390x__) +# define Py_C_RECURSION_LIMIT 1500 #else // This value is duplicated in Lib/test/support/__init__.py -# define Py_C_RECURSION_LIMIT 9000 +# define Py_C_RECURSION_LIMIT 8000 #endif diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index d00e7c568bb456..c8f73cede230d8 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2372,7 +2372,7 @@ def _get_c_recursion_limit(): return _testcapi.Py_C_RECURSION_LIMIT except (ImportError, AttributeError): # Originally taken from Include/cpython/pystate.h . - return 7000 + return 8000 # The default C recursion limit. Py_C_RECURSION_LIMIT = _get_c_recursion_limit() From 2a4c6a5b62e0823feebaa38cb1db7354a101c358 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 15 Dec 2023 16:02:08 +0000 Subject: [PATCH 06/10] Remove unused import --- Lib/test/test_ast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 35011dfe972188..64fcb02309de77 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -19,7 +19,7 @@ from test import support from test.support.import_helper import import_fresh_module -from test.support import os_helper, script_helper, Py_DEBUG +from test.support import os_helper, script_helper from test.support.ast_helper import ASTTestMixin def to_tuple(t): From 1345860f0137ae2d83195c5674a622584b3337d1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 15 Dec 2023 16:21:19 +0000 Subject: [PATCH 07/10] Add test for deep recursion with LRU cache --- Lib/test/test_functools.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index e4de2c5ede15f1..2da7ede5a8ae00 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1862,6 +1862,19 @@ def orig(): ... self.assertEqual(str(Signature.from_callable(lru.cache_info)), '()') self.assertEqual(str(Signature.from_callable(lru.cache_clear)), '()') + @support.skip_on_s390x + def test_lru_recursion(self): + + @self.module.lru_cache + def fib(n): + if n <= 1: + return n + return fib(n-1) + fib(n-2) + + if not support.Py_DEBUG: + with support.infinite_recursion(): + fib(2500) + @py_functools.lru_cache() def py_cached_func(x, y): From 661118309102119a8eca2a8e56a76a893076284f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 15 Dec 2023 16:26:10 +0000 Subject: [PATCH 08/10] Add news --- .../2023-12-15-16-26-01.gh-issue-112215.xJS6_6.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-12-15-16-26-01.gh-issue-112215.xJS6_6.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-15-16-26-01.gh-issue-112215.xJS6_6.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-15-16-26-01.gh-issue-112215.xJS6_6.rst new file mode 100644 index 00000000000000..01ca1cc7f79b8f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-15-16-26-01.gh-issue-112215.xJS6_6.rst @@ -0,0 +1,3 @@ +Increase the C recursion limit by a factor of 3 for non-debug builds, except +for webassembly and s390 platforms which are unchanged. This mitigates some +regressions in 3.12 with deep recursion mixing builtin (C) and Python code. From a8c8eca27de336ff0d86ce5d75205e069cf26574 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 15 Dec 2023 16:31:04 +0000 Subject: [PATCH 09/10] Exclude WASI from test --- Lib/test/test_functools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 2da7ede5a8ae00..b95afe0bcc86e4 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1863,6 +1863,7 @@ def orig(): ... self.assertEqual(str(Signature.from_callable(lru.cache_clear)), '()') @support.skip_on_s390x + @unittest.skipIf(support.is_wasi, "WASI has limited C stack") def test_lru_recursion(self): @self.module.lru_cache From bbdac949d918601a9f7707a02b3526ee6b922c6a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 15 Dec 2023 17:37:34 +0000 Subject: [PATCH 10/10] Further reduce C recursion limit for s390 --- Include/cpython/pystate.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 1614ff8f85580a..ed7dd829d4b6f0 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -224,7 +224,7 @@ struct _ts { // recursions, sometimes less. 500 is a more conservative limit. # define Py_C_RECURSION_LIMIT 500 #elif defined(__s390x__) -# define Py_C_RECURSION_LIMIT 1500 +# define Py_C_RECURSION_LIMIT 1200 #else // This value is duplicated in Lib/test/support/__init__.py # define Py_C_RECURSION_LIMIT 8000