[3.12] GH-112215: Increase C recursion limit for non debug builds (GH-113397)

Include/cpython/pystate.h

@@ -252,11 +252,17 @@ struct _ts {
    layout, optimization, and WASI runtime. Wasmtime can handle about 700
    recursions, sometimes less. 500 is a more conservative limit. */
 #ifndef C_RECURSION_LIMIT
-#  ifdef __wasi__
+#  ifdef Py_DEBUG
+     // A debug build is likely built with low optimization level which implies
+     // higher stack memory usage than a release build: use a lower limit.
 #    define C_RECURSION_LIMIT 500
+#  elif defined(__wasi__)
+#    define C_RECURSION_LIMIT 500
+#  elif defined(__s390x__)
+#    define C_RECURSION_LIMIT 1200
 #  else
     // This value is duplicated in Lib/test/support/__init__.py
-#    define C_RECURSION_LIMIT 1500
+#    define C_RECURSION_LIMIT 8000
 #  endif
 #endif
 

Lib/test/support/__init__.py

@@ -2112,12 +2112,7 @@ def set_recursion_limit(limit):
     finally:
         sys.setrecursionlimit(original_limit)
 
-def infinite_recursion(max_depth=100):
-    """Set a lower limit for tests that interact with infinite recursions
-    (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some
-    debug windows builds, due to not enough functions being inlined the
-    stack size might not handle the default recursion limit (1000). See
-    bpo-11105 for details."""
+def infinite_recursion(max_depth=20_000):
     if max_depth < 3:
         raise ValueError("max_depth must be at least 3, got {max_depth}")
     depth = get_recursion_depth()
@@ -2356,11 +2351,15 @@ def adjust_int_max_str_digits(max_digits):
     finally:
         sys.set_int_max_str_digits(current)
 
-#For recursion tests, easily exceeds default recursion limit
-EXCEEDS_RECURSION_LIMIT = 5000
 
 # The default C recursion limit (from Include/cpython/pystate.h).
-C_RECURSION_LIMIT = 1500
+if Py_DEBUG:
+    C_RECURSION_LIMIT = 500
+else:
+    C_RECURSION_LIMIT = 8000
+
+#For recursion tests, easily exceeds default recursion limit
+EXCEEDS_RECURSION_LIMIT = C_RECURSION_LIMIT * 3
 
 #Windows doesn't have os.uname() but it doesn't support s390x.
 skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x',

Lib/test/test_ast.py

@@ -1089,7 +1089,7 @@ def next(self):
     def test_ast_recursion_limit(self):
         fail_depth = support.EXCEEDS_RECURSION_LIMIT
         crash_depth = 100_000
-        success_depth = 1200
+        success_depth = support.C_RECURSION_LIMIT
 
         def check_limit(prefix, repeated):
             expect_ok = prefix + repeated * success_depth

Lib/test/test_functools.py

@@ -1862,6 +1862,20 @@ def orig(): ...
         self.assertEqual(str(Signature.from_callable(lru.cache_info)), '()')
         self.assertEqual(str(Signature.from_callable(lru.cache_clear)), '()')
 
+    @support.skip_on_s390x
+    @unittest.skipIf(support.is_wasi, "WASI has limited C stack")
+    def test_lru_recursion(self):
+
+        @self.module.lru_cache
+        def fib(n):
+            if n <= 1:
+                return n
+            return fib(n-1) + fib(n-2)
+
+        if not support.Py_DEBUG:
+            with support.infinite_recursion():
+                fib(2000)
+
 
 @py_functools.lru_cache()
 def py_cached_func(x, y):

Lib/test/test_json/test_recursion.py

@@ -85,10 +85,10 @@ def test_highly_nested_objects_encoding(self):
         for x in range(100000):
             l, d = [l], {'k':d}
         with self.assertRaises(RecursionError):
-            with support.infinite_recursion():
+            with support.infinite_recursion(5000):
                 self.dumps(l)
         with self.assertRaises(RecursionError):
-            with support.infinite_recursion():
+            with support.infinite_recursion(5000):
                 self.dumps(d)
 
     def test_endless_recursion(self):
@@ -99,7 +99,7 @@ def default(self, o):
                 return [o]
 
         with self.assertRaises(RecursionError):
-            with support.infinite_recursion():
+            with support.infinite_recursion(1000):
                 EndlessJSONEncoder(check_circular=False).encode(5j)
 
 

Lib/test/test_support.py

@@ -633,7 +633,7 @@ def recursive_function(depth):
             if depth:
                 recursive_function(depth - 1)
 
-        for max_depth in (5, 25, 250):
+        for max_depth in (5, 25, 250, 2500):
             with support.infinite_recursion(max_depth):
                 available = support.get_recursion_available()
 

Lib/test/test_sys_settrace.py

@@ -7,7 +7,7 @@
 import gc
 from functools import wraps
 import asyncio
-from test.support import import_helper
+from test.support import import_helper, Py_DEBUG
 import contextlib
 import warnings
 
@@ -2964,6 +2964,7 @@ def test_trace_unpack_long_sequence(self):
         counts = self.count_traces(ns["f"])
         self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1})
 
+    @unittest.skipIf(Py_DEBUG, "Debug build cannot handle deep recursion")
     def test_trace_lots_of_globals(self):
         code = """if 1:
             def f():

Lib/test/test_xml_etree.py

@@ -2535,7 +2535,7 @@ def __eq__(self, o):
         e.extend([ET.Element('bar')])
         self.assertRaises(ValueError, e.remove, X('baz'))
 
-    @support.infinite_recursion(25)
+    @support.infinite_recursion()
     def test_recursive_repr(self):
         # Issue #25455
         e = ET.Element('foo')

Misc/NEWS.d/next/Core and Builtins/2023-12-15-16-26-01.gh-issue-112215.xJS6_6.rst

@@ -0,0 +1,3 @@
+Increase the C recursion limit by a factor of 3 for non-debug builds, except
+for webassembly and s390 platforms which are unchanged. This mitigates some
+regressions in 3.12 with deep recursion mixing builtin (C) and Python code.