Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit0142236

Browse files
authored
GH-130396: Use computed stack limits on linux (GH-130398)
* Implement C recursion protection with limit pointers for Linux, MacOS and Windows* Remove calls to PyOS_CheckStack* Add stack protection to parser* Make tests more robust to low stacks* Improve error messages for stack overflow
1 parent99088ab commit0142236

File tree

58 files changed

+1295
-1482
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+1295
-1482
lines changed

‎Doc/c-api/exceptions.rst

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -921,11 +921,7 @@ because the :ref:`call protocol <call>` takes care of recursion handling.
921921
922922
Marks a point where a recursive C-level call is about to be performed.
923923
924-
If:c:macro:`!USE_STACKCHECK` is defined, this function checks if the OS
925-
stack overflowed using:c:func:`PyOS_CheckStack`. If this is the case, it
926-
sets a:exc:`MemoryError` and returns a nonzero value.
927-
928-
The function then checks if the recursion limit is reached. If this is the
924+
The function then checks if the stack limit is reached. If this is the
929925
case, a:exc:`RecursionError` is set and a nonzero value is returned.
930926
Otherwise, zero is returned.
931927

‎Include/cpython/object.h

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -487,18 +487,19 @@ PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(PyThreadState *tstate);
487487
* we have headroom above the trigger limit */
488488
#definePy_TRASHCAN_HEADROOM 50
489489

490+
/* Helper function for Py_TRASHCAN_BEGIN */
491+
PyAPI_FUNC(int)_Py_ReachedRecursionLimitWithMargin(PyThreadState*tstate,intmargin_count);
492+
490493
#definePy_TRASHCAN_BEGIN(op,dealloc) \
491494
do { \
492495
PyThreadState *tstate = PyThreadState_Get(); \
493-
if (tstate->c_recursion_remaining <= Py_TRASHCAN_HEADROOM && Py_TYPE(op)->tp_dealloc == (destructor)dealloc) { \
496+
if (_Py_ReachedRecursionLimitWithMargin(tstate, 1) && Py_TYPE(op)->tp_dealloc == (destructor)dealloc) { \
494497
_PyTrash_thread_deposit_object(tstate, (PyObject *)op); \
495498
break; \
496-
} \
497-
tstate->c_recursion_remaining--;
499+
}
498500
/* The body of the deallocator is here. */
499501
#definePy_TRASHCAN_END \
500-
tstate->c_recursion_remaining++; \
501-
if (tstate->delete_later && tstate->c_recursion_remaining > (Py_TRASHCAN_HEADROOM*2)) { \
502+
if (tstate->delete_later && !_Py_ReachedRecursionLimitWithMargin(tstate, 2)) { \
502503
_PyTrash_thread_destroy_chain(tstate); \
503504
} \
504505
} while (0);

‎Include/cpython/pystate.h

Lines changed: 2 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ struct _ts {
112112
intpy_recursion_remaining;
113113
intpy_recursion_limit;
114114

115-
intc_recursion_remaining;
115+
intc_recursion_remaining;/* Retained for backwards compatibility. Do not use */
116116
intrecursion_headroom;/* Allow 50 more calls to handle any errors. */
117117

118118
/* 'tracing' keeps track of the execution depth when tracing/profiling.
@@ -202,36 +202,7 @@ struct _ts {
202202
PyObject*threading_local_sentinel;
203203
};
204204

205-
#ifdefPy_DEBUG
206-
// A debug build is likely built with low optimization level which implies
207-
// higher stack memory usage than a release build: use a lower limit.
208-
# definePy_C_RECURSION_LIMIT 500
209-
#elif defined(__s390x__)
210-
# definePy_C_RECURSION_LIMIT 800
211-
#elif defined(_WIN32)&& defined(_M_ARM64)
212-
# definePy_C_RECURSION_LIMIT 1000
213-
#elif defined(_WIN32)
214-
# definePy_C_RECURSION_LIMIT 3000
215-
#elif defined(__ANDROID__)
216-
// On an ARM64 emulator, API level 34 was OK with 10000, but API level 21
217-
// crashed in test_compiler_recursion_limit.
218-
# definePy_C_RECURSION_LIMIT 3000
219-
#elif defined(_Py_ADDRESS_SANITIZER)
220-
# definePy_C_RECURSION_LIMIT 4000
221-
#elif defined(__sparc__)
222-
// test_descr crashed on sparc64 with >7000 but let's keep a margin of error.
223-
# definePy_C_RECURSION_LIMIT 4000
224-
#elif defined(__wasi__)
225-
// Based on wasmtime 16.
226-
# definePy_C_RECURSION_LIMIT 5000
227-
#elif defined(__hppa__)|| defined(__powerpc64__)
228-
// test_descr crashed with >8000 but let's keep a margin of error.
229-
# definePy_C_RECURSION_LIMIT 5000
230-
#else
231-
// This value is duplicated in Lib/test/support/__init__.py
232-
# definePy_C_RECURSION_LIMIT 10000
233-
#endif
234-
205+
# definePy_C_RECURSION_LIMIT 5000
235206

236207
/* other API */
237208

@@ -246,7 +217,6 @@ _PyThreadState_UncheckedGet(void)
246217
returnPyThreadState_GetUnchecked();
247218
}
248219

249-
250220
// Disable tracing and profiling.
251221
PyAPI_FUNC(void)PyThreadState_EnterTracing(PyThreadState*tstate);
252222

‎Include/internal/pycore_ceval.h

Lines changed: 35 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -193,18 +193,28 @@ extern void _PyEval_DeactivateOpCache(void);
193193

194194
/* --- _Py_EnterRecursiveCall() ----------------------------------------- */
195195

196-
#ifdefUSE_STACKCHECK
197-
/* With USE_STACKCHECK macro defined, trigger stack checks in
198-
_Py_CheckRecursiveCall() on every 64th call to _Py_EnterRecursiveCall. */
199-
staticinlineint_Py_MakeRecCheck(PyThreadState*tstate) {
200-
return (tstate->c_recursion_remaining--<0
201-
|| (tstate->c_recursion_remaining&63)==0);
196+
#if !_Py__has_builtin(__builtin_frame_address)
197+
staticuintptr_treturn_pointer_as_int(char*p) {
198+
return (uintptr_t)p;
202199
}
200+
#endif
201+
202+
staticinlineuintptr_t
203+
_Py_get_machine_stack_pointer(void) {
204+
#if_Py__has_builtin(__builtin_frame_address)
205+
return (uintptr_t)__builtin_frame_address(0);
203206
#else
204-
staticinlineint_Py_MakeRecCheck(PyThreadState*tstate) {
205-
returntstate->c_recursion_remaining--<0;
206-
}
207+
charhere;
208+
/* Avoid compiler warning about returning stack address */
209+
returnreturn_pointer_as_int(&here);
207210
#endif
211+
}
212+
213+
staticinlineint_Py_MakeRecCheck(PyThreadState*tstate) {
214+
uintptr_there_addr=_Py_get_machine_stack_pointer();
215+
_PyThreadStateImpl*_tstate= (_PyThreadStateImpl*)tstate;
216+
returnhere_addr<_tstate->c_stack_soft_limit;
217+
}
208218

209219
// Export for '_json' shared extension, used via _Py_EnterRecursiveCall()
210220
// static inline function.
@@ -220,23 +230,30 @@ static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate,
220230
return (_Py_MakeRecCheck(tstate)&&_Py_CheckRecursiveCall(tstate,where));
221231
}
222232

223-
staticinlinevoid_Py_EnterRecursiveCallTstateUnchecked(PyThreadState*tstate) {
224-
assert(tstate->c_recursion_remaining>0);
225-
tstate->c_recursion_remaining--;
226-
}
227-
228233
staticinlineint_Py_EnterRecursiveCall(constchar*where) {
229234
PyThreadState*tstate=_PyThreadState_GET();
230235
return_Py_EnterRecursiveCallTstate(tstate,where);
231236
}
232237

233-
staticinlinevoid_Py_LeaveRecursiveCallTstate(PyThreadState*tstate) {
234-
tstate->c_recursion_remaining++;
238+
staticinlinevoid_Py_LeaveRecursiveCallTstate(PyThreadState*tstate) {
239+
(void)tstate;
240+
}
241+
242+
PyAPI_FUNC(void)_Py_InitializeRecursionLimits(PyThreadState*tstate);
243+
244+
staticinlineint_Py_ReachedRecursionLimit(PyThreadState*tstate) {
245+
uintptr_there_addr=_Py_get_machine_stack_pointer();
246+
_PyThreadStateImpl*_tstate= (_PyThreadStateImpl*)tstate;
247+
if (here_addr>_tstate->c_stack_soft_limit) {
248+
return0;
249+
}
250+
if (_tstate->c_stack_hard_limit==0) {
251+
_Py_InitializeRecursionLimits(tstate);
252+
}
253+
returnhere_addr <=_tstate->c_stack_soft_limit;
235254
}
236255

237256
staticinlinevoid_Py_LeaveRecursiveCall(void) {
238-
PyThreadState*tstate=_PyThreadState_GET();
239-
_Py_LeaveRecursiveCallTstate(tstate);
240257
}
241258

242259
externstruct_PyInterpreterFrame*_PyEval_GetFrame(void);
@@ -327,7 +344,6 @@ void _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit);
327344

328345
PyAPI_FUNC(PyObject*)_PyFloat_FromDouble_ConsumeInputs(_PyStackRefleft,_PyStackRefright,doublevalue);
329346

330-
331347
#ifdef__cplusplus
332348
}
333349
#endif

‎Include/internal/pycore_symtable.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,8 +82,6 @@ struct symtable {
8282
PyObject*st_private;/* name of current class or NULL */
8383
_PyFutureFeatures*st_future;/* module's future features that affect
8484
the symbol table */
85-
intrecursion_depth;/* current recursion depth */
86-
intrecursion_limit;/* recursion limit */
8785
};
8886

8987
typedefstruct_symtable_entry {

‎Include/internal/pycore_tstate.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,11 @@ typedef struct _PyThreadStateImpl {
2121
// semi-public fields are in PyThreadState.
2222
PyThreadStatebase;
2323

24+
// These are addresses, but we need to convert to ints to avoid UB.
25+
uintptr_tc_stack_top;
26+
uintptr_tc_stack_soft_limit;
27+
uintptr_tc_stack_hard_limit;
28+
2429
PyObject*asyncio_running_loop;// Strong reference
2530
PyObject*asyncio_running_task;// Strong reference
2631

‎Include/pythonrun.h

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,23 @@ PyAPI_FUNC(void) PyErr_DisplayException(PyObject *);
2121
/* Stuff with no proper home (yet) */
2222
PyAPI_DATA(int) (*PyOS_InputHook)(void);
2323

24-
/* Stack size, in "pointers" (so we get extra safety margins
25-
on 64-bit platforms). On a 32-bit platform, this translates
26-
to an 8k margin. */
27-
#definePYOS_STACK_MARGIN 2048
28-
29-
#if defined(WIN32)&& !defined(MS_WIN64)&& !defined(_M_ARM)&& defined(_MSC_VER)&&_MSC_VER >=1300
30-
/* Enable stack checking under Microsoft C */
31-
// When changing the platforms, ensure PyOS_CheckStack() docs are still correct
24+
/* Stack size, in "pointers". This must be large enough, so
25+
* no two calls to check recursion depth are more than this far
26+
* apart. In practice, that means it must be larger than the C
27+
* stack consumption of PyEval_EvalDefault */
28+
#if defined(_Py_ADDRESS_SANITIZER)|| defined(_Py_THREAD_SANITIZER)
29+
# definePYOS_STACK_MARGIN 4096
30+
#elif defined(Py_DEBUG)&& defined(WIN32)
31+
# definePYOS_STACK_MARGIN 3072
32+
#elif defined(__wasi__)
33+
/* Web assembly has two stacks, so this isn't really a size */
34+
# definePYOS_STACK_MARGIN 500
35+
#else
36+
# definePYOS_STACK_MARGIN 2048
37+
#endif
38+
#definePYOS_STACK_MARGIN_BYTES (PYOS_STACK_MARGIN * sizeof(void *))
39+
40+
#if defined(WIN32)
3241
#defineUSE_STACKCHECK
3342
#endif
3443

‎Lib/test/list_tests.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
fromfunctoolsimportcmp_to_key
77

88
fromtestimportseq_tests
9-
fromtest.supportimportALWAYS_EQ,NEVER_EQ,get_c_recursion_limit,skip_emscripten_stack_overflow
9+
fromtest.supportimportALWAYS_EQ,NEVER_EQ
10+
fromtest.supportimportskip_emscripten_stack_overflow,skip_wasi_stack_overflow
1011

1112

1213
classCommonTest(seq_tests.CommonTest):
@@ -59,10 +60,11 @@ def test_repr(self):
5960
self.assertEqual(str(a2),"[0, 1, 2, [...], 3]")
6061
self.assertEqual(repr(a2),"[0, 1, 2, [...], 3]")
6162

63+
@skip_wasi_stack_overflow()
6264
@skip_emscripten_stack_overflow()
6365
deftest_repr_deep(self):
6466
a=self.type2test([])
65-
foriinrange(get_c_recursion_limit()+1):
67+
foriinrange(200_000):
6668
a=self.type2test([a])
6769
self.assertRaises(RecursionError,repr,a)
6870

‎Lib/test/mapping_tests.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# tests common to dict and UserDict
22
importunittest
33
importcollections
4-
fromtest.supportimportget_c_recursion_limit,skip_emscripten_stack_overflow
4+
fromtestimportsupport
55

66

77
classBasicTestMappingProtocol(unittest.TestCase):
@@ -622,10 +622,11 @@ def __repr__(self):
622622
d=self._full_mapping({1:BadRepr()})
623623
self.assertRaises(Exc,repr,d)
624624

625-
@skip_emscripten_stack_overflow()
625+
@support.skip_wasi_stack_overflow()
626+
@support.skip_emscripten_stack_overflow()
626627
deftest_repr_deep(self):
627628
d=self._empty_mapping()
628-
foriinrange(get_c_recursion_limit()+1):
629+
foriinrange(support.exceeds_recursion_limit()):
629630
d0=d
630631
d=self._empty_mapping()
631632
d[1]=d0

‎Lib/test/pythoninfo.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -684,7 +684,6 @@ def collect_testcapi(info_add):
684684
fornamein (
685685
'LONG_MAX',# always 32-bit on Windows, 64-bit on 64-bit Unix
686686
'PY_SSIZE_T_MAX',
687-
'Py_C_RECURSION_LIMIT',
688687
'SIZEOF_TIME_T',# 32-bit or 64-bit depending on the platform
689688
'SIZEOF_WCHAR_T',# 16-bit or 32-bit depending on the platform
690689
):

‎Lib/test/support/__init__.py

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,7 @@
5656
"run_with_tz","PGO","missing_compiler_executable",
5757
"ALWAYS_EQ","NEVER_EQ","LARGEST","SMALLEST",
5858
"LOOPBACK_TIMEOUT","INTERNET_TIMEOUT","SHORT_TIMEOUT","LONG_TIMEOUT",
59-
"Py_DEBUG","exceeds_recursion_limit","get_c_recursion_limit",
60-
"skip_on_s390x",
59+
"Py_DEBUG","exceeds_recursion_limit","skip_on_s390x",
6160
"requires_jit_enabled",
6261
"requires_jit_disabled",
6362
"force_not_colorized",
@@ -558,6 +557,9 @@ def skip_android_selinux(name):
558557
defskip_emscripten_stack_overflow():
559558
returnunittest.skipIf(is_emscripten,"Exhausts limited stack on Emscripten")
560559

560+
defskip_wasi_stack_overflow():
561+
returnunittest.skipIf(is_wasi,"Exhausts stack on WASI")
562+
561563
is_apple_mobile=sys.platformin {"ios","tvos","watchos"}
562564
is_apple=is_apple_mobileorsys.platform=="darwin"
563565

@@ -2624,17 +2626,9 @@ def adjust_int_max_str_digits(max_digits):
26242626
sys.set_int_max_str_digits(current)
26252627

26262628

2627-
defget_c_recursion_limit():
2628-
try:
2629-
import_testcapi
2630-
return_testcapi.Py_C_RECURSION_LIMIT
2631-
exceptImportError:
2632-
raiseunittest.SkipTest('requires _testcapi')
2633-
2634-
26352629
defexceeds_recursion_limit():
26362630
"""For recursion tests, easily exceeds default recursion limit."""
2637-
returnget_c_recursion_limit()*3
2631+
return150_000
26382632

26392633

26402634
# Windows doesn't have os.uname() but it doesn't support s390x.

‎Lib/test/test_ast/test_ast.py

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@
1818
_testinternalcapi=None
1919

2020
fromtestimportsupport
21-
fromtest.supportimportos_helper,script_helper,skip_emscripten_stack_overflow
21+
fromtest.supportimportos_helper,script_helper
22+
fromtest.supportimportskip_emscripten_stack_overflow,skip_wasi_stack_overflow
2223
fromtest.support.ast_helperimportASTTestMixin
2324
fromtest.test_ast.utilsimportto_tuple
2425
fromtest.test_ast.snippetsimport (
@@ -750,25 +751,25 @@ def next(self):
750751
enum._test_simple_enum(_Precedence,ast._Precedence)
751752

752753
@support.cpython_only
754+
@skip_wasi_stack_overflow()
753755
@skip_emscripten_stack_overflow()
754756
deftest_ast_recursion_limit(self):
755-
fail_depth=support.exceeds_recursion_limit()
756-
crash_depth=100_000
757-
success_depth=int(support.get_c_recursion_limit()*0.8)
757+
crash_depth=500_000
758+
success_depth=200
758759
if_testinternalcapiisnotNone:
759760
remaining=_testinternalcapi.get_c_recursion_remaining()
760761
success_depth=min(success_depth,remaining)
761762

762763
defcheck_limit(prefix,repeated):
763764
expect_ok=prefix+repeated*success_depth
764765
ast.parse(expect_ok)
765-
fordepthin (fail_depth,crash_depth):
766-
broken=prefix+repeated*depth
767-
details="Compiling ({!r} + {!r} * {})".format(
768-
prefix,repeated,depth)
769-
withself.assertRaises(RecursionError,msg=details):
770-
withsupport.infinite_recursion():
771-
ast.parse(broken)
766+
767+
broken=prefix+repeated*crash_depth
768+
details="Compiling ({!r} + {!r} * {})".format(
769+
prefix,repeated,crash_depth)
770+
withself.assertRaises(RecursionError,msg=details):
771+
withsupport.infinite_recursion():
772+
ast.parse(broken)
772773

773774
check_limit("a","()")
774775
check_limit("a",".b")

‎Lib/test/test_builtin.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1052,6 +1052,7 @@ def test_filter_pickle(self):
10521052
f2=filter(filter_char,"abcdeabcde")
10531053
self.check_iter_pickle(f1,list(f2),proto)
10541054

1055+
@support.skip_wasi_stack_overflow()
10551056
@support.requires_resource('cpu')
10561057
deftest_filter_dealloc(self):
10571058
# Tests recursive deallocation of nested filter objects using the

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp