Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit2498c22

Browse files
authored
GH-91079: Implement C stack limits using addresses, not counters. (GH-130007)
* Implement C recursion protection with limit pointers* Remove calls to PyOS_CheckStack* Add stack protection to parser* Make tests more robust to low stacks* Improve error messages for stack overflow
1 parentc637bce commit2498c22

File tree

47 files changed

+1218
-1464
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+1218
-1464
lines changed

‎Doc/c-api/exceptions.rst

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -921,11 +921,7 @@ because the :ref:`call protocol <call>` takes care of recursion handling.
921921
922922
Marks a point where a recursive C-level call is about to be performed.
923923
924-
If:c:macro:`!USE_STACKCHECK` is defined, this function checks if the OS
925-
stack overflowed using:c:func:`PyOS_CheckStack`. If this is the case, it
926-
sets a:exc:`MemoryError` and returns a nonzero value.
927-
928-
The function then checks if the recursion limit is reached. If this is the
924+
The function then checks if the stack limit is reached. If this is the
929925
case, a:exc:`RecursionError` is set and a nonzero value is returned.
930926
Otherwise, zero is returned.
931927

‎Include/cpython/object.h

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -487,18 +487,19 @@ PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(PyThreadState *tstate);
487487
* we have headroom above the trigger limit */
488488
#definePy_TRASHCAN_HEADROOM 50
489489

490+
/* Helper function for Py_TRASHCAN_BEGIN */
491+
PyAPI_FUNC(int)_Py_ReachedRecursionLimitWithMargin(PyThreadState*tstate,intmargin_count);
492+
490493
#definePy_TRASHCAN_BEGIN(op,dealloc) \
491494
do { \
492495
PyThreadState *tstate = PyThreadState_Get(); \
493-
if (tstate->c_recursion_remaining <= Py_TRASHCAN_HEADROOM && Py_TYPE(op)->tp_dealloc == (destructor)dealloc) { \
496+
if (_Py_ReachedRecursionLimitWithMargin(tstate, 1) && Py_TYPE(op)->tp_dealloc == (destructor)dealloc) { \
494497
_PyTrash_thread_deposit_object(tstate, (PyObject *)op); \
495498
break; \
496-
} \
497-
tstate->c_recursion_remaining--;
499+
}
498500
/* The body of the deallocator is here. */
499501
#definePy_TRASHCAN_END \
500-
tstate->c_recursion_remaining++; \
501-
if (tstate->delete_later && tstate->c_recursion_remaining > (Py_TRASHCAN_HEADROOM*2)) { \
502+
if (tstate->delete_later && !_Py_ReachedRecursionLimitWithMargin(tstate, 2)) { \
502503
_PyTrash_thread_destroy_chain(tstate); \
503504
} \
504505
} while (0);

‎Include/cpython/pystate.h

Lines changed: 2 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ struct _ts {
112112
intpy_recursion_remaining;
113113
intpy_recursion_limit;
114114

115-
intc_recursion_remaining;
115+
intc_recursion_remaining;/* Retained for backwards compatibility. Do not use */
116116
intrecursion_headroom;/* Allow 50 more calls to handle any errors. */
117117

118118
/* 'tracing' keeps track of the execution depth when tracing/profiling.
@@ -202,36 +202,7 @@ struct _ts {
202202
PyObject*threading_local_sentinel;
203203
};
204204

205-
#ifdefPy_DEBUG
206-
// A debug build is likely built with low optimization level which implies
207-
// higher stack memory usage than a release build: use a lower limit.
208-
# definePy_C_RECURSION_LIMIT 500
209-
#elif defined(__s390x__)
210-
# definePy_C_RECURSION_LIMIT 800
211-
#elif defined(_WIN32)&& defined(_M_ARM64)
212-
# definePy_C_RECURSION_LIMIT 1000
213-
#elif defined(_WIN32)
214-
# definePy_C_RECURSION_LIMIT 3000
215-
#elif defined(__ANDROID__)
216-
// On an ARM64 emulator, API level 34 was OK with 10000, but API level 21
217-
// crashed in test_compiler_recursion_limit.
218-
# definePy_C_RECURSION_LIMIT 3000
219-
#elif defined(_Py_ADDRESS_SANITIZER)
220-
# definePy_C_RECURSION_LIMIT 4000
221-
#elif defined(__sparc__)
222-
// test_descr crashed on sparc64 with >7000 but let's keep a margin of error.
223-
# definePy_C_RECURSION_LIMIT 4000
224-
#elif defined(__wasi__)
225-
// Based on wasmtime 16.
226-
# definePy_C_RECURSION_LIMIT 5000
227-
#elif defined(__hppa__)|| defined(__powerpc64__)
228-
// test_descr crashed with >8000 but let's keep a margin of error.
229-
# definePy_C_RECURSION_LIMIT 5000
230-
#else
231-
// This value is duplicated in Lib/test/support/__init__.py
232-
# definePy_C_RECURSION_LIMIT 10000
233-
#endif
234-
205+
# definePy_C_RECURSION_LIMIT 5000
235206

236207
/* other API */
237208

@@ -246,7 +217,6 @@ _PyThreadState_UncheckedGet(void)
246217
returnPyThreadState_GetUnchecked();
247218
}
248219

249-
250220
// Disable tracing and profiling.
251221
PyAPI_FUNC(void)PyThreadState_EnterTracing(PyThreadState*tstate);
252222

‎Include/internal/pycore_ceval.h

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -193,18 +193,12 @@ extern void _PyEval_DeactivateOpCache(void);
193193

194194
/* --- _Py_EnterRecursiveCall() ----------------------------------------- */
195195

196-
#ifdefUSE_STACKCHECK
197-
/* With USE_STACKCHECK macro defined, trigger stack checks in
198-
_Py_CheckRecursiveCall() on every 64th call to _Py_EnterRecursiveCall. */
199196
staticinlineint_Py_MakeRecCheck(PyThreadState*tstate) {
200-
return (tstate->c_recursion_remaining--<0
201-
|| (tstate->c_recursion_remaining&63)==0);
197+
charhere;
198+
uintptr_there_addr= (uintptr_t)&here;
199+
_PyThreadStateImpl*_tstate= (_PyThreadStateImpl*)tstate;
200+
returnhere_addr<_tstate->c_stack_soft_limit;
202201
}
203-
#else
204-
staticinlineint_Py_MakeRecCheck(PyThreadState*tstate) {
205-
returntstate->c_recursion_remaining--<0;
206-
}
207-
#endif
208202

209203
// Export for '_json' shared extension, used via _Py_EnterRecursiveCall()
210204
// static inline function.
@@ -220,23 +214,31 @@ static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate,
220214
return (_Py_MakeRecCheck(tstate)&&_Py_CheckRecursiveCall(tstate,where));
221215
}
222216

223-
staticinlinevoid_Py_EnterRecursiveCallTstateUnchecked(PyThreadState*tstate) {
224-
assert(tstate->c_recursion_remaining>0);
225-
tstate->c_recursion_remaining--;
226-
}
227-
228217
staticinlineint_Py_EnterRecursiveCall(constchar*where) {
229218
PyThreadState*tstate=_PyThreadState_GET();
230219
return_Py_EnterRecursiveCallTstate(tstate,where);
231220
}
232221

233-
staticinlinevoid_Py_LeaveRecursiveCallTstate(PyThreadState*tstate) {
234-
tstate->c_recursion_remaining++;
222+
staticinlinevoid_Py_LeaveRecursiveCallTstate(PyThreadState*tstate) {
223+
(void)tstate;
224+
}
225+
226+
PyAPI_FUNC(void)_Py_InitializeRecursionLimits(PyThreadState*tstate);
227+
228+
staticinlineint_Py_ReachedRecursionLimit(PyThreadState*tstate) {
229+
charhere;
230+
uintptr_there_addr= (uintptr_t)&here;
231+
_PyThreadStateImpl*_tstate= (_PyThreadStateImpl*)tstate;
232+
if (here_addr>_tstate->c_stack_soft_limit) {
233+
return0;
234+
}
235+
if (_tstate->c_stack_hard_limit==0) {
236+
_Py_InitializeRecursionLimits(tstate);
237+
}
238+
returnhere_addr <=_tstate->c_stack_soft_limit;
235239
}
236240

237241
staticinlinevoid_Py_LeaveRecursiveCall(void) {
238-
PyThreadState*tstate=_PyThreadState_GET();
239-
_Py_LeaveRecursiveCallTstate(tstate);
240242
}
241243

242244
externstruct_PyInterpreterFrame*_PyEval_GetFrame(void);
@@ -327,7 +329,6 @@ void _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit);
327329

328330
PyAPI_FUNC(PyObject*)_PyFloat_FromDouble_ConsumeInputs(_PyStackRefleft,_PyStackRefright,doublevalue);
329331

330-
331332
#ifdef__cplusplus
332333
}
333334
#endif

‎Include/internal/pycore_symtable.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,8 +82,6 @@ struct symtable {
8282
PyObject*st_private;/* name of current class or NULL */
8383
_PyFutureFeatures*st_future;/* module's future features that affect
8484
the symbol table */
85-
intrecursion_depth;/* current recursion depth */
86-
intrecursion_limit;/* recursion limit */
8785
};
8886

8987
typedefstruct_symtable_entry {

‎Include/internal/pycore_tstate.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,11 @@ typedef struct _PyThreadStateImpl {
2121
// semi-public fields are in PyThreadState.
2222
PyThreadStatebase;
2323

24+
// These are addresses, but we need to convert to ints to avoid UB.
25+
uintptr_tc_stack_top;
26+
uintptr_tc_stack_soft_limit;
27+
uintptr_tc_stack_hard_limit;
28+
2429
PyObject*asyncio_running_loop;// Strong reference
2530
PyObject*asyncio_running_task;// Strong reference
2631

‎Include/pythonrun.h

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,18 @@ PyAPI_FUNC(void) PyErr_DisplayException(PyObject *);
2121
/* Stuff with no proper home (yet) */
2222
PyAPI_DATA(int) (*PyOS_InputHook)(void);
2323

24-
/* Stack size, in "pointers" (so we get extra safety margins
25-
on 64-bit platforms). On a 32-bit platform, this translates
26-
to an 8k margin. */
27-
#definePYOS_STACK_MARGIN 2048
28-
29-
#if defined(WIN32)&& !defined(MS_WIN64)&& !defined(_M_ARM)&& defined(_MSC_VER)&&_MSC_VER >=1300
30-
/* Enable stack checking under Microsoft C */
31-
// When changing the platforms, ensure PyOS_CheckStack() docs are still correct
24+
/* Stack size, in "pointers". This must be large enough, so
25+
* no two calls to check recursion depth are more than this far
26+
* apart. In practice, that means it must be larger than the C
27+
* stack consumption of PyEval_EvalDefault */
28+
#if defined(Py_DEBUG)&& defined(WIN32)
29+
# definePYOS_STACK_MARGIN 3072
30+
#else
31+
# definePYOS_STACK_MARGIN 2048
32+
#endif
33+
#definePYOS_STACK_MARGIN_BYTES (PYOS_STACK_MARGIN * sizeof(void *))
34+
35+
#if defined(WIN32)
3236
#defineUSE_STACKCHECK
3337
#endif
3438

‎Lib/test/list_tests.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
fromfunctoolsimportcmp_to_key
77

88
fromtestimportseq_tests
9-
fromtest.supportimportALWAYS_EQ,NEVER_EQ,get_c_recursion_limit,skip_emscripten_stack_overflow
9+
fromtest.supportimportALWAYS_EQ,NEVER_EQ
10+
fromtest.supportimportskip_emscripten_stack_overflow,skip_wasi_stack_overflow
1011

1112

1213
classCommonTest(seq_tests.CommonTest):
@@ -59,10 +60,11 @@ def test_repr(self):
5960
self.assertEqual(str(a2),"[0, 1, 2, [...], 3]")
6061
self.assertEqual(repr(a2),"[0, 1, 2, [...], 3]")
6162

63+
@skip_wasi_stack_overflow()
6264
@skip_emscripten_stack_overflow()
6365
deftest_repr_deep(self):
6466
a=self.type2test([])
65-
foriinrange(get_c_recursion_limit()+1):
67+
foriinrange(100_000):
6668
a=self.type2test([a])
6769
self.assertRaises(RecursionError,repr,a)
6870

‎Lib/test/mapping_tests.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# tests common to dict and UserDict
22
importunittest
33
importcollections
4-
fromtest.supportimportget_c_recursion_limit,skip_emscripten_stack_overflow
4+
fromtestimportsupport
55

66

77
classBasicTestMappingProtocol(unittest.TestCase):
@@ -622,10 +622,11 @@ def __repr__(self):
622622
d=self._full_mapping({1:BadRepr()})
623623
self.assertRaises(Exc,repr,d)
624624

625-
@skip_emscripten_stack_overflow()
625+
@support.skip_wasi_stack_overflow()
626+
@support.skip_emscripten_stack_overflow()
626627
deftest_repr_deep(self):
627628
d=self._empty_mapping()
628-
foriinrange(get_c_recursion_limit()+1):
629+
foriinrange(support.exceeds_recursion_limit()):
629630
d0=d
630631
d=self._empty_mapping()
631632
d[1]=d0

‎Lib/test/pythoninfo.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -684,7 +684,6 @@ def collect_testcapi(info_add):
684684
fornamein (
685685
'LONG_MAX',# always 32-bit on Windows, 64-bit on 64-bit Unix
686686
'PY_SSIZE_T_MAX',
687-
'Py_C_RECURSION_LIMIT',
688687
'SIZEOF_TIME_T',# 32-bit or 64-bit depending on the platform
689688
'SIZEOF_WCHAR_T',# 16-bit or 32-bit depending on the platform
690689
):

‎Lib/test/support/__init__.py

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,7 @@
5656
"run_with_tz","PGO","missing_compiler_executable",
5757
"ALWAYS_EQ","NEVER_EQ","LARGEST","SMALLEST",
5858
"LOOPBACK_TIMEOUT","INTERNET_TIMEOUT","SHORT_TIMEOUT","LONG_TIMEOUT",
59-
"Py_DEBUG","exceeds_recursion_limit","get_c_recursion_limit",
60-
"skip_on_s390x",
59+
"Py_DEBUG","exceeds_recursion_limit","skip_on_s390x",
6160
"requires_jit_enabled",
6261
"requires_jit_disabled",
6362
"force_not_colorized",
@@ -558,6 +557,9 @@ def skip_android_selinux(name):
558557
defskip_emscripten_stack_overflow():
559558
returnunittest.skipIf(is_emscripten,"Exhausts limited stack on Emscripten")
560559

560+
defskip_wasi_stack_overflow():
561+
returnunittest.skipIf(is_wasi,"Exhausts stack on WASI")
562+
561563
is_apple_mobile=sys.platformin {"ios","tvos","watchos"}
562564
is_apple=is_apple_mobileorsys.platform=="darwin"
563565

@@ -2624,17 +2626,9 @@ def adjust_int_max_str_digits(max_digits):
26242626
sys.set_int_max_str_digits(current)
26252627

26262628

2627-
defget_c_recursion_limit():
2628-
try:
2629-
import_testcapi
2630-
return_testcapi.Py_C_RECURSION_LIMIT
2631-
exceptImportError:
2632-
raiseunittest.SkipTest('requires _testcapi')
2633-
2634-
26352629
defexceeds_recursion_limit():
26362630
"""For recursion tests, easily exceeds default recursion limit."""
2637-
returnget_c_recursion_limit()*3
2631+
return100_000
26382632

26392633

26402634
# Windows doesn't have os.uname() but it doesn't support s390x.

‎Lib/test/test_ast/test_ast.py

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@
1818
_testinternalcapi=None
1919

2020
fromtestimportsupport
21-
fromtest.supportimportos_helper,script_helper,skip_emscripten_stack_overflow
21+
fromtest.supportimportos_helper,script_helper
22+
fromtest.supportimportskip_emscripten_stack_overflow,skip_wasi_stack_overflow
2223
fromtest.support.ast_helperimportASTTestMixin
2324
fromtest.test_ast.utilsimportto_tuple
2425
fromtest.test_ast.snippetsimport (
@@ -751,25 +752,25 @@ def next(self):
751752
enum._test_simple_enum(_Precedence,ast._Precedence)
752753

753754
@support.cpython_only
755+
@skip_wasi_stack_overflow()
754756
@skip_emscripten_stack_overflow()
755757
deftest_ast_recursion_limit(self):
756-
fail_depth=support.exceeds_recursion_limit()
757-
crash_depth=100_000
758-
success_depth=int(support.get_c_recursion_limit()*0.8)
758+
crash_depth=200_000
759+
success_depth=200
759760
if_testinternalcapiisnotNone:
760761
remaining=_testinternalcapi.get_c_recursion_remaining()
761762
success_depth=min(success_depth,remaining)
762763

763764
defcheck_limit(prefix,repeated):
764765
expect_ok=prefix+repeated*success_depth
765766
ast.parse(expect_ok)
766-
fordepthin (fail_depth,crash_depth):
767-
broken=prefix+repeated*depth
768-
details="Compiling ({!r} + {!r} * {})".format(
769-
prefix,repeated,depth)
770-
withself.assertRaises(RecursionError,msg=details):
771-
withsupport.infinite_recursion():
772-
ast.parse(broken)
767+
768+
broken=prefix+repeated*crash_depth
769+
details="Compiling ({!r} + {!r} * {})".format(
770+
prefix,repeated,crash_depth)
771+
withself.assertRaises(RecursionError,msg=details):
772+
withsupport.infinite_recursion():
773+
ast.parse(broken)
773774

774775
check_limit("a","()")
775776
check_limit("a",".b")

‎Lib/test/test_call.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
importunittest
22
fromtest.supportimport (cpython_only,is_wasi,requires_limited_api,Py_DEBUG,
3-
set_recursion_limit,skip_on_s390x,skip_emscripten_stack_overflow,
3+
set_recursion_limit,skip_on_s390x,exceeds_recursion_limit,skip_emscripten_stack_overflow,
44
skip_if_sanitizer,import_helper)
55
try:
66
import_testcapi
@@ -1064,10 +1064,10 @@ def c_py_recurse(m):
10641064
recurse(90_000)
10651065
withself.assertRaises(RecursionError):
10661066
recurse(101_000)
1067-
c_recurse(100)
1067+
c_recurse(50)
10681068
withself.assertRaises(RecursionError):
10691069
c_recurse(90_000)
1070-
c_py_recurse(90)
1070+
c_py_recurse(50)
10711071
withself.assertRaises(RecursionError):
10721072
c_py_recurse(100_000)
10731073

‎Lib/test/test_capi/test_misc.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -408,7 +408,7 @@ def test_trashcan_subclass(self):
408408
# activated when its tp_dealloc is being called by a subclass
409409
from_testcapiimportMyList
410410
L=None
411-
foriinrange(1000):
411+
foriinrange(100):
412412
L=MyList((L,))
413413

414414
@support.requires_resource('cpu')

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp