Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit5d46122

Browse files
authored
[3.11]gh-101765: Fix SystemError / segmentation fault in iter__reduce__ when internal access ofbuiltins.__dict__ exhausts the iterator (GH-101769) (#102228)
(cherry picked from commit54dfa14)
1 parent9da3e7f commit5d46122

File tree

9 files changed

+148
-23
lines changed

9 files changed

+148
-23
lines changed

‎Lib/test/test_iter.py‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77
fromtest.supportimportcheck_free_after_iterating,ALWAYS_EQ,NEVER_EQ
88
importpickle
99
importcollections.abc
10+
importfunctools
11+
importcontextlib
12+
importbuiltins
1013

1114
# Test result of triple loop (too big to inline)
1215
TRIPLETS= [(0,0,0), (0,0,1), (0,0,2),
@@ -91,6 +94,12 @@ def __call__(self):
9194
raiseIndexError# Emergency stop
9295
returni
9396

97+
classEmptyIterClass:
98+
def__len__(self):
99+
return0
100+
def__getitem__(self,i):
101+
raiseStopIteration
102+
94103
# Main test suite
95104

96105
classTestCase(unittest.TestCase):
@@ -238,6 +247,78 @@ def test_mutating_seq_class_exhausted_iter(self):
238247
self.assertEqual(list(empit), [5,6])
239248
self.assertEqual(list(a), [0,1,2,3,4,5,6])
240249

250+
deftest_reduce_mutating_builtins_iter(self):
251+
# This is a reproducer of issue #101765
252+
# where iter `__reduce__` calls could lead to a segfault or SystemError
253+
# depending on the order of C argument evaluation, which is undefined
254+
255+
# Backup builtins
256+
builtins_dict=builtins.__dict__
257+
orig= {"iter":iter,"reversed":reversed}
258+
259+
defrun(builtin_name,item,sentinel=None):
260+
it=iter(item)ifsentinelisNoneelseiter(item,sentinel)
261+
262+
classCustomStr:
263+
def__init__(self,name,iterator):
264+
self.name=name
265+
self.iterator=iterator
266+
def__hash__(self):
267+
returnhash(self.name)
268+
def__eq__(self,other):
269+
# Here we exhaust our iterator, possibly changing
270+
# its `it_seq` pointer to NULL
271+
# The `__reduce__` call should correctly get
272+
# the pointers after this call
273+
list(self.iterator)
274+
returnother==self.name
275+
276+
# del is required here
277+
# to not prematurely call __eq__ from
278+
# the hash collision with the old key
279+
delbuiltins_dict[builtin_name]
280+
builtins_dict[CustomStr(builtin_name,it)]=orig[builtin_name]
281+
282+
returnit.__reduce__()
283+
284+
types= [
285+
(EmptyIterClass(),),
286+
(bytes(8),),
287+
(bytearray(8),),
288+
((1,2,3),),
289+
(lambda:0,0),
290+
(tuple[int],)# GenericAlias
291+
]
292+
293+
try:
294+
run_iter=functools.partial(run,"iter")
295+
# The returned value of `__reduce__` should not only be valid
296+
# but also *empty*, as `it` was exhausted during `__eq__`
297+
# i.e "xyz" returns (iter, ("",))
298+
self.assertEqual(run_iter("xyz"), (orig["iter"], ("",)))
299+
self.assertEqual(run_iter([1,2,3]), (orig["iter"], ([],)))
300+
301+
# _PyEval_GetBuiltin is also called for `reversed` in a branch of
302+
# listiter_reduce_general
303+
self.assertEqual(
304+
run("reversed",orig["reversed"](list(range(8)))),
305+
(iter, ([],))
306+
)
307+
308+
forcaseintypes:
309+
self.assertEqual(run_iter(*case), (orig["iter"], ((),)))
310+
finally:
311+
# Restore original builtins
312+
forkey,funcinorig.items():
313+
# need to suppress KeyErrors in case
314+
# a failed test deletes the key without setting anything
315+
withcontextlib.suppress(KeyError):
316+
# del is required here
317+
# to not invoke our custom __eq__ from
318+
# the hash collision with the old key
319+
delbuiltins_dict[key]
320+
builtins_dict[key]=func
321+
241322
# Test a new_style class with __iter__ but no next() method
242323
deftest_new_style_iter_class(self):
243324
classIterClass(object):
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fix SystemError / segmentation fault in iter ``__reduce__`` when internal access of ``builtins.__dict__`` keys mutates the iter object.

‎Objects/bytearrayobject.c‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2394,11 +2394,16 @@ PyDoc_STRVAR(length_hint_doc,
23942394
staticPyObject*
23952395
bytearrayiter_reduce(bytesiterobject*it,PyObject*Py_UNUSED(ignored))
23962396
{
2397+
PyObject*iter=_PyEval_GetBuiltin(&_Py_ID(iter));
2398+
2399+
/* _PyEval_GetBuiltin can invoke arbitrary code,
2400+
* call must be before access of iterator pointers.
2401+
* see issue #101765 */
2402+
23972403
if (it->it_seq!=NULL) {
2398-
returnPy_BuildValue("N(O)n",_PyEval_GetBuiltin(&_Py_ID(iter)),
2399-
it->it_seq,it->it_index);
2404+
returnPy_BuildValue("N(O)n",iter,it->it_seq,it->it_index);
24002405
}else {
2401-
returnPy_BuildValue("N(())",_PyEval_GetBuiltin(&_Py_ID(iter)));
2406+
returnPy_BuildValue("N(())",iter);
24022407
}
24032408
}
24042409

‎Objects/bytesobject.c‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3191,11 +3191,16 @@ PyDoc_STRVAR(length_hint_doc,
31913191
staticPyObject*
31923192
striter_reduce(striterobject*it,PyObject*Py_UNUSED(ignored))
31933193
{
3194+
PyObject*iter=_PyEval_GetBuiltin(&_Py_ID(iter));
3195+
3196+
/* _PyEval_GetBuiltin can invoke arbitrary code,
3197+
* call must be before access of iterator pointers.
3198+
* see issue #101765 */
3199+
31943200
if (it->it_seq!=NULL) {
3195-
returnPy_BuildValue("N(O)n",_PyEval_GetBuiltin(&_Py_ID(iter)),
3196-
it->it_seq,it->it_index);
3201+
returnPy_BuildValue("N(O)n",iter,it->it_seq,it->it_index);
31973202
}else {
3198-
returnPy_BuildValue("N(())",_PyEval_GetBuiltin(&_Py_ID(iter)));
3203+
returnPy_BuildValue("N(())",iter);
31993204
}
32003205
}
32013206

‎Objects/genericaliasobject.c‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -885,8 +885,17 @@ ga_iter_clear(PyObject *self) {
885885
staticPyObject*
886886
ga_iter_reduce(PyObject*self,PyObject*Py_UNUSED(ignored))
887887
{
888+
PyObject*iter=_PyEval_GetBuiltin(&_Py_ID(iter));
888889
gaiterobject*gi= (gaiterobject*)self;
889-
returnPy_BuildValue("N(O)",_PyEval_GetBuiltin(&_Py_ID(iter)),gi->obj);
890+
891+
/* _PyEval_GetBuiltin can invoke arbitrary code,
892+
* call must be before access of iterator pointers.
893+
* see issue #101765 */
894+
895+
if (gi->obj)
896+
returnPy_BuildValue("N(O)",iter,gi->obj);
897+
else
898+
returnPy_BuildValue("N(())",iter);
890899
}
891900

892901
staticPyMethodDefga_iter_methods[]= {

‎Objects/iterobject.c‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -103,11 +103,16 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(
103103
staticPyObject*
104104
iter_reduce(seqiterobject*it,PyObject*Py_UNUSED(ignored))
105105
{
106+
PyObject*iter=_PyEval_GetBuiltin(&_Py_ID(iter));
107+
108+
/* _PyEval_GetBuiltin can invoke arbitrary code,
109+
* call must be before access of iterator pointers.
110+
* see issue #101765 */
111+
106112
if (it->it_seq!=NULL)
107-
returnPy_BuildValue("N(O)n",_PyEval_GetBuiltin(&_Py_ID(iter)),
108-
it->it_seq,it->it_index);
113+
returnPy_BuildValue("N(O)n",iter,it->it_seq,it->it_index);
109114
else
110-
returnPy_BuildValue("N(())",_PyEval_GetBuiltin(&_Py_ID(iter)));
115+
returnPy_BuildValue("N(())",iter);
111116
}
112117

113118
PyDoc_STRVAR(reduce_doc,"Return state information for pickling.");
@@ -242,11 +247,16 @@ calliter_iternext(calliterobject *it)
242247
staticPyObject*
243248
calliter_reduce(calliterobject*it,PyObject*Py_UNUSED(ignored))
244249
{
250+
PyObject*iter=_PyEval_GetBuiltin(&_Py_ID(iter));
251+
252+
/* _PyEval_GetBuiltin can invoke arbitrary code,
253+
* call must be before access of iterator pointers.
254+
* see issue #101765 */
255+
245256
if (it->it_callable!=NULL&&it->it_sentinel!=NULL)
246-
returnPy_BuildValue("N(OO)",_PyEval_GetBuiltin(&_Py_ID(iter)),
247-
it->it_callable,it->it_sentinel);
257+
returnPy_BuildValue("N(OO)",iter,it->it_callable,it->it_sentinel);
248258
else
249-
returnPy_BuildValue("N(())",_PyEval_GetBuiltin(&_Py_ID(iter)));
259+
returnPy_BuildValue("N(())",iter);
250260
}
251261

252262
staticPyMethodDefcalliter_methods[]= {

‎Objects/listobject.c‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3452,18 +3452,22 @@ listiter_reduce_general(void *_it, int forward)
34523452
{
34533453
PyObject*list;
34543454

3455+
/* _PyEval_GetBuiltin can invoke arbitrary code,
3456+
* call must be before access of iterator pointers.
3457+
* see issue #101765 */
3458+
34553459
/* the objects are not the same, index is of different types! */
34563460
if (forward) {
3461+
PyObject*iter=_PyEval_GetBuiltin(&_Py_ID(iter));
34573462
listiterobject*it= (listiterobject*)_it;
34583463
if (it->it_seq) {
3459-
returnPy_BuildValue("N(O)n",_PyEval_GetBuiltin(&_Py_ID(iter)),
3460-
it->it_seq,it->it_index);
3464+
returnPy_BuildValue("N(O)n",iter,it->it_seq,it->it_index);
34613465
}
34623466
}else {
3467+
PyObject*reversed=_PyEval_GetBuiltin(&_Py_ID(reversed));
34633468
listreviterobject*it= (listreviterobject*)_it;
34643469
if (it->it_seq) {
3465-
returnPy_BuildValue("N(O)n",_PyEval_GetBuiltin(&_Py_ID(reversed)),
3466-
it->it_seq,it->it_index);
3470+
returnPy_BuildValue("N(O)n",reversed,it->it_seq,it->it_index);
34673471
}
34683472
}
34693473
/* empty iterator, create an empty list */

‎Objects/tupleobject.c‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1072,11 +1072,16 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(
10721072
staticPyObject*
10731073
tupleiter_reduce(tupleiterobject*it,PyObject*Py_UNUSED(ignored))
10741074
{
1075+
PyObject*iter=_PyEval_GetBuiltin(&_Py_ID(iter));
1076+
1077+
/* _PyEval_GetBuiltin can invoke arbitrary code,
1078+
* call must be before access of iterator pointers.
1079+
* see issue #101765 */
1080+
10751081
if (it->it_seq)
1076-
returnPy_BuildValue("N(O)n",_PyEval_GetBuiltin(&_Py_ID(iter)),
1077-
it->it_seq,it->it_index);
1082+
returnPy_BuildValue("N(O)n",iter,it->it_seq,it->it_index);
10781083
else
1079-
returnPy_BuildValue("N(())",_PyEval_GetBuiltin(&_Py_ID(iter)));
1084+
returnPy_BuildValue("N(())",iter);
10801085
}
10811086

10821087
staticPyObject*

‎Objects/unicodeobject.c‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15757,14 +15757,19 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(
1575715757
staticPyObject*
1575815758
unicodeiter_reduce(unicodeiterobject*it,PyObject*Py_UNUSED(ignored))
1575915759
{
15760+
PyObject*iter=_PyEval_GetBuiltin(&_Py_ID(iter));
15761+
15762+
/* _PyEval_GetBuiltin can invoke arbitrary code,
15763+
* call must be before access of iterator pointers.
15764+
* see issue #101765 */
15765+
1576015766
if (it->it_seq!=NULL) {
15761-
returnPy_BuildValue("N(O)n",_PyEval_GetBuiltin(&_Py_ID(iter)),
15762-
it->it_seq,it->it_index);
15767+
returnPy_BuildValue("N(O)n",iter,it->it_seq,it->it_index);
1576315768
}else {
1576415769
PyObject*u= (PyObject*)_PyUnicode_New(0);
1576515770
if (u==NULL)
1576615771
returnNULL;
15767-
returnPy_BuildValue("N(N)",_PyEval_GetBuiltin(&_Py_ID(iter)),u);
15772+
returnPy_BuildValue("N(N)",iter,u);
1576815773
}
1576915774
}
1577015775

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp