Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit54dfa14

Browse files
authored
gh-101765: Fix SystemError / segmentation fault in iter__reduce__ when internal access ofbuiltins.__dict__ exhausts the iterator (#101769)
1 parent89b4c12 commit54dfa14

File tree

9 files changed

+148
-23
lines changed

9 files changed

+148
-23
lines changed

‎Lib/test/test_iter.py‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77
fromtest.supportimportcheck_free_after_iterating,ALWAYS_EQ,NEVER_EQ
88
importpickle
99
importcollections.abc
10+
importfunctools
11+
importcontextlib
12+
importbuiltins
1013

1114
# Test result of triple loop (too big to inline)
1215
TRIPLETS= [(0,0,0), (0,0,1), (0,0,2),
@@ -91,6 +94,12 @@ def __call__(self):
9194
raiseIndexError# Emergency stop
9295
returni
9396

97+
classEmptyIterClass:
98+
def__len__(self):
99+
return0
100+
def__getitem__(self,i):
101+
raiseStopIteration
102+
94103
# Main test suite
95104

96105
classTestCase(unittest.TestCase):
@@ -238,6 +247,78 @@ def test_mutating_seq_class_exhausted_iter(self):
238247
self.assertEqual(list(empit), [5,6])
239248
self.assertEqual(list(a), [0,1,2,3,4,5,6])
240249

250+
deftest_reduce_mutating_builtins_iter(self):
251+
# This is a reproducer of issue #101765
252+
# where iter `__reduce__` calls could lead to a segfault or SystemError
253+
# depending on the order of C argument evaluation, which is undefined
254+
255+
# Backup builtins
256+
builtins_dict=builtins.__dict__
257+
orig= {"iter":iter,"reversed":reversed}
258+
259+
defrun(builtin_name,item,sentinel=None):
260+
it=iter(item)ifsentinelisNoneelseiter(item,sentinel)
261+
262+
classCustomStr:
263+
def__init__(self,name,iterator):
264+
self.name=name
265+
self.iterator=iterator
266+
def__hash__(self):
267+
returnhash(self.name)
268+
def__eq__(self,other):
269+
# Here we exhaust our iterator, possibly changing
270+
# its `it_seq` pointer to NULL
271+
# The `__reduce__` call should correctly get
272+
# the pointers after this call
273+
list(self.iterator)
274+
returnother==self.name
275+
276+
# del is required here
277+
# to not prematurely call __eq__ from
278+
# the hash collision with the old key
279+
delbuiltins_dict[builtin_name]
280+
builtins_dict[CustomStr(builtin_name,it)]=orig[builtin_name]
281+
282+
returnit.__reduce__()
283+
284+
types= [
285+
(EmptyIterClass(),),
286+
(bytes(8),),
287+
(bytearray(8),),
288+
((1,2,3),),
289+
(lambda:0,0),
290+
(tuple[int],)# GenericAlias
291+
]
292+
293+
try:
294+
run_iter=functools.partial(run,"iter")
295+
# The returned value of `__reduce__` should not only be valid
296+
# but also *empty*, as `it` was exhausted during `__eq__`
297+
# i.e "xyz" returns (iter, ("",))
298+
self.assertEqual(run_iter("xyz"), (orig["iter"], ("",)))
299+
self.assertEqual(run_iter([1,2,3]), (orig["iter"], ([],)))
300+
301+
# _PyEval_GetBuiltin is also called for `reversed` in a branch of
302+
# listiter_reduce_general
303+
self.assertEqual(
304+
run("reversed",orig["reversed"](list(range(8)))),
305+
(iter, ([],))
306+
)
307+
308+
forcaseintypes:
309+
self.assertEqual(run_iter(*case), (orig["iter"], ((),)))
310+
finally:
311+
# Restore original builtins
312+
forkey,funcinorig.items():
313+
# need to suppress KeyErrors in case
314+
# a failed test deletes the key without setting anything
315+
withcontextlib.suppress(KeyError):
316+
# del is required here
317+
# to not invoke our custom __eq__ from
318+
# the hash collision with the old key
319+
delbuiltins_dict[key]
320+
builtins_dict[key]=func
321+
241322
# Test a new_style class with __iter__ but no next() method
242323
deftest_new_style_iter_class(self):
243324
classIterClass(object):
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fix SystemError / segmentation fault in iter ``__reduce__`` when internal access of ``builtins.__dict__`` keys mutates the iter object.

‎Objects/bytearrayobject.c‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2391,11 +2391,16 @@ PyDoc_STRVAR(length_hint_doc,
23912391
staticPyObject*
23922392
bytearrayiter_reduce(bytesiterobject*it,PyObject*Py_UNUSED(ignored))
23932393
{
2394+
PyObject*iter=_PyEval_GetBuiltin(&_Py_ID(iter));
2395+
2396+
/* _PyEval_GetBuiltin can invoke arbitrary code,
2397+
* call must be before access of iterator pointers.
2398+
* see issue #101765 */
2399+
23942400
if (it->it_seq!=NULL) {
2395-
returnPy_BuildValue("N(O)n",_PyEval_GetBuiltin(&_Py_ID(iter)),
2396-
it->it_seq,it->it_index);
2401+
returnPy_BuildValue("N(O)n",iter,it->it_seq,it->it_index);
23972402
}else {
2398-
returnPy_BuildValue("N(())",_PyEval_GetBuiltin(&_Py_ID(iter)));
2403+
returnPy_BuildValue("N(())",iter);
23992404
}
24002405
}
24012406

‎Objects/bytesobject.c‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3169,11 +3169,16 @@ PyDoc_STRVAR(length_hint_doc,
31693169
staticPyObject*
31703170
striter_reduce(striterobject*it,PyObject*Py_UNUSED(ignored))
31713171
{
3172+
PyObject*iter=_PyEval_GetBuiltin(&_Py_ID(iter));
3173+
3174+
/* _PyEval_GetBuiltin can invoke arbitrary code,
3175+
* call must be before access of iterator pointers.
3176+
* see issue #101765 */
3177+
31723178
if (it->it_seq!=NULL) {
3173-
returnPy_BuildValue("N(O)n",_PyEval_GetBuiltin(&_Py_ID(iter)),
3174-
it->it_seq,it->it_index);
3179+
returnPy_BuildValue("N(O)n",iter,it->it_seq,it->it_index);
31753180
}else {
3176-
returnPy_BuildValue("N(())",_PyEval_GetBuiltin(&_Py_ID(iter)));
3181+
returnPy_BuildValue("N(())",iter);
31773182
}
31783183
}
31793184

‎Objects/genericaliasobject.c‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -877,8 +877,17 @@ ga_iter_clear(PyObject *self) {
877877
staticPyObject*
878878
ga_iter_reduce(PyObject*self,PyObject*Py_UNUSED(ignored))
879879
{
880+
PyObject*iter=_PyEval_GetBuiltin(&_Py_ID(iter));
880881
gaiterobject*gi= (gaiterobject*)self;
881-
returnPy_BuildValue("N(O)",_PyEval_GetBuiltin(&_Py_ID(iter)),gi->obj);
882+
883+
/* _PyEval_GetBuiltin can invoke arbitrary code,
884+
* call must be before access of iterator pointers.
885+
* see issue #101765 */
886+
887+
if (gi->obj)
888+
returnPy_BuildValue("N(O)",iter,gi->obj);
889+
else
890+
returnPy_BuildValue("N(())",iter);
882891
}
883892

884893
staticPyMethodDefga_iter_methods[]= {

‎Objects/iterobject.c‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -102,11 +102,16 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(
102102
staticPyObject*
103103
iter_reduce(seqiterobject*it,PyObject*Py_UNUSED(ignored))
104104
{
105+
PyObject*iter=_PyEval_GetBuiltin(&_Py_ID(iter));
106+
107+
/* _PyEval_GetBuiltin can invoke arbitrary code,
108+
* call must be before access of iterator pointers.
109+
* see issue #101765 */
110+
105111
if (it->it_seq!=NULL)
106-
returnPy_BuildValue("N(O)n",_PyEval_GetBuiltin(&_Py_ID(iter)),
107-
it->it_seq,it->it_index);
112+
returnPy_BuildValue("N(O)n",iter,it->it_seq,it->it_index);
108113
else
109-
returnPy_BuildValue("N(())",_PyEval_GetBuiltin(&_Py_ID(iter)));
114+
returnPy_BuildValue("N(())",iter);
110115
}
111116

112117
PyDoc_STRVAR(reduce_doc,"Return state information for pickling.");
@@ -239,11 +244,16 @@ calliter_iternext(calliterobject *it)
239244
staticPyObject*
240245
calliter_reduce(calliterobject*it,PyObject*Py_UNUSED(ignored))
241246
{
247+
PyObject*iter=_PyEval_GetBuiltin(&_Py_ID(iter));
248+
249+
/* _PyEval_GetBuiltin can invoke arbitrary code,
250+
* call must be before access of iterator pointers.
251+
* see issue #101765 */
252+
242253
if (it->it_callable!=NULL&&it->it_sentinel!=NULL)
243-
returnPy_BuildValue("N(OO)",_PyEval_GetBuiltin(&_Py_ID(iter)),
244-
it->it_callable,it->it_sentinel);
254+
returnPy_BuildValue("N(OO)",iter,it->it_callable,it->it_sentinel);
245255
else
246-
returnPy_BuildValue("N(())",_PyEval_GetBuiltin(&_Py_ID(iter)));
256+
returnPy_BuildValue("N(())",iter);
247257
}
248258

249259
staticPyMethodDefcalliter_methods[]= {

‎Objects/listobject.c‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3444,18 +3444,22 @@ listiter_reduce_general(void *_it, int forward)
34443444
{
34453445
PyObject*list;
34463446

3447+
/* _PyEval_GetBuiltin can invoke arbitrary code,
3448+
* call must be before access of iterator pointers.
3449+
* see issue #101765 */
3450+
34473451
/* the objects are not the same, index is of different types! */
34483452
if (forward) {
3453+
PyObject*iter=_PyEval_GetBuiltin(&_Py_ID(iter));
34493454
_PyListIterObject*it= (_PyListIterObject*)_it;
34503455
if (it->it_seq) {
3451-
returnPy_BuildValue("N(O)n",_PyEval_GetBuiltin(&_Py_ID(iter)),
3452-
it->it_seq,it->it_index);
3456+
returnPy_BuildValue("N(O)n",iter,it->it_seq,it->it_index);
34533457
}
34543458
}else {
3459+
PyObject*reversed=_PyEval_GetBuiltin(&_Py_ID(reversed));
34553460
listreviterobject*it= (listreviterobject*)_it;
34563461
if (it->it_seq) {
3457-
returnPy_BuildValue("N(O)n",_PyEval_GetBuiltin(&_Py_ID(reversed)),
3458-
it->it_seq,it->it_index);
3462+
returnPy_BuildValue("N(O)n",reversed,it->it_seq,it->it_index);
34593463
}
34603464
}
34613465
/* empty iterator, create an empty list */

‎Objects/tupleobject.c‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1048,11 +1048,16 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(
10481048
staticPyObject*
10491049
tupleiter_reduce(_PyTupleIterObject*it,PyObject*Py_UNUSED(ignored))
10501050
{
1051+
PyObject*iter=_PyEval_GetBuiltin(&_Py_ID(iter));
1052+
1053+
/* _PyEval_GetBuiltin can invoke arbitrary code,
1054+
* call must be before access of iterator pointers.
1055+
* see issue #101765 */
1056+
10511057
if (it->it_seq)
1052-
returnPy_BuildValue("N(O)n",_PyEval_GetBuiltin(&_Py_ID(iter)),
1053-
it->it_seq,it->it_index);
1058+
returnPy_BuildValue("N(O)n",iter,it->it_seq,it->it_index);
10541059
else
1055-
returnPy_BuildValue("N(())",_PyEval_GetBuiltin(&_Py_ID(iter)));
1060+
returnPy_BuildValue("N(())",iter);
10561061
}
10571062

10581063
staticPyObject*

‎Objects/unicodeobject.c‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14784,14 +14784,19 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(
1478414784
staticPyObject*
1478514785
unicodeiter_reduce(unicodeiterobject*it,PyObject*Py_UNUSED(ignored))
1478614786
{
14787+
PyObject*iter=_PyEval_GetBuiltin(&_Py_ID(iter));
14788+
14789+
/* _PyEval_GetBuiltin can invoke arbitrary code,
14790+
* call must be before access of iterator pointers.
14791+
* see issue #101765 */
14792+
1478714793
if (it->it_seq!=NULL) {
14788-
returnPy_BuildValue("N(O)n",_PyEval_GetBuiltin(&_Py_ID(iter)),
14789-
it->it_seq,it->it_index);
14794+
returnPy_BuildValue("N(O)n",iter,it->it_seq,it->it_index);
1479014795
}else {
1479114796
PyObject*u=unicode_new_empty();
1479214797
if (u==NULL)
1479314798
returnNULL;
14794-
returnPy_BuildValue("N(N)",_PyEval_GetBuiltin(&_Py_ID(iter)),u);
14799+
returnPy_BuildValue("N(N)",iter,u);
1479514800
}
1479614801
}
1479714802

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp