Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit87be8d9

Browse files
gh-100227: Make the Global Interned Dict Safe for Isolated Interpreters (gh-102925)
This is effectively two changes. The first (the bulk of the change) is where we add _Py_AddToGlobalDict() (and _PyRuntime.cached_objects.main_tstate, etc.). The second (much smaller) change is where we update PyUnicode_InternInPlace() to use _Py_AddToGlobalDict() instead of calling PyDict_SetDefault() directly.Basically, _Py_AddToGlobalDict() is a wrapper around PyDict_SetDefault() that should be used whenever we need to add a value to a runtime-global dict object (in the few cases where we are leaving the container global rather than moving it to PyInterpreterState, e.g. the interned strings dict). _Py_AddToGlobalDict() does all the necessary work to make sure the target global dict is shared safely between isolated interpreters. This is especially important as we move the obmalloc state to each interpreter (gh-101660), as well as, potentially, the GIL (PEP 684).#100227
1 parent8709697 commit87be8d9

File tree

7 files changed

+204
-30
lines changed

7 files changed

+204
-30
lines changed

‎Include/internal/pycore_global_objects.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,10 @@ extern "C" {
2828

2929
struct_Py_cached_objects {
3030
PyObject*interned_strings;
31+
/* A thread state tied to the main interpreter,
32+
used exclusively for when a global object (e.g. interned strings)
33+
is resized (i.e. deallocated + allocated) from an arbitrary thread. */
34+
PyThreadStatemain_tstate;
3135
};
3236

3337
#define_Py_GLOBAL_OBJECT(NAME) \

‎Include/internal/pycore_pystate.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,11 @@ PyAPI_FUNC(void) _PyThreadState_Init(
127127
PyThreadState*tstate);
128128
PyAPI_FUNC(void)_PyThreadState_DeleteExcept(PyThreadState*tstate);
129129

130+
externvoid_PyThreadState_InitDetached(PyThreadState*,PyInterpreterState*);
131+
externvoid_PyThreadState_ClearDetached(PyThreadState*);
132+
133+
externPyObject*_Py_AddToGlobalDict(PyObject*,PyObject*,PyObject*);
134+
130135

131136
staticinlinevoid
132137
_PyThreadState_UpdateTracingState(PyThreadState*tstate)

‎Include/internal/pycore_runtime_init.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ extern PyTypeObject _PyExc_MemoryError;
5959
.types = { \
6060
.next_version_tag = 1, \
6161
}, \
62+
.cached_objects = { \
63+
.main_tstate = _PyThreadState_INIT, \
64+
}, \
6265
.static_objects = { \
6366
.singletons = { \
6467
.small_ints = _Py_small_ints_INIT, \

‎Include/internal/pycore_unicodeobject.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ struct _Py_unicode_runtime_ids {
3434

3535
struct_Py_unicode_runtime_state {
3636
struct_Py_unicode_runtime_idsids;
37+
/* The interned dict is at _PyRuntime.cached_objects.interned_strings. */
3738
};
3839

3940
/* fs_codec.encoding is initialized to NULL.

‎Objects/unicodeobject.c‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14609,16 +14609,11 @@ PyUnicode_InternInPlace(PyObject **p)
1460914609
}
1461014610

1461114611
PyObject*interned=get_interned_dict();
14612-
assert(interned!=NULL);
14613-
14614-
PyObject*t=PyDict_SetDefault(interned,s,s);
14615-
if (t==NULL) {
14616-
PyErr_Clear();
14617-
return;
14618-
}
14619-
14612+
PyObject*t=_Py_AddToGlobalDict(interned,s,s);
1462014613
if (t!=s) {
14621-
Py_SETREF(*p,Py_NewRef(t));
14614+
if (t!=NULL) {
14615+
Py_SETREF(*p,Py_NewRef(t));
14616+
}
1462214617
return;
1462314618
}
1462414619

‎Python/pylifecycle.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -636,6 +636,8 @@ pycore_create_interpreter(_PyRuntimeState *runtime,
636636
returnstatus;
637637
}
638638

639+
_PyThreadState_InitDetached(&runtime->cached_objects.main_tstate,interp);
640+
639641
*tstate_p=tstate;
640642
return_PyStatus_OK();
641643
}
@@ -1932,6 +1934,8 @@ Py_FinalizeEx(void)
19321934
// XXX Do this sooner during finalization.
19331935
// XXX Ensure finalizer errors are handled properly.
19341936

1937+
_PyThreadState_ClearDetached(&runtime->cached_objects.main_tstate);
1938+
19351939
finalize_interp_clear(tstate);
19361940
finalize_interp_delete(tstate->interp);
19371941

‎Python/pystate.c‎

Lines changed: 183 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -565,6 +565,124 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime)
565565
#endif
566566

567567

568+
//---------------
569+
// global objects
570+
//---------------
571+
572+
/* The global objects thread state is meant to be used in a very limited
573+
way and should not be used to actually run any Python code. */
574+
575+
staticPyThreadState*
576+
bind_global_objects_state(_PyRuntimeState*runtime)
577+
{
578+
PyThreadState*main_tstate=&runtime->cached_objects.main_tstate;
579+
580+
bind_tstate(main_tstate);
581+
/* Unlike _PyThreadState_Bind(), we do not modify gilstate TSS. */
582+
583+
returnmain_tstate;
584+
}
585+
586+
staticvoid
587+
unbind_global_objects_state(_PyRuntimeState*runtime)
588+
{
589+
PyThreadState*main_tstate=&runtime->cached_objects.main_tstate;
590+
assert(tstate_is_alive(main_tstate));
591+
assert(!main_tstate->_status.active);
592+
assert(gilstate_tss_get(runtime)!=main_tstate);
593+
594+
unbind_tstate(main_tstate);
595+
596+
/* This thread state may be bound/unbound repeatedly,
597+
so we must erase evidence that it was ever bound (or unbound). */
598+
main_tstate->_status.bound=0;
599+
main_tstate->_status.unbound=0;
600+
601+
/* We must fully unlink the thread state from any OS thread,
602+
to allow it to be bound more than once. */
603+
main_tstate->thread_id=0;
604+
#ifdefPY_HAVE_THREAD_NATIVE_ID
605+
main_tstate->native_thread_id=0;
606+
#endif
607+
}
608+
609+
staticinlinevoid
610+
acquire_global_objects_lock(_PyRuntimeState*runtime)
611+
{
612+
/* For now we can rely on the GIL, so we don't actually
613+
acquire a global lock here. */
614+
assert(current_fast_get(runtime)!=NULL);
615+
}
616+
617+
staticinlinevoid
618+
release_global_objects_lock(_PyRuntimeState*runtime)
619+
{
620+
/* For now we can rely on the GIL, so we don't actually
621+
release a global lock here. */
622+
assert(current_fast_get(runtime)!=NULL);
623+
}
624+
625+
PyObject*
626+
_Py_AddToGlobalDict(PyObject*dict,PyObject*key,PyObject*value)
627+
{
628+
assert(dict!=NULL);
629+
assert(PyDict_CheckExact(dict));
630+
631+
/* All global objects are stored in _PyRuntime
632+
and owned by the main interpreter. */
633+
_PyRuntimeState*runtime=&_PyRuntime;
634+
PyThreadState*curts=current_fast_get(runtime);
635+
PyInterpreterState*interp=curts->interp;
636+
assert(interp!=NULL);// The GIL must be held.
637+
638+
/* Due to interpreter isolation we must hold a global lock,
639+
starting at this point and ending before we return.
640+
Note that the operations in this function are very fucused
641+
and we should not expect any reentrancy. */
642+
acquire_global_objects_lock(runtime);
643+
644+
/* Swap to the main interpreter, if necessary. */
645+
PyThreadState*oldts=NULL;
646+
if (!_Py_IsMainInterpreter(interp)) {
647+
PyThreadState*main_tstate=bind_global_objects_state(runtime);
648+
649+
oldts=_PyThreadState_Swap(runtime,main_tstate);
650+
assert(oldts!=NULL);
651+
assert(!_Py_IsMainInterpreter(oldts->interp));
652+
653+
/* The limitations of the global objects thread state apply
654+
from this point to the point we swap back to oldts. */
655+
}
656+
657+
/* This might trigger a resize, which is why we must "acquire"
658+
the global object state. Also note that PyDict_SetDefault()
659+
must be compatible with our reentrancy and global objects state
660+
constraints. */
661+
PyObject*actual=PyDict_SetDefault(dict,key,value);
662+
if (actual==NULL) {
663+
/* Raising an exception from one interpreter in another
664+
is problematic, so we clear it and let the caller deal
665+
with the returned NULL. */
666+
assert(PyErr_ExceptionMatches(PyExc_MemoryError));
667+
PyErr_Clear();
668+
}
669+
670+
/* Swap back, it it wasn't in the main interpreter already. */
671+
if (oldts!=NULL) {
672+
// The returned tstate should be _PyRuntime.cached_objects.main_tstate.
673+
_PyThreadState_Swap(runtime,oldts);
674+
675+
unbind_global_objects_state(runtime);
676+
}
677+
678+
release_global_objects_lock(runtime);
679+
680+
// XXX Immortalize the key and value.
681+
682+
returnactual;
683+
}
684+
685+
568686
/*************************************/
569687
/* the per-interpreter runtime state */
570688
/*************************************/
@@ -1217,8 +1335,7 @@ free_threadstate(PyThreadState *tstate)
12171335

12181336
staticvoid
12191337
init_threadstate(PyThreadState*tstate,
1220-
PyInterpreterState*interp,uint64_tid,
1221-
PyThreadState*next)
1338+
PyInterpreterState*interp,uint64_tid)
12221339
{
12231340
if (tstate->_status.initialized) {
12241341
Py_FatalError("thread state already initialized");
@@ -1227,18 +1344,13 @@ init_threadstate(PyThreadState *tstate,
12271344
assert(interp!=NULL);
12281345
tstate->interp=interp;
12291346

1347+
// next/prev are set in add_threadstate().
1348+
assert(tstate->next==NULL);
1349+
assert(tstate->prev==NULL);
1350+
12301351
assert(id>0);
12311352
tstate->id=id;
12321353

1233-
assert(interp->threads.head==tstate);
1234-
assert((next!=NULL&&id!=1)|| (next==NULL&&id==1));
1235-
if (next!=NULL) {
1236-
assert(next->prev==NULL||next->prev==tstate);
1237-
next->prev=tstate;
1238-
}
1239-
tstate->next=next;
1240-
assert(tstate->prev==NULL);
1241-
12421354
// thread_id and native_thread_id are set in bind_tstate().
12431355

12441356
tstate->py_recursion_limit=interp->ceval.recursion_limit,
@@ -1259,6 +1371,22 @@ init_threadstate(PyThreadState *tstate,
12591371
tstate->_status.initialized=1;
12601372
}
12611373

1374+
staticvoid
1375+
add_threadstate(PyInterpreterState*interp,PyThreadState*tstate,
1376+
PyThreadState*next)
1377+
{
1378+
assert(interp->threads.head!=tstate);
1379+
assert((next!=NULL&&tstate->id!=1)||
1380+
(next==NULL&&tstate->id==1));
1381+
if (next!=NULL) {
1382+
assert(next->prev==NULL||next->prev==tstate);
1383+
next->prev=tstate;
1384+
}
1385+
tstate->next=next;
1386+
assert(tstate->prev==NULL);
1387+
interp->threads.head=tstate;
1388+
}
1389+
12621390
staticPyThreadState*
12631391
new_threadstate(PyInterpreterState*interp)
12641392
{
@@ -1298,9 +1426,9 @@ new_threadstate(PyInterpreterState *interp)
12981426
&initial._main_interpreter._initial_thread,
12991427
sizeof(*tstate));
13001428
}
1301-
interp->threads.head=tstate;
13021429

1303-
init_threadstate(tstate,interp,id,old_head);
1430+
init_threadstate(tstate,interp,id);
1431+
add_threadstate(interp,tstate,old_head);
13041432

13051433
HEAD_UNLOCK(runtime);
13061434
if (!used_newtstate) {
@@ -1347,6 +1475,33 @@ _PyThreadState_Init(PyThreadState *tstate)
13471475
Py_FatalError("_PyThreadState_Init() is for internal use only");
13481476
}
13491477

1478+
void
1479+
_PyThreadState_InitDetached(PyThreadState*tstate,PyInterpreterState*interp)
1480+
{
1481+
_PyRuntimeState*runtime=interp->runtime;
1482+
1483+
HEAD_LOCK(runtime);
1484+
interp->threads.next_unique_id+=1;
1485+
uint64_tid=interp->threads.next_unique_id;
1486+
HEAD_UNLOCK(runtime);
1487+
1488+
init_threadstate(tstate,interp,id);
1489+
// We do not call add_threadstate().
1490+
}
1491+
1492+
1493+
staticvoid
1494+
clear_datastack(PyThreadState*tstate)
1495+
{
1496+
_PyStackChunk*chunk=tstate->datastack_chunk;
1497+
tstate->datastack_chunk=NULL;
1498+
while (chunk!=NULL) {
1499+
_PyStackChunk*prev=chunk->previous;
1500+
_PyObject_VirtualFree(chunk,chunk->size);
1501+
chunk=prev;
1502+
}
1503+
}
1504+
13501505
void
13511506
PyThreadState_Clear(PyThreadState*tstate)
13521507
{
@@ -1421,7 +1576,6 @@ PyThreadState_Clear(PyThreadState *tstate)
14211576
// XXX Do it as early in the function as possible.
14221577
}
14231578

1424-
14251579
/* Common code for PyThreadState_Delete() and PyThreadState_DeleteCurrent() */
14261580
staticvoid
14271581
tstate_delete_common(PyThreadState*tstate)
@@ -1454,17 +1608,25 @@ tstate_delete_common(PyThreadState *tstate)
14541608
unbind_tstate(tstate);
14551609

14561610
// XXX Move to PyThreadState_Clear()?
1457-
_PyStackChunk*chunk=tstate->datastack_chunk;
1458-
tstate->datastack_chunk=NULL;
1459-
while (chunk!=NULL) {
1460-
_PyStackChunk*prev=chunk->previous;
1461-
_PyObject_VirtualFree(chunk,chunk->size);
1462-
chunk=prev;
1463-
}
1611+
clear_datastack(tstate);
14641612

14651613
tstate->_status.finalized=1;
14661614
}
14671615

1616+
void
1617+
_PyThreadState_ClearDetached(PyThreadState*tstate)
1618+
{
1619+
assert(!tstate->_status.bound);
1620+
assert(!tstate->_status.bound_gilstate);
1621+
assert(tstate->datastack_chunk==NULL);
1622+
assert(tstate->thread_id==0);
1623+
assert(tstate->native_thread_id==0);
1624+
assert(tstate->next==NULL);
1625+
assert(tstate->prev==NULL);
1626+
1627+
PyThreadState_Clear(tstate);
1628+
clear_datastack(tstate);
1629+
}
14681630

14691631
staticvoid
14701632
zapthreads(PyInterpreterState*interp)

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp