Mar 23, 2024 · Mar 23, 2024 · Mar 23, 2024 · Mar 23, 2024 · Mar 26, 2024 · Mar 26, 2024
diff --git a/Include/cpython/tupleobject.h b/Include/cpython/tupleobject.h
    PyTupleObject *tuple = _PyTuple_CAST(op);
    assert(0 <= index);
    assert(index < Py_SIZE(tuple));
 #ifdefPy_TEST_TAG
 #ifdefPy_TAG_TEST
    // Make sure it's not a tagged pointer
    assert(((uintptr_t)op &Py_TEST_TAG) == 0);
    assert(((uintptr_t)op &Py_TAG_TEST) == 0);
 #endif
    tuple->ob_item[index] = value;
 }
diff --git a/Include/internal/pycore_tagged.h b/Include/internal/pycore_tagged.h
 } _PyStackRef;

 #define Py_TAG (0b0)
 #define Py_TEST_TAG (0b1)
 #define Py_TAG_TEST (0b1)
 #define Py_TAG_DEFERRED (0b1)

 #if defined(Py_TEST_TAG)
    #define Py_STACK_UNTAG_BORROWED(tagged) ((PyObject *)(uintptr_t)((tagged).bits & (~Py_TEST_TAG)))
 #if defined(Py_TAG_TEST)
    #define Py_STACK_UNTAG_BORROWED(tagged) ((PyObject *)(uintptr_t)((tagged).bits & (~Py_TAG_TEST)))
 #elif defined(Py_GIL_DISABLED)
    #define Py_STACK_UNTAG_BORROWED(tagged) ((PyObject *)((tagged).bits & (~Py_TAG)))
 #else
    #define Py_STACK_UNTAG_BORROWED(tagged) ((PyObject *)(uintptr_t)((tagged).bits))
 #endif

 #if defined(Py_TEST_TAG)
    #define Py_STACK_TAG(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj) |Py_TEST_TAG)})
 #if defined(Py_TAG_TEST)
    #define Py_STACK_TAG(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj) |Py_TAG_TEST)})
 #elif defined(Py_GIL_DISABLED)
    #define Py_STACK_TAG(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj) | Py_TAG}))
 #else
    #define Py_STACK_TAG(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj))})
 #endif

 static inline PyObject *
 _Py_Stack_Untag_Owned(_PyStackRef tagged) {
 #if defined(Py_TAG_TEST)
    // Test tag takes up same tag as deferred tag!
    Py_UNREACHABLE();
 #endif
    if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) {
        return Py_NewRef(Py_STACK_UNTAG_BORROWED(tagged));
    }
    return Py_STACK_UNTAG_BORROWED(tagged);
 }

 #if defined(Py_TAG_TEST)
    #define Py_STACK_UNTAG_OWNED(tagged) Py_STACK_UNTAG_BORROWED(tagged)
 #elif defined(Py_GIL_DISABLED)
    #define Py_STACK_UNTAG_OWNED(tagged) _Py_Stack_Untag_Owned(tagged)
 #else
    #define Py_STACK_UNTAG_OWNED(tagged) Py_STACK_UNTAG_BORROWED(tagged)
 #endif

 #define MAX_UNTAG_SCRATCH 10

 static inline void
 _Py_untag_stack(PyObject **dst, const _PyStackRef *src, size_t length) {
 _Py_untag_stack_borrowed(PyObject **dst, const _PyStackRef *src, size_t length) {
    for (size_t i = 0; i < length; i++) {
        dst[i] = Py_STACK_UNTAG_BORROWED(src[i]);
    }
 }

 static inline void
 _Py_untag_stack_owned(PyObject **dst, const _PyStackRef *src, size_t length) {
    for (size_t i = 0; i < length; i++) {
        dst[i] = Py_STACK_UNTAG_OWNED(src[i]);
    }
 }


 #define Py_XSETREF_STACKREF(dst, src) \
    do { \
diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h
    Py_ssize_t seqlen
    );

 PyAPI_FUNC(PyObject*)_PyUnicode_JoinTaggedArray(
 PyAPI_FUNC(PyObject*)_PyUnicode_JoinStack(
    PyObject *separator,
    _PyStackRef const *items,
    Py_ssize_t seqlen
diff --git a/Objects/call.c b/Objects/call.c
        PyErr_NoMemory();
        return NULL;
    }
 _Py_untag_stack(args + 1, tagged, nargs);
 _Py_untag_stack_owned(args + 1, tagged, nargs);
    PyObject *res = PyObject_Vectorcall(callable, args + 1, nargsf, kwnames);
    PyMem_Free(args);
    return res;
 PyObject_Vectorcall_Tagged(PyObject *callable,
                           const _PyStackRef *tagged, size_t nargsf, PyObject *kwnames)
 {
 #if defined(Py_GIL_DISABLED) || defined(Py_TEST_TAG)
 #if defined(Py_GIL_DISABLED) || defined(Py_TAG_TEST)
    size_t nargs = nargsf & ~PY_VECTORCALL_ARGUMENTS_OFFSET;
    if (kwnames != NULL && PyTuple_CheckExact(kwnames)) {
        nargs += PyTuple_GET_SIZE(kwnames);
        return PyObject_VectorcallTaggedSlow(callable, tagged, nargsf, kwnames);
    }
    // + 1 to allow args[-1] to be used by PY_VECTORCALL_ARGUMENTS_OFFSET
 _Py_untag_stack(args + 1, tagged, nargs);
 _Py_untag_stack_owned(args + 1, tagged, nargs);
    return PyObject_Vectorcall(callable, args + 1, nargsf, kwnames);
 #else
    (void)(PyObject_VectorcallTaggedSlow);
        return NULL;
    }
    // + 1 to allow args[-1] to be used by PY_VECTORCALL_ARGUMENTS_OFFSET
 _Py_untag_stack(args + 1, tagged, nargs);
 _Py_untag_stack_owned(args + 1, tagged, nargs);
    PyObject *res = callable->tp_vectorcall((PyObject *)callable,
                                            args + 1, nargsf, kwnames);
    PyMem_Free(args);
 PyObject_TypeVectorcall_Tagged(PyTypeObject *callable,
                               const _PyStackRef *tagged, size_t nargsf, PyObject *kwnames)
 {
 #if defined(Py_GIL_DISABLED) || defined(Py_TEST_TAG)
 #if defined(Py_GIL_DISABLED) || defined(Py_TAG_TEST)
    size_t nargs = nargsf & ~PY_VECTORCALL_ARGUMENTS_OFFSET;
    PyObject *args[MAX_UNTAG_SCRATCH];
    if (nargs >= MAX_UNTAG_SCRATCH) {
        return PyObject_TypeVectorcall_TaggedSlow(callable, tagged, nargsf, kwnames);
    }
    // + 1 to allow args[-1] to be used by PY_VECTORCALL_ARGUMENTS_OFFSET
 _Py_untag_stack(args + 1, tagged, nargs);
 _Py_untag_stack_owned(args + 1, tagged, nargs);
    return callable->tp_vectorcall((PyObject *)callable, args + 1, nargsf, kwnames);
 #else
    (void)PyObject_TypeVectorcall_TaggedSlow;
        PyErr_NoMemory();
        return NULL;
    }
 _Py_untag_stack(args + 1, tagged, nargs);
 _Py_untag_stack_owned(args + 1, tagged, nargs);
    PyObject *res = cfunc(self, args + 1, nargsf);
    PyMem_Free(args);
    return res;
                                    PyObject *self,
                                    const _PyStackRef *tagged, Py_ssize_t nargsf)
 {
 #if defined(Py_GIL_DISABLED) || defined(Py_TEST_TAG)
 #if defined(Py_GIL_DISABLED) || defined(Py_TAG_TEST)
    size_t nargs = nargsf & ~PY_VECTORCALL_ARGUMENTS_OFFSET;
    PyObject *args[MAX_UNTAG_SCRATCH];
    if (nargs >= MAX_UNTAG_SCRATCH) {
        return PyObject_PyCFunctionFastCall_TaggedSlow(cfunc, self, tagged, nargsf);
    }
 _Py_untag_stack(args + 1, tagged, nargs);
 _Py_untag_stack_owned(args + 1, tagged, nargs);
    return cfunc(self, args + 1, nargsf);
 #else
    (void)PyObject_PyCFunctionFastCall_TaggedSlow;
        PyErr_NoMemory();
        return NULL;
    }
 _Py_untag_stack(args + 1, tagged, nargs);
 _Py_untag_stack_owned(args + 1, tagged, nargs);
    PyObject *res = cfunc(self, args + 1, nargsf, kwds);
    PyMem_Free(args);
    return res;
                                    const _PyStackRef *tagged, Py_ssize_t nargsf,
                                    PyObject *kwds)
 {
 #if defined(Py_GIL_DISABLED) || defined(Py_TEST_TAG)
 #if defined(Py_GIL_DISABLED) || defined(Py_TAG_TEST)
    size_t nargs = nargsf & ~PY_VECTORCALL_ARGUMENTS_OFFSET;
    PyObject *args[MAX_UNTAG_SCRATCH];
    if (nargs >= MAX_UNTAG_SCRATCH) {
        return PyObject_PyCFunctionFastWithKeywordsCall_TaggedSlow(
            cfunc, self, tagged, nargsf, kwds
        );
    }
 _Py_untag_stack(args + 1, tagged, nargs);
 _Py_untag_stack_owned(args + 1, tagged, nargs);
    return cfunc(self, args + 1, nargsf, kwds);
 #else
    (void)PyObject_PyCFunctionFastWithKeywordsCall_TaggedSlow;
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
    _PyStackRef const *vs = values;

    for (Py_ssize_t i = 0; i < length; i++) {
        PyObject *key =Py_STACK_UNTAG_BORROWED(*ks);
        PyObject *value =Py_STACK_UNTAG_BORROWED(*vs);
        PyObject *key =Py_STACK_UNTAG_OWNED(*ks);
        PyObject *value =Py_STACK_UNTAG_OWNED(*vs);
        if (setitem_lock_held((PyDictObject *)dict, key, value) < 0) {
            Py_DECREF(dict);
            return NULL;

    for (Py_ssize_t i = 0; i < length; i++) {
        PyObject *key = *ks;
        PyObject *value =Py_STACK_UNTAG_BORROWED(*vs);
        PyObject *value =Py_STACK_UNTAG_OWNED(*vs);
        if (setitem_lock_held((PyDictObject *)dict, key, value) < 0) {
            Py_DECREF(dict);
            return NULL;
diff --git a/Objects/listobject.c b/Objects/listobject.c

    PyObject **dst = list->ob_item;
    for (Py_ssize_t i = 0; i < n; i++) {
        PyObject *item =Py_STACK_UNTAG_BORROWED(src[i]);
        PyObject *item =Py_STACK_UNTAG_OWNED(src[i]);
        dst[i] = item;
    }

diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c
    }
    PyObject **dst = tuple->ob_item;
    for (Py_ssize_t i = 0; i < n; i++) {
        PyObject *item =Py_STACK_UNTAG_BORROWED(src[i]);
        PyObject *item =Py_STACK_UNTAG_OWNED(src[i]);
        dst[i] = item;
    }
    _PyObject_GC_TRACK(tuple);
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
 }

 PyObject*
 _PyUnicode_JoinTaggedArray_Slow(PyObject *separator, _PyStackRef const *tagged, Py_ssize_t seqlen)
 _PyUnicode_JoinStack_Slow(PyObject *separator, _PyStackRef const *tagged, Py_ssize_t seqlen)
 {
    PyObject **args = PyMem_Malloc(seqlen * sizeof(PyObject *));
    if (args == NULL) {
        PyErr_NoMemory();
        return NULL;
    }
 _Py_untag_stack(args, tagged, seqlen);
 _Py_untag_stack_borrowed(args, tagged, seqlen);
    PyObject *res = _PyUnicode_JoinArray(separator, args, seqlen);
    PyMem_Free(args);
    return res;
 }

 PyObject *
 _PyUnicode_JoinTaggedArray(PyObject *separator, _PyStackRef const *items_tagged, Py_ssize_t seqlen)
 _PyUnicode_JoinStack(PyObject *separator, _PyStackRef const *items_tagged, Py_ssize_t seqlen)
 {
 #if defined(Py_GIL_DISABLED) || defined(Py_TEST_TAG)
 #if defined(Py_GIL_DISABLED) || defined(Py_TAG_TEST)
    PyObject *args[MAX_UNTAG_SCRATCH];
    if (seqlen > MAX_UNTAG_SCRATCH) {
        return_PyUnicode_JoinTaggedArray_Slow(separator, items_tagged, seqlen);
        return_PyUnicode_JoinStack_Slow(separator, items_tagged, seqlen);
    }
 _Py_untag_stack(args, items_tagged, seqlen);
 _Py_untag_stack_borrowed(args, items_tagged, seqlen);
    return _PyUnicode_JoinArray(separator, args, seqlen);
 #else
    (void)_PyUnicode_JoinTaggedArray_Slow;
    (void)_PyUnicode_JoinStack_Slow;
    return _PyUnicode_JoinArray(separator, (PyObject **)items_tagged, seqlen);
 #endif
 }
diff --git a/Python/bytecodes.c b/Python/bytecodes.c
        }

        inst(BUILD_STRING, (pieces[oparg] -- str)) {
            str =_PyUnicode_JoinTaggedArray(&_Py_STR(empty), pieces, oparg);
            str =_PyUnicode_JoinStack(&_Py_STR(empty), pieces, oparg);
            DECREF_INPUTS();
            ERROR_IF(str == NULL, error);
        }
            for (int i = 0; i < oparg; i++) {
                _PyStackRef item = values[i];
                if (err == 0) {
                    // TODO steals reference, needs TO_OWNED
                    err = PySet_Add(set, Py_STACK_UNTAG_BORROWED(item));
                    err = PySet_Add(set, Py_STACK_UNTAG_OWNED(item));
                }
                Py_DECREF_STACKREF(item);
            }
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py
    "CALL_STAT_INC",
    "maybe_lltrace_resume_frame",
    "_PyUnicode_JoinArray",
    "_PyUnicode_JoinTaggedArray",
    "_PyUnicode_JoinStack",
    "Py_STACK_TAG",
    "Py_STACK_UNTAG_BORROWED",
    "Py_DECREF_STACKREF",
Original file line number	Diff line number	Diff line change
Expand Up		@@ -32,9 +32,9 @@ PyTuple_SET_ITEM(PyObject op, Py_ssize_t index, PyObject value) {
		PyTupleObject *tuple = _PyTuple_CAST(op);
		assert(0 <= index);
		assert(index < Py_SIZE(tuple));
		#ifdefPy_TEST_TAG
		#ifdefPy_TAG_TEST
		// Make sure it's not a tagged pointer
		assert(((uintptr_t)op &Py_TEST_TAG) == 0);
		assert(((uintptr_t)op &Py_TAG_TEST) == 0);
		#endif
		tuple->ob_item[index] = value;
		}
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -16,33 +16,61 @@ typedef union {
		} _PyStackRef;

		#define Py_TAG (0b0)
		#define Py_TEST_TAG (0b1)
		#define Py_TAG_TEST (0b1)
		#define Py_TAG_DEFERRED (0b1)

		#if defined(Py_TEST_TAG)
		#define Py_STACK_UNTAG_BORROWED(tagged) ((PyObject *)(uintptr_t)((tagged).bits & (~Py_TEST_TAG)))
		#if defined(Py_TAG_TEST)
		#define Py_STACK_UNTAG_BORROWED(tagged) ((PyObject *)(uintptr_t)((tagged).bits & (~Py_TAG_TEST)))
		#elif defined(Py_GIL_DISABLED)
		#define Py_STACK_UNTAG_BORROWED(tagged) ((PyObject *)((tagged).bits & (~Py_TAG)))
		#else
		#define Py_STACK_UNTAG_BORROWED(tagged) ((PyObject *)(uintptr_t)((tagged).bits))
		#endif

		#if defined(Py_TEST_TAG)
		#define Py_STACK_TAG(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj) \|Py_TEST_TAG)})
		#if defined(Py_TAG_TEST)
		#define Py_STACK_TAG(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj) \|Py_TAG_TEST)})
		#elif defined(Py_GIL_DISABLED)
		#define Py_STACK_TAG(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj) \| Py_TAG}))
		#else
		#define Py_STACK_TAG(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj))})
		#endif

		static inline PyObject *
		_Py_Stack_Untag_Owned(_PyStackRef tagged) {
		#if defined(Py_TAG_TEST)
		// Test tag takes up same tag as deferred tag!
		Py_UNREACHABLE();
		#endif
		if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) {
		return Py_NewRef(Py_STACK_UNTAG_BORROWED(tagged));
		}
		return Py_STACK_UNTAG_BORROWED(tagged);
		}

		#if defined(Py_TAG_TEST)
		#define Py_STACK_UNTAG_OWNED(tagged) Py_STACK_UNTAG_BORROWED(tagged)
		#elif defined(Py_GIL_DISABLED)
		#define Py_STACK_UNTAG_OWNED(tagged) _Py_Stack_Untag_Owned(tagged)
		#else
		#define Py_STACK_UNTAG_OWNED(tagged) Py_STACK_UNTAG_BORROWED(tagged)
		#endif

		#define MAX_UNTAG_SCRATCH 10

		static inline void
		_Py_untag_stack(PyObject *dst, const _PyStackRef src, size_t length) {
		_Py_untag_stack_borrowed(PyObject *dst, const _PyStackRef src, size_t length) {
		for (size_t i = 0; i < length; i++) {
		dst[i] = Py_STACK_UNTAG_BORROWED(src[i]);
		}
		}

		static inline void
		_Py_untag_stack_owned(PyObject *dst, const _PyStackRef src, size_t length) {
		for (size_t i = 0; i < length; i++) {
		dst[i] = Py_STACK_UNTAG_OWNED(src[i]);
		}
		}


		#define Py_XSETREF_STACKREF(dst, src) \
		do { \
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -209,7 +209,7 @@ PyAPI_FUNC(PyObject*) _PyUnicode_JoinArray(
		Py_ssize_t seqlen
		);

		PyAPI_FUNC(PyObject*)_PyUnicode_JoinTaggedArray(
		PyAPI_FUNC(PyObject*)_PyUnicode_JoinStack(
		PyObject *separator,
		_PyStackRef const *items,
		Py_ssize_t seqlen
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -1059,7 +1059,7 @@ PyObject_VectorcallTaggedSlow(PyObject *callable,
		PyErr_NoMemory();
		return NULL;
		}
		_Py_untag_stack(args + 1, tagged, nargs);
		_Py_untag_stack_owned(args + 1, tagged, nargs);
		PyObject *res = PyObject_Vectorcall(callable, args + 1, nargsf, kwnames);
		PyMem_Free(args);
		return res;
Expand All		@@ -1069,7 +1069,7 @@ PyObject *
		PyObject_Vectorcall_Tagged(PyObject *callable,
		const _PyStackRef tagged, size_t nargsf, PyObject kwnames)
		{
		#if defined(Py_GIL_DISABLED) \|\| defined(Py_TEST_TAG)
		#if defined(Py_GIL_DISABLED) \|\| defined(Py_TAG_TEST)
		size_t nargs = nargsf & ~PY_VECTORCALL_ARGUMENTS_OFFSET;
		if (kwnames != NULL && PyTuple_CheckExact(kwnames)) {
Fidget-Spinner marked this conversation as resolved. OutdatedShow resolvedHide resolved
		nargs += PyTuple_GET_SIZE(kwnames);
Expand All		@@ -1079,7 +1079,7 @@ PyObject_Vectorcall_Tagged(PyObject *callable,
		return PyObject_VectorcallTaggedSlow(callable, tagged, nargsf, kwnames);
		}
		// + 1 to allow args[-1] to be used by PY_VECTORCALL_ARGUMENTS_OFFSET
		_Py_untag_stack(args + 1, tagged, nargs);
		_Py_untag_stack_owned(args + 1, tagged, nargs);
		return PyObject_Vectorcall(callable, args + 1, nargsf, kwnames);
		#else
		(void)(PyObject_VectorcallTaggedSlow);
Expand All		@@ -1098,7 +1098,7 @@ PyObject_TypeVectorcall_TaggedSlow(PyTypeObject *callable,
		return NULL;
		}
		// + 1 to allow args[-1] to be used by PY_VECTORCALL_ARGUMENTS_OFFSET
		_Py_untag_stack(args + 1, tagged, nargs);
		_Py_untag_stack_owned(args + 1, tagged, nargs);
		PyObject res = callable->tp_vectorcall((PyObject )callable,
		args + 1, nargsf, kwnames);
		PyMem_Free(args);
Expand All		@@ -1109,14 +1109,14 @@ PyObject *
		PyObject_TypeVectorcall_Tagged(PyTypeObject *callable,
		const _PyStackRef tagged, size_t nargsf, PyObject kwnames)
		{
		#if defined(Py_GIL_DISABLED) \|\| defined(Py_TEST_TAG)
		#if defined(Py_GIL_DISABLED) \|\| defined(Py_TAG_TEST)
		size_t nargs = nargsf & ~PY_VECTORCALL_ARGUMENTS_OFFSET;
		PyObject *args[MAX_UNTAG_SCRATCH];
		if (nargs >= MAX_UNTAG_SCRATCH) {
		return PyObject_TypeVectorcall_TaggedSlow(callable, tagged, nargsf, kwnames);
		}
		// + 1 to allow args[-1] to be used by PY_VECTORCALL_ARGUMENTS_OFFSET
		_Py_untag_stack(args + 1, tagged, nargs);
		_Py_untag_stack_owned(args + 1, tagged, nargs);
		return callable->tp_vectorcall((PyObject *)callable, args + 1, nargsf, kwnames);
		#else
		(void)PyObject_TypeVectorcall_TaggedSlow;
Expand All		@@ -1135,7 +1135,7 @@ PyObject_PyCFunctionFastCall_TaggedSlow(PyCFunctionFast cfunc,
		PyErr_NoMemory();
		return NULL;
		}
		_Py_untag_stack(args + 1, tagged, nargs);
		_Py_untag_stack_owned(args + 1, tagged, nargs);
		PyObject *res = cfunc(self, args + 1, nargsf);
		PyMem_Free(args);
		return res;
Expand All		@@ -1146,13 +1146,13 @@ PyObject_PyCFunctionFastCall_Tagged(PyCFunctionFast cfunc,
		PyObject *self,
		const _PyStackRef *tagged, Py_ssize_t nargsf)
		{
		#if defined(Py_GIL_DISABLED) \|\| defined(Py_TEST_TAG)
		#if defined(Py_GIL_DISABLED) \|\| defined(Py_TAG_TEST)
		size_t nargs = nargsf & ~PY_VECTORCALL_ARGUMENTS_OFFSET;
		PyObject *args[MAX_UNTAG_SCRATCH];
		if (nargs >= MAX_UNTAG_SCRATCH) {
		return PyObject_PyCFunctionFastCall_TaggedSlow(cfunc, self, tagged, nargsf);
		}
		_Py_untag_stack(args + 1, tagged, nargs);
		_Py_untag_stack_owned(args + 1, tagged, nargs);
		return cfunc(self, args + 1, nargsf);
		#else
		(void)PyObject_PyCFunctionFastCall_TaggedSlow;
Expand All		@@ -1172,7 +1172,7 @@ PyObject_PyCFunctionFastWithKeywordsCall_TaggedSlow(PyCFunctionFastWithKeywords
		PyErr_NoMemory();
		return NULL;
		}
		_Py_untag_stack(args + 1, tagged, nargs);
		_Py_untag_stack_owned(args + 1, tagged, nargs);
		PyObject *res = cfunc(self, args + 1, nargsf, kwds);
		PyMem_Free(args);
		return res;
Expand All		@@ -1184,15 +1184,15 @@ PyObject_PyCFunctionFastWithKeywordsCall_Tagged(PyCFunctionFastWithKeywords cfun
		const _PyStackRef *tagged, Py_ssize_t nargsf,
		PyObject *kwds)
		{
		#if defined(Py_GIL_DISABLED) \|\| defined(Py_TEST_TAG)
		#if defined(Py_GIL_DISABLED) \|\| defined(Py_TAG_TEST)
		size_t nargs = nargsf & ~PY_VECTORCALL_ARGUMENTS_OFFSET;
		PyObject *args[MAX_UNTAG_SCRATCH];
		if (nargs >= MAX_UNTAG_SCRATCH) {
		return PyObject_PyCFunctionFastWithKeywordsCall_TaggedSlow(
		cfunc, self, tagged, nargsf, kwds
		);
		}
		_Py_untag_stack(args + 1, tagged, nargs);
		_Py_untag_stack_owned(args + 1, tagged, nargs);
		return cfunc(self, args + 1, nargsf, kwds);
		#else
		(void)PyObject_PyCFunctionFastWithKeywordsCall_TaggedSlow;
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2136,8 +2136,8 @@ _PyDict_FromStackItems(_PyStackRef const *keys, Py_ssize_t keys_offset,
		_PyStackRef const *vs = values;

		for (Py_ssize_t i = 0; i < length; i++) {
		PyObject key =Py_STACK_UNTAG_BORROWED(ks);
		PyObject value =Py_STACK_UNTAG_BORROWED(vs);
		PyObject key =Py_STACK_UNTAG_OWNED(ks);
		PyObject value =Py_STACK_UNTAG_OWNED(vs);
		if (setitem_lock_held((PyDictObject *)dict, key, value) < 0) {
		Py_DECREF(dict);
		return NULL;
Expand DownExpand Up		@@ -2177,7 +2177,7 @@ _PyDict_FromStackItemsUntaggedKeys(

		for (Py_ssize_t i = 0; i < length; i++) {
		PyObject key = ks;
		PyObject value =Py_STACK_UNTAG_BORROWED(vs);
		PyObject value =Py_STACK_UNTAG_OWNED(vs);
		if (setitem_lock_held((PyDictObject *)dict, key, value) < 0) {
		Py_DECREF(dict);
		return NULL;
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -3191,7 +3191,7 @@ _PyList_FromStackSteal(_PyStackRef const *src, Py_ssize_t n)

		PyObject **dst = list->ob_item;
		for (Py_ssize_t i = 0; i < n; i++) {
		PyObject *item =Py_STACK_UNTAG_BORROWED(src[i]);
		PyObject *item =Py_STACK_UNTAG_OWNED(src[i]);
		dst[i] = item;
		}

Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -427,7 +427,7 @@ _PyTuple_FromStackSteal(_PyStackRef const *src, Py_ssize_t n)
		}
		PyObject **dst = tuple->ob_item;
		for (Py_ssize_t i = 0; i < n; i++) {
		PyObject *item =Py_STACK_UNTAG_BORROWED(src[i]);
		PyObject *item =Py_STACK_UNTAG_OWNED(src[i]);
		dst[i] = item;
		}
		_PyObject_GC_TRACK(tuple);
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -9739,31 +9739,31 @@ _PyUnicode_JoinArray(PyObject separator, PyObject const *items, Py_ssize_t seq
		}

		PyObject*
		_PyUnicode_JoinTaggedArray_Slow(PyObject separator, _PyStackRef const tagged, Py_ssize_t seqlen)
		_PyUnicode_JoinStack_Slow(PyObject separator, _PyStackRef const tagged, Py_ssize_t seqlen)
		{
		PyObject *args = PyMem_Malloc(seqlen sizeof(PyObject *));
		if (args == NULL) {
		PyErr_NoMemory();
		return NULL;
		}
		_Py_untag_stack(args, tagged, seqlen);
		_Py_untag_stack_borrowed(args, tagged, seqlen);
		PyObject *res = _PyUnicode_JoinArray(separator, args, seqlen);
		PyMem_Free(args);
		return res;
		}

		PyObject *
		_PyUnicode_JoinTaggedArray(PyObject separator, _PyStackRef const items_tagged, Py_ssize_t seqlen)
		_PyUnicode_JoinStack(PyObject separator, _PyStackRef const items_tagged, Py_ssize_t seqlen)
		{
		#if defined(Py_GIL_DISABLED) \|\| defined(Py_TEST_TAG)
		#if defined(Py_GIL_DISABLED) \|\| defined(Py_TAG_TEST)
		PyObject *args[MAX_UNTAG_SCRATCH];
		if (seqlen > MAX_UNTAG_SCRATCH) {
		return_PyUnicode_JoinTaggedArray_Slow(separator, items_tagged, seqlen);
		return_PyUnicode_JoinStack_Slow(separator, items_tagged, seqlen);
		}
		_Py_untag_stack(args, items_tagged, seqlen);
		_Py_untag_stack_borrowed(args, items_tagged, seqlen);
		return _PyUnicode_JoinArray(separator, args, seqlen);
		#else
		(void)_PyUnicode_JoinTaggedArray_Slow;
		(void)_PyUnicode_JoinStack_Slow;
		return _PyUnicode_JoinArray(separator, (PyObject **)items_tagged, seqlen);
		#endif
		}
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -1587,7 +1587,7 @@ dummy_func(
		}

		inst(BUILD_STRING, (pieces[oparg] -- str)) {
		str =_PyUnicode_JoinTaggedArray(&_Py_STR(empty), pieces, oparg);
		str =_PyUnicode_JoinStack(&_Py_STR(empty), pieces, oparg);
		DECREF_INPUTS();
		ERROR_IF(str == NULL, error);
		}
Expand DownExpand Up		@@ -1634,8 +1634,7 @@ dummy_func(
		for (int i = 0; i < oparg; i++) {
		_PyStackRef item = values[i];
		if (err == 0) {
		// TODO steals reference, needs TO_OWNED
		err = PySet_Add(set, Py_STACK_UNTAG_BORROWED(item));
		err = PySet_Add(set, Py_STACK_UNTAG_OWNED(item));
		}
		Py_DECREF_STACKREF(item);
		}
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -409,7 +409,7 @@ def has_error_without_pop(op: parser.InstDef) -> bool:
		"CALL_STAT_INC",
		"maybe_lltrace_resume_frame",
		"_PyUnicode_JoinArray",
		"_PyUnicode_JoinTaggedArray",
		"_PyUnicode_JoinStack",
		"Py_STACK_TAG",
		"Py_STACK_UNTAG_BORROWED",
		"Py_DECREF_STACKREF",
Expand Down