May 2, 2025 · Apr 28, 2025 · Apr 29, 2025 · Apr 29, 2025 · May 2, 2025 · Apr 28, 2025
diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h
    PyObject_HEAD
    Py_ssize_t length;          /* Number of code points in the string */
    Py_hash_t hash;             /* Hash value; -1 if not set */
 #ifdef Py_GIL_DISABLED
    /* Ensure 4 byte alignment for PyUnicode_DATA(), see gh-63736 on m68k.
       In the non-free-threaded build, we'll use explicit padding instead */
   _Py_ALIGN_AS(4)
 #endif
    struct {
        /* If interned is non-zero, the two references from the
           dictionary to this object are *not* counted in ob_refcnt.
               3: Interned, Immortal, and Static
           This categorization allows the runtime to determine the right
           cleanup mechanism at runtime shutdown. */
        uint16_t interned;
 #ifdef Py_GIL_DISABLED
        // Needs to be accessed atomically, so can't be a bit field.
        unsigned char interned;
 #else
        unsigned int interned:2;
 #endif
        /* Character size:

           - PyUnicode_1BYTE_KIND (1):
             * all characters are in the range U+0000-U+10FFFF
             * at least one character is in the range U+10000-U+10FFFF
         */
        unsignedshort kind:3;
        unsignedint kind:3;
        /* Compact is with respect to the allocation scheme. Compact unicode
           objects only require one memory block while non-compact objects use
           one block for the PyUnicodeObject struct and another for its data
           buffer. */
        unsignedshort compact:1;
        unsignedint compact:1;
        /* The string only contains characters in the range U+0000-U+007F (ASCII)
           and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is
           set, use the PyASCIIObject structure. */
        unsignedshort ascii:1;
        unsignedint ascii:1;
        /* The object is statically allocated. */
        unsigned short statically_allocated:1;
        unsigned int statically_allocated:1;
 #ifndef Py_GIL_DISABLED
        /* Padding to ensure that PyUnicode_DATA() is always aligned to
           4 bytes (see issue #19537 on m68k) and we use unsigned short to avoid
           the extra four bytes on 32-bit Windows. This is restricted features
           for specific compilers including GCC, MSVC, Clang and IBM's XL compiler. */
        unsigned short :10;
           4 bytes (see issue gh-63736 on m68k) */
        unsigned int :24;
 #endif
    } state;
 } PyASCIIObject;

 /* Use only if you know it's a string */
 static inline unsigned int PyUnicode_CHECK_INTERNED(PyObject *op) {
 #ifdef Py_GIL_DISABLED
    return_Py_atomic_load_uint16_relaxed(&_PyASCIIObject_CAST(op)->state.interned);
    return_Py_atomic_load_uint8_relaxed(&_PyASCIIObject_CAST(op)->state.interned);
 #else
    return _PyASCIIObject_CAST(op)->state.interned;
 #endif
diff --git a/Include/pymacro.h b/Include/pymacro.h
 #  define static_assert _Static_assert
 #endif


 // _Py_ALIGN_AS: this compiler's spelling of `alignas` keyword,
 // We currently use alignas for free-threaded builds only; additional compat
 // checking would be great before we add it to the default build.
 // Standards/compiler support:
 // - `alignas` is a keyword in C23 and C++11.
 // - `_Alignas` is a keyword in C11
 // - GCC & clang has __attribute__((aligned))
 //   (use that for older standards in pedantic mode)
 // - MSVC has __declspec(align)
 // - `_Alignas` is common C compiler extension
 // Older compilers may name it differently; to allow compilation on such
 // unsupported platforms, we don't redefine _Py_ALIGN_AS if it's already
 // defined. Note that defining it wrong (including defining it to nothing) will
 // cause ABI incompatibilities.
 #ifdef Py_GIL_DISABLED
 #   ifndef _Py_ALIGN_AS
 #       ifdef __cplusplus
 #           if __cplusplus >= 201103L
 #               define _Py_ALIGN_AS(V) alignas(V)
 #           elif defined(__GNUC__) || defined(__clang__)
 #               define _Py_ALIGN_AS(V) __attribute__((aligned(V)))
 #           elif defined(_MSC_VER)
 #               define _Py_ALIGN_AS(V) __declspec(align(V))
 #           else
 #               define _Py_ALIGN_AS(V) alignas(V)
 #           endif
 #       elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L
 #           define _Py_ALIGN_AS(V) alignas(V)
 #       elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
 #           define _Py_ALIGN_AS(V) _Alignas(V)
 #       elif (defined(__GNUC__) || defined(__clang__))
 #           define _Py_ALIGN_AS(V) __attribute__((aligned(V)))
 #       elif defined(_MSC_VER)
 #           define _Py_ALIGN_AS(V) __declspec(align(V))
 #       else
 #           define _Py_ALIGN_AS(V) _Alignas(V)
 #       endif
 #   endif
 #endif

 /* Minimum value between x and y */
 #define Py_MIN(x, y) (((x) > (y)) ? (y) : (x))

diff --git a/Misc/NEWS.d/next/C_API/2025-04-28-15-36-01.gh-issue-128972.8bZMIm.rst b/Misc/NEWS.d/next/C_API/2025-04-28-15-36-01.gh-issue-128972.8bZMIm.rst
 For non-free-threaded builds, the memory layout of :c:struct:`PyASCIIObject`
 is reverted to match Python 3.13. (Note that the structure is not part of
 stable ABI and so its memory layout is *guaranteed* to remain stable.)
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
        _Py_DecRefTotal(_PyThreadState_GET());
    }
 #endif
 FT_ATOMIC_STORE_UINT16_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_IMMORTAL);
 FT_ATOMIC_STORE_UINT8_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_IMMORTAL);
    _Py_SetImmortal(s);
 }

        Py_DECREF(s);
        Py_DECREF(s);
    }
 FT_ATOMIC_STORE_UINT16_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_MORTAL);
 FT_ATOMIC_STORE_UINT8_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_MORTAL);

    /* INTERNED_MORTAL -> INTERNED_IMMORTAL (if needed) */

            Py_UNREACHABLE();
        }
        if (!shared) {
 FT_ATOMIC_STORE_UINT16_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_NOT_INTERNED);
 FT_ATOMIC_STORE_UINT8_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_NOT_INTERNED);
        }
    }
 #ifdef INTERNED_STATS
Original file line number	Diff line number	Diff line change
Expand Up		@@ -99,6 +99,11 @@ typedef struct {
		PyObject_HEAD
		Py_ssize_t length; /* Number of code points in the string */
		Py_hash_t hash; /* Hash value; -1 if not set */
		#ifdef Py_GIL_DISABLED
		/* Ensure 4 byte alignment for PyUnicode_DATA(), see gh-63736 on m68k.
		In the non-free-threaded build, we'll use explicit padding instead */
		_Py_ALIGN_AS(4)
Comment on lines +103 to +105 Copy link Member serhiy-storchakaApr 28, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. It does not ensure alignment for`PyUnicode_DATA()`, it ensures alignment for`state`. To ensure alignment for`PyUnicode_DATA()`, you need to add`_Py_ALIGN_AS` before the`data` field in`PyUnicodeObject`. Copy link Contributor colesburyApr 28, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. Am I missing something? This sounds like a distinction without a difference. The`PyUnicode_DATA()` macro depends on the size of`PyASCIIObject`. The size is a multiple of the alignment. The alignment of`PyASCIIObject` must be at least the alignment of every member, so it must be at least 4 bytes with the`_Py_ALIGN_AS(4)` macro. Copy link Member serhiy-storchakaApr 28, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. The alignment of`PyASCIIObject` should be the same as the alignment of PyObject. The definition of the`_PyUnicode_COMPACT_DATA()` macro and the code that calculates the size of the`PyASCIIObject` objects should be changed to guarantee the alignment of data. Perhaps`_Py_ALIGN_AS(4)` should be added in`PyObject_HEAD`. Copy link Contributor colesburyApr 28, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. Huh? Why? Copy link Member serhiy-storchakaApr 28, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. What exactly are you asking about? Copy link MemberAuthor encukouApr 30, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. Why? AFAIK, for`sizeof(PyASCIIObject)` it doesn't really matter. I'd rather avoid setting a 4-byte alignment for 64-bit pointers. FWIW, since 3.3 the header says "the data immediately follow the structure". We can't switch users to`_Py_SIZE_ROUND_UP`, we need to make sure the size is aligned. Copy link Member serhiy-storchakaApr 30, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. The data immediately following the structure (PyASCIIObject or PyCompactUnicodeObject) has the same alignment as the structure. Adding`_Py_ALIGN_AS()` before the`state` member does not change the alignment of the structure. If we add a byte at the end of the structure, the alignment of the data immediately following the structure will change. It works for current structures by accident, but we can just add padding for the same effect. Copy link Contributor colesburyApr 30, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. I'd rather avoid setting a 4-byte alignment for 64-bit pointers. `_Alignas(4)` sets theminimum alignment -- it only makes the alignmentstricter. It won't reduce the alignment of 64-bit pointers. Adding`_Py_ALIGN_AS()` before the state member does not change the alignment of the structure. It does. Seehttps://gcc.godbolt.org/z/evh3hjxM8, for example. Arrays of structs (both`Foo arr[N]` and`malloc(sizeof(Foo) * N)`) would not work if that wasn't the case. Copy link Member serhiy-storchakaApr 30, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. Hmm, it seems that the alignment of a structure is determined by the largest alignment of its members. I didn't find an explicit statement in the C11 standard, but it seems logical. If the structure has less alignment, then we can't guarantee the alignment of the members. So we can apply _Py_ALIGN_AS() to any member, and it will work independently of the compiler. The original issue (#63736) was because on some platforms`Py_ssize_t` and pointers have alignment less than 4. Copy link MemberAuthor encukouMay 1, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. Yeah. FWIW, I also didn't find an explicit statement that structsize is a multiple of its alignment, but it's implied by the fact that`malloc(sizeof(element_type) * n)` allocates an array.
		#endif
		struct {
		/* If interned is non-zero, the two references from the
		dictionary to this object are not counted in ob_refcnt.
Expand All		@@ -109,7 +114,12 @@ typedef struct {
		3: Interned, Immortal, and Static
		This categorization allows the runtime to determine the right
		cleanup mechanism at runtime shutdown. */
		uint16_t interned;
		#ifdef Py_GIL_DISABLED
		// Needs to be accessed atomically, so can't be a bit field.
		unsigned char interned;
		#else
		unsigned int interned:2;
		#endif
		/* Character size:

		- PyUnicode_1BYTE_KIND (1):
Expand All		@@ -132,23 +142,23 @@ typedef struct {
		* all characters are in the range U+0000-U+10FFFF
		* at least one character is in the range U+10000-U+10FFFF
		*/
		unsignedshort kind:3;
		unsignedint kind:3;
		/* Compact is with respect to the allocation scheme. Compact unicode
		objects only require one memory block while non-compact objects use
		one block for the PyUnicodeObject struct and another for its data
		buffer. */
		unsignedshort compact:1;
		unsignedint compact:1;
		/* The string only contains characters in the range U+0000-U+007F (ASCII)
		and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is
		set, use the PyASCIIObject structure. */
		unsignedshort ascii:1;
		unsignedint ascii:1;
		/* The object is statically allocated. */
		unsigned short statically_allocated:1;
		unsigned int statically_allocated:1;
		#ifndef Py_GIL_DISABLED
		/* Padding to ensure that PyUnicode_DATA() is always aligned to
		4 bytes (see issue #19537 on m68k) and we use unsigned short to avoid
		the extra four bytes on 32-bit Windows. This is restricted features
		for specific compilers including GCC, MSVC, Clang and IBM's XL compiler. */
		unsigned short :10;
		4 bytes (see issue gh-63736 on m68k) */
		unsigned int :24;
		#endif
		} state;
		} PyASCIIObject;

Expand DownExpand Up		@@ -198,7 +208,7 @@ typedef struct {
		/* Use only if you know it's a string */
		static inline unsigned int PyUnicode_CHECK_INTERNED(PyObject *op) {
		#ifdef Py_GIL_DISABLED
		return_Py_atomic_load_uint16_relaxed(&_PyASCIIObject_CAST(op)->state.interned);
		return_Py_atomic_load_uint8_relaxed(&_PyASCIIObject_CAST(op)->state.interned);
		#else
		return _PyASCIIObject_CAST(op)->state.interned;
		#endif
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -23,6 +23,47 @@
		# define static_assert _Static_assert
		#endif


		// _Py_ALIGN_AS: this compiler's spelling of `alignas` keyword,
		// We currently use alignas for free-threaded builds only; additional compat
		// checking would be great before we add it to the default build.
		// Standards/compiler support:
		// - `alignas` is a keyword in C23 and C++11.
		// - `_Alignas` is a keyword in C11
		// - GCC & clang has __attribute__((aligned))
		// (use that for older standards in pedantic mode)
		// - MSVC has __declspec(align)
		// - `_Alignas` is common C compiler extension
		// Older compilers may name it differently; to allow compilation on such
		// unsupported platforms, we don't redefine _Py_ALIGN_AS if it's already
		// defined. Note that defining it wrong (including defining it to nothing) will
		// cause ABI incompatibilities.
		#ifdef Py_GIL_DISABLED
		# ifndef _Py_ALIGN_AS
		# ifdef __cplusplus
		# if __cplusplus >= 201103L
		# define _Py_ALIGN_AS(V) alignas(V)
		# elif defined(__GNUC__) \|\| defined(__clang__)
		# define _Py_ALIGN_AS(V) __attribute__((aligned(V)))
		# elif defined(_MSC_VER)
		# define _Py_ALIGN_AS(V) __declspec(align(V))
		# else
		# define _Py_ALIGN_AS(V) alignas(V)
		# endif
		# elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L
		# define _Py_ALIGN_AS(V) alignas(V)
		# elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
		# define _Py_ALIGN_AS(V) _Alignas(V)
		# elif (defined(__GNUC__) \|\| defined(__clang__))
		# define _Py_ALIGN_AS(V) __attribute__((aligned(V)))
		# elif defined(_MSC_VER)
		# define _Py_ALIGN_AS(V) __declspec(align(V))
		# else
		# define _Py_ALIGN_AS(V) _Alignas(V)
		# endif
		# endif
		#endif

		/* Minimum value between x and y */
		#define Py_MIN(x, y) (((x) > (y)) ? (y) : (x))

Expand Down
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		For non-free-threaded builds, the memory layout of :c:struct:`PyASCIIObject`
		is reverted to match Python 3.13. (Note that the structure is not part of
		stable ABI and so its memory layout is guaranteed to remain stable.)
Original file line number	Diff line number	Diff line change
Expand Up		@@ -15918,7 +15918,7 @@ immortalize_interned(PyObject *s)
		_Py_DecRefTotal(_PyThreadState_GET());
		}
		#endif
		FT_ATOMIC_STORE_UINT16_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_IMMORTAL);
		FT_ATOMIC_STORE_UINT8_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_IMMORTAL);
		_Py_SetImmortal(s);
		}

Expand DownExpand Up		@@ -16036,7 +16036,7 @@ intern_common(PyInterpreterState interp, PyObject s /* stolen */,
		Py_DECREF(s);
		Py_DECREF(s);
		}
		FT_ATOMIC_STORE_UINT16_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_MORTAL);
		FT_ATOMIC_STORE_UINT8_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_MORTAL);

		/* INTERNED_MORTAL -> INTERNED_IMMORTAL (if needed) */

Expand DownExpand Up		@@ -16172,7 +16172,7 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp)
		Py_UNREACHABLE();
		}
		if (!shared) {
		FT_ATOMIC_STORE_UINT16_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_NOT_INTERNED);
		FT_ATOMIC_STORE_UINT8_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_NOT_INTERNED);
		}
		}
		#ifdef INTERNED_STATS
Expand Down