Commit407e3df

authored and

noahbkim

committed

pythongh-113993: Allow interned strings to be mortal, and fix related issues (pythonGH-120520)

* Add an InternalDocs file describing how interning should work and how to use it.* Add internal functions to *explicitly* request what kind of interning is done: - `_PyUnicode_InternMortal` - `_PyUnicode_InternImmortal` - `_PyUnicode_InternStatic`* Switch uses of `PyUnicode_InternInPlace` to those.* Disallow using `_Py_SetImmortal` on strings directly. You should use `_PyUnicode_InternImmortal` instead: - Strings should be interned before immortalization, otherwise you're possibly interning a immortalizing copy. - `_Py_SetImmortal` doesn't handle the `SSTATE_INTERNED_MORTAL` to `SSTATE_INTERNED_IMMORTAL` update, and those flags can't be changed in backports, as they are now part of public API and version-specific ABI.* Add private `_only_immortal` argument for `sys.getunicodeinternedsize`, used in refleak test machinery.* Make sure the statically allocated string singletons are unique. This means these sets are now disjoint: - `_Py_ID` - `_Py_STR` (including the empty string) - one-character latin-1 singletons Now, when you intern a singleton, that exact singleton will be interned.* Add a `_Py_LATIN1_CHR` macro, use it instead of `_Py_ID`/`_Py_STR` for one-character latin-1 singletons everywhere (including Clinic).* Intern `_Py_STR` singletons at startup.* For free-threaded builds, intern `_Py_LATIN1_CHR` singletons at startup.* Beef up the tests. Cover internal details (marked with `@cpython_only`).* Add lots of assertionsCo-Authored-By: Eric Snow <ericsnowcurrently@gmail.com>

1 parent48d7ef9 commit407e3dfCopy full SHA for 407e3df

File tree

42 files changed

+2460

-1136

lines changed

Include/internal
InternalDocs
- string_interning.md
Lib/test
- clinic.test.c
- libregrtest
  - refleak.py
- test_clinic.py
- test_sys.py
Misc/NEWS.d/next/Core and Builtins
- 2024-06-14-22-02-25.gh-issue-113993.MiA0vX.rst
Modules
- _io
  - stringio.c
- _operator.c
- _pickle.c
- _sqlite/clinic
  - connection.c.h
- clinic
Objects
Parser
- action_helpers.c
- pegen.c
Python
Tools
- build
  - generate_global_objects.py
- clinic/libclinic
  - clanguage.py

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+2460

-1136

lines changed

`‎Include/internal/pycore_global_objects_fini_generated.h`

Lines changed: 1 addition & 19 deletions

Some generated files are not rendered by default. Learn more aboutcustomizing how changed files appear on GitHub.

`‎Include/internal/pycore_global_strings.h`

Lines changed: 5 additions & 19 deletions

Original file line number	Diff line number	Diff line change
`@@ -37,22 +37,17 @@ struct _Py_global_strings {`
`37`	`37`	`STRUCT_FOR_STR(anon_setcomp, "<setcomp>")`
`38`	`38`	`STRUCT_FOR_STR(anon_string, "<string>")`
`39`	`39`	`STRUCT_FOR_STR(anon_unknown, "<unknown>")`
`40`		`-STRUCT_FOR_STR(close_br, "}")`
`41`	`40`	`STRUCT_FOR_STR(dbl_close_br, "}}")`
`42`	`41`	`STRUCT_FOR_STR(dbl_open_br, "{{")`
`43`	`42`	`STRUCT_FOR_STR(dbl_percent, "%%")`
`44`	`43`	`STRUCT_FOR_STR(defaults, ".defaults")`
`45`		`-STRUCT_FOR_STR(dot, ".")`
`46`	`44`	`STRUCT_FOR_STR(dot_locals, ".<locals>")`
`47`	`45`	`STRUCT_FOR_STR(empty, "")`
`48`	`46`	`STRUCT_FOR_STR(format, ".format")`
`49`	`47`	`STRUCT_FOR_STR(generic_base, ".generic_base")`
`50`	`48`	`STRUCT_FOR_STR(json_decoder, "json.decoder")`
`51`	`49`	`STRUCT_FOR_STR(kwdefaults, ".kwdefaults")`
`52`	`50`	`STRUCT_FOR_STR(list_err, "listindexoutofrange")`
`53`		`-STRUCT_FOR_STR(newline, "\n")`
`54`		`-STRUCT_FOR_STR(open_br, "{")`
`55`		`-STRUCT_FOR_STR(percent, "%")`
`56`	`51`	`STRUCT_FOR_STR(type_params, ".type_params")`
`57`	`52`	`STRUCT_FOR_STR(utf_8, "utf-8")`
`58`	`53`	`}literals;`
`@@ -67,7 +62,6 @@ struct _Py_global_strings {`
`67`	`62`	`STRUCT_FOR_ID(TextIOWrapper)`
`68`	`63`	`STRUCT_FOR_ID(True)`
`69`	`64`	`STRUCT_FOR_ID(WarningMessage)`
`70`		`-STRUCT_FOR_ID(_)`
`71`	`65`	`STRUCT_FOR_ID(_WindowsConsoleIO)`
`72`	`66`	`STRUCT_FOR_ID(__IOBase_closed)`
`73`	`67`	`STRUCT_FOR_ID(__abc_tpflags__)`
`@@ -261,6 +255,7 @@ struct _Py_global_strings {`
`261`	`255`	`STRUCT_FOR_ID(_lock_unlock_module)`
`262`	`256`	`STRUCT_FOR_ID(_loop)`
`263`	`257`	`STRUCT_FOR_ID(_needs_com_addref_)`
	`258`	`+STRUCT_FOR_ID(_only_immortal)`
`264`	`259`	`STRUCT_FOR_ID(_pack_)`
`265`	`260`	`STRUCT_FOR_ID(_restype_)`
`266`	`261`	`STRUCT_FOR_ID(_showwarnmsg)`
`@@ -273,7 +268,6 @@ struct _Py_global_strings {`
`273`	`268`	`STRUCT_FOR_ID(_uninitialized_submodules)`
`274`	`269`	`STRUCT_FOR_ID(_warn_unawaited_coroutine)`
`275`	`270`	`STRUCT_FOR_ID(_xoptions)`
`276`		`-STRUCT_FOR_ID(a)`
`277`	`271`	`STRUCT_FOR_ID(abs_tol)`
`278`	`272`	`STRUCT_FOR_ID(access)`
`279`	`273`	`STRUCT_FOR_ID(aclose)`
`@@ -297,7 +291,6 @@ struct _Py_global_strings {`
`297`	`291`	`STRUCT_FOR_ID(attribute)`
`298`	`292`	`STRUCT_FOR_ID(authorizer_callback)`
`299`	`293`	`STRUCT_FOR_ID(autocommit)`
`300`		`-STRUCT_FOR_ID(b)`
`301`	`294`	`STRUCT_FOR_ID(backtick)`
`302`	`295`	`STRUCT_FOR_ID(base)`
`303`	`296`	`STRUCT_FOR_ID(before)`
`@@ -315,7 +308,6 @@ struct _Py_global_strings {`
`315`	`308`	`STRUCT_FOR_ID(byteorder)`
`316`	`309`	`STRUCT_FOR_ID(bytes)`
`317`	`310`	`STRUCT_FOR_ID(bytes_per_sep)`
`318`		`-STRUCT_FOR_ID(c)`
`319`	`311`	`STRUCT_FOR_ID(c_call)`
`320`	`312`	`STRUCT_FOR_ID(c_exception)`
`321`	`313`	`STRUCT_FOR_ID(c_return)`
`@@ -371,7 +363,6 @@ struct _Py_global_strings {`
`371`	`363`	`STRUCT_FOR_ID(count)`
`372`	`364`	`STRUCT_FOR_ID(covariant)`
`373`	`365`	`STRUCT_FOR_ID(cwd)`
`374`		`-STRUCT_FOR_ID(d)`
`375`	`366`	`STRUCT_FOR_ID(data)`
`376`	`367`	`STRUCT_FOR_ID(database)`
`377`	`368`	`STRUCT_FOR_ID(day)`
`@@ -400,7 +391,6 @@ struct _Py_global_strings {`
`400`	`391`	`STRUCT_FOR_ID(dont_inherit)`
`401`	`392`	`STRUCT_FOR_ID(dst)`
`402`	`393`	`STRUCT_FOR_ID(dst_dir_fd)`
`403`		`-STRUCT_FOR_ID(e)`
`404`	`394`	`STRUCT_FOR_ID(eager_start)`
`405`	`395`	`STRUCT_FOR_ID(effective_ids)`
`406`	`396`	`STRUCT_FOR_ID(element_factory)`
`@@ -424,7 +414,6 @@ struct _Py_global_strings {`
`424`	`414`	`STRUCT_FOR_ID(exp)`
`425`	`415`	`STRUCT_FOR_ID(extend)`
`426`	`416`	`STRUCT_FOR_ID(extra_tokens)`
`427`		`-STRUCT_FOR_ID(f)`
`428`	`417`	`STRUCT_FOR_ID(facility)`
`429`	`418`	`STRUCT_FOR_ID(factory)`
`430`	`419`	`STRUCT_FOR_ID(false)`
`@@ -457,7 +446,6 @@ struct _Py_global_strings {`
`457`	`446`	`STRUCT_FOR_ID(fset)`
`458`	`447`	`STRUCT_FOR_ID(func)`
`459`	`448`	`STRUCT_FOR_ID(future)`
`460`		`-STRUCT_FOR_ID(g)`
`461`	`449`	`STRUCT_FOR_ID(generation)`
`462`	`450`	`STRUCT_FOR_ID(genexpr)`
`463`	`451`	`STRUCT_FOR_ID(get)`
`@@ -471,7 +459,6 @@ struct _Py_global_strings {`
`471`	`459`	`STRUCT_FOR_ID(globals)`
`472`	`460`	`STRUCT_FOR_ID(groupindex)`
`473`	`461`	`STRUCT_FOR_ID(groups)`
`474`		`-STRUCT_FOR_ID(h)`
`475`	`462`	`STRUCT_FOR_ID(handle)`
`476`	`463`	`STRUCT_FOR_ID(handle_seq)`
`477`	`464`	`STRUCT_FOR_ID(has_location)`
`@@ -582,7 +569,6 @@ struct _Py_global_strings {`
`582`	`569`	`STRUCT_FOR_ID(msg)`
`583`	`570`	`STRUCT_FOR_ID(mutex)`
`584`	`571`	`STRUCT_FOR_ID(mycmp)`
`585`		`-STRUCT_FOR_ID(n)`
`586`	`572`	`STRUCT_FOR_ID(n_arg)`
`587`	`573`	`STRUCT_FOR_ID(n_fields)`
`588`	`574`	`STRUCT_FOR_ID(n_sequence_fields)`
`@@ -628,7 +614,6 @@ struct _Py_global_strings {`
`628`	`614`	`STRUCT_FOR_ID(outgoing)`
`629`	`615`	`STRUCT_FOR_ID(overlapped)`
`630`	`616`	`STRUCT_FOR_ID(owner)`
`631`		`-STRUCT_FOR_ID(p)`
`632`	`617`	`STRUCT_FOR_ID(pages)`
`633`	`618`	`STRUCT_FOR_ID(parent)`
`634`	`619`	`STRUCT_FOR_ID(password)`
`@@ -656,7 +641,6 @@ struct _Py_global_strings {`
`656`	`641`	`STRUCT_FOR_ID(ps2)`
`657`	`642`	`STRUCT_FOR_ID(query)`
`658`	`643`	`STRUCT_FOR_ID(quotetabs)`
`659`		`-STRUCT_FOR_ID(r)`
`660`	`644`	`STRUCT_FOR_ID(raw)`
`661`	`645`	`STRUCT_FOR_ID(read)`
`662`	`646`	`STRUCT_FOR_ID(read1)`
`@@ -680,7 +664,6 @@ struct _Py_global_strings {`
`680`	`664`	`STRUCT_FOR_ID(return)`
`681`	`665`	`STRUCT_FOR_ID(reverse)`
`682`	`666`	`STRUCT_FOR_ID(reversed)`
`683`		`-STRUCT_FOR_ID(s)`
`684`	`667`	`STRUCT_FOR_ID(salt)`
`685`	`668`	`STRUCT_FOR_ID(sched_priority)`
`686`	`669`	`STRUCT_FOR_ID(scheduler)`
`@@ -787,7 +770,6 @@ struct _Py_global_strings {`
`787`	`770`	`STRUCT_FOR_ID(writable)`
`788`	`771`	`STRUCT_FOR_ID(write)`
`789`	`772`	`STRUCT_FOR_ID(write_through)`
`790`		`-STRUCT_FOR_ID(x)`
`791`	`773`	`STRUCT_FOR_ID(year)`
`792`	`774`	`STRUCT_FOR_ID(zdict)`
`793`	`775`	`}identifiers;`
`@@ -810,6 +792,10 @@ struct _Py_global_strings {`
`810`	`792`	`(_Py_SINGLETON(strings.identifiers._py_ ## NAME._ascii.ob_base))`
`811`	`793`	`#define_Py_STR(NAME) \`
`812`	`794`	`(_Py_SINGLETON(strings.literals._py_ ## NAME._ascii.ob_base))`
	`795`	`+#define_Py_LATIN1_CHR(CH) \`
	`796`	`+ ((CH)<128 \`
	`797`	`+ ? (PyObject*)&_Py_SINGLETON(strings).ascii[(CH)] \`
	`798`	`+ : (PyObject*)&_Py_SINGLETON(strings).latin1[(CH)-128])`
`813`	`799`
`814`	`800`	`/* _Py_DECLARE_STR() should precede all uses of _Py_STR() in a function.`
`815`	`801`

0 commit comments

Comments

(0)

Movatterモバイル変換

Navigation Menu

Search code, repositories, users, issues, pull requests...

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Commit407e3df

File tree

42 files changed

Some content is hidden

42 files changed

`‎Include/internal/pycore_global_objects_fini_generated.h`

`‎Include/internal/pycore_global_strings.h`

0 commit comments