Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitfe4373d

Browse files
committed
gh-144319: Add huge pages support for pymalloc
1 parent9b154ab commitfe4373d

File tree

7 files changed

+164
-4
lines changed

7 files changed

+164
-4
lines changed

‎Doc/c-api/memory.rst‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -677,7 +677,11 @@ The pymalloc allocator
677677
Python has a *pymalloc* allocator optimized for small objects (smaller or equal
678678
to 512 bytes) with a short lifetime. It uses memory mappings called "arenas"
679679
with a fixed size of either 256 KiB on 32-bit platforms or 1 MiB on 64-bit
680-
platforms. It falls back to :c:func:`PyMem_RawMalloc` and
680+
platforms. When Python is configured with :option:`--with-pymalloc-hugepages`,
681+
the arena size on 64-bit platforms is increased to 2 MiB to match the huge page
682+
size, and arena allocation will attempt to use huge pages (``MAP_HUGETLB`` on
683+
Linux, ``MEM_LARGE_PAGES`` on Windows) with automatic fallback to regular pages.
684+
It falls back to :c:func:`PyMem_RawMalloc` and
681685
:c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes.
682686
683687
*pymalloc* is the :ref:`default allocator <default-memory-allocators>` of the

‎Include/internal/pycore_obmalloc.h‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,11 @@ typedef unsigned int pymem_uint; /* assuming >= 16 bits */
208208
* mappings to reduce heap fragmentation.
209209
*/
210210
#ifdefUSE_LARGE_ARENAS
211-
#defineARENA_BITS 20/* 1 MiB */
211+
# ifdefPYMALLOC_USE_HUGEPAGES
212+
# defineARENA_BITS 21/* 2 MiB */
213+
# else
214+
# defineARENA_BITS 20/* 1 MiB */
215+
# endif
212216
#else
213217
#defineARENA_BITS 18/* 256 KiB */
214218
#endif
@@ -469,7 +473,7 @@ nfp free pools in usable_arenas.
469473
*/
470474

471475
/* How many arena_objects do we initially allocate?
472-
* 16 = can allocate 16 arenas = 16 * ARENA_SIZE= 4MBbefore growing the
476+
* 16 = can allocate 16 arenas = 16 * ARENA_SIZE before growing the
473477
* `arenas` vector.
474478
*/
475479
#defineINITIAL_ARENA_OBJECTS 16
@@ -512,14 +516,26 @@ struct _obmalloc_mgmt {
512516
513517
memory address bit allocation for keys
514518
515-
64-bit pointers, IGNORE_BITS=0 and 2^20 arena size:
519+
ARENA_BITS is configurable: 20 (1 MiB) by default on 64-bit, or
520+
21 (2 MiB) when PYMALLOC_USE_HUGEPAGES is enabled. All bit widths
521+
below are derived from ARENA_BITS automatically.
522+
523+
64-bit pointers, IGNORE_BITS=0 and 2^20 arena size (default):
516524
15 -> MAP_TOP_BITS
517525
15 -> MAP_MID_BITS
518526
14 -> MAP_BOT_BITS
519527
20 -> ideal aligned arena
520528
----
521529
64
522530
531+
64-bit pointers, IGNORE_BITS=0 and 2^21 arena size (hugepages):
532+
15 -> MAP_TOP_BITS
533+
15 -> MAP_MID_BITS
534+
13 -> MAP_BOT_BITS
535+
21 -> ideal aligned arena
536+
----
537+
64
538+
523539
64-bit pointers, IGNORE_BITS=16, and 2^20 arena size:
524540
16 -> IGNORE_BITS
525541
10 -> MAP_TOP_BITS
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Add huge pages support for the pymalloc allocator. Patch by Pablo Galindo

‎Objects/obmalloc.c‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -496,10 +496,30 @@ void *
496496
_PyMem_ArenaAlloc(void*Py_UNUSED(ctx),size_tsize)
497497
{
498498
#ifdefMS_WINDOWS
499+
# ifdefPYMALLOC_USE_HUGEPAGES
500+
void*ptr=VirtualAlloc(NULL,size,
501+
MEM_COMMIT |MEM_RESERVE |MEM_LARGE_PAGES,
502+
PAGE_READWRITE);
503+
if (ptr!=NULL)
504+
returnptr;
505+
/* Fall back to regular pages */
506+
# endif
499507
returnVirtualAlloc(NULL,size,
500508
MEM_COMMIT |MEM_RESERVE,PAGE_READWRITE);
501509
#elif defined(ARENAS_USE_MMAP)
502510
void*ptr;
511+
# ifdefPYMALLOC_USE_HUGEPAGES
512+
# ifdefMAP_HUGETLB
513+
ptr=mmap(NULL,size,PROT_READ|PROT_WRITE,
514+
MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB,-1,0);
515+
if (ptr!=MAP_FAILED) {
516+
assert(ptr!=NULL);
517+
(void)_PyAnnotateMemoryMap(ptr,size,"cpython:pymalloc:hugepage");
518+
returnptr;
519+
}
520+
/* Fall back to regular pages */
521+
# endif
522+
# endif
503523
ptr=mmap(NULL,size,PROT_READ|PROT_WRITE,
504524
MAP_PRIVATE|MAP_ANONYMOUS,-1,0);
505525
if (ptr==MAP_FAILED)

‎configure‎

Lines changed: 78 additions & 0 deletions
Some generated files are not rendered by default. Learn more aboutcustomizing how changed files appear on GitHub.

‎configure.ac‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5061,6 +5061,44 @@ then
50615061
fi
50625062
AC_MSG_RESULT([$with_pymalloc])
50635063

5064+
AC_MSG_CHECKING([for --with-pymalloc-hugepages])
5065+
AC_ARG_WITH(
5066+
[pymalloc-hugepages],
5067+
[AS_HELP_STRING([--with-pymalloc-hugepages],
5068+
[enable huge page support for pymalloc arenas (default is no)])])
5069+
if test "$with_pymalloc_hugepages" = "yes"
5070+
then
5071+
dnl Check for MAP_HUGETLB (Linux) or MEM_LARGE_PAGES (Windows)
5072+
have_hugepage_support=no
5073+
AC_COMPILE_IFELSE(
5074+
[AC_LANG_PROGRAM([[
5075+
#include <sys/mman.h>
5076+
]],[[
5077+
int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB;
5078+
(void)flags;
5079+
]])],
5080+
[have_hugepage_support=yes],
5081+
[AC_COMPILE_IFELSE(
5082+
[AC_LANG_PROGRAM([[
5083+
#include <windows.h>
5084+
]],[[
5085+
int flags = MEM_COMMIT | MEM_RESERVE | MEM_LARGE_PAGES;
5086+
(void)flags;
5087+
]])],
5088+
[have_hugepage_support=yes],
5089+
[have_hugepage_support=no])])
5090+
if test "$have_hugepage_support" = "yes"
5091+
then
5092+
AC_DEFINE([PYMALLOC_USE_HUGEPAGES],[1],
5093+
[Define to use huge pages for pymalloc arenas])
5094+
with_pymalloc_hugepages=yes
5095+
else
5096+
AC_MSG_WARN([--with-pymalloc-hugepages requested but neither MAP_HUGETLB nor MEM_LARGE_PAGES found])
5097+
with_pymalloc_hugepages=no
5098+
fi
5099+
fi
5100+
AC_MSG_RESULT([${with_pymalloc_hugepages:-no}])
5101+
50645102
# Check for --with-c-locale-coercion
50655103
AC_MSG_CHECKING([for --with-c-locale-coercion])
50665104
AC_ARG_WITH(

‎pyconfig.h.in‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1739,6 +1739,9 @@
17391739
/* Define as the preferred size in bits of long digits */
17401740
#undef PYLONG_BITS_IN_DIGIT
17411741

1742+
/* Define to use huge pages for pymalloc arenas */
1743+
#undef PYMALLOC_USE_HUGEPAGES
1744+
17421745
/* enabled builtin hash modules */
17431746
#undef PY_BUILTIN_HASHLIB_HASHES
17441747

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2026 Movatter.jp