Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit5dd7fc1

Browse files
committed
Add transparent block-level memory accounting
Adds accounting of memory allocated in a memory context. Compared tovarious ad hoc solutions, the main advantage is that the accounting istransparent and does not require direct control over allocations (thismatters for use cases where the allocations happen in user code, likefor example aggregate states allocated in a transition functions).To reduce overhead, the accounting happens at the block level (not forindividual chunks) and only the context immediately owning the block isupdated. When inquiring about amount of memory allocated in a context,we have to recursively walk all children contexts.This "lazy" accounting works well for cases with relatively small numberof contexts in the relevant subtree and/or with infrequent inquiries.Author: Jeff DavisReivewed-by: Tomas Vondra, Melanie Plageman, Soumyadeep ChakrabortyDiscussion:https://www.postgresql.org/message-id/flat/027a129b8525601c6a680d27ce3a7172dab61aab.camel@j-davis.com
1 parent36d22dd commit5dd7fc1

File tree

7 files changed

+121
-0
lines changed

7 files changed

+121
-0
lines changed

‎src/backend/utils/mmgr/README

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ The basic operations on a memory context are:
2323
* reset a context (free all memory allocated in the context, but not the
2424
context object itself)
2525

26+
* inquire about the total amount of memory allocated to the context
27+
(the raw memory from which the context allocates chunks; not the
28+
chunks themselves)
29+
2630
Given a chunk of memory previously allocated from a context, one can
2731
free it or reallocate it larger or smaller (corresponding to standard C
2832
library's free() and realloc() routines). These operations return memory
@@ -452,3 +456,33 @@ returns the memory when reset/deleted).
452456

453457
These memory contexts were initially developed for ReorderBuffer, but
454458
may be useful elsewhere as long as the allocation patterns match.
459+
460+
461+
Memory Accounting
462+
-----------------
463+
464+
One of the basic memory context operations is determining the amount of
465+
memory used in the context (and it's children). We have multiple places
466+
that implement their own ad hoc memory accounting, and this is meant to
467+
provide a unified approach. Ad hoc accounting solutions work for places
468+
with tight control over the allocations or when it's easy to determine
469+
sizes of allocated chunks (e.g. places that only work with tuples).
470+
471+
The accounting built into the memory contexts is transparent and works
472+
transparently for all allocations as long as they end up in the right
473+
memory context subtree.
474+
475+
Consider for example aggregate functions - the aggregate state is often
476+
represented by an arbitrary structure, allocated from the transition
477+
function, so the ad hoc accounting is unlikely to work. The built-in
478+
accounting will however handle such cases just fine.
479+
480+
To minimize overhead, the accounting is done at the block level, not for
481+
individual allocation chunks.
482+
483+
The accounting is lazy - after a block is allocated (or freed), only the
484+
context owning that block is updated. This means that when inquiring
485+
about the memory usage in a given context, we have to walk all children
486+
contexts recursively. This means the memory accounting is not intended
487+
for cases with too many memory contexts (in the relevant subtree).
488+

‎src/backend/utils/mmgr/aset.c

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -458,6 +458,9 @@ AllocSetContextCreateInternal(MemoryContext parent,
458458
parent,
459459
name);
460460

461+
((MemoryContext)set)->mem_allocated=
462+
set->keeper->endptr- ((char*)set);
463+
461464
return (MemoryContext)set;
462465
}
463466
}
@@ -546,6 +549,8 @@ AllocSetContextCreateInternal(MemoryContext parent,
546549
parent,
547550
name);
548551

552+
((MemoryContext)set)->mem_allocated=firstBlockSize;
553+
549554
return (MemoryContext)set;
550555
}
551556

@@ -566,6 +571,7 @@ AllocSetReset(MemoryContext context)
566571
{
567572
AllocSetset= (AllocSet)context;
568573
AllocBlockblock;
574+
Sizekeepersize=set->keeper->endptr- ((char*)set);
569575

570576
AssertArg(AllocSetIsValid(set));
571577

@@ -604,6 +610,8 @@ AllocSetReset(MemoryContext context)
604610
else
605611
{
606612
/* Normal case, release the block */
613+
context->mem_allocated-=block->endptr- ((char*)block);
614+
607615
#ifdefCLOBBER_FREED_MEMORY
608616
wipe_mem(block,block->freeptr- ((char*)block));
609617
#endif
@@ -612,6 +620,8 @@ AllocSetReset(MemoryContext context)
612620
block=next;
613621
}
614622

623+
Assert(context->mem_allocated==keepersize);
624+
615625
/* Reset block size allocation sequence, too */
616626
set->nextBlockSize=set->initBlockSize;
617627
}
@@ -628,6 +638,7 @@ AllocSetDelete(MemoryContext context)
628638
{
629639
AllocSetset= (AllocSet)context;
630640
AllocBlockblock=set->blocks;
641+
Sizekeepersize=set->keeper->endptr- ((char*)set);
631642

632643
AssertArg(AllocSetIsValid(set));
633644

@@ -683,6 +694,9 @@ AllocSetDelete(MemoryContext context)
683694
{
684695
AllocBlocknext=block->next;
685696

697+
if (block!=set->keeper)
698+
context->mem_allocated-=block->endptr- ((char*)block);
699+
686700
#ifdefCLOBBER_FREED_MEMORY
687701
wipe_mem(block,block->freeptr- ((char*)block));
688702
#endif
@@ -693,6 +707,8 @@ AllocSetDelete(MemoryContext context)
693707
block=next;
694708
}
695709

710+
Assert(context->mem_allocated==keepersize);
711+
696712
/* Finally, free the context header, including the keeper block */
697713
free(set);
698714
}
@@ -733,6 +749,9 @@ AllocSetAlloc(MemoryContext context, Size size)
733749
block= (AllocBlock)malloc(blksize);
734750
if (block==NULL)
735751
returnNULL;
752+
753+
context->mem_allocated+=blksize;
754+
736755
block->aset=set;
737756
block->freeptr=block->endptr= ((char*)block)+blksize;
738757

@@ -928,6 +947,8 @@ AllocSetAlloc(MemoryContext context, Size size)
928947
if (block==NULL)
929948
returnNULL;
930949

950+
context->mem_allocated+=blksize;
951+
931952
block->aset=set;
932953
block->freeptr= ((char*)block)+ALLOC_BLOCKHDRSZ;
933954
block->endptr= ((char*)block)+blksize;
@@ -1028,6 +1049,9 @@ AllocSetFree(MemoryContext context, void *pointer)
10281049
set->blocks=block->next;
10291050
if (block->next)
10301051
block->next->prev=block->prev;
1052+
1053+
context->mem_allocated-=block->endptr- ((char*)block);
1054+
10311055
#ifdefCLOBBER_FREED_MEMORY
10321056
wipe_mem(block,block->freeptr- ((char*)block));
10331057
#endif
@@ -1144,6 +1168,7 @@ AllocSetRealloc(MemoryContext context, void *pointer, Size size)
11441168
AllocBlockblock= (AllocBlock) (((char*)chunk)-ALLOC_BLOCKHDRSZ);
11451169
Sizechksize;
11461170
Sizeblksize;
1171+
Sizeoldblksize;
11471172

11481173
/*
11491174
* Try to verify that we have a sane block pointer: it should
@@ -1159,13 +1184,18 @@ AllocSetRealloc(MemoryContext context, void *pointer, Size size)
11591184
/* Do the realloc */
11601185
chksize=MAXALIGN(size);
11611186
blksize=chksize+ALLOC_BLOCKHDRSZ+ALLOC_CHUNKHDRSZ;
1187+
oldblksize=block->endptr- ((char*)block);
1188+
11621189
block= (AllocBlock)realloc(block,blksize);
11631190
if (block==NULL)
11641191
{
11651192
/* Disallow external access to private part of chunk header. */
11661193
VALGRIND_MAKE_MEM_NOACCESS(chunk,ALLOCCHUNK_PRIVATE_LEN);
11671194
returnNULL;
11681195
}
1196+
1197+
context->mem_allocated+=blksize-oldblksize;
1198+
11691199
block->freeptr=block->endptr= ((char*)block)+blksize;
11701200

11711201
/* Update pointers since block has likely been moved */
@@ -1383,6 +1413,7 @@ AllocSetCheck(MemoryContext context)
13831413
constchar*name=set->header.name;
13841414
AllocBlockprevblock;
13851415
AllocBlockblock;
1416+
int64total_allocated=0;
13861417

13871418
for (prevblock=NULL,block=set->blocks;
13881419
block!=NULL;
@@ -1393,6 +1424,11 @@ AllocSetCheck(MemoryContext context)
13931424
longblk_data=0;
13941425
longnchunks=0;
13951426

1427+
if (set->keeper==block)
1428+
total_allocated+=block->endptr- ((char*)set);
1429+
else
1430+
total_allocated+=block->endptr- ((char*)block);
1431+
13961432
/*
13971433
* Empty block - empty can be keeper-block only
13981434
*/
@@ -1479,6 +1515,8 @@ AllocSetCheck(MemoryContext context)
14791515
elog(WARNING,"problem in alloc set %s: found inconsistent memory block %p",
14801516
name,block);
14811517
}
1518+
1519+
Assert(total_allocated==context->mem_allocated);
14821520
}
14831521

14841522
#endif/* MEMORY_CONTEXT_CHECKING */

‎src/backend/utils/mmgr/generation.c

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,8 @@ GenerationReset(MemoryContext context)
297297

298298
dlist_delete(miter.cur);
299299

300+
context->mem_allocated-=block->blksize;
301+
300302
#ifdefCLOBBER_FREED_MEMORY
301303
wipe_mem(block,block->blksize);
302304
#endif
@@ -352,6 +354,8 @@ GenerationAlloc(MemoryContext context, Size size)
352354
if (block==NULL)
353355
returnNULL;
354356

357+
context->mem_allocated+=blksize;
358+
355359
/* block with a single (used) chunk */
356360
block->blksize=blksize;
357361
block->nchunks=1;
@@ -407,6 +411,8 @@ GenerationAlloc(MemoryContext context, Size size)
407411
if (block==NULL)
408412
returnNULL;
409413

414+
context->mem_allocated+=blksize;
415+
410416
block->blksize=blksize;
411417
block->nchunks=0;
412418
block->nfree=0;
@@ -522,6 +528,7 @@ GenerationFree(MemoryContext context, void *pointer)
522528
if (set->block==block)
523529
set->block=NULL;
524530

531+
context->mem_allocated-=block->blksize;
525532
free(block);
526533
}
527534

@@ -746,6 +753,7 @@ GenerationCheck(MemoryContext context)
746753
GenerationContext*gen= (GenerationContext*)context;
747754
constchar*name=context->name;
748755
dlist_iteriter;
756+
int64total_allocated=0;
749757

750758
/* walk all blocks in this context */
751759
dlist_foreach(iter,&gen->blocks)
@@ -755,6 +763,8 @@ GenerationCheck(MemoryContext context)
755763
nchunks;
756764
char*ptr;
757765

766+
total_allocated+=block->blksize;
767+
758768
/*
759769
* nfree > nchunks is surely wrong, and we don't expect to see
760770
* equality either, because such a block should have gotten freed.
@@ -833,6 +843,8 @@ GenerationCheck(MemoryContext context)
833843
elog(WARNING,"problem in Generation %s: number of free chunks %d in block %p does not match header %d",
834844
name,nfree,block,block->nfree);
835845
}
846+
847+
Assert(total_allocated==context->mem_allocated);
836848
}
837849

838850
#endif/* MEMORY_CONTEXT_CHECKING */

‎src/backend/utils/mmgr/mcxt.c

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -462,6 +462,30 @@ MemoryContextIsEmpty(MemoryContext context)
462462
returncontext->methods->is_empty(context);
463463
}
464464

465+
/*
466+
* Find the memory allocated to blocks for this memory context. If recurse is
467+
* true, also include children.
468+
*/
469+
int64
470+
MemoryContextMemAllocated(MemoryContextcontext,boolrecurse)
471+
{
472+
int64total=context->mem_allocated;
473+
474+
AssertArg(MemoryContextIsValid(context));
475+
476+
if (recurse)
477+
{
478+
MemoryContextchild=context->firstchild;
479+
480+
for (child=context->firstchild;
481+
child!=NULL;
482+
child=child->nextchild)
483+
total+=MemoryContextMemAllocated(child, true);
484+
}
485+
486+
returntotal;
487+
}
488+
465489
/*
466490
* MemoryContextStats
467491
*Print statistics about the named context and all its descendants.
@@ -736,6 +760,7 @@ MemoryContextCreate(MemoryContext node,
736760
node->methods=methods;
737761
node->parent=parent;
738762
node->firstchild=NULL;
763+
node->mem_allocated=0;
739764
node->prevchild=NULL;
740765
node->name=name;
741766
node->ident=NULL;

‎src/backend/utils/mmgr/slab.c

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -305,12 +305,14 @@ SlabReset(MemoryContext context)
305305
#endif
306306
free(block);
307307
slab->nblocks--;
308+
context->mem_allocated-=slab->blockSize;
308309
}
309310
}
310311

311312
slab->minFreeChunks=0;
312313

313314
Assert(slab->nblocks==0);
315+
Assert(context->mem_allocated==0);
314316
}
315317

316318
/*
@@ -388,6 +390,7 @@ SlabAlloc(MemoryContext context, Size size)
388390

389391
slab->minFreeChunks=slab->chunksPerBlock;
390392
slab->nblocks+=1;
393+
context->mem_allocated+=slab->blockSize;
391394
}
392395

393396
/* grab the block from the freelist (even the new block is there) */
@@ -480,6 +483,9 @@ SlabAlloc(MemoryContext context, Size size)
480483
#endif
481484

482485
SlabAllocInfo(slab,chunk);
486+
487+
Assert(slab->nblocks*slab->blockSize==context->mem_allocated);
488+
483489
returnSlabChunkGetPointer(chunk);
484490
}
485491

@@ -555,11 +561,13 @@ SlabFree(MemoryContext context, void *pointer)
555561
{
556562
free(block);
557563
slab->nblocks--;
564+
context->mem_allocated-=slab->blockSize;
558565
}
559566
else
560567
dlist_push_head(&slab->freelist[block->nfree],&block->node);
561568

562569
Assert(slab->nblocks >=0);
570+
Assert(slab->nblocks*slab->blockSize==context->mem_allocated);
563571
}
564572

565573
/*
@@ -782,6 +790,8 @@ SlabCheck(MemoryContext context)
782790
name,block->nfree,block,nfree);
783791
}
784792
}
793+
794+
Assert(slab->nblocks*slab->blockSize==context->mem_allocated);
785795
}
786796

787797
#endif/* MEMORY_CONTEXT_CHECKING */

‎src/include/nodes/memnodes.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ typedef struct MemoryContextData
7979
/* these two fields are placed here to minimize alignment wastage: */
8080
boolisReset;/* T = no space alloced since last reset */
8181
boolallowInCritSection;/* allow palloc in critical section */
82+
int64mem_allocated;/* track memory allocated for this context */
8283
constMemoryContextMethods*methods;/* virtual function table */
8384
MemoryContextparent;/* NULL if no parent (toplevel context) */
8485
MemoryContextfirstchild;/* head of linked list of children */

‎src/include/utils/memutils.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ extern void MemoryContextSetParent(MemoryContext context,
8282
externSizeGetMemoryChunkSpace(void*pointer);
8383
externMemoryContextMemoryContextGetParent(MemoryContextcontext);
8484
externboolMemoryContextIsEmpty(MemoryContextcontext);
85+
externint64MemoryContextMemAllocated(MemoryContextcontext,boolrecurse);
8586
externvoidMemoryContextStats(MemoryContextcontext);
8687
externvoidMemoryContextStatsDetail(MemoryContextcontext,intmax_children);
8788
externvoidMemoryContextAllowInCriticalSection(MemoryContextcontext,

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp