Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit263865a

Browse files
committed
Permit super-MaxAllocSize allocations with MemoryContextAllocHuge().
The MaxAllocSize guard is convenient for most callers, because itreduces the need for careful attention to overflow, data type selection,and the SET_VARSIZE() limit. A handful of callers are happy to navigatethose hazards in exchange for the ability to allocate a larger chunk.Introduce MemoryContextAllocHuge() and repalloc_huge(). Use this intuplesort.c and tuplestore.c, enabling internal sorts of up to INT_MAXtuples, a factor-of-48 increase. In particular, B-tree index builds cannow benefit from much-larger maintenance_work_mem settings.Reviewed by Stephen Frost, Simon Riggs and Jeff Janes.
1 parent9ef86cd commit263865a

File tree

7 files changed

+183
-98
lines changed

7 files changed

+183
-98
lines changed

‎src/backend/utils/mmgr/aset.c

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -458,6 +458,7 @@ AllocSetContextCreate(MemoryContext parent,
458458
maxBlockSize=MAXALIGN(maxBlockSize);
459459
if (maxBlockSize<initBlockSize)
460460
maxBlockSize=initBlockSize;
461+
Assert(AllocHugeSizeIsValid(maxBlockSize));/* must be safe to double */
461462
context->initBlockSize=initBlockSize;
462463
context->maxBlockSize=maxBlockSize;
463464
context->nextBlockSize=initBlockSize;
@@ -643,6 +644,10 @@ AllocSetDelete(MemoryContext context)
643644
* AllocSetAlloc
644645
*Returns pointer to allocated memory of given size; memory is added
645646
*to the set.
647+
*
648+
* No request may exceed:
649+
*MAXALIGN_DOWN(SIZE_MAX) - ALLOC_BLOCKHDRSZ - ALLOC_CHUNKHDRSZ
650+
* All callers use a much-lower limit.
646651
*/
647652
staticvoid*
648653
AllocSetAlloc(MemoryContextcontext,Sizesize)

‎src/backend/utils/mmgr/mcxt.c

Lines changed: 66 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -455,14 +455,7 @@ MemoryContextContains(MemoryContext context, void *pointer)
455455
header= (StandardChunkHeader*)
456456
((char*)pointer-STANDARDCHUNKHEADERSIZE);
457457

458-
/*
459-
* If the context link doesn't match then we certainly have a non-member
460-
* chunk. Also check for a reasonable-looking size as extra guard against
461-
* being fooled by bogus pointers.
462-
*/
463-
if (header->context==context&&AllocSizeIsValid(header->size))
464-
return true;
465-
return false;
458+
returnheader->context==context;
466459
}
467460

468461
/*--------------------
@@ -757,6 +750,71 @@ repalloc(void *pointer, Size size)
757750
returnret;
758751
}
759752

753+
/*
754+
* MemoryContextAllocHuge
755+
*Allocate (possibly-expansive) space within the specified context.
756+
*
757+
* See considerations in comment at MaxAllocHugeSize.
758+
*/
759+
void*
760+
MemoryContextAllocHuge(MemoryContextcontext,Sizesize)
761+
{
762+
void*ret;
763+
764+
AssertArg(MemoryContextIsValid(context));
765+
766+
if (!AllocHugeSizeIsValid(size))
767+
elog(ERROR,"invalid memory alloc request size %lu",
768+
(unsigned long)size);
769+
770+
context->isReset= false;
771+
772+
ret= (*context->methods->alloc) (context,size);
773+
VALGRIND_MEMPOOL_ALLOC(context,ret,size);
774+
775+
returnret;
776+
}
777+
778+
/*
779+
* repalloc_huge
780+
*Adjust the size of a previously allocated chunk, permitting a large
781+
*value. The previous allocation need not have been "huge".
782+
*/
783+
void*
784+
repalloc_huge(void*pointer,Sizesize)
785+
{
786+
MemoryContextcontext;
787+
void*ret;
788+
789+
if (!AllocHugeSizeIsValid(size))
790+
elog(ERROR,"invalid memory alloc request size %lu",
791+
(unsigned long)size);
792+
793+
/*
794+
* Try to detect bogus pointers handed to us, poorly though we can.
795+
* Presumably, a pointer that isn't MAXALIGNED isn't pointing at an
796+
* allocated chunk.
797+
*/
798+
Assert(pointer!=NULL);
799+
Assert(pointer== (void*)MAXALIGN(pointer));
800+
801+
/*
802+
* OK, it's probably safe to look at the chunk header.
803+
*/
804+
context= ((StandardChunkHeader*)
805+
((char*)pointer-STANDARDCHUNKHEADERSIZE))->context;
806+
807+
AssertArg(MemoryContextIsValid(context));
808+
809+
/* isReset must be false already */
810+
Assert(!context->isReset);
811+
812+
ret= (*context->methods->realloc) (context,pointer,size);
813+
VALGRIND_MEMPOOL_CHANGE(context,pointer,ret,size);
814+
815+
returnret;
816+
}
817+
760818
/*
761819
* MemoryContextStrdup
762820
*Like strdup(), but allocate from the specified context

‎src/backend/utils/sort/tuplesort.c

Lines changed: 49 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -211,8 +211,8 @@ struct Tuplesortstate
211211
* tuples to return? */
212212
boolboundUsed;/* true if we made use of a bounded heap */
213213
intbound;/* if bounded, the maximum number of tuples */
214-
longavailMem;/* remaining memory available, in bytes */
215-
longallowedMem;/* total memory allowed, in bytes */
214+
SizeavailMem;/* remaining memory available, in bytes */
215+
SizeallowedMem;/* total memory allowed, in bytes */
216216
intmaxTapes;/* number of tapes (Knuth's T) */
217217
inttapeRange;/* maxTapes-1 (Knuth's P) */
218218
MemoryContextsortcontext;/* memory context holding all sort data */
@@ -308,7 +308,7 @@ struct Tuplesortstate
308308
int*mergenext;/* first preread tuple for each source */
309309
int*mergelast;/* last preread tuple for each source */
310310
int*mergeavailslots;/* slots left for prereading each tape */
311-
long*mergeavailmem;/* availMem for prereading each tape */
311+
Size*mergeavailmem;/* availMem for prereading each tape */
312312
intmergefreelist;/* head of freelist of recycled slots */
313313
intmergefirstfree;/* first slot never used in this merge */
314314

@@ -961,25 +961,26 @@ tuplesort_end(Tuplesortstate *state)
961961
}
962962

963963
/*
964-
* Grow the memtuples[] array, if possible within our memory constraint.
965-
* Return TRUE if we were able to enlarge the array, FALSE if not.
964+
* Grow the memtuples[] array, if possible within our memory constraint. We
965+
* must not exceed INT_MAX tuples in memory or the caller-provided memory
966+
* limit. Return TRUE if we were able to enlarge the array, FALSE if not.
966967
*
967-
* Normally, at each increment we double the size of the array. Whenwe no
968-
*longer have enough memory to do that, we attempt one last, smaller increase
969-
*(and thenclear the growmemtuples flag so we don't try any more). That
970-
*allows us touseallowedMem as fully aspossible; sticking to the pure
971-
*doubling rule couldresult in almost halfof allowedMemgoing unused.
972-
*Because availMem moves around withtuple addition/removal, we need some
973-
*rule to prevent making repeated smallincreases in memtupsize, which would
974-
*just be useless thrashing. Thegrowmemtuples flag accomplishes that and
975-
*also prevents uselessrecalculations in this function.
968+
* Normally, at each increment we double the size of the array. Whendoing
969+
*that would exceed a limit, we attempt one last, smaller increase (and then
970+
* clear the growmemtuples flag so we don't try any more). That allows us to
971+
* usememory as fully aspermitted; sticking to the pure doubling rule could
972+
* result in almost half going unused. Because availMem moves around with
973+
* tuple addition/removal, we need some rule to prevent making repeated small
974+
* increases in memtupsize, which would just be useless thrashing. The
975+
* growmemtuples flag accomplishes that and also prevents useless
976+
* recalculations in this function.
976977
*/
977978
staticbool
978979
grow_memtuples(Tuplesortstate*state)
979980
{
980981
intnewmemtupsize;
981982
intmemtupsize=state->memtupsize;
982-
longmemNowUsed=state->allowedMem-state->availMem;
983+
SizememNowUsed=state->allowedMem-state->availMem;
983984

984985
/* Forget it if we've already maxed out memtuples, per comment above */
985986
if (!state->growmemtuples)
@@ -989,14 +990,16 @@ grow_memtuples(Tuplesortstate *state)
989990
if (memNowUsed <=state->availMem)
990991
{
991992
/*
992-
* It is surely safe to double memtupsize if we've used no more than
993-
* half of allowedMem.
994-
*
995-
* Note: it might seem that we need to worry about memtupsize * 2
996-
* overflowing an int, but the MaxAllocSize clamp applied below
997-
* ensures the existing memtupsize can't be large enough for that.
993+
* We've used no more than half of allowedMem; double our usage,
994+
* clamping at INT_MAX.
998995
*/
999-
newmemtupsize=memtupsize*2;
996+
if (memtupsize<INT_MAX /2)
997+
newmemtupsize=memtupsize*2;
998+
else
999+
{
1000+
newmemtupsize=INT_MAX;
1001+
state->growmemtuples= false;
1002+
}
10001003
}
10011004
else
10021005
{
@@ -1012,24 +1015,27 @@ grow_memtuples(Tuplesortstate *state)
10121015
* we've already seen, and thus we can extrapolate from the space
10131016
* consumption so far to estimate an appropriate new size for the
10141017
* memtuples array. The optimal value might be higher or lower than
1015-
* this estimate, but it's hard to know that in advance.
1018+
* this estimate, but it's hard to know that in advance. We again
1019+
* clamp at INT_MAX tuples.
10161020
*
10171021
* This calculation is safe against enlarging the array so much that
10181022
* LACKMEM becomes true, because the memory currently used includes
10191023
* the present array; thus, there would be enough allowedMem for the
10201024
* new array elements even if no other memory were currently used.
10211025
*
10221026
* We do the arithmetic in float8, because otherwise the product of
1023-
* memtupsize and allowedMem could overflow. (A little algebra shows
1024-
* that grow_ratio must be less than 2 here, so we are not risking
1025-
* integer overflow this way.)Any inaccuracy in the result should be
1026-
* insignificant; but even if we computed a completely insane result,
1027-
* the checks below will prevent anything really bad from happening.
1027+
* memtupsize and allowedMem could overflow. Any inaccuracy in the
1028+
* result should be insignificant; but even if we computed a
1029+
* completely insane result, the checks below will prevent anything
1030+
* really bad from happening.
10281031
*/
10291032
doublegrow_ratio;
10301033

10311034
grow_ratio= (double)state->allowedMem / (double)memNowUsed;
1032-
newmemtupsize= (int) (memtupsize*grow_ratio);
1035+
if (memtupsize*grow_ratio<INT_MAX)
1036+
newmemtupsize= (int) (memtupsize*grow_ratio);
1037+
else
1038+
newmemtupsize=INT_MAX;
10331039

10341040
/* We won't make any further enlargement attempts */
10351041
state->growmemtuples= false;
@@ -1040,12 +1046,13 @@ grow_memtuples(Tuplesortstate *state)
10401046
gotonoalloc;
10411047

10421048
/*
1043-
* On a 64-bit machine, allowedMem could be more than MaxAllocSize. Clamp
1044-
* to ensure our request won't be rejected by palloc.
1049+
* On a 32-bit machine, allowedMem could exceed MaxAllocHugeSize. Clamp
1050+
* to ensure our request won't be rejected. Note that we can easily
1051+
* exhaust address space before facing this outcome.
10451052
*/
1046-
if ((Size)newmemtupsize >=MaxAllocSize /sizeof(SortTuple))
1053+
if ((Size)newmemtupsize >=MaxAllocHugeSize /sizeof(SortTuple))
10471054
{
1048-
newmemtupsize= (int) (MaxAllocSize /sizeof(SortTuple));
1055+
newmemtupsize= (int) (MaxAllocHugeSize /sizeof(SortTuple));
10491056
state->growmemtuples= false;/* can't grow any more */
10501057
}
10511058

@@ -1060,15 +1067,15 @@ grow_memtuples(Tuplesortstate *state)
10601067
* palloc would be treating both old and new arrays as separate chunks.
10611068
* But we'll check LACKMEM explicitly below just in case.)
10621069
*/
1063-
if (state->availMem< (long) ((newmemtupsize-memtupsize)*sizeof(SortTuple)))
1070+
if (state->availMem< (Size) ((newmemtupsize-memtupsize)*sizeof(SortTuple)))
10641071
gotonoalloc;
10651072

10661073
/* OK, do it */
10671074
FREEMEM(state,GetMemoryChunkSpace(state->memtuples));
10681075
state->memtupsize=newmemtupsize;
10691076
state->memtuples= (SortTuple*)
1070-
repalloc(state->memtuples,
1071-
state->memtupsize*sizeof(SortTuple));
1077+
repalloc_huge(state->memtuples,
1078+
state->memtupsize*sizeof(SortTuple));
10721079
USEMEM(state,GetMemoryChunkSpace(state->memtuples));
10731080
if (LACKMEM(state))
10741081
elog(ERROR,"unexpected out-of-memory situation during sort");
@@ -1715,7 +1722,7 @@ tuplesort_getdatum(Tuplesortstate *state, bool forward,
17151722
* This is exported for use by the planner. allowedMem is in bytes.
17161723
*/
17171724
int
1718-
tuplesort_merge_order(longallowedMem)
1725+
tuplesort_merge_order(SizeallowedMem)
17191726
{
17201727
intmOrder;
17211728

@@ -1749,7 +1756,7 @@ inittapes(Tuplesortstate *state)
17491756
intmaxTapes,
17501757
ntuples,
17511758
j;
1752-
longtapeSpace;
1759+
SizetapeSpace;
17531760

17541761
/* Compute number of tapes to use: merge order plus 1 */
17551762
maxTapes=tuplesort_merge_order(state->allowedMem)+1;
@@ -1798,7 +1805,7 @@ inittapes(Tuplesortstate *state)
17981805
state->mergenext= (int*)palloc0(maxTapes*sizeof(int));
17991806
state->mergelast= (int*)palloc0(maxTapes*sizeof(int));
18001807
state->mergeavailslots= (int*)palloc0(maxTapes*sizeof(int));
1801-
state->mergeavailmem= (long*)palloc0(maxTapes*sizeof(long));
1808+
state->mergeavailmem= (Size*)palloc0(maxTapes*sizeof(Size));
18021809
state->tp_fib= (int*)palloc0(maxTapes*sizeof(int));
18031810
state->tp_runs= (int*)palloc0(maxTapes*sizeof(int));
18041811
state->tp_dummy= (int*)palloc0(maxTapes*sizeof(int));
@@ -2026,7 +2033,7 @@ mergeonerun(Tuplesortstate *state)
20262033
intsrcTape;
20272034
inttupIndex;
20282035
SortTuple*tup;
2029-
longpriorAvail,
2036+
SizepriorAvail,
20302037
spaceFreed;
20312038

20322039
/*
@@ -2100,7 +2107,7 @@ beginmerge(Tuplesortstate *state)
21002107
inttapenum;
21012108
intsrcTape;
21022109
intslotsPerTape;
2103-
longspacePerTape;
2110+
SizespacePerTape;
21042111

21052112
/* Heap should be empty here */
21062113
Assert(state->memtupcount==0);
@@ -2221,7 +2228,7 @@ mergeprereadone(Tuplesortstate *state, int srcTape)
22212228
unsignedinttuplen;
22222229
SortTuplestup;
22232230
inttupIndex;
2224-
longpriorAvail,
2231+
SizepriorAvail,
22252232
spaceUsed;
22262233

22272234
if (!state->mergeactive[srcTape])

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp