Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitb1e5c9f

Browse files
committed
Change logtape/tuplestore code to use int64 for block numbers
The code previously relied on "long" as type to track block numbers,which would be 4 bytes in all Windows builds or any 32-bit builds. Thislimited the code to be able to handle up to 16TB of data with thedefault block size of 8kB, like during a CLUSTER. This code now relieson a more portable int64, which should be more than enough for at leastthe next 20 years to come.This issue has been reported back in 2017, but nothing was done about itback then, so here we go now.Reported-by: Peter GeogheganReviewed-by: Heikki LinnakangasDiscussion:https://postgr.es/m/CAH2-WznCscXnWmnj=STC0aSa7QG+BRedDnZsP=Jo_R9GUZvUrg@mail.gmail.com
1 parentc99c7a4 commitb1e5c9f

File tree

5 files changed

+80
-80
lines changed

5 files changed

+80
-80
lines changed

‎src/backend/storage/file/buffile.c

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -841,14 +841,14 @@ BufFileTell(BufFile *file, int *fileno, off_t *offset)
841841
*
842842
* Performs absolute seek to the start of the n'th BLCKSZ-sized block of
843843
* the file. Note that users of this interface will fail if their files
844-
* exceed BLCKSZ *LONG_MAX bytes, but that is quite a lot; we don't work
845-
* with tables bigger than that, either...
844+
* exceed BLCKSZ *PG_INT64_MAX bytes, but that is quite a lot; we don't
845+
*workwith tables bigger than that, either...
846846
*
847847
* Result is 0 if OK, EOF if not. Logical position is not moved if an
848848
* impossible seek is attempted.
849849
*/
850850
int
851-
BufFileSeekBlock(BufFile*file,longblknum)
851+
BufFileSeekBlock(BufFile*file,int64blknum)
852852
{
853853
returnBufFileSeek(file,
854854
(int) (blknum /BUFFILE_SEG_SIZE),
@@ -901,10 +901,10 @@ BufFileSize(BufFile *file)
901901
* begins. Caller should apply this as an offset when working off block
902902
* positions that are in terms of the original BufFile space.
903903
*/
904-
long
904+
int64
905905
BufFileAppend(BufFile*target,BufFile*source)
906906
{
907-
longstartBlock=target->numFiles*BUFFILE_SEG_SIZE;
907+
int64startBlock=target->numFiles*BUFFILE_SEG_SIZE;
908908
intnewNumFiles=target->numFiles+source->numFiles;
909909
inti;
910910

‎src/backend/utils/sort/logtape.c

Lines changed: 63 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -94,9 +94,9 @@
9494
*/
9595
typedefstructTapeBlockTrailer
9696
{
97-
longprev;/* previous block on this tape, or -1 on first
97+
int64prev;/* previous block on this tape, or -1 on first
9898
* block */
99-
longnext;/* next block on this tape, or # of valid
99+
int64next;/* next block on this tape, or # of valid
100100
* bytes on last block (if < 0) */
101101
}TapeBlockTrailer;
102102

@@ -153,10 +153,10 @@ struct LogicalTape
153153
* When concatenation of worker tape BufFiles is performed, an offset to
154154
* the first block in the unified BufFile space is applied during reads.
155155
*/
156-
longfirstBlockNumber;
157-
longcurBlockNumber;
158-
longnextBlockNumber;
159-
longoffsetBlockNumber;
156+
int64firstBlockNumber;
157+
int64curBlockNumber;
158+
int64nextBlockNumber;
159+
int64offsetBlockNumber;
160160

161161
/*
162162
* Buffer for current data block(s).
@@ -172,7 +172,7 @@ struct LogicalTape
172172
* order; blocks are consumed from the end of the array (lowest block
173173
* numbers first).
174174
*/
175-
long*prealloc;
175+
int64*prealloc;
176176
intnprealloc;/* number of elements in list */
177177
intprealloc_size;/* number of elements list can hold */
178178
};
@@ -200,9 +200,9 @@ struct LogicalTapeSet
200200
* blocks that are in unused holes between worker spaces following BufFile
201201
* concatenation.
202202
*/
203-
longnBlocksAllocated;/* # of blocks allocated */
204-
longnBlocksWritten;/* # of blocks used in underlying file */
205-
longnHoleBlocks;/* # of "hole" blocks left */
203+
int64nBlocksAllocated;/* # of blocks allocated */
204+
int64nBlocksWritten;/* # of blocks used in underlying file */
205+
int64nHoleBlocks;/* # of "hole" blocks left */
206206

207207
/*
208208
* We store the numbers of recycled-and-available blocks in freeBlocks[].
@@ -213,19 +213,19 @@ struct LogicalTapeSet
213213
* LogicalTapeSetForgetFreeSpace().
214214
*/
215215
boolforgetFreeSpace;/* are we remembering free blocks? */
216-
long*freeBlocks;/* resizable array holding minheap */
217-
longnFreeBlocks;/* # of currently free blocks */
216+
int64*freeBlocks;/* resizable array holding minheap */
217+
int64nFreeBlocks;/* # of currently free blocks */
218218
SizefreeBlocksLen;/* current allocated length of freeBlocks[] */
219219
boolenable_prealloc;/* preallocate write blocks? */
220220
};
221221

222222
staticLogicalTape*ltsCreateTape(LogicalTapeSet*lts);
223-
staticvoidltsWriteBlock(LogicalTapeSet*lts,longblocknum,constvoid*buffer);
224-
staticvoidltsReadBlock(LogicalTapeSet*lts,longblocknum,void*buffer);
225-
staticlongltsGetBlock(LogicalTapeSet*lts,LogicalTape*lt);
226-
staticlongltsGetFreeBlock(LogicalTapeSet*lts);
227-
staticlongltsGetPreallocBlock(LogicalTapeSet*lts,LogicalTape*lt);
228-
staticvoidltsReleaseBlock(LogicalTapeSet*lts,longblocknum);
223+
staticvoidltsWriteBlock(LogicalTapeSet*lts,int64blocknum,constvoid*buffer);
224+
staticvoidltsReadBlock(LogicalTapeSet*lts,int64blocknum,void*buffer);
225+
staticint64ltsGetBlock(LogicalTapeSet*lts,LogicalTape*lt);
226+
staticint64ltsGetFreeBlock(LogicalTapeSet*lts);
227+
staticint64ltsGetPreallocBlock(LogicalTapeSet*lts,LogicalTape*lt);
228+
staticvoidltsReleaseBlock(LogicalTapeSet*lts,int64blocknum);
229229
staticvoidltsInitReadBuffer(LogicalTape*lt);
230230

231231

@@ -235,7 +235,7 @@ static void ltsInitReadBuffer(LogicalTape *lt);
235235
* No need for an error return convention; we ereport() on any error.
236236
*/
237237
staticvoid
238-
ltsWriteBlock(LogicalTapeSet*lts,longblocknum,constvoid*buffer)
238+
ltsWriteBlock(LogicalTapeSet*lts,int64blocknum,constvoid*buffer)
239239
{
240240
/*
241241
* BufFile does not support "holes", so if we're about to write a block
@@ -263,8 +263,8 @@ ltsWriteBlock(LogicalTapeSet *lts, long blocknum, const void *buffer)
263263
if (BufFileSeekBlock(lts->pfile,blocknum)!=0)
264264
ereport(ERROR,
265265
(errcode_for_file_access(),
266-
errmsg("could not seek to block %ld of temporary file",
267-
blocknum)));
266+
errmsg("could not seek to block %lld of temporary file",
267+
(long long)blocknum)));
268268
BufFileWrite(lts->pfile,buffer,BLCKSZ);
269269

270270
/* Update nBlocksWritten, if we extended the file */
@@ -279,13 +279,13 @@ ltsWriteBlock(LogicalTapeSet *lts, long blocknum, const void *buffer)
279279
* module should never attempt to read a block it doesn't know is there.
280280
*/
281281
staticvoid
282-
ltsReadBlock(LogicalTapeSet*lts,longblocknum,void*buffer)
282+
ltsReadBlock(LogicalTapeSet*lts,int64blocknum,void*buffer)
283283
{
284284
if (BufFileSeekBlock(lts->pfile,blocknum)!=0)
285285
ereport(ERROR,
286286
(errcode_for_file_access(),
287-
errmsg("could not seek to block %ld of temporary file",
288-
blocknum)));
287+
errmsg("could not seek to block %lld of temporary file",
288+
(long long)blocknum)));
289289
BufFileReadExact(lts->pfile,buffer,BLCKSZ);
290290
}
291291

@@ -303,7 +303,7 @@ ltsReadFillBuffer(LogicalTape *lt)
303303
do
304304
{
305305
char*thisbuf=lt->buffer+lt->nbytes;
306-
longdatablocknum=lt->nextBlockNumber;
306+
int64datablocknum=lt->nextBlockNumber;
307307

308308
/* Fetch next block number */
309309
if (datablocknum==-1L)
@@ -333,28 +333,28 @@ ltsReadFillBuffer(LogicalTape *lt)
333333
return (lt->nbytes>0);
334334
}
335335

336-
staticinlineunsigned long
337-
left_offset(unsigned longi)
336+
staticinlineuint64
337+
left_offset(uint64i)
338338
{
339339
return2*i+1;
340340
}
341341

342-
staticinlineunsigned long
343-
right_offset(unsigned longi)
342+
staticinlineuint64
343+
right_offset(uint64i)
344344
{
345345
return2*i+2;
346346
}
347347

348-
staticinlineunsigned long
349-
parent_offset(unsigned longi)
348+
staticinlineuint64
349+
parent_offset(uint64i)
350350
{
351351
return (i-1) /2;
352352
}
353353

354354
/*
355355
* Get the next block for writing.
356356
*/
357-
staticlong
357+
staticint64
358358
ltsGetBlock(LogicalTapeSet*lts,LogicalTape*lt)
359359
{
360360
if (lts->enable_prealloc)
@@ -367,14 +367,14 @@ ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt)
367367
* Select the lowest currently unused block from the tape set's global free
368368
* list min heap.
369369
*/
370-
staticlong
370+
staticint64
371371
ltsGetFreeBlock(LogicalTapeSet*lts)
372372
{
373-
long*heap=lts->freeBlocks;
374-
longblocknum;
375-
longheapsize;
376-
longholeval;
377-
unsigned longholepos;
373+
int64*heap=lts->freeBlocks;
374+
int64blocknum;
375+
int64heapsize;
376+
int64holeval;
377+
uint64holepos;
378378

379379
/* freelist empty; allocate a new block */
380380
if (lts->nFreeBlocks==0)
@@ -398,9 +398,9 @@ ltsGetFreeBlock(LogicalTapeSet *lts)
398398
heapsize=lts->nFreeBlocks;
399399
while (true)
400400
{
401-
unsigned longleft=left_offset(holepos);
402-
unsigned longright=right_offset(holepos);
403-
unsigned longmin_child;
401+
uint64left=left_offset(holepos);
402+
uint64right=right_offset(holepos);
403+
uint64min_child;
404404

405405
if (left<heapsize&&right<heapsize)
406406
min_child= (heap[left]<heap[right]) ?left :right;
@@ -427,7 +427,7 @@ ltsGetFreeBlock(LogicalTapeSet *lts)
427427
* Refill the preallocation list with blocks from the tape set's free list if
428428
* necessary.
429429
*/
430-
staticlong
430+
staticint64
431431
ltsGetPreallocBlock(LogicalTapeSet*lts,LogicalTape*lt)
432432
{
433433
/* sorted in descending order, so return the last element */
@@ -437,16 +437,16 @@ ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt)
437437
if (lt->prealloc==NULL)
438438
{
439439
lt->prealloc_size=TAPE_WRITE_PREALLOC_MIN;
440-
lt->prealloc= (long*)palloc(sizeof(long)*lt->prealloc_size);
440+
lt->prealloc= (int64*)palloc(sizeof(int64)*lt->prealloc_size);
441441
}
442442
elseif (lt->prealloc_size<TAPE_WRITE_PREALLOC_MAX)
443443
{
444444
/* when the preallocation list runs out, double the size */
445445
lt->prealloc_size *=2;
446446
if (lt->prealloc_size>TAPE_WRITE_PREALLOC_MAX)
447447
lt->prealloc_size=TAPE_WRITE_PREALLOC_MAX;
448-
lt->prealloc= (long*)repalloc(lt->prealloc,
449-
sizeof(long)*lt->prealloc_size);
448+
lt->prealloc= (int64*)repalloc(lt->prealloc,
449+
sizeof(int64)*lt->prealloc_size);
450450
}
451451

452452
/* refill preallocation list */
@@ -466,10 +466,10 @@ ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt)
466466
* Return a block# to the freelist.
467467
*/
468468
staticvoid
469-
ltsReleaseBlock(LogicalTapeSet*lts,longblocknum)
469+
ltsReleaseBlock(LogicalTapeSet*lts,int64blocknum)
470470
{
471-
long*heap;
472-
unsigned longholepos;
471+
int64*heap;
472+
uint64holepos;
473473

474474
/*
475475
* Do nothing if we're no longer interested in remembering free space.
@@ -486,12 +486,12 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
486486
* If the freelist becomes very large, just return and leak this free
487487
* block.
488488
*/
489-
if (lts->freeBlocksLen*2*sizeof(long)>MaxAllocSize)
489+
if (lts->freeBlocksLen*2*sizeof(int64)>MaxAllocSize)
490490
return;
491491

492492
lts->freeBlocksLen *=2;
493-
lts->freeBlocks= (long*)repalloc(lts->freeBlocks,
494-
lts->freeBlocksLen*sizeof(long));
493+
lts->freeBlocks= (int64*)repalloc(lts->freeBlocks,
494+
lts->freeBlocksLen*sizeof(int64));
495495
}
496496

497497
/* create a "hole" at end of minheap array */
@@ -502,7 +502,7 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
502502
/* sift up to insert blocknum */
503503
while (holepos!=0)
504504
{
505-
unsigned longparent=parent_offset(holepos);
505+
uint64parent=parent_offset(holepos);
506506

507507
if (heap[parent]<blocknum)
508508
break;
@@ -566,7 +566,7 @@ LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
566566
lts->nHoleBlocks=0L;
567567
lts->forgetFreeSpace= false;
568568
lts->freeBlocksLen=32;/* reasonable initial guess */
569-
lts->freeBlocks= (long*)palloc(lts->freeBlocksLen*sizeof(long));
569+
lts->freeBlocks= (int64*)palloc(lts->freeBlocksLen*sizeof(int64));
570570
lts->nFreeBlocks=0;
571571
lts->enable_prealloc=preallocate;
572572

@@ -609,7 +609,7 @@ LogicalTape *
609609
LogicalTapeImport(LogicalTapeSet*lts,intworker,TapeShare*shared)
610610
{
611611
LogicalTape*lt;
612-
longtapeblocks;
612+
int64tapeblocks;
613613
charfilename[MAXPGPATH];
614614
BufFile*file;
615615
int64filesize;
@@ -789,7 +789,7 @@ LogicalTapeWrite(LogicalTape *lt, const void *ptr, size_t size)
789789
if (lt->pos >= (int)TapeBlockPayloadSize)
790790
{
791791
/* Buffer full, dump it out */
792-
longnextBlockNumber;
792+
int64nextBlockNumber;
793793

794794
if (!lt->dirty)
795795
{
@@ -1086,7 +1086,7 @@ LogicalTapeBackspace(LogicalTape *lt, size_t size)
10861086
seekpos= (size_t)lt->pos;/* part within this block */
10871087
while (size>seekpos)
10881088
{
1089-
longprev=TapeBlockGetTrailer(lt->buffer)->prev;
1089+
int64prev=TapeBlockGetTrailer(lt->buffer)->prev;
10901090

10911091
if (prev==-1L)
10921092
{
@@ -1100,10 +1100,10 @@ LogicalTapeBackspace(LogicalTape *lt, size_t size)
11001100
ltsReadBlock(lt->tapeSet,prev,lt->buffer);
11011101

11021102
if (TapeBlockGetTrailer(lt->buffer)->next!=lt->curBlockNumber)
1103-
elog(ERROR,"broken tape, next of block %ld is %ld, expected %ld",
1104-
prev,
1105-
TapeBlockGetTrailer(lt->buffer)->next,
1106-
lt->curBlockNumber);
1103+
elog(ERROR,"broken tape, next of block %lld is %lld, expected %lld",
1104+
(long long)prev,
1105+
(long long) (TapeBlockGetTrailer(lt->buffer)->next),
1106+
(long long)lt->curBlockNumber);
11071107

11081108
lt->nbytes=TapeBlockPayloadSize;
11091109
lt->curBlockNumber=prev;
@@ -1130,7 +1130,7 @@ LogicalTapeBackspace(LogicalTape *lt, size_t size)
11301130
* LogicalTapeTell().
11311131
*/
11321132
void
1133-
LogicalTapeSeek(LogicalTape*lt,longblocknum,intoffset)
1133+
LogicalTapeSeek(LogicalTape*lt,int64blocknum,intoffset)
11341134
{
11351135
Assert(lt->frozen);
11361136
Assert(offset >=0&&offset <=TapeBlockPayloadSize);
@@ -1159,7 +1159,7 @@ LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
11591159
* the position for a seek after freezing. Not clear if anyone needs that.
11601160
*/
11611161
void
1162-
LogicalTapeTell(LogicalTape*lt,long*blocknum,int*offset)
1162+
LogicalTapeTell(LogicalTape*lt,int64*blocknum,int*offset)
11631163
{
11641164
if (lt->buffer==NULL)
11651165
ltsInitReadBuffer(lt);
@@ -1179,7 +1179,7 @@ LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
11791179
* This should not be called while there are open write buffers; otherwise it
11801180
* may not account for buffered data.
11811181
*/
1182-
long
1182+
int64
11831183
LogicalTapeSetBlocks(LogicalTapeSet*lts)
11841184
{
11851185
returnlts->nBlocksWritten-lts->nHoleBlocks;

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp