Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit8fcb32d

Browse files
committed
Add more protections in WAL record APIs against overflows
This commit adds a limit to the size of an XLogRecord at 1020MB, basedon a suggestion by Heikki Linnakangas. This counts for the overheadneeded by the XLogReader when allocating the memory it needs to read arecord in DecodeXLogRecordRequiredSpace(), based on the record size. Anassertion based on that is added to detect that any additions in theXLogReader facilities would not cause any overflows. If that's ever thecase, the upper bound allowed would need to be adjusted.Before this, it was possible for an external module to create WALrecords large enough to be assembled but not replayable, causingfailures when replaying such WAL records on standbys. One casementioned where this is possible is the in-core functionpg_logical_emit_message() (wrapper for LogLogicalMessage), that allowsto emit WAL records with an arbitrary amount of data potentially higherthan the replay limit of approximately 1GB (limit of a palloc, minus theoverhead needed by a XLogReader).This commit is a follow-up offfd1b6b that has added similar protectionsfor the block-level data. Here, the checks are extended to the wholerecord length, mainrdata_len being extended from uint32 to uint64 withthe routines registering buffer and record data still limited to uint32to minimize the checks when assembling a record. All the error messagesrelated to overflow checks are improved to provide more context aboutthe error happening.Author: Matthias van de MeentReviewed-by: Andres Freund, Heikki Linnakangas, Michael PaquierDiscussion:https://postgr.es/m/CAEze2WgGiw+LZt+vHf8tWqB_6VxeLsMeoAuod0N=ij1q17n5pw@mail.gmail.com
1 parent26158b8 commit8fcb32d

File tree

2 files changed

+65
-8
lines changed

2 files changed

+65
-8
lines changed

‎src/backend/access/transam/xloginsert.c

Lines changed: 54 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ static intmax_registered_block_id = 0;/* highest block_id + 1 currently
9898
*/
9999
staticXLogRecData*mainrdata_head;
100100
staticXLogRecData*mainrdata_last= (XLogRecData*)&mainrdata_head;
101-
staticuint32mainrdata_len;/* total # of bytes in chain */
101+
staticuint64mainrdata_len;/* total # of bytes in chain */
102102

103103
/* flags for the in-progress insertion */
104104
staticuint8curinsert_flags=0;
@@ -355,7 +355,10 @@ XLogRegisterData(char *data, uint32 len)
355355
Assert(begininsert_called);
356356

357357
if (num_rdatas >=max_rdatas)
358-
elog(ERROR,"too much WAL data");
358+
ereport(ERROR,
359+
(errmsg_internal("too much WAL data"),
360+
errdetail_internal("%u out of %u data segments are already in use.",
361+
num_rdatas,max_rdatas)));
359362
rdata=&rdatas[num_rdatas++];
360363

361364
rdata->data=data;
@@ -405,9 +408,16 @@ XLogRegisterBufData(uint8 block_id, char *data, uint32 len)
405408
* regbuf->rdata_len does not grow beyond what
406409
* XLogRecordBlockHeader->data_length can hold.
407410
*/
408-
if (num_rdatas >=max_rdatas||
409-
regbuf->rdata_len+len>UINT16_MAX)
410-
elog(ERROR,"too much WAL data");
411+
if (num_rdatas >=max_rdatas)
412+
ereport(ERROR,
413+
(errmsg_internal("too much WAL data"),
414+
errdetail_internal("%u out of %u data segments are already in use.",
415+
num_rdatas,max_rdatas)));
416+
if (regbuf->rdata_len+len>UINT16_MAX||len>UINT16_MAX)
417+
ereport(ERROR,
418+
(errmsg_internal("too much WAL data"),
419+
errdetail_internal("Registering more than maximum %u bytes allowed to block %u: current %u bytes, adding %u bytes.",
420+
UINT16_MAX,block_id,regbuf->rdata_len,len)));
411421

412422
rdata=&rdatas[num_rdatas++];
413423

@@ -527,7 +537,7 @@ XLogRecordAssemble(RmgrId rmid, uint8 info,
527537
XLogRecPtr*fpw_lsn,int*num_fpi,bool*topxid_included)
528538
{
529539
XLogRecData*rdt;
530-
uint32total_len=0;
540+
uint64total_len=0;
531541
intblock_id;
532542
pg_crc32crdata_crc;
533543
registered_buffer*prev_regbuf=NULL;
@@ -841,8 +851,18 @@ XLogRecordAssemble(RmgrId rmid, uint8 info,
841851
{
842852
if (mainrdata_len>255)
843853
{
854+
uint32mainrdata_len_4b;
855+
856+
if (mainrdata_len>PG_UINT32_MAX)
857+
ereport(ERROR,
858+
(errmsg_internal("too much WAL data"),
859+
errdetail_internal("Main data length is %llu bytes for a maximum of %u bytes.",
860+
(unsigned long long)mainrdata_len,
861+
PG_UINT32_MAX)));
862+
863+
mainrdata_len_4b= (uint32)mainrdata_len;
844864
*(scratch++)= (char)XLR_BLOCK_ID_DATA_LONG;
845-
memcpy(scratch,&mainrdata_len,sizeof(uint32));
865+
memcpy(scratch,&mainrdata_len_4b,sizeof(uint32));
846866
scratch+=sizeof(uint32);
847867
}
848868
else
@@ -872,13 +892,27 @@ XLogRecordAssemble(RmgrId rmid, uint8 info,
872892
for (rdt=hdr_rdt.next;rdt!=NULL;rdt=rdt->next)
873893
COMP_CRC32C(rdata_crc,rdt->data,rdt->len);
874894

895+
/*
896+
* Ensure that the XLogRecord is not too large.
897+
*
898+
* XLogReader machinery is only able to handle records up to a certain
899+
* size (ignoring machine resource limitations), so make sure that we will
900+
* not emit records larger than the sizes advertised to be supported.
901+
* This cap is based on DecodeXLogRecordRequiredSpace().
902+
*/
903+
if (total_len >=XLogRecordMaxSize)
904+
ereport(ERROR,
905+
(errmsg_internal("oversized WAL record"),
906+
errdetail_internal("WAL record would be %llu bytes (of maximum %u bytes); rmid %u flags %u.",
907+
(unsigned long long)total_len,XLogRecordMaxSize,rmid,info)));
908+
875909
/*
876910
* Fill in the fields in the record header. Prev-link is filled in later,
877911
* once we know where in the WAL the record will be inserted. The CRC does
878912
* not include the record header yet.
879913
*/
880914
rechdr->xl_xid=GetCurrentTransactionIdIfAny();
881-
rechdr->xl_tot_len=total_len;
915+
rechdr->xl_tot_len=(uint32)total_len;
882916
rechdr->xl_info=info;
883917
rechdr->xl_rmid=rmid;
884918
rechdr->xl_prev=InvalidXLogRecPtr;
@@ -1297,6 +1331,18 @@ log_newpage_range(Relation rel, ForkNumber forknum,
12971331
void
12981332
InitXLogInsert(void)
12991333
{
1334+
#ifdefUSE_ASSERT_CHECKING
1335+
1336+
/*
1337+
* Check that any records assembled can be decoded. This is capped based
1338+
* on what XLogReader would require at its maximum bound. This code path
1339+
* is called once per backend, more than enough for this check.
1340+
*/
1341+
size_tmax_required=DecodeXLogRecordRequiredSpace(XLogRecordMaxSize);
1342+
1343+
Assert(AllocSizeIsValid(max_required));
1344+
#endif
1345+
13001346
/* Initialize the working areas */
13011347
if (xloginsert_cxt==NULL)
13021348
{

‎src/include/access/xlogrecord.h

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,17 @@ typedef struct XLogRecord
6262
#defineXLR_INFO_MASK0x0F
6363
#defineXLR_RMGR_INFO_MASK0xF0
6464

65+
/*
66+
* XLogReader needs to allocate all the data of a WAL record in a single
67+
* chunk. This means that a single XLogRecord cannot exceed MaxAllocSize
68+
* in length if we ignore any allocation overhead of the XLogReader.
69+
*
70+
* To accommodate some overhead, this value allows for 4M of allocation
71+
* overhead, that should be plenty enough for what
72+
* DecodeXLogRecordRequiredSpace() expects as extra.
73+
*/
74+
#defineXLogRecordMaxSize(1020 * 1024 * 1024)
75+
6576
/*
6677
* If a WAL record modifies any relation files, in ways not covered by the
6778
* usual block references, this flag is set. This is not used for anything

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp