Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitff9f111

Browse files
committed
Fix WAL replay in presence of an incomplete record
Physical replication always ships WAL segment files to replicas oncethey are complete. This is a problem if one WAL record is split acrossa segment boundary and the primary server crashes before writing downthe segment with the next portion of the WAL record: WAL writing aftercrash recovery would happily resume at the point where the broken recordstarted, overwriting that record ... but any standby or backup may havealready received a copy of that segment, and they are not rewinding.This causes standbys to stop following the primary after the lattercrashes: LOG: invalid contrecord length 7262 at A8/D9FFFBC8because the standby is still trying to read the continuation record(contrecord) for the original long WAL record, but it is not there andit will never be. A workaround is to stop the replica, delete the WALfile, and restart it -- at which point a fresh copy is brought over fromthe primary. But that's pretty labor intensive, and I bet many userswould just give up and re-clone the standby instead.A fix for this problem was already attempted in commit515e3d8, butit only addressed the case for the scenario of WAL archiving, sostreaming replication would still be a problem (as well as other thingssuch as taking a filesystem-level backup while the server is down afterhaving crashed), and it had performance scalability problems too; so ithad to be reverted.This commit fixes the problem using an approach suggested by AndresFreund, whereby the initial portion(s) of the split-up WAL record arekept, and a special type of WAL record is written where the contrecordwas lost, so that WAL replay in the replica knows to skip the brokenparts. With this approach, we can continue to stream/archive segmentfiles as soon as they are complete, and replay of the broken recordswill proceed across the crash point without a hitch.Because a new type of WAL record is added, users should be careful toupgrade standbys first, primaries later. Otherwise they risk the standbybeing unable to start if the primary happens to write such a record.A new TAP test that exercises this is added, but the portability of itis yet to be seen.This has been wrong since the introduction of physical replication, sobackpatch all the way back. In stable branches, keep the newXLogReaderState members at the end of the struct, to avoid an ABIbreak.Author: Álvaro Herrera <alvherre@alvh.no-ip.org>Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com>Reviewed-by: Nathan Bossart <bossartn@amazon.com>Discussion:https://postgr.es/m/202108232252.dh7uxf6oxwcy@alvherre.pgsql
1 parent2acb7cc commitff9f111

File tree

9 files changed

+450
-7
lines changed

9 files changed

+450
-7
lines changed

‎src/backend/access/rmgrdesc/xlogdesc.c

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,15 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
139139
xlrec.ThisTimeLineID,xlrec.PrevTimeLineID,
140140
timestamptz_to_str(xlrec.end_time));
141141
}
142+
elseif (info==XLOG_OVERWRITE_CONTRECORD)
143+
{
144+
xl_overwrite_contrecordxlrec;
145+
146+
memcpy(&xlrec,rec,sizeof(xl_overwrite_contrecord));
147+
appendStringInfo(buf,"lsn %X/%X; time %s",
148+
LSN_FORMAT_ARGS(xlrec.overwritten_lsn),
149+
timestamptz_to_str(xlrec.overwrite_time));
150+
}
142151
}
143152

144153
constchar*
@@ -178,6 +187,9 @@ xlog_identify(uint8 info)
178187
caseXLOG_END_OF_RECOVERY:
179188
id="END_OF_RECOVERY";
180189
break;
190+
caseXLOG_OVERWRITE_CONTRECORD:
191+
id="OVERWRITE_CONTRECORD";
192+
break;
181193
caseXLOG_FPI:
182194
id="FPI";
183195
break;

‎src/backend/access/transam/xlog.c

Lines changed: 149 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,15 @@ static XLogRecPtr LastRec;
199199
staticXLogRecPtrflushedUpto=0;
200200
staticTimeLineIDreceiveTLI=0;
201201

202+
/*
203+
* abortedRecPtr is the start pointer of a broken record at end of WAL when
204+
* recovery completes; missingContrecPtr is the location of the first
205+
* contrecord that went missing. See CreateOverwriteContrecordRecord for
206+
* details.
207+
*/
208+
staticXLogRecPtrabortedRecPtr;
209+
staticXLogRecPtrmissingContrecPtr;
210+
202211
/*
203212
* During recovery, lastFullPageWrites keeps track of full_page_writes that
204213
* the replayed WAL records indicate. It's initialized with full_page_writes
@@ -892,8 +901,11 @@ static void CheckRequiredParameterValues(void);
892901
staticvoidXLogReportParameters(void);
893902
staticvoidcheckTimeLineSwitch(XLogRecPtrlsn,TimeLineIDnewTLI,
894903
TimeLineIDprevTLI);
904+
staticvoidVerifyOverwriteContrecord(xl_overwrite_contrecord*xlrec,
905+
XLogReaderState*state);
895906
staticvoidLocalSetXLogInsertAllowed(void);
896907
staticvoidCreateEndOfRecoveryRecord(void);
908+
staticXLogRecPtrCreateOverwriteContrecordRecord(XLogRecPtraborted_lsn);
897909
staticvoidCheckPointGuts(XLogRecPtrcheckPointRedo,intflags);
898910
staticvoidKeepLogSeg(XLogRecPtrrecptr,XLogSegNo*logSegNo);
899911
staticXLogRecPtrXLogGetReplicationSlotMinimumLSN(void);
@@ -2246,6 +2258,18 @@ AdvanceXLInsertBuffer(XLogRecPtr upto, bool opportunistic)
22462258
if (!Insert->forcePageWrites)
22472259
NewPage->xlp_info |=XLP_BKP_REMOVABLE;
22482260

2261+
/*
2262+
* If a record was found to be broken at the end of recovery, and
2263+
* we're going to write on the page where its first contrecord was
2264+
* lost, set the XLP_FIRST_IS_OVERWRITE_CONTRECORD flag on the page
2265+
* header. See CreateOverwriteContrecordRecord().
2266+
*/
2267+
if (missingContrecPtr==NewPageBeginPtr)
2268+
{
2269+
NewPage->xlp_info |=XLP_FIRST_IS_OVERWRITE_CONTRECORD;
2270+
missingContrecPtr=InvalidXLogRecPtr;
2271+
}
2272+
22492273
/*
22502274
* If first page of an XLOG segment file, make it a long header.
22512275
*/
@@ -4394,6 +4418,19 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
43944418
EndRecPtr=xlogreader->EndRecPtr;
43954419
if (record==NULL)
43964420
{
4421+
/*
4422+
* When not in standby mode we find that WAL ends in an incomplete
4423+
* record, keep track of that record. After recovery is done,
4424+
* we'll write a record to indicate downstream WAL readers that
4425+
* that portion is to be ignored.
4426+
*/
4427+
if (!StandbyMode&&
4428+
!XLogRecPtrIsInvalid(xlogreader->abortedRecPtr))
4429+
{
4430+
abortedRecPtr=xlogreader->abortedRecPtr;
4431+
missingContrecPtr=xlogreader->missingContrecPtr;
4432+
}
4433+
43974434
if (readFile >=0)
43984435
{
43994436
close(readFile);
@@ -7069,6 +7106,12 @@ StartupXLOG(void)
70697106
InRecovery= true;
70707107
}
70717108

7109+
/*
7110+
* Start recovery assuming that the final record isn't lost.
7111+
*/
7112+
abortedRecPtr=InvalidXLogRecPtr;
7113+
missingContrecPtr=InvalidXLogRecPtr;
7114+
70727115
/* REDO */
70737116
if (InRecovery)
70747117
{
@@ -7655,8 +7698,9 @@ StartupXLOG(void)
76557698

76567699
/*
76577700
* Kill WAL receiver, if it's still running, before we continue to write
7658-
* the startup checkpoint record. It will trump over the checkpoint and
7659-
* subsequent records if it's still alive when we start writing WAL.
7701+
* the startup checkpoint and aborted-contrecord records. It will trump
7702+
* over these records and subsequent ones if it's still alive when we
7703+
* start writing WAL.
76607704
*/
76617705
XLogShutdownWalRcv();
76627706

@@ -7689,8 +7733,12 @@ StartupXLOG(void)
76897733
StandbyMode= false;
76907734

76917735
/*
7692-
* Re-fetch the last valid or last applied record, so we can identify the
7693-
* exact endpoint of what we consider the valid portion of WAL.
7736+
* Determine where to start writing WAL next.
7737+
*
7738+
* When recovery ended in an incomplete record, write a WAL record about
7739+
* that and continue after it. In all other cases, re-fetch the last
7740+
* valid or last applied record, so we can identify the exact endpoint of
7741+
* what we consider the valid portion of WAL.
76947742
*/
76957743
XLogBeginRead(xlogreader,LastRec);
76967744
record=ReadRecord(xlogreader,PANIC, false);
@@ -7821,6 +7869,18 @@ StartupXLOG(void)
78217869
XLogCtl->ThisTimeLineID=ThisTimeLineID;
78227870
XLogCtl->PrevTimeLineID=PrevTimeLineID;
78237871

7872+
/*
7873+
* Actually, if WAL ended in an incomplete record, skip the parts that
7874+
* made it through and start writing after the portion that persisted.
7875+
* (It's critical to first write an OVERWRITE_CONTRECORD message, which
7876+
* we'll do as soon as we're open for writing new WAL.)
7877+
*/
7878+
if (!XLogRecPtrIsInvalid(missingContrecPtr))
7879+
{
7880+
Assert(!XLogRecPtrIsInvalid(abortedRecPtr));
7881+
EndOfLog=missingContrecPtr;
7882+
}
7883+
78247884
/*
78257885
* Prepare to write WAL starting at EndOfLog location, and init xlog
78267886
* buffer cache using the block containing the last record from the
@@ -7873,13 +7933,23 @@ StartupXLOG(void)
78737933
XLogCtl->LogwrtRqst.Write=EndOfLog;
78747934
XLogCtl->LogwrtRqst.Flush=EndOfLog;
78757935

7936+
LocalSetXLogInsertAllowed();
7937+
7938+
/* If necessary, write overwrite-contrecord before doing anything else */
7939+
if (!XLogRecPtrIsInvalid(abortedRecPtr))
7940+
{
7941+
Assert(!XLogRecPtrIsInvalid(missingContrecPtr));
7942+
CreateOverwriteContrecordRecord(abortedRecPtr);
7943+
abortedRecPtr=InvalidXLogRecPtr;
7944+
missingContrecPtr=InvalidXLogRecPtr;
7945+
}
7946+
78767947
/*
78777948
* Update full_page_writes in shared memory and write an XLOG_FPW_CHANGE
78787949
* record before resource manager writes cleanup WAL records or checkpoint
78797950
* record is written.
78807951
*/
78817952
Insert->fullPageWrites=lastFullPageWrites;
7882-
LocalSetXLogInsertAllowed();
78837953
UpdateFullPageWrites();
78847954
LocalXLogInsertAllowed=-1;
78857955

@@ -9365,6 +9435,53 @@ CreateEndOfRecoveryRecord(void)
93659435
LocalXLogInsertAllowed=-1;/* return to "check" state */
93669436
}
93679437

9438+
/*
9439+
* Write an OVERWRITE_CONTRECORD message.
9440+
*
9441+
* When on WAL replay we expect a continuation record at the start of a page
9442+
* that is not there, recovery ends and WAL writing resumes at that point.
9443+
* But it's wrong to resume writing new WAL back at the start of the record
9444+
* that was broken, because downstream consumers of that WAL (physical
9445+
* replicas) are not prepared to "rewind". So the first action after
9446+
* finishing replay of all valid WAL must be to write a record of this type
9447+
* at the point where the contrecord was missing; to support xlogreader
9448+
* detecting the special case, XLP_FIRST_IS_OVERWRITE_CONTRECORD is also added
9449+
* to the page header where the record occurs. xlogreader has an ad-hoc
9450+
* mechanism to report metadata about the broken record, which is what we
9451+
* use here.
9452+
*
9453+
* At replay time, XLP_FIRST_IS_OVERWRITE_CONTRECORD instructs xlogreader to
9454+
* skip the record it was reading, and pass back the LSN of the skipped
9455+
* record, so that its caller can verify (on "replay" of that record) that the
9456+
* XLOG_OVERWRITE_CONTRECORD matches what was effectively overwritten.
9457+
*/
9458+
staticXLogRecPtr
9459+
CreateOverwriteContrecordRecord(XLogRecPtraborted_lsn)
9460+
{
9461+
xl_overwrite_contrecordxlrec;
9462+
XLogRecPtrrecptr;
9463+
9464+
/* sanity check */
9465+
if (!RecoveryInProgress())
9466+
elog(ERROR,"can only be used at end of recovery");
9467+
9468+
xlrec.overwritten_lsn=aborted_lsn;
9469+
xlrec.overwrite_time=GetCurrentTimestamp();
9470+
9471+
START_CRIT_SECTION();
9472+
9473+
XLogBeginInsert();
9474+
XLogRegisterData((char*)&xlrec,sizeof(xl_overwrite_contrecord));
9475+
9476+
recptr=XLogInsert(RM_XLOG_ID,XLOG_OVERWRITE_CONTRECORD);
9477+
9478+
XLogFlush(recptr);
9479+
9480+
END_CRIT_SECTION();
9481+
9482+
returnrecptr;
9483+
}
9484+
93689485
/*
93699486
* Flush all data in shared memory to disk, and fsync
93709487
*
@@ -10295,6 +10412,13 @@ xlog_redo(XLogReaderState *record)
1029510412

1029610413
RecoveryRestartPoint(&checkPoint);
1029710414
}
10415+
elseif (info==XLOG_OVERWRITE_CONTRECORD)
10416+
{
10417+
xl_overwrite_contrecordxlrec;
10418+
10419+
memcpy(&xlrec,XLogRecGetData(record),sizeof(xl_overwrite_contrecord));
10420+
VerifyOverwriteContrecord(&xlrec,record);
10421+
}
1029810422
elseif (info==XLOG_END_OF_RECOVERY)
1029910423
{
1030010424
xl_end_of_recoveryxlrec;
@@ -10462,6 +10586,26 @@ xlog_redo(XLogReaderState *record)
1046210586
}
1046310587
}
1046410588

10589+
/*
10590+
* Verify the payload of a XLOG_OVERWRITE_CONTRECORD record.
10591+
*/
10592+
staticvoid
10593+
VerifyOverwriteContrecord(xl_overwrite_contrecord*xlrec,XLogReaderState*state)
10594+
{
10595+
if (xlrec->overwritten_lsn!=state->overwrittenRecPtr)
10596+
elog(FATAL,"mismatching overwritten LSN %X/%X -> %X/%X",
10597+
LSN_FORMAT_ARGS(xlrec->overwritten_lsn),
10598+
LSN_FORMAT_ARGS(state->overwrittenRecPtr));
10599+
10600+
ereport(LOG,
10601+
(errmsg("sucessfully skipped missing contrecord at %X/%X, overwritten at %s",
10602+
LSN_FORMAT_ARGS(xlrec->overwritten_lsn),
10603+
timestamptz_to_str(xlrec->overwrite_time))));
10604+
10605+
/* Verifying the record should only happen once */
10606+
state->overwrittenRecPtr=InvalidXLogRecPtr;
10607+
}
10608+
1046510609
#ifdefWAL_DEBUG
1046610610

1046710611
staticvoid

‎src/backend/access/transam/xlogreader.c

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -278,6 +278,7 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
278278
total_len;
279279
uint32targetRecOff;
280280
uint32pageHeaderSize;
281+
boolassembled;
281282
boolgotheader;
282283
intreadOff;
283284

@@ -293,6 +294,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
293294
state->errormsg_buf[0]='\0';
294295

295296
ResetDecoder(state);
297+
state->abortedRecPtr=InvalidXLogRecPtr;
298+
state->missingContrecPtr=InvalidXLogRecPtr;
296299

297300
RecPtr=state->EndRecPtr;
298301

@@ -319,7 +322,9 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
319322
randAccess= true;
320323
}
321324

325+
restart:
322326
state->currRecPtr=RecPtr;
327+
assembled= false;
323328

324329
targetPagePtr=RecPtr- (RecPtr %XLOG_BLCKSZ);
325330
targetRecOff=RecPtr %XLOG_BLCKSZ;
@@ -415,6 +420,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
415420
char*buffer;
416421
uint32gotlen;
417422

423+
assembled= true;
424+
418425
/*
419426
* Enlarge readRecordBuf as needed.
420427
*/
@@ -448,8 +455,25 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
448455

449456
Assert(SizeOfXLogShortPHD <=readOff);
450457

451-
/* Check that the continuation on next page looks valid */
452458
pageHeader= (XLogPageHeader)state->readBuf;
459+
460+
/*
461+
* If we were expecting a continuation record and got an
462+
* "overwrite contrecord" flag, that means the continuation record
463+
* was overwritten with a different record. Restart the read by
464+
* assuming the address to read is the location where we found
465+
* this flag; but keep track of the LSN of the record we were
466+
* reading, for later verification.
467+
*/
468+
if (pageHeader->xlp_info&XLP_FIRST_IS_OVERWRITE_CONTRECORD)
469+
{
470+
state->overwrittenRecPtr=state->currRecPtr;
471+
ResetDecoder(state);
472+
RecPtr=targetPagePtr;
473+
gotorestart;
474+
}
475+
476+
/* Check that the continuation on next page looks valid */
453477
if (!(pageHeader->xlp_info&XLP_FIRST_IS_CONTRECORD))
454478
{
455479
report_invalid_record(state,
@@ -551,6 +575,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
551575
returnNULL;
552576

553577
err:
578+
if (assembled)
579+
{
580+
/*
581+
* We get here when a record that spans multiple pages needs to be
582+
* assembled, but something went wrong -- perhaps a contrecord piece
583+
* was lost. If caller is WAL replay, it will know where the aborted
584+
* record was and where to direct followup WAL to be written, marking
585+
* the next piece with XLP_FIRST_IS_OVERWRITE_CONTRECORD, which will
586+
* in turn signal downstream WAL consumers that the broken WAL record
587+
* is to be ignored.
588+
*/
589+
state->abortedRecPtr=RecPtr;
590+
state->missingContrecPtr=targetPagePtr;
591+
}
554592

555593
/*
556594
* Invalidate the read state. We might read from a different source after

‎src/include/access/xlog_internal.h

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,10 @@ typedef XLogLongPageHeaderData *XLogLongPageHeader;
7676
#defineXLP_LONG_HEADER0x0002
7777
/* This flag indicates backup blocks starting in this page are optional */
7878
#defineXLP_BKP_REMOVABLE0x0004
79+
/* Replaces a missing contrecord; see CreateOverwriteContrecordRecord */
80+
#defineXLP_FIRST_IS_OVERWRITE_CONTRECORD 0x0008
7981
/* All defined flag bits in xlp_info (used for validity checking of header) */
80-
#defineXLP_ALL_FLAGS0x0007
82+
#defineXLP_ALL_FLAGS0x000F
8183

8284
#defineXLogPageHeaderSize(hdr)\
8385
(((hdr)->xlp_info & XLP_LONG_HEADER) ? SizeOfXLogLongPHD : SizeOfXLogShortPHD)
@@ -249,6 +251,13 @@ typedef struct xl_restore_point
249251
charrp_name[MAXFNAMELEN];
250252
}xl_restore_point;
251253

254+
/* Overwrite of prior contrecord */
255+
typedefstructxl_overwrite_contrecord
256+
{
257+
XLogRecPtroverwritten_lsn;
258+
TimestampTzoverwrite_time;
259+
}xl_overwrite_contrecord;
260+
252261
/* End of recovery mark, when we don't do an END_OF_RECOVERY checkpoint */
253262
typedefstructxl_end_of_recovery
254263
{

‎src/include/access/xlogreader.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,16 @@ struct XLogReaderState
175175
XLogRecPtrReadRecPtr;/* start of last record read */
176176
XLogRecPtrEndRecPtr;/* end+1 of last record read */
177177

178+
/*
179+
* Set at the end of recovery: the start point of a partial record at the
180+
* end of WAL (InvalidXLogRecPtr if there wasn't one), and the start
181+
* location of its first contrecord that went missing.
182+
*/
183+
XLogRecPtrabortedRecPtr;
184+
XLogRecPtrmissingContrecPtr;
185+
/* Set when XLP_FIRST_IS_OVERWRITE_CONTRECORD is found */
186+
XLogRecPtroverwrittenRecPtr;
187+
178188

179189
/* ----------------------------------------
180190
* Decoded representation of current record

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp