Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitd72792d

Browse files
committed
Don't archive bogus recycled or preallocated files after timeline switch.
After a timeline switch, we would leave behind recycled WAL segments thatare in the future, but on the old timeline. After promotion, and after theybecome old enough to be recycled again, we would notice that they don't havea .ready or .done file, create a .ready file for them, and archive them.That's bogus, because the files contain garbage, recycled from an oldertimeline (or prealloced as zeros). We shouldn't archive such files.This could happen when we're following a timeline switch during replay, orwhen we switch to new timeline at end-of-recovery.To fix, whenever we switch to a new timeline, scan the data directory forWAL segments on the old timeline, but with a higher segment number, andremove them. Those don't belong to our timeline history, and are mostlikely bogus recycled or preallocated files. They could also be valid filesthat we streamed from the primary ahead of time, but in any case, they'renot needed to recover to the new timeline.
1 parent131fb7f commitd72792d

File tree

3 files changed

+209
-94
lines changed

3 files changed

+209
-94
lines changed

‎src/backend/access/transam/xlog.c

Lines changed: 189 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -782,6 +782,8 @@ static intemode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
782782
staticvoidXLogFileClose(void);
783783
staticvoidPreallocXlogFiles(XLogRecPtrendptr);
784784
staticvoidRemoveOldXlogFiles(XLogSegNosegno,XLogRecPtrendptr);
785+
staticvoidRemoveXlogFile(constchar*segname,XLogRecPtrendptr);
786+
staticvoidRemoveNonParentXlogFiles(XLogRecPtrswitchpoint,TimeLineIDnewTLI);
785787
staticvoidUpdateLastRemovedPtr(char*filename);
786788
staticvoidValidateXLOGDirectoryStructure(void);
787789
staticvoidCleanupBackupHistory(void);
@@ -3743,32 +3745,17 @@ UpdateLastRemovedPtr(char *filename)
37433745
}
37443746

37453747
/*
3746-
* Recycle or remove all log files older or equal to passed segno
3748+
* Recycle or remove all log files older or equal to passed segno.
37473749
*
37483750
* endptr is current (or recent) end of xlog; this is used to determine
37493751
* whether we want to recycle rather than delete no-longer-wanted log files.
37503752
*/
37513753
staticvoid
37523754
RemoveOldXlogFiles(XLogSegNosegno,XLogRecPtrendptr)
37533755
{
3754-
XLogSegNoendlogSegNo;
3755-
intmax_advance;
37563756
DIR*xldir;
37573757
structdirent*xlde;
37583758
charlastoff[MAXFNAMELEN];
3759-
charpath[MAXPGPATH];
3760-
3761-
#ifdefWIN32
3762-
charnewpath[MAXPGPATH];
3763-
#endif
3764-
structstatstatbuf;
3765-
3766-
/*
3767-
* Initialize info about where to try to recycle to. We allow recycling
3768-
* segments up to XLOGfileslop segments beyond the current XLOG location.
3769-
*/
3770-
XLByteToPrevSeg(endptr,endlogSegNo);
3771-
max_advance=XLOGfileslop;
37723759

37733760
xldir=AllocateDir(XLOGDIR);
37743761
if (xldir==NULL)
@@ -3789,6 +3776,11 @@ RemoveOldXlogFiles(XLogSegNo segno, XLogRecPtr endptr)
37893776

37903777
while ((xlde=ReadDir(xldir,XLOGDIR))!=NULL)
37913778
{
3779+
/* Ignore files that are not XLOG segments */
3780+
if (strlen(xlde->d_name)!=24||
3781+
strspn(xlde->d_name,"0123456789ABCDEF")!=24)
3782+
continue;
3783+
37923784
/*
37933785
* We ignore the timeline part of the XLOG segment identifiers in
37943786
* deciding whether a segment is still needed. This ensures that we
@@ -3800,92 +3792,110 @@ RemoveOldXlogFiles(XLogSegNo segno, XLogRecPtr endptr)
38003792
* We use the alphanumeric sorting property of the filenames to decide
38013793
* which ones are earlier than the lastoff segment.
38023794
*/
3803-
if (strlen(xlde->d_name)==24&&
3804-
strspn(xlde->d_name,"0123456789ABCDEF")==24&&
3805-
strcmp(xlde->d_name+8,lastoff+8) <=0)
3795+
if (strcmp(xlde->d_name+8,lastoff+8) <=0)
38063796
{
38073797
if (XLogArchiveCheckDone(xlde->d_name))
38083798
{
3809-
snprintf(path,MAXPGPATH,XLOGDIR"/%s",xlde->d_name);
3810-
38113799
/* Update the last removed location in shared memory first */
38123800
UpdateLastRemovedPtr(xlde->d_name);
38133801

3814-
/*
3815-
* Before deleting the file, see if it can be recycled as a
3816-
* future log segment. Only recycle normal files, pg_standby
3817-
* for example can create symbolic links pointing to a
3818-
* separate archive directory.
3819-
*/
3820-
if (lstat(path,&statbuf)==0&&S_ISREG(statbuf.st_mode)&&
3821-
InstallXLogFileSegment(&endlogSegNo,path,
3822-
true,&max_advance, true))
3823-
{
3824-
ereport(DEBUG2,
3825-
(errmsg("recycled transaction log file \"%s\"",
3826-
xlde->d_name)));
3827-
CheckpointStats.ckpt_segs_recycled++;
3828-
/* Needn't recheck that slot on future iterations */
3829-
if (max_advance>0)
3830-
{
3831-
endlogSegNo++;
3832-
max_advance--;
3833-
}
3834-
}
3835-
else
3836-
{
3837-
/* No need for any more future segments... */
3838-
intrc;
3802+
RemoveXlogFile(xlde->d_name,endptr);
3803+
}
3804+
}
3805+
}
38393806

3840-
ereport(DEBUG2,
3841-
(errmsg("removing transaction log file \"%s\"",
3842-
xlde->d_name)));
3807+
FreeDir(xldir);
3808+
}
38433809

3810+
/*
3811+
* Recycle or remove a log file that's no longer needed.
3812+
*
3813+
* endptr is current (or recent) end of xlog; this is used to determine
3814+
* whether we want to recycle rather than delete no-longer-wanted log files.
3815+
*/
3816+
staticvoid
3817+
RemoveXlogFile(constchar*segname,XLogRecPtrendptr)
3818+
{
3819+
charpath[MAXPGPATH];
38443820
#ifdefWIN32
3821+
charnewpath[MAXPGPATH];
3822+
#endif
3823+
structstatstatbuf;
3824+
XLogSegNoendlogSegNo;
3825+
intmax_advance;
38453826

3846-
/*
3847-
* On Windows, if another process (e.g another backend)
3848-
* holds the file open in FILE_SHARE_DELETE mode, unlink
3849-
* will succeed, but the file will still show up in
3850-
* directory listing until the last handle is closed. To
3851-
* avoid confusing the lingering deleted file for a live
3852-
* WAL file that needs to be archived, rename it before
3853-
* deleting it.
3854-
*
3855-
* If another process holds the file open without
3856-
* FILE_SHARE_DELETE flag, rename will fail. We'll try
3857-
* again at the next checkpoint.
3858-
*/
3859-
snprintf(newpath,MAXPGPATH,"%s.deleted",path);
3860-
if (rename(path,newpath)!=0)
3861-
{
3862-
ereport(LOG,
3863-
(errcode_for_file_access(),
3864-
errmsg("could not rename old transaction log file \"%s\": %m",
3865-
path)));
3866-
continue;
3867-
}
3868-
rc=unlink(newpath);
3827+
/*
3828+
* Initialize info about where to try to recycle to. We allow recycling
3829+
* segments up to XLOGfileslop segments beyond the current XLOG location.
3830+
*/
3831+
XLByteToPrevSeg(endptr,endlogSegNo);
3832+
max_advance=XLOGfileslop;
3833+
3834+
snprintf(path,MAXPGPATH,XLOGDIR"/%s",segname);
3835+
3836+
/*
3837+
* Before deleting the file, see if it can be recycled as a future log
3838+
* segment. Only recycle normal files, pg_standby for example can create
3839+
* symbolic links pointing to a separate archive directory.
3840+
*/
3841+
if (lstat(path,&statbuf)==0&&S_ISREG(statbuf.st_mode)&&
3842+
InstallXLogFileSegment(&endlogSegNo,path,
3843+
true,&max_advance, true))
3844+
{
3845+
ereport(DEBUG2,
3846+
(errmsg("recycled transaction log file \"%s\"",segname)));
3847+
CheckpointStats.ckpt_segs_recycled++;
3848+
/* Needn't recheck that slot on future iterations */
3849+
if (max_advance>0)
3850+
{
3851+
endlogSegNo++;
3852+
max_advance--;
3853+
}
3854+
}
3855+
else
3856+
{
3857+
/* No need for any more future segments... */
3858+
intrc;
3859+
3860+
ereport(DEBUG2,
3861+
(errmsg("removing transaction log file \"%s\"",segname)));
3862+
3863+
#ifdefWIN32
3864+
/*
3865+
* On Windows, if another process (e.g another backend) holds the file
3866+
* open in FILE_SHARE_DELETE mode, unlink will succeed, but the file
3867+
* will still show up in directory listing until the last handle is
3868+
* closed. To avoid confusing the lingering deleted file for a live
3869+
* WAL file that needs to be archived, rename it before deleting it.
3870+
*
3871+
* If another process holds the file open without FILE_SHARE_DELETE
3872+
* flag, rename will fail. We'll try again at the next checkpoint.
3873+
*/
3874+
snprintf(newpath,MAXPGPATH,"%s.deleted",path);
3875+
if (rename(path,newpath)!=0)
3876+
{
3877+
ereport(LOG,
3878+
(errcode_for_file_access(),
3879+
errmsg("could not rename old transaction log file \"%s\": %m",
3880+
path)));
3881+
return;
3882+
}
3883+
rc=unlink(newpath);
38693884
#else
3870-
rc=unlink(path);
3885+
rc=unlink(path);
38713886
#endif
3872-
if (rc!=0)
3873-
{
3874-
ereport(LOG,
3875-
(errcode_for_file_access(),
3876-
errmsg("could not remove old transaction log file \"%s\": %m",
3877-
path)));
3878-
continue;
3879-
}
3880-
CheckpointStats.ckpt_segs_removed++;
3881-
}
3882-
3883-
XLogArchiveCleanup(xlde->d_name);
3884-
}
3887+
if (rc!=0)
3888+
{
3889+
ereport(LOG,
3890+
(errcode_for_file_access(),
3891+
errmsg("could not remove old transaction log file \"%s\": %m",
3892+
path)));
3893+
return;
38853894
}
3895+
CheckpointStats.ckpt_segs_removed++;
38863896
}
38873897

3888-
FreeDir(xldir);
3898+
XLogArchiveCleanup(segname);
38893899
}
38903900

38913901
/*
@@ -5407,6 +5417,76 @@ exitArchiveRecovery(TimeLineID endTLI, XLogSegNo endLogSegNo)
54075417
(errmsg("archive recovery complete")));
54085418
}
54095419

5420+
/*
5421+
* Remove WAL files that are not part of the given timeline's history.
5422+
*
5423+
* This is called during recovery, whenever we switch to follow a new
5424+
* timeline, and at the end of recovery when we create a new timeline. We
5425+
* wouldn't otherwise care about extra WAL files lying in pg_xlog, but they
5426+
* can be pre-allocated or recycled WAL segments on the old timeline that we
5427+
* haven't used yet, and contain garbage. If we just leave them in pg_xlog,
5428+
* they will eventually be archived, and we can't let that happen. Files that
5429+
* belong to our timeline history are valid, because we have successfully
5430+
* replayed them, but from others we can't be sure.
5431+
*
5432+
* 'switchpoint' is the current point in WAL where we switch to new timeline,
5433+
* and 'newTLI' is the new timeline we switch to.
5434+
*/
5435+
staticvoid
5436+
RemoveNonParentXlogFiles(XLogRecPtrswitchpoint,TimeLineIDnewTLI)
5437+
{
5438+
DIR*xldir;
5439+
structdirent*xlde;
5440+
charswitchseg[MAXFNAMELEN];
5441+
XLogSegNoendLogSegNo;
5442+
5443+
XLByteToPrevSeg(switchpoint,endLogSegNo);
5444+
5445+
xldir=AllocateDir(XLOGDIR);
5446+
if (xldir==NULL)
5447+
ereport(ERROR,
5448+
(errcode_for_file_access(),
5449+
errmsg("could not open transaction log directory \"%s\": %m",
5450+
XLOGDIR)));
5451+
5452+
/*
5453+
* Construct a filename of the last segment to be kept.
5454+
*/
5455+
XLogFileName(switchseg,newTLI,endLogSegNo);
5456+
5457+
elog(DEBUG2,"attempting to remove WAL segments newer than log file %s",
5458+
switchseg);
5459+
5460+
while ((xlde=ReadDir(xldir,XLOGDIR))!=NULL)
5461+
{
5462+
/* Ignore files that are not XLOG segments */
5463+
if (strlen(xlde->d_name)!=24||
5464+
strspn(xlde->d_name,"0123456789ABCDEF")!=24)
5465+
continue;
5466+
5467+
/*
5468+
* Remove files that are on a timeline older than the new one we're
5469+
* switching to, but with a segment number >= the first segment on
5470+
* the new timeline.
5471+
*/
5472+
if (strncmp(xlde->d_name,switchseg,8)<0&&
5473+
strcmp(xlde->d_name+8,switchseg+8)>0)
5474+
{
5475+
/*
5476+
* If the file has already been marked as .ready, however, don't
5477+
* remove it yet. It should be OK to remove it - files that are
5478+
* not part of our timeline history are not required for recovery
5479+
* - but seems safer to let them be archived and removed later.
5480+
*/
5481+
if (!XLogArchiveIsReady(xlde->d_name))
5482+
RemoveXlogFile(xlde->d_name,switchpoint);
5483+
}
5484+
}
5485+
5486+
FreeDir(xldir);
5487+
}
5488+
5489+
54105490
/*
54115491
* Extract timestamp from WAL record.
54125492
*
@@ -6775,9 +6855,9 @@ StartupXLOG(void)
67756855
*/
67766856
if (record->xl_rmid==RM_XLOG_ID)
67776857
{
6858+
uint8info=record->xl_info& ~XLR_INFO_MASK;
67786859
TimeLineIDnewTLI=ThisTimeLineID;
67796860
TimeLineIDprevTLI=ThisTimeLineID;
6780-
uint8info=record->xl_info& ~XLR_INFO_MASK;
67816861

67826862
if (info==XLOG_CHECKPOINT_SHUTDOWN)
67836863
{
@@ -6845,12 +6925,21 @@ StartupXLOG(void)
68456925
/* Allow read-only connections if we're consistent now */
68466926
CheckRecoveryConsistency();
68476927

6848-
/*
6849-
* If this record was a timeline switch, wake up any
6850-
* walsenders to notice that we are on a new timeline.
6851-
*/
6852-
if (switchedTLI&&AllowCascadeReplication())
6853-
WalSndWakeup();
6928+
if (switchedTLI)
6929+
{
6930+
/*
6931+
* Before we go further on the new timeline, clean up any
6932+
* (possibly bogus) future WAL segments on the old one.
6933+
*/
6934+
RemoveNonParentXlogFiles(EndRecPtr,ThisTimeLineID);
6935+
6936+
/*
6937+
* Wake up any walsenders to notice that we are on a new
6938+
* timeline.
6939+
*/
6940+
if (AllowCascadeReplication())
6941+
WalSndWakeup();
6942+
}
68546943

68556944
/* Exit loop if we reached inclusive recovery target */
68566945
if (recoveryStopsAfter(record))
@@ -7177,6 +7266,12 @@ StartupXLOG(void)
71777266
true);
71787267
}
71797268

7269+
/*
7270+
* Clean up any (possibly bogus) future WAL segments on the old timeline.
7271+
*/
7272+
if (ArchiveRecoveryRequested)
7273+
RemoveNonParentXlogFiles(EndOfLog,ThisTimeLineID);
7274+
71807275
/*
71817276
* Preallocate additional log files, if wanted.
71827277
*/

‎src/backend/access/transam/xlogarchive.c

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -693,6 +693,25 @@ XLogArchiveIsBusy(const char *xlog)
693693
return true;
694694
}
695695

696+
/*
697+
* XLogArchiveIsReady
698+
*
699+
* Check to see if an XLOG segment file has an archive notification (.ready)
700+
* file.
701+
*/
702+
bool
703+
XLogArchiveIsReady(constchar*xlog)
704+
{
705+
chararchiveStatusPath[MAXPGPATH];
706+
structstatstat_buf;
707+
708+
StatusFilePath(archiveStatusPath,xlog,".ready");
709+
if (stat(archiveStatusPath,&stat_buf)==0)
710+
return true;
711+
712+
return false;
713+
}
714+
696715
/*
697716
* XLogArchiveCleanup
698717
*

‎src/include/access/xlog_internal.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,6 +283,7 @@ extern void XLogArchiveNotifySeg(XLogSegNo segno);
283283
externvoidXLogArchiveForceDone(constchar*xlog);
284284
externboolXLogArchiveCheckDone(constchar*xlog);
285285
externboolXLogArchiveIsBusy(constchar*xlog);
286+
externboolXLogArchiveIsReady(constchar*xlog);
286287
externvoidXLogArchiveCleanup(constchar*xlog);
287288

288289
#endif/* XLOG_INTERNAL_H */

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp