Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitb2a5545

Browse files
committed
Don't archive bogus recycled or preallocated files after timeline switch.
After a timeline switch, we would leave behind recycled WAL segments thatare in the future, but on the old timeline. After promotion, and after theybecome old enough to be recycled again, we would notice that they don't havea .ready or .done file, create a .ready file for them, and archive them.That's bogus, because the files contain garbage, recycled from an oldertimeline (or prealloced as zeros). We shouldn't archive such files.This could happen when we're following a timeline switch during replay, orwhen we switch to new timeline at end-of-recovery.To fix, whenever we switch to a new timeline, scan the data directory forWAL segments on the old timeline, but with a higher segment number, andremove them. Those don't belong to our timeline history, and are mostlikely bogus recycled or preallocated files. They could also be valid filesthat we streamed from the primary ahead of time, but in any case, they'renot needed to recover to the new timeline.
1 parent1f94bec commitb2a5545

File tree

3 files changed

+211
-89
lines changed

3 files changed

+211
-89
lines changed

‎src/backend/access/transam/xlog.c

Lines changed: 191 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -791,6 +791,7 @@ static intemode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
791791
staticvoidXLogFileClose(void);
792792
staticvoidPreallocXlogFiles(XLogRecPtrendptr);
793793
staticvoidRemoveOldXlogFiles(XLogSegNosegno,XLogRecPtrPriorRedoPtr,XLogRecPtrendptr);
794+
staticvoidRemoveXlogFile(constchar*segname,XLogRecPtrPriorRedoPtr,XLogRecPtrendptr);
794795
staticvoidUpdateLastRemovedPtr(char*filename);
795796
staticvoidValidateXLOGDirectoryStructure(void);
796797
staticvoidCleanupBackupHistory(void);
@@ -3531,7 +3532,7 @@ UpdateLastRemovedPtr(char *filename)
35313532
}
35323533

35333534
/*
3534-
* Recycle or remove all log files older or equal to passed segno
3535+
* Recycle or remove all log files older or equal to passed segno.
35353536
*
35363537
* endptr is current (or recent) end of xlog, and PriorRedoRecPtr is the
35373538
* redo pointer of the previous checkpoint. These are used to determine
@@ -3540,23 +3541,9 @@ UpdateLastRemovedPtr(char *filename)
35403541
staticvoid
35413542
RemoveOldXlogFiles(XLogSegNosegno,XLogRecPtrPriorRedoPtr,XLogRecPtrendptr)
35423543
{
3543-
XLogSegNoendlogSegNo;
3544-
XLogSegNorecycleSegNo;
35453544
DIR*xldir;
35463545
structdirent*xlde;
35473546
charlastoff[MAXFNAMELEN];
3548-
charpath[MAXPGPATH];
3549-
3550-
#ifdefWIN32
3551-
charnewpath[MAXPGPATH];
3552-
#endif
3553-
structstatstatbuf;
3554-
3555-
/*
3556-
* Initialize info about where to try to recycle to.
3557-
*/
3558-
XLByteToPrevSeg(endptr,endlogSegNo);
3559-
recycleSegNo=XLOGfileslop(PriorRedoPtr);
35603547

35613548
xldir=AllocateDir(XLOGDIR);
35623549
if (xldir==NULL)
@@ -3577,6 +3564,11 @@ RemoveOldXlogFiles(XLogSegNo segno, XLogRecPtr PriorRedoPtr, XLogRecPtr endptr)
35773564

35783565
while ((xlde=ReadDir(xldir,XLOGDIR))!=NULL)
35793566
{
3567+
/* Ignore files that are not XLOG segments */
3568+
if (strlen(xlde->d_name)!=24||
3569+
strspn(xlde->d_name,"0123456789ABCDEF")!=24)
3570+
continue;
3571+
35803572
/*
35813573
* We ignore the timeline part of the XLOG segment identifiers in
35823574
* deciding whether a segment is still needed. This ensures that we
@@ -3588,89 +3580,183 @@ RemoveOldXlogFiles(XLogSegNo segno, XLogRecPtr PriorRedoPtr, XLogRecPtr endptr)
35883580
* We use the alphanumeric sorting property of the filenames to decide
35893581
* which ones are earlier than the lastoff segment.
35903582
*/
3591-
if (strlen(xlde->d_name)==24&&
3592-
strspn(xlde->d_name,"0123456789ABCDEF")==24&&
3593-
strcmp(xlde->d_name+8,lastoff+8) <=0)
3583+
if (strcmp(xlde->d_name+8,lastoff+8) <=0)
35943584
{
35953585
if (XLogArchiveCheckDone(xlde->d_name))
35963586
{
3597-
snprintf(path,MAXPGPATH,XLOGDIR"/%s",xlde->d_name);
3598-
35993587
/* Update the last removed location in shared memory first */
36003588
UpdateLastRemovedPtr(xlde->d_name);
36013589

3602-
/*
3603-
* Before deleting the file, see if it can be recycled as a
3604-
* future log segment. Only recycle normal files, pg_standby
3605-
* for example can create symbolic links pointing to a
3606-
* separate archive directory.
3607-
*/
3608-
if (endlogSegNo <=recycleSegNo&&
3609-
lstat(path,&statbuf)==0&&S_ISREG(statbuf.st_mode)&&
3610-
InstallXLogFileSegment(&endlogSegNo,path,
3611-
true,recycleSegNo, true))
3612-
{
3613-
ereport(DEBUG2,
3614-
(errmsg("recycled transaction log file \"%s\"",
3615-
xlde->d_name)));
3616-
CheckpointStats.ckpt_segs_recycled++;
3617-
/* Needn't recheck that slot on future iterations */
3618-
endlogSegNo++;
3619-
}
3620-
else
3621-
{
3622-
/* No need for any more future segments... */
3623-
intrc;
3590+
RemoveXlogFile(xlde->d_name,PriorRedoPtr,endptr);
3591+
}
3592+
}
3593+
}
3594+
3595+
FreeDir(xldir);
3596+
}
3597+
3598+
/*
3599+
* Remove WAL files that are not part of the given timeline's history.
3600+
*
3601+
* This is called during recovery, whenever we switch to follow a new
3602+
* timeline, and at the end of recovery when we create a new timeline. We
3603+
* wouldn't otherwise care about extra WAL files lying in pg_xlog, but they
3604+
* might be leftover pre-allocated or recycled WAL segments on the old timeline
3605+
* that we haven't used yet, and contain garbage. If we just leave them in
3606+
* pg_xlog, they will eventually be archived, and we can't let that happen.
3607+
* Files that belong to our timeline history are valid, because we have
3608+
* successfully replayed them, but from others we can't be sure.
3609+
*
3610+
* 'switchpoint' is the current point in WAL where we switch to new timeline,
3611+
* and 'newTLI' is the new timeline we switch to.
3612+
*/
3613+
staticvoid
3614+
RemoveNonParentXlogFiles(XLogRecPtrswitchpoint,TimeLineIDnewTLI)
3615+
{
3616+
DIR*xldir;
3617+
structdirent*xlde;
3618+
charswitchseg[MAXFNAMELEN];
3619+
XLogSegNoendLogSegNo;
3620+
3621+
XLByteToPrevSeg(switchpoint,endLogSegNo);
3622+
3623+
xldir=AllocateDir(XLOGDIR);
3624+
if (xldir==NULL)
3625+
ereport(ERROR,
3626+
(errcode_for_file_access(),
3627+
errmsg("could not open transaction log directory \"%s\": %m",
3628+
XLOGDIR)));
36243629

3625-
ereport(DEBUG2,
3626-
(errmsg("removing transaction log file \"%s\"",
3627-
xlde->d_name)));
3630+
/*
3631+
* Construct a filename of the last segment to be kept.
3632+
*/
3633+
XLogFileName(switchseg,newTLI,endLogSegNo);
3634+
3635+
elog(DEBUG2,"attempting to remove WAL segments newer than log file %s",
3636+
switchseg);
36283637

3638+
while ((xlde=ReadDir(xldir,XLOGDIR))!=NULL)
3639+
{
3640+
/* Ignore files that are not XLOG segments */
3641+
if (strlen(xlde->d_name)!=24||
3642+
strspn(xlde->d_name,"0123456789ABCDEF")!=24)
3643+
continue;
3644+
3645+
/*
3646+
* Remove files that are on a timeline older than the new one we're
3647+
* switching to, but with a segment number >= the first segment on
3648+
* the new timeline.
3649+
*/
3650+
if (strncmp(xlde->d_name,switchseg,8)<0&&
3651+
strcmp(xlde->d_name+8,switchseg+8)>0)
3652+
{
3653+
/*
3654+
* If the file has already been marked as .ready, however, don't
3655+
* remove it yet. It should be OK to remove it - files that are
3656+
* not part of our timeline history are not required for recovery
3657+
* - but seems safer to let them be archived and removed later.
3658+
*/
3659+
if (!XLogArchiveIsReady(xlde->d_name))
3660+
RemoveXlogFile(xlde->d_name,InvalidXLogRecPtr,switchpoint);
3661+
}
3662+
}
3663+
3664+
FreeDir(xldir);
3665+
}
3666+
3667+
/*
3668+
* Recycle or remove a log file that's no longer needed.
3669+
*
3670+
* endptr is current (or recent) end of xlog, and PriorRedoRecPtr is the
3671+
* redo pointer of the previous checkpoint. These are used to determine
3672+
* whether we want to recycle rather than delete no-longer-wanted log files.
3673+
* If PriorRedoRecPtr is not known, pass invalid, and the function will
3674+
* recycle, somewhat arbitrarily, 10 future segments.
3675+
*/
3676+
staticvoid
3677+
RemoveXlogFile(constchar*segname,XLogRecPtrPriorRedoPtr,XLogRecPtrendptr)
3678+
{
3679+
charpath[MAXPGPATH];
36293680
#ifdefWIN32
3681+
charnewpath[MAXPGPATH];
3682+
#endif
3683+
structstatstatbuf;
3684+
XLogSegNoendlogSegNo;
3685+
XLogSegNorecycleSegNo;
36303686

3631-
/*
3632-
* On Windows, if another process (e.g another backend)
3633-
* holds the file open in FILE_SHARE_DELETE mode, unlink
3634-
* will succeed, but the file will still show up in
3635-
* directory listing until the last handle is closed. To
3636-
* avoid confusing the lingering deleted file for a live
3637-
* WAL file that needs to be archived, rename it before
3638-
* deleting it.
3639-
*
3640-
* If another process holds the file open without
3641-
* FILE_SHARE_DELETE flag, rename will fail. We'll try
3642-
* again at the next checkpoint.
3643-
*/
3644-
snprintf(newpath,MAXPGPATH,"%s.deleted",path);
3645-
if (rename(path,newpath)!=0)
3646-
{
3647-
ereport(LOG,
3648-
(errcode_for_file_access(),
3649-
errmsg("could not rename old transaction log file \"%s\": %m",
3650-
path)));
3651-
continue;
3652-
}
3653-
rc=unlink(newpath);
3687+
/*
3688+
* Initialize info about where to try to recycle to.
3689+
*/
3690+
XLByteToPrevSeg(endptr,endlogSegNo);
3691+
if (PriorRedoPtr==InvalidXLogRecPtr)
3692+
recycleSegNo=endlogSegNo+10;
3693+
else
3694+
recycleSegNo=XLOGfileslop(PriorRedoPtr);
3695+
3696+
snprintf(path,MAXPGPATH,XLOGDIR"/%s",segname);
3697+
3698+
/*
3699+
* Before deleting the file, see if it can be recycled as a future log
3700+
* segment. Only recycle normal files, pg_standby for example can create
3701+
* symbolic links pointing to a separate archive directory.
3702+
*/
3703+
if (endlogSegNo <=recycleSegNo&&
3704+
lstat(path,&statbuf)==0&&S_ISREG(statbuf.st_mode)&&
3705+
InstallXLogFileSegment(&endlogSegNo,path,
3706+
true,recycleSegNo, true))
3707+
{
3708+
ereport(DEBUG2,
3709+
(errmsg("recycled transaction log file \"%s\"",
3710+
segname)));
3711+
CheckpointStats.ckpt_segs_recycled++;
3712+
/* Needn't recheck that slot on future iterations */
3713+
endlogSegNo++;
3714+
}
3715+
else
3716+
{
3717+
/* No need for any more future segments... */
3718+
intrc;
3719+
3720+
ereport(DEBUG2,
3721+
(errmsg("removing transaction log file \"%s\"",
3722+
segname)));
3723+
3724+
#ifdefWIN32
3725+
/*
3726+
* On Windows, if another process (e.g another backend) holds the file
3727+
* open in FILE_SHARE_DELETE mode, unlink will succeed, but the file
3728+
* will still show up in directory listing until the last handle is
3729+
* closed. To avoid confusing the lingering deleted file for a live WAL
3730+
* file that needs to be archived, rename it before deleting it.
3731+
*
3732+
* If another process holds the file open without FILE_SHARE_DELETE
3733+
* flag, rename will fail. We'll try again at the next checkpoint.
3734+
*/
3735+
snprintf(newpath,MAXPGPATH,"%s.deleted",path);
3736+
if (rename(path,newpath)!=0)
3737+
{
3738+
ereport(LOG,
3739+
(errcode_for_file_access(),
3740+
errmsg("could not rename old transaction log file \"%s\": %m",
3741+
path)));
3742+
return;
3743+
}
3744+
rc=unlink(newpath);
36543745
#else
3655-
rc=unlink(path);
3746+
rc=unlink(path);
36563747
#endif
3657-
if (rc!=0)
3658-
{
3659-
ereport(LOG,
3660-
(errcode_for_file_access(),
3661-
errmsg("could not remove old transaction log file \"%s\": %m",
3662-
path)));
3663-
continue;
3664-
}
3665-
CheckpointStats.ckpt_segs_removed++;
3666-
}
3667-
3668-
XLogArchiveCleanup(xlde->d_name);
3669-
}
3748+
if (rc!=0)
3749+
{
3750+
ereport(LOG,
3751+
(errcode_for_file_access(),
3752+
errmsg("could not remove old transaction log file \"%s\": %m",
3753+
path)));
3754+
return;
36703755
}
3756+
CheckpointStats.ckpt_segs_removed++;
36713757
}
36723758

3673-
FreeDir(xldir);
3759+
XLogArchiveCleanup(segname);
36743760
}
36753761

36763762
/*
@@ -6626,12 +6712,22 @@ StartupXLOG(void)
66266712
/* Allow read-only connections if we're consistent now */
66276713
CheckRecoveryConsistency();
66286714

6629-
/*
6630-
* If this record was a timeline switch, wake up any
6631-
* walsenders to notice that we are on a new timeline.
6632-
*/
6633-
if (switchedTLI&&AllowCascadeReplication())
6634-
WalSndWakeup();
6715+
/* Is this a timeline switch? */
6716+
if (switchedTLI)
6717+
{
6718+
/*
6719+
* Before we continue on the new timeline, clean up any
6720+
* (possibly bogus) future WAL segments on the old timeline.
6721+
*/
6722+
RemoveNonParentXlogFiles(EndRecPtr,ThisTimeLineID);
6723+
6724+
/*
6725+
* Wake up any walsenders to notice that we are on a new
6726+
* timeline.
6727+
*/
6728+
if (switchedTLI&&AllowCascadeReplication())
6729+
WalSndWakeup();
6730+
}
66356731

66366732
/* Exit loop if we reached inclusive recovery target */
66376733
if (recoveryStopsAfter(xlogreader))
@@ -6975,6 +7071,12 @@ StartupXLOG(void)
69757071
true);
69767072
}
69777073

7074+
/*
7075+
* Clean up any (possibly bogus) future WAL segments on the old timeline.
7076+
*/
7077+
if (ArchiveRecoveryRequested)
7078+
RemoveNonParentXlogFiles(EndOfLog,ThisTimeLineID);
7079+
69787080
/*
69797081
* Preallocate additional log files, if wanted.
69807082
*/

‎src/backend/access/transam/xlogarchive.c

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -694,6 +694,25 @@ XLogArchiveIsBusy(const char *xlog)
694694
return true;
695695
}
696696

697+
/*
698+
* XLogArchiveIsReady
699+
*
700+
* Check to see if an XLOG segment file has an archive notification (.ready)
701+
* file.
702+
*/
703+
bool
704+
XLogArchiveIsReady(constchar*xlog)
705+
{
706+
chararchiveStatusPath[MAXPGPATH];
707+
structstatstat_buf;
708+
709+
StatusFilePath(archiveStatusPath,xlog,".ready");
710+
if (stat(archiveStatusPath,&stat_buf)==0)
711+
return true;
712+
713+
return false;
714+
}
715+
697716
/*
698717
* XLogArchiveCleanup
699718
*

‎src/include/access/xlog_internal.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,7 @@ extern void XLogArchiveNotifySeg(XLogSegNo segno);
281281
externvoidXLogArchiveForceDone(constchar*xlog);
282282
externboolXLogArchiveCheckDone(constchar*xlog);
283283
externboolXLogArchiveIsBusy(constchar*xlog);
284+
externboolXLogArchiveIsReady(constchar*xlog);
284285
externvoidXLogArchiveCleanup(constchar*xlog);
285286

286287
#endif/* XLOG_INTERNAL_H */

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp