Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit5b69381

Browse files
committed
Don't archive bogus recycled or preallocated files after timeline switch.
After a timeline switch, we would leave behind recycled WAL segments thatare in the future, but on the old timeline. After promotion, and after theybecome old enough to be recycled again, we would notice that they don't havea .ready or .done file, create a .ready file for them, and archive them.That's bogus, because the files contain garbage, recycled from an oldertimeline (or prealloced as zeros). We shouldn't archive such files.This could happen when we're following a timeline switch during replay, orwhen we switch to new timeline at end-of-recovery.To fix, whenever we switch to a new timeline, scan the data directory forWAL segments on the old timeline, but with a higher segment number, andremove them. Those don't belong to our timeline history, and are mostlikely bogus recycled or preallocated files. They could also be valid filesthat we streamed from the primary ahead of time, but in any case, they'renot needed to recover to the new timeline.
1 parent595bc97 commit5b69381

File tree

1 file changed

+197
-87
lines changed
  • src/backend/access/transam

1 file changed

+197
-87
lines changed

‎src/backend/access/transam/xlog.c

Lines changed: 197 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -564,6 +564,7 @@ static void XLogArchiveNotify(const char *xlog);
564564
staticvoidXLogArchiveNotifySeg(uint32log,uint32seg);
565565
staticboolXLogArchiveCheckDone(constchar*xlog);
566566
staticboolXLogArchiveIsBusy(constchar*xlog);
567+
externboolXLogArchiveIsReady(constchar*xlog);
567568
staticvoidXLogArchiveCleanup(constchar*xlog);
568569
staticvoidreadRecoveryCommandFile(void);
569570
staticvoidexitArchiveRecovery(TimeLineIDendTLI,
@@ -598,6 +599,8 @@ static void ExecuteRecoveryCommand(char *command, char *commandName,
598599
boolfailOnerror);
599600
staticvoidPreallocXlogFiles(XLogRecPtrendptr);
600601
staticvoidRemoveOldXlogFiles(uint32log,uint32seg,XLogRecPtrendptr);
602+
staticvoidRemoveXlogFile(constchar*segname,XLogRecPtrendptr);
603+
staticvoidRemoveNonParentXlogFiles(XLogRecPtrswitchpoint,TimeLineIDnewTLI);
601604
staticvoidUpdateLastRemovedPtr(char*filename);
602605
staticvoidValidateXLOGDirectoryStructure(void);
603606
staticvoidCleanupBackupHistory(void);
@@ -1405,6 +1408,25 @@ XLogArchiveIsBusy(const char *xlog)
14051408
return true;
14061409
}
14071410

1411+
/*
1412+
* XLogArchiveIsReady
1413+
*
1414+
* Check to see if an XLOG segment file has an archive notification (.ready)
1415+
* file.
1416+
*/
1417+
bool
1418+
XLogArchiveIsReady(constchar*xlog)
1419+
{
1420+
chararchiveStatusPath[MAXPGPATH];
1421+
structstatstat_buf;
1422+
1423+
StatusFilePath(archiveStatusPath,xlog,".ready");
1424+
if (stat(archiveStatusPath,&stat_buf)==0)
1425+
return true;
1426+
1427+
return false;
1428+
}
1429+
14081430
/*
14091431
* XLogArchiveCleanup
14101432
*
@@ -3274,25 +3296,9 @@ UpdateLastRemovedPtr(char *filename)
32743296
staticvoid
32753297
RemoveOldXlogFiles(uint32log,uint32seg,XLogRecPtrendptr)
32763298
{
3277-
uint32endlogId;
3278-
uint32endlogSeg;
3279-
intmax_advance;
32803299
DIR*xldir;
32813300
structdirent*xlde;
32823301
charlastoff[MAXFNAMELEN];
3283-
charpath[MAXPGPATH];
3284-
3285-
#ifdefWIN32
3286-
charnewpath[MAXPGPATH];
3287-
#endif
3288-
structstatstatbuf;
3289-
3290-
/*
3291-
* Initialize info about where to try to recycle to. We allow recycling
3292-
* segments up to XLOGfileslop segments beyond the current XLOG location.
3293-
*/
3294-
XLByteToPrevSeg(endptr,endlogId,endlogSeg);
3295-
max_advance=XLOGfileslop;
32963302

32973303
xldir=AllocateDir(XLOGDIR);
32983304
if (xldir==NULL)
@@ -3308,6 +3314,11 @@ RemoveOldXlogFiles(uint32 log, uint32 seg, XLogRecPtr endptr)
33083314

33093315
while ((xlde=ReadDir(xldir,XLOGDIR))!=NULL)
33103316
{
3317+
/* Ignore files that are not XLOG segments */
3318+
if (strlen(xlde->d_name)!=24||
3319+
strspn(xlde->d_name,"0123456789ABCDEF")!=24)
3320+
continue;
3321+
33113322
/*
33123323
* We ignore the timeline part of the XLOG segment identifiers in
33133324
* deciding whether a segment is still needed. This ensures that we
@@ -3319,9 +3330,7 @@ RemoveOldXlogFiles(uint32 log, uint32 seg, XLogRecPtr endptr)
33193330
* We use the alphanumeric sorting property of the filenames to decide
33203331
* which ones are earlier than the lastoff segment.
33213332
*/
3322-
if (strlen(xlde->d_name)==24&&
3323-
strspn(xlde->d_name,"0123456789ABCDEF")==24&&
3324-
strcmp(xlde->d_name+8,lastoff+8) <=0)
3333+
if (strcmp(xlde->d_name+8,lastoff+8) <=0)
33253334
{
33263335
/*
33273336
* Normally we don't delete old XLOG files during recovery to
@@ -3336,86 +3345,107 @@ RemoveOldXlogFiles(uint32 log, uint32 seg, XLogRecPtr endptr)
33363345
*/
33373346
if (WalRcvInProgress()||XLogArchiveCheckDone(xlde->d_name))
33383347
{
3339-
snprintf(path,MAXPGPATH,XLOGDIR"/%s",xlde->d_name);
3340-
33413348
/* Update the last removed location in shared memory first */
33423349
UpdateLastRemovedPtr(xlde->d_name);
33433350

3344-
/*
3345-
* Before deleting the file, see if it can be recycled as a
3346-
* future log segment. Only recycle normal files, pg_standby
3347-
* for example can create symbolic links pointing to a
3348-
* separate archive directory.
3349-
*/
3350-
if (lstat(path,&statbuf)==0&&S_ISREG(statbuf.st_mode)&&
3351-
InstallXLogFileSegment(&endlogId,&endlogSeg,path,
3352-
true,&max_advance, true))
3353-
{
3354-
ereport(DEBUG2,
3355-
(errmsg("recycled transaction log file \"%s\"",
3356-
xlde->d_name)));
3357-
CheckpointStats.ckpt_segs_recycled++;
3358-
/* Needn't recheck that slot on future iterations */
3359-
if (max_advance>0)
3360-
{
3361-
NextLogSeg(endlogId,endlogSeg);
3362-
max_advance--;
3363-
}
3364-
}
3365-
else
3366-
{
3367-
/* No need for any more future segments... */
3368-
intrc;
3351+
RemoveXlogFile(xlde->d_name,endptr);
3352+
}
3353+
}
3354+
}
33693355

3370-
ereport(DEBUG2,
3371-
(errmsg("removing transaction log file \"%s\"",
3372-
xlde->d_name)));
3356+
FreeDir(xldir);
3357+
}
33733358

3359+
/*
3360+
* Recycle or remove a log file that's no longer needed.
3361+
*
3362+
* endptr is current (or recent) end of xlog; this is used to determine
3363+
* whether we want to recycle rather than delete no-longer-wanted log files.
3364+
*/
3365+
staticvoid
3366+
RemoveXlogFile(constchar*segname,XLogRecPtrendptr)
3367+
{
3368+
charpath[MAXPGPATH];
33743369
#ifdefWIN32
3370+
charnewpath[MAXPGPATH];
3371+
#endif
3372+
structstatstatbuf;
3373+
uint32endlogId;
3374+
uint32endlogSeg;
3375+
intmax_advance;
33753376

3376-
/*
3377-
* On Windows, if another process (e.g another backend)
3378-
* holds the file open in FILE_SHARE_DELETE mode, unlink
3379-
* will succeed, but the file will still show up in
3380-
* directory listing until the last handle is closed. To
3381-
* avoid confusing the lingering deleted file for a live
3382-
* WAL file that needs to be archived, rename it before
3383-
* deleting it.
3384-
*
3385-
* If another process holds the file open without
3386-
* FILE_SHARE_DELETE flag, rename will fail. We'll try
3387-
* again at the next checkpoint.
3388-
*/
3389-
snprintf(newpath,MAXPGPATH,"%s.deleted",path);
3390-
if (rename(path,newpath)!=0)
3391-
{
3392-
ereport(LOG,
3393-
(errcode_for_file_access(),
3394-
errmsg("could not rename old transaction log file \"%s\": %m",
3395-
path)));
3396-
continue;
3397-
}
3398-
rc=unlink(newpath);
3377+
/*
3378+
* Initialize info about where to try to recycle to. We allow recycling
3379+
* segments up to XLOGfileslop segments beyond the current XLOG location.
3380+
*/
3381+
XLByteToPrevSeg(endptr,endlogId,endlogSeg);
3382+
max_advance=XLOGfileslop;
3383+
3384+
snprintf(path,MAXPGPATH,XLOGDIR"/%s",segname);
3385+
3386+
/*
3387+
* Before deleting the file, see if it can be recycled as a future log
3388+
* segment. Only recycle normal files, pg_standby for example can create
3389+
* symbolic links pointing to a separate archive directory.
3390+
*/
3391+
if (lstat(path,&statbuf)==0&&S_ISREG(statbuf.st_mode)&&
3392+
InstallXLogFileSegment(&endlogId,&endlogSeg,path,
3393+
true,&max_advance, true))
3394+
{
3395+
ereport(DEBUG2,
3396+
(errmsg("recycled transaction log file \"%s\"",segname)));
3397+
CheckpointStats.ckpt_segs_recycled++;
3398+
/* Needn't recheck that slot on future iterations */
3399+
if (max_advance>0)
3400+
{
3401+
NextLogSeg(endlogId,endlogSeg);
3402+
max_advance--;
3403+
}
3404+
}
3405+
else
3406+
{
3407+
/* No need for any more future segments... */
3408+
intrc;
3409+
3410+
ereport(DEBUG2,
3411+
(errmsg("removing transaction log file \"%s\"",segname)));
3412+
3413+
#ifdefWIN32
3414+
/*
3415+
* On Windows, if another process (e.g another backend) holds the file
3416+
* open in FILE_SHARE_DELETE mode, unlink will succeed, but the file
3417+
* will still show up in directory listing until the last handle is
3418+
* closed. To avoid confusing the lingering deleted file for a live
3419+
* WAL file that needs to be archived, rename it before deleting it.
3420+
*
3421+
* If another process holds the file open without FILE_SHARE_DELETE
3422+
* flag, rename will fail. We'll try again at the next checkpoint.
3423+
*/
3424+
snprintf(newpath,MAXPGPATH,"%s.deleted",path);
3425+
if (rename(path,newpath)!=0)
3426+
{
3427+
ereport(LOG,
3428+
(errcode_for_file_access(),
3429+
errmsg("could not rename old transaction log file \"%s\": %m",
3430+
path)));
3431+
return;
3432+
}
3433+
rc=unlink(newpath);
33993434
#else
3400-
rc=unlink(path);
3435+
rc=unlink(path);
34013436
#endif
3402-
if (rc!=0)
3403-
{
3404-
ereport(LOG,
3405-
(errcode_for_file_access(),
3406-
errmsg("could not remove old transaction log file \"%s\": %m",
3407-
path)));
3408-
continue;
3409-
}
3410-
CheckpointStats.ckpt_segs_removed++;
3411-
}
3412-
3413-
XLogArchiveCleanup(xlde->d_name);
3414-
}
3437+
if (rc!=0)
3438+
{
3439+
ereport(LOG,
3440+
(errcode_for_file_access(),
3441+
errmsg("could not remove old transaction log file \"%s\": %m",
3442+
path)));
3443+
return;
34153444
}
3445+
CheckpointStats.ckpt_segs_removed++;
34163446
}
34173447

3418-
FreeDir(xldir);
3448+
XLogArchiveCleanup(segname);
34193449
}
34203450

34213451
/*
@@ -5501,6 +5531,76 @@ exitArchiveRecovery(TimeLineID endTLI, uint32 endLogId, uint32 endLogSeg)
55015531
(errmsg("archive recovery complete")));
55025532
}
55035533

5534+
/*
5535+
* Remove WAL files that are not part of the given timeline's history.
5536+
*
5537+
* This is called during recovery, whenever we switch to follow a new
5538+
* timeline, and at the end of recovery when we create a new timeline. We
5539+
* wouldn't otherwise care about extra WAL files lying in pg_xlog, but they
5540+
* can be pre-allocated or recycled WAL segments on the old timeline that we
5541+
* haven't used yet, and contain garbage. If we just leave them in pg_xlog,
5542+
* they will eventually be archived, and we can't let that happen. Files that
5543+
* belong to our timeline history are valid, because we have successfully
5544+
* replayed them, but from others we can't be sure.
5545+
*
5546+
* 'switchpoint' is the current point in WAL where we switch to new timeline,
5547+
* and 'newTLI' is the new timeline we switch to.
5548+
*/
5549+
staticvoid
5550+
RemoveNonParentXlogFiles(XLogRecPtrswitchpoint,TimeLineIDnewTLI)
5551+
{
5552+
DIR*xldir;
5553+
structdirent*xlde;
5554+
charswitchseg[MAXFNAMELEN];
5555+
uint32endlogId;
5556+
uint32endlogSeg;
5557+
5558+
XLByteToPrevSeg(switchpoint,endlogId,endlogSeg);
5559+
5560+
xldir=AllocateDir(XLOGDIR);
5561+
if (xldir==NULL)
5562+
ereport(ERROR,
5563+
(errcode_for_file_access(),
5564+
errmsg("could not open transaction log directory \"%s\": %m",
5565+
XLOGDIR)));
5566+
5567+
/*
5568+
* Construct a filename of the last segment to be kept.
5569+
*/
5570+
XLogFileName(switchseg,newTLI,endlogId,endlogSeg);
5571+
5572+
elog(DEBUG2,"attempting to remove WAL segments newer than log file %s",
5573+
switchseg);
5574+
5575+
while ((xlde=ReadDir(xldir,XLOGDIR))!=NULL)
5576+
{
5577+
/* Ignore files that are not XLOG segments */
5578+
if (strlen(xlde->d_name)!=24||
5579+
strspn(xlde->d_name,"0123456789ABCDEF")!=24)
5580+
continue;
5581+
5582+
/*
5583+
* Remove files that are on a timeline older than the new one we're
5584+
* switching to, but with a segment number >= the first segment on
5585+
* the new timeline.
5586+
*/
5587+
if (strncmp(xlde->d_name,switchseg,8)<0&&
5588+
strcmp(xlde->d_name+8,switchseg+8)>0)
5589+
{
5590+
/*
5591+
* If the file has already been marked as .ready, however, don't
5592+
* remove it yet. It should be OK to remove it - files that are
5593+
* not part of our timeline history are not required for recovery
5594+
* - but seems safer to let them be archived and removed later.
5595+
*/
5596+
if (!XLogArchiveIsReady(xlde->d_name))
5597+
RemoveXlogFile(xlde->d_name,switchpoint);
5598+
}
5599+
}
5600+
5601+
FreeDir(xldir);
5602+
}
5603+
55045604
/*
55055605
* For point-in-time recovery, this function decides whether we want to
55065606
* stop applying the XLOG at or after the current record.
@@ -5771,6 +5871,7 @@ StartupXLOG(void)
57715871
boolwasShutdown;
57725872
boolreachedStopPoint= false;
57735873
boolhaveBackupLabel= false;
5874+
booldidArchiveRecovery= false;
57745875
XLogRecPtrRecPtr,
57755876
checkPointLoc,
57765877
EndOfLog;
@@ -6469,7 +6570,10 @@ StartupXLOG(void)
64696570
* we will use that below.)
64706571
*/
64716572
if (InArchiveRecovery)
6573+
{
6574+
didArchiveRecovery= true;
64726575
exitArchiveRecovery(curFileTLI,endLogId,endLogSeg);
6576+
}
64736577

64746578
/*
64756579
* Prepare to write WAL starting at EndOfLog position, and init xlog
@@ -6582,6 +6686,12 @@ StartupXLOG(void)
65826686
true);
65836687
}
65846688

6689+
/*
6690+
* Clean up any (possibly bogus) future WAL segments on the old timeline.
6691+
*/
6692+
if (didArchiveRecovery)
6693+
RemoveNonParentXlogFiles(EndOfLog,ThisTimeLineID);
6694+
65856695
/*
65866696
* Preallocate additional log files, if wanted.
65876697
*/

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp