Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitad2925e

Browse files
committed
Don't archive bogus recycled or preallocated files after timeline switch.
After a timeline switch, we would leave behind recycled WAL segments thatare in the future, but on the old timeline. After promotion, and after theybecome old enough to be recycled again, we would notice that they don't havea .ready or .done file, create a .ready file for them, and archive them.That's bogus, because the files contain garbage, recycled from an oldertimeline (or prealloced as zeros). We shouldn't archive such files.This could happen when we're following a timeline switch during replay, orwhen we switch to new timeline at end-of-recovery.To fix, whenever we switch to a new timeline, scan the data directory forWAL segments on the old timeline, but with a higher segment number, andremove them. Those don't belong to our timeline history, and are mostlikely bogus recycled or preallocated files. They could also be valid filesthat we streamed from the primary ahead of time, but in any case, they'renot needed to recover to the new timeline.
1 parentd75e094 commitad2925e

File tree

1 file changed

+197
-87
lines changed
  • src/backend/access/transam

1 file changed

+197
-87
lines changed

‎src/backend/access/transam/xlog.c

Lines changed: 197 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -609,6 +609,7 @@ static void XLogArchiveNotify(const char *xlog);
609609
staticvoidXLogArchiveNotifySeg(uint32log,uint32seg);
610610
staticboolXLogArchiveCheckDone(constchar*xlog);
611611
staticboolXLogArchiveIsBusy(constchar*xlog);
612+
externboolXLogArchiveIsReady(constchar*xlog);
612613
staticvoidXLogArchiveCleanup(constchar*xlog);
613614
staticvoidreadRecoveryCommandFile(void);
614615
staticvoidexitArchiveRecovery(TimeLineIDendTLI,
@@ -646,6 +647,8 @@ static void ExecuteRecoveryCommand(char *command, char *commandName,
646647
boolfailOnerror);
647648
staticvoidPreallocXlogFiles(XLogRecPtrendptr);
648649
staticvoidRemoveOldXlogFiles(uint32log,uint32seg,XLogRecPtrendptr);
650+
staticvoidRemoveXlogFile(constchar*segname,XLogRecPtrendptr);
651+
staticvoidRemoveNonParentXlogFiles(XLogRecPtrswitchpoint,TimeLineIDnewTLI);
649652
staticvoidUpdateLastRemovedPtr(char*filename);
650653
staticvoidValidateXLOGDirectoryStructure(void);
651654
staticvoidCleanupBackupHistory(void);
@@ -1455,6 +1458,25 @@ XLogArchiveIsBusy(const char *xlog)
14551458
return true;
14561459
}
14571460

1461+
/*
1462+
* XLogArchiveIsReady
1463+
*
1464+
* Check to see if an XLOG segment file has an archive notification (.ready)
1465+
* file.
1466+
*/
1467+
bool
1468+
XLogArchiveIsReady(constchar*xlog)
1469+
{
1470+
chararchiveStatusPath[MAXPGPATH];
1471+
structstatstat_buf;
1472+
1473+
StatusFilePath(archiveStatusPath,xlog,".ready");
1474+
if (stat(archiveStatusPath,&stat_buf)==0)
1475+
return true;
1476+
1477+
return false;
1478+
}
1479+
14581480
/*
14591481
* XLogArchiveCleanup
14601482
*
@@ -3332,25 +3354,9 @@ UpdateLastRemovedPtr(char *filename)
33323354
staticvoid
33333355
RemoveOldXlogFiles(uint32log,uint32seg,XLogRecPtrendptr)
33343356
{
3335-
uint32endlogId;
3336-
uint32endlogSeg;
3337-
intmax_advance;
33383357
DIR*xldir;
33393358
structdirent*xlde;
33403359
charlastoff[MAXFNAMELEN];
3341-
charpath[MAXPGPATH];
3342-
3343-
#ifdefWIN32
3344-
charnewpath[MAXPGPATH];
3345-
#endif
3346-
structstatstatbuf;
3347-
3348-
/*
3349-
* Initialize info about where to try to recycle to. We allow recycling
3350-
* segments up to XLOGfileslop segments beyond the current XLOG location.
3351-
*/
3352-
XLByteToPrevSeg(endptr,endlogId,endlogSeg);
3353-
max_advance=XLOGfileslop;
33543360

33553361
xldir=AllocateDir(XLOGDIR);
33563362
if (xldir==NULL)
@@ -3366,6 +3372,11 @@ RemoveOldXlogFiles(uint32 log, uint32 seg, XLogRecPtr endptr)
33663372

33673373
while ((xlde=ReadDir(xldir,XLOGDIR))!=NULL)
33683374
{
3375+
/* Ignore files that are not XLOG segments */
3376+
if (strlen(xlde->d_name)!=24||
3377+
strspn(xlde->d_name,"0123456789ABCDEF")!=24)
3378+
continue;
3379+
33693380
/*
33703381
* We ignore the timeline part of the XLOG segment identifiers in
33713382
* deciding whether a segment is still needed. This ensures that we
@@ -3377,9 +3388,7 @@ RemoveOldXlogFiles(uint32 log, uint32 seg, XLogRecPtr endptr)
33773388
* We use the alphanumeric sorting property of the filenames to decide
33783389
* which ones are earlier than the lastoff segment.
33793390
*/
3380-
if (strlen(xlde->d_name)==24&&
3381-
strspn(xlde->d_name,"0123456789ABCDEF")==24&&
3382-
strcmp(xlde->d_name+8,lastoff+8) <=0)
3391+
if (strcmp(xlde->d_name+8,lastoff+8) <=0)
33833392
{
33843393
/*
33853394
* Normally we don't delete old XLOG files during recovery to
@@ -3394,86 +3403,107 @@ RemoveOldXlogFiles(uint32 log, uint32 seg, XLogRecPtr endptr)
33943403
*/
33953404
if (WalRcvInProgress()||XLogArchiveCheckDone(xlde->d_name))
33963405
{
3397-
snprintf(path,MAXPGPATH,XLOGDIR"/%s",xlde->d_name);
3398-
33993406
/* Update the last removed location in shared memory first */
34003407
UpdateLastRemovedPtr(xlde->d_name);
34013408

3402-
/*
3403-
* Before deleting the file, see if it can be recycled as a
3404-
* future log segment. Only recycle normal files, pg_standby
3405-
* for example can create symbolic links pointing to a
3406-
* separate archive directory.
3407-
*/
3408-
if (lstat(path,&statbuf)==0&&S_ISREG(statbuf.st_mode)&&
3409-
InstallXLogFileSegment(&endlogId,&endlogSeg,path,
3410-
true,&max_advance, true))
3411-
{
3412-
ereport(DEBUG2,
3413-
(errmsg("recycled transaction log file \"%s\"",
3414-
xlde->d_name)));
3415-
CheckpointStats.ckpt_segs_recycled++;
3416-
/* Needn't recheck that slot on future iterations */
3417-
if (max_advance>0)
3418-
{
3419-
NextLogSeg(endlogId,endlogSeg);
3420-
max_advance--;
3421-
}
3422-
}
3423-
else
3424-
{
3425-
/* No need for any more future segments... */
3426-
intrc;
3409+
RemoveXlogFile(xlde->d_name,endptr);
3410+
}
3411+
}
3412+
}
34273413

3428-
ereport(DEBUG2,
3429-
(errmsg("removing transaction log file \"%s\"",
3430-
xlde->d_name)));
3414+
FreeDir(xldir);
3415+
}
34313416

3417+
/*
3418+
* Recycle or remove a log file that's no longer needed.
3419+
*
3420+
* endptr is current (or recent) end of xlog; this is used to determine
3421+
* whether we want to recycle rather than delete no-longer-wanted log files.
3422+
*/
3423+
staticvoid
3424+
RemoveXlogFile(constchar*segname,XLogRecPtrendptr)
3425+
{
3426+
charpath[MAXPGPATH];
34323427
#ifdefWIN32
3428+
charnewpath[MAXPGPATH];
3429+
#endif
3430+
structstatstatbuf;
3431+
uint32endlogId;
3432+
uint32endlogSeg;
3433+
intmax_advance;
34333434

3434-
/*
3435-
* On Windows, if another process (e.g another backend)
3436-
* holds the file open in FILE_SHARE_DELETE mode, unlink
3437-
* will succeed, but the file will still show up in
3438-
* directory listing until the last handle is closed. To
3439-
* avoid confusing the lingering deleted file for a live
3440-
* WAL file that needs to be archived, rename it before
3441-
* deleting it.
3442-
*
3443-
* If another process holds the file open without
3444-
* FILE_SHARE_DELETE flag, rename will fail. We'll try
3445-
* again at the next checkpoint.
3446-
*/
3447-
snprintf(newpath,MAXPGPATH,"%s.deleted",path);
3448-
if (rename(path,newpath)!=0)
3449-
{
3450-
ereport(LOG,
3451-
(errcode_for_file_access(),
3452-
errmsg("could not rename old transaction log file \"%s\": %m",
3453-
path)));
3454-
continue;
3455-
}
3456-
rc=unlink(newpath);
3435+
/*
3436+
* Initialize info about where to try to recycle to. We allow recycling
3437+
* segments up to XLOGfileslop segments beyond the current XLOG location.
3438+
*/
3439+
XLByteToPrevSeg(endptr,endlogId,endlogSeg);
3440+
max_advance=XLOGfileslop;
3441+
3442+
snprintf(path,MAXPGPATH,XLOGDIR"/%s",segname);
3443+
3444+
/*
3445+
* Before deleting the file, see if it can be recycled as a future log
3446+
* segment. Only recycle normal files, pg_standby for example can create
3447+
* symbolic links pointing to a separate archive directory.
3448+
*/
3449+
if (lstat(path,&statbuf)==0&&S_ISREG(statbuf.st_mode)&&
3450+
InstallXLogFileSegment(&endlogId,&endlogSeg,path,
3451+
true,&max_advance, true))
3452+
{
3453+
ereport(DEBUG2,
3454+
(errmsg("recycled transaction log file \"%s\"",segname)));
3455+
CheckpointStats.ckpt_segs_recycled++;
3456+
/* Needn't recheck that slot on future iterations */
3457+
if (max_advance>0)
3458+
{
3459+
NextLogSeg(endlogId,endlogSeg);
3460+
max_advance--;
3461+
}
3462+
}
3463+
else
3464+
{
3465+
/* No need for any more future segments... */
3466+
intrc;
3467+
3468+
ereport(DEBUG2,
3469+
(errmsg("removing transaction log file \"%s\"",segname)));
3470+
3471+
#ifdefWIN32
3472+
/*
3473+
* On Windows, if another process (e.g another backend) holds the file
3474+
* open in FILE_SHARE_DELETE mode, unlink will succeed, but the file
3475+
* will still show up in directory listing until the last handle is
3476+
* closed. To avoid confusing the lingering deleted file for a live
3477+
* WAL file that needs to be archived, rename it before deleting it.
3478+
*
3479+
* If another process holds the file open without FILE_SHARE_DELETE
3480+
* flag, rename will fail. We'll try again at the next checkpoint.
3481+
*/
3482+
snprintf(newpath,MAXPGPATH,"%s.deleted",path);
3483+
if (rename(path,newpath)!=0)
3484+
{
3485+
ereport(LOG,
3486+
(errcode_for_file_access(),
3487+
errmsg("could not rename old transaction log file \"%s\": %m",
3488+
path)));
3489+
return;
3490+
}
3491+
rc=unlink(newpath);
34573492
#else
3458-
rc=unlink(path);
3493+
rc=unlink(path);
34593494
#endif
3460-
if (rc!=0)
3461-
{
3462-
ereport(LOG,
3463-
(errcode_for_file_access(),
3464-
errmsg("could not remove old transaction log file \"%s\": %m",
3465-
path)));
3466-
continue;
3467-
}
3468-
CheckpointStats.ckpt_segs_removed++;
3469-
}
3470-
3471-
XLogArchiveCleanup(xlde->d_name);
3472-
}
3495+
if (rc!=0)
3496+
{
3497+
ereport(LOG,
3498+
(errcode_for_file_access(),
3499+
errmsg("could not remove old transaction log file \"%s\": %m",
3500+
path)));
3501+
return;
34733502
}
3503+
CheckpointStats.ckpt_segs_removed++;
34743504
}
34753505

3476-
FreeDir(xldir);
3506+
XLogArchiveCleanup(segname);
34773507
}
34783508

34793509
/*
@@ -5629,6 +5659,76 @@ exitArchiveRecovery(TimeLineID endTLI, uint32 endLogId, uint32 endLogSeg)
56295659
(errmsg("archive recovery complete")));
56305660
}
56315661

5662+
/*
5663+
* Remove WAL files that are not part of the given timeline's history.
5664+
*
5665+
* This is called during recovery, whenever we switch to follow a new
5666+
* timeline, and at the end of recovery when we create a new timeline. We
5667+
* wouldn't otherwise care about extra WAL files lying in pg_xlog, but they
5668+
* can be pre-allocated or recycled WAL segments on the old timeline that we
5669+
* haven't used yet, and contain garbage. If we just leave them in pg_xlog,
5670+
* they will eventually be archived, and we can't let that happen. Files that
5671+
* belong to our timeline history are valid, because we have successfully
5672+
* replayed them, but from others we can't be sure.
5673+
*
5674+
* 'switchpoint' is the current point in WAL where we switch to new timeline,
5675+
* and 'newTLI' is the new timeline we switch to.
5676+
*/
5677+
staticvoid
5678+
RemoveNonParentXlogFiles(XLogRecPtrswitchpoint,TimeLineIDnewTLI)
5679+
{
5680+
DIR*xldir;
5681+
structdirent*xlde;
5682+
charswitchseg[MAXFNAMELEN];
5683+
uint32endlogId;
5684+
uint32endlogSeg;
5685+
5686+
XLByteToPrevSeg(switchpoint,endlogId,endlogSeg);
5687+
5688+
xldir=AllocateDir(XLOGDIR);
5689+
if (xldir==NULL)
5690+
ereport(ERROR,
5691+
(errcode_for_file_access(),
5692+
errmsg("could not open transaction log directory \"%s\": %m",
5693+
XLOGDIR)));
5694+
5695+
/*
5696+
* Construct a filename of the last segment to be kept.
5697+
*/
5698+
XLogFileName(switchseg,newTLI,endlogId,endlogSeg);
5699+
5700+
elog(DEBUG2,"attempting to remove WAL segments newer than log file %s",
5701+
switchseg);
5702+
5703+
while ((xlde=ReadDir(xldir,XLOGDIR))!=NULL)
5704+
{
5705+
/* Ignore files that are not XLOG segments */
5706+
if (strlen(xlde->d_name)!=24||
5707+
strspn(xlde->d_name,"0123456789ABCDEF")!=24)
5708+
continue;
5709+
5710+
/*
5711+
* Remove files that are on a timeline older than the new one we're
5712+
* switching to, but with a segment number >= the first segment on
5713+
* the new timeline.
5714+
*/
5715+
if (strncmp(xlde->d_name,switchseg,8)<0&&
5716+
strcmp(xlde->d_name+8,switchseg+8)>0)
5717+
{
5718+
/*
5719+
* If the file has already been marked as .ready, however, don't
5720+
* remove it yet. It should be OK to remove it - files that are
5721+
* not part of our timeline history are not required for recovery
5722+
* - but seems safer to let them be archived and removed later.
5723+
*/
5724+
if (!XLogArchiveIsReady(xlde->d_name))
5725+
RemoveXlogFile(xlde->d_name,switchpoint);
5726+
}
5727+
}
5728+
5729+
FreeDir(xldir);
5730+
}
5731+
56325732
/*
56335733
* For point-in-time recovery, this function decides whether we want to
56345734
* stop applying the XLOG at or after the current record.
@@ -6075,6 +6175,7 @@ StartupXLOG(void)
60756175
boolwasShutdown;
60766176
boolreachedStopPoint= false;
60776177
boolhaveBackupLabel= false;
6178+
booldidArchiveRecovery= false;
60786179
XLogRecPtrRecPtr,
60796180
checkPointLoc,
60806181
EndOfLog;
@@ -6844,7 +6945,10 @@ StartupXLOG(void)
68446945
* we will use that below.)
68456946
*/
68466947
if (InArchiveRecovery)
6948+
{
6949+
didArchiveRecovery= true;
68476950
exitArchiveRecovery(curFileTLI,endLogId,endLogSeg);
6951+
}
68486952

68496953
/*
68506954
* Prepare to write WAL starting at EndOfLog position, and init xlog
@@ -6957,6 +7061,12 @@ StartupXLOG(void)
69577061
true);
69587062
}
69597063

7064+
/*
7065+
* Clean up any (possibly bogus) future WAL segments on the old timeline.
7066+
*/
7067+
if (didArchiveRecovery)
7068+
RemoveNonParentXlogFiles(EndOfLog,ThisTimeLineID);
7069+
69607070
/*
69617071
* Preallocate additional log files, if wanted.
69627072
*/

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp