Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commita800267

Browse files
committed
Don't archive bogus recycled or preallocated files after timeline switch.
After a timeline switch, we would leave behind recycled WAL segments thatare in the future, but on the old timeline. After promotion, and after theybecome old enough to be recycled again, we would notice that they don't havea .ready or .done file, create a .ready file for them, and archive them.That's bogus, because the files contain garbage, recycled from an oldertimeline (or prealloced as zeros). We shouldn't archive such files.This could happen when we're following a timeline switch during replay, orwhen we switch to new timeline at end-of-recovery.To fix, whenever we switch to a new timeline, scan the data directory forWAL segments on the old timeline, but with a higher segment number, andremove them. Those don't belong to our timeline history, and are mostlikely bogus recycled or preallocated files. They could also be valid filesthat we streamed from the primary ahead of time, but in any case, they'renot needed to recover to the new timeline.
1 parent8dfddf1 commita800267

File tree

3 files changed

+208
-94
lines changed

3 files changed

+208
-94
lines changed

‎src/backend/access/transam/xlog.c

Lines changed: 188 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -669,6 +669,8 @@ static intemode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
669669
staticvoidXLogFileClose(void);
670670
staticvoidPreallocXlogFiles(XLogRecPtrendptr);
671671
staticvoidRemoveOldXlogFiles(XLogSegNosegno,XLogRecPtrendptr);
672+
staticvoidRemoveXlogFile(constchar*segname,XLogRecPtrendptr);
673+
staticvoidRemoveNonParentXlogFiles(XLogRecPtrswitchpoint,TimeLineIDnewTLI);
672674
staticvoidUpdateLastRemovedPtr(char*filename);
673675
staticvoidValidateXLOGDirectoryStructure(void);
674676
staticvoidCleanupBackupHistory(void);
@@ -2876,32 +2878,17 @@ UpdateLastRemovedPtr(char *filename)
28762878
}
28772879

28782880
/*
2879-
* Recycle or remove all log files older or equal to passed segno
2881+
* Recycle or remove all log files older or equal to passed segno.
28802882
*
28812883
* endptr is current (or recent) end of xlog; this is used to determine
28822884
* whether we want to recycle rather than delete no-longer-wanted log files.
28832885
*/
28842886
staticvoid
28852887
RemoveOldXlogFiles(XLogSegNosegno,XLogRecPtrendptr)
28862888
{
2887-
XLogSegNoendlogSegNo;
2888-
intmax_advance;
28892889
DIR*xldir;
28902890
structdirent*xlde;
28912891
charlastoff[MAXFNAMELEN];
2892-
charpath[MAXPGPATH];
2893-
2894-
#ifdefWIN32
2895-
charnewpath[MAXPGPATH];
2896-
#endif
2897-
structstatstatbuf;
2898-
2899-
/*
2900-
* Initialize info about where to try to recycle to. We allow recycling
2901-
* segments up to XLOGfileslop segments beyond the current XLOG location.
2902-
*/
2903-
XLByteToPrevSeg(endptr,endlogSegNo);
2904-
max_advance=XLOGfileslop;
29052892

29062893
xldir=AllocateDir(XLOGDIR);
29072894
if (xldir==NULL)
@@ -2922,6 +2909,11 @@ RemoveOldXlogFiles(XLogSegNo segno, XLogRecPtr endptr)
29222909

29232910
while ((xlde=ReadDir(xldir,XLOGDIR))!=NULL)
29242911
{
2912+
/* Ignore files that are not XLOG segments */
2913+
if (strlen(xlde->d_name)!=24||
2914+
strspn(xlde->d_name,"0123456789ABCDEF")!=24)
2915+
continue;
2916+
29252917
/*
29262918
* We ignore the timeline part of the XLOG segment identifiers in
29272919
* deciding whether a segment is still needed. This ensures that we
@@ -2933,92 +2925,110 @@ RemoveOldXlogFiles(XLogSegNo segno, XLogRecPtr endptr)
29332925
* We use the alphanumeric sorting property of the filenames to decide
29342926
* which ones are earlier than the lastoff segment.
29352927
*/
2936-
if (strlen(xlde->d_name)==24&&
2937-
strspn(xlde->d_name,"0123456789ABCDEF")==24&&
2938-
strcmp(xlde->d_name+8,lastoff+8) <=0)
2928+
if (strcmp(xlde->d_name+8,lastoff+8) <=0)
29392929
{
29402930
if (XLogArchiveCheckDone(xlde->d_name))
29412931
{
2942-
snprintf(path,MAXPGPATH,XLOGDIR"/%s",xlde->d_name);
2943-
29442932
/* Update the last removed location in shared memory first */
29452933
UpdateLastRemovedPtr(xlde->d_name);
29462934

2947-
/*
2948-
* Before deleting the file, see if it can be recycled as a
2949-
* future log segment. Only recycle normal files, pg_standby
2950-
* for example can create symbolic links pointing to a
2951-
* separate archive directory.
2952-
*/
2953-
if (lstat(path,&statbuf)==0&&S_ISREG(statbuf.st_mode)&&
2954-
InstallXLogFileSegment(&endlogSegNo,path,
2955-
true,&max_advance, true))
2956-
{
2957-
ereport(DEBUG2,
2958-
(errmsg("recycled transaction log file \"%s\"",
2959-
xlde->d_name)));
2960-
CheckpointStats.ckpt_segs_recycled++;
2961-
/* Needn't recheck that slot on future iterations */
2962-
if (max_advance>0)
2963-
{
2964-
endlogSegNo++;
2965-
max_advance--;
2966-
}
2967-
}
2968-
else
2969-
{
2970-
/* No need for any more future segments... */
2971-
intrc;
2935+
RemoveXlogFile(xlde->d_name,endptr);
2936+
}
2937+
}
2938+
}
29722939

2973-
ereport(DEBUG2,
2974-
(errmsg("removing transaction log file \"%s\"",
2975-
xlde->d_name)));
2940+
FreeDir(xldir);
2941+
}
29762942

2943+
/*
2944+
* Recycle or remove a log file that's no longer needed.
2945+
*
2946+
* endptr is current (or recent) end of xlog; this is used to determine
2947+
* whether we want to recycle rather than delete no-longer-wanted log files.
2948+
*/
2949+
staticvoid
2950+
RemoveXlogFile(constchar*segname,XLogRecPtrendptr)
2951+
{
2952+
charpath[MAXPGPATH];
29772953
#ifdefWIN32
2954+
charnewpath[MAXPGPATH];
2955+
#endif
2956+
structstatstatbuf;
2957+
XLogSegNoendlogSegNo;
2958+
intmax_advance;
29782959

2979-
/*
2980-
* On Windows, if another process (e.g another backend)
2981-
* holds the file open in FILE_SHARE_DELETE mode, unlink
2982-
* will succeed, but the file will still show up in
2983-
* directory listing until the last handle is closed. To
2984-
* avoid confusing the lingering deleted file for a live
2985-
* WAL file that needs to be archived, rename it before
2986-
* deleting it.
2987-
*
2988-
* If another process holds the file open without
2989-
* FILE_SHARE_DELETE flag, rename will fail. We'll try
2990-
* again at the next checkpoint.
2991-
*/
2992-
snprintf(newpath,MAXPGPATH,"%s.deleted",path);
2993-
if (rename(path,newpath)!=0)
2994-
{
2995-
ereport(LOG,
2996-
(errcode_for_file_access(),
2997-
errmsg("could not rename old transaction log file \"%s\": %m",
2998-
path)));
2999-
continue;
3000-
}
3001-
rc=unlink(newpath);
2960+
/*
2961+
* Initialize info about where to try to recycle to. We allow recycling
2962+
* segments up to XLOGfileslop segments beyond the current XLOG location.
2963+
*/
2964+
XLByteToPrevSeg(endptr,endlogSegNo);
2965+
max_advance=XLOGfileslop;
2966+
2967+
snprintf(path,MAXPGPATH,XLOGDIR"/%s",segname);
2968+
2969+
/*
2970+
* Before deleting the file, see if it can be recycled as a future log
2971+
* segment. Only recycle normal files, pg_standby for example can create
2972+
* symbolic links pointing to a separate archive directory.
2973+
*/
2974+
if (lstat(path,&statbuf)==0&&S_ISREG(statbuf.st_mode)&&
2975+
InstallXLogFileSegment(&endlogSegNo,path,
2976+
true,&max_advance, true))
2977+
{
2978+
ereport(DEBUG2,
2979+
(errmsg("recycled transaction log file \"%s\"",segname)));
2980+
CheckpointStats.ckpt_segs_recycled++;
2981+
/* Needn't recheck that slot on future iterations */
2982+
if (max_advance>0)
2983+
{
2984+
endlogSegNo++;
2985+
max_advance--;
2986+
}
2987+
}
2988+
else
2989+
{
2990+
/* No need for any more future segments... */
2991+
intrc;
2992+
2993+
ereport(DEBUG2,
2994+
(errmsg("removing transaction log file \"%s\"",segname)));
2995+
2996+
#ifdefWIN32
2997+
/*
2998+
* On Windows, if another process (e.g another backend) holds the file
2999+
* open in FILE_SHARE_DELETE mode, unlink will succeed, but the file
3000+
* will still show up in directory listing until the last handle is
3001+
* closed. To avoid confusing the lingering deleted file for a live
3002+
* WAL file that needs to be archived, rename it before deleting it.
3003+
*
3004+
* If another process holds the file open without FILE_SHARE_DELETE
3005+
* flag, rename will fail. We'll try again at the next checkpoint.
3006+
*/
3007+
snprintf(newpath,MAXPGPATH,"%s.deleted",path);
3008+
if (rename(path,newpath)!=0)
3009+
{
3010+
ereport(LOG,
3011+
(errcode_for_file_access(),
3012+
errmsg("could not rename old transaction log file \"%s\": %m",
3013+
path)));
3014+
return;
3015+
}
3016+
rc=unlink(newpath);
30023017
#else
3003-
rc=unlink(path);
3018+
rc=unlink(path);
30043019
#endif
3005-
if (rc!=0)
3006-
{
3007-
ereport(LOG,
3008-
(errcode_for_file_access(),
3009-
errmsg("could not remove old transaction log file \"%s\": %m",
3010-
path)));
3011-
continue;
3012-
}
3013-
CheckpointStats.ckpt_segs_removed++;
3014-
}
3015-
3016-
XLogArchiveCleanup(xlde->d_name);
3017-
}
3020+
if (rc!=0)
3021+
{
3022+
ereport(LOG,
3023+
(errcode_for_file_access(),
3024+
errmsg("could not remove old transaction log file \"%s\": %m",
3025+
path)));
3026+
return;
30183027
}
3028+
CheckpointStats.ckpt_segs_removed++;
30193029
}
30203030

3021-
FreeDir(xldir);
3031+
XLogArchiveCleanup(segname);
30223032
}
30233033

30243034
/*
@@ -4474,6 +4484,75 @@ exitArchiveRecovery(TimeLineID endTLI, XLogSegNo endLogSegNo)
44744484
(errmsg("archive recovery complete")));
44754485
}
44764486

4487+
/*
4488+
* Remove WAL files that are not part of the given timeline's history.
4489+
*
4490+
* This is called during recovery, whenever we switch to follow a new
4491+
* timeline, and at the end of recovery when we create a new timeline. We
4492+
* wouldn't otherwise care about extra WAL files lying in pg_xlog, but they
4493+
* can be pre-allocated or recycled WAL segments on the old timeline that we
4494+
* haven't used yet, and contain garbage. If we just leave them in pg_xlog,
4495+
* they will eventually be archived, and we can't let that happen. Files that
4496+
* belong to our timeline history are valid, because we have successfully
4497+
* replayed them, but from others we can't be sure.
4498+
*
4499+
* 'switchpoint' is the current point in WAL where we switch to new timeline,
4500+
* and 'newTLI' is the new timeline we switch to.
4501+
*/
4502+
staticvoid
4503+
RemoveNonParentXlogFiles(XLogRecPtrswitchpoint,TimeLineIDnewTLI)
4504+
{
4505+
DIR*xldir;
4506+
structdirent*xlde;
4507+
charswitchseg[MAXFNAMELEN];
4508+
XLogSegNoendLogSegNo;
4509+
4510+
XLByteToPrevSeg(switchpoint,endLogSegNo);
4511+
4512+
xldir=AllocateDir(XLOGDIR);
4513+
if (xldir==NULL)
4514+
ereport(ERROR,
4515+
(errcode_for_file_access(),
4516+
errmsg("could not open transaction log directory \"%s\": %m",
4517+
XLOGDIR)));
4518+
4519+
/*
4520+
* Construct a filename of the last segment to be kept.
4521+
*/
4522+
XLogFileName(switchseg,newTLI,endLogSegNo);
4523+
4524+
elog(DEBUG2,"attempting to remove WAL segments newer than log file %s",
4525+
switchseg);
4526+
4527+
while ((xlde=ReadDir(xldir,XLOGDIR))!=NULL)
4528+
{
4529+
/* Ignore files that are not XLOG segments */
4530+
if (strlen(xlde->d_name)!=24||
4531+
strspn(xlde->d_name,"0123456789ABCDEF")!=24)
4532+
continue;
4533+
4534+
/*
4535+
* Remove files that are on a timeline older than the new one we're
4536+
* switching to, but with a segment number >= the first segment on
4537+
* the new timeline.
4538+
*/
4539+
if (strncmp(xlde->d_name,switchseg,8)<0&&
4540+
strcmp(xlde->d_name+8,switchseg+8)>0)
4541+
{
4542+
/*
4543+
* If the file has already been marked as .ready, however, don't
4544+
* remove it yet. It should be OK to remove it - files that are
4545+
* not part of our timeline history are not required for recovery
4546+
* - but seems safer to let them be archived and removed later.
4547+
*/
4548+
if (!XLogArchiveIsReady(xlde->d_name))
4549+
RemoveXlogFile(xlde->d_name,switchpoint);
4550+
}
4551+
}
4552+
4553+
FreeDir(xldir);
4554+
}
4555+
44774556
/*
44784557
* For point-in-time recovery, this function decides whether we want to
44794558
* stop applying the XLOG at or after the current record.
@@ -5647,9 +5726,9 @@ StartupXLOG(void)
56475726
*/
56485727
if (record->xl_rmid==RM_XLOG_ID)
56495728
{
5729+
uint8info=record->xl_info& ~XLR_INFO_MASK;
56505730
TimeLineIDnewTLI=ThisTimeLineID;
56515731
TimeLineIDprevTLI=ThisTimeLineID;
5652-
uint8info=record->xl_info& ~XLR_INFO_MASK;
56535732

56545733
if (info==XLOG_CHECKPOINT_SHUTDOWN)
56555734
{
@@ -5717,12 +5796,21 @@ StartupXLOG(void)
57175796
/* Allow read-only connections if we're consistent now */
57185797
CheckRecoveryConsistency();
57195798

5720-
/*
5721-
* If this record was a timeline switch, wake up any
5722-
* walsenders to notice that we are on a new timeline.
5723-
*/
5724-
if (switchedTLI&&AllowCascadeReplication())
5725-
WalSndWakeup();
5799+
if (switchedTLI)
5800+
{
5801+
/*
5802+
* Before we go further on the new timeline, clean up any
5803+
* (possibly bogus) future WAL segments on the old one.
5804+
*/
5805+
RemoveNonParentXlogFiles(EndRecPtr,ThisTimeLineID);
5806+
5807+
/*
5808+
* Wake up any walsenders to notice that we are on a new
5809+
* timeline.
5810+
*/
5811+
if (AllowCascadeReplication())
5812+
WalSndWakeup();
5813+
}
57265814

57275815
/* Exit loop if we reached inclusive recovery target */
57285816
if (!recoveryContinue)
@@ -6059,6 +6147,12 @@ StartupXLOG(void)
60596147
true);
60606148
}
60616149

6150+
/*
6151+
* Clean up any (possibly bogus) future WAL segments on the old timeline.
6152+
*/
6153+
if (ArchiveRecoveryRequested)
6154+
RemoveNonParentXlogFiles(EndOfLog,ThisTimeLineID);
6155+
60626156
/*
60636157
* Preallocate additional log files, if wanted.
60646158
*/

‎src/backend/access/transam/xlogarchive.c

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -692,6 +692,25 @@ XLogArchiveIsBusy(const char *xlog)
692692
return true;
693693
}
694694

695+
/*
696+
* XLogArchiveIsReady
697+
*
698+
* Check to see if an XLOG segment file has an archive notification (.ready)
699+
* file.
700+
*/
701+
bool
702+
XLogArchiveIsReady(constchar*xlog)
703+
{
704+
chararchiveStatusPath[MAXPGPATH];
705+
structstatstat_buf;
706+
707+
StatusFilePath(archiveStatusPath,xlog,".ready");
708+
if (stat(archiveStatusPath,&stat_buf)==0)
709+
return true;
710+
711+
return false;
712+
}
713+
695714
/*
696715
* XLogArchiveCleanup
697716
*

‎src/include/access/xlog_internal.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,7 @@ extern void XLogArchiveNotifySeg(XLogSegNo segno);
282282
externvoidXLogArchiveForceDone(constchar*xlog);
283283
externboolXLogArchiveCheckDone(constchar*xlog);
284284
externboolXLogArchiveIsBusy(constchar*xlog);
285+
externboolXLogArchiveIsReady(constchar*xlog);
285286
externvoidXLogArchiveCleanup(constchar*xlog);
286287

287288
#endif/* XLOG_INTERNAL_H */

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp