Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitc3de0f9

Browse files
committed
Fix failures with incorrect epoch handling for 2PC files at recovery
At the beginning of recovery, an orphaned two-phase file in an epochdifferent than the one defined in the checkpoint record could not beremoved based on the assumptions that AdjustToFullTransactionId() relieson, assuming that all files would be either from the current epoch orfrom the previous epoch.If the checkpoint epoch was 0 while the 2PC file was orphaned and in thefuture, AdjustToFullTransactionId() would underflow the epoch used tobuild the 2PC file path. In non-assert builds, this would create aWARNING message referring to a 2PC file with an epoch of "FFFFFFFF" (orUINT32_MAX), as an effect of the underflow calculation, leaving theorphaned file around.Some tests are added with dummy 2PC files in the past and the future,checking that these are properly removed.Issue introduced by5a1dfde, that has switched two-phase statefiles to use FullTransactionIds.Reported-by: Vitaly DavydovAuthor: Michael PaquierReviewed-by: Vitaly DavydovDiscussion:https://postgr.es/m/13b5b6-676c3080-4d-531db900@47931709Backpatch-through: 17
1 parent03c46e1 commitc3de0f9

File tree

2 files changed

+150
-51
lines changed

2 files changed

+150
-51
lines changed

‎src/backend/access/transam/twophase.c‎

Lines changed: 116 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -221,13 +221,13 @@ static void ProcessRecords(char *bufptr, TransactionId xid,
221221
staticvoidRemoveGXact(GlobalTransactiongxact);
222222

223223
staticvoidXlogReadTwoPhaseData(XLogRecPtrlsn,char**buf,int*len);
224-
staticchar*ProcessTwoPhaseBuffer(TransactionIdxid,
224+
staticchar*ProcessTwoPhaseBuffer(FullTransactionIdxid,
225225
XLogRecPtrprepare_start_lsn,
226226
boolfromdisk,boolsetParent,boolsetNextXid);
227227
staticvoidMarkAsPreparingGuts(GlobalTransactiongxact,TransactionIdxid,
228228
constchar*gid,TimestampTzprepared_at,Oidowner,
229229
Oiddatabaseid);
230-
staticvoidRemoveTwoPhaseFile(TransactionIdxid,boolgiveWarning);
230+
staticvoidRemoveTwoPhaseFile(FullTransactionIdfxid,boolgiveWarning);
231231
staticvoidRecreateTwoPhaseFile(TransactionIdxid,void*content,intlen);
232232

233233
/*
@@ -927,41 +927,26 @@ TwoPhaseGetDummyProc(TransactionId xid, bool lock_held)
927927
/************************************************************************/
928928

929929
/*
930-
* Compute the FullTransactionId for the given TransactionId.
931-
*
932-
* The wrap logic is safe here because the span of active xids cannot exceed one
933-
* epoch at any given time.
930+
* Compute FullTransactionId for the given TransactionId, using the current
931+
* epoch.
934932
*/
935933
staticinlineFullTransactionId
936-
AdjustToFullTransactionId(TransactionIdxid)
934+
FullTransactionIdFromCurrentEpoch(TransactionIdxid)
937935
{
936+
FullTransactionIdfxid;
938937
FullTransactionIdnextFullXid;
939-
TransactionIdnextXid;
940938
uint32epoch;
941939

942-
Assert(TransactionIdIsValid(xid));
943-
944-
LWLockAcquire(XidGenLock,LW_SHARED);
945-
nextFullXid=TransamVariables->nextXid;
946-
LWLockRelease(XidGenLock);
947-
948-
nextXid=XidFromFullTransactionId(nextFullXid);
940+
nextFullXid=ReadNextFullTransactionId();
949941
epoch=EpochFromFullTransactionId(nextFullXid);
950-
if (unlikely(xid>nextXid))
951-
{
952-
/* Wraparound occurred, must be from a prev epoch. */
953-
Assert(epoch>0);
954-
epoch--;
955-
}
956942

957-
returnFullTransactionIdFromEpochAndXid(epoch,xid);
943+
fxid=FullTransactionIdFromEpochAndXid(epoch,xid);
944+
returnfxid;
958945
}
959946

960947
staticinlineint
961-
TwoPhaseFilePath(char*path,TransactionIdxid)
948+
TwoPhaseFilePath(char*path,FullTransactionIdfxid)
962949
{
963-
FullTransactionIdfxid=AdjustToFullTransactionId(xid);
964-
965950
returnsnprintf(path,MAXPGPATH,TWOPHASE_DIR"/%08X%08X",
966951
EpochFromFullTransactionId(fxid),
967952
XidFromFullTransactionId(fxid));
@@ -1297,7 +1282,8 @@ RegisterTwoPhaseRecord(TwoPhaseRmgrId rmid, uint16 info,
12971282
* If it looks OK (has a valid magic number and CRC), return the palloc'd
12981283
* contents of the file, issuing an error when finding corrupted data. If
12991284
* missing_ok is true, which indicates that missing files can be safely
1300-
* ignored, then return NULL. This state can be reached when doing recovery.
1285+
* ignored, then return NULL. This state can be reached when doing recovery
1286+
* after discarding two-phase files from other epochs.
13011287
*/
13021288
staticchar*
13031289
ReadTwoPhaseFile(TransactionIdxid,boolmissing_ok)
@@ -1311,8 +1297,10 @@ ReadTwoPhaseFile(TransactionId xid, bool missing_ok)
13111297
pg_crc32ccalc_crc,
13121298
file_crc;
13131299
intr;
1300+
FullTransactionIdfxid;
13141301

1315-
TwoPhaseFilePath(path,xid);
1302+
fxid=FullTransactionIdFromCurrentEpoch(xid);
1303+
TwoPhaseFilePath(path,fxid);
13161304

13171305
fd=OpenTransientFile(path,O_RDONLY |PG_BINARY);
13181306
if (fd<0)
@@ -1677,10 +1665,16 @@ FinishPreparedTransaction(const char *gid, bool isCommit)
16771665
AtEOXact_PgStat(isCommit, false);
16781666

16791667
/*
1680-
* And now we can clean up any files we may have left.
1668+
* And now we can clean up any files we may have left. These should be
1669+
* from the current epoch.
16811670
*/
16821671
if (ondisk)
1683-
RemoveTwoPhaseFile(xid, true);
1672+
{
1673+
FullTransactionIdfxid;
1674+
1675+
fxid=FullTransactionIdFromCurrentEpoch(xid);
1676+
RemoveTwoPhaseFile(fxid, true);
1677+
}
16841678

16851679
MyLockedGxact=NULL;
16861680

@@ -1719,13 +1713,17 @@ ProcessRecords(char *bufptr, TransactionId xid,
17191713
*
17201714
* If giveWarning is false, do not complain about file-not-present;
17211715
* this is an expected case during WAL replay.
1716+
*
1717+
* This routine is used at early stages at recovery where future and
1718+
* past orphaned files are checked, hence the FullTransactionId to build
1719+
* a complete file name fit for the removal.
17221720
*/
17231721
staticvoid
1724-
RemoveTwoPhaseFile(TransactionIdxid,boolgiveWarning)
1722+
RemoveTwoPhaseFile(FullTransactionIdfxid,boolgiveWarning)
17251723
{
17261724
charpath[MAXPGPATH];
17271725

1728-
TwoPhaseFilePath(path,xid);
1726+
TwoPhaseFilePath(path,fxid);
17291727
if (unlink(path))
17301728
if (errno!=ENOENT||giveWarning)
17311729
ereport(WARNING,
@@ -1745,13 +1743,16 @@ RecreateTwoPhaseFile(TransactionId xid, void *content, int len)
17451743
charpath[MAXPGPATH];
17461744
pg_crc32cstatefile_crc;
17471745
intfd;
1746+
FullTransactionIdfxid;
17481747

17491748
/* Recompute CRC */
17501749
INIT_CRC32C(statefile_crc);
17511750
COMP_CRC32C(statefile_crc,content,len);
17521751
FIN_CRC32C(statefile_crc);
17531752

1754-
TwoPhaseFilePath(path,xid);
1753+
/* Use current epoch */
1754+
fxid=FullTransactionIdFromCurrentEpoch(xid);
1755+
TwoPhaseFilePath(path,fxid);
17551756

17561757
fd=OpenTransientFile(path,
17571758
O_CREAT |O_TRUNC |O_WRONLY |PG_BINARY);
@@ -1899,7 +1900,9 @@ CheckPointTwoPhase(XLogRecPtr redo_horizon)
18991900
* Scan pg_twophase and fill TwoPhaseState depending on the on-disk data.
19001901
* This is called once at the beginning of recovery, saving any extra
19011902
* lookups in the future. Two-phase files that are newer than the
1902-
* minimum XID horizon are discarded on the way.
1903+
* minimum XID horizon are discarded on the way. Two-phase files with
1904+
* an epoch older or newer than the current checkpoint's record epoch
1905+
* are also discarded.
19031906
*/
19041907
void
19051908
restoreTwoPhaseData(void)
@@ -1914,14 +1917,11 @@ restoreTwoPhaseData(void)
19141917
if (strlen(clde->d_name)==16&&
19151918
strspn(clde->d_name,"0123456789ABCDEF")==16)
19161919
{
1917-
TransactionIdxid;
19181920
FullTransactionIdfxid;
19191921
char*buf;
19201922

19211923
fxid=FullTransactionIdFromU64(strtou64(clde->d_name,NULL,16));
1922-
xid=XidFromFullTransactionId(fxid);
1923-
1924-
buf=ProcessTwoPhaseBuffer(xid,InvalidXLogRecPtr,
1924+
buf=ProcessTwoPhaseBuffer(fxid,InvalidXLogRecPtr,
19251925
true, false, false);
19261926
if (buf==NULL)
19271927
continue;
@@ -1972,6 +1972,7 @@ PrescanPreparedTransactions(TransactionId **xids_p, int *nxids_p)
19721972
TransactionIdorigNextXid=XidFromFullTransactionId(nextXid);
19731973
TransactionIdresult=origNextXid;
19741974
TransactionId*xids=NULL;
1975+
uint32epoch=EpochFromFullTransactionId(nextXid);
19751976
intnxids=0;
19761977
intallocsize=0;
19771978
inti;
@@ -1980,14 +1981,20 @@ PrescanPreparedTransactions(TransactionId **xids_p, int *nxids_p)
19801981
for (i=0;i<TwoPhaseState->numPrepXacts;i++)
19811982
{
19821983
TransactionIdxid;
1984+
FullTransactionIdfxid;
19831985
char*buf;
19841986
GlobalTransactiongxact=TwoPhaseState->prepXacts[i];
19851987

19861988
Assert(gxact->inredo);
19871989

19881990
xid=gxact->xid;
19891991

1990-
buf=ProcessTwoPhaseBuffer(xid,
1992+
/*
1993+
* All two-phase files with past and future epoch in pg_twophase are
1994+
* gone at this point, so we're OK to rely on only the current epoch.
1995+
*/
1996+
fxid=FullTransactionIdFromEpochAndXid(epoch,xid);
1997+
buf=ProcessTwoPhaseBuffer(fxid,
19911998
gxact->prepare_start_lsn,
19921999
gxact->ondisk, false, true);
19932000

@@ -2049,19 +2056,31 @@ void
20492056
StandbyRecoverPreparedTransactions(void)
20502057
{
20512058
inti;
2059+
uint32epoch;
2060+
FullTransactionIdnextFullXid;
2061+
2062+
/* get current epoch */
2063+
nextFullXid=ReadNextFullTransactionId();
2064+
epoch=EpochFromFullTransactionId(nextFullXid);
20522065

20532066
LWLockAcquire(TwoPhaseStateLock,LW_EXCLUSIVE);
20542067
for (i=0;i<TwoPhaseState->numPrepXacts;i++)
20552068
{
20562069
TransactionIdxid;
2070+
FullTransactionIdfxid;
20572071
char*buf;
20582072
GlobalTransactiongxact=TwoPhaseState->prepXacts[i];
20592073

20602074
Assert(gxact->inredo);
20612075

20622076
xid=gxact->xid;
20632077

2064-
buf=ProcessTwoPhaseBuffer(xid,
2078+
/*
2079+
* At this stage, we're OK to work with the current epoch as all past
2080+
* and future files have been already discarded.
2081+
*/
2082+
fxid=FullTransactionIdFromEpochAndXid(epoch,xid);
2083+
buf=ProcessTwoPhaseBuffer(fxid,
20652084
gxact->prepare_start_lsn,
20662085
gxact->ondisk, true, false);
20672086
if (buf!=NULL)
@@ -2090,18 +2109,29 @@ void
20902109
RecoverPreparedTransactions(void)
20912110
{
20922111
inti;
2112+
uint32epoch;
2113+
FullTransactionIdnextFullXid;
2114+
2115+
/* get current epoch */
2116+
nextFullXid=ReadNextFullTransactionId();
2117+
epoch=EpochFromFullTransactionId(nextFullXid);
20932118

20942119
LWLockAcquire(TwoPhaseStateLock,LW_EXCLUSIVE);
20952120
for (i=0;i<TwoPhaseState->numPrepXacts;i++)
20962121
{
20972122
TransactionIdxid;
2123+
FullTransactionIdfxid;
20982124
char*buf;
20992125
GlobalTransactiongxact=TwoPhaseState->prepXacts[i];
21002126
char*bufptr;
21012127
TwoPhaseFileHeader*hdr;
21022128
TransactionId*subxids;
21032129
constchar*gid;
21042130

2131+
/*
2132+
* At this stage, we're OK to work with the current epoch as all past
2133+
* and future files have been already discarded.
2134+
*/
21052135
xid=gxact->xid;
21062136

21072137
/*
@@ -2113,7 +2143,8 @@ RecoverPreparedTransactions(void)
21132143
* SubTransSetParent has been set before, if the prepared transaction
21142144
* generated xid assignment records.
21152145
*/
2116-
buf=ProcessTwoPhaseBuffer(xid,
2146+
fxid=FullTransactionIdFromEpochAndXid(epoch,xid);
2147+
buf=ProcessTwoPhaseBuffer(fxid,
21172148
gxact->prepare_start_lsn,
21182149
gxact->ondisk, true, false);
21192150
if (buf==NULL)
@@ -2181,7 +2212,7 @@ RecoverPreparedTransactions(void)
21812212
/*
21822213
* ProcessTwoPhaseBuffer
21832214
*
2184-
* Given atransaction id, read it either from disk or read it directly
2215+
* Given aFullTransactionId, read it either from disk or read it directly
21852216
* via shmem xlog record pointer using the provided "prepare_start_lsn".
21862217
*
21872218
* If setParent is true, set up subtransaction parent linkages.
@@ -2190,32 +2221,35 @@ RecoverPreparedTransactions(void)
21902221
* value scanned.
21912222
*/
21922223
staticchar*
2193-
ProcessTwoPhaseBuffer(TransactionIdxid,
2224+
ProcessTwoPhaseBuffer(FullTransactionIdfxid,
21942225
XLogRecPtrprepare_start_lsn,
21952226
boolfromdisk,
21962227
boolsetParent,boolsetNextXid)
21972228
{
21982229
FullTransactionIdnextXid=TransamVariables->nextXid;
2199-
TransactionIdorigNextXid=XidFromFullTransactionId(nextXid);
22002230
TransactionId*subxids;
22012231
char*buf;
22022232
TwoPhaseFileHeader*hdr;
22032233
inti;
2234+
TransactionIdxid=XidFromFullTransactionId(fxid);
22042235

22052236
Assert(LWLockHeldByMeInMode(TwoPhaseStateLock,LW_EXCLUSIVE));
22062237

22072238
if (!fromdisk)
22082239
Assert(prepare_start_lsn!=InvalidXLogRecPtr);
22092240

2210-
/* Reject XID if too new */
2211-
if (TransactionIdFollowsOrEquals(xid,origNextXid))
2241+
/*
2242+
* Reject full XID if too new. Note that this discards files from future
2243+
* epochs.
2244+
*/
2245+
if (FullTransactionIdFollowsOrEquals(fxid,nextXid))
22122246
{
22132247
if (fromdisk)
22142248
{
22152249
ereport(WARNING,
2216-
(errmsg("removing future two-phase state file for transaction %u",
2217-
xid)));
2218-
RemoveTwoPhaseFile(xid, true);
2250+
(errmsg("removing future two-phase state fileof epoch %ufor transaction %u",
2251+
EpochFromFullTransactionId(fxid),xid)));
2252+
RemoveTwoPhaseFile(fxid, true);
22192253
}
22202254
else
22212255
{
@@ -2227,6 +2261,26 @@ ProcessTwoPhaseBuffer(TransactionId xid,
22272261
returnNULL;
22282262
}
22292263

2264+
/* Discard files from past epochs */
2265+
if (EpochFromFullTransactionId(fxid)<EpochFromFullTransactionId(nextXid))
2266+
{
2267+
if (fromdisk)
2268+
{
2269+
ereport(WARNING,
2270+
(errmsg("removing past two-phase state file of epoch %u for transaction %u",
2271+
EpochFromFullTransactionId(fxid),xid)));
2272+
RemoveTwoPhaseFile(fxid, true);
2273+
}
2274+
else
2275+
{
2276+
ereport(WARNING,
2277+
(errmsg("removing past two-phase state from memory for transaction %u",
2278+
xid)));
2279+
PrepareRedoRemove(xid, true);
2280+
}
2281+
returnNULL;
2282+
}
2283+
22302284
/* Already processed? */
22312285
if (TransactionIdDidCommit(xid)||TransactionIdDidAbort(xid))
22322286
{
@@ -2235,7 +2289,7 @@ ProcessTwoPhaseBuffer(TransactionId xid,
22352289
ereport(WARNING,
22362290
(errmsg("removing stale two-phase state file for transaction %u",
22372291
xid)));
2238-
RemoveTwoPhaseFile(xid, true);
2292+
RemoveTwoPhaseFile(fxid, true);
22392293
}
22402294
else
22412295
{
@@ -2521,8 +2575,11 @@ PrepareRedoAdd(char *buf, XLogRecPtr start_lsn,
25212575
if (!XLogRecPtrIsInvalid(start_lsn))
25222576
{
25232577
charpath[MAXPGPATH];
2578+
FullTransactionIdfxid;
25242579

2525-
TwoPhaseFilePath(path,hdr->xid);
2580+
/* Use current epoch */
2581+
fxid=FullTransactionIdFromCurrentEpoch(hdr->xid);
2582+
TwoPhaseFilePath(path,fxid);
25262583

25272584
if (access(path,F_OK)==0)
25282585
{
@@ -2617,7 +2674,15 @@ PrepareRedoRemove(TransactionId xid, bool giveWarning)
26172674
*/
26182675
elog(DEBUG2,"removing 2PC data for transaction %u",xid);
26192676
if (gxact->ondisk)
2620-
RemoveTwoPhaseFile(xid,giveWarning);
2677+
{
2678+
FullTransactionIdfxid;
2679+
2680+
/*
2681+
* We should deal with a file at the current epoch here.
2682+
*/
2683+
fxid=FullTransactionIdFromCurrentEpoch(xid);
2684+
RemoveTwoPhaseFile(fxid,giveWarning);
2685+
}
26212686
RemoveGXact(gxact);
26222687
}
26232688

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp