Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitc6c3334

Browse files
committed
Prevent possibility of panics during shutdown checkpoint.
When the checkpointer writes the shutdown checkpoint, it checksafterwards whether any WAL has been written since it started andthrows a PANIC if so. At that point, only walsenders are stillactive, so one might think this could not happen, but walsenders canalso generate WAL, for instance in BASE_BACKUP and logical decodingrelated commands (e.g. via hint bits). So they can trigger this panicif such a command is run while the shutdown checkpoint is beingwritten.To fix this, divide the walsender shutdown into two phases. First,checkpointer, itself triggered by postmaster, sends aPROCSIG_WALSND_INIT_STOPPING signal to all walsenders. If the backendis idle or runs an SQL query this causes the backend to shutdown, iflogical replication is in progress all existing WAL records areprocessed followed by a shutdown. Otherwise this causes the walsenderto switch to the "stopping" state. In this state, the walsender willreject any further replication commands. The checkpointer begins theshutdown checkpoint once all walsenders are confirmed asstopping. When the shutdown checkpoint finishes, the postmaster sendsus SIGUSR2. This instructs walsender to send any outstanding WAL,including the shutdown checkpoint record, wait for it to be replicatedto the standby, and then exit.Author: Andres Freund, based on an earlier patch by Michael PaquierReported-By: Fujii Masao, Andres FreundReviewed-By: Michael PaquierDiscussion:https://postgr.es/m/20170602002912.tqlwn4gymzlxpvs2@alap3.anarazel.deBackpatch: 9.4, where logical decoding was introduced
1 parent47fd420 commitc6c3334

File tree

7 files changed

+186
-28
lines changed

7 files changed

+186
-28
lines changed

‎doc/src/sgml/monitoring.sgml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1690,6 +1690,11 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
16901690
<literal>backup</>: This WAL sender is sending a backup.
16911691
</para>
16921692
</listitem>
1693+
<listitem>
1694+
<para>
1695+
<literal>stopping</>: This WAL sender is stopping.
1696+
</para>
1697+
</listitem>
16931698
</itemizedlist>
16941699
</entry>
16951700
</row>

‎src/backend/access/transam/xlog.c

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8324,6 +8324,17 @@ ShutdownXLOG(int code, Datum arg)
83248324
ereport(IsPostmasterEnvironment ?LOG :NOTICE,
83258325
(errmsg("shutting down")));
83268326

8327+
/*
8328+
* Signal walsenders to move to stopping state.
8329+
*/
8330+
WalSndInitStopping();
8331+
8332+
/*
8333+
* Wait for WAL senders to be in stopping state. This prevents commands
8334+
* from writing new WAL.
8335+
*/
8336+
WalSndWaitStopping();
8337+
83278338
if (RecoveryInProgress())
83288339
CreateRestartPoint(CHECKPOINT_IS_SHUTDOWN |CHECKPOINT_IMMEDIATE);
83298340
else

‎src/backend/replication/walsender.c

Lines changed: 160 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,17 @@
2424
* are treated as not a crash but approximately normal termination;
2525
* the walsender will exit quickly without sending any more XLOG records.
2626
*
27-
* If the server is shut down, postmaster sends us SIGUSR2 after all
28-
* regular backends have exited and the shutdown checkpoint has been written.
29-
* This instructs walsender to send any outstanding WAL, including the
30-
* shutdown checkpoint record, wait for it to be replicated to the standby,
31-
* and then exit.
27+
* If the server is shut down, checkpointer sends us
28+
* PROCSIG_WALSND_INIT_STOPPING after all regular backends have exited. If
29+
* the backend is idle or runs an SQL query this causes the backend to
30+
* shutdown, if logical replication is in progress all existing WAL records
31+
* are processed followed by a shutdown. Otherwise this causes the walsender
32+
* to switch to the "stopping" state. In this state, the walsender will reject
33+
* any further replication commands. The checkpointer begins the shutdown
34+
* checkpoint once all walsenders are confirmed as stopping. When the shutdown
35+
* checkpoint finishes, the postmaster sends us SIGUSR2. This instructs
36+
* walsender to send any outstanding WAL, including the shutdown checkpoint
37+
* record, wait for it to be replicated to the standby, and then exit.
3238
*
3339
*
3440
* Portions Copyright (c) 2010-2017, PostgreSQL Global Development Group
@@ -177,13 +183,14 @@ static bool WalSndCaughtUp = false;
177183

178184
/* Flags set by signal handlers for later service in main loop */
179185
staticvolatilesig_atomic_tgot_SIGHUP= false;
180-
staticvolatilesig_atomic_twalsender_ready_to_stop= false;
186+
staticvolatilesig_atomic_tgot_SIGUSR2= false;
187+
staticvolatilesig_atomic_tgot_STOPPING= false;
181188

182189
/*
183-
* This is set while we are streaming. When not set, SIGUSR2 signal will be
184-
* handled like SIGTERM. When set, the main loop is responsible for checking
185-
*walsender_ready_to_stop and terminating when it's set (after streaming any
186-
* remaining WAL).
190+
* This is set while we are streaming. When not set
191+
*PROCSIG_WALSND_INIT_STOPPING signal will behandled like SIGTERM. When set,
192+
*the main loop is responsible for checking got_STOPPING and terminating when
193+
*it's set (after streaming anyremaining WAL).
187194
*/
188195
staticvolatilesig_atomic_treplication_active= false;
189196

@@ -300,7 +307,8 @@ WalSndErrorCleanup(void)
300307
ReplicationSlotCleanup();
301308

302309
replication_active= false;
303-
if (walsender_ready_to_stop)
310+
311+
if (got_STOPPING||got_SIGUSR2)
304312
proc_exit(0);
305313

306314
/* Revert back to startup state */
@@ -677,7 +685,7 @@ StartReplication(StartReplicationCmd *cmd)
677685
WalSndLoop(XLogSendPhysical);
678686

679687
replication_active= false;
680-
if (walsender_ready_to_stop)
688+
if (got_STOPPING)
681689
proc_exit(0);
682690
WalSndSetState(WALSNDSTATE_STARTUP);
683691

@@ -1055,7 +1063,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
10551063
{
10561064
ereport(LOG,
10571065
(errmsg("terminating walsender process after promotion")));
1058-
walsender_ready_to_stop= true;
1066+
got_STOPPING= true;
10591067
}
10601068

10611069
WalSndSetState(WALSNDSTATE_CATCHUP);
@@ -1106,7 +1114,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
11061114
ReplicationSlotRelease();
11071115

11081116
replication_active= false;
1109-
if (walsender_ready_to_stop)
1117+
if (got_STOPPING)
11101118
proc_exit(0);
11111119
WalSndSetState(WALSNDSTATE_STARTUP);
11121120

@@ -1311,6 +1319,14 @@ WalSndWaitForWal(XLogRecPtr loc)
13111319
/* Check for input from the client */
13121320
ProcessRepliesIfAny();
13131321

1322+
/*
1323+
* If we're shutting down, trigger pending WAL to be written out,
1324+
* otherwise we'd possibly end up waiting for WAL that never gets
1325+
* written, because walwriter has shut down already.
1326+
*/
1327+
if (got_STOPPING)
1328+
XLogBackgroundFlush();
1329+
13141330
/* Update our idea of the currently flushed position. */
13151331
if (!RecoveryInProgress())
13161332
RecentFlushPtr=GetFlushRecPtr();
@@ -1326,7 +1342,7 @@ WalSndWaitForWal(XLogRecPtr loc)
13261342
* RecentFlushPtr, so we can send all remaining data before shutting
13271343
* down.
13281344
*/
1329-
if (walsender_ready_to_stop)
1345+
if (got_STOPPING)
13301346
break;
13311347

13321348
/*
@@ -1400,6 +1416,22 @@ exec_replication_command(const char *cmd_string)
14001416
MemoryContextcmd_context;
14011417
MemoryContextold_context;
14021418

1419+
/*
1420+
* If WAL sender has been told that shutdown is getting close, switch its
1421+
* status accordingly to handle the next replication commands correctly.
1422+
*/
1423+
if (got_STOPPING)
1424+
WalSndSetState(WALSNDSTATE_STOPPING);
1425+
1426+
/*
1427+
* Throw error if in stopping mode. We need prevent commands that could
1428+
* generate WAL while the shutdown checkpoint is being written. To be
1429+
* safe, we just prohibit all new commands.
1430+
*/
1431+
if (MyWalSnd->state==WALSNDSTATE_STOPPING)
1432+
ereport(ERROR,
1433+
(errmsg("cannot execute new commands while WAL sender is in stopping mode")));
1434+
14031435
/*
14041436
* CREATE_REPLICATION_SLOT ... LOGICAL exports a snapshot until the next
14051437
* command arrives. Clean up the old stuff if there's anything.
@@ -2128,7 +2160,7 @@ WalSndLoop(WalSndSendDataCallback send_data)
21282160
* normal termination at shutdown, or a promotion, the walsender
21292161
* is not sure which.
21302162
*/
2131-
if (walsender_ready_to_stop)
2163+
if (got_SIGUSR2)
21322164
WalSndDone(send_data);
21332165
}
21342166

@@ -2443,6 +2475,10 @@ XLogSendPhysical(void)
24432475
XLogRecPtrendptr;
24442476
Sizenbytes;
24452477

2478+
/* If requested switch the WAL sender to the stopping state. */
2479+
if (got_STOPPING)
2480+
WalSndSetState(WALSNDSTATE_STOPPING);
2481+
24462482
if (streamingDoneSending)
24472483
{
24482484
WalSndCaughtUp= true;
@@ -2733,7 +2769,16 @@ XLogSendLogical(void)
27332769
* point, then we're caught up.
27342770
*/
27352771
if (logical_decoding_ctx->reader->EndRecPtr >=GetFlushRecPtr())
2772+
{
27362773
WalSndCaughtUp= true;
2774+
2775+
/*
2776+
* Have WalSndLoop() terminate the connection in an orderly
2777+
* manner, after writing out all the pending data.
2778+
*/
2779+
if (got_STOPPING)
2780+
got_SIGUSR2= true;
2781+
}
27372782
}
27382783

27392784
/* Update shared memory status */
@@ -2843,6 +2888,26 @@ WalSndRqstFileReload(void)
28432888
}
28442889
}
28452890

2891+
/*
2892+
* Handle PROCSIG_WALSND_INIT_STOPPING signal.
2893+
*/
2894+
void
2895+
HandleWalSndInitStopping(void)
2896+
{
2897+
Assert(am_walsender);
2898+
2899+
/*
2900+
* If replication has not yet started, die like with SIGTERM. If
2901+
* replication is active, only set a flag and wake up the main loop. It
2902+
* will send any outstanding WAL, wait for it to be replicated to the
2903+
* standby, and then exit gracefully.
2904+
*/
2905+
if (!replication_active)
2906+
kill(MyProcPid,SIGTERM);
2907+
else
2908+
got_STOPPING= true;
2909+
}
2910+
28462911
/* SIGHUP: set flag to re-read config file at next convenient time */
28472912
staticvoid
28482913
WalSndSigHupHandler(SIGNAL_ARGS)
@@ -2856,22 +2921,17 @@ WalSndSigHupHandler(SIGNAL_ARGS)
28562921
errno=save_errno;
28572922
}
28582923

2859-
/* SIGUSR2: set flag to do a last cycle and shut down afterwards */
2924+
/*
2925+
* SIGUSR2: set flag to do a last cycle and shut down afterwards. The WAL
2926+
* sender should already have been switched to WALSNDSTATE_STOPPING at
2927+
* this point.
2928+
*/
28602929
staticvoid
28612930
WalSndLastCycleHandler(SIGNAL_ARGS)
28622931
{
28632932
intsave_errno=errno;
28642933

2865-
/*
2866-
* If replication has not yet started, die like with SIGTERM. If
2867-
* replication is active, only set a flag and wake up the main loop. It
2868-
* will send any outstanding WAL, wait for it to be replicated to the
2869-
* standby, and then exit gracefully.
2870-
*/
2871-
if (!replication_active)
2872-
kill(MyProcPid,SIGTERM);
2873-
2874-
walsender_ready_to_stop= true;
2934+
got_SIGUSR2= true;
28752935
SetLatch(MyLatch);
28762936

28772937
errno=save_errno;
@@ -2969,6 +3029,77 @@ WalSndWakeup(void)
29693029
}
29703030
}
29713031

3032+
/*
3033+
* Signal all walsenders to move to stopping state.
3034+
*
3035+
* This will trigger walsenders to move to a state where no further WAL can be
3036+
* generated. See this file's header for details.
3037+
*/
3038+
void
3039+
WalSndInitStopping(void)
3040+
{
3041+
inti;
3042+
3043+
for (i=0;i<max_wal_senders;i++)
3044+
{
3045+
WalSnd*walsnd=&WalSndCtl->walsnds[i];
3046+
pid_tpid;
3047+
3048+
SpinLockAcquire(&walsnd->mutex);
3049+
pid=walsnd->pid;
3050+
SpinLockRelease(&walsnd->mutex);
3051+
3052+
if (pid==0)
3053+
continue;
3054+
3055+
SendProcSignal(pid,PROCSIG_WALSND_INIT_STOPPING,InvalidBackendId);
3056+
}
3057+
}
3058+
3059+
/*
3060+
* Wait that all the WAL senders have quit or reached the stopping state. This
3061+
* is used by the checkpointer to control when the shutdown checkpoint can
3062+
* safely be performed.
3063+
*/
3064+
void
3065+
WalSndWaitStopping(void)
3066+
{
3067+
for (;;)
3068+
{
3069+
inti;
3070+
boolall_stopped= true;
3071+
3072+
for (i=0;i<max_wal_senders;i++)
3073+
{
3074+
WalSndStatestate;
3075+
WalSnd*walsnd=&WalSndCtl->walsnds[i];
3076+
3077+
SpinLockAcquire(&walsnd->mutex);
3078+
3079+
if (walsnd->pid==0)
3080+
{
3081+
SpinLockRelease(&walsnd->mutex);
3082+
continue;
3083+
}
3084+
3085+
state=walsnd->state;
3086+
SpinLockRelease(&walsnd->mutex);
3087+
3088+
if (state!=WALSNDSTATE_STOPPING)
3089+
{
3090+
all_stopped= false;
3091+
break;
3092+
}
3093+
}
3094+
3095+
/* safe to leave if confirmation is done for all WAL senders */
3096+
if (all_stopped)
3097+
return;
3098+
3099+
pg_usleep(10000L);/* wait for 10 msec */
3100+
}
3101+
}
3102+
29723103
/* Set state for current walsender (only called in walsender) */
29733104
void
29743105
WalSndSetState(WalSndStatestate)
@@ -3002,6 +3133,8 @@ WalSndGetStateString(WalSndState state)
30023133
return"catchup";
30033134
caseWALSNDSTATE_STREAMING:
30043135
return"streaming";
3136+
caseWALSNDSTATE_STOPPING:
3137+
return"stopping";
30053138
}
30063139
return"UNKNOWN";
30073140
}

‎src/backend/storage/ipc/procsignal.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include"access/parallel.h"
2121
#include"commands/async.h"
2222
#include"miscadmin.h"
23+
#include"replication/walsender.h"
2324
#include"storage/latch.h"
2425
#include"storage/ipc.h"
2526
#include"storage/proc.h"
@@ -270,6 +271,9 @@ procsignal_sigusr1_handler(SIGNAL_ARGS)
270271
if (CheckProcSignal(PROCSIG_PARALLEL_MESSAGE))
271272
HandleParallelMessageInterrupt();
272273

274+
if (CheckProcSignal(PROCSIG_WALSND_INIT_STOPPING))
275+
HandleWalSndInitStopping();
276+
273277
if (CheckProcSignal(PROCSIG_RECOVERY_CONFLICT_DATABASE))
274278
RecoveryConflictInterrupt(PROCSIG_RECOVERY_CONFLICT_DATABASE);
275279

‎src/include/replication/walsender.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ extern void WalSndSignals(void);
4444
externSizeWalSndShmemSize(void);
4545
externvoidWalSndShmemInit(void);
4646
externvoidWalSndWakeup(void);
47+
externvoidWalSndInitStopping(void);
48+
externvoidWalSndWaitStopping(void);
49+
externvoidHandleWalSndInitStopping(void);
4750
externvoidWalSndRqstFileReload(void);
4851

4952
/*

‎src/include/replication/walsender_private.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ typedef enum WalSndState
2424
WALSNDSTATE_STARTUP=0,
2525
WALSNDSTATE_BACKUP,
2626
WALSNDSTATE_CATCHUP,
27-
WALSNDSTATE_STREAMING
27+
WALSNDSTATE_STREAMING,
28+
WALSNDSTATE_STOPPING
2829
}WalSndState;
2930

3031
/*

‎src/include/storage/procsignal.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ typedef enum
3232
PROCSIG_CATCHUP_INTERRUPT,/* sinval catchup interrupt */
3333
PROCSIG_NOTIFY_INTERRUPT,/* listen/notify interrupt */
3434
PROCSIG_PARALLEL_MESSAGE,/* message from cooperating parallel backend */
35+
PROCSIG_WALSND_INIT_STOPPING,/* ask walsenders to prepare for shutdown */
3536

3637
/* Recovery conflict reasons */
3738
PROCSIG_RECOVERY_CONFLICT_DATABASE,

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp