Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit90bcc7c

Browse files
alvherrehughcapetCyberDem0n
committed
Avoid deleting critical WAL segments during pg_rewind
Previously, in unlucky cases, it was possible for pg_rewind to removecertain WAL segments from the rewound demoted primary. In particularthis happens if those files have been marked for archival (i.e., their.ready files were created) but not yet archived; the newly promoted nodeno longer has such files because of them having been recycled, but theyare likely critical for recovery in the demoted node. If pg_rewindremoves them, recovery is not possible anymore.Fix this by maintaining a hash table of files in this situation in thescan that looks for a checkpoint, which the decide_file_actions phasecan consult so that it knows to preserve them.Backpatch to 14. The problem also exists in 13, but that branch was notblessed with commiteb00f1d, so this patch is difficult to applythere. Users of older releases will just have to continue to be extracareful when rewinding.Co-authored-by: Полина Бунгина (Polina Bungina) <bungina@gmail.com>Co-authored-by: Alexander Kukushkin <cyberdemn@gmail.com>Reviewed-by: Kyotaro Horiguchi <horikyota.ntt@gmail.com>Reviewed-by: Atsushi Torikoshi <torikoshia@oss.nttdata.com>Discussion:https://postgr.es/m/CAAtGL4AhzmBRsEsaDdz7065T+k+BscNadfTqP1NcPmsqwA5HBw@mail.gmail.com
1 parentd31bbfb commit90bcc7c

File tree

7 files changed

+169
-7
lines changed

7 files changed

+169
-7
lines changed

‎src/bin/pg_rewind/filemap.c

Lines changed: 77 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,14 +38,14 @@
3838
* Define a hash table which we can use to store information about the files
3939
* appearing in source and target systems.
4040
*/
41-
#defineSH_PREFIXfilehash
42-
#defineSH_ELEMENT_TYPEfile_entry_t
43-
#defineSH_KEY_TYPEconst char *
44-
#defineSH_KEYpath
41+
#defineSH_PREFIXfilehash
42+
#defineSH_ELEMENT_TYPEfile_entry_t
43+
#defineSH_KEY_TYPEconst char *
44+
#defineSH_KEYpath
4545
#defineSH_HASH_KEY(tb,key)hash_string(key)
4646
#defineSH_EQUAL(tb,a,b)(strcmp(a, b) == 0)
47-
#defineSH_SCOPEstatic inline
48-
#defineSH_RAW_ALLOCATORpg_malloc0
47+
#defineSH_SCOPEstatic inline
48+
#defineSH_RAW_ALLOCATORpg_malloc0
4949
#defineSH_DECLARE
5050
#defineSH_DEFINE
5151
#include"lib/simplehash.h"
@@ -60,7 +60,36 @@ static char *datasegpath(RelFileLocator rlocator, ForkNumber forknum,
6060

6161
staticfile_entry_t*insert_filehash_entry(constchar*path);
6262
staticfile_entry_t*lookup_filehash_entry(constchar*path);
63+
64+
/*
65+
* A separate hash table which tracks WAL files that must not be deleted.
66+
*/
67+
typedefstructkeepwal_entry
68+
{
69+
constchar*path;
70+
uint32status;
71+
}keepwal_entry;
72+
73+
#defineSH_PREFIXkeepwal
74+
#defineSH_ELEMENT_TYPEkeepwal_entry
75+
#defineSH_KEY_TYPEconst char *
76+
#defineSH_KEYpath
77+
#defineSH_HASH_KEY(tb,key)hash_string(key)
78+
#defineSH_EQUAL(tb,a,b)(strcmp(a, b) == 0)
79+
#defineSH_SCOPEstatic inline
80+
#defineSH_RAW_ALLOCATORpg_malloc0
81+
#defineSH_DECLARE
82+
#defineSH_DEFINE
83+
#include"lib/simplehash.h"
84+
85+
#defineKEEPWAL_INITIAL_SIZE1000
86+
87+
88+
statickeepwal_hash*keepwal=NULL;
89+
staticboolkeepwal_entry_exists(constchar*path);
90+
6391
staticintfinal_filemap_cmp(constvoid*a,constvoid*b);
92+
6493
staticboolcheck_file_excluded(constchar*path,boolis_source);
6594

6695
/*
@@ -206,6 +235,39 @@ lookup_filehash_entry(const char *path)
206235
returnfilehash_lookup(filehash,path);
207236
}
208237

238+
/*
239+
* Initialize a hash table to store WAL file names that must be kept.
240+
*/
241+
void
242+
keepwal_init(void)
243+
{
244+
/* An initial hash size out of thin air */
245+
keepwal=keepwal_create(KEEPWAL_INITIAL_SIZE,NULL);
246+
}
247+
248+
/* Mark the given file to prevent its removal */
249+
void
250+
keepwal_add_entry(constchar*path)
251+
{
252+
keepwal_entry*entry;
253+
boolfound;
254+
255+
/* Should only be called with keepwal initialized */
256+
Assert(keepwal!=NULL);
257+
258+
entry=keepwal_insert(keepwal,path,&found);
259+
260+
if (!found)
261+
entry->path=pg_strdup(path);
262+
}
263+
264+
/* Return true if file is marked as not to be removed, false otherwise */
265+
staticbool
266+
keepwal_entry_exists(constchar*path)
267+
{
268+
returnkeepwal_lookup(keepwal,path)!=NULL;
269+
}
270+
209271
/*
210272
* Callback for processing source file list.
211273
*
@@ -685,7 +747,15 @@ decide_file_action(file_entry_t *entry)
685747
}
686748
elseif (entry->target_exists&& !entry->source_exists)
687749
{
688-
/* File exists in target, but not source. Remove it. */
750+
/*
751+
* For files that exist in target but not in source, we check the
752+
* keepwal hash table; any files listed therein must not be removed.
753+
*/
754+
if (keepwal_entry_exists(path))
755+
{
756+
pg_log_debug("Not removing file \"%s\" because it is required for recovery",path);
757+
returnFILE_ACTION_NONE;
758+
}
689759
returnFILE_ACTION_REMOVE;
690760
}
691761
elseif (!entry->target_exists&& !entry->source_exists)

‎src/bin/pg_rewind/filemap.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,4 +110,7 @@ extern filemap_t *decide_file_actions(void);
110110
externvoidcalculate_totals(filemap_t*filemap);
111111
externvoidprint_filemap(filemap_t*filemap);
112112

113+
externvoidkeepwal_init(void);
114+
externvoidkeepwal_add_entry(constchar*path);
115+
113116
#endif/* FILEMAP_H */

‎src/bin/pg_rewind/meson.build

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ tests += {
4343
't/007_standby_source.pl',
4444
't/008_min_recovery_point.pl',
4545
't/009_growing_files.pl',
46+
't/010_keep_recycled_wals.pl',
4647
],
4748
},
4849
}

‎src/bin/pg_rewind/parsexlog.c

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,8 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
175175
XLogReaderState*xlogreader;
176176
char*errormsg;
177177
XLogPageReadPrivateprivate;
178+
XLogSegNocurrent_segno=0;
179+
TimeLineIDcurrent_tli=0;
178180

179181
/*
180182
* The given fork pointer points to the end of the last common record,
@@ -217,6 +219,25 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
217219
LSN_FORMAT_ARGS(searchptr));
218220
}
219221

222+
/* Detect if a new WAL file has been opened */
223+
if (xlogreader->seg.ws_tli!=current_tli||
224+
xlogreader->seg.ws_segno!=current_segno)
225+
{
226+
charxlogfname[MAXFNAMELEN];
227+
228+
snprintf(xlogfname,MAXFNAMELEN,XLOGDIR"/");
229+
230+
/* update curent values */
231+
current_tli=xlogreader->seg.ws_tli;
232+
current_segno=xlogreader->seg.ws_segno;
233+
234+
XLogFileName(xlogfname+sizeof(XLOGDIR),
235+
current_tli,current_segno,WalSegSz);
236+
237+
/* Track this filename as one to not remove */
238+
keepwal_add_entry(xlogfname);
239+
}
240+
220241
/*
221242
* Check if it is a checkpoint record. This checkpoint record needs to
222243
* be the latest checkpoint before WAL forked and not the checkpoint

‎src/bin/pg_rewind/pg_rewind.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -455,6 +455,9 @@ main(int argc, char **argv)
455455
exit(0);
456456
}
457457

458+
/* Initialize hashtable that tracks WAL files protected from removal */
459+
keepwal_init();
460+
458461
findLastCheckpoint(datadir_target,divergerec,lastcommontliIndex,
459462
&chkptrec,&chkpttli,&chkptredo,restore_command);
460463
pg_log_info("rewinding from last common checkpoint at %X/%X on timeline %u",
Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# Copyright (c) 2021-2024, PostgreSQL Global Development Group
2+
#
3+
# Test situation where a target data directory contains
4+
# WAL files that were already recycled by the new primary.
5+
#
6+
7+
use strict;
8+
use warningsFATAL=>'all';
9+
use PostgreSQL::Test::Utils;
10+
use Test::More;
11+
12+
use FindBin;
13+
use lib$FindBin::RealBin;
14+
use RewindTest;
15+
16+
RewindTest::setup_cluster();
17+
$node_primary->enable_archiving();
18+
RewindTest::start_primary();
19+
20+
RewindTest::create_standby();
21+
$node_standby->enable_restoring($node_primary, 0);
22+
$node_standby->reload();
23+
24+
RewindTest::primary_psql("CHECKPOINT");# last common checkpoint
25+
26+
# We use "perl -e 'exit(1)'" as an alternative to "false", because the latter
27+
# might not be available on Windows.
28+
my$false ="$^X -e 'exit(1)'";
29+
$node_primary->append_conf(
30+
'postgresql.conf',qq(
31+
archive_command = '$false'
32+
));
33+
$node_primary->reload();
34+
35+
# advance WAL on primary; this WAL segment will never make it to the archive
36+
RewindTest::primary_psql("CREATE TABLE t(a int)");
37+
RewindTest::primary_psql("INSERT INTO t VALUES(0)");
38+
RewindTest::primary_psql("SELECT pg_switch_wal()");
39+
40+
RewindTest::promote_standby;
41+
42+
# new primary loses diverging WAL segment
43+
RewindTest::standby_psql("INSERT INTO t values(0)");
44+
RewindTest::standby_psql("SELECT pg_switch_wal()");
45+
46+
$node_standby->stop();
47+
$node_primary->stop();
48+
49+
my ($stdout,$stderr) = run_command(
50+
[
51+
'pg_rewind','--debug',
52+
'--source-pgdata',$node_standby->data_dir,
53+
'--target-pgdata',$node_primary->data_dir,
54+
'--no-sync',
55+
]);
56+
57+
like(
58+
$stderr,
59+
qr/Not removing file .* because it is required for recovery/,
60+
"some WAL files were skipped");
61+
62+
done_testing();

‎src/tools/pgindent/typedefs.list

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3596,6 +3596,8 @@ json_manifest_version_callback
35963596
json_ofield_action
35973597
json_scalar_action
35983598
json_struct_action
3599+
keepwal_entry
3600+
keepwal_hash
35993601
keyEntryData
36003602
key_t
36013603
lclContext

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp