Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitda2a718

Browse files
committed
Add prefix checks in exclude lists for pg_rewind, pg_checksums and base backups
An instance of PostgreSQL crashing with a bad timing could leave behindtemporary pg_internal.init files, potentially causing failures whenverifying checksums. As the same exclusion lists are used betweenpg_rewind, pg_checksums and basebackup.c, all those tools are extendedwith prefix checks to keep everything in sync, with dedicated checksadded for pg_internal.init.Backpatch down to 11, where pg_checksums (pg_verify_checksums in 11) andchecksum verification for base backups have been introduced.Reported-by: Michael BanckAuthor: Michael PaquierReviewed-by: Kyotaro Horiguchi, David SteeleDiscussion:https://postgr.es/m/62031974fd8e941dd8351fbc8c7eff60d59c5338.camel@credativ.deBackpatch-through: 11
1 parent2eadd00 commitda2a718

File tree

4 files changed

+113
-52
lines changed

4 files changed

+113
-52
lines changed

‎src/backend/replication/basebackup.c‎

Lines changed: 50 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,18 @@ static int64 total_checksum_failures;
123123
/* Do not verify checksums. */
124124
staticboolnoverify_checksums= false;
125125

126+
/*
127+
* Definition of one element part of an exclusion list, used for paths part
128+
* of checksum validation or base backups. "name" is the name of the file
129+
* or path to check for exclusion. If "match_prefix" is true, any items
130+
* matching the name as prefix are excluded.
131+
*/
132+
structexclude_list_item
133+
{
134+
constchar*name;
135+
boolmatch_prefix;
136+
};
137+
126138
/*
127139
* The contents of these directories are removed or recreated during server
128140
* start so they are not included in backups. The directories themselves are
@@ -172,31 +184,34 @@ static const char *excludeDirContents[] =
172184
/*
173185
* List of files excluded from backups.
174186
*/
175-
staticconstchar*excludeFiles[]=
187+
staticconststructexclude_list_itemexcludeFiles[]=
176188
{
177189
/* Skip auto conf temporary file. */
178-
PG_AUTOCONF_FILENAME".tmp",
190+
{PG_AUTOCONF_FILENAME".tmp", false},
179191

180192
/* Skip current log file temporary file */
181-
LOG_METAINFO_DATAFILE_TMP,
193+
{LOG_METAINFO_DATAFILE_TMP, false},
182194

183-
/* Skip relation cache because it is rebuilt on startup */
184-
RELCACHE_INIT_FILENAME,
195+
/*
196+
* Skip relation cache because it is rebuilt on startup. This includes
197+
* temporary files.
198+
*/
199+
{RELCACHE_INIT_FILENAME, true},
185200

186201
/*
187202
* If there's a backup_label or tablespace_map file, it belongs to a
188203
* backup started by the user with pg_start_backup(). It is *not* correct
189204
* for this backup. Our backup_label/tablespace_map is injected into the
190205
* tar separately.
191206
*/
192-
BACKUP_LABEL_FILE,
193-
TABLESPACE_MAP,
207+
{BACKUP_LABEL_FILE, false},
208+
{TABLESPACE_MAP, false},
194209

195-
"postmaster.pid",
196-
"postmaster.opts",
210+
{"postmaster.pid", false},
211+
{"postmaster.opts", false},
197212

198213
/* end of list */
199-
NULL
214+
{NULL, false}
200215
};
201216

202217
/*
@@ -205,16 +220,15 @@ static const char *excludeFiles[] =
205220
* Note: this list should be kept in sync with what pg_verify_checksums.c
206221
* includes.
207222
*/
208-
staticconstchar*noChecksumFiles[]= {
209-
"pg_control",
210-
"pg_filenode.map",
211-
"pg_internal.init",
212-
"PG_VERSION",
223+
staticconststructexclude_list_itemnoChecksumFiles[]= {
224+
{"pg_control", false},
225+
{"pg_filenode.map", false},
226+
{"pg_internal.init", true},
227+
{"PG_VERSION", false},
213228
#ifdefEXEC_BACKEND
214-
"config_exec_params",
215-
"config_exec_params.new",
229+
{"config_exec_params", true},
216230
#endif
217-
NULL,
231+
{NULL, false}
218232
};
219233

220234

@@ -1107,9 +1121,13 @@ sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces,
11071121

11081122
/* Scan for files that should be excluded */
11091123
excludeFound= false;
1110-
for (excludeIdx=0;excludeFiles[excludeIdx]!=NULL;excludeIdx++)
1124+
for (excludeIdx=0;excludeFiles[excludeIdx].name!=NULL;excludeIdx++)
11111125
{
1112-
if (strcmp(de->d_name,excludeFiles[excludeIdx])==0)
1126+
intcmplen=strlen(excludeFiles[excludeIdx].name);
1127+
1128+
if (!excludeFiles[excludeIdx].match_prefix)
1129+
cmplen++;
1130+
if (strncmp(de->d_name,excludeFiles[excludeIdx].name,cmplen)==0)
11131131
{
11141132
elog(DEBUG1,"file \"%s\" excluded from backup",de->d_name);
11151133
excludeFound= true;
@@ -1342,17 +1360,24 @@ sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces,
13421360
staticbool
13431361
is_checksummed_file(constchar*fullpath,constchar*filename)
13441362
{
1345-
constchar**f;
1346-
13471363
/* Check that the file is in a tablespace */
13481364
if (strncmp(fullpath,"./global/",9)==0||
13491365
strncmp(fullpath,"./base/",7)==0||
13501366
strncmp(fullpath,"/",1)==0)
13511367
{
1352-
/* Compare file against noChecksumFiles skiplist */
1353-
for (f=noChecksumFiles;*f;f++)
1354-
if (strcmp(*f,filename)==0)
1368+
intexcludeIdx;
1369+
1370+
/* Compare file against noChecksumFiles skip list */
1371+
for (excludeIdx=0;noChecksumFiles[excludeIdx].name!=NULL;excludeIdx++)
1372+
{
1373+
intcmplen=strlen(noChecksumFiles[excludeIdx].name);
1374+
1375+
if (!noChecksumFiles[excludeIdx].match_prefix)
1376+
cmplen++;
1377+
if (strncmp(filename,noChecksumFiles[excludeIdx].name,
1378+
cmplen)==0)
13551379
return false;
1380+
}
13561381

13571382
return true;
13581383
}

‎src/bin/pg_basebackup/t/010_pg_basebackup.pl‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
use File::Pathqw(rmtree);
77
use PostgresNode;
88
use TestLib;
9-
use Test::Moretests=>106;
9+
use Test::Moretests=>107;
1010

1111
program_help_ok('pg_basebackup');
1212
program_version_ok('pg_basebackup');
@@ -65,8 +65,8 @@
6565

6666
# Write some files to test that they are not copied.
6767
foreachmy$filename (
68-
qw(backup_label tablespace_map postgresql.auto.conf.tmp current_logfiles.tmp)
69-
)
68+
qw(backup_label tablespace_map postgresql.auto.conf.tmp
69+
current_logfiles.tmp global/pg_internal.init.123))
7070
{
7171
openmy$file,'>>',"$pgdata/$filename";
7272
print$file"DONOTCOPY";
@@ -135,7 +135,7 @@
135135
# These files should not be copied.
136136
foreachmy$filename (
137137
qw(postgresql.auto.conf.tmp postmaster.opts postmaster.pid tablespace_map current_logfiles.tmp
138-
global/pg_internal.init))
138+
global/pg_internal.init global/pg_internal.init.123))
139139
{
140140
ok(!-f"$tempdir/backup/$filename","$filename not copied");
141141
}

‎src/bin/pg_rewind/filemap.c‎

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,18 @@ static intfinal_filemap_cmp(const void *a, const void *b);
3232
staticvoidfilemap_list_to_array(filemap_t*map);
3333
staticboolcheck_file_excluded(constchar*path,boolis_source);
3434

35+
/*
36+
* Definition of one element part of an exclusion list, used to exclude
37+
* contents when rewinding. "name" is the name of the file or path to
38+
* check for exclusion. If "match_prefix" is true, any items matching
39+
* the name as prefix are excluded.
40+
*/
41+
structexclude_list_item
42+
{
43+
constchar*name;
44+
boolmatch_prefix;
45+
};
46+
3547
/*
3648
* The contents of these directories are removed or recreated during server
3749
* start so they are not included in data processed by pg_rewind.
@@ -80,32 +92,34 @@ static const char *excludeDirContents[] =
8092
};
8193

8294
/*
83-
* List of files excluded from filemap processing.
95+
* List of files excluded from filemap processing. Files are excluded
96+
* if their prefix match.
8497
*/
85-
staticconstchar*excludeFiles[]=
98+
staticconststructexclude_list_itemexcludeFiles[]=
8699
{
87100
/* Skip auto conf temporary file. */
88-
"postgresql.auto.conf.tmp",/* defined as PG_AUTOCONF_FILENAME */
101+
{"postgresql.auto.conf.tmp",false},/* defined as PG_AUTOCONF_FILENAME */
89102

90103
/* Skip current log file temporary file */
91-
"current_logfiles.tmp",/* defined as LOG_METAINFO_DATAFILE_TMP */
104+
{"current_logfiles.tmp", false},/* defined as
105+
* LOG_METAINFO_DATAFILE_TMP */
92106

93107
/* Skip relation cache because it is rebuilt on startup */
94-
"pg_internal.init",/* defined as RELCACHE_INIT_FILENAME */
108+
{"pg_internal.init", true},/* defined as RELCACHE_INIT_FILENAME */
95109

96110
/*
97111
* If there's a backup_label or tablespace_map file, it belongs to a
98112
* backup started by the user with pg_start_backup(). It is *not* correct
99113
* for this backup. Our backup_label is written later on separately.
100114
*/
101-
"backup_label",/* defined as BACKUP_LABEL_FILE */
102-
"tablespace_map",/* defined as TABLESPACE_MAP */
115+
{"backup_label", false},/* defined as BACKUP_LABEL_FILE */
116+
{"tablespace_map", false},/* defined as TABLESPACE_MAP */
103117

104-
"postmaster.pid",
105-
"postmaster.opts",
118+
{"postmaster.pid", false},
119+
{"postmaster.opts", false},
106120

107121
/* end of list */
108-
NULL
122+
{NULL, false}
109123
};
110124

111125
/*
@@ -498,14 +512,19 @@ check_file_excluded(const char *path, bool is_source)
498512
constchar*filename;
499513

500514
/* check individual files... */
501-
for (excludeIdx=0;excludeFiles[excludeIdx]!=NULL;excludeIdx++)
515+
for (excludeIdx=0;excludeFiles[excludeIdx].name!=NULL;excludeIdx++)
502516
{
517+
intcmplen=strlen(excludeFiles[excludeIdx].name);
518+
503519
filename=last_dir_separator(path);
504520
if (filename==NULL)
505521
filename=path;
506522
else
507523
filename++;
508-
if (strcmp(filename,excludeFiles[excludeIdx])==0)
524+
525+
if (!excludeFiles[excludeIdx].match_prefix)
526+
cmplen++;
527+
if (strncmp(filename,excludeFiles[excludeIdx].name,cmplen)==0)
509528
{
510529
if (is_source)
511530
pg_log(PG_DEBUG,"entry \"%s\" excluded from source file list\n",

‎src/bin/pg_verify_checksums/pg_verify_checksums.c‎

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -50,31 +50,48 @@ usage(void)
5050
printf(_("Report bugs to <pgsql-bugs@postgresql.org>.\n"));
5151
}
5252

53+
/*
54+
* Definition of one element part of an exclusion list, used for files
55+
* to exclude from checksum validation. "name" is the name of the file
56+
* or path to check for exclusion. If "match_prefix" is true, any items
57+
* matching the name as prefix are excluded.
58+
*/
59+
structexclude_list_item
60+
{
61+
constchar*name;
62+
boolmatch_prefix;
63+
};
64+
5365
/*
5466
* List of files excluded from checksum validation.
5567
*
5668
* Note: this list should be kept in sync with what basebackup.c includes.
5769
*/
58-
staticconstchar*constskip[]= {
59-
"pg_control",
60-
"pg_filenode.map",
61-
"pg_internal.init",
62-
"PG_VERSION",
70+
staticconststructexclude_list_itemskip[]= {
71+
{"pg_control", false},
72+
{"pg_filenode.map", false},
73+
{"pg_internal.init", true},
74+
{"PG_VERSION", false},
6375
#ifdefEXEC_BACKEND
64-
"config_exec_params",
65-
"config_exec_params.new",
76+
{"config_exec_params", true},
6677
#endif
67-
NULL,
78+
{NULL, false}
6879
};
6980

7081
staticbool
7182
skipfile(constchar*fn)
7283
{
73-
constchar*const*f;
84+
intexcludeIdx;
85+
86+
for (excludeIdx=0;skip[excludeIdx].name!=NULL;excludeIdx++)
87+
{
88+
intcmplen=strlen(skip[excludeIdx].name);
7489

75-
for (f=skip;*f;f++)
76-
if (strcmp(*f,fn)==0)
90+
if (!skip[excludeIdx].match_prefix)
91+
cmplen++;
92+
if (strncmp(skip[excludeIdx].name,fn,cmplen)==0)
7793
return true;
94+
}
7895

7996
return false;
8097
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp