Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitbc34223

Browse files
committed
pg_basebackup pg_receivexlog: Issue fsync more carefully
Several places weren't careful about fsyncing in the way. See1d4a0aband606e0f9 for details about required fsyncs.This adds a couple of functions in src/common/ that have an equivalentin the backend: durable_rename(), fsync_parent_path()From: Michael Paquier <michael.paquier@gmail.com>
1 parentbf5bb2e commitbc34223

File tree

4 files changed

+162
-29
lines changed

4 files changed

+162
-29
lines changed

‎src/bin/pg_basebackup/pg_basebackup.c

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include<zlib.h>
2828
#endif
2929

30+
#include"common/file_utils.h"
3031
#include"common/string.h"
3132
#include"fe_utils/string_utils.h"
3233
#include"getopt_long.h"
@@ -1196,6 +1197,10 @@ ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
11961197

11971198
if (copybuf!=NULL)
11981199
PQfreemem(copybuf);
1200+
1201+
/* sync the resulting tar file, errors are not considered fatal */
1202+
if (strcmp(basedir,"-")!=0)
1203+
(void)fsync_fname(filename, false,progname);
11991204
}
12001205

12011206

@@ -1472,6 +1477,11 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum)
14721477

14731478
if (basetablespace&&writerecoveryconf)
14741479
WriteRecoveryConf();
1480+
1481+
/*
1482+
* No data is synced here, everything is done for all tablespaces at the
1483+
* end.
1484+
*/
14751485
}
14761486

14771487
/*
@@ -1950,6 +1960,23 @@ BaseBackup(void)
19501960
PQclear(res);
19511961
PQfinish(conn);
19521962

1963+
/*
1964+
* Make data persistent on disk once backup is completed. For tar
1965+
* format once syncing the parent directory is fine, each tar file
1966+
* created per tablespace has been already synced. In plain format,
1967+
* all the data of the base directory is synced, taking into account
1968+
* all the tablespaces. Errors are not considered fatal.
1969+
*/
1970+
if (format=='t')
1971+
{
1972+
if (strcmp(basedir,"-")!=0)
1973+
(void)fsync_fname(basedir, true,progname);
1974+
}
1975+
else
1976+
{
1977+
(void)fsync_pgdata(basedir,progname);
1978+
}
1979+
19531980
if (verbose)
19541981
fprintf(stderr,"%s: base backup completed\n",progname);
19551982
}

‎src/bin/pg_basebackup/receivelog.c

Lines changed: 30 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
#include"libpq-fe.h"
2828
#include"access/xlog_internal.h"
29+
#include"common/file_utils.h"
2930

3031

3132
/* fd and filename for currently open WAL file */
@@ -71,17 +72,13 @@ mark_file_as_archived(const char *basedir, const char *fname)
7172
return false;
7273
}
7374

74-
if (fsync(fd)!=0)
75-
{
76-
fprintf(stderr,_("%s: could not fsync file \"%s\": %s\n"),
77-
progname,tmppath,strerror(errno));
78-
79-
close(fd);
75+
close(fd);
8076

77+
if (fsync_fname(tmppath, false,progname)!=0)
8178
return false;
82-
}
8379

84-
close(fd);
80+
if (fsync_parent_path(tmppath,progname)!=0)
81+
return false;
8582

8683
return true;
8784
}
@@ -132,6 +129,16 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint)
132129
{
133130
/* File is open and ready to use */
134131
walfile=f;
132+
133+
/*
134+
* fsync, in case of a previous crash between padding and fsyncing the
135+
* file.
136+
*/
137+
if (fsync_fname(fn, false,progname)!=0)
138+
return false;
139+
if (fsync_parent_path(fn,progname)!=0)
140+
return false;
141+
135142
return true;
136143
}
137144
if (statbuf.st_size!=0)
@@ -160,6 +167,17 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint)
160167
}
161168
free(zerobuf);
162169

170+
/*
171+
* fsync WAL file and containing directory, to ensure the file is
172+
* persistently created and zeroed. That's particularly important when
173+
* using synchronous mode, where the file is modified and fsynced
174+
* in-place, without a directory fsync.
175+
*/
176+
if (fsync_fname(fn, false,progname)!=0)
177+
return false;
178+
if (fsync_parent_path(fn,progname)!=0)
179+
return false;
180+
163181
if (lseek(f,SEEK_SET,0)!=0)
164182
{
165183
fprintf(stderr,
@@ -220,10 +238,9 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos)
220238

221239
snprintf(oldfn,sizeof(oldfn),"%s/%s%s",stream->basedir,current_walfile_name,stream->partial_suffix);
222240
snprintf(newfn,sizeof(newfn),"%s/%s",stream->basedir,current_walfile_name);
223-
if (rename(oldfn,newfn)!=0)
241+
if (durable_rename(oldfn,newfn,progname)!=0)
224242
{
225-
fprintf(stderr,_("%s: could not rename file \"%s\": %s\n"),
226-
progname,current_walfile_name,strerror(errno));
243+
/* durable_rename produced a log entry */
227244
return false;
228245
}
229246
}
@@ -341,14 +358,6 @@ writeTimeLineHistoryFile(StreamCtl *stream, char *filename, char *content)
341358
return false;
342359
}
343360

344-
if (fsync(fd)!=0)
345-
{
346-
close(fd);
347-
fprintf(stderr,_("%s: could not fsync file \"%s\": %s\n"),
348-
progname,tmppath,strerror(errno));
349-
return false;
350-
}
351-
352361
if (close(fd)!=0)
353362
{
354363
fprintf(stderr,_("%s: could not close file \"%s\": %s\n"),
@@ -359,10 +368,9 @@ writeTimeLineHistoryFile(StreamCtl *stream, char *filename, char *content)
359368
/*
360369
* Now move the completed history file into place with its final name.
361370
*/
362-
if (rename(tmppath,path)<0)
371+
if (durable_rename(tmppath,path,progname)<0)
363372
{
364-
fprintf(stderr,_("%s: could not rename file \"%s\" to \"%s\": %s\n"),
365-
progname,tmppath,path,strerror(errno));
373+
/* durable_rename produced a log entry */
366374
return false;
367375
}
368376

‎src/common/file_utils.c

Lines changed: 100 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ static void pre_sync_fname(const char *fname, bool isdir,
3434
constchar*progname);
3535
#endif
3636
staticvoidwalkdir(constchar*path,
37-
void (*action) (constchar*fname,boolisdir,constchar*progname),
37+
int (*action) (constchar*fname,boolisdir,constchar*progname),
3838
boolprocess_symlinks,constchar*progname);
3939

4040
/*
@@ -120,7 +120,7 @@ fsync_pgdata(const char *pg_data, const char *progname)
120120
*/
121121
staticvoid
122122
walkdir(constchar*path,
123-
void (*action) (constchar*fname,boolisdir,constchar*progname),
123+
int (*action) (constchar*fname,boolisdir,constchar*progname),
124124
boolprocess_symlinks,constchar*progname)
125125
{
126126
DIR*dir;
@@ -228,7 +228,7 @@ pre_sync_fname(const char *fname, bool isdir, const char *progname)
228228
* directories on systems where that isn't allowed/required. Reports
229229
* other errors non-fatally.
230230
*/
231-
void
231+
int
232232
fsync_fname(constchar*fname,boolisdir,constchar*progname)
233233
{
234234
intfd;
@@ -256,10 +256,10 @@ fsync_fname(const char *fname, bool isdir, const char *progname)
256256
if (fd<0)
257257
{
258258
if (errno==EACCES|| (isdir&&errno==EISDIR))
259-
return;
259+
return0;
260260
fprintf(stderr,_("%s: could not open file \"%s\": %s\n"),
261261
progname,fname,strerror(errno));
262-
return;
262+
return-1;
263263
}
264264

265265
returncode=fsync(fd);
@@ -269,8 +269,103 @@ fsync_fname(const char *fname, bool isdir, const char *progname)
269269
* those errors. Anything else needs to be reported.
270270
*/
271271
if (returncode!=0&& !(isdir&&errno==EBADF))
272+
{
272273
fprintf(stderr,_("%s: could not fsync file \"%s\": %s\n"),
273274
progname,fname,strerror(errno));
275+
return-1;
276+
}
274277

275278
(void)close(fd);
279+
return0;
280+
}
281+
282+
/*
283+
* fsync_parent_path -- fsync the parent path of a file or directory
284+
*
285+
* This is aimed at making file operations persistent on disk in case of
286+
* an OS crash or power failure.
287+
*/
288+
int
289+
fsync_parent_path(constchar*fname,constchar*progname)
290+
{
291+
charparentpath[MAXPGPATH];
292+
293+
strlcpy(parentpath,fname,MAXPGPATH);
294+
get_parent_directory(parentpath);
295+
296+
/*
297+
* get_parent_directory() returns an empty string if the input argument is
298+
* just a file name (see comments in path.c), so handle that as being the
299+
* current directory.
300+
*/
301+
if (strlen(parentpath)==0)
302+
strlcpy(parentpath,".",MAXPGPATH);
303+
304+
if (fsync_fname(parentpath, true,progname)!=0)
305+
return-1;
306+
307+
return0;
308+
}
309+
310+
/*
311+
* durable_rename -- rename(2) wrapper, issuing fsyncs required for durability
312+
*
313+
* Wrapper around rename, similar to the backend version.
314+
*/
315+
int
316+
durable_rename(constchar*oldfile,constchar*newfile,constchar*progname)
317+
{
318+
intfd;
319+
320+
/*
321+
* First fsync the old and target path (if it exists), to ensure that they
322+
* are properly persistent on disk. Syncing the target file is not
323+
* strictly necessary, but it makes it easier to reason about crashes;
324+
* because it's then guaranteed that either source or target file exists
325+
* after a crash.
326+
*/
327+
if (fsync_fname(oldfile, false,progname)!=0)
328+
return-1;
329+
330+
fd=open(newfile,PG_BINARY |O_RDWR,0);
331+
if (fd<0)
332+
{
333+
if (errno!=ENOENT)
334+
{
335+
fprintf(stderr,_("%s: could not open file \"%s\": %s\n"),
336+
progname,newfile,strerror(errno));
337+
return-1;
338+
}
339+
}
340+
else
341+
{
342+
if (fsync(fd)!=0)
343+
{
344+
fprintf(stderr,_("%s: could not fsync file \"%s\": %s\n"),
345+
progname,newfile,strerror(errno));
346+
close(fd);
347+
return-1;
348+
}
349+
close(fd);
350+
}
351+
352+
/* Time to do the real deal... */
353+
if (rename(oldfile,newfile)!=0)
354+
{
355+
fprintf(stderr,_("%s: could not rename file \"%s\" to \"%s\": %s\n"),
356+
progname,oldfile,newfile,strerror(errno));
357+
return-1;
358+
}
359+
360+
/*
361+
* To guarantee renaming the file is persistent, fsync the file with its
362+
* new name, and its containing directory.
363+
*/
364+
if (fsync_fname(newfile, false,progname)!=0)
365+
return-1;
366+
367+
if (fsync_parent_path(newfile,progname)!=0)
368+
return-1;
369+
370+
return0;
276371
}

‎src/include/common/file_utils.h

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,11 @@
1515
#ifndefFILE_UTILS_H
1616
#defineFILE_UTILS_H
1717

18-
externvoidfsync_fname(constchar*fname,boolisdir,
19-
constchar*progname);
18+
externintfsync_fname(constchar*fname,boolisdir,
19+
constchar*progname);
2020
externvoidfsync_pgdata(constchar*pg_data,constchar*progname);
21+
externintdurable_rename(constchar*oldfile,constchar*newfile,
22+
constchar*progname);
23+
externintfsync_parent_path(constchar*fname,constchar*progname);
2124

2225
#endif/* FILE_UTILS_H */

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp