Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit04834e7

Browse files
author
Michael Paquier
committed
Page-level backup using block tracking in WAL records
This commit improves the performance of page-level, or differentialbackup, by not having to scan anymore all the pages of a relation file,something that can be very long on large data sets, but by scanning thelist of blocks changed by WAL records since the last full or differentialbackup.As a restriction and to avoid potential data corruption should hint-bitupdates occur on a page, backups can only be taken from a server that haswal_log_hints or data checksums enabled.Base patch by Yury Zhuravlev, heavily modified by me.
1 parent47d0b60 commit04834e7

File tree

12 files changed

+578
-119
lines changed

12 files changed

+578
-119
lines changed

‎.gitignore‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,8 @@
2525
/regression.diffs
2626
/regression.out
2727
/results
28+
29+
# Extra files
30+
/datapagemap.c
31+
/datapagemap.h
32+
/xlogreader.c

‎Makefile‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,16 @@ OBJS = backup.o \
1414
util.o\
1515
validate.o\
1616
xlog.o\
17+
datapagemap.o\
18+
parsexlog.o\
19+
xlogreader.o\
1720
pgut/pgut.o\
1821
pgut/pgut-port.o
1922

2023
DOCS = doc/pg_arman.txt
2124

25+
EXTRA_CLEAN = datapagemap.c datapagemap.h xlogreader.c
26+
2227
# asciidoc and xmlto are present, so install the html documentation and man
2328
# pages as well. html is part of the vanilla documentation. Man pages need a
2429
# special handling at installation.
@@ -35,6 +40,27 @@ PG_LIBS = $(libpq_pgport)
3540

3641
REGRESS = init option show delete backup restore
3742

43+
all: checksrcdir docs datapagemap.h pg_arman
44+
45+
# This rule's only purpose is to give the user instructions on how to pass
46+
# the path to PostgreSQL source tree to the makefile.
47+
.PHONY: checksrcdir
48+
checksrcdir:
49+
ifndeftop_srcdir
50+
@echo "You must have PostgreSQL source tree available to compile."
51+
@echo "Pass the path to the PostgreSQL source tree to make, in the top_srcdir"
52+
@echo "variable: \"make top_srcdir=<path to PostgreSQL source tree>\""
53+
@exit 1
54+
endif
55+
56+
# Those files are symlinked from the PostgreSQL sources.
57+
xlogreader.c:% :$(top_srcdir)/src/backend/access/transam/%
58+
rm -f$@&&$(LN_S)$<.
59+
datapagemap.c:% :$(top_srcdir)/src/bin/pg_rewind/%
60+
rm -f$@&&$(LN_S)$<.
61+
datapagemap.h:% :$(top_srcdir)/src/bin/pg_rewind/%
62+
rm -f&&$(LN_S)$<.
63+
3864
PG_CONFIG = pg_config
3965
PGXS :=$(shell$(PG_CONFIG) --pgxs)
4066
include$(PGXS)
@@ -43,7 +69,6 @@ include $(PGXS)
4369
# Compile documentation as well is ASCIIDOC and XMLTO are defined
4470
ifneq ($(ASCIIDOC),)
4571
ifneq ($(XMLTO),)
46-
all: docs
4772
docs:
4873
$(MAKE) -C doc/
4974

@@ -53,7 +78,13 @@ install: install-man
5378
install-man:
5479
$(MKDIR_P)'$(DESTDIR)$(mandir)/man1/'
5580
$(INSTALL_DATA)$(man_DOCS)'$(DESTDIR)$(mandir)/man1/'
81+
else
82+
docs:
83+
@echo"No docs to build"
5684
endif# XMLTO
85+
else
86+
docs:
87+
@echo"No docs to build"
5788
endif# ASCIIDOC
5889

5990
# Clean up documentation as well

‎backup.c‎

Lines changed: 120 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ static int server_version = 0;
2828

2929
staticboolin_backup= false;/* TODO: more robust logic */
3030

31+
/* list of files contained in backup */
32+
parray*backup_files_list;
33+
3134
/*
3235
* Backup routines
3336
*/
@@ -48,6 +51,7 @@ static void create_file_list(parray *files,
4851
constchar*subdir,
4952
constchar*prefix,
5053
boolis_append);
54+
staticvoidwait_for_archive(pgBackup*backup,constchar*sql);
5155

5256
/*
5357
* Take a backup of database and return the list of files backed up.
@@ -56,7 +60,6 @@ static parray *
5660
do_backup_database(parray*backup_list,pgBackupOptionbkupopt)
5761
{
5862
inti;
59-
parray*files;/* backup file list from non-snapshot */
6063
parray*prev_files=NULL;/* file list of previous database backup */
6164
FILE*fp;
6265
charpath[MAXPGPATH];
@@ -68,6 +71,7 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
6871

6972
/* repack the options */
7073
boolsmooth_checkpoint=bkupopt.smooth_checkpoint;
74+
pgBackup*prev_backup=NULL;
7175

7276
/* Block backup operations on a standby */
7377
if (pg_is_standby())
@@ -78,6 +82,9 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
7882
/* Initialize size summary */
7983
current.data_bytes=0;
8084

85+
/* do some checks on the node */
86+
sanityChecks();
87+
8188
/*
8289
* Obtain current timeline by scanning control file, theh LSN
8390
* obtained at output of pg_start_backup or pg_stop_backup does
@@ -123,8 +130,8 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
123130
* List directories and symbolic links with the physical path to make
124131
* mkdirs.sh, then sort them in order of path. Omit $PGDATA.
125132
*/
126-
files=parray_new();
127-
dir_list_file(files,pgdata,NULL, false, false);
133+
backup_files_list=parray_new();
134+
dir_list_file(backup_files_list,pgdata,NULL, false, false);
128135

129136
if (!check)
130137
{
@@ -133,26 +140,24 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
133140
if (fp==NULL)
134141
elog(ERROR_SYSTEM,"can't open make directory script \"%s\": %s",
135142
path,strerror(errno));
136-
dir_print_mkdirs_sh(fp,files,pgdata);
143+
dir_print_mkdirs_sh(fp,backup_files_list,pgdata);
137144
fclose(fp);
138145
if (chmod(path,DIR_PERMISSION)==-1)
139146
elog(ERROR_SYSTEM,"can't change mode of \"%s\": %s",path,
140147
strerror(errno));
141148
}
142149

143150
/* clear directory list */
144-
parray_walk(files,pgFileFree);
145-
parray_free(files);
146-
files=NULL;
151+
parray_walk(backup_files_list,pgFileFree);
152+
parray_free(backup_files_list);
153+
backup_files_list=NULL;
147154

148155
/*
149156
* To take differential backup, the file list of the last completed database
150157
* backup is needed.
151158
*/
152159
if (current.backup_mode==BACKUP_MODE_DIFF_PAGE)
153160
{
154-
pgBackup*prev_backup;
155-
156161
/* find last completed database backup */
157162
prev_backup=catalog_get_last_data_backup(backup_list,current.tli);
158163
pgBackupGetPath(prev_backup,prev_file_txt,lengthof(prev_file_txt),
@@ -167,26 +172,55 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
167172
(uint32) (*lsn >>32), (uint32)*lsn);
168173
}
169174

170-
/* initialize backup listfrom non-snapshot*/
171-
files=parray_new();
175+
/* initialize backup list */
176+
backup_files_list=parray_new();
172177

173178
/* list files with the logical path. omit $PGDATA */
174-
add_files(files,pgdata, false, true);
179+
add_files(backup_files_list,pgdata, false, true);
175180

176181
/* backup files */
177182
pgBackupGetPath(&current,path,lengthof(path),DATABASE_DIR);
178-
backup_files(pgdata,path,files,prev_files,lsn,NULL);
183+
184+
/*
185+
* Build page mapping in differential mode. When using this mode, the
186+
* list of blocks to be taken is known by scanning the WAL segments
187+
* present in archives up to the point where start backup has begun.
188+
* However, normally this segment is not yet available in the archives,
189+
* leading to failures when building the page map. Hence before doing
190+
* anything and in order to ensure that all the segments needed for the
191+
* scan are here, for a switch of the last segment with pg_switch_xlog.
192+
*/
193+
if (current.backup_mode==BACKUP_MODE_DIFF_PAGE)
194+
{
195+
/* Enforce archiving of last segment and wait for it to be here */
196+
wait_for_archive(&current,"SELECT * FROM pg_switch_xlog()");
197+
198+
/* Now build the page map */
199+
parray_qsort(backup_files_list,pgFileComparePathDesc);
200+
elog(LOG,"extractPageMap");
201+
elog(LOG,"current_tli:%X",current.tli);
202+
elog(LOG,"prev_backup->start_lsn: %X/%X",
203+
(uint32) (prev_backup->start_lsn >>32),
204+
(uint32) (prev_backup->start_lsn));
205+
elog(LOG,"current.start_lsn: %X/%X",
206+
(uint32) (current.start_lsn >>32),
207+
(uint32) (current.start_lsn));
208+
extractPageMap(arclog_path,prev_backup->start_lsn,current.tli,
209+
current.start_lsn);
210+
}
211+
212+
backup_files(pgdata,path,backup_files_list,prev_files,lsn,NULL);
179213

180214
/* notify end of backup */
181215
pg_stop_backup(&current);
182216

183217
/* create file list */
184-
create_file_list(files,pgdata,DATABASE_FILE_LIST,NULL, false);
218+
create_file_list(backup_files_list,pgdata,DATABASE_FILE_LIST,NULL, false);
185219

186220
/* print summary of size of backup mode files */
187-
for (i=0;i<parray_num(files);i++)
221+
for (i=0;i<parray_num(backup_files_list);i++)
188222
{
189-
pgFile*file= (pgFile*)parray_get(files,i);
223+
pgFile*file= (pgFile*)parray_get(backup_files_list,i);
190224
if (!S_ISREG(file->mode))
191225
continue;
192226
/*
@@ -204,7 +238,7 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
204238
current.data_bytes);
205239
elog(LOG,"========================================");
206240

207-
returnfiles;
241+
returnbackup_files_list;
208242
}
209243

210244

@@ -654,7 +688,6 @@ backup_files(const char *from_root,
654688
}
655689
else
656690
{
657-
elog(LOG,"\n");
658691
elog(ERROR_SYSTEM,
659692
"can't stat backup mode. \"%s\": %s",
660693
file->path,strerror(errno));
@@ -825,3 +858,72 @@ create_file_list(parray *files,
825858
fclose(fp);
826859
}
827860
}
861+
862+
/*
863+
* A helper function to create the path of a relation file and segment.
864+
*
865+
* The returned path is palloc'd
866+
*/
867+
staticchar*
868+
datasegpath(RelFileNodernode,ForkNumberforknum,BlockNumbersegno)
869+
{
870+
char*path;
871+
char*segpath;
872+
873+
path=relpathperm(rnode,forknum);
874+
if (segno>0)
875+
{
876+
segpath=psprintf("%s.%u",path,segno);
877+
pfree(path);
878+
returnsegpath;
879+
}
880+
else
881+
returnpath;
882+
}
883+
884+
/*
885+
* This routine gets called while reading WAL segments from the WAL archive,
886+
* for every block that have changed in the target system. It makes note of
887+
* all the changed blocks in the pagemap of the file and adds them in the
888+
* things to track for the backup.
889+
*/
890+
void
891+
process_block_change(ForkNumberforknum,RelFileNodernode,BlockNumberblkno)
892+
{
893+
char*path;
894+
char*rel_path;
895+
BlockNumberblkno_inseg;
896+
intsegno;
897+
pgFile*file_item=NULL;
898+
intj;
899+
900+
segno=blkno /RELSEG_SIZE;
901+
blkno_inseg=blkno %RELSEG_SIZE;
902+
903+
rel_path=datasegpath(rnode,forknum,segno);
904+
path=pg_malloc(strlen(rel_path)+strlen(pgdata)+2);
905+
sprintf(path,"%s/%s",pgdata,rel_path);
906+
907+
for (j=0;j<parray_num(backup_files_list);j++)
908+
{
909+
pgFile*p= (pgFile*)parray_get(backup_files_list,j);
910+
911+
if (strcmp(p->path,path)==0)
912+
{
913+
file_item=p;
914+
break;
915+
}
916+
}
917+
918+
/*
919+
* If we don't have any record of this file in the file map, it means
920+
* that it's a relation that did not have much activity since the last
921+
* backup. We can safely ignore it. If it is a new relation file, the
922+
* backup would simply copy it as-is.
923+
*/
924+
if (file_item)
925+
datapagemap_add(&file_item->pagemap,blkno_inseg);
926+
927+
pg_free(path);
928+
pg_free(rel_path);
929+
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp