Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commita6f3b12

Browse files
committed
[Issue#313] restore pg_control last
1 parentda2cbcb commita6f3b12

File tree

3 files changed

+170
-102
lines changed

3 files changed

+170
-102
lines changed

‎src/restore.c

Lines changed: 162 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* restore.c: restore DB cluster and archived WAL.
44
*
55
* Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION
6-
* Portions Copyright (c) 2015-2019, Postgres Professional
6+
* Portions Copyright (c) 2015-2021, Postgres Professional
77
*
88
*-------------------------------------------------------------------------
99
*/
@@ -61,6 +61,8 @@ static void create_recovery_conf(InstanceState *instanceState, time_t backup_id,
6161
pgBackup*backup,
6262
pgRestoreParams*params);
6363
staticvoid*restore_files(void*arg);
64+
staticsize_trestore_file(pgFile*dest_file,constchar*to_fullpath,boolalready_exists,char*out_buf,
65+
pgBackup*dest_backup,parray*parent_chain,booluse_bitmap,IncrRestoreModeincremental_mode,XLogRecPtrshift_lsn);
6466
staticvoidset_orphan_status(parray*backups,pgBackup*parent_backup);
6567

6668
staticvoidrestore_chain(pgBackup*dest_backup,parray*parent_chain,
@@ -710,6 +712,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
710712
parray*pgdata_files=NULL;
711713
parray*dest_files=NULL;
712714
parray*external_dirs=NULL;
715+
pgFile*dest_pg_control_file=NULL;
716+
chardest_pg_control_fullpath[MAXPGPATH];
713717
/* arrays with meta info for multi threaded backup */
714718
pthread_t*threads;
715719
restore_files_arg*threads_args;
@@ -965,37 +969,61 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
965969
dest_bytes=dest_backup->pgdata_bytes;
966970

967971
pretty_size(dest_bytes,pretty_dest_bytes,lengthof(pretty_dest_bytes));
972+
973+
/*
974+
* [Issue #313]
975+
* find pg_control file (in already sorted earlier dest_files, see parray_qsort(backup->files...))
976+
* and exclude it from list for future special processing
977+
*/
978+
{
979+
intcontrol_file_elem_index;
980+
pgFilesearch_key;
981+
MemSet(&search_key,0,sizeof(pgFile));
982+
/* pgFileCompareRelPathWithExternal uses only .rel_path and .external_dir_num for comparision */
983+
search_key.rel_path=XLOG_CONTROL_FILE;
984+
search_key.external_dir_num=0;
985+
control_file_elem_index=parray_bsearch_index(dest_files,&search_key,pgFileCompareRelPathWithExternal);
986+
if(control_file_elem_index<0)
987+
elog(ERROR,"\"%s\" not found in backup %s",XLOG_CONTROL_FILE,base36enc(dest_backup->start_time));
988+
dest_pg_control_file=parray_remove(dest_files,control_file_elem_index);
989+
990+
join_path_components(dest_pg_control_fullpath,pgdata_path,dest_pg_control_file->rel_path);
991+
/* remove dest control file before restoring */
992+
if (params->incremental_mode!=INCR_NONE)
993+
fio_unlink(dest_pg_control_fullpath,FIO_DB_HOST);
994+
}
995+
968996
elog(INFO,"Start restoring backup files. PGDATA size: %s",pretty_dest_bytes);
969997
time(&start_time);
970998
thread_interrupted= false;
971999

9721000
/* Restore files into target directory */
9731001
for (i=0;i<num_threads;i++)
9741002
{
975-
restore_files_arg*arg=&(threads_args[i]);
976-
977-
arg->dest_files=dest_files;
978-
arg->pgdata_files=pgdata_files;
979-
arg->dest_backup=dest_backup;
980-
arg->dest_external_dirs=external_dirs;
981-
arg->parent_chain=parent_chain;
982-
arg->dbOid_exclude_list=dbOid_exclude_list;
983-
arg->skip_external_dirs=params->skip_external_dirs;
984-
arg->to_root=pgdata_path;
985-
arg->use_bitmap=use_bitmap;
986-
arg->incremental_mode=params->incremental_mode;
987-
arg->shift_lsn=params->shift_lsn;
988-
threads_args[i].restored_bytes=0;
989-
/* By default there are some error */
990-
threads_args[i].ret=1;
1003+
threads_args[i]= (restore_files_arg){
1004+
.dest_files=dest_files,
1005+
.pgdata_files=pgdata_files,
1006+
.dest_backup=dest_backup,
1007+
.dest_external_dirs=external_dirs,
1008+
.parent_chain=parent_chain,
1009+
.dbOid_exclude_list=dbOid_exclude_list,
1010+
.skip_external_dirs=params->skip_external_dirs,
1011+
.to_root=pgdata_path,
1012+
.use_bitmap=use_bitmap,
1013+
.incremental_mode=params->incremental_mode,
1014+
.shift_lsn=params->shift_lsn,
1015+
.restored_bytes=0,
1016+
/* By default there are some error */
1017+
.ret=1,
1018+
};
9911019

9921020
/* Useless message TODO: rewrite */
9931021
elog(LOG,"Start thread %i",i+1);
9941022

995-
pthread_create(&threads[i],NULL,restore_files,arg);
1023+
pthread_create(&threads[i],NULL,restore_files,&(threads_args[i]));
9961024
}
9971025

998-
/* Waittheads */
1026+
/* Waitthreads */
9991027
for (i=0;i<num_threads;i++)
10001028
{
10011029
pthread_join(threads[i],NULL);
@@ -1005,6 +1033,15 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
10051033
total_bytes+=threads_args[i].restored_bytes;
10061034
}
10071035

1036+
/* [Issue #313] copy pg_control at very end */
1037+
if (restore_isok)
1038+
{
1039+
fio_is_remote(FIO_DB_HOST);/* reopen already closed ssh connection */
1040+
total_bytes+=restore_file(dest_pg_control_file,dest_pg_control_fullpath, false,NULL,
1041+
dest_backup,parent_chain,use_bitmap,params->incremental_mode,params->shift_lsn);
1042+
fio_disconnect();
1043+
}
1044+
10081045
time(&end_time);
10091046
pretty_time_interval(difftime(end_time,start_time),
10101047
pretty_time,lengthof(pretty_time));
@@ -1066,10 +1103,15 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
10661103
}
10671104

10681105
/* TODO: write test for case: file to be synced is missing */
1106+
/* MKulagin question: where is fio connection reopened? */
10691107
if (fio_sync(to_fullpath,FIO_DB_HOST)!=0)
10701108
elog(ERROR,"Failed to sync file \"%s\": %s",to_fullpath,strerror(errno));
10711109
}
10721110

1111+
/* sync control file */
1112+
if (fio_sync(dest_pg_control_fullpath,FIO_DB_HOST)!=0)
1113+
elog(ERROR,"Failed to sync file \"%s\": %s",dest_pg_control_fullpath,strerror(errno));
1114+
10731115
time(&end_time);
10741116
pretty_time_interval(difftime(end_time,start_time),
10751117
pretty_time,lengthof(pretty_time));
@@ -1089,6 +1131,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
10891131
parray_free(pgdata_files);
10901132
}
10911133

1134+
pgFileFree(dest_pg_control_file);
1135+
10921136
for (i=parray_num(parent_chain)-1;i >=0;i--)
10931137
{
10941138
pgBackup*backup= (pgBackup*)parray_get(parent_chain,i);
@@ -1107,7 +1151,6 @@ restore_files(void *arg)
11071151
inti;
11081152
uint64n_files;
11091153
charto_fullpath[MAXPGPATH];
1110-
FILE*out=NULL;
11111154
char*out_buf=pgut_malloc(STDIO_BUFSIZE);
11121155

11131156
restore_files_arg*arguments= (restore_files_arg*)arg;
@@ -1117,9 +1160,6 @@ restore_files(void *arg)
11171160
for (i=0;i<parray_num(arguments->dest_files);i++)
11181161
{
11191162
boolalready_exists= false;
1120-
PageState*checksum_map=NULL;/* it should take ~1.5MB at most */
1121-
datapagemap_t*lsn_map=NULL;/* it should take 16kB at most */
1122-
char*errmsg=NULL;/* remote agent error message */
11231163
pgFile*dest_file= (pgFile*)parray_get(arguments->dest_files,i);
11241164

11251165
/* Directories were created before */
@@ -1193,114 +1233,134 @@ restore_files(void *arg)
11931233
already_exists= true;
11941234
}
11951235

1196-
/*
1197-
* Handle incremental restore case for data files.
1198-
* If file is already exists in pgdata, then
1199-
* we scan it block by block and get
1200-
* array of checksums for every page.
1201-
*/
1202-
if (already_exists&&
1203-
dest_file->is_datafile&& !dest_file->is_cfs&&
1204-
dest_file->n_blocks>0)
1236+
arguments->restored_bytes+=restore_file(dest_file,to_fullpath,already_exists,out_buf,
1237+
arguments->dest_backup,arguments->parent_chain,arguments->use_bitmap,
1238+
arguments->incremental_mode,arguments->shift_lsn);
1239+
}
1240+
1241+
free(out_buf);
1242+
1243+
/* ssh connection to longer needed */
1244+
fio_disconnect();
1245+
1246+
/* Data files restoring is successful */
1247+
arguments->ret=0;
1248+
1249+
returnNULL;
1250+
}
1251+
1252+
/*
1253+
* Restore one file into $PGDATA.
1254+
*/
1255+
staticsize_t
1256+
restore_file(pgFile*dest_file,constchar*to_fullpath,boolalready_exists,char*out_buf,
1257+
pgBackup*dest_backup,parray*parent_chain,booluse_bitmap,IncrRestoreModeincremental_mode,XLogRecPtrshift_lsn)
1258+
{
1259+
FILE*out=NULL;
1260+
size_trestored_bytes=0;
1261+
PageState*checksum_map=NULL;/* it should take ~1.5MB at most */
1262+
datapagemap_t*lsn_map=NULL;/* it should take 16kB at most */
1263+
char*errmsg=NULL;/* remote agent error message */
1264+
1265+
/*
1266+
* Handle incremental restore case for data files.
1267+
* If file is already exists in pgdata, then
1268+
* we scan it block by block and get
1269+
* array of checksums for every page.
1270+
*/
1271+
if (already_exists&&
1272+
dest_file->is_datafile&& !dest_file->is_cfs&&
1273+
dest_file->n_blocks>0)
1274+
{
1275+
if (incremental_mode==INCR_LSN)
12051276
{
1206-
if (arguments->incremental_mode==INCR_LSN)
1207-
{
1208-
lsn_map=fio_get_lsn_map(to_fullpath,arguments->dest_backup->checksum_version,
1209-
dest_file->n_blocks,arguments->shift_lsn,
1210-
dest_file->segno*RELSEG_SIZE,FIO_DB_HOST);
1211-
}
1212-
elseif (arguments->incremental_mode==INCR_CHECKSUM)
1213-
{
1214-
checksum_map=fio_get_checksum_map(to_fullpath,arguments->dest_backup->checksum_version,
1215-
dest_file->n_blocks,arguments->dest_backup->stop_lsn,
1216-
dest_file->segno*RELSEG_SIZE,FIO_DB_HOST);
1217-
}
1277+
lsn_map=fio_get_lsn_map(to_fullpath,dest_backup->checksum_version,
1278+
dest_file->n_blocks,shift_lsn,
1279+
dest_file->segno*RELSEG_SIZE,FIO_DB_HOST);
12181280
}
1281+
elseif (incremental_mode==INCR_CHECKSUM)
1282+
{
1283+
checksum_map=fio_get_checksum_map(to_fullpath,dest_backup->checksum_version,
1284+
dest_file->n_blocks,dest_backup->stop_lsn,
1285+
dest_file->segno*RELSEG_SIZE,FIO_DB_HOST);
1286+
}
1287+
}
12191288

1220-
/*
1221-
* Open dest file and truncate it to zero, if destination
1222-
* file already exists and dest file size is zero, or
1223-
* if file do not exist
1224-
*/
1225-
if ((already_exists&&dest_file->write_size==0)|| !already_exists)
1226-
out=fio_fopen(to_fullpath,PG_BINARY_W,FIO_DB_HOST);
1227-
/*
1228-
* If file already exists and dest size is not zero,
1229-
* then open it for reading and writing.
1230-
*/
1231-
else
1232-
out=fio_fopen(to_fullpath,PG_BINARY_R"+",FIO_DB_HOST);
1233-
1234-
if (out==NULL)
1235-
elog(ERROR,"Cannot open restore target file \"%s\": %s",
1236-
to_fullpath,strerror(errno));
1289+
/*
1290+
* Open dest file and truncate it to zero, if destination
1291+
* file already exists and dest file size is zero, or
1292+
* if file do not exist
1293+
*/
1294+
if ((already_exists&&dest_file->write_size==0)|| !already_exists)
1295+
out=fio_fopen(to_fullpath,PG_BINARY_W,FIO_DB_HOST);
1296+
/*
1297+
* If file already exists and dest size is not zero,
1298+
* then open it for reading and writing.
1299+
*/
1300+
else
1301+
out=fio_fopen(to_fullpath,PG_BINARY_R"+",FIO_DB_HOST);
12371302

1238-
/* update file permission */
1239-
if (fio_chmod(to_fullpath,dest_file->mode,FIO_DB_HOST)==-1)
1240-
elog(ERROR,"Cannot change mode of \"%s\": %s",to_fullpath,
1241-
strerror(errno));
1303+
if (out==NULL)
1304+
elog(ERROR,"Cannot open restore target file \"%s\": %s",
1305+
to_fullpath,strerror(errno));
12421306

1243-
if (!dest_file->is_datafile||dest_file->is_cfs)
1244-
elog(VERBOSE,"Restoring nonedata file: \"%s\"",to_fullpath);
1245-
else
1246-
elog(VERBOSE,"Restoring data file: \"%s\"",to_fullpath);
1307+
/* update file permission */
1308+
if (fio_chmod(to_fullpath,dest_file->mode,FIO_DB_HOST)==-1)
1309+
elog(ERROR,"Cannot change mode of \"%s\": %s",to_fullpath,
1310+
strerror(errno));
12471311

1248-
// If destination file is 0 sized, then just close it and go for the next
1249-
if (dest_file->write_size==0)
1250-
gotodone;
1312+
if (!dest_file->is_datafile||dest_file->is_cfs)
1313+
elog(VERBOSE,"Restoring nonedata file: \"%s\"",to_fullpath);
1314+
else
1315+
elog(VERBOSE,"Restoring data file: \"%s\"",to_fullpath);
12511316

1317+
// If destination file is 0 sized, then just close it and go for the next
1318+
if (dest_file->write_size!=0)
1319+
{
12521320
/* Restore destination file */
12531321
if (dest_file->is_datafile&& !dest_file->is_cfs)
12541322
{
12551323
/* enable stdio buffering for local destination data file */
1256-
if (!fio_is_remote_file(out))
1324+
if (!fio_is_remote_file(out)&&out_buf!=NULL)
12571325
setvbuf(out,out_buf,_IOFBF,STDIO_BUFSIZE);
12581326
/* Destination file is data file */
1259-
arguments->restored_bytes+=restore_data_file(arguments->parent_chain,
1327+
restored_bytes+=restore_data_file(parent_chain,
12601328
dest_file,out,to_fullpath,
1261-
arguments->use_bitmap,checksum_map,
1262-
arguments->shift_lsn,lsn_map, true);
1329+
use_bitmap,checksum_map,
1330+
shift_lsn,lsn_map, true);
12631331
}
12641332
else
12651333
{
12661334
/* disable stdio buffering for local destination nonedata file */
12671335
if (!fio_is_remote_file(out))
12681336
setvbuf(out,NULL,_IONBF,BUFSIZ);
12691337
/* Destination file is nonedata file */
1270-
arguments->restored_bytes+=restore_non_data_file(arguments->parent_chain,
1271-
arguments->dest_backup,dest_file,out,to_fullpath,
1338+
restored_bytes+=restore_non_data_file(parent_chain,
1339+
dest_backup,dest_file,out,to_fullpath,
12721340
already_exists);
12731341
}
1342+
}
12741343

1275-
done:
1276-
/* Writing is asynchronous in case of restore in remote mode, so check the agent status */
1277-
if (fio_check_error_file(out,&errmsg))
1278-
elog(ERROR,"Cannot write to the remote file \"%s\": %s",to_fullpath,errmsg);
1279-
1280-
/* close file */
1281-
if (fio_fclose(out)!=0)
1282-
elog(ERROR,"Cannot close file \"%s\": %s",to_fullpath,
1283-
strerror(errno));
1284-
1285-
/* free pagemap used for restore optimization */
1286-
pg_free(dest_file->pagemap.bitmap);
1287-
1288-
if (lsn_map)
1289-
pg_free(lsn_map->bitmap);
1344+
/* Writing is asynchronous in case of restore in remote mode, so check the agent status */
1345+
if (fio_check_error_file(out,&errmsg))
1346+
elog(ERROR,"Cannot write to the remote file \"%s\": %s",to_fullpath,errmsg);
12901347

1291-
pg_free(lsn_map);
1292-
pg_free(checksum_map);
1293-
}
1348+
/* close file */
1349+
if (fio_fclose(out)!=0)
1350+
elog(ERROR,"Cannot close file \"%s\": %s",to_fullpath,
1351+
strerror(errno));
12941352

1295-
free(out_buf);
1353+
if (lsn_map)
1354+
pg_free(lsn_map->bitmap);
12961355

1297-
/* ssh connection to longer needed */
1298-
fio_disconnect();
1356+
pg_free(lsn_map);
1357+
pg_free(checksum_map);
12991358

1300-
/* Data files restoring is successful */
1301-
arguments->ret=0;
1359+
/* free pagemap used for restore optimization */
1360+
pg_free(dest_file->pagemap.bitmap);
1361+
dest_file->pagemap.bitmap=NULL;
13021362

1303-
returnNULL;
1363+
returnrestored_bytes;
13041364
}
13051365

13061366
/*

‎src/utils/parray.c

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,13 @@ parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const
198198
returnbsearch(&key,array->data,array->used,sizeof(void*),compare);
199199
}
200200

201+
int
202+
parray_bsearch_index(parray*array,constvoid*key,int(*compare)(constvoid*,constvoid*))
203+
{
204+
void**elem=parray_bsearch(array,key,compare);
205+
returnelem!=NULL ?elem-array->data :-1;
206+
}
207+
201208
/* checks that parray contains element */
202209
boolparray_contains(parray*array,void*elem)
203210
{

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp