Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitb26b0f0

Browse files
committed
Merge commit '9d273330d3fa633b650d30048d2822abd93ccfd0' into PGPRO9_6
2 parents75532ba +9d27333 commitb26b0f0

File tree

11 files changed

+135
-51
lines changed

11 files changed

+135
-51
lines changed

‎contrib/pg_arman/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ OBJS = backup.o \
2121
pgut/pgut.o\
2222
pgut/pgut-port.o
2323

24-
EXTRA_CLEAN = datapagemap.c datapagemap.h xlogreader.c receivelog.h streamutil.h logging.h
24+
EXTRA_CLEAN = datapagemap.c datapagemap.h xlogreader.c receivelog.c receivelog.h streamutil.c streamutil.h logging.h
2525

2626
PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS}
2727
overrideCPPFLAGS := -DFRONTEND$(CPPFLAGS)

‎contrib/pg_arman/README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,12 +75,12 @@ some overhead to PostgreSQL performance. On our experiments it appears to be
7575
less than 3%.
7676

7777
These two approaches were implemented in this fork of pg_arman. The second
78-
approach requires[patch for PostgreSQL](https://gist.github.com/stalkerg/7ce2394c4f3b36f995895190a633b4fa).
78+
approach requires[patch for PostgreSQL 9.5](https://gist.github.com/stalkerg/44703dbcbac1da08f448b7e6966646c0).
7979

8080
Testing block level incremental backup
8181
--------------------------------------
8282

83-
You need build and install[PGPRO9_5_ptrack branch of PostgreSQL](https://github.com/postgrespro/postgrespro/tree/PGPRO9_5_ptrack) or[apply this patch to PostgreSQL 9.5](https://gist.github.com/stalkerg/7ce2394c4f3b36f995895190a633b4fa).
83+
You need build and install[PGPRO9_5 branch of PostgreSQL](https://github.com/postgrespro/postgrespro) or[apply this patch to PostgreSQL 9.5](https://gist.github.com/stalkerg/44703dbcbac1da08f448b7e6966646c0).
8484

8585
###Retrieving changed blocks from WAL archive
8686

‎contrib/pg_arman/backup.c

Lines changed: 49 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,9 @@ static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr;
3838
constchar*progname="pg_arman";
3939

4040
/* list of files contained in backup */
41-
parray*backup_files_list;
41+
parray*backup_files_list;
42+
staticvolatileuint32total_copy_files_increment;
43+
staticuint32total_files_num;
4244

4345
typedefstruct
4446
{
@@ -47,8 +49,6 @@ typedef struct
4749
parray*files;
4850
parray*prev_files;
4951
constXLogRecPtr*lsn;
50-
unsignedintstart_file_idx;
51-
unsignedintend_file_idx;
5252
}backup_files_args;
5353

5454
/*
@@ -106,6 +106,7 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
106106
pthread_tstream_thread;
107107
backup_files_args*backup_threads_args[num_threads];
108108

109+
109110
/* repack the options */
110111
boolsmooth_checkpoint=bkupopt.smooth_checkpoint;
111112
pgBackup*prev_backup=NULL;
@@ -146,7 +147,7 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
146147
}
147148

148149
/* clear ptrack files for FULL and DIFF backup */
149-
if (current.backup_mode!=BACKUP_MODE_DIFF_PTRACK)
150+
if (current.backup_mode!=BACKUP_MODE_DIFF_PTRACK&& !disable_ptrack_clear)
150151
pg_ptrack_clear();
151152

152153
/* start stream replication */
@@ -310,11 +311,21 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
310311
if (!check)
311312
dir_create_dir(dirpath,DIR_PERMISSION);
312313
}
314+
else
315+
{
316+
total_files_num++;
317+
}
318+
319+
__sync_lock_release(&file->lock);
313320
}
314321

315322
if (num_threads<1)
316323
num_threads=1;
317324

325+
/* sort by size for load balancing */
326+
parray_qsort(backup_files_list,pgFileCompareSize);
327+
328+
/* init thread args with own file lists */
318329
for (i=0;i<num_threads;i++)
319330
{
320331
backup_files_args*arg=pg_malloc(sizeof(backup_files_args));
@@ -323,19 +334,17 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
323334
arg->files=backup_files_list;
324335
arg->prev_files=prev_files;
325336
arg->lsn=lsn;
326-
arg->start_file_idx=i* (parray_num(backup_files_list)/num_threads);
327-
if (i==num_threads-1)
328-
arg->end_file_idx=parray_num(backup_files_list);
329-
else
330-
arg->end_file_idx= (i+1)* (parray_num(backup_files_list)/num_threads);
337+
backup_threads_args[i]=arg;
338+
}
331339

340+
total_copy_files_increment=0;
341+
342+
/* Run threads */
343+
for (i=0;i<num_threads;i++)
344+
{
332345
if (verbose)
333-
elog(WARNING,"Start thread for start_file_idx:%i end_file_idx:%i num:%li",
334-
arg->start_file_idx,
335-
arg->end_file_idx,
336-
parray_num(backup_files_list));
337-
backup_threads_args[i]=arg;
338-
pthread_create(&backup_threads[i],NULL, (void*(*)(void*))backup_files,arg);
346+
elog(WARNING,"Start thread num:%li",parray_num(backup_threads_args[i]->files));
347+
pthread_create(&backup_threads[i],NULL, (void*(*)(void*))backup_files,backup_threads_args[i]);
339348
}
340349

341350
/* Wait theads */
@@ -345,6 +354,9 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
345354
pg_free(backup_threads_args[i]);
346355
}
347356

357+
if (progress)
358+
fprintf(stderr,"\n");
359+
348360
/* Notify end of backup */
349361
pg_stop_backup(&current);
350362

@@ -911,12 +923,14 @@ backup_files(void *arg)
911923
gettimeofday(&tv,NULL);
912924

913925
/* backup a file or create a directory */
914-
for (i=arguments->start_file_idx;i<arguments->end_file_idx;i++)
926+
for (i=0;i<parray_num(arguments->files);i++)
915927
{
916928
intret;
917929
structstatbuf;
918930

919931
pgFile*file= (pgFile*)parray_get(arguments->files,i);
932+
if (__sync_lock_test_and_set(&file->lock,1)!=0)
933+
continue;
920934

921935
/* If current time is rewinded, abort this backup. */
922936
if (tv.tv_sec<file->mtime)
@@ -1007,6 +1021,9 @@ backup_files(void *arg)
10071021
}
10081022
else
10091023
elog(LOG,"unexpected file type %d",buf.st_mode);
1024+
if (progress)
1025+
fprintf(stderr,"\rProgress %i/%u",total_copy_files_increment,total_files_num-1);
1026+
__sync_fetch_and_add(&total_copy_files_increment,1);
10101027
}
10111028
}
10121029

@@ -1089,6 +1106,20 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
10891106
continue;
10901107

10911108
file->is_datafile= true;
1109+
{
1110+
intfind_dot;
1111+
char*text_segno;
1112+
for(find_dot=path_len-1;file->path[find_dot]!='.'&&find_dot >=0;find_dot--);
1113+
if (find_dot <=0)
1114+
continue;
1115+
1116+
text_segno=file->path+find_dot+1;
1117+
/* in future we will need check all chars */
1118+
if (!isdigit(text_segno[0]))
1119+
continue;
1120+
1121+
file->segno= (int)strtol(text_segno,NULL,10);
1122+
}
10921123
}
10931124
parray_concat(files,list_file);
10941125
}
@@ -1334,13 +1365,13 @@ StreamLog(void *arg)
13341365
ctl.basedir=basedir;
13351366
ctl.stream_stop=stop_streaming;
13361367
ctl.standby_message_timeout=standby_message_timeout;
1337-
ctl.partial_suffix=".partial";
1368+
ctl.partial_suffix=NULL;
13381369
ctl.synchronous= false;
13391370
ctl.mark_done= false;
13401371
ReceiveXlogStream(conn,&ctl);
13411372
#else
13421373
ReceiveXlogStream(conn,startpos,starttli,NULL,basedir,
1343-
stop_streaming,standby_message_timeout,".partial",
1374+
stop_streaming,standby_message_timeout,NULL,
13441375
false, false);
13451376
#endif
13461377

‎contrib/pg_arman/data.c

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ backup_data_file(const char *from_root, const char *to_root,
7272
size_tread_len=0;
7373
pg_crc32crc;
7474
off_toffset;
75+
charwrite_buffer[sizeof(header)+BLCKSZ];
76+
size_twrite_buffer_real_size;
7577

7678
INIT_CRC32C(crc);
7779

@@ -185,12 +187,14 @@ backup_data_file(const char *from_root, const char *to_root,
185187
elog(ERROR,"File: %s blknum %u have wrong page header.",file->path,blknum);
186188
}
187189

190+
188191
if(current.checksum_version&&
189-
pg_checksum_page(page.data,header.block)!= ((PageHeader)page.data)->pd_checksum)
192+
pg_checksum_page(page.data,file->segno*RELSEG_SIZE+blknum)!= ((PageHeader)page.data)->pd_checksum)
190193
{
191194
if (try_checksum)
192195
{
193196
elog(WARNING,"File: %s blknum %u have wrong checksum, try again",file->path,blknum);
197+
usleep(100);
194198
fseek(in,-sizeof(page),SEEK_CUR);
195199
fread(&page,1,sizeof(page),in);
196200
}
@@ -211,10 +215,15 @@ backup_data_file(const char *from_root, const char *to_root,
211215
upper_offset=header.hole_offset+header.hole_length;
212216
upper_length=BLCKSZ-upper_offset;
213217

218+
write_buffer_real_size=sizeof(header)+header.hole_offset+upper_length;
219+
memcpy(write_buffer,&header,sizeof(header));
220+
if (header.hole_offset)
221+
memcpy(write_buffer+sizeof(header),page.data,header.hole_offset);
222+
if (upper_length)
223+
memcpy(write_buffer+sizeof(header)+header.hole_offset,page.data+upper_offset,upper_length);
224+
214225
/* write data page excluding hole */
215-
if (fwrite(&header,1,sizeof(header),out)!=sizeof(header)||
216-
fwrite(page.data,1,header.hole_offset,out)!=header.hole_offset||
217-
fwrite(page.data+upper_offset,1,upper_length,out)!=upper_length)
226+
if(fwrite(write_buffer,1,write_buffer_real_size,out)!=write_buffer_real_size)
218227
{
219228
interrno_tmp=errno;
220229
/* oops */
@@ -273,8 +282,6 @@ backup_data_file(const char *from_root, const char *to_root,
273282
inti;
274283

275284
for(i=0;i<BLCKSZ&&page.data[i]==0;i++);
276-
277-
278285
if (i==BLCKSZ)
279286
{
280287
elog(WARNING,"File: %s blknum %u, empty page",file->path,blknum);
@@ -298,6 +305,7 @@ backup_data_file(const char *from_root, const char *to_root,
298305
if (try_checksum)
299306
{
300307
elog(WARNING,"File: %s blknum %u have wrong page header, try again",file->path,blknum);
308+
usleep(100);
301309
fseek(in,-sizeof(page),SEEK_CUR);
302310
fread(&page,1,sizeof(page),in);
303311
continue;
@@ -307,7 +315,7 @@ backup_data_file(const char *from_root, const char *to_root,
307315
}
308316

309317
if(current.checksum_version&&
310-
pg_checksum_page(page.data,header.block)!= ((PageHeader)page.data)->pd_checksum)
318+
pg_checksum_page(page.data,file->segno*RELSEG_SIZE+blknum)!= ((PageHeader)page.data)->pd_checksum)
311319
{
312320
if (try_checksum)
313321
elog(WARNING,"File: %s blknum %u have wrong checksum, try again",file->path,blknum);
@@ -330,10 +338,15 @@ backup_data_file(const char *from_root, const char *to_root,
330338
upper_offset=header.hole_offset+header.hole_length;
331339
upper_length=BLCKSZ-upper_offset;
332340

341+
write_buffer_real_size=sizeof(header)+header.hole_offset+upper_length;
342+
memcpy(write_buffer,&header,sizeof(header));
343+
if (header.hole_offset)
344+
memcpy(write_buffer+sizeof(header),page.data,header.hole_offset);
345+
if (upper_length)
346+
memcpy(write_buffer+sizeof(header)+header.hole_offset,page.data+upper_offset,upper_length);
347+
333348
/* write data page excluding hole */
334-
if (fwrite(&header,1,sizeof(header),out)!=sizeof(header)||
335-
fwrite(page.data,1,header.hole_offset,out)!=header.hole_offset||
336-
fwrite(page.data+upper_offset,1,upper_length,out)!=upper_length)
349+
if(fwrite(write_buffer,1,write_buffer_real_size,out)!=write_buffer_real_size)
337350
{
338351
interrno_tmp=errno;
339352
/* oops */

‎contrib/pg_arman/dir.c

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ const char *pgdata_exclude[] =
2828
NULL,/* arclog_path will be set later */
2929
NULL,/* 'pg_tblspc' will be set later */
3030
NULL,/* sentinel */
31+
NULL
3132
};
3233

3334
staticpgFile*pgFileNew(constchar*path,boolomit_symlink);
@@ -192,6 +193,21 @@ pgFileComparePathDesc(const void *f1, const void *f2)
192193
return-pgFileComparePath(f1,f2);
193194
}
194195

196+
/* Compare two pgFile with their size */
197+
int
198+
pgFileCompareSize(constvoid*f1,constvoid*f2)
199+
{
200+
pgFile*f1p=*(pgFile**)f1;
201+
pgFile*f2p=*(pgFile**)f2;
202+
203+
if (f1p->size>f2p->size)
204+
return1;
205+
elseif (f1p->size<f2p->size)
206+
return-1;
207+
else
208+
return0;
209+
}
210+
195211
/* Compare two pgFile with their modify timestamp. */
196212
int
197213
pgFileCompareMtime(constvoid*f1,constvoid*f2)
@@ -260,9 +276,13 @@ dir_list_file(parray *files, const char *root, const char *exclude[], bool omit_
260276
fclose(black_list_file);
261277
parray_qsort(black_list,BlackListCompare);
262278
dir_list_file_internal(files,root,exclude,omit_symlink,add_root,black_list);
279+
parray_qsort(files,pgFileComparePath);
263280
}
264281
else
282+
{
265283
dir_list_file_internal(files,root,exclude,omit_symlink,add_root,NULL);
284+
parray_qsort(files,pgFileComparePath);
285+
}
266286
}
267287

268288
void
@@ -406,8 +426,6 @@ dir_list_file_internal(parray *files, const char *root, const char *exclude[],
406426

407427
break;/* pseudo loop */
408428
}
409-
410-
parray_qsort(files,pgFileComparePath);
411429
}
412430

413431
/* print mkdirs.sh */

‎contrib/pg_arman/doc/pg_arman.md

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -244,7 +244,10 @@ absolute paths; relative paths are not allowed.
244244
Number of threads for backup.
245245

246246
**--stream**:
247-
Enable stream replication for save WAL during backup process.
247+
Enable stream replication for save WAL during backup process.
248+
249+
**--disable-ptrack-clear**:
250+
Disable clear ptrack files for postgres without ptrack patch.
248251

249252
###RESTORE OPTIONS
250253

@@ -272,7 +275,7 @@ the same parameters as the ones in recovery.confin recovery.conf.
272275
**--stream**:
273276
Restore without recovery.conf and use pg_xlog WALs. Before you need
274277
backup with**--stream** option. This option will disable all**--recovery-**
275-
options.
278+
options.
276279

277280
###CATALOG OPTIONS
278281

‎contrib/pg_arman/expected/option.out

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,16 @@ Common Options:
1818
-c, --check show what would have been done
1919
-j, --threads=NUM num threads for backup and restore
2020
--stream use stream for save/restore WAL during backup
21+
--progress show progress copy files
2122

2223
Backup options:
2324
-b, --backup-mode=MODE full,page,ptrack
2425
-C, --smooth-checkpoint do smooth checkpoint before backup
2526
--validate validate backup after taking it
2627
--keep-data-generations=N keep GENERATION of full data backup
2728
--keep-data-days=DAY keep enough data backup to recover to DAY days age
29+
--disable-ptrack-clear disable clear ptrack for postgres without ptrack
30+
--backup-pg-log start backup pg_log directory
2831

2932
Restore options:
3033
--recovery-target-time time stamp up to which recovery will proceed

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp