Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commita9aab29

Browse files
committed
improvement: use dynamically allocated buffers for stdio buffering
1 parent54f6331 commita9aab29

File tree

5 files changed

+177
-78
lines changed

5 files changed

+177
-78
lines changed

‎src/archive.c‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1556,6 +1556,9 @@ get_wal_file_internal(const char *from_path, const char *to_path, FILE *out,
15561556
}
15571557
gotocleanup;
15581558
}
1559+
1560+
/* disable stdio buffering */
1561+
setvbuf(out,NULL,_IONBF,BUFSIZ);
15591562
}
15601563
#ifdefHAVE_LIBZ
15611564
else

‎src/data.c‎

Lines changed: 122 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -566,8 +566,8 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file,
566566
datapagemap_iterator_t*iter=NULL;
567567

568568
/* stdio buffers */
569-
charin_buffer[STDIO_BUFSIZE];
570-
charout_buffer[STDIO_BUFSIZE];
569+
char*in_buf=NULL;
570+
char*out_buf=NULL;
571571

572572
/* sanity */
573573
if (file->size %BLCKSZ!=0)
@@ -634,17 +634,12 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file,
634634
from_fullpath,strerror(errno));
635635
}
636636

637-
if (!fio_is_remote_file(in))
638-
setvbuf(in,in_buffer,_IOFBF,STDIO_BUFSIZE);
639-
640637
/* open backup file for write */
641638
out=fopen(to_fullpath,PG_BINARY_W);
642639
if (out==NULL)
643640
elog(ERROR,"Cannot open backup file \"%s\": %s",
644641
to_fullpath,strerror(errno));
645642

646-
setvbuf(out,out_buffer,_IOFBF,STDIO_BUFSIZE);
647-
648643
/* update file permission */
649644
if (chmod(to_fullpath,FILE_PERMISSION)==-1)
650645
elog(ERROR,"Cannot change mode of \"%s\": %s",to_fullpath,
@@ -667,6 +662,24 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file,
667662
else
668663
use_pagemap= true;
669664

665+
if (!fio_is_remote_file(in))
666+
{
667+
/* enable stdio buffering for local input file,
668+
* unless the pagemap is involved, which
669+
* imply a lot of random access.
670+
*/
671+
if (use_pagemap)
672+
setvbuf(in,NULL,_IONBF,BUFSIZ);
673+
else
674+
{
675+
in_buf=pgut_malloc(STDIO_BUFSIZE);
676+
setvbuf(in,in_buf,_IOFBF,STDIO_BUFSIZE);
677+
}
678+
}
679+
680+
/* enable stdio buffering for output file */
681+
out_buf=pgut_malloc(STDIO_BUFSIZE);
682+
setvbuf(out,out_buf,_IOFBF,STDIO_BUFSIZE);
670683

671684
/* Remote mode */
672685
if (fio_is_remote_file(in))
@@ -789,6 +802,9 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file,
789802
elog(ERROR,"Cannot remove file \"%s\": %s",to_fullpath,
790803
strerror(errno));
791804
}
805+
806+
pg_free(in_buf);
807+
pg_free(out_buf);
792808
}
793809

794810
/*
@@ -837,18 +853,18 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
837853
size_t
838854
restore_data_file(parray*parent_chain,pgFile*dest_file,FILE*out,constchar*to_fullpath)
839855
{
840-
inti;
856+
inti;
841857
size_ttotal_write_len=0;
842-
charbuffer[STDIO_BUFSIZE];
858+
char*in_buf;
843859

844860
for (i=parray_num(parent_chain)-1;i >=0;i--)
845861
{
846-
charfrom_root[MAXPGPATH];
847-
charfrom_fullpath[MAXPGPATH];
848-
FILE*in=NULL;
862+
charfrom_root[MAXPGPATH];
863+
charfrom_fullpath[MAXPGPATH];
864+
FILE*in=NULL;
849865

850-
pgFile**res_file=NULL;
851-
pgFile*tmp_file=NULL;
866+
pgFile**res_file=NULL;
867+
pgFile*tmp_file=NULL;
852868

853869
pgBackup*backup= (pgBackup*)parray_get(parent_chain,i);
854870

@@ -886,7 +902,8 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char
886902
elog(ERROR,"Cannot open backup file \"%s\": %s",from_fullpath,
887903
strerror(errno));
888904

889-
setvbuf(in,buffer,_IOFBF,STDIO_BUFSIZE);
905+
in_buf=pgut_malloc(STDIO_BUFSIZE);
906+
setvbuf(in,in_buf,_IOFBF,STDIO_BUFSIZE);
890907

891908
/*
892909
* Restore the file.
@@ -902,6 +919,8 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char
902919
elog(ERROR,"Cannot close file \"%s\": %s",from_fullpath,
903920
strerror(errno));
904921
}
922+
pg_free(in_buf);
923+
905924
returntotal_write_len;
906925
}
907926

@@ -912,6 +931,21 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
912931
BackupPageHeaderheader;
913932
BlockNumberblknum=0;
914933
size_twrite_len=0;
934+
off_tcur_pos=0;
935+
936+
/*
937+
* We rely on stdio buffering of input and output.
938+
* For buffering to be efficient, we try to minimize the
939+
* number of lseek syscalls, because it forces buffer flush.
940+
* For that, we track current write position in
941+
* output file and issue fseek only when offset of block to be
942+
* written not equal to current write position, which happens
943+
* a lot when blocks from incremental backup are restored,
944+
* but should never happen in case of blocks from FULL backup.
945+
*/
946+
if (fio_fseek(out,cur_pos)<0)
947+
elog(ERROR,"Cannot seek block %u of \"%s\": %s",
948+
blknum,to_fullpath,strerror(errno));
915949

916950
for (;;)
917951
{
@@ -928,23 +962,24 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
928962
/* read BackupPageHeader */
929963
read_len=fread(&header,1,sizeof(header),in);
930964

965+
if (ferror(in))
966+
elog(ERROR,"Cannot read header of block %u of \"%s\": %s",
967+
blknum,from_fullpath,strerror(errno));
968+
931969
if (read_len!=sizeof(header))
932970
{
933-
interrno_tmp=errno;
934971
if (read_len==0&&feof(in))
935972
break;/* EOF found */
936-
elseif (read_len!=0&&feof(in))
973+
974+
if (read_len!=0&&feof(in))
937975
elog(ERROR,"Odd size page found at block %u of \"%s\"",
938976
blknum,from_fullpath);
939-
else
940-
elog(ERROR,"Cannot read header of block %u of \"%s\": %s",
941-
blknum,from_fullpath,strerror(errno_tmp));
942977
}
943978

944979
/* Consider empty blockm. wtf empty block ? */
945980
if (header.block==0&&header.compressed_size==0)
946981
{
947-
elog(VERBOSE,"Skip empty block of \"%s\"",from_fullpath);
982+
elog(WARNING,"Skip empty block of \"%s\"",from_fullpath);
948983
continue;
949984
}
950985

@@ -1019,14 +1054,19 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
10191054
is_compressed= true;
10201055
}
10211056

1022-
write_pos=blknum*BLCKSZ;
1023-
10241057
/*
10251058
* Seek and write the restored page.
1059+
* When restoring file from FULL backup, pages are written sequentially,
1060+
* so there is no need to issue fseek for every page.
10261061
*/
1027-
if (fio_fseek(out,write_pos)<0)
1028-
elog(ERROR,"Cannot seek block %u of \"%s\": %s",
1029-
blknum,to_fullpath,strerror(errno));
1062+
write_pos=blknum*BLCKSZ;
1063+
1064+
if (cur_pos!=write_pos)
1065+
{
1066+
if (fio_fseek(out,blknum*BLCKSZ)<0)
1067+
elog(ERROR,"Cannot seek block %u of \"%s\": %s",
1068+
blknum,to_fullpath,strerror(errno));
1069+
}
10301070

10311071
/* If page is compressed and restore is in remote mode, send compressed
10321072
* page to the remote side.
@@ -1048,6 +1088,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
10481088
}
10491089

10501090
write_len+=BLCKSZ;
1091+
cur_pos=write_pos+BLCKSZ;/* update current write position */
10511092
}
10521093

10531094
elog(VERBOSE,"Copied file \"%s\": %lu bytes",from_fullpath,write_len);
@@ -1063,8 +1104,8 @@ void
10631104
restore_non_data_file_internal(FILE*in,FILE*out,pgFile*file,
10641105
constchar*from_fullpath,constchar*to_fullpath)
10651106
{
1066-
ssize_tread_len=0;
1067-
charbuf[STDIO_BUFSIZE];/* 64kB buffer */
1107+
size_tread_len=0;
1108+
char*buf=pgut_malloc(STDIO_BUFSIZE);/* 64kB buffer */
10681109

10691110
/* copy content */
10701111
for (;;)
@@ -1075,20 +1116,25 @@ restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file,
10751116
if (interrupted||thread_interrupted)
10761117
elog(ERROR,"Interrupted during non-data file restore");
10771118

1078-
read_len=fread(buf,1,sizeof(buf),in);
1079-
1080-
if (read_len==0&&feof(in))
1081-
break;
1119+
read_len=fread(buf,1,STDIO_BUFSIZE,in);
10821120

1083-
if (read_len<0)
1121+
if (ferror(in))
10841122
elog(ERROR,"Cannot read backup file \"%s\": %s",
1085-
from_fullpath,strerror(errno));
1123+
from_fullpath,strerror(errno));
10861124

1087-
if (fio_fwrite(out,buf,read_len)!=read_len)
1088-
elog(ERROR,"Cannot write to \"%s\": %s",to_fullpath,
1089-
strerror(errno));
1125+
if (read_len>0)
1126+
{
1127+
if (fio_fwrite(out,buf,read_len)!=read_len)
1128+
elog(ERROR,"Cannot write to \"%s\": %s",to_fullpath,
1129+
strerror(errno));
1130+
}
1131+
1132+
if (feof(in))
1133+
break;
10901134
}
10911135

1136+
pg_free(buf);
1137+
10921138
elog(VERBOSE,"Copied file \"%s\": %lu bytes",from_fullpath,file->write_size);
10931139
}
10941140

@@ -1103,7 +1149,6 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
11031149

11041150
pgFile*tmp_file=NULL;
11051151
pgBackup*tmp_backup=NULL;
1106-
charbuffer[STDIO_BUFSIZE];
11071152

11081153
/* Check if full copy of destination file is available in destination backup */
11091154
if (dest_file->write_size>0)
@@ -1176,7 +1221,8 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
11761221
elog(ERROR,"Cannot open backup file \"%s\": %s",from_fullpath,
11771222
strerror(errno));
11781223

1179-
setvbuf(in,buffer,_IOFBF,STDIO_BUFSIZE);
1224+
/* disable stdio buffering for non-data files */
1225+
setvbuf(in,NULL,_IONBF,BUFSIZ);
11801226

11811227
/* do actual work */
11821228
restore_non_data_file_internal(in,out,tmp_file,from_fullpath,to_fullpath);
@@ -1192,17 +1238,18 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
11921238
* Copy file to backup.
11931239
* We do not apply compression to these files, because
11941240
* it is either small control file or already compressed cfs file.
1241+
* TODO: optimize remote copying
11951242
*/
11961243
void
11971244
backup_non_data_file_internal(constchar*from_fullpath,
11981245
fio_locationfrom_location,
11991246
constchar*to_fullpath,pgFile*file,
12001247
boolmissing_ok)
12011248
{
1202-
FILE*in;
1203-
FILE*out;
1204-
ssize_tread_len=0;
1205-
charbuf[STDIO_BUFSIZE];/* 64kB buffer */
1249+
FILE*in;
1250+
FILE*out;
1251+
ssize_tread_len=0;
1252+
char*buf;
12061253
pg_crc32crc;
12071254

12081255
INIT_FILE_CRC32(true,crc);
@@ -1247,18 +1294,26 @@ backup_non_data_file_internal(const char *from_fullpath,
12471294
elog(ERROR,"Cannot change mode of \"%s\": %s",to_fullpath,
12481295
strerror(errno));
12491296

1297+
/* disable stdio buffering for local input/output files */
1298+
if (!fio_is_remote_file(in))
1299+
setvbuf(in,NULL,_IONBF,BUFSIZ);
1300+
setvbuf(out,NULL,_IONBF,BUFSIZ);
1301+
1302+
/* allocate 64kB buffer */
1303+
buf=pgut_malloc(STDIO_BUFSIZE);
1304+
12501305
/* copy content and calc CRC */
12511306
for (;;)
12521307
{
1253-
read_len=fio_fread(in,buf,sizeof(buf));
1254-
1255-
if (read_len==0)
1256-
break;
1308+
read_len=fio_fread(in,buf,STDIO_BUFSIZE);
12571309

12581310
if (read_len<0)
12591311
elog(ERROR,"Cannot read from source file \"%s\": %s",
12601312
from_fullpath,strerror(errno));
12611313

1314+
if (read_len==0)
1315+
break;
1316+
12621317
if (fwrite(buf,1,read_len,out)!=read_len)
12631318
elog(ERROR,"Cannot write to \"%s\": %s",to_fullpath,
12641319
strerror(errno));
@@ -1267,6 +1322,19 @@ backup_non_data_file_internal(const char *from_fullpath,
12671322
COMP_FILE_CRC32(true,crc,buf,read_len);
12681323

12691324
file->read_size+=read_len;
1325+
1326+
//if (read_len < STDIO_BUFSIZE)
1327+
//{
1328+
//if (!fio_is_remote_file(in))
1329+
//{
1330+
//if (ferror(in))
1331+
//elog(ERROR, "Cannot read from source file \"%s\": %s",
1332+
//from_fullpath, strerror(errno));
1333+
//
1334+
//if (feof(in))
1335+
//break;
1336+
//}
1337+
//}
12701338
}
12711339

12721340
file->write_size= (int64)file->read_size;
@@ -1280,6 +1348,7 @@ backup_non_data_file_internal(const char *from_fullpath,
12801348
if (fclose(out))
12811349
elog(ERROR,"Cannot write \"%s\": %s",to_fullpath,strerror(errno));
12821350
fio_fclose(in);
1351+
pg_free(buf);
12831352
}
12841353

12851354
/*
@@ -1478,9 +1547,13 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
14781547

14791548
/* read BackupPageHeader */
14801549
read_len=fread(&header,1,sizeof(header),in);
1550+
1551+
if (ferror(in))
1552+
elog(ERROR,"Cannot read header of block %u of \"%s\": %s",
1553+
blknum,file->path,strerror(errno));
1554+
14811555
if (read_len!=sizeof(header))
14821556
{
1483-
interrno_tmp=errno;
14841557
if (read_len==0&&feof(in))
14851558
break;/* EOF found */
14861559
elseif (read_len!=0&&feof(in))
@@ -1489,7 +1562,7 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
14891562
blknum,file->path);
14901563
else
14911564
elog(WARNING,"Cannot read header of block %u of \"%s\": %s",
1492-
blknum,file->path,strerror(errno_tmp));
1565+
blknum,file->path,strerror(errno));
14931566
return false;
14941567
}
14951568

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp