Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitf081a48

Browse files
committed
Unify buffer sizes in pg_dump compression API
Prior to the introduction of the compression API ine996073, pg_dumpwould use the ZLIB_IN_SIZE/ZLIB_OUT_SIZE to size input/output buffers.Commit0da243f introduced similar constants for LZ4, but while gzipdefined both buffers to be 4kB, LZ4 used 4kB and 16kB without any clearreasoning why that's desirable.Furthermore, parts of the code unaware of which compression is used(e.g. pg_backup_directory.c) continued to use ZLIB_OUT_SIZE directly.Simplify by replacing the various constants with DEFAULT_IO_BUFFER_SIZE,set to 4kB. The compression implementations still have an option to usea custom value, but considering 4kB was fine for 20+ years, I find thatunlikely (and we'd probably just increase the default buffer size).Author: Georgios KokolatosReviewed-by: Tomas Vondra, Justin PryzbyDiscussion:https://postgr.es/m/33496f7c-3449-1426-d568-63f6bca2ac1f@gmail.com
1 parentd3b5775 commitf081a48

File tree

5 files changed

+21
-25
lines changed

5 files changed

+21
-25
lines changed

‎src/bin/pg_dump/compress_gzip.c

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -120,8 +120,8 @@ WriteDataToArchiveGzip(ArchiveHandle *AH, CompressorState *cs,
120120
* actually allocate one extra byte because some routines want to
121121
* append a trailing zero byte to the zlib output.
122122
*/
123-
gzipcs->outbuf=pg_malloc(ZLIB_OUT_SIZE+1);
124-
gzipcs->outsize=ZLIB_OUT_SIZE;
123+
gzipcs->outsize=DEFAULT_IO_BUFFER_SIZE;
124+
gzipcs->outbuf=pg_malloc(gzipcs->outsize+1);
125125

126126
/*
127127
* A level of zero simply copies the input one block at the time. This
@@ -158,10 +158,10 @@ ReadDataFromArchiveGzip(ArchiveHandle *AH, CompressorState *cs)
158158
zp->zfree=Z_NULL;
159159
zp->opaque=Z_NULL;
160160

161-
buf=pg_malloc(ZLIB_IN_SIZE);
162-
buflen=ZLIB_IN_SIZE;
161+
buflen=DEFAULT_IO_BUFFER_SIZE;
162+
buf=pg_malloc(buflen);
163163

164-
out=pg_malloc(ZLIB_OUT_SIZE+1);
164+
out=pg_malloc(DEFAULT_IO_BUFFER_SIZE+1);
165165

166166
if (inflateInit(zp)!=Z_OK)
167167
pg_fatal("could not initialize compression library: %s",
@@ -176,14 +176,14 @@ ReadDataFromArchiveGzip(ArchiveHandle *AH, CompressorState *cs)
176176
while (zp->avail_in>0)
177177
{
178178
zp->next_out= (void*)out;
179-
zp->avail_out=ZLIB_OUT_SIZE;
179+
zp->avail_out=DEFAULT_IO_BUFFER_SIZE;
180180

181181
res=inflate(zp,0);
182182
if (res!=Z_OK&&res!=Z_STREAM_END)
183183
pg_fatal("could not uncompress data: %s",zp->msg);
184184

185-
out[ZLIB_OUT_SIZE-zp->avail_out]='\0';
186-
ahwrite(out,1,ZLIB_OUT_SIZE-zp->avail_out,AH);
185+
out[DEFAULT_IO_BUFFER_SIZE-zp->avail_out]='\0';
186+
ahwrite(out,1,DEFAULT_IO_BUFFER_SIZE-zp->avail_out,AH);
187187
}
188188
}
189189

@@ -192,13 +192,13 @@ ReadDataFromArchiveGzip(ArchiveHandle *AH, CompressorState *cs)
192192
while (res!=Z_STREAM_END)
193193
{
194194
zp->next_out= (void*)out;
195-
zp->avail_out=ZLIB_OUT_SIZE;
195+
zp->avail_out=DEFAULT_IO_BUFFER_SIZE;
196196
res=inflate(zp,0);
197197
if (res!=Z_OK&&res!=Z_STREAM_END)
198198
pg_fatal("could not uncompress data: %s",zp->msg);
199199

200-
out[ZLIB_OUT_SIZE-zp->avail_out]='\0';
201-
ahwrite(out,1,ZLIB_OUT_SIZE-zp->avail_out,AH);
200+
out[DEFAULT_IO_BUFFER_SIZE-zp->avail_out]='\0';
201+
ahwrite(out,1,DEFAULT_IO_BUFFER_SIZE-zp->avail_out,AH);
202202
}
203203

204204
if (inflateEnd(zp)!=Z_OK)

‎src/bin/pg_dump/compress_io.h

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,8 @@
1717

1818
#include"pg_backup_archiver.h"
1919

20-
/* Initial buffer sizes used in zlib compression. */
21-
#defineZLIB_OUT_SIZE4096
22-
#defineZLIB_IN_SIZE4096
20+
/* Default size used for IO buffers */
21+
#defineDEFAULT_IO_BUFFER_SIZE4096
2322

2423
externchar*supports_compression(constpg_compress_specificationcompression_spec);
2524

‎src/bin/pg_dump/compress_lz4.c

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,6 @@
2020
#include<lz4.h>
2121
#include<lz4frame.h>
2222

23-
#defineLZ4_OUT_SIZE(4 * 1024)
24-
#defineLZ4_IN_SIZE(16 * 1024)
25-
2623
/*
2724
* LZ4F_HEADER_SIZE_MAX first appeared in v1.7.5 of the library.
2825
* Redefine it for installations with a lesser version.
@@ -57,7 +54,7 @@ ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs)
5754
size_tbuflen;
5855
size_tcnt;
5956

60-
buflen=LZ4_IN_SIZE;
57+
buflen=DEFAULT_IO_BUFFER_SIZE;
6158
buf=pg_malloc(buflen);
6259
decbuf=pg_malloc(buflen);
6360

@@ -208,7 +205,7 @@ LZ4File_init(LZ4File *fs, int size, bool compressing)
208205

209206
if (fs->compressing)
210207
{
211-
fs->buflen=LZ4F_compressBound(LZ4_IN_SIZE,&fs->prefs);
208+
fs->buflen=LZ4F_compressBound(DEFAULT_IO_BUFFER_SIZE,&fs->prefs);
212209
if (fs->buflen<LZ4F_HEADER_SIZE_MAX)
213210
fs->buflen=LZ4F_HEADER_SIZE_MAX;
214211

@@ -244,7 +241,7 @@ LZ4File_init(LZ4File *fs, int size, bool compressing)
244241
return false;
245242
}
246243

247-
fs->buflen=size>LZ4_OUT_SIZE ?size :LZ4_OUT_SIZE;
244+
fs->buflen=Max(size,DEFAULT_IO_BUFFER_SIZE);
248245
fs->buffer=pg_malloc(fs->buflen);
249246

250247
fs->overflowalloclen=fs->buflen;
@@ -423,7 +420,7 @@ LZ4File_write(const void *ptr, size_t size, CompressFileHandle *CFH)
423420

424421
while (remaining>0)
425422
{
426-
intchunk=remaining<LZ4_IN_SIZE ?remaining :LZ4_IN_SIZE;
423+
intchunk=Min(remaining,DEFAULT_IO_BUFFER_SIZE);
427424

428425
remaining-=chunk;
429426

‎src/bin/pg_dump/compress_none.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@ ReadDataFromArchiveNone(ArchiveHandle *AH, CompressorState *cs)
3333
char*buf;
3434
size_tbuflen;
3535

36-
buf=pg_malloc(ZLIB_OUT_SIZE);
37-
buflen=ZLIB_OUT_SIZE;
36+
buflen=DEFAULT_IO_BUFFER_SIZE;
37+
buf=pg_malloc(buflen);
3838

3939
while ((cnt=cs->readF(AH,&buf,&buflen)))
4040
{

‎src/bin/pg_dump/pg_backup_directory.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -394,8 +394,8 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
394394
if (!CFH)
395395
pg_fatal("could not open input file \"%s\": %m",filename);
396396

397-
buf=pg_malloc(ZLIB_OUT_SIZE);
398-
buflen=ZLIB_OUT_SIZE;
397+
buflen=DEFAULT_IO_BUFFER_SIZE;
398+
buf=pg_malloc(buflen);
399399

400400
while (CFH->read_func(buf,buflen,&cnt,CFH)&&cnt>0)
401401
{

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp