Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitd62bcc8

Browse files
committed
Rework compression options of pg_receivewal
pg_receivewal includes sincecada1af the option --compress, to allow thecompression of WAL segments using gzip, with a value of 0 (the default)meaning that no compression can be used.This commit introduces a new option, called --compression-method, ableto use as values "none", the default, and "gzip", to make things moreextensible. The case of --compress=0 becomes fuzzy with this optionlayer, so we have made the choice to make pg_receivewal return an errorwhen using "none" and a non-zero compression level, meaning that theauthorized values of --compress are now [1,9] instead of [0,9]. Notspecifying --compress with "gzip" as compression method makespg_receivewal use the default of zlib instead (Z_DEFAULT_COMPRESSION).The code in charge of finding the streaming start LSN when scanning theexisting archives is refactored and made more extensible. While on it,rename "compression" to "compression_level" in walmethods.c, to reducethe confusion with the introduction of the compression method, even ifthe tar method used by pg_basebackup does not rely on the compressionmethod (yet, at least), but just on the compression level (this areacould be improved more, actually).This is in preparation for an upcoming patch that adds LZ4 support topg_receivewal.Author: Georgios KokolatosReviewed-by: Michael Paquier, Jian Guo, Magnus Hagander, Dilip Kumar,Robert HaasDiscussion:https://postgr.es/m/ZCm1J5vfyQ2E6dYvXz8si39HQ2gwxSZ3IpYaVgYa3lUwY88SLapx9EEnOf5uEwrddhx2twG7zYKjVeuP5MwZXCNPybtsGouDsAD1o2L_I5E=@pm.me
1 parent581055c commitd62bcc8

File tree

8 files changed

+231
-103
lines changed

8 files changed

+231
-103
lines changed

‎doc/src/sgml/ref/pg_receivewal.sgml

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -263,15 +263,36 @@ PostgreSQL documentation
263263
</listitem>
264264
</varlistentry>
265265

266+
<varlistentry>
267+
<term><option>--compression-method=<replaceable class="parameter">level</replaceable></option></term>
268+
<listitem>
269+
<para>
270+
Enables compression of write-ahead logs using the specified method.
271+
Supported values <literal>gzip</literal>, and
272+
<literal>none</literal>.
273+
</para>
274+
275+
<para>
276+
The suffix <filename>.gz</filename> will automatically be added to
277+
all filenames when using <literal>gzip</literal>
278+
</para>
279+
</listitem>
280+
</varlistentry>
281+
266282
<varlistentry>
267283
<term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
268284
<term><option>--compress=<replaceable class="parameter">level</replaceable></option></term>
269285
<listitem>
270286
<para>
271-
Enables gzip compression of write-ahead logs, and specifies the
272-
compression level (0 through 9, 0 being no compression and 9 being best
273-
compression). The suffix <filename>.gz</filename> will
274-
automatically be added to all filenames.
287+
Specifies the compression level (<literal>1</literal> through
288+
<literal>9</literal>, <literal>1</literal> being worst compression
289+
and <literal>9</literal> being best compression) for WAL segments
290+
compressed with <application>gzip</application>.
291+
</para>
292+
293+
<para>
294+
This option requires <option>--compression-method</option> to be
295+
specified with <literal>gzip</literal>.
275296
</para>
276297
</listitem>
277298
</varlistentry>

‎src/bin/pg_basebackup/pg_basebackup.c

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -555,10 +555,13 @@ LogStreamerMain(logstreamer_param *param)
555555
stream.replication_slot=replication_slot;
556556

557557
if (format=='p')
558-
stream.walmethod=CreateWalDirectoryMethod(param->xlog,0,
558+
stream.walmethod=CreateWalDirectoryMethod(param->xlog,
559+
COMPRESSION_NONE,0,
559560
stream.do_sync);
560561
else
561-
stream.walmethod=CreateWalTarMethod(param->xlog,compresslevel,
562+
stream.walmethod=CreateWalTarMethod(param->xlog,
563+
COMPRESSION_NONE,/* ignored */
564+
compresslevel,
562565
stream.do_sync);
563566

564567
if (!ReceiveXlogStream(param->bgconn,&stream))

‎src/bin/pg_basebackup/pg_receivewal.c

Lines changed: 117 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@
1919
#include<signal.h>
2020
#include<sys/stat.h>
2121
#include<unistd.h>
22+
#ifdefHAVE_LIBZ
23+
#include<zlib.h>
24+
#endif
2225

2326
#include"access/xlog_internal.h"
2427
#include"common/file_perm.h"
@@ -45,6 +48,7 @@ static bool do_drop_slot = false;
4548
staticbooldo_sync= true;
4649
staticboolsynchronous= false;
4750
staticchar*replication_slot=NULL;
51+
staticWalCompressionMethodcompression_method=COMPRESSION_NONE;
4852
staticXLogRecPtrendpos=InvalidXLogRecPtr;
4953

5054

@@ -63,16 +67,6 @@ disconnect_atexit(void)
6367
PQfinish(conn);
6468
}
6569

66-
/* Routines to evaluate segment file format */
67-
#defineIsCompressXLogFileName(fname) \
68-
(strlen(fname) == XLOG_FNAME_LEN + strlen(".gz") && \
69-
strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN &&\
70-
strcmp((fname) + XLOG_FNAME_LEN, ".gz") == 0)
71-
#defineIsPartialCompressXLogFileName(fname)\
72-
(strlen(fname) == XLOG_FNAME_LEN + strlen(".gz.partial") && \
73-
strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN &&\
74-
strcmp((fname) + XLOG_FNAME_LEN, ".gz.partial") == 0)
75-
7670
staticvoid
7771
usage(void)
7872
{
@@ -92,7 +86,9 @@ usage(void)
9286
printf(_(" --synchronous flush write-ahead log immediately after writing\n"));
9387
printf(_(" -v, --verbose output verbose messages\n"));
9488
printf(_(" -V, --version output version information, then exit\n"));
95-
printf(_(" -Z, --compress=0-9 compress logs with given compression level\n"));
89+
printf(_(" --compression-method=METHOD\n"
90+
" method to compress logs\n"));
91+
printf(_(" -Z, --compress=1-9 compress logs with given compression level\n"));
9692
printf(_(" -?, --help show this help, then exit\n"));
9793
printf(_("\nConnection options:\n"));
9894
printf(_(" -d, --dbname=CONNSTR connection string\n"));
@@ -108,6 +104,60 @@ usage(void)
108104
printf(_("%s home page: <%s>\n"),PACKAGE_NAME,PACKAGE_URL);
109105
}
110106

107+
/*
108+
* Check if the filename looks like a WAL file, letting caller know if this
109+
* WAL segment is partial and/or compressed.
110+
*/
111+
staticbool
112+
is_xlogfilename(constchar*filename,bool*ispartial,
113+
WalCompressionMethod*wal_compression_method)
114+
{
115+
size_tfname_len=strlen(filename);
116+
size_txlog_pattern_len=strspn(filename,"0123456789ABCDEF");
117+
118+
/* File does not look like a WAL file */
119+
if (xlog_pattern_len!=XLOG_FNAME_LEN)
120+
return false;
121+
122+
/* File looks like a completed uncompressed WAL file */
123+
if (fname_len==XLOG_FNAME_LEN)
124+
{
125+
*ispartial= false;
126+
*wal_compression_method=COMPRESSION_NONE;
127+
return true;
128+
}
129+
130+
/* File looks like a completed gzip-compressed WAL file */
131+
if (fname_len==XLOG_FNAME_LEN+strlen(".gz")&&
132+
strcmp(filename+XLOG_FNAME_LEN,".gz")==0)
133+
{
134+
*ispartial= false;
135+
*wal_compression_method=COMPRESSION_GZIP;
136+
return true;
137+
}
138+
139+
/* File looks like a partial uncompressed WAL file */
140+
if (fname_len==XLOG_FNAME_LEN+strlen(".partial")&&
141+
strcmp(filename+XLOG_FNAME_LEN,".partial")==0)
142+
{
143+
*ispartial= true;
144+
*wal_compression_method=COMPRESSION_NONE;
145+
return true;
146+
}
147+
148+
/* File looks like a partial gzip-compressed WAL file */
149+
if (fname_len==XLOG_FNAME_LEN+strlen(".gz.partial")&&
150+
strcmp(filename+XLOG_FNAME_LEN,".gz.partial")==0)
151+
{
152+
*ispartial= true;
153+
*wal_compression_method=COMPRESSION_GZIP;
154+
return true;
155+
}
156+
157+
/* File does not look like something we know */
158+
return false;
159+
}
160+
111161
staticbool
112162
stop_streaming(XLogRecPtrxlogpos,uint32timeline,boolsegment_finished)
113163
{
@@ -213,33 +263,11 @@ FindStreamingStart(uint32 *tli)
213263
{
214264
uint32tli;
215265
XLogSegNosegno;
266+
WalCompressionMethodwal_compression_method;
216267
boolispartial;
217-
booliscompress;
218268

219-
/*
220-
* Check if the filename looks like an xlog file, or a .partial file.
221-
*/
222-
if (IsXLogFileName(dirent->d_name))
223-
{
224-
ispartial= false;
225-
iscompress= false;
226-
}
227-
elseif (IsPartialXLogFileName(dirent->d_name))
228-
{
229-
ispartial= true;
230-
iscompress= false;
231-
}
232-
elseif (IsCompressXLogFileName(dirent->d_name))
233-
{
234-
ispartial= false;
235-
iscompress= true;
236-
}
237-
elseif (IsPartialCompressXLogFileName(dirent->d_name))
238-
{
239-
ispartial= true;
240-
iscompress= true;
241-
}
242-
else
269+
if (!is_xlogfilename(dirent->d_name,
270+
&ispartial,&wal_compression_method))
243271
continue;
244272

245273
/*
@@ -250,14 +278,14 @@ FindStreamingStart(uint32 *tli)
250278
/*
251279
* Check that the segment has the right size, if it's supposed to be
252280
* completed. For non-compressed segments just check the on-disk size
253-
* and see if it matches a completed segment. For compressed segments,
254-
* look at the last 4 bytes of the compressed file, which is where the
255-
* uncompressed size is located forgzfiles with a size lower than
256-
* 4GB, and then compare it to the size of a completed segment. The 4
257-
* last bytes correspond to the ISIZE member according to
281+
* and see if it matches a completed segment. Forgzip-compressed
282+
*segments,look at the last 4 bytes of the compressed file, which is
283+
*where theuncompressed size is located for files with a size lower
284+
*than4GB, and then compare it to the size of a completed segment.
285+
*The 4last bytes correspond to the ISIZE member according to
258286
* http://www.zlib.org/rfc-gzip.html.
259287
*/
260-
if (!ispartial&&!iscompress)
288+
if (!ispartial&&wal_compression_method==COMPRESSION_NONE)
261289
{
262290
structstatstatbuf;
263291
charfullpath[MAXPGPATH*2];
@@ -276,7 +304,7 @@ FindStreamingStart(uint32 *tli)
276304
continue;
277305
}
278306
}
279-
elseif (!ispartial&&iscompress)
307+
elseif (!ispartial&&wal_compression_method==COMPRESSION_GZIP)
280308
{
281309
intfd;
282310
charbuf[4];
@@ -457,7 +485,9 @@ StreamLog(void)
457485
stream.synchronous=synchronous;
458486
stream.do_sync=do_sync;
459487
stream.mark_done= false;
460-
stream.walmethod=CreateWalDirectoryMethod(basedir,compresslevel,
488+
stream.walmethod=CreateWalDirectoryMethod(basedir,
489+
compression_method,
490+
compresslevel,
461491
stream.do_sync);
462492
stream.partial_suffix=".partial";
463493
stream.replication_slot=replication_slot;
@@ -510,6 +540,7 @@ main(int argc, char **argv)
510540
{"status-interval",required_argument,NULL,'s'},
511541
{"slot",required_argument,NULL,'S'},
512542
{"verbose",no_argument,NULL,'v'},
543+
{"compression-method",required_argument,NULL,'I'},
513544
{"compress",required_argument,NULL,'Z'},
514545
/* action */
515546
{"create-slot",no_argument,NULL,1},
@@ -595,8 +626,20 @@ main(int argc, char **argv)
595626
case'v':
596627
verbose++;
597628
break;
629+
case'I':
630+
if (pg_strcasecmp(optarg,"gzip")==0)
631+
compression_method=COMPRESSION_GZIP;
632+
elseif (pg_strcasecmp(optarg,"none")==0)
633+
compression_method=COMPRESSION_NONE;
634+
else
635+
{
636+
pg_log_error("invalid value \"%s\" for option %s",
637+
optarg,"--compress-method");
638+
exit(1);
639+
}
640+
break;
598641
case'Z':
599-
if (!option_parse_int(optarg,"-Z/--compress",0,9,
642+
if (!option_parse_int(optarg,"-Z/--compress",1,9,
600643
&compresslevel))
601644
exit(1);
602645
break;
@@ -676,13 +719,37 @@ main(int argc, char **argv)
676719
exit(1);
677720
}
678721

679-
#ifndefHAVE_LIBZ
680-
if (compresslevel!=0)
722+
723+
/*
724+
* Compression-related options.
725+
*/
726+
switch (compression_method)
681727
{
682-
pg_log_error("this build does not support compression");
683-
exit(1);
684-
}
728+
caseCOMPRESSION_NONE:
729+
if (compresslevel!=0)
730+
{
731+
pg_log_error("cannot use --compress with --compression-method=%s",
732+
"none");
733+
fprintf(stderr,_("Try \"%s --help\" for more information.\n"),
734+
progname);
735+
exit(1);
736+
}
737+
break;
738+
caseCOMPRESSION_GZIP:
739+
#ifdefHAVE_LIBZ
740+
if (compresslevel==0)
741+
{
742+
pg_log_info("no value specified for --compress, switching to default");
743+
compresslevel=Z_DEFAULT_COMPRESSION;
744+
}
745+
#else
746+
pg_log_error("this build does not support compression with %s",
747+
"gzip");
748+
exit(1);
685749
#endif
750+
break;
751+
}
752+
686753

687754
/*
688755
* Check existence of destination folder.

‎src/bin/pg_basebackup/receivelog.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint)
109109
* When streaming to tar, no file with this name will exist before, so we
110110
* never have to verify a size.
111111
*/
112-
if (stream->walmethod->compression()==0&&
112+
if (stream->walmethod->compression_method()==COMPRESSION_NONE&&
113113
stream->walmethod->existsfile(fn))
114114
{
115115
size=stream->walmethod->get_file_size(fn);

‎src/bin/pg_basebackup/t/020_pg_receivewal.pl

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
use warnings;
66
use PostgreSQL::Test::Utils;
77
use PostgreSQL::Test::Cluster;
8-
use Test::Moretests=>35;
8+
use Test::Moretests=>37;
99

1010
program_help_ok('pg_receivewal');
1111
program_version_ok('pg_receivewal');
@@ -33,6 +33,13 @@
3333
$primary->command_fails(
3434
['pg_receivewal','-D',$stream_dir,'--synchronous','--no-sync' ],
3535
'failure if --synchronous specified with --no-sync');
36+
$primary->command_fails_like(
37+
[
38+
'pg_receivewal','-D',$stream_dir,'--compression-method','none',
39+
'--compress','1'
40+
],
41+
qr/\Qpg_receivewal: error: cannot use --compress with --compression-method=none/,
42+
'failure if --compress spwcified with --compression-method=none');
3643

3744
# Slot creation and drop
3845
my$slot_name ='test';
@@ -90,8 +97,11 @@
9097
# a valid value.
9198
$primary->command_ok(
9299
[
93-
'pg_receivewal','-D',$stream_dir,'--verbose',
94-
'--endpos',$nextlsn,'--compress','1',
100+
'pg_receivewal','-D',
101+
$stream_dir,'--verbose',
102+
'--endpos',$nextlsn,
103+
'--compression-method','gzip',
104+
'--compress','1',
95105
'--no-loop'
96106
],
97107
"streaming some WAL using ZLIB compression");

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp