Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit5c649fe

Browse files
committed
Extend the options of pg_basebackup to control compression
The option --compress is extended to accept a compression method and anoptional compression level, as of the grammar METHOD[:LEVEL]. Themethods currently support are "none" and "gzip", for client-sidecompression. Any of those methods use only an integer value for thecompression level, but any method implemented in the future could usemore specific keywords if necessary.This commit keeps the logic backward-compatible. Hence, the followingcompatibility rules apply for the new format of the option --compress:* -z/--gzip is a synonym of --compress=gzip.* --compress=NUM implies:** --compress=none if NUM = 0.** --compress=gzip:NUM if NUM > 0.Note that there are also plans to extend more this grammar withserver-side compression.Reviewed-by: Robert Haas, Magnus Hagander, Álvaro Herrera, DavidG. Johnston, Georgios KokolatosDiscussion:https://postgr.es/m/Yb3GEgWwcu4wZDuA@paquier.xyz
1 parent512fc2d commit5c649fe

File tree

3 files changed

+179
-24
lines changed

3 files changed

+179
-24
lines changed

‎doc/src/sgml/ref/pg_basebackup.sgml

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -398,15 +398,24 @@ PostgreSQL documentation
398398

399399
<varlistentry>
400400
<term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
401+
<term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
401402
<term><option>--compress=<replaceable class="parameter">level</replaceable></option></term>
403+
<term><option>--compress=<replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
402404
<listitem>
403405
<para>
404-
Enablesgzipcompression of tar file output, and specifies the
406+
Enables compression of tar file output, and specifies the
405407
compression level (0 through 9, 0 being no compression and 9 being best
406408
compression). Compression is only available when using the tar
407409
format, and the suffix <filename>.gz</filename> will
408410
automatically be added to all tar filenames.
409411
</para>
412+
<para>
413+
The compression method can be set to either <literal>gzip</literal>
414+
for compression with <application>gzip</application>, or
415+
<literal>none</literal> for no compression. A compression level
416+
can be optionally specified, by appending the level number after a
417+
colon (<literal>:</literal>).
418+
</para>
410419
</listitem>
411420
</varlistentry>
412421
</variablelist>
@@ -942,6 +951,16 @@ PostgreSQL documentation
942951
<screen>
943952
<prompt>$</prompt> <userinput>pg_basebackup -D backup/data -T /opt/ts=$(pwd)/backup/ts</userinput>
944953
</screen></para>
954+
955+
<para>
956+
To create a backup of a local server with one tar file for each tablespace
957+
compressed with <application>gzip</application> at level 9, stored in the
958+
directory <filename>backup</filename>:
959+
<screen>
960+
<prompt>$</prompt> <userinput>pg_basebackup -D backup -Ft --compress=gzip:9</userinput>
961+
</screen>
962+
</para>
963+
945964
</refsect1>
946965

947966
<refsect1>

‎src/bin/pg_basebackup/pg_basebackup.c

Lines changed: 127 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ static bool showprogress = false;
123123
staticboolestimatesize= true;
124124
staticintverbose=0;
125125
staticintcompresslevel=0;
126+
staticWalCompressionMethodcompressmethod=COMPRESSION_NONE;
126127
staticIncludeWalincludewal=STREAM_WAL;
127128
staticboolfastcheckpoint= false;
128129
staticboolwriterecoveryconf= false;
@@ -379,7 +380,8 @@ usage(void)
379380
printf(_(" -X, --wal-method=none|fetch|stream\n"
380381
" include required WAL files with specified method\n"));
381382
printf(_(" -z, --gzip compress tar output\n"));
382-
printf(_(" -Z, --compress=0-9 compress tar output with given compression level\n"));
383+
printf(_(" -Z, --compress={gzip,none}[:LEVEL] or [LEVEL]\n"
384+
" compress tar output with given compression method or level\n"));
383385
printf(_("\nGeneral options:\n"));
384386
printf(_(" -c, --checkpoint=fast|spread\n"
385387
" set fast or spread checkpointing\n"));
@@ -544,8 +546,7 @@ LogStreamerMain(logstreamer_param *param)
544546
stream.do_sync);
545547
else
546548
stream.walmethod=CreateWalTarMethod(param->xlog,
547-
(compresslevel!=0) ?
548-
COMPRESSION_GZIP :COMPRESSION_NONE,
549+
compressmethod,
549550
compresslevel,
550551
stream.do_sync);
551552

@@ -936,6 +937,81 @@ parse_max_rate(char *src)
936937
return (int32)result;
937938
}
938939

940+
/*
941+
* Utility wrapper to parse the values specified for -Z/--compress.
942+
* *methodres and *levelres will be optionally filled with values coming
943+
* from the parsed results.
944+
*/
945+
staticvoid
946+
parse_compress_options(char*src,WalCompressionMethod*methodres,
947+
int*levelres)
948+
{
949+
char*sep;
950+
intfirstlen;
951+
char*firstpart=NULL;
952+
953+
/* check if the option is split in two */
954+
sep=strchr(src,':');
955+
956+
/*
957+
* The first part of the option value could be a method name, or just a
958+
* level value.
959+
*/
960+
firstlen= (sep!=NULL) ? (sep-src) :strlen(src);
961+
firstpart=pg_malloc(firstlen+1);
962+
strncpy(firstpart,src,firstlen);
963+
firstpart[firstlen]='\0';
964+
965+
/*
966+
* Check if the first part of the string matches with a supported
967+
* compression method.
968+
*/
969+
if (pg_strcasecmp(firstpart,"gzip")==0)
970+
*methodres=COMPRESSION_GZIP;
971+
elseif (pg_strcasecmp(firstpart,"none")==0)
972+
*methodres=COMPRESSION_NONE;
973+
else
974+
{
975+
/*
976+
* It does not match anything known, so check for the
977+
* backward-compatible case of only an integer where the implied
978+
* compression method changes depending on the level value.
979+
*/
980+
if (!option_parse_int(firstpart,"-Z/--compress",0,
981+
INT_MAX,levelres))
982+
exit(1);
983+
984+
*methodres= (*levelres>0) ?
985+
COMPRESSION_GZIP :COMPRESSION_NONE;
986+
return;
987+
}
988+
989+
if (sep==NULL)
990+
{
991+
/*
992+
* The caller specified a method without a colon separator, so let any
993+
* subsequent checks assign a default level.
994+
*/
995+
return;
996+
}
997+
998+
/* Check the contents after the colon separator. */
999+
sep++;
1000+
if (*sep=='\0')
1001+
{
1002+
pg_log_error("no compression level defined for method %s",firstpart);
1003+
exit(1);
1004+
}
1005+
1006+
/*
1007+
* For any of the methods currently supported, the data after the
1008+
* separator can just be an integer.
1009+
*/
1010+
if (!option_parse_int(sep,"-Z/--compress",0,INT_MAX,
1011+
levelres))
1012+
exit(1);
1013+
}
1014+
9391015
/*
9401016
* Read a stream of COPY data and invoke the provided callback for each
9411017
* chunk.
@@ -996,7 +1072,7 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
9961072
boolis_recovery_guc_supported,
9971073
boolexpect_unterminated_tarfile)
9981074
{
999-
bbstreamer*streamer;
1075+
bbstreamer*streamer=NULL;
10001076
bbstreamer*manifest_inject_streamer=NULL;
10011077
boolinject_manifest;
10021078
boolmust_parse_archive;
@@ -1055,19 +1131,22 @@ CreateBackupStreamer(char *archive_name, char *spclocation,
10551131
archive_file=NULL;
10561132
}
10571133

1134+
if (compressmethod==COMPRESSION_NONE)
1135+
streamer=bbstreamer_plain_writer_new(archive_filename,
1136+
archive_file);
10581137
#ifdefHAVE_LIBZ
1059-
if (compresslevel!=0)
1138+
elseif (compressmethod==COMPRESSION_GZIP)
10601139
{
10611140
strlcat(archive_filename,".gz",sizeof(archive_filename));
10621141
streamer=bbstreamer_gzip_writer_new(archive_filename,
10631142
archive_file,
10641143
compresslevel);
10651144
}
1066-
else
10671145
#endif
1068-
streamer=bbstreamer_plain_writer_new(archive_filename,
1069-
archive_file);
1070-
1146+
else
1147+
{
1148+
Assert(false);/* not reachable */
1149+
}
10711150

10721151
/*
10731152
* If we need to parse the archive for whatever reason, then we'll
@@ -2279,11 +2358,11 @@ main(int argc, char **argv)
22792358
#else
22802359
compresslevel=1;/* will be rejected below */
22812360
#endif
2361+
compressmethod=COMPRESSION_GZIP;
22822362
break;
22832363
case'Z':
2284-
if (!option_parse_int(optarg,"-Z/--compress",0,9,
2285-
&compresslevel))
2286-
exit(1);
2364+
parse_compress_options(optarg,&compressmethod,
2365+
&compresslevel);
22872366
break;
22882367
case'c':
22892368
if (pg_strcasecmp(optarg,"fast")==0)
@@ -2412,7 +2491,7 @@ main(int argc, char **argv)
24122491
/*
24132492
* Compression doesn't make sense unless tar format is in use.
24142493
*/
2415-
if (format=='p'&&compresslevel!=0)
2494+
if (format=='p'&&compressmethod!=COMPRESSION_NONE)
24162495
{
24172496
if (backup_target==NULL)
24182497
pg_log_error("only tar mode backups can be compressed");
@@ -2516,14 +2595,43 @@ main(int argc, char **argv)
25162595
}
25172596
}
25182597

2519-
#ifndefHAVE_LIBZ
2520-
/* Sanity checks for compression level. */
2521-
if (compresslevel!=0)
2598+
/* Sanity checks for compression-related options. */
2599+
switch (compressmethod)
25222600
{
2523-
pg_log_error("this build does not support compression");
2524-
exit(1);
2525-
}
2601+
caseCOMPRESSION_NONE:
2602+
if (compresslevel!=0)
2603+
{
2604+
pg_log_error("cannot use compression level with method %s",
2605+
"none");
2606+
fprintf(stderr,_("Try \"%s --help\" for more information.\n"),
2607+
progname);
2608+
exit(1);
2609+
}
2610+
break;
2611+
caseCOMPRESSION_GZIP:
2612+
#ifdefHAVE_LIBZ
2613+
if (compresslevel==0)
2614+
{
2615+
pg_log_info("no value specified for compression level, switching to default");
2616+
compresslevel=Z_DEFAULT_COMPRESSION;
2617+
}
2618+
if (compresslevel>9)
2619+
{
2620+
pg_log_error("compression level %d of method %s higher than maximum of 9",
2621+
compresslevel,"gzip");
2622+
exit(1);
2623+
}
2624+
#else
2625+
pg_log_error("this build does not support compression with %s",
2626+
"gzip");
2627+
exit(1);
25262628
#endif
2629+
break;
2630+
caseCOMPRESSION_LZ4:
2631+
/* option not supported */
2632+
Assert(false);
2633+
break;
2634+
}
25272635

25282636
/*
25292637
* Sanity checks for progress reporting options.

‎src/bin/pg_basebackup/t/010_pg_basebackup.pl

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
use Fcntlqw(:seek);
1111
use PostgreSQL::Test::Cluster;
1212
use PostgreSQL::Test::Utils;
13-
use Test::Moretests=>135;
13+
use Test::Moretests=>143;
1414

1515
program_help_ok('pg_basebackup');
1616
program_version_ok('pg_basebackup');
@@ -38,6 +38,20 @@
3838
$node->command_fails(['pg_basebackup'],
3939
'pg_basebackup needs target directory specified');
4040

41+
# Sanity checks for options
42+
$node->command_fails_like(
43+
['pg_basebackup','-D',"$tempdir/backup",'--compress','none:1' ],
44+
qr/\Qpg_basebackup: error: cannot use compression level with method none/,
45+
'failure if method "none" specified with compression level');
46+
$node->command_fails_like(
47+
['pg_basebackup','-D',"$tempdir/backup",'--compress','none+' ],
48+
qr/\Qpg_basebackup: error: invalid value "none+" for option/,
49+
'failure on incorrect separator to define compression level');
50+
$node->command_fails_like(
51+
['pg_basebackup','-D',"$tempdir/backup",'--compress','none:' ],
52+
qr/\Qpg_basebackup: error: no compression level defined for method none/,
53+
'failure on missing compression level value');
54+
4155
# Some Windows ANSI code pages may reject this filename, in which case we
4256
# quietly proceed without this bit of test coverage.
4357
if (openmy$badchars,'>>',"$tempdir/pgdata/FOO\xe0\xe0\xe0BAR")
@@ -699,7 +713,7 @@
699713
# Check ZLIB compression if available.
700714
SKIP:
701715
{
702-
skip"postgres was not built with ZLIB support",5
716+
skip"postgres was not built with ZLIB support",7
703717
if (!check_pg_config("#define HAVE_LIBZ 1"));
704718

705719
$node->command_ok(
@@ -717,15 +731,28 @@
717731
'--format','t'
718732
],
719733
'pg_basebackup with --gzip');
734+
$node->command_ok(
735+
[
736+
@pg_basebackup_defs,'-D',
737+
"$tempdir/backup_gzip3",'--compress',
738+
'gzip:1','--format',
739+
't'
740+
],
741+
'pg_basebackup with --compress=gzip:1');
720742

721743
# Verify that the stored files are generated with their expected
722744
# names.
723745
my@zlib_files =glob"$tempdir/backup_gzip/*.tar.gz";
724746
is(scalar(@zlib_files), 2,
725-
"two files created with --compress (base.tar.gz and pg_wal.tar.gz)");
747+
"two files created with --compress=NUM (base.tar.gz and pg_wal.tar.gz)"
748+
);
726749
my@zlib_files2 =glob"$tempdir/backup_gzip2/*.tar.gz";
727750
is(scalar(@zlib_files2), 2,
728751
"two files created with --gzip (base.tar.gz and pg_wal.tar.gz)");
752+
my@zlib_files3 =glob"$tempdir/backup_gzip3/*.tar.gz";
753+
is(scalar(@zlib_files3), 2,
754+
"two files created with --compress=gzip:NUM (base.tar.gz and pg_wal.tar.gz)"
755+
);
729756

730757
# Check the integrity of the files generated.
731758
my$gzip =$ENV{GZIP_PROGRAM};
@@ -735,8 +762,9 @@
735762
|| system_log($gzip,'--version') != 0);
736763

737764
my$gzip_is_valid =
738-
system_log($gzip,'--test',@zlib_files,@zlib_files2);
765+
system_log($gzip,'--test',@zlib_files,@zlib_files2,@zlib_files3);
739766
is($gzip_is_valid, 0,"gzip verified the integrity of compressed data");
740767
rmtree("$tempdir/backup_gzip");
741768
rmtree("$tempdir/backup_gzip2");
769+
rmtree("$tempdir/backup_gzip3");
742770
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp