Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit84adc8e

Browse files
committed
pg_dump: Add support for zstd compression
Allow pg_dump to use the zstd compression, in addition to gzip/lz4. Bulkof the new compression method is implemented in compress_zstd.{c,h},covering the pg_dump compression APIs. The rest of the patch adds testand makes various places aware of the new compression method.The zstd library (which this patch relies on) supports multithreadedcompression since version 1.5. We however disallow that feature for now,as it might interfere with parallel backups on platforms that rely onthreads (e.g. Windows). This can be improved / relaxed in the future.This also fixes a minor issue in InitDiscoverCompressFileHandle(), whichwas not updated to check if the file already has the .lz4 extension.Adding zstd compression was originally proposed in 2020 (see the secondthread), but then was reworked to use the new compression API introducedine996073. I've considered both threads when compiling the list ofreviewers.Author: Justin PryzbyReviewed-by: Tomas Vondra, Jacob Champion, Andreas KarlssonDiscussion:https://postgr.es/m/20230224191840.GD1653@telsasoft.comDiscussion:https://postgr.es/m/20201221194924.GI30237@telsasoft.com
1 parent794f259 commit84adc8e

File tree

12 files changed

+713
-54
lines changed

12 files changed

+713
-54
lines changed

‎doc/src/sgml/ref/pg_dump.sgml

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -330,8 +330,9 @@ PostgreSQL documentation
330330
machine-readable format that <application>pg_restore</application>
331331
can read. A directory format archive can be manipulated with
332332
standard Unix tools; for example, files in an uncompressed archive
333-
can be compressed with the <application>gzip</application> or
334-
<application>lz4</application> tools.
333+
can be compressed with the <application>gzip</application>,
334+
<application>lz4</application>, or
335+
<application>zstd</application> tools.
335336
This format is compressed by default using <literal>gzip</literal>
336337
and also supports parallel dumps.
337338
</para>
@@ -655,7 +656,8 @@ PostgreSQL documentation
655656
<para>
656657
Specify the compression method and/or the compression level to use.
657658
The compression method can be set to <literal>gzip</literal>,
658-
<literal>lz4</literal>, or <literal>none</literal> for no compression.
659+
<literal>lz4</literal>, <literal>zstd</literal>,
660+
or <literal>none</literal> for no compression.
659661
A compression detail string can optionally be specified. If the
660662
detail string is an integer, it specifies the compression level.
661663
Otherwise, it should be a comma-separated list of items, each of the
@@ -676,8 +678,9 @@ PostgreSQL documentation
676678
individual table-data segments, and the default is to compress using
677679
<literal>gzip</literal> at a moderate level. For plain text output,
678680
setting a nonzero compression level causes the entire output file to be compressed,
679-
as though it had been fed through <application>gzip</application> or
680-
<application>lz4</application>; but the default is not to compress.
681+
as though it had been fed through <application>gzip</application>,
682+
<application>lz4</application>, or <application>zstd</application>;
683+
but the default is not to compress.
681684
</para>
682685
<para>
683686
The tar archive format currently does not support compression at all.

‎src/bin/pg_dump/Makefile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ include $(top_builddir)/src/Makefile.global
1818

1919
exportGZIP_PROGRAM=$(GZIP)
2020
exportLZ4
21+
exportZSTD
2122
exportwith_icu
2223

2324
overrideCPPFLAGS := -I$(libpq_srcdir)$(CPPFLAGS)
@@ -29,6 +30,7 @@ OBJS = \
2930
compress_io.o\
3031
compress_lz4.o\
3132
compress_none.o\
33+
compress_zstd.o\
3234
dumputils.o\
3335
parallel.o\
3436
pg_backup_archiver.o\

‎src/bin/pg_dump/compress_io.c

Lines changed: 40 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,8 @@
5252
*
5353
*InitDiscoverCompressFileHandle tries to infer the compression by the
5454
*filename suffix. If the suffix is not yet known then it tries to simply
55-
*open the file and if it fails, it tries to open the same file with the .gz
56-
*suffix, and then again with the .lz4 suffix.
55+
*open the file and if it fails, it tries to open the same file with
56+
*compressed suffixes (.gz, .lz4 and .zst, in this order).
5757
*
5858
* IDENTIFICATION
5959
* src/bin/pg_dump/compress_io.c
@@ -69,6 +69,7 @@
6969
#include"compress_io.h"
7070
#include"compress_lz4.h"
7171
#include"compress_none.h"
72+
#include"compress_zstd.h"
7273
#include"pg_backup_utils.h"
7374

7475
/*----------------------
@@ -77,7 +78,8 @@
7778
*/
7879

7980
/*
80-
* Checks whether a compression algorithm is supported.
81+
* Checks whether support for a compression algorithm is implemented in
82+
* pg_dump/restore.
8183
*
8284
* On success returns NULL, otherwise returns a malloc'ed string which can be
8385
* used by the caller in an error message.
@@ -98,6 +100,10 @@ supports_compression(const pg_compress_specification compression_spec)
98100
if (algorithm==PG_COMPRESSION_LZ4)
99101
supported= true;
100102
#endif
103+
#ifdefUSE_ZSTD
104+
if (algorithm==PG_COMPRESSION_ZSTD)
105+
supported= true;
106+
#endif
101107

102108
if (!supported)
103109
returnpsprintf("this build does not support compression with %s",
@@ -130,6 +136,8 @@ AllocateCompressor(const pg_compress_specification compression_spec,
130136
InitCompressorGzip(cs,compression_spec);
131137
elseif (compression_spec.algorithm==PG_COMPRESSION_LZ4)
132138
InitCompressorLZ4(cs,compression_spec);
139+
elseif (compression_spec.algorithm==PG_COMPRESSION_ZSTD)
140+
InitCompressorZstd(cs,compression_spec);
133141

134142
returncs;
135143
}
@@ -196,20 +204,36 @@ InitCompressFileHandle(const pg_compress_specification compression_spec)
196204
InitCompressFileHandleGzip(CFH,compression_spec);
197205
elseif (compression_spec.algorithm==PG_COMPRESSION_LZ4)
198206
InitCompressFileHandleLZ4(CFH,compression_spec);
207+
elseif (compression_spec.algorithm==PG_COMPRESSION_ZSTD)
208+
InitCompressFileHandleZstd(CFH,compression_spec);
199209

200210
returnCFH;
201211
}
202212

213+
/*
214+
* Checks if a compressed file (with the specified extension) exists.
215+
*
216+
* The filename of the tested file is stored to fname buffer (the existing
217+
* buffer is freed, new buffer is allocated and returned through the pointer).
218+
*/
219+
staticbool
220+
check_compressed_file(constchar*path,char**fname,char*ext)
221+
{
222+
free_keep_errno(*fname);
223+
*fname=psprintf("%s.%s",path,ext);
224+
return (access(*fname,F_OK)==0);
225+
}
226+
203227
/*
204228
* Open a file for reading. 'path' is the file to open, and 'mode' should
205229
* be either "r" or "rb".
206230
*
207231
* If the file at 'path' contains the suffix of a supported compression method,
208-
* currently this includes ".gz"and ".lz4", then this compression will be used
232+
* currently this includes ".gz", ".lz4"and ".zst", then this compression will be used
209233
* throughout. Otherwise the compression will be inferred by iteratively trying
210234
* to open the file at 'path', first as is, then by appending known compression
211235
* suffixes. So if you pass "foo" as 'path', this will open either "foo" or
212-
* "foo.gz" or "foo.lz4", trying in that order.
236+
* "foo.{gz,lz4,zst}", trying in that order.
213237
*
214238
* On failure, return NULL with an error code in errno.
215239
*/
@@ -229,36 +253,20 @@ InitDiscoverCompressFileHandle(const char *path, const char *mode)
229253

230254
if (hasSuffix(fname,".gz"))
231255
compression_spec.algorithm=PG_COMPRESSION_GZIP;
256+
elseif (hasSuffix(fname,".lz4"))
257+
compression_spec.algorithm=PG_COMPRESSION_LZ4;
258+
elseif (hasSuffix(fname,".zst"))
259+
compression_spec.algorithm=PG_COMPRESSION_ZSTD;
232260
else
233261
{
234-
boolexists;
235-
236-
exists= (stat(path,&st)==0);
237-
/* avoid unused warning if it is not built with compression */
238-
if (exists)
262+
if (stat(path,&st)==0)
239263
compression_spec.algorithm=PG_COMPRESSION_NONE;
240-
#ifdefHAVE_LIBZ
241-
if (!exists)
242-
{
243-
free_keep_errno(fname);
244-
fname=psprintf("%s.gz",path);
245-
exists= (stat(fname,&st)==0);
246-
247-
if (exists)
248-
compression_spec.algorithm=PG_COMPRESSION_GZIP;
249-
}
250-
#endif
251-
#ifdefUSE_LZ4
252-
if (!exists)
253-
{
254-
free_keep_errno(fname);
255-
fname=psprintf("%s.lz4",path);
256-
exists= (stat(fname,&st)==0);
257-
258-
if (exists)
259-
compression_spec.algorithm=PG_COMPRESSION_LZ4;
260-
}
261-
#endif
264+
elseif (check_compressed_file(path,&fname,"gz"))
265+
compression_spec.algorithm=PG_COMPRESSION_GZIP;
266+
elseif (check_compressed_file(path,&fname,"lz4"))
267+
compression_spec.algorithm=PG_COMPRESSION_LZ4;
268+
elseif (check_compressed_file(path,&fname,"zst"))
269+
compression_spec.algorithm=PG_COMPRESSION_ZSTD;
262270
}
263271

264272
CFH=InitCompressFileHandle(compression_spec);

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp