Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commite953732

Browse files
committed
Add support for zstd with compression of full-page writes in WAL
wal_compression gains a new value, "zstd", to allow the compression offull-page images using the compression method of the same name.Compression is done using the default level recommended by the library,as of ZSTD_CLEVEL_DEFAULT = 3. Some benchmarking has shown that itcould make sense to use a level lower for the FPI compression, like 1 or2, as the compression rate did not change much with a bit less CPUconsumed, but any tests done would only cover few scenarios so it ishard to come to a clear conclusion. Anyway, there is no reason to notuse the default level instead, which is the level recommended by thelibrary so it should be fine for most cases.zstd outclasses easily pglz, and is better than LZ4 where one wants tohave more compression at the cost of extra CPU but both are good enoughin their own scenarios, so the choice between one or the other of thesecomes to a study of the workload patterns and the schema involved,mainly.This commit relies heavily on4035cd5, that reshaped the code creatingand restoring full-page writes to be aware of the compression type,making this integration straight-forward.This patch borrows some early work from Andrey Borodin, though the patchgot a complete rewrite.Author: Justin PryzbyDiscussion:https://postgr.es/m/20220222231948.GJ9008@telsasoft.com
1 parent0071fc7 commite953732

File tree

9 files changed

+78
-8
lines changed

9 files changed

+78
-8
lines changed

‎doc/src/sgml/config.sgml

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3154,10 +3154,13 @@ include_dir 'conf.d'
31543154
server compresses full page images written to WAL when
31553155
<xref linkend="guc-full-page-writes"/> is on or during a base backup.
31563156
A compressed page image will be decompressed during WAL replay.
3157-
The supported methods are <literal>pglz</literal> and
3158-
<literal>lz4</literal> (if <productname>PostgreSQL</productname> was
3159-
compiled with <option>--with-lz4</option>). The default value is
3160-
<literal>off</literal>. Only superusers can change this setting.
3157+
The supported methods are <literal>pglz</literal>,
3158+
<literal>lz4</literal> (if <productname>PostgreSQL</productname>
3159+
was compiled with <option>--with-lz4</option>) and
3160+
<literal>zstd</literal> (if <productname>PostgreSQL</productname>
3161+
was compiled with <option>--with-zstd</option>) and
3162+
The default value is <literal>off</literal>.
3163+
Only superusers can change this setting.
31613164
</para>
31623165

31633166
<para>

‎doc/src/sgml/installation.sgml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,14 @@ su - postgres
271271
</para>
272272
</listitem>
273273

274+
<listitem>
275+
<para>
276+
You need <productname>zstd</productname>, if you want to support
277+
compression of data with this method; see
278+
<xref linkend="guc-wal-compression"/>.
279+
</para>
280+
</listitem>
281+
274282
<listitem>
275283
<para>
276284
To build the <productname>PostgreSQL</productname> documentation,

‎src/backend/access/transam/xloginsert.c

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@
2323
#include<lz4.h>
2424
#endif
2525

26+
#ifdefUSE_ZSTD
27+
#include<zstd.h>
28+
#endif
29+
2630
#include"access/xact.h"
2731
#include"access/xlog.h"
2832
#include"access/xlog_internal.h"
@@ -47,9 +51,16 @@
4751
#defineLZ4_MAX_BLCKSZ0
4852
#endif
4953

54+
#ifdefUSE_ZSTD
55+
#defineZSTD_MAX_BLCKSZZSTD_COMPRESSBOUND(BLCKSZ)
56+
#else
57+
#defineZSTD_MAX_BLCKSZ0
58+
#endif
59+
5060
#definePGLZ_MAX_BLCKSZPGLZ_MAX_OUTPUT(BLCKSZ)
5161

52-
#defineCOMPRESS_BUFSIZEMax(PGLZ_MAX_BLCKSZ, LZ4_MAX_BLCKSZ)
62+
/* Buffer size required to store a compressed version of backup block image */
63+
#defineCOMPRESS_BUFSIZEMax(Max(PGLZ_MAX_BLCKSZ, LZ4_MAX_BLCKSZ), ZSTD_MAX_BLCKSZ)
5364

5465
/*
5566
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -698,6 +709,14 @@ XLogRecordAssemble(RmgrId rmid, uint8 info,
698709
#endif
699710
break;
700711

712+
caseWAL_COMPRESSION_ZSTD:
713+
#ifdefUSE_ZSTD
714+
bimg.bimg_info |=BKPIMAGE_COMPRESS_ZSTD;
715+
#else
716+
elog(ERROR,"zstd is not supported by this build");
717+
#endif
718+
break;
719+
701720
caseWAL_COMPRESSION_NONE:
702721
Assert(false);/* cannot happen */
703722
break;
@@ -906,6 +925,17 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
906925
#endif
907926
break;
908927

928+
caseWAL_COMPRESSION_ZSTD:
929+
#ifdefUSE_ZSTD
930+
len=ZSTD_compress(dest,COMPRESS_BUFSIZE,source,orig_len,
931+
ZSTD_CLEVEL_DEFAULT);
932+
if (ZSTD_isError(len))
933+
len=-1;/* failure */
934+
#else
935+
elog(ERROR,"zstd is not supported by this build");
936+
#endif
937+
break;
938+
909939
caseWAL_COMPRESSION_NONE:
910940
Assert(false);/* cannot happen */
911941
break;

‎src/backend/access/transam/xlogreader.c

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@
2121
#ifdefUSE_LZ4
2222
#include<lz4.h>
2323
#endif
24+
#ifdefUSE_ZSTD
25+
#include<zstd.h>
26+
#endif
2427

2528
#include"access/transam.h"
2629
#include"access/xlog_internal.h"
@@ -1618,6 +1621,23 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
16181621
"LZ4",
16191622
block_id);
16201623
return false;
1624+
#endif
1625+
}
1626+
elseif ((bkpb->bimg_info&BKPIMAGE_COMPRESS_ZSTD)!=0)
1627+
{
1628+
#ifdefUSE_ZSTD
1629+
size_tdecomp_result=ZSTD_decompress(tmp.data,
1630+
BLCKSZ-bkpb->hole_length,
1631+
ptr,bkpb->bimg_len);
1632+
1633+
if (ZSTD_isError(decomp_result))
1634+
decomp_success= false;
1635+
#else
1636+
report_invalid_record(record,"image at %X/%X compressed with %s not supported by build, block %d",
1637+
LSN_FORMAT_ARGS(record->ReadRecPtr),
1638+
"zstd",
1639+
block_id);
1640+
return false;
16211641
#endif
16221642
}
16231643
else

‎src/backend/utils/misc/guc.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -550,6 +550,9 @@ static const struct config_enum_entry wal_compression_options[] = {
550550
{"pglz",WAL_COMPRESSION_PGLZ, false},
551551
#ifdefUSE_LZ4
552552
{"lz4",WAL_COMPRESSION_LZ4, false},
553+
#endif
554+
#ifdefUSE_ZSTD
555+
{"zstd",WAL_COMPRESSION_ZSTD, false},
553556
#endif
554557
{"on",WAL_COMPRESSION_PGLZ, false},
555558
{"off",WAL_COMPRESSION_NONE, false},

‎src/backend/utils/misc/postgresql.conf.sample

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,7 @@
220220
#wal_log_hints = off# also do full page writes of non-critical updates
221221
# (change requires restart)
222222
#wal_compression = off# enables compression of full-page writes;
223-
# off, pglz, lz4, or on
223+
# off, pglz, lz4,zstd,or on
224224
#wal_init_zero = on# zero-fill new WAL files
225225
#wal_recycle = on# recycle WAL files
226226
#wal_buffers = -1# min 32kB, -1 sets based on shared_buffers

‎src/bin/pg_waldump/pg_waldump.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -562,6 +562,8 @@ XLogDumpDisplayRecord(XLogDumpConfig *config, XLogReaderState *record)
562562
method="pglz";
563563
elseif ((bimg_info&BKPIMAGE_COMPRESS_LZ4)!=0)
564564
method="lz4";
565+
elseif ((bimg_info&BKPIMAGE_COMPRESS_ZSTD)!=0)
566+
method="zstd";
565567
else
566568
method="unknown";
567569

‎src/include/access/xlog.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,8 @@ typedef enum WalCompression
7575
{
7676
WAL_COMPRESSION_NONE=0,
7777
WAL_COMPRESSION_PGLZ,
78-
WAL_COMPRESSION_LZ4
78+
WAL_COMPRESSION_LZ4,
79+
WAL_COMPRESSION_ZSTD
7980
}WalCompression;
8081

8182
/* Recovery states */

‎src/include/access/xlogrecord.h

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,8 +149,11 @@ typedef struct XLogRecordBlockImageHeader
149149
/* compression methods supported */
150150
#defineBKPIMAGE_COMPRESS_PGLZ0x04
151151
#defineBKPIMAGE_COMPRESS_LZ40x08
152+
#defineBKPIMAGE_COMPRESS_ZSTD0x10
153+
152154
#defineBKPIMAGE_COMPRESSED(info) \
153-
((info & (BKPIMAGE_COMPRESS_PGLZ | BKPIMAGE_COMPRESS_LZ4)) != 0)
155+
((info & (BKPIMAGE_COMPRESS_PGLZ | BKPIMAGE_COMPRESS_LZ4 | \
156+
BKPIMAGE_COMPRESS_ZSTD)) != 0)
154157

155158
/*
156159
* Extra header information used when page image has "hole" and

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp