Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit40bede5

Browse files
committed
Move pg_lzcompress.c to src/common.
The meta data of PGLZ symbolized by PGLZ_Header is removed, to makethe compression and decompression code independent on the backend-onlyvarlena facility. PGLZ_Header is being used to store some meta datarelated to the data being compressed like the raw length of the uncompressedrecord or some varlena-related data, making it unpluggable once PGLZ isstored in src/common as it contains some backend-only code paths withthe management of varlena structures. The APIs of PGLZ are reworkedat the same time to do only compression and decompression of bufferswithout the meta-data layer, simplifying its use for a more general usage.On-disk format is preserved as well, so there is no incompatibility withprevious major versions of PostgreSQL for TOAST entries.Exposing compression and decompression APIs of pglz makes possible itsuse by extensions and contrib modules. Especially this commit is requiredfor upcoming WAL compression feature so that the WAL reader facility candecompress the WAL data by using pglz_decompress.Michael Paquier, reviewed by me.
1 parent237795a commit40bede5

File tree

7 files changed

+124
-102
lines changed

7 files changed

+124
-102
lines changed

‎src/backend/access/heap/tuptoaster.c

Lines changed: 65 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -35,16 +35,36 @@
3535
#include"access/tuptoaster.h"
3636
#include"access/xact.h"
3737
#include"catalog/catalog.h"
38+
#include"common/pg_lzcompress.h"
3839
#include"miscadmin.h"
3940
#include"utils/fmgroids.h"
40-
#include"utils/pg_lzcompress.h"
4141
#include"utils/rel.h"
4242
#include"utils/typcache.h"
4343
#include"utils/tqual.h"
4444

4545

4646
#undef TOAST_DEBUG
4747

48+
/*
49+
*The information at the start of the compressed toast data.
50+
*/
51+
typedefstructtoast_compress_header
52+
{
53+
int32vl_len_;/* varlena header (do not touch directly!) */
54+
int32rawsize;
55+
}toast_compress_header;
56+
57+
/*
58+
* Utilities for manipulation of header information for compressed
59+
* toast entries.
60+
*/
61+
#defineTOAST_COMPRESS_HDRSZ((int32) sizeof(toast_compress_header))
62+
#defineTOAST_COMPRESS_RAWSIZE(ptr)(((toast_compress_header *) ptr)->rawsize)
63+
#defineTOAST_COMPRESS_RAWDATA(ptr) \
64+
(((char *) ptr) + TOAST_COMPRESS_HDRSZ)
65+
#defineTOAST_COMPRESS_SET_RAWSIZE(ptr,len) \
66+
(((toast_compress_header *) ptr)->rawsize = len)
67+
4868
staticvoidtoast_delete_datum(Relationrel,Datumvalue);
4969
staticDatumtoast_save_datum(Relationrel,Datumvalue,
5070
structvarlena*oldexternal,intoptions);
@@ -53,6 +73,7 @@ static bool toastid_valueid_exists(Oid toastrelid, Oid valueid);
5373
staticstructvarlena*toast_fetch_datum(structvarlena*attr);
5474
staticstructvarlena*toast_fetch_datum_slice(structvarlena*attr,
5575
int32sliceoffset,int32length);
76+
staticstructvarlena*toast_decompress_datum(structvarlena*attr);
5677
staticinttoast_open_indexes(Relationtoastrel,
5778
LOCKMODElock,
5879
Relation**toastidxs,
@@ -138,11 +159,8 @@ heap_tuple_untoast_attr(struct varlena * attr)
138159
/* If it's compressed, decompress it */
139160
if (VARATT_IS_COMPRESSED(attr))
140161
{
141-
PGLZ_Header*tmp= (PGLZ_Header*)attr;
142-
143-
attr= (structvarlena*)palloc(PGLZ_RAW_SIZE(tmp)+VARHDRSZ);
144-
SET_VARSIZE(attr,PGLZ_RAW_SIZE(tmp)+VARHDRSZ);
145-
pglz_decompress(tmp,VARDATA(attr));
162+
structvarlena*tmp=attr;
163+
attr=toast_decompress_datum(tmp);
146164
pfree(tmp);
147165
}
148166
}
@@ -163,11 +181,7 @@ heap_tuple_untoast_attr(struct varlena * attr)
163181
/*
164182
* This is a compressed value inside of the main tuple
165183
*/
166-
PGLZ_Header*tmp= (PGLZ_Header*)attr;
167-
168-
attr= (structvarlena*)palloc(PGLZ_RAW_SIZE(tmp)+VARHDRSZ);
169-
SET_VARSIZE(attr,PGLZ_RAW_SIZE(tmp)+VARHDRSZ);
170-
pglz_decompress(tmp,VARDATA(attr));
184+
attr=toast_decompress_datum(attr);
171185
}
172186
elseif (VARATT_IS_SHORT(attr))
173187
{
@@ -234,14 +248,10 @@ heap_tuple_untoast_attr_slice(struct varlena * attr,
234248

235249
if (VARATT_IS_COMPRESSED(preslice))
236250
{
237-
PGLZ_Header*tmp= (PGLZ_Header*)preslice;
238-
Sizesize=PGLZ_RAW_SIZE(tmp)+VARHDRSZ;
239-
240-
preslice= (structvarlena*)palloc(size);
241-
SET_VARSIZE(preslice,size);
242-
pglz_decompress(tmp,VARDATA(preslice));
251+
structvarlena*tmp=preslice;
252+
preslice=toast_decompress_datum(tmp);
243253

244-
if (tmp!=(PGLZ_Header*)attr)
254+
if (tmp!=attr)
245255
pfree(tmp);
246256
}
247257

@@ -1228,6 +1238,7 @@ toast_compress_datum(Datum value)
12281238
{
12291239
structvarlena*tmp;
12301240
int32valsize=VARSIZE_ANY_EXHDR(DatumGetPointer(value));
1241+
int32len;
12311242

12321243
Assert(!VARATT_IS_EXTERNAL(DatumGetPointer(value)));
12331244
Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value)));
@@ -1240,7 +1251,8 @@ toast_compress_datum(Datum value)
12401251
valsize>PGLZ_strategy_default->max_input_size)
12411252
returnPointerGetDatum(NULL);
12421253

1243-
tmp= (structvarlena*)palloc(PGLZ_MAX_OUTPUT(valsize));
1254+
tmp= (structvarlena*)palloc(PGLZ_MAX_OUTPUT(valsize)+
1255+
TOAST_COMPRESS_HDRSZ);
12441256

12451257
/*
12461258
* We recheck the actual size even if pglz_compress() reports success,
@@ -1252,10 +1264,15 @@ toast_compress_datum(Datum value)
12521264
* only one header byte and no padding if the value is short enough. So
12531265
* we insist on a savings of more than 2 bytes to ensure we have a gain.
12541266
*/
1255-
if (pglz_compress(VARDATA_ANY(DatumGetPointer(value)),valsize,
1256-
(PGLZ_Header*)tmp,PGLZ_strategy_default)&&
1257-
VARSIZE(tmp)<valsize-2)
1267+
len=pglz_compress(VARDATA_ANY(DatumGetPointer(value)),
1268+
valsize,
1269+
TOAST_COMPRESS_RAWDATA(tmp),
1270+
PGLZ_strategy_default);
1271+
if (len >=0&&
1272+
len+TOAST_COMPRESS_HDRSZ<valsize-2)
12581273
{
1274+
TOAST_COMPRESS_SET_RAWSIZE(tmp,valsize);
1275+
SET_VARSIZE_COMPRESSED(tmp,len+TOAST_COMPRESS_HDRSZ);
12591276
/* successful compression */
12601277
returnPointerGetDatum(tmp);
12611278
}
@@ -2100,6 +2117,32 @@ toast_fetch_datum_slice(struct varlena * attr, int32 sliceoffset, int32 length)
21002117
returnresult;
21012118
}
21022119

2120+
/* ----------
2121+
* toast_decompress_datum -
2122+
*
2123+
* Decompress a compressed version of a varlena datum
2124+
*/
2125+
staticstructvarlena*
2126+
toast_decompress_datum(structvarlena*attr)
2127+
{
2128+
structvarlena*result;
2129+
2130+
Assert(VARATT_IS_COMPRESSED(attr));
2131+
2132+
result= (structvarlena*)
2133+
palloc(TOAST_COMPRESS_RAWSIZE(attr)+VARHDRSZ);
2134+
SET_VARSIZE(result,TOAST_COMPRESS_RAWSIZE(attr)+VARHDRSZ);
2135+
2136+
if (pglz_decompress(TOAST_COMPRESS_RAWDATA(attr),
2137+
VARSIZE(attr)-TOAST_COMPRESS_HDRSZ,
2138+
VARDATA(result),
2139+
TOAST_COMPRESS_RAWSIZE(attr))<0)
2140+
elog(ERROR,"compressed data is corrupted");
2141+
2142+
returnresult;
2143+
}
2144+
2145+
21032146
/* ----------
21042147
* toast_open_indexes
21052148
*

‎src/backend/utils/adt/Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ OBJS = acl.o arrayfuncs.o array_selfuncs.o array_typanalyze.o \
2525
jsonfuncs.o like.o lockfuncs.o mac.o misc.o nabstime.o name.o\
2626
network.o network_gist.o network_selfuncs.o\
2727
numeric.o numutils.o oid.o oracle_compat.o\
28-
orderedsetaggs.opg_lzcompress.opg_locale.opg_lsn.o\
29-
pgstatfuncs.opseudotypes.o quote.o rangetypes.o rangetypes_gist.o\
28+
orderedsetaggs.opg_locale.opg_lsn.opgstatfuncs.o\
29+
pseudotypes.o quote.o rangetypes.o rangetypes_gist.o\
3030
rangetypes_selfuncs.o rangetypes_spgist.o rangetypes_typanalyze.o\
3131
regexp.o regproc.o ri_triggers.o rowtypes.o ruleutils.o\
3232
selfuncs.o tid.o timestamp.o trigfuncs.o\

‎src/common/Makefile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ include $(top_builddir)/src/Makefile.global
2323
overrideCPPFLAGS := -DFRONTEND$(CPPFLAGS)
2424
LIBS +=$(PTHREAD_LIBS)
2525

26-
OBJS_COMMON = exec.o pgfnames.o psprintf.o relpath.o rmtree.o string.o username.o wait_error.o
26+
OBJS_COMMON = exec.o pg_lzcompress.o pgfnames.o psprintf.o relpath.o\
27+
rmtree.o string.o username.o wait_error.o
2728

2829
OBJS_FRONTEND =$(OBJS_COMMON) fe_memutils.o
2930

‎src/backend/utils/adt/pg_lzcompress.crenamed to‎src/common/pg_lzcompress.c

Lines changed: 48 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@
88
*
99
*Entry routines:
1010
*
11-
*bool
12-
*pglz_compress(const char *source, int32 slen,PGLZ_Header *dest,
11+
*int32
12+
*pglz_compress(const char *source, int32 slen,char *dest,
1313
* const PGLZ_Strategy *strategy);
1414
*
1515
*source is the input data to be compressed.
@@ -23,44 +23,43 @@
2323
*the compression algorithm. If NULL, the compiled
2424
*in default strategy is used.
2525
*
26-
*The return value isTRUE if compression succeeded,
27-
*FALSEifnot; in the latter case the contents of dest
28-
*are undefined.
26+
*The return value isthe number of bytes written in the
27+
*buffer dest, or -1ifcompression fails; in the latter
28+
*case the contents of destare undefined.
2929
*
30-
*void
31-
*pglz_decompress(const PGLZ_Header *source, char *dest)
30+
*int32
31+
*pglz_decompress(const char *source, int32 slen, char *dest,
32+
*int32 rawsize)
3233
*
3334
*source is the compressed input.
3435
*
36+
*slen is the length of the compressed input.
37+
*
3538
*dest is the area where the uncompressed data will be
3639
*written to. It is the callers responsibility to
37-
*provide enough space. The required amount can be
38-
*obtained with the macro PGLZ_RAW_SIZE(source).
40+
*provide enough space.
3941
*
4042
*The data is written to buff exactly as it was handed
4143
*to pglz_compress(). No terminating zero byte is added.
4244
*
43-
*The decompression algorithm and internal data format:
45+
*rawsize is the length of the uncompressed data.
4446
*
45-
*PGLZ_Header is defined as
47+
*The return value is the number of bytes written in the
48+
*buffer dest, or -1 if decompression fails.
4649
*
47-
*typedef struct PGLZ_Header {
48-
*int32vl_len_;
49-
*int32rawsize;
50-
*}
50+
*The decompression algorithm and internal data format:
5151
*
52-
*The headerisfollowed by the compressed data itself.
52+
*Itismade with the compressed data itself.
5353
*
5454
*The data representation is easiest explained by describing
5555
*the process of decompression.
5656
*
57-
*IfVARSIZE(x) == rawsize + sizeof(PGLZ_Header), then the data
57+
*Ifcompressed_size == rawsize, then the data
5858
*is stored uncompressed as plain bytes. Thus, the decompressor
59-
*simply copies rawsize bytes from the location after the
60-
*header to the destination.
59+
*simply copies rawsize bytes to the destination.
6160
*
62-
*Otherwise the first byteafter the headertells what to do
63-
*the next 8 times.We call this the control byte.
61+
*Otherwise the first byte tells what to do the next 8 times.
62+
*We call this the control byte.
6463
*
6564
*An unset bit in the control byte means, that one uncompressed
6665
*byte follows, which is copied from input to output.
@@ -169,14 +168,18 @@
169168
*
170169
* Copyright (c) 1999-2015, PostgreSQL Global Development Group
171170
*
172-
* src/backend/utils/adt/pg_lzcompress.c
171+
* src/common/pg_lzcompress.c
173172
* ----------
174173
*/
174+
#ifndefFRONTEND
175175
#include"postgres.h"
176+
#else
177+
#include"postgres_fe.h"
178+
#endif
176179

177180
#include<limits.h>
178181

179-
#include"utils/pg_lzcompress.h"
182+
#include"common/pg_lzcompress.h"
180183

181184

182185
/* ----------
@@ -492,14 +495,15 @@ pglz_find_match(int16 *hstart, const char *input, const char *end,
492495
/* ----------
493496
* pglz_compress -
494497
*
495-
*Compresses source into dest using strategy.
498+
*Compresses source into dest using strategy. Returns the number of
499+
*bytes written in buffer dest, or -1 if compression fails.
496500
* ----------
497501
*/
498-
bool
499-
pglz_compress(constchar*source,int32slen,PGLZ_Header*dest,
502+
int32
503+
pglz_compress(constchar*source,int32slen,char*dest,
500504
constPGLZ_Strategy*strategy)
501505
{
502-
unsignedchar*bp= ((unsignedchar*)dest)+sizeof(PGLZ_Header);
506+
unsignedchar*bp= (unsignedchar*)dest;
503507
unsignedchar*bstart=bp;
504508
inthist_next=1;
505509
boolhist_recycle= false;
@@ -533,12 +537,7 @@ pglz_compress(const char *source, int32 slen, PGLZ_Header *dest,
533537
if (strategy->match_size_good <=0||
534538
slen<strategy->min_input_size||
535539
slen>strategy->max_input_size)
536-
return false;
537-
538-
/*
539-
* Save the original source size in the header.
540-
*/
541-
dest->rawsize=slen;
540+
return-1;
542541

543542
/*
544543
* Limit the match parameters to the supported range.
@@ -611,7 +610,7 @@ pglz_compress(const char *source, int32 slen, PGLZ_Header *dest,
611610
* allow 4 slop bytes.
612611
*/
613612
if (bp-bstart >=result_max)
614-
returnfalse;
613+
return-1;
615614

616615
/*
617616
* If we've emitted more than first_success_by bytes without finding
@@ -620,7 +619,7 @@ pglz_compress(const char *source, int32 slen, PGLZ_Header *dest,
620619
* pre-compressed data).
621620
*/
622621
if (!found_match&&bp-bstart >=strategy->first_success_by)
623-
returnfalse;
622+
return-1;
624623

625624
/*
626625
* Try to find a match in the history
@@ -664,35 +663,34 @@ pglz_compress(const char *source, int32 slen, PGLZ_Header *dest,
664663
*ctrlp=ctrlb;
665664
result_size=bp-bstart;
666665
if (result_size >=result_max)
667-
return false;
668-
669-
/*
670-
* Success - need only fill in the actual length of the compressed datum.
671-
*/
672-
SET_VARSIZE_COMPRESSED(dest,result_size+sizeof(PGLZ_Header));
666+
return-1;
673667

674-
return true;
668+
/* success */
669+
returnresult_size;
675670
}
676671

677672

678673
/* ----------
679674
* pglz_decompress -
680675
*
681-
*Decompresses source into dest.
676+
*Decompresses source into dest. Returns the number of bytes
677+
*decompressed in the destination buffer, or -1 if decompression
678+
*fails.
682679
* ----------
683680
*/
684-
void
685-
pglz_decompress(constPGLZ_Header*source,char*dest)
681+
int32
682+
pglz_decompress(constchar*source,int32slen,char*dest,
683+
int32rawsize)
686684
{
687685
constunsignedchar*sp;
688686
constunsignedchar*srcend;
689687
unsignedchar*dp;
690688
unsignedchar*destend;
691689

692-
sp= ((constunsignedchar*)source)+sizeof(PGLZ_Header);
693-
srcend= ((constunsignedchar*)source)+VARSIZE(source);
690+
sp= (constunsignedchar*)source;
691+
srcend= ((constunsignedchar*)source)+slen;
694692
dp= (unsignedchar*)dest;
695-
destend=dp+source->rawsize;
693+
destend=dp+rawsize;
696694

697695
while (sp<srcend&&dp<destend)
698696
{
@@ -771,9 +769,10 @@ pglz_decompress(const PGLZ_Header *source, char *dest)
771769
* Check we decompressed the right amount.
772770
*/
773771
if (dp!=destend||sp!=srcend)
774-
elog(ERROR,"compressed data is corrupt");
772+
return-1;
775773

776774
/*
777775
* That's it.
778776
*/
777+
returnrawsize;
779778
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp