Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit11a078c

Browse files
committed
Optimize partial TOAST decompression
Commit4d0e994 added support for partial TOAST decompression, so thedecompression is interrupted after producing the requested prefix. Forprefix and slices near the beginning of the entry, this may saves a lotof decompression work.That however only deals with decompression - the whole compressed entrywas still fetched and re-assembled, even though the compression usedonly a small fraction of it. This commit improves that by computing howmuch compressed data may be needed to decompress the requested prefix,and then fetches only the necessary part.We always need to fetch a bit more compressed data than the requested(uncompressed) prefix, because the prefix may not be compressible at alland pglz itself adds a bit of overhead. That means this optimization ismost effective when the requested prefix is much smaller than the wholecompressed entry.Author: Binguo BaoReviewed-by: Andrey Borodin, Tomas Vondra, Paul RamseyDiscussion:https://www.postgresql.org/message-id/flat/CAL-OGkthU9Gs7TZchf5OWaL-Gsi=hXqufTxKv9qpNG73d5na_g@mail.gmail.com
1 parent002962d commit11a078c

File tree

4 files changed

+86
-8
lines changed

4 files changed

+86
-8
lines changed

‎src/backend/access/common/detoast.c

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,8 @@ heap_tuple_untoast_attr(struct varlena *attr)
196196
*
197197
*Public entry point to get back part of a toasted value
198198
*from compression or external storage.
199+
*
200+
* Note: When slicelength is negative, return suffix of the value.
199201
* ----------
200202
*/
201203
structvarlena*
@@ -217,8 +219,30 @@ heap_tuple_untoast_attr_slice(struct varlena *attr,
217219
if (!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
218220
returntoast_fetch_datum_slice(attr,sliceoffset,slicelength);
219221

220-
/* fetch it back (compressed marker will get set automatically) */
221-
preslice=toast_fetch_datum(attr);
222+
/*
223+
* For compressed values, we need to fetch enough slices to decompress
224+
* at least the requested part (when a prefix is requested). Otherwise,
225+
* just fetch all slices.
226+
*/
227+
if (slicelength>0&&sliceoffset >=0)
228+
{
229+
int32max_size;
230+
231+
/*
232+
* Determine maximum amount of compressed data needed for a prefix
233+
* of a given length (after decompression).
234+
*/
235+
max_size=pglz_maximum_compressed_size(sliceoffset+slicelength,
236+
TOAST_COMPRESS_SIZE(attr));
237+
238+
/*
239+
* Fetch enough compressed slices (compressed marker will get set
240+
* automatically).
241+
*/
242+
preslice=toast_fetch_datum_slice(attr,0,max_size);
243+
}
244+
else
245+
preslice=toast_fetch_datum(attr);
222246
}
223247
elseif (VARATT_IS_EXTERNAL_INDIRECT(attr))
224248
{
@@ -476,7 +500,9 @@ toast_fetch_datum(struct varlena *attr)
476500
*Reconstruct a segment of a Datum from the chunks saved
477501
*in the toast relation
478502
*
479-
*Note that this function only supports non-compressed external datums.
503+
*Note that this function supports non-compressed external datums
504+
*and compressed external datums (in which case the requrested slice
505+
* has to be a prefix, i.e. sliceoffset has to be 0).
480506
* ----------
481507
*/
482508
staticstructvarlena*
@@ -517,10 +543,11 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, int32 length)
517543
VARATT_EXTERNAL_GET_POINTER(toast_pointer,attr);
518544

519545
/*
520-
* It's nonsense to fetch slices of a compressed datum -- this isn't lo_*
521-
* we can't return a compressed datum which is meaningful to toast later
546+
* It's nonsense to fetch slices of a compressed datum unless when it's
547+
* a prefix -- this isn't lo_* we can't return a compressed datum which
548+
* is meaningful to toast later.
522549
*/
523-
Assert(!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer));
550+
Assert(!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)||0==sliceoffset);
524551

525552
attrsize=toast_pointer.va_extsize;
526553
totalchunks= ((attrsize-1) /TOAST_MAX_CHUNK_SIZE)+1;
@@ -531,12 +558,23 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, int32 length)
531558
length=0;
532559
}
533560

561+
/*
562+
* When fetching a prefix of a compressed external datum, account for the
563+
* rawsize tracking amount of raw data, which is stored at the beginning
564+
* as an int32 value).
565+
*/
566+
if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)&&length>0)
567+
length=length+sizeof(int32);
568+
534569
if (((sliceoffset+length)>attrsize)||length<0)
535570
length=attrsize-sliceoffset;
536571

537572
result= (structvarlena*)palloc(length+VARHDRSZ);
538573

539-
SET_VARSIZE(result,length+VARHDRSZ);
574+
if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
575+
SET_VARSIZE_COMPRESSED(result,length+VARHDRSZ);
576+
else
577+
SET_VARSIZE(result,length+VARHDRSZ);
540578

541579
if (length==0)
542580
returnresult;/* Can save a lot of work at this point! */
@@ -720,7 +758,7 @@ toast_decompress_datum(struct varlena *attr)
720758
SET_VARSIZE(result,TOAST_COMPRESS_RAWSIZE(attr)+VARHDRSZ);
721759

722760
if (pglz_decompress(TOAST_COMPRESS_RAWDATA(attr),
723-
VARSIZE(attr)-TOAST_COMPRESS_HDRSZ,
761+
TOAST_COMPRESS_SIZE(attr),
724762
VARDATA(result),
725763
TOAST_COMPRESS_RAWSIZE(attr), true)<0)
726764
elog(ERROR,"compressed data is corrupted");

‎src/common/pg_lzcompress.c

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -771,3 +771,40 @@ pglz_decompress(const char *source, int32 slen, char *dest,
771771
*/
772772
return (char*)dp-dest;
773773
}
774+
775+
776+
/* ----------
777+
* pglz_max_compressed_size -
778+
*
779+
*Calculate the maximum compressed size for a given amount of raw data.
780+
*Return the maximum size, or total compressed size if maximum size is
781+
*larger than total compressed size.
782+
*
783+
* We can't use PGLZ_MAX_OUTPUT for this purpose, because that's used to size
784+
* the compression buffer (and abort the compression). It does not really say
785+
* what's the maximum compressed size for an input of a given length, and it
786+
* may happen that while the whole value is compressible (and thus fits into
787+
* PGLZ_MAX_OUTPUT nicely), the prefix is not compressible at all.
788+
* ----------
789+
*/
790+
int32
791+
pglz_maximum_compressed_size(int32rawsize,int32total_compressed_size)
792+
{
793+
int32compressed_size;
794+
795+
/*
796+
* pglz uses one control bit per byte, so we need (rawsize * 9) bits. We
797+
* care about bytes though, so we add 7 to make sure we include the last
798+
* incomplete byte (integer division rounds down).
799+
*
800+
* XXX Use int64 to prevent overflow during calculation.
801+
*/
802+
compressed_size= (int32) ((int64)rawsize*9+7) /8;
803+
804+
/*
805+
* Maximum compressed size can't be larger than total compressed size.
806+
*/
807+
compressed_size=Min(compressed_size,total_compressed_size);
808+
809+
returncompressed_size;
810+
}

‎src/include/access/toast_internals.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ typedef struct toast_compress_header
3131
*/
3232
#defineTOAST_COMPRESS_HDRSZ((int32) sizeof(toast_compress_header))
3333
#defineTOAST_COMPRESS_RAWSIZE(ptr) (((toast_compress_header *) (ptr))->rawsize)
34+
#defineTOAST_COMPRESS_SIZE(ptr)((int32) VARSIZE(ptr) - TOAST_COMPRESS_HDRSZ)
3435
#defineTOAST_COMPRESS_RAWDATA(ptr) \
3536
(((char *) (ptr)) + TOAST_COMPRESS_HDRSZ)
3637
#defineTOAST_COMPRESS_SET_RAWSIZE(ptr,len) \

‎src/include/common/pg_lzcompress.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,5 +87,7 @@ extern int32 pglz_compress(const char *source, int32 slen, char *dest,
8787
constPGLZ_Strategy*strategy);
8888
externint32pglz_decompress(constchar*source,int32slen,char*dest,
8989
int32rawsize,boolcheck_complete);
90+
externint32pglz_maximum_compressed_size(int32rawsize,
91+
int32total_compressed_size);
9092

9193
#endif/* _PG_LZCOMPRESS_H_ */

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp