Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit4d0e994

Browse files
committed
Add support for partial TOAST decompression
When asked for a slice of a TOAST entry, decompress enough to return theslice instead of decompressing the entire object.For use cases where the slice is at, or near, the beginning of the entry,this avoids a lot of unnecessary decompression work.This changes the signature of pglz_decompress() by adding a boolean toindicate if it's ok for the call to finish before consuming all of thesource or destination buffers.Author: Paul RamseyReviewed-By: Rafia Sabih, Darafei Praliaskouski, Regina ObeDiscussion:https://postgr.es/m/CACowWR07EDm7Y4m2kbhN_jnys%3DBBf9A6768RyQdKm_%3DNpkcaWg%40mail.gmail.com
1 parentd50d172 commit4d0e994

File tree

5 files changed

+70
-36
lines changed

5 files changed

+70
-36
lines changed

‎src/backend/access/heap/tuptoaster.c

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ static struct varlena *toast_fetch_datum(struct varlena *attr);
7575
staticstructvarlena*toast_fetch_datum_slice(structvarlena*attr,
7676
int32sliceoffset,int32length);
7777
staticstructvarlena*toast_decompress_datum(structvarlena*attr);
78+
staticstructvarlena*toast_decompress_datum_slice(structvarlena*attr,int32slicelength);
7879
staticinttoast_open_indexes(Relationtoastrel,
7980
LOCKMODElock,
8081
Relation**toastidxs,
@@ -301,7 +302,11 @@ heap_tuple_untoast_attr_slice(struct varlena *attr,
301302
{
302303
structvarlena*tmp=preslice;
303304

304-
preslice=toast_decompress_datum(tmp);
305+
/* Decompress enough to encompass the slice and the offset */
306+
if (slicelength>0&&sliceoffset >=0)
307+
preslice=toast_decompress_datum_slice(tmp,slicelength+sliceoffset);
308+
else
309+
preslice=toast_decompress_datum(tmp);
305310

306311
if (tmp!=attr)
307312
pfree(tmp);
@@ -2272,9 +2277,38 @@ toast_decompress_datum(struct varlena *attr)
22722277
if (pglz_decompress(TOAST_COMPRESS_RAWDATA(attr),
22732278
VARSIZE(attr)-TOAST_COMPRESS_HDRSZ,
22742279
VARDATA(result),
2275-
TOAST_COMPRESS_RAWSIZE(attr))<0)
2280+
TOAST_COMPRESS_RAWSIZE(attr), true)<0)
2281+
elog(ERROR,"compressed data is corrupted");
2282+
2283+
returnresult;
2284+
}
2285+
2286+
2287+
/* ----------
2288+
* toast_decompress_datum_slice -
2289+
*
2290+
* Decompress the front of a compressed version of a varlena datum.
2291+
* offset handling happens in heap_tuple_untoast_attr_slice.
2292+
* Here we just decompress a slice from the front.
2293+
*/
2294+
staticstructvarlena*
2295+
toast_decompress_datum_slice(structvarlena*attr,int32slicelength)
2296+
{
2297+
structvarlena*result;
2298+
int32rawsize;
2299+
2300+
Assert(VARATT_IS_COMPRESSED(attr));
2301+
2302+
result= (structvarlena*)palloc(slicelength+VARHDRSZ);
2303+
2304+
rawsize=pglz_decompress(TOAST_COMPRESS_RAWDATA(attr),
2305+
VARSIZE(attr)-TOAST_COMPRESS_HDRSZ,
2306+
VARDATA(result),
2307+
slicelength, false);
2308+
if (rawsize<0)
22762309
elog(ERROR,"compressed data is corrupted");
22772310

2311+
SET_VARSIZE(result,rawsize+VARHDRSZ);
22782312
returnresult;
22792313
}
22802314

‎src/backend/access/transam/xlogreader.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1425,7 +1425,7 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
14251425
{
14261426
/* If a backup block image is compressed, decompress it */
14271427
if (pglz_decompress(ptr,bkpb->bimg_len,tmp.data,
1428-
BLCKSZ-bkpb->hole_length)<0)
1428+
BLCKSZ-bkpb->hole_length, true)<0)
14291429
{
14301430
report_invalid_record(record,"invalid compressed image at %X/%X, block %d",
14311431
(uint32) (record->ReadRecPtr >>32),

‎src/backend/utils/adt/varlena.c

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1894,7 +1894,7 @@ text_starts_with(PG_FUNCTION_ARGS)
18941894
result= false;
18951895
else
18961896
{
1897-
text*targ1=DatumGetTextPP(arg1);
1897+
text*targ1=text_substring(arg1,1,len2, false);
18981898
text*targ2=DatumGetTextPP(arg2);
18991899

19001900
result= (memcmp(VARDATA_ANY(targ1),VARDATA_ANY(targ2),
@@ -5346,17 +5346,21 @@ text_concat_ws(PG_FUNCTION_ARGS)
53465346
Datum
53475347
text_left(PG_FUNCTION_ARGS)
53485348
{
5349-
text*str=PG_GETARG_TEXT_PP(0);
5350-
constchar*p=VARDATA_ANY(str);
5351-
intlen=VARSIZE_ANY_EXHDR(str);
5352-
intn=PG_GETARG_INT32(1);
5353-
intrlen;
5349+
intn=PG_GETARG_INT32(1);
53545350

53555351
if (n<0)
5356-
n=pg_mbstrlen_with_len(p,len)+n;
5357-
rlen=pg_mbcharcliplen(p,len,n);
5352+
{
5353+
text*str=PG_GETARG_TEXT_PP(0);
5354+
constchar*p=VARDATA_ANY(str);
5355+
intlen=VARSIZE_ANY_EXHDR(str);
5356+
intrlen;
53585357

5359-
PG_RETURN_TEXT_P(cstring_to_text_with_len(p,rlen));
5358+
n=pg_mbstrlen_with_len(p,len)+n;
5359+
rlen=pg_mbcharcliplen(p,len,n);
5360+
PG_RETURN_TEXT_P(cstring_to_text_with_len(p,rlen));
5361+
}
5362+
else
5363+
PG_RETURN_TEXT_P(text_substring(PG_GETARG_DATUM(0),1,n, false));
53605364
}
53615365

53625366
/*

‎src/common/pg_lzcompress.c

Lines changed: 19 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
*
3030
*int32
3131
*pglz_decompress(const char *source, int32 slen, char *dest,
32-
*int32 rawsize)
32+
*int32 rawsize, bool check_complete)
3333
*
3434
*source is the compressed input.
3535
*
@@ -44,6 +44,12 @@
4444
*
4545
*rawsize is the length of the uncompressed data.
4646
*
47+
*check_complete is a flag to let us know if -1 should be
48+
*returned in cases where we don't reach the end of the
49+
*source or dest buffers, or not. This should be false
50+
*if the caller is asking for only a partial result and
51+
*true otherwise.
52+
*
4753
*The return value is the number of bytes written in the
4854
*buffer dest, or -1 if decompression fails.
4955
*
@@ -674,13 +680,14 @@ pglz_compress(const char *source, int32 slen, char *dest,
674680
* pglz_decompress -
675681
*
676682
*Decompresses source into dest. Returns the number of bytes
677-
*decompressed in the destination buffer, or -1 if decompression
678-
*fails.
683+
*decompressed in the destination buffer, and *optionally*
684+
*checks that both the source and dest buffers have been
685+
*fully read and written to, respectively.
679686
* ----------
680687
*/
681688
int32
682689
pglz_decompress(constchar*source,int32slen,char*dest,
683-
int32rawsize)
690+
int32rawsize,boolcheck_complete)
684691
{
685692
constunsignedchar*sp;
686693
constunsignedchar*srcend;
@@ -701,8 +708,9 @@ pglz_decompress(const char *source, int32 slen, char *dest,
701708
unsignedcharctrl=*sp++;
702709
intctrlc;
703710

704-
for (ctrlc=0;ctrlc<8&&sp<srcend;ctrlc++)
711+
for (ctrlc=0;ctrlc<8&&sp<srcend&&dp<destend;ctrlc++)
705712
{
713+
706714
if (ctrl&1)
707715
{
708716
/*
@@ -721,25 +729,13 @@ pglz_decompress(const char *source, int32 slen, char *dest,
721729
if (len==18)
722730
len+=*sp++;
723731

724-
/*
725-
* Check for output buffer overrun, to ensure we don't clobber
726-
* memory in case of corrupt input. Note: we must advance dp
727-
* here to ensure the error is detected below the loop. We
728-
* don't simply put the elog inside the loop since that will
729-
* probably interfere with optimization.
730-
*/
731-
if (dp+len>destend)
732-
{
733-
dp+=len;
734-
break;
735-
}
736-
737732
/*
738733
* Now we copy the bytes specified by the tag from OUTPUT to
739734
* OUTPUT. It is dangerous and platform dependent to use
740735
* memcpy() here, because the copied areas could overlap
741736
* extremely!
742737
*/
738+
len=Min(len,destend-dp);
743739
while (len--)
744740
{
745741
*dp=dp[-off];
@@ -752,9 +748,6 @@ pglz_decompress(const char *source, int32 slen, char *dest,
752748
* An unset control bit means LITERAL BYTE. So we just copy
753749
* one from INPUT to OUTPUT.
754750
*/
755-
if (dp >=destend)/* check for buffer overrun */
756-
break;/* do not clobber memory */
757-
758751
*dp++=*sp++;
759752
}
760753

@@ -767,12 +760,15 @@ pglz_decompress(const char *source, int32 slen, char *dest,
767760

768761
/*
769762
* Check we decompressed the right amount.
763+
* If we are slicing, then we won't necessarily
764+
* be at the end of the source or dest buffers
765+
* when we hit a stop, so we don't test them.
770766
*/
771-
if (dp!=destend||sp!=srcend)
767+
if (check_complete&& (dp!=destend||sp!=srcend))
772768
return-1;
773769

774770
/*
775771
* That's it.
776772
*/
777-
returnrawsize;
773+
return(char*)dp-dest;
778774
}

‎src/include/common/pg_lzcompress.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,6 @@ extern const PGLZ_Strategy *const PGLZ_strategy_always;
8686
externint32pglz_compress(constchar*source,int32slen,char*dest,
8787
constPGLZ_Strategy*strategy);
8888
externint32pglz_decompress(constchar*source,int32slen,char*dest,
89-
int32rawsize);
89+
int32rawsize,boolcheck_complete);
9090

9191
#endif/* _PG_LZCOMPRESS_H_ */

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp