Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitf061c82

Browse files
author
Nikita Glukhov
committed
Partial decompression in TOAST iterators
1 parent01ee7ea commitf061c82

File tree

6 files changed

+126
-16
lines changed

6 files changed

+126
-16
lines changed

‎src/backend/access/common/detoast.c

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ create_detoast_iterator(struct varlena *attr)
5656
/* initialize state for pglz_decompress_iterate() */
5757
iter->ctrl=0;
5858
iter->ctrlc=INVALID_CTRLC;
59+
iter->len=0;
60+
iter->off=0;
5961
}
6062
else
6163
{
@@ -81,7 +83,7 @@ create_detoast_iterator(struct varlena *attr)
8183
returncreate_detoast_iterator(attr);
8284

8385
}
84-
elseif (VARATT_IS_COMPRESSED(attr))
86+
elseif (1&&VARATT_IS_COMPRESSED(attr))
8587
{
8688
ToastBuffer*buf;
8789

‎src/backend/access/common/toast_compression.c

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,82 @@ lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength)
246246
#endif
247247
}
248248

249+
void
250+
toast_decompress_iterate(ToastBuffer*source,ToastBuffer*dest,
251+
DetoastIteratoriter,constchar*destend)
252+
{
253+
constchar*sp;
254+
constchar*srcend;
255+
char*dp;
256+
int32dlen;
257+
int32slen;
258+
boollast_source_chunk;
259+
260+
/*
261+
* In the while loop, sp may be incremented such that it points beyond
262+
* srcend. To guard against reading beyond the end of the current chunk,
263+
* we set srcend such that we exit the loop when we are within four bytes
264+
* of the end of the current chunk. When source->limit reaches
265+
* source->capacity, we are decompressing the last chunk, so we can (and
266+
* need to) read every byte.
267+
*/
268+
last_source_chunk=source->limit==source->capacity;
269+
srcend=last_source_chunk ?source->limit :source->limit-4;
270+
sp=source->position;
271+
dp=dest->limit;
272+
if (destend>dest->capacity)
273+
destend=dest->capacity;
274+
275+
slen=srcend-source->position;
276+
277+
/*
278+
* Decompress the data using the appropriate decompression routine.
279+
*/
280+
switch (iter->compression_method)
281+
{
282+
caseTOAST_PGLZ_COMPRESSION_ID:
283+
dlen=pglz_decompress_state(sp,&slen,dp,destend-dp,
284+
last_source_chunk&&destend==dest->capacity,
285+
last_source_chunk,
286+
&iter->decompression_state);
287+
break;
288+
caseTOAST_LZ4_COMPRESSION_ID:
289+
if (source->limit<source->capacity)
290+
dlen=0;/* LZ4 needs need full data to decompress */
291+
else
292+
{
293+
/* decompress the data */
294+
#ifndefUSE_LZ4
295+
NO_LZ4_SUPPORT();
296+
dlen=0;
297+
#else
298+
dlen=LZ4_decompress_safe(source->buf+VARHDRSZ_COMPRESSED,
299+
VARDATA(dest->buf),
300+
VARSIZE(source->buf)-VARHDRSZ_COMPRESSED,
301+
VARDATA_COMPRESSED_GET_EXTSIZE(source->buf));
302+
303+
if (dlen<0)
304+
ereport(ERROR,
305+
(errcode(ERRCODE_DATA_CORRUPTED),
306+
errmsg_internal("compressed lz4 data is corrupt")));
307+
#endif
308+
slen=0;
309+
}
310+
break;
311+
default:
312+
elog(ERROR,"invalid compression method id %d",iter->compression_method);
313+
return;/* keep compiler quiet */
314+
}
315+
316+
if (dlen<0)
317+
ereport(ERROR,
318+
(errcode(ERRCODE_DATA_CORRUPTED),
319+
errmsg_internal("compressed data is corrupt")));
320+
321+
source->position+=slen;
322+
dest->limit+=dlen;
323+
}
324+
249325
/*
250326
* Extract compression ID from a varlena.
251327
*

‎src/backend/access/common/toast_internals.c

Lines changed: 35 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1410,9 +1410,33 @@ pglz_decompress_iterate(ToastBuffer *source, ToastBuffer *dest,
14101410
(source->limit==source->capacity ?source->limit : (source->limit-4));
14111411
sp= (constunsignedchar*)source->position;
14121412
dp= (unsignedchar*)dest->limit;
1413-
if (destend< (unsignedchar*)dest->capacity)
1413+
if (destend> (unsignedchar*)dest->capacity)
14141414
destend= (unsignedchar*)dest->capacity;
14151415

1416+
if (iter->len)
1417+
{
1418+
int32len=iter->len;
1419+
int32off=iter->off;
1420+
int32copylen=Min(len,destend-dp);
1421+
int32remlen=len-copylen;
1422+
1423+
while (copylen--)
1424+
{
1425+
*dp=dp[-off];
1426+
dp++;
1427+
}
1428+
1429+
iter->len=remlen;
1430+
1431+
if (dp >=destend)
1432+
{
1433+
dest->limit= (char*)dp;
1434+
return;
1435+
}
1436+
1437+
Assert(remlen==0);
1438+
}
1439+
14161440
while (sp<srcend&&dp<destend)
14171441
{
14181442
/*
@@ -1433,7 +1457,6 @@ pglz_decompress_iterate(ToastBuffer *source, ToastBuffer *dest,
14331457
ctrlc=0;
14341458
}
14351459

1436-
14371460
for (;ctrlc<INVALID_CTRLC&&sp<srcend&&dp<destend;ctrlc++)
14381461
{
14391462

@@ -1450,6 +1473,7 @@ pglz_decompress_iterate(ToastBuffer *source, ToastBuffer *dest,
14501473
*/
14511474
int32len;
14521475
int32off;
1476+
int32copylen;
14531477

14541478
len= (sp[0]&0x0f)+3;
14551479
off= ((sp[0]&0xf0) <<4) |sp[1];
@@ -1463,17 +1487,21 @@ pglz_decompress_iterate(ToastBuffer *source, ToastBuffer *dest,
14631487
* areas could overlap; to prevent possible uncertainty, we
14641488
* copy only non-overlapping regions.
14651489
*/
1466-
len=Min(len,destend-dp);
1467-
while (off<len)
1490+
copylen=Min(len,destend-dp);
1491+
iter->len=len-copylen;
1492+
1493+
while (off<copylen)
14681494
{
14691495
/* see comments in common/pg_lzcompress.c */
14701496
memcpy(dp,dp-off,off);
1471-
len-=off;
1497+
copylen-=off;
14721498
dp+=off;
14731499
off+=off;
14741500
}
1475-
memcpy(dp,dp-off,len);
1476-
dp+=len;
1501+
memcpy(dp,dp-off,copylen);
1502+
dp+=copylen;
1503+
1504+
iter->off=off;
14771505
}
14781506
else
14791507
{

‎src/common/pg_lzcompress.c

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -700,7 +700,8 @@ typedef struct pglz_state
700700
*/
701701
int32
702702
pglz_decompress_state(constchar*source,int32slen,char*dest,
703-
int32rawsize,boolcheck_complete,void**pstate)
703+
int32dlen,boolcheck_complete,boollast_cource_chunk,
704+
void**pstate)
704705
{
705706
pglz_state*state=pstate ?*pstate :NULL;
706707
constunsignedchar*sp;
@@ -800,7 +801,7 @@ pglz_decompress_state(const char *source, int32 slen, char *dest,
800801
* must check this, else we risk an infinite loop below in the
801802
* face of corrupt data.)
802803
*/
803-
if (unlikely(sp>srcend||off==0))
804+
if (unlikely((sp>srcend&&last_cource_chunk)||off==0))
804805
return-1;
805806

806807
/*
@@ -947,5 +948,5 @@ int32
947948
pglz_decompress(constchar*source,int32slen,char*dest,int32rawsize,
948949
boolcheck_complete)
949950
{
950-
returnpglz_decompress_state(source,slen,dest,rawsize,check_complete,NULL);
951+
returnpglz_decompress_state(source,slen,dest,rawsize,check_complete,true,NULL);
951952
}

‎src/include/access/detoast.h

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,8 @@ typedef struct DetoastIteratorData
105105
unsignedcharctrl;
106106
intctrlc;
107107
intnrefs;
108+
int32len;
109+
int32off;
108110
boolcompressed;/* toast value is compressed? */
109111
booldone;
110112
}DetoastIteratorData;
@@ -156,16 +158,16 @@ detoast_iterate(DetoastIterator detoast_iter, const char *destend)
156158
if (!detoast_iter->compressed)
157159
destend=NULL;
158160

159-
if (destend)
161+
if (1&&destend)
160162
{
161163
constchar*srcend= (constchar*)
162164
(fetch_iter->buf->limit==fetch_iter->buf->capacity ?
163165
fetch_iter->buf->limit :fetch_iter->buf->limit-4);
164166

165-
if (fetch_iter->buf->position >=srcend)
167+
if (fetch_iter->buf->position >=srcend&& !fetch_iter->done)
166168
fetch_datum_iterate(fetch_iter);
167169
}
168-
else
170+
elseif (!fetch_iter->done)
169171
fetch_datum_iterate(fetch_iter);
170172

171173
if (detoast_iter->compressed)

‎src/include/common/pg_lzcompress.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,9 @@ extern int32 pglz_compress(const char *source, int32 slen, char *dest,
8787
constPGLZ_Strategy*strategy);
8888
externint32pglz_decompress(constchar*source,int32slen,char*dest,
8989
int32rawsize,boolcheck_complete);
90-
externint32pglz_decompress_state(constchar*source,int32slen,char*dest,
91-
int32rawsize,boolcheck_complete,
90+
externint32pglz_decompress_state(constchar*source,int32slen,
91+
char*dest,int32dlen,
92+
boolcheck_complete,boollast_source_chunk,
9293
void**state);
9394
externint32pglz_maximum_compressed_size(int32rawsize,
9495
int32total_compressed_size);

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp