Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit89e6e7a

Browse files
author
Nikita Glukhov
committed
Inline TOAST: compression
1 parentf061c82 commit89e6e7a

File tree

8 files changed

+86
-68
lines changed

8 files changed

+86
-68
lines changed

‎src/backend/access/common/detoast.c

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -49,19 +49,15 @@ create_detoast_iterator(struct varlena *attr)
4949
if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
5050
{
5151
iter->compressed= true;
52+
iter->compression_method=VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
5253

5354
/* prepare buffer to received decompressed data */
5455
iter->buf=create_toast_buffer(toast_pointer.va_rawsize, false);
55-
56-
/* initialize state for pglz_decompress_iterate() */
57-
iter->ctrl=0;
58-
iter->ctrlc=INVALID_CTRLC;
59-
iter->len=0;
60-
iter->off=0;
6156
}
6257
else
6358
{
6459
iter->compressed= false;
60+
iter->compression_method=TOAST_INVALID_COMPRESSION_ID;
6561

6662
/* point the buffer directly at the raw data */
6763
iter->buf=fetch_iter->buf;
@@ -95,16 +91,14 @@ create_detoast_iterator(struct varlena *attr)
9591
iter->fetch_datum_iterator->buf=buf=create_toast_buffer(VARSIZE_ANY(attr), true);
9692
iter->fetch_datum_iterator->done= true;
9793
iter->compressed= true;
94+
iter->compression_method=VARDATA_COMPRESSED_GET_COMPRESS_METHOD(attr);
9895

9996
memcpy((void*)buf->buf,attr,VARSIZE_ANY(attr));
10097
buf->limit= (char*)buf->capacity;
10198

10299
/* prepare buffer to received decompressed data */
103100
iter->buf=create_toast_buffer(TOAST_COMPRESS_EXTSIZE(attr)+VARHDRSZ, false);
104101

105-
/* initialize state for pglz_decompress_iterate() */
106-
iter->ctrl=0;
107-
iter->ctrlc=INVALID_CTRLC;
108102
returniter;
109103
}
110104
else

‎src/backend/access/common/toast_compression.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ pglz_compress_datum(const struct varlena *value)
6464
len=pglz_compress(VARDATA_ANY(value),
6565
valsize,
6666
(char*)tmp+VARHDRSZ_COMPRESSED,
67+
NULL,
6768
NULL);
6869
if (len<0)
6970
{

‎src/backend/access/common/toast_internals.c

Lines changed: 40 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1135,28 +1135,12 @@ toast_extract_chunk_fields(Relation toastrel, TupleDesc toasttupDesc,
11351135
}
11361136
}
11371137

1138-
/* ----------
1139-
* create_fetch_datum_iterator -
1140-
*
1141-
* Initialize fetch datum iterator.
1142-
* ----------
1143-
*/
1144-
FetchDatumIterator
1145-
create_fetch_datum_iterator(structvarlena*attr)
1138+
staticvoid
1139+
create_fetch_datum_iterator_scan(FetchDatumIteratoriter)
11461140
{
11471141
intvalidIndex;
1148-
FetchDatumIteratoriter;
1149-
1150-
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
1151-
elog(ERROR,"create_fetch_datum_iterator shouldn't be called for non-ondisk datums");
1152-
1153-
iter= (FetchDatumIterator)palloc0(sizeof(FetchDatumIteratorData));
1154-
1155-
/* Must copy to access aligned fields */
1156-
VARATT_EXTERNAL_GET_POINTER(iter->toast_pointer,attr);
11571142

1158-
iter->ressize=VARATT_EXTERNAL_GET_EXTSIZE(iter->toast_pointer);
1159-
iter->numchunks= ((iter->ressize-1) /TOAST_MAX_CHUNK_SIZE)+1;
1143+
MemoryContextoldcxt=MemoryContextSwitchTo(iter->mcxt);
11601144

11611145
/*
11621146
* Open the toast relation and its indexes
@@ -1189,6 +1173,33 @@ create_fetch_datum_iterator(struct varlena *attr)
11891173
iter->toastscan=systable_beginscan_ordered(iter->toastrel,iter->toastidxs[validIndex],
11901174
&iter->snapshot,1,&iter->toastkey);
11911175

1176+
MemoryContextSwitchTo(oldcxt);
1177+
}
1178+
1179+
/* ----------
1180+
* create_fetch_datum_iterator -
1181+
*
1182+
* Initialize fetch datum iterator.
1183+
* ----------
1184+
*/
1185+
FetchDatumIterator
1186+
create_fetch_datum_iterator(structvarlena*attr)
1187+
{
1188+
FetchDatumIteratoriter;
1189+
1190+
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
1191+
elog(ERROR,"create_fetch_datum_iterator shouldn't be called for non-ondisk datums");
1192+
1193+
iter= (FetchDatumIterator)palloc0(sizeof(FetchDatumIteratorData));
1194+
1195+
iter->mcxt=CurrentMemoryContext;
1196+
1197+
/* Must copy to access aligned fields */
1198+
VARATT_EXTERNAL_GET_POINTER(iter->toast_pointer,attr);
1199+
1200+
iter->ressize=VARATT_EXTERNAL_GET_EXTSIZE(iter->toast_pointer);
1201+
iter->numchunks= ((iter->ressize-1) /TOAST_MAX_CHUNK_SIZE)+1;
1202+
11921203
iter->buf=create_toast_buffer(iter->ressize+VARHDRSZ,
11931204
VARATT_EXTERNAL_IS_COMPRESSED(iter->toast_pointer));
11941205

@@ -1204,7 +1215,7 @@ free_fetch_datum_iterator(FetchDatumIterator iter)
12041215
if (iter==NULL)
12051216
return;
12061217

1207-
if (!iter->done)
1218+
if (!iter->done&&iter->toastscan)
12081219
{
12091220
systable_endscan_ordered(iter->toastscan);
12101221
toast_close_indexes(iter->toastidxs,iter->num_indexes,AccessShareLock);
@@ -1236,6 +1247,9 @@ fetch_datum_iterate(FetchDatumIterator iter)
12361247

12371248
Assert(iter!=NULL&& !iter->done);
12381249

1250+
if (!iter->toastscan)
1251+
create_fetch_datum_iterator_scan(iter);
1252+
12391253
ttup=systable_getnext_ordered(iter->toastscan,ForwardScanDirection);
12401254
if (ttup==NULL)
12411255
{
@@ -1344,8 +1358,9 @@ ToastBuffer *
13441358
create_toast_buffer(int32size,boolcompressed)
13451359
{
13461360
ToastBuffer*buf= (ToastBuffer*)palloc0(sizeof(ToastBuffer));
1347-
buf->buf= (constchar*)palloc0(size);
1348-
if (compressed) {
1361+
buf->buf= (constchar*)palloc(size);
1362+
if (compressed)
1363+
{
13491364
SET_VARSIZE_COMPRESSED(buf->buf,size);
13501365
/*
13511366
* Note the constraint buf->position <= buf->limit may be broken
@@ -1375,6 +1390,7 @@ free_toast_buffer(ToastBuffer *buf)
13751390
pfree(buf);
13761391
}
13771392

1393+
#if0
13781394
/* ----------
13791395
* pglz_decompress_iterate -
13801396
*
@@ -1392,7 +1408,7 @@ free_toast_buffer(ToastBuffer *buf)
13921408
*/
13931409
void
13941410
pglz_decompress_iterate(ToastBuffer*source,ToastBuffer*dest,
1395-
DetoastIteratoriter,unsignedchar*destend)
1411+
DetoastIteratoriter,constchar*destend)
13961412
{
13971413
constunsignedchar*sp;
13981414
constunsignedchar*srcend;
@@ -1525,3 +1541,4 @@ pglz_decompress_iterate(ToastBuffer *source, ToastBuffer *dest,
15251541
source->position= (char*)sp;
15261542
dest->limit= (char*)dp;
15271543
}
1544+
#endif

‎src/backend/access/transam/xloginsert.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -911,7 +911,7 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
911911
switch ((WalCompression)wal_compression)
912912
{
913913
caseWAL_COMPRESSION_PGLZ:
914-
len=pglz_compress(source,orig_len,dest,PGLZ_strategy_default);
914+
len=pglz_compress(source,orig_len,dest,PGLZ_strategy_default,NULL);
915915
break;
916916

917917
caseWAL_COMPRESSION_LZ4:

‎src/common/pg_lzcompress.c

Lines changed: 27 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -507,9 +507,10 @@ pglz_find_match(int16 *hstart, const char *input, const char *end,
507507
*/
508508
int32
509509
pglz_compress(constchar*source,int32slen,char*dest,
510-
constPGLZ_Strategy*strategy)
510+
constPGLZ_Strategy*strategy,int32*dlen)
511511
{
512512
unsignedchar*bp= (unsignedchar*)dest;
513+
unsignedchar*bend;
513514
unsignedchar*bstart=bp;
514515
inthist_next=1;
515516
boolhist_recycle= false;
@@ -545,6 +546,16 @@ pglz_compress(const char *source, int32 slen, char *dest,
545546
slen>strategy->max_input_size)
546547
return-1;
547548

549+
if (dlen)
550+
{
551+
if (*dlen<4)
552+
return-1;
553+
554+
bend=bstart+*dlen-4;
555+
}
556+
else
557+
bend=bstart+PGLZ_MAX_OUTPUT(slen);
558+
548559
/*
549560
* Limit the match parameters to the supported range.
550561
*/
@@ -627,6 +638,9 @@ pglz_compress(const char *source, int32 slen, char *dest,
627638
if (!found_match&&bp-bstart >=strategy->first_success_by)
628639
return-1;
629640

641+
if (bp>bend)
642+
break;
643+
630644
/*
631645
* Try to find a match in the history
632646
*/
@@ -671,15 +685,16 @@ pglz_compress(const char *source, int32 slen, char *dest,
671685
if (result_size >=result_max)
672686
return-1;
673687

688+
if (dlen)
689+
*dlen=dp-source;
690+
674691
/* success */
675692
returnresult_size;
676693
}
677694

678695
/* Opaque pglz decompression state */
679696
typedefstructpglz_state
680697
{
681-
constunsignedchar*sp;
682-
unsignedchar*dp;
683698
int32len;
684699
int32off;
685700
intctrlc;
@@ -699,28 +714,23 @@ typedef struct pglz_state
699714
* ----------
700715
*/
701716
int32
702-
pglz_decompress_state(constchar*source,int32slen,char*dest,
717+
pglz_decompress_state(constchar*source,int32*slen,char*dest,
703718
int32dlen,boolcheck_complete,boollast_cource_chunk,
704719
void**pstate)
705720
{
706721
pglz_state*state=pstate ?*pstate :NULL;
707-
constunsignedchar*sp;
708-
constunsignedchar*srcend;
709-
unsignedchar*dp;
710-
unsignedchar*destend;
722+
constunsignedchar*sp= (constunsignedchar*)source;
723+
constunsignedchar*srcend=sp+*slen;
724+
unsignedchar*dp= (unsignedchar*)dest;
725+
unsignedchar*destend=dp+dlen;
711726
unsignedcharctrl;
712727
intctrlc;
713728
int32len;
714729
int32remlen;
715730
int32off;
716731

717-
srcend= ((constunsignedchar*)source)+slen;
718-
destend= ((unsignedchar*)dest)+rawsize;
719-
720732
if (state)
721733
{
722-
sp=state->sp;
723-
dp=state->dp;
724734
ctrl=state->ctrl;
725735
ctrlc=state->ctrlc;
726736

@@ -742,7 +752,7 @@ pglz_decompress_state(const char *source, int32 slen, char *dest,
742752
if (dp >=destend)
743753
{
744754
state->len=remlen;
745-
state->dp=dp;
755+
*slen=0;
746756
return (char*)dp-dest;
747757
}
748758

@@ -757,8 +767,6 @@ pglz_decompress_state(const char *source, int32 slen, char *dest,
757767
}
758768
else
759769
{
760-
sp= (constunsignedchar*)source;
761-
dp= (unsignedchar*)dest;
762770
ctrl=0;
763771
ctrlc=8;
764772
remlen=0;
@@ -884,10 +892,10 @@ pglz_decompress_state(const char *source, int32 slen, char *dest,
884892

885893
state->ctrl=ctrl;
886894
state->ctrlc=ctrlc;
887-
state->sp=sp;
888-
state->dp=dp;
889895
state->len=remlen;
890896
state->off=off;
897+
898+
*slen= (constchar*)sp-source;
891899
}
892900

893901
/*
@@ -948,5 +956,5 @@ int32
948956
pglz_decompress(constchar*source,int32slen,char*dest,int32rawsize,
949957
boolcheck_complete)
950958
{
951-
returnpglz_decompress_state(source,slen,dest,rawsize,check_complete, true,NULL);
959+
returnpglz_decompress_state(source,&slen,dest,rawsize,check_complete, true,NULL);
952960
}

‎src/include/access/detoast.h

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ do { \
4747
#include"postgres.h"
4848
#include"access/genam.h"
4949
#include"access/toasterapi.h"
50+
#include"access/toast_compression.h"
5051

5152
/*
5253
* TOAST buffer is a producer consumer buffer.
@@ -78,10 +79,11 @@ typedef struct FetchDatumIteratorData
7879
ToastBuffer*buf;
7980
Relationtoastrel;
8081
Relation*toastidxs;
82+
MemoryContextmcxt;
8183
SysScanDesctoastscan;
8284
ScanKeyDatatoastkey;
8385
SnapshotDatasnapshot;
84-
structvaratt_externaltoast_pointer;
86+
structvaratt_externaltoast_pointer;
8587
int32ressize;
8688
int32nextidx;
8789
int32numchunks;
@@ -91,22 +93,13 @@ typedef struct FetchDatumIteratorData
9193

9294
typedefstructFetchDatumIteratorData*FetchDatumIterator;
9395

94-
/*
95-
* If "ctrlc" field in iterator is equal to INVALID_CTRLC, it means that
96-
* the field is invalid and need to read the control byte from the
97-
* source buffer in the next iteration, see pglz_decompress_iterate().
98-
*/
99-
#defineINVALID_CTRLC 8
100-
10196
typedefstructDetoastIteratorData
10297
{
10398
ToastBuffer*buf;
10499
FetchDatumIteratorfetch_datum_iterator;
105-
unsignedcharctrl;
106-
intctrlc;
107100
intnrefs;
108-
int32len;
109-
int32off;
101+
void*decompression_state;
102+
ToastCompressionIdcompression_method;
110103
boolcompressed;/* toast value is compressed? */
111104
booldone;
112105
}DetoastIteratorData;
@@ -118,8 +111,10 @@ extern void free_fetch_datum_iterator(FetchDatumIterator iter);
118111
externvoidfetch_datum_iterate(FetchDatumIteratoriter);
119112
externToastBuffer*create_toast_buffer(int32size,boolcompressed);
120113
externvoidfree_toast_buffer(ToastBuffer*buf);
114+
externvoidtoast_decompress_iterate(ToastBuffer*source,ToastBuffer*dest,
115+
DetoastIteratoriter,constchar*destend);
121116
externvoidpglz_decompress_iterate(ToastBuffer*source,ToastBuffer*dest,
122-
DetoastIteratoriter,unsignedchar*destend);
117+
DetoastIteratoriter,char*destend);
123118

124119
/* ----------
125120
* create_detoast_iterator -
@@ -171,7 +166,7 @@ detoast_iterate(DetoastIterator detoast_iter, const char *destend)
171166
fetch_datum_iterate(fetch_iter);
172167

173168
if (detoast_iter->compressed)
174-
pglz_decompress_iterate(fetch_iter->buf,detoast_iter->buf,detoast_iter, (unsignedchar*)destend);
169+
toast_decompress_iterate(fetch_iter->buf,detoast_iter->buf,detoast_iter,destend);
175170

176171
if (detoast_iter->buf->limit==detoast_iter->buf->capacity)
177172
detoast_iter->done= true;

‎src/include/access/toast_internals.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@ typedef struct toast_compress_header
4040
#defineTOAST_COMPRESS_METHOD(ptr) \
4141
(((toast_compress_header *) (ptr))->tcinfo >> VARLENA_EXTSIZE_BITS)
4242

43+
#defineTOAST_COMPRESS_HDRSZVARHDRSZ_COMPRESSED
44+
#defineTOAST_COMPRESS_RAWDATA(attr)((char *)(attr) + TOAST_COMPRESS_HDRSZ)
45+
4346
#defineTOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(ptr,len,cm_method) \
4447
do { \
4548
Assert((len) > 0 && (len) <= VARLENA_EXTSIZE_MASK); \

‎src/include/common/pg_lzcompress.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,10 +84,10 @@ extern PGDLLIMPORT const PGLZ_Strategy *const PGLZ_strategy_always;
8484
* ----------
8585
*/
8686
externint32pglz_compress(constchar*source,int32slen,char*dest,
87-
constPGLZ_Strategy*strategy);
87+
constPGLZ_Strategy*strategy,int32*dlen);
8888
externint32pglz_decompress(constchar*source,int32slen,char*dest,
8989
int32rawsize,boolcheck_complete);
90-
externint32pglz_decompress_state(constchar*source,int32slen,
90+
externint32pglz_decompress_state(constchar*source,int32*slen,
9191
char*dest,int32dlen,
9292
boolcheck_complete,boollast_source_chunk,
9393
void**state);

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp