Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit3682025

Browse files
committed
Add support for multiple kinds of external toast datums.
To that end, support tags rather than lengths for external datums.As an example of how this can be used, add support or "indirect"tuples which point to some externally allocated memory containinga toast tuple. Similar infrastructure could be used for otherpurposes, including, perhaps, support for alternative compressionalgorithms.Andres Freund, reviewed by Hitoshi Harada and myself
1 parent148326b commit3682025

File tree

10 files changed

+472
-43
lines changed

10 files changed

+472
-43
lines changed

‎src/backend/access/heap/tuptoaster.c‎

Lines changed: 90 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,6 @@
4444

4545
#undef TOAST_DEBUG
4646

47-
/* Size of an EXTERNAL datum that contains a standard TOAST pointer */
48-
#defineTOAST_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(struct varatt_external))
49-
5047
/*
5148
* Testing whether an externally-stored value is compressed now requires
5249
* comparing extsize (the actual length of the external data) to rawsize
@@ -87,25 +84,47 @@ static struct varlena *toast_fetch_datum_slice(struct varlena * attr,
8784
* heap_tuple_fetch_attr -
8885
*
8986
*Public entry point to get back a toasted value from
90-
*externalstorage (possibly still in compressed format).
87+
*externalsource (possibly still in compressed format).
9188
*
9289
* This will return a datum that contains all the data internally, ie, not
93-
* relying on external storage, but it can still be compressed or have a short
94-
* header.
90+
* relying on external storage or memory, but it can still be compressed or
91+
*have a shortheader.
9592
----------
9693
*/
9794
structvarlena*
9895
heap_tuple_fetch_attr(structvarlena*attr)
9996
{
10097
structvarlena*result;
10198

102-
if (VARATT_IS_EXTERNAL(attr))
99+
if (VARATT_IS_EXTERNAL_ONDISK(attr))
103100
{
104101
/*
105102
* This is an external stored plain value
106103
*/
107104
result=toast_fetch_datum(attr);
108105
}
106+
elseif (VARATT_IS_EXTERNAL_INDIRECT(attr))
107+
{
108+
/*
109+
* copy into the caller's memory context. That's not required in all
110+
* cases but sufficient for now since this is mainly used when we need
111+
* to persist a Datum for unusually long time, like in a HOLD cursor.
112+
*/
113+
structvaratt_indirectredirect;
114+
VARATT_EXTERNAL_GET_POINTER(redirect,attr);
115+
attr= (structvarlena*)redirect.pointer;
116+
117+
/* nested indirect Datums aren't allowed */
118+
Assert(!VARATT_IS_EXTERNAL_INDIRECT(attr));
119+
120+
/* doesn't make much sense, but better handle it */
121+
if (VARATT_IS_EXTERNAL_ONDISK(attr))
122+
returnheap_tuple_fetch_attr(attr);
123+
124+
/* copy datum verbatim */
125+
result= (structvarlena*)palloc(VARSIZE_ANY(attr));
126+
memcpy(result,attr,VARSIZE_ANY(attr));
127+
}
109128
else
110129
{
111130
/*
@@ -128,7 +147,7 @@ heap_tuple_fetch_attr(struct varlena * attr)
128147
structvarlena*
129148
heap_tuple_untoast_attr(structvarlena*attr)
130149
{
131-
if (VARATT_IS_EXTERNAL(attr))
150+
if (VARATT_IS_EXTERNAL_ONDISK(attr))
132151
{
133152
/*
134153
* This is an externally stored datum --- fetch it back from there
@@ -145,6 +164,17 @@ heap_tuple_untoast_attr(struct varlena * attr)
145164
pfree(tmp);
146165
}
147166
}
167+
elseif (VARATT_IS_EXTERNAL_INDIRECT(attr))
168+
{
169+
structvaratt_indirectredirect;
170+
VARATT_EXTERNAL_GET_POINTER(redirect,attr);
171+
attr= (structvarlena*)redirect.pointer;
172+
173+
/* nested indirect Datums aren't allowed */
174+
Assert(!VARATT_IS_EXTERNAL_INDIRECT(attr));
175+
176+
attr=heap_tuple_untoast_attr(attr);
177+
}
148178
elseif (VARATT_IS_COMPRESSED(attr))
149179
{
150180
/*
@@ -191,7 +221,7 @@ heap_tuple_untoast_attr_slice(struct varlena * attr,
191221
char*attrdata;
192222
int32attrsize;
193223

194-
if (VARATT_IS_EXTERNAL(attr))
224+
if (VARATT_IS_EXTERNAL_ONDISK(attr))
195225
{
196226
structvaratt_externaltoast_pointer;
197227

@@ -204,6 +234,17 @@ heap_tuple_untoast_attr_slice(struct varlena * attr,
204234
/* fetch it back (compressed marker will get set automatically) */
205235
preslice=toast_fetch_datum(attr);
206236
}
237+
elseif (VARATT_IS_EXTERNAL_INDIRECT(attr))
238+
{
239+
structvaratt_indirectredirect;
240+
VARATT_EXTERNAL_GET_POINTER(redirect,attr);
241+
242+
/* nested indirect Datums aren't allowed */
243+
Assert(!VARATT_IS_EXTERNAL_INDIRECT(redirect.pointer));
244+
245+
returnheap_tuple_untoast_attr_slice(redirect.pointer,
246+
sliceoffset,slicelength);
247+
}
207248
else
208249
preslice=attr;
209250

@@ -267,14 +308,24 @@ toast_raw_datum_size(Datum value)
267308
structvarlena*attr= (structvarlena*)DatumGetPointer(value);
268309
Sizeresult;
269310

270-
if (VARATT_IS_EXTERNAL(attr))
311+
if (VARATT_IS_EXTERNAL_ONDISK(attr))
271312
{
272313
/* va_rawsize is the size of the original datum -- including header */
273314
structvaratt_externaltoast_pointer;
274315

275316
VARATT_EXTERNAL_GET_POINTER(toast_pointer,attr);
276317
result=toast_pointer.va_rawsize;
277318
}
319+
elseif (VARATT_IS_EXTERNAL_INDIRECT(attr))
320+
{
321+
structvaratt_indirecttoast_pointer;
322+
VARATT_EXTERNAL_GET_POINTER(toast_pointer,attr);
323+
324+
/* nested indirect Datums aren't allowed */
325+
Assert(!VARATT_IS_EXTERNAL_INDIRECT(toast_pointer.pointer));
326+
327+
returntoast_raw_datum_size(PointerGetDatum(toast_pointer.pointer));
328+
}
278329
elseif (VARATT_IS_COMPRESSED(attr))
279330
{
280331
/* here, va_rawsize is just the payload size */
@@ -308,7 +359,7 @@ toast_datum_size(Datum value)
308359
structvarlena*attr= (structvarlena*)DatumGetPointer(value);
309360
Sizeresult;
310361

311-
if (VARATT_IS_EXTERNAL(attr))
362+
if (VARATT_IS_EXTERNAL_ONDISK(attr))
312363
{
313364
/*
314365
* Attribute is stored externally - return the extsize whether
@@ -320,6 +371,16 @@ toast_datum_size(Datum value)
320371
VARATT_EXTERNAL_GET_POINTER(toast_pointer,attr);
321372
result=toast_pointer.va_extsize;
322373
}
374+
elseif (VARATT_IS_EXTERNAL_INDIRECT(attr))
375+
{
376+
structvaratt_indirecttoast_pointer;
377+
VARATT_EXTERNAL_GET_POINTER(toast_pointer,attr);
378+
379+
/* nested indirect Datums aren't allowed */
380+
Assert(!VARATT_IS_EXTERNAL_INDIRECT(attr));
381+
382+
returntoast_datum_size(PointerGetDatum(toast_pointer.pointer));
383+
}
323384
elseif (VARATT_IS_SHORT(attr))
324385
{
325386
result=VARSIZE_SHORT(attr);
@@ -387,8 +448,12 @@ toast_delete(Relation rel, HeapTuple oldtup)
387448
{
388449
Datumvalue=toast_values[i];
389450

390-
if (!toast_isnull[i]&&VARATT_IS_EXTERNAL(PointerGetDatum(value)))
451+
if (toast_isnull[i])
452+
continue;
453+
elseif (VARATT_IS_EXTERNAL_ONDISK(PointerGetDatum(value)))
391454
toast_delete_datum(rel,value);
455+
elseif (VARATT_IS_EXTERNAL_INDIRECT(PointerGetDatum(value)))
456+
elog(ERROR,"attempt to delete tuple containing indirect datums");
392457
}
393458
}
394459
}
@@ -490,13 +555,13 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
490555
new_value= (structvarlena*)DatumGetPointer(toast_values[i]);
491556

492557
/*
493-
* If the old value isan externalstoredone, check if it has
494-
*changed sowe have to delete it later.
558+
* If the old value is storedon disk, check if it has changed so
559+
* we have to delete it later.
495560
*/
496561
if (att[i]->attlen==-1&& !toast_oldisnull[i]&&
497-
VARATT_IS_EXTERNAL(old_value))
562+
VARATT_IS_EXTERNAL_ONDISK(old_value))
498563
{
499-
if (toast_isnull[i]|| !VARATT_IS_EXTERNAL(new_value)||
564+
if (toast_isnull[i]|| !VARATT_IS_EXTERNAL_ONDISK(new_value)||
500565
memcmp((char*)old_value, (char*)new_value,
501566
VARSIZE_EXTERNAL(old_value))!=0)
502567
{
@@ -1258,6 +1323,8 @@ toast_save_datum(Relation rel, Datum value,
12581323
int32data_todo;
12591324
Pointerdval=DatumGetPointer(value);
12601325

1326+
Assert(!VARATT_IS_EXTERNAL(value));
1327+
12611328
/*
12621329
* Open the toast relation and its index. We can use the index to check
12631330
* uniqueness of the OID we assign to the toasted item, even though it has
@@ -1341,7 +1408,7 @@ toast_save_datum(Relation rel, Datum value,
13411408
{
13421409
structvaratt_externalold_toast_pointer;
13431410

1344-
Assert(VARATT_IS_EXTERNAL(oldexternal));
1411+
Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal));
13451412
/* Must copy to access aligned fields */
13461413
VARATT_EXTERNAL_GET_POINTER(old_toast_pointer,oldexternal);
13471414
if (old_toast_pointer.va_toastrelid==rel->rd_toastoid)
@@ -1456,7 +1523,7 @@ toast_save_datum(Relation rel, Datum value,
14561523
* Create the TOAST pointer value that we'll return
14571524
*/
14581525
result= (structvarlena*)palloc(TOAST_POINTER_SIZE);
1459-
SET_VARSIZE_EXTERNAL(result,TOAST_POINTER_SIZE);
1526+
SET_VARTAG_EXTERNAL(result,VARTAG_ONDISK);
14601527
memcpy(VARDATA_EXTERNAL(result),&toast_pointer,sizeof(toast_pointer));
14611528

14621529
returnPointerGetDatum(result);
@@ -1480,7 +1547,7 @@ toast_delete_datum(Relation rel, Datum value)
14801547
SysScanDesctoastscan;
14811548
HeapTupletoasttup;
14821549

1483-
if (!VARATT_IS_EXTERNAL(attr))
1550+
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
14841551
return;
14851552

14861553
/* Must copy to access aligned fields */
@@ -1608,6 +1675,9 @@ toast_fetch_datum(struct varlena * attr)
16081675
char*chunkdata;
16091676
int32chunksize;
16101677

1678+
if (VARATT_IS_EXTERNAL_INDIRECT(attr))
1679+
elog(ERROR,"shouldn't be called for indirect tuples");
1680+
16111681
/* Must copy to access aligned fields */
16121682
VARATT_EXTERNAL_GET_POINTER(toast_pointer,attr);
16131683

@@ -1775,7 +1845,7 @@ toast_fetch_datum_slice(struct varlena * attr, int32 sliceoffset, int32 length)
17751845
int32chcpystrt;
17761846
int32chcpyend;
17771847

1778-
Assert(VARATT_IS_EXTERNAL(attr));
1848+
Assert(VARATT_IS_EXTERNAL_ONDISK(attr));
17791849

17801850
/* Must copy to access aligned fields */
17811851
VARATT_EXTERNAL_GET_POINTER(toast_pointer,attr);

‎src/include/access/tuptoaster.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,11 @@
9494
sizeof(int32) -\
9595
VARHDRSZ)
9696

97+
/* Size of an EXTERNAL datum that contains a standard TOAST pointer */
98+
#defineTOAST_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(struct varatt_external))
99+
100+
/* Size of an indirect datum that contains an indirect TOAST pointer */
101+
#defineINDIRECT_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(struct varatt_indirect))
97102

98103
/* ----------
99104
* toast_insert_or_update -

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp