Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitc8f621c

Browse files
committed
logical decoding: Fix handling of large old tuples with replica identity full.
When decoding the old version of an UPDATE or DELETE change, and if thattuple was bigger than MaxHeapTupleSize, we either Assert'ed out, orfailed in more subtle ways in non-assert builds. Normally individualtuples aren't bigger than MaxHeapTupleSize, with big datums toasted.But that's not the case for the old version of a tuple for logicaldecoding; the replica identity is logged as one piece. With the defaultreplica identity btree limits that to small tuples, but that's not thecase for FULL.Change the tuple buffer infrastructure to separate allocate over-largetuples, instead of always going through the slab cache.This unfortunately requires changing the ReorderBufferTupleBufdefinition, we need to store the allocated size someplace. To avoidrequiring output plugins to recompile, don't store HeapTupleHeaderDatadirectly after HeapTupleData, but point to it via t_data; that leavesrooms for the allocated size. As there's no reason for an output pluginto look at ReorderBufferTupleBuf->t_data.header, remove the field. Itwas just a minor convenience having it directly accessible.Reported-By: Adam DratwińskiDiscussion: CAKg6ypLd7773AOX4DiOGRwQk1TVOQKhNwjYiVjJnpq8Wo+i62Q@mail.gmail.com
1 parent0bda14d commitc8f621c

File tree

5 files changed

+222
-63
lines changed

5 files changed

+222
-63
lines changed

‎contrib/test_decoding/expected/toast.out

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -292,6 +292,64 @@ SELECT substr(data, 1, 200) FROM pg_logical_slot_get_changes('regression_slot',
292292
COMMIT
293293
(235 rows)
294294

295+
-- test we can decode "old" tuples bigger than the max heap tuple size correctly
296+
DROP TABLE IF EXISTS toasted_several;
297+
NOTICE: table "toasted_several" does not exist, skipping
298+
CREATE TABLE toasted_several (
299+
id serial unique not null,
300+
toasted_key text primary key,
301+
toasted_col1 text,
302+
toasted_col2 text
303+
);
304+
ALTER TABLE toasted_several REPLICA IDENTITY FULL;
305+
ALTER TABLE toasted_several ALTER COLUMN toasted_key SET STORAGE EXTERNAL;
306+
ALTER TABLE toasted_several ALTER COLUMN toasted_col1 SET STORAGE EXTERNAL;
307+
ALTER TABLE toasted_several ALTER COLUMN toasted_col2 SET STORAGE EXTERNAL;
308+
INSERT INTO toasted_several(toasted_key) VALUES(repeat('9876543210', 2000));
309+
SELECT regexp_replace(data, '^(.{100}).*(.{100})$', '\1..\2') FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');
310+
regexp_replace
311+
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
312+
BEGIN
313+
table public.toasted_several: INSERT: id[integer]:1 toasted_key[text]:'98765432109876543210987654321..098765432109876543210987654321098765432109876543210' toasted_col1[text]:null toasted_col2[text]:null
314+
COMMIT
315+
(3 rows)
316+
317+
-- test update of a toasted key without changing it
318+
UPDATE toasted_several SET toasted_col1 = toasted_key;
319+
UPDATE toasted_several SET toasted_col2 = toasted_col1;
320+
SELECT regexp_replace(data, '^(.{100}).*(.{100})$', '\1..\2') FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');
321+
regexp_replace
322+
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
323+
BEGIN
324+
table public.toasted_several: INSERT: id[integer]:1 toasted_key[text]:'98765432109876543210987654321..098765432109876543210987654321098765432109876543210' toasted_col1[text]:null toasted_col2[text]:null
325+
COMMIT
326+
BEGIN
327+
table public.toasted_several: UPDATE: old-key: id[integer]:1 toasted_key[text]:'98765432109876543210..432109876543210987654321098765432109876543210987654321098765432109876543210' toasted_col2[text]:null
328+
COMMIT
329+
BEGIN
330+
table public.toasted_several: UPDATE: old-key: id[integer]:1 toasted_key[text]:'98765432109876543210..876543210987654321098765432109876543210987654321098765432109876543210987654321098765432109876543210'
331+
COMMIT
332+
(9 rows)
333+
334+
/*
335+
* update with large tuplebuf, in a transaction large enough to force to spool to disk
336+
*/
337+
BEGIN;
338+
INSERT INTO toasted_several(toasted_key) SELECT * FROM generate_series(1, 10234);
339+
UPDATE toasted_several SET toasted_col1 = toasted_col2 WHERE id = 1;
340+
DELETE FROM toasted_several WHERE id = 1;
341+
COMMIT;
342+
DROP TABLE toasted_several;
343+
SELECT regexp_replace(data, '^(.{100}).*(.{100})$', '\1..\2') FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1')
344+
WHERE data NOT LIKE '%INSERT: %';
345+
regexp_replace
346+
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
347+
BEGIN
348+
table public.toasted_several: UPDATE: old-key: id[integer]:1 toasted_key[text]:'98765432109876543210..7654321098765432109876543210987654321098765432109876543210' toasted_col2[text]:unchanged-toast-datum
349+
table public.toasted_several: DELETE: id[integer]:1 toasted_key[text]:'98765432109876543210987654321..876543210987654321098765432109876543210987654321098765432109876543210987654321098765432109876543210'
350+
COMMIT
351+
(4 rows)
352+
295353
SELECT pg_drop_replication_slot('regression_slot');
296354
pg_drop_replication_slot
297355
--------------------------

‎contrib/test_decoding/sql/toast.sql

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,4 +265,41 @@ ALTER TABLE toasted_copy ALTER COLUMN data SET STORAGE EXTERNAL;
265265
203untoasted200
266266
\.
267267
SELECT substr(data,1,200)FROM pg_logical_slot_get_changes('regression_slot',NULL,NULL,'include-xids','0','skip-empty-xacts','1');
268+
269+
-- test we can decode "old" tuples bigger than the max heap tuple size correctly
270+
DROPTABLE IF EXISTS toasted_several;
271+
CREATETABLEtoasted_several (
272+
idserial uniquenot null,
273+
toasted_keytextprimary key,
274+
toasted_col1text,
275+
toasted_col2text
276+
);
277+
ALTERTABLE toasted_several REPLICA IDENTITY FULL;
278+
ALTERTABLE toasted_several ALTER COLUMN toasted_keySET STORAGE EXTERNAL;
279+
ALTERTABLE toasted_several ALTER COLUMN toasted_col1SET STORAGE EXTERNAL;
280+
ALTERTABLE toasted_several ALTER COLUMN toasted_col2SET STORAGE EXTERNAL;
281+
282+
INSERT INTO toasted_several(toasted_key)VALUES(repeat('9876543210',2000));
283+
284+
SELECT regexp_replace(data,'^(.{100}).*(.{100})$','\1..\2')FROM pg_logical_slot_peek_changes('regression_slot',NULL,NULL,'include-xids','0','skip-empty-xacts','1');
285+
286+
-- test update of a toasted key without changing it
287+
UPDATE toasted_severalSET toasted_col1= toasted_key;
288+
UPDATE toasted_severalSET toasted_col2= toasted_col1;
289+
290+
SELECT regexp_replace(data,'^(.{100}).*(.{100})$','\1..\2')FROM pg_logical_slot_get_changes('regression_slot',NULL,NULL,'include-xids','0','skip-empty-xacts','1');
291+
292+
/*
293+
* update with large tuplebuf, in a transaction large enough to force to spool to disk
294+
*/
295+
BEGIN;
296+
INSERT INTO toasted_several(toasted_key)SELECT*FROM generate_series(1,10234);
297+
UPDATE toasted_severalSET toasted_col1= toasted_col2WHERE id=1;
298+
DELETEFROM toasted_severalWHERE id=1;
299+
COMMIT;
300+
301+
DROPTABLE toasted_several;
302+
303+
SELECT regexp_replace(data,'^(.{100}).*(.{100})$','\1..\2')FROM pg_logical_slot_get_changes('regression_slot',NULL,NULL,'include-xids','0','skip-empty-xacts','1')
304+
WHERE data NOTLIKE'%INSERT: %';
268305
SELECT pg_drop_replication_slot('regression_slot');

‎src/backend/replication/logical/decode.c

Lines changed: 33 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -610,7 +610,8 @@ DecodeInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
610610
Sizetuplelen;
611611
char*tupledata=XLogRecGetBlockData(r,0,&tuplelen);
612612

613-
change->data.tp.newtuple=ReorderBufferGetTupleBuf(ctx->reorder);
613+
change->data.tp.newtuple=
614+
ReorderBufferGetTupleBuf(ctx->reorder,tuplelen);
614615

615616
DecodeXLogTuple(tupledata,tuplelen,change->data.tp.newtuple);
616617
}
@@ -656,7 +657,8 @@ DecodeUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
656657
{
657658
data=XLogRecGetBlockData(r,0,&datalen);
658659

659-
change->data.tp.newtuple=ReorderBufferGetTupleBuf(ctx->reorder);
660+
change->data.tp.newtuple=
661+
ReorderBufferGetTupleBuf(ctx->reorder,datalen);
660662

661663
DecodeXLogTuple(data,datalen,change->data.tp.newtuple);
662664
}
@@ -667,7 +669,8 @@ DecodeUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
667669
data=XLogRecGetData(r)+SizeOfHeapUpdate;
668670
datalen=XLogRecGetDataLen(r)-SizeOfHeapUpdate;
669671

670-
change->data.tp.oldtuple=ReorderBufferGetTupleBuf(ctx->reorder);
672+
change->data.tp.oldtuple=
673+
ReorderBufferGetTupleBuf(ctx->reorder,datalen);
671674

672675
DecodeXLogTuple(data,datalen,change->data.tp.oldtuple);
673676
}
@@ -717,13 +720,15 @@ DecodeDelete(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
717720
/* old primary key stored */
718721
if (xlrec->flags&XLH_DELETE_CONTAINS_OLD)
719722
{
723+
Sizelen=XLogRecGetDataLen(r)-SizeOfHeapDelete;
724+
720725
Assert(XLogRecGetDataLen(r)> (SizeOfHeapDelete+SizeOfHeapHeader));
721726

722-
change->data.tp.oldtuple=ReorderBufferGetTupleBuf(ctx->reorder);
727+
change->data.tp.oldtuple=
728+
ReorderBufferGetTupleBuf(ctx->reorder,len);
723729

724730
DecodeXLogTuple((char*)xlrec+SizeOfHeapDelete,
725-
XLogRecGetDataLen(r)-SizeOfHeapDelete,
726-
change->data.tp.oldtuple);
731+
len,change->data.tp.oldtuple);
727732
}
728733

729734
change->data.tp.clear_toast_afterwards= true;
@@ -783,35 +788,39 @@ DecodeMultiInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
783788
*/
784789
if (xlrec->flags&XLH_INSERT_CONTAINS_NEW_TUPLE)
785790
{
786-
change->data.tp.newtuple=ReorderBufferGetTupleBuf(ctx->reorder);
791+
HeapTupleHeaderheader;
792+
793+
xlhdr= (xl_multi_insert_tuple*)SHORTALIGN(data);
794+
data= ((char*)xlhdr)+SizeOfMultiInsertTuple;
795+
datalen=xlhdr->datalen;
796+
797+
change->data.tp.newtuple=
798+
ReorderBufferGetTupleBuf(ctx->reorder,datalen);
787799

788800
tuple=change->data.tp.newtuple;
801+
header=tuple->tuple.t_data;
789802

790803
/* not a disk based tuple */
791804
ItemPointerSetInvalid(&tuple->tuple.t_self);
792805

793-
xlhdr= (xl_multi_insert_tuple*)SHORTALIGN(data);
794-
data= ((char*)xlhdr)+SizeOfMultiInsertTuple;
795-
datalen=xlhdr->datalen;
796-
797806
/*
798807
* We can only figure this out after reassembling the
799808
* transactions.
800809
*/
801810
tuple->tuple.t_tableOid=InvalidOid;
802-
tuple->tuple.t_data=&tuple->t_data.header;
811+
803812
tuple->tuple.t_len=datalen+SizeofHeapTupleHeader;
804813

805-
memset(&tuple->t_data.header,0,SizeofHeapTupleHeader);
814+
memset(header,0,SizeofHeapTupleHeader);
806815

807-
memcpy((char*)&tuple->t_data.header+SizeofHeapTupleHeader,
816+
memcpy((char*)tuple->tuple.t_data+SizeofHeapTupleHeader,
808817
(char*)data,
809818
datalen);
810819
data+=datalen;
811820

812-
tuple->t_data.header.t_infomask=xlhdr->t_infomask;
813-
tuple->t_data.header.t_infomask2=xlhdr->t_infomask2;
814-
tuple->t_data.header.t_hoff=xlhdr->t_hoff;
821+
header->t_infomask=xlhdr->t_infomask;
822+
header->t_infomask2=xlhdr->t_infomask2;
823+
header->t_hoff=xlhdr->t_hoff;
815824
}
816825

817826
/*
@@ -877,31 +886,31 @@ DecodeXLogTuple(char *data, Size len, ReorderBufferTupleBuf *tuple)
877886
{
878887
xl_heap_headerxlhdr;
879888
intdatalen=len-SizeOfHeapHeader;
889+
HeapTupleHeaderheader;
880890

881891
Assert(datalen >=0);
882-
Assert(datalen <=MaxHeapTupleSize);
883892

884893
tuple->tuple.t_len=datalen+SizeofHeapTupleHeader;
894+
header=tuple->tuple.t_data;
885895

886896
/* not a disk based tuple */
887897
ItemPointerSetInvalid(&tuple->tuple.t_self);
888898

889899
/* we can only figure this out after reassembling the transactions */
890900
tuple->tuple.t_tableOid=InvalidOid;
891-
tuple->tuple.t_data=&tuple->t_data.header;
892901

893902
/* data is not stored aligned, copy to aligned storage */
894903
memcpy((char*)&xlhdr,
895904
data,
896905
SizeOfHeapHeader);
897906

898-
memset(&tuple->t_data.header,0,SizeofHeapTupleHeader);
907+
memset(header,0,SizeofHeapTupleHeader);
899908

900-
memcpy((char*)&tuple->t_data.header+SizeofHeapTupleHeader,
909+
memcpy(((char*)tuple->tuple.t_data)+SizeofHeapTupleHeader,
901910
data+SizeOfHeapHeader,
902911
datalen);
903912

904-
tuple->t_data.header.t_infomask=xlhdr.t_infomask;
905-
tuple->t_data.header.t_infomask2=xlhdr.t_infomask2;
906-
tuple->t_data.header.t_hoff=xlhdr.t_hoff;
913+
header->t_infomask=xlhdr.t_infomask;
914+
header->t_infomask2=xlhdr.t_infomask2;
915+
header->t_hoff=xlhdr.t_hoff;
907916
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp