Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit61633f7

Browse files
committed
Correct logical decoding restore behaviour for subtransactions.
Before initializing iteration over a subtransaction's changes, the lastfew changes were not spilled to disk. That's correct if the transactiondidn't spill to disk, but otherwise... This bug can lead to missed ormisorderd subtransaction contents when they were spilled to disk.Move spilling of the remaining in-memory changes toReorderBufferIterTXNInit(), where it can easily be applied to the toptransaction and, if present, subtransactions.Since this code had too many bugs already, noticeably increase testcoverage.Fixes: #14319Reported-By: Huan RuanDiscussion: <20160909012610.20024.58169@wrigleys.postgresql.org>Backport: 9,4-, where logical decoding was added
1 parentd51924b commit61633f7

File tree

4 files changed

+445
-6
lines changed

4 files changed

+445
-6
lines changed

‎contrib/test_decoding/Makefile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,8 @@ submake-test_decoding:
3838
$(MAKE) -C$(top_builddir)/contrib/test_decoding
3939

4040
REGRESSCHECKS=ddl xact rewrite toast permissions decoding_in_xact\
41-
decoding_into_rel binary prepared replorigin time messages
41+
decoding_into_rel binary prepared replorigin time messages\
42+
spill
4243

4344
regresscheck: | submake-regress submake-test_decoding temp-install
4445
$(MKDIR_P) regression_output
Lines changed: 256 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,256 @@
1+
-- predictability
2+
SET synchronous_commit = on;
3+
SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_decoding');
4+
?column?
5+
----------
6+
init
7+
(1 row)
8+
9+
CREATE TABLE spill_test(data text);
10+
-- consume DDL
11+
SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');
12+
data
13+
------
14+
(0 rows)
15+
16+
-- spilling main xact
17+
BEGIN;
18+
INSERT INTO spill_test SELECT 'serialize-topbig--1:'||g.i FROM generate_series(1, 5000) g(i);
19+
COMMIT;
20+
SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)]
21+
FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT'
22+
GROUP BY 1 ORDER BY 1;
23+
regexp_split_to_array | count | array_agg | array_agg
24+
-----------------------+-------+---------------------------------------------------------------------+------------------------------------------------------------------------
25+
'serialize-topbig--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-topbig--1:1' | table public.spill_test: INSERT: data[text]:'serialize-topbig--1:5000'
26+
(1 row)
27+
28+
-- spilling subxact, nothing in main
29+
BEGIN;
30+
SAVEPOINT s;
31+
INSERT INTO spill_test SELECT 'serialize-subbig--1:'||g.i FROM generate_series(1, 5000) g(i);
32+
RELEASE SAVEPOINT s;
33+
COMMIT;
34+
SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)]
35+
FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT'
36+
GROUP BY 1 ORDER BY 1;
37+
regexp_split_to_array | count | array_agg | array_agg
38+
-----------------------+-------+---------------------------------------------------------------------+------------------------------------------------------------------------
39+
'serialize-subbig--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subbig--1:1' | table public.spill_test: INSERT: data[text]:'serialize-subbig--1:5000'
40+
(1 row)
41+
42+
-- spilling subxact, spilling main xact
43+
BEGIN;
44+
SAVEPOINT s;
45+
INSERT INTO spill_test SELECT 'serialize-subbig-topbig--1:'||g.i FROM generate_series(1, 5000) g(i);
46+
RELEASE SAVEPOINT s;
47+
INSERT INTO spill_test SELECT 'serialize-subbig-topbig--2:'||g.i FROM generate_series(5001, 10000) g(i);
48+
COMMIT;
49+
SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)]
50+
FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT'
51+
GROUP BY 1 ORDER BY 1;
52+
regexp_split_to_array | count | array_agg | array_agg
53+
-----------------------------+-------+-------------------------------------------------------------------------------+--------------------------------------------------------------------------------
54+
'serialize-subbig-topbig--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subbig-topbig--1:1' | table public.spill_test: INSERT: data[text]:'serialize-subbig-topbig--1:5000'
55+
'serialize-subbig-topbig--2 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subbig-topbig--2:5001' | table public.spill_test: INSERT: data[text]:'serialize-subbig-topbig--2:10000'
56+
(2 rows)
57+
58+
-- spilling subxact, non-spilling main xact
59+
BEGIN;
60+
SAVEPOINT s;
61+
INSERT INTO spill_test SELECT 'serialize-subbig-topsmall--1:'||g.i FROM generate_series(1, 5000) g(i);
62+
RELEASE SAVEPOINT s;
63+
INSERT INTO spill_test SELECT 'serialize-subbig-topsmall--2:'||g.i FROM generate_series(5001, 5001) g(i);
64+
COMMIT;
65+
SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)]
66+
FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT'
67+
GROUP BY 1 ORDER BY 1;
68+
regexp_split_to_array | count | array_agg | array_agg
69+
-------------------------------+-------+---------------------------------------------------------------------------------+---------------------------------------------------------------------------------
70+
'serialize-subbig-topsmall--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subbig-topsmall--1:1' | table public.spill_test: INSERT: data[text]:'serialize-subbig-topsmall--1:5000'
71+
'serialize-subbig-topsmall--2 | 1 | table public.spill_test: INSERT: data[text]:'serialize-subbig-topsmall--2:5001' | table public.spill_test: INSERT: data[text]:'serialize-subbig-topsmall--2:5001'
72+
(2 rows)
73+
74+
-- not-spilling subxact, spilling main xact
75+
BEGIN;
76+
SAVEPOINT s;
77+
INSERT INTO spill_test SELECT 'serialize-subbig-topbig--1:'||g.i FROM generate_series(1, 5000) g(i);
78+
RELEASE SAVEPOINT s;
79+
INSERT INTO spill_test SELECT 'serialize-subbig-topbig--2:'||g.i FROM generate_series(5001, 10000) g(i);
80+
COMMIT;
81+
SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)]
82+
FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT'
83+
GROUP BY 1 ORDER BY 1;
84+
regexp_split_to_array | count | array_agg | array_agg
85+
-----------------------------+-------+-------------------------------------------------------------------------------+--------------------------------------------------------------------------------
86+
'serialize-subbig-topbig--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subbig-topbig--1:1' | table public.spill_test: INSERT: data[text]:'serialize-subbig-topbig--1:5000'
87+
'serialize-subbig-topbig--2 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subbig-topbig--2:5001' | table public.spill_test: INSERT: data[text]:'serialize-subbig-topbig--2:10000'
88+
(2 rows)
89+
90+
-- spilling main xact, spilling subxact
91+
BEGIN;
92+
INSERT INTO spill_test SELECT 'serialize-topbig-subbig--1:'||g.i FROM generate_series(1, 5000) g(i);
93+
SAVEPOINT s;
94+
INSERT INTO spill_test SELECT 'serialize-topbig-subbig--2:'||g.i FROM generate_series(5001, 10000) g(i);
95+
RELEASE SAVEPOINT s;
96+
COMMIT;
97+
SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)]
98+
FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT'
99+
GROUP BY 1 ORDER BY 1;
100+
regexp_split_to_array | count | array_agg | array_agg
101+
-----------------------------+-------+-------------------------------------------------------------------------------+--------------------------------------------------------------------------------
102+
'serialize-topbig-subbig--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-topbig-subbig--1:1' | table public.spill_test: INSERT: data[text]:'serialize-topbig-subbig--1:5000'
103+
'serialize-topbig-subbig--2 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-topbig-subbig--2:5001' | table public.spill_test: INSERT: data[text]:'serialize-topbig-subbig--2:10000'
104+
(2 rows)
105+
106+
-- spilling main xact, not spilling subxact
107+
BEGIN;
108+
INSERT INTO spill_test SELECT 'serialize-topbig-subsmall--1:'||g.i FROM generate_series(1, 5000) g(i);
109+
SAVEPOINT s;
110+
INSERT INTO spill_test SELECT 'serialize-topbig-subsmall--2:'||g.i FROM generate_series(5001, 5001) g(i);
111+
RELEASE SAVEPOINT s;
112+
COMMIT;
113+
SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)]
114+
FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT'
115+
GROUP BY 1 ORDER BY 1;
116+
regexp_split_to_array | count | array_agg | array_agg
117+
-------------------------------+-------+---------------------------------------------------------------------------------+---------------------------------------------------------------------------------
118+
'serialize-topbig-subsmall--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-topbig-subsmall--1:1' | table public.spill_test: INSERT: data[text]:'serialize-topbig-subsmall--1:5000'
119+
'serialize-topbig-subsmall--2 | 1 | table public.spill_test: INSERT: data[text]:'serialize-topbig-subsmall--2:5001' | table public.spill_test: INSERT: data[text]:'serialize-topbig-subsmall--2:5001'
120+
(2 rows)
121+
122+
-- spilling subxact, followed by another spilling subxact
123+
BEGIN;
124+
SAVEPOINT s1;
125+
INSERT INTO spill_test SELECT 'serialize-subbig-subbig--1:'||g.i FROM generate_series(1, 5000) g(i);
126+
RELEASE SAVEPOINT s1;
127+
SAVEPOINT s2;
128+
INSERT INTO spill_test SELECT 'serialize-subbig-subbig--2:'||g.i FROM generate_series(5001, 10000) g(i);
129+
RELEASE SAVEPOINT s2;
130+
COMMIT;
131+
SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)]
132+
FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT'
133+
GROUP BY 1 ORDER BY 1;
134+
regexp_split_to_array | count | array_agg | array_agg
135+
-----------------------------+-------+-------------------------------------------------------------------------------+--------------------------------------------------------------------------------
136+
'serialize-subbig-subbig--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subbig-subbig--1:1' | table public.spill_test: INSERT: data[text]:'serialize-subbig-subbig--1:5000'
137+
'serialize-subbig-subbig--2 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subbig-subbig--2:5001' | table public.spill_test: INSERT: data[text]:'serialize-subbig-subbig--2:10000'
138+
(2 rows)
139+
140+
-- spilling subxact, followed by not spilling subxact
141+
BEGIN;
142+
SAVEPOINT s1;
143+
INSERT INTO spill_test SELECT 'serialize-subbig-subsmall--1:'||g.i FROM generate_series(1, 5000) g(i);
144+
RELEASE SAVEPOINT s1;
145+
SAVEPOINT s2;
146+
INSERT INTO spill_test SELECT 'serialize-subbig-subsmall--2:'||g.i FROM generate_series(5001, 5001) g(i);
147+
RELEASE SAVEPOINT s2;
148+
COMMIT;
149+
SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)]
150+
FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT'
151+
GROUP BY 1 ORDER BY 1;
152+
regexp_split_to_array | count | array_agg | array_agg
153+
-------------------------------+-------+---------------------------------------------------------------------------------+---------------------------------------------------------------------------------
154+
'serialize-subbig-subsmall--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subbig-subsmall--1:1' | table public.spill_test: INSERT: data[text]:'serialize-subbig-subsmall--1:5000'
155+
'serialize-subbig-subsmall--2 | 1 | table public.spill_test: INSERT: data[text]:'serialize-subbig-subsmall--2:5001' | table public.spill_test: INSERT: data[text]:'serialize-subbig-subsmall--2:5001'
156+
(2 rows)
157+
158+
-- not spilling subxact, followed by spilling subxact
159+
BEGIN;
160+
SAVEPOINT s1;
161+
INSERT INTO spill_test SELECT 'serialize-subsmall-subbig--1:'||g.i FROM generate_series(1, 1) g(i);
162+
RELEASE SAVEPOINT s1;
163+
SAVEPOINT s2;
164+
INSERT INTO spill_test SELECT 'serialize-subsmall-subbig--2:'||g.i FROM generate_series(2, 5001) g(i);
165+
RELEASE SAVEPOINT s2;
166+
COMMIT;
167+
SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)]
168+
FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT'
169+
GROUP BY 1 ORDER BY 1;
170+
regexp_split_to_array | count | array_agg | array_agg
171+
-------------------------------+-------+------------------------------------------------------------------------------+---------------------------------------------------------------------------------
172+
'serialize-subsmall-subbig--1 | 1 | table public.spill_test: INSERT: data[text]:'serialize-subsmall-subbig--1:1' | table public.spill_test: INSERT: data[text]:'serialize-subsmall-subbig--1:1'
173+
'serialize-subsmall-subbig--2 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-subsmall-subbig--2:2' | table public.spill_test: INSERT: data[text]:'serialize-subsmall-subbig--2:5001'
174+
(2 rows)
175+
176+
-- spilling subxact, containing another spilling subxact
177+
BEGIN;
178+
SAVEPOINT s1;
179+
INSERT INTO spill_test SELECT 'serialize-nested-subbig-subbig--1:'||g.i FROM generate_series(1, 5000) g(i);
180+
SAVEPOINT s2;
181+
INSERT INTO spill_test SELECT 'serialize-nested-subbig-subbig--2:'||g.i FROM generate_series(5001, 10000) g(i);
182+
RELEASE SAVEPOINT s2;
183+
RELEASE SAVEPOINT s1;
184+
COMMIT;
185+
SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)]
186+
FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT'
187+
GROUP BY 1 ORDER BY 1;
188+
regexp_split_to_array | count | array_agg | array_agg
189+
------------------------------------+-------+--------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------
190+
'serialize-nested-subbig-subbig--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subbig--1:1' | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subbig--1:5000'
191+
'serialize-nested-subbig-subbig--2 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subbig--2:5001' | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subbig--2:10000'
192+
(2 rows)
193+
194+
-- spilling subxact, containing a not spilling subxact
195+
BEGIN;
196+
SAVEPOINT s1;
197+
INSERT INTO spill_test SELECT 'serialize-nested-subbig-subsmall--1:'||g.i FROM generate_series(1, 5000) g(i);
198+
SAVEPOINT s2;
199+
INSERT INTO spill_test SELECT 'serialize-nested-subbig-subsmall--2:'||g.i FROM generate_series(5001, 5001) g(i);
200+
RELEASE SAVEPOINT s2;
201+
RELEASE SAVEPOINT s1;
202+
COMMIT;
203+
SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)]
204+
FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT'
205+
GROUP BY 1 ORDER BY 1;
206+
regexp_split_to_array | count | array_agg | array_agg
207+
--------------------------------------+-------+----------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------
208+
'serialize-nested-subbig-subsmall--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subsmall--1:1' | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subsmall--1:5000'
209+
'serialize-nested-subbig-subsmall--2 | 1 | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subsmall--2:5001' | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subsmall--2:5001'
210+
(2 rows)
211+
212+
-- not spilling subxact, containing a spilling subxact
213+
BEGIN;
214+
SAVEPOINT s1;
215+
INSERT INTO spill_test SELECT 'serialize-nested-subsmall-subbig--1:'||g.i FROM generate_series(1, 1) g(i);
216+
SAVEPOINT s2;
217+
INSERT INTO spill_test SELECT 'serialize-nested-subsmall-subbig--2:'||g.i FROM generate_series(2, 5001) g(i);
218+
RELEASE SAVEPOINT s2;
219+
RELEASE SAVEPOINT s1;
220+
COMMIT;
221+
SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)]
222+
FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT'
223+
GROUP BY 1 ORDER BY 1;
224+
regexp_split_to_array | count | array_agg | array_agg
225+
--------------------------------------+-------+-------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------
226+
'serialize-nested-subsmall-subbig--1 | 1 | table public.spill_test: INSERT: data[text]:'serialize-nested-subsmall-subbig--1:1' | table public.spill_test: INSERT: data[text]:'serialize-nested-subsmall-subbig--1:1'
227+
'serialize-nested-subsmall-subbig--2 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-nested-subsmall-subbig--2:2' | table public.spill_test: INSERT: data[text]:'serialize-nested-subsmall-subbig--2:5001'
228+
(2 rows)
229+
230+
-- not spilling subxact, containing a spilling subxact that aborts and one that commits
231+
BEGIN;
232+
SAVEPOINT s1;
233+
INSERT INTO spill_test SELECT 'serialize-nested-subbig-subbigabort--1:'||g.i FROM generate_series(1, 5000) g(i);
234+
SAVEPOINT s2;
235+
INSERT INTO spill_test SELECT 'serialize-nested-subbig-subbigabort--2:'||g.i FROM generate_series(5001, 10000) g(i);
236+
ROLLBACK TO SAVEPOINT s2;
237+
SAVEPOINT s3;
238+
INSERT INTO spill_test SELECT 'serialize-nested-subbig-subbigabort-subbig-3:'||g.i FROM generate_series(5001, 10000) g(i);
239+
RELEASE SAVEPOINT s1;
240+
COMMIT;
241+
SELECT (regexp_split_to_array(data, ':'))[4], COUNT(*), (array_agg(data))[1], (array_agg(data))[count(*)]
242+
FROM pg_logical_slot_get_changes('regression_slot', NULL,NULL) WHERE data ~ 'INSERT'
243+
GROUP BY 1 ORDER BY 1;
244+
regexp_split_to_array | count | array_agg | array_agg
245+
-----------------------------------------------+-------+-------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------
246+
'serialize-nested-subbig-subbigabort--1 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subbigabort--1:1' | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subbigabort--1:5000'
247+
'serialize-nested-subbig-subbigabort-subbig-3 | 5000 | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subbigabort-subbig-3:5001' | table public.spill_test: INSERT: data[text]:'serialize-nested-subbig-subbigabort-subbig-3:10000'
248+
(2 rows)
249+
250+
DROP TABLE spill_test;
251+
SELECT pg_drop_replication_slot('regression_slot');
252+
pg_drop_replication_slot
253+
--------------------------
254+
255+
(1 row)
256+

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp