Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commita507b86

Browse files
committed
Add WAL consistency checking facility.
When the new GUC wal_consistency_checking is set to a non-empty value,it triggers recording of additional full-page images, which arecompared on the standby against the results of applying the WAL record(without regard to those full-page images). Allowable differencessuch as hints are masked out, and the resulting pages are compared;any difference results in a FATAL error on the standby.Kuntal Ghosh, based on earlier patches by Michael Paquier and HeikkiLinnakangas. Extensively reviewed and revised by Michael Paquier andby me, with additional reviews and comments from Amit Kapila, ÁlvaroHerrera, Simon Riggs, and Peter Eisentraut.
1 parent115cb31 commita507b86

36 files changed

+811
-48
lines changed

‎doc/src/sgml/config.sgml

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8184,6 +8184,38 @@ LOG: CleanUpLock: deleting: lock(0xb7acd844) id(24688,24696,0,0,0,1)
81848184
</listitem>
81858185
</varlistentry>
81868186

8187+
<varlistentry id="guc-wal-consistency-checking" xreflabel="wal_consistency_checking">
8188+
<term><varname>wal_consistency_checking</varname> (<type>string</type>)
8189+
<indexterm>
8190+
<primary><varname>wal_consistency_checking</> configuration parameter</primary>
8191+
</indexterm>
8192+
</term>
8193+
<listitem>
8194+
<para>
8195+
This parameter is intended to be used to check for bugs in the WAL
8196+
redo routines. When enabled, full-page images of any buffers modified
8197+
in conjunction with the WAL record are added to the record.
8198+
If the record is subsequently replayed, the system will first apply
8199+
each record and then test whether the buffers modified by the record
8200+
match the stored images. In certain cases (such as hint bits), minor
8201+
variations are acceptable, and will be ignored. Any unexpected
8202+
differences will result in a fatal error, terminating recovery.
8203+
</para>
8204+
8205+
<para>
8206+
The default value of this setting is the empty string, which disables
8207+
the feature. It can be set to <literal>all</literal> to check all
8208+
records, or to a comma-separated list of resource managers to check
8209+
only records originating from those resource managers. Currently,
8210+
the supported resource managers are <literal>heap</>,
8211+
<literal>heap2</>, <literal>btree</>, <literal>gin</>,
8212+
<literal>gist</>, <literal>sequence</>, <literal>spgist</>,
8213+
<literal>brin</>, and <literal>generic</>. Only
8214+
superusers can change this setting.
8215+
</para>
8216+
</listitem>
8217+
</varlistentry>
8218+
81878219
<varlistentry id="guc-wal-debug" xreflabel="wal_debug">
81888220
<term><varname>wal_debug</varname> (<type>boolean</type>)
81898221
<indexterm>

‎src/backend/access/brin/brin_xlog.c

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include"access/brin_page.h"
1414
#include"access/brin_pageops.h"
1515
#include"access/brin_xlog.h"
16+
#include"access/bufmask.h"
1617
#include"access/xlogutils.h"
1718

1819

@@ -279,3 +280,22 @@ brin_redo(XLogReaderState *record)
279280
elog(PANIC,"brin_redo: unknown op code %u",info);
280281
}
281282
}
283+
284+
/*
285+
* Mask a BRIN page before doing consistency checks.
286+
*/
287+
void
288+
brin_mask(char*pagedata,BlockNumberblkno)
289+
{
290+
Pagepage= (Page)pagedata;
291+
292+
mask_page_lsn(page);
293+
294+
mask_page_hint_bits(page);
295+
296+
if (BRIN_IS_REGULAR_PAGE(page))
297+
{
298+
/* Regular brin pages contain unused space which needs to be masked. */
299+
mask_unused_space(page);
300+
}
301+
}

‎src/backend/access/common/Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ subdir = src/backend/access/common
1212
top_builddir = ../../../..
1313
include$(top_builddir)/src/Makefile.global
1414

15-
OBJS = heaptuple.o indextuple.o printsimple.o printtup.o reloptions.o\
16-
scankey.o tupconvert.o tupdesc.o
15+
OBJS =bufmask.oheaptuple.o indextuple.o printsimple.o printtup.o\
16+
reloptions.oscankey.o tupconvert.o tupdesc.o
1717

1818
include$(top_srcdir)/src/backend/common.mk

‎src/backend/access/common/bufmask.c

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
/*-------------------------------------------------------------------------
2+
*
3+
* bufmask.c
4+
* Routines for buffer masking. Used to mask certain bits
5+
* in a page which can be different when the WAL is generated
6+
* and when the WAL is applied.
7+
*
8+
* Portions Copyright (c) 2016, PostgreSQL Global Development Group
9+
*
10+
* Contains common routines required for masking a page.
11+
*
12+
* IDENTIFICATION
13+
* src/backend/storage/buffer/bufmask.c
14+
*
15+
*-------------------------------------------------------------------------
16+
*/
17+
18+
#include"postgres.h"
19+
20+
#include"access/bufmask.h"
21+
22+
/*
23+
* mask_page_lsn
24+
*
25+
* In consistency checks, the LSN of the two pages compared will likely be
26+
* different because of concurrent operations when the WAL is generated
27+
* and the state of the page when WAL is applied.
28+
*/
29+
void
30+
mask_page_lsn(Pagepage)
31+
{
32+
PageHeaderphdr= (PageHeader)page;
33+
34+
PageXLogRecPtrSet(phdr->pd_lsn, (uint64)MASK_MARKER);
35+
}
36+
37+
/*
38+
* mask_page_hint_bits
39+
*
40+
* Mask hint bits in PageHeader. We want to ignore differences in hint bits,
41+
* since they can be set without emitting any WAL.
42+
*/
43+
void
44+
mask_page_hint_bits(Pagepage)
45+
{
46+
PageHeaderphdr= (PageHeader)page;
47+
48+
/* Ignore prune_xid (it's like a hint-bit) */
49+
phdr->pd_prune_xid=MASK_MARKER;
50+
51+
/* Ignore PD_PAGE_FULL and PD_HAS_FREE_LINES flags, they are just hints. */
52+
PageClearFull(page);
53+
PageClearHasFreeLinePointers(page);
54+
55+
/*
56+
* During replay, if the page LSN has advanced past our XLOG record's LSN,
57+
* we don't mark the page all-visible. See heap_xlog_visible() for
58+
* details.
59+
*/
60+
PageClearAllVisible(page);
61+
}
62+
63+
/*
64+
* mask_unused_space
65+
*
66+
* Mask the unused space of a page between pd_lower and pd_upper.
67+
*/
68+
void
69+
mask_unused_space(Pagepage)
70+
{
71+
intpd_lower= ((PageHeader)page)->pd_lower;
72+
intpd_upper= ((PageHeader)page)->pd_upper;
73+
intpd_special= ((PageHeader)page)->pd_special;
74+
75+
/* Sanity check */
76+
if (pd_lower>pd_upper||pd_special<pd_upper||
77+
pd_lower<SizeOfPageHeaderData||pd_special>BLCKSZ)
78+
{
79+
elog(ERROR,"invalid page pd_lower %u pd_upper %u pd_special %u\n",
80+
pd_lower,pd_upper,pd_special);
81+
}
82+
83+
memset(page+pd_lower,MASK_MARKER,pd_upper-pd_lower);
84+
}
85+
86+
/*
87+
* mask_lp_flags
88+
*
89+
* In some index AMs, line pointer flags can be modified in master without
90+
* emitting any WAL record.
91+
*/
92+
void
93+
mask_lp_flags(Pagepage)
94+
{
95+
OffsetNumberoffnum,
96+
maxoff;
97+
98+
maxoff=PageGetMaxOffsetNumber(page);
99+
for (offnum=FirstOffsetNumber;
100+
offnum <=maxoff;
101+
offnum=OffsetNumberNext(offnum))
102+
{
103+
ItemIditemId=PageGetItemId(page,offnum);
104+
105+
if (ItemIdIsUsed(itemId))
106+
itemId->lp_flags=LP_UNUSED;
107+
}
108+
}
109+
110+
/*
111+
* mask_page_content
112+
*
113+
* In some index AMs, the contents of deleted pages need to be almost
114+
* completely ignored.
115+
*/
116+
void
117+
mask_page_content(Pagepage)
118+
{
119+
/* Mask Page Content */
120+
memset(page+SizeOfPageHeaderData,MASK_MARKER,
121+
BLCKSZ-SizeOfPageHeaderData);
122+
123+
/* Mask pd_lower and pd_upper */
124+
memset(&((PageHeader)page)->pd_lower,MASK_MARKER,
125+
sizeof(uint16));
126+
memset(&((PageHeader)page)->pd_upper,MASK_MARKER,
127+
sizeof(uint16));
128+
}

‎src/backend/access/gin/ginxlog.c

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
*/
1414
#include"postgres.h"
1515

16+
#include"access/bufmask.h"
1617
#include"access/gin_private.h"
1718
#include"access/xlogutils.h"
1819
#include"utils/memutils.h"
@@ -758,3 +759,34 @@ gin_xlog_cleanup(void)
758759
MemoryContextDelete(opCtx);
759760
opCtx=NULL;
760761
}
762+
763+
/*
764+
* Mask a GIN page before running consistency checks on it.
765+
*/
766+
void
767+
gin_mask(char*pagedata,BlockNumberblkno)
768+
{
769+
Pagepage= (Page)pagedata;
770+
GinPageOpaqueopaque;
771+
772+
mask_page_lsn(page);
773+
opaque=GinPageGetOpaque(page);
774+
775+
mask_page_hint_bits(page);
776+
777+
/*
778+
* GIN metapage doesn't use pd_lower/pd_upper. Other page types do. Hence,
779+
* we need to apply masking for those pages.
780+
*/
781+
if (opaque->flags!=GIN_META)
782+
{
783+
/*
784+
* For GIN_DELETED page, the page is initialized to empty. Hence, mask
785+
* the page content.
786+
*/
787+
if (opaque->flags&GIN_DELETED)
788+
mask_page_content(page);
789+
else
790+
mask_unused_space(page);
791+
}
792+
}

‎src/backend/access/gist/gistxlog.c

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
*/
1414
#include"postgres.h"
1515

16+
#include"access/bufmask.h"
1617
#include"access/gist_private.h"
1718
#include"access/xloginsert.h"
1819
#include"access/xlogutils.h"
@@ -342,6 +343,48 @@ gist_xlog_cleanup(void)
342343
MemoryContextDelete(opCtx);
343344
}
344345

346+
/*
347+
* Mask a Gist page before running consistency checks on it.
348+
*/
349+
void
350+
gist_mask(char*pagedata,BlockNumberblkno)
351+
{
352+
Pagepage= (Page)pagedata;
353+
354+
mask_page_lsn(page);
355+
356+
mask_page_hint_bits(page);
357+
mask_unused_space(page);
358+
359+
/*
360+
* NSN is nothing but a special purpose LSN. Hence, mask it for the same
361+
* reason as mask_page_lsn.
362+
*/
363+
GistPageSetNSN(page, (uint64)MASK_MARKER);
364+
365+
/*
366+
* We update F_FOLLOW_RIGHT flag on the left child after writing WAL
367+
* record. Hence, mask this flag. See gistplacetopage() for details.
368+
*/
369+
GistMarkFollowRight(page);
370+
371+
if (GistPageIsLeaf(page))
372+
{
373+
/*
374+
* In gist leaf pages, it is possible to modify the LP_FLAGS without
375+
* emitting any WAL record. Hence, mask the line pointer flags. See
376+
* gistkillitems() for details.
377+
*/
378+
mask_lp_flags(page);
379+
}
380+
381+
/*
382+
* During gist redo, we never mark a page as garbage. Hence, mask it to
383+
* ignore any differences.
384+
*/
385+
GistClearPageHasGarbage(page);
386+
}
387+
345388
/*
346389
* Write WAL record of a page split.
347390
*/

‎src/backend/access/heap/heapam.c

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
*/
3939
#include"postgres.h"
4040

41+
#include"access/bufmask.h"
4142
#include"access/heapam.h"
4243
#include"access/heapam_xlog.h"
4344
#include"access/hio.h"
@@ -9142,3 +9143,81 @@ heap_sync(Relation rel)
91429143
heap_close(toastrel,AccessShareLock);
91439144
}
91449145
}
9146+
9147+
/*
9148+
* Mask a heap page before performing consistency checks on it.
9149+
*/
9150+
void
9151+
heap_mask(char*pagedata,BlockNumberblkno)
9152+
{
9153+
Pagepage= (Page)pagedata;
9154+
OffsetNumberoff;
9155+
9156+
mask_page_lsn(page);
9157+
9158+
mask_page_hint_bits(page);
9159+
mask_unused_space(page);
9160+
9161+
for (off=1;off <=PageGetMaxOffsetNumber(page);off++)
9162+
{
9163+
ItemIdiid=PageGetItemId(page,off);
9164+
char*page_item;
9165+
9166+
page_item= (char*) (page+ItemIdGetOffset(iid));
9167+
9168+
if (ItemIdIsNormal(iid))
9169+
{
9170+
9171+
HeapTupleHeaderpage_htup= (HeapTupleHeader)page_item;
9172+
9173+
/*
9174+
* If xmin of a tuple is not yet frozen, we should ignore
9175+
* differences in hint bits, since they can be set without
9176+
* emitting WAL.
9177+
*/
9178+
if (!HeapTupleHeaderXminFrozen(page_htup))
9179+
page_htup->t_infomask &= ~HEAP_XACT_MASK;
9180+
else
9181+
{
9182+
/* Still we need to mask xmax hint bits. */
9183+
page_htup->t_infomask &= ~HEAP_XMAX_INVALID;
9184+
page_htup->t_infomask &= ~HEAP_XMAX_COMMITTED;
9185+
}
9186+
9187+
/*
9188+
* During replay, we set Command Id to FirstCommandId. Hence, mask
9189+
* it. See heap_xlog_insert() for details.
9190+
*/
9191+
page_htup->t_choice.t_heap.t_field3.t_cid=MASK_MARKER;
9192+
9193+
/*
9194+
* For a speculative tuple, heap_insert() does not set ctid in the
9195+
* caller-passed heap tuple itself, leaving the ctid field to
9196+
* contain a speculative token value - a per-backend monotonically
9197+
* increasing identifier. Besides, it does not WAL-log ctid under
9198+
* any circumstances.
9199+
*
9200+
* During redo, heap_xlog_insert() sets t_ctid to current block
9201+
* number and self offset number. It doesn't care about any
9202+
* speculative insertions in master. Hence, we set t_ctid to
9203+
* current block number and self offset number to ignore any
9204+
* inconsistency.
9205+
*/
9206+
if (HeapTupleHeaderIsSpeculative(page_htup))
9207+
ItemPointerSet(&page_htup->t_ctid,blkno,off);
9208+
}
9209+
9210+
/*
9211+
* Ignore any padding bytes after the tuple, when the length of the
9212+
* item is not MAXALIGNed.
9213+
*/
9214+
if (ItemIdHasStorage(iid))
9215+
{
9216+
intlen=ItemIdGetLength(iid);
9217+
intpadlen=MAXALIGN(len)-len;
9218+
9219+
if (padlen>0)
9220+
memset(page_item+len,MASK_MARKER,padlen);
9221+
}
9222+
}
9223+
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp