Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitb300527

Browse files
committed
Decouple the values of TOAST_TUPLE_THRESHOLD and TOAST_MAX_CHUNK_SIZE.
Add the latter to the values checked in pg_control, since it can't be changedwithout invalidating toast table content. This commit in itself shouldn'tchange any behavior, but it lays some necessary groundwork for experimentationwith these toast-control numbers.Note: while TOAST_TUPLE_THRESHOLD can now be changed without initdb, somethought still needs to be given to needs_toast_table() in toasting.c beforeunleashing random changes.
1 parent4fea0ca commitb300527

File tree

8 files changed

+114
-48
lines changed

8 files changed

+114
-48
lines changed

‎doc/src/sgml/storage.sgml‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
<!-- $PostgreSQL: pgsql/doc/src/sgml/storage.sgml,v 1.15 2007/03/02 00:48:44 tgl Exp $ -->
1+
<!-- $PostgreSQL: pgsql/doc/src/sgml/storage.sgml,v 1.16 2007/04/03 04:14:26 tgl Exp $ -->
22

33
<chapter id="storage">
44

@@ -240,8 +240,9 @@ of the LZ family of compression techniques. See
240240

241241
<para>
242242
Out-of-line values are divided (after compression if used) into chunks of at
243-
most <literal>TOAST_MAX_CHUNK_SIZE</> bytes (this value is a little less than
244-
<literal>BLCKSZ/4</>, or about 2000 bytes by default). Each chunk is stored
243+
most <symbol>TOAST_MAX_CHUNK_SIZE</> bytes (by default this value is chosen
244+
so that four chunk rows will fit on a page, making it about 2000 bytes).
245+
Each chunk is stored
245246
as a separate row in the <acronym>TOAST</> table for the owning table. Every
246247
<acronym>TOAST</> table has the columns <structfield>chunk_id</> (an OID
247248
identifying the particular <acronym>TOAST</>ed value),
@@ -260,10 +261,12 @@ regardless of the actual size of the represented value.
260261

261262
<para>
262263
The <acronym>TOAST</> code is triggered only
263-
when a row value to be stored in a table is wider than <literal>BLCKSZ/4</>
264-
bytes (normally 2 kB). The <acronym>TOAST</> code will compress and/or move
264+
when a row value to be stored in a table is wider than
265+
<symbol>TOAST_TUPLE_THRESHOLD</> bytes (normally 2 kB).
266+
The <acronym>TOAST</> code will compress and/or move
265267
field values out-of-line until the row value is shorter than
266-
<literal>BLCKSZ/4</> bytes or no more gains can be had. During an UPDATE
268+
<symbol>TOAST_TUPLE_TARGET</> bytes (also normally 2 kB)
269+
or no more gains can be had. During an UPDATE
267270
operation, values of unchanged fields are normally preserved as-is; so an
268271
UPDATE of a row with out-of-line values incurs no <acronym>TOAST</> costs if
269272
none of the out-of-line values change.

‎src/backend/access/heap/heapam.c‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
*
99
*
1010
* IDENTIFICATION
11-
* $PostgreSQL: pgsql/src/backend/access/heap/heapam.c,v 1.230 2007/03/29 00:15:37 tgl Exp $
11+
* $PostgreSQL: pgsql/src/backend/access/heap/heapam.c,v 1.231 2007/04/03 04:14:26 tgl Exp $
1212
*
1313
*
1414
* INTERFACE ROUTINES
@@ -1420,7 +1420,13 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
14201420
* Note: below this point, heaptup is the data we actually intend to store
14211421
* into the relation; tup is the caller's original untoasted data.
14221422
*/
1423-
if (HeapTupleHasExternal(tup)||tup->t_len>TOAST_TUPLE_THRESHOLD)
1423+
if (relation->rd_rel->relkind==RELKIND_TOASTVALUE)
1424+
{
1425+
/* toast table entries should never be recursively toasted */
1426+
Assert(!HeapTupleHasExternal(tup));
1427+
heaptup=tup;
1428+
}
1429+
elseif (HeapTupleHasExternal(tup)||tup->t_len>TOAST_TUPLE_THRESHOLD)
14241430
heaptup=toast_insert_or_update(relation,tup,NULL,
14251431
use_wal,use_fsm);
14261432
else
@@ -1777,7 +1783,12 @@ heap_delete(Relation relation, ItemPointer tid,
17771783
* because we need to look at the contents of the tuple, but it's OK to
17781784
* release the content lock on the buffer first.
17791785
*/
1780-
if (HeapTupleHasExternal(&tp))
1786+
if (relation->rd_rel->relkind==RELKIND_TOASTVALUE)
1787+
{
1788+
/* toast table entries should never be recursively toasted */
1789+
Assert(!HeapTupleHasExternal(&tp));
1790+
}
1791+
elseif (HeapTupleHasExternal(&tp))
17811792
toast_delete(relation,&tp);
17821793

17831794
/*
@@ -2075,9 +2086,17 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
20752086
* We need to invoke the toaster if there are already any out-of-line
20762087
* toasted values present, or if the new tuple is over-threshold.
20772088
*/
2078-
need_toast= (HeapTupleHasExternal(&oldtup)||
2079-
HeapTupleHasExternal(newtup)||
2080-
newtup->t_len>TOAST_TUPLE_THRESHOLD);
2089+
if (relation->rd_rel->relkind==RELKIND_TOASTVALUE)
2090+
{
2091+
/* toast table entries should never be recursively toasted */
2092+
Assert(!HeapTupleHasExternal(&oldtup));
2093+
Assert(!HeapTupleHasExternal(newtup));
2094+
need_toast= false;
2095+
}
2096+
else
2097+
need_toast= (HeapTupleHasExternal(&oldtup)||
2098+
HeapTupleHasExternal(newtup)||
2099+
newtup->t_len>TOAST_TUPLE_THRESHOLD);
20812100

20822101
pagefree=PageGetFreeSpace((Page)dp);
20832102

‎src/backend/access/heap/tuptoaster.c‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
*
99
*
1010
* IDENTIFICATION
11-
* $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.72 2007/03/29 00:15:37 tgl Exp $
11+
* $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.73 2007/04/03 04:14:26 tgl Exp $
1212
*
1313
*
1414
* INTERFACE ROUTINES
@@ -291,6 +291,12 @@ toast_delete(Relation rel, HeapTuple oldtup)
291291
Datumtoast_values[MaxHeapAttributeNumber];
292292
booltoast_isnull[MaxHeapAttributeNumber];
293293

294+
/*
295+
* We should only ever be called for tuples of plain relations ---
296+
* recursing on a toast rel is bad news.
297+
*/
298+
Assert(rel->rd_rel->relkind==RELKIND_RELATION);
299+
294300
/*
295301
* Get the tuple descriptor and break down the tuple into fields.
296302
*
@@ -360,6 +366,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
360366
boolhas_nulls= false;
361367

362368
SizemaxDataLen;
369+
Sizehoff;
363370

364371
chartoast_action[MaxHeapAttributeNumber];
365372
booltoast_isnull[MaxHeapAttributeNumber];
@@ -370,6 +377,12 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
370377
booltoast_free[MaxHeapAttributeNumber];
371378
booltoast_delold[MaxHeapAttributeNumber];
372379

380+
/*
381+
* We should only ever be called for tuples of plain relations ---
382+
* recursing on a toast rel is bad news.
383+
*/
384+
Assert(rel->rd_rel->relkind==RELKIND_RELATION);
385+
373386
/*
374387
* Get the tuple descriptor and break down the tuple(s) into fields.
375388
*/
@@ -512,15 +525,15 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
512525
*/
513526

514527
/* compute header overhead --- this should match heap_form_tuple() */
515-
maxDataLen= offsetof(HeapTupleHeaderData,t_bits);
528+
hoff= offsetof(HeapTupleHeaderData,t_bits);
516529
if (has_nulls)
517-
maxDataLen+=BITMAPLEN(numAttrs);
530+
hoff+=BITMAPLEN(numAttrs);
518531
if (newtup->t_data->t_infomask&HEAP_HASOID)
519-
maxDataLen+=sizeof(Oid);
520-
maxDataLen=MAXALIGN(maxDataLen);
521-
Assert(maxDataLen==newtup->t_data->t_hoff);
532+
hoff+=sizeof(Oid);
533+
hoff=MAXALIGN(hoff);
534+
Assert(hoff==newtup->t_data->t_hoff);
522535
/* now convert to a limit on the tuple data size */
523-
maxDataLen=TOAST_TUPLE_TARGET-maxDataLen;
536+
maxDataLen=TOAST_TUPLE_TARGET-hoff;
524537

525538
/*
526539
* Look for attributes with attstorage 'x' to compress
@@ -583,7 +596,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
583596

584597
/*
585598
* Second we look for attributes of attstorage 'x' or 'e' that are still
586-
* inline.
599+
* inline. But skip this if there's no toast table to push them to.
587600
*/
588601
while (heap_compute_data_size(tupleDesc,
589602
toast_values,toast_isnull)>maxDataLen&&
@@ -695,7 +708,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
695708
}
696709

697710
/*
698-
* Finally we store attributes of type 'm' external
711+
* Finally we store attributes of type 'm' external, if possible.
699712
*/
700713
while (heap_compute_data_size(tupleDesc,
701714
toast_values,toast_isnull)>maxDataLen&&

‎src/backend/access/transam/xlog.c‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
* Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
88
* Portions Copyright (c) 1994, Regents of the University of California
99
*
10-
* $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.265 2007/03/0320:02:26momjian Exp $
10+
* $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.266 2007/04/0304:14:26tgl Exp $
1111
*
1212
*-------------------------------------------------------------------------
1313
*/
@@ -28,6 +28,7 @@
2828
#include"access/multixact.h"
2929
#include"access/subtrans.h"
3030
#include"access/transam.h"
31+
#include"access/tuptoaster.h"
3132
#include"access/twophase.h"
3233
#include"access/xact.h"
3334
#include"access/xlog_internal.h"
@@ -3634,6 +3635,8 @@ WriteControlFile(void)
36343635
ControlFile->nameDataLen=NAMEDATALEN;
36353636
ControlFile->indexMaxKeys=INDEX_MAX_KEYS;
36363637

3638+
ControlFile->toast_max_chunk_size=TOAST_MAX_CHUNK_SIZE;
3639+
36373640
#ifdefHAVE_INT64_TIMESTAMP
36383641
ControlFile->enableIntTimes= TRUE;
36393642
#else
@@ -3824,6 +3827,13 @@ ReadControlFile(void)
38243827
" but the server was compiled with INDEX_MAX_KEYS %d.",
38253828
ControlFile->indexMaxKeys,INDEX_MAX_KEYS),
38263829
errhint("It looks like you need to recompile or initdb.")));
3830+
if (ControlFile->toast_max_chunk_size!=TOAST_MAX_CHUNK_SIZE)
3831+
ereport(FATAL,
3832+
(errmsg("database files are incompatible with server"),
3833+
errdetail("The database cluster was initialized with TOAST_MAX_CHUNK_SIZE %d,"
3834+
" but the server was compiled with TOAST_MAX_CHUNK_SIZE %d.",
3835+
ControlFile->toast_max_chunk_size, (int)TOAST_MAX_CHUNK_SIZE),
3836+
errhint("It looks like you need to recompile or initdb.")));
38273837

38283838
#ifdefHAVE_INT64_TIMESTAMP
38293839
if (ControlFile->enableIntTimes!= TRUE)

‎src/bin/pg_controldata/pg_controldata.c‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
* copyright (c) Oliver Elphick <olly@lfix.co.uk>, 2001;
77
* licence: BSD
88
*
9-
* $PostgreSQL: pgsql/src/bin/pg_controldata/pg_controldata.c,v 1.34 2007/03/18 16:50:43 neilc Exp $
9+
* $PostgreSQL: pgsql/src/bin/pg_controldata/pg_controldata.c,v 1.35 2007/04/03 04:14:26 tgl Exp $
1010
*/
1111
#include"postgres.h"
1212

@@ -199,6 +199,8 @@ main(int argc, char *argv[])
199199
ControlFile.nameDataLen);
200200
printf(_("Maximum columns in an index: %u\n"),
201201
ControlFile.indexMaxKeys);
202+
printf(_("Maximum size of a TOAST chunk: %u\n"),
203+
ControlFile.toast_max_chunk_size);
202204
printf(_("Date/time type storage: %s\n"),
203205
(ControlFile.enableIntTimes ?_("64-bit integers") :_("floating-point numbers")));
204206
printf(_("Maximum length of locale name: %u\n"),

‎src/bin/pg_resetxlog/pg_resetxlog.c‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
* Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
2424
* Portions Copyright (c) 1994, Regents of the University of California
2525
*
26-
* $PostgreSQL: pgsql/src/bin/pg_resetxlog/pg_resetxlog.c,v 1.58 2007/03/0320:02:27 momjian Exp $
26+
* $PostgreSQL: pgsql/src/bin/pg_resetxlog/pg_resetxlog.c,v 1.59 2007/04/0304:14:26 tgl Exp $
2727
*
2828
*-------------------------------------------------------------------------
2929
*/
@@ -41,6 +41,7 @@
4141
#endif
4242

4343
#include"access/transam.h"
44+
#include"access/tuptoaster.h"
4445
#include"access/multixact.h"
4546
#include"access/xlog_internal.h"
4647
#include"catalog/catversion.h"
@@ -484,6 +485,7 @@ GuessControlValues(void)
484485
ControlFile.xlog_seg_size=XLOG_SEG_SIZE;
485486
ControlFile.nameDataLen=NAMEDATALEN;
486487
ControlFile.indexMaxKeys=INDEX_MAX_KEYS;
488+
ControlFile.toast_max_chunk_size=TOAST_MAX_CHUNK_SIZE;
487489
#ifdefHAVE_INT64_TIMESTAMP
488490
ControlFile.enableIntTimes= TRUE;
489491
#else
@@ -572,6 +574,8 @@ PrintControlValues(bool guessed)
572574
ControlFile.nameDataLen);
573575
printf(_("Maximum columns in an index: %u\n"),
574576
ControlFile.indexMaxKeys);
577+
printf(_("Maximum size of a TOAST chunk: %u\n"),
578+
ControlFile.toast_max_chunk_size);
575579
printf(_("Date/time type storage: %s\n"),
576580
(ControlFile.enableIntTimes ?_("64-bit integers") :_("floating-point numbers")));
577581
printf(_("Maximum length of locale name: %u\n"),

‎src/include/access/tuptoaster.h‎

Lines changed: 35 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
*
77
* Copyright (c) 2000-2007, PostgreSQL Global Development Group
88
*
9-
* $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.33 2007/03/29 00:15:39 tgl Exp $
9+
* $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.34 2007/04/03 04:14:26 tgl Exp $
1010
*
1111
*-------------------------------------------------------------------------
1212
*/
@@ -29,19 +29,26 @@
2929
* TOAST_TUPLE_TARGET bytes. Both numbers include all tuple header overhead
3030
* and between-fields alignment padding, but we do *not* consider any
3131
* end-of-tuple alignment padding; hence the values can be compared directly
32-
* to a tuple's t_len field. We choose TOAST_TUPLE_THRESHOLD with the
33-
* knowledge that toast-table tuples will be exactly that size, and we'd
34-
* like to fit four of them per page with minimal space wastage.
32+
* to a tuple's t_len field.
3533
*
36-
* The numbers need not be the same, though they currently are.
34+
* The numbers need not be the same, though they currently are. It doesn't
35+
* make sense for TARGET to exceed THRESHOLD, but it could be useful to make
36+
* it be smaller.
3737
*
38-
* Note: sizeof(PageHeaderData) includes the first ItemId, but we have
39-
* to allow for 3 more, if we want to fit 4 tuples on a page.
38+
* Currently we choose both values to match the largest tuple size for which
39+
* TOAST_TUPLES_PER_PAGE tuples can fit on a disk page.
40+
*
41+
* XXX while these can be modified without initdb, some thought needs to be
42+
* given to needs_toast_table() in toasting.c before unleashing random
43+
* changes.
4044
*/
45+
#defineTOAST_TUPLES_PER_PAGE4
46+
47+
/* Note: sizeof(PageHeaderData) includes the first ItemId on the page */
4148
#defineTOAST_TUPLE_THRESHOLD\
4249
MAXALIGN_DOWN((BLCKSZ - \
43-
MAXALIGN(sizeof(PageHeaderData) +3 * sizeof(ItemIdData))) \
44-
/4)
50+
MAXALIGN(sizeof(PageHeaderData) +(TOAST_TUPLES_PER_PAGE-1) * sizeof(ItemIdData))) \
51+
/TOAST_TUPLES_PER_PAGE)
4552

4653
#defineTOAST_TUPLE_TARGETTOAST_TUPLE_THRESHOLD
4754

@@ -56,20 +63,26 @@
5663
* When we store an oversize datum externally, we divide it into chunks
5764
* containing at most TOAST_MAX_CHUNK_SIZE data bytes.This number *must*
5865
* be small enough that the completed toast-table tuple (including the
59-
* ID and sequence fields and all overhead) is no more than MaxHeapTupleSize
60-
* bytes. It *should* be small enough to make toast-table tuples no more
61-
* than TOAST_TUPLE_THRESHOLD bytes, else heapam.c will uselessly invoke
62-
* the toaster on toast-table tuples. The current coding ensures that the
63-
* maximum tuple length is exactly TOAST_TUPLE_THRESHOLD bytes.
64-
*
65-
* NB: you cannot change this value without forcing initdb, at least not
66-
* if your DB contains any multi-chunk toasted values.
66+
* ID and sequence fields and all overhead) will fit on a page.
67+
* The coding here sets the size on the theory that we want to fit
68+
* EXTERN_TUPLES_PER_PAGE tuples of maximum size onto a page.
69+
*
70+
* NB: Changing TOAST_MAX_CHUNK_SIZE requires an initdb.
6771
*/
68-
#defineTOAST_MAX_CHUNK_SIZE(TOAST_TUPLE_THRESHOLD -\
69-
MAXALIGN(offsetof(HeapTupleHeaderData, t_bits)) -\
70-
sizeof(Oid) -\
71-
sizeof(int32) -\
72-
VARHDRSZ)
72+
#defineEXTERN_TUPLES_PER_PAGE4/* tweak only this */
73+
74+
/* Note: sizeof(PageHeaderData) includes the first ItemId on the page */
75+
#defineEXTERN_TUPLE_MAX_SIZE\
76+
MAXALIGN_DOWN((BLCKSZ - \
77+
MAXALIGN(sizeof(PageHeaderData) + (EXTERN_TUPLES_PER_PAGE-1) * sizeof(ItemIdData))) \
78+
/ EXTERN_TUPLES_PER_PAGE)
79+
80+
#defineTOAST_MAX_CHUNK_SIZE\
81+
(EXTERN_TUPLE_MAX_SIZE -\
82+
MAXALIGN(offsetof(HeapTupleHeaderData, t_bits)) -\
83+
sizeof(Oid) -\
84+
sizeof(int32) -\
85+
VARHDRSZ)
7386

7487

7588
/* ----------

‎src/include/catalog/pg_control.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
* Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
99
* Portions Copyright (c) 1994, Regents of the University of California
1010
*
11-
* $PostgreSQL: pgsql/src/include/catalog/pg_control.h,v 1.36 2007/03/0320:02:27 momjian Exp $
11+
* $PostgreSQL: pgsql/src/include/catalog/pg_control.h,v 1.37 2007/04/0304:14:26 tgl Exp $
1212
*
1313
*-------------------------------------------------------------------------
1414
*/
@@ -22,7 +22,7 @@
2222

2323

2424
/* Version identifier for this pg_control format */
25-
#definePG_CONTROL_VERSION832
25+
#definePG_CONTROL_VERSION833
2626

2727
/*
2828
* Body of CheckPoint XLOG records. This is declared here because we keep
@@ -135,6 +135,8 @@ typedef struct ControlFileData
135135
uint32nameDataLen;/* catalog name field width */
136136
uint32indexMaxKeys;/* max number of columns in an index */
137137

138+
uint32toast_max_chunk_size;/* chunk size in TOAST tables */
139+
138140
/* flag indicating internal format of timestamp, interval, time */
139141
uint32enableIntTimes;/* int64 storage enabled? */
140142

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp