Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit10e3226

Browse files
committed
Align blocks in incremental backups to BLCKSZ
Align blocks stored in incremental files to BLCKSZ, so that theincremental backups work well with CoW filesystems.The header of the incremental file is padded with \0 to a multiple ofBLCKSZ, so that the block data (also BLCKSZ) is aligned to BLCKSZ. Thepadding is added only to files containing block data, so files with justthe header remain small. This adds a bit of extra space, but as thenumber of blocks increases the overhead gets negligible very quickly.And as the padding is \0 bytes, it does compress extremely well.The alignment is important for CoW filesystems that usually require theblocks to be aligned to filesystem page size for features like blocksharing, deduplication etc. to work well. With the variable sized headerthe blocks in the increments were not aligned at all, negating thebenefits of the CoW filesystems.This matters even for non-CoW filesystems, for example when placed on aRAID array. If the block is not aligned, it may easily span multipledevices, causing read and write amplification.It might be better to align the blocks to the filesystem page, notBLCKSZ, but we have no good way to determine that. Even if we determinethe page size at the time of taking the backup, the backup may move. Fornow the BLCKSZ seems sufficient - the filesystem page is usually 4K, sothe default BLCKSZ (8K by default) is aligned to that.Author: Tomas VondraReviewed-by: Robert Haas, Jakub WartakDiscussion:https://postgr.es/m/3024283a-7491-4240-80d0-421575f6bb23%40enterprisedb.com
1 parentee1cbe8 commit10e3226

File tree

4 files changed

+70
-4
lines changed

4 files changed

+70
-4
lines changed

‎src/backend/backup/basebackup.c

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1623,6 +1623,8 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
16231623
{
16241624
unsignedmagic=INCREMENTAL_MAGIC;
16251625
size_theader_bytes_done=0;
1626+
charpadding[BLCKSZ];
1627+
size_tpaddinglen;
16261628

16271629
/* Emit header data. */
16281630
push_to_sink(sink,&checksum_ctx,&header_bytes_done,
@@ -1635,6 +1637,23 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
16351637
incremental_blocks,
16361638
sizeof(BlockNumber)*num_incremental_blocks);
16371639

1640+
/*
1641+
* Add padding to align header to a multiple of BLCKSZ, but only if
1642+
* the incremental file has some blocks. If there are no blocks we
1643+
* don't want make the file unnecessarily large, as that might make
1644+
* some filesystem optimizations impossible.
1645+
*/
1646+
if (num_incremental_blocks>0)
1647+
{
1648+
paddinglen= (BLCKSZ- (header_bytes_done %BLCKSZ));
1649+
1650+
memset(padding,0,paddinglen);
1651+
bytes_done+=paddinglen;
1652+
1653+
push_to_sink(sink,&checksum_ctx,&header_bytes_done,
1654+
padding,paddinglen);
1655+
}
1656+
16381657
/* Flush out any data still in the buffer so it's again empty. */
16391658
if (header_bytes_done>0)
16401659
{
@@ -1748,6 +1767,13 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
17481767
blkno+=cnt /BLCKSZ;
17491768
bytes_done+=cnt;
17501769

1770+
/*
1771+
* Make sure incremental files with block data are properly aligned
1772+
* (header is a multiple of BLCKSZ, blocks are BLCKSZ too).
1773+
*/
1774+
Assert(!((incremental_blocks!=NULL&&num_incremental_blocks>0)&&
1775+
(bytes_done %BLCKSZ!=0)));
1776+
17511777
/* Archive the data we just read. */
17521778
bbsink_archive_contents(sink,cnt);
17531779

‎src/backend/backup/basebackup_incremental.c

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -928,6 +928,36 @@ GetFileBackupMethod(IncrementalBackupInfo *ib, const char *path,
928928
returnBACK_UP_FILE_INCREMENTALLY;
929929
}
930930

931+
/*
932+
* Compute the size for a header of an incremental file containing a given
933+
* number of blocks. The header is rounded to a multiple of BLCKSZ, but
934+
* only if the file will store some block data.
935+
*/
936+
externsize_t
937+
GetIncrementalHeaderSize(unsignednum_blocks_required)
938+
{
939+
size_tresult;
940+
941+
/* Make sure we're not going to overflow. */
942+
Assert(num_blocks_required <=RELSEG_SIZE);
943+
944+
/*
945+
* Three four byte quantities (magic number, truncation block length,
946+
* block count) followed by block numbers.
947+
*/
948+
result=3*sizeof(uint32)+ (sizeof(BlockNumber)*num_blocks_required);
949+
950+
/*
951+
* Round the header size to a multiple of BLCKSZ - when not a multiple of
952+
* BLCKSZ, add the missing fraction of a block. But do this only if the
953+
* file will store data for some blocks, otherwise keep it small.
954+
*/
955+
if ((num_blocks_required>0)&& (result %BLCKSZ!=0))
956+
result+=BLCKSZ- (result %BLCKSZ);
957+
958+
returnresult;
959+
}
960+
931961
/*
932962
* Compute the size for an incremental file containing a given number of blocks.
933963
*/
@@ -940,11 +970,12 @@ GetIncrementalFileSize(unsigned num_blocks_required)
940970
Assert(num_blocks_required <=RELSEG_SIZE);
941971

942972
/*
943-
* Three four byte quantities (magic number, truncation block length,
944-
* block count) followed by block numbers followed by block contents.
973+
* Header with three four byte quantities (magic number, truncation block
974+
* length, block count) followed by block numbers, rounded to a multiple
975+
* of BLCKSZ (for files with block data), followed by block contents.
945976
*/
946-
result=3*sizeof(uint32);
947-
result+=(BLCKSZ+sizeof(BlockNumber))*num_blocks_required;
977+
result=GetIncrementalHeaderSize(num_blocks_required);
978+
result+=BLCKSZ*num_blocks_required;
948979

949980
returnresult;
950981
}

‎src/bin/pg_combinebackup/reconstruct.c

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -472,6 +472,14 @@ make_incremental_rfile(char *filename)
472472
sizeof(rf->truncation_block_length)+
473473
sizeof(BlockNumber)*rf->num_blocks;
474474

475+
/*
476+
* Round header length to a multiple of BLCKSZ, so that blocks contents
477+
* are properly aligned. Only do this when the file actually has data for
478+
* some blocks.
479+
*/
480+
if ((rf->num_blocks>0)&& ((rf->header_length %BLCKSZ)!=0))
481+
rf->header_length+= (BLCKSZ- (rf->header_length %BLCKSZ));
482+
475483
returnrf;
476484
}
477485

‎src/include/backup/basebackup_incremental.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,5 +51,6 @@ extern FileBackupMethod GetFileBackupMethod(IncrementalBackupInfo *ib,
5151
BlockNumber*relative_block_numbers,
5252
unsigned*truncation_block_length);
5353
externsize_tGetIncrementalFileSize(unsignednum_blocks_required);
54+
externsize_tGetIncrementalHeaderSize(unsignednum_blocks_required);
5455

5556
#endif

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp