Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit708c8e2

Browse files
authored
Merge pull request#6 from postgrespro/double_slot
Resolve issues#5 and#1: reduce number of collisions in the ptrack map
2 parentsde8d655 +9c132a3 commit708c8e2

File tree

11 files changed

+162
-83
lines changed

11 files changed

+162
-83
lines changed

‎.gitignore

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
11
.deps
22
*.so
33
*.o
4-
ptrack--2.0.sql
54
Dockerfile
6-

‎Makefile

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,11 @@
22

33
MODULE_big = ptrack
44
OBJS = ptrack.o datapagemap.o engine.o$(WIN32RES)
5-
EXTENSION = ptrack
6-
EXTVERSION = 2.1
7-
DATA = ptrack.sql ptrack--2.0--2.1.sql
8-
DATA_built =$(EXTENSION)--$(EXTVERSION).sql
95
PGFILEDESC = "ptrack - block-level incremental backup engine"
106

11-
EXTRA_CLEAN =$(EXTENSION)--$(EXTVERSION).sql
7+
EXTENSION = ptrack
8+
EXTVERSION = 2.2
9+
DATA = ptrack--2.1.sql ptrack--2.0--2.1.sql ptrack--2.1--2.2.sql
1210

1311
TAP_TESTS = 1
1412

@@ -22,13 +20,3 @@ top_builddir = ../..
2220
include$(top_builddir)/src/Makefile.global
2321
include$(top_srcdir)/contrib/contrib-global.mk
2422
endif
25-
26-
$(EXTENSION)--$(EXTVERSION).sql: ptrack.sql
27-
cat$^>$@
28-
29-
# temp-install: EXTRA_INSTALL=contrib/ptrack
30-
31-
# check-tap: temp-install
32-
# $(prove_check)
33-
34-
# check: check-tap

‎README.md

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,8 @@ To disable `ptrack` and clean up all remaining service files set `ptrack.map_siz
6565

6666
* ptrack_version() — returns ptrack version string.
6767
* ptrack_init_lsn() — returns LSN of the last ptrack map initialization.
68-
* ptrack_get_pagemapset('LSN') — returns a set of changed data files with bitmaps of changed blocks since specified LSN.
68+
* ptrack_get_pagemapset(start_lsn pg_lsn) — returns a set of changed data files with bitmaps of changed blocks since specified`start_lsn`.
69+
* ptrack_get_change_stat(start_lsn pg_lsn) — returns statistic of changes (number of files, pages and size in MB) since specified`start_lsn`.
6970

7071
Usage example:
7172

@@ -102,6 +103,10 @@ Usually, you have to only install new version of `ptrack` and do `ALTER EXTENSIO
102103
* Do`ALTER EXTENSION 'ptrack' UPDATE;`.
103104
* Restart your server.
104105

106+
####Upgrading from 2.1.* to 2.2.*:
107+
108+
Since version 2.2 we use a different algorithm for tracking changed pages. Thus, data recorded in the`ptrack.map` using pre 2.2 versions of`ptrack` is incompatible with newer versions. After extension upgrade and server restart old`ptrack.map` will be discarded with`WARNING` and initialized from the scratch.
109+
105110
##Limitations
106111

107112
1. You can only use`ptrack` safely with`wal_level >= 'replica'`. Otherwise, you can lose tracking of some changes if crash-recovery occurs, since[certain commands are designed not to write WAL at all if wal_level is minimal](https://www.postgresql.org/docs/12/populate.html#POPULATE-PITR), but we only durably flush`ptrack` map at checkpoint time.

‎engine.c

Lines changed: 61 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,8 @@ ptrackMapInit(void)
156156
sprintf(ptrack_path,"%s/%s",DataDir,PTRACK_PATH);
157157
sprintf(ptrack_mmap_path,"%s/%s",DataDir,PTRACK_MMAP_PATH);
158158

159+
ptrack_map_reinit:
160+
159161
/* Remove old PTRACK_MMAP_PATH file, if exists */
160162
if (ptrack_file_exists(ptrack_mmap_path))
161163
durable_unlink(ptrack_mmap_path,LOG);
@@ -175,18 +177,15 @@ ptrackMapInit(void)
175177
if (stat(ptrack_path,&stat_buf)==0)
176178
{
177179
copy_file(ptrack_path,ptrack_mmap_path);
178-
is_new_map= false;/* flag to check checksum */
180+
is_new_map= false;/* flag to checkmap file format andchecksum */
179181
ptrack_fd=BasicOpenFile(ptrack_mmap_path,O_RDWR |PG_BINARY);
180-
if (ptrack_fd<0)
181-
elog(ERROR,"ptrack init: failed to open map file \"%s\": %m",ptrack_mmap_path);
182182
}
183183
else
184-
{
185184
/* Create new file for PTRACK_MMAP_PATH */
186185
ptrack_fd=BasicOpenFile(ptrack_mmap_path,O_RDWR |O_CREAT |PG_BINARY);
187-
if (ptrack_fd<0)
188-
elog(ERROR,"ptrack init: failed to open map file \"%s\": %m",ptrack_mmap_path);
189-
}
186+
187+
if (ptrack_fd<0)
188+
elog(ERROR,"ptrack init: failed to open map file \"%s\": %m",ptrack_mmap_path);
190189

191190
#ifdefWIN32
192191
{
@@ -227,7 +226,20 @@ ptrackMapInit(void)
227226
elog(ERROR,"ptrack init: wrong map format of file \"%s\"",ptrack_path);
228227

229228
/* Check ptrack version inside old ptrack map */
230-
/* No-op for now, but may be used for future compatibility checks */
229+
if (ptrack_map->version_num!=PTRACK_VERSION_NUM)
230+
{
231+
ereport(WARNING,
232+
(errcode(ERRCODE_DATA_CORRUPTED),
233+
errmsg("ptrack init: map format version %d in the file \"%s\" is incompatible with loaded version %d",
234+
ptrack_map->version_num,ptrack_path,PTRACK_VERSION_NUM),
235+
errdetail("Deleting file \"%s\" and reinitializing ptrack map.",ptrack_path)));
236+
237+
/* Clean up everything and try again */
238+
ptrackCleanFilesAndMap();
239+
240+
is_new_map= true;
241+
gotoptrack_map_reinit;
242+
}
231243

232244
/* Check CRC */
233245
INIT_CRC32C(crc);
@@ -378,7 +390,7 @@ ptrackCheckpoint(void)
378390
/*
379391
* We are writing ptrack map values to file, but we want to simply map it
380392
* into the memory with mmap after a crash/restart. That way, we have to
381-
* write values taking into account all paddings/allignments.
393+
* write values taking into account all paddings/alignments.
382394
*
383395
* Write both magic and varsion_num at once.
384396
*/
@@ -435,7 +447,7 @@ ptrackCheckpoint(void)
435447
* going to overflow. */
436448

437449
/*
438-
* We should not have anyallignment issues here, since sizeof()
450+
* We should not have anyalignment issues here, since sizeof()
439451
* takes into account all paddings for us.
440452
*/
441453
ptrack_write_chunk(ptrack_tmp_fd,&crc, (char*)buf,writesz);
@@ -446,7 +458,7 @@ ptrackCheckpoint(void)
446458
}
447459
}
448460

449-
/* Write ifanythig left */
461+
/* Write ifanything left */
450462
if ((i+1) %PTRACK_BUF_SIZE!=0)
451463
{
452464
size_twritesz=sizeof(pg_atomic_uint64)*j;
@@ -641,48 +653,56 @@ void
641653
ptrack_mark_block(RelFileNodeBackendsmgr_rnode,
642654
ForkNumberforknum,BlockNumberblocknum)
643655
{
656+
PtBlockIdbid;
644657
size_thash;
658+
size_tslot1;
659+
size_tslot2;
645660
XLogRecPtrnew_lsn;
646-
PtBlockIdbid;
647661
/*
648662
* We use pg_atomic_uint64 here only for alignment purposes, because
649-
* pg_atomic_uint64 isforcely aligned on 8 bytes during the MSVC build.
663+
* pg_atomic_uint64 isforcedly aligned on 8 bytes during the MSVC build.
650664
*/
651665
pg_atomic_uint64old_lsn;
652666
pg_atomic_uint64old_init_lsn;
653667

654-
if (ptrack_map_size!=0&& (ptrack_map!=NULL)&&
655-
smgr_rnode.backend==InvalidBackendId)/* do not track temporary
656-
* relations */
657-
{
658-
bid.relnode=smgr_rnode.node;
659-
bid.forknum=forknum;
660-
bid.blocknum=blocknum;
661-
hash=BID_HASH_FUNC(bid);
662-
663-
if (RecoveryInProgress())
664-
new_lsn=GetXLogReplayRecPtr(NULL);
665-
else
666-
new_lsn=GetXLogInsertRecPtr();
667-
668-
old_lsn.value=pg_atomic_read_u64(&ptrack_map->entries[hash]);
668+
if (ptrack_map_size==0
669+
||ptrack_map==NULL
670+
||smgr_rnode.backend!=InvalidBackendId)/* do not track temporary
671+
* relations */
672+
return;
669673

670-
/* Atomically assign new init LSN value */
671-
old_init_lsn.value=pg_atomic_read_u64(&ptrack_map->init_lsn);
674+
bid.relnode=smgr_rnode.node;
675+
bid.forknum=forknum;
676+
bid.blocknum=blocknum;
672677

673-
if (old_init_lsn.value==InvalidXLogRecPtr)
674-
{
675-
elog(DEBUG1,"ptrack_mark_block: init_lsn "UINT64_FORMAT" <- "UINT64_FORMAT,old_init_lsn.value,new_lsn);
678+
hash=BID_HASH_FUNC(bid);
679+
slot1=hash %PtrackContentNblocks;
680+
slot2= ((hash <<32) | (hash >>32)) %PtrackContentNblocks;
676681

677-
while (old_init_lsn.value<new_lsn&&
678-
!pg_atomic_compare_exchange_u64(&ptrack_map->init_lsn, (uint64*)&old_init_lsn.value,new_lsn));
679-
}
682+
if (RecoveryInProgress())
683+
new_lsn=GetXLogReplayRecPtr(NULL);
684+
else
685+
new_lsn=GetXLogInsertRecPtr();
680686

681-
elog(DEBUG3,"ptrack_mark_block: map[%zu]="UINT64_FORMAT" <- "UINT64_FORMAT,hash,old_lsn.value,new_lsn);
687+
/* Atomically assign new init LSN value */
688+
old_init_lsn.value=pg_atomic_read_u64(&ptrack_map->init_lsn);
689+
if (old_init_lsn.value==InvalidXLogRecPtr)
690+
{
691+
elog(DEBUG1,"ptrack_mark_block: init_lsn "UINT64_FORMAT" <- "UINT64_FORMAT,old_init_lsn.value,new_lsn);
682692

683-
/* Atomically assign new LSN value */
684-
while (old_lsn.value<new_lsn&&
685-
!pg_atomic_compare_exchange_u64(&ptrack_map->entries[hash], (uint64*)&old_lsn.value,new_lsn));
686-
elog(DEBUG3,"ptrack_mark_block: map[%zu]="UINT64_FORMAT,hash,pg_atomic_read_u64(&ptrack_map->entries[hash]));
693+
while (old_init_lsn.value<new_lsn&&
694+
!pg_atomic_compare_exchange_u64(&ptrack_map->init_lsn, (uint64*)&old_init_lsn.value,new_lsn));
687695
}
696+
697+
/* Atomically assign new LSN value to the first slot */
698+
old_lsn.value=pg_atomic_read_u64(&ptrack_map->entries[slot1]);
699+
elog(DEBUG3,"ptrack_mark_block: map[%zu]="UINT64_FORMAT" <- "UINT64_FORMAT,slot1,old_lsn.value,new_lsn);
700+
while (old_lsn.value<new_lsn&&
701+
!pg_atomic_compare_exchange_u64(&ptrack_map->entries[slot1], (uint64*)&old_lsn.value,new_lsn));
702+
elog(DEBUG3,"ptrack_mark_block: map[%zu]="UINT64_FORMAT,hash,pg_atomic_read_u64(&ptrack_map->entries[slot1]));
703+
704+
/* And to the second */
705+
old_lsn.value=pg_atomic_read_u64(&ptrack_map->entries[slot2]);
706+
while (old_lsn.value<new_lsn&&
707+
!pg_atomic_compare_exchange_u64(&ptrack_map->entries[slot2], (uint64*)&old_lsn.value,new_lsn));
688708
}

‎engine.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ typedef struct PtrackMapHdr
5050
{
5151
/*
5252
* Three magic bytes (+ \0) to be sure, that we are reading ptrack.map
53-
* with a right PtrackMapHdrstrucutre.
53+
* with a right PtrackMapHdrstructure.
5454
*/
5555
charmagic[PTRACK_MAGIC_SIZE];
5656

@@ -72,7 +72,6 @@ typedef struct PtrackMapHdr
7272

7373
typedefPtrackMapHdr*PtrackMap;
7474

75-
/* TODO: check MAXALIGN usage below */
7675
/* Number of elements in ptrack map (LSN array) */
7776
#definePtrackContentNblocks \
7877
((ptrack_map_size - offsetof(PtrackMapHdr, entries) - sizeof(pg_crc32c)) / sizeof(pg_atomic_uint64))
@@ -84,9 +83,10 @@ typedef PtrackMapHdr * PtrackMap;
8483
/* CRC32 value offset in order to directly access it in the mmap'ed memory chunk */
8584
#definePtrackCrcOffset (PtrackActualSize - sizeof(pg_crc32c))
8685

87-
/* Map block address 'bid' to map slot */
86+
/* Block address 'bid' to hash. To get slot position in map should be divided
87+
* with '% PtrackContentNblocks' */
8888
#defineBID_HASH_FUNC(bid) \
89-
(size_t)(DatumGetUInt64(hash_any_extended((unsigned char *)&bid, sizeof(bid), 0)) % PtrackContentNblocks)
89+
(size_t)(DatumGetUInt64(hash_any_extended((unsigned char *)&bid, sizeof(bid), 0)))
9090

9191
/*
9292
* Per process pointer to shared ptrack_map

‎ptrack--2.1--2.2.sql

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/* ptrack/ptrack--2.1--2.2.sql*/
2+
3+
-- Complain if script is sourced in psql, rather than via ALTER EXTENSION
4+
\echo Use"ALTER EXTENSION ptrack UPDATE;" to load this file.\ quit
5+
6+
DROPFUNCTION ptrack_get_pagemapset(start_lsn pg_lsn);
7+
CREATEFUNCTIONptrack_get_pagemapset(start_lsn pg_lsn)
8+
RETURNS TABLE (pathtext,
9+
pagecountbigint,
10+
pagemapbytea)
11+
AS'MODULE_PATHNAME'
12+
LANGUAGE C STRICT VOLATILE;
13+
14+
CREATEFUNCTIONptrack_get_change_stat(start_lsn pg_lsn)
15+
RETURNS TABLE (
16+
filesbigint,
17+
pagesnumeric,
18+
"size, MB"numeric
19+
)AS
20+
$func$
21+
DECLARE
22+
block_sizebigint;
23+
BEGIN
24+
block_size := (SELECT settingFROM pg_settingsWHERE name='block_size');
25+
26+
RETURN QUERY
27+
SELECT changed_files,
28+
changed_pages,
29+
block_size* changed_pages/ (1024.0*1024)
30+
FROM
31+
(SELECTcount(path)AS changed_files,
32+
sum(pagecount)AS changed_pages
33+
FROM ptrack_get_pagemapset(start_lsn)) s;
34+
END
35+
$func$ LANGUAGE plpgsql;

‎ptrack.sqlrenamed to‎ptrack--2.1.sql

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
/* ptrack/ptrack--2.1.sql*/
2+
13
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
24
\echo Use"CREATE EXTENSION ptrack" to load this file. \quit
35

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp