Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitd4e7b32

Browse files
author
Daniel Shelepanov
committed
[PBCKP-278] ptrack adapted to hadling cfs relations
tags: cfs, ptrack
1 parent936db26 commitd4e7b32

File tree

4 files changed

+274
-12
lines changed

4 files changed

+274
-12
lines changed

‎engine.c‎

Lines changed: 100 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
*/
1919

2020
#include"postgres.h"
21-
21+
#include<ctype.h>
2222
#include<unistd.h>
2323
#include<sys/stat.h>
2424

@@ -36,6 +36,10 @@
3636
#include"catalog/pg_tablespace.h"
3737
#include"miscadmin.h"
3838
#include"port/pg_crc32c.h"
39+
#ifdefPGPRO_EE
40+
/* For file_is_in_cfs_tablespace() only. */
41+
#include"replication/basebackup.h"
42+
#endif
3943
#include"storage/copydir.h"
4044
#ifPG_VERSION_NUM >=120000
4145
#include"storage/md.h"
@@ -91,6 +95,81 @@ ptrack_write_chunk(int fd, pg_crc32c *crc, char *chunk, size_t size)
9195
}
9296
}
9397

98+
/*
99+
* Determines a file type which is one of the following:
100+
* * regular relation file
101+
* * regular relation file which is a part of a multifile relation
102+
* * free space map
103+
* * visibility map
104+
* * compressed file map
105+
*/
106+
FileType
107+
file_type_from_path(constchar*filename) {
108+
ssize_tlen=strlen(filename);
109+
boolmet_dot;
110+
111+
// For further length checks we assume that the filename is at least
112+
// 1 character longer than the corresponding extension. For example,
113+
// strlen("_vm") == 3 therefore we assume that the filename can't be
114+
// shorter than 4 bytes, for example: "5_vm".
115+
// visibility map
116+
if(len >=4&&strcmp(&filename[len-3],"_vm")==0)
117+
returnFileTypeVM;
118+
119+
// free space map
120+
if(len >=5&&strcmp(&filename[len-4],"_fsm")==0)
121+
returnFileTypeFSM;
122+
123+
// compressed file map
124+
if(len >=5&&strcmp(&filename[len-4],".cfm")==0)
125+
returnFileTypeCFM;
126+
127+
// regular and multipart regular files
128+
met_dot= false;
129+
130+
// we expect filename to be ****/12345 or ****/12345.12
131+
for(ssize_ti=len-1;i >=0&&filename[i]!='/';i--) {
132+
if(isdigit(filename[i])!=0)
133+
continue;
134+
135+
// we expect the dot to appear only once
136+
if(filename[i]=='.') {
137+
if(met_dot)
138+
returnFileTypeUnknown;
139+
140+
met_dot= true;
141+
}else
142+
// met anything except digit and dot => unknown file type
143+
returnFileTypeUnknown;
144+
}
145+
146+
returnmet_dot ?FileTypeMainMulti :FileTypeMain;
147+
}
148+
149+
#ifdefPGPRO_EE
150+
/*
151+
* Determines the relation file size specified by fullpath as if it
152+
* was not compressed.
153+
*/
154+
off_t
155+
get_cfs_relation_decompressed_size(RelFileNodeBackendrnode,constchar*fullpath,ForkNumberforknum) {
156+
Filefd;
157+
intcompressor;
158+
off_tsize;
159+
160+
compressor=md_get_compressor_internal(rnode.node,rnode.backend,forknum);
161+
fd=PathNameOpenFile(fullpath,O_RDWR |PG_BINARY,compressor);
162+
163+
if(fd<0)
164+
return (off_t)-1;
165+
166+
size=FileSize(fd);
167+
FileClose(fd);
168+
169+
returnsize;
170+
}
171+
#endif
172+
94173
/*
95174
* Delete ptrack files when ptrack is disabled.
96175
*
@@ -508,6 +587,12 @@ ptrack_mark_file(Oid dbOid, Oid tablespaceOid,
508587
structstatstat_buf;
509588
intoidchars;
510589
charoidbuf[OIDCHARS+1];
590+
#ifdefPGPRO_EE
591+
FileTypefile_type;
592+
off_trel_size;
593+
boolis_cfs;
594+
is_cfs=file_is_in_cfs_tablespace(filepath);
595+
#endif
511596

512597
/* Do not track temporary relations */
513598
if (looks_like_temp_rel_name(filename))
@@ -526,6 +611,20 @@ ptrack_mark_file(Oid dbOid, Oid tablespaceOid,
526611
oidbuf[oidchars]='\0';
527612
nodeRel(nodeOf(rnode))=atooid(oidbuf);
528613

614+
#ifdefPGPRO_EE
615+
file_type=file_type_from_path(filepath);
616+
617+
if(is_cfs&&file_type!=FileTypeCFM) {
618+
rel_size=get_cfs_relation_decompressed_size(rnode,filepath,forknum);
619+
620+
if(rel_size== (off_t)-1) {
621+
elog(WARNING,"ptrack: could not open cfs-compressed relation file: %s",filepath);
622+
return;
623+
}
624+
625+
nblocks=rel_size /BLCKSZ;
626+
}else
627+
#endif
529628
/* Compute number of blocks based on file size */
530629
if (stat(filepath,&stat_buf)==0)
531630
nblocks=stat_buf.st_size /BLCKSZ;

‎engine.h‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,15 @@ typedef struct PtrackMapHdr
7373

7474
typedefPtrackMapHdr*PtrackMap;
7575

76+
typedefenum {
77+
FileTypeUnknown,
78+
FileTypeMain,
79+
FileTypeMainMulti,
80+
FileTypeFSM,
81+
FileTypeVM,
82+
FileTypeCFM
83+
}FileType;
84+
7685
/* Number of elements in ptrack map (LSN array) */
7786
#definePtrackContentNblocks \
7887
((ptrack_map_size - offsetof(PtrackMapHdr, entries) - sizeof(pg_crc32c)) / sizeof(pg_atomic_uint64))
@@ -111,4 +120,10 @@ extern void ptrack_walkdir(const char *path, Oid tablespaceOid, Oid dbOid);
111120
externvoidptrack_mark_block(RelFileNodeBackendsmgr_rnode,
112121
ForkNumberforkno,BlockNumberblkno);
113122

123+
externFileTypefile_type_from_path(constchar*path);
124+
#ifdefPGPRO_EE
125+
externoff_tget_cfs_relation_decompressed_size(RelFileNodeBackendrnode,
126+
constchar*fullpath,ForkNumberforknum);
127+
#endif
128+
114129
#endif/* PTRACK_ENGINE_H */

‎ptrack.c‎

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -251,14 +251,6 @@ ptrack_copydir_hook(const char *path)
251251

252252
elog(DEBUG1,"ptrack_copydir_hook: spcOid %u, dbOid %u",spcOid,dbOid);
253253

254-
#ifdefPGPRO_EE
255-
/*
256-
* Currently, we do not track files from compressed tablespaces in ptrack.
257-
*/
258-
if (file_is_in_cfs_tablespace(path))
259-
elog(DEBUG1,"ptrack_copydir_hook: skipping changes tracking in the CFS tablespace %u",spcOid);
260-
else
261-
#endif
262254
ptrack_walkdir(path,spcOid,dbOid);
263255

264256
if (prev_copydir_hook)
@@ -315,7 +307,8 @@ ptrack_gather_filelist(List **filelist, char *path, Oid spcOid, Oid dbOid)
315307

316308
if (strcmp(de->d_name,".")==0||
317309
strcmp(de->d_name,"..")==0||
318-
looks_like_temp_rel_name(de->d_name))
310+
looks_like_temp_rel_name(de->d_name)||
311+
file_type_from_path(de->d_name)==FileTypeCFM)
319312
continue;
320313

321314
snprintf(subpath,sizeof(subpath),"%s/%s",path,de->d_name);
@@ -403,6 +396,10 @@ ptrack_filelist_getnext(PtScanCtx * ctx)
403396
ListCell*cell;
404397
char*fullpath;
405398
structstatfst;
399+
off_trel_st_size;
400+
#ifdefPGPRO_EE
401+
RelFileNodeBackendrnodebackend;
402+
#endif
406403

407404
/* No more file in the list */
408405
if (list_length(ctx->filelist)==0)
@@ -449,14 +446,23 @@ ptrack_filelist_getnext(PtScanCtx * ctx)
449446
returnptrack_filelist_getnext(ctx);
450447
}
451448

449+
#ifdefPGPRO_EE
450+
rnodebackend.node=ctx->bid.relnode;
451+
rnodebackend.backend=InvalidBackendId;
452+
453+
rel_st_size=get_cfs_relation_decompressed_size(rnodebackend,fullpath,pfl->forknum);
454+
#else
455+
rel_st_size=fst.st_size;
456+
#endif
457+
452458
if (pfl->segno>0)
453459
{
454-
ctx->relsize=pfl->segno*RELSEG_SIZE+fst.st_size /BLCKSZ;
460+
ctx->relsize=pfl->segno*RELSEG_SIZE+rel_st_size /BLCKSZ;
455461
ctx->bid.blocknum=pfl->segno*RELSEG_SIZE;
456462
}
457463
else
458464
/* Estimate relsize as size of first segment in blocks */
459-
ctx->relsize=fst.st_size /BLCKSZ;
465+
ctx->relsize=rel_st_size /BLCKSZ;
460466

461467
elog(DEBUG3,"ptrack: got file %s with size %u from the file list",pfl->path,ctx->relsize);
462468

‎t/002_cfs_compatibility.pl‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
use strict;
2+
use warnings;
3+
use Test::More;
4+
5+
my$pg_15_modules;
6+
7+
BEGIN
8+
{
9+
$pg_15_modules =eval
10+
{
11+
require PostgreSQL::Test::Cluster;
12+
require PostgreSQL::Test::Utils;
13+
return 1;
14+
};
15+
16+
unless (defined$pg_15_modules)
17+
{
18+
$pg_15_modules = 0;
19+
20+
require PostgresNode;
21+
require TestLib;
22+
}
23+
}
24+
25+
note('PostgreSQL 15 modules are used:' . ($pg_15_modules ?'yes' :'no'));
26+
27+
my$node;
28+
my$res_stdout;
29+
my$res_stderr;
30+
31+
# Create node.
32+
# Older versions of PostgreSQL modules use get_new_node function.
33+
# Newer use standard perl object constructor syntax.
34+
eval
35+
{
36+
if ($pg_15_modules)
37+
{
38+
$node = PostgreSQL::Test::Cluster->new("node");
39+
}
40+
else
41+
{
42+
$node = PostgresNode::get_new_node("node");
43+
}
44+
};
45+
46+
note"Test for handling a ptrack map in compressed relations";
47+
48+
my$psql_stdout;
49+
50+
# Starting the node
51+
$node->init;
52+
53+
# Could not load ptrack module after postmaster start
54+
55+
my$cfs_tblspc1 =$node->basedir."/cfs_tblspc1";
56+
my$cfs_tblspc2 =$node->basedir."/cfs_tblspc2";
57+
mkdir$cfs_tblspc1ordie;
58+
mkdir$cfs_tblspc2ordie;
59+
my$no_cfs_tblspc1 =$node->basedir."/no_cfs_tblspc1";
60+
my$no_cfs_tblspc2 =$node->basedir."/no_cfs_tblspc2";
61+
mkdir$no_cfs_tblspc1ordie;
62+
mkdir$no_cfs_tblspc2ordie;
63+
64+
$node->append_conf('postgresql.conf',qq{
65+
shared_preload_libraries = 'ptrack'
66+
ptrack.map_size = 16
67+
wal_level = 'replica'
68+
});
69+
70+
$node->start;
71+
72+
# check cfs availability first
73+
my$cfs_available =$node->safe_psql('postgres',
74+
"select count(oid) from pg_proc where proname = 'cfs_version'");
75+
76+
if($cfs_availableeq"0") {
77+
$node->stop;
78+
planskip_all=>"CFS is not supported by this PostgreSQL build";
79+
}else {
80+
plantests=> 2;
81+
}
82+
83+
# Creating content
84+
$node->safe_psql('postgres',qq|
85+
create tablespace cfs_tblspc1 location '$cfs_tblspc1' with (compression=true);
86+
create tablespace cfs_tblspc2 location '$cfs_tblspc2' with (compression=true);
87+
create tablespace no_cfs_tblspc1 location '$no_cfs_tblspc1';
88+
create tablespace no_cfs_tblspc2 location '$no_cfs_tblspc2';
89+
90+
create database testing_cfs tablespace cfs_tblspc1;
91+
create database testing_no_cfs tablespace no_cfs_tblspc1;
92+
|);
93+
94+
$node->safe_psql('testing_cfs',qq{
95+
create table testing(i int, text varchar);
96+
insert into testing select 1, '1111111111111111111111111' from generate_series(1,10000000);
97+
});
98+
99+
$node->safe_psql('testing_no_cfs',qq{
100+
create table testing_no(i int, text varchar);
101+
insert into testing_no select 1, '1111111111111111111111111' from generate_series(1,10000000);
102+
});
103+
104+
# creating ptrack
105+
$node->safe_psql('postgres',"create extension ptrack");
106+
107+
# obtaining init lsn for further usage in ptrack_get_pagemapset
108+
my$init_lsn =$node->safe_psql('postgres','select ptrack_init_lsn()');
109+
110+
# obtaining relpath for cfs table
111+
my$cfs_relpath =$node->safe_psql('testing_cfs',"select pg_relation_filepath('testing');");
112+
113+
# obtaining relpath for no-cfs table
114+
my$no_cfs_relpath =$node->safe_psql('testing_no_cfs',"select pg_relation_filepath('testing_no');");
115+
116+
# forcing copydir() hook by altering dbs tablespaces
117+
$node->safe_psql('postgres',"alter database testing_cfs set tablespace cfs_tblspc2;");
118+
$node->safe_psql('postgres',"alter database testing_no_cfs set tablespace no_cfs_tblspc2;");
119+
120+
# select the pagecount sums and compare them (should be equal)
121+
my$pagecount_sum_cfs =$node->safe_psql('postgres',
122+
"select sum(pagecount) from ptrack_get_pagemapset('$init_lsn'::pg_lsn) where path like '%$cfs_relpath%';");
123+
my$pagecount_sum_no_cfs =$node->safe_psql('postgres',
124+
"select sum(pagecount) from ptrack_get_pagemapset('$init_lsn'::pg_lsn) where path like '%$no_cfs_relpath%';");
125+
126+
is($pagecount_sum_cfs,$pagecount_sum_no_cfs,"pagecount sums don't match");
127+
128+
# forcing copydir() hook by altering dbs tablespaces back
129+
$node->safe_psql('postgres',"alter database testing_cfs set tablespace cfs_tblspc1;");
130+
$node->safe_psql('postgres',"alter database testing_no_cfs set tablespace no_cfs_tblspc1;");
131+
132+
# select the pagecount sums and compare them (again, they should be equal)
133+
$pagecount_sum_cfs =$node->safe_psql('postgres',
134+
"select sum(pagecount) from ptrack_get_pagemapset('$init_lsn'::pg_lsn) where path like '%$cfs_relpath%';");
135+
$pagecount_sum_no_cfs =$node->safe_psql('postgres',
136+
"select sum(pagecount) from ptrack_get_pagemapset('$init_lsn'::pg_lsn) where path like '%$no_cfs_relpath%';");
137+
138+
is($pagecount_sum_cfs,$pagecount_sum_no_cfs,"pagecount sums don't match");
139+
140+
141+
$node->stop;
142+

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp