Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit2ec6bc6

Browse files
committed
Implement incremental backup of compressed files, both page and ptrack do the same. Adds two columns to the list of files.
1 parent976694f commit2ec6bc6

File tree

5 files changed

+507
-23
lines changed

5 files changed

+507
-23
lines changed

‎backup.c

Lines changed: 176 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1102,6 +1102,75 @@ backup_cleanup(bool fatal, void *userdata)
11021102
}
11031103
}
11041104

1105+
/* Count bytes in file */
1106+
staticlong
1107+
file_size(constchar*file)
1108+
{
1109+
longr;
1110+
FILE*f=fopen(file,"r");
1111+
1112+
if (!f)
1113+
{
1114+
elog(ERROR,"pg_probackup: could not open file \"%s\" for reading: %s\n",
1115+
file,strerror(errno));
1116+
return-1;
1117+
}
1118+
fseek(f,0,SEEK_END);
1119+
r=ftell(f);
1120+
fclose(f);
1121+
returnr;
1122+
}
1123+
1124+
/*
1125+
* Find corresponding file in previous backup.
1126+
* Compare generations and return true if we don't need full copy
1127+
* of the file, but just part of it.
1128+
*
1129+
* skip_size - size of the file in previous backup. We can skip it
1130+
* and copy just remaining part of the file.
1131+
*/
1132+
bool
1133+
backup_compressed_file_partially(pgFile*file,void*arg,size_t*skip_size)
1134+
{
1135+
boolresult= false;
1136+
pgFile*prev_file=NULL;
1137+
size_tcurrent_file_size;
1138+
backup_files_args*arguments= (backup_files_args*)arg;
1139+
1140+
if (arguments->prev_files)
1141+
{
1142+
pgFile**p= (pgFile**)parray_bsearch(arguments->prev_files,
1143+
file,pgFileComparePath);
1144+
if (p)
1145+
prev_file=*p;
1146+
1147+
/* If file's gc generation has changed since last backup, just copy it*/
1148+
if (prev_file&&prev_file->generation==file->generation)
1149+
{
1150+
current_file_size=file_size(file->path);
1151+
1152+
if (prev_file->write_size==BYTES_INVALID)
1153+
return false;
1154+
1155+
*skip_size=prev_file->write_size;
1156+
1157+
if (current_file_size >=prev_file->write_size)
1158+
{
1159+
elog(LOG,"Backup file %s partially: prev_size %lu, current_size %lu",
1160+
file->path,prev_file->write_size,current_file_size);
1161+
result= true;
1162+
}
1163+
else
1164+
elog(ERROR,"Something is wrong with %s. current_file_size %lu, prev %lu",
1165+
file->path,current_file_size,prev_file->write_size);
1166+
}
1167+
else
1168+
elog(LOG,"Copy full %s.",file->path);
1169+
}
1170+
1171+
returnresult;
1172+
}
1173+
11051174
/*
11061175
* Take differential backup at page level.
11071176
*/
@@ -1200,9 +1269,47 @@ backup_files(void *arg)
12001269
}
12011270

12021271
/* copy the file into backup */
1203-
if (!(file->is_datafile
1204-
?backup_data_file(arguments->from_root,arguments->to_root,file,arguments->lsn)
1205-
:copy_file(arguments->from_root,arguments->to_root,file)))
1272+
if (file->is_datafile)
1273+
{
1274+
if (!backup_data_file(arguments->from_root,
1275+
arguments->to_root,file,
1276+
arguments->lsn))
1277+
{
1278+
/* record as skipped file in file_xxx.txt */
1279+
file->write_size=BYTES_INVALID;
1280+
elog(LOG,"skip");
1281+
continue;
1282+
}
1283+
}
1284+
elseif (is_compressed_data_file(file))
1285+
{
1286+
size_tskip_size=0;
1287+
if (backup_compressed_file_partially(file,arguments,&skip_size))
1288+
{
1289+
/* backup cfs segment partly */
1290+
if (!copy_file_partly(arguments->from_root,
1291+
arguments->to_root,
1292+
file,skip_size))
1293+
{
1294+
/* record as skipped file in file_xxx.txt */
1295+
file->write_size=BYTES_INVALID;
1296+
elog(LOG,"skip");
1297+
continue;
1298+
}
1299+
}
1300+
elseif (!copy_file(arguments->from_root,
1301+
arguments->to_root,
1302+
file))
1303+
{
1304+
/* record as skipped file in file_xxx.txt */
1305+
file->write_size=BYTES_INVALID;
1306+
elog(LOG,"skip");
1307+
continue;
1308+
}
1309+
}
1310+
elseif (!copy_file(arguments->from_root,
1311+
arguments->to_root,
1312+
file))
12061313
{
12071314
/* record as skipped file in file_xxx.txt */
12081315
file->write_size=BYTES_INVALID;
@@ -1251,14 +1358,14 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
12511358
relative=file->path+strlen(root)+1;
12521359
if (is_pgdata&&
12531360
!path_is_prefix_of_path("base",relative)&&
1254-
/*!path_is_prefix_of_path("global", relative) &&*/
1361+
/*!path_is_prefix_of_path("global", relative) &&*///TODO What's wrong with this line?
12551362
!path_is_prefix_of_path("pg_tblspc",relative))
12561363
continue;
12571364

12581365
/* Get file name from path */
12591366
fname=last_dir_separator(relative);
12601367

1261-
/* Remove temp tables */
1368+
/* Remove temp tablesfrom the list*/
12621369
if (fname[0]=='t'&&isdigit(fname[1]))
12631370
{
12641371
pgFileFree(file);
@@ -1268,7 +1375,7 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
12681375
}
12691376

12701377
path_len=strlen(file->path);
1271-
/* Get link ptrack file torealations files */
1378+
/* Get link ptrack file torelations files */
12721379
if (path_len>6&&strncmp(file->path+(path_len-6),"ptrack",6)==0)
12731380
{
12741381
pgFile*search_file;
@@ -1277,12 +1384,15 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
12771384
while(true) {
12781385
pgFiletmp_file;
12791386
tmp_file.path=pg_strdup(file->path);
1280-
/* I hope segno not more than 999999 */
1387+
1388+
/* Segno fits into 6 digits since it is not more than 4000 */
12811389
if (segno>0)
12821390
sprintf(tmp_file.path+path_len-7,".%d",segno);
12831391
else
12841392
tmp_file.path[path_len-7]='\0';
1393+
12851394
pre_search_file= (pgFile**)parray_bsearch(list_file,&tmp_file,pgFileComparePath);
1395+
12861396
if (pre_search_file!=NULL)
12871397
{
12881398
search_file=*pre_search_file;
@@ -1296,6 +1406,7 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
12961406
segno++;
12971407
}
12981408

1409+
/* Remove ptrack file itself from backup list */
12991410
pgFileFree(file);
13001411
parray_remove(list_file,i);
13011412
i--;
@@ -1305,9 +1416,9 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
13051416
/* compress map file it is not data file */
13061417
if (path_len>4&&strncmp(file->path+(path_len-4),".cfm",4)==0)
13071418
{
1308-
if (current.backup_mode==BACKUP_MODE_DIFF_PTRACK||
1309-
current.backup_mode==BACKUP_MODE_DIFF_PAGE)
1310-
elog(ERROR,"You can't use incremental backup with compress tablespace");
1419+
if (current.backup_mode==BACKUP_MODE_DIFF_PAGE)
1420+
elog(ERROR,"You can't use PAGE mode backup with compressed tablespace.\n"
1421+
"Try FULL or PTRACK mode instead.");
13111422
continue;
13121423
}
13131424

@@ -1355,11 +1466,34 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
13551466
pgFiletmp_file;
13561467
tmp_file.path=pg_strdup(file->path);
13571468
tmp_file.path[path_len-4]='\0';
1358-
pre_search_file= (pgFile**)parray_bsearch(list_file,&tmp_file,pgFileComparePath);
1469+
pre_search_file= (pgFile**)parray_bsearch(list_file,
1470+
&tmp_file,pgFileComparePath);
13591471
if (pre_search_file!=NULL)
13601472
{
1473+
FileMap*map;
1474+
intmd=open(file->path,O_RDWR|PG_BINARY,0);
1475+
if (md<0)
1476+
elog(ERROR,"add_files(). cannot open cfm file '%s'",file->path);
1477+
1478+
map=cfs_mmap(md);
1479+
if (map==MAP_FAILED)
1480+
{
1481+
elog(LOG,"add_files(). cfs_compression_ration failed to map file %s: %m",file->path);
1482+
close(md);
1483+
break;
1484+
}
1485+
1486+
(*pre_search_file)->generation=map->generation;
13611487
(*pre_search_file)->is_datafile= false;
1488+
1489+
if (cfs_munmap(map)<0)
1490+
elog(LOG,"add_files(). CFS failed to unmap file %s: %m",file->path);
1491+
if (close(md)<0)
1492+
elog(LOG,"add_files(). CFS failed to close file %s: %m",file->path);
13621493
}
1494+
else
1495+
elog(ERROR,"corresponding segment '%s' is not found",tmp_file.path);
1496+
13631497
pg_free(tmp_file.path);
13641498
}
13651499
}
@@ -1633,3 +1767,34 @@ StreamLog(void *arg)
16331767
PQfinish(conn);
16341768
conn=NULL;
16351769
}
1770+
1771+
1772+
FileMap*cfs_mmap(intmd)
1773+
{
1774+
FileMap*map;
1775+
#ifdefWIN32
1776+
HANDLEmh=CreateFileMapping(_get_osfhandle(md),NULL,PAGE_READWRITE,
1777+
0, (DWORD)sizeof(FileMap),NULL);
1778+
if (mh==NULL)
1779+
return (FileMap*)MAP_FAILED;
1780+
1781+
map= (FileMap*)MapViewOfFile(mh,FILE_MAP_ALL_ACCESS,0,0,0);
1782+
CloseHandle(mh);
1783+
if (map==NULL)
1784+
return (FileMap*)MAP_FAILED;
1785+
1786+
#else
1787+
map= (FileMap*)mmap(NULL,sizeof(FileMap),
1788+
PROT_WRITE |PROT_READ,MAP_SHARED,md,0);
1789+
#endif
1790+
returnmap;
1791+
}
1792+
1793+
intcfs_munmap(FileMap*map)
1794+
{
1795+
#ifdefWIN32
1796+
returnUnmapViewOfFile(map) ?0 :-1;
1797+
#else
1798+
returnmunmap(map,sizeof(FileMap));
1799+
#endif
1800+
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp