Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit693bffe

Browse files
committed
[PBCKP-146] fix cfm truncated crc calculation in delta/page backup
- On backup we should compare truncated crc with previous version.- copying remote file didn't honor "don't truncate first 64 bytes" rule.- crc calculation didn't honoer "don't truncate first 64 bytes" rule.
1 parent8570825 commit693bffe

File tree

4 files changed

+48
-13
lines changed

4 files changed

+48
-13
lines changed

‎src/data.c‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -806,7 +806,10 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
806806
* file could be deleted under our feets.
807807
* But then backup_non_data_file_internal will handle it safely
808808
*/
809-
file->crc=fio_get_crc32(from_fullpath,FIO_DB_HOST, false, true);
809+
if (file->forkName!=cfm)
810+
file->crc=fio_get_crc32(from_fullpath,FIO_DB_HOST, false, true);
811+
else
812+
file->crc=fio_get_crc32_truncated(from_fullpath,FIO_DB_HOST, true);
810813

811814
/* ...and checksum is the same... */
812815
if (EQ_TRADITIONAL_CRC32(file->crc,prev_file->crc))
@@ -1334,7 +1337,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
13341337
pg_crc32file_crc;
13351338
if (tmp_file->forkName==cfm&&
13361339
tmp_file->uncompressed_size>tmp_file->write_size)
1337-
file_crc=fio_get_crc32_truncated(to_fullpath,FIO_DB_HOST);
1340+
file_crc=fio_get_crc32_truncated(to_fullpath,FIO_DB_HOST, false);
13381341
else
13391342
file_crc=fio_get_crc32(to_fullpath,FIO_DB_HOST, false, false);
13401343

‎src/pg_probackup.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1082,7 +1082,7 @@ extern void fio_pgFileDelete(pgFile *file, const char *full_path);
10821082
externvoidpgFileFree(void*file);
10831083

10841084
externpg_crc32pgFileGetCRC(constchar*file_path,booluse_crc32c,boolmissing_ok);
1085-
externpg_crc32pgFileGetCRCTruncated(constchar*file_path,booluse_crc32c);
1085+
externpg_crc32pgFileGetCRCTruncated(constchar*file_path,booluse_crc32c,boolmissing_ok);
10861086
externpg_crc32pgFileGetCRCgz(constchar*file_path,booluse_crc32c,boolmissing_ok);
10871087

10881088
externintpgFileMapComparePath(constvoid*f1,constvoid*f2);

‎src/utils/file.c‎

Lines changed: 40 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1377,8 +1377,6 @@ fio_get_crc32_ex(const char *file_path, fio_location location,
13771377
{
13781378
if (decompress&&truncated)
13791379
elog(ERROR,"Could not calculate CRC for compressed truncated file");
1380-
if (missing_ok&&truncated)
1381-
elog(ERROR,"CRC calculation for missing truncated file is forbidden");
13821380

13831381
if (fio_is_remote(location))
13841382
{
@@ -1408,7 +1406,7 @@ fio_get_crc32_ex(const char *file_path, fio_location location,
14081406
if (decompress)
14091407
returnpgFileGetCRCgz(file_path, true,missing_ok);
14101408
elseif (truncated)
1411-
returnpgFileGetCRCTruncated(file_path, true);
1409+
returnpgFileGetCRCTruncated(file_path, true,missing_ok);
14121410
else
14131411
returnpgFileGetCRC(file_path, true,missing_ok);
14141412
}
@@ -1422,9 +1420,10 @@ fio_get_crc32(const char *file_path, fio_location location,
14221420
}
14231421

14241422
pg_crc32
1425-
fio_get_crc32_truncated(constchar*file_path,fio_locationlocation)
1423+
fio_get_crc32_truncated(constchar*file_path,fio_locationlocation,
1424+
boolmissing_ok)
14261425
{
1427-
returnfio_get_crc32_ex(file_path,location, false,false, true);
1426+
returnfio_get_crc32_ex(file_path,location, false,missing_ok, true);
14281427
}
14291428

14301429
/* Remove file */
@@ -3003,6 +3002,7 @@ fio_send_file_impl(int out, char const* path)
30033002
fio_headerhdr;
30043003
char*buf=pgut_malloc(CHUNK_SIZE);
30053004
size_tread_len=0;
3005+
int64_tread_size=0;
30063006
char*errormsg=NULL;
30073007

30083008
/* open source file for read */
@@ -3066,7 +3066,19 @@ fio_send_file_impl(int out, char const* path)
30663066
if (read_len>0)
30673067
{
30683068
/* send chunk */
3069-
size_tnon_zero_len=find_zero_tail(buf,read_len);
3069+
int64_tnon_zero_len=find_zero_tail(buf,read_len);
3070+
/*
3071+
* It is dirty trick to silence warnings in CFS GC process:
3072+
* backup at least cfs header size bytes.
3073+
*/
3074+
if (read_size+non_zero_len<PAGE_ZEROSEARCH_FINE_GRANULARITY&&
3075+
read_size+read_len>0)
3076+
{
3077+
non_zero_len=Min(PAGE_ZEROSEARCH_FINE_GRANULARITY,
3078+
read_size+read_len);
3079+
non_zero_len-=read_size;
3080+
}
3081+
30703082
if (non_zero_len>0)
30713083
{
30723084
hdr.cop=FIO_PAGE;
@@ -3082,6 +3094,8 @@ fio_send_file_impl(int out, char const* path)
30823094
hdr.arg=read_len-non_zero_len;
30833095
IO_CHECK(fio_write_all(out,&hdr,sizeof(hdr)),sizeof(hdr));
30843096
}
3097+
3098+
read_size+=read_len;
30853099
}
30863100

30873101
if (feof(fp))
@@ -3166,7 +3180,7 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok)
31663180
* Read the local file to compute CRC for it extened to real_size.
31673181
*/
31683182
pg_crc32
3169-
pgFileGetCRCTruncated(constchar*file_path,booluse_crc32c)
3183+
pgFileGetCRCTruncated(constchar*file_path,booluse_crc32c,boolmissing_ok)
31703184
{
31713185
FILE*fp;
31723186
char*buf;
@@ -3180,6 +3194,15 @@ pgFileGetCRCTruncated(const char *file_path, bool use_crc32c)
31803194
fp=fopen(file_path,PG_BINARY_R);
31813195
if (fp==NULL)
31823196
{
3197+
if (errno==ENOENT)
3198+
{
3199+
if (missing_ok)
3200+
{
3201+
FIN_FILE_CRC32(use_crc32c,st.crc);
3202+
returnst.crc;
3203+
}
3204+
}
3205+
31833206
elog(ERROR,"Cannot open file \"%s\": %s",
31843207
file_path,strerror(errno));
31853208
}
@@ -3200,6 +3223,14 @@ pgFileGetCRCTruncated(const char *file_path, bool use_crc32c)
32003223
elog(ERROR,"Cannot read \"%s\": %s",file_path,strerror(errno));
32013224

32023225
non_zero_len=find_zero_tail(buf,len);
3226+
/* same trick as in fio_send_file */
3227+
if (st.read_size+non_zero_len<PAGE_ZEROSEARCH_FINE_GRANULARITY&&
3228+
st.read_size+len>0)
3229+
{
3230+
non_zero_len=Min(PAGE_ZEROSEARCH_FINE_GRANULARITY,
3231+
st.read_size+len);
3232+
non_zero_len-=st.read_size;
3233+
}
32033234
if (non_zero_len)
32043235
{
32053236
fio_send_file_crc(&st,buf,non_zero_len);
@@ -3894,12 +3925,12 @@ fio_communicate(int in, int out)
38943925
break;
38953926
caseFIO_GET_CRC32:
38963927
Assert((hdr.arg&GET_CRC32_TRUNCATED)==0||
3897-
(hdr.arg&GET_CRC32_TRUNCATED)==GET_CRC32_TRUNCATED);
3928+
(hdr.arg&(GET_CRC32_TRUNCATED|GET_CRC32_DECOMPRESS))==GET_CRC32_TRUNCATED);
38983929
/* calculate crc32 for a file */
38993930
if ((hdr.arg&GET_CRC32_DECOMPRESS))
39003931
crc=pgFileGetCRCgz(buf, true, (hdr.arg&GET_CRC32_MISSING_OK)!=0);
39013932
elseif ((hdr.arg&GET_CRC32_TRUNCATED))
3902-
crc=pgFileGetCRCTruncated(buf, true);
3933+
crc=pgFileGetCRCTruncated(buf, true, (hdr.arg&GET_CRC32_MISSING_OK)!=0);
39033934
else
39043935
crc=pgFileGetCRC(buf, true, (hdr.arg&GET_CRC32_MISSING_OK)!=0);
39053936
IO_CHECK(fio_write_all(out,&crc,sizeof(crc)),sizeof(crc));

‎src/utils/file.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,8 @@ extern void fio_disconnect(void);
123123
externintfio_sync(charconst*path,fio_locationlocation);
124124
externpg_crc32fio_get_crc32(constchar*file_path,fio_locationlocation,
125125
booldecompress,boolmissing_ok);
126-
externpg_crc32fio_get_crc32_truncated(constchar*file_path,fio_locationlocation);
126+
externpg_crc32fio_get_crc32_truncated(constchar*file_path,fio_locationlocation,
127+
boolmissing_ok);
127128

128129
externintfio_rename(charconst*old_path,charconst*new_path,fio_locationlocation);
129130
externintfio_symlink(charconst*target,charconst*link_path,booloverwrite,fio_locationlocation);

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp