Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit0c7c78b

Browse files
committed
Merge branch 'REL_2_5-PBCKP-146-2' into REL_2_5
2 parents34af025 +e36924a commit0c7c78b

File tree

12 files changed

+727
-256
lines changed

12 files changed

+727
-256
lines changed

‎src/archive.c

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1375,11 +1375,11 @@ get_wal_file(const char *filename, const char *from_fullpath,
13751375
#ifdefHAVE_LIBZ
13761376
/* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */
13771377
if (IsXLogFileName(filename))
1378-
rc=fio_send_file_gz(from_fullpath_gz,to_fullpath,out,&errmsg);
1378+
rc=fio_send_file_gz(from_fullpath_gz,out,&errmsg);
13791379
if (rc==FILE_MISSING)
13801380
#endif
13811381
/* ... failing that, use uncompressed */
1382-
rc=fio_send_file(from_fullpath,to_fullpath,out,NULL,&errmsg);
1382+
rc=fio_send_file(from_fullpath,out, false,NULL,&errmsg);
13831383

13841384
/* When not in prefetch mode, try to use partial file */
13851385
if (rc==FILE_MISSING&& !prefetch_mode&&IsXLogFileName(filename))
@@ -1389,13 +1389,13 @@ get_wal_file(const char *filename, const char *from_fullpath,
13891389
#ifdefHAVE_LIBZ
13901390
/* '.gz.partial' goes first ... */
13911391
snprintf(from_partial,sizeof(from_partial),"%s.gz.partial",from_fullpath);
1392-
rc=fio_send_file_gz(from_partial,to_fullpath,out,&errmsg);
1392+
rc=fio_send_file_gz(from_partial,out,&errmsg);
13931393
if (rc==FILE_MISSING)
13941394
#endif
13951395
{
13961396
/* ... failing that, use '.partial' */
13971397
snprintf(from_partial,sizeof(from_partial),"%s.partial",from_fullpath);
1398-
rc=fio_send_file(from_partial,to_fullpath,out,NULL,&errmsg);
1398+
rc=fio_send_file(from_partial,out, false,NULL,&errmsg);
13991399
}
14001400

14011401
if (rc==SEND_OK)

‎src/catalog.c

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1069,6 +1069,7 @@ get_backup_filelist(pgBackup *backup, bool strict)
10691069
charlinked[MAXPGPATH];
10701070
charcompress_alg_string[MAXPGPATH];
10711071
int64write_size,
1072+
uncompressed_size,
10721073
mode,/* bit length of mode_t depends on platforms */
10731074
is_datafile,
10741075
is_cfs,
@@ -1132,6 +1133,11 @@ get_backup_filelist(pgBackup *backup, bool strict)
11321133
if (get_control_value_int64(buf,"hdr_size",&hdr_size, false))
11331134
file->hdr_size= (int)hdr_size;
11341135

1136+
if (get_control_value_int64(buf,"full_size",&uncompressed_size, false))
1137+
file->uncompressed_size=uncompressed_size;
1138+
else
1139+
file->uncompressed_size=write_size;
1140+
11351141
if (file->external_dir_num==0)
11361142
set_forkname(file);
11371143

@@ -2561,6 +2567,11 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
25612567
file->external_dir_num,
25622568
file->dbOid);
25632569

2570+
if (file->uncompressed_size!=0&&
2571+
file->uncompressed_size!=file->write_size)
2572+
len+=sprintf(line+len,",\"full_size\":\""INT64_FORMAT"\"",
2573+
file->uncompressed_size);
2574+
25642575
if (file->is_datafile)
25652576
len+=sprintf(line+len,",\"segno\":\"%d\"",file->segno);
25662577

‎src/data.c

Lines changed: 36 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -799,6 +799,7 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
799799
* and its mtime is less than parent backup start time ... */
800800
if ((pg_strcasecmp(file->name,RELMAPPER_FILENAME)!=0)&&
801801
(prev_file&&file->exists_in_prev&&
802+
file->size==prev_file->size&&
802803
file->mtime <=parent_backup_time))
803804
{
804805
/*
@@ -1330,7 +1331,12 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
13301331
if (already_exists)
13311332
{
13321333
/* compare checksums of already existing file and backup file */
1333-
pg_crc32file_crc=fio_get_crc32(to_fullpath,FIO_DB_HOST, false, false);
1334+
pg_crc32file_crc;
1335+
if (tmp_file->forkName==cfm&&
1336+
tmp_file->uncompressed_size>tmp_file->write_size)
1337+
file_crc=fio_get_crc32_truncated(to_fullpath,FIO_DB_HOST);
1338+
else
1339+
file_crc=fio_get_crc32(to_fullpath,FIO_DB_HOST, false, false);
13341340

13351341
if (file_crc==tmp_file->crc)
13361342
{
@@ -1387,10 +1393,12 @@ backup_non_data_file_internal(const char *from_fullpath,
13871393
constchar*to_fullpath,pgFile*file,
13881394
boolmissing_ok)
13891395
{
1390-
FILE*in=NULL;
13911396
FILE*out=NULL;
1392-
ssize_tread_len=0;
1393-
char*buf=NULL;
1397+
char*errmsg=NULL;
1398+
intrc;
1399+
boolcut_zero_tail;
1400+
1401+
cut_zero_tail=file->forkName==cfm;
13941402

13951403
INIT_FILE_CRC32(true,file->crc);
13961404

@@ -1412,107 +1420,44 @@ backup_non_data_file_internal(const char *from_fullpath,
14121420

14131421
/* backup remote file */
14141422
if (fio_is_remote(FIO_DB_HOST))
1415-
{
1416-
char*errmsg=NULL;
1417-
intrc=fio_send_file(from_fullpath,to_fullpath,out,file,&errmsg);
1423+
rc=fio_send_file(from_fullpath,out,cut_zero_tail,file,&errmsg);
1424+
else
1425+
rc=fio_send_file_local(from_fullpath,out,cut_zero_tail,file,&errmsg);
14181426

1419-
/* handle errors */
1420-
if (rc==FILE_MISSING)
1421-
{
1422-
/* maybe deleted, it's not error in case of backup */
1423-
if (missing_ok)
1424-
{
1425-
elog(LOG,"File \"%s\" is not found",from_fullpath);
1426-
file->write_size=FILE_NOT_FOUND;
1427-
gotocleanup;
1428-
}
1429-
else
1430-
elog(ERROR,"File \"%s\" is not found",from_fullpath);
1431-
}
1432-
elseif (rc==WRITE_FAILED)
1433-
elog(ERROR,"Cannot write to \"%s\": %s",to_fullpath,strerror(errno));
1434-
elseif (rc!=SEND_OK)
1427+
/* handle errors */
1428+
if (rc==FILE_MISSING)
1429+
{
1430+
/* maybe deleted, it's not error in case of backup */
1431+
if (missing_ok)
14351432
{
1436-
if (errmsg)
1437-
elog(ERROR,"%s",errmsg);
1438-
else
1439-
elog(ERROR,"Cannot access remote file \"%s\"",from_fullpath);
1433+
elog(LOG,"File \"%s\" is not found",from_fullpath);
1434+
file->write_size=FILE_NOT_FOUND;
1435+
gotocleanup;
14401436
}
1441-
1442-
pg_free(errmsg);
1437+
else
1438+
elog(ERROR,"File \"%s\" is not found",from_fullpath);
14431439
}
1444-
/* backup local file */
1445-
else
1440+
elseif (rc==WRITE_FAILED)
1441+
elog(ERROR,"Cannot write to \"%s\": %s",to_fullpath,strerror(errno));
1442+
elseif (rc!=SEND_OK)
14461443
{
1447-
/* open source file for read */
1448-
in=fopen(from_fullpath,PG_BINARY_R);
1449-
if (in==NULL)
1450-
{
1451-
/* maybe deleted, it's not error in case of backup */
1452-
if (errno==ENOENT)
1453-
{
1454-
if (missing_ok)
1455-
{
1456-
elog(LOG,"File \"%s\" is not found",from_fullpath);
1457-
file->write_size=FILE_NOT_FOUND;
1458-
gotocleanup;
1459-
}
1460-
else
1461-
elog(ERROR,"File \"%s\" is not found",from_fullpath);
1462-
}
1463-
1464-
elog(ERROR,"Cannot open file \"%s\": %s",from_fullpath,
1465-
strerror(errno));
1466-
}
1467-
1468-
/* disable stdio buffering for local input/output files to avoid triple buffering */
1469-
setvbuf(in,NULL,_IONBF,BUFSIZ);
1470-
setvbuf(out,NULL,_IONBF,BUFSIZ);
1471-
1472-
/* allocate 64kB buffer */
1473-
buf=pgut_malloc(CHUNK_SIZE);
1474-
1475-
/* copy content and calc CRC */
1476-
for (;;)
1477-
{
1478-
read_len=fread(buf,1,CHUNK_SIZE,in);
1479-
1480-
if (ferror(in))
1481-
elog(ERROR,"Cannot read from file \"%s\": %s",
1482-
from_fullpath,strerror(errno));
1483-
1484-
if (read_len>0)
1485-
{
1486-
if (fwrite(buf,1,read_len,out)!=read_len)
1487-
elog(ERROR,"Cannot write to file \"%s\": %s",to_fullpath,
1488-
strerror(errno));
1489-
1490-
/* update CRC */
1491-
COMP_FILE_CRC32(true,file->crc,buf,read_len);
1492-
file->read_size+=read_len;
1493-
}
1494-
1495-
if (feof(in))
1496-
break;
1497-
}
1444+
if (errmsg)
1445+
elog(ERROR,"%s",errmsg);
1446+
else
1447+
elog(ERROR,"Cannot access remote file \"%s\"",from_fullpath);
14981448
}
14991449

1500-
file->write_size= (int64)file->read_size;
1501-
1502-
if (file->write_size>0)
1503-
file->uncompressed_size=file->write_size;
1450+
file->uncompressed_size=file->read_size;
15041451

15051452
cleanup:
1453+
if (errmsg!=NULL)
1454+
pg_free(errmsg);
1455+
15061456
/* finish CRC calculation and store into pgFile */
15071457
FIN_FILE_CRC32(true,file->crc);
15081458

1509-
if (in&&fclose(in))
1510-
elog(ERROR,"Cannot close the file \"%s\": %s",from_fullpath,strerror(errno));
1511-
15121459
if (out&&fclose(out))
15131460
elog(ERROR,"Cannot close the file \"%s\": %s",to_fullpath,strerror(errno));
1514-
1515-
pg_free(buf);
15161461
}
15171462

15181463
/*

‎src/dir.c

Lines changed: 1 addition & 132 deletions
Original file line numberDiff line numberDiff line change
@@ -262,137 +262,6 @@ pgFileDelete(mode_t mode, const char *full_path)
262262
}
263263
}
264264

265-
/*
266-
* Read the local file to compute its CRC.
267-
* We cannot make decision about file decompression because
268-
* user may ask to backup already compressed files and we should be
269-
* obvious about it.
270-
*/
271-
pg_crc32
272-
pgFileGetCRC(constchar*file_path,booluse_crc32c,boolmissing_ok)
273-
{
274-
FILE*fp;
275-
pg_crc32crc=0;
276-
char*buf;
277-
size_tlen=0;
278-
279-
INIT_FILE_CRC32(use_crc32c,crc);
280-
281-
/* open file in binary read mode */
282-
fp=fopen(file_path,PG_BINARY_R);
283-
if (fp==NULL)
284-
{
285-
if (errno==ENOENT)
286-
{
287-
if (missing_ok)
288-
{
289-
FIN_FILE_CRC32(use_crc32c,crc);
290-
returncrc;
291-
}
292-
}
293-
294-
elog(ERROR,"Cannot open file \"%s\": %s",
295-
file_path,strerror(errno));
296-
}
297-
298-
/* disable stdio buffering */
299-
setvbuf(fp,NULL,_IONBF,BUFSIZ);
300-
buf=pgut_malloc(STDIO_BUFSIZE);
301-
302-
/* calc CRC of file */
303-
for (;;)
304-
{
305-
if (interrupted)
306-
elog(ERROR,"interrupted during CRC calculation");
307-
308-
len=fread(buf,1,STDIO_BUFSIZE,fp);
309-
310-
if (ferror(fp))
311-
elog(ERROR,"Cannot read \"%s\": %s",file_path,strerror(errno));
312-
313-
/* update CRC */
314-
COMP_FILE_CRC32(use_crc32c,crc,buf,len);
315-
316-
if (feof(fp))
317-
break;
318-
}
319-
320-
FIN_FILE_CRC32(use_crc32c,crc);
321-
fclose(fp);
322-
pg_free(buf);
323-
324-
returncrc;
325-
}
326-
327-
/*
328-
* Read the local file to compute its CRC.
329-
* We cannot make decision about file decompression because
330-
* user may ask to backup already compressed files and we should be
331-
* obvious about it.
332-
*/
333-
pg_crc32
334-
pgFileGetCRCgz(constchar*file_path,booluse_crc32c,boolmissing_ok)
335-
{
336-
gzFilefp;
337-
pg_crc32crc=0;
338-
intlen=0;
339-
interr;
340-
char*buf;
341-
342-
INIT_FILE_CRC32(use_crc32c,crc);
343-
344-
/* open file in binary read mode */
345-
fp=gzopen(file_path,PG_BINARY_R);
346-
if (fp==NULL)
347-
{
348-
if (errno==ENOENT)
349-
{
350-
if (missing_ok)
351-
{
352-
FIN_FILE_CRC32(use_crc32c,crc);
353-
returncrc;
354-
}
355-
}
356-
357-
elog(ERROR,"Cannot open file \"%s\": %s",
358-
file_path,strerror(errno));
359-
}
360-
361-
buf=pgut_malloc(STDIO_BUFSIZE);
362-
363-
/* calc CRC of file */
364-
for (;;)
365-
{
366-
if (interrupted)
367-
elog(ERROR,"interrupted during CRC calculation");
368-
369-
len=gzread(fp,buf,STDIO_BUFSIZE);
370-
371-
if (len <=0)
372-
{
373-
/* we either run into eof or error */
374-
if (gzeof(fp))
375-
break;
376-
else
377-
{
378-
constchar*err_str=NULL;
379-
380-
err_str=gzerror(fp,&err);
381-
elog(ERROR,"Cannot read from compressed file %s",err_str);
382-
}
383-
}
384-
385-
/* update CRC */
386-
COMP_FILE_CRC32(use_crc32c,crc,buf,len);
387-
}
388-
389-
FIN_FILE_CRC32(use_crc32c,crc);
390-
gzclose(fp);
391-
pg_free(buf);
392-
393-
returncrc;
394-
}
395-
396265
void
397266
pgFileFree(void*file)
398267
{
@@ -1812,7 +1681,7 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_
18121681
FIO_BACKUP_HOST);
18131682
file->crc=pgFileGetCRC(database_map_path, true, false);
18141683
file->write_size=file->size;
1815-
file->uncompressed_size=file->read_size;
1684+
file->uncompressed_size=file->size;
18161685

18171686
parray_append(backup_files_list,file);
18181687
}

‎src/merge.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1078,7 +1078,7 @@ merge_files(void *arg)
10781078
tmp_file->hdr_crc=file->hdr_crc;
10791079
}
10801080
else
1081-
tmp_file->uncompressed_size=tmp_file->write_size;
1081+
tmp_file->uncompressed_size=tmp_file->uncompressed_size;
10821082

10831083
/* Copy header metadata from old map into a new one */
10841084
tmp_file->n_headers=file->n_headers;

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp