Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitd4f9aa3

Browse files
committed
[Issue#449] fix stream backups, PAGE backup, WAL parsing, delete and validate
1 parentc7a751d commitd4f9aa3

File tree

8 files changed

+223
-46
lines changed

8 files changed

+223
-46
lines changed

‎src/backup.c‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1238,7 +1238,7 @@ wait_wal_lsn(const char *wal_segment_dir, XLogRecPtr target_lsn, bool is_start_l
12381238
{
12391239
XLogSegNotargetSegNo;
12401240
charwal_segment_path[MAXPGPATH],
1241-
wal_segment_subdir[MAXPGPATH],
1241+
wal_segment_subdir[MAXPGPATH],// used only to check file existence, not actual parsing
12421242
wal_segment[MAXFNAMELEN];
12431243
boolfile_exists= false;
12441244
uint32try_count=0,
@@ -1257,10 +1257,10 @@ wait_wal_lsn(const char *wal_segment_dir, XLogRecPtr target_lsn, bool is_start_l
12571257
instance_config.xlog_seg_size);
12581258

12591259
// obtain WAL archive subdir for ARCHIVE backup
1260-
if (!in_stream_dir)
1261-
get_archive_subdir(wal_segment_subdir,wal_segment_dir,wal_segment,SEGMENT);
1262-
else
1260+
if (in_stream_dir)
12631261
strcpy(wal_segment_subdir,wal_segment_dir);
1262+
else
1263+
get_archive_subdir(wal_segment_subdir,wal_segment_dir,wal_segment,SEGMENT);
12641264

12651265
join_path_components(wal_segment_path,wal_segment_subdir,wal_segment);
12661266
/*
@@ -1319,7 +1319,7 @@ wait_wal_lsn(const char *wal_segment_dir, XLogRecPtr target_lsn, bool is_start_l
13191319
*/
13201320
if (!XRecOffIsNull(target_lsn)&&
13211321
wal_contains_lsn(wal_segment_dir,target_lsn,tli,
1322-
instance_config.xlog_seg_size))
1322+
instance_config.xlog_seg_size, !in_stream_dir))
13231323
/* Target LSN was found */
13241324
{
13251325
elog(LOG,"Found LSN: %X/%X", (uint32) (target_lsn >>32), (uint32)target_lsn);
@@ -1915,7 +1915,8 @@ pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startb
19151915
if (!read_recovery_info(xlog_path,backup->tli,
19161916
instance_config.xlog_seg_size,
19171917
backup->start_lsn,backup->stop_lsn,
1918-
&backup->recovery_time))
1918+
&backup->recovery_time,
1919+
!backup->stream))
19191920
{
19201921
elog(LOG,"Failed to find Recovery Time in WAL, forced to trust current_timestamp");
19211922
backup->recovery_time=stop_backup_result.invocation_time;

‎src/catalog.c‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1628,7 +1628,9 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance)
16281628
elseif (strcmp(suffix,"gz")!=0)
16291629
{
16301630
elog(WARNING,"unexpected WAL file name \"%s\"",file->name);
1631-
// TODO: free file
1631+
pgFileFree(file);
1632+
parray_remove(xlog_files_list,i);
1633+
i--;
16321634
continue;
16331635
}
16341636
}
@@ -1735,7 +1737,9 @@ catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance)
17351737
else
17361738
{
17371739
elog(WARNING,"unexpected WAL file name \"%s\"",file->name);
1738-
// TODO: free file
1740+
pgFileFree(file);
1741+
parray_remove(xlog_files_list,i);
1742+
i--;
17391743
}
17401744
}
17411745

‎src/delete.c‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -970,9 +970,12 @@ delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timeli
970970
}
971971

972972
wal_deleted= true;
973-
}
974973

975-
//TODO: cleanup
974+
/* cleanup */
975+
pgXlogFileFree(wal_file);
976+
parray_remove(tlinfo->xlog_filelist,i);
977+
i--;
978+
}
976979
}
977980

978981
/* Remove empty subdirectories */
@@ -987,10 +990,12 @@ delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timeli
987990
join_path_components(fullpath,instanceState->instance_wal_subdir_path,file->name);
988991

989992
if (dir_is_empty(fullpath,FIO_LOCAL_HOST))
993+
{
990994
pgFileDelete(file->mode,fullpath);
991-
992-
pgFileFree(file);
993-
// TODO: maintain instanceState->wal_archive_subdirs
995+
pgFileFree(file);
996+
parray_remove(instanceState->wal_archive_subdirs,i);
997+
i--;
998+
}
994999
}
9951000
}
9961001

‎src/dir.c‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -405,6 +405,19 @@ pgFileFree(void *file)
405405
pfree(file);
406406
}
407407

408+
void
409+
pgXlogFileFree(void*xlogfile)
410+
{
411+
xlogFile*xlogfile_ptr;
412+
413+
if (xlogfile==NULL)
414+
return;
415+
416+
xlogfile_ptr= (xlogFile*)xlogfile;
417+
418+
pg_free(xlogfile_ptr);
419+
}
420+
408421
/* Compare two pgFile with their path in ascending order of ASCII code. */
409422
int
410423
pgFileMapComparePath(constvoid*f1,constvoid*f2)

‎src/parsexlog.c‎

Lines changed: 107 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ typedef struct XLogReaderData
115115
gzFilegz_xlogfile;
116116
chargz_xlogpath[MAXPGPATH];
117117
#endif
118-
boolis_stream;
118+
boolhonor_subdirs;
119119
}XLogReaderData;
120120

121121
/* Function to process a WAL record */
@@ -174,7 +174,7 @@ static bool RunXLogThreads(const char *archivedir,
174174
xlog_record_functionprocess_record,
175175
XLogRecTarget*last_rec,
176176
boolinclusive_endpoint,
177-
boolis_stream);
177+
boolhonor_subdirs);
178178
//static XLogReaderState *InitXLogThreadRead(xlog_thread_arg *arg);
179179
staticboolSwitchThreadToNextWal(XLogReaderState*xlogreader,
180180
xlog_thread_arg*arg);
@@ -256,7 +256,7 @@ extractPageMap(const char *archivedir, uint32 wal_seg_size,
256256
extract_isok=RunXLogThreads(archivedir,0,InvalidTransactionId,
257257
InvalidXLogRecPtr,end_tli,wal_seg_size,
258258
startpoint,endpoint, false,extractPageInfo,
259-
NULL, true,false);
259+
NULL, true,true);
260260
else
261261
{
262262
/* We have to process WAL located on several different xlog intervals,
@@ -350,7 +350,7 @@ extractPageMap(const char *archivedir, uint32 wal_seg_size,
350350
extract_isok=RunXLogThreads(archivedir,0,InvalidTransactionId,
351351
InvalidXLogRecPtr,tmp_interval->tli,wal_seg_size,
352352
tmp_interval->begin_lsn,tmp_interval->end_lsn,
353-
false,extractPageInfo,NULL,inclusive_endpoint,false);
353+
false,extractPageInfo,NULL,inclusive_endpoint,true);
354354
if (!extract_isok)
355355
break;
356356

@@ -379,7 +379,7 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup,
379379
got_endpoint=RunXLogThreads(archivedir,0,InvalidTransactionId,
380380
InvalidXLogRecPtr,tli,xlog_seg_size,
381381
backup->start_lsn,backup->stop_lsn,
382-
false,NULL,NULL, true,backup->stream);
382+
false,NULL,NULL, true,!backup->stream);
383383

384384
if (!got_endpoint)
385385
{
@@ -452,6 +452,7 @@ validate_wal(pgBackup *backup, const char *archivedir,
452452
elog(WARNING,"Backup %s WAL segments are corrupted",backup_id);
453453
return;
454454
}
455+
455456
/*
456457
* If recovery target is provided check that we can restore backup to a
457458
* recovery target time or xid.
@@ -493,7 +494,7 @@ validate_wal(pgBackup *backup, const char *archivedir,
493494
RunXLogThreads(archivedir,target_time,target_xid,target_lsn,
494495
tli,wal_seg_size,backup->stop_lsn,
495496
InvalidXLogRecPtr, true,validateXLogRecord,&last_rec, true,
496-
backup->stream);
497+
true);
497498
if (last_rec.rec_time>0)
498499
time2iso(last_timestamp,lengthof(last_timestamp),
499500
timestamptz_to_time_t(last_rec.rec_time), false);
@@ -535,7 +536,7 @@ validate_wal(pgBackup *backup, const char *archivedir,
535536
bool
536537
read_recovery_info(constchar*archivedir,TimeLineIDtli,uint32wal_seg_size,
537538
XLogRecPtrstart_lsn,XLogRecPtrstop_lsn,
538-
time_t*recovery_time)
539+
time_t*recovery_time,boolhonor_subdirs)
539540
{
540541
XLogRecPtrstartpoint=stop_lsn;
541542
XLogReaderState*xlogreader;
@@ -552,6 +553,7 @@ read_recovery_info(const char *archivedir, TimeLineID tli, uint32 wal_seg_size,
552553

553554
xlogreader=InitXLogPageRead(&reader_data,archivedir,tli,wal_seg_size,
554555
false, true, true);
556+
reader_data.honor_subdirs=honor_subdirs;
555557

556558
/* Read records from stop_lsn down to start_lsn */
557559
do
@@ -611,7 +613,7 @@ read_recovery_info(const char *archivedir, TimeLineID tli, uint32 wal_seg_size,
611613
*/
612614
bool
613615
wal_contains_lsn(constchar*archivedir,XLogRecPtrtarget_lsn,
614-
TimeLineIDtarget_tli,uint32wal_seg_size)
616+
TimeLineIDtarget_tli,uint32wal_seg_size,boolhonor_subdirs)
615617
{
616618
XLogReaderState*xlogreader;
617619
XLogReaderDatareader_data;
@@ -629,6 +631,7 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn,
629631
elog(ERROR,"Out of memory");
630632

631633
xlogreader->system_identifier=instance_config.system_identifier;
634+
reader_data.honor_subdirs=honor_subdirs;
632635

633636
#ifPG_VERSION_NUM >=130000
634637
if (XLogRecPtrIsInvalid(target_lsn))
@@ -1015,38 +1018,114 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
10151018
/* Try to switch to the next WAL segment */
10161019
if (!reader_data->xlogexists)
10171020
{
1018-
charxlogfname[MAXFNAMELEN];
1019-
charpartial_file[MAXPGPATH];
1021+
boolcompressed= false;
1022+
charxlogfname[MAXFNAMELEN];
1023+
//charpartial_file[MAXPGPATH];
1024+
charfullpath[MAXPGPATH];
1025+
charfullpath_gz[MAXPGPATH];
1026+
charfullpath_partial_gz[MAXPGPATH];
10201027

10211028
GetXLogFileName(xlogfname,reader_data->tli,reader_data->xlogsegno,wal_seg_size);
10221029

1023-
if (reader_data->is_stream)
1024-
join_path_components(reader_data->xlogpath,wal_archivedir,xlogfname);
10251030
/* obtain WAL archive subdir for ARCHIVE backup */
1026-
else
1031+
if (reader_data->honor_subdirs)
10271032
{
10281033
chararchive_subdir[MAXPGPATH];
10291034
get_archive_subdir(archive_subdir,wal_archivedir,xlogfname,SEGMENT);
1030-
join_path_components(reader_data->xlogpath,archive_subdir,xlogfname);
1035+
1036+
/* check existence of wal_dir/xlogid/segment.gz file ... */
1037+
snprintf(fullpath_gz,MAXPGPATH,"%s/%s.gz",archive_subdir,xlogfname);
1038+
1039+
//TODO: rewrite it to something less ugly
1040+
#ifdefHAVE_LIBZ
1041+
if (fileExists(fullpath_gz,FIO_LOCAL_HOST))
1042+
{
1043+
snprintf(reader_data->xlogpath,MAXPGPATH,"%s/%s",archive_subdir,xlogfname);
1044+
snprintf(reader_data->gz_xlogpath,MAXPGPATH,"%s",fullpath_gz);
1045+
compressed= true;
1046+
gotofile_found;
1047+
}
1048+
1049+
/* ... failing that check existence of wal_dir/xlogid/segment.partial.gz ... */
1050+
snprintf(fullpath_partial_gz,MAXPGPATH,"%s/%s.partial.gz",archive_subdir,xlogfname);
1051+
if (fileExists(fullpath_partial_gz,FIO_LOCAL_HOST))
1052+
{
1053+
snprintf(reader_data->xlogpath,MAXPGPATH,"%s/%s.partial",archive_subdir,xlogfname);
1054+
snprintf(reader_data->gz_xlogpath,MAXPGPATH,"%s",fullpath_partial_gz);
1055+
compressed= true;
1056+
gotofile_found;
1057+
}
1058+
#endif
1059+
/* ... failing that check existence of wal_dir/xlogid/segment ... */
1060+
snprintf(fullpath,MAXPGPATH,"%s/%s",archive_subdir,xlogfname);
1061+
if (fileExists(fullpath,FIO_LOCAL_HOST))
1062+
{
1063+
snprintf(reader_data->xlogpath,MAXPGPATH,"%s",fullpath);
1064+
gotofile_found;
1065+
}
1066+
1067+
gotoarchive_dir;
10311068
}
1069+
/* use directory as-is */
1070+
else
1071+
{
1072+
archive_dir:
1073+
#ifdefHAVE_LIBZ
1074+
/* ... failing that check existence of wal_dir/segment.gz ... */
1075+
snprintf(fullpath_gz,MAXPGPATH,"%s/%s.gz",wal_archivedir,xlogfname);
1076+
if (fileExists(fullpath_gz,FIO_LOCAL_HOST))
1077+
{
1078+
snprintf(reader_data->gz_xlogpath,MAXPGPATH,"%s",fullpath_gz);
1079+
snprintf(reader_data->xlogpath,MAXPGPATH,"%s/%s",wal_archivedir,xlogfname);
1080+
compressed= true;
10321081

1033-
snprintf(reader_data->gz_xlogpath,MAXPGPATH,"%s.gz",reader_data->xlogpath);
1082+
gotofile_found;
1083+
}
1084+
1085+
/* ... failing that check existence of wal_dir/segment.partial.gz ... */
1086+
snprintf(fullpath_partial_gz,MAXPGPATH,"%s/%s.partial.gz",wal_archivedir,xlogfname);
1087+
if (fileExists(wal_archivedir,FIO_LOCAL_HOST))
1088+
{
1089+
snprintf(reader_data->xlogpath,MAXPGPATH,"%s/%s.partial",wal_archivedir,xlogfname);
1090+
snprintf(reader_data->gz_xlogpath,MAXPGPATH,"%s",fullpath_partial_gz);
1091+
compressed= true;
1092+
gotofile_found;
1093+
}
1094+
#endif
1095+
/* ... failing that check existence of wal_dir/segment ... */
1096+
snprintf(fullpath,MAXPGPATH,"%s/%s",wal_archivedir,xlogfname);
1097+
if (fileExists(fullpath,FIO_LOCAL_HOST))
1098+
{
1099+
snprintf(reader_data->xlogpath,MAXPGPATH,"%s",fullpath);
1100+
gotofile_found;
1101+
}
1102+
}
1103+
1104+
file_found:
1105+
canonicalize_path(reader_data->xlogpath);
1106+
1107+
#ifdefHAVE_LIBZ
1108+
if (compressed)
1109+
canonicalize_path(reader_data->gz_xlogpath);
1110+
#endif
1111+
1112+
//snprintf(reader_data->gz_xlogpath, MAXPGPATH, "%s.gz", reader_data->xlogpath);
10341113

10351114
/* We fall back to using .partial segment in case if we are running
10361115
* multi-timeline incremental backup right after standby promotion.
10371116
* TODO: it should be explicitly enabled.
10381117
*/
1039-
snprintf(partial_file,MAXPGPATH,"%s.partial",reader_data->xlogpath);
1118+
//snprintf(partial_file, MAXPGPATH, "%s.partial", reader_data->xlogpath);
10401119

10411120
/* If segment do not exists, but the same
10421121
* segment with '.partial' suffix does, use it instead */
1043-
if (!fileExists(reader_data->xlogpath,FIO_LOCAL_HOST)&&
1044-
fileExists(partial_file,FIO_LOCAL_HOST))
1045-
{
1046-
snprintf(reader_data->xlogpath,MAXPGPATH,"%s",partial_file);
1047-
}
1122+
//if (!fileExists(reader_data->xlogpath, FIO_LOCAL_HOST) &&
1123+
//fileExists(partial_file, FIO_LOCAL_HOST))
1124+
//{
1125+
//snprintf(reader_data->xlogpath, MAXPGPATH, "%s", partial_file);
1126+
//}
10481127

1049-
if (fileExists(reader_data->xlogpath,FIO_LOCAL_HOST))
1128+
if (!compressed)
10501129
{
10511130
elog(LOG,"Thread [%d]: Opening WAL segment \"%s\"",
10521131
reader_data->thread_num,reader_data->xlogpath);
@@ -1065,7 +1144,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
10651144
}
10661145
#ifdefHAVE_LIBZ
10671146
/* Try to open compressed WAL segment */
1068-
elseif (fileExists(reader_data->gz_xlogpath,FIO_LOCAL_HOST))
1147+
else
10691148
{
10701149
elog(LOG,"Thread [%d]: Opening compressed WAL segment \"%s\"",
10711150
reader_data->thread_num,reader_data->gz_xlogpath);
@@ -1203,7 +1282,7 @@ RunXLogThreads(const char *archivedir, time_t target_time,
12031282
TransactionIdtarget_xid,XLogRecPtrtarget_lsn,TimeLineIDtli,
12041283
uint32segment_size,XLogRecPtrstartpoint,XLogRecPtrendpoint,
12051284
boolconsistent_read,xlog_record_functionprocess_record,
1206-
XLogRecTarget*last_rec,boolinclusive_endpoint,boolis_stream)
1285+
XLogRecTarget*last_rec,boolinclusive_endpoint,boolhonor_subdirs)
12071286
{
12081287
pthread_t*threads;
12091288
xlog_thread_arg*thread_args;
@@ -1267,7 +1346,7 @@ RunXLogThreads(const char *archivedir, time_t target_time,
12671346
consistent_read, false);
12681347
arg->reader_data.xlogsegno=segno_next;
12691348
arg->reader_data.thread_num=i+1;
1270-
arg->reader_data.is_stream=is_stream;
1349+
arg->reader_data.honor_subdirs=honor_subdirs;
12711350
arg->process_record=process_record;
12721351
arg->startpoint=startpoint;
12731352
arg->endpoint=endpoint;
@@ -1495,7 +1574,7 @@ XLogThreadWorker(void *arg)
14951574
reader_data->thread_num,
14961575
(uint32) (errptr >>32), (uint32) (errptr));
14971576

1498-
/*In we failed to read record located at endpoint position,
1577+
/*If we failed to read record located at endpoint position,
14991578
* and endpoint is not inclusive, do not consider this as an error.
15001579
*/
15011580
if (!thread_arg->inclusive_endpoint&&
@@ -1522,6 +1601,7 @@ XLogThreadWorker(void *arg)
15221601

15231602
if (thread_arg->process_record)
15241603
thread_arg->process_record(xlogreader,reader_data,&stop_reading);
1604+
15251605
if (stop_reading)
15261606
{
15271607
thread_arg->got_target= true;
@@ -1928,7 +2008,7 @@ bool validate_wal_segment(TimeLineID tli, XLogSegNo segno, const char *prefetch_
19282008

19292009
rc=RunXLogThreads(prefetch_dir,0,InvalidTransactionId,
19302010
InvalidXLogRecPtr,tli,wal_seg_size,
1931-
startpoint,endpoint, false,NULL,NULL, true,true);
2011+
startpoint,endpoint, false,NULL,NULL, true,false);
19322012

19332013
num_threads=tmp_num_threads;
19342014

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp