Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitbeb4e9b

Browse files
committed
Improve performance of pgarch_readyXlog() with many status files.
Presently, the archive_status directory was scanned for each file toarchive. When there are many status files, say because archive_commandhas been failing for a long time, these directory scans can get veryslow. With this change, the archiver remembers several files to archiveduring each directory scan, speeding things up.To ensure timeline history files are archived as quickly as possible,XLogArchiveNotify() forces the archiver to do a new directory scan assoon as the .ready file for one is created.Nathan Bossart, per a long discussion involving many people. It isnot clear to me exactly who out of all those people reviewed thisparticular patch.Discussion:http://postgr.es/m/CA+TgmobhAbs2yabTuTRkJTq_kkC80-+jw=pfpypdOJ7+gAbQbw@mail.gmail.comDiscussion:http://postgr.es/m/620F3CE1-0255-4D66-9D87-0EADE866985A@amazon.com
1 parent01ec41a commitbeb4e9b

File tree

3 files changed

+182
-26
lines changed

3 files changed

+182
-26
lines changed

‎src/backend/access/transam/xlogarchive.c‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -489,6 +489,20 @@ XLogArchiveNotify(const char *xlog)
489489
return;
490490
}
491491

492+
/*
493+
* Timeline history files are given the highest archival priority to
494+
* lower the chance that a promoted standby will choose a timeline that
495+
* is already in use. However, the archiver ordinarily tries to gather
496+
* multiple files to archive from each scan of the archive_status
497+
* directory, which means that newly created timeline history files
498+
* could be left unarchived for a while. To ensure that the archiver
499+
* picks up timeline history files as soon as possible, we force the
500+
* archiver to scan the archive_status directory the next time it looks
501+
* for a file to archive.
502+
*/
503+
if (IsTLHistoryFileName(xlog))
504+
PgArchForceDirScan();
505+
492506
/* Notify archiver that it's got something to do */
493507
if (IsUnderPostmaster)
494508
PgArchWakeup();

‎src/backend/postmaster/pgarch.c‎

Lines changed: 167 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535

3636
#include"access/xlog.h"
3737
#include"access/xlog_internal.h"
38+
#include"lib/binaryheap.h"
3839
#include"libpq/pqsignal.h"
3940
#include"miscadmin.h"
4041
#include"pgstat.h"
@@ -47,6 +48,7 @@
4748
#include"storage/proc.h"
4849
#include"storage/procsignal.h"
4950
#include"storage/shmem.h"
51+
#include"storage/spin.h"
5052
#include"utils/guc.h"
5153
#include"utils/ps_status.h"
5254

@@ -72,10 +74,23 @@
7274
*/
7375
#defineNUM_ORPHAN_CLEANUP_RETRIES 3
7476

77+
/*
78+
* Maximum number of .ready files to gather per directory scan.
79+
*/
80+
#defineNUM_FILES_PER_DIRECTORY_SCAN 64
81+
7582
/* Shared memory area for archiver process */
7683
typedefstructPgArchData
7784
{
7885
intpgprocno;/* pgprocno of archiver process */
86+
87+
/*
88+
* Forces a directory scan in pgarch_readyXlog(). Protected by
89+
* arch_lck.
90+
*/
91+
boolforce_dir_scan;
92+
93+
slock_tarch_lck;
7994
}PgArchData;
8095

8196

@@ -86,6 +101,22 @@ typedef struct PgArchData
86101
statictime_tlast_sigterm_time=0;
87102
staticPgArchData*PgArch=NULL;
88103

104+
/*
105+
* Stuff for tracking multiple files to archive from each scan of
106+
* archive_status. Minimizing the number of directory scans when there are
107+
* many files to archive can significantly improve archival rate.
108+
*
109+
* arch_heap is a max-heap that is used during the directory scan to track
110+
* the highest-priority files to archive. After the directory scan
111+
* completes, the file names are stored in ascending order of priority in
112+
* arch_files. pgarch_readyXlog() returns files from arch_files until it
113+
* is empty, at which point another directory scan must be performed.
114+
*/
115+
staticbinaryheap*arch_heap=NULL;
116+
staticchararch_filenames[NUM_FILES_PER_DIRECTORY_SCAN][MAX_XFN_CHARS];
117+
staticchar*arch_files[NUM_FILES_PER_DIRECTORY_SCAN];
118+
staticintarch_files_size=0;
119+
89120
/*
90121
* Flags set by interrupt handlers for later service in the main loop.
91122
*/
@@ -103,6 +134,7 @@ static bool pgarch_readyXlog(char *xlog);
103134
staticvoidpgarch_archiveDone(char*xlog);
104135
staticvoidpgarch_die(intcode,Datumarg);
105136
staticvoidHandlePgArchInterrupts(void);
137+
staticintready_file_comparator(Datuma,Datumb,void*arg);
106138

107139
/* Report shared memory space needed by PgArchShmemInit */
108140
Size
@@ -129,6 +161,7 @@ PgArchShmemInit(void)
129161
/* First time through, so initialize */
130162
MemSet(PgArch,0,PgArchShmemSize());
131163
PgArch->pgprocno=INVALID_PGPROCNO;
164+
SpinLockInit(&PgArch->arch_lck);
132165
}
133166
}
134167

@@ -198,6 +231,10 @@ PgArchiverMain(void)
198231
*/
199232
PgArch->pgprocno=MyProc->pgprocno;
200233

234+
/* Initialize our max-heap for prioritizing files to archive. */
235+
arch_heap=binaryheap_allocate(NUM_FILES_PER_DIRECTORY_SCAN,
236+
ready_file_comparator,NULL);
237+
201238
pgarch_MainLoop();
202239

203240
proc_exit(0);
@@ -325,6 +362,9 @@ pgarch_ArchiverCopyLoop(void)
325362
{
326363
charxlog[MAX_XFN_CHARS+1];
327364

365+
/* force directory scan in the first call to pgarch_readyXlog() */
366+
arch_files_size=0;
367+
328368
/*
329369
* loop through all xlogs with archive_status of .ready and archive
330370
* them...mostly we expect this to be a single file, though it is possible
@@ -600,26 +640,62 @@ pgarch_archiveXlog(char *xlog)
600640
staticbool
601641
pgarch_readyXlog(char*xlog)
602642
{
603-
/*
604-
* open xlog status directory and read through list of xlogs that have the
605-
* .ready suffix, looking for earliest file. It is possible to optimise
606-
* this code, though only a single file is expected on the vast majority
607-
* of calls, so....
608-
*/
609643
charXLogArchiveStatusDir[MAXPGPATH];
610644
DIR*rldir;
611645
structdirent*rlde;
612-
boolfound= false;
613-
boolhistoryFound= false;
646+
boolforce_dir_scan;
614647

648+
/*
649+
* If a directory scan was requested, clear the stored file names and
650+
* proceed.
651+
*/
652+
SpinLockAcquire(&PgArch->arch_lck);
653+
force_dir_scan=PgArch->force_dir_scan;
654+
PgArch->force_dir_scan= false;
655+
SpinLockRelease(&PgArch->arch_lck);
656+
657+
if (force_dir_scan)
658+
arch_files_size=0;
659+
660+
/*
661+
* If we still have stored file names from the previous directory scan,
662+
* try to return one of those. We check to make sure the status file
663+
* is still present, as the archive_command for a previous file may
664+
* have already marked it done.
665+
*/
666+
while (arch_files_size>0)
667+
{
668+
structstatst;
669+
charstatus_file[MAXPGPATH];
670+
char*arch_file;
671+
672+
arch_files_size--;
673+
arch_file=arch_files[arch_files_size];
674+
StatusFilePath(status_file,arch_file,".ready");
675+
676+
if (stat(status_file,&st)==0)
677+
{
678+
strcpy(xlog,arch_file);
679+
return true;
680+
}
681+
elseif (errno!=ENOENT)
682+
ereport(ERROR,
683+
(errcode_for_file_access(),
684+
errmsg("could not stat file \"%s\": %m",status_file)));
685+
}
686+
687+
/*
688+
* Open the archive status directory and read through the list of files
689+
* with the .ready suffix, looking for the earliest files.
690+
*/
615691
snprintf(XLogArchiveStatusDir,MAXPGPATH,XLOGDIR"/archive_status");
616692
rldir=AllocateDir(XLogArchiveStatusDir);
617693

618694
while ((rlde=ReadDir(rldir,XLogArchiveStatusDir))!=NULL)
619695
{
620696
intbasenamelen= (int)strlen(rlde->d_name)-6;
621697
charbasename[MAX_XFN_CHARS+1];
622-
boolishistory;
698+
char*arch_file;
623699

624700
/* Ignore entries with unexpected number of characters */
625701
if (basenamelen<MIN_XFN_CHARS||
@@ -638,32 +714,97 @@ pgarch_readyXlog(char *xlog)
638714
memcpy(basename,rlde->d_name,basenamelen);
639715
basename[basenamelen]='\0';
640716

641-
/* Is this a history file? */
642-
ishistory=IsTLHistoryFileName(basename);
643-
644717
/*
645-
* Consume the file to archive. History files have the highest
646-
* priority. If this is the first file or the first history file
647-
* ever, copy it. In the presence of a history file already chosen as
648-
* target, ignore all other files except history files which have been
649-
* generated for an older timeline than what is already chosen as
650-
* target to archive.
718+
* Store the file in our max-heap if it has a high enough priority.
651719
*/
652-
if (!found|| (ishistory&& !historyFound))
720+
if (arch_heap->bh_size<NUM_FILES_PER_DIRECTORY_SCAN)
653721
{
654-
strcpy(xlog,basename);
655-
found= true;
656-
historyFound=ishistory;
722+
/* If the heap isn't full yet, quickly add it. */
723+
arch_file=arch_filenames[arch_heap->bh_size];
724+
strcpy(arch_file,basename);
725+
binaryheap_add_unordered(arch_heap,CStringGetDatum(arch_file));
726+
727+
/* If we just filled the heap, make it a valid one. */
728+
if (arch_heap->bh_size==NUM_FILES_PER_DIRECTORY_SCAN)
729+
binaryheap_build(arch_heap);
657730
}
658-
elseif (ishistory|| !historyFound)
731+
elseif (ready_file_comparator(binaryheap_first(arch_heap),
732+
CStringGetDatum(basename),NULL)>0)
659733
{
660-
if (strcmp(basename,xlog)<0)
661-
strcpy(xlog,basename);
734+
/*
735+
* Remove the lowest priority file and add the current one to
736+
* the heap.
737+
*/
738+
arch_file=DatumGetCString(binaryheap_remove_first(arch_heap));
739+
strcpy(arch_file,basename);
740+
binaryheap_add(arch_heap,CStringGetDatum(arch_file));
662741
}
663742
}
664743
FreeDir(rldir);
665744

666-
returnfound;
745+
/* If no files were found, simply return. */
746+
if (arch_heap->bh_size==0)
747+
return false;
748+
749+
/*
750+
* If we didn't fill the heap, we didn't make it a valid one. Do that
751+
* now.
752+
*/
753+
if (arch_heap->bh_size<NUM_FILES_PER_DIRECTORY_SCAN)
754+
binaryheap_build(arch_heap);
755+
756+
/*
757+
* Fill arch_files array with the files to archive in ascending order
758+
* of priority.
759+
*/
760+
arch_files_size=arch_heap->bh_size;
761+
for (inti=0;i<arch_files_size;i++)
762+
arch_files[i]=DatumGetCString(binaryheap_remove_first(arch_heap));
763+
764+
/* Return the highest priority file. */
765+
arch_files_size--;
766+
strcpy(xlog,arch_files[arch_files_size]);
767+
768+
return true;
769+
}
770+
771+
/*
772+
* ready_file_comparator
773+
*
774+
* Compares the archival priority of the given files to archive. If "a"
775+
* has a higher priority than "b", a negative value will be returned. If
776+
* "b" has a higher priority than "a", a positive value will be returned.
777+
* If "a" and "b" have equivalent values, 0 will be returned.
778+
*/
779+
staticint
780+
ready_file_comparator(Datuma,Datumb,void*arg)
781+
{
782+
char*a_str=DatumGetCString(a);
783+
char*b_str=DatumGetCString(b);
784+
boola_history=IsTLHistoryFileName(a_str);
785+
boolb_history=IsTLHistoryFileName(b_str);
786+
787+
/* Timeline history files always have the highest priority. */
788+
if (a_history!=b_history)
789+
returna_history ?-1 :1;
790+
791+
/* Priority is given to older files. */
792+
returnstrcmp(a_str,b_str);
793+
}
794+
795+
/*
796+
* PgArchForceDirScan
797+
*
798+
* When called, the next call to pgarch_readyXlog() will perform a
799+
* directory scan. This is useful for ensuring that important files such
800+
* as timeline history files are archived as quickly as possible.
801+
*/
802+
void
803+
PgArchForceDirScan(void)
804+
{
805+
SpinLockAcquire(&PgArch->arch_lck);
806+
PgArch->force_dir_scan= true;
807+
SpinLockRelease(&PgArch->arch_lck);
667808
}
668809

669810
/*

‎src/include/postmaster/pgarch.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,5 +31,6 @@ extern void PgArchShmemInit(void);
3131
externboolPgArchCanRestart(void);
3232
externvoidPgArchiverMain(void)pg_attribute_noreturn();
3333
externvoidPgArchWakeup(void);
34+
externvoidPgArchForceDirScan(void);
3435

3536
#endif/* _PGARCH_H */

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp