Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit3fce98e

Browse files
arssherololobus
authored andcommitted
Add --priority keeper option.
Sentinel will promote keeper with higher priority than the current one if thisis possible. In async mode this is a bit non-deterministic because we alwayselect node with highest LSN, and under heavy load prioritized node might neverreport LSN higher than its stronger competitors. However, if nodes are equalthis should happen at some moment. In sync mode, we can just elect any ofsynchronous standbies.Priority can be set during keeper start (--priority) or later with new command'stolonctl set keeperpriority'. The latter allows to update priority withoutrestarting the keeper (and its Postgres instance), which can be used forcontrolled failover.Implementssorintlab#492
1 parent8b42a97 commit3fce98e

File tree

8 files changed

+467
-40
lines changed

8 files changed

+467
-40
lines changed

‎cmd/keeper/cmd/keeper.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,8 @@ type config struct {
9999
uidstring
100100
dataDirstring
101101
debugbool
102+
priorityint
103+
prioritySpecifiedbool// true iff explicitly set by user
102104
pgListenAddressstring
103105
pgAdvertiseAddressstring
104106
pgPortstring
@@ -139,6 +141,7 @@ func init() {
139141
CmdKeeper.PersistentFlags().StringVar(&cfg.pgSUPassword,"pg-su-password","","postgres superuser password. Only one of --pg-su-password or --pg-su-passwordfile must be provided. Must be the same for all keepers.")
140142
CmdKeeper.PersistentFlags().StringVar(&cfg.pgSUPasswordFile,"pg-su-passwordfile","","postgres superuser password file. Only one of --pg-su-password or --pg-su-passwordfile must be provided. Must be the same for all keepers)")
141143
CmdKeeper.PersistentFlags().BoolVar(&cfg.debug,"debug",false,"enable debug logging")
144+
CmdKeeper.PersistentFlags().IntVar(&cfg.priority,"priority",0,"keeper priority, integer. Stolon will promote available keeper with higher priority than current master, if this is possible. Healthy keeper with higher priority will be elected even if current master is online. If not specified, priority is set to "+strconv.Itoa(cluster.DefaultPriority)+" on first keeper invocation; on subsequent invocations, last value (which could be also set with 'stolonctl setkeeperpriority') is reused.")
142145

143146
CmdKeeper.PersistentFlags().BoolVar(&cfg.canBeMaster,"can-be-master",true,"prevent keeper from being elected as master")
144147
CmdKeeper.PersistentFlags().BoolVar(&cfg.canBeSynchronousReplica,"can-be-synchronous-replica",true,"prevent keeper from being chosen as synchronous replica")
@@ -451,6 +454,8 @@ type PostgresKeeper struct {
451454
pgSUUsernamestring
452455
pgSUPasswordstring
453456

457+
priority*int// nil means not specified
458+
454459
sleepInterval time.Duration
455460
requestTimeout time.Duration
456461

@@ -484,6 +489,10 @@ func NewPostgresKeeper(cfg *config, end chan error) (*PostgresKeeper, error) {
484489
returnnil,fmt.Errorf("cannot get absolute datadir path for %q: %v",cfg.dataDir,err)
485490
}
486491

492+
varpriority*int=nil
493+
ifcfg.prioritySpecified {
494+
priority=&cfg.priority
495+
}
487496
p:=&PostgresKeeper{
488497
cfg:cfg,
489498

@@ -503,6 +512,8 @@ func NewPostgresKeeper(cfg *config, end chan error) (*PostgresKeeper, error) {
503512
pgSUUsername:cfg.pgSUUsername,
504513
pgSUPassword:cfg.pgSUPassword,
505514

515+
priority:priority,
516+
506517
sleepInterval:cluster.DefaultSleepInterval,
507518
requestTimeout:cluster.DefaultRequestTimeout,
508519

@@ -578,6 +589,7 @@ func (p *PostgresKeeper) updateKeeperInfo() error {
578589
Maj:maj,
579590
Min:min,
580591
},
592+
Priority:p.priority,
581593
PostgresState:p.getLastPGState(),
582594

583595
CanBeMaster:p.canBeMaster,
@@ -2029,6 +2041,10 @@ func keeper(c *cobra.Command, args []string) {
20292041
}
20302042
}
20312043

2044+
// if --priority wasn't specified explictily, last value is reused, so
2045+
// remember it
2046+
cfg.prioritySpecified=c.Flags().Changed("priority")
2047+
20322048
// Open (and create if needed) the lock file.
20332049
// There is no need to clean up this file since we don't use the file as an actual lock. We get a lock
20342050
// on the file. So the lock get released when our process stops (or log.Fatalfs).

‎cmd/sentinel/cmd/sentinel.go

Lines changed: 73 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,11 @@ func (s *Sentinel) updateKeepersStatus(cd *cluster.ClusterData, keepersInfo clus
243243
}else {
244244
s.CleanKeeperError(keeperUID)
245245
// Update keeper status infos
246+
// If keeper restarted with specified priority, update it
247+
ifki.Priority!=nil&&
248+
k.Status.BootUUID!=ki.BootUUID {
249+
k.Spec.Priority=*ki.Priority
250+
}
246251
k.Status.BootUUID=ki.BootUUID
247252
k.Status.PostgresBinaryVersion.Maj=ki.PostgresBinaryVersion.Maj
248253
k.Status.PostgresBinaryVersion.Min=ki.PostgresBinaryVersion.Min
@@ -699,12 +704,17 @@ func (s *Sentinel) validStandbysByStatus(cd *cluster.ClusterData) (map[string]*c
699704
returngoodStandbys,failedStandbys,convergingStandbys
700705
}
701706

702-
// dbSlice implements sort interface to sort by XLogPos
703-
typedbSlice []*cluster.DB
704-
705-
func (pdbSlice)Len()int {returnlen(p) }
706-
func (pdbSlice)Less(i,jint)bool {returnp[i].Status.XLogPos<p[j].Status.XLogPos }
707-
func (pdbSlice)Swap(i,jint) {p[i],p[j]=p[j],p[i] }
707+
// sort dbs by XLogPos and keeper's priority
708+
funcsortDBs(cd*cluster.ClusterData,dbs []*cluster.DB) {
709+
sort.Slice(dbs,func(i,jint)bool {
710+
ifdbs[i].Status.XLogPos!=dbs[j].Status.XLogPos {
711+
returndbs[i].Status.XLogPos<dbs[j].Status.XLogPos
712+
}
713+
pi:=cd.Keepers[dbs[i].Spec.KeeperUID].Spec.Priority
714+
pj:=cd.Keepers[dbs[j].Spec.KeeperUID].Spec.Priority
715+
returnpi<pj
716+
})
717+
}
708718

709719
func (s*Sentinel)findBestStandbys(cd*cluster.ClusterData,masterDB*cluster.DB) []*cluster.DB {
710720
goodStandbys,_,_:=s.validStandbysByStatus(cd)
@@ -726,7 +736,7 @@ func (s *Sentinel) findBestStandbys(cd *cluster.ClusterData, masterDB *cluster.D
726736
bestDBs=append(bestDBs,db)
727737
}
728738
// Sort by XLogPos
729-
sort.Sort(dbSlice(bestDBs))
739+
sortDBs(cd,bestDBs)
730740
returnbestDBs
731741
}
732742

@@ -773,11 +783,54 @@ func (s *Sentinel) findBestNewMasters(cd *cluster.ClusterData, masterDB *cluster
773783
}
774784

775785
// Sort by XLogPos
776-
sort.Sort(dbSlice(bestNewMasters))
786+
sortDBs(cd,bestNewMasters)
777787
log.Debugf("bestNewMasters: %s",spew.Sdump(bestNewMasters))
778788
returnbestNewMasters
779789
}
780790

791+
// findBestNewMaster returns the DB who can be a new master. This function mostly takes care of
792+
// sync mode; in async case new master is just a first element of findBestNewMasters.
793+
func (s*Sentinel)findBestNewMaster(cd*cluster.ClusterData,curMasterDB*cluster.DB,logErrorsbool)*cluster.DB {
794+
bestNewMasters:=s.findBestNewMasters(cd,curMasterDB)
795+
iflen(bestNewMasters)==0 {
796+
iflogErrors {
797+
log.Errorw("no eligible masters")
798+
}
799+
returnnil
800+
}
801+
802+
// if synchronous replication is enabled, only choose new master in the synchronous replication standbys.
803+
varbestNewMasterDB*cluster.DB=nil
804+
ifcurMasterDB.Spec.SynchronousReplication {
805+
commonSyncStandbys:=util.CommonElements(curMasterDB.Status.SynchronousStandbys,curMasterDB.Spec.SynchronousStandbys)
806+
iflen(commonSyncStandbys)==0 {
807+
iflogErrors {
808+
log.Warnw("cannot choose synchronous standby since there are no common elements between the latest master reported synchronous standbys and the db spec ones","reported",curMasterDB.Status.SynchronousStandbys,"spec",curMasterDB.Spec.SynchronousStandbys)
809+
}
810+
returnnil
811+
}
812+
// In synchronous mode there is no need to choose DB with
813+
// highest LSN; all found dbs must be in sync, so pick the one
814+
// with highest priority.
815+
varnewMasterPriorityint
816+
for_,nm:=rangebestNewMasters {
817+
ifutil.StringInSlice(commonSyncStandbys,nm.UID) {
818+
nmPriority:=cd.Keepers[nm.Spec.KeeperUID].Spec.Priority
819+
if (bestNewMasterDB==nil)|| (nmPriority>newMasterPriority) {
820+
bestNewMasterDB=nm
821+
newMasterPriority=nmPriority
822+
}
823+
}
824+
}
825+
ifbestNewMasterDB==nil&&logErrors {
826+
log.Warnw("cannot choose synchronous standby since there's not match between the possible masters and the usable synchronousStandbys","reported",curMasterDB.Status.SynchronousStandbys,"spec",curMasterDB.Spec.SynchronousStandbys,"common",commonSyncStandbys,"possibleMasters",bestNewMasters)
827+
}
828+
}else {
829+
bestNewMasterDB=bestNewMasters[0]
830+
}
831+
returnbestNewMasterDB
832+
}
833+
781834
func (s*Sentinel)updateCluster(cd*cluster.ClusterData,pis cluster.ProxiesInfo) (*cluster.ClusterData,error) {
782835
// take a cd deepCopy to check that the code isn't changing it (it'll be a bug)
783836
origcd:=cd.DeepCopy()
@@ -1002,37 +1055,20 @@ func (s *Sentinel) updateCluster(cd *cluster.ClusterData, pis cluster.ProxiesInf
10021055
masterOK=false
10031056
}
10041057

1005-
if!masterOK {
1006-
log.Infow("trying to find a new master to replace failed master")
1007-
bestNewMasters:=s.findBestNewMasters(newcd,curMasterDB)
1008-
iflen(bestNewMasters)==0 {
1009-
log.Errorw("no eligible masters")
1058+
bestNewMasterDB:=s.findBestNewMaster(newcd,curMasterDB,!masterOK)
1059+
ifbestNewMasterDB!=nil {
1060+
if!masterOK {
1061+
log.Infow("electing db as the new master","db",bestNewMasterDB.UID,"keeper",bestNewMasterDB.Spec.KeeperUID)
1062+
wantedMasterDBUID=bestNewMasterDB.UID
10101063
}else {
1011-
// if synchronous replication is enabled, only choose new master in the synchronous replication standbys.
1012-
varbestNewMasterDB*cluster.DB
1013-
ifcurMasterDB.Spec.SynchronousReplication {
1014-
commonSyncStandbys:=util.CommonElements(curMasterDB.Status.SynchronousStandbys,curMasterDB.Spec.SynchronousStandbys)
1015-
iflen(commonSyncStandbys)==0 {
1016-
log.Warnw("cannot choose synchronous standby since there are no common elements between the latest master reported synchronous standbys and the db spec ones","reported",curMasterDB.Status.SynchronousStandbys,"spec",curMasterDB.Spec.SynchronousStandbys)
1017-
}else {
1018-
for_,nm:=rangebestNewMasters {
1019-
ifutil.StringInSlice(commonSyncStandbys,nm.UID) {
1020-
bestNewMasterDB=nm
1021-
break
1022-
}
1023-
}
1024-
ifbestNewMasterDB==nil {
1025-
log.Warnw("cannot choose synchronous standby since there's not match between the possible masters and the usable synchronousStandbys","reported",curMasterDB.Status.SynchronousStandbys,"spec",curMasterDB.Spec.SynchronousStandbys,"common",commonSyncStandbys,"possibleMasters",bestNewMasters)
1026-
}
1027-
}
1028-
}else {
1029-
bestNewMasterDB=bestNewMasters[0]
1030-
}
1031-
ifbestNewMasterDB!=nil {
1032-
log.Infow("electing db as the new master","db",bestNewMasterDB.UID,"keeper",bestNewMasterDB.Spec.KeeperUID)
1064+
// Even if current master is ok, we probably still
1065+
// want to change it if there is ready DB with higher
1066+
// keeper priority.
1067+
curMasterPriority:=cd.Keepers[curMasterDB.Spec.KeeperUID].Spec.Priority
1068+
newMasterPriority:=cd.Keepers[bestNewMasterDB.Spec.KeeperUID].Spec.Priority
1069+
ifnewMasterPriority>curMasterPriority {
1070+
log.Infow("electing db as the new master because it has higher priority","db",bestNewMasterDB.UID,"keeper",bestNewMasterDB.Spec.KeeperUID,"currPriority",curMasterPriority,"newPriority",newMasterPriority)
10331071
wantedMasterDBUID=bestNewMasterDB.UID
1034-
}else {
1035-
log.Errorw("no eligible masters")
10361072
}
10371073
}
10381074
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp