@@ -129,6 +129,10 @@ func newPGCoordInternal(
129129// signals when first heartbeat has been sent, so it's safe to start binding.
130130fHB := make (chan struct {})
131131
132+ // we need to arrange for the querier to stop _after_ the tunneler and binder, since we delete
133+ // the coordinator when the querier stops (via the heartbeats). If the tunneler and binder are
134+ // still running, they could run afoul of foreign key constraints.
135+ querierCtx ,querierCancel := context .WithCancel (dbauthz .As (context .Background (),pgCoordSubject ))
132136c := & pgCoord {
133137ctx :ctx ,
134138cancel :cancel ,
@@ -142,9 +146,17 @@ func newPGCoordInternal(
142146tunneler :newTunneler (ctx ,logger ,id ,store ,sCh ,fHB ),
143147tunnelerCh :sCh ,
144148id :id ,
145- querier :newQuerier (ctx ,logger ,id ,ps ,store ,id ,cCh ,ccCh ,numQuerierWorkers ,fHB ),
149+ querier :newQuerier (querierCtx ,logger ,id ,ps ,store ,id ,cCh ,ccCh ,numQuerierWorkers ,fHB ),
146150closed :make (chan struct {}),
147151}
152+ go func () {
153+ // when the main context is canceled, or the coordinator closed, the binder and tunneler
154+ // always eventually stop. Once they stop it's safe to cancel the querier context, which
155+ // has the effect of deleting the coordinator from the database and ceasing heartbeats.
156+ c .binder .workerWG .Wait ()
157+ c .tunneler .workerWG .Wait ()
158+ querierCancel ()
159+ }()
148160logger .Info (ctx ,"starting coordinator" )
149161return c ,nil
150162}
@@ -255,6 +267,8 @@ type tunneler struct {
255267mu sync.Mutex
256268latest map [uuid.UUID ]map [uuid.UUID ]tunnel
257269workQ * workQ [tKey ]
270+
271+ workerWG sync.WaitGroup
258272}
259273
260274func newTunneler (ctx context.Context ,
@@ -274,6 +288,9 @@ func newTunneler(ctx context.Context,
274288workQ :newWorkQ [tKey ](ctx ),
275289}
276290go s .handle ()
291+ // add to the waitgroup immediately to avoid any races waiting for it before
292+ // the workers start.
293+ s .workerWG .Add (numTunnelerWorkers )
277294go func () {
278295<- startWorkers
279296for i := 0 ;i < numTunnelerWorkers ;i ++ {
@@ -297,6 +314,7 @@ func (t *tunneler) handle() {
297314}
298315
299316func (t * tunneler )worker () {
317+ defer t .workerWG .Done ()
300318eb := backoff .NewExponentialBackOff ()
301319eb .MaxElapsedTime = 0 // retry indefinitely
302320eb .MaxInterval = dbMaxBackoff
@@ -435,6 +453,8 @@ type binder struct {
435453mu sync.Mutex
436454latest map [bKey ]binding
437455workQ * workQ [bKey ]
456+
457+ workerWG sync.WaitGroup
438458}
439459
440460func newBinder (ctx context.Context ,
@@ -454,6 +474,9 @@ func newBinder(ctx context.Context,
454474workQ :newWorkQ [bKey ](ctx ),
455475}
456476go b .handleBindings ()
477+ // add to the waitgroup immediately to avoid any races waiting for it before
478+ // the workers start.
479+ b .workerWG .Add (numBinderWorkers )
457480go func () {
458481<- startWorkers
459482for i := 0 ;i < numBinderWorkers ;i ++ {
@@ -477,6 +500,7 @@ func (b *binder) handleBindings() {
477500}
478501
479502func (b * binder )worker () {
503+ defer b .workerWG .Done ()
480504eb := backoff .NewExponentialBackOff ()
481505eb .MaxElapsedTime = 0 // retry indefinitely
482506eb .MaxInterval = dbMaxBackoff