@@ -156,6 +156,9 @@ func (p *provisionerDaemon) connect(ctx context.Context) {
156
156
157
157
// Locks a job in the database, and runs it!
158
158
func (p * provisionerDaemon )acquireJob (ctx context.Context ) {
159
+ if p .isClosed () {
160
+ return
161
+ }
159
162
p .acquiredJobMutex .Lock ()
160
163
defer p .acquiredJobMutex .Unlock ()
161
164
if p .isRunningJob () {
@@ -174,16 +177,14 @@ func (p *provisionerDaemon) acquireJob(ctx context.Context) {
174
177
p .opts .Logger .Warn (context .Background (),"acquire job" ,slog .Error (err ))
175
178
return
176
179
}
177
- if p .isClosed () {
178
- return
179
- }
180
180
if p .acquiredJob .JobId == "" {
181
181
p .opts .Logger .Debug (context .Background (),"no jobs available" )
182
182
return
183
183
}
184
184
ctx ,p .acquiredJobCancel = context .WithCancel (ctx )
185
185
p .acquiredJobCancelled .Store (false )
186
186
p .acquiredJobRunning .Store (true )
187
+
187
188
p .acquiredJobDone = make (chan struct {})
188
189
189
190
p .opts .Logger .Info (context .Background (),"acquired job" ,
@@ -234,8 +235,6 @@ func (p *provisionerDaemon) runJob(ctx context.Context) {
234
235
return
235
236
}
236
237
p .opts .Logger .Debug (ctx ,"cleaned up work directory" )
237
- p .acquiredJobMutex .Lock ()
238
- defer p .acquiredJobMutex .Unlock ()
239
238
p .acquiredJobRunning .Store (false )
240
239
close (p .acquiredJobDone )
241
240
}()
@@ -510,11 +509,22 @@ func (p *provisionerDaemon) Close() error {
510
509
func (p * provisionerDaemon )closeWithError (err error )error {
511
510
p .closeMutex .Lock ()
512
511
defer p .closeMutex .Unlock ()
512
+
513
513
if p .isClosed () {
514
514
return p .closeError
515
515
}
516
516
517
517
if p .isRunningJob () {
518
+
519
+ // We also need the 'acquire job' mutex here,
520
+ // so that a new `p.acquiredJobDone` channel isn't created
521
+ // while we're waiting on the mutex.
522
+
523
+ // Note the mutex order - it's important that we always use the same order of acquisition
524
+ // to avoid deadlocks
525
+ p .acquiredJobMutex .Lock ()
526
+ defer p .acquiredJobMutex .Unlock ()
527
+
518
528
errMsg := "provisioner daemon was shutdown gracefully"
519
529
if err != nil {
520
530
errMsg = err .Error ()