@@ -156,6 +156,9 @@ func (p *provisionerDaemon) connect(ctx context.Context) {
156156
157157// Locks a job in the database, and runs it!
158158func (p * provisionerDaemon )acquireJob (ctx context.Context ) {
159+ if p .isClosed () {
160+ return
161+ }
159162p .acquiredJobMutex .Lock ()
160163defer p .acquiredJobMutex .Unlock ()
161164if p .isRunningJob () {
@@ -174,16 +177,14 @@ func (p *provisionerDaemon) acquireJob(ctx context.Context) {
174177p .opts .Logger .Warn (context .Background (),"acquire job" ,slog .Error (err ))
175178return
176179}
177- if p .isClosed () {
178- return
179- }
180180if p .acquiredJob .JobId == "" {
181181p .opts .Logger .Debug (context .Background (),"no jobs available" )
182182return
183183}
184184ctx ,p .acquiredJobCancel = context .WithCancel (ctx )
185185p .acquiredJobCancelled .Store (false )
186186p .acquiredJobRunning .Store (true )
187+
187188p .acquiredJobDone = make (chan struct {})
188189
189190p .opts .Logger .Info (context .Background (),"acquired job" ,
@@ -234,8 +235,6 @@ func (p *provisionerDaemon) runJob(ctx context.Context) {
234235return
235236}
236237p .opts .Logger .Debug (ctx ,"cleaned up work directory" )
237- p .acquiredJobMutex .Lock ()
238- defer p .acquiredJobMutex .Unlock ()
239238p .acquiredJobRunning .Store (false )
240239close (p .acquiredJobDone )
241240}()
@@ -510,11 +509,22 @@ func (p *provisionerDaemon) Close() error {
510509func (p * provisionerDaemon )closeWithError (err error )error {
511510p .closeMutex .Lock ()
512511defer p .closeMutex .Unlock ()
512+
513513if p .isClosed () {
514514return p .closeError
515515}
516516
517517if p .isRunningJob () {
518+
519+ // We also need the 'acquire job' mutex here,
520+ // so that a new `p.acquiredJobDone` channel isn't created
521+ // while we're waiting on the mutex.
522+
523+ // Note the mutex order - it's important that we always use the same order of acquisition
524+ // to avoid deadlocks
525+ p .acquiredJobMutex .Lock ()
526+ defer p .acquiredJobMutex .Unlock ()
527+
518528errMsg := "provisioner daemon was shutdown gracefully"
519529if err != nil {
520530errMsg = err .Error ()