Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitd45515d

Browse files
committed
feat: orphan delete pending canceled prebuilds
1 parent88b7372 commitd45515d

File tree

9 files changed

+199
-96
lines changed

9 files changed

+199
-96
lines changed

‎coderd/database/dbauthz/dbauthz.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4933,10 +4933,10 @@ func (q *querier) UpdateOrganizationDeletedByID(ctx context.Context, arg databas
49334933
returndeleteQ(q.log,q.auth,q.db.GetOrganizationByID,deleteF)(ctx,arg.ID)
49344934
}
49354935

4936-
func (q*querier)UpdatePrebuildProvisionerJobWithCancel(ctx context.Context,arg database.UpdatePrebuildProvisionerJobWithCancelParams) ([]uuid.UUID,error) {
4936+
func (q*querier)UpdatePrebuildProvisionerJobWithCancel(ctx context.Context,arg database.UpdatePrebuildProvisionerJobWithCancelParams) ([]database.UpdatePrebuildProvisionerJobWithCancelRow,error) {
49374937
// Prebuild operation for canceling pending prebuild jobs from non-active template versions
49384938
iferr:=q.authorizeContext(ctx,policy.ActionUpdate,rbac.ResourcePrebuiltWorkspace);err!=nil {
4939-
return []uuid.UUID{},err
4939+
return []database.UpdatePrebuildProvisionerJobWithCancelRow{},err
49404940
}
49414941
returnq.db.UpdatePrebuildProvisionerJobWithCancel(ctx,arg)
49424942
}

‎coderd/database/dbauthz/dbauthz_test.go‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -646,10 +646,13 @@ func (s *MethodTestSuite) TestProvisionerJob() {
646646
PresetID: uuid.NullUUID{UUID:uuid.New(),Valid:true},
647647
Now:dbtime.Now(),
648648
}
649-
jobIDs:= []uuid.UUID{uuid.New(),uuid.New()}
649+
canceledJobs:= []database.UpdatePrebuildProvisionerJobWithCancelRow{
650+
{ID:uuid.New(),WorkspaceID:uuid.New(),TemplateID:uuid.New(),TemplateVersionPresetID: uuid.NullUUID{UUID:uuid.New(),Valid:true}},
651+
{ID:uuid.New(),WorkspaceID:uuid.New(),TemplateID:uuid.New(),TemplateVersionPresetID: uuid.NullUUID{UUID:uuid.New(),Valid:true}},
652+
}
650653

651-
dbm.EXPECT().UpdatePrebuildProvisionerJobWithCancel(gomock.Any(),arg).Return(jobIDs,nil).AnyTimes()
652-
check.Args(arg).Asserts(rbac.ResourcePrebuiltWorkspace,policy.ActionUpdate).Returns(jobIDs)
654+
dbm.EXPECT().UpdatePrebuildProvisionerJobWithCancel(gomock.Any(),arg).Return(canceledJobs,nil).AnyTimes()
655+
check.Args(arg).Asserts(rbac.ResourcePrebuiltWorkspace,policy.ActionUpdate).Returns(canceledJobs)
653656
}))
654657
s.Run("GetProvisionerJobsByIDs",s.Mocked(func(dbm*dbmock.MockStore,faker*gofakeit.Faker,check*expects) {
655658
org:=testutil.Fake(s.T(),faker, database.Organization{})

‎coderd/database/dbmetrics/querymetrics.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more aboutcustomizing how changed files appear on GitHub.

‎coderd/database/dbmock/dbmock.go‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more aboutcustomizing how changed files appear on GitHub.

‎coderd/database/querier.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more aboutcustomizing how changed files appear on GitHub.

‎coderd/database/queries.sql.go‎

Lines changed: 26 additions & 12 deletions
Some generated files are not rendered by default. Learn more aboutcustomizing how changed files appear on GitHub.

‎coderd/database/queries/prebuilds.sql‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -300,12 +300,8 @@ GROUP BY wpb.template_version_preset_id;
300300
-- Cancels all pending provisioner jobs for prebuilt workspaces on a specific preset from an
301301
-- inactive template version.
302302
-- This is an optimization to clean up stale pending jobs.
303-
UPDATE provisioner_jobs
304-
SET
305-
canceled_at= @now::timestamptz,
306-
completed_at= @now::timestamptz
307-
WHERE idIN (
308-
SELECTpj.id
303+
WITH jobs_to_cancelAS (
304+
SELECTpj.id,w.idAS workspace_id,w.template_id,wpb.template_version_preset_id
309305
FROM provisioner_jobs pj
310306
INNER JOIN workspace_prebuild_builds wpbONwpb.job_id=pj.id
311307
INNER JOIN workspaces wONw.id=wpb.workspace_id
@@ -324,4 +320,10 @@ WHERE id IN (
324320
ANDpj.canceled_at ISNULL
325321
ANDpj.completed_at ISNULL
326322
)
327-
RETURNING id;
323+
UPDATE provisioner_jobs
324+
SET
325+
canceled_at= @now::timestamptz,
326+
completed_at= @now::timestamptz
327+
FROM jobs_to_cancel
328+
WHEREprovisioner_jobs.id=jobs_to_cancel.id
329+
RETURNINGjobs_to_cancel.id,jobs_to_cancel.workspace_id,jobs_to_cancel.template_id,jobs_to_cancel.template_version_preset_id;

‎enterprise/coderd/prebuilds/reconcile.go‎

Lines changed: 84 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -642,40 +642,20 @@ func (c *StoreReconciler) executeReconciliationAction(ctx context.Context, logge
642642
returnmultiErr.ErrorOrNil()
643643

644644
caseprebuilds.ActionTypeCancelPending:
645-
// Cancel pending prebuild jobs from non-active template versions to avoid
646-
// provisioning obsolete workspaces that would immediately be deprovisioned.
647-
// This uses a criteria-based update to ensure only jobs that are still pending
648-
// at execution time are canceled, avoiding race conditions where jobs may have
649-
// transitioned to running status between query and update.
650-
canceledJobs,err:=c.store.UpdatePrebuildProvisionerJobWithCancel(
651-
ctx,
652-
database.UpdatePrebuildProvisionerJobWithCancelParams{
653-
Now:c.clock.Now(),
654-
PresetID: uuid.NullUUID{
655-
UUID:ps.Preset.ID,
656-
Valid:true,
657-
},
658-
})
659-
iferr!=nil {
660-
logger.Error(ctx,"failed to cancel pending prebuild jobs",
661-
slog.F("template_version_id",ps.Preset.TemplateVersionID.String()),
662-
slog.F("preset_id",ps.Preset.ID),
663-
slog.Error(err))
664-
returnerr
665-
}
666-
iflen(canceledJobs)>0 {
667-
logger.Info(ctx,"canceled pending prebuild jobs for inactive version",
668-
slog.F("template_version_id",ps.Preset.TemplateVersionID.String()),
669-
slog.F("preset_id",ps.Preset.ID),
670-
slog.F("count",len(canceledJobs)))
671-
}
672-
returnnil
645+
returnc.cancelAndOrphanDeletePendingPrebuilds(ctx,ps.Preset.TemplateID,ps.Preset.TemplateVersionID,ps.Preset.ID)
673646

674647
default:
675648
returnxerrors.Errorf("unknown action type: %v",action.ActionType)
676649
}
677650
}
678651

652+
typeDeprovisionModeint
653+
654+
const (
655+
DeprovisionModeNormalDeprovisionMode=iota
656+
DeprovisionModeOrphan
657+
)
658+
679659
func (c*StoreReconciler)createPrebuiltWorkspace(ctx context.Context,prebuiltWorkspaceID uuid.UUID,templateID uuid.UUID,presetID uuid.UUID)error {
680660
name,err:=prebuilds.GenerateName()
681661
iferr!=nil {
@@ -717,33 +697,91 @@ func (c *StoreReconciler) createPrebuiltWorkspace(ctx context.Context, prebuiltW
717697
c.logger.Info(ctx,"attempting to create prebuild",slog.F("name",name),
718698
slog.F("workspace_id",prebuiltWorkspaceID.String()),slog.F("preset_id",presetID.String()))
719699

720-
returnc.provision(ctx,db,prebuiltWorkspaceID,template,presetID,database.WorkspaceTransitionStart,workspace)
700+
returnc.provision(ctx,db,prebuiltWorkspaceID,template,presetID,database.WorkspaceTransitionStart,workspace,DeprovisionModeNormal)
721701
},&database.TxOptions{
722702
Isolation:sql.LevelRepeatableRead,
723703
ReadOnly:false,
724704
})
725705
}
726706

727-
func (c*StoreReconciler)deletePrebuiltWorkspace(ctx context.Context,prebuiltWorkspaceID uuid.UUID,templateID uuid.UUID,presetID uuid.UUID)error {
707+
// provisionDelete provisions a delete transition for a prebuilt workspace.
708+
// If mode is DeprovisionModeOrphan, the builder will not send Terraform state to the provisioner,
709+
// allowing the workspace to be deleted even when no provisioners are available.
710+
// This is safe for canceled pending prebuilds since no Terraform resources were actually provisioned.
711+
//
712+
// IMPORTANT: This function must be called within a database transaction. It does not create its own transaction.
713+
// The caller is responsible for managing the transaction boundary via db.InTx().
714+
func (c*StoreReconciler)provisionDelete(ctx context.Context,db database.Store,workspaceID uuid.UUID,templateID uuid.UUID,presetID uuid.UUID,modeDeprovisionMode)error {
715+
workspace,err:=db.GetWorkspaceByID(ctx,workspaceID)
716+
iferr!=nil {
717+
returnxerrors.Errorf("get workspace: %w",err)
718+
}
719+
720+
template,err:=db.GetTemplateByID(ctx,templateID)
721+
iferr!=nil {
722+
returnxerrors.Errorf("get template: %w",err)
723+
}
724+
725+
returnc.provision(ctx,db,workspaceID,template,presetID,
726+
database.WorkspaceTransitionDelete,workspace,mode)
727+
}
728+
729+
// Cancel and delete pending prebuild jobs from non-active template versions to avoid provisioning obsolete workspaces
730+
// that would immediately be deprovisioned.
731+
// The cancel operation uses a criteria-based update to ensure only jobs that are still pending at execution time are
732+
// canceled, avoiding race conditions where jobs may have transitioned to running status between query and update.
733+
// Since the canceled jobs were not yet processed by a provisioner, no Terraform resources were created, so it is safe
734+
// to orphan delete the associated workspaces. By orphan deleting these workspaces, we avoid overwhelming the provisioner
735+
// queue, since in case no provisioner daemon is available for processing a job, no job is actually created. In case,
736+
// a job is created, since the terraform state is empty it should be processed quite quick by the daemon.
737+
func (c*StoreReconciler)cancelAndOrphanDeletePendingPrebuilds(ctx context.Context,templateID uuid.UUID,templateVersionID uuid.UUID,presetID uuid.UUID)error {
728738
returnc.store.InTx(func(db database.Store)error {
729-
workspace,err:=db.GetWorkspaceByID(ctx,prebuiltWorkspaceID)
739+
canceledJobs,err:=db.UpdatePrebuildProvisionerJobWithCancel(
740+
ctx,
741+
database.UpdatePrebuildProvisionerJobWithCancelParams{
742+
Now:c.clock.Now(),
743+
PresetID: uuid.NullUUID{
744+
UUID:presetID,
745+
Valid:true,
746+
},
747+
})
730748
iferr!=nil {
731-
returnxerrors.Errorf("get workspace by ID: %w",err)
749+
c.logger.Error(ctx,"failed to cancel pending prebuild jobs",
750+
slog.F("template_id",templateID),
751+
slog.F("template_version_id",templateVersionID),
752+
slog.F("preset_id",presetID),
753+
slog.Error(err))
754+
returnerr
732755
}
733756

734-
template,err:=db.GetTemplateByID(ctx,templateID)
735-
iferr!=nil {
736-
returnxerrors.Errorf("failed to get template: %w",err)
757+
iflen(canceledJobs)>0 {
758+
c.logger.Info(ctx,"canceled pending prebuild jobs for inactive version",
759+
slog.F("template_id",templateID),
760+
slog.F("template_version_id",templateVersionID),
761+
slog.F("preset_id",presetID),
762+
slog.F("count",len(canceledJobs)))
737763
}
738764

739-
ifworkspace.OwnerID!=database.PrebuildsSystemUserID {
740-
returnxerrors.Errorf("prebuilt workspace is not owned by prebuild user anymore, probably it was claimed")
765+
varmultiErr multierror.Error
766+
for_,job:=rangecanceledJobs {
767+
err=c.provisionDelete(ctx,db,job.WorkspaceID,job.TemplateID,presetID,DeprovisionModeOrphan)
768+
iferr!=nil {
769+
c.logger.Error(ctx,"failed to orphan delete canceled prebuild",
770+
slog.F("workspace_id",job.WorkspaceID),slog.Error(err))
771+
multiErr.Errors=append(multiErr.Errors,err)
772+
}
741773
}
742774

743-
c.logger.Info(ctx,"attempting to delete prebuild",
744-
slog.F("workspace_id",prebuiltWorkspaceID.String()),slog.F("preset_id",presetID.String()))
775+
returnmultiErr.ErrorOrNil()
776+
},&database.TxOptions{
777+
Isolation:sql.LevelRepeatableRead,
778+
ReadOnly:false,
779+
})
780+
}
745781

746-
returnc.provision(ctx,db,prebuiltWorkspaceID,template,presetID,database.WorkspaceTransitionDelete,workspace)
782+
func (c*StoreReconciler)deletePrebuiltWorkspace(ctx context.Context,prebuiltWorkspaceID uuid.UUID,templateID uuid.UUID,presetID uuid.UUID)error {
783+
returnc.store.InTx(func(db database.Store)error {
784+
returnc.provisionDelete(ctx,db,prebuiltWorkspaceID,templateID,presetID,DeprovisionModeNormal)
747785
},&database.TxOptions{
748786
Isolation:sql.LevelRepeatableRead,
749787
ReadOnly:false,
@@ -758,6 +796,7 @@ func (c *StoreReconciler) provision(
758796
presetID uuid.UUID,
759797
transition database.WorkspaceTransition,
760798
workspace database.Workspace,
799+
modeDeprovisionMode,
761800
)error {
762801
tvp,err:=db.GetPresetParametersByTemplateVersionID(ctx,template.ActiveVersionID)
763802
iferr!=nil {
@@ -795,6 +834,11 @@ func (c *StoreReconciler) provision(
795834
builder=builder.RichParameterValues(params)
796835
}
797836

837+
// Orphan delete canceled pending prebuilds
838+
iftransition==database.WorkspaceTransitionDelete&&mode==DeprovisionModeOrphan {
839+
builder=builder.Orphan()
840+
}
841+
798842
_,provisionerJob,_,err:=builder.Build(
799843
ctx,
800844
db,

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp