Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

feat: delete pending canceled prebuilds#20499

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.

Already on GitHub?Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes fromall commits
Commits
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletionscoderd/database/dbauthz/dbauthz.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -4933,10 +4933,10 @@ func (q *querier) UpdateOrganizationDeletedByID(ctx context.Context, arg databas
return deleteQ(q.log, q.auth, q.db.GetOrganizationByID, deleteF)(ctx, arg.ID)
}

func (q *querier) UpdatePrebuildProvisionerJobWithCancel(ctx context.Context, arg database.UpdatePrebuildProvisionerJobWithCancelParams) ([]uuid.UUID, error) {
func (q *querier) UpdatePrebuildProvisionerJobWithCancel(ctx context.Context, arg database.UpdatePrebuildProvisionerJobWithCancelParams) ([]database.UpdatePrebuildProvisionerJobWithCancelRow, error) {
// Prebuild operation for canceling pending prebuild jobs from non-active template versions
if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourcePrebuiltWorkspace); err != nil {
return []uuid.UUID{}, err
return []database.UpdatePrebuildProvisionerJobWithCancelRow{}, err
}
return q.db.UpdatePrebuildProvisionerJobWithCancel(ctx, arg)
}
Expand Down
9 changes: 6 additions & 3 deletionscoderd/database/dbauthz/dbauthz_test.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -646,10 +646,13 @@ func (s *MethodTestSuite) TestProvisionerJob() {
PresetID: uuid.NullUUID{UUID: uuid.New(), Valid: true},
Now: dbtime.Now(),
}
jobIDs := []uuid.UUID{uuid.New(), uuid.New()}
canceledJobs := []database.UpdatePrebuildProvisionerJobWithCancelRow{
{ID: uuid.New(), WorkspaceID: uuid.New(), TemplateID: uuid.New(), TemplateVersionPresetID: uuid.NullUUID{UUID: uuid.New(), Valid: true}},
{ID: uuid.New(), WorkspaceID: uuid.New(), TemplateID: uuid.New(), TemplateVersionPresetID: uuid.NullUUID{UUID: uuid.New(), Valid: true}},
}

dbm.EXPECT().UpdatePrebuildProvisionerJobWithCancel(gomock.Any(), arg).Return(jobIDs, nil).AnyTimes()
check.Args(arg).Asserts(rbac.ResourcePrebuiltWorkspace, policy.ActionUpdate).Returns(jobIDs)
dbm.EXPECT().UpdatePrebuildProvisionerJobWithCancel(gomock.Any(), arg).Return(canceledJobs, nil).AnyTimes()
check.Args(arg).Asserts(rbac.ResourcePrebuiltWorkspace, policy.ActionUpdate).Returns(canceledJobs)
}))
s.Run("GetProvisionerJobsByIDs", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
org := testutil.Fake(s.T(), faker, database.Organization{})
Expand Down
2 changes: 1 addition & 1 deletioncoderd/database/dbmetrics/querymetrics.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more abouthow customized files appear on GitHub.

4 changes: 2 additions & 2 deletionscoderd/database/dbmock/dbmock.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more abouthow customized files appear on GitHub.

2 changes: 1 addition & 1 deletioncoderd/database/querier.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more abouthow customized files appear on GitHub.

38 changes: 26 additions & 12 deletionscoderd/database/queries.sql.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more abouthow customized files appear on GitHub.

16 changes: 9 additions & 7 deletionscoderd/database/queries/prebuilds.sql
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -300,12 +300,8 @@ GROUP BY wpb.template_version_preset_id;
-- Cancels all pending provisioner jobs for prebuilt workspaces on a specific preset from an
-- inactive template version.
-- This is an optimization to clean up stale pending jobs.
UPDATE provisioner_jobs
SET
canceled_at = @now::timestamptz,
completed_at = @now::timestamptz
WHERE id IN (
SELECT pj.id
WITH jobs_to_cancel AS (
SELECT pj.id, w.id AS workspace_id, w.template_id, wpb.template_version_preset_id
FROM provisioner_jobs pj
INNER JOIN workspace_prebuild_builds wpb ON wpb.job_id = pj.id
INNER JOIN workspaces w ON w.id = wpb.workspace_id
Expand All@@ -324,4 +320,10 @@ WHERE id IN (
AND pj.canceled_at IS NULL
AND pj.completed_at IS NULL
)
RETURNING id;
UPDATE provisioner_jobs
SET
canceled_at = @now::timestamptz,
completed_at = @now::timestamptz
FROM jobs_to_cancel
WHERE provisioner_jobs.id = jobs_to_cancel.id
RETURNING jobs_to_cancel.id, jobs_to_cancel.workspace_id, jobs_to_cancel.template_id, jobs_to_cancel.template_version_preset_id;
144 changes: 104 additions & 40 deletionsenterprise/coderd/prebuilds/reconcile.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -57,6 +57,24 @@ type StoreReconciler struct {

var _ prebuilds.ReconciliationOrchestrator = &StoreReconciler{}

type DeprovisionMode int

const (
DeprovisionModeNormal DeprovisionMode = iota
DeprovisionModeOrphan
)

func (d DeprovisionMode) String() string {
switch d {
case DeprovisionModeOrphan:
return "orphan"
case DeprovisionModeNormal:
return "normal"
default:
return "unknown"
}
}

func NewStoreReconciler(store database.Store,
ps pubsub.Pubsub,
fileCache *files.Cache,
Expand DownExpand Up@@ -642,34 +660,7 @@ func (c *StoreReconciler) executeReconciliationAction(ctx context.Context, logge
return multiErr.ErrorOrNil()

case prebuilds.ActionTypeCancelPending:
// Cancel pending prebuild jobs from non-active template versions to avoid
// provisioning obsolete workspaces that would immediately be deprovisioned.
// This uses a criteria-based update to ensure only jobs that are still pending
// at execution time are canceled, avoiding race conditions where jobs may have
// transitioned to running status between query and update.
canceledJobs, err := c.store.UpdatePrebuildProvisionerJobWithCancel(
ctx,
database.UpdatePrebuildProvisionerJobWithCancelParams{
Now: c.clock.Now(),
PresetID: uuid.NullUUID{
UUID: ps.Preset.ID,
Valid: true,
},
})
if err != nil {
logger.Error(ctx, "failed to cancel pending prebuild jobs",
slog.F("template_version_id", ps.Preset.TemplateVersionID.String()),
slog.F("preset_id", ps.Preset.ID),
slog.Error(err))
return err
}
if len(canceledJobs) > 0 {
logger.Info(ctx, "canceled pending prebuild jobs for inactive version",
slog.F("template_version_id", ps.Preset.TemplateVersionID.String()),
slog.F("preset_id", ps.Preset.ID),
slog.F("count", len(canceledJobs)))
}
return nil
return c.cancelAndOrphanDeletePendingPrebuilds(ctx, ps.Preset.TemplateID, ps.Preset.TemplateVersionID, ps.Preset.ID)

default:
return xerrors.Errorf("unknown action type: %v", action.ActionType)
Expand DownExpand Up@@ -717,33 +708,100 @@ func (c *StoreReconciler) createPrebuiltWorkspace(ctx context.Context, prebuiltW
c.logger.Info(ctx, "attempting to create prebuild", slog.F("name", name),
slog.F("workspace_id", prebuiltWorkspaceID.String()), slog.F("preset_id", presetID.String()))

return c.provision(ctx, db, prebuiltWorkspaceID, template, presetID, database.WorkspaceTransitionStart, workspace)
return c.provision(ctx, db, prebuiltWorkspaceID, template, presetID, database.WorkspaceTransitionStart, workspace, DeprovisionModeNormal)
}, &database.TxOptions{
Isolation: sql.LevelRepeatableRead,
ReadOnly: false,
})
}

func (c *StoreReconciler) deletePrebuiltWorkspace(ctx context.Context, prebuiltWorkspaceID uuid.UUID, templateID uuid.UUID, presetID uuid.UUID) error {
// provisionDelete provisions a delete transition for a prebuilt workspace.
//
// If mode is DeprovisionModeOrphan, the builder will not send Terraform state to the provisioner.
// This allows the workspace to be deleted even when no provisioners are available, and is safe
// when no Terraform resources were actually created (e.g., for pending prebuilds that were canceled
// before provisioning started).
//
// IMPORTANT: This function must be called within a database transaction. It does not create its own transaction.
// The caller is responsible for managing the transaction boundary via db.InTx().
func (c *StoreReconciler) provisionDelete(ctx context.Context, db database.Store, workspaceID uuid.UUID, templateID uuid.UUID, presetID uuid.UUID, mode DeprovisionMode) error {
workspace, err := db.GetWorkspaceByID(ctx, workspaceID)
if err != nil {
return xerrors.Errorf("get workspace by ID: %w", err)
}
Comment on lines +728 to +731
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others.Learn more.

I was going to mention that since you changed theUpdatePrebuildProvisionerJobWithCancel query to return workspace_id, we could also returnowner_id there as well so we can use it here. Unfortunately it looks likewsbuilder needs the workspace in any case.


template, err := db.GetTemplateByID(ctx, templateID)
if err != nil {
return xerrors.Errorf("failed to get template: %w", err)
}

if workspace.OwnerID != database.PrebuildsSystemUserID {
return xerrors.Errorf("prebuilt workspace is not owned by prebuild user anymore, probably it was claimed")
}

c.logger.Info(ctx, "attempting to delete prebuild", slog.F("orphan", mode.String()),
slog.F("name", workspace.Name), slog.F("workspace_id", workspaceID.String()), slog.F("preset_id", presetID.String()))

return c.provision(ctx, db, workspaceID, template, presetID,
database.WorkspaceTransitionDelete, workspace, mode)
}

// cancelAndOrphanDeletePendingPrebuilds cancels pending prebuild jobs from inactive template versions
// and orphan-deletes their associated workspaces.
//
// The cancel operation uses a criteria-based update to ensure only jobs that are still pending at
// execution time are canceled, avoiding race conditions where jobs may have transitioned to running.
//
// Since these jobs were never processed by a provisioner, no Terraform resources were created,
// making it safe to orphan-delete the workspaces (skipping Terraform destroy).
func (c *StoreReconciler) cancelAndOrphanDeletePendingPrebuilds(ctx context.Context, templateID uuid.UUID, templateVersionID uuid.UUID, presetID uuid.UUID) error {
return c.store.InTx(func(db database.Store) error {
workspace, err := db.GetWorkspaceByID(ctx, prebuiltWorkspaceID)
canceledJobs, err := db.UpdatePrebuildProvisionerJobWithCancel(
ctx,
database.UpdatePrebuildProvisionerJobWithCancelParams{
Now: c.clock.Now(),
PresetID: uuid.NullUUID{
UUID: presetID,
Valid: true,
},
})
if err != nil {
return xerrors.Errorf("get workspace by ID: %w", err)
c.logger.Error(ctx, "failed to cancel pending prebuild jobs",
slog.F("template_id", templateID.String()),
slog.F("template_version_id", templateVersionID.String()),
slog.F("preset_id", presetID.String()),
slog.Error(err))
return err
}

template, err := db.GetTemplateByID(ctx, templateID)
if err != nil {
return xerrors.Errorf("failed to get template: %w", err)
if len(canceledJobs) > 0 {
c.logger.Info(ctx, "canceled pending prebuild jobs for inactive version",
slog.F("template_id", templateID.String()),
slog.F("template_version_id", templateVersionID.String()),
slog.F("preset_id", presetID.String()),
slog.F("count", len(canceledJobs)))
}

if workspace.OwnerID != database.PrebuildsSystemUserID {
return xerrors.Errorf("prebuilt workspace is not owned by prebuild user anymore, probably it was claimed")
var multiErr multierror.Error
for _, job := range canceledJobs {
err = c.provisionDelete(ctx, db, job.WorkspaceID, job.TemplateID, presetID, DeprovisionModeOrphan)
if err != nil {
c.logger.Error(ctx, "failed to orphan delete canceled prebuild",
slog.F("workspace_id", job.WorkspaceID.String()), slog.Error(err))
multiErr.Errors = append(multiErr.Errors, err)
}
}

c.logger.Info(ctx, "attempting to delete prebuild",
slog.F("workspace_id", prebuiltWorkspaceID.String()), slog.F("preset_id", presetID.String()))
return multiErr.ErrorOrNil()
}, &database.TxOptions{
Isolation: sql.LevelRepeatableRead,
ReadOnly: false,
})
}

return c.provision(ctx, db, prebuiltWorkspaceID, template, presetID, database.WorkspaceTransitionDelete, workspace)
func (c *StoreReconciler) deletePrebuiltWorkspace(ctx context.Context, prebuiltWorkspaceID uuid.UUID, templateID uuid.UUID, presetID uuid.UUID) error {
return c.store.InTx(func(db database.Store) error {
return c.provisionDelete(ctx, db, prebuiltWorkspaceID, templateID, presetID, DeprovisionModeNormal)
}, &database.TxOptions{
Isolation: sql.LevelRepeatableRead,
ReadOnly: false,
Expand All@@ -758,6 +816,7 @@ func (c *StoreReconciler) provision(
presetID uuid.UUID,
transition database.WorkspaceTransition,
workspace database.Workspace,
mode DeprovisionMode,
) error {
tvp, err := db.GetPresetParametersByTemplateVersionID(ctx, template.ActiveVersionID)
if err != nil {
Expand DownExpand Up@@ -795,6 +854,11 @@ func (c *StoreReconciler) provision(
builder = builder.RichParameterValues(params)
}

// Use orphan mode for deletes when no Terraform resources exist
if transition == database.WorkspaceTransitionDelete && mode == DeprovisionModeOrphan {
builder = builder.Orphan()
}

_, provisionerJob, _, err := builder.Build(
ctx,
db,
Expand Down
Loading
Loading
[8]ページ先頭
©2009-2025 Movatter.jp