- Notifications
You must be signed in to change notification settings - Fork1k
fix: don't create autostart workspace builds with no available provisioners#19067
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.
Already on GitHub?Sign in to your account
Merged
Uh oh!
There was an error while loading.Please reload this page.
Merged
Changes fromall commits
Commits
Show all changes
18 commits Select commitHold shift + click to select a range
314bb66
add test that fails since we still trigger autobuilds even if a
cstyan310511e
add check for available provisioners before we try to do a workspace
cstyan1256877
fix provisioner check for tests which don't fully set up/wait for the
cstyan26f5001
use the runOnce time
cstyan78a8f5e
refactor: remove SkipProvisionerCheck and improve test provisioner ha…
cstyana983dc2
more fixes for tests; mostly around ensuring that tests which use the
cstyan0bc4cce
significantly simplify
cstyan9a3f534
more refactor and clean up
cstyana98ef1d
fix lint
cstyane48c724
hasValidProvisioner only needs tags
cstyan883c748
more cleanup of function params
cstyan3c90c6d
already waiting for the workspaces provisioner, don't need to wait for
cstyan5a6493c
fix print statement missed in earlier commit
cstyanc5f0d41
address more review feedback; mostly around provisioner closers in the
cstyan224acfc
remove MustWaitForAnyProvisionerWithClient
cstyane7d748e
Merge branch 'main' into callum-autostart-no-provisioner
cstyan3275c57
not sure how this was passing before, the prebuilds NextStartAt time is
cstyanbae5c80
this test also needs to have the provisioner time updated
cstyanFile filter
Filter by extension
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Diff view
Diff view
There are no files selected for viewing
50 changes: 50 additions & 0 deletionscoderd/autobuild/lifecycle_executor.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -29,6 +29,7 @@ import ( | ||
"github.com/coder/coder/v2/coderd/database/provisionerjobs" | ||
"github.com/coder/coder/v2/coderd/database/pubsub" | ||
"github.com/coder/coder/v2/coderd/notifications" | ||
"github.com/coder/coder/v2/coderd/provisionerdserver" | ||
"github.com/coder/coder/v2/coderd/schedule" | ||
"github.com/coder/coder/v2/coderd/wsbuilder" | ||
"github.com/coder/coder/v2/codersdk" | ||
@@ -132,6 +133,39 @@ func (e *Executor) Run() { | ||
}) | ||
} | ||
// hasValidProvisioner checks whether there is at least one valid (non-stale, correct tags) provisioner | ||
// based on time t and the tags maps (such as from a templateVersionJob). | ||
func (e *Executor) hasValidProvisioner(ctx context.Context, tx database.Store, t time.Time, ws database.Workspace, tags map[string]string) (bool, error) { | ||
queryParams := database.GetProvisionerDaemonsByOrganizationParams{ | ||
OrganizationID: ws.OrganizationID, | ||
WantTags: tags, | ||
} | ||
cstyan marked this conversation as resolved. Show resolvedHide resolvedUh oh!There was an error while loading.Please reload this page. | ||
// nolint: gocritic // The user (in this case, the user/context for autostart builds) may not have the full | ||
// permissions to read provisioner daemons, but we need to check if there's any for the job prior to the | ||
// execution of the job via autostart to fix: https://github.com/coder/coder/issues/17941 | ||
provisionerDaemons, err := tx.GetProvisionerDaemonsByOrganization(dbauthz.AsSystemReadProvisionerDaemons(ctx), queryParams) | ||
if err != nil { | ||
return false, xerrors.Errorf("get provisioner daemons: %w", err) | ||
} | ||
logger := e.log.With(slog.F("tags", tags)) | ||
// Check if any provisioners are active (not stale) | ||
for _, pd := range provisionerDaemons { | ||
if pd.LastSeenAt.Valid { | ||
age := t.Sub(pd.LastSeenAt.Time) | ||
if age <= provisionerdserver.StaleInterval { | ||
logger.Debug(ctx, "hasValidProvisioner: found active provisioner", | ||
slog.F("daemon_id", pd.ID), | ||
) | ||
return true, nil | ||
} | ||
} | ||
} | ||
logger.Debug(ctx, "hasValidProvisioner: no active provisioners found") | ||
return false, nil | ||
} | ||
func (e *Executor) runOnce(t time.Time) Stats { | ||
stats := Stats{ | ||
Transitions: make(map[uuid.UUID]database.WorkspaceTransition), | ||
@@ -281,6 +315,22 @@ func (e *Executor) runOnce(t time.Time) Stats { | ||
return nil | ||
} | ||
// Get the template version job to access tags | ||
templateVersionJob, err := tx.GetProvisionerJobByID(e.ctx, activeTemplateVersion.JobID) | ||
if err != nil { | ||
return xerrors.Errorf("get template version job: %w", err) | ||
} | ||
// Before creating the workspace build, check for available provisioners | ||
hasProvisioners, err := e.hasValidProvisioner(e.ctx, tx, t, ws, templateVersionJob.Tags) | ||
if err != nil { | ||
return xerrors.Errorf("check provisioner availability: %w", err) | ||
} | ||
if !hasProvisioners { | ||
log.Warn(e.ctx, "skipping autostart - no available provisioners") | ||
return nil // Skip this workspace | ||
} | ||
if nextTransition != "" { | ||
builder := wsbuilder.New(ws, nextTransition, *e.buildUsageChecker.Load()). | ||
SetLastWorkspaceBuildInTx(&latestBuild). | ||
Oops, something went wrong.
Uh oh!
There was an error while loading.Please reload this page.
Oops, something went wrong.
Uh oh!
There was an error while loading.Please reload this page.
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.