- Notifications
You must be signed in to change notification settings - Fork1.1k
fix: don't create autostart workspace builds with no available provisioners#19067
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.
Already on GitHub?Sign in to your account
Merged
Uh oh!
There was an error while loading.Please reload this page.
Merged
Changes fromall commits
Commits
Show all changes
18 commits Select commitHold shift + click to select a range
314bb66 add test that fails since we still trigger autobuilds even if a
cstyan310511e add check for available provisioners before we try to do a workspace
cstyan1256877 fix provisioner check for tests which don't fully set up/wait for the
cstyan26f5001 use the runOnce time
cstyan78a8f5e refactor: remove SkipProvisionerCheck and improve test provisioner ha…
cstyana983dc2 more fixes for tests; mostly around ensuring that tests which use the
cstyan0bc4cce significantly simplify
cstyan9a3f534 more refactor and clean up
cstyana98ef1d fix lint
cstyane48c724 hasValidProvisioner only needs tags
cstyan883c748 more cleanup of function params
cstyan3c90c6d already waiting for the workspaces provisioner, don't need to wait for
cstyan5a6493c fix print statement missed in earlier commit
cstyanc5f0d41 address more review feedback; mostly around provisioner closers in the
cstyan224acfc remove MustWaitForAnyProvisionerWithClient
cstyane7d748e Merge branch 'main' into callum-autostart-no-provisioner
cstyan3275c57 not sure how this was passing before, the prebuilds NextStartAt time is
cstyanbae5c80 this test also needs to have the provisioner time updated
cstyanFile filter
Filter by extension
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Diff view
Diff view
There are no files selected for viewing
50 changes: 50 additions & 0 deletionscoderd/autobuild/lifecycle_executor.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -29,6 +29,7 @@ import ( | ||
| "github.com/coder/coder/v2/coderd/database/provisionerjobs" | ||
| "github.com/coder/coder/v2/coderd/database/pubsub" | ||
| "github.com/coder/coder/v2/coderd/notifications" | ||
| "github.com/coder/coder/v2/coderd/provisionerdserver" | ||
| "github.com/coder/coder/v2/coderd/schedule" | ||
| "github.com/coder/coder/v2/coderd/wsbuilder" | ||
| "github.com/coder/coder/v2/codersdk" | ||
| @@ -132,6 +133,39 @@ func (e *Executor) Run() { | ||
| }) | ||
| } | ||
| // hasValidProvisioner checks whether there is at least one valid (non-stale, correct tags) provisioner | ||
| // based on time t and the tags maps (such as from a templateVersionJob). | ||
| func (e *Executor) hasValidProvisioner(ctx context.Context, tx database.Store, t time.Time, ws database.Workspace, tags map[string]string) (bool, error) { | ||
| queryParams := database.GetProvisionerDaemonsByOrganizationParams{ | ||
| OrganizationID: ws.OrganizationID, | ||
| WantTags: tags, | ||
| } | ||
cstyan marked this conversation as resolved. Show resolvedHide resolvedUh oh!There was an error while loading.Please reload this page. | ||
| // nolint: gocritic // The user (in this case, the user/context for autostart builds) may not have the full | ||
| // permissions to read provisioner daemons, but we need to check if there's any for the job prior to the | ||
| // execution of the job via autostart to fix: https://github.com/coder/coder/issues/17941 | ||
| provisionerDaemons, err := tx.GetProvisionerDaemonsByOrganization(dbauthz.AsSystemReadProvisionerDaemons(ctx), queryParams) | ||
| if err != nil { | ||
| return false, xerrors.Errorf("get provisioner daemons: %w", err) | ||
| } | ||
| logger := e.log.With(slog.F("tags", tags)) | ||
| // Check if any provisioners are active (not stale) | ||
| for _, pd := range provisionerDaemons { | ||
| if pd.LastSeenAt.Valid { | ||
| age := t.Sub(pd.LastSeenAt.Time) | ||
| if age <= provisionerdserver.StaleInterval { | ||
| logger.Debug(ctx, "hasValidProvisioner: found active provisioner", | ||
| slog.F("daemon_id", pd.ID), | ||
| ) | ||
| return true, nil | ||
| } | ||
| } | ||
| } | ||
| logger.Debug(ctx, "hasValidProvisioner: no active provisioners found") | ||
| return false, nil | ||
| } | ||
| func (e *Executor) runOnce(t time.Time) Stats { | ||
| stats := Stats{ | ||
| Transitions: make(map[uuid.UUID]database.WorkspaceTransition), | ||
| @@ -281,6 +315,22 @@ func (e *Executor) runOnce(t time.Time) Stats { | ||
| return nil | ||
| } | ||
| // Get the template version job to access tags | ||
| templateVersionJob, err := tx.GetProvisionerJobByID(e.ctx, activeTemplateVersion.JobID) | ||
| if err != nil { | ||
| return xerrors.Errorf("get template version job: %w", err) | ||
| } | ||
| // Before creating the workspace build, check for available provisioners | ||
| hasProvisioners, err := e.hasValidProvisioner(e.ctx, tx, t, ws, templateVersionJob.Tags) | ||
| if err != nil { | ||
| return xerrors.Errorf("check provisioner availability: %w", err) | ||
| } | ||
| if !hasProvisioners { | ||
| log.Warn(e.ctx, "skipping autostart - no available provisioners") | ||
| return nil // Skip this workspace | ||
| } | ||
| if nextTransition != "" { | ||
| builder := wsbuilder.New(ws, nextTransition, *e.buildUsageChecker.Load()). | ||
| SetLastWorkspaceBuildInTx(&latestBuild). | ||
Oops, something went wrong.
Uh oh!
There was an error while loading.Please reload this page.
Oops, something went wrong.
Uh oh!
There was an error while loading.Please reload this page.
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.