Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit3885d9a

Browse files
committed
feat: POC to limit prebuild actions
1 parentf31e6e0 commit3885d9a

File tree

4 files changed

+213
-59
lines changed

4 files changed

+213
-59
lines changed

‎coderd/prebuilds/global_snapshot.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,15 @@ func NewGlobalSnapshot(
5353
}
5454
}
5555

56+
// TotalPendingCount get total number of prebuild-related jobs in the queue
57+
func (sGlobalSnapshot)TotalPendingCount()int {
58+
vartotalint
59+
for_,row:=ranges.PrebuildsInProgress {
60+
total+=int(row.Count)
61+
}
62+
returntotal
63+
}
64+
5665
func (sGlobalSnapshot)FilterByPreset(presetID uuid.UUID) (*PresetSnapshot,error) {
5766
preset,found:=slice.Find(s.Presets,func(preset database.GetTemplatePresetsWithPrebuildsRow)bool {
5867
returnpreset.ID==presetID

‎coderd/prebuilds/preset_snapshot.go‎

Lines changed: 42 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package prebuilds
33
import (
44
"context"
55
"fmt"
6+
"math"
67
"slices"
78
"time"
89

@@ -31,9 +32,6 @@ const (
3132

3233
// ActionTypeDelete indicates that existing prebuilds should be deleted.
3334
ActionTypeDelete
34-
35-
// ActionTypeBackoff indicates that prebuild creation should be delayed.
36-
ActionTypeBackoff
3735
)
3836

3937
// PresetSnapshot is a filtered view of GlobalSnapshot focused on a single preset.
@@ -306,17 +304,29 @@ func (p PresetSnapshot) isActive() bool {
306304
func (pPresetSnapshot)handleActiveTemplateVersion() (actions []*ReconciliationActions,errerror) {
307305
state:=p.CalculateState()
308306

307+
// If preset has pending prebuilds in the provisioner queue do not add new actions
308+
ifstate.Starting>0||state.Stopping>0||state.Deleting>0 {
309+
p.logger.Warn(context.Background(),"skipping preset with pending prebuilds",
310+
slog.F("preset_id",p.Preset.ID),
311+
slog.F("starting",state.Starting),
312+
slog.F("stopping",state.Stopping),
313+
slog.F("deleting",state.Deleting))
314+
returnnil,nil
315+
}
316+
309317
// If we have expired prebuilds, delete them
310318
ifstate.Expired>0 {
311319
vardeleteIDs []uuid.UUID
312320
for_,expired:=rangep.Expired {
313321
deleteIDs=append(deleteIDs,expired.ID)
314322
}
315-
actions=append(actions,
316-
&ReconciliationActions{
317-
ActionType:ActionTypeDelete,
318-
DeleteIDs:deleteIDs,
319-
})
323+
iflen(deleteIDs)>0 {
324+
actions=append(actions,
325+
&ReconciliationActions{
326+
ActionType:ActionTypeDelete,
327+
DeleteIDs:deleteIDs,
328+
})
329+
}
320330
}
321331

322332
// If we still have more prebuilds than desired, delete the oldest ones
@@ -328,6 +338,14 @@ func (p PresetSnapshot) handleActiveTemplateVersion() (actions []*Reconciliation
328338
})
329339
}
330340

341+
// If preset is hard-limited, and it's a create operation, log it and exit early.
342+
// Creation operation is disallowed for hard-limited preset.
343+
ifp.IsHardLimited {
344+
p.logger.Warn(context.Background(),"skipping hard limited preset for creation action",
345+
slog.F("preset_id",p.Preset.ID))
346+
returnactions,nil
347+
}
348+
331349
// Number of running prebuilds excluding the recently deleted Expired
332350
runningValid:=state.Actual-state.Expired
333351

@@ -350,13 +368,16 @@ func (p PresetSnapshot) handleActiveTemplateVersion() (actions []*Reconciliation
350368
func (pPresetSnapshot)handleInactiveTemplateVersion() ([]*ReconciliationActions,error) {
351369
prebuildsToDelete:=len(p.Running)
352370
deleteIDs:=p.getOldestPrebuildIDs(prebuildsToDelete)
371+
iflen(deleteIDs)>0 {
372+
return []*ReconciliationActions{
373+
{
374+
ActionType:ActionTypeDelete,
375+
DeleteIDs:deleteIDs,
376+
},
377+
},nil
378+
}
353379

354-
return []*ReconciliationActions{
355-
{
356-
ActionType:ActionTypeDelete,
357-
DeleteIDs:deleteIDs,
358-
},
359-
},nil
380+
returnnil,nil
360381
}
361382

362383
// needsBackoffPeriod checks if we should delay prebuild creation due to recent failures.
@@ -371,12 +392,13 @@ func (p PresetSnapshot) needsBackoffPeriod(clock quartz.Clock, backoffInterval t
371392
returnnil,false
372393
}
373394

374-
return []*ReconciliationActions{
375-
{
376-
ActionType:ActionTypeBackoff,
377-
BackoffUntil:backoffUntil,
378-
},
379-
},true
395+
// If there is anything to backoff for (usually a cycle of failed prebuilds), then log and bail out.
396+
p.logger.Warn(context.Background(),"template prebuild state retrieved, backing off",
397+
slog.F("preset_id",p.Preset.ID),
398+
slog.F("backoff_until",backoffUntil.Format(time.RFC3339)),
399+
slog.F("backoff_secs",math.Round(backoffUntil.Sub(p.clock.Now()).Seconds())))
400+
401+
returnnil,true
380402
}
381403

382404
// countEligible returns the number of prebuilds that are ready to be claimed.

‎enterprise/coderd/prebuilds/reconcile.go‎

Lines changed: 56 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import (
66
"encoding/json"
77
"errors"
88
"fmt"
9-
"math"
109
"strings"
1110
"sync"
1211
"sync/atomic"
@@ -300,6 +299,16 @@ func (c *StoreReconciler) ReconcileAll(ctx context.Context) error {
300299
returnxerrors.Errorf("reconcile prebuild membership: %w",err)
301300
}
302301

302+
// If the number of pending prebuilds is greater than the number allowed, skip reconciliation
303+
sched,err:=NewJobsScheduler(c.logger,snapshot.TotalPendingCount(),c.executeReconciliationAction)
304+
iferr!=nil {
305+
logger.Info(ctx,"number of pending prebuilds is greater than the number allowed, skipping reconciliation")
306+
returnnil
307+
}
308+
309+
// Collect preset actions
310+
resultsCh:=make(chanPresetActions,len(snapshot.Presets))
311+
303312
vareg errgroup.Group
304313
// Reconcile presets in parallel. Each preset in its own goroutine.
305314
for_,preset:=rangesnapshot.Presets {
@@ -311,7 +320,7 @@ func (c *StoreReconciler) ReconcileAll(ctx context.Context) error {
311320

312321
eg.Go(func()error {
313322
// Pass outer context.
314-
err=c.ReconcilePreset(ctx,*ps)
323+
actions,err:=c.ReconcilePreset(ctx,*ps)
315324
iferr!=nil {
316325
logger.Error(
317326
ctx,
@@ -321,12 +330,31 @@ func (c *StoreReconciler) ReconcileAll(ctx context.Context) error {
321330
)
322331
}
323332
// DO NOT return error otherwise the tx will end.
324-
returnnil
333+
select {
334+
case<-ctx.Done():
335+
returnnil
336+
caseresultsCh<-PresetActions{Preset:*ps,Actions:actions}:
337+
returnnil
338+
}
325339
})
326340
}
327341

328-
// Release lock only when all preset reconciliation goroutines are finished.
329-
returneg.Wait()
342+
// Wait for all goroutines to finish
343+
_=eg.Wait()
344+
close(resultsCh)
345+
346+
results:=make([]PresetActions,0,len(snapshot.Presets))
347+
forr:=rangeresultsCh {
348+
results=append(results,r)
349+
}
350+
351+
// Scheduler executes reconciliation actions
352+
iferr:=sched.Run(ctx,results);err!=nil {
353+
logger.Error(ctx,"scheduler returned errors",slog.Error(err))
354+
returnerr
355+
}
356+
357+
returnnil
330358
})
331359
iferr!=nil {
332360
logger.Error(ctx,"failed to reconcile",slog.Error(err))
@@ -442,7 +470,7 @@ func (c *StoreReconciler) SnapshotState(ctx context.Context, store database.Stor
442470
return&state,err
443471
}
444472

445-
func (c*StoreReconciler)ReconcilePreset(ctx context.Context,ps prebuilds.PresetSnapshot)error {
473+
func (c*StoreReconciler)ReconcilePreset(ctx context.Context,ps prebuilds.PresetSnapshot)([]*prebuilds.ReconciliationActions,error) {
446474
logger:=c.logger.With(
447475
slog.F("template_id",ps.Preset.TemplateID.String()),
448476
slog.F("template_name",ps.Preset.TemplateName),
@@ -463,36 +491,37 @@ func (c *StoreReconciler) ReconcilePreset(ctx context.Context, ps prebuilds.Pres
463491
PresetID:ps.Preset.ID,
464492
})
465493
iferr!=nil {
466-
returnxerrors.Errorf("failed to update preset prebuild status: %w",err)
494+
returnnil,xerrors.Errorf("failed to update preset prebuild status: %w",err)
467495
}
468496
}
469497

470498
state:=ps.CalculateState()
499+
logger.Debug(ctx,"calculated reconciliation state for preset",
500+
slog.F("desired",state.Desired),
501+
slog.F("actual",state.Actual),
502+
slog.F("extraneous",state.Extraneous),
503+
slog.F("starting",state.Starting),
504+
slog.F("stopping",state.Stopping),
505+
slog.F("deleting",state.Deleting),
506+
slog.F("eligible",state.Eligible))
507+
471508
actions,err:=c.CalculateActions(ctx,ps)
472509
iferr!=nil {
473510
logger.Error(ctx,"failed to calculate actions for preset",slog.Error(err))
474-
returnerr
475-
}
476-
477-
fields:= []any{
478-
slog.F("desired",state.Desired),slog.F("actual",state.Actual),
479-
slog.F("extraneous",state.Extraneous),slog.F("starting",state.Starting),
480-
slog.F("stopping",state.Stopping),slog.F("deleting",state.Deleting),
481-
slog.F("eligible",state.Eligible),
511+
returnnil,err
482512
}
483513

484-
levelFn:=logger.Debug
485-
levelFn(ctx,"calculated reconciliation state for preset",fields...)
514+
returnactions,nil
486515

487-
varmultiErr multierror.Error
488-
for_,action:=rangeactions {
489-
err=c.executeReconciliationAction(ctx,logger,ps,action)
490-
iferr!=nil {
491-
logger.Error(ctx,"failed to execute action","type",action.ActionType,slog.Error(err))
492-
multiErr.Errors=append(multiErr.Errors,err)
493-
}
494-
}
495-
returnmultiErr.ErrorOrNil()
516+
//var multiErr multierror.Error
517+
//for _, action := range actions {
518+
//err = c.executeReconciliationAction(ctx, logger, ps, action)
519+
//if err != nil {
520+
//logger.Error(ctx, "failed to execute action", "type", action.ActionType, slog.Error(err))
521+
//multiErr.Errors = append(multiErr.Errors, err)
522+
//}
523+
//}
524+
//return multiErr.ErrorOrNil()
496525
}
497526

498527
func (c*StoreReconciler)CalculateActions(ctx context.Context,snapshot prebuilds.PresetSnapshot) ([]*prebuilds.ReconciliationActions,error) {
@@ -574,8 +603,6 @@ func (c *StoreReconciler) executeReconciliationAction(ctx context.Context, logge
574603
levelFn(ctx,"calculated reconciliation action for preset",fields...)
575604

576605
switch {
577-
caseaction.ActionType==prebuilds.ActionTypeBackoff:
578-
levelFn=logger.Warn
579606
// Log at info level when there's a change to be effected.
580607
caseaction.ActionType==prebuilds.ActionTypeCreate&&action.Create>0:
581608
levelFn=logger.Info
@@ -584,16 +611,6 @@ func (c *StoreReconciler) executeReconciliationAction(ctx context.Context, logge
584611
}
585612

586613
switchaction.ActionType {
587-
caseprebuilds.ActionTypeBackoff:
588-
// If there is anything to backoff for (usually a cycle of failed prebuilds), then log and bail out.
589-
levelFn(ctx,"template prebuild state retrieved, backing off",
590-
append(fields,
591-
slog.F("backoff_until",action.BackoffUntil.Format(time.RFC3339)),
592-
slog.F("backoff_secs",math.Round(action.BackoffUntil.Sub(c.clock.Now()).Seconds())),
593-
)...)
594-
595-
returnnil
596-
597614
caseprebuilds.ActionTypeCreate:
598615
// Unexpected things happen (i.e. bugs or bitflips); let's defend against disastrous outcomes.
599616
// See https://blog.robertelder.org/causes-of-bit-flips-in-computer-memory/.
@@ -662,7 +679,7 @@ func (c *StoreReconciler) createPrebuiltWorkspace(ctx context.Context, prebuiltW
662679
OrganizationID:template.OrganizationID,
663680
TemplateID:template.ID,
664681
Name:name,
665-
LastUsedAt:c.clock.Now(),
682+
LastUsedAt:now,
666683
AutomaticUpdates:database.AutomaticUpdatesNever,
667684
AutostartSchedule: sql.NullString{},
668685
Ttl: sql.NullInt64{},

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp