Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit7163c58

Browse files
committed
fix: limit concurrent database connections in prebuild reconciliation
1 parentf8d9a80 commit7163c58

File tree

3 files changed

+184
-0
lines changed

3 files changed

+184
-0
lines changed

‎coderd/prebuilds/preset_snapshot.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,29 @@ func (p PresetSnapshot) CalculateState() *ReconciliationState {
258258
}
259259
}
260260

261+
// CanSkipReconciliation returns true if this preset can safely be skipped during
262+
// the reconciliation loop.
263+
//
264+
// This is a performance optimization to avoid spawning goroutines for presets
265+
// that have no work to do. It only returns true for presets from inactive
266+
// template versions that have no running workspaces, no pending jobs, and no
267+
// in-progress builds.
268+
//
269+
// Presets from active template versions always go through the reconciliation loop
270+
// to ensure desired_instances is maintained correctly.
271+
func (pPresetSnapshot)CanSkipReconciliation()bool {
272+
// Only skip presets from inactive template versions that have absolutely nothing to clean up
273+
if!p.isActive()&&
274+
len(p.Running)==0&&
275+
len(p.Expired)==0&&
276+
// len(p.InProgress) == 0 CountInProgressPrebuilds only queries active template versions
277+
p.PendingCount==0&&
278+
p.Backoff==nil {
279+
returntrue
280+
}
281+
returnfalse
282+
}
283+
261284
// CalculateActions determines what actions are needed to reconcile the current state with the desired state.
262285
// The function:
263286
// 1. First checks if a backoff period is needed (if previous builds failed)

‎coderd/prebuilds/preset_snapshot_test.go‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ import (
1010
"github.com/stretchr/testify/assert"
1111
"github.com/stretchr/testify/require"
1212

13+
"cdr.dev/slog/sloggers/slogtest"
14+
1315
"github.com/coder/coder/v2/coderd/database"
1416
"github.com/coder/coder/v2/coderd/prebuilds"
1517
"github.com/coder/coder/v2/testutil"
@@ -1527,6 +1529,152 @@ func TestCalculateDesiredInstances(t *testing.T) {
15271529
}
15281530
}
15291531

1532+
// TestCanSkipReconciliation ensures that CanSkipReconciliation only returns true
1533+
// when CalculateActions would return no actions.
1534+
funcTestCanSkipReconciliation(t*testing.T) {
1535+
t.Parallel()
1536+
1537+
clock:=quartz.NewMock(t)
1538+
logger:=slogtest.Make(t,nil)
1539+
backoffInterval:=5*time.Minute
1540+
1541+
tests:= []struct {
1542+
namestring
1543+
preset database.GetTemplatePresetsWithPrebuildsRow
1544+
running []database.GetRunningPrebuiltWorkspacesRow
1545+
expired []database.GetRunningPrebuiltWorkspacesRow
1546+
inProgress []database.CountInProgressPrebuildsRow
1547+
pendingCountint
1548+
backoff*database.GetPresetsBackoffRow
1549+
expectedCanSkipbool
1550+
expectedActionNoOpbool
1551+
}{
1552+
{
1553+
name:"inactive_with_nothing_to_cleanup",
1554+
preset: database.GetTemplatePresetsWithPrebuildsRow{
1555+
UsingActiveVersion:false,
1556+
Deleted:false,
1557+
Deprecated:false,
1558+
DesiredInstances: sql.NullInt32{Int32:5,Valid:true},
1559+
},
1560+
running: []database.GetRunningPrebuiltWorkspacesRow{},
1561+
expired: []database.GetRunningPrebuiltWorkspacesRow{},
1562+
inProgress: []database.CountInProgressPrebuildsRow{},
1563+
pendingCount:0,
1564+
backoff:nil,
1565+
expectedCanSkip:true,
1566+
expectedActionNoOp:true,
1567+
},
1568+
{
1569+
name:"inactive_with_running_workspaces",
1570+
preset: database.GetTemplatePresetsWithPrebuildsRow{
1571+
UsingActiveVersion:false,
1572+
Deleted:false,
1573+
Deprecated:false,
1574+
},
1575+
running: []database.GetRunningPrebuiltWorkspacesRow{
1576+
{ID:uuid.New()},
1577+
},
1578+
expired: []database.GetRunningPrebuiltWorkspacesRow{},
1579+
inProgress: []database.CountInProgressPrebuildsRow{},
1580+
pendingCount:0,
1581+
backoff:nil,
1582+
expectedCanSkip:false,
1583+
expectedActionNoOp:false,
1584+
},
1585+
{
1586+
name:"inactive_with_pending_jobs",
1587+
preset: database.GetTemplatePresetsWithPrebuildsRow{
1588+
UsingActiveVersion:false,
1589+
Deleted:false,
1590+
Deprecated:false,
1591+
},
1592+
running: []database.GetRunningPrebuiltWorkspacesRow{},
1593+
expired: []database.GetRunningPrebuiltWorkspacesRow{},
1594+
inProgress: []database.CountInProgressPrebuildsRow{},
1595+
pendingCount:3,
1596+
backoff:nil,
1597+
expectedCanSkip:false,
1598+
expectedActionNoOp:false,
1599+
},
1600+
{
1601+
name:"active_with_no_workspaces",
1602+
preset: database.GetTemplatePresetsWithPrebuildsRow{
1603+
UsingActiveVersion:true,
1604+
Deleted:false,
1605+
Deprecated:false,
1606+
DesiredInstances: sql.NullInt32{Int32:5,Valid:true},
1607+
},
1608+
running: []database.GetRunningPrebuiltWorkspacesRow{},
1609+
expired: []database.GetRunningPrebuiltWorkspacesRow{},
1610+
inProgress: []database.CountInProgressPrebuildsRow{},
1611+
pendingCount:0,
1612+
backoff:nil,
1613+
expectedCanSkip:false,
1614+
expectedActionNoOp:false,// Should create 5 workspaces
1615+
},
1616+
{
1617+
name:"active_with_backoff",
1618+
preset: database.GetTemplatePresetsWithPrebuildsRow{
1619+
UsingActiveVersion:true,
1620+
Deleted:false,
1621+
Deprecated:false,
1622+
DesiredInstances: sql.NullInt32{Int32:5,Valid:true},
1623+
},
1624+
running: []database.GetRunningPrebuiltWorkspacesRow{},
1625+
expired: []database.GetRunningPrebuiltWorkspacesRow{},
1626+
inProgress: []database.CountInProgressPrebuildsRow{},
1627+
pendingCount:0,
1628+
backoff:&database.GetPresetsBackoffRow{
1629+
NumFailed:3,
1630+
LastBuildAt:clock.Now().Add(-1*time.Minute),
1631+
},
1632+
expectedCanSkip:false,
1633+
expectedActionNoOp:false,// Should backoff
1634+
},
1635+
}
1636+
1637+
for_,tt:=rangetests {
1638+
t.Run(tt.name,func(t*testing.T) {
1639+
t.Parallel()
1640+
1641+
ps:=prebuilds.NewPresetSnapshot(
1642+
tt.preset,
1643+
[]database.TemplateVersionPresetPrebuildSchedule{},
1644+
tt.running,
1645+
tt.expired,
1646+
tt.inProgress,
1647+
tt.pendingCount,
1648+
tt.backoff,
1649+
false,
1650+
clock,
1651+
logger,
1652+
)
1653+
1654+
canSkip:=ps.CanSkipReconciliation()
1655+
require.Equal(t,tt.expectedCanSkip,canSkip)
1656+
1657+
actions,err:=ps.CalculateActions(backoffInterval)
1658+
require.NoError(t,err)
1659+
1660+
actionNoOp:=true
1661+
for_,action:=rangeactions {
1662+
if!action.IsNoop() {
1663+
actionNoOp=false
1664+
break
1665+
}
1666+
}
1667+
require.Equal(t,tt.expectedActionNoOp,actionNoOp,
1668+
"CalculateActions() isNoOp mismatch")
1669+
1670+
// IMPORTANT: If CanSkipReconciliation is true, CalculateActions must return no actions
1671+
ifcanSkip {
1672+
require.True(t,actionNoOp)
1673+
}
1674+
})
1675+
}
1676+
}
1677+
15301678
funcmustParseTime(t*testing.T,layout,valuestring) time.Time {
15311679
t.Helper()
15321680
parsedTime,err:=time.Parse(layout,value)

‎enterprise/coderd/prebuilds/reconcile.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -341,13 +341,26 @@ func (c *StoreReconciler) ReconcileAll(ctx context.Context) (stats prebuilds.Rec
341341
}
342342

343343
vareg errgroup.Group
344+
// Limit concurrent goroutines to avoid exhausting the database connection pool.
345+
// Each preset reconciliation may perform multiple sequential database operations
346+
// (creates/deletes), and with a pool limit of 10 connections, allowing unlimited
347+
// concurrency would cause connection contention and thrashing. A limit of 5 ensures
348+
// we stay below the pool limit while maintaining reasonable parallelism.
349+
eg.SetLimit(5)
350+
344351
// Reconcile presets in parallel. Each preset in its own goroutine.
345352
for_,preset:=rangesnapshot.Presets {
346353
ps,err:=snapshot.FilterByPreset(preset.ID)
347354
iferr!=nil {
348355
logger.Warn(ctx,"failed to find preset snapshot",slog.Error(err),slog.F("preset_id",preset.ID.String()))
349356
continue
350357
}
358+
// Performance optimization: Skip presets from inactive template versions that have
359+
// no running workspaces, pending jobs, or in-progress builds. This avoids spawning
360+
// goroutines for presets that don't need reconciliation actions.
361+
ifps.CanSkipReconciliation() {
362+
continue
363+
}
351364

352365
eg.Go(func()error {
353366
// Pass outer context.

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp