Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit562b56d

Browse files
committed
feat: fetch prebuilds metrics state in background
Signed-off-by: Danny Kopping <dannykopping@gmail.com>
1 parent7f056da commit562b56d

File tree

3 files changed

+84
-17
lines changed

3 files changed

+84
-17
lines changed

‎enterprise/coderd/prebuilds/metricscollector.go

Lines changed: 74 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,13 @@ package prebuilds
22

33
import (
44
"context"
5+
"sync/atomic"
56
"time"
67

7-
"cdr.dev/slog"
8-
98
"github.com/prometheus/client_golang/prometheus"
9+
"golang.org/x/xerrors"
10+
11+
"cdr.dev/slog"
1012

1113
"github.com/coder/coder/v2/coderd/database"
1214
"github.com/coder/coder/v2/coderd/database/dbauthz"
@@ -57,18 +59,27 @@ var (
5759
)
5860
)
5961

62+
const (
63+
metricsUpdateInterval=time.Second*15
64+
metricsUpdateTimeout=time.Second*10
65+
)
66+
6067
typeMetricsCollectorstruct {
6168
database database.Store
6269
logger slog.Logger
6370
snapshotter prebuilds.StateSnapshotter
71+
72+
latestState atomic.Pointer[state]
6473
}
6574

6675
var_ prometheus.Collector=new(MetricsCollector)
6776

77+
// NewMetricsCollector returns a
6878
funcNewMetricsCollector(db database.Store,logger slog.Logger,snapshotter prebuilds.StateSnapshotter)*MetricsCollector {
79+
log:=logger.Named("prebuilds_metrics_collector")
6980
return&MetricsCollector{
7081
database:db,
71-
logger:logger.Named("prebuilds_metrics_collector"),
82+
logger:log,
7283
snapshotter:snapshotter,
7384
}
7485
}
@@ -82,34 +93,31 @@ func (*MetricsCollector) Describe(descCh chan<- *prometheus.Desc) {
8293
descCh<-eligiblePrebuildsDesc
8394
}
8495

96+
// Collect uses the cached state to set configured metrics.
97+
// The state is cached because this function can be called multiple times per second and retrieving the current state
98+
// is an expensive operation.
8599
func (mc*MetricsCollector)Collect(metricsChchan<- prometheus.Metric) {
86100
// nolint:gocritic // We need to set an authz context to read metrics from the db.
87-
ctx,cancel:=context.WithTimeout(dbauthz.AsPrebuildsOrchestrator(context.Background()),10*time.Second)
88-
defercancel()
89-
prebuildMetrics,err:=mc.database.GetPrebuildMetrics(ctx)
90-
iferr!=nil {
91-
mc.logger.Error(ctx,"failed toget prebuild metrics",slog.Error(err))
101+
ctx:=dbauthz.AsPrebuildsOrchestrator(context.Background())
102+
103+
currentState:=mc.latestState.Load()
104+
ifcurrentState==nil {
105+
mc.logger.Warn(ctx,"failed toset prebuilds metrics; state not set")
92106
return
93107
}
94108

95-
for_,metric:=rangeprebuildMetrics {
109+
for_,metric:=rangecurrentState.prebuildMetrics {
96110
metricsCh<-prometheus.MustNewConstMetric(createdPrebuildsDesc,prometheus.CounterValue,float64(metric.CreatedCount),metric.TemplateName,metric.PresetName,metric.OrganizationName)
97111
metricsCh<-prometheus.MustNewConstMetric(failedPrebuildsDesc,prometheus.CounterValue,float64(metric.FailedCount),metric.TemplateName,metric.PresetName,metric.OrganizationName)
98112
metricsCh<-prometheus.MustNewConstMetric(claimedPrebuildsDesc,prometheus.CounterValue,float64(metric.ClaimedCount),metric.TemplateName,metric.PresetName,metric.OrganizationName)
99113
}
100114

101-
snapshot,err:=mc.snapshotter.SnapshotState(ctx,mc.database)
102-
iferr!=nil {
103-
mc.logger.Error(ctx,"failed to get latest prebuild state",slog.Error(err))
104-
return
105-
}
106-
107-
for_,preset:=rangesnapshot.Presets {
115+
for_,preset:=rangecurrentState.snapshot.Presets {
108116
if!preset.UsingActiveVersion {
109117
continue
110118
}
111119

112-
presetSnapshot,err:=snapshot.FilterByPreset(preset.ID)
120+
presetSnapshot,err:=currentState.snapshot.FilterByPreset(preset.ID)
113121
iferr!=nil {
114122
mc.logger.Error(ctx,"failed to filter by preset",slog.Error(err))
115123
continue
@@ -121,3 +129,52 @@ func (mc *MetricsCollector) Collect(metricsCh chan<- prometheus.Metric) {
121129
metricsCh<-prometheus.MustNewConstMetric(eligiblePrebuildsDesc,prometheus.GaugeValue,float64(state.Eligible),preset.TemplateName,preset.Name,preset.OrganizationName)
122130
}
123131
}
132+
133+
typestatestruct {
134+
prebuildMetrics []database.GetPrebuildMetricsRow
135+
snapshot*prebuilds.GlobalSnapshot
136+
}
137+
138+
// BackgroundFetch updates the metrics state every given interval.
139+
func (mc*MetricsCollector)BackgroundFetch(ctx context.Context,updateInterval,updateTimeout time.Duration) {
140+
tick:=time.NewTicker(time.Nanosecond)
141+
defertick.Stop()
142+
143+
for {
144+
select {
145+
case<-ctx.Done():
146+
return
147+
case<-tick.C:
148+
// Tick immediately, then set regular interval.
149+
tick.Reset(updateInterval)
150+
151+
iferr:=mc.UpdateState(ctx,updateTimeout);err!=nil {
152+
mc.logger.Error(ctx,"failed to update prebuilds metrics state",slog.Error(err))
153+
}
154+
}
155+
}
156+
}
157+
158+
// UpdateState builds the current metrics state.
159+
func (mc*MetricsCollector)UpdateState(ctx context.Context,timeout time.Duration)error {
160+
mc.logger.Debug(ctx,"fetching prebuilds metrics state")
161+
fetchCtx,fetchCancel:=context.WithTimeout(ctx,timeout)
162+
deferfetchCancel()
163+
164+
prebuildMetrics,err:=mc.database.GetPrebuildMetrics(fetchCtx)
165+
iferr!=nil {
166+
returnxerrors.Errorf("fetch prebuild metrics: %w",err)
167+
}
168+
169+
snapshot,err:=mc.snapshotter.SnapshotState(fetchCtx,mc.database)
170+
iferr!=nil {
171+
returnxerrors.Errorf("snapshot state: %w",err)
172+
}
173+
mc.logger.Debug(ctx,"fetched prebuilds metrics state")
174+
175+
mc.latestState.Store(&state{
176+
prebuildMetrics:prebuildMetrics,
177+
snapshot:snapshot,
178+
})
179+
returnnil
180+
}

‎enterprise/coderd/prebuilds/metricscollector_test.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"github.com/coder/quartz"
1717

1818
"github.com/coder/coder/v2/coderd/database"
19+
"github.com/coder/coder/v2/coderd/database/dbauthz"
1920
"github.com/coder/coder/v2/coderd/database/dbgen"
2021
"github.com/coder/coder/v2/coderd/database/dbtestutil"
2122
agplprebuilds"github.com/coder/coder/v2/coderd/prebuilds"
@@ -248,6 +249,10 @@ func TestMetricsCollector(t *testing.T) {
248249
setupTestDBWorkspaceAgent(t,db,workspace.ID,eligible)
249250
}
250251

252+
// Force an update to the metrics state to allow the collector to collect fresh metrics.
253+
// nolint:gocritic // Authz context needed to retrieve state.
254+
require.NoError(t,collector.UpdateState(dbauthz.AsPrebuildsOrchestrator(ctx),testutil.WaitLong))
255+
251256
metricsFamilies,err:=registry.Gather()
252257
require.NoError(t,err)
253258

‎enterprise/coderd/prebuilds/reconcile.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,11 @@ func (c *StoreReconciler) Run(ctx context.Context) {
9797
ctx,cancel:=context.WithCancelCause(dbauthz.AsPrebuildsOrchestrator(ctx))
9898
c.cancelFn=cancel
9999

100+
// Start updating metrics in the background.
101+
ifc.metrics!=nil {
102+
goc.metrics.BackgroundFetch(ctx,metricsUpdateInterval,metricsUpdateTimeout)
103+
}
104+
100105
// Everything is in place, reconciler can now be considered as running.
101106
//
102107
// NOTE: without this atomic bool, Stop might race with Run for the c.cancelFn above.

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp