@@ -2,6 +2,9 @@ package prebuilds
22
33import (
44"context"
5+ "fmt"
6+ "sync"
7+ "sync/atomic"
58"time"
69
710"cdr.dev/slog"
3538labels ,
3639nil ,
3740)
41+ resourceReplacementsDesc = prometheus .NewDesc (
42+ "coderd_prebuilt_workspaces_resource_replacements_total" ,
43+ "Total number of prebuilt workspaces whose resource(s) got replaced upon being claimed. " +
44+ "In Terraform, drift on immutable attributes results in resource replacement. " +
45+ "This represents a worst-case scenario for prebuilt workspaces because the pre-provisioned resource " +
46+ "would have been recreated when claiming, thus obviating the point of pre-provisioning. " +
47+ "See https://coder.com/docs/admin/templates/extending-templates/prebuilt-workspaces.md#preventing-resource-replacement" ,
48+ labels ,
49+ nil ,
50+ )
3851desiredPrebuildsDesc = prometheus .NewDesc (
3952"coderd_prebuilt_workspaces_desired" ,
4053"Target number of prebuilt workspaces that should be available for each template preset." ,
@@ -61,22 +74,27 @@ type MetricsCollector struct {
6174database database.Store
6275logger slog.Logger
6376snapshotter prebuilds.StateSnapshotter
77+
78+ replacementsCounter map [replacementKey ]* atomic.Int64
79+ replacementsCounterMu sync.Mutex
6480}
6581
6682var _ prometheus.Collector = new (MetricsCollector )
6783
6884func NewMetricsCollector (db database.Store ,logger slog.Logger ,snapshotter prebuilds.StateSnapshotter )* MetricsCollector {
6985return & MetricsCollector {
70- database :db ,
71- logger :logger .Named ("prebuilds_metrics_collector" ),
72- snapshotter :snapshotter ,
86+ database :db ,
87+ logger :logger .Named ("prebuilds_metrics_collector" ),
88+ snapshotter :snapshotter ,
89+ replacementsCounter :make (map [replacementKey ]* atomic.Int64 ),
7390}
7491}
7592
7693func (* MetricsCollector )Describe (descCh chan <- * prometheus.Desc ) {
7794descCh <- createdPrebuildsDesc
7895descCh <- failedPrebuildsDesc
7996descCh <- claimedPrebuildsDesc
97+ descCh <- resourceReplacementsDesc
8098descCh <- desiredPrebuildsDesc
8199descCh <- runningPrebuildsDesc
82100descCh <- eligiblePrebuildsDesc
@@ -98,6 +116,12 @@ func (mc *MetricsCollector) Collect(metricsCh chan<- prometheus.Metric) {
98116metricsCh <- prometheus .MustNewConstMetric (claimedPrebuildsDesc ,prometheus .CounterValue ,float64 (metric .ClaimedCount ),metric .TemplateName ,metric .PresetName ,metric .OrganizationName )
99117}
100118
119+ mc .replacementsCounterMu .Lock ()
120+ for key ,val := range mc .replacementsCounter {
121+ metricsCh <- prometheus .MustNewConstMetric (resourceReplacementsDesc ,prometheus .CounterValue ,float64 (val .Load ()),key .templateName ,key .presetName ,key .orgName )
122+ }
123+ mc .replacementsCounterMu .Unlock ()
124+
101125snapshot ,err := mc .snapshotter .SnapshotState (ctx ,mc .database )
102126if err != nil {
103127mc .logger .Error (ctx ,"failed to get latest prebuild state" ,slog .Error (err ))
@@ -121,3 +145,22 @@ func (mc *MetricsCollector) Collect(metricsCh chan<- prometheus.Metric) {
121145metricsCh <- prometheus .MustNewConstMetric (eligiblePrebuildsDesc ,prometheus .GaugeValue ,float64 (state .Eligible ),preset .TemplateName ,preset .Name ,preset .OrganizationName )
122146}
123147}
148+
149+ type replacementKey struct {
150+ orgName ,templateName ,presetName string
151+ }
152+
153+ func (k replacementKey )String ()string {
154+ return fmt .Sprintf ("%s:%s:%s" ,k .orgName ,k .templateName ,k .presetName )
155+ }
156+
157+ func (mc * MetricsCollector )trackResourceReplacement (orgName ,templateName ,presetName string ) {
158+ mc .replacementsCounterMu .Lock ()
159+ defer mc .replacementsCounterMu .Unlock ()
160+
161+ key := replacementKey {orgName :orgName ,templateName :templateName ,presetName :presetName }
162+ if _ ,ok := mc .replacementsCounter [key ];! ok {
163+ mc .replacementsCounter [key ]= & atomic.Int64 {}
164+ }
165+ mc .replacementsCounter [key ].Add (1 )
166+ }