Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit8ccae5f

Browse files
committed
feat(coderd/database): adddbrollup service to rollup insights
1 parentcba0c08 commit8ccae5f

File tree

7 files changed

+350
-5
lines changed

7 files changed

+350
-5
lines changed

‎coderd/coderd.go

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ import (
4747
"github.com/coder/coder/v2/coderd/batchstats"
4848
"github.com/coder/coder/v2/coderd/database"
4949
"github.com/coder/coder/v2/coderd/database/dbauthz"
50+
"github.com/coder/coder/v2/coderd/database/dbrollup"
5051
"github.com/coder/coder/v2/coderd/database/dbtime"
5152
"github.com/coder/coder/v2/coderd/database/pubsub"
5253
"github.com/coder/coder/v2/coderd/externalauth"
@@ -180,6 +181,7 @@ type Options struct {
180181

181182
UpdateAgentMetricsfunc(ctx context.Context,labels prometheusmetrics.AgentMetricLabels,metrics []*agentproto.Stats_Metric)
182183
StatsBatcher*batchstats.Batcher
184+
DBRollupInterval time.Duration
183185

184186
WorkspaceAppsStatsCollectorOptions workspaceapps.StatsCollectorOptions
185187

@@ -342,6 +344,9 @@ func New(options *Options) *API {
342344
ifoptions.StatsBatcher==nil {
343345
panic("developer error: options.StatsBatcher is nil")
344346
}
347+
ifoptions.DBRollupInterval==0 {
348+
options.DBRollupInterval=dbrollup.DefaultInterval
349+
}
345350

346351
siteCacheDir:=options.CacheDir
347352
ifsiteCacheDir!="" {
@@ -414,7 +419,13 @@ func New(options *Options) *API {
414419
ctx,
415420
options.Logger.Named("acquirer"),
416421
options.Database,
417-
options.Pubsub),
422+
options.Pubsub,
423+
),
424+
rolluper:dbrollup.New(
425+
options.Logger,
426+
options.Database,
427+
options.DBRollupInterval,
428+
),
418429
workspaceUsageTracker:options.WorkspaceUsageTracker,
419430
}
420431

@@ -1190,7 +1201,9 @@ type API struct {
11901201
statsBatcher*batchstats.Batcher
11911202

11921203
Acquirer*provisionerdserver.Acquirer
1193-
1204+
// rolluper rolls up template usage stats from raw agent and app
1205+
// stats. This is used to provide insights in the WebUI.
1206+
rolluper*dbrollup.Rolluper
11941207
workspaceUsageTracker*workspaceusage.Tracker
11951208
}
11961209

@@ -1203,6 +1216,7 @@ func (api *API) Close() error {
12031216
api.WebsocketWaitGroup.Wait()
12041217
api.WebsocketWaitMutex.Unlock()
12051218

1219+
api.rolluper.Close()
12061220
api.metricsCache.Close()
12071221
ifapi.updateChecker!=nil {
12081222
api.updateChecker.Close()

‎coderd/coderdtest/coderdtest.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@ type Options struct {
147147
WorkspaceAppsStatsCollectorOptions workspaceapps.StatsCollectorOptions
148148
AllowWorkspaceRenamesbool
149149
NewTickerfunc(duration time.Duration) (<-chan time.Time,func())
150+
DBRollupInterval time.Duration
150151
WorkspaceUsageTrackerFlushchanint
151152
WorkspaceUsageTrackerTickchan time.Time
152153
}
@@ -487,6 +488,7 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can
487488
WorkspaceAppsStatsCollectorOptions:options.WorkspaceAppsStatsCollectorOptions,
488489
AllowWorkspaceRenames:options.AllowWorkspaceRenames,
489490
NewTicker:options.NewTicker,
491+
DBRollupInterval:options.DBRollupInterval,
490492
WorkspaceUsageTracker:wuTracker,
491493
}
492494
}

‎coderd/database/dbgen/dbgen.go

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -489,6 +489,38 @@ func WorkspaceApp(t testing.TB, db database.Store, orig database.WorkspaceApp) d
489489
returnresource
490490
}
491491

492+
funcWorkspaceAppStat(t testing.TB,db database.Store,orig database.WorkspaceAppStat) database.WorkspaceAppStat {
493+
// This is not going to be correct, but our query doesn't return the ID.
494+
id,err:=cryptorand.Int63()
495+
require.NoError(t,err,"generate id")
496+
497+
scheme:= database.WorkspaceAppStat{
498+
ID:takeFirst(orig.ID,id),
499+
UserID:takeFirst(orig.UserID,uuid.New()),
500+
WorkspaceID:takeFirst(orig.WorkspaceID,uuid.New()),
501+
AgentID:takeFirst(orig.AgentID,uuid.New()),
502+
AccessMethod:takeFirst(orig.AccessMethod,""),
503+
SlugOrPort:takeFirst(orig.SlugOrPort,""),
504+
SessionID:takeFirst(orig.SessionID,uuid.New()),
505+
SessionStartedAt:takeFirst(orig.SessionStartedAt,dbtime.Now().Add(-time.Minute)),
506+
SessionEndedAt:takeFirst(orig.SessionEndedAt,dbtime.Now()),
507+
Requests:takeFirst(orig.Requests,1),
508+
}
509+
err=db.InsertWorkspaceAppStats(genCtx, database.InsertWorkspaceAppStatsParams{
510+
UserID: []uuid.UUID{scheme.UserID},
511+
WorkspaceID: []uuid.UUID{scheme.WorkspaceID},
512+
AgentID: []uuid.UUID{scheme.AgentID},
513+
AccessMethod: []string{scheme.AccessMethod},
514+
SlugOrPort: []string{scheme.SlugOrPort},
515+
SessionID: []uuid.UUID{scheme.SessionID},
516+
SessionStartedAt: []time.Time{scheme.SessionStartedAt},
517+
SessionEndedAt: []time.Time{scheme.SessionEndedAt},
518+
Requests: []int32{scheme.Requests},
519+
})
520+
require.NoError(t,err,"insert workspace agent stat")
521+
returnscheme
522+
}
523+
492524
funcWorkspaceResource(t testing.TB,db database.Store,orig database.WorkspaceResource) database.WorkspaceResource {
493525
resource,err:=db.InsertWorkspaceResource(genCtx, database.InsertWorkspaceResourceParams{
494526
ID:takeFirst(orig.ID,uuid.New()),

‎coderd/database/dbpurge/dbpurge.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ const (
2424
// This is for cleaning up old, unused resources from the database that take up space.
2525
funcNew(ctx context.Context,logger slog.Logger,db database.Store) io.Closer {
2626
closed:=make(chanstruct{})
27+
logger=logger.Named("dbpurge")
2728

2829
ctx,cancelFunc:=context.WithCancel(ctx)
2930
//nolint:gocritic // The system purges old db records without user input.

‎coderd/database/dbrollup/dbrollup.go

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
package dbrollup
2+
3+
import (
4+
"context"
5+
"time"
6+
7+
"golang.org/x/sync/errgroup"
8+
9+
"cdr.dev/slog"
10+
11+
"github.com/coder/coder/v2/coderd/database"
12+
"github.com/coder/coder/v2/coderd/database/dbauthz"
13+
)
14+
15+
const (
16+
// DefaultInterval is the default time between rollups.
17+
// Rollups will be synchronized with the clock so that
18+
// they happen 13:00, 13:05, 13:10, etc.
19+
DefaultInterval=5*time.Minute
20+
)
21+
22+
typeRolluperstruct {
23+
cancel context.CancelFunc
24+
closedchanstruct{}
25+
db database.Store
26+
logger slog.Logger
27+
}
28+
29+
// New creates a new DB rollup service that periodically runs rollup queries.
30+
// It is the caller's responsibility to call Close on the returned instance.
31+
//
32+
// This is for e.g. generating insights data (template_usage_stats) from
33+
// raw data (workspace_agent_stats, workspace_app_stats).
34+
funcNew(logger slog.Logger,db database.Store,interval time.Duration)*Rolluper {
35+
ctx,cancel:=context.WithCancel(context.Background())
36+
37+
r:=&Rolluper{
38+
cancel:cancel,
39+
closed:make(chanstruct{}),
40+
db:db,
41+
logger:logger.Named("dbrollup"),
42+
}
43+
44+
//nolint:gocritic // The system rolls up database tables without user input.
45+
ctx=dbauthz.AsSystemRestricted(ctx)
46+
gor.start(ctx,interval)
47+
48+
returnr
49+
}
50+
51+
func (r*Rolluper)start(ctx context.Context,interval time.Duration) {
52+
deferclose(r.closed)
53+
54+
do:=func() {
55+
vareg errgroup.Group
56+
57+
r.logger.Debug(ctx,"rolling up data")
58+
now:=time.Now()
59+
60+
// Track whether or not we performed a rollup (we got the advisory lock).
61+
templateUsageStats:=false
62+
63+
eg.Go(func()error {
64+
returnr.db.InTx(func(tx database.Store)error {
65+
// Acquire a lock to ensure that only one instance of
66+
// the rollup is running at a time.
67+
ok,err:=tx.TryAcquireLock(ctx,database.LockIDDBRollup)
68+
iferr!=nil {
69+
returnerr
70+
}
71+
if!ok {
72+
returnnil
73+
}
74+
75+
templateUsageStats=true
76+
returntx.UpsertTemplateUsageStats(ctx)
77+
},nil)
78+
})
79+
80+
err:=eg.Wait()
81+
iferr!=nil {
82+
ifdatabase.IsQueryCanceledError(err) {
83+
return
84+
}
85+
// Only log if Close hasn't been called.
86+
ifctx.Err()==nil {
87+
r.logger.Error(ctx,"failed to rollup data",slog.Error(err))
88+
}
89+
}else {
90+
r.logger.Debug(ctx,
91+
"rolled up data",
92+
slog.F("took",time.Since(now)),
93+
slog.F("template_usage_stats",templateUsageStats),
94+
)
95+
}
96+
}
97+
98+
// Perform do immediately and on every tick of the ticker,
99+
// disregarding the execution time of do. This ensure that
100+
// the rollup is performed every interval assuming do does
101+
// not take longer than the interval to execute.
102+
t:=time.NewTicker(time.Microsecond)
103+
defert.Stop()
104+
for {
105+
select {
106+
case<-ctx.Done():
107+
return
108+
case<-t.C:
109+
// Ensure we're on the interval.
110+
now:=time.Now()
111+
next:=now.Add(interval).Truncate(interval)// Ensure we're on the interval and synced with the clock.
112+
d:=next.Sub(now)
113+
// Safety check (shouldn't be possible).
114+
ifd<=0 {
115+
d=interval
116+
}
117+
t.Reset(d)
118+
119+
do()
120+
121+
r.logger.Debug(ctx,"next rollup at",slog.F("next",next))
122+
}
123+
}
124+
}
125+
126+
func (r*Rolluper)Close()error {
127+
r.cancel()
128+
<-r.closed
129+
returnnil
130+
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp