Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitb63a2e0

Browse files
committed
review
1 parent0e592b2 commitb63a2e0

File tree

6 files changed

+179
-97
lines changed

6 files changed

+179
-97
lines changed

‎scaletest/autostart/config.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,8 @@ func (c Config) Validate() error {
6363
returnxerrors.New("workspace_job_timeout must be greater than 0")
6464
}
6565

66-
ifc.AutostartDelay<=0 {
67-
returnxerrors.New("autostart_delay must begreater than 0")
66+
ifc.AutostartDelay<time.Minute*2 {
67+
returnxerrors.New("autostart_delay must beat least 2 minutes")
6868
}
6969

7070
ifc.AutostartTimeout<=0 {

‎scaletest/autostart/metrics.go‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,30 @@ import (
77
)
88

99
typeMetricsstruct {
10-
AutostartLatencySeconds prometheus.HistogramVec
11-
AutostartErrorsTotal prometheus.CounterVec
10+
AutostartJobCreationLatencySeconds prometheus.HistogramVec
11+
AutostartJobAcquiredLatencySeconds prometheus.HistogramVec
12+
AutostartTotalLatencySeconds prometheus.HistogramVec
13+
AutostartErrorsTotal prometheus.CounterVec
1214
}
1315

1416
funcNewMetrics(reg prometheus.Registerer)*Metrics {
1517
m:=&Metrics{
16-
AutostartLatencySeconds:*prometheus.NewHistogramVec(prometheus.HistogramOpts{
18+
AutostartJobCreationLatencySeconds:*prometheus.NewHistogramVec(prometheus.HistogramOpts{
1719
Namespace:"coderd",
1820
Subsystem:"scaletest",
19-
Name:"autostart_latency_seconds",
21+
Name:"autostart_job_creation_latency_seconds",
22+
Help:"Time from when the workspace is scheduled to be autostarted to when the autostart job has been created.",
23+
}, []string{"username","workspace_name"}),
24+
AutostartJobAcquiredLatencySeconds:*prometheus.NewHistogramVec(prometheus.HistogramOpts{
25+
Namespace:"coderd",
26+
Subsystem:"scaletest",
27+
Name:"autostart_job_acquired_latency_seconds",
28+
Help:"Time from when the autostart job is created to when the autostart job has been acquired by a provisioner daemon.",
29+
}, []string{"username","workspace_name"}),
30+
AutostartTotalLatencySeconds:*prometheus.NewHistogramVec(prometheus.HistogramOpts{
31+
Namespace:"coderd",
32+
Subsystem:"scaletest",
33+
Name:"autostart_total_latency_seconds",
2034
Help:"Time from when the workspace is scheduled to be autostarted to when the autostart build has finished.",
2135
}, []string{"username","workspace_name"}),
2236
AutostartErrorsTotal:*prometheus.NewCounterVec(prometheus.CounterOpts{
@@ -27,13 +41,23 @@ func NewMetrics(reg prometheus.Registerer) *Metrics {
2741
}, []string{"username","action"}),
2842
}
2943

30-
reg.MustRegister(m.AutostartLatencySeconds)
44+
reg.MustRegister(m.AutostartTotalLatencySeconds)
45+
reg.MustRegister(m.AutostartJobCreationLatencySeconds)
46+
reg.MustRegister(m.AutostartJobAcquiredLatencySeconds)
3147
reg.MustRegister(m.AutostartErrorsTotal)
3248
returnm
3349
}
3450

3551
func (m*Metrics)RecordCompletion(elapsed time.Duration,usernamestring,workspacestring) {
36-
m.AutostartLatencySeconds.WithLabelValues(username,workspace).Observe(elapsed.Seconds())
52+
m.AutostartTotalLatencySeconds.WithLabelValues(username,workspace).Observe(elapsed.Seconds())
53+
}
54+
55+
func (m*Metrics)RecordJobCreation(elapsed time.Duration,usernamestring,workspacestring) {
56+
m.AutostartJobCreationLatencySeconds.WithLabelValues(username,workspace).Observe(elapsed.Seconds())
57+
}
58+
59+
func (m*Metrics)RecordJobAcquired(elapsed time.Duration,usernamestring,workspacestring) {
60+
m.AutostartJobAcquiredLatencySeconds.WithLabelValues(username,workspace).Observe(elapsed.Seconds())
3761
}
3862

3963
func (m*Metrics)AddError(usernamestring,actionstring) {

‎scaletest/autostart/run.go‎

Lines changed: 92 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@ type Runner struct {
2525
createUserRunner*createusers.Runner
2626
workspacebuildRunner*workspacebuild.Runner
2727

28-
autostartDuration time.Duration
28+
autostartTotalLatency time.Duration
29+
autostartJobCreationLatency time.Duration
30+
autostartJobAcquiredLatency time.Duration
2931
// Closed when the autostart schedule has been set.
3032
// Used in tests.
3133
autostartSetchanstruct{}
@@ -49,6 +51,13 @@ func (r *Runner) Run(ctx context.Context, id string, logs io.Writer) error {
4951
ctx,span:=tracing.StartSpan(ctx)
5052
deferspan.End()
5153

54+
reachedBarrier:=false
55+
deferfunc() {
56+
if!reachedBarrier {
57+
r.cfg.SetupBarrier.Done()
58+
}
59+
}()
60+
5261
logs=loadtestutil.NewSyncWriter(logs)
5362
logger:=slog.Make(sloghuman.Sink(logs)).Leveled(slog.LevelDebug)
5463
r.client.SetLogger(logger)
@@ -72,6 +81,8 @@ func (r *Runner) Run(ctx context.Context, id string, logs io.Writer) error {
7281
workspaceBuildConfig:=r.cfg.Workspace
7382
workspaceBuildConfig.OrganizationID=r.cfg.User.OrganizationID
7483
workspaceBuildConfig.UserID=newUser.ID.String()
84+
// We'll wait for the build ourselves to avoid multiple API requests
85+
workspaceBuildConfig.NoWaitForBuild=true
7586

7687
r.workspacebuildRunner=workspacebuild.NewRunner(newUserClient,workspaceBuildConfig)
7788
workspace,err:=r.workspacebuildRunner.RunReturningWorkspace(ctx,id,logs)
@@ -80,36 +91,60 @@ func (r *Runner) Run(ctx context.Context, id string, logs io.Writer) error {
8091
returnxerrors.Errorf("create workspace: %w",err)
8192
}
8293

83-
logger.Info(ctx,fmt.Sprintf("workspace %q created",workspace.Name))
94+
watchCtx,cancel:=context.WithCancel(ctx)
95+
defercancel()
96+
workspaceUpdates,err:=newUserClient.WatchWorkspace(watchCtx,workspace.ID)
97+
iferr!=nil {
98+
r.cfg.Metrics.AddError(newUser.Username,"watch_workspace")
99+
returnxerrors.Errorf("watch workspace: %w",err)
100+
}
101+
102+
createWorkspaceCtx,cancel2:=context.WithTimeout(ctx,r.cfg.WorkspaceJobTimeout)
103+
defercancel2()
104+
105+
err=waitForWorkspaceUpdate(createWorkspaceCtx,logger,workspaceUpdates,func(ws codersdk.Workspace)bool {
106+
returnws.LatestBuild.Transition==codersdk.WorkspaceTransitionStart&&
107+
ws.LatestBuild.Job.Status==codersdk.ProvisionerJobSucceeded
108+
})
109+
iferr!=nil {
110+
r.cfg.Metrics.AddError(newUser.Username,"wait_for_initial_build")
111+
returnxerrors.Errorf("timeout waiting for initial workspace build to complete: %w",err)
112+
}
84113

85114
logger.Info(ctx,fmt.Sprintf("stopping workspace %q",workspace.Name))
86115

87-
stopBuild,err:=newUserClient.CreateWorkspaceBuild(ctx,workspace.ID, codersdk.CreateWorkspaceBuildRequest{
116+
_,err=newUserClient.CreateWorkspaceBuild(ctx,workspace.ID, codersdk.CreateWorkspaceBuildRequest{
88117
Transition:codersdk.WorkspaceTransitionStop,
89118
})
90119
iferr!=nil {
91120
r.cfg.Metrics.AddError(newUser.Username,"create_stop_build")
92121
returnxerrors.Errorf("create stop build: %w",err)
93122
}
94123

95-
stopBuildCtx,cancel2:=context.WithTimeout(ctx,r.cfg.WorkspaceJobTimeout)
96-
defercancel2()
124+
stopBuildCtx,cancel3:=context.WithTimeout(ctx,r.cfg.WorkspaceJobTimeout)
125+
defercancel3()
97126

98-
err=workspacebuild.WaitForBuild(stopBuildCtx,logs,newUserClient,stopBuild.ID)
127+
err=waitForWorkspaceUpdate(stopBuildCtx,logger,workspaceUpdates,func(ws codersdk.Workspace)bool {
128+
returnws.LatestBuild.Transition==codersdk.WorkspaceTransitionStop&&
129+
ws.LatestBuild.Job.Status==codersdk.ProvisionerJobSucceeded
130+
})
99131
iferr!=nil {
100132
r.cfg.Metrics.AddError(newUser.Username,"wait_for_stop_build")
101-
returnxerrors.Errorf("wait for stop build to complete: %w",err)
133+
returnxerrors.Errorf("timeout waiting for stop build to complete: %w",err)
102134
}
103135

104136
logger.Info(ctx,fmt.Sprintf("workspace %q stopped successfully",workspace.Name))
105137

106138
logger.Info(ctx,"waiting for all runners to reach barrier")
139+
reachedBarrier=true
107140
r.cfg.SetupBarrier.Done()
108141
r.cfg.SetupBarrier.Wait()
109142
logger.Info(ctx,"all runners reached barrier, proceeding with autostart schedule")
110143

111-
autoStartTime:=r.cfg.Clock.Now().Add(r.cfg.AutostartDelay)
112-
schedule:=fmt.Sprintf("CRON_TZ=UTC %d %d * * *",autoStartTime.Minute(),autoStartTime.Hour())
144+
testStartTime:=r.cfg.Clock.Now()
145+
autostartTime:=testStartTime.Add(r.cfg.AutostartDelay).Round(time.Minute)
146+
timeUntilAutostart:=autostartTime.Sub(testStartTime)
147+
schedule:=fmt.Sprintf("CRON_TZ=UTC %d %d * * *",autostartTime.Minute(),autostartTime.Hour())
113148

114149
logger.Info(ctx,fmt.Sprintf("setting autostart schedule for workspace %q: %s",workspace.Name,schedule))
115150

@@ -122,61 +157,66 @@ func (r *Runner) Run(ctx context.Context, id string, logs io.Writer) error {
122157
}
123158
close(r.autostartSet)
124159

125-
logger.Info(ctx,fmt.Sprintf("autostart schedule set for workspace %q",workspace.Name))
126-
127160
logger.Info(ctx,fmt.Sprintf("waiting for workspace %q to autostart",workspace.Name))
128161

129-
autostartInitiateCtx,cancel2:=context.WithTimeout(ctx,r.cfg.AutostartTimeout+r.cfg.AutostartDelay)
130-
defercancel2()
162+
autostartInitiateCtx,cancel4:=context.WithTimeout(ctx,timeUntilAutostart+r.cfg.AutostartTimeout)
163+
defercancel4()
131164

132-
workspaceUpdates,err:=newUserClient.WatchWorkspace(autostartInitiateCtx,workspace.ID)
133-
iferr!=nil {
134-
r.cfg.Metrics.AddError(newUser.Username,"watch_workspace")
135-
returnxerrors.Errorf("watch workspace: %w",err)
136-
}
165+
logger.Info(ctx,"listening for workspace updates to detect autostart build")
137166

138-
varautoStartBuild codersdk.WorkspaceBuild
167+
err=waitForWorkspaceUpdate(autostartInitiateCtx,logger,workspaceUpdates,func(ws codersdk.Workspace)bool {
168+
ifws.LatestBuild.Transition!=codersdk.WorkspaceTransitionStart {
169+
returnfalse
170+
}
139171

140-
logger.Info(ctx,"listening for workspace updates to detect autostart build")
141-
waitNewBuildLoop:
142-
for {
143-
select {
144-
case<-autostartInitiateCtx.Done():
145-
returnxerrors.Errorf("timeout waiting for autostart build to be created: %w",autostartInitiateCtx.Err())
146-
caseupdatedWorkspace,ok:=<-workspaceUpdates:
147-
if!ok {
148-
r.cfg.Metrics.AddError(newUser.Username,"workspace_updates_channel_closed")
149-
returnxerrors.Errorf("workspace updates channel closed")
150-
}
172+
// The job has been created, but it might be pending
173+
ifr.autostartJobCreationLatency==0 {
174+
r.autostartJobCreationLatency=r.cfg.Clock.Since(autostartTime)
175+
r.cfg.Metrics.RecordJobCreation(r.autostartJobCreationLatency,newUser.Username,workspace.Name)
176+
}
151177

152-
ifupdatedWorkspace.LatestBuild.ID!=stopBuild.ID&&
153-
updatedWorkspace.LatestBuild.Transition==codersdk.WorkspaceTransitionStart {
154-
autoStartBuild=updatedWorkspace.LatestBuild
155-
logger.Info(ctx,fmt.Sprintf("autostart build created with ID %s",autoStartBuild.ID))
156-
break waitNewBuildLoop
178+
ifws.LatestBuild.Job.Status==codersdk.ProvisionerJobRunning||
179+
ws.LatestBuild.Job.Status==codersdk.ProvisionerJobSucceeded {
180+
// Job is no longer pending, but it might not have finished
181+
ifr.autostartJobAcquiredLatency==0 {
182+
r.autostartJobAcquiredLatency=r.cfg.Clock.Since(autostartTime)
183+
r.cfg.Metrics.RecordJobAcquired(r.autostartJobAcquiredLatency,newUser.Username,workspace.Name)
157184
}
185+
returnws.LatestBuild.Job.Status==codersdk.ProvisionerJobSucceeded
158186
}
159-
}
160-
161-
logger.Info(ctx,"waiting for autostart build to complete")
162-
buildCompleteCtx,cancel3:=context.WithTimeout(ctx,r.cfg.WorkspaceJobTimeout)
163-
defercancel3()
164187

165-
err=workspacebuild.WaitForBuild(buildCompleteCtx,logs,newUserClient,autoStartBuild.ID)
188+
returnfalse
189+
})
166190
iferr!=nil {
167191
r.cfg.Metrics.AddError(newUser.Username,"wait_for_autostart_build")
168-
returnxerrors.Errorf("waitfor autostart build tocomplete: %w",err)
192+
returnxerrors.Errorf("timeout waitingfor autostart build tobe created: %w",err)
169193
}
170194

171-
r.autostartDuration=r.cfg.Clock.Since(autoStartTime)
172-
logger.Info(ctx,fmt.Sprintf("workspace %q autostarted successfully",workspace.Name))
195+
r.autostartTotalLatency=r.cfg.Clock.Since(autostartTime)
173196

174-
logger.Info(ctx,fmt.Sprintf("autostart completed in %v",r.autostartDuration))
175-
r.cfg.Metrics.RecordCompletion(r.autostartDuration,newUser.Username,workspace.Name)
197+
logger.Info(ctx,fmt.Sprintf("autostart completed in %v",r.autostartTotalLatency))
198+
r.cfg.Metrics.RecordCompletion(r.autostartTotalLatency,newUser.Username,workspace.Name)
176199

177200
returnnil
178201
}
179202

203+
funcwaitForWorkspaceUpdate(ctx context.Context,logger slog.Logger,updates<-chan codersdk.Workspace,shouldBreakfunc(codersdk.Workspace)bool)error {
204+
for {
205+
select {
206+
case<-ctx.Done():
207+
returnctx.Err()
208+
caseupdatedWorkspace,ok:=<-updates:
209+
if!ok {
210+
returnxerrors.New("workspace updates channel closed")
211+
}
212+
logger.Debug(ctx,"received workspace update",slog.F("update",updatedWorkspace))
213+
ifshouldBreak(updatedWorkspace) {
214+
returnnil
215+
}
216+
}
217+
}
218+
}
219+
180220
func (r*Runner)Cleanup(ctx context.Context,idstring,logs io.Writer)error {
181221
ifr.workspacebuildRunner!=nil {
182222
_,_=fmt.Fprintln(logs,"Cleaning up workspace...")
@@ -196,11 +236,15 @@ func (r *Runner) Cleanup(ctx context.Context, id string, logs io.Writer) error {
196236
}
197237

198238
const (
199-
AutostartLatencyMetric="autostart_latency_seconds"
239+
AutostartTotalLatencyMetric="autostart_total_latency_seconds"
240+
AutostartJobCreationLatencyMetric="autostart_job_creation_latency_seconds"
241+
AutostartJobAcquiredLatencyMetric="autostart_job_acquired_latency_seconds"
200242
)
201243

202244
func (r*Runner)GetMetrics()map[string]any {
203245
returnmap[string]any{
204-
AutostartLatencyMetric:r.autostartDuration.Seconds(),
246+
AutostartTotalLatencyMetric:r.autostartTotalLatency.Seconds(),
247+
AutostartJobCreationLatencyMetric:r.autostartJobCreationLatency.Seconds(),
248+
AutostartJobAcquiredLatencyMetric:r.autostartJobAcquiredLatency.Seconds(),
205249
}
206250
}

‎scaletest/autostart/run_internal_test.go‎

Lines changed: 30 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"golang.org/x/sync/errgroup"
1414

1515
"github.com/coder/coder/v2/coderd/coderdtest"
16+
"github.com/coder/coder/v2/coderd/database/dbauthz"
1617
"github.com/coder/coder/v2/codersdk"
1718
"github.com/coder/coder/v2/provisioner/echo"
1819
"github.com/coder/coder/v2/provisionersdk/proto"
@@ -25,20 +26,25 @@ import (
2526
funcTestRun(t*testing.T) {
2627
t.Parallel()
2728
numUsers:=2
28-
autoStartDelay:=time.Minute
29+
autoStartDelay:=2*time.Minute
2930
expectedLatency:=5*time.Minute
3031

3132
mClock:=quartz.NewMock(t)
3233
mClock.Set(time.Now())
3334

34-
ctx:=testutil.Context(t,testutil.WaitSuperLong)
35+
ctx:=testutil.Context(t,testutil.WaitLong)
3536
tickCh:=make(chan time.Time)
3637

37-
client:=coderdtest.New(t,&coderdtest.Options{
38+
client,db:=coderdtest.NewWithDatabase(t,&coderdtest.Options{
3839
IncludeProvisionerDaemon:true,
3940
AutobuildTicker:tickCh,
4041
})
4142
user:=coderdtest.CreateFirstUser(t,client)
43+
// System context required to get provisioner daemons.
44+
provisioners,err:=db.GetProvisionerDaemons(dbauthz.AsSystemRestricted(ctx))
45+
require.NoError(t,err)
46+
require.Len(t,provisioners,1)
47+
provisioner:=provisioners[0]
4248

4349
authToken:=uuid.NewString()
4450
version:=coderdtest.CreateTemplateVersion(t,client,user.OrganizationID,&echo.Responses{
@@ -109,25 +115,20 @@ func TestRun(t *testing.T) {
109115
})
110116
}
111117

112-
gofunc() {
113-
// Wait for each of the runners to have set autostart schedules.
114-
for_,runner:=rangerunners {
115-
select {
116-
case<-ctx.Done():
117-
return
118-
case<-runner.autostartSet:
119-
}
120-
}
121-
mClock.Advance(autoStartDelay)
122-
tickTime:=mClock.Now().UTC()
123-
mClock.Advance(expectedLatency)
124-
select {
125-
case<-ctx.Done():
126-
casetickCh<-tickTime:
127-
}
128-
}()
118+
// Wait for each of the runners to have set autostart schedules.
119+
for_,runner:=rangerunners {
120+
testutil.TryReceive(ctx,t,runner.autostartSet)
121+
}
122+
mClock.Advance(autoStartDelay)
123+
// The runner rounds now + autostartdelay to the nearest minute
124+
mClock.Set(mClock.Now().Round(time.Minute))
125+
tickTime:=mClock.Now().UTC()
126+
mClock.Advance(expectedLatency)
127+
128+
coderdtest.UpdateProvisionerLastSeenAt(t,db,provisioner.ID,tickTime)
129+
testutil.RequireSend(ctx,t,tickCh,tickTime)
129130

130-
err:=eg.Wait()
131+
err=eg.Wait()
131132
require.NoError(t,err)
132133

133134
users,err:=client.Users(ctx, codersdk.UsersRequest{})
@@ -164,9 +165,15 @@ func TestRun(t *testing.T) {
164165

165166
for_,runner:=rangerunners {
166167
metrics:=runner.GetMetrics()
167-
require.Contains(t,metrics,AutostartLatencyMetric)
168-
latency,ok:=metrics[AutostartLatencyMetric].(float64)
168+
require.Contains(t,metrics,AutostartTotalLatencyMetric)
169+
latency,ok:=metrics[AutostartTotalLatencyMetric].(float64)
170+
require.True(t,ok)
171+
jobCreationLatency,ok:=metrics[AutostartJobCreationLatencyMetric].(float64)
172+
require.True(t,ok)
173+
jobAcquiredLatency,ok:=metrics[AutostartJobAcquiredLatencyMetric].(float64)
169174
require.True(t,ok)
170175
require.Equal(t,expectedLatency.Seconds(),latency)
176+
require.Equal(t,expectedLatency.Seconds(),jobCreationLatency)
177+
require.Equal(t,expectedLatency.Seconds(),jobAcquiredLatency)
171178
}
172179
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp