Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit3a82249

Browse files
committed
feat(scaletest): add runner for thundering herd autostart
1 parentc50e86c commit3a82249

File tree

5 files changed

+499
-5
lines changed

5 files changed

+499
-5
lines changed

‎scaletest/autostart/config.go‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
package autostart
2+
3+
import (
4+
"sync"
5+
"time"
6+
7+
"golang.org/x/xerrors"
8+
9+
"github.com/coder/coder/v2/codersdk"
10+
"github.com/coder/coder/v2/scaletest/createusers"
11+
"github.com/coder/coder/v2/scaletest/workspacebuild"
12+
"github.com/coder/quartz"
13+
)
14+
15+
typeConfigstruct {
16+
// User is the configuration for the user to create.
17+
User createusers.Config`json:"user"`
18+
19+
// Workspace is the configuration for the workspace to create. The workspace
20+
// will be built using the new user.
21+
//
22+
// OrganizationID is ignored and set to the new user's organization ID.
23+
Workspace workspacebuild.Config`json:"workspace"`
24+
25+
// WorkspaceJobTimeout is how long to wait for any one workspace job
26+
// (start or stop) to complete.
27+
WorkspaceJobTimeout time.Duration`json:"workspace_job_timeout"`
28+
29+
// AutostartDelay is how long after all the workspaces have been stopped
30+
// to schedule them to be started again.
31+
AutostartDelay time.Duration`json:"autostart_delay"`
32+
33+
// AutostartTimeout is how long to wait for the autostart build to be
34+
// initiated after the scheduled time.
35+
AutostartTimeout time.Duration`json:"autostart_timeout"`
36+
37+
Metrics*Metrics`json:"-"`
38+
39+
// SetupBarrier is used to ensure all runners own stopped workspaces
40+
// before setting the autostart schedule on each.
41+
SetupBarrier*sync.WaitGroup`json:"-"`
42+
43+
// Clock is mocked in tests.
44+
Clock quartz.Clock`json:"-"`
45+
}
46+
47+
func (cConfig)Validate()error {
48+
iferr:=c.User.Validate();err!=nil {
49+
returnxerrors.Errorf("user config: %w",err)
50+
}
51+
c.Workspace.OrganizationID=c.User.OrganizationID
52+
// This value will be overwritten during the test.
53+
c.Workspace.UserID=codersdk.Me
54+
iferr:=c.Workspace.Validate();err!=nil {
55+
returnxerrors.Errorf("workspace config: %w",err)
56+
}
57+
58+
ifc.SetupBarrier==nil {
59+
returnxerrors.New("setup barrier must be set")
60+
}
61+
62+
ifc.WorkspaceJobTimeout<=0 {
63+
returnxerrors.New("workspace_job_timeout must be greater than 0")
64+
}
65+
66+
ifc.AutostartDelay<=0 {
67+
returnxerrors.New("autostart_delay must be greater than 0")
68+
}
69+
70+
ifc.AutostartTimeout<=0 {
71+
returnxerrors.New("autostart_timeout must be greater than 0")
72+
}
73+
74+
ifc.Metrics==nil {
75+
returnxerrors.New("metrics must be set")
76+
}
77+
78+
ifc.Clock==nil {
79+
returnxerrors.New("clock must be set")
80+
}
81+
82+
returnnil
83+
}

‎scaletest/autostart/metrics.go‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package autostart
2+
3+
import (
4+
"time"
5+
6+
"github.com/prometheus/client_golang/prometheus"
7+
)
8+
9+
typeMetricsstruct {
10+
AutostartLatencySeconds prometheus.HistogramVec
11+
AutostartErrorsTotal prometheus.CounterVec
12+
}
13+
14+
funcNewMetrics(reg prometheus.Registerer)*Metrics {
15+
m:=&Metrics{
16+
AutostartLatencySeconds:*prometheus.NewHistogramVec(prometheus.HistogramOpts{
17+
Namespace:"coderd",
18+
Subsystem:"scaletest",
19+
Name:"autostart_latency_seconds",
20+
Help:"Time from when the workspace is scheduled to be autostarted to when the autostart build has finished.",
21+
}, []string{"username","workspace_name"}),
22+
AutostartErrorsTotal:*prometheus.NewCounterVec(prometheus.CounterOpts{
23+
Namespace:"coderd",
24+
Subsystem:"scaletest",
25+
Name:"autostart_errors_total",
26+
Help:"Total number of autostart errors",
27+
}, []string{"username","action"}),
28+
}
29+
30+
reg.MustRegister(m.AutostartLatencySeconds)
31+
reg.MustRegister(m.AutostartErrorsTotal)
32+
returnm
33+
}
34+
35+
func (m*Metrics)RecordCompletion(elapsed time.Duration,usernamestring,workspacestring) {
36+
m.AutostartLatencySeconds.WithLabelValues(username,workspace).Observe(elapsed.Seconds())
37+
}
38+
39+
func (m*Metrics)AddError(usernamestring,actionstring) {
40+
m.AutostartErrorsTotal.WithLabelValues(username,action).Inc()
41+
}

‎scaletest/autostart/run.go‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
package autostart
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io"
7+
"time"
8+
9+
"golang.org/x/xerrors"
10+
11+
"cdr.dev/slog"
12+
"cdr.dev/slog/sloggers/sloghuman"
13+
"github.com/coder/coder/v2/coderd/tracing"
14+
"github.com/coder/coder/v2/codersdk"
15+
"github.com/coder/coder/v2/scaletest/createusers"
16+
"github.com/coder/coder/v2/scaletest/harness"
17+
"github.com/coder/coder/v2/scaletest/loadtestutil"
18+
"github.com/coder/coder/v2/scaletest/workspacebuild"
19+
)
20+
21+
typeRunnerstruct {
22+
client*codersdk.Client
23+
cfgConfig
24+
25+
createUserRunner*createusers.Runner
26+
workspacebuildRunner*workspacebuild.Runner
27+
28+
autostartDuration time.Duration
29+
// Closed when the autostart schedule has been set.
30+
// Used in tests.
31+
autostartSetchanstruct{}
32+
}
33+
34+
funcNewRunner(client*codersdk.Client,cfgConfig)*Runner {
35+
return&Runner{
36+
client:client,
37+
cfg:cfg,
38+
autostartSet:make(chanstruct{}),
39+
}
40+
}
41+
42+
var (
43+
_ harness.Runnable=&Runner{}
44+
_ harness.Cleanable=&Runner{}
45+
_ harness.Collectable=&Runner{}
46+
)
47+
48+
func (r*Runner)Run(ctx context.Context,idstring,logs io.Writer)error {
49+
ctx,span:=tracing.StartSpan(ctx)
50+
deferspan.End()
51+
52+
logs=loadtestutil.NewSyncWriter(logs)
53+
logger:=slog.Make(sloghuman.Sink(logs)).Leveled(slog.LevelDebug)
54+
r.client.SetLogger(logger)
55+
r.client.SetLogBodies(true)
56+
57+
r.createUserRunner=createusers.NewRunner(r.client,r.cfg.User)
58+
newUserAndToken,err:=r.createUserRunner.RunReturningUser(ctx,id,logs)
59+
iferr!=nil {
60+
returnxerrors.Errorf("create user: %w",err)
61+
}
62+
newUser:=newUserAndToken.User
63+
64+
newUserClient:=codersdk.New(r.client.URL,
65+
codersdk.WithSessionToken(newUserAndToken.SessionToken),
66+
codersdk.WithLogger(logger),
67+
codersdk.WithLogBodies())
68+
69+
logger.Info(ctx,fmt.Sprintf("user %q created",newUser.Username),slog.F("id",newUser.ID.String()))
70+
71+
workspaceBuildConfig:=r.cfg.Workspace
72+
workspaceBuildConfig.OrganizationID=r.cfg.User.OrganizationID
73+
workspaceBuildConfig.UserID=newUser.ID.String()
74+
75+
r.workspacebuildRunner=workspacebuild.NewRunner(newUserClient,workspaceBuildConfig)
76+
workspace,err:=r.workspacebuildRunner.RunReturningWorkspace(ctx,id,logs)
77+
iferr!=nil {
78+
returnxerrors.Errorf("create workspace: %w",err)
79+
}
80+
81+
logger.Info(ctx,fmt.Sprintf("workspace %q created",workspace.Name))
82+
83+
logger.Info(ctx,fmt.Sprintf("stopping workspace %q",workspace.Name))
84+
85+
stopBuild,err:=newUserClient.CreateWorkspaceBuild(ctx,workspace.ID, codersdk.CreateWorkspaceBuildRequest{
86+
Transition:codersdk.WorkspaceTransitionStop,
87+
})
88+
iferr!=nil {
89+
returnxerrors.Errorf("create stop build: %w",err)
90+
}
91+
92+
stopBuildCtx,cancel2:=context.WithTimeout(ctx,r.cfg.WorkspaceJobTimeout)
93+
defercancel2()
94+
95+
err=workspacebuild.WaitForBuild(stopBuildCtx,logs,newUserClient,stopBuild.ID)
96+
iferr!=nil {
97+
returnxerrors.Errorf("wait for stop build to complete: %w",err)
98+
}
99+
100+
logger.Info(ctx,fmt.Sprintf("workspace %q stopped successfully",workspace.Name))
101+
102+
logger.Info(ctx,"waiting for all runners to reach barrier")
103+
r.cfg.SetupBarrier.Done()
104+
r.cfg.SetupBarrier.Wait()
105+
logger.Info(ctx,"all runners reached barrier, proceeding with autostart schedule")
106+
107+
autoStartTime:=r.cfg.Clock.Now().Add(r.cfg.AutostartDelay)
108+
schedule:=fmt.Sprintf("CRON_TZ=UTC %d %d * * *",autoStartTime.Minute(),autoStartTime.Hour())
109+
110+
logger.Info(ctx,fmt.Sprintf("setting autostart schedule for workspace %q: %s",workspace.Name,schedule))
111+
112+
err=newUserClient.UpdateWorkspaceAutostart(ctx,workspace.ID, codersdk.UpdateWorkspaceAutostartRequest{
113+
Schedule:&schedule,
114+
})
115+
iferr!=nil {
116+
returnxerrors.Errorf("update workspace autostart: %w",err)
117+
}
118+
close(r.autostartSet)
119+
120+
logger.Info(ctx,fmt.Sprintf("autostart schedule set for workspace %q",workspace.Name))
121+
122+
logger.Info(ctx,fmt.Sprintf("waiting for workspace %q to autostart",workspace.Name))
123+
124+
autostartInitiateCtx,cancel2:=context.WithTimeout(ctx,r.cfg.AutostartTimeout+r.cfg.AutostartDelay)
125+
defercancel2()
126+
127+
workspaceUpdates,err:=newUserClient.WatchWorkspace(autostartInitiateCtx,workspace.ID)
128+
iferr!=nil {
129+
returnxerrors.Errorf("watch workspace: %w",err)
130+
}
131+
132+
varautoStartBuild codersdk.WorkspaceBuild
133+
134+
logger.Info(ctx,"listening for workspace updates to detect autostart build")
135+
waitNewBuildLoop:
136+
for {
137+
select {
138+
case<-autostartInitiateCtx.Done():
139+
returnxerrors.Errorf("timeout waiting for autostart build to be created: %w",autostartInitiateCtx.Err())
140+
caseupdatedWorkspace,ok:=<-workspaceUpdates:
141+
if!ok {
142+
returnxerrors.Errorf("workspace updates channel closed")
143+
}
144+
145+
ifupdatedWorkspace.LatestBuild.ID!=stopBuild.ID&&
146+
updatedWorkspace.LatestBuild.Transition==codersdk.WorkspaceTransitionStart {
147+
autoStartBuild=updatedWorkspace.LatestBuild
148+
logger.Info(ctx,fmt.Sprintf("autostart build created with ID %s",autoStartBuild.ID))
149+
break waitNewBuildLoop
150+
}
151+
}
152+
}
153+
154+
logger.Info(ctx,"waiting for autostart build to complete")
155+
buildCompleteCtx,cancel3:=context.WithTimeout(ctx,r.cfg.WorkspaceJobTimeout)
156+
defercancel3()
157+
158+
err=workspacebuild.WaitForBuild(buildCompleteCtx,logs,newUserClient,autoStartBuild.ID)
159+
iferr!=nil {
160+
returnxerrors.Errorf("wait for autostart build to complete: %w",err)
161+
}
162+
163+
r.autostartDuration=r.cfg.Clock.Since(autoStartTime)
164+
logger.Info(ctx,fmt.Sprintf("workspace %q autostarted successfully",workspace.Name))
165+
166+
logger.Info(ctx,fmt.Sprintf("autostart completed in %v",r.autostartDuration))
167+
r.cfg.Metrics.RecordCompletion(r.autostartDuration,newUser.Username,workspace.Name)
168+
169+
returnnil
170+
}
171+
172+
func (r*Runner)Cleanup(ctx context.Context,idstring,logs io.Writer)error {
173+
ifr.workspacebuildRunner!=nil {
174+
_,_=fmt.Fprintln(logs,"Cleaning up workspace...")
175+
iferr:=r.workspacebuildRunner.Cleanup(ctx,id,logs);err!=nil {
176+
returnxerrors.Errorf("cleanup workspace: %w",err)
177+
}
178+
}
179+
180+
ifr.createUserRunner!=nil {
181+
_,_=fmt.Fprintln(logs,"Cleaning up user...")
182+
iferr:=r.createUserRunner.Cleanup(ctx,id,logs);err!=nil {
183+
returnxerrors.Errorf("cleanup user: %w",err)
184+
}
185+
}
186+
187+
returnnil
188+
}
189+
190+
const (
191+
AutostartLatencyMetric="autostart_latency_seconds"
192+
)
193+
194+
func (r*Runner)GetMetrics()map[string]any {
195+
returnmap[string]any{
196+
AutostartLatencyMetric:r.autostartDuration.Seconds(),
197+
}
198+
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp