- Notifications
You must be signed in to change notification settings - Fork1k
fix(agent): start devcontainers through agentcontainers package#18471
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.
Already on GitHub?Sign in to your account
Uh oh!
There was an error while loading.Please reload this page.
Changes fromall commits
978c871
fe99bd6
aeae6e2
916f7e8
53e256b
91bb43a
81fe11d
5c70a8c
8437ca4
2c6a2b1
a512ad4
c50dc6e
4d40ef2
ce32e2e
32ac48a
738b755
3714fec
996d440
ae5dd1e
9d76cf6
7285c39
3fce51c
54aa84a
3d08d0e
fa479fc
File filter
Filter by extension
Conversations
Uh oh!
There was an error while loading.Please reload this page.
Jump to
Uh oh!
There was an error while loading.Please reload this page.
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -91,6 +91,7 @@ type Options struct { | ||
Execer agentexec.Execer | ||
Devcontainers bool | ||
DevcontainerAPIOptions []agentcontainers.Option // Enable Devcontainers for these to be effective. | ||
Clock quartz.Clock | ||
} | ||
type Client interface { | ||
@@ -144,6 +145,9 @@ func New(options Options) Agent { | ||
if options.PortCacheDuration == 0 { | ||
options.PortCacheDuration = 1 * time.Second | ||
} | ||
if options.Clock == nil { | ||
options.Clock = quartz.NewReal() | ||
} | ||
prometheusRegistry := options.PrometheusRegistry | ||
if prometheusRegistry == nil { | ||
@@ -157,6 +161,7 @@ func New(options Options) Agent { | ||
hardCtx, hardCancel := context.WithCancel(context.Background()) | ||
gracefulCtx, gracefulCancel := context.WithCancel(hardCtx) | ||
a := &agent{ | ||
clock: options.Clock, | ||
tailnetListenPort: options.TailnetListenPort, | ||
reconnectingPTYTimeout: options.ReconnectingPTYTimeout, | ||
logger: options.Logger, | ||
@@ -204,6 +209,7 @@ func New(options Options) Agent { | ||
} | ||
type agent struct { | ||
clock quartz.Clock | ||
logger slog.Logger | ||
client Client | ||
exchangeToken func(ctx context.Context) (string, error) | ||
@@ -273,7 +279,7 @@ type agent struct { | ||
devcontainers bool | ||
containerAPIOptions []agentcontainers.Option | ||
containerAPI*agentcontainers.API | ||
} | ||
func (a *agent) TailnetConn() *tailnet.Conn { | ||
@@ -330,6 +336,19 @@ func (a *agent) init() { | ||
// will not report anywhere. | ||
a.scriptRunner.RegisterMetrics(a.prometheusRegistry) | ||
if a.devcontainers { | ||
containerAPIOpts := []agentcontainers.Option{ | ||
agentcontainers.WithExecer(a.execer), | ||
agentcontainers.WithCommandEnv(a.sshServer.CommandEnv), | ||
agentcontainers.WithScriptLogger(func(logSourceID uuid.UUID) agentcontainers.ScriptLogger { | ||
return a.logSender.GetScriptLogger(logSourceID) | ||
}), | ||
} | ||
containerAPIOpts = append(containerAPIOpts, a.containerAPIOptions...) | ||
a.containerAPI = agentcontainers.NewAPI(a.logger.Named("containers"), containerAPIOpts...) | ||
} | ||
a.reconnectingPTYServer = reconnectingpty.NewServer( | ||
a.logger.Named("reconnecting-pty"), | ||
a.sshServer, | ||
@@ -1141,15 +1160,18 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context, | ||
} | ||
var ( | ||
scripts = manifest.Scripts | ||
scriptRunnerOpts []agentscripts.InitOption | ||
devcontainerScripts map[uuid.UUID]codersdk.WorkspaceAgentScript | ||
) | ||
if a.devcontainers { | ||
a.containerAPI.Init( | ||
agentcontainers.WithManifestInfo(manifest.OwnerName, manifest.WorkspaceName), | ||
agentcontainers.WithDevcontainers(manifest.Devcontainers, scripts), | ||
agentcontainers.WithSubAgentClient(agentcontainers.NewSubAgentClientFromAPI(a.logger, aAPI)), | ||
) | ||
scripts, devcontainerScripts = agentcontainers.ExtractDevcontainerScripts(manifest.Devcontainers, scripts) | ||
} | ||
err = a.scriptRunner.Init(scripts, aAPI.ScriptCompleted, scriptRunnerOpts...) | ||
if err != nil { | ||
@@ -1168,7 +1190,12 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context, | ||
// finished (both start and post start). For instance, an | ||
// autostarted devcontainer will be included in this time. | ||
err := a.scriptRunner.Execute(a.gracefulCtx, agentscripts.ExecuteStartScripts) | ||
for _, dc := range manifest.Devcontainers { | ||
cErr := a.createDevcontainer(ctx, aAPI, dc, devcontainerScripts[dc.ID]) | ||
err = errors.Join(err, cErr) | ||
} | ||
dur := time.Since(start).Seconds() | ||
if err != nil { | ||
a.logger.Warn(ctx, "startup script(s) failed", slog.Error(err)) | ||
@@ -1187,14 +1214,6 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context, | ||
} | ||
a.metrics.startupScriptSeconds.WithLabelValues(label).Set(dur) | ||
a.scriptRunner.StartCron() | ||
}) | ||
if err != nil { | ||
return xerrors.Errorf("track conn goroutine: %w", err) | ||
@@ -1204,6 +1223,38 @@ func (a *agent) handleManifest(manifestOK *checkpoint) func(ctx context.Context, | ||
} | ||
} | ||
func (a *agent) createDevcontainer( | ||
ctx context.Context, | ||
aAPI proto.DRPCAgentClient26, | ||
dc codersdk.WorkspaceAgentDevcontainer, | ||
script codersdk.WorkspaceAgentScript, | ||
) (err error) { | ||
var ( | ||
exitCode = int32(0) | ||
startTime = a.clock.Now() | ||
status = proto.Timing_OK | ||
) | ||
if err = a.containerAPI.CreateDevcontainer(dc.WorkspaceFolder, dc.ConfigPath); err != nil { | ||
exitCode = 1 | ||
status = proto.Timing_EXIT_FAILURE | ||
} | ||
endTime := a.clock.Now() | ||
if _, scriptErr := aAPI.ScriptCompleted(ctx, &proto.WorkspaceAgentScriptCompletedRequest{ | ||
Timing: &proto.Timing{ | ||
ScriptId: script.ID[:], | ||
Start: timestamppb.New(startTime), | ||
End: timestamppb.New(endTime), | ||
ExitCode: exitCode, | ||
Stage: proto.Timing_START, | ||
DanielleMaywood marked this conversation as resolved. Show resolvedHide resolvedUh oh!There was an error while loading.Please reload this page. | ||
Status: status, | ||
}, | ||
}); scriptErr != nil { | ||
a.logger.Warn(ctx, "reporting script completed failed", slog.Error(scriptErr)) | ||
} | ||
return err | ||
} | ||
// createOrUpdateNetwork waits for the manifest to be set using manifestOK, then creates or updates | ||
// the tailnet using the information in the manifest | ||
func (a *agent) createOrUpdateNetwork(manifestOK, networkOK *checkpoint) func(context.Context, proto.DRPCAgentClient26) error { | ||
@@ -1227,7 +1278,6 @@ func (a *agent) createOrUpdateNetwork(manifestOK, networkOK *checkpoint) func(co | ||
// agent API. | ||
network, err = a.createTailnet( | ||
a.gracefulCtx, | ||
manifest.AgentID, | ||
manifest.DERPMap, | ||
manifest.DERPForceWebSockets, | ||
@@ -1262,9 +1312,9 @@ func (a *agent) createOrUpdateNetwork(manifestOK, networkOK *checkpoint) func(co | ||
network.SetBlockEndpoints(manifest.DisableDirectConnections) | ||
// Update the subagent client if the container API is available. | ||
if a.containerAPI != nil { | ||
client := agentcontainers.NewSubAgentClientFromAPI(a.logger, aAPI) | ||
a.containerAPI.UpdateSubAgentClient(client) | ||
} | ||
} | ||
return nil | ||
@@ -1382,7 +1432,6 @@ func (a *agent) trackGoroutine(fn func()) error { | ||
func (a *agent) createTailnet( | ||
ctx context.Context, | ||
agentID uuid.UUID, | ||
derpMap *tailcfg.DERPMap, | ||
derpForceWebSockets, disableDirectConnections bool, | ||
@@ -1515,10 +1564,7 @@ func (a *agent) createTailnet( | ||
}() | ||
if err = a.trackGoroutine(func() { | ||
defer apiListener.Close() | ||
apiHandler := a.apiHandler() | ||
server := &http.Server{ | ||
BaseContext: func(net.Listener) context.Context { return ctx }, | ||
Handler: apiHandler, | ||
@@ -1532,7 +1578,6 @@ func (a *agent) createTailnet( | ||
case <-ctx.Done(): | ||
case <-a.hardCtx.Done(): | ||
} | ||
_ = server.Close() | ||
}() | ||
@@ -1871,6 +1916,12 @@ func (a *agent) Close() error { | ||
a.logger.Error(a.hardCtx, "script runner close", slog.Error(err)) | ||
} | ||
if a.containerAPI != nil { | ||
if err := a.containerAPI.Close(); err != nil { | ||
a.logger.Error(a.hardCtx, "container API close", slog.Error(err)) | ||
} | ||
} | ||
// Wait for the graceful shutdown to complete, but don't wait forever so | ||
// that we don't break user expectations. | ||
go func() { | ||
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -207,6 +207,10 @@ func WithDevcontainers(devcontainers []codersdk.WorkspaceAgentDevcontainer, scri | ||
api.devcontainerNames = make(map[string]bool, len(devcontainers)) | ||
api.devcontainerLogSourceIDs = make(map[string]uuid.UUID) | ||
for _, dc := range devcontainers { | ||
if dc.Status == "" { | ||
dc.Status = codersdk.WorkspaceAgentDevcontainerStatusStarting | ||
} | ||
api.knownDevcontainers[dc.WorkspaceFolder] = dc | ||
api.devcontainerNames[dc.Name] = true | ||
for _, script := range scripts { | ||
@@ -265,8 +269,6 @@ func NewAPI(logger slog.Logger, options ...Option) *API { | ||
api := &API{ | ||
ctx: ctx, | ||
cancel: cancel, | ||
initialUpdateDone: make(chan struct{}), | ||
updateTrigger: make(chan chan error), | ||
updateInterval: defaultUpdateInterval, | ||
@@ -315,10 +317,28 @@ func NewAPI(logger slog.Logger, options ...Option) *API { | ||
api.subAgentClient.Store(&c) | ||
} | ||
return api | ||
} | ||
// Init applies a final set of options to the API and then | ||
// begins the watcherLoop and updaterLoop. This function | ||
// must only be called once. | ||
func (api *API) Init(opts ...Option) { | ||
DanielleMaywood marked this conversation as resolved. Show resolvedHide resolvedUh oh!There was an error while loading.Please reload this page. | ||
api.mu.Lock() | ||
defer api.mu.Unlock() | ||
if api.closed { | ||
return | ||
} | ||
for _, opt := range opts { | ||
DanielleMaywood marked this conversation as resolved. Show resolvedHide resolvedUh oh!There was an error while loading.Please reload this page. | ||
opt(api) | ||
} | ||
api.watcherDone = make(chan struct{}) | ||
api.updaterDone = make(chan struct{}) | ||
go api.watcherLoop() | ||
go api.updaterLoop() | ||
} | ||
func (api *API) watcherLoop() { | ||
@@ -909,8 +929,9 @@ func (api *API) handleDevcontainerRecreate(w http.ResponseWriter, r *http.Reques | ||
dc.Status = codersdk.WorkspaceAgentDevcontainerStatusStarting | ||
dc.Container = nil | ||
api.knownDevcontainers[dc.WorkspaceFolder] = dc | ||
go func() { | ||
_ = api.CreateDevcontainer(dc.WorkspaceFolder, configPath, WithRemoveExistingContainer()) | ||
}() | ||
api.mu.Unlock() | ||
@@ -920,15 +941,29 @@ func (api *API) handleDevcontainerRecreate(w http.ResponseWriter, r *http.Reques | ||
}) | ||
} | ||
//createDevcontainer should run in its own goroutine and is responsible for | ||
// recreating a devcontainer based on the provided devcontainer configuration. | ||
// It updates the devcontainer status and logs the process. The configPath is | ||
// passed as a parameter for the odd chance that the container being recreated | ||
// has a different config file than the one stored in the devcontainer state. | ||
// The devcontainer state must be set to starting and the asyncWg must be | ||
// incremented before calling this function. | ||
func (api *API) CreateDevcontainer(workspaceFolder, configPath string, opts ...DevcontainerCLIUpOptions) error { | ||
api.mu.Lock() | ||
if api.closed { | ||
api.mu.Unlock() | ||
return nil | ||
} | ||
dc, found := api.knownDevcontainers[workspaceFolder] | ||
if !found { | ||
api.mu.Unlock() | ||
return xerrors.Errorf("devcontainer not found") | ||
} | ||
DanielleMaywood marked this conversation as resolved. Show resolvedHide resolvedUh oh!There was an error while loading.Please reload this page. | ||
api.asyncWg.Add(1) | ||
defer api.asyncWg.Done() | ||
api.mu.Unlock() | ||
var ( | ||
err error | ||
@@ -969,12 +1004,15 @@ func (api *API) recreateDevcontainer(dc codersdk.WorkspaceAgentDevcontainer, con | ||
logger.Debug(ctx, "starting devcontainer recreation") | ||
upOptions := []DevcontainerCLIUpOptions{WithUpOutput(infoW, errW)} | ||
upOptions = append(upOptions, opts...) | ||
_, err = api.dccli.Up(ctx, dc.WorkspaceFolder, configPath, upOptions...) | ||
if err != nil { | ||
// No need to log if the API is closing (context canceled), as this | ||
// is expected behavior when the API is shutting down. | ||
if !errors.Is(err, context.Canceled) { | ||
logger.Error(ctx, "devcontainercreation failed", slog.Error(err)) | ||
} | ||
api.mu.Lock() | ||
@@ -983,10 +1021,11 @@ func (api *API) recreateDevcontainer(dc codersdk.WorkspaceAgentDevcontainer, con | ||
api.knownDevcontainers[dc.WorkspaceFolder] = dc | ||
api.recreateErrorTimes[dc.WorkspaceFolder] = api.clock.Now("agentcontainers", "recreate", "errorTimes") | ||
api.mu.Unlock() | ||
return xerrors.Errorf("start devcontainer: %w", err) | ||
} | ||
logger.Info(ctx, "devcontainercreated successfully") | ||
api.mu.Lock() | ||
dc = api.knownDevcontainers[dc.WorkspaceFolder] | ||
@@ -1009,8 +1048,11 @@ func (api *API) recreateDevcontainer(dc codersdk.WorkspaceAgentDevcontainer, con | ||
// Ensure an immediate refresh to accurately reflect the | ||
// devcontainer state after recreation. | ||
if err := api.RefreshContainers(ctx); err != nil { | ||
logger.Error(ctx, "failed to trigger immediate refresh after devcontainer creation", slog.Error(err)) | ||
return xerrors.Errorf("refresh containers: %w", err) | ||
} | ||
return nil | ||
} | ||
// markDevcontainerDirty finds the devcontainer with the given config file path | ||
@@ -1609,8 +1651,12 @@ func (api *API) Close() error { | ||
err := api.watcher.Close() | ||
// Wait for loops to finish. | ||
if api.watcherDone != nil { | ||
<-api.watcherDone | ||
} | ||
if api.updaterDone != nil { | ||
<-api.updaterDone | ||
} | ||
// Wait for all async tasks to complete. | ||
api.asyncWg.Wait() | ||
Uh oh!
There was an error while loading.Please reload this page.
Uh oh!
There was an error while loading.Please reload this page.