- Notifications
You must be signed in to change notification settings - Fork928
feat: add startup script logs to the ui#6558
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.
Already on GitHub?Sign in to your account
Merged
Uh oh!
There was an error while loading.Please reload this page.
Merged
Changes fromall commits
Commits
Show all changes
34 commits Select commitHold shift + click to select a range
99d510c Add startup script logs to the database
code-asher66c8ec3 Add coderd endpoints for startup script logs
code-asher1cc3e9d Push startup script logs from agent
code-asher45d250f Pull startup script logs on frontend
code-asher7fed360 Merge branch 'main' into startuplogs
kylecarbs7ce73aa Rename queries
kylecarbsb86c400 Add constraint
kylecarbs0c4d2c3 Start creating log sending loop
kylecarbs1bb700f Add log sending to the agent
kylecarbs736705f Add tests for streaming logs
kylecarbsf741523 Shorten notify channel name
kylecarbs54c30be Add FE
kylecarbsadb06ea Improve bulk log performance
kylecarbs4061b13 Finish UI display
kylecarbs4c5b630 Fix startup log visibility
kylecarbs05d536c Add warning for overflow
kylecarbs34fde1a Fix agent queue logs overflow
kylecarbs379f1f4 Display staartup logs in a virtual DOM for performance
kylecarbsdecde5c Fix agent queue with loads of logs
kylecarbsd74457c Merge branch 'main' into startuplogs
kylecarbsac55f48 Fix authorize test
kylecarbs8d75963 Remove faulty test
kylecarbscc715cd Fix startup and shutdown reporting error
kylecarbse3a4b2c Fix gen
kylecarbs399dad7 Merge branch 'main' into startuplogs
kylecarbs45c0aca Fix comments
kylecarbs5a0b15d Periodically purge old database entries
kylecarbsb1b3fcb Add test fixture for migration
kylecarbs6e1032c Add Storybook
kylecarbs3762e8d Check if there are logs when displaying features
kylecarbsf6b9fce Fix startup component overflow gap
kylecarbsc48658c Fix startup log wrapping
kylecarbs4ec1a0e Merge branch 'main' into startuplogs
kylecarbsb55b7a1 Merge branch 'main' into startuplogs
kylecarbsFile filter
Filter by extension
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Diff view
Diff view
There are no files selected for viewing
141 changes: 138 additions & 3 deletionsagent/agent.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -41,6 +41,7 @@ import ( | ||
| "cdr.dev/slog" | ||
| "github.com/coder/coder/agent/usershell" | ||
| "github.com/coder/coder/buildinfo" | ||
| "github.com/coder/coder/coderd/database" | ||
| "github.com/coder/coder/coderd/gitauth" | ||
| "github.com/coder/coder/codersdk" | ||
| "github.com/coder/coder/codersdk/agentsdk" | ||
| @@ -88,6 +89,7 @@ type Client interface { | ||
| PostLifecycle(ctx context.Context, state agentsdk.PostLifecycleRequest) error | ||
| PostAppHealth(ctx context.Context, req agentsdk.PostAppHealthsRequest) error | ||
| PostStartup(ctx context.Context, req agentsdk.PostStartupRequest) error | ||
| PatchStartupLogs(ctx context.Context, req agentsdk.PatchStartupLogs) error | ||
| } | ||
| func New(options Options) io.Closer { | ||
| @@ -642,13 +644,32 @@ func (a *agent) runScript(ctx context.Context, lifecycle, script string) error { | ||
| } | ||
| a.logger.Info(ctx, "running script", slog.F("lifecycle", lifecycle), slog.F("script", script)) | ||
| fileWriter, err := a.filesystem.OpenFile(filepath.Join(a.logDir, fmt.Sprintf("coder-%s-script.log", lifecycle)), os.O_CREATE|os.O_RDWR, 0o600) | ||
| if err != nil { | ||
| return xerrors.Errorf("open %s script log file: %w", lifecycle, err) | ||
| } | ||
| defer func() { | ||
| _ =fileWriter.Close() | ||
| }() | ||
| var writer io.Writer = fileWriter | ||
| if lifecycle == "startup" { | ||
| // Create pipes for startup logs reader and writer | ||
| logsReader, logsWriter := io.Pipe() | ||
| defer func() { | ||
| _ = logsReader.Close() | ||
| }() | ||
| writer = io.MultiWriter(fileWriter, logsWriter) | ||
| flushedLogs, err := a.trackScriptLogs(ctx, logsReader) | ||
| if err != nil { | ||
| return xerrors.Errorf("track script logs: %w", err) | ||
| } | ||
| defer func() { | ||
| _ = logsWriter.Close() | ||
| <-flushedLogs | ||
| }() | ||
| } | ||
| cmd, err := a.createCommand(ctx, script, nil) | ||
| if err != nil { | ||
| return xerrors.Errorf("create command: %w", err) | ||
| @@ -664,10 +685,124 @@ func (a *agent) runScript(ctx context.Context, lifecycle, script string) error { | ||
| return xerrors.Errorf("run: %w", err) | ||
| } | ||
| return nil | ||
| } | ||
| func (a *agent) trackScriptLogs(ctx context.Context, reader io.Reader) (chan struct{}, error) { | ||
| // Initialize variables for log management | ||
| queuedLogs := make([]agentsdk.StartupLog, 0) | ||
| var flushLogsTimer *time.Timer | ||
| var logMutex sync.Mutex | ||
| logsFlushed := sync.NewCond(&sync.Mutex{}) | ||
| var logsSending bool | ||
| defer func() { | ||
| logMutex.Lock() | ||
| if flushLogsTimer != nil { | ||
| flushLogsTimer.Stop() | ||
| } | ||
| logMutex.Unlock() | ||
| }() | ||
| // sendLogs function uploads the queued logs to the server | ||
| sendLogs := func() { | ||
| // Lock logMutex and check if logs are already being sent | ||
| logMutex.Lock() | ||
| if logsSending { | ||
| logMutex.Unlock() | ||
| return | ||
| } | ||
| if flushLogsTimer != nil { | ||
| flushLogsTimer.Stop() | ||
| } | ||
| if len(queuedLogs) == 0 { | ||
| logMutex.Unlock() | ||
| return | ||
| } | ||
| // Move the current queued logs to logsToSend and clear the queue | ||
| logsToSend := queuedLogs | ||
| logsSending = true | ||
| queuedLogs = make([]agentsdk.StartupLog, 0) | ||
| logMutex.Unlock() | ||
| // Retry uploading logs until successful or a specific error occurs | ||
| for r := retry.New(time.Second, 5*time.Second); r.Wait(ctx); { | ||
| err := a.client.PatchStartupLogs(ctx, agentsdk.PatchStartupLogs{ | ||
| Logs: logsToSend, | ||
| }) | ||
kylecarbs marked this conversation as resolved. Show resolvedHide resolvedUh oh!There was an error while loading.Please reload this page. | ||
| if err == nil { | ||
| break | ||
| } | ||
| var sdkErr *codersdk.Error | ||
| if errors.As(err, &sdkErr) { | ||
| if sdkErr.StatusCode() == http.StatusRequestEntityTooLarge { | ||
| a.logger.Warn(ctx, "startup logs too large, dropping logs") | ||
| break | ||
| } | ||
| } | ||
| a.logger.Error(ctx, "upload startup logs", slog.Error(err), slog.F("to_send", logsToSend)) | ||
| } | ||
| // Reset logsSending flag | ||
| logMutex.Lock() | ||
| logsSending = false | ||
| flushLogsTimer.Reset(100 * time.Millisecond) | ||
| logMutex.Unlock() | ||
| logsFlushed.Broadcast() | ||
| } | ||
| // queueLog function appends a log to the queue and triggers sendLogs if necessary | ||
| queueLog := func(log agentsdk.StartupLog) { | ||
| logMutex.Lock() | ||
| defer logMutex.Unlock() | ||
| // Append log to the queue | ||
| queuedLogs = append(queuedLogs, log) | ||
| // If there are more than 100 logs, send them immediately | ||
| if len(queuedLogs) > 100 { | ||
| // Don't early return after this, because we still want | ||
| // to reset the timer just in case logs come in while | ||
| // we're sending. | ||
| go sendLogs() | ||
| } | ||
| // Reset or set the flushLogsTimer to trigger sendLogs after 100 milliseconds | ||
| if flushLogsTimer != nil { | ||
| flushLogsTimer.Reset(100 * time.Millisecond) | ||
| return | ||
| } | ||
| flushLogsTimer = time.AfterFunc(100*time.Millisecond, sendLogs) | ||
| } | ||
| // It's important that we either flush or drop all logs before returning | ||
| // because the startup state is reported after flush. | ||
| // | ||
| // It'd be weird for the startup state to be ready, but logs are still | ||
| // coming in. | ||
| logsFinished := make(chan struct{}) | ||
| err := a.trackConnGoroutine(func() { | ||
| scanner := bufio.NewScanner(reader) | ||
| for scanner.Scan() { | ||
| queueLog(agentsdk.StartupLog{ | ||
| CreatedAt: database.Now(), | ||
| Output: scanner.Text(), | ||
| }) | ||
| } | ||
| defer close(logsFinished) | ||
| logsFlushed.L.Lock() | ||
| for { | ||
| logMutex.Lock() | ||
| if len(queuedLogs) == 0 { | ||
| logMutex.Unlock() | ||
| break | ||
| } | ||
| logMutex.Unlock() | ||
| logsFlushed.Wait() | ||
| } | ||
| }) | ||
| if err != nil { | ||
| return nil, xerrors.Errorf("track conn goroutine: %w", err) | ||
| } | ||
| return logsFinished, nil | ||
| } | ||
| func (a *agent) init(ctx context.Context) { | ||
| // Clients' should ignore the host key when connecting. | ||
| // The agent needs to authenticate with coderd to SSH, | ||
118 changes: 89 additions & 29 deletionsagent/agent_test.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
5 changes: 4 additions & 1 deletioncli/agent.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
5 changes: 5 additions & 0 deletionscli/server.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
Oops, something went wrong.
Uh oh!
There was an error while loading.Please reload this page.
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.