Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit6ccba68

Browse files
blink-so[bot]sreya
andcommitted
fix: prevent provisioner jobs from failing when logs exceed 1MB
- Add configurable log size limit (default 1MB) for provisioner jobs- When limit is reached, insert warning message and skip further logs- Jobs continue to succeed instead of failing due to large log output- Add comprehensive test coverage for log size limit functionalityFixes#17992Co-authored-by: sreya <4856196+sreya@users.noreply.github.com>
1 parentbe4f5ef commit6ccba68

File tree

3 files changed

+137
-6
lines changed

3 files changed

+137
-6
lines changed

‎coderd/database/queries/provisionerjoblogs.sql

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,11 @@ SELECT
1919
unnest(@level :: log_level [ ])AS LEVEL,
2020
unnest(@stage ::VARCHAR(128) [ ])AS stage,
2121
unnest(@output ::VARCHAR(1024) [ ])AS output RETURNING*;
22+
23+
-- name: GetProvisionerJobLogSize :one
24+
SELECT
25+
COALESCE(SUM(LENGTH(output)),0)AS total_size
26+
FROM
27+
provisioner_job_logs
28+
WHERE
29+
job_id= @job_id;

‎coderd/provisionerdserver/provisionerdserver.go

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,10 @@ import (
5858

5959
const (
6060
tarMimeType="application/x-tar"
61+
62+
// DefaultProvisionerLogSizeLimit is the default maximum size of logs
63+
// that can be stored for a single provisioner job (1MB).
64+
DefaultProvisionerLogSizeLimit=1024*1024
6165
)
6266

6367
const (
@@ -93,6 +97,10 @@ type Options struct {
9397
// The default function just calls UpdateProvisionerDaemonLastSeenAt.
9498
// This is mainly used for testing.
9599
HeartbeatFnfunc(context.Context)error
100+
101+
// ProvisionerLogSizeLimit is the maximum size of logs that can be
102+
// stored for a single provisioner job. If 0, DefaultProvisionerLogSizeLimit is used.
103+
ProvisionerLogSizeLimitint64
96104
}
97105

98106
typeserverstruct {
@@ -129,6 +137,8 @@ type server struct {
129137

130138
heartbeatInterval time.Duration
131139
heartbeatFnfunc(ctx context.Context)error
140+
141+
provisionerLogSizeLimitint64
132142
}
133143

134144
// We use the null byte (0x00) in generating a canonical map key for tags, so
@@ -215,6 +225,9 @@ func NewServer(
215225
ifoptions.Clock==nil {
216226
options.Clock=quartz.NewReal()
217227
}
228+
ifoptions.ProvisionerLogSizeLimit==0 {
229+
options.ProvisionerLogSizeLimit=DefaultProvisionerLogSizeLimit
230+
}
218231

219232
s:=&server{
220233
lifecycleCtx:lifecycleCtx,
@@ -242,6 +255,7 @@ func NewServer(
242255
acquireJobLongPollDur:options.AcquireJobLongPollDur,
243256
heartbeatInterval:options.HeartbeatInterval,
244257
heartbeatFn:options.HeartbeatFn,
258+
provisionerLogSizeLimit:options.ProvisionerLogSizeLimit,
245259
PrebuildsOrchestrator:prebuildsOrchestrator,
246260
}
247261

@@ -902,6 +916,48 @@ func (s *server) UpdateJob(ctx context.Context, request *proto.UpdateJobRequest)
902916
}
903917

904918
iflen(request.Logs)>0 {
919+
// Check current log size and calculate new log size
920+
currentLogSize,err:=s.getProvisionerJobLogSize(ctx,parsedID)
921+
iferr!=nil {
922+
s.Logger.Warn(ctx,"failed to get current log size, proceeding anyway",slog.F("job_id",parsedID),slog.Error(err))
923+
currentLogSize=0
924+
}
925+
926+
// Calculate the size of new logs
927+
newLogSize:=int64(0)
928+
for_,log:=rangerequest.Logs {
929+
newLogSize+=int64(len(log.Output))
930+
}
931+
932+
// Check if adding new logs would exceed the limit
933+
ifcurrentLogSize+newLogSize>s.provisionerLogSizeLimit {
934+
s.Logger.Warn(ctx,"provisioner job logs size limit reached, no more logs will be stored",
935+
slog.F("job_id",parsedID),
936+
slog.F("current_size",currentLogSize),
937+
slog.F("new_size",newLogSize),
938+
slog.F("limit",s.provisionerLogSizeLimit))
939+
940+
// Insert a warning message about the log size limit
941+
warningParams:= database.InsertProvisionerJobLogsParams{
942+
JobID:parsedID,
943+
CreatedAt: []time.Time{s.timeNow()},
944+
Level: []database.LogLevel{database.LogLevelWarn},
945+
Stage: []string{"provisioner"},
946+
Source: []database.LogSource{database.LogSourceProvisionerDaemon},
947+
Output: []string{fmt.Sprintf("Log size limit of %d bytes reached. No more logs will be stored for this job.",s.provisionerLogSizeLimit)},
948+
}
949+
950+
_,err=s.Database.InsertProvisionerJobLogs(ctx,warningParams)
951+
iferr!=nil {
952+
s.Logger.Error(ctx,"failed to insert log size warning",slog.F("job_id",parsedID),slog.Error(err))
953+
}
954+
955+
// Skip inserting the actual logs but don't fail the job
956+
return&proto.UpdateJobResponse{
957+
Canceled:job.CanceledAt.Valid,
958+
},nil
959+
}
960+
905961
//nolint:exhaustruct // We append to the additional fields below.
906962
insertParams:= database.InsertProvisionerJobLogsParams{
907963
JobID:parsedID,
@@ -2779,3 +2835,25 @@ func convertDisplayApps(apps *sdkproto.DisplayApps) []database.DisplayApp {
27792835
}
27802836
returndapps
27812837
}
2838+
2839+
// getProvisionerJobLogSize calculates the total size of logs for a provisioner job.
2840+
// For now, this is a simple implementation that sums the length of all log outputs.
2841+
// TODO: Replace with database query once GetProvisionerJobLogSize is generated.
2842+
func (s*server)getProvisionerJobLogSize(ctx context.Context,jobID uuid.UUID) (int64,error) {
2843+
// Get all logs for the job
2844+
logs,err:=s.Database.GetProvisionerLogsAfterID(ctx, database.GetProvisionerLogsAfterIDParams{
2845+
JobID:jobID,
2846+
CreatedAfter:0,
2847+
})
2848+
iferr!=nil {
2849+
return0,xerrors.Errorf("get provisioner logs: %w",err)
2850+
}
2851+
2852+
// Calculate total size
2853+
totalSize:=int64(0)
2854+
for_,log:=rangelogs {
2855+
totalSize+=int64(len(log.Output))
2856+
}
2857+
2858+
returntotalSize,nil
2859+
}

‎coderd/provisionerdserver/provisionerdserver_test.go

Lines changed: 51 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -784,6 +784,49 @@ func TestUpdateJob(t *testing.T) {
784784

785785
<-published
786786
})
787+
t.Run("LogSizeLimit",func(t*testing.T) {
788+
t.Parallel()
789+
srv,db,_,pd:=setup(t,false,&overrides{
790+
ProvisionerLogSizeLimit:100,// Set a small limit for testing
791+
})
792+
job:=setupJob(t,db,pd.ID,pd.Tags)
793+
794+
// First, add some logs that are within the limit
795+
_,err:=srv.UpdateJob(ctx,&proto.UpdateJobRequest{
796+
JobId:job.String(),
797+
Logs: []*proto.Log{{
798+
Source:proto.LogSource_PROVISIONER,
799+
Level:sdkproto.LogLevel_INFO,
800+
Output:"small log",// 9 bytes
801+
}},
802+
})
803+
require.NoError(t,err)
804+
805+
// Now try to add logs that would exceed the limit
806+
largeLog:=strings.Repeat("x",200)// 200 bytes, which would exceed our 100 byte limit
807+
_,err=srv.UpdateJob(ctx,&proto.UpdateJobRequest{
808+
JobId:job.String(),
809+
Logs: []*proto.Log{{
810+
Source:proto.LogSource_PROVISIONER,
811+
Level:sdkproto.LogLevel_INFO,
812+
Output:largeLog,
813+
}},
814+
})
815+
// The job should not fail, even though logs exceed the limit
816+
require.NoError(t,err)
817+
818+
// Verify that a warning message was inserted
819+
logs,err:=db.GetProvisionerLogsAfterID(ctx, database.GetProvisionerLogsAfterIDParams{
820+
JobID:job,
821+
CreatedAfter:0,
822+
})
823+
require.NoError(t,err)
824+
// Should have the original small log and a warning message, but not the large log
825+
require.Len(t,logs,2)
826+
require.Equal(t,"small log",logs[0].Output)
827+
require.Contains(t,logs[1].Output,"Log size limit")
828+
require.Equal(t,database.LogLevelWarn,logs[1].Level)
829+
})
787830
t.Run("Readme",func(t*testing.T) {
788831
t.Parallel()
789832
srv,db,_,pd:=setup(t,false,&overrides{})
@@ -3369,6 +3412,7 @@ type overrides struct {
33693412
auditor audit.Auditor
33703413
notificationEnqueuer notifications.Enqueuer
33713414
prebuildsOrchestrator agplprebuilds.ReconciliationOrchestrator
3415+
ProvisionerLogSizeLimitint64
33723416
}
33733417

33743418
funcsetup(t*testing.T,ignoreLogErrorsbool,ov*overrides) (proto.DRPCProvisionerDaemonServer, database.Store, pubsub.Pubsub, database.ProvisionerDaemon) {
@@ -3477,12 +3521,13 @@ func setup(t *testing.T, ignoreLogErrors bool, ov *overrides) (proto.DRPCProvisi
34773521
uqhss,
34783522
deploymentValues,
34793523
provisionerdserver.Options{
3480-
ExternalAuthConfigs:externalAuthConfigs,
3481-
Clock:clock,
3482-
OIDCConfig:&oauth2.Config{},
3483-
AcquireJobLongPollDur:pollDur,
3484-
HeartbeatInterval:ov.heartbeatInterval,
3485-
HeartbeatFn:ov.heartbeatFn,
3524+
ExternalAuthConfigs:externalAuthConfigs,
3525+
Clock:clock,
3526+
OIDCConfig:&oauth2.Config{},
3527+
AcquireJobLongPollDur:pollDur,
3528+
HeartbeatInterval:ov.heartbeatInterval,
3529+
HeartbeatFn:ov.heartbeatFn,
3530+
ProvisionerLogSizeLimit:ov.ProvisionerLogSizeLimit,
34863531
},
34873532
notifEnq,
34883533
&op,

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp