Commitdc82ff2

committed

chore: implement first part of file streaming

1 parent360aa2f commitdc82ff2Copy full SHA for dc82ff2

File tree

+75

-7

lines changed

+75

-7

lines changed

Lines changed: 4 additions & 6 deletions

Original file line number	Diff line number	Diff line change
`@@ -19,13 +19,11 @@ import (`
`19`	`19`	`tfjson"github.com/hashicorp/terraform-json"`
`20`	`20`	`"go.opentelemetry.io/otel/attribute"`
`21`	`21`	`"golang.org/x/xerrors"`
`22`		`-protobuf"google.golang.org/protobuf/proto"`
`23`	`22`
`24`	`23`	`"cdr.dev/slog"`
`25`	`24`
`26`	`25`	`"github.com/coder/coder/v2/coderd/database"`
`27`	`26`	`"github.com/coder/coder/v2/coderd/tracing"`
`28`		`-"github.com/coder/coder/v2/codersdk/drpcsdk"`
`29`	`27`	`"github.com/coder/coder/v2/provisionersdk/proto"`
`30`	`28`	`)`
`31`	`29`
`@@ -357,10 +355,10 @@ func (e *executor) plan(ctx, killCtx context.Context, env, vars []string, logr l`
`357`	`355`	`ModuleFiles:moduleFiles,`
`358`	`356`	`}`
`359`	`357`
`360`		`-ifprotobuf.Size(msg)>drpcsdk.MaxMessageSize {`
`361`		`-e.logger.Warn(ctx,"cannot persist terraform modules, message payload too big",slog.F("archive_size",len(msg.ModuleFiles)))`
`362`		`-msg.ModuleFiles=nil`
`363`		`-}`
	`358`	`+//if protobuf.Size(msg) > drpcsdk.MaxMessageSize {`
	`359`	`+//e.logger.Warn(ctx, "cannot persist terraform modules, message payload too big", slog.F("archive_size", len(msg.ModuleFiles)))`
	`360`	`+//msg.ModuleFiles = nil`
	`361`	`+//}`
`364`	`362`
`365`	`363`	`returnmsg,nil`
`366`	`364`	`}`

Lines changed: 6 additions & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -13,6 +13,7 @@ import (`
`13`	`13`
`14`	`14`	`"golang.org/x/xerrors"`
`15`	`15`
	`16`	`+"github.com/coder/coder/v2/coderd/util/xio"`
`16`	`17`	`"github.com/coder/coder/v2/provisionersdk/proto"`
`17`	`18`	`)`
`18`	`19`
`@@ -85,7 +86,11 @@ func GetModulesArchive(root fs.FS) ([]byte, error) {`
`85`	`86`
`86`	`87`	`empty:=true`
`87`	`88`	`varb bytes.Buffer`
`88`		`-w:=tar.NewWriter(&b)`
	`89`	`+`
	`90`	`+// Limit to 20MB for now.`
	`91`	`+// TODO: Determine what a reasonable limit is for modules`
	`92`	`+lw:=xio.NewLimitWriter(&b,20<<20)`
	`93`	`+w:=tar.NewWriter(lw)`
`89`	`94`
`90`	`95`	`for_,it:=rangem.Modules {`
`91`	`96`	`// Check to make sure that the module is a remote module fetched by`

Lines changed: 1 addition & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+package provisionerd`

Lines changed: 38 additions & 0 deletions

Some generated files are not rendered by default. Learn more aboutcustomizing how changed files appear on GitHub.

Lines changed: 26 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -17,6 +17,9 @@ import (`
`17`	`17`	`"golang.org/x/xerrors"`
`18`	`18`
`19`	`19`	`"cdr.dev/slog"`
	`20`	`+"github.com/coder/coder/v2/codersdk/drpcsdk"`
	`21`	`+`
	`22`	`+protobuf"google.golang.org/protobuf/proto"`
`20`	`23`
`21`	`24`	`"github.com/coder/coder/v2/provisionersdk/proto"`
`22`	`25`	`)`
`@@ -161,6 +164,29 @@ func (s *Session) handleRequests() error {`
`161`	`164`	`returnerr`
`162`	`165`	`}`
`163`	`166`	`resp.Type=&proto.Response_Plan{Plan:complete}`
	`167`	`+`
	`168`	`+ifprotobuf.Size(resp)>drpcsdk.MaxMessageSize {`
	`169`	`+// Send the modules over as a stream`
	`170`	`+s.Logger.Info(s.Context(),"plan response too large, sending modules as stream")`
	`171`	`+dataUp,chunks:=proto.BytesToDataUpload(complete.ModuleFiles)`
	`172`	`+`
	`173`	`+complete.ModuleFiles=nil// sent over the stream`
	`174`	`+resp.Type=&proto.Response_Plan{Plan:complete}`
	`175`	`+`
	`176`	`+err:=s.stream.Send(&proto.Response{Type:&proto.Response_DataUpload{DataUpload:dataUp}})`
	`177`	`+iferr!=nil {`
	`178`	`+complete.Error=fmt.Sprintf("send data upload: %s",err.Error())`
	`179`	`+}else {`
	`180`	`+fori,chunk:=rangechunks {`
	`181`	`+err:=s.stream.Send(&proto.Response{Type:&proto.Response_ChunkPiece{ChunkPiece:chunk}})`
	`182`	`+iferr!=nil {`
	`183`	`+complete.Error=fmt.Sprintf("send data piece upload %d/%d: %s",i,dataUp.Chunks,err.Error())`
	`184`	`+break`
	`185`	`+}`
	`186`	`+}`
	`187`	`+}`
	`188`	`+}`
	`189`	`+`
`164`	`190`	`ifcomplete.Error=="" {`
`165`	`191`	`planned=true`
`166`	`192`	`}`

Comments

(0)