Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

feat: add scaletest Runner for dynamicparameters load gen#19890

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.

Already on GitHub?Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes fromall commits
Commits
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

Some comments aren't visible on the classic Files Changed page.

1 change: 1 addition & 0 deletions.github/workflows/typos.toml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
[default]
extend-ignore-identifiers-re = ["gho_.*"]
extend-ignore-re = ["(#|//)\\s*spellchecker:ignore-next-line\\n.*"]

[default.extend-identifiers]
alog = "alog"
Expand Down
15 changes: 13 additions & 2 deletionscoderd/coderdtest/dynamicparameters.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,9 @@ type DynamicParameterTemplateParams struct {
Plan json.RawMessage
ModulesArchive []byte

// ExtraFiles are additional files to include in the template, beyond the MainTF.
ExtraFiles map[string][]byte

// Uses a zip archive instead of a tar
Zip bool

Expand All@@ -36,9 +39,17 @@ type DynamicParameterTemplateParams struct {
func DynamicParameterTemplate(t *testing.T, client *codersdk.Client, org uuid.UUID, args DynamicParameterTemplateParams) (codersdk.Template, codersdk.TemplateVersion) {
t.Helper()

files := echo.WithExtraFiles(map[string][]byte{
// Start with main.tf
extraFiles := map[string][]byte{
"main.tf": []byte(args.MainTF),
})
}

// Add any additional files
for name, content := range args.ExtraFiles {
extraFiles[name] = content
}

files := echo.WithExtraFiles(extraFiles)
files.ProvisionPlan = []*proto.Response{{
Type: &proto.Response_Plan{
Plan: &proto.PlanComplete{
Expand Down
18 changes: 18 additions & 0 deletionsprovisioner/echo/serve.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,6 +7,7 @@ import (
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"text/template"

Expand DownExpand Up@@ -359,9 +360,26 @@ func TarWithOptions(ctx context.Context, logger slog.Logger, responses *Response
}
}
}
dirs := []string{}
for name, content := range responses.ExtraFiles {
logger.Debug(ctx, "extra file", slog.F("name", name))

// We need to add directories before any files that use them. But, we only need to do this
// once.
dir := filepath.Dir(name)
if dir != "." && !slices.Contains(dirs, dir) {
logger.Debug(ctx, "adding extra file directory", slog.F("dir", dir))
dirs = append(dirs, dir)
err := writer.WriteHeader(&tar.Header{
Name: dir,
Mode: 0o755,
Typeflag: tar.TypeDir,
})
if err != nil {
return nil, err
}
}

err := writer.WriteHeader(&tar.Header{
Name: name,
Size: int64(len(content)),
Expand Down
10 changes: 10 additions & 0 deletionsscaletest/dynamicparameters/config.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
package dynamicparameters

import"github.com/google/uuid"

typeConfigstruct {
TemplateVersion uuid.UUID`json:"template_version"`
SessionTokenstring`json:"session_token"`
Metrics*Metrics`json:"-"`
MetricLabelValues []string`json:"metric_label_values"`
}
Comment on lines +5 to +10
Copy link
Member

@ethanndicksonethanndicksonSep 22, 2025•
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others.Learn more.

Optional: could use a Validate function here, I've found the one on my load generator somewhat useful, such as for checking that my time.Duration timeouts aren't zero.

28 changes: 28 additions & 0 deletionsscaletest/dynamicparameters/metrics.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
package dynamicparameters

import "github.com/prometheus/client_golang/prometheus"

type Metrics struct {
LatencyInitialResponseSeconds prometheus.HistogramVec
LatencyChangeResponseSeconds prometheus.HistogramVec
}

func NewMetrics(reg prometheus.Registerer, labelNames ...string) *Metrics {
m := &Metrics{
LatencyInitialResponseSeconds: *prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "coderd",
Subsystem: "scaletest",
Name: "dynamic_parameters_latency_initial_response_seconds",
Help: "Time in seconds to get the initial dynamic parameters response from start of request.",
}, labelNames),
LatencyChangeResponseSeconds: *prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "coderd",
Subsystem: "scaletest",
Name: "dynamic_parameters_latency_change_response_seconds",
Help: "Time in seconds to between sending a dynamic parameters change request and receiving the response.",
}, labelNames),
}
reg.MustRegister(m.LatencyInitialResponseSeconds)
reg.MustRegister(m.LatencyChangeResponseSeconds)
return m
}
114 changes: 114 additions & 0 deletionsscaletest/dynamicparameters/run.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
package dynamicparameters

import (
"context"
"fmt"
"io"
"slices"
"time"

"golang.org/x/xerrors"

"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/scaletest/harness"
"github.com/coder/websocket"
)

type Runner struct {
client *codersdk.Client
cfg Config
}

var _ harness.Runnable = &Runner{}

func NewRunner(client *codersdk.Client, cfg Config) *Runner {
clone := codersdk.New(client.URL)
clone.HTTPClient = client.HTTPClient
clone.SetLogger(client.Logger())
clone.SetSessionToken(cfg.SessionToken)
return &Runner{
client: clone,
cfg: cfg,
}
}

// Run executes the dynamic parameters test, which:
//
// 1. connects to the dynamic parameters stream
// 2. waits for the initial response
// 3. sends a change request
// 4. waits for the change response
// 5. closes the stream
func (r *Runner) Run(ctx context.Context, _ string, logs io.Writer) (retErr error) {
startTime := time.Now()
stream, err := r.client.TemplateVersionDynamicParameters(ctx, codersdk.Me, r.cfg.TemplateVersion)
if err != nil {
return xerrors.Errorf("connect to dynamic parameters stream: %w", err)
}
defer stream.Close(websocket.StatusNormalClosure)
respCh := stream.Chan()

var initTime time.Time
select {
case <-ctx.Done():
return ctx.Err()
case resp, ok := <-respCh:
if !ok {
return xerrors.Errorf("dynamic parameters stream closed before initial response")
}
initTime = time.Now()
r.cfg.Metrics.LatencyInitialResponseSeconds.
WithLabelValues(r.cfg.MetricLabelValues...).
Observe(initTime.Sub(startTime).Seconds())
_, _ = fmt.Fprintf(logs, "initial response: %+v\n", resp)
if !slices.ContainsFunc(resp.Parameters, func(p codersdk.PreviewParameter) bool {
return p.Name == "zero"
}) {
return xerrors.Errorf("missing expected parameter: 'zero'")
}
if err := checkNoDiagnostics(resp); err != nil {
return xerrors.Errorf("unexpected initial response diagnostics: %w", err)
}
}

err = stream.Send(codersdk.DynamicParametersRequest{
ID: 1,
Inputs: map[string]string{
"zero": "B",
},
})
if err != nil {
return xerrors.Errorf("send change request: %w", err)
}
select {
case <-ctx.Done():
return ctx.Err()
case resp, ok := <-respCh:
if !ok {
return xerrors.Errorf("dynamic parameters stream closed before change response")
}
_, _ = fmt.Fprintf(logs, "change response: %+v\n", resp)
r.cfg.Metrics.LatencyChangeResponseSeconds.
WithLabelValues(r.cfg.MetricLabelValues...).
Observe(time.Since(initTime).Seconds())
if resp.ID != 1 {
return xerrors.Errorf("unexpected response ID: %d", resp.ID)
}
if err := checkNoDiagnostics(resp); err != nil {
return xerrors.Errorf("unexpected change response diagnostics: %w", err)
}
return nil
Comment on lines +86 to +100
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others.Learn more.

We do not assert any param condition on the response. I wonder if we can refactor theParameterAsserter structure to be useful here:

funcAssertParameter(t*testing.T,namestring,params []codersdk.PreviewParameter)*ParameterAsserter {

It really streamlines the assertion code to something like:

coderdtest.AssertParameter(t,"groups",resp.Parameters).Exists().Options(database.EveryoneGroup,"admin","auditor").Value("admin")

It just currently only works in unit tests.

Copy link
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others.Learn more.

I feel like this is missing the point --- we are scale testing, not functional testing. I'm not looking to check that the response we get is "correct" in any detailed sense of the term, just that the response has not thrown errors and is doing the computation we want it to.

Emyrk reacted with thumbs up emoji
}
}

func checkNoDiagnostics(resp codersdk.DynamicParametersResponse) error {
if len(resp.Diagnostics) != 0 {
return xerrors.Errorf("unexpected response diagnostics: %v", resp.Diagnostics)
}
for _, param := range resp.Parameters {
if len(param.Diagnostics) != 0 {
return xerrors.Errorf("unexpected parameter diagnostics for '%s': %v", param.Name, param.Diagnostics)
}
}
return nil
}
49 changes: 49 additions & 0 deletionsscaletest/dynamicparameters/run_test.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
package dynamicparameters_test

import (
"strings"
"testing"

"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/require"

"cdr.dev/slog"
"github.com/coder/coder/v2/coderd/coderdtest"
"github.com/coder/coder/v2/scaletest/dynamicparameters"
"github.com/coder/coder/v2/testutil"
)

func TestRun(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)

client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true})
client.SetLogger(testutil.Logger(t).Leveled(slog.LevelDebug))
first := coderdtest.CreateFirstUser(t, client)
userClient, _ := coderdtest.CreateAnotherUser(t, client, first.OrganizationID)
orgID := first.OrganizationID

dynamicParametersTerraformSource, err := dynamicparameters.TemplateContent()
require.NoError(t, err)

template, version := coderdtest.DynamicParameterTemplate(t, client, orgID, coderdtest.DynamicParameterTemplateParams{
MainTF: dynamicParametersTerraformSource,
Plan: nil,
ModulesArchive: nil,
StaticParams: nil,
ExtraFiles: dynamicparameters.GetModuleFiles(),
})

reg := prometheus.NewRegistry()
cfg := dynamicparameters.Config{
TemplateVersion: version.ID,
SessionToken: userClient.SessionToken(),
Metrics: dynamicparameters.NewMetrics(reg, "template", "test_label_name"),
MetricLabelValues: []string{template.Name, "test_label_value"},
}
runner := dynamicparameters.NewRunner(userClient, cfg)
var logs strings.Builder
err = runner.Run(ctx, t.Name(), &logs)
t.Log("Runner logs:\n\n" + logs.String())
require.NoError(t, err)
}
74 changes: 74 additions & 0 deletionsscaletest/dynamicparameters/template.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
package dynamicparameters

import (
_ "embed"
"encoding/json"
"strings"
"text/template"

"github.com/coder/coder/v2/cryptorand"
)

//go:embed tf/main.tf
var templateContent string

func TemplateContent() (string, error) {
randomString, err := cryptorand.String(8)
if err != nil {
return "", err
}
tmpl, err := template.New("workspace-template").Parse(templateContent)
if err != nil {
return "", err
}
var result strings.Builder
err = tmpl.Execute(&result, map[string]string{
"RandomString": randomString,
})
if err != nil {
return "", err
}
return result.String(), nil
}

//go:embed tf/modules/two/main.tf
var moduleTwoMainTF string

// GetModuleFiles returns a map of module files to be used with ExtraFiles
func GetModuleFiles() map[string][]byte {
// Create the modules.json that Terraform needs to see the module
modulesJSON := struct {
Modules []struct {
Key string `json:"Key"`
Source string `json:"Source"`
Dir string `json:"Dir"`
} `json:"Modules"`
}{
Modules: []struct {
Key string `json:"Key"`
Source string `json:"Source"`
Dir string `json:"Dir"`
}{
{
Key: "",
Source: "",
Dir: ".",
},
{
Key: "two",
Source: "./modules/two",
Dir: "modules/two",
},
},
}

modulesJSONBytes, err := json.Marshal(modulesJSON)
if err != nil {
panic(err) // This should never happen with static data
}

return map[string][]byte{
"modules/two/main.tf": []byte(moduleTwoMainTF),
".terraform/modules/modules.json": modulesJSONBytes,
}
}
Loading
Loading
[8]ページ先頭
©2009-2025 Movatter.jp