Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

feat: add scaletest Runner for dynamicparameters load gen#19890

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.

Already on GitHub?Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from1 commit
Commits
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
NextNext commit
feat: add scaletest Runner for dynamicparameters load gen
  • Loading branch information
@spikecurtis
spikecurtis committedSep 23, 2025
commit15ed27bc3825f3c4b44913600c61e65ceb821634
10 changes: 10 additions & 0 deletionsscaletest/dynamicparameters/config.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
package dynamicparameters

import"github.com/google/uuid"

typeConfigstruct {
TemplateVersion uuid.UUID`json:"template_version"`
SessionTokenstring`json:"session_token"`
Metrics*Metrics`json:"-"`
MetricLabelValues []string`json:"metric_label_values"`
}
Comment on lines +5 to +10
Copy link
Member

@ethanndicksonethanndicksonSep 22, 2025•
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others.Learn more.

Optional: could use a Validate function here, I've found the one on my load generator somewhat useful, such as for checking that my time.Duration timeouts aren't zero.

28 changes: 28 additions & 0 deletionsscaletest/dynamicparameters/metrics.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
package dynamicparameters

import "github.com/prometheus/client_golang/prometheus"

type Metrics struct {
LatencyInitialResponseSeconds prometheus.HistogramVec
LatencyChangeResponseSeconds prometheus.HistogramVec
}

func NewMetrics(reg prometheus.Registerer, labelNames ...string) *Metrics {
m := &Metrics{
LatencyInitialResponseSeconds: *prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "coderd",
Subsystem: "scaletest",
Name: "dynamic_parameters_latency_initial_response_seconds",
Help: "Time in seconds to get the initial dynamic parameters response from start of request.",
}, labelNames),
LatencyChangeResponseSeconds: *prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "coderd",
Subsystem: "scaletest",
Name: "dynamic_parameters_latency_change_response_seconds",
Help: "Time in seconds to between sending a dynamic parameters change request and receiving the response.",
}, labelNames),
}
reg.MustRegister(m.LatencyInitialResponseSeconds)
reg.MustRegister(m.LatencyChangeResponseSeconds)
return m
}
114 changes: 114 additions & 0 deletionsscaletest/dynamicparameters/run.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
package dynamicparameters

import (
"context"
"fmt"
"io"
"slices"
"time"

"golang.org/x/xerrors"

"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/scaletest/harness"
"github.com/coder/websocket"
)

type Runner struct {
client *codersdk.Client
cfg Config
}

var _ harness.Runnable = &Runner{}

func NewRunner(client *codersdk.Client, cfg Config) *Runner {
clone := codersdk.New(client.URL)
clone.HTTPClient = client.HTTPClient
clone.SetLogger(client.Logger())
clone.SetSessionToken(cfg.SessionToken)
return &Runner{
client: clone,
cfg: cfg,
}
}

// Run executes the dynamic parameters test, which:
//
// 1. connects to the dynamic parameters stream
// 2. waits for the initial response
// 3. sends a change request
// 4. waits for the change response
// 5. closes the stream
func (r *Runner) Run(ctx context.Context, _ string, logs io.Writer) (retErr error) {
startTime := time.Now()
stream, err := r.client.TemplateVersionDynamicParameters(ctx, codersdk.Me, r.cfg.TemplateVersion)
if err != nil {
return xerrors.Errorf("connect to dynamic parameters stream: %w", err)
}
defer stream.Close(websocket.StatusNormalClosure)
respCh := stream.Chan()

var initTime time.Time
select {
case <-ctx.Done():
return ctx.Err()
case resp, ok := <-respCh:
if !ok {
return xerrors.Errorf("dynamic parameters stream closed before initial response")
}
initTime = time.Now()
r.cfg.Metrics.LatencyInitialResponseSeconds.
WithLabelValues(r.cfg.MetricLabelValues...).
Observe(initTime.Sub(startTime).Seconds())
_, _ = fmt.Fprintf(logs, "initial response: %+v\n", resp)
if !slices.ContainsFunc(resp.Parameters, func(p codersdk.PreviewParameter) bool {
return p.Name == "zero"
}) {
return xerrors.Errorf("mising expected parameter: 'zero'")
}
if err := checkNoDiagnostics(resp); err != nil {
return xerrors.Errorf("unexpected initial response diagnostics: %w", err)
}
}

err = stream.Send(codersdk.DynamicParametersRequest{
ID: 1,
Inputs: map[string]string{
"zero": "B",
},
})
if err != nil {
return xerrors.Errorf("send change request: %w", err)
}
select {
case <-ctx.Done():
return ctx.Err()
case resp, ok := <-respCh:
if !ok {
return xerrors.Errorf("dynamic parameters stream closed before change response")
}
_, _ = fmt.Fprintf(logs, "change response: %+v\n", resp)
r.cfg.Metrics.LatencyChangeResponseSeconds.
WithLabelValues(r.cfg.MetricLabelValues...).
Observe(time.Since(initTime).Seconds())
if resp.ID != 1 {
return xerrors.Errorf("unexpected response ID: %d", resp.ID)
}
if err := checkNoDiagnostics(resp); err != nil {
return xerrors.Errorf("unexpected change response diagnostics: %w", err)
}
return nil
Comment on lines +86 to +100
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others.Learn more.

We do not assert any param condition on the response. I wonder if we can refactor theParameterAsserter structure to be useful here:

funcAssertParameter(t*testing.T,namestring,params []codersdk.PreviewParameter)*ParameterAsserter {

It really streamlines the assertion code to something like:

coderdtest.AssertParameter(t,"groups",resp.Parameters).Exists().Options(database.EveryoneGroup,"admin","auditor").Value("admin")

It just currently only works in unit tests.

Copy link
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others.Learn more.

I feel like this is missing the point --- we are scale testing, not functional testing. I'm not looking to check that the response we get is "correct" in any detailed sense of the term, just that the response has not thrown errors and is doing the computation we want it to.

Emyrk reacted with thumbs up emoji
}
}

func checkNoDiagnostics(resp codersdk.DynamicParametersResponse) error {
if len(resp.Diagnostics) != 0 {
return xerrors.Errorf("unexpected response diagnostics: %v", resp.Diagnostics)
}
for _, param := range resp.Parameters {
if len(param.Diagnostics) != 0 {
return xerrors.Errorf("unexpected parameter diagnostics for '%s': %v", param.Name, param.Diagnostics)
}
}
return nil
}
46 changes: 46 additions & 0 deletionsscaletest/dynamicparameters/run_test.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
package dynamicparameters_test

import (
"strings"
"testing"

"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/require"

"github.com/coder/coder/v2/coderd/coderdtest"
"github.com/coder/coder/v2/scaletest/dynamicparameters"
"github.com/coder/coder/v2/testutil"
)

func TestRun(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)

client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true})
first := coderdtest.CreateFirstUser(t, client)
userClient, _ := coderdtest.CreateAnotherUser(t, client, first.OrganizationID)
orgID := first.OrganizationID

dynamicParametersTerraformSource, err := dynamicparameters.TemplateContent()
require.NoError(t, err)

template, version := coderdtest.DynamicParameterTemplate(t, client, orgID, coderdtest.DynamicParameterTemplateParams{
MainTF: dynamicParametersTerraformSource,
Plan: nil,
ModulesArchive: nil,
StaticParams: nil,
})

reg := prometheus.NewRegistry()
cfg := dynamicparameters.Config{
TemplateVersion: version.ID,
SessionToken: userClient.SessionToken(),
Metrics: dynamicparameters.NewMetrics(reg, "template", "test_label_name"),
MetricLabelValues: []string{template.Name, "test_label_value"},
}
runner := dynamicparameters.NewRunner(userClient, cfg)
var logs strings.Builder
err = runner.Run(ctx, t.Name(), &logs)
t.Log("Runner logs:\n\n" + logs.String())
require.NoError(t, err)
}
37 changes: 37 additions & 0 deletionsscaletest/dynamicparameters/template.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
package dynamicparameters

import (
_ "embed"
"strings"
"text/template"

"github.com/coder/coder/v2/cryptorand"
)

//go:embed workspace-template.tf
var templateContent string

func TemplateContent() (string, error) {
randomString, err := cryptorand.String(8)
if err != nil {
return "", err
}

// Parse the template
tmpl, err := template.New("workspace-template").Parse(templateContent)
if err != nil {
return "", err
}

// Execute the template with the random string
var result strings.Builder
err = tmpl.Execute(&result, map[string]string{
"RandomString": randomString,
})
if err != nil {
// Return the original template if execution fails
return "", err
}

return result.String(), nil
}
126 changes: 126 additions & 0 deletionsscaletest/dynamicparameters/workspace-template.tf
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
# Cache busting string so each copy of the template is unique: {{.RandomString}}
terraform {
required_providers {
coder = {
source = "coder/coder"
version = "2.5.3"
}
}
}

locals {
one_options = {
"A" = ["AA", "AB"]
"B" = ["BA", "BB"]
}

three_options = {
"AA" = ["AAA", "AAB"]
"AB" = ["ABA", "ABB"]
"BA" = ["BAA", "BAB"]
"BB" = ["BBA", "BBB"]
}

username = data.coder_workspace_owner.me.name
}

data "coder_workspace_owner" "me" {}

data "coder_parameter" "zero" {
name = "zero"
display_name = "Root"
description = "Hello ${local.username}, pick your next parameter using this `dropdown` parameter."
form_type = "dropdown"
mutable = true
default = "A"

option {
value = "A"
name = "A"
}

option {
value = "B"
name = "B"
}
}

data "coder_parameter" "one" {

name = "One"
display_name = "Level One"
description = "This is the first level."

type = "list(string)"
form_type = "multi-select"
order = 2
mutable = true
default = "[\"${local.one_options[data.coder_parameter.zero.value][0]}\"]"

dynamic "option" {
for_each = local.one_options[data.coder_parameter.zero.value]
content {
name = option.value
value = option.value
}
}
}

data "coder_parameter" "two" {

name = "Two"
display_name = "Level Two"
description = "This is the second level."

type = "string"
form_type = "textarea"
order = 3
mutable = true

default = trim(data.coder_parameter.one.value, "[\"]")
}

data "coder_parameter" "three" {

name = "Three"
display_name = "Level Three"
description = "This is the first level."

type = "string"
form_type = "radio"
order = 4
mutable = true
default = local.three_options[data.coder_parameter.two.value][0]

dynamic "option" {
for_each = local.three_options[data.coder_parameter.two.value]
content {
name = option.value
value = option.value
}
}
}

data "coder_parameter" "four" {
name = "four"
display_name = "Level Four"
description = "This is the last level."
order = 5

type = "string"
form_type = "radio"
default = "a_fake_value_to_satify_import"

option {
name = format("%s-%s", local.username, data.coder_parameter.three.value)
value = "a_fake_value_to_satify_import"
}

dynamic "option" {
for_each = data.coder_workspace_owner.me.rbac_roles
content {
name = format("%s-%s", option.value.name, data.coder_parameter.three.value)
value = option.value.name
}
}
}
[8]ページ先頭
©2009-2025 Movatter.jp