Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

fix(coderd): extract provisionerdserver.StaleInterval to 90 seconds#15643

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.

Already on GitHub?Sign in to your account

Merged
johnstcn merged 3 commits intomainfromcj/provisionerdserver-staleheartbeats
Nov 28, 2024
Merged
Show file tree
Hide file tree
Changes fromall commits
Commits
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletioncoderd/coderd.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -628,7 +628,8 @@ func New(options *Options) *API {
CurrentVersion: buildinfo.Version(),
CurrentAPIMajorVersion: proto.CurrentMajor,
Store: options.Database,
// TimeNow and StaleInterval set to defaults, see healthcheck/provisioner.go
StaleInterval: provisionerdserver.StaleInterval,
// TimeNow set to default, see healthcheck/provisioner.go
},
})
}
Expand Down
2 changes: 1 addition & 1 deletioncoderd/healthcheck/provisioner.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -50,7 +50,7 @@ func (r *ProvisionerDaemonsReport) Run(ctx context.Context, opts *ProvisionerDae
now := opts.TimeNow()

if opts.StaleInterval == 0 {
opts.StaleInterval = provisionerdserver.DefaultHeartbeatInterval * 3
opts.StaleInterval = provisionerdserver.StaleInterval
}

if opts.CurrentVersion == "" {
Expand Down
106 changes: 80 additions & 26 deletionscoderd/healthcheck/provisioner_test.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -15,15 +15,21 @@ import (
"github.com/coder/coder/v2/coderd/database/dbtime"
"github.com/coder/coder/v2/coderd/healthcheck"
"github.com/coder/coder/v2/coderd/healthcheck/health"
"github.com/coder/coder/v2/coderd/provisionerdserver"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/codersdk/healthsdk"
"github.com/coder/coder/v2/provisionerd/proto"
"github.com/coder/coder/v2/testutil"
)

func TestProvisionerDaemonReport(t *testing.T) {
t.Parallel()

now := dbtime.Now()
var (
now = dbtime.Now()
oneHourAgo = now.Add(-time.Hour)
staleThreshold = now.Add(-provisionerdserver.StaleInterval).Add(-time.Second)
)

for _, tt := range []struct {
name string
Expand DownExpand Up@@ -65,7 +71,9 @@ func TestProvisionerDaemonReport(t *testing.T) {
currentVersion: "v1.2.3",
currentAPIMajorVersion: proto.CurrentMajor,
expectedSeverity: health.SeverityOK,
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-ok", "v1.2.3", "1.0", now)},
provisionerDaemons: []database.ProvisionerDaemon{
fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v1.2.3"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
},
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
{
ProvisionerDaemon: codersdk.ProvisionerDaemon{
Expand All@@ -88,7 +96,9 @@ func TestProvisionerDaemonReport(t *testing.T) {
currentAPIMajorVersion: proto.CurrentMajor,
expectedSeverity: health.SeverityWarning,
expectedWarningCode: health.CodeProvisionerDaemonVersionMismatch,
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-old", "v1.1.2", "1.0", now)},
provisionerDaemons: []database.ProvisionerDaemon{
fakeProvisionerDaemon(t, withName("pd-old"), withVersion("v1.1.2"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
},
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
{
ProvisionerDaemon: codersdk.ProvisionerDaemon{
Expand DownExpand Up@@ -116,7 +126,9 @@ func TestProvisionerDaemonReport(t *testing.T) {
currentAPIMajorVersion: proto.CurrentMajor,
expectedSeverity: health.SeverityError,
expectedWarningCode: health.CodeUnknown,
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-invalid-version", "invalid", "1.0", now)},
provisionerDaemons: []database.ProvisionerDaemon{
fakeProvisionerDaemon(t, withName("pd-invalid-version"), withVersion("invalid"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
},
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
{
ProvisionerDaemon: codersdk.ProvisionerDaemon{
Expand DownExpand Up@@ -144,7 +156,9 @@ func TestProvisionerDaemonReport(t *testing.T) {
currentAPIMajorVersion: proto.CurrentMajor,
expectedSeverity: health.SeverityError,
expectedWarningCode: health.CodeUnknown,
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-invalid-api", "v1.2.3", "invalid", now)},
provisionerDaemons: []database.ProvisionerDaemon{
fakeProvisionerDaemon(t, withName("pd-invalid-api"), withVersion("v1.2.3"), withAPIVersion("invalid"), withCreatedAt(now), withLastSeenAt(now)),
},
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
{
ProvisionerDaemon: codersdk.ProvisionerDaemon{
Expand DownExpand Up@@ -172,7 +186,9 @@ func TestProvisionerDaemonReport(t *testing.T) {
currentAPIMajorVersion: 2,
expectedSeverity: health.SeverityWarning,
expectedWarningCode: health.CodeProvisionerDaemonAPIMajorVersionDeprecated,
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-old-api", "v2.3.4", "1.0", now)},
provisionerDaemons: []database.ProvisionerDaemon{
fakeProvisionerDaemon(t, withName("pd-old-api"), withVersion("v2.3.4"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
},
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
{
ProvisionerDaemon: codersdk.ProvisionerDaemon{
Expand DownExpand Up@@ -200,7 +216,10 @@ func TestProvisionerDaemonReport(t *testing.T) {
currentAPIMajorVersion: proto.CurrentMajor,
expectedSeverity: health.SeverityWarning,
expectedWarningCode: health.CodeProvisionerDaemonVersionMismatch,
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-ok", "v1.2.3", "1.0", now), fakeProvisionerDaemon(t, "pd-old", "v1.1.2", "1.0", now)},
provisionerDaemons: []database.ProvisionerDaemon{
fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v1.2.3"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
fakeProvisionerDaemon(t, withName("pd-old"), withVersion("v1.1.2"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
},
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
{
ProvisionerDaemon: codersdk.ProvisionerDaemon{
Expand DownExpand Up@@ -241,7 +260,10 @@ func TestProvisionerDaemonReport(t *testing.T) {
currentAPIMajorVersion: proto.CurrentMajor,
expectedSeverity: health.SeverityWarning,
expectedWarningCode: health.CodeProvisionerDaemonVersionMismatch,
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-ok", "v1.2.3", "1.0", now), fakeProvisionerDaemon(t, "pd-new", "v2.3.4", "1.0", now)},
provisionerDaemons: []database.ProvisionerDaemon{
fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v1.2.3"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
fakeProvisionerDaemon(t, withName("pd-new"), withVersion("v2.3.4"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
},
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
{
ProvisionerDaemon: codersdk.ProvisionerDaemon{
Expand DownExpand Up@@ -281,7 +303,10 @@ func TestProvisionerDaemonReport(t *testing.T) {
currentVersion: "v2.3.4",
currentAPIMajorVersion: proto.CurrentMajor,
expectedSeverity: health.SeverityOK,
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemonStale(t, "pd-stale", "v1.2.3", "0.9", now.Add(-5*time.Minute), now), fakeProvisionerDaemon(t, "pd-ok", "v2.3.4", "1.0", now)},
provisionerDaemons: []database.ProvisionerDaemon{
fakeProvisionerDaemon(t, withName("pd-stale"), withVersion("v1.2.3"), withAPIVersion("0.9"), withCreatedAt(oneHourAgo), withLastSeenAt(staleThreshold)),
fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v2.3.4"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
},
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
{
ProvisionerDaemon: codersdk.ProvisionerDaemon{
Expand All@@ -304,8 +329,10 @@ func TestProvisionerDaemonReport(t *testing.T) {
currentAPIMajorVersion: proto.CurrentMajor,
expectedSeverity: health.SeverityError,
expectedWarningCode: health.CodeProvisionerDaemonsNoProvisionerDaemons,
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemonStale(t, "pd-ok", "v1.2.3", "0.9", now.Add(-5*time.Minute), now)},
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{},
provisionerDaemons: []database.ProvisionerDaemon{
fakeProvisionerDaemon(t, withName("pd-stale"), withVersion("v1.2.3"), withAPIVersion("0.9"), withCreatedAt(oneHourAgo), withLastSeenAt(staleThreshold)),
},
expectedItems: []healthsdk.ProvisionerDaemonsReportItem{},
},
} {
tt := tt
Expand DownExpand Up@@ -353,25 +380,52 @@ func TestProvisionerDaemonReport(t *testing.T) {
}
}

func fakeProvisionerDaemon(t *testing.T, name, version, apiVersion string, now time.Time) database.ProvisionerDaemon {
func withName(s string) func(*database.ProvisionerDaemon) {
return func(pd *database.ProvisionerDaemon) {
pd.Name = s
}
}

func withCreatedAt(at time.Time) func(*database.ProvisionerDaemon) {
return func(pd *database.ProvisionerDaemon) {
pd.CreatedAt = at
}
}

func withLastSeenAt(at time.Time) func(*database.ProvisionerDaemon) {
return func(pd *database.ProvisionerDaemon) {
pd.LastSeenAt.Valid = true
pd.LastSeenAt.Time = at
}
}

func withVersion(v string) func(*database.ProvisionerDaemon) {
return func(pd *database.ProvisionerDaemon) {
pd.Version = v
}
}

func withAPIVersion(v string) func(*database.ProvisionerDaemon) {
return func(pd *database.ProvisionerDaemon) {
pd.APIVersion = v
}
}

func fakeProvisionerDaemon(t *testing.T, opts ...func(*database.ProvisionerDaemon)) database.ProvisionerDaemon {
t.Helper()
return database.ProvisionerDaemon{
pd := database.ProvisionerDaemon{
ID: uuid.Nil,
Name:name,
CreatedAt:now,
LastSeenAt: sql.NullTime{Time: now, Valid: true},
Name:testutil.GetRandomName(t),
CreatedAt:time.Time{},
LastSeenAt: sql.NullTime{},
Provisioners: []database.ProvisionerType{database.ProvisionerTypeEcho, database.ProvisionerTypeTerraform},
ReplicaID: uuid.NullUUID{},
Tags: map[string]string{},
Version:version,
APIVersion:apiVersion,
Version:"",
APIVersion:"",
}
}

func fakeProvisionerDaemonStale(t *testing.T, name, version, apiVersion string, lastSeenAt, now time.Time) database.ProvisionerDaemon {
t.Helper()
d := fakeProvisionerDaemon(t, name, version, apiVersion, now)
d.LastSeenAt.Valid = true
d.LastSeenAt.Time = lastSeenAt
return d
for _, o := range opts {
o(&pd)
}
return pd
}
4 changes: 4 additions & 0 deletionscoderd/provisionerdserver/provisionerdserver.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -57,6 +57,10 @@ const (
// DefaultHeartbeatInterval is the interval at which the provisioner daemon
// will update its last seen at timestamp in the database.
DefaultHeartbeatInterval = time.Minute

// StaleInterval is the amount of time after the last heartbeat for which
// the provisioner will be reported as 'stale'.
StaleInterval = 90 * time.Second
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others.Learn more.

If I configure my heartbeat interval to be greater than 90 sec, it reads like this will introduce false negatives where a heartbeat simply hasn't run but the provisioner presents as stale. Am I missing something?

Copy link
MemberAuthor

@johnstcnjohnstcnNov 28, 2024•
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others.Learn more.

Yep, but we don't expose that as acoderd configuration knob yet. The only way you could configure that is by editing the source code right now.

)

type Options struct {
Expand Down
8 changes: 4 additions & 4 deletionscoderd/templateversions.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1515,7 +1515,7 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht

// Check for eligible provisioners. This allows us to log a message warning deployment administrators
// of users submitting jobs for which no provisioners are available.
matchedProvisioners, err = checkProvisioners(ctx, tx, organization.ID, tags, api.DeploymentValues.Provisioner.DaemonPollInterval.Value())
matchedProvisioners, err = checkProvisioners(ctx, tx, organization.ID, tags)
if err != nil {
api.Logger.Error(ctx, "failed to check eligible provisioner daemons for job", slog.Error(err))
} else if matchedProvisioners.Count == 0 {
Expand DownExpand Up@@ -1819,7 +1819,7 @@ func (api *API) publishTemplateUpdate(ctx context.Context, templateID uuid.UUID)
}
}

func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUID, wantTags map[string]string, pollInterval time.Duration) (codersdk.MatchedProvisioners, error) {
func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUID, wantTags map[string]string) (codersdk.MatchedProvisioners, error) {
// Check for eligible provisioners. This allows us to return a warning to the user if they
// submit a job for which no provisioner is available.
eligibleProvisioners, err := store.GetProvisionerDaemonsByOrganization(ctx, database.GetProvisionerDaemonsByOrganizationParams{
Expand All@@ -1831,15 +1831,15 @@ func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUI
return codersdk.MatchedProvisioners{}, xerrors.Errorf("provisioner daemons by organization: %w", err)
}

threePollsAgo := time.Now().Add(-3 * pollInterval)
staleInterval := time.Now().Add(-provisionerdserver.StaleInterval)
mostRecentlySeen := codersdk.NullTime{}
var matched codersdk.MatchedProvisioners
for _, provisioner := range eligibleProvisioners {
if !provisioner.LastSeenAt.Valid {
continue
}
matched.Count++
if provisioner.LastSeenAt.Time.After(threePollsAgo) {
if provisioner.LastSeenAt.Time.After(staleInterval) {
matched.Available++
}
if provisioner.LastSeenAt.Time.After(mostRecentlySeen.Time) {
Expand Down
Loading
[8]ページ先頭
©2009-2025 Movatter.jp