Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

fix(coderd/prometheusmetrics)!: filter deleted wsbuilds to reduce db load#19197

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.

Already on GitHub?Sign in to your account

Merged
mafredri merged 3 commits intomainfrommafredri/fix-prommetrics-db-load
Aug 11, 2025
Merged
Show file tree
Hide file tree
Changes fromall commits
Commits
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 0 additions & 11 deletionscoderd/database/dbauthz/dbauthz.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2178,17 +2178,6 @@ func (q *querier) GetLatestWorkspaceBuildByWorkspaceID(ctx context.Context, work
return q.db.GetLatestWorkspaceBuildByWorkspaceID(ctx, workspaceID)
}

func (q *querier) GetLatestWorkspaceBuilds(ctx context.Context) ([]database.WorkspaceBuild, error) {
// This function is a system function until we implement a join for workspace builds.
// This is because we need to query for all related workspaces to the returned builds.
// This is a very inefficient method of fetching the latest workspace builds.
// We should just join the rbac properties.
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
return nil, err
}
return q.db.GetLatestWorkspaceBuilds(ctx)
}

func (q *querier) GetLatestWorkspaceBuildsByWorkspaceIDs(ctx context.Context, ids []uuid.UUID) ([]database.WorkspaceBuild, error) {
// This function is a system function until we implement a join for workspace builds.
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
Expand Down
6 changes: 0 additions & 6 deletionscoderd/database/dbauthz/dbauthz_test.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -4033,12 +4033,6 @@ func (s *MethodTestSuite) TestSystemFunctions() {
LoginType: l.LoginType,
}).Asserts(rbac.ResourceSystem, policy.ActionRead).Returns(l)
}))
s.Run("GetLatestWorkspaceBuilds", s.Subtest(func(db database.Store, check *expects) {
dbtestutil.DisableForeignKeysAndTriggers(s.T(), db)
dbgen.WorkspaceBuild(s.T(), db, database.WorkspaceBuild{})
dbgen.WorkspaceBuild(s.T(), db, database.WorkspaceBuild{})
check.Args().Asserts(rbac.ResourceSystem, policy.ActionRead)
}))
s.Run("GetActiveUserCount", s.Subtest(func(db database.Store, check *expects) {
check.Args(false).Asserts(rbac.ResourceSystem, policy.ActionRead).Returns(int64(0))
}))
Expand Down
7 changes: 0 additions & 7 deletionscoderd/database/dbmetrics/querymetrics.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more abouthow customized files appear on GitHub.

15 changes: 0 additions & 15 deletionscoderd/database/dbmock/dbmock.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more abouthow customized files appear on GitHub.

1 change: 0 additions & 1 deletioncoderd/database/querier.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more abouthow customized files appear on GitHub.

59 changes: 0 additions & 59 deletionscoderd/database/queries.sql.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more abouthow customized files appear on GitHub.

14 changes: 0 additions & 14 deletionscoderd/database/queries/workspacebuilds.sql
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -91,20 +91,6 @@ JOIN
workspace_build_with_user AS wb
ON m.workspace_id = wb.workspace_id AND m.max_build_number = wb.build_number;

-- name: GetLatestWorkspaceBuilds :many
SELECT wb.*
FROM (
SELECT
workspace_id, MAX(build_number) as max_build_number
FROM
workspace_build_with_user AS workspace_builds
GROUP BY
workspace_id
) m
JOIN
workspace_build_with_user AS wb
ON m.workspace_id = wb.workspace_id AND m.max_build_number = wb.build_number;

-- name: InsertWorkspaceBuild :exec
INSERT INTO
workspace_builds (
Expand Down
65 changes: 21 additions & 44 deletionscoderd/prometheusmetrics/prometheusmetrics.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -150,7 +150,7 @@ func Workspaces(ctx context.Context, logger slog.Logger, registerer prometheus.R
Namespace: "coderd",
Subsystem: "api",
Name: "workspace_latest_build",
Help: "The current number of workspace builds by status.",
Help: "The current number of workspace builds by status for all non-deleted workspaces.",
}, []string{"status"})
if err := registerer.Register(workspaceLatestBuildTotals); err != nil {
return nil, err
Expand All@@ -159,7 +159,7 @@ func Workspaces(ctx context.Context, logger slog.Logger, registerer prometheus.R
workspaceLatestBuildStatuses := prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "coderd",
Name: "workspace_latest_build_status",
Help: "The current workspace statuses by template, transition, and owner.",
Help: "The current workspace statuses by template, transition, and owner for all non-deleted workspaces.",
}, []string{"status", "template_name", "template_version", "workspace_owner", "workspace_transition"})
if err := registerer.Register(workspaceLatestBuildStatuses); err != nil {
return nil, err
Expand All@@ -168,59 +168,37 @@ func Workspaces(ctx context.Context, logger slog.Logger, registerer prometheus.R
ctx, cancelFunc := context.WithCancel(ctx)
done := make(chan struct{})

updateWorkspaceTotals := func() {
builds, err := db.GetLatestWorkspaceBuilds(ctx)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
// clear all series if there are no database entries
workspaceLatestBuildTotals.Reset()
} else {
logger.Warn(ctx, "failed to load latest workspace builds", slog.Error(err))
}
return
}
jobIDs := make([]uuid.UUID, 0, len(builds))
for _, build := range builds {
jobIDs = append(jobIDs, build.JobID)
}
jobs, err := db.GetProvisionerJobsByIDs(ctx, jobIDs)
if err != nil {
ids := make([]string, 0, len(jobIDs))
for _, id := range jobIDs {
ids = append(ids, id.String())
}

logger.Warn(ctx, "failed to load provisioner jobs", slog.F("ids", ids), slog.Error(err))
return
}

workspaceLatestBuildTotals.Reset()
for _, job := range jobs {
status := codersdk.ProvisionerJobStatus(job.JobStatus)
workspaceLatestBuildTotals.WithLabelValues(string(status)).Add(1)
// TODO: deprecated: remove in the future
workspaceLatestBuildTotalsDeprecated.WithLabelValues(string(status)).Add(1)
}
}

updateWorkspaceStatuses := func() {
updateWorkspaceMetrics := func() {
ws, err := db.GetWorkspaces(ctx, database.GetWorkspacesParams{
Deleted: false,
WithSummary: false,
})
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
// clear all series if there are no database entries
workspaceLatestBuildTotals.Reset()
workspaceLatestBuildStatuses.Reset()
} else {
logger.Warn(ctx, "failed to load active workspaces for metrics", slog.Error(err))
}

logger.Warn(ctx, "failed to load active workspaces", slog.Error(err))
return
}

workspaceLatestBuildTotals.Reset()
workspaceLatestBuildStatuses.Reset()

for _, w := range ws {
workspaceLatestBuildStatuses.WithLabelValues(string(w.LatestBuildStatus), w.TemplateName, w.TemplateVersionName.String, w.OwnerUsername, string(w.LatestBuildTransition)).Add(1)
status := string(w.LatestBuildStatus)
workspaceLatestBuildTotals.WithLabelValues(status).Add(1)
// TODO: deprecated: remove in the future
workspaceLatestBuildTotalsDeprecated.WithLabelValues(status).Add(1)

workspaceLatestBuildStatuses.WithLabelValues(
status,
w.TemplateName,
w.TemplateVersionName.String,
w.OwnerUsername,
string(w.LatestBuildTransition),
).Add(1)
}
}

Expand All@@ -230,8 +208,7 @@ func Workspaces(ctx context.Context, logger slog.Logger, registerer prometheus.R
doTick := func() {
defer ticker.Reset(duration)

updateWorkspaceTotals()
updateWorkspaceStatuses()
updateWorkspaceMetrics()
}

go func() {
Expand Down
74 changes: 74 additions & 0 deletionscoderd/prometheusmetrics/prometheusmetrics_test.go
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -247,6 +247,32 @@ func TestWorkspaceLatestBuildTotals(t *testing.T) {
codersdk.ProvisionerJobSucceeded: 3,
codersdk.ProvisionerJobRunning: 1,
},
}, {
Name: "MultipleWithDeleted",
Database: func() database.Store {
db, _ := dbtestutil.NewDB(t)
u := dbgen.User(t, db, database.User{})
org := dbgen.Organization(t, db, database.Organization{})
insertCanceled(t, db, u, org)
insertFailed(t, db, u, org)
insertSuccess(t, db, u, org)
insertRunning(t, db, u, org)

// Verify that deleted workspaces/builds are NOT counted in metrics.
n, err := cryptorand.Intn(5)
require.NoError(t, err)
for range 1 + n {
insertDeleted(t, db, u, org)
}
return db
},
Total: 4, // Only non-deleted workspaces should be counted
Status: map[codersdk.ProvisionerJobStatus]int{
codersdk.ProvisionerJobCanceled: 1,
codersdk.ProvisionerJobFailed: 1,
codersdk.ProvisionerJobSucceeded: 1,
codersdk.ProvisionerJobRunning: 1,
},
}} {
t.Run(tc.Name, func(t *testing.T) {
t.Parallel()
Expand DownExpand Up@@ -323,6 +349,33 @@ func TestWorkspaceLatestBuildStatuses(t *testing.T) {
codersdk.ProvisionerJobSucceeded: 3,
codersdk.ProvisionerJobRunning: 1,
},
}, {
Name: "MultipleWithDeleted",
Database: func() database.Store {
db, _ := dbtestutil.NewDB(t)
u := dbgen.User(t, db, database.User{})
org := dbgen.Organization(t, db, database.Organization{})
insertTemplates(t, db, u, org)
insertCanceled(t, db, u, org)
insertFailed(t, db, u, org)
insertSuccess(t, db, u, org)
insertRunning(t, db, u, org)

// Verify that deleted workspaces/builds are NOT counted in metrics.
n, err := cryptorand.Intn(5)
require.NoError(t, err)
for range 1 + n {
insertDeleted(t, db, u, org)
}
Copy link
MemberAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others.Learn more.

Review: This fails in the old implementation, but I thought it best to formalize the assumption. Considering the old implementation did not test for this, it seems like validation for the new approach.

johnstcn reacted with thumbs up emoji
return db
},
ExpectedWorkspaces: 4, // Only non-deleted workspaces should be counted
ExpectedStatuses: map[codersdk.ProvisionerJobStatus]int{
codersdk.ProvisionerJobCanceled: 1,
codersdk.ProvisionerJobFailed: 1,
codersdk.ProvisionerJobSucceeded: 1,
codersdk.ProvisionerJobRunning: 1,
},
}} {
t.Run(tc.Name, func(t *testing.T) {
t.Parallel()
Expand DownExpand Up@@ -907,3 +960,24 @@ func insertSuccess(t *testing.T, db database.Store, u database.User, org databas
})
require.NoError(t, err)
}

func insertDeleted(t *testing.T, db database.Store, u database.User, org database.Organization) {
job := insertRunning(t, db, u, org)
err := db.UpdateProvisionerJobWithCompleteByID(context.Background(), database.UpdateProvisionerJobWithCompleteByIDParams{
ID: job.ID,
CompletedAt: sql.NullTime{
Time: dbtime.Now(),
Valid: true,
},
})
require.NoError(t, err)

build, err := db.GetWorkspaceBuildByJobID(context.Background(), job.ID)
require.NoError(t, err)

err = db.UpdateWorkspaceDeletedByID(context.Background(), database.UpdateWorkspaceDeletedByIDParams{
ID: build.WorkspaceID,
Deleted: true,
})
require.NoError(t, err)
}
Loading
[8]ページ先頭
©2009-2025 Movatter.jp