May 21, 2025 · May 6, 2025 · May 12, 2025 · May 12, 2025 · May 12, 2025 · May 12, 2025
diff --git a/cli/testdata/server-config.yaml.golden b/cli/testdata/server-config.yaml.golden
  # backoff.
  # (default: 1h0m0s, type: duration)
  reconciliation_backoff_lookback_period: 1h0m0s
  # Maximum number of consecutive failed prebuilds before a preset hits the hard
  # limit; disabled when set to zero.
  # (default: 3, type: int)
  failure_hard_limit: 3
diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go
diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json
diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go
 return q.db.GetPresetParametersByTemplateVersionID(ctx, args)
 }

 func (q *querier) GetPresetsAtFailureLimit(ctx context.Context, hardLimit int64) ([]database.GetPresetsAtFailureLimitRow, error) {
 // GetPresetsAtFailureLimit returns a list of template version presets that have reached the hard failure limit.
 // Request the same authorization permissions as GetPresetsBackoff, since the methods are similar.
 if err := q.authorizeContext(ctx, policy.ActionViewInsights, rbac.ResourceTemplate.All()); err != nil {
 return nil, err
 }
 return q.db.GetPresetsAtFailureLimit(ctx, hardLimit)
 }

 func (q *querier) GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]database.GetPresetsBackoffRow, error) {
 // GetPresetsBackoff returns a list of template version presets along with metadata such as the number of failed prebuilds.
 if err := q.authorizeContext(ctx, policy.ActionViewInsights, rbac.ResourceTemplate.All()); err != nil {
 return deleteQ(q.log, q.auth, q.db.GetOrganizationByID, deleteF)(ctx, arg.ID)
 }

 func (q *querier) UpdatePresetPrebuildStatus(ctx context.Context, arg database.UpdatePresetPrebuildStatusParams) error {
 preset, err := q.db.GetPresetByID(ctx, arg.PresetID)
 if err != nil {
 return err
 }

 object := rbac.ResourceTemplate.
 WithID(preset.TemplateID.UUID).
 InOrg(preset.OrganizationID)

 err = q.authorizeContext(ctx, policy.ActionUpdate, object)
 if err != nil {
 return err
 }

 return q.db.UpdatePresetPrebuildStatus(ctx, arg)
 }

 func (q *querier) UpdateProvisionerDaemonLastSeenAt(ctx context.Context, arg database.UpdateProvisionerDaemonLastSeenAtParams) error {
 if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceProvisionerDaemon); err != nil {
 return err
diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go
 Asserts(rbac.ResourceWorkspace.All(), policy.ActionRead).
 ErrorsWithInMemDB(dbmem.ErrUnimplemented)
 }))
 s.Run("GetPresetsAtFailureLimit", s.Subtest(func(_ database.Store, check *expects) {
 check.Args(int64(0)).
 Asserts(rbac.ResourceTemplate.All(), policy.ActionViewInsights).
 ErrorsWithInMemDB(dbmem.ErrUnimplemented)
 }))
 s.Run("GetPresetsBackoff", s.Subtest(func(_ database.Store, check *expects) {
 check.Args(time.Time{}).
 Asserts(rbac.ResourceTemplate.All(), policy.ActionViewInsights).
 },
 InvalidateAfterSecs: preset.InvalidateAfterSecs,
 OrganizationID:      org.ID,
 PrebuildStatus:      database.PrebuildStatusHealthy,
 })
 }))
 s.Run("UpdatePresetPrebuildStatus", s.Subtest(func(db database.Store, check *expects) {
 org := dbgen.Organization(s.T(), db, database.Organization{})
 user := dbgen.User(s.T(), db, database.User{})
 template := dbgen.Template(s.T(), db, database.Template{
 OrganizationID: org.ID,
 CreatedBy:      user.ID,
 })
 templateVersion := dbgen.TemplateVersion(s.T(), db, database.TemplateVersion{
 TemplateID: uuid.NullUUID{
 UUID:  template.ID,
 Valid: true,
 },
 OrganizationID: org.ID,
 CreatedBy:      user.ID,
 })
 preset := dbgen.Preset(s.T(), db, database.InsertPresetParams{
 TemplateVersionID: templateVersion.ID,
 })
 req := database.UpdatePresetPrebuildStatusParams{
 PresetID: preset.ID,
 Status:   database.PrebuildStatusHealthy,
 }
 check.Args(req).
 Asserts(rbac.ResourceTemplate.WithID(template.ID).InOrg(org.ID), policy.ActionUpdate)
 }))
 }

 func (s *MethodTestSuite) TestOAuth2ProviderApps() {
diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go
 CreatedAt:           preset.CreatedAt,
 DesiredInstances:    preset.DesiredInstances,
 InvalidateAfterSecs: preset.InvalidateAfterSecs,
 PrebuildStatus:      preset.PrebuildStatus,
 TemplateID:          tv.TemplateID,
 OrganizationID:      tv.OrganizationID,
 }, nil
 return parameters, nil
 }

 func (q *FakeQuerier) GetPresetsAtFailureLimit(ctx context.Context, hardLimit int64) ([]database.GetPresetsAtFailureLimitRow, error) {
 return nil, ErrUnimplemented
 }

 func (*FakeQuerier) GetPresetsBackoff(_ context.Context, _ time.Time) ([]database.GetPresetsBackoffRow, error) {
 return nil, ErrUnimplemented
 }
 Int32: 0,
 Valid: true,
 },
 PrebuildStatus: database.PrebuildStatusHealthy,
 }
 q.presets = append(q.presets, preset)
 return preset, nil
 return sql.ErrNoRows
 }

 func (q *FakeQuerier) UpdatePresetPrebuildStatus(ctx context.Context, arg database.UpdatePresetPrebuildStatusParams) error {
 err := validateDatabaseType(arg)
 if err != nil {
 return err
 }

 q.mutex.RLock()
 defer q.mutex.RUnlock()

 for _, preset := range q.presets {
 if preset.ID == arg.PresetID {
 preset.PrebuildStatus = arg.Status
 return nil
 }
 }

 return xerrors.Errorf("preset %v does not exist", arg.PresetID)
 }

 func (q *FakeQuerier) UpdateProvisionerDaemonLastSeenAt(_ context.Context, arg database.UpdateProvisionerDaemonLastSeenAtParams) error {
 err := validateDatabaseType(arg)
 if err != nil {
diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go
diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go
diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql
diff --git a/coderd/database/migrations/000328_prebuild_failure_limit_notification.down.sql b/coderd/database/migrations/000328_prebuild_failure_limit_notification.down.sql
 DELETE FROM notification_templates WHERE id = '414d9331-c1fc-4761-b40c-d1f4702279eb';
diff --git a/coderd/database/migrations/000328_prebuild_failure_limit_notification.up.sql b/coderd/database/migrations/000328_prebuild_failure_limit_notification.up.sql
 INSERT INTO notification_templates
 (id, name, title_template, body_template, "group", actions)
 VALUES ('414d9331-c1fc-4761-b40c-d1f4702279eb',
 'Prebuild Failure Limit Reached',
 E'There is a problem creating prebuilt workspaces',
 $$
 The number of failed prebuild attempts has reached the hard limit for template **{{ .Labels.template }}** and preset **{{ .Labels.preset }}**.

 To resume prebuilds, fix the underlying issue and upload a new template version.

 Refer to the documentation for more details:
 - [Troubleshooting templates](https://coder.com/docs/admin/templates/troubleshooting)
 - [Troubleshooting of prebuilt workspaces](https://coder.com/docs/admin/templates/extending-templates/prebuilt-workspaces#administration-and-troubleshooting)
 $$,
 'Template Events',
 '[
 {
 "label": "View failed prebuilt workspaces",
 "url": "{{base_url}}/workspaces?filter=owner:prebuilds+status:failed+template:{{.Labels.template}}"
 },
 {
 "label": "View template version",
 "url": "{{base_url}}/templates/{{.Labels.org}}/{{.Labels.template}}/versions/{{.Labels.template_version}}"
 }
 ]'::jsonb);
diff --git a/coderd/database/migrations/000329_add_status_to_template_presets.down.sql b/coderd/database/migrations/000329_add_status_to_template_presets.down.sql
 -- Remove the column from the table first (must happen before dropping the enum type)
 ALTER TABLE template_version_presets DROP COLUMN prebuild_status;

 -- Then drop the enum type
 DROP TYPE prebuild_status;
diff --git a/coderd/database/migrations/000329_add_status_to_template_presets.up.sql b/coderd/database/migrations/000329_add_status_to_template_presets.up.sql
 CREATE TYPE prebuild_status AS ENUM (
  'healthy',          -- Prebuilds are working as expected; this is the default, healthy state.
  'hard_limited',     -- Prebuilds have failed repeatedly and hit the configured hard failure limit; won't be retried anymore.
  'validation_failed' -- Prebuilds failed due to a non-retryable validation error (e.g. template misconfiguration); won't be retried.
 );

 ALTER TABLE template_version_presets ADD COLUMN prebuild_status prebuild_status NOT NULL DEFAULT 'healthy'::prebuild_status;
diff --git a/coderd/database/models.go b/coderd/database/models.go
Original file line number	Diff line number	Diff line change
Expand Up		@@ -704,3 +704,7 @@ workspace_prebuilds:
		# backoff.
		# (default: 1h0m0s, type: duration)
		reconciliation_backoff_lookback_period: 1h0m0s
		# Maximum number of consecutive failed prebuilds before a preset hits the hard
		# limit; disabled when set to zero.
		# (default: 3, type: int)
		failure_hard_limit: 3
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2226,6 +2226,15 @@ func (q *querier) GetPresetParametersByTemplateVersionID(ctx context.Context, ar
		return q.db.GetPresetParametersByTemplateVersionID(ctx, args)
		}

		func (q *querier) GetPresetsAtFailureLimit(ctx context.Context, hardLimit int64) ([]database.GetPresetsAtFailureLimitRow, error) {
		// GetPresetsAtFailureLimit returns a list of template version presets that have reached the hard failure limit.
		// Request the same authorization permissions as GetPresetsBackoff, since the methods are similar.
		if err := q.authorizeContext(ctx, policy.ActionViewInsights, rbac.ResourceTemplate.All()); err != nil {
Copy link Contributor dannykoppingMay 12, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. I still think "insights" is a weird auth context to use here. Why did we choose this again@SasSwart? Copy link ContributorAuthor evgeniy-scherbinaMay 12, 2025• edited Loading Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. @dannykopping Basically we have 3 actions: `Read` Template.All() `Use` Template.All() `ViewInsights` Template.All() `Read` is for reading the content of template `Use` is for creating workspaces `ViewInsights` is for gathering stats & metrics across templates Looks like`GetPresetsBackoff` falls into`ViewInsights` category - because we gathering statistics across all template presets, calculating number of failed builds, etc... I had a discussion with Steven regarding this, but I'm open to suggestions. Copy link Contributor dannykoppingMay 12, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. Repurposing other RBAC roles just because they overlap logically feels like a bad design choice. Each feature should have its own RBAC policy. Copy link ContributorAuthor evgeniy-scherbinaMay 12, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. We can introduce`new resource type` here:https://github.com/coder/coder/blob/main/coderd/rbac/object_gen.go Then use it like this: iferr:=q.authorizeContext(ctx,policy.ActionViewInsights,rbac.ResourcePrebuilds.All());err!=nil {} Optionally introduce new`action type` as well, and use it like this: iferr:=q.authorizeContext(ctx,policy.ActionCollectStatistics,rbac.ResourcePrebuilds.All());err!=nil {} But I think we don't want to introduce too many action types. cc@Emyrk Copy link Contributor dannykoppingMay 12, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. We can do this in a follow-up, by the way 👍 evgeniy-scherbina reacted with thumbs up emoji Copy link Member EmyrkMay 12, 2025 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. I don't have all the context for prebuilds. Can we throw a quick disc chat on the calendar?
		return nil, err
		}
		return q.db.GetPresetsAtFailureLimit(ctx, hardLimit)
		}

		func (q *querier) GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]database.GetPresetsBackoffRow, error) {
		// GetPresetsBackoff returns a list of template version presets along with metadata such as the number of failed prebuilds.
		if err := q.authorizeContext(ctx, policy.ActionViewInsights, rbac.ResourceTemplate.All()); err != nil {
Expand DownExpand Up		@@ -4201,6 +4210,24 @@ func (q *querier) UpdateOrganizationDeletedByID(ctx context.Context, arg databas
		return deleteQ(q.log, q.auth, q.db.GetOrganizationByID, deleteF)(ctx, arg.ID)
		}

		func (q *querier) UpdatePresetPrebuildStatus(ctx context.Context, arg database.UpdatePresetPrebuildStatusParams) error {
		preset, err := q.db.GetPresetByID(ctx, arg.PresetID)
		if err != nil {
		return err
		}

		object := rbac.ResourceTemplate.
		WithID(preset.TemplateID.UUID).
		InOrg(preset.OrganizationID)

		err = q.authorizeContext(ctx, policy.ActionUpdate, object)
		if err != nil {
		return err
		}

		return q.db.UpdatePresetPrebuildStatus(ctx, arg)
		}

		func (q *querier) UpdateProvisionerDaemonLastSeenAt(ctx context.Context, arg database.UpdateProvisionerDaemonLastSeenAtParams) error {
		if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceProvisionerDaemon); err != nil {
		return err
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -4924,6 +4924,11 @@ func (s *MethodTestSuite) TestPrebuilds() {
		Asserts(rbac.ResourceWorkspace.All(), policy.ActionRead).
		ErrorsWithInMemDB(dbmem.ErrUnimplemented)
		}))
		s.Run("GetPresetsAtFailureLimit", s.Subtest(func(_ database.Store, check *expects) {
		check.Args(int64(0)).
		Asserts(rbac.ResourceTemplate.All(), policy.ActionViewInsights).
		ErrorsWithInMemDB(dbmem.ErrUnimplemented)
		}))
		s.Run("GetPresetsBackoff", s.Subtest(func(_ database.Store, check *expects) {
		check.Args(time.Time{}).
		Asserts(rbac.ResourceTemplate.All(), policy.ActionViewInsights).
Expand DownExpand Up		@@ -4971,8 +4976,34 @@ func (s *MethodTestSuite) TestPrebuilds() {
		},
		InvalidateAfterSecs: preset.InvalidateAfterSecs,
		OrganizationID: org.ID,
		PrebuildStatus: database.PrebuildStatusHealthy,
		})
		}))
		s.Run("UpdatePresetPrebuildStatus", s.Subtest(func(db database.Store, check *expects) {
		org := dbgen.Organization(s.T(), db, database.Organization{})
		user := dbgen.User(s.T(), db, database.User{})
		template := dbgen.Template(s.T(), db, database.Template{
		OrganizationID: org.ID,
		CreatedBy: user.ID,
		})
		templateVersion := dbgen.TemplateVersion(s.T(), db, database.TemplateVersion{
		TemplateID: uuid.NullUUID{
		UUID: template.ID,
		Valid: true,
		},
		OrganizationID: org.ID,
		CreatedBy: user.ID,
		})
		preset := dbgen.Preset(s.T(), db, database.InsertPresetParams{
		TemplateVersionID: templateVersion.ID,
		})
		req := database.UpdatePresetPrebuildStatusParams{
		PresetID: preset.ID,
		Status: database.PrebuildStatusHealthy,
		}
		check.Args(req).
		Asserts(rbac.ResourceTemplate.WithID(template.ID).InOrg(org.ID), policy.ActionUpdate)
		}))
		}

		func (s *MethodTestSuite) TestOAuth2ProviderApps() {
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -4287,6 +4287,7 @@ func (q *FakeQuerier) GetPresetByID(ctx context.Context, presetID uuid.UUID) (da
		CreatedAt: preset.CreatedAt,
		DesiredInstances: preset.DesiredInstances,
		InvalidateAfterSecs: preset.InvalidateAfterSecs,
		PrebuildStatus: preset.PrebuildStatus,
		TemplateID: tv.TemplateID,
		OrganizationID: tv.OrganizationID,
		}, nil
Expand DownExpand Up		@@ -4352,6 +4353,10 @@ func (q *FakeQuerier) GetPresetParametersByTemplateVersionID(_ context.Context,
		return parameters, nil
		}

		func (q *FakeQuerier) GetPresetsAtFailureLimit(ctx context.Context, hardLimit int64) ([]database.GetPresetsAtFailureLimitRow, error) {
		return nil, ErrUnimplemented
		}

		func (*FakeQuerier) GetPresetsBackoff(_ context.Context, _ time.Time) ([]database.GetPresetsBackoffRow, error) {
		return nil, ErrUnimplemented
		}
Expand DownExpand Up		@@ -9089,6 +9094,7 @@ func (q *FakeQuerier) InsertPreset(_ context.Context, arg database.InsertPresetP
		Int32: 0,
		Valid: true,
		},
		PrebuildStatus: database.PrebuildStatusHealthy,
		}
		q.presets = append(q.presets, preset)
		return preset, nil
Expand DownExpand Up		@@ -10917,6 +10923,25 @@ func (q *FakeQuerier) UpdateOrganizationDeletedByID(_ context.Context, arg datab
		return sql.ErrNoRows
		}

		func (q *FakeQuerier) UpdatePresetPrebuildStatus(ctx context.Context, arg database.UpdatePresetPrebuildStatusParams) error {
		err := validateDatabaseType(arg)
		if err != nil {
		return err
		}

		q.mutex.RLock()
		defer q.mutex.RUnlock()

		for _, preset := range q.presets {
		if preset.ID == arg.PresetID {
		preset.PrebuildStatus = arg.Status
		return nil
		}
		}

		return xerrors.Errorf("preset %v does not exist", arg.PresetID)
		}

		func (q *FakeQuerier) UpdateProvisionerDaemonLastSeenAt(_ context.Context, arg database.UpdateProvisionerDaemonLastSeenAtParams) error {
		err := validateDatabaseType(arg)
		if err != nil {
Expand Down
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		DELETE FROM notification_templates WHERE id = '414d9331-c1fc-4761-b40c-d1f4702279eb';
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,25 @@
		INSERT INTO notification_templates
		(id, name, title_template, body_template, "group", actions)
		VALUES ('414d9331-c1fc-4761-b40c-d1f4702279eb',
		'Prebuild Failure Limit Reached',
		E'There is a problem creating prebuilt workspaces',
		$$
		The number of failed prebuild attempts has reached the hard limit for template {{ .Labels.template }} and preset {{ .Labels.preset }}.

		To resume prebuilds, fix the underlying issue and upload a new template version.
dannykopping marked this conversation as resolved. Show resolvedHide resolved

		Refer to the documentation for more details:
		- [Troubleshooting templates](https://coder.com/docs/admin/templates/troubleshooting)
		- [Troubleshooting of prebuilt workspaces](https://coder.com/docs/admin/templates/extending-templates/prebuilt-workspaces#administration-and-troubleshooting)
		$$,
		'Template Events',
		'[
		{
		"label": "View failed prebuilt workspaces",
		"url": "{{base_url}}/workspaces?filter=owner:prebuilds+status:failed+template:{{.Labels.template}}"
		},
		{
		"label": "View template version",
		"url": "{{base_url}}/templates/{{.Labels.org}}/{{.Labels.template}}/versions/{{.Labels.template_version}}"
		}
		]'::jsonb);
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,5 @@
		-- Remove the column from the table first (must happen before dropping the enum type)
		ALTER TABLE template_version_presets DROP COLUMN prebuild_status;

		-- Then drop the enum type
		DROP TYPE prebuild_status;
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,7 @@
		CREATE TYPE prebuild_status AS ENUM (
		'healthy', -- Prebuilds are working as expected; this is the default, healthy state.
		'hard_limited', -- Prebuilds have failed repeatedly and hit the configured hard failure limit; won't be retried anymore.
dannykopping marked this conversation as resolved. Show resolvedHide resolved
		'validation_failed' -- Prebuilds failed due to a non-retryable validation error (e.g. template misconfiguration); won't be retried.
		);

		ALTER TABLE template_version_presets ADD COLUMN prebuild_status prebuild_status NOT NULL DEFAULT 'healthy'::prebuild_status;