Jun 24, 2025 · Jun 24, 2025
diff --git a/enterprise/coderd/prebuilds/reconcile.go b/enterprise/coderd/prebuilds/reconcile.go

 // If the preset reached the hard failure limit for the first time during this iteration:
 // - Mark it as hard-limited in the database
 // - Send notifications to template admins
 // - Continue execution, we disallow only creation operation for hard-limited presets. Deletion is allowed.
 if ps.Preset.PrebuildStatus != database.PrebuildStatusHardLimited && ps.IsHardLimited {
 logger.Warn(ctx, "preset is hard limited, notifying template admins")
 if err != nil {
 return xerrors.Errorf("failed to update preset prebuild status: %w", err)
 }

 err = c.notifyPrebuildFailureLimitReached(ctx, ps)
 if err != nil {
 logger.Error(ctx, "failed to notify that number of prebuild failures reached the limit", slog.Error(err))
 }
 }

 state := ps.CalculateState()
 return multiErr.ErrorOrNil()
 }

 func (c *StoreReconciler) notifyPrebuildFailureLimitReached(ctx context.Context, ps prebuilds.PresetSnapshot) error {
 // nolint:gocritic // Necessary to query all the required data.
 ctx = dbauthz.AsSystemRestricted(ctx)

 // Send notification to template admins.
 if c.notifEnq == nil {
 c.logger.Warn(ctx, "notification enqueuer not set, cannot send prebuild is hard limited notification(s)")
 return nil
 }

 templateAdmins, err := c.store.GetUsers(ctx, database.GetUsersParams{
 RbacRole: []string{codersdk.RoleTemplateAdmin},
 })
 if err != nil {
 return xerrors.Errorf("fetch template admins: %w", err)
 }

 for _, templateAdmin := range templateAdmins {
 if _, err := c.notifEnq.EnqueueWithData(ctx, templateAdmin.ID, notifications.PrebuildFailureLimitReached,
 map[string]string{
 "org":              ps.Preset.OrganizationName,
 "template":         ps.Preset.TemplateName,
 "template_version": ps.Preset.TemplateVersionName,
 "preset":           ps.Preset.Name,
 },
 map[string]any{},
 "prebuilds_reconciler",
 // Associate this notification with all the related entities.
 ps.Preset.TemplateID, ps.Preset.TemplateVersionID, ps.Preset.ID, ps.Preset.OrganizationID,
 ); err != nil {
 c.logger.Error(ctx,
 "failed to send notification",
 slog.Error(err),
 slog.F("template_admin_id", templateAdmin.ID.String()),
 )

 continue
 }
 }

 return nil
 }

 func (c *StoreReconciler) CalculateActions(ctx context.Context, snapshot prebuilds.PresetSnapshot) ([]*prebuilds.ReconciliationActions, error) {
 if ctx.Err() != nil {
 return nil, ctx.Err()
diff --git a/enterprise/coderd/prebuilds/reconcile_test.go b/enterprise/coderd/prebuilds/reconcile_test.go
 cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
 controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, clock, registry, fakeEnqueuer)

 // Template admin to receive a notification.
 templateAdmin := dbgen.User(t, db, database.User{
 RBACRoles: []string{codersdk.RoleTemplateAdmin},
 })

 // Set up test environment with a template, version, and preset.
 ownerID := uuid.New()
 dbgen.User(t, db, database.User{
 require.Equal(t, 1, len(workspaces))
 require.Equal(t, database.PrebuildStatusHardLimited, updatedPreset.PrebuildStatus)

 // When hard limit is reached, a notification should be sent.
 matching := fakeEnqueuer.Sent(func(notification *notificationstest.FakeNotification) bool {
 if !assert.Equal(t, notifications.PrebuildFailureLimitReached, notification.TemplateID, "unexpected template") {
 return false
 }

 if !assert.Equal(t, templateAdmin.ID, notification.UserID, "unexpected receiver") {
 return false
 }

 return true
 })
 require.Len(t, matching, 1)

 // When hard limit is reached, metric is set to 1.
 mf, err = registry.Gather()
 require.NoError(t, err)
 cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
 controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, clock, registry, fakeEnqueuer)

 // Template admin to receive a notification.
 templateAdmin := dbgen.User(t, db, database.User{
 RBACRoles: []string{codersdk.RoleTemplateAdmin},
 })

 // Set up test environment with a template, version, and preset.
 ownerID := uuid.New()
 dbgen.User(t, db, database.User{
 require.NoError(t, err)
 require.Equal(t, database.PrebuildStatusHardLimited, updatedPreset.PrebuildStatus)

 // When hard limit is reached, a notification should be sent.
 matching := fakeEnqueuer.Sent(func(notification *notificationstest.FakeNotification) bool {
 if !assert.Equal(t, notifications.PrebuildFailureLimitReached, notification.TemplateID, "unexpected template") {
 return false
 }

 if !assert.Equal(t, templateAdmin.ID, notification.UserID, "unexpected receiver") {
 return false
 }

 return true
 })
 require.Len(t, matching, 1)

 // When hard limit is reached, metric is set to 1.
 mf, err = registry.Gather()
 require.NoError(t, err)
Original file line number	Diff line number	Diff line change
Expand Up		@@ -427,7 +427,6 @@ func (c *StoreReconciler) ReconcilePreset(ctx context.Context, ps prebuilds.Pres

		// If the preset reached the hard failure limit for the first time during this iteration:
		// - Mark it as hard-limited in the database
		// - Send notifications to template admins
		// - Continue execution, we disallow only creation operation for hard-limited presets. Deletion is allowed.
		if ps.Preset.PrebuildStatus != database.PrebuildStatusHardLimited && ps.IsHardLimited {
		logger.Warn(ctx, "preset is hard limited, notifying template admins")
Expand All		@@ -439,11 +438,6 @@ func (c *StoreReconciler) ReconcilePreset(ctx context.Context, ps prebuilds.Pres
		if err != nil {
		return xerrors.Errorf("failed to update preset prebuild status: %w", err)
		}

		err = c.notifyPrebuildFailureLimitReached(ctx, ps)
		if err != nil {
		logger.Error(ctx, "failed to notify that number of prebuild failures reached the limit", slog.Error(err))
		}
		}

		state := ps.CalculateState()
Expand DownExpand Up		@@ -474,49 +468,6 @@ func (c *StoreReconciler) ReconcilePreset(ctx context.Context, ps prebuilds.Pres
		return multiErr.ErrorOrNil()
		}

		func (c *StoreReconciler) notifyPrebuildFailureLimitReached(ctx context.Context, ps prebuilds.PresetSnapshot) error {
		// nolint:gocritic // Necessary to query all the required data.
		ctx = dbauthz.AsSystemRestricted(ctx)

		// Send notification to template admins.
		if c.notifEnq == nil {
		c.logger.Warn(ctx, "notification enqueuer not set, cannot send prebuild is hard limited notification(s)")
		return nil
		}

		templateAdmins, err := c.store.GetUsers(ctx, database.GetUsersParams{
		RbacRole: []string{codersdk.RoleTemplateAdmin},
		})
		if err != nil {
		return xerrors.Errorf("fetch template admins: %w", err)
		}

		for _, templateAdmin := range templateAdmins {
		if _, err := c.notifEnq.EnqueueWithData(ctx, templateAdmin.ID, notifications.PrebuildFailureLimitReached,
		map[string]string{
		"org": ps.Preset.OrganizationName,
		"template": ps.Preset.TemplateName,
		"template_version": ps.Preset.TemplateVersionName,
		"preset": ps.Preset.Name,
		},
		map[string]any{},
		"prebuilds_reconciler",
		// Associate this notification with all the related entities.
		ps.Preset.TemplateID, ps.Preset.TemplateVersionID, ps.Preset.ID, ps.Preset.OrganizationID,
		); err != nil {
		c.logger.Error(ctx,
		"failed to send notification",
		slog.Error(err),
		slog.F("template_admin_id", templateAdmin.ID.String()),
		)

		continue
		}
		}

		return nil
		}

		func (c StoreReconciler) CalculateActions(ctx context.Context, snapshot prebuilds.PresetSnapshot) ([]prebuilds.ReconciliationActions, error) {
		if ctx.Err() != nil {
		return nil, ctx.Err()
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -853,11 +853,6 @@ func TestSkippingHardLimitedPresets(t *testing.T) {
		cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
		controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, clock, registry, fakeEnqueuer)

		// Template admin to receive a notification.
		templateAdmin := dbgen.User(t, db, database.User{
		RBACRoles: []string{codersdk.RoleTemplateAdmin},
		})

		// Set up test environment with a template, version, and preset.
		ownerID := uuid.New()
		dbgen.User(t, db, database.User{
Expand DownExpand Up		@@ -938,20 +933,6 @@ func TestSkippingHardLimitedPresets(t *testing.T) {
		require.Equal(t, 1, len(workspaces))
		require.Equal(t, database.PrebuildStatusHardLimited, updatedPreset.PrebuildStatus)

		// When hard limit is reached, a notification should be sent.
		matching := fakeEnqueuer.Sent(func(notification *notificationstest.FakeNotification) bool {
		if !assert.Equal(t, notifications.PrebuildFailureLimitReached, notification.TemplateID, "unexpected template") {
		return false
		}

		if !assert.Equal(t, templateAdmin.ID, notification.UserID, "unexpected receiver") {
		return false
		}

		return true
		})
		require.Len(t, matching, 1)

		// When hard limit is reached, metric is set to 1.
		mf, err = registry.Gather()
		require.NoError(t, err)
Expand DownExpand Up		@@ -1016,11 +997,6 @@ func TestHardLimitedPresetShouldNotBlockDeletion(t *testing.T) {
		cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
		controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, clock, registry, fakeEnqueuer)

		// Template admin to receive a notification.
		templateAdmin := dbgen.User(t, db, database.User{
		RBACRoles: []string{codersdk.RoleTemplateAdmin},
		})

		// Set up test environment with a template, version, and preset.
		ownerID := uuid.New()
		dbgen.User(t, db, database.User{
Expand DownExpand Up		@@ -1125,20 +1101,6 @@ func TestHardLimitedPresetShouldNotBlockDeletion(t *testing.T) {
		require.NoError(t, err)
		require.Equal(t, database.PrebuildStatusHardLimited, updatedPreset.PrebuildStatus)

		// When hard limit is reached, a notification should be sent.
		matching := fakeEnqueuer.Sent(func(notification *notificationstest.FakeNotification) bool {
		if !assert.Equal(t, notifications.PrebuildFailureLimitReached, notification.TemplateID, "unexpected template") {
		return false
		}

		if !assert.Equal(t, templateAdmin.ID, notification.UserID, "unexpected receiver") {
		return false
		}

		return true
		})
		require.Len(t, matching, 1)

		// When hard limit is reached, metric is set to 1.
		mf, err = registry.Gather()
		require.NoError(t, err)
Expand Down