@@ -815,6 +815,213 @@ func TestSkippingHardLimitedPresets(t *testing.T) {
815
815
}
816
816
}
817
817
818
+ func TestHardLimitedPresetShouldNotBlockDeletion (t * testing.T ) {
819
+ t .Parallel ()
820
+
821
+ if ! dbtestutil .WillUsePostgres () {
822
+ t .Skip ("This test requires postgres" )
823
+ }
824
+
825
+ // Test cases verify the behavior of prebuild creation depending on configured failure limits.
826
+ testCases := []struct {
827
+ name string
828
+ hardLimit int64
829
+ isHardLimitHit bool
830
+ }{
831
+ {
832
+ name :"hard limit is hit - skip creation of prebuilt workspace" ,
833
+ hardLimit :1 ,
834
+ isHardLimitHit :true ,
835
+ },
836
+ }
837
+
838
+ for _ ,tc := range testCases {
839
+ t .Run (tc .name ,func (t * testing.T ) {
840
+ t .Parallel ()
841
+
842
+ clock := quartz .NewMock (t )
843
+ ctx := testutil .Context (t ,testutil .WaitShort )
844
+ cfg := codersdk.PrebuildsConfig {
845
+ FailureHardLimit :serpent .Int64 (tc .hardLimit ),
846
+ ReconciliationBackoffInterval :0 ,
847
+ }
848
+ logger := slogtest .Make (
849
+ t ,& slogtest.Options {IgnoreErrors :true },
850
+ ).Leveled (slog .LevelDebug )
851
+ db ,pubSub := dbtestutil .NewDB (t )
852
+ fakeEnqueuer := newFakeEnqueuer ()
853
+ registry := prometheus .NewRegistry ()
854
+ controller := prebuilds .NewStoreReconciler (db ,pubSub ,cfg ,logger ,clock ,registry ,fakeEnqueuer )
855
+
856
+ // Template admin to receive a notification.
857
+ templateAdmin := dbgen .User (t ,db , database.User {
858
+ RBACRoles : []string {codersdk .RoleTemplateAdmin },
859
+ })
860
+
861
+ // Set up test environment with a template, version, and preset.
862
+ ownerID := uuid .New ()
863
+ dbgen .User (t ,db , database.User {
864
+ ID :ownerID ,
865
+ })
866
+ org ,template := setupTestDBTemplate (t ,db ,ownerID ,false )
867
+ templateVersionID := setupTestDBTemplateVersion (ctx ,t ,clock ,db ,pubSub ,org .ID ,ownerID ,template .ID )
868
+ preset := setupTestDBPreset (t ,db ,templateVersionID ,2 ,uuid .New ().String ())
869
+
870
+ // Create a successful prebuilt workspace.
871
+ successfulWorkspace ,_ := setupTestDBPrebuild (
872
+ t ,
873
+ clock ,
874
+ db ,
875
+ pubSub ,
876
+ database .WorkspaceTransitionStart ,
877
+ database .ProvisionerJobStatusSucceeded ,
878
+ org .ID ,
879
+ preset ,
880
+ template .ID ,
881
+ templateVersionID ,
882
+ )
883
+
884
+ // Make sure that prebuilt workspaces created in such order: [successful, failed].
885
+ clock .Advance (time .Second ).MustWait (ctx )
886
+
887
+ // Create a failed prebuilt workspace that counts toward the hard failure limit.
888
+ setupTestDBPrebuild (
889
+ t ,
890
+ clock ,
891
+ db ,
892
+ pubSub ,
893
+ database .WorkspaceTransitionStart ,
894
+ database .ProvisionerJobStatusFailed ,
895
+ org .ID ,
896
+ preset ,
897
+ template .ID ,
898
+ templateVersionID ,
899
+ )
900
+
901
+ getJobStatusMap := func (workspaces []database.WorkspaceTable )map [database.ProvisionerJobStatus ]int {
902
+ jobStatusMap := make (map [database.ProvisionerJobStatus ]int )
903
+ for _ ,workspace := range workspaces {
904
+ workspaceBuilds ,err := db .GetWorkspaceBuildsByWorkspaceID (ctx , database.GetWorkspaceBuildsByWorkspaceIDParams {
905
+ WorkspaceID :workspace .ID ,
906
+ })
907
+ require .NoError (t ,err )
908
+
909
+ for _ ,workspaceBuild := range workspaceBuilds {
910
+ job ,err := db .GetProvisionerJobByID (ctx ,workspaceBuild .JobID )
911
+ require .NoError (t ,err )
912
+ jobStatusMap [job .JobStatus ]++
913
+ }
914
+ }
915
+ return jobStatusMap
916
+ }
917
+
918
+ // Verify initial state: two workspaces exist, one successful, one failed.
919
+ workspaces ,err := db .GetWorkspacesByTemplateID (ctx ,template .ID )
920
+ require .NoError (t ,err )
921
+ require .Equal (t ,2 ,len (workspaces ))
922
+ jobStatusMap := getJobStatusMap (workspaces )
923
+ require .Len (t ,jobStatusMap ,2 )
924
+ require .Equal (t ,1 ,jobStatusMap [database .ProvisionerJobStatusSucceeded ])
925
+ require .Equal (t ,1 ,jobStatusMap [database .ProvisionerJobStatusFailed ])
926
+
927
+ //Verify initial state: metric is not set - meaning preset is not hard limited.
928
+ require .NoError (t ,controller .ForceMetricsUpdate (ctx ))
929
+ mf ,err := registry .Gather ()
930
+ require .NoError (t ,err )
931
+ metric := findMetric (mf ,prebuilds .MetricPresetHardLimitedGauge ,map [string ]string {
932
+ "template_name" :template .Name ,
933
+ "preset_name" :preset .Name ,
934
+ "org_name" :org .Name ,
935
+ })
936
+ require .Nil (t ,metric )
937
+
938
+ // We simulate a failed prebuild in the test; Consequently, the backoff mechanism is triggered when ReconcileAll is called.
939
+ // Even though ReconciliationBackoffInterval is set to zero, we still need to advance the clock by at least one nanosecond.
940
+ clock .Advance (time .Nanosecond ).MustWait (ctx )
941
+
942
+ // Trigger reconciliation to attempt creating a new prebuild.
943
+ // The outcome depends on whether the hard limit has been reached.
944
+ require .NoError (t ,controller .ReconcileAll (ctx ))
945
+
946
+ // These two additional calls to ReconcileAll should not trigger any notifications.
947
+ // A notification is only sent once.
948
+ require .NoError (t ,controller .ReconcileAll (ctx ))
949
+ require .NoError (t ,controller .ReconcileAll (ctx ))
950
+
951
+ // Verify the final state after reconciliation.
952
+ // When hard limit is reached, no new workspace should be created.
953
+ workspaces ,err = db .GetWorkspacesByTemplateID (ctx ,template .ID )
954
+ require .NoError (t ,err )
955
+ require .Equal (t ,2 ,len (workspaces ))
956
+ jobStatusMap = getJobStatusMap (workspaces )
957
+ require .Len (t ,jobStatusMap ,2 )
958
+ require .Equal (t ,1 ,jobStatusMap [database .ProvisionerJobStatusSucceeded ])
959
+ require .Equal (t ,1 ,jobStatusMap [database .ProvisionerJobStatusFailed ])
960
+
961
+ updatedPreset ,err := db .GetPresetByID (ctx ,preset .ID )
962
+ require .NoError (t ,err )
963
+ require .Equal (t ,database .PrebuildStatusHardLimited ,updatedPreset .PrebuildStatus )
964
+
965
+ // When hard limit is reached, a notification should be sent.
966
+ matching := fakeEnqueuer .Sent (func (notification * notificationstest.FakeNotification )bool {
967
+ if ! assert .Equal (t ,notifications .PrebuildFailureLimitReached ,notification .TemplateID ,"unexpected template" ) {
968
+ return false
969
+ }
970
+
971
+ if ! assert .Equal (t ,templateAdmin .ID ,notification .UserID ,"unexpected receiver" ) {
972
+ return false
973
+ }
974
+
975
+ return true
976
+ })
977
+ require .Len (t ,matching ,1 )
978
+
979
+ // When hard limit is reached, metric is set to 1.
980
+ mf ,err = registry .Gather ()
981
+ require .NoError (t ,err )
982
+ metric = findMetric (mf ,prebuilds .MetricPresetHardLimitedGauge ,map [string ]string {
983
+ "template_name" :template .Name ,
984
+ "preset_name" :preset .Name ,
985
+ "org_name" :org .Name ,
986
+ })
987
+ require .NotNil (t ,metric )
988
+ require .NotNil (t ,metric .GetGauge ())
989
+ require .EqualValues (t ,1 ,metric .GetGauge ().GetValue ())
990
+
991
+ // When: the template is deleted.
992
+ require .NoError (t ,db .UpdateTemplateDeletedByID (ctx , database.UpdateTemplateDeletedByIDParams {
993
+ ID :template .ID ,
994
+ Deleted :true ,
995
+ UpdatedAt :dbtime .Now (),
996
+ }))
997
+
998
+ // Trigger reconciliation to make sure that successful, but outdated prebuilt workspace will be deleted.
999
+ require .NoError (t ,controller .ReconcileAll (ctx ))
1000
+
1001
+ workspaces ,err = db .GetWorkspacesByTemplateID (ctx ,template .ID )
1002
+ require .NoError (t ,err )
1003
+ require .Equal (t ,2 ,len (workspaces ))
1004
+
1005
+ jobStatusMap = getJobStatusMap (workspaces )
1006
+ require .Len (t ,jobStatusMap ,3 )
1007
+ require .Equal (t ,1 ,jobStatusMap [database .ProvisionerJobStatusSucceeded ])
1008
+ require .Equal (t ,1 ,jobStatusMap [database .ProvisionerJobStatusFailed ])
1009
+ // Pending job should be the job that deletes successful, but outdated prebuilt workspace.
1010
+ // Prebuilt workspace MUST be deleted, despite the fact that preset is marked as hard limited.
1011
+ require .Equal (t ,1 ,jobStatusMap [database .ProvisionerJobStatusPending ])
1012
+
1013
+ workspaceBuilds ,err := db .GetWorkspaceBuildsByWorkspaceID (ctx , database.GetWorkspaceBuildsByWorkspaceIDParams {
1014
+ WorkspaceID :successfulWorkspace .ID ,
1015
+ })
1016
+ require .NoError (t ,err )
1017
+ require .Equal (t ,2 ,len (workspaceBuilds ))
1018
+ // Make sure that successfully created, but outdated prebuilt workspace was scheduled for deletion.
1019
+ require .Equal (t ,database .WorkspaceTransitionDelete ,workspaceBuilds [0 ].Transition )
1020
+ require .Equal (t ,database .WorkspaceTransitionStart ,workspaceBuilds [1 ].Transition )
1021
+ })
1022
+ }
1023
+ }
1024
+
818
1025
func TestRunLoop (t * testing.T ) {
819
1026
t .Parallel ()
820
1027