@@ -533,6 +533,108 @@ func TestDetectorPendingWorkspaceBuildNoOverrideStateIfNoExistingBuild(t *testin
533
533
detector .Wait ()
534
534
}
535
535
536
+ // TestDetectorWorkspaceBuildForDormantWorkspace ensures that the jobreaper has
537
+ // enough permissions to fix dormant workspaces.
538
+ //
539
+ // Dormant workspaces are treated as rbac.ResourceWorkspaceDormant rather than
540
+ // rbac.ResourceWorkspace, which resulted in a bug where the jobreaper would
541
+ // be able to see but not fix dormant workspaces.
542
+ func TestDetectorWorkspaceBuildForDormantWorkspace (t * testing.T ) {
543
+ t .Parallel ()
544
+
545
+ var (
546
+ ctx = testutil .Context (t ,testutil .WaitLong )
547
+ db ,pubsub = dbtestutil .NewDB (t )
548
+ log = testutil .Logger (t )
549
+ tickCh = make (chan time.Time )
550
+ statsCh = make (chan jobreaper.Stats )
551
+ )
552
+
553
+ var (
554
+ now = time .Now ()
555
+ tenMinAgo = now .Add (- time .Minute * 10 )
556
+ sixMinAgo = now .Add (- time .Minute * 6 )
557
+ org = dbgen .Organization (t ,db , database.Organization {})
558
+ user = dbgen .User (t ,db , database.User {})
559
+ file = dbgen .File (t ,db , database.File {})
560
+ template = dbgen .Template (t ,db , database.Template {
561
+ OrganizationID :org .ID ,
562
+ CreatedBy :user .ID ,
563
+ })
564
+ templateVersion = dbgen .TemplateVersion (t ,db , database.TemplateVersion {
565
+ OrganizationID :org .ID ,
566
+ TemplateID : uuid.NullUUID {
567
+ UUID :template .ID ,
568
+ Valid :true ,
569
+ },
570
+ CreatedBy :user .ID ,
571
+ })
572
+ workspace = dbgen .Workspace (t ,db , database.WorkspaceTable {
573
+ OwnerID :user .ID ,
574
+ OrganizationID :org .ID ,
575
+ TemplateID :template .ID ,
576
+ DormantAt : sql.NullTime {
577
+ Time :now .Add (- time .Hour ),
578
+ Valid :true ,
579
+ },
580
+ })
581
+
582
+ // First build.
583
+ expectedWorkspaceBuildState = []byte (`{"dean":"cool","colin":"also cool"}` )
584
+ currentWorkspaceBuildJob = dbgen .ProvisionerJob (t ,db ,pubsub , database.ProvisionerJob {
585
+ CreatedAt :tenMinAgo ,
586
+ UpdatedAt :sixMinAgo ,
587
+ StartedAt : sql.NullTime {
588
+ Time :tenMinAgo ,
589
+ Valid :true ,
590
+ },
591
+ OrganizationID :org .ID ,
592
+ InitiatorID :user .ID ,
593
+ Provisioner :database .ProvisionerTypeEcho ,
594
+ StorageMethod :database .ProvisionerStorageMethodFile ,
595
+ FileID :file .ID ,
596
+ Type :database .ProvisionerJobTypeWorkspaceBuild ,
597
+ Input : []byte ("{}" ),
598
+ })
599
+ _ = dbgen .WorkspaceBuild (t ,db , database.WorkspaceBuild {
600
+ WorkspaceID :workspace .ID ,
601
+ TemplateVersionID :templateVersion .ID ,
602
+ BuildNumber :1 ,
603
+ JobID :currentWorkspaceBuildJob .ID ,
604
+ // Should not be overridden.
605
+ ProvisionerState :expectedWorkspaceBuildState ,
606
+ })
607
+ )
608
+
609
+ t .Log ("current job ID: " ,currentWorkspaceBuildJob .ID )
610
+
611
+ // Ensure the RBAC is the dormant type to ensure we're testing the right
612
+ // thing.
613
+ require .Equal (t ,rbac .ResourceWorkspaceDormant .Type ,workspace .RBACObject ().Type )
614
+
615
+ detector := jobreaper .New (ctx ,wrapDBAuthz (db ,log ),pubsub ,log ,tickCh ).WithStatsChannel (statsCh )
616
+ detector .Start ()
617
+ tickCh <- now
618
+
619
+ stats := <- statsCh
620
+ require .NoError (t ,stats .Error )
621
+ require .Len (t ,stats .TerminatedJobIDs ,1 )
622
+ require .Equal (t ,currentWorkspaceBuildJob .ID ,stats .TerminatedJobIDs [0 ])
623
+
624
+ // Check that the current provisioner job was updated.
625
+ job ,err := db .GetProvisionerJobByID (ctx ,currentWorkspaceBuildJob .ID )
626
+ require .NoError (t ,err )
627
+ require .WithinDuration (t ,now ,job .UpdatedAt ,30 * time .Second )
628
+ require .True (t ,job .CompletedAt .Valid )
629
+ require .WithinDuration (t ,now ,job .CompletedAt .Time ,30 * time .Second )
630
+ require .True (t ,job .Error .Valid )
631
+ require .Contains (t ,job .Error .String ,"Build has been detected as hung" )
632
+ require .False (t ,job .ErrorCode .Valid )
633
+
634
+ detector .Close ()
635
+ detector .Wait ()
636
+ }
637
+
536
638
func TestDetectorHungOtherJobTypes (t * testing.T ) {
537
639
t .Parallel ()
538
640