Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitadd96f7

Browse files
michelvocksSkarlso
authored andcommitted
Implemented pipeline integrity check during pipeline creation process (#96)
* Implemented pipeline integrity check during pipeline creation process* Fixed tests. Pipelines will be now removed if integrity check fails* Added one more test
1 parent07cf7a3 commitadd96f7

File tree

9 files changed

+130
-27
lines changed

9 files changed

+130
-27
lines changed

‎frontend/client/views/overview/index.vue‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<template>
22
<divclass="columns is-multiline">
33
<templatev-for="(pipeline,index)inpipelines">
4-
<divclass="column is-one-third":key="index">
4+
<divclass="column is-one-third":key="index"v-if="!pipeline.p.notvalid">
55
<divclass="pipeline-box notification content-article">
66
<divclass="status-display-success"v-if="pipeline.r.status === 'success'"></div>
77
<divclass="status-display-fail"v-else-if="pipeline.r.status === 'failed'"></div>

‎frontend/client/views/pipeline/create.vue‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -369,11 +369,16 @@ export default {
369369
this.historyRows[i].output+='Starting build process. This may take some time...\n'
370370
}
371371
372-
if (this.historyRows[i].status>=75) {
372+
if (this.historyRows[i].status>=50) {
373373
this.historyRows[i].output+='Pipeline has been successfully compiled.\n'
374374
this.historyRows[i].output+='Copy binary to pipelines folder...\n'
375375
}
376376
377+
if (this.historyRows[i].status>=75) {
378+
this.historyRows[i].output+='Pipeline binary has been copied to pipelines folder.\n'
379+
this.historyRows[i].output+='Starting integrity check...\n'
380+
}
381+
377382
if (this.historyRows[i].status===100) {
378383
this.historyRows[i].output+='Copy was successful.\n'
379384
this.historyRows[i].output+='Finished.\n'

‎gaia.go‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -101,15 +101,16 @@ type User struct {
101101

102102
// Pipeline represents a single pipeline
103103
typePipelinestruct {
104-
IDint`json:"id,omitempty"`
105-
Namestring`json:"name,omitempty"`
106-
RepoGitRepo`json:"repo,omitempty"`
107-
TypePipelineType`json:"type,omitempty"`
108-
ExecPathstring`json:"execpath,omitempty"`
109-
SHA256Sum []byte`json:"sha256sum,omitempty"`
110-
Jobs []Job`json:"jobs,omitempty"`
111-
Created time.Time`json:"created,omitempty"`
112-
UUIDstring`json:"uuid,omitempty"`
104+
IDint`json:"id,omitempty"`
105+
Namestring`json:"name,omitempty"`
106+
RepoGitRepo`json:"repo,omitempty"`
107+
TypePipelineType`json:"type,omitempty"`
108+
ExecPathstring`json:"execpath,omitempty"`
109+
SHA256Sum []byte`json:"sha256sum,omitempty"`
110+
Jobs []Job`json:"jobs,omitempty"`
111+
Created time.Time`json:"created,omitempty"`
112+
UUIDstring`json:"uuid,omitempty"`
113+
IsNotValidbool`json:"notvalid,omitempty"`
113114
}
114115

115116
// GitRepo represents a single git repository

‎pipeline/create_pipeline.go‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package pipeline
22

33
import (
44
"fmt"
5+
"path/filepath"
56

67
"github.com/gaia-pipeline/gaia"
78
"github.com/gaia-pipeline/gaia/services"
@@ -12,7 +13,10 @@ const (
1213
pipelineCloneStatus=25
1314

1415
// Percent of pipeline creation progress after compile process done
15-
pipelineCompileStatus=75
16+
pipelineCompileStatus=50
17+
18+
// Percent of pipeline creation progress after validation binary copy
19+
pipelineCopyStatus=75
1620

1721
// Completed percent progress
1822
pipelineCompleteStatus=100
@@ -86,6 +90,39 @@ func CreatePipeline(p *gaia.CreatePipeline) {
8690
return
8791
}
8892

93+
// Update status of our pipeline build
94+
p.Status=pipelineCopyStatus
95+
err=storeService.CreatePipelinePut(p)
96+
iferr!=nil {
97+
gaia.Cfg.Logger.Error("cannot put create pipeline into store","error",err.Error())
98+
return
99+
}
100+
101+
// Run update if needed
102+
p.Pipeline.ExecPath=filepath.Join(gaia.Cfg.PipelinePath,appendTypeToName(p.Pipeline.Name,p.Pipeline.Type))
103+
err=updatePipeline(&p.Pipeline)
104+
iferr!=nil {
105+
p.StatusType=gaia.CreatePipelineFailed
106+
p.Output=fmt.Sprintf("cannot update pipeline: %s",err.Error())
107+
storeService.CreatePipelinePut(p)
108+
109+
// Creation failed. Remove broken pipeline.
110+
DeleteBinary(p.Pipeline)
111+
return
112+
}
113+
114+
// Try to get pipeline jobs to check if this pipeline is valid.
115+
schedulerService,_:=services.SchedulerService()
116+
iferr=schedulerService.SetPipelineJobs(&p.Pipeline);err!=nil {
117+
p.StatusType=gaia.CreatePipelineFailed
118+
p.Output=fmt.Sprintf("cannot validate pipeline: %s",err.Error())
119+
storeService.CreatePipelinePut(p)
120+
121+
// Creation failed. Remove broken pipeline.
122+
DeleteBinary(p.Pipeline)
123+
return
124+
}
125+
89126
// Save the generated pipeline data
90127
err=bP.SavePipeline(&p.Pipeline)
91128
iferr!=nil {

‎pipeline/create_pipeline_test.go‎

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,12 @@ import (
44
"bytes"
55
"errors"
66
"io/ioutil"
7+
"strings"
78
"testing"
89

10+
"github.com/gaia-pipeline/gaia"
911
"github.com/gaia-pipeline/gaia/services"
10-
1112
"github.com/gaia-pipeline/gaia/store"
12-
13-
"github.com/gaia-pipeline/gaia"
1413
"github.com/hashicorp/go-hclog"
1514
)
1615

@@ -28,6 +27,16 @@ func (mcp *mockCreatePipelineStore) PipelinePut(p *gaia.Pipeline) error {
2827
returnmcp.Error
2928
}
3029

30+
typemockSchedulerstruct {
31+
Errorerror
32+
}
33+
34+
func (ms*mockScheduler)Init()error {returnnil }
35+
func (ms*mockScheduler)SchedulePipeline(p*gaia.Pipeline,args []gaia.Argument) (*gaia.PipelineRun,error) {
36+
returnnil,nil
37+
}
38+
func (ms*mockScheduler)SetPipelineJobs(p*gaia.Pipeline)error {returnms.Error }
39+
3140
funcTestCreatePipelineUnknownType(t*testing.T) {
3241
tmp,_:=ioutil.TempDir("","TestCreatePipelineUnknownType")
3342
gaia.Cfg=new(gaia.Config)
@@ -101,6 +110,7 @@ func TestCreatePipeline(t *testing.T) {
101110
tmp,_:=ioutil.TempDir("","TestCreatePipeline")
102111
gaia.Cfg=new(gaia.Config)
103112
gaia.Cfg.HomePath=tmp
113+
gaia.Cfg.PipelinePath=tmp
104114
buf:=new(bytes.Buffer)
105115
gaia.Cfg.Logger=hclog.New(&hclog.LoggerOptions{
106116
Level:hclog.Trace,
@@ -110,11 +120,43 @@ func TestCreatePipeline(t *testing.T) {
110120
mcp:=new(mockCreatePipelineStore)
111121
services.MockStorageService(mcp)
112122
deferfunc() {services.MockStorageService(nil) }()
123+
ms:=new(mockScheduler)
124+
services.MockSchedulerService(ms)
125+
deferfunc() {services.MockSchedulerService(nil) }()
113126
cp:=new(gaia.CreatePipeline)
127+
cp.Pipeline.Name="test"
114128
cp.Pipeline.Type=gaia.PTypeGolang
115129
cp.Pipeline.Repo.URL="https://github.com/gaia-pipeline/pipeline-test"
116130
CreatePipeline(cp)
117131
ifcp.StatusType!=gaia.CreatePipelineSuccess {
118132
t.Fatal("pipeline status was not success. was: ",cp.StatusType)
119133
}
120134
}
135+
136+
funcTestCreatePipelineSetPipelineJobsFail(t*testing.T) {
137+
tmp,_:=ioutil.TempDir("","TestCreatePipelineSetPipelineJobsFail")
138+
gaia.Cfg=new(gaia.Config)
139+
gaia.Cfg.HomePath=tmp
140+
gaia.Cfg.PipelinePath=tmp
141+
buf:=new(bytes.Buffer)
142+
gaia.Cfg.Logger=hclog.New(&hclog.LoggerOptions{
143+
Level:hclog.Trace,
144+
Output:buf,
145+
Name:"Gaia",
146+
})
147+
mcp:=new(mockCreatePipelineStore)
148+
services.MockStorageService(mcp)
149+
deferfunc() {services.MockStorageService(nil) }()
150+
ms:=new(mockScheduler)
151+
ms.Error=errors.New("error")
152+
services.MockSchedulerService(ms)
153+
deferfunc() {services.MockSchedulerService(nil) }()
154+
cp:=new(gaia.CreatePipeline)
155+
cp.Pipeline.Name="test"
156+
cp.Pipeline.Type=gaia.PTypeGolang
157+
cp.Pipeline.Repo.URL="https://github.com/gaia-pipeline/pipeline-test"
158+
CreatePipeline(cp)
159+
if!strings.Contains(cp.Output,"cannot validate pipeline") {
160+
t.Fatalf("error thrown should contain 'cannot validate pipeline' but its %s",cp.Output)
161+
}
162+
}

‎pipeline/ticker.go‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -111,10 +111,10 @@ func checkActivePipelines() {
111111
}
112112

113113
// Let us try again to start the plugin and receive all implemented jobs
114-
schedulerService.SetPipelineJobs(p)
115-
116-
// Set new pipeline hash
117-
p.SHA256Sum=checksum
114+
iferr=schedulerService.SetPipelineJobs(p);err!=nil {
115+
// Mark that this pipeline is broken.
116+
p.IsNotValid=true
117+
}
118118

119119
// Replace pipeline
120120
ifok:=GlobalActivePipelines.Replace(*p);!ok {
@@ -157,16 +157,19 @@ func checkActivePipelines() {
157157

158158
// update pipeline if needed
159159
ifbytes.Compare(pipeline.SHA256Sum,pipelineCheckSum)!=0 {
160-
pipeline.SHA256Sum=pipelineCheckSum
161160
iferr=updatePipeline(pipeline);err!=nil {
162161
storeService.PipelinePut(pipeline)
163162
gaia.Cfg.Logger.Error("cannot update pipeline","error",err.Error(),"pipeline",pipeline)
164163
continue
165164
}
165+
storeService.PipelinePut(pipeline)
166166
}
167167

168168
// Let us try to start the plugin and receive all implemented jobs
169-
schedulerService.SetPipelineJobs(pipeline)
169+
iferr=schedulerService.SetPipelineJobs(pipeline);err!=nil {
170+
// Mark that this pipeline is broken.
171+
pipeline.IsNotValid=true
172+
}
170173

171174
// We encountered a drop-in pipeline previously. Now is the time to save it.
172175
ifshouldStore {

‎pipeline/update_pipeline.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,5 +52,13 @@ func updatePipeline(p *gaia.Pipeline) error {
5252
returnerr
5353
}
5454
}
55+
56+
// Update checksum
57+
checksum,err:=getSHA256Sum(p.ExecPath)
58+
iferr!=nil {
59+
returnerr
60+
}
61+
p.SHA256Sum=checksum
62+
5563
returnnil
5664
}

‎plugin/plugin.go‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@ package plugin
22

33
import (
44
"bufio"
5+
"bytes"
56
"errors"
7+
"fmt"
68
"io"
79
"os"
810
"os/exec"
@@ -46,8 +48,9 @@ type Plugin struct {
4648
// Log file where all output is stored.
4749
logFile*os.File
4850

49-
// Writer used to write logs from execution to file
51+
// Writer used to write logs from execution to file or buffer
5052
writer*bufio.Writer
53+
buffer*bytes.Buffer
5154

5255
// CA instance used to handle certificates
5356
ca security.CAAPI
@@ -85,10 +88,14 @@ func (p *Plugin) Connect(command *exec.Cmd, logPath *string) error {
8588
iferr!=nil {
8689
returnerr
8790
}
88-
}
8991

90-
// Create new writer
91-
p.writer=bufio.NewWriter(p.logFile)
92+
// Create new writer
93+
p.writer=bufio.NewWriter(p.logFile)
94+
}else {
95+
// If no path is provided, write output to buffer
96+
p.buffer=new(bytes.Buffer)
97+
p.writer=bufio.NewWriter(p.buffer)
98+
}
9299

93100
// Create and sign a new pair of certificates for the server
94101
varerrerror
@@ -129,7 +136,8 @@ func (p *Plugin) Connect(command *exec.Cmd, logPath *string) error {
129136
// Connect via gRPC
130137
gRPCClient,err:=p.client.Client()
131138
iferr!=nil {
132-
returnerr
139+
p.writer.Flush()
140+
returnfmt.Errorf("%s\n\n--- output ---\n%s",err.Error(),p.buffer.String())
133141
}
134142

135143
// Request the plugin

‎scheduler/scheduler.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -608,7 +608,6 @@ func (s *Scheduler) SetPipelineJobs(p *gaia.Pipeline) error {
608608
// Get jobs
609609
jobs,err:=s.getPipelineJobs(p)
610610
iferr!=nil {
611-
gaia.Cfg.Logger.Debug("cannot get jobs from pipeline","error",err.Error(),"pipeline",p)
612611
returnerr
613612
}
614613
p.Jobs=jobs

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp