Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commita1cbf49

Browse files
authored
Python create pipeline race condition (#132)
* Changed order of pipeline creation process* Added execpath set after build execution* Removed manual set for execpath and tested race condition fix* Fixed failing test
1 parenta2042c4 commita1cbf49

File tree

7 files changed

+71
-38
lines changed

7 files changed

+71
-38
lines changed

‎frontend/client/views/pipeline/create.vue‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -371,12 +371,12 @@ export default {
371371
372372
if (this.historyRows[i].status>=50) {
373373
this.historyRows[i].output+='Pipeline has been successfully compiled.\n'
374-
this.historyRows[i].output+='Copy binary to pipelines folder...\n'
374+
this.historyRows[i].output+='Verifying pipeline...\n'
375375
}
376376
377377
if (this.historyRows[i].status>=75) {
378-
this.historyRows[i].output+='Pipelinebinaryhas beencopied to pipelines folder.\n'
379-
this.historyRows[i].output+='Starting integrity check...\n'
378+
this.historyRows[i].output+='Pipeline has beenverified.\n'
379+
this.historyRows[i].output+='Finalizing creation process...\n'
380380
}
381381
382382
if (this.historyRows[i].status===100) {

‎workers/pipeline/build_cpp.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,10 @@ func (b *BuildPipelineCpp) ExecuteBuild(p *gaia.CreatePipeline) error {
6060
returnerr
6161
}
6262

63+
// Build has been finished. Set execution path to the build result archive.
64+
// This will be used during pipeline verification phase which will happen after this step.
65+
p.Pipeline.ExecPath=filepath.Join(p.Pipeline.Repo.LocalDest,cppFinalBinaryName)
66+
6367
returnnil
6468
}
6569

‎workers/pipeline/build_golang.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,10 @@ func (b *BuildPipelineGolang) ExecuteBuild(p *gaia.CreatePipeline) error {
8282
returnerr
8383
}
8484

85+
// Build has been finished. Set execution path to the build result archive.
86+
// This will be used during pipeline verification phase which will happen after this step.
87+
p.Pipeline.ExecPath=filepath.Join(p.Pipeline.Repo.LocalDest,appendTypeToName(p.Pipeline.Name,p.Pipeline.Type))
88+
8589
returnnil
8690
}
8791

‎workers/pipeline/build_java.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,10 @@ func (b *BuildPipelineJava) ExecuteBuild(p *gaia.CreatePipeline) error {
7171
returnerr
7272
}
7373

74+
// Build has been finished. Set execution path to the build result archive.
75+
// This will be used during pipeline verification phase which will happen after this step.
76+
p.Pipeline.ExecPath=filepath.Join(p.Pipeline.Repo.LocalDest,mavenTargetFolder,javaFinalJarName)
77+
7478
returnnil
7579
}
7680

‎workers/pipeline/build_python.go‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -65,17 +65,25 @@ func (b *BuildPipelinePython) ExecuteBuild(p *gaia.CreatePipeline) error {
6565
returnerr
6666
}
6767

68+
// Build has been finished. Set execution path to the build result archive.
69+
// This will be used during pipeline verification phase which will happen after this step.
70+
p.Pipeline.ExecPath,err=findPythonArchivePath(p)
71+
iferr!=nil {
72+
returnerr
73+
}
74+
6875
returnnil
6976
}
7077

71-
// CopyBinary copies the final compiled archive to the
72-
// destination folder.
73-
func (b*BuildPipelinePython)CopyBinary(p*gaia.CreatePipeline)error {
78+
// findPythonArchivePath filters the archives in the generated dist folder
79+
// and looks for the final archive. It will return an error if less or more
80+
// than one file(s) are found otherwise the full path to the file.
81+
funcfindPythonArchivePath(p*gaia.CreatePipeline) (srcstring,errerror) {
7482
// find all files in dist folder
7583
distFolder:=filepath.Join(p.Pipeline.Repo.LocalDest,"dist")
7684
files,err:=ioutil.ReadDir(distFolder)
7785
iferr!=nil {
78-
returnerr
86+
return
7987
}
8088

8189
// filter for archives
@@ -88,12 +96,24 @@ func (b *BuildPipelinePython) CopyBinary(p *gaia.CreatePipeline) error {
8896

8997
// if we found more or less than one archive we have a problem
9098
iflen(archive)!=1 {
91-
gaia.Cfg.Logger.Debug("cannot copy python package","foundPackages",len(archive),"archives",files)
92-
returnerrors.New("cannot copy python package: not found")
99+
gaia.Cfg.Logger.Debug("cannot find python package","foundPackages",len(archive),"archives",files)
100+
err=errors.New("cannot find python package")
101+
return
93102
}
94103

104+
// Return full path
105+
src=filepath.Join(distFolder,archive[0].Name())
106+
return
107+
}
108+
109+
// CopyBinary copies the final compiled archive to the
110+
// destination folder.
111+
func (b*BuildPipelinePython)CopyBinary(p*gaia.CreatePipeline)error {
95112
// Define src and destination
96-
src:=filepath.Join(distFolder,archive[0].Name())
113+
src,err:=findPythonArchivePath(p)
114+
iferr!=nil {
115+
returnerr
116+
}
97117
dest:=filepath.Join(gaia.Cfg.PipelinePath,appendTypeToName(p.Pipeline.Name,p.Pipeline.Type))
98118

99119
// Copy binary

‎workers/pipeline/build_python_test.go‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,22 @@ func TestExecuteBuildPython(t *testing.T) {
5151
tmp,_:=ioutil.TempDir("","TestExecuteBuildPython")
5252
gaia.Cfg=new(gaia.Config)
5353
gaia.Cfg.HomePath=tmp
54-
b:=new(BuildPipelinePython)
5554
p:=new(gaia.CreatePipeline)
55+
p.Pipeline.Name="main"
56+
p.Pipeline.Type=gaia.PTypePython
57+
p.Pipeline.Repo.LocalDest=tmp
58+
os.Mkdir(filepath.Join(tmp,"dist"),0744)
59+
src:=filepath.Join(tmp,"dist",p.Pipeline.Name+".tar.gz")
60+
f,_:=os.Create(src)
61+
deferos.RemoveAll(tmp)
62+
deferf.Close()
63+
ioutil.WriteFile(src, []byte("testcontent"),0666)
64+
b:=new(BuildPipelinePython)
5665
// go must be existent, python maybe not.
5766
pythonBinaryName="go"
5867
err:=b.ExecuteBuild(p)
5968
iferr!=nil {
60-
t.Fatal("error while running executebuild. none was expected")
69+
t.Fatalf("error while running executebuild. none was expected but got %s",err.Error())
6170
}
6271
expectedBuildArgs:="setup.py,sdist"
6372
actualArgs:=os.Getenv("CMD_ARGS")

‎workers/pipeline/create_pipeline.go‎

Lines changed: 18 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package pipeline
22

33
import (
44
"fmt"
5-
"path/filepath"
65

76
"github.com/gaia-pipeline/gaia"
87
"github.com/gaia-pipeline/gaia/services"
@@ -15,8 +14,8 @@ const (
1514
// Percent of pipeline creation progress after compile process done
1615
pipelineCompileStatus=50
1716

18-
// Percent of pipeline creation progress after validation binary copy
19-
pipelineCopyStatus=75
17+
// Percent of pipeline creation progress after validation
18+
pipelineValidateStatus=75
2019

2120
// Completed percent progress
2221
pipelineCompleteStatus=100
@@ -81,33 +80,12 @@ func CreatePipeline(p *gaia.CreatePipeline) {
8180
return
8281
}
8382

84-
// Copy compiled binary to plugins folder
85-
err=bP.CopyBinary(p)
86-
iferr!=nil {
87-
p.StatusType=gaia.CreatePipelineFailed
88-
p.Output=fmt.Sprintf("cannot copy compiled binary: %s",err.Error())
89-
storeService.CreatePipelinePut(p)
90-
return
91-
}
92-
93-
// Update status of our pipeline build
94-
p.Status=pipelineCopyStatus
95-
err=storeService.CreatePipelinePut(p)
96-
iferr!=nil {
97-
gaia.Cfg.Logger.Error("cannot put create pipeline into store","error",err.Error())
98-
return
99-
}
100-
10183
// Run update if needed
102-
p.Pipeline.ExecPath=filepath.Join(gaia.Cfg.PipelinePath,appendTypeToName(p.Pipeline.Name,p.Pipeline.Type))
10384
err=updatePipeline(&p.Pipeline)
10485
iferr!=nil {
10586
p.StatusType=gaia.CreatePipelineFailed
10687
p.Output=fmt.Sprintf("cannot update pipeline: %s",err.Error())
10788
storeService.CreatePipelinePut(p)
108-
109-
// Creation failed. Remove broken pipeline.
110-
DeleteBinary(p.Pipeline)
11189
return
11290
}
11391

@@ -117,9 +95,14 @@ func CreatePipeline(p *gaia.CreatePipeline) {
11795
p.StatusType=gaia.CreatePipelineFailed
11896
p.Output=fmt.Sprintf("cannot validate pipeline: %s",err.Error())
11997
storeService.CreatePipelinePut(p)
98+
return
99+
}
120100

121-
// Creation failed. Remove broken pipeline.
122-
DeleteBinary(p.Pipeline)
101+
// Update status of our pipeline build
102+
p.Status=pipelineValidateStatus
103+
err=storeService.CreatePipelinePut(p)
104+
iferr!=nil {
105+
gaia.Cfg.Logger.Error("cannot put create pipeline into store","error",err.Error())
123106
return
124107
}
125108

@@ -132,6 +115,15 @@ func CreatePipeline(p *gaia.CreatePipeline) {
132115
return
133116
}
134117

118+
// Copy compiled binary to plugins folder which is the final step
119+
err=bP.CopyBinary(p)
120+
iferr!=nil {
121+
p.StatusType=gaia.CreatePipelineFailed
122+
p.Output=fmt.Sprintf("cannot copy compiled binary: %s",err.Error())
123+
storeService.CreatePipelinePut(p)
124+
return
125+
}
126+
135127
// Set create pipeline status to complete
136128
p.Status=pipelineCompleteStatus
137129
p.StatusType=gaia.CreatePipelineSuccess

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp