Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitd9d44c1

Browse files
authored
ci: Print go test stats (#6855)
Fixes#6676
1 parent7738274 commitd9d44c1

18 files changed

+4255
-18
lines changed

‎.github/workflows/ci.yaml

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -299,7 +299,14 @@ jobs:
299299
echo "cover=false" >> $GITHUB_OUTPUT
300300
fi
301301
302-
gotestsum --junitfile="gotests.xml" --packages="./..." -- -parallel=8 -timeout=7m -short -failfast $COVERAGE_FLAGS
302+
gotestsum --junitfile="gotests.xml" --jsonfile="gotests.json" --packages="./..." -- -parallel=8 -timeout=7m -short -failfast $COVERAGE_FLAGS
303+
304+
-name:Print test stats
305+
if:success() || failure()
306+
run:|
307+
# Artifacts are not available after rerunning a job,
308+
# so we need to print the test stats to the log.
309+
go run ./scripts/ci-report/main.go gotests.json | tee gotests_stats.json
303310
304311
-uses:actions/upload-artifact@v3
305312
if:success() || failure()
@@ -369,6 +376,13 @@ jobs:
369376
run:|
370377
make test-postgres
371378
379+
-name:Print test stats
380+
if:success() || failure()
381+
run:|
382+
# Artifacts are not available after rerunning a job,
383+
# so we need to print the test stats to the log.
384+
go run ./scripts/ci-report/main.go gotests.json | tee gotests_stats.json
385+
372386
-uses:actions/upload-artifact@v3
373387
if:success() || failure()
374388
with:

‎.github/workflows/typos.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,5 @@ extend-exclude = [
2424
# These files contain base64 strings that confuse the detector
2525
"**XService**.ts",
2626
"**identity.go",
27+
"scripts/ci-report/testdata/**",
2728
]

‎.gitignore

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
**/*.swp
77
gotests.coverage
88
gotests.xml
9-
gotestsum.json
9+
gotests_stats.json
10+
gotests.json
1011
node_modules/
1112
vendor/
1213
yarn-error.log
@@ -29,9 +30,8 @@ site/e2e/states/*.json
2930
site/playwright-report/*
3031
site/.swc
3132

32-
# Make target for updating golden files.
33-
cli/testdata/.gen-golden
34-
helm/tests/testdata/.gen-golden
33+
# Make target for updating golden files (any dir).
34+
.gen-golden
3535

3636
# Build
3737
/build/

‎.prettierignore

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99
**/*.swp
1010
gotests.coverage
1111
gotests.xml
12-
gotestsum.json
12+
gotests_stats.json
13+
gotests.json
1314
node_modules/
1415
vendor/
1516
yarn-error.log
@@ -32,9 +33,8 @@ site/e2e/states/*.json
3233
site/playwright-report/*
3334
site/.swc
3435

35-
# Make target for updating golden files.
36-
cli/testdata/.gen-golden
37-
helm/tests/testdata/.gen-golden
36+
# Make target for updating golden files (any dir).
37+
.gen-golden
3838

3939
# Build
4040
/build/

‎Makefile

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -514,7 +514,7 @@ coderd/apidoc/swagger.json: $(shell find ./scripts/apidocgen $(FIND_EXCLUSIONS)
514514
./scripts/apidocgen/generate.sh
515515
yarn run --cwd=site format:write:only ../docs/api ../docs/manifest.json ../coderd/apidoc/swagger.json
516516

517-
update-golden-files: cli/testdata/.gen-golden helm/tests/testdata/.gen-golden
517+
update-golden-files: cli/testdata/.gen-golden helm/tests/testdata/.gen-golden scripts/ci-report/testdata/.gen-golden
518518
.PHONY: update-golden-files
519519

520520
cli/testdata/.gen-golden:$(wildcard cli/testdata/*.golden)$(wildcard cli/*.tpl)$(GO_SRC_FILES)
@@ -525,6 +525,10 @@ helm/tests/testdata/.gen-golden: $(wildcard helm/tests/testdata/*.golden) $(GO_S
525525
gotest ./helm/tests -run=TestUpdateGoldenFiles -update
526526
touch"$@"
527527

528+
scripts/ci-report/testdata/.gen-golden:$(wildcard scripts/ci-report/testdata/*)$(wildcard scripts/ci-report/*.go)
529+
gotest ./scripts/ci-report -run=TestOutputMatchesGoldenFile -update
530+
touch"$@"
531+
528532
# Generate a prettierrc for the site package that uses relative paths for
529533
# overrides. This allows us to share the same prettier config between the
530534
# site and the root of the repo.
@@ -596,6 +600,7 @@ test-postgres: test-clean test-postgres-docker
596600
# more consistent execution.
597601
DB=ci DB_FROM=$(shell go run scripts/migrate-ci/main.go) gotestsum\
598602
--junitfile="gotests.xml"\
603+
--jsonfile="gotests.json"\
599604
--packages="./..." --\
600605
-covermode=atomic -coverprofile="gotests.coverage" -timeout=20m\
601606
-parallel=4\

‎scripts/ci-report/README.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#ci-report
2+
3+
This program generates a CI report from the`gotests.json` generated by`go test -json` (we use`gotestsum` as a frontend).
4+
5+
##Limitations
6+
7+
We won't generate any report/stats for tests that weren't run. To find all existing tests, we could use:`go test ./... -list=. -json`, but the time it takes is probably not worth it. Usually most tests will run, even if there are errors and we're using`-failfast`.

‎scripts/ci-report/main.go

Lines changed: 258 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,258 @@
1+
package main
2+
3+
import (
4+
"bufio"
5+
"encoding/json"
6+
"errors"
7+
"fmt"
8+
"io"
9+
"os"
10+
"strings"
11+
"time"
12+
13+
"golang.org/x/exp/slices"
14+
"golang.org/x/xerrors"
15+
)
16+
17+
funcmain() {
18+
iflen(os.Args)!=2 {
19+
_,_=fmt.Println("usage: ci-report <gotests.json>")
20+
os.Exit(1)
21+
}
22+
name:=os.Args[1]
23+
24+
goTests,err:=parseGoTestJSON(name)
25+
iferr!=nil {
26+
_,_=fmt.Printf("error parsing gotestsum report: %v",err)
27+
os.Exit(1)
28+
}
29+
30+
rep,err:=parseCIReport(goTests)
31+
iferr!=nil {
32+
_,_=fmt.Printf("error parsing ci report: %v",err)
33+
os.Exit(1)
34+
}
35+
36+
err=printCIReport(os.Stdout,rep)
37+
iferr!=nil {
38+
_,_=fmt.Printf("error printing report: %v",err)
39+
os.Exit(1)
40+
}
41+
}
42+
43+
funcparseGoTestJSON(namestring) (GotestsumReport,error) {
44+
f,err:=os.Open(name)
45+
iferr!=nil {
46+
returnGotestsumReport{},xerrors.Errorf("error opening gotestsum json file: %w",err)
47+
}
48+
deferf.Close()
49+
50+
dec:=json.NewDecoder(f)
51+
varreportGotestsumReport
52+
for {
53+
vareGotestsumReportEntry
54+
err=dec.Decode(&e)
55+
iferrors.Is(err,io.EOF) {
56+
break
57+
}
58+
iferr!=nil {
59+
returnGotestsumReport{},xerrors.Errorf("error decoding json: %w",err)
60+
}
61+
e.Package=strings.TrimPrefix(e.Package,"github.com/coder/coder/")
62+
report=append(report,e)
63+
}
64+
65+
returnreport,nil
66+
}
67+
68+
funcparseCIReport(reportGotestsumReport) (CIReport,error) {
69+
packagesSortedByName:= []string{}
70+
packageTimes:=map[string]float64{}
71+
packageFail:=map[string]int{}
72+
packageSkip:=map[string]bool{}
73+
testTimes:=map[string]float64{}
74+
testSkip:=map[string]bool{}
75+
testOutput:=map[string]string{}
76+
testSortedByName:= []string{}
77+
timeouts:=map[string]string{}
78+
timeoutRunningTests:=map[string]bool{}
79+
fori,e:=rangereport {
80+
switche.Action {
81+
// A package/test may fail or pass.
82+
caseFail:
83+
ife.Test=="" {
84+
packageTimes[e.Package]=*e.Elapsed
85+
}else {
86+
packageFail[e.Package]++
87+
name:=e.Package+"."+e.Test
88+
testTimes[name]=*e.Elapsed
89+
}
90+
casePass:
91+
ife.Test=="" {
92+
packageTimes[e.Package]=*e.Elapsed
93+
}else {
94+
name:=e.Package+"."+e.Test
95+
delete(testOutput,name)
96+
testTimes[name]=*e.Elapsed
97+
}
98+
99+
// Gather all output (deleted when irrelevant).
100+
caseOutput:
101+
name:=e.Package+"."+e.Test// May be pkg.Test or pkg.
102+
if_,ok:=timeouts[name];ok||strings.HasPrefix(e.Output,"panic: test timed out") {
103+
timeouts[name]+=e.Output
104+
continue
105+
}
106+
ife.Test!="" {
107+
name:=e.Package+"."+e.Test
108+
testOutput[name]+=e.Output
109+
}
110+
111+
// Packages start, tests run and either may be skipped.
112+
caseStart:
113+
packagesSortedByName=append(packagesSortedByName,e.Package)
114+
caseRun:
115+
name:=e.Package+"."+e.Test
116+
testSortedByName=append(testSortedByName,name)
117+
caseSkip:
118+
ife.Test=="" {
119+
packageSkip[e.Package]=true
120+
}else {
121+
name:=e.Package+"."+e.Test
122+
testSkip[name]=true
123+
delete(testOutput,name)
124+
}
125+
126+
// Ignore.
127+
caseCont:
128+
casePause:
129+
130+
default:
131+
returnCIReport{},xerrors.Errorf("unknown action: %v in entry %d (%v)",e.Action,i,e)
132+
}
133+
}
134+
135+
// Normalize timeout from "pkg." or "pkg.Test" to "pkg".
136+
timeoutsNorm:=make(map[string]string)
137+
fork,v:=rangetimeouts {
138+
names:=strings.SplitN(k,".",2)
139+
pkg:=names[0]
140+
if_,ok:=timeoutsNorm[pkg];ok {
141+
panic("multiple timeouts for package: "+pkg)
142+
}
143+
timeoutsNorm[pkg]=v
144+
145+
// Mark all running tests as timed out.
146+
// panic: test timed out after 2s\nrunning tests:\n\tTestAgent_Session_TTY_Hushlogin (0s)\n\n ...
147+
parts:=strings.SplitN(v,"\n",3)
148+
iflen(parts)==3&&strings.HasPrefix(parts[1],"running tests:") {
149+
s:=bufio.NewScanner(strings.NewReader(parts[2]))
150+
fors.Scan() {
151+
name:=s.Text()
152+
if!strings.HasPrefix(name,"\tTest") {
153+
break
154+
}
155+
name=strings.TrimPrefix(name,"\t")
156+
name=strings.SplitN(name," ",2)[0]
157+
timeoutRunningTests[pkg+"."+name]=true
158+
packageFail[pkg]++
159+
}
160+
}
161+
}
162+
timeouts=timeoutsNorm
163+
164+
sortAZ:=func(a,bstring)bool {returna<b }
165+
slices.SortFunc(packagesSortedByName,sortAZ)
166+
slices.SortFunc(testSortedByName,sortAZ)
167+
168+
varrepCIReport
169+
170+
for_,pkg:=rangepackagesSortedByName {
171+
output,timeout:=timeouts[pkg]
172+
rep.Packages=append(rep.Packages,PackageReport{
173+
Name:pkg,
174+
Time:packageTimes[pkg],
175+
Skip:packageSkip[pkg],
176+
Fail:packageFail[pkg]>0,
177+
Timeout:timeout,
178+
Output:output,
179+
NumFailed:packageFail[pkg],
180+
})
181+
}
182+
183+
for_,test:=rangetestSortedByName {
184+
names:=strings.SplitN(test,".",2)
185+
skip:=testSkip[test]
186+
out,fail:=testOutput[test]
187+
rep.Tests=append(rep.Tests,TestReport{
188+
Package:names[0],
189+
Name:names[1],
190+
Time:testTimes[test],
191+
Skip:skip,
192+
Fail:fail,
193+
Timeout:timeoutRunningTests[test],
194+
Output:out,
195+
})
196+
}
197+
198+
returnrep,nil
199+
}
200+
201+
funcprintCIReport(dst io.Writer,repCIReport)error {
202+
enc:=json.NewEncoder(dst)
203+
enc.SetIndent(""," ")
204+
err:=enc.Encode(rep)
205+
iferr!=nil {
206+
returnxerrors.Errorf("error encoding json: %w",err)
207+
}
208+
returnnil
209+
}
210+
211+
typeCIReportstruct {
212+
Packages []PackageReport`json:"packages"`
213+
Tests []TestReport`json:"tests"`
214+
}
215+
216+
typePackageReportstruct {
217+
Namestring`json:"name"`
218+
Timefloat64`json:"time"`
219+
Skipbool`json:"skip,omitempty"`
220+
Failbool`json:"fail,omitempty"`
221+
NumFailedint`json:"num_failed,omitempty"`
222+
Timeoutbool`json:"timeout,omitempty"`
223+
Outputstring`json:"output,omitempty"`// Output present e.g. for timeout.
224+
}
225+
226+
typeTestReportstruct {
227+
Packagestring`json:"package"`
228+
Namestring`json:"name"`
229+
Timefloat64`json:"time"`
230+
Skipbool`json:"skip,omitempty"`
231+
Failbool`json:"fail,omitempty"`
232+
Timeoutbool`json:"timeout,omitempty"`
233+
Outputstring`json:"output,omitempty"`
234+
}
235+
236+
typeGotestsumReport []GotestsumReportEntry
237+
238+
typeGotestsumReportEntrystruct {
239+
Time time.Time`json:"Time"`
240+
ActionAction`json:"Action"`
241+
Packagestring`json:"Package"`
242+
Teststring`json:"Test,omitempty"`
243+
Outputstring`json:"Output,omitempty"`
244+
Elapsed*float64`json:"Elapsed,omitempty"`
245+
}
246+
247+
typeActionstring
248+
249+
const (
250+
ContAction="cont"
251+
FailAction="fail"
252+
OutputAction="output"
253+
PassAction="pass"
254+
PauseAction="pause"
255+
RunAction="run"
256+
SkipAction="skip"
257+
StartAction="start"
258+
)

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp