Movatterモバイル変換


[0]ホーム

URL:


Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Sign up
Appearance settings

Commit2207ac0

Browse files
mtojekpull[bot]
authored andcommitted
feat: clean stale provisioner files (#9545)
1 parentcebb098 commit2207ac0

File tree

13 files changed

+594
-9
lines changed

13 files changed

+594
-9
lines changed

‎Makefile

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -570,7 +570,7 @@ coderd/apidoc/swagger.json: $(shell find ./scripts/apidocgen $(FIND_EXCLUSIONS)
570570
./scripts/apidocgen/generate.sh
571571
pnpm run format:write:only ./docs/api ./docs/manifest.json ./coderd/apidoc/swagger.json
572572

573-
update-golden-files: cli/testdata/.gen-golden helm/coder/tests/testdata/.gen-golden helm/provisioner/tests/testdata/.gen-golden scripts/ci-report/testdata/.gen-golden enterprise/cli/testdata/.gen-golden coderd/.gen-golden
573+
update-golden-files: cli/testdata/.gen-golden helm/coder/tests/testdata/.gen-golden helm/provisioner/tests/testdata/.gen-golden scripts/ci-report/testdata/.gen-golden enterprise/cli/testdata/.gen-golden coderd/.gen-golden provisioner/terraform/testdata/.gen-golden
574574
.PHONY: update-golden-files
575575

576576
cli/testdata/.gen-golden:$(wildcard cli/testdata/*.golden)$(wildcard cli/*.tpl)$(GO_SRC_FILES)$(wildcard cli/*_test.go)
@@ -593,6 +593,10 @@ coderd/.gen-golden: $(wildcard coderd/testdata/*/*.golden) $(GO_SRC_FILES) $(wil
593593
gotest ./coderd -run="Test.*Golden$$" -update
594594
touch"$@"
595595

596+
provisioner/terraform/testdata/.gen-golden:$(wildcard provisioner/terraform/testdata/*/*.golden)$(GO_SRC_FILES)$(wildcard provisioner/terraform/*_test.go)
597+
gotest ./provisioner/terraform -run="Test.*Golden$$" -update
598+
touch"$@"
599+
596600
scripts/ci-report/testdata/.gen-golden:$(wildcard scripts/ci-report/testdata/*)$(wildcard scripts/ci-report/*.go)
597601
gotest ./scripts/ci-report -run=TestOutputMatchesGoldenFile -update
598602
touch"$@"

‎go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,8 @@ require (
196196
tailscale.comv1.46.1
197197
)
198198

199+
requiregithub.com/djherbis/timesv1.5.0
200+
199201
require (
200202
cloud.google.com/go/computev1.23.0// indirect
201203
cloud.google.com/go/loggingv1.8.1// indirect

‎go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,8 @@ github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WA
267267
github.com/dgryski/triflesv0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g=
268268
github.com/dgryski/triflesv0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
269269
github.com/dhui/dktestv0.3.16 h1:i6gq2YQEtcrjKbeJpBkWjE8MmLZPYllcjOFbTZuPDnw=
270+
github.com/djherbis/timesv1.5.0 h1:79myA211VwPhFTqUk8xehWrsEO+zcIZj0zT8mXPVARU=
271+
github.com/djherbis/timesv1.5.0/go.mod h1:5q7FDLvbNg1L/KaBmPcWlVR9NmoKo3+ucqUA3ijQhA0=
270272
github.com/dlclark/regexp2v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo=
271273
github.com/dlclark/regexp2v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
272274
github.com/docker/cliv23.0.5+incompatible h1:ufWmAOuD3Vmr7JP2G5K3cyuNC4YZWiAsuDEvFVVDafE=

‎provisioner/terraform/cleanup.go

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
package terraform
2+
3+
import (
4+
"context"
5+
"os"
6+
"path/filepath"
7+
"strings"
8+
"time"
9+
10+
"github.com/djherbis/times"
11+
"github.com/spf13/afero"
12+
"golang.org/x/xerrors"
13+
14+
"cdr.dev/slog"
15+
)
16+
17+
// CleanStaleTerraformPlugins browses the Terraform cache directory
18+
// and remove stale plugins that haven't been used for a while.
19+
// Additionally, it sweeps empty, old directory trees.
20+
//
21+
// Sample cachePath:
22+
//
23+
///Users/john.doe/Library/Caches/coder/provisioner-1/tf
24+
///tmp/coder/provisioner-0/tf
25+
funcCleanStaleTerraformPlugins(ctx context.Context,cachePathstring,fs afero.Fs,now time.Time,logger slog.Logger)error {
26+
cachePath,err:=filepath.Abs(cachePath)// sanity check in case the path is e.g. ../../../cache
27+
iferr!=nil {
28+
returnxerrors.Errorf("unable to determine absolute path %q: %w",cachePath,err)
29+
}
30+
31+
// Firstly, check if the cache path exists.
32+
_,err=fs.Stat(cachePath)
33+
ifos.IsNotExist(err) {
34+
returnnil
35+
}elseiferr!=nil {
36+
returnxerrors.Errorf("unable to stat cache path %q: %w",cachePath,err)
37+
}
38+
39+
logger.Info(ctx,"clean stale Terraform plugins",slog.F("cache_path",cachePath))
40+
41+
// Filter directory trees matching pattern: <repositoryURL>/<company>/<plugin>/<version>/<distribution>
42+
filterFunc:=func(pathstring,info os.FileInfo)bool {
43+
if!info.IsDir() {
44+
returnfalse
45+
}
46+
47+
relativePath,err:=filepath.Rel(cachePath,path)
48+
iferr!=nil {
49+
logger.Error(ctx,"unable to evaluate a relative path",slog.F("base",cachePath),slog.F("target",path),slog.Error(err))
50+
returnfalse
51+
}
52+
53+
parts:=strings.Split(relativePath,string(filepath.Separator))
54+
returnlen(parts)==5
55+
}
56+
57+
// Review cached Terraform plugins
58+
varpluginPaths []string
59+
err=afero.Walk(fs,cachePath,func(pathstring,info os.FileInfo,errerror)error {
60+
iferr!=nil {
61+
returnerr
62+
}
63+
64+
if!filterFunc(path,info) {
65+
returnnil
66+
}
67+
68+
logger.Debug(ctx,"plugin directory discovered",slog.F("path",path))
69+
pluginPaths=append(pluginPaths,path)
70+
returnnil
71+
})
72+
iferr!=nil {
73+
returnxerrors.Errorf("unable to walk through cache directory %q: %w",cachePath,err)
74+
}
75+
76+
// Identify stale plugins
77+
varstalePlugins []string
78+
for_,pluginPath:=rangepluginPaths {
79+
accessTime,err:=latestAccessTime(fs,pluginPath)
80+
iferr!=nil {
81+
returnxerrors.Errorf("unable to evaluate latest access time for directory %q: %w",pluginPath,err)
82+
}
83+
84+
ifaccessTime.Add(staleTerraformPluginRetention).Before(now) {
85+
logger.Info(ctx,"plugin directory is stale and will be removed",slog.F("plugin_path",pluginPath))
86+
stalePlugins=append(stalePlugins,pluginPath)
87+
}else {
88+
logger.Debug(ctx,"plugin directory is not stale",slog.F("plugin_path",pluginPath))
89+
}
90+
}
91+
92+
// Remove stale plugins
93+
for_,stalePluginPath:=rangestalePlugins {
94+
// Remove the plugin directory
95+
err=fs.RemoveAll(stalePluginPath)
96+
iferr!=nil {
97+
returnxerrors.Errorf("unable to remove stale plugin %q: %w",stalePluginPath,err)
98+
}
99+
100+
// Compact the plugin structure by removing empty directories.
101+
wd:=stalePluginPath
102+
level:=5// <repositoryURL>/<company>/<plugin>/<version>/<distribution>
103+
for {
104+
level--
105+
iflevel==0 {
106+
break// do not compact further
107+
}
108+
109+
wd=filepath.Dir(wd)
110+
111+
files,err:=afero.ReadDir(fs,wd)
112+
iferr!=nil {
113+
returnxerrors.Errorf("unable to read directory content %q: %w",wd,err)
114+
}
115+
116+
iflen(files)>0 {
117+
break// there are still other plugins
118+
}
119+
120+
logger.Debug(ctx,"remove empty directory",slog.F("path",wd))
121+
err=fs.Remove(wd)
122+
iferr!=nil {
123+
returnxerrors.Errorf("unable to remove directory %q: %w",wd,err)
124+
}
125+
}
126+
}
127+
returnnil
128+
}
129+
130+
// latestAccessTime walks recursively through the directory content, and locates
131+
// the last accessed file.
132+
funclatestAccessTime(fs afero.Fs,pluginPathstring) (time.Time,error) {
133+
varlatest time.Time
134+
err:=afero.Walk(fs,pluginPath,func(pathstring,info os.FileInfo,errerror)error {
135+
iferr!=nil {
136+
returnerr
137+
}
138+
139+
accessTime:=info.ModTime()// fallback to modTime if accessTime is not available (afero)
140+
ifinfo.Sys()!=nil {
141+
timeSpec:=times.Get(info)
142+
accessTime=timeSpec.AccessTime()
143+
}
144+
iflatest.Before(accessTime) {
145+
latest=accessTime
146+
}
147+
returnnil
148+
})
149+
iferr!=nil {
150+
return time.Time{},xerrors.Errorf("unable to walk the plugin path %q: %w",pluginPath,err)
151+
}
152+
returnlatest,nil
153+
}

‎provisioner/terraform/cleanup_test.go

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
//go:build linux || darwin
2+
3+
package terraform_test
4+
5+
import (
6+
"bytes"
7+
"context"
8+
"flag"
9+
"os"
10+
"path/filepath"
11+
"strings"
12+
"testing"
13+
"time"
14+
15+
"github.com/google/go-cmp/cmp"
16+
"github.com/spf13/afero"
17+
"github.com/stretchr/testify/assert"
18+
"github.com/stretchr/testify/require"
19+
20+
"cdr.dev/slog"
21+
"cdr.dev/slog/sloggers/slogtest"
22+
"github.com/coder/coder/v2/provisioner/terraform"
23+
"github.com/coder/coder/v2/testutil"
24+
)
25+
26+
constcachePath="/tmp/coder/provisioner-0/tf"
27+
28+
// updateGoldenFiles is a flag that can be set to update golden files.
29+
varupdateGoldenFiles=flag.Bool("update",false,"Update golden files")
30+
31+
var (
32+
coderPluginPath=filepath.Join("registry.terraform.io","coder","coder","0.11.1","darwin_arm64")
33+
dockerPluginPath=filepath.Join("registry.terraform.io","kreuzwerker","docker","2.25.0","darwin_arm64")
34+
)
35+
36+
funcTestPluginCache_Golden(t*testing.T) {
37+
t.Parallel()
38+
39+
prepare:=func() (afero.Fs, time.Time, slog.Logger) {
40+
fs:=afero.NewMemMapFs()
41+
now:=time.Date(2023,time.June,3,4,5,6,0,time.UTC)
42+
logger:=slogtest.Make(t,nil).
43+
Leveled(slog.LevelDebug).
44+
Named("cleanup-test")
45+
returnfs,now,logger
46+
}
47+
48+
t.Run("all plugins are stale",func(t*testing.T) {
49+
t.Parallel()
50+
51+
ctx,cancel:=context.WithTimeout(context.Background(),testutil.WaitShort)
52+
defercancel()
53+
54+
fs,now,logger:=prepare()
55+
56+
// given
57+
// This plugin is older than 30 days.
58+
addPluginFile(t,fs,coderPluginPath,"terraform-provider-coder_v0.11.1",now.Add(-63*24*time.Hour))
59+
addPluginFile(t,fs,coderPluginPath,"LICENSE",now.Add(-33*24*time.Hour))
60+
addPluginFile(t,fs,coderPluginPath,"README.md",now.Add(-31*24*time.Hour))
61+
addPluginFolder(t,fs,coderPluginPath,"new_folder",now.Add(-31*24*time.Hour))
62+
addPluginFile(t,fs,coderPluginPath,filepath.Join("new_folder","foobar.tf"),now.Add(-43*24*time.Hour))
63+
64+
// This plugin is older than 30 days.
65+
addPluginFile(t,fs,dockerPluginPath,"terraform-provider-docker_v2.25.0",now.Add(-31*24*time.Hour))
66+
addPluginFile(t,fs,dockerPluginPath,"LICENSE",now.Add(-32*24*time.Hour))
67+
addPluginFile(t,fs,dockerPluginPath,"README.md",now.Add(-33*24*time.Hour))
68+
69+
// when
70+
terraform.CleanStaleTerraformPlugins(ctx,cachePath,fs,now,logger)
71+
72+
// then
73+
diffFileSystem(t,fs)
74+
})
75+
76+
t.Run("one plugin is stale",func(t*testing.T) {
77+
t.Parallel()
78+
79+
ctx,cancel:=context.WithTimeout(context.Background(),testutil.WaitShort)
80+
defercancel()
81+
82+
fs,now,logger:=prepare()
83+
84+
// given
85+
addPluginFile(t,fs,coderPluginPath,"terraform-provider-coder_v0.11.1",now.Add(-2*time.Hour))
86+
addPluginFile(t,fs,coderPluginPath,"LICENSE",now.Add(-3*time.Hour))
87+
addPluginFile(t,fs,coderPluginPath,"README.md",now.Add(-4*time.Hour))
88+
addPluginFolder(t,fs,coderPluginPath,"new_folder",now.Add(-5*time.Hour))
89+
addPluginFile(t,fs,coderPluginPath,filepath.Join("new_folder","foobar.tf"),now.Add(-4*time.Hour))
90+
91+
// This plugin is older than 30 days.
92+
addPluginFile(t,fs,dockerPluginPath,"terraform-provider-docker_v2.25.0",now.Add(-31*24*time.Hour))
93+
addPluginFile(t,fs,dockerPluginPath,"LICENSE",now.Add(-32*24*time.Hour))
94+
addPluginFile(t,fs,dockerPluginPath,"README.md",now.Add(-33*24*time.Hour))
95+
96+
// when
97+
terraform.CleanStaleTerraformPlugins(ctx,cachePath,fs,now,logger)
98+
99+
// then
100+
diffFileSystem(t,fs)
101+
})
102+
103+
t.Run("one plugin file is touched",func(t*testing.T) {
104+
t.Parallel()
105+
106+
ctx,cancel:=context.WithTimeout(context.Background(),testutil.WaitShort)
107+
defercancel()
108+
109+
fs,now,logger:=prepare()
110+
111+
// given
112+
addPluginFile(t,fs,coderPluginPath,"terraform-provider-coder_v0.11.1",now.Add(-63*24*time.Hour))
113+
addPluginFile(t,fs,coderPluginPath,"LICENSE",now.Add(-33*24*time.Hour))
114+
addPluginFile(t,fs,coderPluginPath,"README.md",now.Add(-31*24*time.Hour))
115+
addPluginFolder(t,fs,coderPluginPath,"new_folder",now.Add(-4*time.Hour))// touched
116+
addPluginFile(t,fs,coderPluginPath,filepath.Join("new_folder","foobar.tf"),now.Add(-43*24*time.Hour))
117+
118+
addPluginFile(t,fs,dockerPluginPath,"terraform-provider-docker_v2.25.0",now.Add(-31*24*time.Hour))
119+
addPluginFile(t,fs,dockerPluginPath,"LICENSE",now.Add(-2*time.Hour))
120+
addPluginFile(t,fs,dockerPluginPath,"README.md",now.Add(-33*24*time.Hour))
121+
122+
// when
123+
terraform.CleanStaleTerraformPlugins(ctx,cachePath,fs,now,logger)
124+
125+
// then
126+
diffFileSystem(t,fs)
127+
})
128+
}
129+
130+
funcaddPluginFile(t*testing.T,fs afero.Fs,pluginPathstring,resourcePathstring,accessTime time.Time) {
131+
err:=fs.MkdirAll(filepath.Join(cachePath,pluginPath),0o755)
132+
require.NoError(t,err,"can't create test folder for plugin file")
133+
134+
err=fs.Chtimes(filepath.Join(cachePath,pluginPath),accessTime,accessTime)
135+
require.NoError(t,err,"can't set times")
136+
137+
err=afero.WriteFile(fs,filepath.Join(cachePath,pluginPath,resourcePath), []byte("foo"),0o644)
138+
require.NoError(t,err,"can't create test file")
139+
140+
err=fs.Chtimes(filepath.Join(cachePath,pluginPath,resourcePath),accessTime,accessTime)
141+
require.NoError(t,err,"can't set times")
142+
}
143+
144+
funcaddPluginFolder(t*testing.T,fs afero.Fs,pluginPathstring,folderPathstring,accessTime time.Time) {
145+
err:=fs.MkdirAll(filepath.Join(cachePath,pluginPath,folderPath),0o755)
146+
require.NoError(t,err,"can't create plugin folder")
147+
148+
err=fs.Chtimes(filepath.Join(cachePath,pluginPath,folderPath),accessTime,accessTime)
149+
require.NoError(t,err,"can't set times")
150+
}
151+
152+
funcdiffFileSystem(t*testing.T,fs afero.Fs) {
153+
actual:=dumpFileSystem(t,fs)
154+
155+
partialName:=strings.Join(strings.Split(t.Name(),"/")[1:],"_")
156+
goldenFile:=filepath.Join("testdata","cleanup-stale-plugins",partialName+".txt.golden")
157+
if*updateGoldenFiles {
158+
err:=os.MkdirAll(filepath.Dir(goldenFile),0o755)
159+
require.NoError(t,err,"want no error creating golden file directory")
160+
161+
err=os.WriteFile(goldenFile,actual,0o600)
162+
require.NoError(t,err,"want no error creating golden file")
163+
return
164+
}
165+
166+
want,err:=os.ReadFile(goldenFile)
167+
require.NoError(t,err,"open golden file, run\"make update-golden-files\" and commit the changes")
168+
assert.Empty(t,cmp.Diff(want,actual),"golden file mismatch (-want +got): %s, run\"make update-golden-files\", verify and commit the changes",goldenFile)
169+
}
170+
171+
funcdumpFileSystem(t*testing.T,fs afero.Fs) []byte {
172+
varbuffer bytes.Buffer
173+
err:=afero.Walk(fs,"/",func(pathstring,info os.FileInfo,errerror)error {
174+
_,_=buffer.WriteString(path)
175+
_=buffer.WriteByte(' ')
176+
ifinfo.IsDir() {
177+
_=buffer.WriteByte('d')
178+
}else {
179+
_=buffer.WriteByte('f')
180+
}
181+
_=buffer.WriteByte('\n')
182+
returnnil
183+
})
184+
require.NoError(t,err,"can't dump the file system")
185+
returnbuffer.Bytes()
186+
}

0 commit comments

Comments
 (0)

[8]ページ先頭

©2009-2025 Movatter.jp