- Notifications
You must be signed in to change notification settings - Fork1.1k
feat: clean stale provisioner files#9545
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.
Already on GitHub?Sign in to your account
Merged
Uh oh!
There was an error while loading.Please reload this page.
Merged
Changes fromall commits
Commits
Show all changes
27 commits Select commitHold shift + click to select a range
277d7c2 TODOs
mtojekafb0982 Remove stale session directories
mtojek1624453 Fix lint
mtojek4bd9a56 log errors
mtojekd395d02 use djherbis/times
mtojeke2c71f8 Stub for cleanStaleTerraformPlugins
mtojek19741fc Review cached Terraform plugins
mtojek2af24c2 WIP access time
mtojek85a74cd WIP
mtojek20bd67c Implementation
mtojek905388b lint
mtojekcd0c134 more fixes
mtojek7c8ca9f polishing
mtojek574f084 move to cleanups
mtojek7aada74 fix stat
mtojekeabdacb fix comment
mtojek1b49d80 WIP use afero
mtojek44e0517 Some golden files
mtojek18c5a1d Polishing
mtojek2fdd756 Merge branch 'main' into 7107-clean-stale-files
mtojekb273cbf make fmt
mtojek6c45602 fix: windows
mtojekb35f7a7 Fix
mtojek1cd5fc2 filepath.Join
mtojek662c32b filepath.Join
mtojek4e6a09c oh do not run on windows
mtojeked2a8fd Merge branch 'main' into 7107-clean-stale-files
mtojekFile filter
Filter by extension
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Diff view
Diff view
There are no files selected for viewing
6 changes: 5 additions & 1 deletionMakefile
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
2 changes: 2 additions & 0 deletionsgo.mod
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -196,6 +196,8 @@ require ( | ||
| tailscale.comv1.46.1 | ||
| ) | ||
| requiregithub.com/djherbis/timesv1.5.0 | ||
Member There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others.Learn more. It's super annoying that this isn't exposed in the standard library. MemberAuthor There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others.Learn more. I did the self-evaluation of the module, and I agree with you. Fortunately, this is just a basic syscall, so in theory, we could "port" it to coder. | ||
| require ( | ||
| cloud.google.com/go/computev1.23.0// indirect | ||
| cloud.google.com/go/loggingv1.8.1// indirect | ||
2 changes: 2 additions & 0 deletionsgo.sum
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
153 changes: 153 additions & 0 deletionsprovisioner/terraform/cleanup.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,153 @@ | ||
| package terraform | ||
| import ( | ||
| "context" | ||
| "os" | ||
| "path/filepath" | ||
| "strings" | ||
| "time" | ||
| "github.com/djherbis/times" | ||
| "github.com/spf13/afero" | ||
| "golang.org/x/xerrors" | ||
| "cdr.dev/slog" | ||
| ) | ||
| // CleanStaleTerraformPlugins browses the Terraform cache directory | ||
| // and remove stale plugins that haven't been used for a while. | ||
| // Additionally, it sweeps empty, old directory trees. | ||
| // | ||
| // Sample cachePath: | ||
| // | ||
| ///Users/john.doe/Library/Caches/coder/provisioner-1/tf | ||
| ///tmp/coder/provisioner-0/tf | ||
| funcCleanStaleTerraformPlugins(ctx context.Context,cachePathstring,fs afero.Fs,now time.Time,logger slog.Logger)error { | ||
| cachePath,err:=filepath.Abs(cachePath)// sanity check in case the path is e.g. ../../../cache | ||
| iferr!=nil { | ||
| returnxerrors.Errorf("unable to determine absolute path %q: %w",cachePath,err) | ||
| } | ||
| // Firstly, check if the cache path exists. | ||
| _,err=fs.Stat(cachePath) | ||
| ifos.IsNotExist(err) { | ||
| returnnil | ||
| }elseiferr!=nil { | ||
| returnxerrors.Errorf("unable to stat cache path %q: %w",cachePath,err) | ||
| } | ||
| logger.Info(ctx,"clean stale Terraform plugins",slog.F("cache_path",cachePath)) | ||
| // Filter directory trees matching pattern: <repositoryURL>/<company>/<plugin>/<version>/<distribution> | ||
| filterFunc:=func(pathstring,info os.FileInfo)bool { | ||
| if!info.IsDir() { | ||
| returnfalse | ||
| } | ||
| relativePath,err:=filepath.Rel(cachePath,path) | ||
| iferr!=nil { | ||
| logger.Error(ctx,"unable to evaluate a relative path",slog.F("base",cachePath),slog.F("target",path),slog.Error(err)) | ||
| returnfalse | ||
| } | ||
| parts:=strings.Split(relativePath,string(filepath.Separator)) | ||
| returnlen(parts)==5 | ||
| } | ||
| // Review cached Terraform plugins | ||
| varpluginPaths []string | ||
| err=afero.Walk(fs,cachePath,func(pathstring,info os.FileInfo,errerror)error { | ||
| iferr!=nil { | ||
| returnerr | ||
| } | ||
| if!filterFunc(path,info) { | ||
| returnnil | ||
| } | ||
| logger.Debug(ctx,"plugin directory discovered",slog.F("path",path)) | ||
| pluginPaths=append(pluginPaths,path) | ||
| returnnil | ||
| }) | ||
| iferr!=nil { | ||
| returnxerrors.Errorf("unable to walk through cache directory %q: %w",cachePath,err) | ||
| } | ||
| // Identify stale plugins | ||
| varstalePlugins []string | ||
| for_,pluginPath:=rangepluginPaths { | ||
| accessTime,err:=latestAccessTime(fs,pluginPath) | ||
| iferr!=nil { | ||
| returnxerrors.Errorf("unable to evaluate latest access time for directory %q: %w",pluginPath,err) | ||
| } | ||
| ifaccessTime.Add(staleTerraformPluginRetention).Before(now) { | ||
| logger.Info(ctx,"plugin directory is stale and will be removed",slog.F("plugin_path",pluginPath)) | ||
| stalePlugins=append(stalePlugins,pluginPath) | ||
| }else { | ||
| logger.Debug(ctx,"plugin directory is not stale",slog.F("plugin_path",pluginPath)) | ||
| } | ||
| } | ||
| // Remove stale plugins | ||
| for_,stalePluginPath:=rangestalePlugins { | ||
| // Remove the plugin directory | ||
| err=fs.RemoveAll(stalePluginPath) | ||
| iferr!=nil { | ||
| returnxerrors.Errorf("unable to remove stale plugin %q: %w",stalePluginPath,err) | ||
| } | ||
| // Compact the plugin structure by removing empty directories. | ||
| wd:=stalePluginPath | ||
| level:=5// <repositoryURL>/<company>/<plugin>/<version>/<distribution> | ||
| for { | ||
| level-- | ||
| iflevel==0 { | ||
| break// do not compact further | ||
| } | ||
| wd=filepath.Dir(wd) | ||
| files,err:=afero.ReadDir(fs,wd) | ||
| iferr!=nil { | ||
| returnxerrors.Errorf("unable to read directory content %q: %w",wd,err) | ||
| } | ||
| iflen(files)>0 { | ||
| break// there are still other plugins | ||
| } | ||
| logger.Debug(ctx,"remove empty directory",slog.F("path",wd)) | ||
| err=fs.Remove(wd) | ||
| iferr!=nil { | ||
| returnxerrors.Errorf("unable to remove directory %q: %w",wd,err) | ||
| } | ||
| } | ||
| } | ||
| returnnil | ||
| } | ||
| // latestAccessTime walks recursively through the directory content, and locates | ||
| // the last accessed file. | ||
| funclatestAccessTime(fs afero.Fs,pluginPathstring) (time.Time,error) { | ||
| varlatest time.Time | ||
| err:=afero.Walk(fs,pluginPath,func(pathstring,info os.FileInfo,errerror)error { | ||
| iferr!=nil { | ||
| returnerr | ||
| } | ||
| accessTime:=info.ModTime()// fallback to modTime if accessTime is not available (afero) | ||
Member There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others.Learn more. Just be aware that modtime is not guaranteed (see:https://www.kernel.org/doc/html/latest/filesystems/api-summary.html?highlight=nocmtime#c.file_update_time) Howver, I wouldassume it to at least be the time the file was created. MemberAuthor There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others.Learn more. Yes, that is correct. I was thinking about running a self-test on the cache directory, but on the other hand I wouldn't like to overcomplicate things. | ||
| ifinfo.Sys()!=nil { | ||
| timeSpec:=times.Get(info) | ||
| accessTime=timeSpec.AccessTime() | ||
| } | ||
| iflatest.Before(accessTime) { | ||
| latest=accessTime | ||
| } | ||
| returnnil | ||
| }) | ||
| iferr!=nil { | ||
| return time.Time{},xerrors.Errorf("unable to walk the plugin path %q: %w",pluginPath,err) | ||
| } | ||
| returnlatest,nil | ||
| } | ||
186 changes: 186 additions & 0 deletionsprovisioner/terraform/cleanup_test.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,186 @@ | ||
| //go:build linux || darwin | ||
| package terraform_test | ||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "flag" | ||
| "os" | ||
| "path/filepath" | ||
| "strings" | ||
| "testing" | ||
| "time" | ||
| "github.com/google/go-cmp/cmp" | ||
| "github.com/spf13/afero" | ||
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/require" | ||
| "cdr.dev/slog" | ||
| "cdr.dev/slog/sloggers/slogtest" | ||
| "github.com/coder/coder/v2/provisioner/terraform" | ||
| "github.com/coder/coder/v2/testutil" | ||
| ) | ||
| const cachePath = "/tmp/coder/provisioner-0/tf" | ||
| // updateGoldenFiles is a flag that can be set to update golden files. | ||
| var updateGoldenFiles = flag.Bool("update", false, "Update golden files") | ||
| var ( | ||
| coderPluginPath = filepath.Join("registry.terraform.io", "coder", "coder", "0.11.1", "darwin_arm64") | ||
| dockerPluginPath = filepath.Join("registry.terraform.io", "kreuzwerker", "docker", "2.25.0", "darwin_arm64") | ||
| ) | ||
| func TestPluginCache_Golden(t *testing.T) { | ||
| t.Parallel() | ||
| prepare := func() (afero.Fs, time.Time, slog.Logger) { | ||
| fs := afero.NewMemMapFs() | ||
| now := time.Date(2023, time.June, 3, 4, 5, 6, 0, time.UTC) | ||
| logger := slogtest.Make(t, nil). | ||
| Leveled(slog.LevelDebug). | ||
| Named("cleanup-test") | ||
| return fs, now, logger | ||
| } | ||
| t.Run("all plugins are stale", func(t *testing.T) { | ||
| t.Parallel() | ||
| ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort) | ||
| defer cancel() | ||
| fs, now, logger := prepare() | ||
| // given | ||
| // This plugin is older than 30 days. | ||
| addPluginFile(t, fs, coderPluginPath, "terraform-provider-coder_v0.11.1", now.Add(-63*24*time.Hour)) | ||
| addPluginFile(t, fs, coderPluginPath, "LICENSE", now.Add(-33*24*time.Hour)) | ||
| addPluginFile(t, fs, coderPluginPath, "README.md", now.Add(-31*24*time.Hour)) | ||
| addPluginFolder(t, fs, coderPluginPath, "new_folder", now.Add(-31*24*time.Hour)) | ||
| addPluginFile(t, fs, coderPluginPath, filepath.Join("new_folder", "foobar.tf"), now.Add(-43*24*time.Hour)) | ||
| // This plugin is older than 30 days. | ||
| addPluginFile(t, fs, dockerPluginPath, "terraform-provider-docker_v2.25.0", now.Add(-31*24*time.Hour)) | ||
| addPluginFile(t, fs, dockerPluginPath, "LICENSE", now.Add(-32*24*time.Hour)) | ||
| addPluginFile(t, fs, dockerPluginPath, "README.md", now.Add(-33*24*time.Hour)) | ||
| // when | ||
| terraform.CleanStaleTerraformPlugins(ctx, cachePath, fs, now, logger) | ||
| // then | ||
| diffFileSystem(t, fs) | ||
| }) | ||
| t.Run("one plugin is stale", func(t *testing.T) { | ||
| t.Parallel() | ||
| ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort) | ||
| defer cancel() | ||
| fs, now, logger := prepare() | ||
| // given | ||
| addPluginFile(t, fs, coderPluginPath, "terraform-provider-coder_v0.11.1", now.Add(-2*time.Hour)) | ||
| addPluginFile(t, fs, coderPluginPath, "LICENSE", now.Add(-3*time.Hour)) | ||
| addPluginFile(t, fs, coderPluginPath, "README.md", now.Add(-4*time.Hour)) | ||
| addPluginFolder(t, fs, coderPluginPath, "new_folder", now.Add(-5*time.Hour)) | ||
| addPluginFile(t, fs, coderPluginPath, filepath.Join("new_folder", "foobar.tf"), now.Add(-4*time.Hour)) | ||
| // This plugin is older than 30 days. | ||
| addPluginFile(t, fs, dockerPluginPath, "terraform-provider-docker_v2.25.0", now.Add(-31*24*time.Hour)) | ||
| addPluginFile(t, fs, dockerPluginPath, "LICENSE", now.Add(-32*24*time.Hour)) | ||
| addPluginFile(t, fs, dockerPluginPath, "README.md", now.Add(-33*24*time.Hour)) | ||
| // when | ||
| terraform.CleanStaleTerraformPlugins(ctx, cachePath, fs, now, logger) | ||
| // then | ||
| diffFileSystem(t, fs) | ||
| }) | ||
| t.Run("one plugin file is touched", func(t *testing.T) { | ||
| t.Parallel() | ||
| ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort) | ||
| defer cancel() | ||
| fs, now, logger := prepare() | ||
| // given | ||
| addPluginFile(t, fs, coderPluginPath, "terraform-provider-coder_v0.11.1", now.Add(-63*24*time.Hour)) | ||
| addPluginFile(t, fs, coderPluginPath, "LICENSE", now.Add(-33*24*time.Hour)) | ||
| addPluginFile(t, fs, coderPluginPath, "README.md", now.Add(-31*24*time.Hour)) | ||
| addPluginFolder(t, fs, coderPluginPath, "new_folder", now.Add(-4*time.Hour)) // touched | ||
| addPluginFile(t, fs, coderPluginPath, filepath.Join("new_folder", "foobar.tf"), now.Add(-43*24*time.Hour)) | ||
| addPluginFile(t, fs, dockerPluginPath, "terraform-provider-docker_v2.25.0", now.Add(-31*24*time.Hour)) | ||
| addPluginFile(t, fs, dockerPluginPath, "LICENSE", now.Add(-2*time.Hour)) | ||
| addPluginFile(t, fs, dockerPluginPath, "README.md", now.Add(-33*24*time.Hour)) | ||
| // when | ||
| terraform.CleanStaleTerraformPlugins(ctx, cachePath, fs, now, logger) | ||
| // then | ||
| diffFileSystem(t, fs) | ||
| }) | ||
| } | ||
| func addPluginFile(t *testing.T, fs afero.Fs, pluginPath string, resourcePath string, accessTime time.Time) { | ||
| err := fs.MkdirAll(filepath.Join(cachePath, pluginPath), 0o755) | ||
| require.NoError(t, err, "can't create test folder for plugin file") | ||
| err = fs.Chtimes(filepath.Join(cachePath, pluginPath), accessTime, accessTime) | ||
| require.NoError(t, err, "can't set times") | ||
| err = afero.WriteFile(fs, filepath.Join(cachePath, pluginPath, resourcePath), []byte("foo"), 0o644) | ||
| require.NoError(t, err, "can't create test file") | ||
| err = fs.Chtimes(filepath.Join(cachePath, pluginPath, resourcePath), accessTime, accessTime) | ||
| require.NoError(t, err, "can't set times") | ||
| } | ||
| func addPluginFolder(t *testing.T, fs afero.Fs, pluginPath string, folderPath string, accessTime time.Time) { | ||
| err := fs.MkdirAll(filepath.Join(cachePath, pluginPath, folderPath), 0o755) | ||
| require.NoError(t, err, "can't create plugin folder") | ||
| err = fs.Chtimes(filepath.Join(cachePath, pluginPath, folderPath), accessTime, accessTime) | ||
| require.NoError(t, err, "can't set times") | ||
| } | ||
| func diffFileSystem(t *testing.T, fs afero.Fs) { | ||
| actual := dumpFileSystem(t, fs) | ||
| partialName := strings.Join(strings.Split(t.Name(), "/")[1:], "_") | ||
| goldenFile := filepath.Join("testdata", "cleanup-stale-plugins", partialName+".txt.golden") | ||
| if *updateGoldenFiles { | ||
| err := os.MkdirAll(filepath.Dir(goldenFile), 0o755) | ||
| require.NoError(t, err, "want no error creating golden file directory") | ||
| err = os.WriteFile(goldenFile, actual, 0o600) | ||
| require.NoError(t, err, "want no error creating golden file") | ||
| return | ||
| } | ||
| want, err := os.ReadFile(goldenFile) | ||
| require.NoError(t, err, "open golden file, run \"make update-golden-files\" and commit the changes") | ||
| assert.Empty(t, cmp.Diff(want, actual), "golden file mismatch (-want +got): %s, run \"make update-golden-files\", verify and commit the changes", goldenFile) | ||
| } | ||
| func dumpFileSystem(t *testing.T, fs afero.Fs) []byte { | ||
| var buffer bytes.Buffer | ||
| err := afero.Walk(fs, "/", func(path string, info os.FileInfo, err error) error { | ||
| _, _ = buffer.WriteString(path) | ||
| _ = buffer.WriteByte(' ') | ||
| if info.IsDir() { | ||
| _ = buffer.WriteByte('d') | ||
| } else { | ||
| _ = buffer.WriteByte('f') | ||
| } | ||
| _ = buffer.WriteByte('\n') | ||
| return nil | ||
| }) | ||
| require.NoError(t, err, "can't dump the file system") | ||
| return buffer.Bytes() | ||
| } |
Oops, something went wrong.
Uh oh!
There was an error while loading.Please reload this page.
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.