Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitf11fea6

Browse files
committed
set up a terraform providers mirror for tests
1 parentf670bc3 commitf11fea6

File tree

4 files changed

+231
-34
lines changed

4 files changed

+231
-34
lines changed

‎provisioner/terraform/executor.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,9 @@ type executor struct {
3535
mut*sync.Mutex
3636
binaryPathstring
3737
// cachePath and workdir must not be used by multiple processes at once.
38-
cachePathstring
39-
workdirstring
38+
cachePathstring
39+
cliConfigPathstring
40+
workdirstring
4041
// used to capture execution times at various stages
4142
timings*timingAggregator
4243
}
@@ -50,6 +51,9 @@ func (e *executor) basicEnv() []string {
5051
ife.cachePath!=""&&runtime.GOOS=="linux" {
5152
env=append(env,"TF_PLUGIN_CACHE_DIR="+e.cachePath)
5253
}
54+
ife.cliConfigPath!="" {
55+
env=append(env,"TF_CLI_CONFIG_FILE="+e.cliConfigPath)
56+
}
5357
returnenv
5458
}
5559

‎provisioner/terraform/provision_test.go

Lines changed: 175 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,17 @@
33
package terraform_test
44

55
import (
6+
"bytes"
67
"context"
8+
"crypto/sha256"
9+
"encoding/hex"
710
"encoding/json"
811
"errors"
912
"fmt"
1013
"net"
1114
"net/http"
1215
"os"
16+
"os/exec"
1317
"path/filepath"
1418
"sort"
1519
"strings"
@@ -29,10 +33,11 @@ import (
2933
)
3034

3135
typeprovisionerServeOptionsstruct {
32-
binaryPathstring
33-
exitTimeout time.Duration
34-
workDirstring
35-
logger*slog.Logger
36+
binaryPathstring
37+
cliConfigPathstring
38+
exitTimeout time.Duration
39+
workDirstring
40+
logger*slog.Logger
3641
}
3742

3843
funcsetupProvisioner(t*testing.T,opts*provisionerServeOptions) (context.Context, proto.DRPCProvisionerClient) {
@@ -66,9 +71,10 @@ func setupProvisioner(t *testing.T, opts *provisionerServeOptions) (context.Cont
6671
Logger:*opts.logger,
6772
WorkDirectory:opts.workDir,
6873
},
69-
BinaryPath:opts.binaryPath,
70-
CachePath:cachePath,
71-
ExitTimeout:opts.exitTimeout,
74+
BinaryPath:opts.binaryPath,
75+
CachePath:cachePath,
76+
ExitTimeout:opts.exitTimeout,
77+
CliConfigPath:opts.cliConfigPath,
7278
})
7379
}()
7480
api:=proto.NewDRPCProvisionerClient(client)
@@ -85,6 +91,149 @@ func configure(ctx context.Context, t *testing.T, client proto.DRPCProvisionerCl
8591
returnsess
8692
}
8793

94+
funchashTemplateFilesAndTestName(t*testing.T,testNamestring,templateFilesmap[string]string)string {
95+
t.Helper()
96+
97+
sortedFileNames:=make([]string,0,len(templateFiles))
98+
forfileName:=rangetemplateFiles {
99+
sortedFileNames=append(sortedFileNames,fileName)
100+
}
101+
sort.Strings(sortedFileNames)
102+
103+
hasher:=sha256.New()
104+
for_,fileName:=rangesortedFileNames {
105+
file:=templateFiles[fileName]
106+
_,err:=hasher.Write([]byte(fileName))
107+
require.NoError(t,err)
108+
_,err=hasher.Write([]byte(file))
109+
require.NoError(t,err)
110+
}
111+
_,err:=hasher.Write([]byte(testName))
112+
require.NoError(t,err)
113+
returnhex.EncodeToString(hasher.Sum(nil))
114+
}
115+
116+
const (
117+
terraformConfigFileName="terraform.rc"
118+
cacheProvidersDirName="providers"
119+
cacheTemplateFilesDirName="files"
120+
)
121+
122+
// Writes a Terraform CLI config file (`terraform.rc`) in `dir` to enforce using the local provider mirror.
123+
// This blocks network access for providers, forcing Terraform to use only what's cached in `dir`.
124+
// Returns the path to the generated config file.
125+
funcwriteCliConfig(t*testing.T,dirstring)string {
126+
t.Helper()
127+
128+
cliConfigPath:=filepath.Join(dir,terraformConfigFileName)
129+
require.NoError(t,os.MkdirAll(filepath.Dir(cliConfigPath),0o700))
130+
131+
content:=fmt.Sprintf(`
132+
provider_installation {
133+
filesystem_mirror {
134+
path = "%s"
135+
include = ["*/*"]
136+
}
137+
direct {
138+
exclude = ["*/*"]
139+
}
140+
}
141+
`,filepath.Join(dir,cacheProvidersDirName))
142+
require.NoError(t,os.WriteFile(cliConfigPath, []byte(content),0o600))
143+
returncliConfigPath
144+
}
145+
146+
funcrunCmd(t*testing.T,dirstring,args...string) {
147+
t.Helper()
148+
149+
stdout,stderr:=bytes.NewBuffer(nil),bytes.NewBuffer(nil)
150+
cmd:=exec.Command(args[0],args[1:]...)//#nosec
151+
cmd.Dir=dir
152+
cmd.Stdout=stdout
153+
cmd.Stderr=stderr
154+
iferr:=cmd.Run();err!=nil {
155+
t.Fatalf("failed to run %s: %s\nstdout: %s\nstderr: %s",strings.Join(args," "),err,stdout.String(),stderr.String())
156+
}
157+
}
158+
159+
// Ensures Terraform providers are downloaded and cached locally in a unique directory for this test.
160+
// Uses `terraform init` then `mirror` to populate the cache if needed.
161+
// Returns the cache directory path.
162+
funcdownloadProviders(t*testing.T,rootDirstring,templateFilesmap[string]string)string {
163+
t.Helper()
164+
165+
// Each test gets a unique cache dir based on its name and template files.
166+
// This ensures that tests can download providers in parallel and that they
167+
// will redownload providers if the template files change.
168+
hash:=hashTemplateFilesAndTestName(t,t.Name(),templateFiles)
169+
dir:=filepath.Join(rootDir,hash[:12])
170+
if_,err:=os.Stat(dir);err==nil {
171+
t.Logf("%s: using cached terraform providers",t.Name())
172+
returndir
173+
}
174+
filesDir:=filepath.Join(dir,cacheTemplateFilesDirName)
175+
deferfunc() {
176+
// The files dir will contain a copy of terraform providers generated
177+
// by the terraform init command. We don't want to persist them since
178+
// we already have a registry mirror in the providers dir.
179+
iferr:=os.RemoveAll(filesDir);err!=nil {
180+
t.Logf("failed to remove files dir %s: %s",filesDir,err)
181+
}
182+
if!t.Failed() {
183+
return
184+
}
185+
iferr:=os.RemoveAll(dir);err!=nil {
186+
t.Logf("failed to remove dir %s: %s",dir,err)
187+
}
188+
}()
189+
190+
require.NoError(t,os.MkdirAll(filesDir,0o700))
191+
192+
forfileName,file:=rangetemplateFiles {
193+
filePath:=filepath.Join(filesDir,fileName)
194+
if_,err:=os.Stat(filePath);os.IsNotExist(err) {
195+
require.NoError(t,os.MkdirAll(filepath.Dir(filePath),0o700))
196+
require.NoError(t,os.WriteFile(filePath, []byte(file),0o600))
197+
}
198+
}
199+
200+
providersDir:=filepath.Join(dir,cacheProvidersDirName)
201+
require.NoError(t,os.MkdirAll(providersDir,0o700))
202+
203+
// We need to run init because if a test uses modules in its template,
204+
// the mirror command will fail without it.
205+
runCmd(t,filesDir,"terraform","init")
206+
// Now, mirror the providers into `providersDir`. We use this explicit mirror
207+
// instead of relying only on the standard Terraform plugin cache.
208+
//
209+
// Why? Because this mirror, when used with the CLI config from `writeCliConfig`,
210+
// prevents Terraform from hitting the network registry during `plan`. This cuts
211+
// down on network calls, making CI tests less flaky.
212+
//
213+
// In contrast, the standard cache *still* contacts the registry for metadata
214+
// during `init`, even if the plugins are already cached locally - see link below.
215+
//
216+
// Ref: https://developer.hashicorp.com/terraform/cli/config/config-file#provider-plugin-cache
217+
// > When a plugin cache directory is enabled, the terraform init command will
218+
// > still use the configured or implied installation methods to obtain metadata
219+
// > about which plugins are available
220+
runCmd(t,filesDir,"terraform","providers","mirror",providersDir)
221+
222+
returndir
223+
}
224+
225+
// Caches providers locally and generates a Terraform CLI config to use *only* that cache.
226+
// This setup prevents network access for providers during `terraform init`, improving reliability
227+
// in subsequent test runs.
228+
// Returns the path to the generated CLI config file.
229+
funccacheProviders(t*testing.T,templateFilesmap[string]string,rootDirstring)string {
230+
t.Helper()
231+
232+
providersParentDir:=downloadProviders(t,rootDir,templateFiles)
233+
cliConfigPath:=writeCliConfig(t,providersParentDir)
234+
returncliConfigPath
235+
}
236+
88237
funcreadProvisionLog(t*testing.T,response proto.DRPCProvisioner_SessionClient)string {
89238
varlogBuf strings.Builder
90239
for {
@@ -352,6 +501,8 @@ func TestProvision(t *testing.T) {
352501
Applybool
353502
// Some tests may need to be skipped until the relevant provider version is released.
354503
SkipReasonstring
504+
// If SkipCacheProviders is true, then skip caching the terraform providers for this test.
505+
SkipCacheProvidersbool
355506
}{
356507
{
357508
Name:"missing-variable",
@@ -422,16 +573,18 @@ func TestProvision(t *testing.T) {
422573
Files:map[string]string{
423574
"main.tf":`a`,
424575
},
425-
ErrorContains:"initialize terraform",
426-
ExpectLogContains:"Argument or block definition required",
576+
ErrorContains:"initialize terraform",
577+
ExpectLogContains:"Argument or block definition required",
578+
SkipCacheProviders:true,
427579
},
428580
{
429581
Name:"bad-syntax-2",
430582
Files:map[string]string{
431583
"main.tf":`;asdf;`,
432584
},
433-
ErrorContains:"initialize terraform",
434-
ExpectLogContains:`The ";" character is not valid.`,
585+
ErrorContains:"initialize terraform",
586+
ExpectLogContains:`The ";" character is not valid.`,
587+
SkipCacheProviders:true,
435588
},
436589
{
437590
Name:"destroy-no-state",
@@ -847,7 +1000,17 @@ func TestProvision(t *testing.T) {
8471000
t.Skip(testCase.SkipReason)
8481001
}
8491002

850-
ctx,api:=setupProvisioner(t,nil)
1003+
cliConfigPath:=""
1004+
if!testCase.SkipCacheProviders {
1005+
cliConfigPath=cacheProviders(
1006+
t,
1007+
testCase.Files,
1008+
filepath.Join(testutil.PersistentCacheDir(t),"terraform_provision_test"),
1009+
)
1010+
}
1011+
ctx,api:=setupProvisioner(t,&provisionerServeOptions{
1012+
cliConfigPath:cliConfigPath,
1013+
})
8511014
sess:=configure(ctx,t,api,&proto.Config{
8521015
TemplateSourceArchive:testutil.CreateTar(t,testCase.Files),
8531016
})

‎provisioner/terraform/serve.go

Lines changed: 25 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@ type ServeOptions struct {
2828
BinaryPathstring
2929
// CachePath must not be used by multiple processes at once.
3030
CachePathstring
31-
Tracer trace.Tracer
31+
// CliConfigPath is the path to the Terraform CLI config file.
32+
CliConfigPathstring
33+
Tracer trace.Tracer
3234

3335
// ExitTimeout defines how long we will wait for a running Terraform
3436
// command to exit (cleanly) if the provision was stopped. This
@@ -132,22 +134,24 @@ func Serve(ctx context.Context, options *ServeOptions) error {
132134
options.ExitTimeout=unhanger.HungJobExitTimeout
133135
}
134136
returnprovisionersdk.Serve(ctx,&server{
135-
execMut:&sync.Mutex{},
136-
binaryPath:options.BinaryPath,
137-
cachePath:options.CachePath,
138-
logger:options.Logger,
139-
tracer:options.Tracer,
140-
exitTimeout:options.ExitTimeout,
137+
execMut:&sync.Mutex{},
138+
binaryPath:options.BinaryPath,
139+
cachePath:options.CachePath,
140+
cliConfigPath:options.CliConfigPath,
141+
logger:options.Logger,
142+
tracer:options.Tracer,
143+
exitTimeout:options.ExitTimeout,
141144
},options.ServeOptions)
142145
}
143146

144147
typeserverstruct {
145-
execMut*sync.Mutex
146-
binaryPathstring
147-
cachePathstring
148-
logger slog.Logger
149-
tracer trace.Tracer
150-
exitTimeout time.Duration
148+
execMut*sync.Mutex
149+
binaryPathstring
150+
cachePathstring
151+
cliConfigPathstring
152+
logger slog.Logger
153+
tracer trace.Tracer
154+
exitTimeout time.Duration
151155
}
152156

153157
func (s*server)startTrace(ctx context.Context,namestring,opts...trace.SpanStartOption) (context.Context, trace.Span) {
@@ -158,12 +162,13 @@ func (s *server) startTrace(ctx context.Context, name string, opts ...trace.Span
158162

159163
func (s*server)executor(workdirstring,stage database.ProvisionerJobTimingStage)*executor {
160164
return&executor{
161-
server:s,
162-
mut:s.execMut,
163-
binaryPath:s.binaryPath,
164-
cachePath:s.cachePath,
165-
workdir:workdir,
166-
logger:s.logger.Named("executor"),
167-
timings:newTimingAggregator(stage),
165+
server:s,
166+
mut:s.execMut,
167+
binaryPath:s.binaryPath,
168+
cachePath:s.cachePath,
169+
cliConfigPath:s.cliConfigPath,
170+
workdir:workdir,
171+
logger:s.logger.Named("executor"),
172+
timings:newTimingAggregator(stage),
168173
}
169174
}

‎testutil/cache.go

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package testutil
2+
3+
import (
4+
"os"
5+
"path/filepath"
6+
"testing"
7+
8+
"github.com/stretchr/testify/require"
9+
)
10+
11+
// PersistentCacheDir returns a path to a directory
12+
// that will be cached between test runs in Github Actions.
13+
funcPersistentCacheDir(t*testing.T)string {
14+
t.Helper()
15+
16+
// We don't use os.UserCacheDir() because the path it
17+
// returns is different on different operating systems.
18+
// This would make it harder to specify which cache dir to use
19+
// in Github Actions.
20+
home,err:=os.UserHomeDir()
21+
require.NoError(t,err)
22+
dir:=filepath.Join(home,".cache","coderv2-test")
23+
24+
returndir
25+
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp