Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit158243d

Browse files
authored
fix: add cache for terraform installer files (#20776)
Replaces not working mocks by simple proxy that caches terraform files using testcachehttps://github.com/coder/coder/blob/16b8e6072fd84f45404e3f84bb2b6fea2424b090/testutil/cache.go#L13Fixes:coder/internal#1126
1 parenteb64473 commit158243d

File tree

3 files changed

+77
-154
lines changed

3 files changed

+77
-154
lines changed

‎provisioner/terraform/install.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ var (
3434
// operation.
3535
//
3636
//nolint:revive // verbose is a control flag that controls the verbosity of the log output.
37-
funcInstall(ctx context.Context,log slog.Logger,verbosebool,dirstring,wantVersion*version.Version,baseUrlstring,verifyChecksumsbool) (string,error) {
37+
funcInstall(ctx context.Context,log slog.Logger,verbosebool,dirstring,wantVersion*version.Version,baseUrlstring) (string,error) {
3838
err:=os.MkdirAll(dir,0o750)
3939
iferr!=nil {
4040
return"",err
@@ -63,10 +63,9 @@ func Install(ctx context.Context, log slog.Logger, verbose bool, dir string, wan
6363
}
6464

6565
installer:=&releases.ExactVersion{
66-
InstallDir:dir,
67-
Product:product.Terraform,
68-
Version:TerraformVersion,
69-
SkipChecksumVerification:!verifyChecksums,
66+
InstallDir:dir,
67+
Product:product.Terraform,
68+
Version:TerraformVersion,
7069
}
7170
installer.SetLogger(slog.Stdlib(ctx,log,slog.LevelDebug))
7271
ifbaseUrl!="" {

‎provisioner/terraform/install_test.go‎

Lines changed: 72 additions & 148 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,12 @@
66
package terraform_test
77

88
import (
9-
"archive/zip"
109
"context"
11-
"encoding/json"
12-
"fmt"
10+
"errors"
11+
"io"
1312
"net"
1413
"net/http"
14+
"net/url"
1515
"os"
1616
"path/filepath"
1717
"strings"
@@ -28,172 +28,93 @@ import (
2828
)
2929

3030
const (
31-
// simple script that mocks `./terraform version -json`
32-
terraformExecutableTemplate=`#!/bin/bash
33-
cat <<EOF
34-
{
35-
"terraform_version": "${ver}",
36-
"platform": "linux_amd64",
37-
"provider_selections": {},
38-
"terraform_outdated": true
39-
}
40-
EOF
41-
`
31+
cacheSubDir="terraform_install_test"
32+
terraformURL="https://releases.hashicorp.com"
4233
)
4334

4435
var (
4536
version1=terraform.TerraformVersion
4637
version2=version.Must(version.NewVersion("1.2.0"))
4738
)
4839

49-
typeproductBuildstruct {
50-
Namestring`json:"name"`
51-
Versionstring`json:"version"`
52-
OSstring`json:"os"`
53-
Archstring`json:"arch"`
54-
Filenamestring`json:"filename"`
55-
URLstring`json:"url"`
56-
}
57-
58-
typeproductVersionstruct {
59-
Namestring`json:"name"`
60-
Version*version.Version`json:"version"`
61-
Builds []productBuild`json:"builds"`
62-
}
63-
64-
typeproductstruct {
65-
Namestring`json:"name"`
66-
Versionsmap[string]productVersion`json:"versions"`
40+
typeterraformProxystruct {
41+
t*testing.T
42+
cacheRootstring
43+
listener net.Listener
44+
srv*http.Server
45+
fsHandler http.Handler
46+
httpClient*http.Client
47+
mutex*sync.Mutex
6748
}
6849

69-
funczipFilename(v*version.Version)string {
70-
returnfmt.Sprintf("terraform_%s_linux_amd64.zip",v)
71-
}
72-
73-
// returns `/${version}/index.json` in struct format
74-
funcversionedJSON(v*version.Version)productVersion {
75-
returnproductVersion{
76-
Name:"terraform",
77-
Version:v,
78-
Builds: []productBuild{
79-
{
80-
Arch:"amd64",
81-
Filename:zipFilename(v),
82-
Name:"terraform",
83-
OS:"linux",
84-
URL:fmt.Sprintf("/terraform/%s/%s",v,zipFilename(v)),
85-
Version:v.String(),
86-
},
87-
},
50+
// Simple cached proxy for terraform files.
51+
// Serves files from persistent cache or forwards requests to releases.hashicorp.com
52+
// Modifies downloaded index.json files so they point to proxy.
53+
funcpersistentlyCachedProxy(t*testing.T)*terraformProxy {
54+
cacheRoot:=filepath.Join(testutil.PersistentCacheDir(t),cacheSubDir)
55+
proxy:=terraformProxy{
56+
t:t,
57+
mutex:&sync.Mutex{},
58+
cacheRoot:cacheRoot,
59+
fsHandler:http.FileServer(http.Dir(cacheRoot)),
60+
httpClient:&http.Client{},
8861
}
89-
}
9062

91-
// returns `/index.json` in struct format
92-
funcmainJSON(versions...*version.Version)product {
93-
vj:=map[string]productVersion{}
94-
for_,v:=rangeversions {
95-
vj[v.String()]=versionedJSON(v)
96-
}
97-
mj:=product{
98-
Name:"terraform",
99-
Versions:vj,
63+
listener,err:=net.Listen("tcp","127.0.0.1:0")
64+
iferr!=nil {
65+
t.Fatalf("failed to create listener")
10066
}
101-
returnmj
102-
}
67+
proxy.listener=listener
10368

104-
funcexeContent(v*version.Version) []byte {
105-
return []byte(strings.ReplaceAll(terraformExecutableTemplate,"${ver}",v.String()))
106-
}
69+
m:=http.NewServeMux()
70+
m.HandleFunc("GET /",proxy.handleGet)
10771

108-
funcmustMarshal(t*testing.T,objany) []byte {
109-
b,err:=json.Marshal(obj)
110-
require.NoError(t,err)
111-
returnb
72+
proxy.srv=&http.Server{
73+
WriteTimeout:30*time.Second,
74+
ReadTimeout:30*time.Second,
75+
Handler:m,
76+
}
77+
return&proxy
11278
}
11379

114-
// Mock files are based on https://releases.hashicorp.com/terraform
115-
// mock directory structure:
116-
//
117-
//${tmpDir}/index.json
118-
//${tmpDir}/${version}/index.json
119-
//${tmpDir}/${version}/terraform_${version}_linux_amd64.zip
120-
// -> zip contains 'terraform' binary and sometimes 'LICENSE.txt'
121-
funccreateFakeTerraformInstallationFiles(t*testing.T)string {
122-
tmpDir:=t.TempDir()
123-
124-
mij:=mustMarshal(t,mainJSON(version1,version2))
125-
jv1:=mustMarshal(t,versionedJSON(version1))
126-
jv2:=mustMarshal(t,versionedJSON(version2))
127-
128-
// `index.json`
129-
require.NoError(t,os.WriteFile(filepath.Join(tmpDir,"index.json"),mij,0o400))
130-
131-
// `${version1}/index.json`
132-
require.NoError(t,os.Mkdir(filepath.Join(tmpDir,version1.String()),0o700))
133-
require.NoError(t,os.WriteFile(filepath.Join(tmpDir,version1.String(),"index.json"),jv1,0o400))
134-
135-
// `${version2}/index.json`
136-
require.NoError(t,os.Mkdir(filepath.Join(tmpDir,version2.String()),0o700))
137-
require.NoError(t,os.WriteFile(filepath.Join(tmpDir,version2.String(),"index.json"),jv2,0o400))
138-
139-
// `${version1}/linux_amd64.zip`
140-
zip1,err:=os.Create(filepath.Join(tmpDir,version1.String(),zipFilename(version1)))
141-
require.NoError(t,err)
142-
zip1Writer:=zip.NewWriter(zip1)
80+
funcuriToFilename(u url.URL)string {
81+
returnstrings.ReplaceAll(u.RequestURI(),"/","_")
82+
}
14383

144-
// `${version1}/linux_amd64.zip/terraform`
145-
exe1,err:=zip1Writer.Create("terraform")
146-
require.NoError(t,err)
147-
n,err:=exe1.Write(exeContent(version1))
148-
require.NoError(t,err)
149-
require.NotZero(t,n)
84+
func (p*terraformProxy)handleGet(w http.ResponseWriter,r*http.Request) {
85+
p.mutex.Lock()
86+
deferp.mutex.Unlock()
15087

151-
// `${version1}/linux_amd64.zip/LICENSE.txt`
152-
lic1,err:=zip1Writer.Create("LICENSE.txt")
153-
require.NoError(t,err)
154-
n,err=lic1.Write([]byte("some license"))
155-
require.NoError(t,err)
156-
require.NotZero(t,n)
157-
require.NoError(t,zip1Writer.Close())
88+
filename:=uriToFilename(*r.URL)
89+
path:=filepath.Join(p.cacheRoot,filename)
90+
if_,err:=os.Stat(path);errors.Is(err,os.ErrNotExist) {
91+
require.NoError(p.t,os.MkdirAll(p.cacheRoot,os.ModeDir|0o700))
15892

159-
// `${version2}/linux_amd64.zip`
160-
zip2,err:=os.Create(filepath.Join(tmpDir,version2.String(),zipFilename(version2)))
161-
require.NoError(t,err)
162-
zip2Writer:=zip.NewWriter(zip2)
93+
// Update cache
94+
req,err:=http.NewRequestWithContext(p.t.Context(),"GET",terraformURL+r.URL.Path,nil)
95+
require.NoError(p.t,err)
16396

164-
// `${version1}/linux_amd64.zip/terraform`
165-
exe2,err:=zip2Writer.Create("terraform")
166-
require.NoError(t,err)
167-
n,err=exe2.Write(exeContent(version2))
168-
require.NoError(t,err)
169-
require.NotZero(t,n)
170-
require.NoError(t,zip2Writer.Close())
97+
resp,err:=p.httpClient.Do(req)
98+
require.NoError(p.t,err)
99+
deferresp.Body.Close()
171100

172-
returntmpDir
173-
}
101+
body,err:=io.ReadAll(resp.Body)
102+
require.NoError(p.t,err)
174103

175-
// starts http server serving fake terraform installation files
176-
funcstartFakeTerraformServer(t*testing.T,tmpDirstring)string {
177-
listener,err:=net.Listen("tcp","127.0.0.1:0")
178-
iferr!=nil {
179-
t.Fatalf("failed to create listener")
104+
// update index.json so urls in it point to proxy by making them relative
105+
// "https://releases.hashicorp.com/terraform/1.13.4/terraform_1.13.4_windows_amd64.zip" -> "/terraform/1.13.4/terraform_1.13.4_windows_amd64.zip"
106+
ifstrings.HasSuffix(r.URL.Path,"index.json") {
107+
body= []byte(strings.ReplaceAll(string(body),terraformURL,""))
108+
}
109+
require.NoError(p.t,os.WriteFile(path,body,0o400))
110+
}elseiferr!=nil {
111+
p.t.Errorf("unexpected error when trying to read file from cache: %v",err)
180112
}
181113

182-
mux:=http.NewServeMux()
183-
fs:=http.FileServer(http.Dir(tmpDir))
184-
mux.Handle("/terraform/",http.StripPrefix("/terraform",fs))
185-
186-
srv:= http.Server{
187-
ReadHeaderTimeout:time.Second,
188-
Handler:mux,
189-
}
190-
gosrv.Serve(listener)
191-
t.Cleanup(func() {
192-
iferr:=srv.Close();err!=nil {
193-
t.Errorf("failed to close server: %v",err)
194-
}
195-
})
196-
return"http://"+listener.Addr().String()
114+
// Serve from cache
115+
r.URL.Path=filename
116+
r.URL.RawPath=filename
117+
p.fsHandler.ServeHTTP(w,r)
197118
}
198119

199120
funcTestInstall(t*testing.T) {
@@ -205,8 +126,11 @@ func TestInstall(t *testing.T) {
205126
dir:=t.TempDir()
206127
log:=testutil.Logger(t)
207128

208-
tmpDir:=createFakeTerraformInstallationFiles(t)
209-
addr:=startFakeTerraformServer(t,tmpDir)
129+
proxy:=persistentlyCachedProxy(t)
130+
goproxy.srv.Serve(proxy.listener)
131+
t.Cleanup(func() {
132+
require.NoError(t,proxy.srv.Close())
133+
})
210134

211135
// Install spins off 8 installs with Version and waits for them all
212136
// to complete. The locking mechanism within Install should
@@ -219,7 +143,7 @@ func TestInstall(t *testing.T) {
219143
wg.Add(1)
220144
gofunc() {
221145
deferwg.Done()
222-
p,err:=terraform.Install(ctx,log,false,dir,version,addr,false)
146+
p,err:=terraform.Install(ctx,log,false,dir,version,"http://"+proxy.listener.Addr().String())
223147
assert.NoError(t,err)
224148
paths<-p
225149
}()

‎provisioner/terraform/serve.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ func Serve(ctx context.Context, options *ServeOptions) error {
103103
slog.F("min_version",minTerraformVersion.String()))
104104
}
105105

106-
binPath,err:=Install(ctx,options.Logger,options.ExternalProvisioner,options.CachePath,TerraformVersion,"",true)
106+
binPath,err:=Install(ctx,options.Logger,options.ExternalProvisioner,options.CachePath,TerraformVersion,"")
107107
iferr!=nil {
108108
returnxerrors.Errorf("install terraform: %w",err)
109109
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp