Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitb19d644

Browse files
authored
feat: add etag to slim binaries endpoint (#5750)
1 parentc377cd0 commitb19d644

File tree

3 files changed

+222
-58
lines changed

3 files changed

+222
-58
lines changed

‎coderd/coderd.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ func New(options *Options) *API {
197197
ifsiteCacheDir!="" {
198198
siteCacheDir=filepath.Join(siteCacheDir,"site")
199199
}
200-
binFS,err:=site.ExtractOrReadBinFS(siteCacheDir,site.FS())
200+
binFS,binHashes,err:=site.ExtractOrReadBinFS(siteCacheDir,site.FS())
201201
iferr!=nil {
202202
panic(xerrors.Errorf("read site bin failed: %w",err))
203203
}
@@ -213,7 +213,7 @@ func New(options *Options) *API {
213213
ID:uuid.New(),
214214
Options:options,
215215
RootHandler:r,
216-
siteHandler:site.Handler(site.FS(),binFS),
216+
siteHandler:site.Handler(site.FS(),binFS,binHashes),
217217
HTTPAuth:&HTTPAuthorizer{
218218
Authorizer:options.Authorizer,
219219
Logger:options.Logger,

‎site/site.go

Lines changed: 162 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"path"
1717
"path/filepath"
1818
"strings"
19+
"sync"
1920
"text/template"// html/template escapes some nonces
2021
"time"
2122

@@ -24,6 +25,7 @@ import (
2425
"github.com/unrolled/secure"
2526
"golang.org/x/exp/slices"
2627
"golang.org/x/sync/errgroup"
28+
"golang.org/x/sync/singleflight"
2729
"golang.org/x/xerrors"
2830

2931
"github.com/coder/coder/coderd/httpapi"
@@ -48,7 +50,7 @@ func init() {
4850
}
4951

5052
// Handler returns an HTTP handler for serving the static site.
51-
funcHandler(siteFS fs.FS,binFS http.FileSystem) http.Handler {
53+
funcHandler(siteFS fs.FS,binFS http.FileSystem,binHashesmap[string]string) http.Handler {
5254
// html files are handled by a text/template. Non-html files
5355
// are served by the default file server.
5456
//
@@ -59,13 +61,43 @@ func Handler(siteFS fs.FS, binFS http.FileSystem) http.Handler {
5961
panic(xerrors.Errorf("Failed to return handler for static files. Html files failed to load: %w",err))
6062
}
6163

64+
binHashCache:=newBinHashCache(binFS,binHashes)
65+
6266
mux:=http.NewServeMux()
6367
mux.Handle("/bin/",http.StripPrefix("/bin",http.HandlerFunc(func(rw http.ResponseWriter,r*http.Request) {
6468
// Convert underscores in the filename to hyphens. We eventually want to
6569
// change our hyphen-based filenames to underscores, but we need to
6670
// support both for now.
6771
r.URL.Path=strings.ReplaceAll(r.URL.Path,"_","-")
6872

73+
// Set ETag header to the SHA1 hash of the file contents.
74+
name:=filePath(r.URL.Path)
75+
ifname==""||name=="/" {
76+
// Serve the directory listing.
77+
http.FileServer(binFS).ServeHTTP(rw,r)
78+
return
79+
}
80+
ifstrings.Contains(name,"/") {
81+
// We only serve files from the root of this directory, so avoid any
82+
// shenanigans by blocking slashes in the URL path.
83+
http.NotFound(rw,r)
84+
return
85+
}
86+
hash,err:=binHashCache.getHash(name)
87+
ifxerrors.Is(err,os.ErrNotExist) {
88+
http.NotFound(rw,r)
89+
return
90+
}
91+
iferr!=nil {
92+
http.Error(rw,err.Error(),http.StatusInternalServerError)
93+
return
94+
}
95+
96+
// ETag header needs to be quoted.
97+
rw.Header().Set("ETag",fmt.Sprintf(`%q`,hash))
98+
99+
// http.FileServer will see the ETag header and automatically handle
100+
// If-Match and If-None-Match headers on the request properly.
69101
http.FileServer(binFS).ServeHTTP(rw,r)
70102
})))
71103
mux.Handle("/",http.FileServer(http.FS(siteFS)))// All other non-html static files.
@@ -409,20 +441,23 @@ func htmlFiles(files fs.FS) (*htmlTemplates, error) {
409441
},nil
410442
}
411443

412-
// ExtractOrReadBinFS checks the provided fs for compressed coder
413-
// binaries and extracts them into dest/bin if found. As a fallback,
414-
// the provided FS is checked for a /bin directory, if it is non-empty
415-
// it is returned. Finally dest/bin is returned as a fallback allowing
416-
// binaries to be manually placed in dest (usually
417-
// ${CODER_CACHE_DIRECTORY}/site/bin).
418-
funcExtractOrReadBinFS(deststring,siteFS fs.FS) (http.FileSystem,error) {
444+
// ExtractOrReadBinFS checks the provided fs for compressed coder binaries and
445+
// extracts them into dest/bin if found. As a fallback, the provided FS is
446+
// checked for a /bin directory, if it is non-empty it is returned. Finally
447+
// dest/bin is returned as a fallback allowing binaries to be manually placed in
448+
// dest (usually ${CODER_CACHE_DIRECTORY}/site/bin).
449+
//
450+
// Returns a http.FileSystem that serves unpacked binaries, and a map of binary
451+
// name to SHA1 hash. The returned hash map may be incomplete or contain hashes
452+
// for missing files.
453+
funcExtractOrReadBinFS(deststring,siteFS fs.FS) (http.FileSystem,map[string]string,error) {
419454
ifdest=="" {
420455
// No destination on fs, embedded fs is the only option.
421456
binFS,err:=fs.Sub(siteFS,"bin")
422457
iferr!=nil {
423-
returnnil,xerrors.Errorf("cache path is empty and embedded fs does not have /bin: %w",err)
458+
returnnil,nil,xerrors.Errorf("cache path is empty and embedded fs does not have /bin: %w",err)
424459
}
425-
returnhttp.FS(binFS),nil
460+
returnhttp.FS(binFS),nil,nil
426461
}
427462

428463
dest=filepath.Join(dest,"bin")
@@ -440,51 +475,63 @@ func ExtractOrReadBinFS(dest string, siteFS fs.FS) (http.FileSystem, error) {
440475
files,err:=fs.ReadDir(siteFS,"bin")
441476
iferr!=nil {
442477
ifxerrors.Is(err,fs.ErrNotExist) {
443-
// Given fs does not have a bin directory,
444-
// serve from cache directory.
445-
returnmkdest()
478+
// Given fs does not have a bin directory, serve from cache
479+
// directory without extracting anything.
480+
binFS,err:=mkdest()
481+
iferr!=nil {
482+
returnnil,nil,xerrors.Errorf("mkdest failed: %w",err)
483+
}
484+
returnbinFS,map[string]string{},nil
446485
}
447-
returnnil,xerrors.Errorf("site fs read dir failed: %w",err)
486+
returnnil,nil,xerrors.Errorf("site fs read dir failed: %w",err)
448487
}
449488

450489
iflen(filterFiles(files,"GITKEEP"))>0 {
451-
// If there are other files than bin/GITKEEP,
452-
// serve the files.
490+
// If there are other files than bin/GITKEEP, serve the files.
453491
binFS,err:=fs.Sub(siteFS,"bin")
454492
iferr!=nil {
455-
returnnil,xerrors.Errorf("site fs sub dir failed: %w",err)
493+
returnnil,nil,xerrors.Errorf("site fs sub dir failed: %w",err)
456494
}
457-
returnhttp.FS(binFS),nil
495+
returnhttp.FS(binFS),nil,nil
458496
}
459497

460-
// Nothing we can do, serve the cache directory,
461-
// thus allowing binaries to be places there.
462-
returnmkdest()
498+
// Nothing we can do, serve the cache directory, thus allowing
499+
// binaries to be placed there.
500+
binFS,err:=mkdest()
501+
iferr!=nil {
502+
returnnil,nil,xerrors.Errorf("mkdest failed: %w",err)
503+
}
504+
returnbinFS,map[string]string{},nil
463505
}
464-
returnnil,xerrors.Errorf("open coder binary archive failed: %w",err)
506+
returnnil,nil,xerrors.Errorf("open coder binary archive failed: %w",err)
465507
}
466508
deferarchive.Close()
467509

468-
dir,err:=mkdest()
510+
binFS,err:=mkdest()
469511
iferr!=nil {
470-
returnnil,err
512+
returnnil,nil,err
513+
}
514+
515+
shaFiles,err:=parseSHA1(siteFS)
516+
iferr!=nil {
517+
returnnil,nil,xerrors.Errorf("parse sha1 file failed: %w",err)
471518
}
472519

473-
ok,err:=verifyBinSha1IsCurrent(dest,siteFS)
520+
ok,err:=verifyBinSha1IsCurrent(dest,siteFS,shaFiles)
474521
iferr!=nil {
475-
returnnil,xerrors.Errorf("verify coder binaries sha1 failed: %w",err)
522+
returnnil,nil,xerrors.Errorf("verify coder binaries sha1 failed: %w",err)
476523
}
477524
if!ok {
478525
n,err:=extractBin(dest,archive)
479526
iferr!=nil {
480-
returnnil,xerrors.Errorf("extract coder binaries failed: %w",err)
527+
returnnil,nil,xerrors.Errorf("extract coder binaries failed: %w",err)
481528
}
482529
ifn==0 {
483-
returnnil,xerrors.New("no files were extracted from coder binaries archive")
530+
returnnil,nil,xerrors.New("no files were extracted from coder binaries archive")
484531
}
485532
}
486533

487-
returndir,nil
534+
returnbinFS,shaFiles,nil
488535
}
489536

490537
funcfilterFiles(files []fs.DirEntry,names...string) []fs.DirEntry {
@@ -501,24 +548,32 @@ func filterFiles(files []fs.DirEntry, names ...string) []fs.DirEntry {
501548
// errHashMismatch is a sentinel error used in verifyBinSha1IsCurrent.
502549
varerrHashMismatch=xerrors.New("hash mismatch")
503550

504-
funcverifyBinSha1IsCurrent(deststring,siteFS fs.FS) (okbool,errerror) {
505-
b1,err:=fs.ReadFile(siteFS,"bin/coder.sha1")
551+
funcparseSHA1(siteFS fs.FS) (map[string]string,error) {
552+
b,err:=fs.ReadFile(siteFS,"bin/coder.sha1")
506553
iferr!=nil {
507-
returnfalse,xerrors.Errorf("read coder sha1 from embedded fs failed: %w",err)
554+
returnnil,xerrors.Errorf("read coder sha1 from embedded fs failed: %w",err)
508555
}
509-
// Parse sha1 file.
510-
shaFiles:=make(map[string][]byte)
511-
for_,line:=rangebytes.Split(bytes.TrimSpace(b1), []byte{'\n'}) {
556+
557+
shaFiles:=make(map[string]string)
558+
for_,line:=rangebytes.Split(bytes.TrimSpace(b), []byte{'\n'}) {
512559
parts:=bytes.Split(line, []byte{' ','*'})
513560
iflen(parts)!=2 {
514-
returnfalse,xerrors.Errorf("malformed sha1 file: %w",err)
561+
returnnil,xerrors.Errorf("malformed sha1 file: %w",err)
515562
}
516-
shaFiles[string(parts[1])]=parts[0]
563+
shaFiles[string(parts[1])]=strings.ToLower(string(parts[0]))
517564
}
518565
iflen(shaFiles)==0 {
519-
returnfalse,xerrors.Errorf("empty sha1 file: %w",err)
566+
returnnil,xerrors.Errorf("empty sha1 file: %w",err)
520567
}
521568

569+
returnshaFiles,nil
570+
}
571+
572+
funcverifyBinSha1IsCurrent(deststring,siteFS fs.FS,shaFilesmap[string]string) (okbool,errerror) {
573+
b1,err:=fs.ReadFile(siteFS,"bin/coder.sha1")
574+
iferr!=nil {
575+
returnfalse,xerrors.Errorf("read coder sha1 from embedded fs failed: %w",err)
576+
}
522577
b2,err:=os.ReadFile(filepath.Join(dest,"coder.sha1"))
523578
iferr!=nil {
524579
ifxerrors.Is(err,fs.ErrNotExist) {
@@ -551,7 +606,7 @@ func verifyBinSha1IsCurrent(dest string, siteFS fs.FS) (ok bool, err error) {
551606
}
552607
returnxerrors.Errorf("hash file failed: %w",err)
553608
}
554-
if!bytes.Equal(hash1,hash2) {
609+
if!strings.EqualFold(hash1,hash2) {
555610
returnerrHashMismatch
556611
}
557612
returnnil
@@ -570,24 +625,24 @@ func verifyBinSha1IsCurrent(dest string, siteFS fs.FS) (ok bool, err error) {
570625

571626
// sha1HashFile computes a SHA1 hash of the file, returning the hex
572627
// representation.
573-
funcsha1HashFile(namestring) ([]byte,error) {
628+
funcsha1HashFile(namestring) (string,error) {
574629
//#nosec // Not used for cryptography.
575630
hash:=sha1.New()
576631
f,err:=os.Open(name)
577632
iferr!=nil {
578-
returnnil,err
633+
return"",err
579634
}
580635
deferf.Close()
581636

582637
_,err=io.Copy(hash,f)
583638
iferr!=nil {
584-
returnnil,err
639+
return"",err
585640
}
586641

587642
b:=make([]byte,hash.Size())
588643
hash.Sum(b[:0])
589644

590-
return[]byte(hex.EncodeToString(b)),nil
645+
returnhex.EncodeToString(b),nil
591646
}
592647

593648
funcextractBin(deststring,r io.Reader) (numExtractedint,errerror) {
@@ -672,3 +727,67 @@ func RenderStaticErrorPage(rw http.ResponseWriter, r *http.Request, data ErrorPa
672727
return
673728
}
674729
}
730+
731+
typebinHashCachestruct {
732+
binFS http.FileSystem
733+
734+
hashesmap[string]string
735+
mut sync.RWMutex
736+
sf singleflight.Group
737+
semchanstruct{}
738+
}
739+
740+
funcnewBinHashCache(binFS http.FileSystem,binHashesmap[string]string)*binHashCache {
741+
b:=&binHashCache{
742+
binFS:binFS,
743+
hashes:make(map[string]string,len(binHashes)),
744+
mut: sync.RWMutex{},
745+
sf: singleflight.Group{},
746+
sem:make(chanstruct{},4),
747+
}
748+
// Make a copy since we're gonna be mutating it.
749+
fork,v:=rangebinHashes {
750+
b.hashes[k]=v
751+
}
752+
753+
returnb
754+
}
755+
756+
func (b*binHashCache)getHash(namestring) (string,error) {
757+
b.mut.RLock()
758+
hash,ok:=b.hashes[name]
759+
b.mut.RUnlock()
760+
ifok {
761+
returnhash,nil
762+
}
763+
764+
// Avoid DOS by using a pool, and only doing work once per file.
765+
v,err,_:=b.sf.Do(name,func() (interface{},error) {
766+
b.sem<-struct{}{}
767+
deferfunc() {<-b.sem }()
768+
769+
f,err:=b.binFS.Open(name)
770+
iferr!=nil {
771+
return"",err
772+
}
773+
deferf.Close()
774+
775+
h:=sha1.New()//#nosec // Not used for cryptography.
776+
_,err=io.Copy(h,f)
777+
iferr!=nil {
778+
return"",err
779+
}
780+
781+
hash:=hex.EncodeToString(h.Sum(nil))
782+
b.mut.Lock()
783+
b.hashes[name]=hash
784+
b.mut.Unlock()
785+
returnhash,nil
786+
})
787+
iferr!=nil {
788+
return"",err
789+
}
790+
791+
//nolint:forcetypeassert
792+
returnstrings.ToLower(v.(string)),nil
793+
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp