Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit42fd1c1

Browse files
authored
ci: cache embedded postgres downloaded binaries (#18477)
Updates CI job definitions to cache downloaded binaries for embedded-postgres.
1 parentc4e4fe8 commit42fd1c1

File tree

5 files changed

+158
-10
lines changed

5 files changed

+158
-10
lines changed
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
name:"Download Embedded Postgres Cache"
2+
description:|
3+
Downloads the embedded postgres cache and outputs today's cache key.
4+
A PR job can use a cache if it was created by its base branch, its current
5+
branch, or the default branch.
6+
https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache
7+
outputs:
8+
cache-key:
9+
description:"Today's cache key"
10+
value:${{ steps.vars.outputs.cache-key }}
11+
inputs:
12+
key-prefix:
13+
description:"Prefix for the cache key"
14+
required:true
15+
cache-path:
16+
description:"Path to the cache directory"
17+
required:true
18+
runs:
19+
using:"composite"
20+
steps:
21+
-name:Get date values and cache key
22+
id:vars
23+
shell:bash
24+
run:|
25+
export YEAR_MONTH=$(date +'%Y-%m')
26+
export PREV_YEAR_MONTH=$(date -d 'last month' +'%Y-%m')
27+
export DAY=$(date +'%d')
28+
echo "year-month=$YEAR_MONTH" >> $GITHUB_OUTPUT
29+
echo "prev-year-month=$PREV_YEAR_MONTH" >> $GITHUB_OUTPUT
30+
echo "cache-key=${{ inputs.key-prefix }}-${YEAR_MONTH}-${DAY}" >> $GITHUB_OUTPUT
31+
32+
# By default, depot keeps caches for 14 days. This is plenty for embedded
33+
# postgres, which changes infrequently.
34+
# https://depot.dev/docs/github-actions/overview#cache-retention-policy
35+
-name:Download embedded Postgres cache
36+
uses:actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684# v4.2.3
37+
with:
38+
path:${{ inputs.cache-path }}
39+
key:${{ steps.vars.outputs.cache-key }}
40+
# > If there are multiple partial matches for a restore key, the action returns the most recently created cache.
41+
# https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/caching-dependencies-to-speed-up-workflows#matching-a-cache-key
42+
# The second restore key allows non-main branches to use the cache from the previous month.
43+
# This prevents PRs from rebuilding the cache on the first day of the month.
44+
# It also makes sure that once a month, the cache is fully reset.
45+
restore-keys:|
46+
${{ inputs.key-prefix }}-${{ steps.vars.outputs.year-month }}-
47+
${{ github.ref != 'refs/heads/main' && format('{0}-{1}-', inputs.key-prefix, steps.vars.outputs.prev-year-month) || '' }}
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
name:"Upload Embedded Postgres Cache"
2+
description:Uploads the embedded Postgres cache. This only runs on the main branch.
3+
inputs:
4+
cache-key:
5+
description:"Cache key"
6+
required:true
7+
cache-path:
8+
description:"Path to the cache directory"
9+
required:true
10+
runs:
11+
using:"composite"
12+
steps:
13+
-name:Upload Embedded Postgres cache
14+
if:${{ github.ref == 'refs/heads/main' }}
15+
uses:actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684# v4.2.3
16+
with:
17+
path:${{ inputs.cache-path }}
18+
key:${{ inputs.cache-key }}
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name:"Setup Embedded Postgres Cache Paths"
2+
description:Sets up a path for cached embedded postgres binaries.
3+
outputs:
4+
embedded-pg-cache:
5+
description:"Value of EMBEDDED_PG_CACHE_DIR"
6+
value:${{ steps.paths.outputs.embedded-pg-cache }}
7+
cached-dirs:
8+
description:"directories that should be cached between CI runs"
9+
value:${{ steps.paths.outputs.cached-dirs }}
10+
runs:
11+
using:"composite"
12+
steps:
13+
-name:Override Go paths
14+
id:paths
15+
uses:actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea# v7
16+
with:
17+
script:|
18+
const path = require('path');
19+
20+
// RUNNER_TEMP should be backed by a RAM disk on Windows if
21+
// coder/setup-ramdisk-action was used
22+
const runnerTemp = process.env.RUNNER_TEMP;
23+
const embeddedPgCacheDir = path.join(runnerTemp, 'embedded-pg-cache');
24+
core.exportVariable('EMBEDDED_PG_CACHE_DIR', embeddedPgCacheDir);
25+
core.setOutput('embedded-pg-cache', embeddedPgCacheDir);
26+
const cachedDirs = `${embeddedPgCacheDir}`;
27+
core.setOutput('cached-dirs', cachedDirs);
28+
29+
-name:Create directories
30+
shell:bash
31+
run:|
32+
set -e
33+
mkdir -p "$EMBEDDED_PG_CACHE_DIR"

‎.github/workflows/ci.yaml

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -473,6 +473,17 @@ jobs:
473473
with:
474474
key-prefix:test-go-pg-${{ runner.os }}-${{ runner.arch }}
475475

476+
-name:Setup Embedded Postgres Cache Paths
477+
id:embedded-pg-cache
478+
uses:./.github/actions/setup-embedded-pg-cache-paths
479+
480+
-name:Download Embedded Postgres Cache
481+
id:download-embedded-pg-cache
482+
uses:./.github/actions/embedded-pg-cache/download
483+
with:
484+
key-prefix:embedded-pg-${{ runner.os }}-${{ runner.arch }}
485+
cache-path:${{ steps.embedded-pg-cache.outputs.cached-dirs }}
486+
476487
-name:Normalize File and Directory Timestamps
477488
shell:bash
478489
# Normalize file modification timestamps so that go test can use the
@@ -497,12 +508,12 @@ jobs:
497508
# Create a temp dir on the R: ramdisk drive for Windows. The default
498509
# C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
499510
mkdir -p "R:/temp/embedded-pg"
500-
go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg"
511+
go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg" -cache "${EMBEDDED_PG_CACHE_DIR}"
501512
elif [ "${{ runner.os }}" == "macOS" ]; then
502513
# Postgres runs faster on a ramdisk on macOS too
503514
mkdir -p /tmp/tmpfs
504515
sudo mount_tmpfs -o noowners -s 8g /tmp/tmpfs
505-
go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg
516+
go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg -cache "${EMBEDDED_PG_CACHE_DIR}"
506517
elif [ "${{ runner.os }}" == "Linux" ]; then
507518
make test-postgres-docker
508519
fi
@@ -571,6 +582,14 @@ jobs:
571582
with:
572583
cache-key:${{ steps.download-cache.outputs.cache-key }}
573584

585+
-name:Upload Embedded Postgres Cache
586+
uses:./.github/actions/embedded-pg-cache/upload
587+
# We only use the embedded Postgres cache on macOS and Windows runners.
588+
if:runner.OS == 'macOS' || runner.OS == 'Windows'
589+
with:
590+
cache-key:${{ steps.download-embedded-pg-cache.outputs.cache-key }}
591+
cache-path:"${{ steps.embedded-pg-cache.outputs.embedded-pg-cache }}"
592+
574593
-name:Upload test stats to Datadog
575594
timeout-minutes:1
576595
continue-on-error:true

‎scripts/embedded-pg/main.go

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,31 +4,43 @@ package main
44
import (
55
"database/sql"
66
"flag"
7+
"log"
78
"os"
89
"path/filepath"
10+
"time"
911

1012
embeddedpostgres"github.com/fergusstrange/embedded-postgres"
1113
)
1214

1315
funcmain() {
1416
varcustomPathstring
17+
varcachePathstring
1518
flag.StringVar(&customPath,"path","","Optional custom path for postgres data directory")
19+
flag.StringVar(&cachePath,"cache","","Optional custom path for embedded postgres binaries")
1620
flag.Parse()
1721

1822
postgresPath:=filepath.Join(os.TempDir(),"coder-test-postgres")
1923
ifcustomPath!="" {
2024
postgresPath=customPath
2125
}
26+
iferr:=os.MkdirAll(postgresPath,os.ModePerm);err!=nil {
27+
log.Fatalf("Failed to create directory %s: %v",postgresPath,err)
28+
}
29+
ifcachePath=="" {
30+
cachePath=filepath.Join(postgresPath,"cache")
31+
}
32+
iferr:=os.MkdirAll(cachePath,os.ModePerm);err!=nil {
33+
log.Fatalf("Failed to create directory %s: %v",cachePath,err)
34+
}
2235

2336
ep:=embeddedpostgres.NewDatabase(
2437
embeddedpostgres.DefaultConfig().
2538
Version(embeddedpostgres.V16).
2639
BinariesPath(filepath.Join(postgresPath,"bin")).
27-
// Default BinaryRepositoryURL repo1.maven.org is flaky.
2840
BinaryRepositoryURL("https://repo.maven.apache.org/maven2").
2941
DataPath(filepath.Join(postgresPath,"data")).
3042
RuntimePath(filepath.Join(postgresPath,"runtime")).
31-
CachePath(filepath.Join(postgresPath,"cache")).
43+
CachePath(cachePath).
3244
Username("postgres").
3345
Password("postgres").
3446
Database("postgres").
@@ -38,8 +50,27 @@ func main() {
3850
)
3951
err:=ep.Start()
4052
iferr!=nil {
41-
panic(err)
53+
log.Fatalf("Failed to start embedded postgres: %v",err)
54+
}
55+
56+
// Troubleshooting: list files in cachePath
57+
iferr:=filepath.Walk(cachePath,func(pathstring,info os.FileInfo,errerror)error {
58+
iferr!=nil {
59+
returnerr
60+
}
61+
switch {
62+
caseinfo.IsDir():
63+
log.Printf("D: %s",path)
64+
caseinfo.Mode().IsRegular():
65+
log.Printf("F: %s [%s] (%d bytes) %s",path,info.Mode().String(),info.Size(),info.ModTime().Format(time.RFC3339))
66+
default:
67+
log.Printf("Other: %s [%s] %s",path,info.Mode(),info.ModTime().Format(time.RFC3339))
68+
}
69+
returnnil
70+
});err!=nil {
71+
log.Printf("Failed to list files in cachePath %s: %v",cachePath,err)
4272
}
73+
4374
// We execute these queries instead of using the embeddedpostgres
4475
// StartParams because it doesn't work on Windows. The library
4576
// seems to have a bug where it sends malformed parameters to
@@ -58,21 +89,21 @@ func main() {
5889
}
5990
db,err:=sql.Open("postgres","postgres://postgres:postgres@127.0.0.1:5432/postgres?sslmode=disable")
6091
iferr!=nil {
61-
panic(err)
92+
log.Fatalf("Failed to connect to embedded postgres: %v",err)
6293
}
6394
for_,query:=rangeparamQueries {
6495
if_,err:=db.Exec(query);err!=nil {
65-
panic(err)
96+
log.Fatalf("Failed to execute setup query %q: %v",query,err)
6697
}
6798
}
6899
iferr:=db.Close();err!=nil {
69-
panic(err)
100+
log.Fatalf("Failed to close database connection: %v",err)
70101
}
71102
// We restart the database to apply all the parameters.
72103
iferr:=ep.Stop();err!=nil {
73-
panic(err)
104+
log.Fatalf("Failed to stop embedded postgres after applying parameters: %v",err)
74105
}
75106
iferr:=ep.Start();err!=nil {
76-
panic(err)
107+
log.Fatalf("Failed to start embedded postgres after applying parameters: %v",err)
77108
}
78109
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp