Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit94e67b7

Browse files
committed
Merge branch '152-pre-release-issues' into 'master'
fix: pre-release Docker container issues (#152):- resolve docker volume mappings and remove indirect dump container dirs- use dynamic names for auxiliary containers to be able to run multiple database lab instances on the same machine- automatic pull absent Docker images- add container logs output on errorSee merge request postgres-ai/database-lab!141
2 parentsd18a294 +ef187f8 commit94e67b7

File tree

7 files changed

+140
-73
lines changed

7 files changed

+140
-73
lines changed

‎pkg/retrieval/engine/postgres/initialize/logical/dump.go‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,7 @@ const (
3636
DumpJobType="logical-dump"
3737

3838
// Defines dump options.
39-
dumpContainerName="retrieval_logical_dump"
40-
dumpContainerDir="/tmp"
39+
dumpContainerPrefix="dblab_ld_"
4140

4241
// Defines dump source types.
4342
sourceTypeLocal="local"
@@ -190,13 +189,17 @@ func (d *DumpJob) setupDumper() error {
190189
returnerrors.Errorf("unknown source type given: %v",d.Source.Type)
191190
}
192191

192+
func (d*DumpJob)dumpContainerName()string {
193+
returndumpContainerPrefix+d.globalCfg.InstanceID
194+
}
195+
193196
// Name returns a name of the job.
194197
func (d*DumpJob)Name()string {
195198
returnd.name
196199
}
197200

198201
// Run starts the job.
199-
func (d*DumpJob)Run(ctx context.Context)error {
202+
func (d*DumpJob)Run(ctx context.Context)(errerror) {
200203
log.Msg(fmt.Sprintf("Run job: %s. Options: %v",d.Name(),d.DumpOptions))
201204

202205
isEmpty,err:=tools.IsEmptyDirectory(d.globalCfg.DataDir)
@@ -212,6 +215,10 @@ func (d *DumpJob) Run(ctx context.Context) error {
212215
log.Msg("The data directory is not empty. Existing data may be overwritten.")
213216
}
214217

218+
iferr:=tools.PullImage(ctx,d.dockerClient,d.DockerImage);err!=nil {
219+
returnerrors.Wrap(err,"failed to scan pulling image response")
220+
}
221+
215222
cont,err:=d.dockerClient.ContainerCreate(ctx,
216223
&container.Config{
217224
Env:d.getEnvironmentVariables(),
@@ -223,21 +230,27 @@ func (d *DumpJob) Run(ctx context.Context) error {
223230
NetworkMode:d.getContainerNetworkMode(),
224231
},
225232
&network.NetworkingConfig{},
226-
dumpContainerName,
233+
d.dumpContainerName(),
227234
)
228235
iferr!=nil {
229236
log.Err(err)
230237

231-
returnerrors.Wrap(err,"failed to create container")
238+
returnerrors.Wrapf(err,"failed to create container %q",d.dumpContainerName())
232239
}
233240

234241
defertools.RemoveContainer(ctx,d.dockerClient,cont.ID,tools.StopTimeout)
235242

243+
deferfunc() {
244+
iferr!=nil {
245+
tools.PrintContainerLogs(ctx,d.dockerClient,d.dumpContainerName())
246+
}
247+
}()
248+
236249
iferr:=d.dockerClient.ContainerStart(ctx,cont.ID, types.ContainerStartOptions{});err!=nil {
237-
returnerrors.Wrap(err,"failed to start container")
250+
returnerrors.Wrapf(err,"failed to start container %q",d.dumpContainerName())
238251
}
239252

240-
log.Msg(fmt.Sprintf("Running container: %s. ID: %v",dumpContainerName,cont.ID))
253+
log.Msg(fmt.Sprintf("Running container: %s. ID: %v",d.dumpContainerName(),cont.ID))
241254

242255
iferr:=tools.CheckContainerReadiness(ctx,d.dockerClient,cont.ID);err!=nil {
243256
returnerrors.Wrap(err,"failed to readiness check")
@@ -258,7 +271,7 @@ func (d *DumpJob) Run(ctx context.Context) error {
258271
})
259272

260273
iferr!=nil {
261-
returnerrors.Wrap(err,"failed to createan exec command")
274+
returnerrors.Wrap(err,"failed to createdump command")
262275
}
263276

264277
iflen(d.Partial.Tables)>0 {
@@ -343,11 +356,12 @@ func (d *DumpJob) getDumpContainerPath() string {
343356

344357
func (d*DumpJob)getEnvironmentVariables() []string {
345358
envs:= []string{
346-
"PGDATA="+pgDataContainerDir,
359+
"PGDATA="+d.globalCfg.DataDir,
347360
"POSTGRES_HOST_AUTH_METHOD=trust",
348361
}
349362

350363
ifd.DumpOptions.Source.Type==sourceTypeLocal&&d.DumpOptions.Source.Connection.Port==defaults.Port {
364+
log.Msg(fmt.Sprintf("The default PostgreSQL port is busy, trying to use an alternative one: %d",reservePort))
351365
envs=append(envs,"PGPORT="+strconv.Itoa(reservePort))
352366
}
353367

@@ -360,15 +374,15 @@ func (d *DumpJob) getMountVolumes() []mount.Mount {
360374
{
361375
Type:mount.TypeBind,
362376
Source:d.globalCfg.DataDir,
363-
Target:pgDataContainerDir,
377+
Target:d.globalCfg.DataDir,
364378
},
365379
}
366380

367381
ifd.DumpOptions.DumpFile!="" {
368382
mounts=append(mounts, mount.Mount{
369383
Type:mount.TypeBind,
370384
Source:filepath.Dir(d.DumpOptions.DumpFile),
371-
Target:dumpContainerDir,
385+
Target:filepath.Dir(d.DumpOptions.DumpFile),
372386
})
373387
}
374388

‎pkg/retrieval/engine/postgres/initialize/logical/restore.go‎

Lines changed: 41 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,8 @@ const (
3333
RestoreJobType="logical-restore"
3434

3535
// const defines restore options.
36-
restoreContainerName="retrieval_logical_restore"
37-
pgDataContainerDir="/var/lib/postgresql/pgdata"
38-
dumpContainerFile="/tmp/db.dump"
39-
defaultParallelJobs=1
36+
restoreContainerPrefix="dblab_lr_"
37+
defaultParallelJobs=1
4038
)
4139

4240
// RestoreJob defines a logical restore job.
@@ -80,10 +78,6 @@ func NewJob(cfg config.JobConfig, docker *client.Client, globalCfg *dblabCfg.Glo
8078

8179
restoreJob.setDefaults()
8280

83-
iferr:=restoreJob.dbMarker.CreateConfig();err!=nil {
84-
returnnil,errors.Wrap(err,"failed to create a DBMarker config of the database")
85-
}
86-
8781
returnrestoreJob,nil
8882
}
8983

@@ -94,32 +88,41 @@ func (r *RestoreJob) setDefaults() {
9488
}
9589
}
9690

91+
func (r*RestoreJob)restoreContainerName()string {
92+
returnrestoreContainerPrefix+r.globalCfg.InstanceID
93+
}
94+
9795
// Name returns a name of the job.
9896
func (r*RestoreJob)Name()string {
9997
returnr.name
10098
}
10199

102100
// Run starts the job.
103-
func (r*RestoreJob)Run(ctx context.Context)error {
101+
func (r*RestoreJob)Run(ctx context.Context)(errerror) {
104102
log.Msg(fmt.Sprintf("Run job: %s. Options: %v",r.Name(),r.RestoreOptions))
105103

106104
isEmpty,err:=tools.IsEmptyDirectory(r.globalCfg.DataDir)
107105
iferr!=nil {
108-
returnerrors.Wrap(err,"failed to explore the data directory")
106+
returnerrors.Wrapf(err,"failed to explore the data directory %q",r.globalCfg.DataDir)
109107
}
110108

111109
if!isEmpty {
112110
if!r.ForceInit {
113-
returnerrors.New("the data directory is not empty. Use 'forceInit' or empty the data directory")
111+
returnerrors.Errorf("the data directory %q is not empty. Use 'forceInit' or empty the data directory",
112+
r.globalCfg.DataDir)
114113
}
115114

116-
log.Msg("The data directory is not empty. Existing data may be overwritten.")
115+
log.Msg(fmt.Sprintf("The data directory %q is not empty. Existing data may be overwritten.",r.globalCfg.DataDir))
116+
}
117+
118+
iferr:=tools.PullImage(ctx,r.dockerClient,r.RestoreOptions.DockerImage);err!=nil {
119+
returnerrors.Wrap(err,"failed to scan image pulling response")
117120
}
118121

119122
cont,err:=r.dockerClient.ContainerCreate(ctx,
120123
&container.Config{
121124
Env: []string{
122-
"PGDATA="+pgDataContainerDir,
125+
"PGDATA="+r.globalCfg.DataDir,
123126
},
124127
Image:r.RestoreOptions.DockerImage,
125128
Healthcheck:health.GetConfig(),
@@ -129,29 +132,35 @@ func (r *RestoreJob) Run(ctx context.Context) error {
129132
{
130133
Type:mount.TypeBind,
131134
Source:r.RestoreOptions.DumpFile,
132-
Target:dumpContainerFile,
135+
Target:r.RestoreOptions.DumpFile,
133136
},
134137
{
135138
Type:mount.TypeBind,
136139
Source:r.globalCfg.DataDir,
137-
Target:pgDataContainerDir,
140+
Target:r.globalCfg.DataDir,
138141
},
139142
},
140143
},
141144
&network.NetworkingConfig{},
142-
restoreContainerName,
145+
r.restoreContainerName(),
143146
)
144147
iferr!=nil {
145-
returnerrors.Wrap(err,"failed to create container")
148+
returnerrors.Wrapf(err,"failed to create container %q",r.restoreContainerName())
146149
}
147150

148151
defertools.RemoveContainer(ctx,r.dockerClient,cont.ID,tools.StopTimeout)
149152

153+
deferfunc() {
154+
iferr!=nil {
155+
tools.PrintContainerLogs(ctx,r.dockerClient,r.restoreContainerName())
156+
}
157+
}()
158+
150159
iferr:=r.dockerClient.ContainerStart(ctx,cont.ID, types.ContainerStartOptions{});err!=nil {
151-
returnerrors.Wrap(err,"failed to startacontainer")
160+
returnerrors.Wrapf(err,"failed to start container %q",r.restoreContainerName())
152161
}
153162

154-
log.Msg(fmt.Sprintf("Running container: %s. ID: %v",restoreContainerName,cont.ID))
163+
log.Msg(fmt.Sprintf("Running container: %s. ID: %v",r.restoreContainerName(),cont.ID))
155164

156165
iferr:=r.markDatabase(ctx,cont.ID);err!=nil {
157166
returnerrors.Wrap(err,"failed to mark the database")
@@ -172,19 +181,19 @@ func (r *RestoreJob) Run(ctx context.Context) error {
172181
})
173182

174183
iferr!=nil {
175-
returnerrors.Wrap(err,"failed to createan exec command")
184+
returnerrors.Wrap(err,"failed to createrestore command")
176185
}
177186

178187
iflen(r.Partial.Tables)>0 {
179188
log.Msg("Partial restore will be run. Tables for restoring: ",strings.Join(r.Partial.Tables,", "))
180189
}
181190

182191
iferr:=r.dockerClient.ContainerExecStart(ctx,execCommand.ID, types.ExecStartCheck{Tty:true});err!=nil {
183-
returnerrors.Wrap(err,"failed to runthe exec command")
192+
returnerrors.Wrap(err,"failed to runrestore command")
184193
}
185194

186195
iferr:=tools.InspectCommandResponse(ctx,r.dockerClient,cont.ID,execCommand.ID);err!=nil {
187-
returnerrors.Wrap(err,"failed to exectherestore command")
196+
returnerrors.Wrap(err,"failed to exec restore command")
188197
}
189198

190199
iferr:=recalculateStats(ctx,r.dockerClient,cont.ID,buildAnalyzeCommand(Connection{
@@ -209,6 +218,10 @@ func (r *RestoreJob) markDatabase(ctx context.Context, contID string) error {
209218
r.dbMark.DataStateAt=dataStateAt
210219
}
211220

221+
iferr:=r.dbMarker.CreateConfig();err!=nil {
222+
returnerrors.Wrap(err,"failed to create a DBMarker config of the database")
223+
}
224+
212225
iferr:=r.dbMarker.SaveConfig(r.dbMark);err!=nil {
213226
returnerrors.Wrap(err,"failed to mark the database")
214227
}
@@ -217,20 +230,22 @@ func (r *RestoreJob) markDatabase(ctx context.Context, contID string) error {
217230
}
218231

219232
func (r*RestoreJob)retrieveDataStateAt(ctx context.Context,contIDstring) (string,error) {
220-
restoreMetaCmd:= []string{"sh","-c","pg_restore -l "+dumpContainerFile+" | head -n 10"}
233+
restoreMetaCmd:= []string{"sh","-c","pg_restore -l "+r.RestoreOptions.DumpFile+" | head -n 10"}
234+
235+
log.Dbg("Running a restore metadata command: ",restoreMetaCmd)
221236

222237
execCommand,err:=r.dockerClient.ContainerExecCreate(ctx,contID, types.ExecConfig{
223238
AttachStdout:true,
224239
AttachStderr:true,
225240
Cmd:restoreMetaCmd,
226241
})
227242
iferr!=nil {
228-
return"",errors.Wrap(err,"failed to create a restoremeta command")
243+
return"",errors.Wrap(err,"failed to create a restoremetadata command")
229244
}
230245

231246
execAttach,err:=r.dockerClient.ContainerExecAttach(ctx,execCommand.ID, types.ExecStartCheck{})
232247
iferr!=nil {
233-
return"",errors.Wrap(err,"failed to exec a restoremeta command")
248+
return"",errors.Wrap(err,"failed to exec a restoremetadata command")
234249
}
235250

236251
deferexecAttach.Close()
@@ -258,7 +273,7 @@ func (r *RestoreJob) buildLogicalRestoreCommand() []string {
258273
restoreCmd=append(restoreCmd,"-t",table)
259274
}
260275

261-
restoreCmd=append(restoreCmd,dumpContainerFile)
276+
restoreCmd=append(restoreCmd,r.RestoreOptions.DumpFile)
262277

263278
returnrestoreCmd
264279
}

‎pkg/retrieval/engine/postgres/initialize/logical/restore_test.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ func TestRestoreCommandBuilding(t *testing.T) {
2222
DBName:"testDB",
2323
ParallelJobs:1,
2424
ForceInit:false,
25+
DumpFile:"/tmp/db.dump",
2526
},
2627
Command: []string{"pg_restore","-U","postgres","-C","-d","testDB","-j","1","/tmp/db.dump"},
2728
},
@@ -30,13 +31,14 @@ func TestRestoreCommandBuilding(t *testing.T) {
3031
ParallelJobs:4,
3132
ForceInit:true,
3233
},
33-
Command: []string{"pg_restore","-U","postgres","-C","-d","postgres","--clean","--if-exists","-j","4","/tmp/db.dump"},
34+
Command: []string{"pg_restore","-U","postgres","-C","-d","postgres","--clean","--if-exists","-j","4",""},
3435
},
3536
{
3637
CopyOptions:RestoreOptions{
3738
DBName:"testDB",
3839
ParallelJobs:1,
3940
Partial:Partial{Tables: []string{"test","users"}},
41+
DumpFile:"/tmp/db.dump",
4042
},
4143
Command: []string{"pg_restore","-U","postgres","-C","-d","testDB","-j","1","-t","test","-t","users","/tmp/db.dump"},
4244
},

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp