Skip to content

Commit 94e67b7

Browse files
agneumagneum
committed
Merge branch '152-pre-release-issues' into 'master'
fix: pre-release Docker container issues (#152): - resolve docker volume mappings and remove indirect dump container dirs - use dynamic names for auxiliary containers to be able to run multiple database lab instances on the same machine - automatic pull absent Docker images - add container logs output on error See merge request postgres-ai/database-lab!141
2 parents d18a294 + ef187f8 commit 94e67b7

File tree

7 files changed

+140
-73
lines changed

7 files changed

+140
-73
lines changed

‎pkg/retrieval/engine/postgres/initialize/logical/dump.go

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,7 @@ const (
3636
DumpJobType = "logical-dump"
3737

3838
// Defines dump options.
39-
dumpContainerName = "retrieval_logical_dump"
40-
dumpContainerDir = "/tmp"
39+
dumpContainerPrefix = "dblab_ld_"
4140

4241
// Defines dump source types.
4342
sourceTypeLocal = "local"
@@ -190,13 +189,17 @@ func (d *DumpJob) setupDumper() error {
190189
return errors.Errorf("unknown source type given: %v", d.Source.Type)
191190
}
192191

192+
func (d *DumpJob) dumpContainerName() string {
193+
return dumpContainerPrefix + d.globalCfg.InstanceID
194+
}
195+
193196
// Name returns a name of the job.
194197
func (d *DumpJob) Name() string {
195198
return d.name
196199
}
197200

198201
// Run starts the job.
199-
func (d *DumpJob) Run(ctx context.Context) error {
202+
func (d *DumpJob) Run(ctx context.Context) (err error) {
200203
log.Msg(fmt.Sprintf("Run job: %s. Options: %v", d.Name(), d.DumpOptions))
201204

202205
isEmpty, err := tools.IsEmptyDirectory(d.globalCfg.DataDir)
@@ -212,6 +215,10 @@ func (d *DumpJob) Run(ctx context.Context) error {
212215
log.Msg("The data directory is not empty. Existing data may be overwritten.")
213216
}
214217

218+
if err := tools.PullImage(ctx, d.dockerClient, d.DockerImage); err != nil {
219+
return errors.Wrap(err, "failed to scan pulling image response")
220+
}
221+
215222
cont, err := d.dockerClient.ContainerCreate(ctx,
216223
&container.Config{
217224
Env: d.getEnvironmentVariables(),
@@ -223,21 +230,27 @@ func (d *DumpJob) Run(ctx context.Context) error {
223230
NetworkMode: d.getContainerNetworkMode(),
224231
},
225232
&network.NetworkingConfig{},
226-
dumpContainerName,
233+
d.dumpContainerName(),
227234
)
228235
if err != nil {
229236
log.Err(err)
230237

231-
return errors.Wrap(err, "failed to create container")
238+
return errors.Wrapf(err, "failed to create container %q", d.dumpContainerName())
232239
}
233240

234241
defer tools.RemoveContainer(ctx, d.dockerClient, cont.ID, tools.StopTimeout)
235242

243+
defer func() {
244+
if err != nil {
245+
tools.PrintContainerLogs(ctx, d.dockerClient, d.dumpContainerName())
246+
}
247+
}()
248+
236249
if err := d.dockerClient.ContainerStart(ctx, cont.ID, types.ContainerStartOptions{}); err != nil {
237-
return errors.Wrap(err, "failed to start container")
250+
return errors.Wrapf(err, "failed to start container %q", d.dumpContainerName())
238251
}
239252

240-
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", dumpContainerName, cont.ID))
253+
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", d.dumpContainerName(), cont.ID))
241254

242255
if err := tools.CheckContainerReadiness(ctx, d.dockerClient, cont.ID); err != nil {
243256
return errors.Wrap(err, "failed to readiness check")
@@ -258,7 +271,7 @@ func (d *DumpJob) Run(ctx context.Context) error {
258271
})
259272

260273
if err != nil {
261-
return errors.Wrap(err, "failed to create an exec command")
274+
return errors.Wrap(err, "failed to create dump command")
262275
}
263276

264277
if len(d.Partial.Tables) > 0 {
@@ -343,11 +356,12 @@ func (d *DumpJob) getDumpContainerPath() string {
343356

344357
func (d *DumpJob) getEnvironmentVariables() []string {
345358
envs := []string{
346-
"PGDATA=" + pgDataContainerDir,
359+
"PGDATA=" + d.globalCfg.DataDir,
347360
"POSTGRES_HOST_AUTH_METHOD=trust",
348361
}
349362

350363
if d.DumpOptions.Source.Type == sourceTypeLocal && d.DumpOptions.Source.Connection.Port == defaults.Port {
364+
log.Msg(fmt.Sprintf("The default PostgreSQL port is busy, trying to use an alternative one: %d", reservePort))
351365
envs = append(envs, "PGPORT="+strconv.Itoa(reservePort))
352366
}
353367

@@ -360,15 +374,15 @@ func (d *DumpJob) getMountVolumes() []mount.Mount {
360374
{
361375
Type: mount.TypeBind,
362376
Source: d.globalCfg.DataDir,
363-
Target: pgDataContainerDir,
377+
Target: d.globalCfg.DataDir,
364378
},
365379
}
366380

367381
if d.DumpOptions.DumpFile != "" {
368382
mounts = append(mounts, mount.Mount{
369383
Type: mount.TypeBind,
370384
Source: filepath.Dir(d.DumpOptions.DumpFile),
371-
Target: dumpContainerDir,
385+
Target: filepath.Dir(d.DumpOptions.DumpFile),
372386
})
373387
}
374388

‎pkg/retrieval/engine/postgres/initialize/logical/restore.go

Lines changed: 41 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,8 @@ const (
3333
RestoreJobType = "logical-restore"
3434

3535
// const defines restore options.
36-
restoreContainerName = "retrieval_logical_restore"
37-
pgDataContainerDir = "/var/lib/postgresql/pgdata"
38-
dumpContainerFile = "/tmp/db.dump"
39-
defaultParallelJobs = 1
36+
restoreContainerPrefix = "dblab_lr_"
37+
defaultParallelJobs = 1
4038
)
4139

4240
// RestoreJob defines a logical restore job.
@@ -80,10 +78,6 @@ func NewJob(cfg config.JobConfig, docker *client.Client, globalCfg *dblabCfg.Glo
8078

8179
restoreJob.setDefaults()
8280

83-
if err := restoreJob.dbMarker.CreateConfig(); err != nil {
84-
return nil, errors.Wrap(err, "failed to create a DBMarker config of the database")
85-
}
86-
8781
return restoreJob, nil
8882
}
8983

@@ -94,32 +88,41 @@ func (r *RestoreJob) setDefaults() {
9488
}
9589
}
9690

91+
func (r *RestoreJob) restoreContainerName() string {
92+
return restoreContainerPrefix + r.globalCfg.InstanceID
93+
}
94+
9795
// Name returns a name of the job.
9896
func (r *RestoreJob) Name() string {
9997
return r.name
10098
}
10199

102100
// Run starts the job.
103-
func (r *RestoreJob) Run(ctx context.Context) error {
101+
func (r *RestoreJob) Run(ctx context.Context) (err error) {
104102
log.Msg(fmt.Sprintf("Run job: %s. Options: %v", r.Name(), r.RestoreOptions))
105103

106104
isEmpty, err := tools.IsEmptyDirectory(r.globalCfg.DataDir)
107105
if err != nil {
108-
return errors.Wrap(err, "failed to explore the data directory")
106+
return errors.Wrapf(err, "failed to explore the data directory %q", r.globalCfg.DataDir)
109107
}
110108

111109
if !isEmpty {
112110
if !r.ForceInit {
113-
return errors.New("the data directory is not empty. Use 'forceInit' or empty the data directory")
111+
return errors.Errorf("the data directory %q is not empty. Use 'forceInit' or empty the data directory",
112+
r.globalCfg.DataDir)
114113
}
115114

116-
log.Msg("The data directory is not empty. Existing data may be overwritten.")
115+
log.Msg(fmt.Sprintf("The data directory %q is not empty. Existing data may be overwritten.", r.globalCfg.DataDir))
116+
}
117+
118+
if err := tools.PullImage(ctx, r.dockerClient, r.RestoreOptions.DockerImage); err != nil {
119+
return errors.Wrap(err, "failed to scan image pulling response")
117120
}
118121

119122
cont, err := r.dockerClient.ContainerCreate(ctx,
120123
&container.Config{
121124
Env: []string{
122-
"PGDATA=" + pgDataContainerDir,
125+
"PGDATA=" + r.globalCfg.DataDir,
123126
},
124127
Image: r.RestoreOptions.DockerImage,
125128
Healthcheck: health.GetConfig(),
@@ -129,29 +132,35 @@ func (r *RestoreJob) Run(ctx context.Context) error {
129132
{
130133
Type: mount.TypeBind,
131134
Source: r.RestoreOptions.DumpFile,
132-
Target: dumpContainerFile,
135+
Target: r.RestoreOptions.DumpFile,
133136
},
134137
{
135138
Type: mount.TypeBind,
136139
Source: r.globalCfg.DataDir,
137-
Target: pgDataContainerDir,
140+
Target: r.globalCfg.DataDir,
138141
},
139142
},
140143
},
141144
&network.NetworkingConfig{},
142-
restoreContainerName,
145+
r.restoreContainerName(),
143146
)
144147
if err != nil {
145-
return errors.Wrap(err, "failed to create container")
148+
return errors.Wrapf(err, "failed to create container %q", r.restoreContainerName())
146149
}
147150

148151
defer tools.RemoveContainer(ctx, r.dockerClient, cont.ID, tools.StopTimeout)
149152

153+
defer func() {
154+
if err != nil {
155+
tools.PrintContainerLogs(ctx, r.dockerClient, r.restoreContainerName())
156+
}
157+
}()
158+
150159
if err := r.dockerClient.ContainerStart(ctx, cont.ID, types.ContainerStartOptions{}); err != nil {
151-
return errors.Wrap(err, "failed to start a container")
160+
return errors.Wrapf(err, "failed to start container %q", r.restoreContainerName())
152161
}
153162

154-
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", restoreContainerName, cont.ID))
163+
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", r.restoreContainerName(), cont.ID))
155164

156165
if err := r.markDatabase(ctx, cont.ID); err != nil {
157166
return errors.Wrap(err, "failed to mark the database")
@@ -172,19 +181,19 @@ func (r *RestoreJob) Run(ctx context.Context) error {
172181
})
173182

174183
if err != nil {
175-
return errors.Wrap(err, "failed to create an exec command")
184+
return errors.Wrap(err, "failed to create restore command")
176185
}
177186

178187
if len(r.Partial.Tables) > 0 {
179188
log.Msg("Partial restore will be run. Tables for restoring: ", strings.Join(r.Partial.Tables, ", "))
180189
}
181190

182191
if err := r.dockerClient.ContainerExecStart(ctx, execCommand.ID, types.ExecStartCheck{Tty: true}); err != nil {
183-
return errors.Wrap(err, "failed to run the exec command")
192+
return errors.Wrap(err, "failed to run restore command")
184193
}
185194

186195
if err := tools.InspectCommandResponse(ctx, r.dockerClient, cont.ID, execCommand.ID); err != nil {
187-
return errors.Wrap(err, "failed to exec the restore command")
196+
return errors.Wrap(err, "failed to exec restore command")
188197
}
189198

190199
if err := recalculateStats(ctx, r.dockerClient, cont.ID, buildAnalyzeCommand(Connection{
@@ -209,6 +218,10 @@ func (r *RestoreJob) markDatabase(ctx context.Context, contID string) error {
209218
r.dbMark.DataStateAt = dataStateAt
210219
}
211220

221+
if err := r.dbMarker.CreateConfig(); err != nil {
222+
return errors.Wrap(err, "failed to create a DBMarker config of the database")
223+
}
224+
212225
if err := r.dbMarker.SaveConfig(r.dbMark); err != nil {
213226
return errors.Wrap(err, "failed to mark the database")
214227
}
@@ -217,20 +230,22 @@ func (r *RestoreJob) markDatabase(ctx context.Context, contID string) error {
217230
}
218231

219232
func (r *RestoreJob) retrieveDataStateAt(ctx context.Context, contID string) (string, error) {
220-
restoreMetaCmd := []string{"sh", "-c", "pg_restore -l " + dumpContainerFile + " | head -n 10"}
233+
restoreMetaCmd := []string{"sh", "-c", "pg_restore -l " + r.RestoreOptions.DumpFile + " | head -n 10"}
234+
235+
log.Dbg("Running a restore metadata command: ", restoreMetaCmd)
221236

222237
execCommand, err := r.dockerClient.ContainerExecCreate(ctx, contID, types.ExecConfig{
223238
AttachStdout: true,
224239
AttachStderr: true,
225240
Cmd: restoreMetaCmd,
226241
})
227242
if err != nil {
228-
return "", errors.Wrap(err, "failed to create a restore meta command")
243+
return "", errors.Wrap(err, "failed to create a restore metadata command")
229244
}
230245

231246
execAttach, err := r.dockerClient.ContainerExecAttach(ctx, execCommand.ID, types.ExecStartCheck{})
232247
if err != nil {
233-
return "", errors.Wrap(err, "failed to exec a restore meta command")
248+
return "", errors.Wrap(err, "failed to exec a restore metadata command")
234249
}
235250

236251
defer execAttach.Close()
@@ -258,7 +273,7 @@ func (r *RestoreJob) buildLogicalRestoreCommand() []string {
258273
restoreCmd = append(restoreCmd, "-t", table)
259274
}
260275

261-
restoreCmd = append(restoreCmd, dumpContainerFile)
276+
restoreCmd = append(restoreCmd, r.RestoreOptions.DumpFile)
262277

263278
return restoreCmd
264279
}

‎pkg/retrieval/engine/postgres/initialize/logical/restore_test.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ func TestRestoreCommandBuilding(t *testing.T) {
2222
DBName: "testDB",
2323
ParallelJobs: 1,
2424
ForceInit: false,
25+
DumpFile: "/tmp/db.dump",
2526
},
2627
Command: []string{"pg_restore", "-U", "postgres", "-C", "-d", "testDB", "-j", "1", "/tmp/db.dump"},
2728
},
@@ -30,13 +31,14 @@ func TestRestoreCommandBuilding(t *testing.T) {
3031
ParallelJobs: 4,
3132
ForceInit: true,
3233
},
33-
Command: []string{"pg_restore", "-U", "postgres", "-C", "-d", "postgres", "--clean", "--if-exists", "-j", "4", "/tmp/db.dump"},
34+
Command: []string{"pg_restore", "-U", "postgres", "-C", "-d", "postgres", "--clean", "--if-exists", "-j", "4", ""},
3435
},
3536
{
3637
CopyOptions: RestoreOptions{
3738
DBName: "testDB",
3839
ParallelJobs: 1,
3940
Partial: Partial{Tables: []string{"test", "users"}},
41+
DumpFile: "/tmp/db.dump",
4042
},
4143
Command: []string{"pg_restore", "-U", "postgres", "-C", "-d", "testDB", "-j", "1", "-t", "test", "-t", "users", "/tmp/db.dump"},
4244
},

0 commit comments

Comments
 (0)