Skip to content

Commit e703907

Browse files
agneumagneum
committed
refactor: service container output (#195)
1 parent c2dfc27 commit e703907

File tree

9 files changed

+301
-357
lines changed

9 files changed

+301
-357
lines changed

‎configs/config.example.physical_generic.yml

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,9 +142,24 @@ retrieval:
142142
# Restores database data from a physical backup.
143143
physicalRestore:
144144
options:
145+
# Defines the tool to restore data.
145146
tool: customTool
147+
148+
# Specifies the Docker image containing the restoring tool.
146149
dockerImage: "postgresai/sync-instance:12"
147-
syncInstance: true
150+
151+
# Sync instance options.
152+
sync:
153+
# Enable running of a sync instance.
154+
enabled: true
155+
156+
# Custom health check options for a sync instance container.
157+
healthCheck:
158+
# Health check interval for a sync instance container (in seconds).
159+
interval: 5
160+
161+
# Maximum number of health check retries.
162+
maxRetries: 200
148163

149164
# Set environment variables here. See https://www.postgresql.org/docs/current/libpq-envars.html
150165
envs:

‎configs/config.example.physical_walg.yml

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,12 +142,31 @@ retrieval:
142142
# Restores database data from a physical backup.
143143
physicalRestore:
144144
options:
145+
# Defines the tool to restore data.
145146
tool: walg
147+
148+
# Specifies the Docker image containing the restoring tool.
146149
dockerImage: "postgresai/sync-instance:12"
147-
syncInstance: true
150+
151+
# Sync instance options.
152+
sync:
153+
# Enable running of a sync instance.
154+
enabled: true
155+
156+
# Custom health check options for a sync instance container.
157+
healthCheck:
158+
# Health check interval for a sync instance container (in seconds).
159+
interval: 5
160+
161+
# Maximum number of health check retries.
162+
maxRetries: 200
163+
164+
# Passes custom environment variables to the Docker container with the restoring tool.
148165
envs:
149166
WALG_GS_PREFIX: "gs://{BUCKET}/{SCOPE}"
150167
GOOGLE_APPLICATION_CREDENTIALS: "/tmp/sa.json"
168+
169+
# Defines WAL-G configuration options.
151170
walg:
152171
backupName: LATEST
153172

‎pkg/retrieval/engine/postgres/logical/dump.go

Lines changed: 19 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ package logical
77
import (
88
"context"
99
"fmt"
10-
"io"
1110
"os"
1211
"strconv"
1312
"strings"
@@ -18,7 +17,6 @@ import (
1817
"github.com/docker/docker/api/types/network"
1918
"github.com/docker/docker/client"
2019
"github.com/pkg/errors"
21-
"github.com/sethvargo/go-password/password"
2220

2321
dblabCfg "gitlab.com/postgres-ai/database-lab/pkg/config"
2422
"gitlab.com/postgres-ai/database-lab/pkg/log"
@@ -230,7 +228,7 @@ func (d *DumpJob) Run(ctx context.Context) (err error) {
230228
return errors.Wrap(err, "failed to build container host config")
231229
}
232230

233-
pwd, err := password.Generate(tools.PasswordLength, tools.PasswordMinDigits, tools.PasswordMinSymbols, false, true)
231+
pwd, err := tools.GeneratePassword()
234232
if err != nil {
235233
return errors.Wrap(err, "failed to generate PostgreSQL password")
236234
}
@@ -248,15 +246,17 @@ func (d *DumpJob) Run(ctx context.Context) (err error) {
248246

249247
defer func() {
250248
if err != nil {
251-
tools.PrintContainerLogs(ctx, d.dockerClient, d.dumpContainerName(), err)
249+
tools.PrintContainerLogs(ctx, d.dockerClient, d.dumpContainerName())
252250
}
253251
}()
254252

253+
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", d.dumpContainerName(), dumpCont.ID))
254+
255255
if err := d.dockerClient.ContainerStart(ctx, dumpCont.ID, types.ContainerStartOptions{}); err != nil {
256256
return errors.Wrapf(err, "failed to start container %q", d.dumpContainerName())
257257
}
258258

259-
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", d.dumpContainerName(), dumpCont.ID))
259+
log.Msg("Waiting for container is ready")
260260

261261
if err := tools.CheckContainerReadiness(ctx, d.dockerClient, dumpCont.ID); err != nil {
262262
return errors.Wrap(err, "failed to readiness check")
@@ -269,22 +269,14 @@ func (d *DumpJob) Run(ctx context.Context) (err error) {
269269
dumpCommand := d.buildLogicalDumpCommand()
270270
log.Msg("Running dump command", dumpCommand)
271271

272-
execCommand, err := d.dockerClient.ContainerExecCreate(ctx, dumpCont.ID, types.ExecConfig{
273-
AttachStdout: true,
274-
AttachStderr: true,
275-
Cmd: dumpCommand,
276-
Env: d.getExecEnvironmentVariables(),
277-
})
278-
279-
if err != nil {
280-
return errors.Wrap(err, "failed to create dump command")
281-
}
282-
283272
if len(d.Partial.Tables) > 0 {
284273
log.Msg("Partial dump will be run. Tables for dumping: ", strings.Join(d.Partial.Tables, ", "))
285274
}
286275

287-
if err := d.performDumpCommand(ctx, os.Stdout, dumpCont.ID, execCommand.ID); err != nil {
276+
if err := d.performDumpCommand(ctx, dumpCont.ID, types.ExecConfig{
277+
Cmd: dumpCommand,
278+
Env: d.getExecEnvironmentVariables(),
279+
}); err != nil {
288280
return errors.Wrap(err, "failed to dump a database")
289281
}
290282

@@ -293,10 +285,14 @@ func (d *DumpJob) Run(ctx context.Context) (err error) {
293285
return errors.Wrap(err, "failed to mark the created dump")
294286
}
295287

296-
if err := recalculateStats(ctx, d.dockerClient, dumpCont.ID, buildAnalyzeCommand(Connection{
297-
Username: d.globalCfg.Database.User(),
298-
DBName: d.globalCfg.Database.Name(),
299-
}, d.DumpOptions.ParallelJobs)); err != nil {
288+
analyzeCmd := buildAnalyzeCommand(
289+
Connection{Username: d.globalCfg.Database.User(), DBName: d.globalCfg.Database.Name()},
290+
d.DumpOptions.ParallelJobs,
291+
)
292+
293+
log.Msg("Running analyze command: ", analyzeCmd)
294+
295+
if err := tools.ExecCommand(ctx, d.dockerClient, dumpCont.ID, types.ExecConfig{Cmd: analyzeCmd}); err != nil {
300296
return errors.Wrap(err, "failed to recalculate statistics after restore")
301297
}
302298
}
@@ -317,26 +313,12 @@ func (d *DumpJob) setupConnectionOptions(ctx context.Context) error {
317313
return nil
318314
}
319315

320-
func (d *DumpJob) performDumpCommand(ctx context.Context, cmdOutput io.Writer, contID, commandID string) error {
316+
func (d *DumpJob) performDumpCommand(ctx context.Context, contID string, commandCfg types.ExecConfig) error {
321317
if d.DumpOptions.Restore != nil {
322318
d.dbMark.DataStateAt = time.Now().Format(tools.DataStateAtFormat)
323319
}
324320

325-
execAttach, err := d.dockerClient.ContainerExecAttach(ctx, commandID, types.ExecStartCheck{})
326-
if err != nil {
327-
return err
328-
}
329-
defer execAttach.Close()
330-
331-
if err := tools.ProcessAttachResponse(ctx, execAttach.Reader, cmdOutput); err != nil {
332-
return err
333-
}
334-
335-
if err := tools.InspectCommandResponse(ctx, d.dockerClient, contID, commandID); err != nil {
336-
return errors.Wrap(err, "failed to exec the dump command")
337-
}
338-
339-
return nil
321+
return tools.ExecCommand(ctx, d.dockerClient, contID, commandCfg)
340322
}
341323

342324
func (d *DumpJob) getEnvironmentVariables(password string) []string {

‎pkg/retrieval/engine/postgres/logical/logical.go

Lines changed: 0 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -6,41 +6,9 @@
66
package logical
77

88
import (
9-
"context"
109
"strconv"
11-
12-
"github.com/docker/docker/api/types"
13-
"github.com/docker/docker/client"
14-
"github.com/pkg/errors"
15-
16-
"gitlab.com/postgres-ai/database-lab/pkg/log"
17-
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/tools"
1810
)
1911

20-
func recalculateStats(ctx context.Context, dockerClient *client.Client, contID string, analyzeCmd []string) error {
21-
log.Msg("Running analyze command: ", analyzeCmd)
22-
23-
exec, err := dockerClient.ContainerExecCreate(ctx, contID, types.ExecConfig{
24-
AttachStdout: true,
25-
AttachStderr: true,
26-
Cmd: analyzeCmd,
27-
})
28-
29-
if err != nil {
30-
return errors.Wrap(err, "failed to create an exec command")
31-
}
32-
33-
if err := dockerClient.ContainerExecStart(ctx, exec.ID, types.ExecStartCheck{Tty: true}); err != nil {
34-
return errors.Wrap(err, "failed to run the exec command")
35-
}
36-
37-
if err := tools.InspectCommandResponse(ctx, dockerClient, contID, exec.ID); err != nil {
38-
return errors.Wrap(err, "failed to exec the restore command")
39-
}
40-
41-
return nil
42-
}
43-
4412
func buildAnalyzeCommand(conn Connection, parallelJobs int) []string {
4513
analyzeCmd := []string{
4614
"vacuumdb",

‎pkg/retrieval/engine/postgres/logical/restore.go

Lines changed: 14 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ import (
1717
"github.com/docker/docker/api/types/network"
1818
"github.com/docker/docker/client"
1919
"github.com/pkg/errors"
20-
"github.com/sethvargo/go-password/password"
2120

2221
dblabCfg "gitlab.com/postgres-ai/database-lab/pkg/config"
2322
"gitlab.com/postgres-ai/database-lab/pkg/log"
@@ -136,7 +135,7 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
136135
return errors.Wrap(err, "failed to build container host config")
137136
}
138137

139-
pwd, err := password.Generate(tools.PasswordLength, tools.PasswordMinDigits, tools.PasswordMinSymbols, false, true)
138+
pwd, err := tools.GeneratePassword()
140139
if err != nil {
141140
return errors.Wrap(err, "failed to generate PostgreSQL password")
142141
}
@@ -155,15 +154,17 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
155154

156155
defer func() {
157156
if err != nil {
158-
tools.PrintContainerLogs(ctx, r.dockerClient, r.restoreContainerName(), err)
157+
tools.PrintContainerLogs(ctx, r.dockerClient, r.restoreContainerName())
159158
}
160159
}()
161160

161+
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", r.restoreContainerName(), restoreCont.ID))
162+
162163
if err := r.dockerClient.ContainerStart(ctx, restoreCont.ID, types.ContainerStartOptions{}); err != nil {
163164
return errors.Wrapf(err, "failed to start container %q", r.restoreContainerName())
164165
}
165166

166-
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", r.restoreContainerName(), restoreCont.ID))
167+
log.Msg("Waiting for container is ready")
167168

168169
if err := tools.CheckContainerReadiness(ctx, r.dockerClient, restoreCont.ID); err != nil {
169170
return errors.Wrap(err, "failed to readiness check")
@@ -172,37 +173,26 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
172173
restoreCommand := r.buildLogicalRestoreCommand()
173174
log.Msg("Running restore command: ", restoreCommand)
174175

175-
execCommand, err := r.dockerClient.ContainerExecCreate(ctx, restoreCont.ID, types.ExecConfig{
176-
AttachStdout: true,
177-
AttachStderr: true,
178-
Tty: true,
179-
Cmd: restoreCommand,
180-
})
181-
182-
if err != nil {
183-
return errors.Wrap(err, "failed to create restore command")
184-
}
185-
186176
if len(r.Partial.Tables) > 0 {
187177
log.Msg("Partial restore will be run. Tables for restoring: ", strings.Join(r.Partial.Tables, ", "))
188178
}
189179

190-
if err := r.dockerClient.ContainerExecStart(ctx, execCommand.ID, types.ExecStartCheck{Tty: true}); err != nil {
191-
return errors.Wrap(err, "failed to run restore command")
192-
}
193-
194-
if err := tools.InspectCommandResponse(ctx, r.dockerClient, restoreCont.ID, execCommand.ID); err != nil {
180+
if err := tools.ExecCommand(ctx, r.dockerClient, restoreCont.ID, types.ExecConfig{Cmd: restoreCommand}); err != nil {
195181
return errors.Wrap(err, "failed to exec restore command")
196182
}
197183

198184
if err := r.markDatabase(ctx, restoreCont.ID); err != nil {
199185
return errors.Wrap(err, "failed to mark the database")
200186
}
201187

202-
if err := recalculateStats(ctx, r.dockerClient, restoreCont.ID, buildAnalyzeCommand(Connection{
203-
Username: r.globalCfg.Database.User(),
204-
DBName: r.globalCfg.Database.Name(),
205-
}, r.RestoreOptions.ParallelJobs)); err != nil {
188+
analyzeCmd := buildAnalyzeCommand(
189+
Connection{Username: r.globalCfg.Database.User(), DBName: r.globalCfg.Database.Name()},
190+
r.RestoreOptions.ParallelJobs,
191+
)
192+
193+
log.Msg("Running analyze command: ", analyzeCmd)
194+
195+
if err := tools.ExecCommand(ctx, r.dockerClient, restoreCont.ID, types.ExecConfig{Cmd: analyzeCmd}); err != nil {
206196
return errors.Wrap(err, "failed to recalculate statistics after restore")
207197
}
208198

0 commit comments

Comments
 (0)